diff --git a/.github/workflows/hil-flash-check.yml b/.github/workflows/hil-flash-check.yml deleted file mode 100644 index a5948dd3..00000000 --- a/.github/workflows/hil-flash-check.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: HIL flash and health check - -# Builds firmware for each attached board, flashes it, and health-checks the -# API over both USB and HTTP once the board boots. -# -# Deliberately no `pull_request` trigger — this targets a self-hosted runner on -# a private network and physically reflashes hardware. Fork gating is designed -# in dev/hil/DESIGN.md §4 and is not built yet. -# -# The `push` trigger exists so this can be validated before merging, since a -# workflow_dispatch workflow is not dispatchable until it reaches the default -# branch. Drop the push trigger once this is on main. - -on: - workflow_dispatch: - inputs: - skip_http: - description: "USB checks only (skip the network stack)" - type: boolean - default: false - skip_flash: - description: "Health-check what is already flashed, do not reflash" - type: boolean - default: false - inventory_only: - description: "Just print each board's chip id, for building VECTOR_HIL_BOARD_MAP" - type: boolean - default: false - identify: - description: "Blink each board in turn so you can see which physical board is which" - type: boolean - default: false - push: - branches: - - claude/hil-testing-design-s564ln - paths: - - dev/hil/flash_and_check.py - - .github/workflows/hil-flash-check.yml - -permissions: - contents: read - -# Flashing is destructive and the bench is one set of boards. Never interleave. -concurrency: - group: hil-bench - cancel-in-progress: false - -jobs: - flash-and-check: - runs-on: [self-hosted, vector-hil] - timeout-minutes: 45 - - steps: - # actions/checkout is safe here ONLY because every trigger above is - # repo-internal. If a fork-reachable trigger is ever added, this must go - # back to running from the runner's pinned clone ($VECTOR_HIL_REPO) — - # otherwise a fork PR would execute its own harness code on the bench. - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - - name: Prepare environment - run: | - test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } - echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" - - # VECTOR_HIL_BOARD_MAP, VECTOR_HIL_WIFI_* and friends arrive from the - # runner's .env, which the runner process exports into every job. They - # are deliberately NOT restated in an `env:` block here: the `env` - # context only covers workflow/job/step-level vars, so `${{ env.X }}` - # would evaluate to empty and shadow the real value. - - name: Flash and health-check every board - run: | - args="" - if [ "${{ inputs.skip_http }}" = "true" ]; then args="$args --skip-http"; fi - if [ "${{ inputs.skip_flash }}" = "true" ]; then args="$args --skip-flash"; fi - if [ "${{ inputs.inventory_only }}" = "true" ]; then args="$args --inventory-only"; fi - if [ "${{ inputs.identify }}" = "true" ]; then args="$args --identify"; fi - python dev/hil/flash_and_check.py $args - - - name: Board serial logs on failure - if: failure() - run: | - # A board that failed its health check may still be printing something - # useful; grab a few seconds of console from each. - for dev in /dev/ttyACM*; do - echo "--- $dev" - timeout 8 cat "$dev" || true - done diff --git a/.github/workflows/hil-smoke.yml b/.github/workflows/hil-smoke.yml deleted file mode 100644 index ca4c9cbc..00000000 --- a/.github/workflows/hil-smoke.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: HIL smoke - -# Proves the bench runner is wired up correctly: the runner picks up jobs, the -# bench environment reaches them, the boards enumerate, and each one answers -# over serial. It does not test firmware — see dev/hil/DESIGN.md for that. -# -# Deliberately no `pull_request` trigger. This targets a self-hosted runner on -# a private network, and a pull_request trigger would let any fork PR run code -# against it. Fork gating is designed in DESIGN.md §4 and is not built yet. -# -# The `push` trigger exists so this can be validated before merging, since a -# workflow_dispatch workflow is not dispatchable until it reaches the default -# branch. Drop the push trigger once this is on main. - -on: - workflow_dispatch: - push: - branches: - - claude/hil-testing-design-s564ln - -permissions: {} - -# The bench is a single set of physical boards. Queue, never interleave. -concurrency: - group: hil-bench - cancel-in-progress: false - -jobs: - smoke: - runs-on: [self-hosted, vector-hil] - timeout-minutes: 15 - - # No actions/checkout on purpose: under the design's trust model the bench - # runs harness code from a trusted checkout the runner already has, not - # from the PR. $VECTOR_HIL_REPO points at it. - - steps: - - name: Runner host - run: | - echo "user: $(whoami)" - echo "kernel: $(uname -srm)" - echo "arch: $(dpkg --print-architecture)" - free -h | awk '/Mem:/ {print "memory: " $2 " total, " $7 " available"}' - echo "uptime:$(uptime -p | sed 's/^up//')" - - - name: Bench environment - run: | - fail=0 - for var in VECTOR_HIL_REPO VECTOR_HIL_VENV VECTOR_HIL_WIFI_SSID VECTOR_HIL_WIFI_PASSWORD; do - eval "val=\${$var:-}" - if [ -z "$val" ]; then - echo "MISSING: $var" - fail=1 - else - echo "set: $var" - fi - done - [ "$fail" -eq 0 ] || { - echo "The runner's .env is incomplete. It is read at service start:" - echo " cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start" - exit 1 - } - test -x "$VECTOR_HIL_VENV/bin/python" || { echo "venv python missing at $VECTOR_HIL_VENV"; exit 1; } - test -d "$VECTOR_HIL_REPO/.git" || { echo "no clone at $VECTOR_HIL_REPO"; exit 1; } - - # dev/detect_boards.py shells out to a bare `mpremote` from PATH and - # silently returns {} when it is missing, so the venv must be ON PATH - # for later steps - invoking its python by absolute path is not enough. - echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" - - - name: Serial access - run: | - if ! ls /dev/ttyACM* >/dev/null 2>&1; then - echo "no /dev/ttyACM* devices - check the USB hub and power" - exit 1 - fi - ls -l /dev/ttyACM* - id -nG | grep -qw dialout || { - echo "runner user is not in the dialout group; reboot or restart the service" - exit 1 - } - - - name: mpremote sees the boards - run: | - # Checked separately because detect_boards.py collapses "mpremote is - # missing", "mpremote failed", and "no boards attached" into the same - # empty result, which makes a real failure indistinguishable from an - # empty bench. - command -v mpremote || { echo "mpremote not on PATH"; exit 1; } - mpremote version - - echo "--- mpremote devs ---" - mpremote devs | tee /tmp/devs.txt - test -s /tmp/devs.txt || { - echo "mpremote sees no devices, though /dev/ttyACM* exist." - echo "Boards may be busy or in an odd state; power-cycle the hub and retry." - exit 1 - } - - - name: Detect and query boards - run: | - # detect_boards.py prints progress text before the JSON, so the - # machine-readable part is the last line only. - raw=$(python "$VECTOR_HIL_REPO/dev/detect_boards.py") - echo "$raw" - json=$(printf '%s\n' "$raw" | tail -1) - - python - "$json" <<'PY' - import json, subprocess, sys - - boards = json.loads(sys.argv[1]) - - if not boards: - sys.exit("no boards detected") - - rc = 0 - for kind, ports in sorted(boards.items()): - for port in ports: - probe = subprocess.run( - ["mpremote", "connect", port, "exec", - "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)"], - capture_output=True, text=True, timeout=60) - if probe.returncode: - print(f"FAIL {kind:12} {port:16} {probe.stderr.strip()}") - rc = 1 - else: - print(f"ok {kind:12} {port:16} {probe.stdout.strip()}") - - print(f"\n{sum(len(p) for p in boards.values())} board(s) responding") - sys.exit(rc) - PY - - - name: Summary - if: always() - run: echo "Bench smoke ${{ job.status }}" diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml new file mode 100644 index 00000000..ec9c3f11 --- /dev/null +++ b/.github/workflows/hil.yml @@ -0,0 +1,268 @@ +name: HIL + +# Everything the hardware bench does, in one workflow: recover the boards, +# flash and health-check them, then boot every board against every game config. +# +# It replaces hil-smoke.yml, hil-flash-check.yml, hil-config-matrix.yml and +# hil-recover.yml. They were separate files sharing one `hil-bench` concurrency +# group, and GitHub keeps only one *pending* run per group - so a push touching +# two of them had them race, with the loser silently cancelled. One workflow +# takes one lease on the bench and runs the stages in order. +# +# It also makes recovery automatic. `recover.py` costs ~30s when every board +# answers and repairs one that does not, so running it first turns "a board +# wedged and the next four runs were useless" into a self-healing bench. +# +# --------------------------------------------------------------------------- +# Why workflow_run, and not pull_request +# --------------------------------------------------------------------------- +# +# This runs on a self-hosted runner on a private network, wired to real +# hardware. A `pull_request` trigger would let any fork PR execute its own code +# there - its own workflow, its own dev/hil/**, its own dependencies. +# +# `workflow_run` is the primitive that makes per-PR hardware testing safe: +# GitHub always runs it *from the default branch, using the default branch's +# code*, whatever the triggering PR contains. A PR cannot change what the Pi +# executes. It supplies exactly one thing - the commit that gets built and +# flashed. See dev/hil/DESIGN.md §4. +# +# Fork PRs are not run automatically even so: the gate job stops them and says +# how to run one deliberately. Making forks automatic-with-approval needs a +# `hardware-lab` GitHub Environment with required reviewers (DESIGN.md §4); +# that is a repository setting, not something this file can create. + +on: + # Fires when "Build and Deploy" finishes, which is every push to a PR. + workflow_run: + workflows: ["Build and Deploy"] + types: [completed] + + workflow_dispatch: + inputs: + stages: + description: "Which stages to run: all, recover, flash-check, config-matrix" + type: string + default: "all" + target: + description: "Limit the config matrix to one target, e.g. wpc (blank = all attached boards)" + type: string + default: "" + configs: + description: "Comma-separated config names for the matrix instead of all of them" + type: string + default: "" + limit: + description: "Stop the matrix after this many configs per board" + type: string + default: "" + + # Temporary, so this can be validated before it reaches the default branch: + # a workflow_run workflow does nothing until it is on main. Drop this once + # merged - after that, workflow_run covers every PR. + push: + branches: + - claude/wpc-hil-config-validation-rc62dn + +permissions: + contents: read + +# One set of physical boards. Queue, never interleave. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + gate: + name: Decide whether to touch the bench + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + run_bench: ${{ steps.decide.outputs.run_bench }} + reason: ${{ steps.decide.outputs.reason }} + commit: ${{ steps.decide.outputs.commit }} + steps: + - name: Decide + id: decide + env: + EVENT: ${{ github.event_name }} + # For workflow_run these describe the run that triggered us. + BUILD_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + BUILD_EVENT: ${{ github.event.workflow_run.event }} + BUILD_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + BUILD_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + THIS_REPO: ${{ github.repository }} + run: | + run_bench=true + reason="running" + commit="${BUILD_HEAD_SHA:-$GITHUB_SHA}" + + if [ "$EVENT" = "workflow_run" ]; then + # No point flashing a build that failed. + if [ "$BUILD_CONCLUSION" != "success" ]; then + run_bench=false + reason="the build it follows concluded '$BUILD_CONCLUSION', so there is nothing good to flash" + # A fork's code must not run on the bench unattended. workflow_run + # protects *this* workflow, but the firmware being flashed is still + # the fork's, and a board runs it on a private network. + elif [ -n "$BUILD_HEAD_REPO" ] && [ "$BUILD_HEAD_REPO" != "$THIS_REPO" ]; then + run_bench=false + reason="the change comes from the fork '$BUILD_HEAD_REPO'. Review it, then run this workflow by hand (Run workflow) to put it on the bench." + fi + fi + + echo "run_bench=$run_bench" >> "$GITHUB_OUTPUT" + echo "reason=$reason" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" + + { + echo "### HIL gate" + echo + echo "- event: \`$EVENT\`" + echo "- commit under test: \`$commit\`" + echo "- bench: **$([ "$run_bench" = true ] && echo "running" || echo "skipped")** - $reason" + } >> "$GITHUB_STEP_SUMMARY" + + bench: + name: Bench + needs: gate + if: needs.gate.outputs.run_bench == 'true' + runs-on: [self-hosted, vector-hil] + + # The bench is a singleton behind `concurrency: hil-bench`, and this job + # holds that lease for as long as it runs. At 180 minutes a wedged job + # blocked every later run for three hours - which is what happened: a + # checkout hung, the runner stopped acknowledging, and the queue behind it + # went nowhere until someone noticed. + # + # So the cap is the healthy run plus headroom, not a number picked to never + # fire. A full healthy pass is ~50 minutes (recover ~30s, flash and health + # check ~4 min, the matrix ~45 for 130 configs across three boards). + # + # cancel-in-progress stays false deliberately: cancelling a live bench job + # can stop it mid-flash and leave a board half-written, which is a worse + # problem than a queue. + timeout-minutes: 90 + + steps: + # On workflow_run this checks out the DEFAULT BRANCH, not the PR - that + # is the whole security property, and it must stay that way. The board + # gets the PR's firmware because the harness builds it from the commit + # below, not because the PR's harness code runs here. + # + # Capped because this is the step that actually wedged: it sat in + # `in_progress` while the runner stopped answering, with 180 minutes of + # job budget to sit in. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + timeout-minutes: 10 + with: + fetch-depth: 0 + + - name: Check out the commit under test + timeout-minutes: 10 + if: github.event_name == 'workflow_run' + env: + COMMIT: ${{ needs.gate.outputs.commit }} + run: | + # Only the firmware sources move to the PR's commit. dev/hil/** and + # this workflow stay on the default branch, which is what makes a + # fork PR unable to change what runs here. + git fetch --quiet origin "$COMMIT" || git fetch --quiet --no-tags origin "+refs/heads/*:refs/remotes/origin/*" + git checkout --quiet "$COMMIT" -- src/ + echo "firmware sources at $COMMIT; harness from ${{ github.ref_name }}" + + - name: Prepare environment + run: | + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + + # 1. Recover. Cheap when the bench is healthy, and the difference between + # a wedged board costing one run and costing every run until someone + # notices. ~30s when every board answers; a ladder still going after + # the cap has failed in a way that needs a person, not more waiting. + - name: Recover any wedged board + timeout-minutes: 15 + id: recover + if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'recover') + continue-on-error: true + run: python dev/hil/recover.py + + # 2. Flash and health-check: DESIGN.md G1/G2. Exercises the API over both + # USB and HTTP, which the config matrix does not. ~4 min: three builds + # on the Zero 2 W, one parallel flash pass, a health check per board. + - name: Flash and health-check every board + timeout-minutes: 25 + id: flash_check + if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'flash-check') + continue-on-error: true + run: python dev/hil/flash_and_check.py + + # 3. The config matrix: DESIGN.md G3. The long pole - ~20s per config, + # 130 configs across three boards. + - name: Boot every config on every board + timeout-minutes: 70 + id: config_matrix + if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'config-matrix') + continue-on-error: true + env: + HIL_TARGET: ${{ inputs.target }} + HIL_CONFIGS: ${{ inputs.configs }} + HIL_LIMIT: ${{ inputs.limit }} + run: | + # An array, not a word-split string: `configs: "Taxi_L4, AttackMars_11"` + # is a natural thing to type, and splitting on that space hands + # argparse a stray positional and fails the run. config_matrix strips + # whitespace around the commas itself, so the value only has to + # arrive as one argument. + args=() + if [ -n "${HIL_TARGET:-}" ]; then args+=(--target "$HIL_TARGET"); fi + if [ -n "${HIL_CONFIGS:-}" ]; then args+=(--configs "$HIL_CONFIGS"); fi + if [ -n "${HIL_LIMIT:-}" ]; then args+=(--limit "$HIL_LIMIT"); fi + + python dev/hil/config_matrix.py "${args[@]}" + + # Diagnostics read from boards that are by definition misbehaving. + - name: Board serial logs on failure + timeout-minutes: 5 + if: failure() || steps.flash_check.outcome == 'failure' || steps.config_matrix.outcome == 'failure' + run: | + # `stty raw -echo` first, and both calls under `timeout`: a tty reverts + # to ECHO-on once every handle closes, so a bare `cat` feeds the board + # its own output back, and opening a tty waits for carrier, so a wedged + # board can block even stty. + for dev in /dev/ttyACM*; do + [ -e "$dev" ] || continue + echo "--- $dev" + timeout 5 stty -F "$dev" raw -echo 115200 || true + timeout 8 cat "$dev" || true + done + + - name: Verdict + if: always() + env: + RECOVER: ${{ steps.recover.outcome }} + FLASH_CHECK: ${{ steps.flash_check.outcome }} + CONFIG_MATRIX: ${{ steps.config_matrix.outcome }} + run: | + { + echo "### HIL stages" + echo + echo "| stage | result |" + echo "|---|---|" + echo "| recover | ${RECOVER:-skipped} |" + echo "| flash + health check | ${FLASH_CHECK:-skipped} |" + echo "| config matrix | ${CONFIG_MATRIX:-skipped} |" + } >> "$GITHUB_STEP_SUMMARY" + + echo "recover: ${RECOVER:-skipped}" + echo "flash-check: ${FLASH_CHECK:-skipped}" + echo "config-matrix: ${CONFIG_MATRIX:-skipped}" + + # Every stage runs even when an earlier one fails - one broken board + # should not cost the signal from the others - so the verdict is + # taken here rather than by the first failure. + failed=0 + for outcome in "$RECOVER" "$FLASH_CHECK" "$CONFIG_MATRIX"; do + [ "$outcome" = "failure" ] && failed=1 + done + exit "$failed" diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index cf0466b9..d5ba08f5 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -1,6 +1,6 @@ # Hardware-in-the-Loop (HIL) Testing — Design -**Status:** design proposal; bench bring-up and the flash/health-check harness are implemented and running +**Status:** design proposal; bench bring-up, the flash/health-check harness (G1/G2) and the config matrix (G3) are implemented and running **Scope:** a self-hosted GitHub Actions runner driving real Vector boards, safely, from a public repository. --- @@ -206,6 +206,110 @@ Tests parameterize over `manifest ∩ dev/ci/targets.json`. Adding `whitestar` l ## 7. Harness structure +### One workflow, three stages + +`.github/workflows/hil.yml` is the whole bench. It replaced four separate +workflows that shared the `hil-bench` concurrency group — and since GitHub keeps +only one *pending* run per group, a push touching two of them had them race with +the loser silently cancelled. One workflow takes one lease and runs: + +| stage | what it is | cost | cap | +|---|---|---|---| +| recover | `recover.py` — repair any wedged board | ~30s healthy | 15 min | +| flash + health check | G1/G2, API over USB *and* HTTP | ~3.5 min | 25 min | +| config matrix | G3, every config on every board | ~45 min | 70 min | + +The caps are per step, under a 90-minute cap on the job as a whole, and they +exist because this job holds the `hil-bench` lease while it runs. At the +original 180 minutes one wedged job — a hung `actions/checkout`, on a runner +that had stopped acknowledging — blocked every run behind it for three hours. +`cancel-in-progress` stays `false` regardless: cancelling a live bench job can +stop it mid-flash and leave a board half-written, which is worse than a queue. + +Every stage runs even when an earlier one fails, and the verdict is taken at the +end: one broken board should not cost the signal from the others. Running +recovery first is the point of combining them — it costs almost nothing on a +healthy bench and turns "a board wedged, so the next four runs were useless" +into a bench that repairs itself. + +### What "the bench is here" means + +Three states have to be told apart before any of the above is worth running, and +getting them confused has cost whole runs: + +- **On serial.** The normal case: a port, a chip id, a system it reports running. +- **In BOOTSEL.** No serial port at all — the ROM bootloader enumerates as USB + mass storage, so `mpremote devs` is empty while `lsusb` lists every board. + This looked exactly like an unplugged bench and was reported as "no boards + found — check the USB hub and power" for a bench that was fine. The bootrom + publishes the board's unique id as its USB serial number, which is the same id + `machine.unique_id()` returns, so these boards are identifiable through + `VECTOR_HIL_BOARD_MAP` like any other: the harness mounts the drive and writes + the pinned TrenchCoat UF2 for the mapped target, and the board rejoins the run. +- **Absent.** Not enumerated either way. This is the state that must fail the + run rather than shrink it: two boards out of three passing is a green run that + proves nothing about the third. `REQUIRED_TARGETS` names the systems the bench + is for; `VECTOR_HIL_REQUIRED_TARGETS` on the runner is how a bench that has + really lost a board says so out loud. + +Recovery is a call, not an instruction. A board that stops answering mid-run +is drained, USB-reset and power cycled by whichever harness found it, then +probed again — "run `dev/hil/recover.py`" is useless advice on a bench that is +supposed to run unattended, and the recover stage only runs once at the start +of a job. The reflash rung stays manual: it is destructive and leaves the board +on TrenchCoat's bundled firmware, so a matrix continuing past it would be +testing firmware other than the build under test. + +A board whose chip id is in none of the map's entries is never guessed at — the +id and the exact line to add are written to the job summary, because the fix is +one edit on the runner host and whoever makes it is reading the run, not the log. + +Boards are flashed concurrently. They are independent devices on independent +ports and `dev/flash.py` is one subprocess each, spending nearly all of its time +waiting on USB, so the stage costs about what one board costs instead of three. +The failure output travels with the exception rather than being printed by the +worker, so a failing board's log is still that board's log. + +**Trigger: `workflow_run` on "Build and Deploy" completion**, which is every +push to a PR. Not `pull_request`, and the distinction is the security model: +`workflow_run` workflows always run *from the default branch, using the default +branch's code*, so a PR cannot change the workflow, `dev/hil/**`, or the pinned +dependencies that the Pi executes. The bench job checks out the default branch +and then takes **only `src/`** from the commit under test, so the firmware is +the PR's and the harness is not. + +Fork PRs are gated: the `gate` job stops them with a message saying to review +and dispatch by hand. Making forks automatic-with-approval needs a +`hardware-lab` Environment with required reviewers (§4) — a repository setting, +not something the workflow can create. + +What exists today: + +``` +dev/hil/ + bench.py # shared plumbing: inventory, resolve, build, flash, + # reset, wait-for-boot, USB request, config select + flash_and_check.py # G1/G2: one flash per board, then the API health + # check over both USB and HTTP + config_matrix.py # G3: one flash per board, then every game config for + # that target in turn + setup-runner.sh # bench Pi provisioning + DESIGN.md, RUNNER_SETUP.md +``` + +`bench.py` deliberately asserts nothing about firmware behaviour — it only gets +a board into a known state and talks to it, including driving the raw REPL over +a connection the caller owns (`bench.Repl`) rather than shelling out to +`mpremote`. Assertions live in the harness that +imports it, so adding a check never means touching the plumbing. The +hardware-free parts (config discovery, selection and ordering, and each +assertion with the board faked out) are covered by +`dev/tests/test_hil_config_matrix.py`. + +The originally proposed structure, for reference — pytest-driven, with a +manifest and a board pool. Worth revisiting when a second board of the same +target arrives and sharding starts to matter: + ``` dev/hil/ bench.py # manifest load + validation @@ -300,18 +404,223 @@ That last one is the valuable one. It turns the API docs into a load-bearing art ### G3 — every config parses and boots -Full matrix on every PR, per the decision above. Per board, loop over that hardware's configs: +**Implemented** in `dev/hil/config_matrix.py`, run as a stage of `.github/workflows/hil.yml`. + +Per board: build and flash that board's target once, so the config bundle under +test is the one this checkout produces, then loop over every config in +`src//config/`: -1. Interrupt to REPL, write `gamename` into `SPI_DataStore` `configuration` record, `machine.reset()` -2. Wait for boot +1. Write `gamename` into the `SPI_DataStore` `configuration` record over the REPL, and **read it back** — see the fixed-width field note below +2. `machine.reset()`, then wait for the ready marker on the console 3. Assert: - - no `CONF00` / `CONF01` fault + - no `CONF00` / `CONF01` fault, and no `HDWR01` either — `HDWR01` puts `main.py` down the `safe_mode` path where the config is never read at all, so it is fatal here even though `flash_and_check.py` only warns about it - `/api/game/active_config` is the config we set - `/api/game/name` matches `GameInfo.GameName` **from the source JSON in the repo** — this cross-checks the on-board `config/all.jsonl.z` against the source and catches build-time config-packing bugs, not just parse errors - `/api/leaders` and `/api/adjustments/status` both return 200 — proves the parsed definition is *usable*, not merely loadable - - free memory after load is above a floor -That last assertion is the one that earns its keep. `sys11_tiny` exists because RAM is tight; a config that parses fine but leaves too little heap is the failure that actually reaches customers. +Each board's bundle is also compared against the source directory once, before +the loop: same set of config names, same game names. That localises a packing +bug to one boot instead of one boot per affected config. + +**Why the game name is the load-bearing assertion.** A config that fails to +apply does not fault or crash from the outside: `GameDefsLoad.go` falls back to +`safe_defaults` and the board serves a generic definition for its hardware, +healthy in every other respect. `/api/game/active_config` does not catch this — +it reads the `gamename` field back out of FRAM, not what actually loaded. The +game name is what separates "loaded my config" from "silently fell back". + +#### Bench results, first full run + +sys11 passed all 39 of its configs, at 20–22s each (~14 min). That is the +harness working end to end: set the config, reboot, and confirm the board comes +up reporting that game. + +WPC did not, and the reason is worth recording because it shaped the design. +The board wedged — silent console, no reply to Ctrl-C — on the first handoff +from our serial connection to `mpremote`, and stayed wedged for the remaining +62 configs. The Pico has a single CDC endpoint, so using `mpremote` mid-run +meant closing our connection, letting a second process open the port, and +reopening afterwards, once per config. sys11 (MicroPython v1.24.1) survived +that 39 times; WPC (v1.26.0-preview) did not survive it once. A running board +printing into a CDC endpoint that nothing is draining is the difference between +them. + +So the harness now **never hands the port to another process and never leaves +it unread while the board is running.** The REPL is driven directly over the +connection the harness already holds (`bench.Repl`), and the port is closed +only while the board is mid-reset. Three further changes came out of the same +run: + +- **Two consecutive setup failures abandon the board.** The first run spent 63 + minutes writing the same 60s timeout 63 times. A board that stops answering + does not start again on its own. +- **Teardown cannot fail the run.** A `TimeoutExpired` escaped `restore_default` + and took the whole process down with a traceback — losing the summary and + skipping data_east, which had never been touched. +- **`stty raw -echo` before dumping a console.** A tty reverts to ECHO-on once + every handle closes, so the `cat /dev/ttyACM*` diagnostic step was echoing + each board's output back into it; the logs show all three boards parsing + their own log lines as USB API requests. Fixed in both HIL workflows. + +#### Why boards are flashed one at a time + +Flashing every board up front and then working through them in turn is the +obvious ordering, and it is wrong. A board flashed first and used last spends +however long the boards ahead of it take — half an hour on a full run — running +the application and printing `SCORE:` / `RESOURCE:` / `DISCOVERY:` lines into a +USB CDC console that nothing is draining. TrenchCoat documents where that ends +(`src/ray.py`, `send_command`): + +> if nothing ever drains the board's output, the USB CDC buffers fill up, +> MicroPython blocks writing to stdout, and the board deadlocks mid-script + +That is not hypothetical. It killed `data_east` 36 minutes into a run — the +board simply stopped answering `mpremote` — and it is the most likely +explanation for the WPC board that went silent for an hour on the very first +bench run and needed recovering. + +So each board is flashed immediately before its own matrix. Boards waiting +their turn sit at the REPL, where inventory's probe leaves them, producing no +output at all. Builds still happen up front; they touch no hardware. + +As a second line, a board that will not take a reset gets its console drained +and one retry: reading is the remedy for exactly this deadlock and costs +seconds, against writing off a whole board's matrix. + +#### Draining reads until the board goes quiet, not for a fixed few seconds + +A blocked board does not hand its backlog over in one gulp. It unblocks, runs +a little further, prints some more, and only then falls silent — so a drain +with a fixed three-second budget catches the first mouthful and declares the +board dead. That is measured, not argued: one bench run wrote off +`/dev/ttyACM0` as unrecoverable after draining 63 bytes from it, and the very +next step in the same job read the port with `cat` for eight seconds and got a +healthy, running web server. + +So `bench.read_until_quiet` reads until the port has been silent for a couple +of seconds, capped at a budget. A port with nothing to say costs the quiet +period; one with a backlog gets as long as it needs. The Ctrl-C comes *after* +the drain, because a firmware blocked writing to stdout is not reading stdin +either — the host's OUT endpoint backs up too, so writing first only times out +against the wedge it is trying to clear. A timed-out interrupt is likewise a +reason to read more and try again, not a reason to give up. + +This matters more than it sounds: on the current runner the drain is the only +recovery rung that can always be reached. The USB reset needs a udev rule and +the power cycle needs `uhubctl`, and neither is installed (see +RUNNER_SETUP.md), so everything above the drain escalates straight to a flash +wipe. + +What the two directions did is then the diagnosis, and they point at different +remedies. Bytes read but the Ctrl-C refused means the board is *talking but not +listening*: producing output normally and never servicing what we send it, so +no amount of reading reaches it — that is what the USB reset and power cycle +rungs are for, and with both unavailable there is genuinely nothing left to +try. Nothing read at all is the opposite: not a board blocked on a full output +buffer, because such a board has a backlog to give up the moment somebody +reads. Both are reported in those words rather than as "still dead", which sent +a maintainer looking for a bricked Pico when the board in question was running +the application and printing to its console the whole time. + +#### A lost connection is repaired, not treated as a verdict + +If a board's boot fails outright, the session is left holding no connection, +and without care every config after it fails instantly with "the board is not +connected" — including the retry that exists precisely to survive a flaky +board. The bench showed the exact shape: the WPC board crashed on boot twice +for one config, and from there the retry failed in no time at all and the next +config failed in 0.0s, two configs blamed on a board that came back on its own +moments later and served the restore step happily. + +So `Session.ensure_connected()` runs before each attempt: if there is no live +connection it drains, resets, and watches the board come up. A board that is +genuinely gone still ends its own matrix through the consecutive-failure +counter; this only stops that happening while the board is still recoverable. + +#### One reset per boot + +`dev/flash.py` ends by resetting the board, so a freshly flashed board is +already booting. Resetting it again lands on top of that boot, while the +firmware is still reading a filesystem written seconds ago — and sys11 wedged +in exactly that window: flashed successfully, then refusing a reset and any +console traffic seconds later. `Session.start(reset=False)` watches the boot +that flashing started instead of forcing another. It is the same shape as the +`ENOENT`-on-a-file-that-exists crashes WPC raises, and suggests both are the +firmware being disturbed while it reads freshly written flash. + +#### Telling a broken config from a flaky board + +They produce the identical symptom on one attempt and need opposite responses, +so every failing config is retried once. Fails twice → the config, and the run +goes red. Passes on the retry → the board, recorded as a **flake** against that +board and reported in the summary, without blaming the config. + +This is not hypothetical either. The WPC board raises `ENOENT` mid-boot on files +that plainly exist; a board that will do that to an import will do it to a route, +and a run where `Congo_21` failed `/api/leaders` with a 500 and `Theatre_13` +failed `/api/adjustments/status` the same way — on a board that also crashed +once on boot in the same run — is exactly the ambiguity this resolves. Without +the retry those read as two broken configs, which would be the wrong conclusion +and would erode trust in the check. + +#### How a boot is watched + +The ready marker (`Server: Loop Forever`) is printed exactly once per boot, which +makes watching for it precise but brittle in two directions. Both are handled: + +- **The board crashes instead of coming up.** MicroPython prints a traceback and + drops to the REPL; nothing will ever print the marker. The watcher recognises + the REPL banner and fails immediately *with the traceback*, rather than burning + the timeout and reporting "never reported its web server" — which is what + happened on the first full bench run and buried the real cause. A crash is + retried once, because it can be intermittent and losing a whole board's matrix + to one flaky boot buys nothing; every crash is counted and reported in the run + summary either way, so a board that only sometimes boots never reads as clean. +- **The marker goes past before we are watching.** Any boot we did not trigger + ourselves looks identical to a board that never came up. On timeout the watcher + asks the board over the USB API before failing: a board that answers is up, + whatever we did or did not see, and says so as a warning. + +Timeouts are set from measurement, not guesswork. Across 70 boots on the bench: +**11.8s minimum, 15.5s mean, 25.6s maximum.** The matrix allows 90s and the flash +harness 150s, so neither is close to marginal — a boot that exceeds them has +stopped, not slowed. + +#### Findings and limits + +- **~~Two WPC configs cannot be selected at all.~~ Fixed in #380.** The bench's + first real finding: `configuration.gamename` is a 16-byte fixed-width field + and `struct.pack` truncates silently, so `HarleyDavidson_L3` and + `GilliganIsland_L9` at 17 characters were offered by the web UI, accepted on + write, truncated into FRAM, and matched nothing on the next boot — `CONF01`, + safe defaults, and a game that could never be selected. Shortened to + `HarleyDavid_L3` and `GilliganIsle_L9`. `test_no_config_name_exceeds_the_gamename_field` + now enforces the rule absolutely, with no allowlist. +- **WPC boots intermittently fail on a missing file.** Four crashes in one + 63-config run, in two flavours — `ImportError: no module named 'origin'` and + `OSError: [Errno 2] ENOENT` — both raised from `phew/server.py:create_schedule` + during boot. Both are "file not found", on files that plainly exist: the same + board boots fine on the retry every time. That points at the on-board + filesystem returning ENOENT under some condition, not at the config bundle and + not (as first guessed) at memory pressure. A `nuke.uf2` wipe before reflashing, + which is what `recover.py`'s reflash rung does, is the obvious thing to try + next: if the littlefs on that board is degraded, it would clear it. +- **`/api/adjustments/status` 500s for configs with no `Adjustments` section.** + `GameDefsLoad` assigns the parsed config straight to `SharedState.gdata` + without merging `safe_defaults` into it, so the key is simply absent and + `Adjustments._get_range_from_gamedef` raises `KeyError`. 14 shipped configs + are affected. Warned about rather than failed, so that a firmware gap the + harness cannot fix does not bury the signal it exists for. +- **Configs that share a `GameName` are not distinguished from each other.** + The four `AddamsFam_*` variants all report "Addams Family", and no route + exposes anything else from `gdata`. For those, a pass means "this config + parsed and loaded without faulting", not "this exact ROM revision's + definition is in memory". +- **The free-memory floor is not implemented.** No route reports heap, and + reading `gc.mem_free()` over the REPL means interrupting the running + firmware, which ends the boot being measured. `sys11_tiny` exists because RAM + is tight, so this assertion is still worth having — it needs a small + firmware-side route first. **Throughput.** Measured on the bench, per board, from the flash/health-check harness: diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 34cf6149..62db3f40 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -118,34 +118,71 @@ hence the explicit `.venv` path here.) Then record what you saw: ```bash +# the value is the whole map - list every board, including ones already correct +sed -i '/^VECTOR_HIL_BOARD_MAP=/d' ~/actions-runner/.env echo 'VECTOR_HIL_BOARD_MAP==sys11,=wpc,=data_east' >> ~/actions-runner/.env cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start ``` +Valid targets are the ones with game configs to boot against: `sys11`, `wpc`, `data_east`, `em`. +Use the directory name, not the name people say out loud — the Data East board is `data_east`, +not `de` (the harness suggests the right one if you get it wrong, and refuses the run rather +than flashing anything). +The tree also builds `classic` and `whitestar`, and the bench **cannot** drive those — they have a +`systemConfig.py` and no `config/` directory, so there is nothing to flash and boot. A board wired +for one of them cannot be mapped at all until that target has configs; mapping it to a different +system would flash the wrong firmware to real hardware, so the harness refuses it by name. The map must cover +*every* board the harness sees - adding a board to the bench means adding it here, or the run +stops with `VECTOR_HIL_BOARD_MAP is set but does not cover: ...`. That failure (and the other +map-related ones) prints these same instructions, pre-filled with the bench's actual chip ids. + With the map set, `flash_and_check.py` uses it and ignores self-report entirely. Without it, the harness falls back to self-report but **refuses to flash when two boards claim the same system**, since that means at least one is running firmware for a system it isn't wired for. `--inventory-only` prints the chip ids without blinking or flashing anything. +A board in BOOTSEL is covered by the same map. Its ROM bootloader publishes the board's unique +id as the USB serial number, which is the id `machine.unique_id()` reports once firmware is +running, so a board can be recognised and put back without ever being a serial device. If one +ever turns up under two different ids, map both — a spare entry costs nothing. + +## Every system, every run + +The bench exists to cover `sys11`, `wpc` and `data_east`, and a run that quietly tests two of +them still reports green about the third. So a missing board fails the run: the boards that are +present are still flashed and checked, and the verdict at the end says which system was absent. + +If the bench has genuinely lost a board for a while, say so deliberately rather than letting the +check rot: + +```bash +echo 'VECTOR_HIL_REQUIRED_TARGETS=sys11,wpc' >> ~/actions-runner/.env +cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start +``` + +Unset, it defaults to all three. Set to an empty value, the check is off entirely. + ## Verify The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` label. -End to end, [`.github/workflows/hil-smoke.yml`](../../.github/workflows/hil-smoke.yml) checks -that the runner picks up jobs, the bench environment reaches them, the serial devices are -present and accessible, and every detected board answers over `mpremote`. It fails if no -boards are found or if any detected board doesn't respond. +End to end, dispatch [`.github/workflows/hil.yml`](../../.github/workflows/hil.yml) with +**stages: `recover`**. That is the cheapest full-path check there is — it proves the runner +picks up jobs and the bench environment reaches them, then prints which recovery rungs this +host can actually use and surveys every board by chip id. It takes under a minute on a healthy +bench and repairs a wedged board on an unhealthy one. -`VECTOR_HIL_VENV` and `VECTOR_HIL_REPO` are exported into every job from `.env`, so workflows -don't hardcode paths. The smoke job deliberately does no `actions/checkout` — under the design's -trust model the bench runs harness code from the clone it already has, not from a PR. +`VECTOR_HIL_VENV` and the `VECTOR_HIL_*` variables are exported into every job from `.env`, so +workflows don't hardcode paths. A missing one fails at the point of use with a message naming +it rather than being pre-flighted separately. ### Running it before this PR merges -A `workflow_dispatch` workflow isn't dispatchable until it exists on the **default branch**, so -the "Run workflow" button won't appear while this is still a PR. The workflow therefore also -triggers on pushes to this branch — push anything to it and the job runs on the bench: +Neither `workflow_dispatch` nor `workflow_run` does anything until the workflow is on the +**default branch** — the Run workflow button doesn't appear, and `workflow_run` only ever runs +default-branch code. The workflow therefore also triggers on pushes to this branch, purely so it +can be validated pre-merge; push anything to it and the job runs on the bench: ```bash git commit --allow-empty -m "trigger hil smoke" && git push @@ -176,4 +213,111 @@ Once this is on `main`, drop the `push:` trigger and use the Run workflow button by querying `systemConfig.vectorSystem`, so port order doesn't matter for distinct board types. Only add udev rules if you have two of the same type. +## Running the harnesses by hand + +Both take the bench venv on `PATH` — the runner's `.env` provides it inside a job, but a +login shell does not read it: + +```bash +cd ~/vector && export PATH="$PWD/.venv/bin:$PATH" + +# flash every board and health-check its API (G1/G2) +.venv/bin/python dev/hil/flash_and_check.py + +# boot every board against every config it can be flashed with (G3) +.venv/bin/python dev/hil/config_matrix.py + +# ...or just the WPC board, first five configs, no reflash +.venv/bin/python dev/hil/config_matrix.py --target wpc --limit 5 --skip-flash +``` + +A full config matrix is roughly 20–22s per config per board (measured on the bench) and WPC +alone has 63, so budget well over half an hour for an unfiltered run. `--configs`, `--limit` and `--target` are there +to keep an iteration loop short; `--changed-since REF` runs the configs a branch touched +first, which is what the workflow does on a push. + +The matrix leaves each board on its generic config when it finishes, including after a +failure, so a run never strands a board on a game config you did not ask for. If a board +stops answering it is abandoned after two consecutive setup failures and the run moves on to +the next board, rather than timing out against a board that is not coming back. + +**Never `cat /dev/ttyACM*` to watch a board.** A tty reverts to ECHO-on once every handle is +closed, so a bare `cat` makes the kernel echo the board's own output back into it, and the +board then tries to parse its log lines as USB API requests. Use `stty -F /dev/ttyACM0 raw +-echo 115200` first, or `mpremote connect /dev/ttyACM0 repl`. + +## Recovering a wedged board + +A board can deadlock with its USB device still enumerated and the firmware gone: the port is +there, `mpremote` opens it, nothing answers. `dev/hil/recover.py` escalates through four +rungs and stops as soon as the board replies. Run it from +the `recover` stage of [`hil.yml`](../../.github/workflows/hil.yml), which runs first on every bench job, or by hand: + +```bash +cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/recover.py +``` + +The ladder is also a library, not just a script. Every harness runs the **cheap rungs +automatically** against a board that stops answering — inventory drains, USB-resets and power +cycles it, then probes again — so a board that wedges *after* the recover stage no longer costs +the rest of the run. The reflash rung is deliberately **not** automatic: it is destructive, and +it leaves the board running TrenchCoat's bundled firmware rather than the build under test, so a +matrix that carried on afterwards would be testing the wrong thing and reporting it as if it +were not. That one stays a decision for a person running `recover.py` by hand — or for the +recover stage itself, where `flash_and_check.py` re-flashes everything immediately afterwards. + +Before the ladder it deals with the one state none of the rungs can reach: a board in +**BOOTSEL/UF2 mode**. Such a board is not a serial device at all — the ROM bootloader +enumerates as USB mass storage, so `mpremote devs` shows nothing and `lsusb` shows everything, +which is exactly how a healthy bench came to report "no boards found". Every harness now looks +for those boards, identifies each one by the id its bootloader publishes, mounts its drive and +writes the TrenchCoat UF2 for the target the map gives it — after which the board is an ordinary +serial device again and the run carries on. A board the map does not cover is left alone, with +its id and the line to add printed to the job summary: there is no way to guess which system's +firmware it wants, and writing the wrong one is worse than leaving it. + +It then opens with a report of which rungs this runner can actually use. As measured on the bench +on 2026-08-28: + +| rung | state | what it needs | +|---|---|---| +| drain the console | works | serial access, which the `dialout` group already gives | +| reset the USB device | works | write access to `/dev/bus/usb/*` — the udev rule below | +| power cycle the hub port | **unavailable** | a hub that switches port power; this bench's is `ganged`, so it cannot | +| reflash via TrenchCoat | works | `udisksctl` to mount the drive, or `picotool` when there isn't one | + +`picotool` is the fallback that matters when a board comes back wrong. Copying a UF2 needs the +board's mass storage to work, and a board power-cycled mid-flash can present a drive nothing +will mount — one on this bench came back with a DOS partition table where a bootloader drive +should have a bare filesystem. `picotool` speaks the bootrom's own USB protocol and needs no +drive at all, so `sudo apt install picotool` is worth doing before you need it. It is always +aimed at a single board by USB bus and address, never at whatever it finds first. + +The two unavailable rungs are worth enabling — they are the non-destructive ones, and without +them a wedged board goes straight from "send it a Ctrl-C" to "wipe its flash". For the USB +reset, a rule like this grants the runner user write access to the boards' USB nodes: + +``` +# /etc/udev/rules.d/60-vector-hil.rules +SUBSYSTEM=="usb", ATTR{idVendor}=="2e8a", MODE="0660", GROUP="dialout" +``` + +then `sudo udevadm control --reload && sudo udevadm trigger`. Note the boards are USB bus +powered, so a hub with per-port power switching makes the power rung a genuine cold boot — +the most reliable recovery short of a reflash. + +> [!IMPORTANT] +> As measured, **a badly wedged board cannot be recovered from software on this +> runner.** The drain reads nothing, the USB reset and power cycle are both +> unavailable for want of the setup above, and a board too far gone to run one +> statement cannot be talked into its ROM bootloader either — TrenchCoat's route +> in goes over the REPL, and the 1200 baud touch fell to `[Errno 110]`. That +> board needs someone to unplug it and plug it back in. Enabling the two rungs +> above is what would change that. + +The reflash rung hands the board to [TrenchCoat](https://github.com/warped-pinball/trench-coat), +pinned by commit in `dev/hil/trench_coat.py`, which resets into the ROM bootloader, wipes the +flash with `nuke.uf2` and writes the real firmware. It is destructive: run +`dev/hil/flash_and_check.py` afterwards to put Vector back on the board. + See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. diff --git a/dev/hil/bench.py b/dev/hil/bench.py new file mode 100644 index 00000000..e33e1613 --- /dev/null +++ b/dev/hil/bench.py @@ -0,0 +1,1592 @@ +#!/usr/bin/env python3 +"""Shared bench plumbing for the hardware-in-the-loop harnesses. + +Everything here is about *getting a board into a known state and talking to +it*: enumerating what is attached, deciding what each board should be flashed +with, building and flashing it, and waiting for the firmware to come up on the +other side of a reset. + +The assertions live in the harnesses that import this: + + flash_and_check.py - one flash per board, then a broad API health check + config_matrix.py - one flash per board, then every game config in turn + +Nothing in here asserts anything about firmware behaviour, so a change to what +a harness checks does not belong in this file. +""" + +import concurrent.futures +import json +import os +import re +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "dev")) + +import serial # noqa: E402 (ships with mpremote) +from usb_coms_demo import UsbApiClient # noqa: E402,F401 (re-exported by this module) + +# A bare bench board has nothing driving the game bus, so this one is correct +# and expected rather than a regression. +EXPECTED_FAULTS = {"HDWR02"} + +# Floating data lines can trip the >250-transition check in main.py and send +# the board down the safe_mode path, where the game config is never loaded. +# Warned about rather than failed, but it invalidates the config assertions - +# see DESIGN.md §8 (G1). +BENCH_WARN_FAULTS = {"HDWR01"} + +# Config key is the config filename without .json (dev/build.py:253). +DEFAULT_GAMENAME = { + "sys11": "GenericSystem11_", + "wpc": "Generic_WPC", + "data_east": "GenericDE_", + "em": "EM_machine_", +} + +# Boot is slow and variable, so we watch the console for the firmware saying +# it is ready rather than guessing at a delay. +# +# "Server: Loop Forever" is the correct marker and the only one: phew prints +# it immediately before loop.run_forever() (phew/server.py:381). The earlier +# "> starting web server on port 80" line is NOT a ready signal - it is +# printed before start_server is even scheduled, let alone bound, so matching +# it returns while the socket is still closed. backend.go() has already run +# connect_to_wifi() by this point, so the marker covers both transports. +READY_MARKER = "Server: Loop Forever" + +# The other way a boot can end. If the application raises, MicroPython prints a +# traceback and drops to the REPL - and then nothing is ever going to print the +# ready marker, so waiting out the timeout only delays a failure we can already +# describe. These lines appear when, and only when, the program has exited: +# main.py never returns on a healthy board. +# +# Watching for this is what turns "never reported its web server within 90s" +# into the traceback that actually explains the boot. +CRASH_MARKERS = ('Type "help()" for more information.', ">>>") + +# Ignore a crash marker in the first moments after opening the port: it can be +# residue from whatever REPL session issued the reset, rather than this boot. +CRASH_MARKER_GRACE = 3 + +# The marker is printed just *before* run_forever(), so give the event loop a +# moment to actually accept the listening socket. http_get's retries cover any +# remainder. +SERVER_SETTLE_SECONDS = 2 + +# A repair mid-run is bounded well below the workflow's per-stage caps: the +# rungs it can reach are a drain, a USB reset and a power cycle, none of which +# is slow. The 600s in recover.py exists for the reflash, which this never runs. +REPAIR_TIMEOUT = 180 + +# Measured across 70 boots on the bench: 11.8s min, 15.5s mean, 25.6s max. The +# flash harness keeps 150s; the matrix runs a tighter one (see +# config_matrix.MATRIX_BOOT_TIMEOUT) because it pays the timeout per config. +BOOT_TIMEOUT = 150 +HTTP_TIMEOUT = 10 + + +class CheckFailure(Exception): + pass + + +class BootCrash(CheckFailure): + """The firmware raised on the way up and dropped to the REPL. + + Distinct from a timeout because it is actionable in a way a timeout is + not: the console holds the traceback, and a retry is worth one attempt + where waiting longer is worth nothing. + """ + + def __init__(self, message, transcript=None): + super().__init__(message) + self.transcript = transcript or [] + + +def log(msg): + print(msg, flush=True) + + +def _append_summary(line): + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + try: + with open(path, "a") as handle: + handle.write(line.replace("::warning::", "WARNING: ").replace("::error::", "ERROR: ") + "\n") + except OSError: + pass + + +def summary(line): + """Put a line in the Actions job summary as well as the log. + + Job summaries are stored separately from the log archive, and two bench + runs proved why that matters: the runner died mid-job, never uploaded its + logs, and every finding went with them. Written incrementally rather than + at the end for the same reason. + """ + log(line) + _append_summary(line) + + +def summary_once(marker, lines): + """Log `lines`, and add them to the summary unless `marker` is there already. + + Three stages share one summary page and reach the same conclusion about + the bench - "this board is not in the map", "this system is missing" - so + without this the reader gets the same block three times, which reads like + three problems. The log still says it every time, where repetition is + exactly what you want: each step's failure explains itself. + """ + for line in lines: + log(line) + path = os.environ.get("GITHUB_STEP_SUMMARY") + if path: + try: + if marker in Path(path).read_text(): + return + except OSError: + pass + for line in lines: + _append_summary(line) + + +def as_block(text): + """A multi-line string as preformatted summary lines.""" + return ["", "```", *text.splitlines(), "```"] + + +_TIMINGS = [] +_group = None + + +def group(title): + global _group + _group = (title, time.monotonic()) + print(f"::group::{title}", flush=True) + + +def endgroup(): + global _group + if _group: + title, started = _group + elapsed = time.monotonic() - started + _TIMINGS.append((title, elapsed)) + print(f" [{elapsed:.1f}s]", flush=True) + _group = None + print("::endgroup::", flush=True) + + +# -------------------------------------------------------------------------- +# 1. inventory +# -------------------------------------------------------------------------- + + +VENV_PYTHON = sys.executable + + +def ensure_tools_on_path(): + """Put the bench venv's bin dir on PATH and pick the interpreter to use. + + Sets VENV_PYTHON, which build() and flash() shell out to, and returns it. + + The harness gets run three ways - from an Actions job, from a login shell, + and by hand - and only the first has the runner's .env applied. dev/build.py + shells out to a bare `mpy-cross` and dev/flash.py to a bare `mpremote`, so + PATH has to be right for subprocesses too, not just for our own calls. + """ + candidates = [] + if os.environ.get("VECTOR_HIL_VENV"): + candidates.append(Path(os.environ["VECTOR_HIL_VENV"]) / "bin") + candidates.append(Path(sys.executable).parent) + candidates.append(REPO_ROOT / ".venv" / "bin") + + global VENV_PYTHON + + for bindir in candidates: + if (bindir / "mpremote").exists(): + os.environ["PATH"] = f"{bindir}{os.pathsep}{os.environ.get('PATH', '')}" + python = bindir / "python" + VENV_PYTHON = str(python) if python.exists() else sys.executable + return VENV_PYTHON + + if shutil.which("mpremote"): + VENV_PYTHON = sys.executable + return VENV_PYTHON + + raise CheckFailure( + "mpremote not found. Run with the bench venv, e.g.\n" + f' cd {REPO_ROOT} && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/.py ...\n' + "(VECTOR_HIL_VENV is exported by the runner service, so it is not set in a login shell.)" + ) + + +def mpremote(*args, timeout=60): + return subprocess.run(["mpremote", *args], capture_output=True, text=True, timeout=timeout) + + +def list_ports(): + result = mpremote("devs", timeout=30) + if result.returncode != 0: + raise CheckFailure(f"`mpremote devs` failed: {result.stderr.strip()}") + return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] + + +# -------------------------------------------------------------------------- +# Boards in the ROM bootloader +# -------------------------------------------------------------------------- +# +# A board in BOOTSEL/UF2 mode is not a serial device at all: the RP2040 ROM +# bootloader enumerates as USB mass storage, so `mpremote devs` shows nothing +# and every serial-based question about it - chip id, running system, is it +# alive - has no answer. A whole bench in that state reported "no boards found +# - check the USB hub and power" while `lsusb` listed all three, which is the +# most misleading thing the harness has ever said: the boards were fine, and +# one UF2 each away from working. +# +# They can still be identified. The bootrom publishes the board's unique id as +# the USB serial number, which is the same id `machine.unique_id()` returns +# once MicroPython is running, so VECTOR_HIL_BOARD_MAP covers a board in this +# state exactly as it covers a running one. + +BOOTSEL_VID = "2e8a" # Raspberry Pi +BOOTSEL_PIDS = {"0003": "RP2040", "000f": "RP2350"} # "RP2 Boot" +RUNNING_PID = "0005" # MicroPython "Board in FS mode" - a normal, working board + +USB_DEVICES = Path("/sys/bus/usb/devices") + + +def bootsel_boards(): + """Boards sitting in the ROM bootloader, from sysfs. + + Read straight out of sysfs rather than by shelling out to lsusb: it needs + no privileges, no package, and it hands us the device directory, which is + what finds the board's mass-storage drive later. + """ + boards = [] + for device in sorted(USB_DEVICES.glob("*")): + try: + vendor = (device / "idVendor").read_text().strip().lower() + product = (device / "idProduct").read_text().strip().lower() + except OSError: + # Interfaces (1-1:1.0) and the like have no ids. Not devices. + continue + if vendor != BOOTSEL_VID or product not in BOOTSEL_PIDS: + continue + try: + chip_id = (device / "serial").read_text().strip().lower() or None + except OSError: + chip_id = None + boards.append( + { + "port": None, + "chip_id": chip_id, + "system": None, + "version": None, + "responsive": False, + "bootsel": True, + "processor": BOOTSEL_PIDS[product], + "usb_device": device, + } + ) + return boards + + +CHIP_ID_SNIPPET = "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())" + + +def _ask_board(port, snippet, timeout=30): + """Run a snippet on a board, treating a hung board as an answer of its own. + + A wedged board makes mpremote hang until its timeout rather than fail, and + letting that propagate means one bad board aborts the whole run before + anything has been tested. It did exactly that: a board left deadlocked by + an earlier run took out the next run in `inventory`, before a single config + was checked. + + One drain and one retry, because a board blocked writing to an undrained + USB endpoint comes back the moment somebody reads it. + """ + try: + return mpremote("connect", port, "exec", snippet, timeout=timeout) + except subprocess.TimeoutExpired: + log(f" {port} did not answer in {timeout}s; draining its console and retrying once") + drain_port(port) + + try: + return mpremote("connect", port, "exec", snippet, timeout=timeout) + except subprocess.TimeoutExpired: + return None + + +def probe(port): + """Return {port, chip_id, system, version, responsive} for one board. + + chip_id comes from the RP2040 itself so it survives any firmware state; + system/version come from the flashed firmware and may be missing if the + board is unflashed or broken. `responsive` is False for a board that never + answered at all - the caller decides what to do about it, but the survey + itself always completes. + """ + board = {"port": port, "chip_id": None, "system": None, "version": None, "responsive": True} + + chip = _ask_board(port, CHIP_ID_SNIPPET) + if chip is None: + board["responsive"] = False + return board + if chip.returncode == 0: + board["chip_id"] = chip.stdout.strip() + + info = _ask_board(port, "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)") + if info is not None and info.returncode == 0 and info.stdout.strip(): + parts = info.stdout.split() + board["system"] = parts[0] + if len(parts) > 1: + board["version"] = parts[1] + + return board + + +def repair_board(port, target=None, allow_reflash=False, step_timeout=REPAIR_TIMEOUT): + """Run the recovery ladder against one board, in-process. + + `recover.py` is a harness, not just a script, so a board that stops + answering mid-run is repaired by calling it rather than by printing "run + dev/hil/recover.py" at somebody who is not there. The bench is meant to + run unattended, and the recover *stage* only runs once at the start of the + job - a board that wedges after it used to cost the rest of the run. + + Reflashing is off by default, and that is the load-bearing part: it is the + one destructive rung, and it leaves the board running TrenchCoat's bundled + firmware rather than the build under test. Harmless in the recover stage, + where flash_and_check re-flashes everything straight afterwards; wrong in + the middle of a matrix, where the run would silently continue against + firmware that is not the one being tested. + + Imported here rather than at the top because recover imports this module, + and its directory is added to sys.path here rather than assumed: every + harness happens to add it today, so relying on that would leave a trap for + the first caller that does not. + """ + here = str(Path(__file__).resolve().parent) + if here not in sys.path: + sys.path.insert(0, here) + + import recover + + options = SimpleNamespace( + reflash=allow_reflash, + no_power_cycle=False, + force_bootsel=False, + step_timeout=step_timeout, + cache_dir=REPO_ROOT / "build" / "hil", + ) + return recover.recover(port, target, options) + + +def repair_unresponsive(boards, board_map=None): + """Give every board that did not answer one pass of the cheap rungs.""" + dead = [b for b in boards if not b.get("responsive", True)] + if not dead: + return boards + + group(f"Repair {len(dead)} board(s) that did not answer") + for b in dead: + # A wedged board cannot tell us its chip id, so there is no target to + # look up - which is fine, because the rungs that need one are the + # rungs this deliberately does not run. + method = repair_board(b["port"], (board_map or {}).get(b.get("chip_id"))) + if not method: + log(f"::warning::{b['port']} is still not answering after the cheap recovery rungs") + continue + log(f"{b['port']} came back after {method} - probing it again") + b.update(probe(b["port"])) + endgroup() + return boards + + +def inventory(board_map=None, repair=True): + """Survey the bench: every board on serial, plus any stuck in BOOTSEL. + + Only the serial boards are returned - a board in the bootloader cannot be + flashed by the normal route and has to be rescued first (see + trench_coat.rescue_bootsel) - but they are always listed, because "no + boards found" is a wrong and expensive answer when three of them are + sitting there as mass-storage devices. + """ + boards = [probe(port) for port in list_ports()] + if repair: + boards = repair_unresponsive(boards, board_map) + stranded = bootsel_boards() + + log(f"{'port':16} {'chip id':18} {'running':12} version") + for b in boards: + if not b.get("responsive", True): + log(f"{b['port']:16} {'NOT ANSWERING':18} {'-':12} -") + continue + log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") + for b in stranded: + log(f"{'(BOOTSEL)':16} {b['chip_id'] or '?':18} {'bootloader':12} {b['processor']} ROM") + + if not boards: + raise CheckFailure(no_boards_message(stranded)) + if stranded: + log(f"::warning::{len(stranded)} board(s) are in BOOTSEL/UF2 mode and were not surveyed - the rescue stage puts firmware back on them, and reports here when it cannot") + return boards + + +def usb_devices(): + """Every USB device sysfs knows about - the harness's own `lsusb`. + + Written out whenever the bench looks empty, because "no boards found" and + "lsusb shows all three" is a contradiction the log has to be able to + settle: it says whether the boards are absent from the bus, present under + an id this harness does not know, or present with no driver bound. + """ + devices = [] + for device in sorted(USB_DEVICES.glob("*")): + try: + vendor = (device / "idVendor").read_text().strip().lower() + product = (device / "idProduct").read_text().strip().lower() + except OSError: + continue + if not vendor or not product: + continue + + def attribute(name): + try: + return (device / name).read_text().strip() + except OSError: + return "" + + devices.append( + { + "path": device.name, + "id": f"{vendor}:{product}", + "name": " ".join(filter(None, (attribute("manufacturer"), attribute("product")))) or "(no product string)", + "serial": attribute("serial"), + } + ) + return devices + + +def usb_bus_report(): + """The USB bus as sysfs sees it, for a log that has to explain an empty bench.""" + devices = usb_devices() + if not devices: + return [ + "Nothing at all is on the USB bus (" + str(USB_DEVICES) + " lists no devices),", + "which is a host-side answer rather than a board-side one: the hub is unplugged,", + "unpowered, or this process cannot read sysfs.", + ] + + lines = ["What is on the USB bus, as sysfs sees it:", ""] + for device in devices: + lines.append(f" {device['path']:12} {device['id']:10} {device['name']}" + (f" serial {device['serial']}" if device["serial"] else "")) + + # Say what each Raspberry Pi device *is*, rather than lumping them together. + # The first version called every 2e8a device unrecognised, which produced a + # flatly self-contradictory report on the bench: it listed a 2e8a:000f + # bootloader and then said none of the devices was "a bootloader this + # harness recognises (2e8a:0003, 2e8a:000f)". + running = [d for d in devices if d["id"] == f"{BOOTSEL_VID}:{RUNNING_PID}"] + bootloaders = [d for d in devices if d["id"].split(":")[1] in BOOTSEL_PIDS and d["id"].startswith(BOOTSEL_VID + ":")] + strangers = [d for d in devices if d["id"].startswith(BOOTSEL_VID + ":") and d not in running and d not in bootloaders] + + if running or bootloaders or strangers: + lines.append("") + lines.append("Raspberry Pi devices on the bus:") + if running: + lines.append(f" {len(running)} running MicroPython - these are the boards that appear as serial ports") + for device in bootloaders: + lines.append(f" 1 in the ROM bootloader: {device['serial'] or '?'} ({device['id']})") + if bootloaders: + lines += [ + " A board here is the rescue stage's job, not a missing board: it is attached and", + " one UF2 away from working. If it is still in this state, the rescue ran and could", + " not finish - its output above says where it stopped.", + ] + for device in strangers: + lines.append(f" 1 in a mode this harness does not handle: {device['id']} - that id is the thing to chase") + return lines + + +def no_boards_message(stranded=None): + """Why the bench looks empty, told apart from a bench that is not there.""" + stranded = bootsel_boards() if stranded is None else stranded + if not stranded: + return "\n".join( + [ + "no boards found: nothing on serial, and nothing in the ROM bootloader either.", + "", + *usb_bus_report(), + ] + ) + ids = ", ".join(b["chip_id"] or "?" for b in stranded) + return ( + f"no board is on serial, but {len(stranded)} are in the ROM bootloader (BOOTSEL/UF2 mode): {ids}.\n" + "They are attached and healthy - they are just running the bootloader instead of firmware,\n" + "which is what a UF2 flash that did not finish leaves behind. dev/hil/recover.py flashes them\n" + "back; if it already ran, its output above says what stopped it." + ) + + +IDENTIFY_SNIPPET = """ +import machine, time +try: + import BoardLED as L + L.startUp() +except Exception: + L = None +led = machine.Pin("LED", machine.Pin.OUT) +for i in range({blinks}): + led.on() + if L: + L.ledColor(L.BLUE) + time.sleep(0.25) + led.off() + if L: + L.ledColor(L.BLACK) + time.sleep(0.25) +""" + + +def identify(boards, seconds=8): + """Blink each board in turn so a human can tell which is which. + + Uses the Pico W onboard LED, which works from the REPL no matter what + firmware is loaded, plus the Vector board's WS2812 in blue when the + flashed firmware happens to provide the driver. + """ + log(f"Blinking each board for ~{seconds}s. Watch the bench and note the order.") + log("") + for index, board in enumerate(boards, 1): + log(f" [{index}/{len(boards)}] BLINKING NOW: {board['port']} chip {board['chip_id']}") + result = mpremote( + "connect", + board["port"], + "exec", + IDENTIFY_SNIPPET.format(blinks=int(seconds / 0.5)), + timeout=seconds + 30, + ) + if result.returncode != 0: + log(f" could not blink this board: {result.stderr.strip()}") + else: + log(" done") + log("") + log("Now map what you saw to the chip ids:") + log(board_map_instructions(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")))) + + +# -------------------------------------------------------------------------- +# 2. resolve +# -------------------------------------------------------------------------- + + +def board_map_instructions(boards, board_map=None): + """How to write or amend VECTOR_HIL_BOARD_MAP for this bench. + + Printed with every map-related failure: the map lives in the runner's + .env on the bench host, so whoever reads the CI log is usually not + looking at the machine that needs editing. + """ + board_map = board_map or {} + + def wanted(chip_id): + """What this board's entry should say - never what a broken one says. + + Echoing the current value back is how a wrong entry survives being + reported: the line offered as the fix contained the very `de` that + failed. A value the bench cannot drive is replaced by the target it + was probably reaching for, or by a blank to fill in. + """ + target = board_map.get(chip_id) + if target in bench_targets(): + return target + suggestion = suggest_target(target) + return suggestion if suggestion in bench_targets() else "" + + # A board whose chip id could not be read cannot be mapped at all - there + # is nothing to key the entry on. Leaving it in produced a suggestion of + # `None=` on the bench, which is a broken value presented as the + # fix. It is named separately instead, with what to do about it. + nameless = [b for b in boards if not b.get("chip_id")] + identified = [b for b in boards if b.get("chip_id")] + suggested = ",".join(f"{b['chip_id']}={wanted(b['chip_id'])}" for b in identified) + lines = [ + "", + "VECTOR_HIL_BOARD_MAP pins each board to the system it is wired to, by RP2040", + "chip id (stable across reflashing). Every board on the bench must appear in it.", + "", + " format: =,=", + " targets: " + ", ".join(bench_targets()), + "", + ] + if nameless: + lines += [ + " " + ", ".join(b.get("port") or "(BOOTSEL)" for b in nameless) + " did not report a chip id, so it cannot be", + " mapped yet - an entry needs an id to key on. A board that enumerates but will not say", + " who it is has usually been left mid-flash; recover it first, then map the id it reports.", + "", + ] + unusable = [target for target in buildable_targets() if target not in bench_targets()] + if unusable: + lines += [ + "This checkout also builds " + ", ".join(unusable) + ", which the bench cannot drive:", + "those targets have no game configs to boot against yet. A board running one of them", + "cannot be mapped to it - and mapping it to a different system would flash the wrong", + "firmware to real hardware.", + "", + ] + if board_map: + lines += ["current map:"] + [f" {chip}={target}" for chip, target in sorted(board_map.items())] + [""] + lines += [ + "Fill in the target for each board and set the whole line - it replaces the old", + "value, so keep the entries that were already right:", + "", + " VECTOR_HIL_BOARD_MAP=" + suggested, + "", + "On the bench host (the map is runner environment, not repo config):", + "", + " # drop any existing entry, then append the new one", + " sed -i '/^VECTOR_HIL_BOARD_MAP=/d' ~/actions-runner/.env", + " echo 'VECTOR_HIL_BOARD_MAP=" + suggested + "' >> ~/actions-runner/.env", + " cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start", + "", + "The service only reads .env at start, so the restart is required.", + "", + "Not sure which physical board is which chip id? Blink them in turn:", + "", + " cd ~/vector && .venv/bin/python dev/hil/flash_and_check.py --identify", + "", + "A board in BOOTSEL cannot blink, and the id above is the one its ROM", + "bootloader publishes. That is the same unique id MicroPython reports, but if", + "a board ever turns up under two, map both - a spare entry costs nothing.", + "", + "See dev/hil/RUNNER_SETUP.md for the full walkthrough.", + ] + return "\n".join(lines) + + +def parse_board_map(raw): + """Parse VECTOR_HIL_BOARD_MAP: 'chipid=target,chipid=target'.""" + mapping = {} + for entry in (raw or "").split(","): + entry = entry.strip() + if not entry: + continue + if "=" not in entry: + raise CheckFailure(f"bad VECTOR_HIL_BOARD_MAP entry {entry!r}, expected chipid=target") + chip, target = entry.split("=", 1) + mapping[chip.strip()] = target.strip() + return mapping + + +def report_unknown_boards(unmapped, boards, board_map): + """Say which boards the map does not cover, and how to add them. + + Written to the Actions job summary as well as the log: a board arriving on + the bench is a one-line edit on the runner host, and whoever has to make + it is reading the run's summary page, not scrolling a log for the chip id. + """ + described = ", ".join(f"{b.get('port') or '(BOOTSEL)'} {b['chip_id'] or '?'}" for b in unmapped) + summary_once( + described, + [ + "", + "### Unrecognised board" + ("s" if len(unmapped) > 1 else ""), + "", + f"`VECTOR_HIL_BOARD_MAP` does not cover: **{described}**", + *as_block(board_map_instructions(boards, board_map)), + ], + ) + return "VECTOR_HIL_BOARD_MAP is set but does not cover: " + described + + +def resolve_targets(boards, board_map): + """Decide the target for each board, refusing to guess when it matters. + + An explicit chip-id map is authoritative. Without one we fall back to what + each board's firmware reports, which is only trustworthy when every board + reports something different - if two boards claim the same system, that is + the signature of a previous mis-flash rather than of the hardware, and + flashing on that basis would silently perpetuate it. + """ + dead = [b for b in boards if not b.get("responsive", True)] + if dead: + raise CheckFailure( + "not answering: " + ", ".join(b["port"] for b in dead) + ".\nA board that will not talk cannot be identified, so it cannot be safely flashed.\n" + "The cheap recovery rungs have already been tried on it. dev/hil/recover.py run by hand\n" + "can also reflash it over the ROM bootloader, which this will not do on its own: that is\n" + "destructive and would leave the board running firmware other than the build under test." + ) + + if board_map: + unmapped = [b for b in boards if b["chip_id"] not in board_map] + if unmapped: + raise CheckFailure(report_unknown_boards(unmapped, boards, board_map)) + for b in boards: + b["target"] = board_map[b["chip_id"]] + check_targets(boards, board_map) + log("targets from VECTOR_HIL_BOARD_MAP") + return boards + + missing = [b for b in boards if not b["system"]] + if missing: + raise CheckFailure("cannot identify " + ", ".join(b["port"] for b in missing) + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP.\n" + board_map_instructions(boards, board_map)) + + systems = [b["system"] for b in boards] + duplicates = {s for s in systems if systems.count(s) > 1} + if duplicates: + raise CheckFailure( + "refusing to flash from autodetection: " + ", ".join(sorted(duplicates)) + " is reported by more than one board.\n" + "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" + "at least one board is running firmware for a system it is not wired for.\n" + "Pin them explicitly instead, using the chip ids above.\n" + board_map_instructions(boards, board_map) + ) + + for b in boards: + b["target"] = b["system"] + check_targets(boards, board_map) + log("targets from firmware self-report (all distinct)") + return boards + + +# Every system the bench exists to cover. A run that quietly tests two of the +# three proves nothing about the third while still reporting green, which is +# worse than not running: it is a check that has stopped checking. Overridable +# for a bench that has genuinely lost a board - deliberately, by whoever runs +# it, and visibly in the log. +REQUIRED_TARGETS = ("sys11", "wpc", "data_east") + + +def required_targets(): + raw = os.environ.get("VECTOR_HIL_REQUIRED_TARGETS") + if raw is None: + return list(REQUIRED_TARGETS) + return [target.strip() for target in raw.split(",") if target.strip()] + + +def check_bench_complete(boards): + """Which required systems are not on the bench, reported where it shows. + + Returns the missing targets rather than raising: the boards that *are* + here still have checks worth running, and a run is expensive. The caller + keeps the result and fails the run at the end - green on two boards out of + three is the outcome this exists to prevent. + """ + wanted = required_targets() + if not wanted: + log("no required targets set - running whatever is attached") + return [] + present = {b.get("target") for b in boards} + missing = [target for target in wanted if target not in present] + if not missing: + log("bench is complete: " + ", ".join(wanted)) + return [] + + have = ", ".join(f"{b['target']} ({b['port']})" for b in boards) or "nothing" + log(f"::error::the bench is missing {', '.join(missing)} - only {have} answered") + summary_once( + "### Incomplete bench", + [ + "", + "### Incomplete bench", + "", + f"Missing: **{', '.join(missing)}**. Attached: {have}.", + "", + "A run proves nothing about a system that is not on the bench, so this fails the run rather", + "than reporting green on two boards out of three. The boards that are here are still checked.", + "", + "Either put the missing board back (`dev/hil/recover.py`, and check it is in", + "`VECTOR_HIL_BOARD_MAP`), or set `VECTOR_HIL_REQUIRED_TARGETS` on the runner to the systems", + "the bench really has.", + "", + # Which of those it is turns on one question the log could not answer: + # is the board on the bus at all? A board that is enumerated but silent + # is a job for the recovery ladder; a board that is simply gone needs + # someone to replug it, and no amount of draining or resetting will + # help. The bus listing already exists for an empty bench - an + # incomplete one deserves it just as much. + *usb_bus_report(), + ], + ) + return missing + + +# -------------------------------------------------------------------------- +# 3. build +# -------------------------------------------------------------------------- + + +# The repo's own list of what it builds - the same file the bench manifest in +# DESIGN.md §6 says a board's target must match. Read rather than restated, +# because a hardcoded copy is a list that goes stale silently. +TARGETS_JSON = REPO_ROOT / "dev" / "ci" / "targets.json" + + +def buildable_targets(): + """Every distinct hardware target this checkout can build. + + `hardware_id` collapses the variants that are one board with two firmware + passes (sys11_tiny is sys11), because this list answers "what can a board + be", not "what can be built". + """ + try: + entries = json.loads(TARGETS_JSON.read_text()) + targets = {entry.get("hardware_id") or entry["id"] for entry in entries} + except (OSError, ValueError, KeyError): + # src/common is shared code, not a system. + targets = {path.parent.name for path in (REPO_ROOT / "src").glob("*/systemConfig.py")} - {"common"} + return sorted(targets) + + +def bench_targets(): + """Targets the bench can actually flash and health-check. + + Narrower than what the tree can build, and the gap is real: `classic` and + `whitestar` have a systemConfig.py and nothing else - no config directory, + so no generic game config to boot them against and nothing for + /api/game/configs_list to return. A board running one of them turned up on + the bench and the harness had no way to say any of this: it offered four + targets with no hint that the tree has seven, and a map entry naming one of + the other three died on a bare KeyError three stages later. + """ + ready = [] + for target in buildable_targets(): + config = DEFAULT_GAMENAME.get(target) + if config and (REPO_ROOT / "src" / target / "config" / f"{config}.json").exists(): + ready.append(target) + return ready + + +def target_labels(): + """{target id: human label} from the repo's target list, for suggestions.""" + labels = {} + try: + for entry in json.loads(TARGETS_JSON.read_text()): + labels.setdefault(entry.get("hardware_id") or entry["id"], entry.get("label", "")) + except (OSError, ValueError, KeyError): + pass + return labels + + +def suggest_target(name): + """The target somebody probably meant when they wrote `name`. + + People write the name they use out loud, not the directory: the bench map + arrived with `de` in it, which is what everyone calls the Data East board + and is nothing like the `data_east` the tree wants. Matching the label and + its initials as well as the id turns that from a puzzle into a correction. + """ + wanted = (name or "").strip().lower().replace("-", "_").replace(" ", "") + if not wanted: + return None + candidates = {} + for target, label in target_labels().items(): + candidates[target.lower()] = target + candidates[target.lower().replace("_", "")] = target + if label: + candidates[label.lower()] = target + candidates["".join(c for c in label if c.isupper()).lower()] = target + return candidates.get(wanted) + + +def describe_bad_target(target): + """Why this target cannot be used, in one sentence plus the way out.""" + if target not in buildable_targets(): + suggestion = suggest_target(target) + did_you_mean = f" - did you mean {suggestion}?" if suggestion else "" + return f"{target!r} is not a target in this checkout{did_you_mean} The bench can drive " + ", ".join(bench_targets()) + "." + return ( + f"{target!r} is a real build target but the bench cannot drive it: src/{target}/config holds no " + f"generic game config to flash and boot against, so there is nothing to boot the board against. " + f"The bench can drive " + ", ".join(bench_targets()) + "." + ) + + +def check_targets(boards, board_map=None): + """Refuse targets the bench cannot drive, while they are still configuration. + + Reported like an unrecognised chip id - to the job summary, with the map + and how to edit it - because it is the same mistake seen from the other + end, and the same person on the same host has to fix it. Left to run, it + would have flashed nothing and died on a bare KeyError two stages later. + """ + bad = sorted({b["target"] for b in boards if b.get("target") not in bench_targets()}, key=str) + if not bad: + return + + described = [describe_bad_target(target) for target in bad] + summary_once( + "### Board map names a target the bench cannot use", + [ + "", + "### Board map names a target the bench cannot use", + "", + *[f"- {line}" for line in described], + *as_block(board_map_instructions(boards, board_map or {})), + ], + ) + raise CheckFailure(" ".join(described)) + + +def source_version(target): + config = REPO_ROOT / "src" / target / "systemConfig.py" + match = re.search(r'SystemVersion\s*=\s*"([^"]+)"', config.read_text()) + if not match: + raise CheckFailure(f"could not read SystemVersion from {config}") + return match.group(1) + + +def build(target): + build_dir = REPO_ROOT / "build" / target + result = subprocess.run( + [VENV_PYTHON, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=900, + ) + if result.returncode != 0: + log(result.stdout[-3000:]) + log(result.stderr[-3000:]) + raise CheckFailure(f"build failed for {target}") + return build_dir + + +# -------------------------------------------------------------------------- +# 4. flash +# -------------------------------------------------------------------------- + + +def write_bench_config(target, workdir): + ssid = os.environ.get("VECTOR_HIL_WIFI_SSID", "") + password = os.environ.get("VECTOR_HIL_WIFI_PASSWORD", "") + game_password = os.environ.get("VECTOR_HIL_GAME_PASSWORD", "hiltest") + + if not ssid or not password: + raise CheckFailure("VECTOR_HIL_WIFI_SSID / VECTOR_HIL_WIFI_PASSWORD are not set") + + # dev/flash.py builds a MicroPython snippet with single-quoted values, so a + # single quote anywhere here would produce a syntax error on the board + # rather than an obvious failure here. + for name, value in (("ssid", ssid), ("password", password), ("game password", game_password)): + if "'" in value or "\\" in value: + raise CheckFailure(f"bench {name} contains a quote or backslash, which dev/flash.py cannot write") + + config = { + "ssid": ssid, + "password": password, + "gamename": DEFAULT_GAMENAME[target], + "Gpassword": game_password, + } + path = workdir / f"hil-config-{target}.json" + path.write_text(json.dumps(config)) + return path + + +def flash(target, port, build_dir, config_path): + result = subprocess.run( + [VENV_PYTHON, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=900, + ) + if result.returncode != 0: + # The output travels with the failure rather than being logged here: + # boards are flashed concurrently, and a line printed from a worker + # thread lands next to another board's output with nothing to say + # which board it came from. + tail = (result.stdout[-2000:] + result.stderr[-2000:]).strip() + raise CheckFailure(f"flash failed for {target} on {port}:\n " + "\n ".join(tail.splitlines()[-20:])) + + +def flash_boards(boards, workdir): + """Flash every board at once, and report per board. + + The boards are independent devices on independent serial ports, and + dev/flash.py is one subprocess per board that spends nearly all its time + waiting on USB - so doing them one after another just adds up the waits. + Three boards on the bench Pi: about a minute, instead of three. + + Returns {port: error message}, empty when every board flashed. + """ + + def flash_one(board): + started = time.monotonic() + config_path = write_bench_config(board["target"], workdir) + flash(board["target"], board["port"], REPO_ROOT / "build" / board["target"], config_path) + return time.monotonic() - started + + errors = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(boards))) as pool: + futures = {pool.submit(flash_one, board): board for board in boards} + for future in concurrent.futures.as_completed(futures): + board = futures[future] + try: + elapsed = future.result() + except CheckFailure as exc: + errors[board["port"]] = str(exc) + except Exception as exc: # noqa: BLE001 - reported per board, never fatal to the others + errors[board["port"]] = f"{type(exc).__name__}: {exc}" + else: + log(f" ok {board['port']:16} {board['target']:12} {elapsed:5.1f}s") + + for port, error in errors.items(): + log(f" FAIL {port}") + log(f"::error::{error}") + return errors + + +# -------------------------------------------------------------------------- +# 5. health +# -------------------------------------------------------------------------- + + +def reset_board(port): + """Reset the board so we own the boot we are about to watch. + + The ready marker is printed exactly once per boot. dev/flash.py already + resets at the end of flashing, but flashing runs over every board before + any health check starts, so by the time we open a console the board booted + a minute ago and the marker is long gone. Resetting here makes the wait + deterministic and the reported boot time meaningful. + """ + result = mpremote( + "connect", + port, + "exec", + "--no-follow", + "import machine; machine.reset()", + timeout=30, + ) + if result.returncode != 0: + raise CheckFailure(f"could not reset {port} before the health check: {result.stderr.strip()}") + + +def wait_for_server(port, timeout=BOOT_TIMEOUT): + """Watch the boot console until the firmware reports its web server is up. + + Polling an API that is not listening yet tells you nothing about why, and + burns the whole timeout when a board fails to boot. Reading the console + instead gives an exact ready signal and, on failure, the boot log that + explains it. + + Two ways this ends other than success: the board never gets there + (timeout), or the firmware raises and drops to the REPL (BootCrash). They + are worth distinguishing - a crash is described by the traceback sitting in + the console right now, and there is no point waiting out the rest of the + budget for a marker a dead program will never print. + + Returns the open serial connection so the USB API can reuse it - the + Pico exposes one CDC endpoint, so a second connection would fight this one. + """ + started = time.monotonic() + deadline = started + timeout + transcript = [] + connection = None + + while time.monotonic() < deadline: + if connection is None: + try: + # The port disappears and re-enumerates across the reset, so a + # failure to open here is expected for the first second or two. + connection = open_serial(port) + except Exception: + time.sleep(1) + continue + try: + raw = connection.readline() + except Exception: + try: + connection.close() + except Exception: + pass + connection = None + continue + + if not raw: + continue + text = raw.decode(errors="replace").rstrip("\r\n") + if not text: + continue + transcript.append(text) + + if READY_MARKER in text: + elapsed = time.monotonic() - started + log(f" server up after {elapsed:.1f}s ({text.strip()!r})") + time.sleep(SERVER_SETTLE_SECONDS) + return connection, transcript + + elapsed = time.monotonic() - started + if elapsed > CRASH_MARKER_GRACE and any(marker in text for marker in CRASH_MARKERS): + try: + connection.close() + except Exception: + pass + raise BootCrash( + f"{port} dropped to the REPL {elapsed:.1f}s into boot - the firmware raised on the way up:\n " + _explain_crash(transcript), + transcript, + ) + + # Before calling it a failure: ask the board directly. The ready marker is + # printed exactly once, so anything that costs us the moment it goes past - + # a reset we did not trigger, a board that booted while we were still + # flashing the next one - looks identical to a board that never came up. + # A board that answers its API is up, whatever we did or did not see. + if connection is not None and _server_is_answering(connection): + log(f"::warning::{port} is answering its API but its ready marker was never seen - " "the marker was probably printed before the console was open") + return connection, transcript + + if connection is not None: + try: + connection.close() + except Exception: + pass + + tail = "\n ".join(transcript[-20:]) or "(nothing on the console)" + raise CheckFailure(f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}") + + +def _server_is_answering(connection): + """One cheap USB request, to tell 'missed the marker' from 'never booted'.""" + try: + prime_usb(connection) + response = UsbApiClient(connection).send_and_receive(route="/api/version", payload=None, timeout=10) + return response.get("status") == 200 + except Exception: + return False + + +def _explain_crash(transcript, lines=14): + """Pull the traceback out of a boot transcript, for the error message. + + The traceback is the whole value of catching this, so it leads; without one + the tail of the console is the next best thing. + """ + for index, line in enumerate(transcript): + if "Traceback (most recent call last)" in line: + return "\n ".join(transcript[index : index + lines]) + return "\n ".join(transcript[-lines:]) or "(nothing on the console)" + + +def prime_usb(connection): + """Clear both ends of the serial line before the first API request. + + usb_comms accumulates stdin characters into a module-level `buffer` until + it sees a newline (usb_comms.py:132). Anything left there without a + terminator - a partial line, stray bytes from the raw-REPL session that + issued the reset - silently prefixes the next request, so the board parses + a route like "\x02/api/version", fails the `_routes` lookup and answers + 404. A lone newline flushes whatever is pending into a discarded request. + """ + try: + connection.reset_input_buffer() + connection.reset_output_buffer() + connection.write(b"\n") + except Exception as exc: + log(f" warning: could not prime the USB link: {exc}") + return + # usb_request_handler is scheduled every 1000ms (phew/server.py:342), so + # give it a turn to consume the flush before the first real request. + time.sleep(1.5) + try: + connection.reset_input_buffer() + except Exception: + pass + + +def get(client, route, expect=200): + response = client.send_and_receive(route=route, payload=None, timeout=15) + status = response.get("status") + if status != expect: + # The board narrates its own routing failures ("USB REQ: route not + # found: ..."), but the client discards every line that is not a + # response. Drain whatever is pending so the reason is visible. + raise CheckFailure(f"{route} returned {status}, expected {expect}" f"{_drain_serial(client.ser)}") + return response.get("body") + + +def _drain_serial(connection, limit=12): + """Return any pending board chatter, formatted for an error message.""" + try: + time.sleep(0.5) + pending = connection.read(connection.in_waiting or 0) + except Exception: + return "" + if not pending: + return "" + lines = [line for line in pending.decode(errors="replace").splitlines() if line.strip()] + if not lines: + return "" + return "\n board said: " + "\n board said: ".join(lines[:limit]) + + +# -------------------------------------------------------------------------- +# reporting +# -------------------------------------------------------------------------- + + +def _dump_boot_log(board, lines=25): + """Show what the board actually said. A failed health check is usually + explained by the boot output, and by this point we already have it.""" + transcript = board.get("boot_log") + if not transcript: + return + log(f" last {min(lines, len(transcript))} lines of {board['port']} boot console:") + for line in transcript[-lines:]: + log(f" {line}") + + +# -------------------------------------------------------------------------- +# talking to a board that might be wedged +# -------------------------------------------------------------------------- +# +# Two traps, both of which cost a bench run before they were understood, and +# both the same shape as the board's own failure: blocking on a buffer that +# nothing is draining. +# +# * `serial.Serial(timeout=...)` sets the READ timeout only. With no +# `write_timeout` a write to a board that has stopped reading its OUT +# endpoint blocks forever - which is exactly the state a wedged board is +# in, and exactly when the recovery tool needs to write to it. +# * `flush()` is termios tcdrain. It waits for the kernel's output buffer to +# reach the device and takes no timeout at all, so it hangs on the same +# board even when the write did not. We never call it: handing the bytes +# to the kernel is enough, and every exchange here is synchronised by a +# read with a deadline rather than by tcdrain. + +SERIAL_READ_TIMEOUT = 1 +SERIAL_WRITE_TIMEOUT = 5 + + +def open_serial(port, baudrate=115200, read_timeout=SERIAL_READ_TIMEOUT, write_timeout=SERIAL_WRITE_TIMEOUT): + """Open a board's port with BOTH timeouts set. Always use this.""" + return serial.Serial(port=port, baudrate=baudrate, timeout=read_timeout, write_timeout=write_timeout) + + +def serial_write(connection, data, what="the board"): + """Write to a board, refusing to wait forever if it has stopped listening.""" + try: + connection.write(data) + except serial.SerialTimeoutException: + raise CheckFailure(f"timed out writing to {what} after {SERIAL_WRITE_TIMEOUT}s - it has stopped draining its USB endpoint") + + +class time_limit: + """Hard ceiling on a block of work, however deep it blocks. + + A backstop rather than a design: every call in here is supposed to be + bounded, but a board in a bad enough state can block a syscall that no + library timeout covers, and one board must never be able to hang a bench + job. SIGALRM interrupts the syscall, so this catches cases the individual + timeouts miss. + + Main thread and POSIX only, which is what the bench is. + """ + + def __init__(self, seconds, what): + self.seconds = int(seconds) + self.what = what + + def _expired(self, _signum, _frame): + raise CheckFailure(f"{self.what} did not finish within {self.seconds}s and was interrupted") + + def __enter__(self): + self.previous = signal.signal(signal.SIGALRM, self._expired) + signal.alarm(self.seconds) + return self + + def __exit__(self, *_exc): + signal.alarm(0) + signal.signal(signal.SIGALRM, self.previous) + return False + + +# -------------------------------------------------------------------------- +# raw REPL, over a connection we already hold +# -------------------------------------------------------------------------- +# +# Everything below drives the REPL over an open pyserial connection instead of +# shelling out to `mpremote`. That is not a stylistic preference - it is the +# fix for a board that wedged on the bench. +# +# The Pico exposes one CDC endpoint, so using mpremote mid-run means closing +# our connection, letting a second process open the port, and reopening +# afterwards - for every config. On the first such handoff the WPC board +# (MicroPython 1.26.0-preview) stopped responding entirely: silent console, no +# reply to Ctrl-C, for the following 63 attempts. sys11 (1.24.1) survived the +# same treatment 39 times. A running board printing to a CDC endpoint that +# nothing is draining is the difference between them, so the harness now never +# leaves the port unread while the board is running, and never lets a second +# process contend for it. The port is closed only while the board is mid-reset. + +CTRL_A = b"\x01" # enter raw REPL +CTRL_B = b"\x02" # back to the friendly REPL +CTRL_C = b"\x03" # interrupt whatever is running +CTRL_D = b"\x04" # execute what was pasted / end-of-output marker + +RAW_REPL_BANNER = b"raw REPL; CTRL-B to exit" +REPL_TIMEOUT = 15 + + +class Repl: + """A raw-REPL session over a connection somebody else owns. + + Owns a pending buffer, which is the whole reason this is a class rather + than a few functions: raw REPL is a sequence of markers (`OK`, \x04, \x04, + `>`) and a read that syncs on one marker almost always pulls in bytes + belonging to the next. Dropping them desynchronises everything that + follows. + """ + + def __init__(self, connection): + self.connection = connection + self._pending = bytearray() + + def read_until(self, marker, timeout, what): + """Return everything up to `marker`, keeping what came after it. + + The board is mid-sentence when we interrupt it, so the stream still + holds application output. Syncing on a marker rather than on a line + count is what makes that harmless. + """ + deadline = time.monotonic() + timeout + while True: + index = self._pending.find(marker) + if index >= 0: + before = bytes(self._pending[:index]) + del self._pending[: index + len(marker)] + return before + if time.monotonic() >= deadline: + break + chunk = self.connection.read(self.connection.in_waiting or 1) + if chunk: + self._pending.extend(chunk) + + tail = bytes(self._pending[-400:]).decode(errors="replace") + raise CheckFailure(f"timed out after {timeout}s waiting for {what}. Last output:\n {tail or '(nothing on the console)'}") + + def enter(self, timeout=REPL_TIMEOUT): + """Interrupt the running firmware and take the raw REPL. + + Ctrl-C raises KeyboardInterrupt in `main.py`, which ends the + application and drops to the REPL - the same thing `mpremote` does, and + the reason this is safe to do to a board we are about to reset anyway. + """ + self._pending.clear() + self.connection.reset_input_buffer() + serial_write(self.connection, CTRL_C + CTRL_C, "the board's REPL") + time.sleep(0.2) + + serial_write(self.connection, CTRL_A, "the board's REPL") + self.read_until(RAW_REPL_BANNER, timeout, "the raw REPL prompt") + self.read_until(b">", timeout, "the raw REPL prompt") + return self + + def exec(self, code, timeout=REPL_TIMEOUT): + """Run one snippet and return what it printed. + + Raw REPL framing: paste the code, Ctrl-D to run, the board answers + `OK`, then stdout, then \x04, then the traceback (empty on success), + then \x04. + """ + serial_write(self.connection, code.encode() + CTRL_D, "the board's REPL") + + self.read_until(b"OK", timeout, "the board to accept the snippet") + output = self.read_until(CTRL_D, timeout, "the snippet to finish") + error = self.read_until(CTRL_D, timeout, "the snippet's exit status") + + if error.strip(): + detail = error.decode(errors="replace").strip().replace("\n", "\n ") + raise CheckFailure(f"the board raised an error running the snippet:\n {detail}") + return output.decode(errors="replace") + + def reset(self): + """Reset the board, without waiting for a reply. + + There is no reply to wait for - the board reboots mid-command and the + port re-enumerates underneath us. The caller reopens it in + wait_for_server(). + """ + try: + serial_write(self.connection, b"import machine; machine.reset()" + CTRL_D, "the board's REPL") + except Exception as exc: + raise CheckFailure(f"could not issue a reset over the REPL: {exc}") + # Let the write reach the board before the port disappears. + time.sleep(0.5) + + +def repl_reset(connection): + """Interrupt whatever the board is doing and reset it, over `connection`.""" + Repl(connection).enter().reset() + + +DRAIN_SECONDS = 20 +DRAIN_QUIET_SECONDS = 2 + + +def read_until_quiet(connection, budget, quiet=DRAIN_QUIET_SECONDS): + """Read from an open port until it stops producing, or the budget runs out. + + A board blocked writing into a CDC endpoint nothing is draining does not + hand over its backlog in one gulp: it unblocks, runs a little further, + prints some more, and only then goes quiet. Reading for a fixed three + seconds catches the first mouthful and calls the board dead. That is not + a guess - a bench run declared /dev/ttyACM0 unrecoverable after draining + 63 bytes from it, and the very next step in the same job read it with + `cat` for eight seconds and got a healthy, running server. + + So read until it has been silent for `quiet` seconds, capped at `budget`. + A port that is already quiet costs `quiet`; one with a backlog gets as + long as it needs to finish coughing it up. + """ + drained = 0 + deadline = time.monotonic() + budget + silent_since = time.monotonic() + while time.monotonic() < deadline: + chunk = connection.read(connection.in_waiting or 1) + if chunk: + drained += len(chunk) + silent_since = time.monotonic() + elif time.monotonic() - silent_since >= quiet: + break + return drained + + +def interrupt_board(connection, port, keys=CTRL_C): + """Send Ctrl-C to a board we have just finished draining. + + Returns whether the board accepted it, which is a diagnosis in its own + right. Both callers read the console to quiet first, because that is the + remedy for the common wedge: a firmware blocked writing to stdout is not + reading stdin either, so the host's OUT endpoint backs up behind it and + the write times out against the very deadlock it is trying to clear. + + A write that still times out after that is a different animal. It means + the board is producing output happily and simply never services what we + send it - and no amount of reading fixes that, because reading is not + what it is waiting for. /dev/ttyACM0 has been in exactly that state for + three runs: 85, 95 and 112 bytes read, every single write timed out. + Retrying the write only spends another five seconds saying so. + """ + try: + serial_write(connection, keys, port) + return True + except CheckFailure as exc: + log(f" {exc}") + return False + + +def drain_port(port, seconds=DRAIN_SECONDS): + """Open a port, read the board's console until it goes quiet, interrupt it. + + A cheap attempt at unsticking a board, and it costs one open. The wedge + worth recovering from is a board blocked writing into a CDC endpoint that + nothing is draining: reading is the whole remedy, and the Ctrl-C afterwards + gets it back to a REPL once the read has freed it - which is why the + interrupt comes after the drain and not before. Reports what it saw either + way; a board that yields zero bytes and a board that yields a backlog are + different problems. + """ + try: + connection = open_serial(port) + except Exception as exc: + log(f" could not reopen {port} to unstick it: {exc}") + return 0 + + drained = 0 + interrupted = False + try: + drained += read_until_quiet(connection, seconds) + interrupted = interrupt_board(connection, port) + # Whatever the interrupt shook loose - a KeyboardInterrupt traceback, + # a REPL banner - is backlog too, and leaving it queued re-wedges the + # board the moment we close. + drained += read_until_quiet(connection, DRAIN_QUIET_SECONDS * 2) + except Exception as exc: + log(f" error while draining {port}: {exc}") + finally: + try: + connection.close() + except Exception: + pass + + log(f" drained {drained} byte(s) from {port} and {'sent Ctrl-C' if interrupted else 'could not send Ctrl-C'}") + return drained + + +# -------------------------------------------------------------------------- +# game configuration +# -------------------------------------------------------------------------- + + +def gamename_field_bytes(): + """Width of the `gamename` field in the FRAM `configuration` record. + + Read out of the firmware source rather than hard-coded, so that widening + the field is picked up here instead of silently leaving a stale limit in + the harness. A filename longer than this cannot round-trip: struct.pack + truncates the value, the truncated name matches no config, and the board + boots on safe defaults with CONF01 raised. + """ + source = (REPO_ROOT / "src" / "common" / "SPI_DataStore.py").read_text() + match = re.search(r'"<32s32s(\d+)s\d+s"', source) + if not match: + raise CheckFailure("could not find the configuration record format in SPI_DataStore.py") + return int(match.group(1)) + + +SET_CONFIG_SNIPPET = ";".join( + [ + "import SPI_DataStore as ds", + "c = ds.read_record('configuration')", + "c['gamename'] = '{gamename}'", + "ds.write_record('configuration', c)", + "print('GAMENAME=' + ds.read_record('configuration')['gamename'])", + ] +) + + +def set_game_config(connection, gamename): + """Point a board at one game config and prove the value survived the write. + + The board picks its config up from the FRAM `configuration` record at boot + (GameDefsLoad.go), so setting it is a REPL write plus a reset - no + authentication, no HTTP, and no reflash. Takes the caller's open connection + rather than a port: see the note above raw REPL for why that matters. + + The read-back is the load-bearing part. `gamename` is a fixed-width field + and struct.pack truncates silently, so a name that is too long is written, + accepted, and then never matches any config on the next boot. Catching it + here costs nothing; catching it after the boot costs a boot cycle and + reports a confusing CONF01. + """ + if "'" in gamename or "\\" in gamename: + raise CheckFailure(f"config name {gamename!r} contains a quote or backslash") + + output = Repl(connection).enter().exec(SET_CONFIG_SNIPPET.format(gamename=gamename)) + + stored = None + for line in output.splitlines(): + if line.startswith("GAMENAME="): + stored = line.split("=", 1)[1].strip() + if stored is None: + raise CheckFailure(f"board did not read back a gamename after the write (said {output.strip()!r})") + + if stored != gamename: + limit = gamename_field_bytes() + detail = "" + if len(gamename) > limit: + detail = f" - the FRAM `gamename` field is {limit} bytes and this filename is {len(gamename)}, so no board can ever store it and the config is unreachable in the field" + raise CheckFailure(f"wrote gamename={gamename!r} but the board stored {stored!r}{detail}") diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py new file mode 100644 index 00000000..a895e1fb --- /dev/null +++ b/dev/hil/config_matrix.py @@ -0,0 +1,768 @@ +#!/usr/bin/env python3 +"""Boot every board against every game config it can be flashed with. + +Run from the repo root on the bench Pi: + + cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/config_matrix.py + +Inside an Actions job the runner's .env already provides VECTOR_HIL_VENV, so +plain `python dev/hil/config_matrix.py` is enough there. + +This is DESIGN.md's G3: "every available config can be parsed and boot". A +config is a JSON file in src//config/ that the build packs into the +firmware's config bundle; the board picks one at boot from the `gamename` field +of the FRAM `configuration` record. So one iteration is: + + write gamename -> reset -> wait for the server -> ask the board what it + loaded + +and the pass condition is that the board reports *that* config, by name. + +Why the game name is the assertion that matters: when a config fails to apply - +missing from the bundle, unparseable, or a filename the board cannot store - +GameDefsLoad falls back to `safe_defaults`, and the board comes up looking +perfectly healthy while running a generic definition for that hardware. Nothing +faults from the outside. Comparing `/api/game/name` against `GameInfo.GameName` +in the *source* JSON is what separates "loaded my config" from "silently fell +back", and it cross-checks the on-board bundle against the repo at the same +time. + +Stages: + + 1. inventory - probe every attached board (shared with flash_and_check.py) + 2. resolve - decide which target each board should be flashed with + 3. build/flash - one firmware flash per board, so the bundle under test is + the one built from this checkout + 4. matrix - per board, per config: set, reset, boot, assert + 5. restore - put each board back on its generic config + +Boards are visited one after another, and a boot cycle is 15-25s, so a full run +is dominated by whichever target has the most configs (WPC, at 63). Use +--target/--configs/--limit to cut it down while iterating. +""" + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import bench # noqa: E402 +import trench_coat # noqa: E402 +from bench import ( # noqa: E402 + _TIMINGS, + BENCH_WARN_FAULTS, + BOOT_TIMEOUT, + DEFAULT_GAMENAME, + EXPECTED_FAULTS, + REPO_ROOT, + BootCrash, + CheckFailure, + UsbApiClient, + _dump_boot_log, + drain_port, + endgroup, + get, + group, + inventory, + log, + parse_board_map, + prime_usb, + repl_reset, + reset_board, + resolve_targets, + set_game_config, + time_limit, + wait_for_server, +) + +# Raised by GameDefsLoad when the configured game cannot be loaded (CONF01) or +# blew up on the way (CONF00). Either one means the board is running +# safe_defaults, which is exactly the failure this harness exists to catch. +CONFIG_FAULTS = {"CONF00", "CONF01"} + +# Measured on the bench: 12.5s (wpc), 15.3s (sys11), and the flash harness's +# 150s default only buys a wedged board more time to waste. +MATRIX_BOOT_TIMEOUT = 90 + +# Hard ceiling on one config, as a backstop to the individual timeouts inside +# it. A healthy config takes ~21s; anything past this is a board that has +# stopped behaving, and it must not be allowed to hang the job. See +# bench.time_limit. +CONFIG_TIMEOUT = 240 + + +def source_configs(target): + """{config filename without .json: {"name": ..., "adjustments": bool}}. + + This is the expectation side of every assertion below - the board's answers + are compared against the source JSON, never against the board's own idea of + what it has. + + `adjustments` records whether the config declares an Adjustments section, + which decides how hard /api/adjustments/status is held to account. See + check_config(). + """ + config_dir = REPO_ROOT / "src" / target / "config" + if not config_dir.is_dir(): + raise CheckFailure(f"no config directory for target {target} at {config_dir}") + + configs = {} + for path in sorted(config_dir.glob("*.json")): + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + raise CheckFailure(f"{path} is not valid JSON: {exc}") + try: + name = data["GameInfo"]["GameName"] + except (KeyError, TypeError): + raise CheckFailure(f"{path} has no GameInfo.GameName") + configs[path.stem] = {"name": name, "adjustments": "Adjustments" in data} + + if not configs: + raise CheckFailure(f"no game configs found in {config_dir}") + return configs + + +def changed_configs(target, ref): + """Config names under test that git says changed since `ref`. + + Ordering these first means a run that is going to fail because of the diff + fails in the first minute rather than the twentieth. + """ + result = subprocess.run( + ["git", "diff", "--name-only", f"{ref}...HEAD", "--", f"src/{target}/config"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + if result.returncode != 0: + log(f"::warning::could not diff against {ref}: {result.stderr.strip()}") + return [] + return [Path(line).stem for line in result.stdout.split()] + + +def order_configs(names, first): + """Put `first` at the front, keeping the rest in their existing order.""" + lead = [name for name in first if name in names] + return lead + [name for name in names if name not in lead] + + +def select_configs(target, args): + configs = source_configs(target) + names = list(configs) + + if args.configs: + requested = [name.strip() for name in args.configs.split(",") if name.strip()] + unknown = [name for name in requested if name not in configs] + if unknown: + raise CheckFailure(f"unknown config(s) for {target}: {', '.join(unknown)}") + names = requested + elif args.changed_since: + names = order_configs(names, changed_configs(target, args.changed_since)) + + if args.limit: + names = names[: args.limit] + return names, configs + + +def check_faults(client, config): + """Fail on anything that makes the config assertions meaningless.""" + faults = get(client, "/api/fault") or [] + if isinstance(faults, dict): + faults = faults.get("faults", []) + codes = {str(f)[:6] for f in faults} + + config_faults = codes & CONFIG_FAULTS + if config_faults: + raise CheckFailure(f"{sorted(config_faults)} raised - the board did not load {config!r} and is on safe defaults") + + # HDWR01 is a warning in flash_and_check.py because it does not stop the + # API working. Here it is fatal: it puts main.py down the safe_mode path, + # where GameDefsLoad never reads the config at all, so every assertion + # below would pass or fail for reasons that have nothing to do with the + # config under test. + if codes & BENCH_WARN_FAULTS: + raise CheckFailure(f"{sorted(codes & BENCH_WARN_FAULTS)} raised - the board booted in safe mode, so no config was loaded and this result would be meaningless") + + unexpected = codes - EXPECTED_FAULTS + if unexpected: + raise CheckFailure(f"unexpected fault(s): {sorted(unexpected)}") + + +def check_adjustments(client, config, declares_adjustments): + """Prove the loaded definition is usable, not merely loadable. + + Held to 200 only for configs that declare an Adjustments section. The rest + are a known firmware gap rather than a config problem: GameDefsLoad assigns + the parsed config straight to SharedState.gdata without merging + safe_defaults into it, so a config with no Adjustments section leaves + gdata["Adjustments"] absent and Adjustments._get_range_from_gamedef raises + KeyError, which route_wrapper turns into a 500. Failing those here would + bury the signal this harness exists for under a defect it cannot fix, so + they are warned about instead - loudly, and once per occurrence. + """ + try: + adjustments = get(client, "/api/adjustments/status") + except CheckFailure as exc: + if declares_adjustments: + raise + log(f"::warning::{config}: /api/adjustments/status failed ({exc}). The config declares no Adjustments section and gdata is not merged with safe_defaults - see check_adjustments().") + return + + if not isinstance(adjustments, dict): + raise CheckFailure(f"/api/adjustments/status returned {type(adjustments).__name__}, expected an object") + + +class Session: + """One board's serial connection, carried across the whole matrix. + + The connection is the point. The Pico has a single CDC endpoint, so every + close is an invitation for something else to grab the port and for the + board to be left printing into an endpoint nothing is draining. The first + bench run wedged the WPC board on exactly that, one config in, and then + spent 63 minutes timing out against a board that was never coming back. + + So: open once per boot, hold it through the assertions AND through setting + the next config, and close it only in `reboot()` - by which point the board + is already resetting and has stopped printing. + """ + + def __init__(self, port, boot_timeout=BOOT_TIMEOUT): + self.port = port + self.boot_timeout = boot_timeout + self.connection = None + self.client = None + self.boot_log = [] + self.crashes = [] + + def start(self, reset=True): + """First boot of the run: watch the board come up, resetting it first + only when something else has not already done so. + + Every later boot is triggered by reboot() over our own connection, but + the first one has to be triggered here, and it must be triggered: + dev/flash.py resets at the end of flashing, and flashing runs over + every board before any of this starts, so the board booted minutes ago + and printed its one ready marker long before we opened a console. That + is bench.reset_board's whole reason for existing, and dropping it is + what made the first matrix run time out on all three boards without + checking a single config. + + `reset=False` is for a board that was just flashed. dev/flash.py ends + by resetting it, so it is already booting - and resetting again lands + on top of that boot, while the firmware is still reading its freshly + written filesystem. That is not theoretical: sys11 wedged in exactly + that window, flashed successfully and then refusing a reset seconds + later, and it is the same shape as the ENOENT-on-a-file-that-exists + crashes the WPC board raises at boot. One reset per boot. + + mpremote is safe here, unlike mid-matrix: no connection of ours is + open yet, so there is no handoff to lose. + + A board that will not take the reset gets one drain and one retry. A + board whose stdout is blocked on an undrained USB endpoint comes back + the moment somebody reads it, and reading costs three seconds - much + less than writing off a board's whole matrix. + """ + if reset: + try: + reset_board(self.port) + except Exception as exc: + log(f"::warning::{self.port} did not take a reset ({exc}); draining its console and retrying once") + drain_port(self.port) + reset_board(self.port) + else: + log(" already booting from the flash; watching that boot rather than forcing another") + return self.wait_for_boot() + + def wait_for_boot(self): + """Watch one boot, retrying once if the firmware raises on the way up. + + A crash is not a timing problem and no amount of waiting fixes it - the + program has exited. But it can be intermittent, and losing a whole + board's matrix to one flaky boot buys nothing: the retry is what lets + the 63 configs actually get checked. + + The crash is never swallowed. Every one is counted, logged with its + traceback, and reported in the run summary, so a board that only + sometimes comes up still shows up as a problem rather than as a clean + run that happened to take longer. + """ + for attempt in range(2): + try: + self.connection, self.boot_log = wait_for_server(self.port, timeout=self.boot_timeout) + break + except BootCrash as crash: + self.crashes.append(str(crash)) + self.boot_log = crash.transcript + if attempt: + raise + log(f"::warning::{self.port} crashed on boot; retrying once. {crash}") + reset_board(self.port) + + prime_usb(self.connection) + self.client = UsbApiClient(self.connection) + return self.client + + def ensure_connected(self): + """Get the board back to a booted, talking state if it is not already. + + Without this, one exhausted boot leaves the session poisoned: the + connection is None, so every later `set_config` raises "the board is + not connected" instantly - including the retry that exists precisely + to survive a flaky board. A bench run showed the shape exactly. The + WPC board crashed on boot twice for FishTales_L4, and from there the + retry failed in no time at all and the next config failed in 0.0s, + two configs blamed on a board that came back on its own moments later + and served the restore step happily. + + So a lost connection is a thing to repair, not a verdict. Drain first, + because the commonest reason a board stops talking is that nothing has + read what it printed, then reset and watch it come up. If it will not, + the caller's consecutive-failure counter still ends the board's matrix + - this only stops that happening while the board is still recoverable. + """ + if self.connection is not None: + return self.client + log(f" {self.port} has no live connection; draining and rebooting it before this config") + drain_port(self.port) + reset_board(self.port) + return self.wait_for_boot() + + def _require_connection(self, what): + if self.connection is None: + raise CheckFailure(f"cannot {what} on {self.port}: the board is not connected (its last boot did not complete)") + return self.connection + + def set_config(self, gamename): + """Point the board at a config over the REPL we already have open.""" + set_game_config(self._require_connection("set a config"), gamename) + + def reboot(self): + """Reset from the REPL, then drop the port while the board is down.""" + self._require_connection("reset the board") + try: + repl_reset(self.connection) + finally: + self.close() + + def nudge(self): + """Try to unstick a board that stopped answering, before giving up.""" + self.close() + drain_port(self.port) + + def close(self): + connection, self.connection, self.client = self.connection, None, None + if connection is not None: + try: + connection.close() + except Exception: + pass + + +def check_bundle(client, target, configs): + """Compare the board's config list against the repo, once per board. + + Cheap, and it localises a whole class of failure before the matrix starts: + if the build dropped or mangled a config, this says so in one boot instead + of once per affected iteration. + """ + on_board = get(client, "/api/game/configs_list") + if not isinstance(on_board, dict) or not on_board: + raise CheckFailure("/api/game/configs_list is empty - the config bundle is missing from the build") + + missing = sorted(set(configs) - set(on_board)) + extra = sorted(set(on_board) - set(configs)) + if missing: + raise CheckFailure(f"{len(missing)} config(s) in src/{target}/config are not in the build's bundle: {', '.join(missing)}") + if extra: + raise CheckFailure(f"the build's bundle carries {len(extra)} config(s) with no source JSON: {', '.join(extra)}") + + mismatched = [f"{name}: bundle says {on_board[name].get('name')!r}, source says {configs[name]['name']!r}" for name in sorted(configs) if on_board[name].get("name") != configs[name]["name"]] + if mismatched: + raise CheckFailure("game name mismatch between the bundle and the source JSON:\n " + "\n ".join(mismatched)) + + log(f" bundle matches source: {len(configs)} configs, names identical") + + +def check_booted_config(client, target, config, expected): + """Assert that the board in front of us booted on `config`.""" + expected_name = expected["name"] + + check_faults(client, config) + + active = get(client, "/api/game/active_config") + active = active.get("active_config") if isinstance(active, dict) else active + + # EM boards answer this route with the game name rather than the filename + # (backend.py:481), so accept either for that target. + expected_active = {config, expected_name} if target == "em" else {config} + if active not in expected_active: + raise CheckFailure(f"active config is {active!r}, expected {config!r}") + + name = get(client, "/api/game/name") + if isinstance(name, dict): + name = name.get("name") + name = str(name).strip() + if name != expected_name: + raise CheckFailure(f"board reports game name {name!r}, but {config}.json says {expected_name!r} - the config did not apply and the board fell back to a generic definition") + + # A config that parses but is not usable still fails a customer. Both of + # these read the loaded definition rather than just its presence. + if get(client, "/api/leaders") is None: + raise CheckFailure("/api/leaders returned no body") + check_adjustments(client, config, expected["adjustments"]) + + return name + + +def restore_default(session, target): + """Leave the board on its generic config, as flash_and_check.py expects. + + Best effort by design: this runs in teardown, including after the board has + stopped answering, and a failure here must not lose the results the matrix + already produced. + """ + default = DEFAULT_GAMENAME[target] + try: + if session.connection is None: + session.start() + session.set_config(default) + session.reboot() + log(f" restored {default}") + except Exception as exc: # noqa: BLE001 - teardown never fails the run + log(f"::warning::could not restore {default} on {session.port}: {exc}") + session.close() + + +# A board that stops answering does not start again on its own, and the bench +# is a shared singleton. Two setup failures in a row is the signal to stop +# spending an hour proving it: the first bench run burned 63 minutes writing +# the same 60s timeout 63 times, which is 62 wasted minutes and one lost +# data_east matrix. +MAX_CONSECUTIVE_SETUP_FAILURES = 2 + + +def flash_before_matrix(board, workdir): + """Flash one board immediately before its own matrix, not all up front. + + The ordering is the point, and it is what the bench taught us. Flashing + every board first leaves the ones further down the queue running the + application for as long as the boards ahead of them take - half an hour or + more - printing to a USB CDC console that nothing is draining. TrenchCoat + documents where that ends (src/ray.py, send_command): "if nothing ever + drains the board's output, the USB CDC buffers fill up, MicroPython blocks + writing to stdout, and the board deadlocks mid-script". + + That is not a hypothetical. It is what killed data_east 36 minutes into a + run, and it is the most likely explanation for the WPC board that went + silent for an hour on the very first bench run. + + Flashed here, a board starts running seconds before we start talking to it. + Boards waiting their turn sit at the REPL, where inventory's probe left + them, producing no output at all. + """ + group(f"Flash {board['target']} on {board['port']}") + try: + config_path = bench.write_bench_config(board["target"], workdir) + bench.flash(board["target"], board["port"], workdir / board["target"], config_path) + log("flashed") + finally: + endgroup() + + +def run_matrix(board, args, just_flashed=False): + """Walk one board through its configs. Returns (passed, failures). + + One connection per boot, held across the assertions and across setting the + next config, closed only while the board is resetting. See Session. + """ + port = board["port"] + target = board["target"] + names, configs = select_configs(target, args) + + session = Session(port, boot_timeout=args.boot_timeout) + passed = [] + failures = [] + flakes = [] + consecutive_setup_failures = 0 + + try: + group(f"Config bundle {target} on {port}") + try: + # A board we just flashed is already booting - see Session.start. + client = session.start(reset=not just_flashed) + check_bundle(client, target, configs) + finally: + endgroup() + + log("") + log(f"{len(names)} config(s) to check on {port} ({target})") + log("") + + for index, config in enumerate(names, 1): + started = time.monotonic() + group(f"[{index}/{len(names)}] {target} {config}") + first_error = None + try: + # Two attempts, because "this config is broken" and "this board + # is flaky" produce the identical symptom on one attempt, and + # they need opposite responses. A config that fails twice is a + # config bug and fails the run; one that passes on the retry is + # the board, and is reported as a flake instead of being + # blamed on the config. The bench has already shown it can do + # this - a WPC board raising ENOENT mid-boot on files that + # exist will do it to a route just as readily. + for attempt in range(2): + try: + with time_limit(args.config_timeout, f"{target} {config}"): + # Set the config on the board we are already talking + # to, then reboot into it. The connection dies with + # the reset; the next wait_for_boot opens a fresh one. + session.ensure_connected() + session.set_config(config) + session.reboot() + client = session.wait_for_boot() + consecutive_setup_failures = 0 + + name = check_booted_config(client, target, config, configs[config]) + break + except CheckFailure as exc: + if attempt: + raise + first_error = exc + log(f"::warning::{target} {config} failed, retrying once to tell a broken config from a flaky board: {exc}") + + elapsed = time.monotonic() - started + if first_error is None: + log(f" ok {config:20} -> {name!r} [{elapsed:.1f}s]") + else: + log(f" ok {config:20} -> {name!r} [{elapsed:.1f}s] (FLAKY - failed once, passed on retry)") + flakes.append((config, str(first_error))) + passed.append(config) + except CheckFailure as exc: + log(f"::error::{target} {config}: failed twice, so this is the config, not the board: {exc}") + _dump_boot_log({"port": port, "boot_log": session.boot_log}) + failures.append((config, str(exc))) + # An assertion that ran is a result about the config. Anything + # that stopped us reaching the assertions is about the board. + if session.client is None: + consecutive_setup_failures += 1 + except Exception as exc: # noqa: BLE001 - one bad config must not end the run + log(f"::error::{target} {config}: unexpected error: {exc}") + failures.append((config, str(exc))) + consecutive_setup_failures += 1 + finally: + endgroup() + + if consecutive_setup_failures: + # One cheap attempt at recovery before spending another cycle, + # and it doubles as diagnosis: whether the board has anything + # queued on its console says a lot about how it is stuck. + session.nudge() + + if consecutive_setup_failures >= MAX_CONSECUTIVE_SETUP_FAILURES: + remaining = names[index:] + log(f"::error::{port} stopped responding - abandoning this board after {consecutive_setup_failures} consecutive setup failures") + if remaining: + log(f" {len(remaining)} config(s) not run: {', '.join(remaining[:8])}{' ...' if len(remaining) > 8 else ''}") + failures.append(("(not run)", f"{len(remaining)} config(s) skipped after {port} stopped responding")) + break + + if not args.keep_going and failures: + break + finally: + group(f"Restore {target} on {port}") + restore_default(session, target) + endgroup() + + return passed, failures, session.crashes, flakes + + +def write_step_summary(results): + """Render the run as a table in the Actions job summary, when there is one.""" + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + + lines = ["## HIL config matrix", "", "| board | target | configs | passed | failed |", "|---|---|---|---:|---:|"] + for board, passed, failures in results: + lines.append(f"| `{board['port']}` | {board['target']} | {len(passed) + len(failures)} | {len(passed)} | {len(failures)} |") + + flaky = [(board, config, reason) for board, _p, _f in results for config, reason in (board.get("flakes") or [])] + if flaky: + lines += ["", "### Flaky (failed once, passed on retry - the board, not the config)", ""] + for board, config, reason in flaky: + lines.append(f"- **{board['target']} `{config}`** - {reason.splitlines()[0]}") + + crashed = [(board, crash) for board, _passed, _failures in results for crash in (board.get("crashes") or [])] + if crashed: + lines += ["", "### Boot crashes (recovered by a retry)", ""] + for board, crash in crashed: + lines.append(f"- **{board['target']}** - {crash.splitlines()[0]}") + + failed = [(board, config, reason) for board, _passed, failures in results for config, reason in failures] + if failed: + lines += ["", "### Failures", ""] + for board, config, reason in failed: + lines.append(f"- **{board['target']} `{config}`** - {reason.splitlines()[0]}") + + with open(path, "a") as handle: + handle.write("\n".join(lines) + "\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--target", action="append", help="only run boards for this target (repeatable), e.g. --target wpc") + parser.add_argument("--configs", help="comma-separated config names to run instead of all of them, e.g. AttackMars_11,Taxi_L4") + parser.add_argument("--limit", type=int, help="stop after this many configs per board - useful for a quick smoke run") + parser.add_argument("--changed-since", metavar="REF", help="run configs changed since REF first, so a config-touching PR fails fast") + parser.add_argument("--skip-flash", action="store_true", help="matrix what is already on the boards instead of building and flashing first") + parser.add_argument("--stop-on-first-failure", dest="keep_going", action="store_false", help="stop a board's matrix at its first failing config (default: run them all)") + # A healthy boot answers in 12-16s on the bench, so the flash harness's + # 150s is generous here and only makes a dead board expensive. + parser.add_argument("--boot-timeout", type=int, default=MATRIX_BOOT_TIMEOUT, help=f"seconds to wait for a board's web server after a reset (default {MATRIX_BOOT_TIMEOUT})") + parser.add_argument("--config-timeout", type=int, default=CONFIG_TIMEOUT, help=f"hard ceiling on one config, in seconds (default {CONFIG_TIMEOUT})") + args = parser.parse_args() + + bench.ensure_tools_on_path() + + workdir = REPO_ROOT / "build" + workdir.mkdir(exist_ok=True) + + board_map = parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")) + trench_coat.rescue_bootsel(board_map, REPO_ROOT / "build" / "hil") + + group("Inventory") + boards = inventory(board_map) + endgroup() + + # A board that will not answer is one board's problem. It cannot be + # identified and so cannot be safely flashed, but the boards that do work + # still have configs worth checking - and aborting the whole run before + # anything is tested is how a single board left wedged by an earlier run + # took out the next one entirely. + # + # Inventory has already run the cheap recovery rungs against these, so + # anything still here needs the destructive one, which is a decision for a + # person rather than for the middle of a matrix. + unresponsive = [b for b in boards if not b.get("responsive", True)] + boards = [b for b in boards if b.get("responsive", True)] + for b in unresponsive: + log(f"::error::{b['port']} is not answering, and draining, resetting and power cycling it did not help - skipping it") + if not boards: + raise CheckFailure("no board on the bench is answering, and the cheap recovery rungs did not bring any back.\nRun dev/hil/recover.py by hand to also reflash them over the ROM bootloader.") + + group("Resolve targets") + boards = resolve_targets(boards, board_map) + for b in boards: + log(f" {b['port']} -> {b['target']}") + # Only when the run is meant to cover the bench: --target says the caller + # asked for one board on purpose, and failing that for incompleteness + # would be answering a question nobody asked. + missing_targets = [] if args.target else bench.check_bench_complete(boards) + endgroup() + + if args.target: + wanted = set(args.target) + boards = [b for b in boards if b["target"] in wanted] + if not boards: + raise CheckFailure(f"no attached board matches --target {', '.join(sorted(wanted))}") + + if not args.skip_flash: + # Building touches no hardware, so it all happens up front. Flashing + # does not - see flash_before_matrix(). + for target in sorted({b["target"] for b in boards}): + group(f"Build {target}") + bench.build(target) + log(f"built {target} at version {bench.source_version(target)}") + endgroup() + + results = [ + ({"port": b["port"], "target": "(unknown)", "crashes": [], "flakes": []}, [], [("(board setup)", "board is not answering, and the cheap recovery rungs did not bring it back")]) + for b in unresponsive + ] + for b in boards: + try: + just_flashed = not args.skip_flash + if just_flashed: + flash_before_matrix(b, workdir) + passed, failures, crashes, flakes = run_matrix(b, args, just_flashed=just_flashed) + except Exception as exc: # noqa: BLE001 + # A board that cannot even be set up is one board's problem. The + # bench is a singleton and a run is expensive, so the other boards + # still get their matrix. Deliberately broad: the first bench run + # died on a TimeoutExpired escaping teardown, which threw away the + # results already in hand and skipped the untouched board entirely. + log(f"::error::{b['target']} on {b['port']}: {exc}") + passed, failures, crashes, flakes = [], [("(board setup)", str(exc))], [], [] + b["crashes"] = crashes + b["flakes"] = flakes + results.append((b, passed, failures)) + + log("") + log("stage timings:") + for title, elapsed in _TIMINGS: + log(f" {elapsed:7.1f}s {title}") + + log("") + log("=" * 60) + total_failures = 0 + total_crashes = 0 + for board, passed, failures in results: + state = "FAIL" if failures else "ok" + crashes = board.get("crashes") or [] + flaky = board.get("flakes") or [] + extra = f", {len(crashes)} boot crash(es)" if crashes else "" + extra += f", {len(flaky)} flaky" if flaky else "" + log(f" {state:5} {board['port']:16} {board['target']:12} {len(passed)} passed, {len(failures)} failed{extra}") + total_failures += len(failures) + total_crashes += len(crashes) + log("=" * 60) + + # A crash that a retry got past is not a passing board. The configs were + # still checked, so it does not fail the run, but it is a firmware fault + # and gets said out loud rather than buried in a green result. + flaky_all = [(board, config, reason) for board, _p, _f in results for config, reason in (board.get("flakes") or [])] + if flaky_all: + log("") + log(f"::warning::{len(flaky_all)} config(s) failed once and passed on retry - the board, not the config:") + for board, config, reason in flaky_all: + log(f" {board['target']} {config}: {reason}") + + if total_crashes: + log("") + log(f"::warning::{total_crashes} boot crash(es) recovered by a retry - the firmware raised on the way up:") + for board, _passed, _failures in results: + for crash in board.get("crashes") or []: + log(f" {board['target']}: {crash}") + + write_step_summary(results) + + if total_failures: + log(f"\n{total_failures} config(s) failed:") + for board, _passed, failures in results: + for config, reason in failures: + log(f" - {board['target']} {config}: {reason}") + return 1 + + checked = sum(len(passed) for _board, passed, _failures in results) + if missing_targets: + log(f"\nall {checked} config(s) booted on the boards that are here, but the bench is missing " + ", ".join(missing_targets)) + return 1 + + log(f"\nall {checked} config(s) booted and reported the right game name") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CheckFailure as exc: + log(f"::error::{exc}") + sys.exit(1) diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index 52a7af9e..caf45486 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -31,57 +31,43 @@ import gzip import json import os -import re -import shutil -import subprocess import sys import time import urllib.error import urllib.request from pathlib import Path -REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(REPO_ROOT / "dev")) - -import serial # noqa: E402 (ships with mpremote) -from usb_coms_demo import UsbApiClient # noqa: E402 - -# A bare bench board has nothing driving the game bus, so this one is correct -# and expected rather than a regression. -EXPECTED_FAULTS = {"HDWR02"} - -# Floating data lines can trip the >250-transition check in main.py and send -# the board down the safe_mode path, where the game config is never loaded. -# Warned about rather than failed, but it invalidates the config assertions - -# see DESIGN.md §8 (G1). -BENCH_WARN_FAULTS = {"HDWR01"} - -# Config key is the config filename without .json (dev/build.py:253). -DEFAULT_GAMENAME = { - "sys11": "GenericSystem11_", - "wpc": "Generic_WPC", - "data_east": "GenericDE_", - "em": "EM_machine_", -} - -# Boot is slow and variable, so we watch the console for the firmware saying -# it is ready rather than guessing at a delay. -# -# "Server: Loop Forever" is the correct marker and the only one: phew prints -# it immediately before loop.run_forever() (phew/server.py:381). The earlier -# "> starting web server on port 80" line is NOT a ready signal - it is -# printed before start_server is even scheduled, let alone bound, so matching -# it returns while the socket is still closed. backend.go() has already run -# connect_to_wifi() by this point, so the marker covers both transports. -READY_MARKER = "Server: Loop Forever" - -# The marker is printed just *before* run_forever(), so give the event loop a -# moment to actually accept the listening socket. http_get's retries cover any -# remainder. -SERVER_SETTLE_SECONDS = 2 - -BOOT_TIMEOUT = 150 -HTTP_TIMEOUT = 10 +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import trench_coat # noqa: E402 +from bench import ( # noqa: E402 + _TIMINGS, + BENCH_WARN_FAULTS, + DEFAULT_GAMENAME, + EXPECTED_FAULTS, + HTTP_TIMEOUT, + REPO_ROOT, + CheckFailure, + UsbApiClient, + _dump_boot_log, + board_map_instructions, + build, + check_bench_complete, + endgroup, + ensure_tools_on_path, + flash_boards, + get, + group, + identify, + inventory, + log, + parse_board_map, + prime_usb, + reset_board, + resolve_targets, + source_version, + wait_for_server, +) # Read-only routes exercised over HTTP. Kept side-effect free so the check can # run against a board repeatedly without changing its state. @@ -104,445 +90,6 @@ ) -class CheckFailure(Exception): - pass - - -def log(msg): - print(msg, flush=True) - - -_TIMINGS = [] -_group = None - - -def group(title): - global _group - _group = (title, time.monotonic()) - print(f"::group::{title}", flush=True) - - -def endgroup(): - global _group - if _group: - title, started = _group - elapsed = time.monotonic() - started - _TIMINGS.append((title, elapsed)) - print(f" [{elapsed:.1f}s]", flush=True) - _group = None - print("::endgroup::", flush=True) - - -# -------------------------------------------------------------------------- -# 1. inventory -# -------------------------------------------------------------------------- - - -def ensure_tools_on_path(): - """Put the bench venv's bin dir on PATH and pick the interpreter to use. - - The harness gets run three ways - from an Actions job, from a login shell, - and by hand - and only the first has the runner's .env applied. dev/build.py - shells out to a bare `mpy-cross` and dev/flash.py to a bare `mpremote`, so - PATH has to be right for subprocesses too, not just for our own calls. - """ - candidates = [] - if os.environ.get("VECTOR_HIL_VENV"): - candidates.append(Path(os.environ["VECTOR_HIL_VENV"]) / "bin") - candidates.append(Path(sys.executable).parent) - candidates.append(REPO_ROOT / ".venv" / "bin") - - for bindir in candidates: - if (bindir / "mpremote").exists(): - os.environ["PATH"] = f"{bindir}{os.pathsep}{os.environ.get('PATH', '')}" - python = bindir / "python" - return str(python) if python.exists() else sys.executable - - if shutil.which("mpremote"): - return sys.executable - - raise CheckFailure( - "mpremote not found. Run with the bench venv, e.g.\n" - f" cd {REPO_ROOT} && PATH=\"$PWD/.venv/bin:$PATH\" .venv/bin/python dev/hil/flash_and_check.py ...\n" - "(VECTOR_HIL_VENV is exported by the runner service, so it is not set in a login shell.)" - ) - - -VENV_PYTHON = sys.executable - - -def mpremote(*args, timeout=60): - return subprocess.run(["mpremote", *args], capture_output=True, text=True, timeout=timeout) - - -def list_ports(): - result = mpremote("devs", timeout=30) - if result.returncode != 0: - raise CheckFailure(f"`mpremote devs` failed: {result.stderr.strip()}") - return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] - - -def probe(port): - """Return {port, chip_id, system, version} for one board. - - chip_id comes from the RP2040 itself so it survives any firmware state; - system/version come from the flashed firmware and may be missing if the - board is unflashed or broken. - """ - board = {"port": port, "chip_id": None, "system": None, "version": None} - - chip = mpremote( - "connect", port, "exec", - "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", - timeout=30, - ) - if chip.returncode == 0: - board["chip_id"] = chip.stdout.strip() - - info = mpremote( - "connect", port, "exec", - "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)", - timeout=30, - ) - if info.returncode == 0 and info.stdout.strip(): - parts = info.stdout.split() - board["system"] = parts[0] - if len(parts) > 1: - board["version"] = parts[1] - - return board - - -def inventory(): - boards = [probe(port) for port in list_ports()] - if not boards: - raise CheckFailure("no boards found - check the USB hub and power") - - log(f"{'port':16} {'chip id':18} {'running':12} version") - for b in boards: - log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") - return boards - - -IDENTIFY_SNIPPET = """ -import machine, time -try: - import BoardLED as L - L.startUp() -except Exception: - L = None -led = machine.Pin("LED", machine.Pin.OUT) -for i in range({blinks}): - led.on() - if L: - L.ledColor(L.BLUE) - time.sleep(0.25) - led.off() - if L: - L.ledColor(L.BLACK) - time.sleep(0.25) -""" - - -def identify(boards, seconds=8): - """Blink each board in turn so a human can tell which is which. - - Uses the Pico W onboard LED, which works from the REPL no matter what - firmware is loaded, plus the Vector board's WS2812 in blue when the - flashed firmware happens to provide the driver. - """ - log(f"Blinking each board for ~{seconds}s. Watch the bench and note the order.") - log("") - for index, board in enumerate(boards, 1): - log(f" [{index}/{len(boards)}] BLINKING NOW: {board['port']} chip {board['chip_id']}") - result = mpremote( - "connect", board["port"], "exec", - IDENTIFY_SNIPPET.format(blinks=int(seconds / 0.5)), - timeout=seconds + 30, - ) - if result.returncode != 0: - log(f" could not blink this board: {result.stderr.strip()}") - else: - log(" done") - log("") - log("Now map what you saw to the chip ids, and put this in the runner's .env:") - log("") - log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) - - -# -------------------------------------------------------------------------- -# 2. resolve -# -------------------------------------------------------------------------- - - -def parse_board_map(raw): - """Parse VECTOR_HIL_BOARD_MAP: 'chipid=target,chipid=target'.""" - mapping = {} - for entry in (raw or "").split(","): - entry = entry.strip() - if not entry: - continue - if "=" not in entry: - raise CheckFailure(f"bad VECTOR_HIL_BOARD_MAP entry {entry!r}, expected chipid=target") - chip, target = entry.split("=", 1) - mapping[chip.strip()] = target.strip() - return mapping - - -def resolve_targets(boards, board_map): - """Decide the target for each board, refusing to guess when it matters. - - An explicit chip-id map is authoritative. Without one we fall back to what - each board's firmware reports, which is only trustworthy when every board - reports something different - if two boards claim the same system, that is - the signature of a previous mis-flash rather than of the hardware, and - flashing on that basis would silently perpetuate it. - """ - if board_map: - unmapped = [b for b in boards if b["chip_id"] not in board_map] - if unmapped: - raise CheckFailure( - "VECTOR_HIL_BOARD_MAP is set but does not cover: " - + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) - ) - for b in boards: - b["target"] = board_map[b["chip_id"]] - log("targets from VECTOR_HIL_BOARD_MAP") - return boards - - missing = [b for b in boards if not b["system"]] - if missing: - raise CheckFailure( - "cannot identify " - + ", ".join(b["port"] for b in missing) - + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP." - ) - - systems = [b["system"] for b in boards] - duplicates = {s for s in systems if systems.count(s) > 1} - if duplicates: - raise CheckFailure( - "refusing to flash from autodetection: " - + ", ".join(sorted(duplicates)) - + " is reported by more than one board.\n" - "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" - "at least one board is running firmware for a system it is not wired for.\n" - "Pin them explicitly instead, using the chip ids above:\n" - " VECTOR_HIL_BOARD_MAP=" - + ",".join(f"{b['chip_id']}=" for b in boards) - ) - - for b in boards: - b["target"] = b["system"] - log("targets from firmware self-report (all distinct)") - return boards - - -# -------------------------------------------------------------------------- -# 3. build -# -------------------------------------------------------------------------- - - -def source_version(target): - config = REPO_ROOT / "src" / target / "systemConfig.py" - match = re.search(r'SystemVersion\s*=\s*"([^"]+)"', config.read_text()) - if not match: - raise CheckFailure(f"could not read SystemVersion from {config}") - return match.group(1) - - -def build(target): - build_dir = REPO_ROOT / "build" / target - result = subprocess.run( - [VENV_PYTHON, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, - ) - if result.returncode != 0: - log(result.stdout[-3000:]) - log(result.stderr[-3000:]) - raise CheckFailure(f"build failed for {target}") - return build_dir - - -# -------------------------------------------------------------------------- -# 4. flash -# -------------------------------------------------------------------------- - - -def write_bench_config(target, workdir): - ssid = os.environ.get("VECTOR_HIL_WIFI_SSID", "") - password = os.environ.get("VECTOR_HIL_WIFI_PASSWORD", "") - game_password = os.environ.get("VECTOR_HIL_GAME_PASSWORD", "hiltest") - - if not ssid or not password: - raise CheckFailure("VECTOR_HIL_WIFI_SSID / VECTOR_HIL_WIFI_PASSWORD are not set") - - # dev/flash.py builds a MicroPython snippet with single-quoted values, so a - # single quote anywhere here would produce a syntax error on the board - # rather than an obvious failure here. - for name, value in (("ssid", ssid), ("password", password), ("game password", game_password)): - if "'" in value or "\\" in value: - raise CheckFailure(f"bench {name} contains a quote or backslash, which dev/flash.py cannot write") - - config = { - "ssid": ssid, - "password": password, - "gamename": DEFAULT_GAMENAME[target], - "Gpassword": game_password, - } - path = workdir / f"hil-config-{target}.json" - path.write_text(json.dumps(config)) - return path - - -def flash(target, port, build_dir, config_path): - result = subprocess.run( - [VENV_PYTHON, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, - ) - if result.returncode != 0: - log(result.stdout[-3000:]) - log(result.stderr[-3000:]) - raise CheckFailure(f"flash failed for {target} on {port}") - - -# -------------------------------------------------------------------------- -# 5. health -# -------------------------------------------------------------------------- - - -def reset_board(port): - """Reset the board so we own the boot we are about to watch. - - The ready marker is printed exactly once per boot. dev/flash.py already - resets at the end of flashing, but flashing runs over every board before - any health check starts, so by the time we open a console the board booted - a minute ago and the marker is long gone. Resetting here makes the wait - deterministic and the reported boot time meaningful. - """ - result = mpremote( - "connect", port, "exec", "--no-follow", "import machine; machine.reset()", - timeout=30, - ) - if result.returncode != 0: - raise CheckFailure(f"could not reset {port} before the health check: {result.stderr.strip()}") - - -def wait_for_server(port, timeout=BOOT_TIMEOUT): - """Watch the boot console until the firmware reports its web server is up. - - Polling an API that is not listening yet tells you nothing about why, and - burns the whole timeout when a board fails to boot. Reading the console - instead gives an exact ready signal and, on failure, the boot log that - explains it. - - Returns the open serial connection so the USB API can reuse it - the - Pico exposes one CDC endpoint, so a second connection would fight this one. - """ - deadline = time.monotonic() + timeout - transcript = [] - connection = None - - while time.monotonic() < deadline: - if connection is None: - try: - # The port disappears and re-enumerates across the reset, so a - # failure to open here is expected for the first second or two. - connection = serial.Serial(port=port, baudrate=115200, timeout=1) - except Exception: - time.sleep(1) - continue - try: - raw = connection.readline() - except Exception: - try: - connection.close() - except Exception: - pass - connection = None - continue - - if not raw: - continue - text = raw.decode(errors="replace").rstrip("\r\n") - if not text: - continue - transcript.append(text) - - if READY_MARKER in text: - elapsed = timeout - (deadline - time.monotonic()) - log(f" server up after {elapsed:.1f}s ({text.strip()!r})") - time.sleep(SERVER_SETTLE_SECONDS) - return connection, transcript - - if connection is not None: - try: - connection.close() - except Exception: - pass - - tail = "\n ".join(transcript[-20:]) or "(nothing on the console)" - raise CheckFailure( - f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}" - ) - - -def prime_usb(connection): - """Clear both ends of the serial line before the first API request. - - usb_comms accumulates stdin characters into a module-level `buffer` until - it sees a newline (usb_comms.py:132). Anything left there without a - terminator - a partial line, stray bytes from the raw-REPL session that - issued the reset - silently prefixes the next request, so the board parses - a route like "\x02/api/version", fails the `_routes` lookup and answers - 404. A lone newline flushes whatever is pending into a discarded request. - """ - try: - connection.reset_input_buffer() - connection.reset_output_buffer() - connection.write(b"\n") - connection.flush() - except Exception as exc: - log(f" warning: could not prime the USB link: {exc}") - return - # usb_request_handler is scheduled every 1000ms (phew/server.py:342), so - # give it a turn to consume the flush before the first real request. - time.sleep(1.5) - try: - connection.reset_input_buffer() - except Exception: - pass - - -def get(client, route, expect=200): - response = client.send_and_receive(route=route, payload=None, timeout=15) - status = response.get("status") - if status != expect: - # The board narrates its own routing failures ("USB REQ: route not - # found: ..."), but the client discards every line that is not a - # response. Drain whatever is pending so the reason is visible. - raise CheckFailure( - f"{route} returned {status}, expected {expect}" - f"{_drain_serial(client.ser)}" - ) - return response.get("body") - - -def _drain_serial(connection, limit=12): - """Return any pending board chatter, formatted for an error message.""" - try: - time.sleep(0.5) - pending = connection.read(connection.in_waiting or 0) - except Exception: - return "" - if not pending: - return "" - lines = [line for line in pending.decode(errors="replace").splitlines() if line.strip()] - if not lines: - return "" - return "\n board said: " + "\n board said: ".join(lines[:limit]) - - def check_faults(board): faults = get(board["client"], "/api/fault") or [] if isinstance(faults, dict): @@ -557,8 +104,7 @@ def check_faults(board): warned = codes & BENCH_WARN_FAULTS if warned: - log(f"::warning::{board['port']} raised {sorted(warned)} - bare-board bus noise, " - "the board is in safe mode and the game config was NOT loaded") + log(f"::warning::{board['port']} raised {sorted(warned)} - bare-board bus noise, " "the board is in safe mode and the game config was NOT loaded") return warned @@ -690,19 +236,14 @@ def health_check_http(board): http_configs = payloads["/api/game/configs_list"] if len(http_configs) != board["usb_config_count"]: - raise CheckFailure( - f"http lists {len(http_configs)} configs, USB lists {board['usb_config_count']}" - ) + raise CheckFailure(f"http lists {len(http_configs)} configs, USB lists {board['usb_config_count']}") # Authentication is enforced over HTTP and deliberately bypassed over USB # (backend.py:280), so this is the only transport that can prove the gate # works. password_check is the one auth route with no side effects. status = http_status(f"{base}/api/auth/password_check") if status != 401: - raise CheckFailure( - f"/api/auth/password_check returned {status} without credentials, expected 401 - " - "HTTP authentication is not being enforced" - ) + raise CheckFailure(f"/api/auth/password_check returned {status} without credentials, expected 401 - " "HTTP authentication is not being enforced") log(f" GET {'/api/auth/password_check':34} 401 (auth enforced, as expected)") # A challenge must not be reusable: the handler deletes it on use. @@ -726,32 +267,16 @@ def _summarise(payload): return repr(payload)[:60] -# -------------------------------------------------------------------------- - - -def _dump_boot_log(board, lines=25): - """Show what the board actually said. A failed health check is usually - explained by the boot output, and by this point we already have it.""" - transcript = board.get("boot_log") - if not transcript: - return - log(f" last {min(lines, len(transcript))} lines of {board['port']} boot console:") - for line in transcript[-lines:]: - log(f" {line}") - - def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--skip-http", action="store_true", help="USB checks only; do not exercise the network stack") parser.add_argument("--skip-flash", action="store_true", help="health-check what is already on the boards") - parser.add_argument("--inventory-only", action="store_true", - help="print each board's chip id and stop - use this to build VECTOR_HIL_BOARD_MAP") - parser.add_argument("--identify", action="store_true", - help="blink each board in turn so you can tell which physical board is which") + parser.add_argument("--inventory-only", action="store_true", help="print each board's chip id and stop - use this to build VECTOR_HIL_BOARD_MAP") + parser.add_argument("--identify", action="store_true", help="blink each board in turn so you can tell which physical board is which") args = parser.parse_args() - global VENV_PYTHON - VENV_PYTHON = ensure_tools_on_path() + ensure_tools_on_path() + board_map = parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")) if args.identify: group("Inventory") @@ -766,30 +291,31 @@ def main(): group("Inventory") boards = inventory() endgroup() - log("") - log("The boards are dedicated to the bench, so pin them by chip id once and") - log("autodetection stops mattering. Put this in the runner's .env, filling in") - log("the target for each (sys11, wpc, data_east, em, whitestar, classic):") - log("") - log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) - log("") - log("Then: cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start") + log(board_map_instructions(boards, board_map)) return 0 workdir = REPO_ROOT / "build" workdir.mkdir(exist_ok=True) failures = [] + # A board in BOOTSEL has no serial port, so it is invisible to every stage + # below. Put firmware back on it first and it joins the run normally. + trench_coat.rescue_bootsel(board_map, REPO_ROOT / "build" / "hil") + group("Inventory") - boards = inventory() + boards = inventory(board_map) endgroup() group("Resolve targets") - boards = resolve_targets(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP"))) + boards = resolve_targets(boards, board_map) for b in boards: log(f" {b['port']} -> {b['target']}") + missing = check_bench_complete(boards) endgroup() + if missing: + failures.append("the bench is missing " + ", ".join(missing)) + if not args.skip_flash: for target in sorted({b["target"] for b in boards}): group(f"Build {target}") @@ -797,17 +323,13 @@ def main(): log(f"built {target} at version {source_version(target)}") endgroup() + group(f"Flash {len(boards)} board(s)") + errors = flash_boards(boards, workdir) for b in boards: - group(f"Flash {b['target']} on {b['port']}") - try: - config_path = write_bench_config(b["target"], workdir) - flash(b["target"], b["port"], REPO_ROOT / "build" / b["target"], config_path) - log("flashed") - except CheckFailure as exc: - log(f"::error::{exc}") - failures.append(f"{b['port']} ({b['target']}): {exc}") + if b["port"] in errors: + failures.append(f"{b['port']} ({b['target']}): {errors[b['port']]}") b["skip"] = True - endgroup() + endgroup() for b in boards: if b.get("skip"): @@ -856,10 +378,12 @@ def main(): for b in boards: state = "FAIL" if any(b["port"] in f for f in failures) else "ok" log(f" {state:5} {b['port']:16} {b['target']:12} {b.get('ip') or ''}") + for target in missing: + log(f" {'ABSENT':5} {'-':16} {target}") log("=" * 60) if failures: - log(f"\n{len(failures)} board(s) failed:") + log(f"\n{len(failures)} failure(s):") for failure in failures: log(f" - {failure}") return 1 diff --git a/dev/hil/recover.py b/dev/hil/recover.py new file mode 100644 index 00000000..4dbbf8e9 --- /dev/null +++ b/dev/hil/recover.py @@ -0,0 +1,545 @@ +#!/usr/bin/env python3 +"""Bring a wedged bench board back, without anyone walking over to it. + + cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/recover.py + +A Vector board can deadlock in a way that leaves the USB device enumerated but +the firmware gone: the port is there, `mpremote` opens it, and nothing ever +answers. TrenchCoat names the mechanism in its own source (`src/ray.py`, +`send_command`): + + if nothing ever drains the board's output, the USB CDC buffers fill up, + MicroPython blocks writing to stdout, and the board deadlocks mid-script + +That is what happened to the WPC board on the bench: silent console, no reply +to Ctrl-C, for an hour. `dev/hil/config_matrix.py` no longer creates the +condition, but a board already in it needs getting out, and the bench is +supposed to run unattended. + +So this escalates, cheapest first, re-testing after each step: + + 1. drain - read whatever the board has queued and send Ctrl-C. If it is + blocked writing to a full CDC buffer, reading is the remedy. + 2. usb reset - ask the kernel to re-enumerate the device (USBDEVFS_RESET). + Resets the host side of the link and TinyUSB's endpoint + state, which can free a write that is stuck on a buffer the + host was not draining. + 3. power - cut and restore power to the board's hub port with uhubctl. + The board is USB bus powered (Trench-Coat-Install-Guide.md), + so this is a real power cycle, not a signal. Needs a hub that + supports per-port power switching. + 4. reflash - hand the board to TrenchCoat, the team's own tool for + recovering a board without the BOOTSEL button. It resets into + the ROM bootloader, wipes the whole flash with nuke.uf2, then + writes the real firmware. Destructive: the board needs + `dev/hil/flash_and_check.py` afterwards to get Vector back. + +Only step 4 needs anything from outside the repo: a checkout of +warped-pinball/trench-coat, pinned by commit. dev/hil/trench_coat.py drives +it - the sequence is not reimplemented here. + +If all four fail, the board needs a person: hold BOOTSEL while replugging it. +""" + +import argparse +import fcntl +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import bench # noqa: E402 +import trench_coat # noqa: E402 +from bench import ( # noqa: E402 + CTRL_C, + DRAIN_QUIET_SECONDS, + DRAIN_SECONDS, + REPO_ROOT, + CheckFailure, + endgroup, + ensure_tools_on_path, + group, + interrupt_board, + list_ports, + log, + mpremote, + open_serial, + parse_board_map, + read_until_quiet, + summary, + time_limit, +) + +# Anything worth saying about a recovery is worth saying in the job summary +# too - see bench.summary for why. +note = summary + +# ioctl number for USBDEVFS_RESET, from : _IO('U', 20). +USBDEVFS_RESET = ord("U") << 8 | 20 + +PROBE_TIMEOUT = 20 +SETTLE_SECONDS = 5 + +# No single step may hang the job. Generous enough for the slowest one - the +# TrenchCoat reflash, which clones, waits for a drive, and writes two UF2s - +# and still short of the workflow's own timeout, so the run always gets to +# print its summary. +STEP_TIMEOUT = 600 + + +def responsive(port, timeout=PROBE_TIMEOUT): + """Can we still get a REPL out of this board? + + Deliberately the cheapest question that distinguishes "wedged" from + "busy": a board running the Vector application answers this, because + mpremote interrupts it to do so. + """ + try: + result = mpremote("connect", port, "exec", "print('alive')", timeout=timeout) + except subprocess.TimeoutExpired: + return False + return result.returncode == 0 and "alive" in result.stdout + + +def survey(board_map): + """Report which ports answer, and work out what each dead one should be. + + A wedged board cannot tell us its chip id, so it cannot be looked up in + VECTOR_HIL_BOARD_MAP directly. What it can be is deduced: ask every board + that does answer who it is, and whatever targets the map still expects are + the dead ones. With one board down and the rest alive, that is exact. + """ + ports = list_ports() + if not ports: + raise CheckFailure(bench.no_boards_message()) + + alive, dead, claimed = [], [], set() + note("") + note("### Boards") + note("") + note("```") + note(f"{'port':16} {'state':14} chip id") + for port in ports: + if not responsive(port): + dead.append(port) + note(f"{port:16} {'NOT ANSWERING':14} -") + continue + alive.append(port) + try: + chip = mpremote("connect", port, "exec", "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", timeout=30) + chip_id = chip.stdout.strip() if chip.returncode == 0 else None + except subprocess.TimeoutExpired: + # Answered once and not the second time. Not our problem to solve + # here, but it does mean we cannot attribute a target to it. + chip_id = None + if chip_id in board_map: + claimed.add(board_map[chip_id]) + note(f"{port:16} {'ok':14} {chip_id or '?'}") + + # Probing a board means interrupting it, which leaves it sitting at a + # bare REPL with the Vector application stopped. A tool that only meant + # to look must put it back, or a "nothing to recover" run quietly + # leaves the whole bench not running its firmware. + try: + mpremote("connect", port, "exec", "--no-follow", "import machine; machine.reset()", timeout=30) + except subprocess.TimeoutExpired: + log(f" warning: could not restart {port} after probing it") + + note("```") + + unclaimed = sorted(set(board_map.values()) - claimed) + targets = {} + if len(dead) == 1 and len(unclaimed) == 1: + targets[dead[0]] = unclaimed[0] + log(f"\n{dead[0]} is the only board not answering and {unclaimed[0]} is the only target unaccounted for, so that is what it is") + elif dead: + log(f"\ncannot tell which target {', '.join(dead)} should be ({len(unclaimed)} unaccounted for: {', '.join(unclaimed) or 'none'})") + log("a reflash needs --target to say which UF2 to use") + + return alive, dead, targets + + +# -------------------------------------------------------------------------- +# 1. drain +# -------------------------------------------------------------------------- + + +def drain(port, seconds=DRAIN_SECONDS): + """Read whatever is queued until the board goes quiet, then interrupt it. + + This is the only rung of the ladder this runner can always reach - the USB + reset needs a udev rule and the power cycle needs uhubctl, and neither is + installed - so it is worth being patient here. A bench run proved the + point: a board written off after a three-second drain of 63 bytes was read + with `cat` moments later and turned out to be a healthy, running server + that had merely blocked on a full output buffer. + """ + try: + connection = open_serial(port) + except Exception as exc: + log(f" could not open {port}: {exc}") + return False + + drained = 0 + interrupted = False + try: + drained += read_until_quiet(connection, seconds) + # Ctrl-C twice: once to break out of a sleep, once for whatever the + # first one dropped us into. + interrupted = interrupt_board(connection, port, CTRL_C + CTRL_C) + drained += read_until_quiet(connection, DRAIN_QUIET_SECONDS * 2) + except Exception as exc: + log(f" error draining {port}: {exc}") + finally: + try: + connection.close() + except Exception: + pass + + log(f" drained {drained} byte(s) and {'sent Ctrl-C' if interrupted else 'could not send Ctrl-C'}") + + # What the two directions did is the diagnosis, and they point at + # different remedies. + if drained and not interrupted: + log(" this board is talking but not listening: it is producing output") + log(" normally and never servicing what we send it, so no amount of") + log(" reading will reach it. That is what the USB reset and power") + log(" cycle rungs are for - see RUNNER_SETUP.md if they are skipped") + log(" below, because without them there is nothing left to try.") + elif not drained: + # A board merely blocked on a full buffer has a backlog to give up the + # moment somebody reads. + log(" (nothing queued - so it is not simply blocked on a full output buffer)") + + return True + + +# -------------------------------------------------------------------------- +# 2. usb reset +# -------------------------------------------------------------------------- + + +def usb_device_path(port): + """Map /dev/ttyACM* to its /dev/bus/usb/BBB/DDD node.""" + name = os.path.basename(port) + link = Path(f"/sys/class/tty/{name}/device") + try: + node = link.resolve() + except OSError as exc: + log(f" cannot resolve {link}: {exc}") + return None + + # The tty hangs off the CDC interface; the device is its parent. + for candidate in [node] + list(node.parents): + busnum = candidate / "busnum" + devnum = candidate / "devnum" + if busnum.exists() and devnum.exists(): + return Path(f"/dev/bus/usb/{int(busnum.read_text()):03d}/{int(devnum.read_text()):03d}") + log(f" could not find the USB device behind {port}") + return None + + +def usb_reset(port): + """Ask the kernel to re-enumerate the device behind `port`.""" + node = usb_device_path(port) + if node is None: + return False + log(f" resetting {node}") + try: + with open(node, "wb") as handle: + fcntl.ioctl(handle.fileno(), USBDEVFS_RESET, 0) + except PermissionError: + log(f" permission denied on {node} - the runner user needs write access to it (a udev rule, or the plugdev group)") + return False + except OSError as exc: + log(f" ioctl failed: {exc}") + return False + return True + + +# -------------------------------------------------------------------------- +# 3. power cycle +# -------------------------------------------------------------------------- + + +def hub_location(port): + """Return (hub, port number) for uhubctl, from the device's sysfs path. + + A USB path looks like 1-1.4:1.0 - bus 1, hub at 1-1, port 4. uhubctl wants + the hub and the port separately. + + Returns None for a device sitting directly on a root hub (1-1:1.0). That is + not a case worth handling: cutting power there takes down the whole bus and + every other board on it, which on this bench means killing the two healthy + boards to recover one. Only a downstream hub port is ours to switch. + """ + name = os.path.basename(port) + try: + node = Path(f"/sys/class/tty/{name}/device").resolve() + except OSError: + return None + + for candidate in [node] + list(node.parents): + if (candidate / "busnum").exists(): + usb_path = candidate.name.split(":")[0] + if "." in usb_path: + hub, _, portnum = usb_path.rpartition(".") + return hub, portnum + log(f" {port} is on a root hub ({usb_path}); power cycling it would cut every board on the bus") + return None + return None + + +def power_cycle(port, off_seconds=3): + """Cut and restore power to the board's hub port. + + The board is powered off the USB cable, so this is the real thing - a cold + boot, with no dependence on the firmware being alive to cooperate. Needs a + hub with per-port power switching; plenty do not have it, which is why this + reports rather than fails. + """ + if not shutil.which("uhubctl"): + log(" uhubctl is not installed - skipping the power cycle (sudo apt install uhubctl)") + return False + + location = hub_location(port) + if location is None: + log(f" could not work out which hub port {port} is on") + return False + hub, portnum = location + + log(f" power cycling hub {hub} port {portnum}") + for action in ("off", "on"): + result = subprocess.run(["uhubctl", "--location", hub, "--ports", portnum, "--action", action], capture_output=True, text=True, timeout=60) + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + log(f" uhubctl {action} failed: {detail}") + if "No compatible" in detail or "not support" in detail: + log(" this hub cannot switch port power - a smart hub is the only way to make this step work") + return False + if action == "off": + time.sleep(off_seconds) + return True + + +# -------------------------------------------------------------------------- +# 4. reflash, using TrenchCoat +# -------------------------------------------------------------------------- + + +def can_complete_a_reflash(): + """Is there any way this runner could write a UF2 once the board is in BOOTSEL? + + Asked *before* anything touches the board into its bootloader, because that + is a one-way door: it takes a board that is at least enumerated as a serial + device and turns it into a mass-storage device that only a UF2 (or a + replug) gets it out of. Doing that with no way to finish the job makes the + board harder to recover, not easier. + """ + # picotool first: it is the only route that does not depend on the board + # presenting a usable drive, which is exactly the case that stranded a + # bench board here (enumerated, in BOOTSEL, no mountable filesystem). + if trench_coat.picotool_available(): + return True, "picotool can write over the bootrom, with or without a drive" + if trench_coat.find_bootloader_drives(): + return True, "a bootloader drive is already mounted" + if shutil.which("udisksctl"): + return True, "udisksctl is available to mount the drive" + if any(Path(root).is_dir() and os.access(root, os.W_OK) for root in trench_coat.MOUNT_ROOTS): + return True, "an automount directory is writable" + return False, "nothing here can flash a board in BOOTSEL (no picotool, no udisksctl, no writable automount directory)" + + +def reflash(port, target, cache_dir, force=False): + """Hand the board to TrenchCoat, which is the tool for exactly this job. + + Not reimplemented here on purpose. TrenchCoat is what the team uses to + recover a board without the BOOTSEL button, and its sequence includes the + step a naive UF2 copy misses: a nuke.uf2 wipe of the whole flash before the + real firmware, so nothing from the old filesystem survives. See + dev/hil/trench_coat.py for how it is pointed at one board instead of all of + them. + """ + possible, why = can_complete_a_reflash() + log(f" {'can' if possible else 'cannot'} finish a reflash here: {why}") + if not possible and not force: + log(" not touching the board into BOOTSEL, because that would leave it as a mass-storage") + log(" device with no way to flash it - strictly worse than how it is now. Install udisksctl") + log(" on the runner (or pass --force-bootsel) to make this step usable.") + return False + + return trench_coat.flash(port, target, cache_dir / "trench-coat") + + +# -------------------------------------------------------------------------- + + +def recover(port, target, args): + """Escalate until the board answers, or until we run out of ideas.""" + steps = [("drain the console", lambda: drain(port)), ("reset the USB device", lambda: usb_reset(port))] + if not args.no_power_cycle: + steps.append(("power cycle the hub port", lambda: power_cycle(port))) + if args.reflash: + if target: + steps.append(("reflash MicroPython over the ROM bootloader", lambda: reflash(port, target, args.cache_dir, args.force_bootsel))) + else: + log(f"::warning::{port}: skipping the reflash step - no target known for this board, pass --target") + + note("") + note(f"### Recovering {port}" + (f" ({target})" if target else "")) + note("") + + for name, step in steps: + group(f"{port}: {name}") + try: + with time_limit(args.step_timeout, name): + attempted = step() + except CheckFailure as exc: + log(f"::error::{exc}") + attempted = False + except Exception as exc: # noqa: BLE001 - a failed recovery step is not a crash + log(f"::error::unexpected error: {exc}") + attempted = False + + if attempted: + time.sleep(SETTLE_SECONDS) + if responsive(port): + note(f"- **{port}: recovered by {name}**") + endgroup() + return name + note(f"- {port}: tried {name} - still not answering") + else: + note(f"- {port}: could not attempt {name}") + endgroup() + + return None + + +def preflight(): + """Report which recovery steps are actually available here. + + A recovery that fails is still worth the run if it says precisely which + one-time bit of runner setup would have made the bench self-healing. Each + line below is a step that either works or names what it needs. + """ + ports = [] + try: + ports = list_ports() + except CheckFailure: + pass + + note("### What this runner can do") + note("") + note("```") + note(f" serial ok {len(ports)} port(s) visible") + + stranded = bench.bootsel_boards() + if stranded: + note(f" bootsel found {len(stranded)} board(s) in the ROM bootloader: " + ", ".join(b["chip_id"] or "?" for b in stranded)) + + node = usb_device_path(ports[0]) if ports else None + if node is None: + note(" usb reset unknown no device to check") + elif os.access(node, os.W_OK): + note(f" usb reset ok {node} is writable") + else: + note(f" usb reset no {node} is not writable - needs a udev rule granting the runner user write access") + + if shutil.which("uhubctl"): + note(" power ok uhubctl is installed (still needs a hub that switches port power)") + else: + note(" power no uhubctl not installed - `sudo apt install uhubctl`") + + possible, why = can_complete_a_reflash() + note(f" reflash {'ok ' if possible else 'no '} {why}") + note(f" {'':11} {'':9} TrenchCoat pinned at {trench_coat.TRENCH_COAT_COMMIT[:8]}") + note("```") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--port", action="append", help="recover only this port (repeatable); default is every board that is not answering") + parser.add_argument("--target", help="target for the dead board, e.g. wpc - only needed when it cannot be deduced") + parser.add_argument("--no-reflash", dest="reflash", action="store_false", help="stop before replacing the firmware; the board is left as found if the cheaper steps fail") + parser.add_argument("--no-power-cycle", action="store_true", help="skip the uhubctl step") + parser.add_argument("--force-bootsel", action="store_true", help="touch the board into BOOTSEL even when nothing here can mount the drive to flash it") + parser.add_argument("--step-timeout", type=int, default=STEP_TIMEOUT, help=f"hard ceiling on any one recovery step, in seconds (default {STEP_TIMEOUT})") + parser.add_argument("--cache-dir", type=Path, default=REPO_ROOT / "build" / "hil", help="where to keep the trench-coat checkout") + args = parser.parse_args() + + ensure_tools_on_path() + + group("What this runner can do") + preflight() + endgroup() + + board_map = parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")) + + # Rung 0, before the ladder proper. A board in BOOTSEL has no serial port, + # so none of the four steps below can even be attempted on it - and it is + # the one state that is always repairable, because the bootloader is + # sitting there waiting for a UF2. + trench_coat.rescue_bootsel(board_map, args.cache_dir) + + group("Survey") + alive, dead, targets = survey(board_map) + endgroup() + + if args.port: + dead = [port for port in args.port] + log(f"recovering {', '.join(dead)} because --port says so") + if not dead: + log(f"\nall {len(alive)} board(s) are answering - nothing to recover") + return 0 + + recovered, lost = [], [] + for port in dead: + target = args.target or targets.get(port) + log("") + log(f"recovering {port}" + (f" (expected to be {target})" if target else " (target unknown)")) + method = recover(port, target, args) + if method: + recovered.append((port, method)) + else: + lost.append(port) + + log("") + log("=" * 60) + for port, method in recovered: + log(f" recovered {port:16} by: {method}") + for port in lost: + log(f" NOT ANSWERING {port}") + log("=" * 60) + + if lost: + log("") + log(f"{len(lost)} board(s) need a person at the bench:") + log(" hold the BOOTSEL button while replugging the USB cable, then run") + log(" dev/hil/flash_and_check.py to put Vector back on it.") + # A board that would not answer is not necessarily a board that is + # gone, and saying so matters: "still dead" sent a maintainer looking + # for a bricked Pico when the board in question was running the + # application and printing to its console the whole time, and only + # ever refused input. + log("") + log(" (a board here has failed every rung this runner can reach, which is not") + log(" the same as being bricked - check the drain output above for what it was") + log(" doing, and RUNNER_SETUP.md for the rungs that were skipped)") + return 1 + + log("") + log("Every board answers again. A board recovered by the reflash step is running") + log("bare MicroPython, so run dev/hil/flash_and_check.py before trusting the bench.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CheckFailure as exc: + log(f"::error::{exc}") + sys.exit(1) diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py new file mode 100644 index 00000000..081c523e --- /dev/null +++ b/dev/hil/trench_coat.py @@ -0,0 +1,662 @@ +#!/usr/bin/env python3 +"""Drive warped-pinball/trench-coat to reflash a bench board. + +TrenchCoat is the tool the Warped Pinball team already uses to recover boards +without touching the BOOTSEL button, so the bench uses *it* rather than a +second implementation of the same delicate sequence. Its `flash_firmware()` +does the part that matters and that a naive "copy a UF2" misses: + + bootloader -> nuke.uf2 -> wait for the drive to cycle -> real UF2 -> wait + for the board to re-enumerate as a serial device + +The `nuke.uf2` wipe is the load-bearing step. It erases the whole flash, so +nothing from the old filesystem survives into the new firmware - which is what +makes this a recovery rather than an upgrade. + +Two things have to be adapted for the bench, and both are done by narrowing +what TrenchCoat can see rather than by changing what it does: + + * It flashes *every* board it finds. On the bench that would nuke the two + healthy boards along with the broken one, so `Ray.find_board_ports` is + narrowed to the single port being recovered. + * It finds bootloader drives by looking for INFO_UF2.TXT under /media and + /Volumes, which assumes a desktop automounter. A headless runner has none, + so `list_rpi_rp2_drives` is wrapped to mount the RPI-RP2 volume with + udisksctl first. + +A board found *already* in BOOTSEL is the one case its `flash_firmware` cannot +be narrowed to - see the second half of this file for why, and for the copy +sequence that replaces it there. It keeps the parts that matter: the nuke.uf2 +wipe, and the pinned UF2 bundle from this checkout. + +Only `src.core`, `src.ray`, `src.ui` and `src.util` are imported, and between +them they need nothing but pyserial - which the bench venv already has because +mpremote ships it. `src.main` and `src.interactive` are the parts that want +InquirerPy and a human, and neither is used here. +""" + +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import bench # noqa: E402 +from bench import ( # noqa: E402 + SERIAL_WRITE_TIMEOUT, + CheckFailure, + endgroup, + group, + log, + open_serial, +) + +# Pinned, like every other third-party input to this bench. Bumping it means +# reviewing what changed in the flashing sequence first. +TRENCH_COAT_COMMIT = "26e6d508c362bed1f6d1323155435c18528de758" +TRENCH_COAT_URL = "https://github.com/warped-pinball/trench-coat" + +# Which bundled UF2 belongs to which bench target. +TARGET_UF2 = { + "wpc": "Vector_WPC_v5.uf2", + "data_east": "Vector_DataEast_v1.uf2", + "sys11": "vector_system_11_and_9_v4.uf2", + "em": "Vector_WPC_v5.uf2", # EM runs on the WPC OS (see TrenchCoat's series menu) +} + + +def clone(root, commit=TRENCH_COAT_COMMIT): + """Make sure `root` holds trench-coat at exactly `commit`.""" + root = Path(root) + if not (root / ".git").is_dir(): + root.parent.mkdir(parents=True, exist_ok=True) + log(f" cloning trench-coat into {root}") + subprocess.run(["git", "clone", "--quiet", TRENCH_COAT_URL, str(root)], check=True, timeout=600) + + subprocess.run(["git", "-C", str(root), "fetch", "--quiet", "origin", commit], check=True, timeout=600) + subprocess.run(["git", "-C", str(root), "checkout", "--quiet", commit], check=True, timeout=120) + + head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], capture_output=True, text=True, timeout=60).stdout.strip() + if head != commit: + raise CheckFailure(f"trench-coat checkout is at {head}, expected the pinned {commit}") + log(f" trench-coat at {commit[:8]}") + return root + + +def load(root): + """Import TrenchCoat's modules from `root` and return (core, ray). + + Guarded because both repositories have a top-level `src` package, and + importing the wrong one would be a confusing way to fail. + """ + root = str(Path(root).resolve()) + if root in sys.path: + sys.path.remove(root) + sys.path.insert(0, root) + + for name in [module for module in sys.modules if module == "src" or module.startswith("src.")]: + del sys.modules[name] + + import src.core as core + import src.ray as ray + + if not str(Path(core.__file__).resolve()).startswith(root): + raise CheckFailure(f"imported the wrong `src` package: got {core.__file__}, expected it under {root}") + return core, ray + + +def bound_serial_writes(ray): + """Give TrenchCoat's serial connections a write timeout. + + `Ray.open` uses `serial.Serial(port, 115200, timeout=0.1)`, which is the + READ timeout only - the same trap this harness had. On a desktop talking to + a healthy board that is fine, which is the case TrenchCoat is written for. + Here the board is wedged by definition, so the Ctrl-C that `open()` writes + blocks with nothing to time it out: one recovery run spent its entire 600s + step budget inside enter_bootloader_mode before the SIGALRM backstop cut it + short. + + Injected rather than patched upstream: it is their code, and this is our + unusual way of using it. + """ + original = ray.serial.Serial + + def bounded(*args, **kwargs): + kwargs.setdefault("write_timeout", SERIAL_WRITE_TIMEOUT) + return original(*args, **kwargs) + + ray.serial.Serial = bounded + + +def bundled_uf2(root, target): + if target not in TARGET_UF2: + raise CheckFailure(f"no TrenchCoat UF2 known for target {target!r} (have: {', '.join(sorted(TARGET_UF2))})") + path = Path(root) / "uf2" / TARGET_UF2[target] + if not path.exists(): + raise CheckFailure(f"{path} is missing from the trench-coat checkout") + return path + + +# Where automounters actually put a volume: /media/LABEL, /media//LABEL, +# /run/media//LABEL. Deliberately NOT a recursive walk - the first +# version used rglob over /media, /run/media and /mnt, which walks whatever +# else happens to be mounted there. On a 512MB Zero 2 W with a drive under +# /mnt that is minutes of I/O and enough memory pressure to take the runner +# down with it, which is the most likely reason a bench job died without +# uploading its logs. +MOUNT_ROOTS = ("/media", "/run/media") + + +def find_bootloader_drives(): + """Mounted RPI-RP2 volumes, found without walking arbitrary filesystems.""" + drives = [] + for root in MOUNT_ROOTS: + base = Path(root) + if not base.is_dir(): + continue + try: + candidates = list(base.iterdir()) + for entry in list(candidates): + if entry.is_dir(): + candidates.extend(entry.iterdir()) + except OSError: + continue + for entry in candidates: + try: + if entry.is_dir() and (entry / "INFO_UF2.TXT").exists(): + drives.append(str(entry)) + except OSError: + continue + return drives + + +def mount_rpi_rp2(): + """Mount an RPI-RP2 volume that nothing automounted. Returns the path or None.""" + by_label = Path("/dev/disk/by-label") + if not by_label.is_dir(): + return None + for link in sorted(by_label.glob("RPI-RP2*")): + device = link.resolve() + result = subprocess.run(["udisksctl", "mount", "-b", str(device)], capture_output=True, text=True, timeout=60) + if result.returncode == 0: + mounted = result.stdout.strip().rsplit(" at ", 1)[-1].rstrip(".") + log(f" mounted {device} at {mounted}") + return mounted + if "AlreadyMounted" in result.stderr: + continue + log(f" udisksctl could not mount {device}: {(result.stderr or result.stdout).strip()}") + return None + + +def bootsel_touch(port): + """Open the port at 1200 baud, which asks the RP2040 to reset to its ROM. + + TrenchCoat's own route into the bootloader is `machine.bootloader()` over + the REPL, fire-and-forget. That is the right first try - it is what the + team uses and it needs no privileges - but it does need the firmware alive + enough to run one statement. The 1200 baud touch is a CDC line-coding + change handled in USB interrupt context, so it can still land when the + Python VM cannot run anything at all. + """ + try: + connection = open_serial(port, baudrate=1200) + connection.dtr = False + time.sleep(0.5) + connection.close() + except Exception as exc: + # The port disappearing underneath us is what a reboot looks like. + log(f" port closed during the 1200 baud touch ({exc})") + + +def wait_for_drive(core, timeout=45): + """Wait for a bootloader drive, mounting it ourselves if nothing else does. + + Uses our bounded search rather than TrenchCoat's os.walk of /media and + /Volumes - same answer on a normal machine, without the risk of walking + into something large. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + drives = find_bootloader_drives() + if drives: + return drives + if mount_rpi_rp2(): + drives = find_bootloader_drives() + if drives: + return drives + time.sleep(1) + return [] + + +def enter_bootloader(core, ray, port): + """Get one board into its ROM bootloader, TrenchCoat's way then ours.""" + log(" asking the board to reset into the bootloader (TrenchCoat's machine.bootloader())") + try: + ray.Ray(port).enter_bootloader_mode() + except Exception as exc: + log(f" that did not go through: {exc}") + + drives = wait_for_drive(core, timeout=20) + if drives: + log(f" board is in bootloader mode: {', '.join(drives)}") + return drives + + log(" no drive yet, falling back to the 1200 baud touch") + bootsel_touch(port) + drives = wait_for_drive(core, timeout=45) + if drives: + log(f" board is in bootloader mode: {', '.join(drives)}") + return drives + + +def flash(port, target, root): + """Recover one board by running TrenchCoat's own firmware flash against it. + + Returns True if TrenchCoat reported the board back as a serial device. + """ + core, ray = load(clone(root)) + bound_serial_writes(ray) + uf2 = bundled_uf2(root, target) + + port_being_recovered = port + drives = enter_bootloader(core, ray, port) + if not drives: + log(" the board never presented a bootloader drive, so there is nothing to flash") + return False + + # From here TrenchCoat drives, on this board only. It sees every serial + # port except the healthy boards' - which is empty right now (the board + # being recovered is a drive, not a port), so its + # get_all_boards_into_bootloader() is a no-op instead of a second attempt, + # and it never touches the rest of the bench. + # + # Hiding *every* port instead would break the other end of the sequence: + # flash_firmware finishes by waiting for as many ports as it flashed + # drives, so a permanently empty list makes that wait unsatisfiable and + # turns a successful reflash into a timeout. Filtering rather than + # emptying also survives the board coming back on a different ttyACM + # number, which it often does. + others = {port for port in serial_ports() if port != port_being_recovered} + ray.Ray.find_board_ports = classmethod(lambda cls: [p for p in serial_ports() if p not in others]) + core.list_rpi_rp2_drives = lambda: find_bootloader_drives() or drives + + # Their failure path prints advice and calls sys.exit; make it an exception + # this harness can report and carry on from. + def refuse_to_exit(now=False): + raise CheckFailure("TrenchCoat could not complete the flash (see its output above)") + + core.graceful_exit = refuse_to_exit + + log(f" handing over to TrenchCoat: nuke.uf2, then {uf2.name}") + core.flash_firmware(str(uf2)) + + # flash_firmware only returns cleanly once the board is back as a serial + # port, so reaching here is the success condition. + log(" TrenchCoat reports the board restarted") + return True + + +# -------------------------------------------------------------------------- +# Boards that are already in the bootloader +# -------------------------------------------------------------------------- +# +# A board found in BOOTSEL needs the second half of the sequence above and not +# the first: there is no port to reset, because the board is a mass-storage +# device already. TrenchCoat's own `core.flash_firmware` cannot be pointed at +# one board here - it wipes every drive it can see and then waits on +# `Ray.find_board_ports()` for as many boards as it flashed, so restricting it +# to one board (which the bench must do, or a rescue takes the healthy boards +# with it) makes its final wait unsatisfiable. So the copy sequence is spelled +# out below, with the parts that matter kept: nuke.uf2 first, and the pinned +# UF2 bundle from the checkout. + +BOOTSEL_SETTLE = 5 +DRIVE_TIMEOUT = 60 +RESTART_TIMEOUT = 90 + + +def serial_ports(): + return sorted(str(path) for path in Path("/dev").glob("ttyACM*")) + + +def block_device(usb_device): + """The /dev node behind a bootloader's mass-storage interface. + + Walked from the USB device's own sysfs directory rather than looked up in + /dev/disk/by-id, because that is exact: with three boards in BOOTSEL the + by-id names differ only by a serial string whose format is the bootrom's + business, while this path belongs to the one device we are holding. + """ + for block in sorted(Path(usb_device).glob("*/host*/target*/*/block/*")): + # A filesystem lives on a partition when the drive has a partition + # table, and udisks rightly refuses to mount the whole disk in that + # case - "/dev/sda is not a mountable filesystem" is exactly what a + # bench board produced, with `blkid` reporting PTTYPE="dos" on it. + # A plain RP2 bootloader drive has no partition table and no + # partitions, so this falls through to the disk as before. + for partition in sorted(block.glob(f"{block.name}[0-9]*")): + return Path("/dev") / partition.name + return Path("/dev") / block.name + return None + + +def mount_point(device): + """Where `device` is mounted, if it is.""" + try: + for line in Path("/proc/mounts").read_text().splitlines(): + fields = line.split() + if len(fields) > 1 and fields[0] == str(device): + return fields[1].replace("\\040", " ") + except OSError: + pass + return None + + +_mount_error = None + + +def mount(device): + """Mount a bootloader drive, whoever has to do it. Returns the path or None.""" + existing = mount_point(device) + if existing: + return existing + result = subprocess.run(["udisksctl", "mount", "--no-user-interaction", "-b", str(device)], capture_output=True, text=True, timeout=120) + if result.returncode == 0: + return result.stdout.strip().rsplit(" at ", 1)[-1].rstrip(".") + if "AlreadyMounted" in result.stderr: + return mount_point(device) + + # Said once. bootsel_drive keeps asking while it waits for a drive that + # may still be settling, and the same permission error sixty times over + # buries everything else in the log. + global _mount_error + detail = (result.stderr or result.stdout).strip() + if detail != _mount_error: + _mount_error = detail + log(f" could not mount {device}: {detail}") + for line in describe_block_device(device): + log(f" {line}") + return None + + +def describe_block_device(device): + """What the kernel thinks of a drive udisks would not mount. + + "not a mountable filesystem" is where a bench recovery actually stopped, + and on its own it does not say whether the board is presenting a broken + filesystem or no drive at all. The size settles it: a bootloader offering + zero sectors is a board-side fault that no amount of retrying fixes, and + it needs picotool or a person rather than this code. + """ + name = Path(device).name + lines = [] + try: + sectors = int((Path("/sys/block") / name / "size").read_text().strip()) + except (OSError, ValueError): + return ["(could not read the drive's size from sysfs)"] + + if sectors == 0: + lines.append("the drive reports ZERO sectors, so there is no filesystem to mount and never will be.") + lines.append("The board is in the bootloader but its mass storage is presenting nothing - a UF2 cannot") + lines.append("be copied to it. picotool talks to the bootrom directly and does not need the drive:") + lines.append(f" picotool load -x # or: picotool info (board is {name})") + else: + lines.append(f"the drive reports {sectors} sectors ({sectors * 512 // 1024} KiB) but no filesystem udisks will mount.") + lines.append("Worth checking by hand on the bench host: sudo blkid " + str(device) + " ; sudo dmesg | tail -30") + lines.append("A `PTTYPE=` in blkid's output means the drive is partitioned and the filesystem is on a") + lines.append("partition; a bootloader drive normally has neither. picotool talks to the bootrom directly") + lines.append("and needs no drive at all, which is the way past this: picotool info ; picotool load -x ") + return lines + + +def unmount(device): + """Best effort, so the next flash does not trip over a stale mount.""" + subprocess.run(["udisksctl", "unmount", "--no-user-interaction", "-b", str(device)], capture_output=True, text=True, timeout=60) + + +def bootsel_drive(chip_id, timeout=DRIVE_TIMEOUT): + """Wait for one board's bootloader drive, by chip id, and mount it. + + Looked up afresh every time rather than remembered: writing a UF2 reboots + the board, so the device node, the sysfs path and the mount point are all + different on the other side of a copy. The chip id is the only handle that + survives. + """ + deadline = time.monotonic() + timeout + while True: + for board in bench.bootsel_boards(): + if board["chip_id"] != chip_id: + continue + device = block_device(board["usb_device"]) + if device is None or not device.exists(): + break + path = mount(device) + if path and (Path(path) / "INFO_UF2.TXT").exists(): + return device, path + break + if time.monotonic() >= deadline: + return None, None + time.sleep(1) + + +def wait_for_bootsel(chip_id, present, timeout=DRIVE_TIMEOUT, poll=0.25): + """Wait until the board is (or is no longer) enumerated as a bootloader. + + Polled quickly, because one of the two things it watches for is a gap: + the board vanishes when it starts running a UF2 and is back moments later, + and a slow poll can step straight over that. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + here = any(board["chip_id"] == chip_id for board in bench.bootsel_boards()) + if here == present: + return True + time.sleep(poll) + return False + + +def copy_uf2(uf2, device, drive): + """Write a UF2 to a mounted bootloader drive. + + An I/O error at the end of the copy is not reported as a failure: the + board reboots the moment the last block lands, which is exactly what a + successful flash looks like from this side - the device goes away + mid-write. Whether it worked is decided by what comes back, below. + """ + log(f" writing {uf2.name} to {drive}") + try: + shutil.copy(str(uf2), drive) + os.sync() + except OSError as exc: + log(f" the drive went away during the copy ({exc}) - that is usually the board rebooting") + unmount(device) + + +PICOTOOL_TIMEOUT = 300 + + +def picotool_available(): + return bool(shutil.which("picotool")) + + +def usb_address(chip_id): + """(bus, address) for one board in BOOTSEL, so picotool can be aimed. + + This is the whole safety story for the fallback. picotool with no device + selection acts on whatever RP2 device it finds first, which on this bench + could be a healthy board someone is mid-flash on. Reading bus and address + out of the sysfs directory we already matched by chip id means the command + can only ever reach the board we identified. + """ + for board in bench.bootsel_boards(): + if board["chip_id"] != chip_id: + continue + try: + bus = int((Path(board["usb_device"]) / "busnum").read_text()) + address = int((Path(board["usb_device"]) / "devnum").read_text()) + except (OSError, ValueError): + return None + return bus, address + return None + + +def picotool(arguments, chip_id, timeout=PICOTOOL_TIMEOUT): + """Run picotool against exactly one board.""" + where = usb_address(chip_id) + if where is None: + return None + command = ["picotool", *arguments, "--bus", str(where[0]), "--address", str(where[1])] + return subprocess.run(command, capture_output=True, text=True, timeout=timeout) + + +def _picotool_output(result, lines=10): + return [line for line in (result.stderr or result.stdout or "").strip().splitlines()[-lines:]] + + +def flash_over_picotool(chip_id, uf2): + """Write a UF2 without a drive, by talking to the ROM bootloader itself. + + The drive-copy route needs the board's mass storage to work. A bench board + came back from a power cycle mid-flash with a drive udisks would not touch + - `blkid` saw a DOS partition table where a bootloader drive should have a + bare filesystem - and no retry was ever going to fix that. picotool speaks + the bootrom's own USB protocol (PICOBOOT) and needs no filesystem at all; + on that same board `picotool info` answered while the drive was unusable. + + The erase keeps the property that makes this a recovery rather than an + upgrade - nothing of the old filesystem survives - and is best effort, + because a board that will not erase may still accept the firmware. + """ + if not picotool_available(): + log(" picotool is not installed, so there is no way past an unusable drive here") + log(" (`sudo apt install picotool` gives the bench a route that needs no filesystem)") + return False + + where = usb_address(chip_id) + if where is None: + log(" could not read the board's USB bus/address, and picotool must be aimed at one board") + return False + + log(f" falling back to picotool on bus {where[0]} address {where[1]} - it needs no drive") + erased = picotool(["erase"], chip_id) + if erased is None or erased.returncode != 0: + detail = "; ".join(_picotool_output(erased, 2)) if erased is not None else "the board moved" + log(f" erase declined ({detail}) - writing the firmware anyway") + + loaded = picotool(["load", "-x", str(uf2)], chip_id) + if loaded is None: + log(" the board stopped being a bootloader before the firmware could be written") + return False + if loaded.returncode != 0: + log(" picotool could not write the firmware:") + for line in _picotool_output(loaded): + log(f" {line}") + return False + + log(f" picotool wrote {uf2.name} and started it") + return True + + +def wait_for_serial(before, uf2): + """Wait for a flashed board to come back as a port it did not hold before.""" + deadline = time.monotonic() + RESTART_TIMEOUT + while time.monotonic() < deadline: + new = [port for port in serial_ports() if port not in before] + if new: + log(f" back as {new[0]} running {uf2.name}") + return new[0] + time.sleep(1) + log(f" {uf2.name} was written but the board never came back as a serial device") + return None + + +def flash_bootsel(chip_id, target, root): + """Put firmware back on a board that is sitting in its ROM bootloader. + + Returns the serial port it came back on, or None. The board is left + running TrenchCoat's bundled firmware for its target, which is a working + Vector board and, more to the point, a board the normal build-and-flash + path can talk to again. + """ + uf2 = bundled_uf2(root, target) + nuke = Path(root) / "uf2" / "nuke.uf2" + if not nuke.exists(): + raise CheckFailure(f"{nuke} is missing from the trench-coat checkout") + + before = set(serial_ports()) + device, drive = bootsel_drive(chip_id) + if drive is None: + log(f" no usable bootloader drive for {chip_id}: it is enumerated and in the bootloader, but") + log(" nothing here could mount a filesystem to copy a UF2 onto - see the reason above") + if not flash_over_picotool(chip_id, uf2): + return None + return wait_for_serial(before, uf2) + + # The wipe is the load-bearing step: it erases the whole flash, so nothing + # from whatever state the board was left in survives into the new firmware. + copy_uf2(nuke, device, drive) + # Not seeing it leave is not a failure: a wipe can start and finish + # between two polls, and what matters is the state it settles in. + if not wait_for_bootsel(chip_id, present=False, timeout=30): + log(" never saw it restart - either the wipe was quick or it never began") + if not wait_for_bootsel(chip_id, present=True, timeout=DRIVE_TIMEOUT): + log(" the board did not come back as a bootloader after the wipe") + return None + time.sleep(BOOTSEL_SETTLE) + + device, drive = bootsel_drive(chip_id) + if drive is None: + log(" the wiped board never presented its drive again") + if not flash_over_picotool(chip_id, uf2): + return None + return wait_for_serial(before, uf2) + copy_uf2(uf2, device, drive) + + return wait_for_serial(before, uf2) + + +def rescue_bootsel(board_map, cache_dir): + """Flash every board found in BOOTSEL back to a serial device. + + Runs before anything else on the bench, because a board in this state is + invisible to every other stage: no serial port, no chip id over the REPL, + nothing to flash or health-check. Boards the map does not cover are + reported and left alone - there is no way to guess which UF2 they need, + and writing the wrong system's firmware is worse than leaving them. + + Returns the number of boards put back. + """ + stranded = bench.bootsel_boards() + if not stranded: + return 0 + + group(f"Rescue {len(stranded)} board(s) in BOOTSEL") + log("These boards are in the ROM bootloader, not running firmware. Flashing them back.") + unmapped = [board for board in stranded if board["chip_id"] not in board_map] + if unmapped: + log(f"::error::{bench.report_unknown_boards(unmapped, stranded, board_map)}") + + rescued = 0 + root = None + for board in stranded: + chip_id = board["chip_id"] + target = board_map.get(chip_id) + if not target: + continue + log(f" {chip_id} {board['processor']} -> {target}") + try: + if root is None: + root = clone(cache_dir / "trench-coat") + if flash_bootsel(chip_id, target, root): + rescued += 1 + except CheckFailure as exc: + log(f"::error::{chip_id} ({target}): {exc}") + except Exception as exc: # noqa: BLE001 - one board's rescue never stops the next + log(f"::error::{chip_id} ({target}): {type(exc).__name__}: {exc}") + + log(f"{rescued} of {len(stranded)} board(s) rescued") + endgroup() + return rescued diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py new file mode 100644 index 00000000..536c35e5 --- /dev/null +++ b/dev/tests/test_hil_bootsel.py @@ -0,0 +1,811 @@ +"""Tests for the states the bench harness has to survive without a serial port. + +A board in BOOTSEL, a board the map has never seen, and a bench that is a board +short are the three ways a run can be worthless while looking fine, and all +three are hardware-free to pin down: sysfs is a directory, the job summary is a +file, and flashing is a subprocess. +""" + +from __future__ import annotations + +import sys +import time +import types +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +_serial_stub = sys.modules.setdefault("serial", types.ModuleType("serial")) +if not hasattr(_serial_stub, "SerialTimeoutException"): + _serial_stub.SerialTimeoutException = type("SerialTimeoutException", (Exception,), {}) +if "usb_coms_demo" not in sys.modules: + stub = types.ModuleType("usb_coms_demo") + stub.UsbApiClient = object + sys.modules["usb_coms_demo"] = stub + +sys.path.insert(0, str(REPO_ROOT / "dev" / "hil")) + +import bench # noqa: E402 +import recover # noqa: E402 +import trench_coat # noqa: E402 + + +def usb_tree(tmp_path, devices): + """Build a fake /sys/bus/usb/devices holding `devices` = {name: (vid, pid, serial)}.""" + root = tmp_path / "usb" + root.mkdir() + for name, fields in devices.items(): + entry = root / name + entry.mkdir() + if fields is None: # an interface, which has no ids at all + continue + vid, pid, serial = fields + (entry / "idVendor").write_text(vid + "\n") + (entry / "idProduct").write_text(pid + "\n") + if serial is not None: + (entry / "serial").write_text(serial + "\n") + return root + + +# -------------------------------------------------------------------------- +# finding a board that has no serial port +# -------------------------------------------------------------------------- + + +def test_bootsel_boards_finds_rp2_bootloaders(monkeypatch, tmp_path): + root = usb_tree( + tmp_path, + { + "1-1": ("2e8a", "0003", "E661A4D4179A5B2F"), # RP2040 in BOOTSEL + "1-2": ("2e8a", "000f", "df13a50c13958980"), # RP2350 in BOOTSEL + "1-3": ("2e8a", "0005", "e66141040380b42e"), # running MicroPython + "1-4": ("1d6b", "0002", None), # a hub + "1-1:1.0": None, # an interface + }, + ) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + found = bench.bootsel_boards() + + assert [b["chip_id"] for b in found] == ["e661a4d4179a5b2f", "df13a50c13958980"] + assert [b["processor"] for b in found] == ["RP2040", "RP2350"] + # The id is the same one the map is keyed by, so a board is identifiable + # here even though nothing can be asked of it. + assert all(b["port"] is None and b["bootsel"] and not b["responsive"] for b in found) + + +def test_bootsel_boards_is_empty_without_sysfs(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "USB_DEVICES", tmp_path / "nothing here") + assert bench.bootsel_boards() == [] + + +def test_an_empty_bench_is_told_apart_from_one_in_the_bootloader(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(bench, "USB_DEVICES", tmp_path / "nothing") + assert "nothing in the ROM bootloader either" in bench.no_boards_message() + + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "e66141040380b42e"}]) + message = bench.no_boards_message() + assert "e66141040380b42e" in message + assert "recover.py" in message + assert "check the USB hub and power" not in message + + +def test_inventory_lists_bootsel_boards_alongside_the_others(monkeypatch, capsys): + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "probe", lambda port: {"port": port, "chip_id": "aaaa", "system": "wpc", "version": "1.7", "responsive": True}) + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "bbbb", "processor": "RP2040"}]) + + boards = bench.inventory() + printed = capsys.readouterr().out + + # Only the usable board is returned, but the other one is not silent. + assert [b["port"] for b in boards] == ["/dev/ttyACM0"] + assert "bbbb" in printed and "BOOTSEL" in printed + assert "::warning::" in printed + + +def test_inventory_explains_a_bench_that_is_entirely_in_the_bootloader(monkeypatch): + monkeypatch.setattr(bench, "list_ports", lambda: []) + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "bbbb", "processor": "RP2040"}]) + + with pytest.raises(bench.CheckFailure, match="ROM bootloader"): + bench.inventory() + + +# -------------------------------------------------------------------------- +# a board the map does not know +# -------------------------------------------------------------------------- + + +@pytest.fixture() +def job_summary(monkeypatch, tmp_path): + path = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(path)) + return path + + +def test_an_unknown_board_puts_its_id_and_the_fix_in_the_job_summary(job_summary): + boards = [ + {"port": "/dev/ttyACM0", "chip_id": "aaaa"}, + {"port": None, "chip_id": "cccc"}, + ] + board_map = {"aaaa": "wpc"} + + message = bench.report_unknown_boards([boards[1]], boards, board_map) + written = job_summary.read_text() + + assert "cccc" in message + # The id, the line to set, and where to set it - the whole remedy, on the + # page somebody reads when a run goes red. + assert "cccc" in written + assert "VECTOR_HIL_BOARD_MAP=aaaa=wpc,cccc=" in written + assert "actions-runner/.env" in written + # A board already mapped keeps its target in the suggested line. + assert "aaaa=" not in written + + +def test_resolve_targets_reports_the_board_it_cannot_place(job_summary): + boards = [ + {"port": "/dev/ttyACM0", "chip_id": "aaaa", "system": "wpc", "responsive": True}, + {"port": "/dev/ttyACM1", "chip_id": "cccc", "system": None, "responsive": True}, + ] + + with pytest.raises(bench.CheckFailure, match="does not cover"): + bench.resolve_targets(boards, {"aaaa": "wpc"}) + + assert "cccc" in job_summary.read_text() + + +def test_the_instructions_name_the_valid_targets(job_summary): + text = bench.board_map_instructions([{"port": None, "chip_id": "cccc"}], {}) + for target in bench.DEFAULT_GAMENAME: + assert target in text + + +# -------------------------------------------------------------------------- +# a bench that is a board short +# -------------------------------------------------------------------------- + + +def boards_for(*targets): + return [{"port": f"/dev/ttyACM{i}", "target": target} for i, target in enumerate(targets)] + + +def test_a_missing_system_fails_the_run(job_summary, monkeypatch): + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + + missing = bench.check_bench_complete(boards_for("wpc", "sys11")) + + assert missing == ["data_east"] + written = job_summary.read_text() + assert "Incomplete bench" in written + assert "data_east" in written + + +def test_a_complete_bench_says_nothing_to_the_summary(job_summary, monkeypatch): + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + + assert bench.check_bench_complete(boards_for("wpc", "sys11", "data_east")) == [] + assert not job_summary.exists() + + +def test_a_bench_that_really_has_lost_a_board_can_say_so(job_summary, monkeypatch): + monkeypatch.setenv("VECTOR_HIL_REQUIRED_TARGETS", "wpc,sys11") + assert bench.check_bench_complete(boards_for("wpc", "sys11")) == [] + + +def test_the_check_can_be_turned_off_entirely(job_summary, monkeypatch): + monkeypatch.setenv("VECTOR_HIL_REQUIRED_TARGETS", "") + assert bench.check_bench_complete([]) == [] + + +# -------------------------------------------------------------------------- +# flashing every board at once +# -------------------------------------------------------------------------- + + +def test_boards_are_flashed_in_parallel(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + + def slow_flash(target, port, build_dir, config_path): + time.sleep(0.3) + + monkeypatch.setattr(bench, "flash", slow_flash) + boards = boards_for("wpc", "sys11", "data_east") + + started = time.monotonic() + assert bench.flash_boards(boards, tmp_path) == {} + elapsed = time.monotonic() - started + + # Three 0.3s flashes, one after another, would be 0.9s. + assert elapsed < 0.6 + + +def test_one_board_failing_to_flash_does_not_take_the_others_with_it(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + + def flash(target, port, build_dir, config_path): + if target == "sys11": + raise bench.CheckFailure("flash failed for sys11: no space left") + + monkeypatch.setattr(bench, "flash", flash) + boards = boards_for("wpc", "sys11", "data_east") + + errors = bench.flash_boards(boards, tmp_path) + + # Attributed to the right board, which is the whole risk of doing this + # concurrently. + assert list(errors) == ["/dev/ttyACM1"] + assert "no space left" in errors["/dev/ttyACM1"] + + +def test_an_unexpected_error_is_reported_rather_than_escaping(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + + def flash(target, port, build_dir, config_path): + raise RuntimeError("the venv moved") + + monkeypatch.setattr(bench, "flash", flash) + + errors = bench.flash_boards(boards_for("wpc"), tmp_path) + + assert "RuntimeError: the venv moved" in errors["/dev/ttyACM0"] + + +# -------------------------------------------------------------------------- +# putting firmware back on a board in the bootloader +# -------------------------------------------------------------------------- + + +def test_the_drive_is_found_through_the_devices_own_sysfs_path(tmp_path): + device = tmp_path / "1-1" + block = device / "1-1:1.0" / "host0" / "target0:0:0" / "0:0:0:0" / "block" / "sda" + block.mkdir(parents=True) + + assert trench_coat.block_device(device) == Path("/dev/sda") + + +def test_no_drive_yet_is_not_an_error(tmp_path): + device = tmp_path / "1-1" + device.mkdir() + assert trench_coat.block_device(device) is None + + +def test_an_already_mounted_drive_is_not_mounted_again(monkeypatch): + monkeypatch.setattr(trench_coat, "mount_point", lambda device: "/media/runner/RPI-RP2") + monkeypatch.setattr(trench_coat.subprocess, "run", lambda *a, **k: pytest.fail("udisksctl must not be called for a drive that is already mounted")) + + assert trench_coat.mount("/dev/sda") == "/media/runner/RPI-RP2" + + +def test_mount_point_reads_proc_mounts(monkeypatch, tmp_path): + mounts = tmp_path / "mounts" + mounts.write_text("proc /proc proc rw 0 0\n/dev/sdb /media/runner/RPI-RP2 vfat rw 0 0\n") + real_read = Path.read_text + monkeypatch.setattr(Path, "read_text", lambda self, *a, **k: real_read(mounts) if str(self) == "/proc/mounts" else real_read(self, *a, **k)) + + assert trench_coat.mount_point("/dev/sdb") == "/media/runner/RPI-RP2" + assert trench_coat.mount_point("/dev/sdc") is None + + +@pytest.fixture() +def rescue(monkeypatch): + """Fake out everything below rescue_bootsel and record what it flashed.""" + flashed = [] + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: Path("/trench-coat")) + monkeypatch.setattr(trench_coat, "flash_bootsel", lambda chip_id, target, root: flashed.append((chip_id, target)) or "/dev/ttyACM9") + return flashed + + +def test_a_board_in_the_bootloader_is_flashed_with_its_own_targets_firmware(monkeypatch, rescue, tmp_path): + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "aaaa", "processor": "RP2040"}, {"chip_id": "bbbb", "processor": "RP2040"}]) + + assert trench_coat.rescue_bootsel({"aaaa": "wpc", "bbbb": "data_east"}, tmp_path) == 2 + assert rescue == [("aaaa", "wpc"), ("bbbb", "data_east")] + + +def test_an_unmapped_board_is_reported_and_left_alone(monkeypatch, rescue, tmp_path, job_summary): + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "aaaa", "processor": "RP2040"}, {"chip_id": "cccc", "processor": "RP2350"}]) + + # Guessing which system's firmware to write would be a good way to flash + # WPC firmware onto the Data East board. + assert trench_coat.rescue_bootsel({"aaaa": "wpc"}, tmp_path) == 1 + assert rescue == [("aaaa", "wpc")] + assert "cccc" in job_summary.read_text() + + +def test_nothing_in_the_bootloader_costs_nothing(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(trench_coat, "clone", lambda *a, **k: pytest.fail("must not clone trench-coat with nothing to rescue")) + + assert trench_coat.rescue_bootsel({"aaaa": "wpc"}, tmp_path) == 0 + + +def test_one_boards_rescue_failing_does_not_stop_the_next(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "aaaa", "processor": "RP2040"}, {"chip_id": "bbbb", "processor": "RP2040"}]) + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: Path("/trench-coat")) + tried = [] + + def flash_bootsel(chip_id, target, root): + tried.append(chip_id) + if chip_id == "aaaa": + raise bench.CheckFailure("no drive ever appeared") + return "/dev/ttyACM1" + + monkeypatch.setattr(trench_coat, "flash_bootsel", flash_bootsel) + + assert trench_coat.rescue_bootsel({"aaaa": "wpc", "bbbb": "sys11"}, tmp_path) == 1 + assert tried == ["aaaa", "bbbb"] + + +def test_the_wipe_comes_before_the_firmware(monkeypatch, tmp_path): + """nuke.uf2 first is the whole difference between a recovery and an upgrade.""" + root = tmp_path / "trench-coat" + (root / "uf2").mkdir(parents=True) + (root / "uf2" / "nuke.uf2").write_text("nuke") + (root / "uf2" / trench_coat.TARGET_UF2["wpc"]).write_text("firmware") + + copied = [] + monkeypatch.setattr(trench_coat, "copy_uf2", lambda uf2, device, drive: copied.append(uf2.name)) + monkeypatch.setattr(trench_coat, "bootsel_drive", lambda chip_id, timeout=None: ("/dev/sda", "/media/RPI-RP2")) + monkeypatch.setattr(trench_coat, "wait_for_bootsel", lambda chip_id, present, timeout=None: True) + monkeypatch.setattr(trench_coat, "BOOTSEL_SETTLE", 0) + ports = iter([[], [], ["/dev/ttyACM0"]]) + monkeypatch.setattr(trench_coat, "serial_ports", lambda: next(ports, ["/dev/ttyACM0"])) + + assert trench_coat.flash_bootsel("aaaa", "wpc", root) == "/dev/ttyACM0" + assert copied == ["nuke.uf2", trench_coat.TARGET_UF2["wpc"]] + + +def test_a_board_that_never_presents_a_drive_is_not_flashed(monkeypatch, tmp_path): + root = tmp_path / "trench-coat" + (root / "uf2").mkdir(parents=True) + (root / "uf2" / "nuke.uf2").write_text("nuke") + (root / "uf2" / trench_coat.TARGET_UF2["wpc"]).write_text("firmware") + + monkeypatch.setattr(trench_coat, "copy_uf2", lambda *a: pytest.fail("nothing to copy to")) + monkeypatch.setattr(trench_coat, "bootsel_drive", lambda chip_id, timeout=None: (None, None)) + monkeypatch.setattr(trench_coat, "serial_ports", lambda: []) + + assert trench_coat.flash_bootsel("aaaa", "wpc", root) is None + + +def test_trench_coat_is_shown_the_recovered_board_and_no_other(monkeypatch, tmp_path): + """The port filter has to do two opposite things at two moments. + + Before the flash it must hide every board, so TrenchCoat resets none of + them; after it, it must show the recovered one, or its own wait for the + board to restart can never be satisfied. + """ + seen = {} + + class FakeRay: + def __init__(self, port): + seen.setdefault("bootloader", []).append(port) + + def enter_bootloader_mode(self): + pass + + @classmethod + def find_board_ports(cls): + return [] + + ray = types.SimpleNamespace(Ray=FakeRay, serial=types.SimpleNamespace(Serial=lambda *a, **k: object())) + core = types.SimpleNamespace( + list_rpi_rp2_drives=lambda: ["/media/RPI-RP2"], + graceful_exit=lambda now=False: None, + flash_firmware=lambda path: seen.update(during=ray.Ray.find_board_ports()), + ) + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: root) + monkeypatch.setattr(trench_coat, "load", lambda root: (core, ray)) + monkeypatch.setattr(trench_coat, "bundled_uf2", lambda root, target: Path("/uf2/wpc.uf2")) + monkeypatch.setattr(trench_coat, "find_bootloader_drives", lambda: ["/media/RPI-RP2"]) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: ["/media/RPI-RP2"]) + + # The two healthy boards are on ACM0 and ACM2; the board being recovered is + # a drive right now and comes back on a number it did not have before. + monkeypatch.setattr(trench_coat, "serial_ports", lambda: ["/dev/ttyACM0", "/dev/ttyACM2"]) + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is True + assert seen["during"] == [] + assert seen["bootloader"] == ["/dev/ttyACM1"] + + hidden = ray.Ray.find_board_ports + monkeypatch.setattr(trench_coat, "serial_ports", lambda: ["/dev/ttyACM0", "/dev/ttyACM2", "/dev/ttyACM3"]) + assert hidden() == ["/dev/ttyACM3"] + + +def test_the_same_finding_reaches_the_summary_once(job_summary, monkeypatch): + """Three stages, one summary page, one problem.""" + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + boards = boards_for("wpc", "sys11") + + for _stage in range(3): + assert bench.check_bench_complete(boards) == ["data_east"] + + assert job_summary.read_text().count("### Incomplete bench") == 1 + + +def test_a_different_finding_still_gets_through(job_summary): + boards = [{"port": None, "chip_id": "cccc"}, {"port": None, "chip_id": "dddd"}] + + bench.report_unknown_boards([boards[0]], boards, {}) + bench.report_unknown_boards([boards[1]], boards, {}) + + written = job_summary.read_text() + assert "cccc" in written and "dddd" in written + + +# -------------------------------------------------------------------------- +# an empty bench has to explain itself +# -------------------------------------------------------------------------- + + +def test_an_empty_bench_prints_the_usb_bus(monkeypatch, tmp_path): + """ "no boards found" and "lsusb shows them" is a contradiction the log must settle.""" + root = usb_tree( + tmp_path, + { + "usb1": ("1d6b", "0002", "0000:01:00.0"), + "1-1": ("2109", "3431", None), + "1-1.1": ("2e8a", "0005", "e661a4d4179a5b2f"), # running, but no serial port for it + "1-1.1:1.0": None, + }, + ) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + message = bench.no_boards_message() + + assert "2e8a:0005" in message + assert "2109:3431" in message + # A board in this state is running firmware, so say that rather than + # calling it unrecognised - the mismatch is that it has no serial port. + assert "1 running MicroPython" in message + # Interfaces carry no ids and are not devices. + assert "1-1.1:1.0" not in message + + +def test_an_empty_sysfs_says_so_rather_than_blaming_the_boards(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "USB_DEVICES", tmp_path / "nothing") + assert "Nothing at all is on the USB bus" in bench.no_boards_message() + + +# -------------------------------------------------------------------------- +# a board wired for a system the bench cannot drive +# -------------------------------------------------------------------------- + + +def test_the_target_list_comes_from_the_repo(monkeypatch): + """src/common is shared code, and sys11_tiny is the sys11 board twice.""" + buildable = bench.buildable_targets() + + assert "classic" in buildable and "whitestar" in buildable + assert "common" not in buildable + assert "sys11_tiny" not in buildable and "sys11" in buildable + + +def test_only_targets_with_configs_can_be_driven(): + ready = bench.bench_targets() + + assert set(ready) == {"sys11", "wpc", "data_east", "em"} + # Real build targets, but nothing to boot them against. + assert "classic" not in ready and "whitestar" not in ready + + +def test_a_board_mapped_to_a_configless_target_is_refused_early(job_summary): + """A classic board turned up on the bench; the map must not be able to lie about it.""" + boards = [{"port": "/dev/ttyACM2", "chip_id": "899f", "system": "classic", "responsive": True}] + + with pytest.raises(bench.CheckFailure, match="bench cannot drive"): + bench.resolve_targets(boards, {"899f": "classic"}) + + # Reported where an unrecognised chip id is reported, for the same person. + assert "target the bench cannot use" in job_summary.read_text() + + +def test_a_target_that_does_not_exist_at_all_says_what_does(job_summary): + boards = [{"port": "/dev/ttyACM2", "chip_id": "899f", "system": None, "responsive": True}] + + with pytest.raises(bench.CheckFailure, match="not a target in this checkout"): + bench.resolve_targets(boards, {"899f": "sys12"}) + + +@pytest.mark.parametrize( + ("written", "meant"), + [("de", "data_east"), ("DE", "data_east"), ("DataEast", "data_east"), ("data-east", "data_east"), ("ws", "whitestar")], +) +def test_the_name_people_actually_use_is_recognised(written, meant): + """The bench map arrived saying `de`, which is what everyone calls that board.""" + assert bench.suggest_target(written) == meant + + +def test_a_name_nobody_meant_gets_no_guess(): + assert bench.suggest_target("nonsense") is None + assert bench.suggest_target("") is None + + +def test_the_suggested_map_line_never_repeats_a_broken_entry(): + """The fix on offer must not contain the value that just failed.""" + boards = [{"port": "/dev/ttyACM0", "chip_id": "899f"}, {"port": "/dev/ttyACM1", "chip_id": "df13"}] + + corrected = bench.board_map_instructions(boards, {"899f": "de", "df13": "wpc"}) + assert "VECTOR_HIL_BOARD_MAP=899f=data_east,df13=wpc" in corrected + + # Nothing to suggest for a target that exists but cannot be driven, so it + # is left blank rather than being offered back. + blanked = bench.board_map_instructions(boards, {"899f": "classic", "df13": "wpc"}) + assert "VECTOR_HIL_BOARD_MAP=899f=,df13=wpc" in blanked + + +def test_self_report_cannot_smuggle_in_an_undriveable_target(): + """Autodetection reads the flashed firmware, which can say 'classic' too.""" + boards = [ + {"port": "/dev/ttyACM0", "chip_id": "aaaa", "system": "wpc", "responsive": True}, + {"port": "/dev/ttyACM2", "chip_id": "899f", "system": "classic", "responsive": True}, + ] + + with pytest.raises(bench.CheckFailure, match="bench cannot drive"): + bench.resolve_targets(boards, {}) + + +def test_the_instructions_name_what_cannot_be_mapped(job_summary): + text = bench.board_map_instructions([{"port": None, "chip_id": "899f"}], {}) + + assert "targets: data_east, em, sys11, wpc" in text + # Naming them matters: the board on the bench is running one of them, and + # the only wrong move is mapping it to a system it is not wired for. + assert "classic" in text and "whitestar" in text + + +def test_an_incomplete_bench_says_whether_the_board_is_even_plugged_in(job_summary, monkeypatch, tmp_path): + """ "Missing" has two remedies and they are opposite. + + A board that is enumerated but silent is the recovery ladder's job. A board + that has dropped off the bus needs a person to replug it - which is exactly + what happened to the wpc board between two runs, with nothing in the log to + tell the two apart. + """ + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + root = usb_tree(tmp_path, {"1-1.1": ("2e8a", "0005", "aaaa"), "1-1.2": ("2e8a", "0005", "bbbb")}) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + assert bench.check_bench_complete(boards_for("sys11", "data_east")) == ["wpc"] + + written = job_summary.read_text() + assert "What is on the USB bus" in written + assert "2e8a:0005" in written + + +def test_a_complete_bench_does_not_print_the_bus(job_summary, monkeypatch): + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + monkeypatch.setattr(bench, "usb_bus_report", lambda: pytest.fail("nothing is missing, so there is nothing to explain")) + + assert bench.check_bench_complete(boards_for("sys11", "wpc", "data_east")) == [] + + +# -------------------------------------------------------------------------- +# the bus report has to describe what it lists +# -------------------------------------------------------------------------- + + +def test_a_bootloader_on_the_bus_is_not_called_unrecognised(monkeypatch, tmp_path): + """The first version contradicted itself on the bench. + + It listed a 2e8a:000f device and then said none of the devices was "a + bootloader this harness recognises (2e8a:0003, 2e8a:000f)". + """ + root = usb_tree( + tmp_path, + { + "1-1.1": ("2e8a", "0005", "e661a4d4179a5b2f"), + "1-1.2": ("2e8a", "000f", "DF13A50C13958980"), + "1-1.3": ("2e8a", "0005", "899fab8c90bfeb9a"), + }, + ) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + report = "\n".join(bench.usb_bus_report()) + + assert "in the ROM bootloader: DF13A50C13958980" in report + assert "2 running MicroPython" in report + assert "does not handle" not in report + assert "rescue stage's job" in report + + +def test_a_genuinely_unknown_mode_is_still_called_out(monkeypatch, tmp_path): + root = usb_tree(tmp_path, {"1-1.1": ("2e8a", "abcd", "aaaa")}) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + report = "\n".join(bench.usb_bus_report()) + + assert "mode this harness does not handle: 2e8a:abcd" in report + + +def test_a_zero_sector_drive_says_so_and_names_picotool(monkeypatch, tmp_path): + """Where a real bench recovery stopped: enumerated, in BOOTSEL, no filesystem.""" + block = tmp_path / "sda" + block.mkdir() + (block / "size").write_text("0\n") + real_read = Path.read_text + monkeypatch.setattr(Path, "read_text", lambda self, *a, **k: real_read(block / "size") if str(self).endswith("/sys/block/sda/size") else real_read(self, *a, **k)) + + described = "\n".join(trench_coat.describe_block_device("/dev/sda")) + + assert "ZERO sectors" in described + assert "picotool" in described + + +def test_a_drive_with_sectors_but_no_filesystem_points_at_blkid(monkeypatch, tmp_path): + block = tmp_path / "sda" + block.mkdir() + (block / "size").write_text("256\n") + real_read = Path.read_text + monkeypatch.setattr(Path, "read_text", lambda self, *a, **k: real_read(block / "size") if str(self).endswith("/sys/block/sda/size") else real_read(self, *a, **k)) + + described = "\n".join(trench_coat.describe_block_device("/dev/sda")) + + assert "256 sectors" in described + assert "blkid" in described + + +def test_the_partition_is_preferred_over_the_whole_disk(tmp_path): + """udisks will not mount a partitioned disk, and a bench board was one. + + `blkid /dev/sda` reported PTUUID/PTTYPE="dos" while udisks said "not a + mountable filesystem" - correct of it, because the filesystem is on the + partition. + """ + device = tmp_path / "1-1" + block = device / "1-1:1.0" / "host0" / "target0:0:0" / "0:0:0:0" / "block" / "sda" + (block / "sda1").mkdir(parents=True) + + assert trench_coat.block_device(device) == Path("/dev/sda1") + + +def test_an_unpartitioned_bootloader_drive_still_uses_the_disk(tmp_path): + """The normal case: an RP2 drive has no partition table and no partitions.""" + device = tmp_path / "1-1" + block = device / "1-1:1.0" / "host0" / "target0:0:0" / "0:0:0:0" / "block" / "sda" + block.mkdir(parents=True) + # sysfs puts other things in here; none of them is a partition. + (block / "queue").mkdir() + (block / "size").write_text("262144\n") + + assert trench_coat.block_device(device) == Path("/dev/sda") + + +# -------------------------------------------------------------------------- +# picotool: the route that needs no drive +# -------------------------------------------------------------------------- + + +@pytest.fixture() +def picotool_calls(monkeypatch, tmp_path): + """Record picotool invocations, with one board in BOOTSEL at a known address.""" + device = tmp_path / "1-1.2" + device.mkdir() + (device / "busnum").write_text("1\n") + (device / "devnum").write_text("22\n") + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "df13", "usb_device": device, "processor": "RP2350"}]) + monkeypatch.setattr(trench_coat.shutil, "which", lambda name: "/usr/bin/picotool") + + calls = [] + + def run(command, **kwargs): + calls.append(command) + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(trench_coat.subprocess, "run", run) + return calls + + +def test_picotool_is_aimed_at_one_board_by_bus_and_address(picotool_calls): + """The safety property: never "whatever RP2 device it finds first".""" + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is True + + for command in picotool_calls: + assert "--bus" in command and command[command.index("--bus") + 1] == "1" + assert "--address" in command and command[command.index("--address") + 1] == "22" + + +def test_picotool_erases_before_it_writes(picotool_calls): + """Keeps the property that makes this a recovery, not an upgrade.""" + trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) + + assert [c[1] for c in picotool_calls][:2] == ["erase", "load"] + assert "-x" in picotool_calls[1] and "/uf2/wpc.uf2" in picotool_calls[1] + + +def test_a_board_that_will_not_erase_is_still_offered_the_firmware(monkeypatch, picotool_calls): + results = iter([types.SimpleNamespace(returncode=1, stdout="", stderr="erase not supported"), types.SimpleNamespace(returncode=0, stdout="", stderr="")]) + monkeypatch.setattr(trench_coat.subprocess, "run", lambda command, **k: picotool_calls.append(command) or next(results)) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is True + assert [c[1] for c in picotool_calls] == ["erase", "load"] + + +def test_a_failed_load_is_reported_as_failure(monkeypatch, picotool_calls): + results = iter([types.SimpleNamespace(returncode=0, stdout="", stderr=""), types.SimpleNamespace(returncode=1, stdout="", stderr="no such device")]) + monkeypatch.setattr(trench_coat.subprocess, "run", lambda command, **k: next(results)) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is False + + +def test_without_picotool_the_fallback_declines_rather_than_guessing(monkeypatch, picotool_calls): + monkeypatch.setattr(trench_coat.shutil, "which", lambda name: None) + monkeypatch.setattr(trench_coat.subprocess, "run", lambda *a, **k: pytest.fail("picotool is not installed")) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is False + + +def test_a_board_that_is_no_longer_in_bootsel_is_never_targeted(monkeypatch): + """No address means no command - picotool is never run unaimed.""" + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(trench_coat.shutil, "which", lambda name: "/usr/bin/picotool") + monkeypatch.setattr(trench_coat.subprocess, "run", lambda *a, **k: pytest.fail("there is no board to aim at")) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is False + + +def test_an_unmountable_drive_falls_through_to_picotool(monkeypatch, tmp_path): + """The bench case end to end: in BOOTSEL, drive unusable, firmware written anyway.""" + root = tmp_path / "trench-coat" + (root / "uf2").mkdir(parents=True) + (root / "uf2" / "nuke.uf2").write_text("nuke") + (root / "uf2" / trench_coat.TARGET_UF2["wpc"]).write_text("firmware") + + monkeypatch.setattr(trench_coat, "bootsel_drive", lambda chip_id, timeout=None: (None, None)) + # Absent before the flash, present after - which is what "it came back" means. + ports = iter([[], ["/dev/ttyACM3"]]) + monkeypatch.setattr(trench_coat, "serial_ports", lambda: next(ports, ["/dev/ttyACM3"])) + tried = [] + monkeypatch.setattr(trench_coat, "flash_over_picotool", lambda chip_id, uf2: tried.append(uf2.name) or True) + + assert trench_coat.flash_bootsel("df13", "wpc", root) == "/dev/ttyACM3" + assert tried == [trench_coat.TARGET_UF2["wpc"]] + + +def test_picotool_makes_a_reflash_possible_without_any_mount(monkeypatch): + monkeypatch.setattr(trench_coat, "picotool_available", lambda: True) + monkeypatch.setattr(trench_coat, "find_bootloader_drives", lambda: []) + monkeypatch.setattr(recover.shutil, "which", lambda name: None) + + possible, why = recover.can_complete_a_reflash() + + assert possible is True + assert "picotool" in why + + +def test_a_board_with_no_chip_id_is_never_put_in_the_suggested_line(): + """The bench printed `VECTOR_HIL_BOARD_MAP=None=,...` as the fix. + + A board that will not say who it is cannot be mapped - an entry needs an + id to key on - so it is named separately rather than rendered into the + line somebody is meant to paste. + """ + boards = [ + {"port": "/dev/ttyACM0", "chip_id": None}, + {"port": "/dev/ttyACM1", "chip_id": "899fab8c90bfeb9a"}, + ] + + text = board_map_instructions_for(boards, {"899fab8c90bfeb9a": "data_east"}) + + assert "None=" not in text + assert "VECTOR_HIL_BOARD_MAP=899fab8c90bfeb9a=data_east" in text + assert "/dev/ttyACM0 did not report a chip id" in text + assert "left mid-flash" in text + + +def board_map_instructions_for(boards, board_map): + return bench.board_map_instructions(boards, board_map) + + +def test_every_board_still_appears_when_they_all_have_ids(): + boards = [{"port": "/dev/ttyACM0", "chip_id": "aaaa"}, {"port": "/dev/ttyACM1", "chip_id": "bbbb"}] + + text = bench.board_map_instructions(boards, {"aaaa": "sys11", "bbbb": "wpc"}) + + assert "VECTOR_HIL_BOARD_MAP=aaaa=sys11,bbbb=wpc" in text + assert "did not report a chip id" not in text diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py new file mode 100644 index 00000000..e416c405 --- /dev/null +++ b/dev/tests/test_hil_config_matrix.py @@ -0,0 +1,1206 @@ +"""Tests for the parts of the HIL config matrix that do not need a board. + +Everything that touches serial, mpremote or a real board is out of reach here; +what is testable is the expectation side - how the harness reads the repo's +config JSON, which configs it decides to run, and in what order - plus the +filename-length rule that decides whether a config can be reached at all. +""" + +from __future__ import annotations + +import json +import subprocess +import sys +import time +import types +from argparse import Namespace +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# bench.py imports pyserial (which ships with mpremote) and dev/usb_coms_demo. +# Neither is needed for the pure helpers under test and neither is guaranteed to +# be installed wherever these tests run, so stand them in before the import. +_serial_stub = sys.modules.setdefault("serial", types.ModuleType("serial")) +if not hasattr(_serial_stub, "SerialTimeoutException"): + _serial_stub.SerialTimeoutException = type("SerialTimeoutException", (Exception,), {}) +if "usb_coms_demo" not in sys.modules: + stub = types.ModuleType("usb_coms_demo") + stub.UsbApiClient = object + sys.modules["usb_coms_demo"] = stub + +sys.path.insert(0, str(REPO_ROOT / "dev" / "hil")) + +import bench # noqa: E402 +import config_matrix as cm # noqa: E402 + + +def write_config(directory: Path, name: str, game_name: str, **sections) -> None: + directory.mkdir(parents=True, exist_ok=True) + payload = {"GameInfo": {"GameName": game_name, "System": "WPC"}} + payload.update(sections) + (directory / f"{name}.json").write_text(json.dumps(payload)) + + +@pytest.fixture() +def fake_repo(tmp_path, monkeypatch): + """A repo-shaped tree with one target and three configs.""" + config_dir = tmp_path / "src" / "wpc" / "config" + write_config(config_dir, "AttackMars_11", "Attack from Mars", Adjustments={"Type": 0}) + write_config(config_dir, "Generic_WPC", "Generic System WPC", Adjustments={"Type": 0}) + write_config(config_dir, "Taxi_L4", "Taxi") + monkeypatch.setattr(cm, "REPO_ROOT", tmp_path) + return tmp_path + + +def args(**overrides) -> Namespace: + defaults = {"configs": None, "limit": None, "changed_since": None} + defaults.update(overrides) + return Namespace(**defaults) + + +def test_source_configs_reads_names_and_adjustments(fake_repo): + configs = cm.source_configs("wpc") + + assert configs["AttackMars_11"] == {"name": "Attack from Mars", "adjustments": True} + # Taxi_L4 declares no Adjustments section, which is what decides whether + # /api/adjustments/status is held to 200 on the bench. + assert configs["Taxi_L4"] == {"name": "Taxi", "adjustments": False} + + +def test_source_configs_rejects_a_config_with_no_game_name(fake_repo): + (fake_repo / "src" / "wpc" / "config" / "Broken_L1.json").write_text(json.dumps({"GameInfo": {}})) + + with pytest.raises(bench.CheckFailure, match="GameInfo.GameName"): + cm.source_configs("wpc") + + +def test_source_configs_rejects_unparseable_json(fake_repo): + (fake_repo / "src" / "wpc" / "config" / "Broken_L1.json").write_text("{nope") + + with pytest.raises(bench.CheckFailure, match="not valid JSON"): + cm.source_configs("wpc") + + +def test_source_configs_rejects_an_unknown_target(fake_repo): + with pytest.raises(bench.CheckFailure, match="no config directory"): + cm.source_configs("nosuchtarget") + + +def test_select_configs_defaults_to_every_config(fake_repo): + names, configs = cm.select_configs("wpc", args()) + + assert names == sorted(configs) == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] + + +def test_select_configs_honours_an_explicit_list(fake_repo): + names, _ = cm.select_configs("wpc", args(configs="Taxi_L4, AttackMars_11")) + + assert names == ["Taxi_L4", "AttackMars_11"] + + +def test_select_configs_rejects_an_unknown_name(fake_repo): + with pytest.raises(bench.CheckFailure, match="Nonexistent_L1"): + cm.select_configs("wpc", args(configs="Nonexistent_L1")) + + +def test_select_configs_applies_the_limit(fake_repo): + names, _ = cm.select_configs("wpc", args(limit=2)) + + assert names == ["AttackMars_11", "Generic_WPC"] + + +def test_select_configs_runs_changed_configs_first(fake_repo, monkeypatch): + monkeypatch.setattr(cm, "changed_configs", lambda target, ref: ["Taxi_L4"]) + + names, _ = cm.select_configs("wpc", args(changed_since="origin/main")) + + assert names[0] == "Taxi_L4" + assert sorted(names) == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] + + +def test_order_configs_ignores_names_that_are_not_under_test(): + ordered = cm.order_configs(["a", "b", "c"], ["c", "deleted"]) + + assert ordered == ["c", "a", "b"] + + +def test_order_configs_is_a_no_op_without_changes(): + assert cm.order_configs(["a", "b"], []) == ["a", "b"] + + +def test_gamename_field_width_comes_from_the_firmware_source(): + # Not hard-coded in the harness: widening the FRAM field in SPI_DataStore.py + # must move this number, or the harness would keep enforcing a stale limit. + assert bench.gamename_field_bytes() == 16 + + +def test_no_config_name_exceeds_the_gamename_field(): + """A config whose filename is longer than the field can never be selected. + + write_record packs `gamename` into a fixed-width field and struct.pack + truncates silently, so the truncated name matches nothing at boot and the + board comes up on safe defaults with CONF01 - while the web UI happily + offers the config. The bench catches this per board; catching it here is + cheaper and fails an ordinary PR instead of 20 minutes of bench time. + + Two configs did exceed it, which is what the bench found first: + GilliganIsland_L9 and HarleyDavidson_L3, both 17 characters. They were + shortened to GilliganIsle_L9 and HarleyDavid_L3 in #380, so the list of + known offenders this test used to carry is gone and the rule is now + absolute. + """ + limit = bench.gamename_field_bytes() + too_long = sorted(path.stem for path in (REPO_ROOT / "src").glob("*/config/*.json") if len(path.stem.encode()) > limit) + + assert too_long == [], f"config filenames longer than the {limit}-byte FRAM gamename field: {', '.join(too_long)}" + + +# -------------------------------------------------------------------------- +# the assertions themselves, with the board faked out +# -------------------------------------------------------------------------- + + +class FakeClient: + def __init__(self, *_args): + pass + + def close(self): + pass + + +def responder(responses): + """A stand-in for bench.get over a fake board.""" + + def fake_get(_client, route, expect=200): + body = responses[route] + if isinstance(body, Exception): + raise body + return body + + return fake_get + + +def healthy(config="AttackMars_11", name="Attack from Mars"): + return { + # HDWR02 is expected on a bare bench board with nothing driving the bus. + "/api/fault": ["HDWR02: No Bus Activity"], + "/api/game/active_config": {"active_config": config}, + "/api/game/name": name, + "/api/leaders": [], + "/api/adjustments/status": {"profiles": [], "adjustments_support": True}, + } + + +def test_check_booted_config_passes_when_the_board_reports_the_configured_game(monkeypatch): + monkeypatch.setattr(cm, "get", responder(healthy())) + + name = cm.check_booted_config(FakeClient(), "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + + assert name == "Attack from Mars" + + +def test_check_booted_config_catches_a_silent_fallback_to_the_generic_config(monkeypatch): + """The failure this harness exists for. + + A config that fails to apply raises nothing an outside observer can see: + GameDefsLoad drops to safe_defaults and the board serves a generic + definition, healthy in every other respect. Only the game name gives it + away. + """ + monkeypatch.setattr(cm, "get", responder(healthy(name="Generic System"))) + + with pytest.raises(bench.CheckFailure, match="fell back to a generic definition"): + cm.check_booted_config(FakeClient(), "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + + +def test_check_booted_config_catches_a_board_running_a_different_config(monkeypatch): + monkeypatch.setattr(cm, "get", responder(healthy(config="Taxi_L4"))) + + with pytest.raises(bench.CheckFailure, match="active config is 'Taxi_L4'"): + cm.check_booted_config(FakeClient(), "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + + +def test_check_booted_config_accepts_the_game_name_as_the_active_config_on_em(monkeypatch): + # EM answers /api/game/active_config with the game name (backend.py:481). + monkeypatch.setattr(cm, "get", responder(healthy(config="EM Machine", name="EM Machine"))) + + name = cm.check_booted_config(FakeClient(), "em", "EM_machine_", {"name": "EM Machine", "adjustments": False}) + + assert name == "EM Machine" + + +@pytest.mark.parametrize( + "faults, expected", + [ + (["CONF01: Invalid Configuration"], "safe defaults"), + (["CONF00: Unknown Configuration Error"], "safe defaults"), + (["HDWR01: Early Bus Activity"], "safe mode"), + (["HDWR00: Unknown Hardware Error"], "unexpected fault"), + ], +) +def test_check_faults_rejects_anything_that_invalidates_the_result(monkeypatch, faults, expected): + monkeypatch.setattr(cm, "get", responder({"/api/fault": faults})) + + with pytest.raises(bench.CheckFailure, match=expected): + cm.check_faults(FakeClient(), "AttackMars_11") + + +def test_check_faults_allows_the_bare_bench_fault(monkeypatch): + monkeypatch.setattr(cm, "get", responder({"/api/fault": ["HDWR02: No Bus Activity"]})) + + cm.check_faults(FakeClient(), "AttackMars_11") + + +def test_check_adjustments_fails_when_the_config_declares_the_section(monkeypatch): + monkeypatch.setattr(cm, "get", responder({"/api/adjustments/status": bench.CheckFailure("returned 500")})) + + with pytest.raises(bench.CheckFailure, match="500"): + cm.check_adjustments(FakeClient(), "AttackMars_11", declares_adjustments=True) + + +def test_check_adjustments_only_warns_for_a_config_with_no_adjustments(monkeypatch, capsys): + # A known firmware gap rather than a config problem - see check_adjustments(). + monkeypatch.setattr(cm, "get", responder({"/api/adjustments/status": bench.CheckFailure("returned 500")})) + + cm.check_adjustments(FakeClient(), "Taxi_L4", declares_adjustments=False) + + assert "::warning::Taxi_L4" in capsys.readouterr().out + + +def test_check_bundle_reports_a_config_missing_from_the_build(monkeypatch): + monkeypatch.setattr(cm, "get", responder({"/api/game/configs_list": {"Taxi_L4": {"name": "Taxi"}}})) + + with pytest.raises(bench.CheckFailure, match="AttackMars_11"): + cm.check_bundle(FakeClient(), "wpc", {"Taxi_L4": {"name": "Taxi"}, "AttackMars_11": {"name": "Attack from Mars"}}) + + +def test_check_bundle_reports_a_game_name_that_drifted_from_the_source(monkeypatch): + monkeypatch.setattr(cm, "get", responder({"/api/game/configs_list": {"Taxi_L4": {"name": "Taksi"}}})) + + with pytest.raises(bench.CheckFailure, match="bundle says 'Taksi'"): + cm.check_bundle(FakeClient(), "wpc", {"Taxi_L4": {"name": "Taxi"}}) + + +# -------------------------------------------------------------------------- +# the matrix loop, and what it does when a board stops answering +# -------------------------------------------------------------------------- + + +class FakeSession: + """Stands in for a board. `dies_after` makes it stop answering mid-run.""" + + def __init__(self, port, boot_timeout=None, dies_after=None): + self.port = port + self.boot_timeout = boot_timeout + self.dies_after = dies_after + self.connection = object() + self.client = FakeClient() + self.boot_log = [] + self.configs_set = [] + self.boots = 0 + self.starts = 0 + self.start_resets = [] + self.nudges = 0 + self.crashes = [] + self.restored = False + + def start(self, reset=True): + self.starts += 1 + self.start_resets.append(reset) + return self.wait_for_boot() + + def wait_for_boot(self): + self.boots += 1 + if self.dies_after is not None and self.boots > self.dies_after: + self.client = None + self.connection = None + raise bench.CheckFailure(f"{self.port} never reported its web server within 90s") + self.client = FakeClient() + self.connection = object() + return self.client + + def ensure_connected(self): + if self.connection is None: + return self.wait_for_boot() + return self.client + + def set_config(self, gamename): + if self.client is None: + raise bench.CheckFailure(f"could not reach the REPL on {self.port}") + self.configs_set.append(gamename) + + def reboot(self): + self.connection = None + + def nudge(self): + self.nudges += 1 + self.close() + + def close(self): + self.connection = None + self.client = None + + +def run_board(monkeypatch, fake_repo, session, check=None, **arg_overrides): + """Drive run_matrix against a fake board. `check` replaces the assertions.""" + if check is None: + + def check(_client, _target, _config, expected): + return expected["name"] + + monkeypatch.setattr(cm, "Session", lambda port, boot_timeout=None: session) + monkeypatch.setattr(cm, "check_bundle", lambda *a, **k: None) + monkeypatch.setattr(cm, "check_booted_config", check) + monkeypatch.setattr(cm, "restore_default", lambda s, _target: setattr(s, "restored", True)) + + defaults = {"configs": None, "limit": None, "changed_since": None, "keep_going": True, "boot_timeout": 90, "config_timeout": 60} + just_flashed = defaults.pop("just_flashed", False) + defaults.update(arg_overrides) + just_flashed = defaults.pop("just_flashed", just_flashed) + return cm.run_matrix({"port": session.port, "target": "wpc"}, Namespace(**defaults), just_flashed=just_flashed) + + +def test_run_matrix_walks_every_config_on_a_healthy_board(monkeypatch, fake_repo): + session = FakeSession("/dev/ttyFAKE") + + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session) + + assert failures == [] + assert passed == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] + assert session.configs_set == passed + assert session.restored is True + + +def test_run_matrix_abandons_a_board_that_stops_answering(monkeypatch, fake_repo): + """The 63-minute failure, in one test. + + The first bench run wrote the same timeout 63 times against a WPC board + that had wedged one config in. Two consecutive setup failures is enough to + know the board is gone. + """ + # Enough configs that abandoning the board leaves some unrun - the point + # being that we stop rather than time out against every one of them. + config_dir = fake_repo / "src" / "wpc" / "config" + for index in range(6): + write_config(config_dir, f"Filler_L{index}", f"Filler {index}", Adjustments={"Type": 0}) + + # Boot 1 is the bundle check and boot 2 is the first config, so the board + # survives exactly one config before going quiet. + session = FakeSession("/dev/ttyFAKE", dies_after=2) + + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session) + + assert len(passed) == 1 + # A dead board costs at most two attempts per config, each of which may + # spend a boot repairing the connection and a boot on the config itself - + # and only MAX_CONSECUTIVE_SETUP_FAILURES configs are attempted at all. + # The point is the ceiling: nine configs must not cost nine timeouts. + assert session.boots <= 1 + 4 * cm.MAX_CONSECUTIVE_SETUP_FAILURES + assert len(passed) + len([f for f in failures if f[0] != "(not run)"]) <= 1 + cm.MAX_CONSECUTIVE_SETUP_FAILURES + assert any("skipped after" in reason for _config, reason in failures) + # Every setup failure gets one cheap recovery attempt before we give up. + assert session.nudges == cm.MAX_CONSECUTIVE_SETUP_FAILURES + # Teardown still runs, so the next job does not inherit a stranded board. + assert session.restored is True + + +def test_run_matrix_keeps_going_after_a_failing_config(monkeypatch, fake_repo): + """An assertion failure is a result about the config, not about the board.""" + + def one_bad_config(_client, _target, config, expected): + if config == "Generic_WPC": + raise bench.CheckFailure("board reports game name 'Generic System' - fell back to a generic definition") + return expected["name"] + + session = FakeSession("/dev/ttyFAKE") + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session, check=one_bad_config) + + assert passed == ["AttackMars_11", "Taxi_L4"] + assert [config for config, _reason in failures] == ["Generic_WPC"] + + +def test_run_matrix_stops_at_the_first_failure_when_asked(monkeypatch, fake_repo): + def always_fails(*_args, **_kwargs): + raise bench.CheckFailure("board reports game name 'Generic System'") + + session = FakeSession("/dev/ttyFAKE") + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session, check=always_fails, keep_going=False) + + assert passed == [] + assert len(failures) == 1 + + +def test_restore_default_never_raises_when_the_board_is_gone(capsys): + """Teardown must not throw away results the matrix already produced. + + The first bench run died exactly here: a TimeoutExpired escaped teardown, + took out the run summary, and skipped the board that had not been touched + yet. + """ + + class DeadSession: + port = "/dev/ttyFAKE" + connection = None + + def wait_for_boot(self): + raise TimeoutError("board is not coming back") + + def close(self): + pass + + cm.restore_default(DeadSession(), "wpc") + + assert "::warning::could not restore Generic_WPC" in capsys.readouterr().out + + +# -------------------------------------------------------------------------- +# raw REPL framing +# -------------------------------------------------------------------------- + + +class FakeSerial: + """Replays a scripted board response and records what was written.""" + + def __init__(self, script=b""): + self.script = bytearray(script) + self.written = bytearray() + + def read(self, size=1): + chunk = bytes(self.script[:size]) + del self.script[: len(chunk)] + return chunk + + @property + def in_waiting(self): + return len(self.script) + + def write(self, data): + self.written.extend(data) + return len(data) + + def flush(self): + pass + + def reset_input_buffer(self): + pass + + def reset_output_buffer(self): + pass + + def close(self): + pass + + +def test_repl_enter_syncs_past_application_output(): + # The board is mid-sentence when we interrupt it, so the banner arrives + # after whatever it was printing. + serial = FakeSerial(b"RESOURCE: RAM=69%\r\nTraceback\r\nraw REPL; CTRL-B to exit\r\n>") + + bench.Repl(serial).enter(timeout=1) + + assert bench.CTRL_C in serial.written + assert bench.CTRL_A in serial.written + + +def test_repl_enter_reports_the_console_when_the_board_does_not_answer(): + serial = FakeSerial(b"RESOURCE: RAM=69%\r\n") + + with pytest.raises(bench.CheckFailure, match="raw REPL prompt"): + bench.Repl(serial).enter(timeout=0.2) + + +def test_repl_keeps_bytes_that_arrive_past_a_marker(): + """The bug that made the first version of this desynchronise. + + A read that syncs on `OK` almost always pulls in the output that follows + it; discarding that made every later marker arrive at an empty stream. + """ + repl = bench.Repl(FakeSerial(b"OKGAMENAME=Taxi_L4\x04\x04>")) + + assert repl.read_until(b"OK", 1, "ok") == b"" + assert repl.read_until(bench.CTRL_D, 1, "output") == b"GAMENAME=Taxi_L4" + + +def test_repl_exec_returns_what_the_snippet_printed(): + serial = FakeSerial(b"OKGAMENAME=Taxi_L4\r\n\x04\x04>") + + output = bench.Repl(serial).exec("print('x')", timeout=1) + + assert "GAMENAME=Taxi_L4" in output + assert serial.written.endswith(bench.CTRL_D) + + +def test_repl_exec_surfaces_a_traceback_from_the_board(): + serial = FakeSerial(b"OK\x04Traceback (most recent call last):\r\n KeyError: nope\x04>") + + with pytest.raises(bench.CheckFailure, match="KeyError: nope"): + bench.Repl(serial).exec("boom", timeout=1) + + +def fake_repl(monkeypatch, output): + class StubRepl: + def __init__(self, _connection): + pass + + def enter(self, timeout=None): + return self + + def exec(self, _code, timeout=None): + return output + + monkeypatch.setattr(bench, "Repl", StubRepl) + + +def test_set_game_config_rejects_a_name_the_board_truncated(monkeypatch): + fake_repl(monkeypatch, "GAMENAME=HarleyDavidson_L\r\n") + + with pytest.raises(bench.CheckFailure, match="16 bytes and this filename is 17"): + bench.set_game_config(FakeSerial(), "HarleyDavidson_L3") + + +def test_set_game_config_accepts_a_clean_round_trip(monkeypatch): + fake_repl(monkeypatch, "GAMENAME=Taxi_L4\r\n") + + bench.set_game_config(FakeSerial(), "Taxi_L4") + + +def test_write_step_summary_renders_a_table_and_the_failures(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + cm.write_step_summary([({"port": "/dev/ttyACM0", "target": "wpc"}, ["Taxi_L4"], [("AttackMars_11", "boom\nsecond line")])]) + + rendered = summary.read_text() + assert "| `/dev/ttyACM0` | wpc | 2 | 1 | 1 |" in rendered + assert "**wpc `AttackMars_11`** - boom" in rendered + assert "second line" not in rendered + + +def test_write_step_summary_is_a_no_op_outside_actions(monkeypatch): + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + + cm.write_step_summary([({"port": "/dev/ttyACM0", "target": "wpc"}, [], [])]) + + +# -------------------------------------------------------------------------- +# not hanging on a board that has stopped listening +# -------------------------------------------------------------------------- + + +def test_open_serial_always_sets_a_write_timeout(monkeypatch): + """The bug that hung a bench job for 30 minutes. + + `serial.Serial(timeout=...)` sets the READ timeout only. Without a + write_timeout, writing to a board that has stopped draining its USB + endpoint blocks forever - which is precisely the board the recovery tool + exists to write to. + """ + opened = {} + + def fake_serial(**kwargs): + opened.update(kwargs) + return FakeSerial() + + monkeypatch.setattr(bench.serial, "Serial", fake_serial, raising=False) + bench.open_serial("/dev/ttyFAKE") + + assert opened["timeout"] == bench.SERIAL_READ_TIMEOUT + assert opened["write_timeout"] == bench.SERIAL_WRITE_TIMEOUT + + +def test_serial_write_turns_a_stuck_board_into_an_error(monkeypatch): + monkeypatch.setattr(bench.serial, "SerialTimeoutException", RuntimeError, raising=False) + + class Deaf: + def write(self, _data): + raise RuntimeError("write timed out") + + with pytest.raises(bench.CheckFailure, match="stopped draining its USB endpoint"): + bench.serial_write(Deaf(), b"\x03", "the board") + + +def test_repl_never_calls_flush(): + """flush() is tcdrain, which takes no timeout and hangs on the same board. + + Handing the bytes to the kernel is enough; every exchange here is + synchronised by a read with a deadline instead. + """ + source = (REPO_ROOT / "dev" / "hil" / "bench.py").read_text() + + assert ".flush()" not in source + + +def test_time_limit_interrupts_work_that_overruns(): + with pytest.raises(bench.CheckFailure, match="did not finish within"): + with bench.time_limit(1, "a step that hangs"): + time.sleep(5) + + +def test_time_limit_is_invisible_when_work_finishes_in_time(): + with bench.time_limit(5, "quick work"): + result = 1 + 1 + + assert result == 2 + + +def test_time_limit_restores_the_previous_handler(): + import signal + + before = signal.getsignal(signal.SIGALRM) + with bench.time_limit(5, "quick work"): + pass + + assert signal.getsignal(signal.SIGALRM) is before + + +def test_run_matrix_resets_the_board_before_watching_its_first_boot(monkeypatch, fake_repo): + """The regression that failed all three boards without checking one config. + + The ready marker is printed once per boot. dev/flash.py resets at the end + of flashing and flashing runs over every board first, so by the time the + matrix opens a console the board booted minutes ago and the marker is gone. + Every later boot is triggered by reboot(); the first one has to be + triggered by start(). + """ + session = FakeSession("/dev/ttyFAKE") + + run_board(monkeypatch, fake_repo, session) + + assert session.starts == 1, "the first boot must be preceded by a reset" + + +def test_session_start_resets_then_waits(monkeypatch): + """start() is a reset plus a wait, in that order.""" + order = [] + monkeypatch.setattr(cm, "reset_board", lambda port: order.append(f"reset {port}")) + monkeypatch.setattr(cm, "wait_for_server", lambda port, timeout=None: (order.append("wait"), (types.SimpleNamespace(close=lambda: None), []))[1]) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + cm.Session("/dev/ttyFAKE").start() + + assert order == ["reset /dev/ttyFAKE", "wait"] + + +# -------------------------------------------------------------------------- +# a boot that crashes rather than one that is slow +# -------------------------------------------------------------------------- + +# The real WPC console from the bench run that prompted this, trimmed. +CRASHED_BOOT = [ + "Connected to wifi with IP address: 192.168.2.175", + "----------", + "Starting server", + "----------", + "2021-01-01 00:00:13 [info ] > starting web server on port 80", + "Traceback (most recent call last):", + ' File "main.py", line 1, in ', + ' File "build/wpc/backend.py", line 1, in go', + ' File "build/wpc/GameStatus.py", line 1, in ', + "ImportError: no module named 'origin'", + "MicroPython v1.26.0-preview.255.g214d6413d.dirty on 2025-07-19; Raspberry Pi Pico 2 W with RP2350", + 'Type "help()" for more information.', + ">>> ", +] + +HEALTHY_BOOT = [ + "Connected to wifi with IP address: 192.168.2.175", + "2021-01-01 00:00:13 [info ] > starting web server on port 80", + "Server: Loop Forever", +] + + +class ScriptedConsole: + """A serial port replaying a boot transcript, one line per readline().""" + + def __init__(self, lines): + self.lines = list(lines) + self.closed = False + + def readline(self): + if not self.lines: + return b"" + return (self.lines.pop(0) + "\r\n").encode() + + def close(self): + self.closed = True + + +def watch_boot(monkeypatch, lines, elapsed=10.0): + """Run wait_for_server against a scripted console.""" + console = ScriptedConsole(lines) + monkeypatch.setattr(bench, "open_serial", lambda *a, **k: console) + monkeypatch.setattr(bench, "SERVER_SETTLE_SECONDS", 0) + monkeypatch.setattr(bench, "CRASH_MARKER_GRACE", -1) + return console + + +def test_a_crashed_boot_fails_immediately_with_the_traceback(monkeypatch): + """The failure that cost 90s and reported the wrong thing. + + A board that raised on the way up will never print the ready marker, so + waiting out the budget only delays a failure the console already explains. + """ + watch_boot(monkeypatch, CRASHED_BOOT) + + with pytest.raises(bench.BootCrash) as caught: + bench.wait_for_server("/dev/ttyFAKE", timeout=90) + + message = str(caught.value) + assert "dropped to the REPL" in message + # The traceback leads, because it is the whole reason to catch this. + assert "ImportError: no module named 'origin'" in message + assert "Traceback (most recent call last)" in message + assert "never reported its web server" not in message + + +def test_a_healthy_boot_is_unaffected(monkeypatch): + console = watch_boot(monkeypatch, HEALTHY_BOOT) + + connection, transcript = bench.wait_for_server("/dev/ttyFAKE", timeout=90) + + assert connection is console + assert "Server: Loop Forever" in transcript[-1] + + +def test_a_repl_prompt_in_the_first_moments_is_not_a_crash(monkeypatch): + """Residue from the REPL session that issued the reset is not this boot.""" + watch_boot(monkeypatch, [">>> "] + HEALTHY_BOOT) + monkeypatch.setattr(bench, "CRASH_MARKER_GRACE", 3600) + + connection, _transcript = bench.wait_for_server("/dev/ttyFAKE", timeout=90) + + assert connection is not None + + +def test_a_boot_that_is_merely_slow_still_times_out(monkeypatch): + """Slow is not the same as crashed, and still reports as a timeout.""" + watch_boot(monkeypatch, ["still booting..."]) + + with pytest.raises(bench.CheckFailure, match="never reported its web server") as caught: + bench.wait_for_server("/dev/ttyFAKE", timeout=0.5) + + assert not isinstance(caught.value, bench.BootCrash) + + +def test_wait_for_boot_retries_once_past_a_crash(monkeypatch): + """An intermittent crash must not cost a whole board's matrix.""" + attempts = [] + + def flaky(_port, timeout=None): + attempts.append(1) + if len(attempts) == 1: + raise bench.BootCrash("crashed on the way up", CRASHED_BOOT) + return types.SimpleNamespace(close=lambda: None), HEALTHY_BOOT + + monkeypatch.setattr(cm, "wait_for_server", flaky) + monkeypatch.setattr(cm, "reset_board", lambda _port: None) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + session = cm.Session("/dev/ttyFAKE") + session.wait_for_boot() + + assert len(attempts) == 2 + # Recovered, but recorded: a board that only sometimes boots is a fault. + assert len(session.crashes) == 1 + + +def test_wait_for_boot_gives_up_after_a_second_crash(monkeypatch): + def always_crashes(_port, timeout=None): + raise bench.BootCrash("crashed on the way up", CRASHED_BOOT) + + monkeypatch.setattr(cm, "wait_for_server", always_crashes) + monkeypatch.setattr(cm, "reset_board", lambda _port: None) + + session = cm.Session("/dev/ttyFAKE") + with pytest.raises(bench.BootCrash): + session.wait_for_boot() + + assert len(session.crashes) == 2 + + +def test_recovered_crashes_are_reported_not_buried(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + board = {"port": "/dev/ttyACM1", "target": "wpc", "crashes": ["dropped to the REPL 13.0s into boot\n ImportError"]} + cm.write_step_summary([(board, ["Taxi_L4"], [])]) + + rendered = summary.read_text() + assert "Boot crashes (recovered by a retry)" in rendered + assert "**wpc**" in rendered + + +def test_a_missed_marker_is_recovered_by_asking_the_board(monkeypatch, capsys): + """The ready marker prints once, so missing it must not read as a dead board. + + Anything that costs us the moment it goes past - a board that booted while + the next one was still being flashed, a reset we did not trigger - looks + exactly like a board that never came up. Asking the API tells them apart. + """ + console = watch_boot(monkeypatch, ["already booted, marker long gone"]) + monkeypatch.setattr(bench, "_server_is_answering", lambda _connection: True) + + connection, _transcript = bench.wait_for_server("/dev/ttyFAKE", timeout=0.5) + + assert connection is console + assert "ready marker was never seen" in capsys.readouterr().out + + +def test_a_board_that_answers_nothing_still_fails(monkeypatch): + watch_boot(monkeypatch, ["nothing useful"]) + monkeypatch.setattr(bench, "_server_is_answering", lambda _connection: False) + + with pytest.raises(bench.CheckFailure, match="never reported its web server"): + bench.wait_for_server("/dev/ttyFAKE", timeout=0.5) + + +def test_server_is_answering_reads_the_usb_api(monkeypatch): + monkeypatch.setattr(bench, "prime_usb", lambda _connection: None) + monkeypatch.setattr(bench, "UsbApiClient", lambda _c: types.SimpleNamespace(send_and_receive=lambda **kw: {"status": 200, "body": {"version": "1.7.13"}})) + + assert bench._server_is_answering(object()) is True + + +def test_server_is_answering_is_false_when_the_board_is_silent(monkeypatch): + monkeypatch.setattr(bench, "prime_usb", lambda _connection: None) + + def times_out(**_kwargs): + raise TimeoutError("no response") + + monkeypatch.setattr(bench, "UsbApiClient", lambda _c: types.SimpleNamespace(send_and_receive=times_out)) + + assert bench._server_is_answering(object()) is False + + +def test_start_drains_and_retries_a_board_that_will_not_reset(monkeypatch): + """A board wedged on its own undrained output comes back when read. + + This is what took out data_east 36 minutes into a run: reading the console + is the remedy, and it costs seconds against a whole board's matrix. + """ + attempts = [] + drained = [] + + def reset(port): + attempts.append(port) + if len(attempts) == 1: + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + + monkeypatch.setattr(cm, "reset_board", reset) + monkeypatch.setattr(cm, "drain_port", lambda port, **kw: drained.append(port)) + monkeypatch.setattr(cm, "wait_for_server", lambda port, timeout=None: (types.SimpleNamespace(close=lambda: None), [])) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + cm.Session("/dev/ttyFAKE").start() + + assert len(attempts) == 2 + assert drained == ["/dev/ttyFAKE"] + + +def test_start_gives_up_if_the_retry_also_fails(monkeypatch): + def always_times_out(_port): + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + + monkeypatch.setattr(cm, "reset_board", always_times_out) + monkeypatch.setattr(cm, "drain_port", lambda port, **kw: None) + + with pytest.raises(subprocess.TimeoutExpired): + cm.Session("/dev/ttyFAKE").start() + + +def test_each_board_is_flashed_immediately_before_its_own_matrix(monkeypatch, fake_repo, tmp_path): + """Flashing every board up front is what wedges the ones waiting their turn. + + A board flashed and then left running for the half hour the boards ahead of + it take is printing into a USB console nothing drains, which is where + MicroPython deadlocks. Flashed here, it runs for seconds before we talk + to it. + """ + flashed = [] + monkeypatch.setattr(cm.bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + monkeypatch.setattr(cm.bench, "flash", lambda target, port, build_dir, config: flashed.append((target, port))) + + cm.flash_before_matrix({"target": "wpc", "port": "/dev/ttyACM1"}, tmp_path) + + assert flashed == [("wpc", "/dev/ttyACM1")] + + +# -------------------------------------------------------------------------- +# one wedged board must not abort the whole bench +# -------------------------------------------------------------------------- + + +def test_probe_recovers_a_board_that_hangs_then_answers(monkeypatch): + """A drain is the remedy for a board blocked on its own undrained output.""" + calls = [] + drained = [] + + def hangs_once(*args, timeout=None): + calls.append(args) + if len(calls) == 1: + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + return types.SimpleNamespace(returncode=0, stdout="df13a50c13958980", stderr="") + + monkeypatch.setattr(bench, "mpremote", hangs_once) + monkeypatch.setattr(bench, "drain_port", lambda port, **kw: drained.append(port)) + + board = bench.probe("/dev/ttyACM2") + + assert board["responsive"] is True + assert board["chip_id"] == "df13a50c13958980" + assert drained == ["/dev/ttyACM2"] + + +def test_probe_reports_a_board_that_never_answers(monkeypatch): + """The survey completes rather than raising - it is one board's problem. + + A board left wedged by an earlier run took out the next run inside + inventory, before a single config was checked. + """ + + def always_hangs(*_args, timeout=None): + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + + monkeypatch.setattr(bench, "mpremote", always_hangs) + monkeypatch.setattr(bench, "drain_port", lambda port, **kw: None) + + board = bench.probe("/dev/ttyACM2") + + assert board["responsive"] is False + assert board["chip_id"] is None + + +def test_resolve_targets_refuses_to_flash_a_board_that_will_not_talk(): + boards = [{"port": "/dev/ttyACM0", "chip_id": "aaa", "system": "sys11", "responsive": True}, {"port": "/dev/ttyACM2", "chip_id": None, "system": None, "responsive": False}] + + with pytest.raises(bench.CheckFailure, match="not answering: /dev/ttyACM2"): + bench.resolve_targets(boards, {"aaa": "sys11"}) + + +def test_a_config_that_fails_once_and_passes_is_the_board_not_the_config(monkeypatch, fake_repo): + """ "Broken config" and "flaky board" look identical on one attempt. + + They need opposite responses, so the matrix asks twice. This board fails + Generic_WPC once and passes it on the retry - the WPC board on the bench + does exactly this, raising ENOENT on files that exist. + """ + seen = [] + + def flaky_once(_client, _target, config, expected): + seen.append(config) + if config == "Generic_WPC" and seen.count("Generic_WPC") == 1: + raise bench.CheckFailure("/api/leaders returned 500, expected 200") + return expected["name"] + + session = FakeSession("/dev/ttyFAKE") + passed, failures, _crashes, flakes = run_board(monkeypatch, fake_repo, session, check=flaky_once) + + # Counted as a pass, because the config is fine... + assert "Generic_WPC" in passed + assert failures == [] + # ...but never silently: the flake is reported against the board. + assert [config for config, _reason in flakes] == ["Generic_WPC"] + assert "500" in flakes[0][1] + + +def test_a_config_that_fails_twice_is_a_real_failure(monkeypatch, fake_repo): + def always_fails(_client, _target, config, expected): + if config == "Generic_WPC": + raise bench.CheckFailure("board reports game name 'Generic System'") + return expected["name"] + + session = FakeSession("/dev/ttyFAKE") + passed, failures, _crashes, flakes = run_board(monkeypatch, fake_repo, session, check=always_fails) + + assert [config for config, _reason in failures] == ["Generic_WPC"] + assert flakes == [] + assert "Generic_WPC" not in passed + + +def test_flaky_configs_are_reported_in_the_job_summary(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + board = {"port": "/dev/ttyACM1", "target": "wpc", "crashes": [], "flakes": [("Congo_21", "/api/leaders returned 500, expected 200")]} + cm.write_step_summary([(board, ["Congo_21"], [])]) + + rendered = summary.read_text() + assert "Flaky (failed once, passed on retry" in rendered + assert "**wpc `Congo_21`**" in rendered + + +def test_a_freshly_flashed_board_is_not_reset_again(monkeypatch, fake_repo): + """dev/flash.py already reset it, so it is mid-boot. + + Resetting again lands on top of that boot while the firmware is still + reading its freshly written filesystem. sys11 wedged in exactly that + window - flashed successfully, then refusing a reset seconds later. + """ + session = FakeSession("/dev/ttyFAKE") + + run_board(monkeypatch, fake_repo, session, just_flashed=True) + + assert session.start_resets == [False] + + +def test_a_board_we_did_not_flash_is_reset_first(monkeypatch, fake_repo): + """--skip-flash means nobody triggered a boot, so we have to.""" + session = FakeSession("/dev/ttyFAKE") + + run_board(monkeypatch, fake_repo, session, just_flashed=False) + + assert session.start_resets == [True] + + +# -------------------------------------------------------------------------- +# a board that stops talking is repaired, not written off +# -------------------------------------------------------------------------- + + +class QuietingSerial: + """A port that hands over a backlog in mouthfuls, then falls silent.""" + + def __init__(self, chunks): + self.chunks = list(chunks) + self.written = [] + self.closed = False + + @property + def in_waiting(self): + return len(self.chunks[0]) if self.chunks else 0 + + def read(self, _size): + return self.chunks.pop(0) if self.chunks else b"" + + def write(self, data): + self.written.append(data) + + def close(self): + self.closed = True + + +def test_read_until_quiet_keeps_reading_while_the_board_is_still_printing(monkeypatch): + """Three seconds catches the first mouthful and calls a live board dead. + + /dev/ttyACM0 was written off after a drain of 63 bytes; the next step in + the same job read it with `cat` and got a healthy running server. + """ + connection = QuietingSerial([b"RESOURCE: ", b"RAM=71%", b"Server: Check Wifi"]) + + drained = bench.read_until_quiet(connection, budget=30, quiet=0) + + assert drained == len(b"RESOURCE: RAM=71%Server: Check Wifi") + assert connection.chunks == [] + + +def test_read_until_quiet_stops_once_the_port_has_been_silent(monkeypatch): + """A port with nothing to say costs `quiet`, not the whole budget.""" + connection = QuietingSerial([]) + started = time.monotonic() + + assert bench.read_until_quiet(connection, budget=30, quiet=0.05) == 0 + assert time.monotonic() - started < 5 + + +def test_drain_port_interrupts_only_after_it_has_read(monkeypatch): + """The Ctrl-C is useless until the read has unblocked the firmware. + + A board blocked writing to stdout is not reading stdin either, so writing + first just times out against the wedge we are trying to clear. + """ + order = [] + + class Recorder(QuietingSerial): + def read(self, size): + chunk = super().read(size) + if chunk: + order.append("read") + return chunk + + def write(self, data): + order.append("write") + super().write(data) + + connection = Recorder([b"backlog"]) + monkeypatch.setattr(bench, "open_serial", lambda port, **kw: connection) + monkeypatch.setattr(bench, "DRAIN_QUIET_SECONDS", 0.05) + + assert bench.drain_port("/dev/ttyFAKE", seconds=5) == len(b"backlog") + assert order[0] == "read" + assert "write" in order + assert connection.closed + + +def test_interrupt_board_reports_a_board_that_will_not_accept_input(monkeypatch): + """Talking but not listening is a diagnosis, not a reason to keep writing. + + Both callers drain to quiet before interrupting, and reading is the remedy + for the wedge where the write times out because the firmware is blocked on + stdout. A write that still times out after that means the board is + producing output happily and simply never services what we send it - + /dev/ttyACM0 sat in exactly that state for three runs. Another five-second + write only spends five more seconds saying so. + """ + attempts = [] + + class Deaf(QuietingSerial): + def write(self, data): + attempts.append(data) + raise bench.serial.SerialTimeoutException("blocked") + + assert bench.interrupt_board(Deaf([]), "/dev/ttyFAKE") is False + assert len(attempts) == 1 + + +def test_interrupt_board_reports_a_board_that_takes_the_ctrl_c(monkeypatch): + connection = QuietingSerial([]) + + assert bench.interrupt_board(connection, "/dev/ttyFAKE") is True + assert connection.written == [bench.CTRL_C] + + +def test_a_session_whose_boot_failed_is_repaired_before_the_next_config(monkeypatch): + """One exhausted boot must not poison every config after it. + + The WPC board crashed on boot twice for one config, and from there the + retry failed instantly and the next config failed in 0.0s - both blamed on + a board that came back on its own moments later. + """ + drained = [] + resets = [] + + session = cm.Session("/dev/ttyFAKE") + monkeypatch.setattr(cm, "drain_port", lambda port, **kw: drained.append(port)) + monkeypatch.setattr(cm, "reset_board", lambda port: resets.append(port)) + monkeypatch.setattr(cm, "wait_for_server", lambda port, timeout=None: (types.SimpleNamespace(close=lambda: None), [])) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + assert session.connection is None + client = session.ensure_connected() + + assert client is not None + assert session.connection is not None + assert drained == ["/dev/ttyFAKE"] + assert resets == ["/dev/ttyFAKE"] + + +def test_ensure_connected_leaves_a_live_session_alone(monkeypatch): + """It repairs a lost connection; it does not add a boot to every config.""" + session = cm.Session("/dev/ttyFAKE") + session.connection = types.SimpleNamespace(close=lambda: None) + session.client = FakeClient() + + def fail(*_args, **_kwargs): + raise AssertionError("a connected session must not be reset") + + monkeypatch.setattr(cm, "drain_port", fail) + monkeypatch.setattr(cm, "reset_board", fail) + + assert session.ensure_connected() is session.client diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py new file mode 100644 index 00000000..a88d65f2 --- /dev/null +++ b/dev/tests/test_hil_recover.py @@ -0,0 +1,606 @@ +"""Tests for the bench recovery ladder, with the hardware faked out. + +The escalation order is the whole design - cheapest and least destructive +first, stopping the moment the board answers - so that is what these pin down, +along with the checksum gate on anything that gets flashed. +""" + +from __future__ import annotations + +import subprocess +import sys +import types +from argparse import Namespace +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +_serial_stub = sys.modules.setdefault("serial", types.ModuleType("serial")) +if not hasattr(_serial_stub, "SerialTimeoutException"): + _serial_stub.SerialTimeoutException = type("SerialTimeoutException", (Exception,), {}) +if "usb_coms_demo" not in sys.modules: + stub = types.ModuleType("usb_coms_demo") + stub.UsbApiClient = object + sys.modules["usb_coms_demo"] = stub + +sys.path.insert(0, str(REPO_ROOT / "dev" / "hil")) + +import bench # noqa: E402 +import recover # noqa: E402 +import trench_coat # noqa: E402 + + +def args(**overrides): + defaults = {"reflash": True, "no_power_cycle": False, "force_bootsel": False, "step_timeout": 30, "cache_dir": Path("/nonexistent")} + defaults.update(overrides) + return Namespace(**defaults) + + +@pytest.fixture() +def ladder(monkeypatch): + """Record which recovery steps ran, and let a test choose which one works.""" + calls = [] + + def step(name, works_at=None): + def run(*_args, **_kwargs): + calls.append(name) + return True + + return run + + monkeypatch.setattr(recover, "drain", step("drain")) + monkeypatch.setattr(recover, "usb_reset", step("usb_reset")) + monkeypatch.setattr(recover, "power_cycle", step("power_cycle")) + monkeypatch.setattr(recover, "reflash", step("reflash")) + monkeypatch.setattr(recover, "SETTLE_SECONDS", 0) + return calls + + +def recovers_after(monkeypatch, calls, step_count): + """Make the board answer once `step_count` steps have run.""" + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: len(calls) >= step_count) + + +def test_the_cheapest_step_that_works_ends_the_ladder(monkeypatch, ladder): + recovers_after(monkeypatch, ladder, 1) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain"] + assert "drain" in method + + +def test_it_escalates_only_as_far_as_it_has_to(monkeypatch, ladder): + recovers_after(monkeypatch, ladder, 3) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain", "usb_reset", "power_cycle"] + assert "power cycle" in method + + +def test_reflashing_is_the_last_resort_and_not_before(monkeypatch, ladder): + recovers_after(monkeypatch, ladder, 4) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain", "usb_reset", "power_cycle", "reflash"] + assert "reflash" in method + + +def test_a_board_that_never_answers_reports_defeat(monkeypatch, ladder): + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: False) + + assert recover.recover("/dev/ttyFAKE", "wpc", args()) is None + assert ladder == ["drain", "usb_reset", "power_cycle", "reflash"] + + +def test_no_reflash_leaves_the_firmware_alone(monkeypatch, ladder): + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: False) + + recover.recover("/dev/ttyFAKE", "wpc", args(reflash=False)) + + assert "reflash" not in ladder + + +def test_reflash_is_skipped_when_the_target_is_unknown(monkeypatch, ladder, capsys): + # Flashing the wrong system's UF2 is worse than leaving the board dead. + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: False) + + recover.recover("/dev/ttyFAKE", None, args()) + + assert "reflash" not in ladder + assert "no target known" in capsys.readouterr().out + + +def test_a_failing_step_does_not_stop_the_ladder(monkeypatch, ladder): + def explode(*_args, **_kwargs): + ladder.append("usb_reset") + raise OSError("ioctl went wrong") + + monkeypatch.setattr(recover, "usb_reset", explode) + recovers_after(monkeypatch, ladder, 3) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain", "usb_reset", "power_cycle"] + assert method is not None + + +# -------------------------------------------------------------------------- +# survey +# -------------------------------------------------------------------------- + + +def fake_survey(monkeypatch, ports, dead_ports, chip_ids): + monkeypatch.setattr(recover, "list_ports", lambda: ports) + monkeypatch.setattr(recover, "responsive", lambda port, timeout=None: port not in dead_ports) + + def fake_mpremote(*call, timeout=None): + port = call[1] + return types.SimpleNamespace(returncode=0, stdout=chip_ids[port], stderr="") + + monkeypatch.setattr(recover, "mpremote", fake_mpremote) + + +def test_survey_deduces_the_dead_board_by_elimination(monkeypatch): + """A wedged board cannot say what it is, but the others can say what it is not.""" + fake_survey( + monkeypatch, + ports=["/dev/ttyACM0", "/dev/ttyACM1", "/dev/ttyACM2"], + dead_ports={"/dev/ttyACM1"}, + chip_ids={"/dev/ttyACM0": "aaa", "/dev/ttyACM2": "ccc"}, + ) + board_map = {"aaa": "sys11", "bbb": "wpc", "ccc": "data_east"} + + alive, dead, targets = recover.survey(board_map) + + assert alive == ["/dev/ttyACM0", "/dev/ttyACM2"] + assert dead == ["/dev/ttyACM1"] + assert targets == {"/dev/ttyACM1": "wpc"} + + +def test_survey_will_not_guess_when_two_boards_are_down(monkeypatch): + fake_survey( + monkeypatch, + ports=["/dev/ttyACM0", "/dev/ttyACM1", "/dev/ttyACM2"], + dead_ports={"/dev/ttyACM1", "/dev/ttyACM2"}, + chip_ids={"/dev/ttyACM0": "aaa"}, + ) + board_map = {"aaa": "sys11", "bbb": "wpc", "ccc": "data_east"} + + _alive, dead, targets = recover.survey(board_map) + + assert sorted(dead) == ["/dev/ttyACM1", "/dev/ttyACM2"] + assert targets == {} + + +def test_survey_fails_when_nothing_is_attached(monkeypatch): + monkeypatch.setattr(recover, "list_ports", lambda: []) + + with pytest.raises(bench.CheckFailure, match="no boards found"): + recover.survey({}) + + +def test_responsive_is_false_when_mpremote_times_out(monkeypatch): + def times_out(*_args, **_kwargs): + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=20) + + monkeypatch.setattr(recover, "mpremote", times_out) + + assert recover.responsive("/dev/ttyFAKE") is False + + +# -------------------------------------------------------------------------- +# handing the board to TrenchCoat +# -------------------------------------------------------------------------- + + +def test_reflash_delegates_to_trench_coat(monkeypatch, tmp_path): + """The sequence is TrenchCoat's, not ours - we only point it at one board.""" + called = {} + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (True, "udisksctl is available")) + monkeypatch.setattr(recover.trench_coat, "flash", lambda port, target, root: called.update(port=port, target=target, root=root) or True) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) is True + assert called["port"] == "/dev/ttyFAKE" + assert called["target"] == "wpc" + + +def test_reflash_will_not_strand_a_board_in_bootsel_it_cannot_flash(monkeypatch, tmp_path, capsys): + """Entering BOOTSEL is a one-way door. + + A wedged board is at least still a serial device. Sending it to the ROM + bootloader with no way to write a UF2 turns it into a mass-storage device + that only a replug gets out of - strictly worse than how it was found. + """ + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) + monkeypatch.setattr(recover.trench_coat, "flash", lambda *a, **k: pytest.fail("must not touch the board")) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) is False + assert "not touching the board into BOOTSEL" in capsys.readouterr().out + + +def test_force_bootsel_overrides_the_guard(monkeypatch, tmp_path): + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) + monkeypatch.setattr(recover.trench_coat, "flash", lambda *a, **k: True) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path, force=True) is True + + +def test_can_complete_a_reflash_accepts_udisksctl(monkeypatch): + monkeypatch.setattr(recover.shutil, "which", lambda name: "/usr/bin/udisksctl" if name == "udisksctl" else None) + monkeypatch.setattr(recover.Path, "is_dir", lambda self: False) + + possible, why = recover.can_complete_a_reflash() + + assert possible is True + assert "udisksctl" in why + + +def test_can_complete_a_reflash_says_no_when_nothing_can_mount(monkeypatch): + monkeypatch.setattr(recover.shutil, "which", lambda _name: None) + monkeypatch.setattr(recover.Path, "is_dir", lambda self: False) + + possible, why = recover.can_complete_a_reflash() + + assert possible is False + # Names every route it looked for, so the message says what to install. + assert "no picotool" in why and "no udisksctl" in why + + +def test_every_target_maps_to_a_bundled_uf2(): + for target, filename in trench_coat.TARGET_UF2.items(): + assert filename.endswith(".uf2"), target + + +def test_bundled_uf2_rejects_an_unknown_target(tmp_path): + with pytest.raises(bench.CheckFailure, match="no TrenchCoat UF2 known"): + trench_coat.bundled_uf2(tmp_path, "whitestar") + + +def test_bundled_uf2_reports_a_checkout_missing_the_file(tmp_path): + with pytest.raises(bench.CheckFailure, match="missing from the trench-coat checkout"): + trench_coat.bundled_uf2(tmp_path, "wpc") + + +def test_load_refuses_the_wrong_src_package(tmp_path, monkeypatch): + """Both repos have a top-level `src`; importing ours would fail confusingly.""" + fake = tmp_path / "src" + fake.mkdir() + (fake / "__init__.py").write_text("") + (fake / "core.py").write_text("") + (fake / "ray.py").write_text("") + + real_import = trench_coat.load + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: root) + + # Loading from the right root works; the guard only fires on a mismatch, + # which is what the assertion inside load() covers. + core, _ray = real_import(tmp_path) + assert str(Path(core.__file__).resolve()).startswith(str(tmp_path.resolve())) + + +# -------------------------------------------------------------------------- +# sysfs plumbing +# -------------------------------------------------------------------------- + + +def test_usb_device_path_walks_up_to_the_device(tmp_path, monkeypatch): + device = tmp_path / "sys" / "devices" / "usb1" / "1-1.4" + interface = device / "1-1.4:1.0" / "tty" / "ttyACM1" + interface.mkdir(parents=True) + (device / "busnum").write_text("1\n") + (device / "devnum").write_text("7\n") + + link = tmp_path / "sys" / "class" / "tty" / "ttyACM1" / "device" + link.parent.mkdir(parents=True) + link.symlink_to(device / "1-1.4:1.0") + + monkeypatch.setattr(recover, "Path", lambda p: link if str(p).endswith("/device") else Path(p)) + + assert recover.usb_device_path("/dev/ttyACM1") == Path("/dev/bus/usb/001/007") + + +@pytest.mark.parametrize( + "usb_path, expected", + [ + ("1-1.4:1.0", ("1-1", "4")), + ("1-1.2.3:1.0", ("1-1.2", "3")), + # A root-hub port (no dot) is deliberately not switchable - see + # test_hub_location_refuses_a_root_hub_port. + ], +) +def test_hub_location_splits_the_usb_path(tmp_path, monkeypatch, usb_path, expected): + device = tmp_path / usb_path + device.mkdir(parents=True) + (device / "busnum").write_text("1\n") + + link = tmp_path / "link" + link.symlink_to(device) + monkeypatch.setattr(recover, "Path", lambda p: link if str(p).endswith("/device") else Path(p)) + + assert recover.hub_location("/dev/ttyACM1") == expected + + +# -------------------------------------------------------------------------- +# narrowing TrenchCoat to one board +# -------------------------------------------------------------------------- + + +def fake_trench_coat(monkeypatch, drives=("/media/RPI-RP2",)): + """Stand in for TrenchCoat's core/ray modules and record how they are used.""" + seen = {"flashed": None, "ports_seen": None, "drives_seen": None, "bootloader": []} + + class FakeRay: + def __init__(self, port): + seen["bootloader"].append(port) + + def enter_bootloader_mode(self): + pass + + @classmethod + def find_board_ports(cls): + return ["/dev/ttyACM0", "/dev/ttyACM1", "/dev/ttyACM2"] + + ray = types.SimpleNamespace(Ray=FakeRay, serial=types.SimpleNamespace(Serial=lambda *a, **k: object())) + core = types.SimpleNamespace( + list_rpi_rp2_drives=lambda: list(drives), + graceful_exit=lambda now=False: None, + flash_firmware=lambda path: seen.update( + flashed=path, + ports_seen=ray.Ray.find_board_ports(), + drives_seen=core.list_rpi_rp2_drives(), + ), + ) + + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: root) + monkeypatch.setattr(trench_coat, "load", lambda root: (core, ray)) + monkeypatch.setattr(trench_coat, "bundled_uf2", lambda root, target: Path(f"/uf2/{target}.uf2")) + # Drive discovery is the real function now, and it neither finds a fake + # drive nor fails fast - without this the tests sit through its full wait. + monkeypatch.setattr(trench_coat, "find_bootloader_drives", lambda: list(drives)) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: list(drives)) + monkeypatch.setattr(trench_coat, "bootsel_touch", lambda port: None) + # The bench's healthy boards. Real serial ports would make what + # find_board_ports is allowed to see depend on the developer's own desk. + monkeypatch.setattr(trench_coat, "serial_ports", lambda: ["/dev/ttyACM0", "/dev/ttyACM2"]) + return seen + + +def test_flash_hides_the_other_boards_from_trench_coat(monkeypatch, tmp_path): + """The one thing that must not go wrong. + + TrenchCoat flashes every board it finds, which on the bench would nuke the + two healthy boards alongside the broken one. It only ever gets to see the + board being recovered. + """ + seen = fake_trench_coat(monkeypatch) + + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is True + # Neither healthy board is visible to it, so neither gets reset. + assert seen["ports_seen"] == [] + assert seen["flashed"] == "/uf2/wpc.uf2" + # Only the board being recovered is asked to enter the bootloader. + assert seen["bootloader"] == ["/dev/ttyACM1"] + + +def test_flash_gives_trench_coat_the_drive_it_could_not_find(monkeypatch, tmp_path): + """A headless runner has no automounter, so we mount and hand the path over.""" + seen = fake_trench_coat(monkeypatch, drives=()) + monkeypatch.setattr(trench_coat, "mount_rpi_rp2", lambda: "/media/runner/RPI-RP2") + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: ["/media/runner/RPI-RP2"]) + + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is True + assert seen["drives_seen"] == ["/media/runner/RPI-RP2"] + + +def test_flash_stops_when_the_board_never_reaches_the_bootloader(monkeypatch, tmp_path, capsys): + seen = fake_trench_coat(monkeypatch, drives=()) + monkeypatch.setattr(trench_coat, "mount_rpi_rp2", lambda: None) + monkeypatch.setattr(trench_coat, "bootsel_touch", lambda port: None) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: []) + + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is False + assert seen["flashed"] is None + assert "never presented a bootloader drive" in capsys.readouterr().out + + +def test_flash_turns_trench_coats_exit_into_an_error(monkeypatch, tmp_path): + """Its failure path calls sys.exit; the harness needs an exception instead.""" + seen = fake_trench_coat(monkeypatch) + core, _ray = trench_coat.load(tmp_path) + + def bail(_path): + core.graceful_exit() + + core.flash_firmware = bail + + with pytest.raises(bench.CheckFailure, match="could not complete the flash"): + trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) + assert seen["flashed"] is None + + +def test_enter_bootloader_falls_back_to_the_1200_baud_touch(monkeypatch): + """TrenchCoat's route needs the VM alive enough to run one statement.""" + touched = [] + attempts = [] + + class FakeRay: + def __init__(self, port): + pass + + def enter_bootloader_mode(self): + attempts.append("machine.bootloader()") + + monkeypatch.setattr(trench_coat, "bootsel_touch", lambda port: touched.append(port)) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: [] if not touched else ["/media/RPI-RP2"]) + + drives = trench_coat.enter_bootloader(types.SimpleNamespace(), types.SimpleNamespace(Ray=FakeRay), "/dev/ttyACM1") + + assert attempts == ["machine.bootloader()"] + assert touched == ["/dev/ttyACM1"] + assert drives == ["/media/RPI-RP2"] + + +def test_hub_location_refuses_a_root_hub_port(tmp_path, monkeypatch, capsys): + """Cutting a root hub takes every board down, not just the broken one.""" + device = tmp_path / "1-1:1.0" + device.mkdir(parents=True) + (device / "busnum").write_text("1\n") + + link = tmp_path / "link" + link.symlink_to(device) + monkeypatch.setattr(recover, "Path", lambda p: link if str(p).endswith("/device") else Path(p)) + + assert recover.hub_location("/dev/ttyACM1") is None + assert "cut every board on the bus" in capsys.readouterr().out + + +def test_power_cycle_stands_down_without_a_switchable_hub_port(monkeypatch, capsys): + monkeypatch.setattr(recover.shutil, "which", lambda _name: "/usr/sbin/uhubctl") + monkeypatch.setattr(recover, "hub_location", lambda _port: None) + + assert recover.power_cycle("/dev/ttyACM1") is False + assert "could not work out which hub port" in capsys.readouterr().out + + +def test_trench_coats_serial_writes_are_bounded(monkeypatch): + """TrenchCoat opens with a read timeout only, which hangs on a wedged board. + + One recovery run spent its whole 600s step budget inside + enter_bootloader_mode because of this, and only the SIGALRM backstop ended + it. Their code is written for a healthy board on a desktop; ours is pointed + at a board that is broken by definition. + """ + opened = {} + + class FakeSerialModule: + @staticmethod + def Serial(*args, **kwargs): + opened.update(kwargs) + return object() + + ray = types.SimpleNamespace(serial=FakeSerialModule) + trench_coat.bound_serial_writes(ray) + ray.serial.Serial("/dev/ttyFAKE", 115200, timeout=0.1) + + assert opened["write_timeout"] == bench.SERIAL_WRITE_TIMEOUT + + +def test_bound_serial_writes_leaves_an_explicit_timeout_alone(monkeypatch): + opened = {} + + class FakeSerialModule: + @staticmethod + def Serial(*args, **kwargs): + opened.update(kwargs) + return object() + + ray = types.SimpleNamespace(serial=FakeSerialModule) + trench_coat.bound_serial_writes(ray) + ray.serial.Serial("/dev/ttyFAKE", write_timeout=99) + + assert opened["write_timeout"] == 99 + + +# -------------------------------------------------------------------------- +# recovery as a call, not an instruction +# -------------------------------------------------------------------------- + + +def test_inventory_repairs_a_board_that_did_not_answer(monkeypatch, capsys): + """The bench runs unattended: printing "run recover.py" helps nobody at 3am.""" + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + + probes = iter( + [ + {"port": "/dev/ttyACM0", "chip_id": None, "system": None, "version": None, "responsive": False}, + {"port": "/dev/ttyACM0", "chip_id": "aaaa", "system": "sys11", "version": "1.10.7", "responsive": True}, + ] + ) + monkeypatch.setattr(bench, "probe", lambda port: next(probes)) + monkeypatch.setattr(bench, "repair_board", lambda port, target=None: "drain the console") + + boards = bench.inventory() + + # Re-probed after the repair, so the run continues with a live board. + assert boards[0]["responsive"] is True + assert boards[0]["chip_id"] == "aaaa" + assert "came back after drain the console" in capsys.readouterr().out + + +def test_a_board_that_stays_dead_is_reported_not_hidden(monkeypatch, capsys): + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(bench, "probe", lambda port: {"port": port, "chip_id": None, "system": None, "version": None, "responsive": False}) + monkeypatch.setattr(bench, "repair_board", lambda port, target=None: None) + + boards = bench.inventory() + + assert boards[0]["responsive"] is False + printed = capsys.readouterr().out + assert "still not answering" in printed + assert "NOT ANSWERING" in printed + + +def test_repair_can_be_turned_off(monkeypatch): + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(bench, "probe", lambda port: {"port": port, "chip_id": None, "system": None, "version": None, "responsive": False}) + monkeypatch.setattr(bench, "repair_board", lambda *a, **k: pytest.fail("must not repair when asked not to")) + + bench.inventory(repair=False) + + +def test_an_automatic_repair_never_reflashes(monkeypatch): + """The one rung that must stay a human decision mid-run. + + A reflash leaves the board running TrenchCoat's bundled firmware, so a + matrix that carried on afterwards would be testing firmware other than the + build under test - and reporting the results as if it were not. + """ + seen = {} + monkeypatch.setattr(recover, "recover", lambda port, target, options: seen.update(port=port, target=target, options=options) or "drain the console") + + assert bench.repair_board("/dev/ttyACM1") == "drain the console" + assert seen["options"].reflash is False + # And the cheap rungs are all still enabled. + assert seen["options"].no_power_cycle is False + + +def test_a_repair_may_be_asked_for_the_full_ladder(monkeypatch): + """The recover stage does want it: flash_and_check re-flashes right after.""" + seen = {} + monkeypatch.setattr(recover, "recover", lambda port, target, options: seen.update(options=options) or None) + + bench.repair_board("/dev/ttyACM1", target="wpc", allow_reflash=True) + + assert seen["options"].reflash is True + + +def test_a_repair_is_bounded_well_below_the_stage_cap(monkeypatch): + seen = {} + monkeypatch.setattr(recover, "recover", lambda port, target, options: seen.update(options=options) or None) + + bench.repair_board("/dev/ttyACM1") + + # The 600s ceiling in recover.py is for the reflash, which this never runs. + assert seen["options"].step_timeout == bench.REPAIR_TIMEOUT < recover.STEP_TIMEOUT + + +def test_repair_does_not_depend_on_the_caller_having_fixed_sys_path(monkeypatch): + """bench does not add its own directory; every harness happens to. + + Relying on that leaves the in-process import broken for the first caller + that does not, which is a failure mode with no signal until a board wedges. + """ + here = str(REPO_ROOT / "dev" / "hil") + monkeypatch.setattr(sys, "path", [p for p in sys.path if p != here]) + monkeypatch.setattr(recover, "recover", lambda port, target, options: "drain the console") + + assert bench.repair_board("/dev/ttyACM1") == "drain the console" + assert here in sys.path