From bb8f4076b3a3ec4f6533636d9fdf3bd1939c541d Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 02:54:43 +0000 Subject: [PATCH 01/32] test(hil): boot every board against every config it can be flashed with Implements DESIGN.md's G3. Per board: build and flash that target once, then loop over every config in src//config/ - write gamename into the FRAM configuration record over the REPL, reset, wait for the ready marker, and ask the board what it loaded. The game name is the assertion that earns its keep. A config that fails to apply does not fault or crash from outside: GameDefsLoad.go falls back to safe_defaults and the board serves the generic definition for its hardware, healthy in every other respect. /api/game/active_config does not catch that either - it reads the gamename field back out of FRAM, not what actually loaded. Comparing /api/game/name against GameInfo.GameName in the source JSON is what separates "loaded my config" from "silently fell back", and it cross-checks the on-board bundle against the repo while it is there. HDWR01 is fatal here even though flash_and_check.py only warns about it: it sends main.py down the safe_mode path where the config is never read, so every downstream assertion would pass or fail for reasons unrelated to the config under test. Shared plumbing moves to bench.py verbatim - inventory, resolve, build, flash, reset, wait-for-boot, USB request - so both harnesses agree on how a board is brought up and neither has to be edited to add a check. flash_and_check.py keeps its own assertions and behaves exactly as before. Two findings from writing this, both pre-existing and neither fixed here: - HarleyDavidson_L3 and GilliganIsland_L9 are 17 characters and the FRAM gamename field is 16, so struct.pack truncates them and the two games can never be selected on a real board - the UI offers them, the write is accepted, and the next boot comes up on safe defaults with CONF01. The harness catches this before spending a boot cycle, and a unit test catches any new offender in ordinary CI. - /api/adjustments/status 500s for the 14 configs that declare no Adjustments section, because gdata is not merged with safe_defaults. Warned about rather than failed, so a firmware gap the harness cannot fix does not bury the signal it exists for. Hardware-free parts are unit tested: config discovery, selection and ordering, and each assertion with the board faked out - including the silent-fallback case the whole harness is built around. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-config-matrix.yml | 126 +++++ dev/hil/DESIGN.md | 80 +++- dev/hil/RUNNER_SETUP.md | 26 ++ dev/hil/bench.py | 598 ++++++++++++++++++++++++ dev/hil/config_matrix.py | 459 ++++++++++++++++++ dev/hil/flash_and_check.py | 530 ++------------------- dev/tests/test_hil_config_matrix.py | 317 +++++++++++++ 7 files changed, 1629 insertions(+), 507 deletions(-) create mode 100644 .github/workflows/hil-config-matrix.yml create mode 100644 dev/hil/bench.py create mode 100644 dev/hil/config_matrix.py create mode 100644 dev/tests/test_hil_config_matrix.py diff --git a/.github/workflows/hil-config-matrix.yml b/.github/workflows/hil-config-matrix.yml new file mode 100644 index 00000000..a05c766d --- /dev/null +++ b/.github/workflows/hil-config-matrix.yml @@ -0,0 +1,126 @@ +name: HIL config matrix + +# Boots every attached board against every game config it can be flashed with, +# and checks that the board reports that config's game name. A config that +# fails to apply does not fault or crash - the board quietly comes up on the +# generic definition for its hardware - so the game name is the assertion that +# separates "loaded" from "silently fell back". +# +# This is DESIGN.md's G3. Runtime is dominated by the board with the most +# configs (WPC, 63) at roughly 15-25s per boot cycle, so a full run is well +# over half an hour; the inputs below exist to make a targeted run cheap. +# +# Deliberately no `pull_request` trigger - this targets a self-hosted runner on +# a private network and physically reflashes hardware. Fork gating is designed +# in dev/hil/DESIGN.md §4 and is not built yet. +# +# The `push` trigger exists so this can be validated before merging, since a +# workflow_dispatch workflow is not dispatchable until it reaches the default +# branch. Drop the push trigger once this is on main. + +on: + workflow_dispatch: + inputs: + target: + description: "Only run boards for this target, e.g. wpc (blank = every attached board)" + type: string + default: "" + configs: + description: "Comma-separated config names to run instead of all of them, e.g. AttackMars_11,Taxi_L4" + type: string + default: "" + limit: + description: "Stop after this many configs per board (blank = no limit)" + type: string + default: "" + skip_flash: + description: "Matrix what is already on the boards instead of building and flashing first" + type: boolean + default: false + stop_on_first_failure: + description: "Stop a board's matrix at its first failing config" + type: boolean + default: false + push: + branches: + - claude/wpc-hil-config-validation-rc62dn + paths: + - dev/hil/bench.py + - dev/hil/config_matrix.py + - .github/workflows/hil-config-matrix.yml + +permissions: + contents: read + +# Flashing is destructive and the bench is one set of boards. Never interleave. +# Shares the group with the other HIL workflows on purpose. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + config-matrix: + runs-on: [self-hosted, vector-hil] + timeout-minutes: 180 + + steps: + # actions/checkout is safe here ONLY because every trigger above is + # repo-internal. If a fork-reachable trigger is ever added, this must go + # back to running from the runner's pinned clone ($VECTOR_HIL_REPO) - + # otherwise a fork PR would execute its own harness code on the bench. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + # --changed-since needs history to diff against; a shallow clone has + # no merge base with the default branch. + fetch-depth: 0 + + - name: Prepare environment + run: | + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + + # VECTOR_HIL_BOARD_MAP, VECTOR_HIL_WIFI_* and friends arrive from the + # runner's .env, which the runner process exports into every job. They + # are deliberately NOT restated in an `env:` block here: the `env` + # context only covers workflow/job/step-level vars, so `${{ env.X }}` + # would evaluate to empty and shadow the real value. + - name: Boot every config on every board + # Inputs go through the environment rather than into the script text: + # `${{ }}` interpolation would splice a dispatcher-supplied string + # straight into the shell. + env: + HIL_TARGET: ${{ inputs.target }} + HIL_CONFIGS: ${{ inputs.configs }} + HIL_LIMIT: ${{ inputs.limit }} + HIL_SKIP_FLASH: ${{ inputs.skip_flash }} + HIL_STOP_ON_FIRST_FAILURE: ${{ inputs.stop_on_first_failure }} + HIL_EVENT: ${{ github.event_name }} + HIL_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: | + args="" + if [ -n "${HIL_TARGET:-}" ]; then args="$args --target $HIL_TARGET"; fi + if [ -n "${HIL_CONFIGS:-}" ]; then args="$args --configs $HIL_CONFIGS"; fi + if [ -n "${HIL_LIMIT:-}" ]; then args="$args --limit $HIL_LIMIT"; fi + if [ "${HIL_SKIP_FLASH:-}" = "true" ]; then args="$args --skip-flash"; fi + if [ "${HIL_STOP_ON_FIRST_FAILURE:-}" = "true" ]; then args="$args --stop-on-first-failure"; fi + + # On a push, run the configs the push touched first so a bad config + # fails in the first minute rather than the fortieth. The ref has to + # be fetched explicitly - a checkout leaves no tracking ref for a + # branch it did not check out. + if [ "$HIL_EVENT" = "push" ] && git fetch --quiet origin "$HIL_DEFAULT_BRANCH"; then + args="$args --changed-since FETCH_HEAD" + fi + + # shellcheck disable=SC2086 # args is a deliberately word-split list + python dev/hil/config_matrix.py $args + + - name: Board serial logs on failure + if: failure() + run: | + # A board left mid-matrix may still be printing something useful; + # grab a few seconds of console from each. + for dev in /dev/ttyACM*; do + echo "--- $dev" + timeout 8 cat "$dev" || true + done diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index cf0466b9..d2f60126 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -1,6 +1,6 @@ # Hardware-in-the-Loop (HIL) Testing — Design -**Status:** design proposal; bench bring-up and the flash/health-check harness are implemented and running +**Status:** design proposal; bench bring-up, the flash/health-check harness (G1/G2) and the config matrix (G3) are implemented and running **Scope:** a self-hosted GitHub Actions runner driving real Vector boards, safely, from a public repository. --- @@ -206,6 +206,31 @@ Tests parameterize over `manifest ∩ dev/ci/targets.json`. Adding `whitestar` l ## 7. Harness structure +What exists today: + +``` +dev/hil/ + bench.py # shared plumbing: inventory, resolve, build, flash, + # reset, wait-for-boot, USB request, config select + flash_and_check.py # G1/G2: one flash per board, then the API health + # check over both USB and HTTP + config_matrix.py # G3: one flash per board, then every game config for + # that target in turn + setup-runner.sh # bench Pi provisioning + DESIGN.md, RUNNER_SETUP.md +``` + +`bench.py` deliberately asserts nothing about firmware behaviour — it only gets +a board into a known state and talks to it. Assertions live in the harness that +imports it, so adding a check never means touching the plumbing. The +hardware-free parts (config discovery, selection and ordering, and each +assertion with the board faked out) are covered by +`dev/tests/test_hil_config_matrix.py`. + +The originally proposed structure, for reference — pytest-driven, with a +manifest and a board pool. Worth revisiting when a second board of the same +target arrives and sharding starts to matter: + ``` dev/hil/ bench.py # manifest load + validation @@ -300,18 +325,59 @@ That last one is the valuable one. It turns the API docs into a load-bearing art ### G3 — every config parses and boots -Full matrix on every PR, per the decision above. Per board, loop over that hardware's configs: +**Implemented** in `dev/hil/config_matrix.py`, run by `.github/workflows/hil-config-matrix.yml`. + +Per board: build and flash that board's target once, so the config bundle under +test is the one this checkout produces, then loop over every config in +`src//config/`: -1. Interrupt to REPL, write `gamename` into `SPI_DataStore` `configuration` record, `machine.reset()` -2. Wait for boot +1. Write `gamename` into the `SPI_DataStore` `configuration` record over the REPL, and **read it back** — see the fixed-width field note below +2. `machine.reset()`, then wait for the ready marker on the console 3. Assert: - - no `CONF00` / `CONF01` fault + - no `CONF00` / `CONF01` fault, and no `HDWR01` either — `HDWR01` puts `main.py` down the `safe_mode` path where the config is never read at all, so it is fatal here even though `flash_and_check.py` only warns about it - `/api/game/active_config` is the config we set - `/api/game/name` matches `GameInfo.GameName` **from the source JSON in the repo** — this cross-checks the on-board `config/all.jsonl.z` against the source and catches build-time config-packing bugs, not just parse errors - `/api/leaders` and `/api/adjustments/status` both return 200 — proves the parsed definition is *usable*, not merely loadable - - free memory after load is above a floor -That last assertion is the one that earns its keep. `sys11_tiny` exists because RAM is tight; a config that parses fine but leaves too little heap is the failure that actually reaches customers. +Each board's bundle is also compared against the source directory once, before +the loop: same set of config names, same game names. That localises a packing +bug to one boot instead of one boot per affected config. + +**Why the game name is the load-bearing assertion.** A config that fails to +apply does not fault or crash from the outside: `GameDefsLoad.go` falls back to +`safe_defaults` and the board serves a generic definition for its hardware, +healthy in every other respect. `/api/game/active_config` does not catch this — +it reads the `gamename` field back out of FRAM, not what actually loaded. The +game name is what separates "loaded my config" from "silently fell back". + +#### Findings and limits + +- **Two WPC configs cannot be selected at all.** `configuration.gamename` is a + 16-byte fixed-width field (`SPI_DataStore.py`), and `struct.pack` truncates + silently. `HarleyDavidson_L3` and `GilliganIsland_L9` are 17 characters, so + the web UI offers them, the write is accepted, the name is truncated on the + way into FRAM, and the next boot matches nothing and comes up on safe + defaults with `CONF01`. The harness catches this before spending a boot + cycle, and `dev/tests/test_hil_config_matrix.py` catches a *new* offender in + ordinary CI. Fixing the two that exist means shortening the filenames or + widening the field (which is a `MapVersion` change — the 96-byte record is + fully used). +- **`/api/adjustments/status` 500s for configs with no `Adjustments` section.** + `GameDefsLoad` assigns the parsed config straight to `SharedState.gdata` + without merging `safe_defaults` into it, so the key is simply absent and + `Adjustments._get_range_from_gamedef` raises `KeyError`. 14 shipped configs + are affected. Warned about rather than failed, so that a firmware gap the + harness cannot fix does not bury the signal it exists for. +- **Configs that share a `GameName` are not distinguished from each other.** + The four `AddamsFam_*` variants all report "Addams Family", and no route + exposes anything else from `gdata`. For those, a pass means "this config + parsed and loaded without faulting", not "this exact ROM revision's + definition is in memory". +- **The free-memory floor is not implemented.** No route reports heap, and + reading `gc.mem_free()` over the REPL means interrupting the running + firmware, which ends the boot being measured. `sys11_tiny` exists because RAM + is tight, so this assertion is still worth having — it needs a small + firmware-side route first. **Throughput.** Measured on the bench, per board, from the flash/health-check harness: diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 34cf6149..9c266e89 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -176,4 +176,30 @@ Once this is on `main`, drop the `push:` trigger and use the Run workflow button by querying `systemConfig.vectorSystem`, so port order doesn't matter for distinct board types. Only add udev rules if you have two of the same type. +## Running the harnesses by hand + +Both take the bench venv on `PATH` — the runner's `.env` provides it inside a job, but a +login shell does not read it: + +```bash +cd ~/vector && export PATH="$PWD/.venv/bin:$PATH" + +# flash every board and health-check its API (G1/G2) +.venv/bin/python dev/hil/flash_and_check.py + +# boot every board against every config it can be flashed with (G3) +.venv/bin/python dev/hil/config_matrix.py + +# ...or just the WPC board, first five configs, no reflash +.venv/bin/python dev/hil/config_matrix.py --target wpc --limit 5 --skip-flash +``` + +A full config matrix is roughly 15–25s per config per board and WPC alone has 63, so budget +well over half an hour for an unfiltered run. `--configs`, `--limit` and `--target` are there +to keep an iteration loop short; `--changed-since REF` runs the configs a branch touched +first, which is what the workflow does on a push. + +The matrix leaves each board on its generic config when it finishes, including after a +failure, so a run never strands a board on a game config you did not ask for. + See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. diff --git a/dev/hil/bench.py b/dev/hil/bench.py new file mode 100644 index 00000000..0008d542 --- /dev/null +++ b/dev/hil/bench.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 +"""Shared bench plumbing for the hardware-in-the-loop harnesses. + +Everything here is about *getting a board into a known state and talking to +it*: enumerating what is attached, deciding what each board should be flashed +with, building and flashing it, and waiting for the firmware to come up on the +other side of a reset. + +The assertions live in the harnesses that import this: + + flash_and_check.py - one flash per board, then a broad API health check + config_matrix.py - one flash per board, then every game config in turn + +Nothing in here asserts anything about firmware behaviour, so a change to what +a harness checks does not belong in this file. +""" + +import json +import os +import re +import shutil +import subprocess +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO_ROOT / "dev")) + +import serial # noqa: E402 (ships with mpremote) +from usb_coms_demo import UsbApiClient # noqa: E402,F401 (re-exported by this module) + +# A bare bench board has nothing driving the game bus, so this one is correct +# and expected rather than a regression. +EXPECTED_FAULTS = {"HDWR02"} + +# Floating data lines can trip the >250-transition check in main.py and send +# the board down the safe_mode path, where the game config is never loaded. +# Warned about rather than failed, but it invalidates the config assertions - +# see DESIGN.md §8 (G1). +BENCH_WARN_FAULTS = {"HDWR01"} + +# Config key is the config filename without .json (dev/build.py:253). +DEFAULT_GAMENAME = { + "sys11": "GenericSystem11_", + "wpc": "Generic_WPC", + "data_east": "GenericDE_", + "em": "EM_machine_", +} + +# Boot is slow and variable, so we watch the console for the firmware saying +# it is ready rather than guessing at a delay. +# +# "Server: Loop Forever" is the correct marker and the only one: phew prints +# it immediately before loop.run_forever() (phew/server.py:381). The earlier +# "> starting web server on port 80" line is NOT a ready signal - it is +# printed before start_server is even scheduled, let alone bound, so matching +# it returns while the socket is still closed. backend.go() has already run +# connect_to_wifi() by this point, so the marker covers both transports. +READY_MARKER = "Server: Loop Forever" + +# The marker is printed just *before* run_forever(), so give the event loop a +# moment to actually accept the listening socket. http_get's retries cover any +# remainder. +SERVER_SETTLE_SECONDS = 2 + +BOOT_TIMEOUT = 150 +HTTP_TIMEOUT = 10 + + +class CheckFailure(Exception): + pass + + +def log(msg): + print(msg, flush=True) + + +_TIMINGS = [] +_group = None + + +def group(title): + global _group + _group = (title, time.monotonic()) + print(f"::group::{title}", flush=True) + + +def endgroup(): + global _group + if _group: + title, started = _group + elapsed = time.monotonic() - started + _TIMINGS.append((title, elapsed)) + print(f" [{elapsed:.1f}s]", flush=True) + _group = None + print("::endgroup::", flush=True) + + +# -------------------------------------------------------------------------- +# 1. inventory +# -------------------------------------------------------------------------- + + +VENV_PYTHON = sys.executable + + +def ensure_tools_on_path(): + """Put the bench venv's bin dir on PATH and pick the interpreter to use. + + Sets VENV_PYTHON, which build() and flash() shell out to, and returns it. + + The harness gets run three ways - from an Actions job, from a login shell, + and by hand - and only the first has the runner's .env applied. dev/build.py + shells out to a bare `mpy-cross` and dev/flash.py to a bare `mpremote`, so + PATH has to be right for subprocesses too, not just for our own calls. + """ + candidates = [] + if os.environ.get("VECTOR_HIL_VENV"): + candidates.append(Path(os.environ["VECTOR_HIL_VENV"]) / "bin") + candidates.append(Path(sys.executable).parent) + candidates.append(REPO_ROOT / ".venv" / "bin") + + global VENV_PYTHON + + for bindir in candidates: + if (bindir / "mpremote").exists(): + os.environ["PATH"] = f"{bindir}{os.pathsep}{os.environ.get('PATH', '')}" + python = bindir / "python" + VENV_PYTHON = str(python) if python.exists() else sys.executable + return VENV_PYTHON + + if shutil.which("mpremote"): + VENV_PYTHON = sys.executable + return VENV_PYTHON + + raise CheckFailure( + "mpremote not found. Run with the bench venv, e.g.\n" + f" cd {REPO_ROOT} && PATH=\"$PWD/.venv/bin:$PATH\" .venv/bin/python dev/hil/.py ...\n" + "(VECTOR_HIL_VENV is exported by the runner service, so it is not set in a login shell.)" + ) + + +def mpremote(*args, timeout=60): + return subprocess.run(["mpremote", *args], capture_output=True, text=True, timeout=timeout) + + +def list_ports(): + result = mpremote("devs", timeout=30) + if result.returncode != 0: + raise CheckFailure(f"`mpremote devs` failed: {result.stderr.strip()}") + return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] + + +def probe(port): + """Return {port, chip_id, system, version} for one board. + + chip_id comes from the RP2040 itself so it survives any firmware state; + system/version come from the flashed firmware and may be missing if the + board is unflashed or broken. + """ + board = {"port": port, "chip_id": None, "system": None, "version": None} + + chip = mpremote( + "connect", port, "exec", + "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", + timeout=30, + ) + if chip.returncode == 0: + board["chip_id"] = chip.stdout.strip() + + info = mpremote( + "connect", port, "exec", + "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)", + timeout=30, + ) + if info.returncode == 0 and info.stdout.strip(): + parts = info.stdout.split() + board["system"] = parts[0] + if len(parts) > 1: + board["version"] = parts[1] + + return board + + +def inventory(): + boards = [probe(port) for port in list_ports()] + if not boards: + raise CheckFailure("no boards found - check the USB hub and power") + + log(f"{'port':16} {'chip id':18} {'running':12} version") + for b in boards: + log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") + return boards + + +IDENTIFY_SNIPPET = """ +import machine, time +try: + import BoardLED as L + L.startUp() +except Exception: + L = None +led = machine.Pin("LED", machine.Pin.OUT) +for i in range({blinks}): + led.on() + if L: + L.ledColor(L.BLUE) + time.sleep(0.25) + led.off() + if L: + L.ledColor(L.BLACK) + time.sleep(0.25) +""" + + +def identify(boards, seconds=8): + """Blink each board in turn so a human can tell which is which. + + Uses the Pico W onboard LED, which works from the REPL no matter what + firmware is loaded, plus the Vector board's WS2812 in blue when the + flashed firmware happens to provide the driver. + """ + log(f"Blinking each board for ~{seconds}s. Watch the bench and note the order.") + log("") + for index, board in enumerate(boards, 1): + log(f" [{index}/{len(boards)}] BLINKING NOW: {board['port']} chip {board['chip_id']}") + result = mpremote( + "connect", board["port"], "exec", + IDENTIFY_SNIPPET.format(blinks=int(seconds / 0.5)), + timeout=seconds + 30, + ) + if result.returncode != 0: + log(f" could not blink this board: {result.stderr.strip()}") + else: + log(" done") + log("") + log("Now map what you saw to the chip ids, and put this in the runner's .env:") + log("") + log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) + + +# -------------------------------------------------------------------------- +# 2. resolve +# -------------------------------------------------------------------------- + + +def parse_board_map(raw): + """Parse VECTOR_HIL_BOARD_MAP: 'chipid=target,chipid=target'.""" + mapping = {} + for entry in (raw or "").split(","): + entry = entry.strip() + if not entry: + continue + if "=" not in entry: + raise CheckFailure(f"bad VECTOR_HIL_BOARD_MAP entry {entry!r}, expected chipid=target") + chip, target = entry.split("=", 1) + mapping[chip.strip()] = target.strip() + return mapping + + +def resolve_targets(boards, board_map): + """Decide the target for each board, refusing to guess when it matters. + + An explicit chip-id map is authoritative. Without one we fall back to what + each board's firmware reports, which is only trustworthy when every board + reports something different - if two boards claim the same system, that is + the signature of a previous mis-flash rather than of the hardware, and + flashing on that basis would silently perpetuate it. + """ + if board_map: + unmapped = [b for b in boards if b["chip_id"] not in board_map] + if unmapped: + raise CheckFailure( + "VECTOR_HIL_BOARD_MAP is set but does not cover: " + + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) + ) + for b in boards: + b["target"] = board_map[b["chip_id"]] + log("targets from VECTOR_HIL_BOARD_MAP") + return boards + + missing = [b for b in boards if not b["system"]] + if missing: + raise CheckFailure( + "cannot identify " + + ", ".join(b["port"] for b in missing) + + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP." + ) + + systems = [b["system"] for b in boards] + duplicates = {s for s in systems if systems.count(s) > 1} + if duplicates: + raise CheckFailure( + "refusing to flash from autodetection: " + + ", ".join(sorted(duplicates)) + + " is reported by more than one board.\n" + "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" + "at least one board is running firmware for a system it is not wired for.\n" + "Pin them explicitly instead, using the chip ids above:\n" + " VECTOR_HIL_BOARD_MAP=" + + ",".join(f"{b['chip_id']}=" for b in boards) + ) + + for b in boards: + b["target"] = b["system"] + log("targets from firmware self-report (all distinct)") + return boards + + +# -------------------------------------------------------------------------- +# 3. build +# -------------------------------------------------------------------------- + + +def source_version(target): + config = REPO_ROOT / "src" / target / "systemConfig.py" + match = re.search(r'SystemVersion\s*=\s*"([^"]+)"', config.read_text()) + if not match: + raise CheckFailure(f"could not read SystemVersion from {config}") + return match.group(1) + + +def build(target): + build_dir = REPO_ROOT / "build" / target + result = subprocess.run( + [VENV_PYTHON, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + ) + if result.returncode != 0: + log(result.stdout[-3000:]) + log(result.stderr[-3000:]) + raise CheckFailure(f"build failed for {target}") + return build_dir + + +# -------------------------------------------------------------------------- +# 4. flash +# -------------------------------------------------------------------------- + + +def write_bench_config(target, workdir): + ssid = os.environ.get("VECTOR_HIL_WIFI_SSID", "") + password = os.environ.get("VECTOR_HIL_WIFI_PASSWORD", "") + game_password = os.environ.get("VECTOR_HIL_GAME_PASSWORD", "hiltest") + + if not ssid or not password: + raise CheckFailure("VECTOR_HIL_WIFI_SSID / VECTOR_HIL_WIFI_PASSWORD are not set") + + # dev/flash.py builds a MicroPython snippet with single-quoted values, so a + # single quote anywhere here would produce a syntax error on the board + # rather than an obvious failure here. + for name, value in (("ssid", ssid), ("password", password), ("game password", game_password)): + if "'" in value or "\\" in value: + raise CheckFailure(f"bench {name} contains a quote or backslash, which dev/flash.py cannot write") + + config = { + "ssid": ssid, + "password": password, + "gamename": DEFAULT_GAMENAME[target], + "Gpassword": game_password, + } + path = workdir / f"hil-config-{target}.json" + path.write_text(json.dumps(config)) + return path + + +def flash(target, port, build_dir, config_path): + result = subprocess.run( + [VENV_PYTHON, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + ) + if result.returncode != 0: + log(result.stdout[-3000:]) + log(result.stderr[-3000:]) + raise CheckFailure(f"flash failed for {target} on {port}") + + +# -------------------------------------------------------------------------- +# 5. health +# -------------------------------------------------------------------------- + + +def reset_board(port): + """Reset the board so we own the boot we are about to watch. + + The ready marker is printed exactly once per boot. dev/flash.py already + resets at the end of flashing, but flashing runs over every board before + any health check starts, so by the time we open a console the board booted + a minute ago and the marker is long gone. Resetting here makes the wait + deterministic and the reported boot time meaningful. + """ + result = mpremote( + "connect", port, "exec", "--no-follow", "import machine; machine.reset()", + timeout=30, + ) + if result.returncode != 0: + raise CheckFailure(f"could not reset {port} before the health check: {result.stderr.strip()}") + + +def wait_for_server(port, timeout=BOOT_TIMEOUT): + """Watch the boot console until the firmware reports its web server is up. + + Polling an API that is not listening yet tells you nothing about why, and + burns the whole timeout when a board fails to boot. Reading the console + instead gives an exact ready signal and, on failure, the boot log that + explains it. + + Returns the open serial connection so the USB API can reuse it - the + Pico exposes one CDC endpoint, so a second connection would fight this one. + """ + deadline = time.monotonic() + timeout + transcript = [] + connection = None + + while time.monotonic() < deadline: + if connection is None: + try: + # The port disappears and re-enumerates across the reset, so a + # failure to open here is expected for the first second or two. + connection = serial.Serial(port=port, baudrate=115200, timeout=1) + except Exception: + time.sleep(1) + continue + try: + raw = connection.readline() + except Exception: + try: + connection.close() + except Exception: + pass + connection = None + continue + + if not raw: + continue + text = raw.decode(errors="replace").rstrip("\r\n") + if not text: + continue + transcript.append(text) + + if READY_MARKER in text: + elapsed = timeout - (deadline - time.monotonic()) + log(f" server up after {elapsed:.1f}s ({text.strip()!r})") + time.sleep(SERVER_SETTLE_SECONDS) + return connection, transcript + + if connection is not None: + try: + connection.close() + except Exception: + pass + + tail = "\n ".join(transcript[-20:]) or "(nothing on the console)" + raise CheckFailure( + f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}" + ) + + +def prime_usb(connection): + """Clear both ends of the serial line before the first API request. + + usb_comms accumulates stdin characters into a module-level `buffer` until + it sees a newline (usb_comms.py:132). Anything left there without a + terminator - a partial line, stray bytes from the raw-REPL session that + issued the reset - silently prefixes the next request, so the board parses + a route like "\x02/api/version", fails the `_routes` lookup and answers + 404. A lone newline flushes whatever is pending into a discarded request. + """ + try: + connection.reset_input_buffer() + connection.reset_output_buffer() + connection.write(b"\n") + connection.flush() + except Exception as exc: + log(f" warning: could not prime the USB link: {exc}") + return + # usb_request_handler is scheduled every 1000ms (phew/server.py:342), so + # give it a turn to consume the flush before the first real request. + time.sleep(1.5) + try: + connection.reset_input_buffer() + except Exception: + pass + + +def get(client, route, expect=200): + response = client.send_and_receive(route=route, payload=None, timeout=15) + status = response.get("status") + if status != expect: + # The board narrates its own routing failures ("USB REQ: route not + # found: ..."), but the client discards every line that is not a + # response. Drain whatever is pending so the reason is visible. + raise CheckFailure( + f"{route} returned {status}, expected {expect}" + f"{_drain_serial(client.ser)}" + ) + return response.get("body") + + +def _drain_serial(connection, limit=12): + """Return any pending board chatter, formatted for an error message.""" + try: + time.sleep(0.5) + pending = connection.read(connection.in_waiting or 0) + except Exception: + return "" + if not pending: + return "" + lines = [line for line in pending.decode(errors="replace").splitlines() if line.strip()] + if not lines: + return "" + return "\n board said: " + "\n board said: ".join(lines[:limit]) + + +# -------------------------------------------------------------------------- +# reporting +# -------------------------------------------------------------------------- + + +def _dump_boot_log(board, lines=25): + """Show what the board actually said. A failed health check is usually + explained by the boot output, and by this point we already have it.""" + transcript = board.get("boot_log") + if not transcript: + return + log(f" last {min(lines, len(transcript))} lines of {board['port']} boot console:") + for line in transcript[-lines:]: + log(f" {line}") + + +# -------------------------------------------------------------------------- +# game configuration +# -------------------------------------------------------------------------- + + +def gamename_field_bytes(): + """Width of the `gamename` field in the FRAM `configuration` record. + + Read out of the firmware source rather than hard-coded, so that widening + the field is picked up here instead of silently leaving a stale limit in + the harness. A filename longer than this cannot round-trip: struct.pack + truncates the value, the truncated name matches no config, and the board + boots on safe defaults with CONF01 raised. + """ + source = (REPO_ROOT / "src" / "common" / "SPI_DataStore.py").read_text() + match = re.search(r'"<32s32s(\d+)s\d+s"', source) + if not match: + raise CheckFailure("could not find the configuration record format in SPI_DataStore.py") + return int(match.group(1)) + + +SET_CONFIG_SNIPPET = ";".join( + [ + "import SPI_DataStore as ds", + "c = ds.read_record('configuration')", + "c['gamename'] = '{gamename}'", + "ds.write_record('configuration', c)", + "print('GAMENAME=' + ds.read_record('configuration')['gamename'])", + ] +) + + +def set_game_config(port, gamename): + """Point a board at one game config and prove the value survived the write. + + The board picks its config up from the FRAM `configuration` record at boot + (GameDefsLoad.go), so setting it is a REPL write plus a reset - no + authentication, no HTTP, and no reflash. mpremote interrupts whatever the + board is running to get the REPL, which is fine here because the caller + resets immediately afterwards. + + The read-back is the load-bearing part. `gamename` is a fixed-width field + and struct.pack truncates silently, so a name that is too long is written, + accepted, and then never matches any config on the next boot. Catching it + here costs nothing; catching it after the boot costs a boot cycle and + reports a confusing CONF01. + """ + if "'" in gamename or "\\" in gamename: + raise CheckFailure(f"config name {gamename!r} contains a quote or backslash") + + result = mpremote("connect", port, "exec", SET_CONFIG_SNIPPET.format(gamename=gamename), timeout=60) + if result.returncode != 0: + raise CheckFailure(f"could not write gamename={gamename!r} to {port}: {result.stderr.strip()}") + + stored = None + for line in result.stdout.splitlines(): + if line.startswith("GAMENAME="): + stored = line.split("=", 1)[1].strip() + if stored is None: + raise CheckFailure(f"board did not read back a gamename after the write (said {result.stdout.strip()!r})") + + if stored != gamename: + limit = gamename_field_bytes() + detail = "" + if len(gamename) > limit: + detail = f" - the FRAM `gamename` field is {limit} bytes and this filename is {len(gamename)}," " so no board can ever store it and the config is unreachable in the field" + raise CheckFailure(f"wrote gamename={gamename!r} but the board stored {stored!r}{detail}") diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py new file mode 100644 index 00000000..36b73046 --- /dev/null +++ b/dev/hil/config_matrix.py @@ -0,0 +1,459 @@ +#!/usr/bin/env python3 +"""Boot every board against every game config it can be flashed with. + +Run from the repo root on the bench Pi: + + cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/config_matrix.py + +Inside an Actions job the runner's .env already provides VECTOR_HIL_VENV, so +plain `python dev/hil/config_matrix.py` is enough there. + +This is DESIGN.md's G3: "every available config can be parsed and boot". A +config is a JSON file in src//config/ that the build packs into the +firmware's config bundle; the board picks one at boot from the `gamename` field +of the FRAM `configuration` record. So one iteration is: + + write gamename -> reset -> wait for the server -> ask the board what it + loaded + +and the pass condition is that the board reports *that* config, by name. + +Why the game name is the assertion that matters: when a config fails to apply - +missing from the bundle, unparseable, or a filename the board cannot store - +GameDefsLoad falls back to `safe_defaults`, and the board comes up looking +perfectly healthy while running a generic definition for that hardware. Nothing +faults from the outside. Comparing `/api/game/name` against `GameInfo.GameName` +in the *source* JSON is what separates "loaded my config" from "silently fell +back", and it cross-checks the on-board bundle against the repo at the same +time. + +Stages: + + 1. inventory - probe every attached board (shared with flash_and_check.py) + 2. resolve - decide which target each board should be flashed with + 3. build/flash - one firmware flash per board, so the bundle under test is + the one built from this checkout + 4. matrix - per board, per config: set, reset, boot, assert + 5. restore - put each board back on its generic config + +Boards are visited one after another, and a boot cycle is 15-25s, so a full run +is dominated by whichever target has the most configs (WPC, at 63). Use +--target/--configs/--limit to cut it down while iterating. +""" + +import argparse +import json +import os +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import bench # noqa: E402 +from bench import ( # noqa: E402 + _TIMINGS, + BENCH_WARN_FAULTS, + DEFAULT_GAMENAME, + EXPECTED_FAULTS, + REPO_ROOT, + CheckFailure, + UsbApiClient, + endgroup, + get, + group, + inventory, + log, + parse_board_map, + prime_usb, + reset_board, + resolve_targets, + set_game_config, + wait_for_server, +) + +# Raised by GameDefsLoad when the configured game cannot be loaded (CONF01) or +# blew up on the way (CONF00). Either one means the board is running +# safe_defaults, which is exactly the failure this harness exists to catch. +CONFIG_FAULTS = {"CONF00", "CONF01"} + + +def source_configs(target): + """{config filename without .json: {"name": ..., "adjustments": bool}}. + + This is the expectation side of every assertion below - the board's answers + are compared against the source JSON, never against the board's own idea of + what it has. + + `adjustments` records whether the config declares an Adjustments section, + which decides how hard /api/adjustments/status is held to account. See + check_config(). + """ + config_dir = REPO_ROOT / "src" / target / "config" + if not config_dir.is_dir(): + raise CheckFailure(f"no config directory for target {target} at {config_dir}") + + configs = {} + for path in sorted(config_dir.glob("*.json")): + try: + data = json.loads(path.read_text()) + except json.JSONDecodeError as exc: + raise CheckFailure(f"{path} is not valid JSON: {exc}") + try: + name = data["GameInfo"]["GameName"] + except (KeyError, TypeError): + raise CheckFailure(f"{path} has no GameInfo.GameName") + configs[path.stem] = {"name": name, "adjustments": "Adjustments" in data} + + if not configs: + raise CheckFailure(f"no game configs found in {config_dir}") + return configs + + +def changed_configs(target, ref): + """Config names under test that git says changed since `ref`. + + Ordering these first means a run that is going to fail because of the diff + fails in the first minute rather than the twentieth. + """ + result = subprocess.run( + ["git", "diff", "--name-only", f"{ref}...HEAD", "--", f"src/{target}/config"], + cwd=REPO_ROOT, + capture_output=True, + text=True, + ) + if result.returncode != 0: + log(f"::warning::could not diff against {ref}: {result.stderr.strip()}") + return [] + return [Path(line).stem for line in result.stdout.split()] + + +def order_configs(names, first): + """Put `first` at the front, keeping the rest in their existing order.""" + lead = [name for name in first if name in names] + return lead + [name for name in names if name not in lead] + + +def select_configs(target, args): + configs = source_configs(target) + names = list(configs) + + if args.configs: + requested = [name.strip() for name in args.configs.split(",") if name.strip()] + unknown = [name for name in requested if name not in configs] + if unknown: + raise CheckFailure(f"unknown config(s) for {target}: {', '.join(unknown)}") + names = requested + elif args.changed_since: + names = order_configs(names, changed_configs(target, args.changed_since)) + + if args.limit: + names = names[: args.limit] + return names, configs + + +def check_faults(client, config): + """Fail on anything that makes the config assertions meaningless.""" + faults = get(client, "/api/fault") or [] + if isinstance(faults, dict): + faults = faults.get("faults", []) + codes = {str(f)[:6] for f in faults} + + config_faults = codes & CONFIG_FAULTS + if config_faults: + raise CheckFailure(f"{sorted(config_faults)} raised - the board did not load {config!r} and is on safe defaults") + + # HDWR01 is a warning in flash_and_check.py because it does not stop the + # API working. Here it is fatal: it puts main.py down the safe_mode path, + # where GameDefsLoad never reads the config at all, so every assertion + # below would pass or fail for reasons that have nothing to do with the + # config under test. + if codes & BENCH_WARN_FAULTS: + raise CheckFailure(f"{sorted(codes & BENCH_WARN_FAULTS)} raised - the board booted in safe mode, so no config was loaded and this result would be meaningless") + + unexpected = codes - EXPECTED_FAULTS + if unexpected: + raise CheckFailure(f"unexpected fault(s): {sorted(unexpected)}") + + +def check_adjustments(client, config, declares_adjustments): + """Prove the loaded definition is usable, not merely loadable. + + Held to 200 only for configs that declare an Adjustments section. The rest + are a known firmware gap rather than a config problem: GameDefsLoad assigns + the parsed config straight to SharedState.gdata without merging + safe_defaults into it, so a config with no Adjustments section leaves + gdata["Adjustments"] absent and Adjustments._get_range_from_gamedef raises + KeyError, which route_wrapper turns into a 500. Failing those here would + bury the signal this harness exists for under a defect it cannot fix, so + they are warned about instead - loudly, and once per occurrence. + """ + try: + adjustments = get(client, "/api/adjustments/status") + except CheckFailure as exc: + if declares_adjustments: + raise + log(f"::warning::{config}: /api/adjustments/status failed ({exc}). The config declares no Adjustments section and gdata is not merged with safe_defaults - see check_adjustments().") + return + + if not isinstance(adjustments, dict): + raise CheckFailure(f"/api/adjustments/status returned {type(adjustments).__name__}, expected an object") + + +def check_config(port, target, config, expected): + """Boot one board on one config and prove it is the config that loaded.""" + expected_name = expected["name"] + + set_game_config(port, config) + reset_board(port) + + connection, boot_log = wait_for_server(port) + client = None + try: + prime_usb(connection) + client = UsbApiClient(connection) + + check_faults(client, config) + + active = get(client, "/api/game/active_config") + active = active.get("active_config") if isinstance(active, dict) else active + + # EM boards answer this route with the game name rather than the + # filename (backend.py:481), so accept either for that target. + expected_active = {config, expected_name} if target == "em" else {config} + if active not in expected_active: + raise CheckFailure(f"active config is {active!r}, expected {config!r}") + + name = get(client, "/api/game/name") + if isinstance(name, dict): + name = name.get("name") + name = str(name).strip() + if name != expected_name: + raise CheckFailure(f"board reports game name {name!r}, but {config}.json says {expected_name!r} - the config did not apply and the board fell back to a generic definition") + + # A config that parses but is not usable still fails a customer. Both + # of these read the loaded definition rather than just its presence. + if get(client, "/api/leaders") is None: + raise CheckFailure("/api/leaders returned no body") + check_adjustments(client, config, expected["adjustments"]) + + return name, boot_log + finally: + if client is not None: + try: + client.close() + except Exception: + pass + else: + try: + connection.close() + except Exception: + pass + + +def check_bundle(port, target, configs): + """Compare the board's config list against the repo, once per board. + + Cheap, and it localises a whole class of failure before the matrix starts: + if the build dropped or mangled a config, this says so in one boot instead + of once per affected iteration. + """ + reset_board(port) + connection, _ = wait_for_server(port) + try: + prime_usb(connection) + client = UsbApiClient(connection) + on_board = get(client, "/api/game/configs_list") + if not isinstance(on_board, dict) or not on_board: + raise CheckFailure("/api/game/configs_list is empty - the config bundle is missing from the build") + + missing = sorted(set(configs) - set(on_board)) + extra = sorted(set(on_board) - set(configs)) + if missing: + raise CheckFailure(f"{len(missing)} config(s) in src/{target}/config are not in the build's bundle: {', '.join(missing)}") + if extra: + raise CheckFailure(f"the build's bundle carries {len(extra)} config(s) with no source JSON: {', '.join(extra)}") + + mismatched = [f"{name}: bundle says {on_board[name].get('name')!r}, source says {configs[name]['name']!r}" for name in sorted(configs) if on_board[name].get("name") != configs[name]["name"]] + if mismatched: + raise CheckFailure("game name mismatch between the bundle and the source JSON:\n " + "\n ".join(mismatched)) + + log(f" bundle matches source: {len(configs)} configs, names identical") + finally: + try: + connection.close() + except Exception: + pass + + +def restore_default(port, target): + """Leave the board on its generic config, as flash_and_check.py expects.""" + default = DEFAULT_GAMENAME[target] + try: + set_game_config(port, default) + reset_board(port) + log(f" restored {default}") + except CheckFailure as exc: + log(f"::warning::could not restore {default} on {port}: {exc}") + + +def run_matrix(board, args): + """Walk one board through its configs. Returns (passed, failures).""" + port = board["port"] + target = board["target"] + names, configs = select_configs(target, args) + + group(f"Config bundle {target} on {port}") + check_bundle(port, target, configs) + endgroup() + + log("") + log(f"{len(names)} config(s) to check on {port} ({target})") + log("") + + passed = [] + failures = [] + for index, config in enumerate(names, 1): + started = time.monotonic() + group(f"[{index}/{len(names)}] {target} {config}") + try: + name, _boot_log = check_config(port, target, config, configs[config]) + elapsed = time.monotonic() - started + log(f" ok {config:20} -> {name!r} [{elapsed:.1f}s]") + passed.append(config) + except CheckFailure as exc: + log(f"::error::{target} {config}: {exc}") + failures.append((config, str(exc))) + if not args.keep_going: + endgroup() + break + except Exception as exc: # noqa: BLE001 - one bad config must not end the run + log(f"::error::{target} {config}: unexpected error: {exc}") + failures.append((config, str(exc))) + if not args.keep_going: + endgroup() + break + endgroup() + + group(f"Restore {target} on {port}") + restore_default(port, target) + endgroup() + + return passed, failures + + +def write_step_summary(results): + """Render the run as a table in the Actions job summary, when there is one.""" + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + + lines = ["## HIL config matrix", "", "| board | target | configs | passed | failed |", "|---|---|---|---:|---:|"] + for board, passed, failures in results: + lines.append(f"| `{board['port']}` | {board['target']} | {len(passed) + len(failures)} | {len(passed)} | {len(failures)} |") + + failed = [(board, config, reason) for board, _passed, failures in results for config, reason in failures] + if failed: + lines += ["", "### Failures", ""] + for board, config, reason in failed: + lines.append(f"- **{board['target']} `{config}`** - {reason.splitlines()[0]}") + + with open(path, "a") as handle: + handle.write("\n".join(lines) + "\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--target", action="append", help="only run boards for this target (repeatable), e.g. --target wpc") + parser.add_argument("--configs", help="comma-separated config names to run instead of all of them, e.g. AttackMars_11,Taxi_L4") + parser.add_argument("--limit", type=int, help="stop after this many configs per board - useful for a quick smoke run") + parser.add_argument("--changed-since", metavar="REF", help="run configs changed since REF first, so a config-touching PR fails fast") + parser.add_argument("--skip-flash", action="store_true", help="matrix what is already on the boards instead of building and flashing first") + parser.add_argument("--stop-on-first-failure", dest="keep_going", action="store_false", help="stop a board's matrix at its first failing config (default: run them all)") + args = parser.parse_args() + + bench.ensure_tools_on_path() + + workdir = REPO_ROOT / "build" + workdir.mkdir(exist_ok=True) + + group("Inventory") + boards = inventory() + endgroup() + + group("Resolve targets") + boards = resolve_targets(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP"))) + for b in boards: + log(f" {b['port']} -> {b['target']}") + endgroup() + + if args.target: + wanted = set(args.target) + boards = [b for b in boards if b["target"] in wanted] + if not boards: + raise CheckFailure(f"no attached board matches --target {', '.join(sorted(wanted))}") + + if not args.skip_flash: + # Flash first so the bundle under test is the one this checkout builds. + # Without it the matrix would validate whatever happened to be on the + # boards, which is the one thing it must not do. + for target in sorted({b["target"] for b in boards}): + group(f"Build {target}") + bench.build(target) + log(f"built {target} at version {bench.source_version(target)}") + endgroup() + + for b in boards: + group(f"Flash {b['target']} on {b['port']}") + config_path = bench.write_bench_config(b["target"], workdir) + bench.flash(b["target"], b["port"], workdir / b["target"], config_path) + log("flashed") + endgroup() + + results = [] + for b in boards: + try: + passed, failures = run_matrix(b, args) + except CheckFailure as exc: + # A board that cannot even be set up is one board's problem. The + # bench is a singleton and a run is expensive, so the other boards + # still get their matrix. + log(f"::error::{b['target']} on {b['port']}: {exc}") + passed, failures = [], [("(board setup)", str(exc))] + results.append((b, passed, failures)) + + log("") + log("stage timings:") + for title, elapsed in _TIMINGS: + log(f" {elapsed:7.1f}s {title}") + + log("") + log("=" * 60) + total_failures = 0 + for board, passed, failures in results: + state = "FAIL" if failures else "ok" + log(f" {state:5} {board['port']:16} {board['target']:12} {len(passed)} passed, {len(failures)} failed") + total_failures += len(failures) + log("=" * 60) + + write_step_summary(results) + + if total_failures: + log(f"\n{total_failures} config(s) failed:") + for board, _passed, failures in results: + for config, reason in failures: + log(f" - {board['target']} {config}: {reason}") + return 1 + + checked = sum(len(passed) for _board, passed, _failures in results) + log(f"\nall {checked} config(s) booted and reported the right game name") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CheckFailure as exc: + log(f"::error::{exc}") + sys.exit(1) diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index 52a7af9e..e9ec2020 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -31,57 +31,41 @@ import gzip import json import os -import re -import shutil -import subprocess import sys import time import urllib.error import urllib.request from pathlib import Path -REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(REPO_ROOT / "dev")) - -import serial # noqa: E402 (ships with mpremote) -from usb_coms_demo import UsbApiClient # noqa: E402 - -# A bare bench board has nothing driving the game bus, so this one is correct -# and expected rather than a regression. -EXPECTED_FAULTS = {"HDWR02"} - -# Floating data lines can trip the >250-transition check in main.py and send -# the board down the safe_mode path, where the game config is never loaded. -# Warned about rather than failed, but it invalidates the config assertions - -# see DESIGN.md §8 (G1). -BENCH_WARN_FAULTS = {"HDWR01"} - -# Config key is the config filename without .json (dev/build.py:253). -DEFAULT_GAMENAME = { - "sys11": "GenericSystem11_", - "wpc": "Generic_WPC", - "data_east": "GenericDE_", - "em": "EM_machine_", -} - -# Boot is slow and variable, so we watch the console for the firmware saying -# it is ready rather than guessing at a delay. -# -# "Server: Loop Forever" is the correct marker and the only one: phew prints -# it immediately before loop.run_forever() (phew/server.py:381). The earlier -# "> starting web server on port 80" line is NOT a ready signal - it is -# printed before start_server is even scheduled, let alone bound, so matching -# it returns while the socket is still closed. backend.go() has already run -# connect_to_wifi() by this point, so the marker covers both transports. -READY_MARKER = "Server: Loop Forever" - -# The marker is printed just *before* run_forever(), so give the event loop a -# moment to actually accept the listening socket. http_get's retries cover any -# remainder. -SERVER_SETTLE_SECONDS = 2 - -BOOT_TIMEOUT = 150 -HTTP_TIMEOUT = 10 +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from bench import ( # noqa: E402 + _TIMINGS, + BENCH_WARN_FAULTS, + DEFAULT_GAMENAME, + EXPECTED_FAULTS, + HTTP_TIMEOUT, + REPO_ROOT, + CheckFailure, + UsbApiClient, + _dump_boot_log, + build, + endgroup, + ensure_tools_on_path, + flash, + get, + group, + identify, + inventory, + log, + parse_board_map, + prime_usb, + reset_board, + resolve_targets, + source_version, + wait_for_server, + write_bench_config, +) # Read-only routes exercised over HTTP. Kept side-effect free so the check can # run against a board repeatedly without changing its state. @@ -104,445 +88,6 @@ ) -class CheckFailure(Exception): - pass - - -def log(msg): - print(msg, flush=True) - - -_TIMINGS = [] -_group = None - - -def group(title): - global _group - _group = (title, time.monotonic()) - print(f"::group::{title}", flush=True) - - -def endgroup(): - global _group - if _group: - title, started = _group - elapsed = time.monotonic() - started - _TIMINGS.append((title, elapsed)) - print(f" [{elapsed:.1f}s]", flush=True) - _group = None - print("::endgroup::", flush=True) - - -# -------------------------------------------------------------------------- -# 1. inventory -# -------------------------------------------------------------------------- - - -def ensure_tools_on_path(): - """Put the bench venv's bin dir on PATH and pick the interpreter to use. - - The harness gets run three ways - from an Actions job, from a login shell, - and by hand - and only the first has the runner's .env applied. dev/build.py - shells out to a bare `mpy-cross` and dev/flash.py to a bare `mpremote`, so - PATH has to be right for subprocesses too, not just for our own calls. - """ - candidates = [] - if os.environ.get("VECTOR_HIL_VENV"): - candidates.append(Path(os.environ["VECTOR_HIL_VENV"]) / "bin") - candidates.append(Path(sys.executable).parent) - candidates.append(REPO_ROOT / ".venv" / "bin") - - for bindir in candidates: - if (bindir / "mpremote").exists(): - os.environ["PATH"] = f"{bindir}{os.pathsep}{os.environ.get('PATH', '')}" - python = bindir / "python" - return str(python) if python.exists() else sys.executable - - if shutil.which("mpremote"): - return sys.executable - - raise CheckFailure( - "mpremote not found. Run with the bench venv, e.g.\n" - f" cd {REPO_ROOT} && PATH=\"$PWD/.venv/bin:$PATH\" .venv/bin/python dev/hil/flash_and_check.py ...\n" - "(VECTOR_HIL_VENV is exported by the runner service, so it is not set in a login shell.)" - ) - - -VENV_PYTHON = sys.executable - - -def mpremote(*args, timeout=60): - return subprocess.run(["mpremote", *args], capture_output=True, text=True, timeout=timeout) - - -def list_ports(): - result = mpremote("devs", timeout=30) - if result.returncode != 0: - raise CheckFailure(f"`mpremote devs` failed: {result.stderr.strip()}") - return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] - - -def probe(port): - """Return {port, chip_id, system, version} for one board. - - chip_id comes from the RP2040 itself so it survives any firmware state; - system/version come from the flashed firmware and may be missing if the - board is unflashed or broken. - """ - board = {"port": port, "chip_id": None, "system": None, "version": None} - - chip = mpremote( - "connect", port, "exec", - "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", - timeout=30, - ) - if chip.returncode == 0: - board["chip_id"] = chip.stdout.strip() - - info = mpremote( - "connect", port, "exec", - "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)", - timeout=30, - ) - if info.returncode == 0 and info.stdout.strip(): - parts = info.stdout.split() - board["system"] = parts[0] - if len(parts) > 1: - board["version"] = parts[1] - - return board - - -def inventory(): - boards = [probe(port) for port in list_ports()] - if not boards: - raise CheckFailure("no boards found - check the USB hub and power") - - log(f"{'port':16} {'chip id':18} {'running':12} version") - for b in boards: - log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") - return boards - - -IDENTIFY_SNIPPET = """ -import machine, time -try: - import BoardLED as L - L.startUp() -except Exception: - L = None -led = machine.Pin("LED", machine.Pin.OUT) -for i in range({blinks}): - led.on() - if L: - L.ledColor(L.BLUE) - time.sleep(0.25) - led.off() - if L: - L.ledColor(L.BLACK) - time.sleep(0.25) -""" - - -def identify(boards, seconds=8): - """Blink each board in turn so a human can tell which is which. - - Uses the Pico W onboard LED, which works from the REPL no matter what - firmware is loaded, plus the Vector board's WS2812 in blue when the - flashed firmware happens to provide the driver. - """ - log(f"Blinking each board for ~{seconds}s. Watch the bench and note the order.") - log("") - for index, board in enumerate(boards, 1): - log(f" [{index}/{len(boards)}] BLINKING NOW: {board['port']} chip {board['chip_id']}") - result = mpremote( - "connect", board["port"], "exec", - IDENTIFY_SNIPPET.format(blinks=int(seconds / 0.5)), - timeout=seconds + 30, - ) - if result.returncode != 0: - log(f" could not blink this board: {result.stderr.strip()}") - else: - log(" done") - log("") - log("Now map what you saw to the chip ids, and put this in the runner's .env:") - log("") - log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) - - -# -------------------------------------------------------------------------- -# 2. resolve -# -------------------------------------------------------------------------- - - -def parse_board_map(raw): - """Parse VECTOR_HIL_BOARD_MAP: 'chipid=target,chipid=target'.""" - mapping = {} - for entry in (raw or "").split(","): - entry = entry.strip() - if not entry: - continue - if "=" not in entry: - raise CheckFailure(f"bad VECTOR_HIL_BOARD_MAP entry {entry!r}, expected chipid=target") - chip, target = entry.split("=", 1) - mapping[chip.strip()] = target.strip() - return mapping - - -def resolve_targets(boards, board_map): - """Decide the target for each board, refusing to guess when it matters. - - An explicit chip-id map is authoritative. Without one we fall back to what - each board's firmware reports, which is only trustworthy when every board - reports something different - if two boards claim the same system, that is - the signature of a previous mis-flash rather than of the hardware, and - flashing on that basis would silently perpetuate it. - """ - if board_map: - unmapped = [b for b in boards if b["chip_id"] not in board_map] - if unmapped: - raise CheckFailure( - "VECTOR_HIL_BOARD_MAP is set but does not cover: " - + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) - ) - for b in boards: - b["target"] = board_map[b["chip_id"]] - log("targets from VECTOR_HIL_BOARD_MAP") - return boards - - missing = [b for b in boards if not b["system"]] - if missing: - raise CheckFailure( - "cannot identify " - + ", ".join(b["port"] for b in missing) - + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP." - ) - - systems = [b["system"] for b in boards] - duplicates = {s for s in systems if systems.count(s) > 1} - if duplicates: - raise CheckFailure( - "refusing to flash from autodetection: " - + ", ".join(sorted(duplicates)) - + " is reported by more than one board.\n" - "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" - "at least one board is running firmware for a system it is not wired for.\n" - "Pin them explicitly instead, using the chip ids above:\n" - " VECTOR_HIL_BOARD_MAP=" - + ",".join(f"{b['chip_id']}=" for b in boards) - ) - - for b in boards: - b["target"] = b["system"] - log("targets from firmware self-report (all distinct)") - return boards - - -# -------------------------------------------------------------------------- -# 3. build -# -------------------------------------------------------------------------- - - -def source_version(target): - config = REPO_ROOT / "src" / target / "systemConfig.py" - match = re.search(r'SystemVersion\s*=\s*"([^"]+)"', config.read_text()) - if not match: - raise CheckFailure(f"could not read SystemVersion from {config}") - return match.group(1) - - -def build(target): - build_dir = REPO_ROOT / "build" / target - result = subprocess.run( - [VENV_PYTHON, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, - ) - if result.returncode != 0: - log(result.stdout[-3000:]) - log(result.stderr[-3000:]) - raise CheckFailure(f"build failed for {target}") - return build_dir - - -# -------------------------------------------------------------------------- -# 4. flash -# -------------------------------------------------------------------------- - - -def write_bench_config(target, workdir): - ssid = os.environ.get("VECTOR_HIL_WIFI_SSID", "") - password = os.environ.get("VECTOR_HIL_WIFI_PASSWORD", "") - game_password = os.environ.get("VECTOR_HIL_GAME_PASSWORD", "hiltest") - - if not ssid or not password: - raise CheckFailure("VECTOR_HIL_WIFI_SSID / VECTOR_HIL_WIFI_PASSWORD are not set") - - # dev/flash.py builds a MicroPython snippet with single-quoted values, so a - # single quote anywhere here would produce a syntax error on the board - # rather than an obvious failure here. - for name, value in (("ssid", ssid), ("password", password), ("game password", game_password)): - if "'" in value or "\\" in value: - raise CheckFailure(f"bench {name} contains a quote or backslash, which dev/flash.py cannot write") - - config = { - "ssid": ssid, - "password": password, - "gamename": DEFAULT_GAMENAME[target], - "Gpassword": game_password, - } - path = workdir / f"hil-config-{target}.json" - path.write_text(json.dumps(config)) - return path - - -def flash(target, port, build_dir, config_path): - result = subprocess.run( - [VENV_PYTHON, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, - ) - if result.returncode != 0: - log(result.stdout[-3000:]) - log(result.stderr[-3000:]) - raise CheckFailure(f"flash failed for {target} on {port}") - - -# -------------------------------------------------------------------------- -# 5. health -# -------------------------------------------------------------------------- - - -def reset_board(port): - """Reset the board so we own the boot we are about to watch. - - The ready marker is printed exactly once per boot. dev/flash.py already - resets at the end of flashing, but flashing runs over every board before - any health check starts, so by the time we open a console the board booted - a minute ago and the marker is long gone. Resetting here makes the wait - deterministic and the reported boot time meaningful. - """ - result = mpremote( - "connect", port, "exec", "--no-follow", "import machine; machine.reset()", - timeout=30, - ) - if result.returncode != 0: - raise CheckFailure(f"could not reset {port} before the health check: {result.stderr.strip()}") - - -def wait_for_server(port, timeout=BOOT_TIMEOUT): - """Watch the boot console until the firmware reports its web server is up. - - Polling an API that is not listening yet tells you nothing about why, and - burns the whole timeout when a board fails to boot. Reading the console - instead gives an exact ready signal and, on failure, the boot log that - explains it. - - Returns the open serial connection so the USB API can reuse it - the - Pico exposes one CDC endpoint, so a second connection would fight this one. - """ - deadline = time.monotonic() + timeout - transcript = [] - connection = None - - while time.monotonic() < deadline: - if connection is None: - try: - # The port disappears and re-enumerates across the reset, so a - # failure to open here is expected for the first second or two. - connection = serial.Serial(port=port, baudrate=115200, timeout=1) - except Exception: - time.sleep(1) - continue - try: - raw = connection.readline() - except Exception: - try: - connection.close() - except Exception: - pass - connection = None - continue - - if not raw: - continue - text = raw.decode(errors="replace").rstrip("\r\n") - if not text: - continue - transcript.append(text) - - if READY_MARKER in text: - elapsed = timeout - (deadline - time.monotonic()) - log(f" server up after {elapsed:.1f}s ({text.strip()!r})") - time.sleep(SERVER_SETTLE_SECONDS) - return connection, transcript - - if connection is not None: - try: - connection.close() - except Exception: - pass - - tail = "\n ".join(transcript[-20:]) or "(nothing on the console)" - raise CheckFailure( - f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}" - ) - - -def prime_usb(connection): - """Clear both ends of the serial line before the first API request. - - usb_comms accumulates stdin characters into a module-level `buffer` until - it sees a newline (usb_comms.py:132). Anything left there without a - terminator - a partial line, stray bytes from the raw-REPL session that - issued the reset - silently prefixes the next request, so the board parses - a route like "\x02/api/version", fails the `_routes` lookup and answers - 404. A lone newline flushes whatever is pending into a discarded request. - """ - try: - connection.reset_input_buffer() - connection.reset_output_buffer() - connection.write(b"\n") - connection.flush() - except Exception as exc: - log(f" warning: could not prime the USB link: {exc}") - return - # usb_request_handler is scheduled every 1000ms (phew/server.py:342), so - # give it a turn to consume the flush before the first real request. - time.sleep(1.5) - try: - connection.reset_input_buffer() - except Exception: - pass - - -def get(client, route, expect=200): - response = client.send_and_receive(route=route, payload=None, timeout=15) - status = response.get("status") - if status != expect: - # The board narrates its own routing failures ("USB REQ: route not - # found: ..."), but the client discards every line that is not a - # response. Drain whatever is pending so the reason is visible. - raise CheckFailure( - f"{route} returned {status}, expected {expect}" - f"{_drain_serial(client.ser)}" - ) - return response.get("body") - - -def _drain_serial(connection, limit=12): - """Return any pending board chatter, formatted for an error message.""" - try: - time.sleep(0.5) - pending = connection.read(connection.in_waiting or 0) - except Exception: - return "" - if not pending: - return "" - lines = [line for line in pending.decode(errors="replace").splitlines() if line.strip()] - if not lines: - return "" - return "\n board said: " + "\n board said: ".join(lines[:limit]) - - def check_faults(board): faults = get(board["client"], "/api/fault") or [] if isinstance(faults, dict): @@ -726,20 +271,6 @@ def _summarise(payload): return repr(payload)[:60] -# -------------------------------------------------------------------------- - - -def _dump_boot_log(board, lines=25): - """Show what the board actually said. A failed health check is usually - explained by the boot output, and by this point we already have it.""" - transcript = board.get("boot_log") - if not transcript: - return - log(f" last {min(lines, len(transcript))} lines of {board['port']} boot console:") - for line in transcript[-lines:]: - log(f" {line}") - - def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--skip-http", action="store_true", help="USB checks only; do not exercise the network stack") @@ -750,8 +281,7 @@ def main(): help="blink each board in turn so you can tell which physical board is which") args = parser.parse_args() - global VENV_PYTHON - VENV_PYTHON = ensure_tools_on_path() + ensure_tools_on_path() if args.identify: group("Inventory") diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py new file mode 100644 index 00000000..26309355 --- /dev/null +++ b/dev/tests/test_hil_config_matrix.py @@ -0,0 +1,317 @@ +"""Tests for the parts of the HIL config matrix that do not need a board. + +Everything that touches serial, mpremote or a real board is out of reach here; +what is testable is the expectation side - how the harness reads the repo's +config JSON, which configs it decides to run, and in what order - plus the +filename-length rule that decides whether a config can be reached at all. +""" + +from __future__ import annotations + +import json +import sys +import types +from argparse import Namespace +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +# bench.py imports pyserial (which ships with mpremote) and dev/usb_coms_demo. +# Neither is needed for the pure helpers under test and neither is guaranteed to +# be installed wherever these tests run, so stand them in before the import. +sys.modules.setdefault("serial", types.ModuleType("serial")) +if "usb_coms_demo" not in sys.modules: + stub = types.ModuleType("usb_coms_demo") + stub.UsbApiClient = object + sys.modules["usb_coms_demo"] = stub + +sys.path.insert(0, str(REPO_ROOT / "dev" / "hil")) + +import bench # noqa: E402 +import config_matrix as cm # noqa: E402 + + +def write_config(directory: Path, name: str, game_name: str, **sections) -> None: + directory.mkdir(parents=True, exist_ok=True) + payload = {"GameInfo": {"GameName": game_name, "System": "WPC"}} + payload.update(sections) + (directory / f"{name}.json").write_text(json.dumps(payload)) + + +@pytest.fixture() +def fake_repo(tmp_path, monkeypatch): + """A repo-shaped tree with one target and three configs.""" + config_dir = tmp_path / "src" / "wpc" / "config" + write_config(config_dir, "AttackMars_11", "Attack from Mars", Adjustments={"Type": 0}) + write_config(config_dir, "Generic_WPC", "Generic System WPC", Adjustments={"Type": 0}) + write_config(config_dir, "Taxi_L4", "Taxi") + monkeypatch.setattr(cm, "REPO_ROOT", tmp_path) + return tmp_path + + +def args(**overrides) -> Namespace: + defaults = {"configs": None, "limit": None, "changed_since": None} + defaults.update(overrides) + return Namespace(**defaults) + + +def test_source_configs_reads_names_and_adjustments(fake_repo): + configs = cm.source_configs("wpc") + + assert configs["AttackMars_11"] == {"name": "Attack from Mars", "adjustments": True} + # Taxi_L4 declares no Adjustments section, which is what decides whether + # /api/adjustments/status is held to 200 on the bench. + assert configs["Taxi_L4"] == {"name": "Taxi", "adjustments": False} + + +def test_source_configs_rejects_a_config_with_no_game_name(fake_repo): + (fake_repo / "src" / "wpc" / "config" / "Broken_L1.json").write_text(json.dumps({"GameInfo": {}})) + + with pytest.raises(bench.CheckFailure, match="GameInfo.GameName"): + cm.source_configs("wpc") + + +def test_source_configs_rejects_unparseable_json(fake_repo): + (fake_repo / "src" / "wpc" / "config" / "Broken_L1.json").write_text("{nope") + + with pytest.raises(bench.CheckFailure, match="not valid JSON"): + cm.source_configs("wpc") + + +def test_source_configs_rejects_an_unknown_target(fake_repo): + with pytest.raises(bench.CheckFailure, match="no config directory"): + cm.source_configs("nosuchtarget") + + +def test_select_configs_defaults_to_every_config(fake_repo): + names, configs = cm.select_configs("wpc", args()) + + assert names == sorted(configs) == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] + + +def test_select_configs_honours_an_explicit_list(fake_repo): + names, _ = cm.select_configs("wpc", args(configs="Taxi_L4, AttackMars_11")) + + assert names == ["Taxi_L4", "AttackMars_11"] + + +def test_select_configs_rejects_an_unknown_name(fake_repo): + with pytest.raises(bench.CheckFailure, match="Nonexistent_L1"): + cm.select_configs("wpc", args(configs="Nonexistent_L1")) + + +def test_select_configs_applies_the_limit(fake_repo): + names, _ = cm.select_configs("wpc", args(limit=2)) + + assert names == ["AttackMars_11", "Generic_WPC"] + + +def test_select_configs_runs_changed_configs_first(fake_repo, monkeypatch): + monkeypatch.setattr(cm, "changed_configs", lambda target, ref: ["Taxi_L4"]) + + names, _ = cm.select_configs("wpc", args(changed_since="origin/main")) + + assert names[0] == "Taxi_L4" + assert sorted(names) == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] + + +def test_order_configs_ignores_names_that_are_not_under_test(): + ordered = cm.order_configs(["a", "b", "c"], ["c", "deleted"]) + + assert ordered == ["c", "a", "b"] + + +def test_order_configs_is_a_no_op_without_changes(): + assert cm.order_configs(["a", "b"], []) == ["a", "b"] + + +def test_gamename_field_width_comes_from_the_firmware_source(): + # Not hard-coded in the harness: widening the FRAM field in SPI_DataStore.py + # must move this number, or the harness would keep enforcing a stale limit. + assert bench.gamename_field_bytes() == 16 + + +# Two shipped WPC configs are longer than the FRAM `gamename` field and so can +# never be selected on a real board: the web UI offers them, the write is +# accepted, the name is truncated on the way into FRAM, and the next boot +# matches nothing and comes up on safe defaults with CONF01 raised. That is a +# pre-existing firmware/config defect, not something this harness introduced - +# it is what the bench found first - so it is recorded here rather than fixed +# here. Shortening either filename (or widening the field) should shorten this +# list; nothing should ever lengthen it. +KNOWN_UNREACHABLE_CONFIGS = {"GilliganIsland_L9", "HarleyDavidson_L3"} + + +def test_no_new_config_name_exceeds_the_gamename_field(): + """A config whose filename is longer than the field can never be selected. + + write_record packs `gamename` into a fixed-width field and struct.pack + truncates silently, so the truncated name matches nothing at boot and the + board comes up on safe defaults with CONF01. The bench catches this per + board, but it is cheaper to catch here, and this way a new offender fails + an ordinary PR rather than 20 minutes of bench time. + """ + limit = bench.gamename_field_bytes() + too_long = {path.stem for path in (REPO_ROOT / "src").glob("*/config/*.json") if len(path.stem.encode()) > limit} + + new_offenders = sorted(too_long - KNOWN_UNREACHABLE_CONFIGS) + assert new_offenders == [], f"config filenames longer than the {limit}-byte FRAM gamename field: {', '.join(new_offenders)}" + + fixed = sorted(KNOWN_UNREACHABLE_CONFIGS - too_long) + assert fixed == [], f"{', '.join(fixed)} now fits - drop it from KNOWN_UNREACHABLE_CONFIGS" + + +# -------------------------------------------------------------------------- +# the assertions themselves, with the board faked out +# -------------------------------------------------------------------------- + + +class FakeClient: + def __init__(self, *_args): + pass + + def close(self): + pass + + +def fake_board(monkeypatch, responses, stored=None): + """Patch out everything that needs hardware; return the recorded writes. + + `responses` maps a route to the body the fake board answers with, or to an + exception to raise. + """ + written = [] + + def fake_get(_client, route, expect=200): + body = responses[route] + if isinstance(body, Exception): + raise body + return body + + def fake_set_game_config(_port, gamename): + if stored is not None and gamename != stored: + raise bench.CheckFailure(f"wrote gamename={gamename!r} but the board stored {stored!r}") + written.append(gamename) + + monkeypatch.setattr(cm, "get", fake_get) + monkeypatch.setattr(cm, "set_game_config", fake_set_game_config) + monkeypatch.setattr(cm, "reset_board", lambda _port: None) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", FakeClient) + monkeypatch.setattr(cm, "wait_for_server", lambda _port, timeout=None: (types.SimpleNamespace(close=lambda: None), ["boot"])) + return written + + +def healthy(config="AttackMars_11", name="Attack from Mars"): + return { + # HDWR02 is expected on a bare bench board with nothing driving the bus. + "/api/fault": ["HDWR02: No Bus Activity"], + "/api/game/active_config": {"active_config": config}, + "/api/game/name": name, + "/api/leaders": [], + "/api/adjustments/status": {"profiles": [], "adjustments_support": True}, + } + + +def test_check_config_passes_when_the_board_reports_the_configured_game(monkeypatch): + written = fake_board(monkeypatch, healthy()) + + name, _boot_log = cm.check_config("/dev/ttyFAKE", "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + + assert name == "Attack from Mars" + assert written == ["AttackMars_11"] + + +def test_check_config_catches_a_silent_fallback_to_the_generic_config(monkeypatch): + """The failure this harness exists for. + + A config that fails to apply raises nothing an outside observer can see: + GameDefsLoad drops to safe_defaults and the board serves a generic + definition, healthy in every other respect. Only the game name gives it + away. + """ + fake_board(monkeypatch, healthy(name="Generic System")) + + with pytest.raises(bench.CheckFailure, match="fell back to a generic definition"): + cm.check_config("/dev/ttyFAKE", "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + + +def test_check_config_catches_a_board_running_a_different_config(monkeypatch): + fake_board(monkeypatch, healthy(config="Taxi_L4")) + + with pytest.raises(bench.CheckFailure, match="active config is 'Taxi_L4'"): + cm.check_config("/dev/ttyFAKE", "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + + +def test_check_config_accepts_the_game_name_as_the_active_config_on_em(monkeypatch): + # EM answers /api/game/active_config with the game name (backend.py:481). + fake_board(monkeypatch, healthy(config="EM Machine", name="EM Machine")) + + name, _ = cm.check_config("/dev/ttyFAKE", "em", "EM_machine_", {"name": "EM Machine", "adjustments": False}) + + assert name == "EM Machine" + + +def test_check_config_reports_a_name_that_cannot_be_stored(monkeypatch): + fake_board(monkeypatch, healthy(), stored="HarleyDavidson_L") + + with pytest.raises(bench.CheckFailure, match="but the board stored"): + cm.check_config("/dev/ttyFAKE", "wpc", "HarleyDavidson_L3", {"name": "Harley Davidson", "adjustments": True}) + + +@pytest.mark.parametrize( + "faults, expected", + [ + (["CONF01: Invalid Configuration"], "safe defaults"), + (["CONF00: Unknown Configuration Error"], "safe defaults"), + (["HDWR01: Early Bus Activity"], "safe mode"), + (["HDWR00: Unknown Hardware Error"], "unexpected fault"), + ], +) +def test_check_faults_rejects_anything_that_invalidates_the_result(monkeypatch, faults, expected): + fake_board(monkeypatch, {"/api/fault": faults}) + + with pytest.raises(bench.CheckFailure, match=expected): + cm.check_faults(FakeClient(), "AttackMars_11") + + +def test_check_faults_allows_the_bare_bench_fault(monkeypatch): + fake_board(monkeypatch, {"/api/fault": ["HDWR02: No Bus Activity"]}) + + cm.check_faults(FakeClient(), "AttackMars_11") + + +def test_check_adjustments_fails_when_the_config_declares_the_section(monkeypatch): + fake_board(monkeypatch, {"/api/adjustments/status": bench.CheckFailure("returned 500")}) + + with pytest.raises(bench.CheckFailure, match="500"): + cm.check_adjustments(FakeClient(), "AttackMars_11", declares_adjustments=True) + + +def test_check_adjustments_only_warns_for_a_config_with_no_adjustments(monkeypatch, capsys): + # A known firmware gap rather than a config problem - see check_adjustments(). + fake_board(monkeypatch, {"/api/adjustments/status": bench.CheckFailure("returned 500")}) + + cm.check_adjustments(FakeClient(), "Taxi_L4", declares_adjustments=False) + + assert "::warning::Taxi_L4" in capsys.readouterr().out + + +def test_write_step_summary_renders_a_table_and_the_failures(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + cm.write_step_summary([({"port": "/dev/ttyACM0", "target": "wpc"}, ["Taxi_L4"], [("AttackMars_11", "boom\nsecond line")])]) + + rendered = summary.read_text() + assert "| `/dev/ttyACM0` | wpc | 2 | 1 | 1 |" in rendered + assert "**wpc `AttackMars_11`** - boom" in rendered + assert "second line" not in rendered + + +def test_write_step_summary_is_a_no_op_outside_actions(monkeypatch): + monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) + + cm.write_step_summary([({"port": "/dev/ttyACM0", "target": "wpc"}, [], [])]) From 621f1b456cd584dcbde85c021b8cfd6141398446 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 04:31:58 +0000 Subject: [PATCH 02/32] fix(hil): drive the REPL over our own connection, and stop grinding on a dead board First bench run: sys11 passed all 39 configs at ~21s each, which is the harness working end to end. WPC wedged - silent console, no reply to Ctrl-C - and then the run spent 63 minutes writing the same 60s timeout 63 times before dying in teardown, taking the summary with it and never touching data_east. Three separate faults, fixed separately. 1. The wedge. The Pico has one CDC endpoint, so using mpremote to write the gamename meant closing our connection, letting a second process open the port, and reopening after - once per config. sys11 (MicroPython 1.24.1) survived that 39 times; WPC (1.26.0-preview) did not survive it once. A running board printing into a CDC endpoint nothing is draining is the difference between them. The REPL is now driven directly over the connection the harness already holds (bench.Repl), so nothing else ever opens the port and the port is closed only while the board is mid-reset. run_matrix became a pipeline - boot, assert, set the next config, reboot - carrying one connection per boot instead of churning three. Repl owns its read buffer, which is the point of it being a class: raw REPL is a sequence of markers and a read that syncs on one almost always pulls in bytes belonging to the next. The first version dropped them and desynchronised; there is a test for it now. 2. 63 timeouts. Two consecutive setup failures abandon the board, after one cheap recovery attempt that drains its console and sends Ctrl-C. Boot timeout for the matrix drops to 90s - a healthy boot answers in 12-16s, so 150s only bought a wedged board more time to waste. A dead board now costs about three minutes instead of an hour, and the boards after it still run. 3. Teardown taking down the run. restore_default caught CheckFailure but not TimeoutExpired, so it raised through main and lost results already in hand. Teardown is best-effort now, and a board that fails outright is one board's failure rather than the run's. Also: stty raw -echo before dumping a console in both HIL workflows. A tty reverts to ECHO-on once every handle closes, so `cat /dev/ttyACM*` was echoing each board's output back into it - the logs show all three boards parsing their own log lines as USB API requests. Simulated end to end against a fake board for the healthy, unstorable-name and wedged cases; the wedge that cost 63 minutes now costs two boots. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-config-matrix.yml | 8 + .github/workflows/hil-flash-check.yml | 6 + dev/hil/DESIGN.md | 37 ++- dev/hil/RUNNER_SETUP.md | 13 +- dev/hil/bench.py | 175 +++++++++++- dev/hil/config_matrix.py | 308 +++++++++++++-------- dev/tests/test_hil_config_matrix.py | 339 +++++++++++++++++++++--- 7 files changed, 721 insertions(+), 165 deletions(-) diff --git a/.github/workflows/hil-config-matrix.yml b/.github/workflows/hil-config-matrix.yml index a05c766d..83508dab 100644 --- a/.github/workflows/hil-config-matrix.yml +++ b/.github/workflows/hil-config-matrix.yml @@ -120,7 +120,15 @@ jobs: run: | # A board left mid-matrix may still be printing something useful; # grab a few seconds of console from each. + # + # `stty raw -echo` first, and it is not optional. A tty reverts to the + # driver default - ECHO on - once every handle is closed, so a bare + # `cat` makes the kernel echo the board's own output straight back at + # it. The last run caught this happening: the boards' consoles filled + # with "USB REQ: invalid request format: RESOURCE: RAM=69..." as they + # parsed their own log lines as USB API requests. for dev in /dev/ttyACM*; do echo "--- $dev" + stty -F "$dev" raw -echo 115200 || true timeout 8 cat "$dev" || true done diff --git a/.github/workflows/hil-flash-check.yml b/.github/workflows/hil-flash-check.yml index a5948dd3..722472c9 100644 --- a/.github/workflows/hil-flash-check.yml +++ b/.github/workflows/hil-flash-check.yml @@ -81,7 +81,13 @@ jobs: run: | # A board that failed its health check may still be printing something # useful; grab a few seconds of console from each. + # + # `stty raw -echo` first, and it is not optional. A tty reverts to the + # driver default - ECHO on - once every handle is closed, so a bare + # `cat` makes the kernel echo the board's own output straight back at + # it, and the board then parses its own log lines as USB API requests. for dev in /dev/ttyACM*; do echo "--- $dev" + stty -F "$dev" raw -echo 115200 || true timeout 8 cat "$dev" || true done diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index d2f60126..73f2d94f 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -221,7 +221,9 @@ dev/hil/ ``` `bench.py` deliberately asserts nothing about firmware behaviour — it only gets -a board into a known state and talks to it. Assertions live in the harness that +a board into a known state and talks to it, including driving the raw REPL over +a connection the caller owns (`bench.Repl`) rather than shelling out to +`mpremote`. Assertions live in the harness that imports it, so adding a check never means touching the plumbing. The hardware-free parts (config discovery, selection and ordering, and each assertion with the board faked out) are covered by @@ -350,6 +352,39 @@ healthy in every other respect. `/api/game/active_config` does not catch this it reads the `gamename` field back out of FRAM, not what actually loaded. The game name is what separates "loaded my config" from "silently fell back". +#### Bench results, first full run + +sys11 passed all 39 of its configs, at 20–22s each (~14 min). That is the +harness working end to end: set the config, reboot, and confirm the board comes +up reporting that game. + +WPC did not, and the reason is worth recording because it shaped the design. +The board wedged — silent console, no reply to Ctrl-C — on the first handoff +from our serial connection to `mpremote`, and stayed wedged for the remaining +62 configs. The Pico has a single CDC endpoint, so using `mpremote` mid-run +meant closing our connection, letting a second process open the port, and +reopening afterwards, once per config. sys11 (MicroPython v1.24.1) survived +that 39 times; WPC (v1.26.0-preview) did not survive it once. A running board +printing into a CDC endpoint that nothing is draining is the difference between +them. + +So the harness now **never hands the port to another process and never leaves +it unread while the board is running.** The REPL is driven directly over the +connection the harness already holds (`bench.Repl`), and the port is closed +only while the board is mid-reset. Three further changes came out of the same +run: + +- **Two consecutive setup failures abandon the board.** The first run spent 63 + minutes writing the same 60s timeout 63 times. A board that stops answering + does not start again on its own. +- **Teardown cannot fail the run.** A `TimeoutExpired` escaped `restore_default` + and took the whole process down with a traceback — losing the summary and + skipping data_east, which had never been touched. +- **`stty raw -echo` before dumping a console.** A tty reverts to ECHO-on once + every handle closes, so the `cat /dev/ttyACM*` diagnostic step was echoing + each board's output back into it; the logs show all three boards parsing + their own log lines as USB API requests. Fixed in both HIL workflows. + #### Findings and limits - **Two WPC configs cannot be selected at all.** `configuration.gamename` is a diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 9c266e89..92846f4b 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -194,12 +194,19 @@ cd ~/vector && export PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/config_matrix.py --target wpc --limit 5 --skip-flash ``` -A full config matrix is roughly 15–25s per config per board and WPC alone has 63, so budget -well over half an hour for an unfiltered run. `--configs`, `--limit` and `--target` are there +A full config matrix is roughly 20–22s per config per board (measured on the bench) and WPC +alone has 63, so budget well over half an hour for an unfiltered run. `--configs`, `--limit` and `--target` are there to keep an iteration loop short; `--changed-since REF` runs the configs a branch touched first, which is what the workflow does on a push. The matrix leaves each board on its generic config when it finishes, including after a -failure, so a run never strands a board on a game config you did not ask for. +failure, so a run never strands a board on a game config you did not ask for. If a board +stops answering it is abandoned after two consecutive setup failures and the run moves on to +the next board, rather than timing out against a board that is not coming back. + +**Never `cat /dev/ttyACM*` to watch a board.** A tty reverts to ECHO-on once every handle is +closed, so a bare `cat` makes the kernel echo the board's own output back into it, and the +board then tries to parse its log lines as USB API requests. Use `stty -F /dev/ttyACM0 raw +-echo 115200` first, or `mpremote connect /dev/ttyACM0 repl`. See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 0008d542..08901ed6 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -529,6 +529,164 @@ def _dump_boot_log(board, lines=25): log(f" {line}") +# -------------------------------------------------------------------------- +# raw REPL, over a connection we already hold +# -------------------------------------------------------------------------- +# +# Everything below drives the REPL over an open pyserial connection instead of +# shelling out to `mpremote`. That is not a stylistic preference - it is the +# fix for a board that wedged on the bench. +# +# The Pico exposes one CDC endpoint, so using mpremote mid-run means closing +# our connection, letting a second process open the port, and reopening +# afterwards - for every config. On the first such handoff the WPC board +# (MicroPython 1.26.0-preview) stopped responding entirely: silent console, no +# reply to Ctrl-C, for the following 63 attempts. sys11 (1.24.1) survived the +# same treatment 39 times. A running board printing to a CDC endpoint that +# nothing is draining is the difference between them, so the harness now never +# leaves the port unread while the board is running, and never lets a second +# process contend for it. The port is closed only while the board is mid-reset. + +CTRL_A = b"\x01" # enter raw REPL +CTRL_B = b"\x02" # back to the friendly REPL +CTRL_C = b"\x03" # interrupt whatever is running +CTRL_D = b"\x04" # execute what was pasted / end-of-output marker + +RAW_REPL_BANNER = b"raw REPL; CTRL-B to exit" +REPL_TIMEOUT = 15 + + +class Repl: + """A raw-REPL session over a connection somebody else owns. + + Owns a pending buffer, which is the whole reason this is a class rather + than a few functions: raw REPL is a sequence of markers (`OK`, \x04, \x04, + `>`) and a read that syncs on one marker almost always pulls in bytes + belonging to the next. Dropping them desynchronises everything that + follows. + """ + + def __init__(self, connection): + self.connection = connection + self._pending = bytearray() + + def read_until(self, marker, timeout, what): + """Return everything up to `marker`, keeping what came after it. + + The board is mid-sentence when we interrupt it, so the stream still + holds application output. Syncing on a marker rather than on a line + count is what makes that harmless. + """ + deadline = time.monotonic() + timeout + while True: + index = self._pending.find(marker) + if index >= 0: + before = bytes(self._pending[:index]) + del self._pending[: index + len(marker)] + return before + if time.monotonic() >= deadline: + break + chunk = self.connection.read(self.connection.in_waiting or 1) + if chunk: + self._pending.extend(chunk) + + tail = bytes(self._pending[-400:]).decode(errors="replace") + raise CheckFailure(f"timed out after {timeout}s waiting for {what}. Last output:\n {tail or '(nothing on the console)'}") + + def enter(self, timeout=REPL_TIMEOUT): + """Interrupt the running firmware and take the raw REPL. + + Ctrl-C raises KeyboardInterrupt in `main.py`, which ends the + application and drops to the REPL - the same thing `mpremote` does, and + the reason this is safe to do to a board we are about to reset anyway. + """ + self._pending.clear() + self.connection.reset_input_buffer() + self.connection.write(CTRL_C + CTRL_C) + self.connection.flush() + time.sleep(0.2) + + self.connection.write(CTRL_A) + self.connection.flush() + self.read_until(RAW_REPL_BANNER, timeout, "the raw REPL prompt") + self.read_until(b">", timeout, "the raw REPL prompt") + return self + + def exec(self, code, timeout=REPL_TIMEOUT): + """Run one snippet and return what it printed. + + Raw REPL framing: paste the code, Ctrl-D to run, the board answers + `OK`, then stdout, then \x04, then the traceback (empty on success), + then \x04. + """ + self.connection.write(code.encode() + CTRL_D) + self.connection.flush() + + self.read_until(b"OK", timeout, "the board to accept the snippet") + output = self.read_until(CTRL_D, timeout, "the snippet to finish") + error = self.read_until(CTRL_D, timeout, "the snippet's exit status") + + if error.strip(): + detail = error.decode(errors="replace").strip().replace("\n", "\n ") + raise CheckFailure(f"the board raised an error running the snippet:\n {detail}") + return output.decode(errors="replace") + + def reset(self): + """Reset the board, without waiting for a reply. + + There is no reply to wait for - the board reboots mid-command and the + port re-enumerates underneath us. The caller reopens it in + wait_for_server(). + """ + try: + self.connection.write(b"import machine; machine.reset()" + CTRL_D) + self.connection.flush() + except Exception as exc: + raise CheckFailure(f"could not issue a reset over the REPL: {exc}") + # Let the write reach the board before the port disappears. + time.sleep(0.5) + + +def repl_reset(connection): + """Interrupt whatever the board is doing and reset it, over `connection`.""" + Repl(connection).enter().reset() + + +def drain_port(port, seconds=3): + """Open a port and read whatever the board has queued, then interrupt it. + + A cheap attempt at unsticking a board that has gone quiet, and it costs one + open and three seconds. The wedge worth recovering from is a board blocked + writing into a CDC endpoint that nothing is draining: reading is the whole + remedy, and the Ctrl-C afterwards gets it back to a REPL if the read freed + it. Reports what it saw either way - a board that yields zero bytes and a + board that yields a backlog are different problems. + """ + try: + connection = serial.Serial(port=port, baudrate=115200, timeout=1) + except Exception as exc: + log(f" could not reopen {port} to unstick it: {exc}") + return 0 + + drained = 0 + try: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + drained += len(connection.read(connection.in_waiting or 1)) + connection.write(CTRL_C) + connection.flush() + except Exception as exc: + log(f" error while draining {port}: {exc}") + finally: + try: + connection.close() + except Exception: + pass + + log(f" drained {drained} byte(s) from {port} and sent Ctrl-C") + return drained + + # -------------------------------------------------------------------------- # game configuration # -------------------------------------------------------------------------- @@ -561,14 +719,13 @@ def gamename_field_bytes(): ) -def set_game_config(port, gamename): +def set_game_config(connection, gamename): """Point a board at one game config and prove the value survived the write. The board picks its config up from the FRAM `configuration` record at boot (GameDefsLoad.go), so setting it is a REPL write plus a reset - no - authentication, no HTTP, and no reflash. mpremote interrupts whatever the - board is running to get the REPL, which is fine here because the caller - resets immediately afterwards. + authentication, no HTTP, and no reflash. Takes the caller's open connection + rather than a port: see the note above raw REPL for why that matters. The read-back is the load-bearing part. `gamename` is a fixed-width field and struct.pack truncates silently, so a name that is too long is written, @@ -579,20 +736,18 @@ def set_game_config(port, gamename): if "'" in gamename or "\\" in gamename: raise CheckFailure(f"config name {gamename!r} contains a quote or backslash") - result = mpremote("connect", port, "exec", SET_CONFIG_SNIPPET.format(gamename=gamename), timeout=60) - if result.returncode != 0: - raise CheckFailure(f"could not write gamename={gamename!r} to {port}: {result.stderr.strip()}") + output = Repl(connection).enter().exec(SET_CONFIG_SNIPPET.format(gamename=gamename)) stored = None - for line in result.stdout.splitlines(): + for line in output.splitlines(): if line.startswith("GAMENAME="): stored = line.split("=", 1)[1].strip() if stored is None: - raise CheckFailure(f"board did not read back a gamename after the write (said {result.stdout.strip()!r})") + raise CheckFailure(f"board did not read back a gamename after the write (said {output.strip()!r})") if stored != gamename: limit = gamename_field_bytes() detail = "" if len(gamename) > limit: - detail = f" - the FRAM `gamename` field is {limit} bytes and this filename is {len(gamename)}," " so no board can ever store it and the config is unreachable in the field" + detail = f" - the FRAM `gamename` field is {limit} bytes and this filename is {len(gamename)}, so no board can ever store it and the config is unreachable in the field" raise CheckFailure(f"wrote gamename={gamename!r} but the board stored {stored!r}{detail}") diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index 36b73046..c7846298 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -55,11 +55,14 @@ from bench import ( # noqa: E402 _TIMINGS, BENCH_WARN_FAULTS, + BOOT_TIMEOUT, DEFAULT_GAMENAME, EXPECTED_FAULTS, REPO_ROOT, CheckFailure, UsbApiClient, + _dump_boot_log, + drain_port, endgroup, get, group, @@ -67,7 +70,7 @@ log, parse_board_map, prime_usb, - reset_board, + repl_reset, resolve_targets, set_game_config, wait_for_server, @@ -78,6 +81,10 @@ # safe_defaults, which is exactly the failure this harness exists to catch. CONFIG_FAULTS = {"CONF00", "CONF01"} +# Measured on the bench: 12.5s (wpc), 15.3s (sys11), and the flash harness's +# 150s default only buys a wedged board more time to waste. +MATRIX_BOOT_TIMEOUT = 90 + def source_configs(target): """{config filename without .json: {"name": ..., "adjustments": bool}}. @@ -201,145 +208,225 @@ def check_adjustments(client, config, declares_adjustments): raise CheckFailure(f"/api/adjustments/status returned {type(adjustments).__name__}, expected an object") -def check_config(port, target, config, expected): - """Boot one board on one config and prove it is the config that loaded.""" - expected_name = expected["name"] +class Session: + """One board's serial connection, carried across the whole matrix. - set_game_config(port, config) - reset_board(port) + The connection is the point. The Pico has a single CDC endpoint, so every + close is an invitation for something else to grab the port and for the + board to be left printing into an endpoint nothing is draining. The first + bench run wedged the WPC board on exactly that, one config in, and then + spent 63 minutes timing out against a board that was never coming back. - connection, boot_log = wait_for_server(port) - client = None - try: - prime_usb(connection) - client = UsbApiClient(connection) - - check_faults(client, config) - - active = get(client, "/api/game/active_config") - active = active.get("active_config") if isinstance(active, dict) else active - - # EM boards answer this route with the game name rather than the - # filename (backend.py:481), so accept either for that target. - expected_active = {config, expected_name} if target == "em" else {config} - if active not in expected_active: - raise CheckFailure(f"active config is {active!r}, expected {config!r}") - - name = get(client, "/api/game/name") - if isinstance(name, dict): - name = name.get("name") - name = str(name).strip() - if name != expected_name: - raise CheckFailure(f"board reports game name {name!r}, but {config}.json says {expected_name!r} - the config did not apply and the board fell back to a generic definition") - - # A config that parses but is not usable still fails a customer. Both - # of these read the loaded definition rather than just its presence. - if get(client, "/api/leaders") is None: - raise CheckFailure("/api/leaders returned no body") - check_adjustments(client, config, expected["adjustments"]) - - return name, boot_log - finally: - if client is not None: - try: - client.close() - except Exception: - pass - else: + So: open once per boot, hold it through the assertions AND through setting + the next config, and close it only in `reboot()` - by which point the board + is already resetting and has stopped printing. + """ + + def __init__(self, port, boot_timeout=BOOT_TIMEOUT): + self.port = port + self.boot_timeout = boot_timeout + self.connection = None + self.client = None + self.boot_log = [] + + def wait_for_boot(self): + self.connection, self.boot_log = wait_for_server(self.port, timeout=self.boot_timeout) + prime_usb(self.connection) + self.client = UsbApiClient(self.connection) + return self.client + + def _require_connection(self, what): + if self.connection is None: + raise CheckFailure(f"cannot {what} on {self.port}: the board is not connected (its last boot did not complete)") + return self.connection + + def set_config(self, gamename): + """Point the board at a config over the REPL we already have open.""" + set_game_config(self._require_connection("set a config"), gamename) + + def reboot(self): + """Reset from the REPL, then drop the port while the board is down.""" + self._require_connection("reset the board") + try: + repl_reset(self.connection) + finally: + self.close() + + def nudge(self): + """Try to unstick a board that stopped answering, before giving up.""" + self.close() + drain_port(self.port) + + def close(self): + connection, self.connection, self.client = self.connection, None, None + if connection is not None: try: connection.close() except Exception: pass -def check_bundle(port, target, configs): +def check_bundle(client, target, configs): """Compare the board's config list against the repo, once per board. Cheap, and it localises a whole class of failure before the matrix starts: if the build dropped or mangled a config, this says so in one boot instead of once per affected iteration. """ - reset_board(port) - connection, _ = wait_for_server(port) - try: - prime_usb(connection) - client = UsbApiClient(connection) - on_board = get(client, "/api/game/configs_list") - if not isinstance(on_board, dict) or not on_board: - raise CheckFailure("/api/game/configs_list is empty - the config bundle is missing from the build") - - missing = sorted(set(configs) - set(on_board)) - extra = sorted(set(on_board) - set(configs)) - if missing: - raise CheckFailure(f"{len(missing)} config(s) in src/{target}/config are not in the build's bundle: {', '.join(missing)}") - if extra: - raise CheckFailure(f"the build's bundle carries {len(extra)} config(s) with no source JSON: {', '.join(extra)}") - - mismatched = [f"{name}: bundle says {on_board[name].get('name')!r}, source says {configs[name]['name']!r}" for name in sorted(configs) if on_board[name].get("name") != configs[name]["name"]] - if mismatched: - raise CheckFailure("game name mismatch between the bundle and the source JSON:\n " + "\n ".join(mismatched)) - - log(f" bundle matches source: {len(configs)} configs, names identical") - finally: - try: - connection.close() - except Exception: - pass + on_board = get(client, "/api/game/configs_list") + if not isinstance(on_board, dict) or not on_board: + raise CheckFailure("/api/game/configs_list is empty - the config bundle is missing from the build") + + missing = sorted(set(configs) - set(on_board)) + extra = sorted(set(on_board) - set(configs)) + if missing: + raise CheckFailure(f"{len(missing)} config(s) in src/{target}/config are not in the build's bundle: {', '.join(missing)}") + if extra: + raise CheckFailure(f"the build's bundle carries {len(extra)} config(s) with no source JSON: {', '.join(extra)}") + + mismatched = [f"{name}: bundle says {on_board[name].get('name')!r}, source says {configs[name]['name']!r}" for name in sorted(configs) if on_board[name].get("name") != configs[name]["name"]] + if mismatched: + raise CheckFailure("game name mismatch between the bundle and the source JSON:\n " + "\n ".join(mismatched)) + + log(f" bundle matches source: {len(configs)} configs, names identical") + + +def check_booted_config(client, target, config, expected): + """Assert that the board in front of us booted on `config`.""" + expected_name = expected["name"] + + check_faults(client, config) + + active = get(client, "/api/game/active_config") + active = active.get("active_config") if isinstance(active, dict) else active + + # EM boards answer this route with the game name rather than the filename + # (backend.py:481), so accept either for that target. + expected_active = {config, expected_name} if target == "em" else {config} + if active not in expected_active: + raise CheckFailure(f"active config is {active!r}, expected {config!r}") + + name = get(client, "/api/game/name") + if isinstance(name, dict): + name = name.get("name") + name = str(name).strip() + if name != expected_name: + raise CheckFailure(f"board reports game name {name!r}, but {config}.json says {expected_name!r} - the config did not apply and the board fell back to a generic definition") + + # A config that parses but is not usable still fails a customer. Both of + # these read the loaded definition rather than just its presence. + if get(client, "/api/leaders") is None: + raise CheckFailure("/api/leaders returned no body") + check_adjustments(client, config, expected["adjustments"]) + + return name -def restore_default(port, target): - """Leave the board on its generic config, as flash_and_check.py expects.""" +def restore_default(session, target): + """Leave the board on its generic config, as flash_and_check.py expects. + + Best effort by design: this runs in teardown, including after the board has + stopped answering, and a failure here must not lose the results the matrix + already produced. + """ default = DEFAULT_GAMENAME[target] try: - set_game_config(port, default) - reset_board(port) + if session.connection is None: + session.wait_for_boot() + session.set_config(default) + session.reboot() log(f" restored {default}") - except CheckFailure as exc: - log(f"::warning::could not restore {default} on {port}: {exc}") + except Exception as exc: # noqa: BLE001 - teardown never fails the run + log(f"::warning::could not restore {default} on {session.port}: {exc}") + session.close() + + +# A board that stops answering does not start again on its own, and the bench +# is a shared singleton. Two setup failures in a row is the signal to stop +# spending an hour proving it: the first bench run burned 63 minutes writing +# the same 60s timeout 63 times, which is 62 wasted minutes and one lost +# data_east matrix. +MAX_CONSECUTIVE_SETUP_FAILURES = 2 def run_matrix(board, args): - """Walk one board through its configs. Returns (passed, failures).""" + """Walk one board through its configs. Returns (passed, failures). + + One connection per boot, held across the assertions and across setting the + next config, closed only while the board is resetting. See Session. + """ port = board["port"] target = board["target"] names, configs = select_configs(target, args) - group(f"Config bundle {target} on {port}") - check_bundle(port, target, configs) - endgroup() - - log("") - log(f"{len(names)} config(s) to check on {port} ({target})") - log("") - + session = Session(port, boot_timeout=args.boot_timeout) passed = [] failures = [] - for index, config in enumerate(names, 1): - started = time.monotonic() - group(f"[{index}/{len(names)}] {target} {config}") + consecutive_setup_failures = 0 + + try: + group(f"Config bundle {target} on {port}") try: - name, _boot_log = check_config(port, target, config, configs[config]) - elapsed = time.monotonic() - started - log(f" ok {config:20} -> {name!r} [{elapsed:.1f}s]") - passed.append(config) - except CheckFailure as exc: - log(f"::error::{target} {config}: {exc}") - failures.append((config, str(exc))) - if not args.keep_going: + client = session.wait_for_boot() + check_bundle(client, target, configs) + finally: + endgroup() + + log("") + log(f"{len(names)} config(s) to check on {port} ({target})") + log("") + + for index, config in enumerate(names, 1): + started = time.monotonic() + group(f"[{index}/{len(names)}] {target} {config}") + try: + # Set the config on the board we are already talking to, then + # reboot into it. The connection dies with the reset; the next + # wait_for_boot opens a fresh one. + session.set_config(config) + session.reboot() + client = session.wait_for_boot() + consecutive_setup_failures = 0 + + name = check_booted_config(client, target, config, configs[config]) + log(f" ok {config:20} -> {name!r} [{time.monotonic() - started:.1f}s]") + passed.append(config) + except CheckFailure as exc: + log(f"::error::{target} {config}: {exc}") + _dump_boot_log({"port": port, "boot_log": session.boot_log}) + failures.append((config, str(exc))) + # An assertion that ran is a result about the config. Anything + # that stopped us reaching the assertions is about the board. + if session.client is None: + consecutive_setup_failures += 1 + except Exception as exc: # noqa: BLE001 - one bad config must not end the run + log(f"::error::{target} {config}: unexpected error: {exc}") + failures.append((config, str(exc))) + consecutive_setup_failures += 1 + finally: endgroup() + + if consecutive_setup_failures: + # One cheap attempt at recovery before spending another cycle, + # and it doubles as diagnosis: whether the board has anything + # queued on its console says a lot about how it is stuck. + session.nudge() + + if consecutive_setup_failures >= MAX_CONSECUTIVE_SETUP_FAILURES: + remaining = names[index:] + log(f"::error::{port} stopped responding - abandoning this board after {consecutive_setup_failures} consecutive setup failures") + if remaining: + log(f" {len(remaining)} config(s) not run: {', '.join(remaining[:8])}{' ...' if len(remaining) > 8 else ''}") + failures.append(("(not run)", f"{len(remaining)} config(s) skipped after {port} stopped responding")) break - except Exception as exc: # noqa: BLE001 - one bad config must not end the run - log(f"::error::{target} {config}: unexpected error: {exc}") - failures.append((config, str(exc))) - if not args.keep_going: - endgroup() + + if not args.keep_going and failures: break + finally: + group(f"Restore {target} on {port}") + restore_default(session, target) endgroup() - group(f"Restore {target} on {port}") - restore_default(port, target) - endgroup() - return passed, failures @@ -371,6 +458,9 @@ def main(): parser.add_argument("--changed-since", metavar="REF", help="run configs changed since REF first, so a config-touching PR fails fast") parser.add_argument("--skip-flash", action="store_true", help="matrix what is already on the boards instead of building and flashing first") parser.add_argument("--stop-on-first-failure", dest="keep_going", action="store_false", help="stop a board's matrix at its first failing config (default: run them all)") + # A healthy boot answers in 12-16s on the bench, so the flash harness's + # 150s is generous here and only makes a dead board expensive. + parser.add_argument("--boot-timeout", type=int, default=MATRIX_BOOT_TIMEOUT, help=f"seconds to wait for a board's web server after a reset (default {MATRIX_BOOT_TIMEOUT})") args = parser.parse_args() bench.ensure_tools_on_path() @@ -415,10 +505,12 @@ def main(): for b in boards: try: passed, failures = run_matrix(b, args) - except CheckFailure as exc: + except Exception as exc: # noqa: BLE001 # A board that cannot even be set up is one board's problem. The # bench is a singleton and a run is expensive, so the other boards - # still get their matrix. + # still get their matrix. Deliberately broad: the first bench run + # died on a TimeoutExpired escaping teardown, which threw away the + # results already in hand and skipped the untouched board entirely. log(f"::error::{b['target']} on {b['port']}: {exc}") passed, failures = [], [("(board setup)", str(exc))] results.append((b, passed, failures)) diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 26309355..3fe1f2b6 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -176,13 +176,8 @@ def close(self): pass -def fake_board(monkeypatch, responses, stored=None): - """Patch out everything that needs hardware; return the recorded writes. - - `responses` maps a route to the body the fake board answers with, or to an - exception to raise. - """ - written = [] +def responder(responses): + """A stand-in for bench.get over a fake board.""" def fake_get(_client, route, expect=200): body = responses[route] @@ -190,18 +185,7 @@ def fake_get(_client, route, expect=200): raise body return body - def fake_set_game_config(_port, gamename): - if stored is not None and gamename != stored: - raise bench.CheckFailure(f"wrote gamename={gamename!r} but the board stored {stored!r}") - written.append(gamename) - - monkeypatch.setattr(cm, "get", fake_get) - monkeypatch.setattr(cm, "set_game_config", fake_set_game_config) - monkeypatch.setattr(cm, "reset_board", lambda _port: None) - monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) - monkeypatch.setattr(cm, "UsbApiClient", FakeClient) - monkeypatch.setattr(cm, "wait_for_server", lambda _port, timeout=None: (types.SimpleNamespace(close=lambda: None), ["boot"])) - return written + return fake_get def healthy(config="AttackMars_11", name="Attack from Mars"): @@ -215,16 +199,15 @@ def healthy(config="AttackMars_11", name="Attack from Mars"): } -def test_check_config_passes_when_the_board_reports_the_configured_game(monkeypatch): - written = fake_board(monkeypatch, healthy()) +def test_check_booted_config_passes_when_the_board_reports_the_configured_game(monkeypatch): + monkeypatch.setattr(cm, "get", responder(healthy())) - name, _boot_log = cm.check_config("/dev/ttyFAKE", "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + name = cm.check_booted_config(FakeClient(), "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) assert name == "Attack from Mars" - assert written == ["AttackMars_11"] -def test_check_config_catches_a_silent_fallback_to_the_generic_config(monkeypatch): +def test_check_booted_config_catches_a_silent_fallback_to_the_generic_config(monkeypatch): """The failure this harness exists for. A config that fails to apply raises nothing an outside observer can see: @@ -232,35 +215,28 @@ def test_check_config_catches_a_silent_fallback_to_the_generic_config(monkeypatc definition, healthy in every other respect. Only the game name gives it away. """ - fake_board(monkeypatch, healthy(name="Generic System")) + monkeypatch.setattr(cm, "get", responder(healthy(name="Generic System"))) with pytest.raises(bench.CheckFailure, match="fell back to a generic definition"): - cm.check_config("/dev/ttyFAKE", "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + cm.check_booted_config(FakeClient(), "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) -def test_check_config_catches_a_board_running_a_different_config(monkeypatch): - fake_board(monkeypatch, healthy(config="Taxi_L4")) +def test_check_booted_config_catches_a_board_running_a_different_config(monkeypatch): + monkeypatch.setattr(cm, "get", responder(healthy(config="Taxi_L4"))) with pytest.raises(bench.CheckFailure, match="active config is 'Taxi_L4'"): - cm.check_config("/dev/ttyFAKE", "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) + cm.check_booted_config(FakeClient(), "wpc", "AttackMars_11", {"name": "Attack from Mars", "adjustments": True}) -def test_check_config_accepts_the_game_name_as_the_active_config_on_em(monkeypatch): +def test_check_booted_config_accepts_the_game_name_as_the_active_config_on_em(monkeypatch): # EM answers /api/game/active_config with the game name (backend.py:481). - fake_board(monkeypatch, healthy(config="EM Machine", name="EM Machine")) + monkeypatch.setattr(cm, "get", responder(healthy(config="EM Machine", name="EM Machine"))) - name, _ = cm.check_config("/dev/ttyFAKE", "em", "EM_machine_", {"name": "EM Machine", "adjustments": False}) + name = cm.check_booted_config(FakeClient(), "em", "EM_machine_", {"name": "EM Machine", "adjustments": False}) assert name == "EM Machine" -def test_check_config_reports_a_name_that_cannot_be_stored(monkeypatch): - fake_board(monkeypatch, healthy(), stored="HarleyDavidson_L") - - with pytest.raises(bench.CheckFailure, match="but the board stored"): - cm.check_config("/dev/ttyFAKE", "wpc", "HarleyDavidson_L3", {"name": "Harley Davidson", "adjustments": True}) - - @pytest.mark.parametrize( "faults, expected", [ @@ -271,20 +247,20 @@ def test_check_config_reports_a_name_that_cannot_be_stored(monkeypatch): ], ) def test_check_faults_rejects_anything_that_invalidates_the_result(monkeypatch, faults, expected): - fake_board(monkeypatch, {"/api/fault": faults}) + monkeypatch.setattr(cm, "get", responder({"/api/fault": faults})) with pytest.raises(bench.CheckFailure, match=expected): cm.check_faults(FakeClient(), "AttackMars_11") def test_check_faults_allows_the_bare_bench_fault(monkeypatch): - fake_board(monkeypatch, {"/api/fault": ["HDWR02: No Bus Activity"]}) + monkeypatch.setattr(cm, "get", responder({"/api/fault": ["HDWR02: No Bus Activity"]})) cm.check_faults(FakeClient(), "AttackMars_11") def test_check_adjustments_fails_when_the_config_declares_the_section(monkeypatch): - fake_board(monkeypatch, {"/api/adjustments/status": bench.CheckFailure("returned 500")}) + monkeypatch.setattr(cm, "get", responder({"/api/adjustments/status": bench.CheckFailure("returned 500")})) with pytest.raises(bench.CheckFailure, match="500"): cm.check_adjustments(FakeClient(), "AttackMars_11", declares_adjustments=True) @@ -292,13 +268,290 @@ def test_check_adjustments_fails_when_the_config_declares_the_section(monkeypatc def test_check_adjustments_only_warns_for_a_config_with_no_adjustments(monkeypatch, capsys): # A known firmware gap rather than a config problem - see check_adjustments(). - fake_board(monkeypatch, {"/api/adjustments/status": bench.CheckFailure("returned 500")}) + monkeypatch.setattr(cm, "get", responder({"/api/adjustments/status": bench.CheckFailure("returned 500")})) cm.check_adjustments(FakeClient(), "Taxi_L4", declares_adjustments=False) assert "::warning::Taxi_L4" in capsys.readouterr().out +def test_check_bundle_reports_a_config_missing_from_the_build(monkeypatch): + monkeypatch.setattr(cm, "get", responder({"/api/game/configs_list": {"Taxi_L4": {"name": "Taxi"}}})) + + with pytest.raises(bench.CheckFailure, match="AttackMars_11"): + cm.check_bundle(FakeClient(), "wpc", {"Taxi_L4": {"name": "Taxi"}, "AttackMars_11": {"name": "Attack from Mars"}}) + + +def test_check_bundle_reports_a_game_name_that_drifted_from_the_source(monkeypatch): + monkeypatch.setattr(cm, "get", responder({"/api/game/configs_list": {"Taxi_L4": {"name": "Taksi"}}})) + + with pytest.raises(bench.CheckFailure, match="bundle says 'Taksi'"): + cm.check_bundle(FakeClient(), "wpc", {"Taxi_L4": {"name": "Taxi"}}) + + +# -------------------------------------------------------------------------- +# the matrix loop, and what it does when a board stops answering +# -------------------------------------------------------------------------- + + +class FakeSession: + """Stands in for a board. `dies_after` makes it stop answering mid-run.""" + + def __init__(self, port, boot_timeout=None, dies_after=None): + self.port = port + self.boot_timeout = boot_timeout + self.dies_after = dies_after + self.connection = object() + self.client = FakeClient() + self.boot_log = [] + self.configs_set = [] + self.boots = 0 + self.nudges = 0 + self.restored = False + + def wait_for_boot(self): + self.boots += 1 + if self.dies_after is not None and self.boots > self.dies_after: + self.client = None + self.connection = None + raise bench.CheckFailure(f"{self.port} never reported its web server within 90s") + self.client = FakeClient() + self.connection = object() + return self.client + + def set_config(self, gamename): + if self.client is None: + raise bench.CheckFailure(f"could not reach the REPL on {self.port}") + self.configs_set.append(gamename) + + def reboot(self): + self.connection = None + + def nudge(self): + self.nudges += 1 + self.close() + + def close(self): + self.connection = None + self.client = None + + +def run_board(monkeypatch, fake_repo, session, check=None, **arg_overrides): + """Drive run_matrix against a fake board. `check` replaces the assertions.""" + if check is None: + + def check(_client, _target, _config, expected): + return expected["name"] + + monkeypatch.setattr(cm, "Session", lambda port, boot_timeout=None: session) + monkeypatch.setattr(cm, "check_bundle", lambda *a, **k: None) + monkeypatch.setattr(cm, "check_booted_config", check) + monkeypatch.setattr(cm, "restore_default", lambda s, _target: setattr(s, "restored", True)) + + defaults = {"configs": None, "limit": None, "changed_since": None, "keep_going": True, "boot_timeout": 90} + defaults.update(arg_overrides) + return cm.run_matrix({"port": session.port, "target": "wpc"}, Namespace(**defaults)) + + +def test_run_matrix_walks_every_config_on_a_healthy_board(monkeypatch, fake_repo): + session = FakeSession("/dev/ttyFAKE") + + passed, failures = run_board(monkeypatch, fake_repo, session) + + assert failures == [] + assert passed == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] + assert session.configs_set == passed + assert session.restored is True + + +def test_run_matrix_abandons_a_board_that_stops_answering(monkeypatch, fake_repo): + """The 63-minute failure, in one test. + + The first bench run wrote the same timeout 63 times against a WPC board + that had wedged one config in. Two consecutive setup failures is enough to + know the board is gone. + """ + # Enough configs that abandoning the board leaves some unrun - the point + # being that we stop rather than time out against every one of them. + config_dir = fake_repo / "src" / "wpc" / "config" + for index in range(6): + write_config(config_dir, f"Filler_L{index}", f"Filler {index}", Adjustments={"Type": 0}) + + # Boot 1 is the bundle check and boot 2 is the first config, so the board + # survives exactly one config before going quiet. + session = FakeSession("/dev/ttyFAKE", dies_after=2) + + passed, failures = run_board(monkeypatch, fake_repo, session) + + assert len(passed) == 1 + assert session.boots <= 1 + cm.MAX_CONSECUTIVE_SETUP_FAILURES + assert any("skipped after" in reason for _config, reason in failures) + # Every setup failure gets one cheap recovery attempt before we give up. + assert session.nudges == cm.MAX_CONSECUTIVE_SETUP_FAILURES + # Teardown still runs, so the next job does not inherit a stranded board. + assert session.restored is True + + +def test_run_matrix_keeps_going_after_a_failing_config(monkeypatch, fake_repo): + """An assertion failure is a result about the config, not about the board.""" + + def one_bad_config(_client, _target, config, expected): + if config == "Generic_WPC": + raise bench.CheckFailure("board reports game name 'Generic System' - fell back to a generic definition") + return expected["name"] + + session = FakeSession("/dev/ttyFAKE") + passed, failures = run_board(monkeypatch, fake_repo, session, check=one_bad_config) + + assert passed == ["AttackMars_11", "Taxi_L4"] + assert [config for config, _reason in failures] == ["Generic_WPC"] + + +def test_run_matrix_stops_at_the_first_failure_when_asked(monkeypatch, fake_repo): + def always_fails(*_args, **_kwargs): + raise bench.CheckFailure("board reports game name 'Generic System'") + + session = FakeSession("/dev/ttyFAKE") + passed, failures = run_board(monkeypatch, fake_repo, session, check=always_fails, keep_going=False) + + assert passed == [] + assert len(failures) == 1 + + +def test_restore_default_never_raises_when_the_board_is_gone(capsys): + """Teardown must not throw away results the matrix already produced. + + The first bench run died exactly here: a TimeoutExpired escaped teardown, + took out the run summary, and skipped the board that had not been touched + yet. + """ + + class DeadSession: + port = "/dev/ttyFAKE" + connection = None + + def wait_for_boot(self): + raise TimeoutError("board is not coming back") + + def close(self): + pass + + cm.restore_default(DeadSession(), "wpc") + + assert "::warning::could not restore Generic_WPC" in capsys.readouterr().out + + +# -------------------------------------------------------------------------- +# raw REPL framing +# -------------------------------------------------------------------------- + + +class FakeSerial: + """Replays a scripted board response and records what was written.""" + + def __init__(self, script=b""): + self.script = bytearray(script) + self.written = bytearray() + + def read(self, size=1): + chunk = bytes(self.script[:size]) + del self.script[: len(chunk)] + return chunk + + @property + def in_waiting(self): + return len(self.script) + + def write(self, data): + self.written.extend(data) + return len(data) + + def flush(self): + pass + + def reset_input_buffer(self): + pass + + def reset_output_buffer(self): + pass + + def close(self): + pass + + +def test_repl_enter_syncs_past_application_output(): + # The board is mid-sentence when we interrupt it, so the banner arrives + # after whatever it was printing. + serial = FakeSerial(b"RESOURCE: RAM=69%\r\nTraceback\r\nraw REPL; CTRL-B to exit\r\n>") + + bench.Repl(serial).enter(timeout=1) + + assert bench.CTRL_C in serial.written + assert bench.CTRL_A in serial.written + + +def test_repl_enter_reports_the_console_when_the_board_does_not_answer(): + serial = FakeSerial(b"RESOURCE: RAM=69%\r\n") + + with pytest.raises(bench.CheckFailure, match="raw REPL prompt"): + bench.Repl(serial).enter(timeout=0.2) + + +def test_repl_keeps_bytes_that_arrive_past_a_marker(): + """The bug that made the first version of this desynchronise. + + A read that syncs on `OK` almost always pulls in the output that follows + it; discarding that made every later marker arrive at an empty stream. + """ + repl = bench.Repl(FakeSerial(b"OKGAMENAME=Taxi_L4\x04\x04>")) + + assert repl.read_until(b"OK", 1, "ok") == b"" + assert repl.read_until(bench.CTRL_D, 1, "output") == b"GAMENAME=Taxi_L4" + + +def test_repl_exec_returns_what_the_snippet_printed(): + serial = FakeSerial(b"OKGAMENAME=Taxi_L4\r\n\x04\x04>") + + output = bench.Repl(serial).exec("print('x')", timeout=1) + + assert "GAMENAME=Taxi_L4" in output + assert serial.written.endswith(bench.CTRL_D) + + +def test_repl_exec_surfaces_a_traceback_from_the_board(): + serial = FakeSerial(b"OK\x04Traceback (most recent call last):\r\n KeyError: nope\x04>") + + with pytest.raises(bench.CheckFailure, match="KeyError: nope"): + bench.Repl(serial).exec("boom", timeout=1) + + +def fake_repl(monkeypatch, output): + class StubRepl: + def __init__(self, _connection): + pass + + def enter(self, timeout=None): + return self + + def exec(self, _code, timeout=None): + return output + + monkeypatch.setattr(bench, "Repl", StubRepl) + + +def test_set_game_config_rejects_a_name_the_board_truncated(monkeypatch): + fake_repl(monkeypatch, "GAMENAME=HarleyDavidson_L\r\n") + + with pytest.raises(bench.CheckFailure, match="16 bytes and this filename is 17"): + bench.set_game_config(FakeSerial(), "HarleyDavidson_L3") + + +def test_set_game_config_accepts_a_clean_round_trip(monkeypatch): + fake_repl(monkeypatch, "GAMENAME=Taxi_L4\r\n") + + bench.set_game_config(FakeSerial(), "Taxi_L4") + + def test_write_step_summary_renders_a_table_and_the_failures(tmp_path, monkeypatch): summary = tmp_path / "summary.md" monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) From 3bc61c4fa9f8a78205da7fe78dca79a06b4e873a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 04:55:25 +0000 Subject: [PATCH 03/32] feat(hil): recover a wedged bench board without a trip to the bench Run 2 died in inventory, before any of the config-matrix fixes could run: `mpremote connect /dev/ttyACM1` timed out at 30s. The WPC board is still deadlocked from run 1, so the bench is down until something revives it. TrenchCoat confirms the mechanism in its own source (src/ray.py, send_command): "if nothing ever drains the board's output, the USB CDC buffers fill up, MicroPython blocks writing to stdout, and the board deadlocks mid-script". Its enter_bootloader_mode() goes through machine.bootloader() on the REPL, which is no use once a board is in that state - hence this. dev/hil/recover.py escalates cheapest-first and stops the moment the board answers: 1. drain read whatever is queued and send Ctrl-C. If it is blocked on a full CDC buffer, reading is the remedy. Zero bytes drained is itself a diagnosis: it is stuck on something else. 2. usb reset USBDEVFS_RESET on the device node, to re-enumerate and reset TinyUSB's endpoint state. 3. power uhubctl on the board's hub port. The board is USB bus powered (Trench-Coat-Install-Guide.md), so this is a genuine cold boot. 4. reflash 1200 baud touch into the ROM bootloader, then a MicroPython UF2 onto the RPI-RP2 drive. The touch is a CDC line-coding change handled in USB interrupt context, so a blocked Python VM does not stop it - which is exactly why it can work when the REPL cannot. Two guards on step 4, because it is the one that can make things worse. It refuses to touch a board into BOOTSEL unless something here can actually mount the drive afterwards - a wedged board is at least still a serial device, while a BOOTSEL board that cannot be flashed needs a physical replug. And the UF2 is downloaded and checksummed *before* the touch, so a bad fetch cannot strand the board either. UF2s come from warped-pinball/trench-coat pinned by commit, with sha256 per file, rather than being vendored. A dead board cannot say which system it is, so the target is deduced by elimination: ask the boards that do answer, and whatever VECTOR_HIL_BOARD_MAP still expects is the one on the floor. With one board down that is exact; with two it refuses to guess, because flashing the wrong system's UF2 is worse than leaving a board dead. The run opens with a capability report - whether the USB node is writable, whether uhubctl is installed, whether anything can mount an RPI-RP2 drive - so even a failed recovery says which one-time bit of runner setup would make the bench self-healing. Expect steps 2 and 3 to be unavailable today: the runner user is in dialout, which covers serial but not raw USB. 22 tests cover the ladder, the elimination logic, both step-4 guards and the sysfs path parsing. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-recover.yml | 101 ++++++ dev/hil/recover.py | 563 ++++++++++++++++++++++++++++++ dev/tests/test_hil_recover.py | 317 +++++++++++++++++ 3 files changed, 981 insertions(+) create mode 100644 .github/workflows/hil-recover.yml create mode 100644 dev/hil/recover.py create mode 100644 dev/tests/test_hil_recover.py diff --git a/.github/workflows/hil-recover.yml b/.github/workflows/hil-recover.yml new file mode 100644 index 00000000..42462d51 --- /dev/null +++ b/.github/workflows/hil-recover.yml @@ -0,0 +1,101 @@ +name: HIL recover a wedged board + +# Gets a bench board that has stopped answering back into a usable state, +# without anyone walking over to it. +# +# A Vector board can deadlock with the USB device still enumerated and the +# firmware gone - the port is there, mpremote opens it, nothing ever answers. +# TrenchCoat names the mechanism in its own source: if nothing drains the +# board's output, the CDC buffers fill, MicroPython blocks writing to stdout, +# and the board deadlocks. That is what took the WPC board out mid-matrix. +# +# dev/hil/recover.py escalates cheapest-first (drain, USB reset, hub power +# cycle, then a UF2 reflash over the ROM bootloader) and stops as soon as the +# board answers. Run it before anything else when a HIL job reports a board +# that never responded. +# +# Deliberately no `pull_request` trigger - self-hosted runner, real hardware. +# The `push` trigger exists so this can be used before it reaches the default +# branch; drop it once this is on main. + +on: + workflow_dispatch: + inputs: + port: + description: "Recover only this port, e.g. /dev/ttyACM1 (blank = every board that is not answering)" + type: string + default: "" + target: + description: "Target for the dead board, e.g. wpc (blank = deduce it from the boards that still answer)" + type: string + default: "" + no_reflash: + description: "Stop before replacing the firmware" + type: boolean + default: false + force_bootsel: + description: "Touch the board into BOOTSEL even if nothing here can mount the drive to flash it" + type: boolean + default: false + push: + branches: + - claude/wpc-hil-config-validation-rc62dn + paths: + - dev/hil/recover.py + - .github/workflows/hil-recover.yml + +permissions: + contents: read + +# Shares the bench with the other HIL workflows. Recovery must never run +# alongside a job that is driving the boards. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + recover: + runs-on: [self-hosted, vector-hil] + timeout-minutes: 30 + + steps: + # Safe here only because every trigger above is repo-internal; see the + # note in hil-config-matrix.yml. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + + - name: Prepare environment + run: | + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + + - name: Recover + env: + HIL_PORT: ${{ inputs.port }} + HIL_TARGET: ${{ inputs.target }} + HIL_NO_REFLASH: ${{ inputs.no_reflash }} + HIL_FORCE_BOOTSEL: ${{ inputs.force_bootsel }} + run: | + args="" + if [ -n "${HIL_PORT:-}" ]; then args="$args --port $HIL_PORT"; fi + if [ -n "${HIL_TARGET:-}" ]; then args="$args --target $HIL_TARGET"; fi + if [ "${HIL_NO_REFLASH:-}" = "true" ]; then args="$args --no-reflash"; fi + if [ "${HIL_FORCE_BOOTSEL:-}" = "true" ]; then args="$args --force-bootsel"; fi + + # shellcheck disable=SC2086 # args is a deliberately word-split list + python dev/hil/recover.py $args + + - name: Board state afterwards + if: always() + run: | + # stty raw -echo first: a tty reverts to ECHO-on once every handle is + # closed, so a bare `cat` feeds the board its own output back. + echo "--- serial ports" + ls -l /dev/ttyACM* 2>/dev/null || echo "(none)" + echo "--- bootloader drives" + ls -l /dev/disk/by-label/ 2>/dev/null | grep -i rpi || echo "(none)" + for dev in /dev/ttyACM*; do + [ -e "$dev" ] || continue + echo "--- $dev" + stty -F "$dev" raw -echo 115200 || true + timeout 5 cat "$dev" || true + done diff --git a/dev/hil/recover.py b/dev/hil/recover.py new file mode 100644 index 00000000..06e2a08e --- /dev/null +++ b/dev/hil/recover.py @@ -0,0 +1,563 @@ +#!/usr/bin/env python3 +"""Bring a wedged bench board back, without anyone walking over to it. + + cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/recover.py + +A Vector board can deadlock in a way that leaves the USB device enumerated but +the firmware gone: the port is there, `mpremote` opens it, and nothing ever +answers. TrenchCoat names the mechanism in its own source (`src/ray.py`, +`send_command`): + + if nothing ever drains the board's output, the USB CDC buffers fill up, + MicroPython blocks writing to stdout, and the board deadlocks mid-script + +That is what happened to the WPC board on the bench: silent console, no reply +to Ctrl-C, for an hour. `dev/hil/config_matrix.py` no longer creates the +condition, but a board already in it needs getting out, and the bench is +supposed to run unattended. + +So this escalates, cheapest first, re-testing after each step: + + 1. drain - read whatever the board has queued and send Ctrl-C. If it is + blocked writing to a full CDC buffer, reading is the remedy. + 2. usb reset - ask the kernel to re-enumerate the device (USBDEVFS_RESET). + Resets the host side of the link and TinyUSB's endpoint + state, which can free a write that is stuck on a buffer the + host was not draining. + 3. power - cut and restore power to the board's hub port with uhubctl. + The board is USB bus powered (Trench-Coat-Install-Guide.md), + so this is a real power cycle, not a signal. Needs a hub that + supports per-port power switching. + 4. reflash - 1200 baud touch to drop the RP2040 into its ROM bootloader, + then copy a MicroPython UF2 onto the RPI-RP2 drive that + appears. The touch is handled in USB interrupt context rather + than by the Python VM, so it can work when everything above + has failed. Destructive: it replaces the firmware, and the + board needs `dev/flash.py` afterwards to get Vector back. + +Only step 4 needs anything from outside the repo - the UF2s come from +warped-pinball/trench-coat, pinned by commit and verified by checksum. + +If all four fail, the board needs a person: hold BOOTSEL while replugging it. +""" + +import argparse +import fcntl +import hashlib +import os +import shutil +import subprocess +import sys +import time +import urllib.request +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +import serial # noqa: E402 +from bench import ( # noqa: E402 + CTRL_C, + REPO_ROOT, + CheckFailure, + endgroup, + ensure_tools_on_path, + group, + list_ports, + log, + mpremote, + parse_board_map, +) + +# warped-pinball/trench-coat, pinned by commit. The checksums are what this +# revision ships; a mismatch means the pin moved under us and the file is not +# flashed. UF2s are large, so they are fetched rather than vendored here. +TRENCH_COAT_COMMIT = "26e6d508c362bed1f6d1323155435c18528de758" +TRENCH_COAT_RAW = f"https://raw.githubusercontent.com/warped-pinball/trench-coat/{TRENCH_COAT_COMMIT}/uf2" + +TARGET_UF2 = { + "wpc": ("Vector_WPC_v5.uf2", "3d02a60de852c11087f76ad61a1baf5921270c9a98ca9542a450aad26fac5191"), + "data_east": ("Vector_DataEast_v1.uf2", "11afc1d22f28099921e63950ba1e86832f47f2c558f8384ff04d9cf6650e7047"), + "sys11": ("vector_system_11_and_9_v4.uf2", "ba63972475f5126c1e5270c30b418510505f5859da9366eb0fac9ef35c9e7a15"), +} + +# ioctl number for USBDEVFS_RESET, from : _IO('U', 20). +USBDEVFS_RESET = ord("U") << 8 | 20 + +PROBE_TIMEOUT = 20 +SETTLE_SECONDS = 5 + + +def responsive(port, timeout=PROBE_TIMEOUT): + """Can we still get a REPL out of this board? + + Deliberately the cheapest question that distinguishes "wedged" from + "busy": a board running the Vector application answers this, because + mpremote interrupts it to do so. + """ + try: + result = mpremote("connect", port, "exec", "print('alive')", timeout=timeout) + except subprocess.TimeoutExpired: + return False + return result.returncode == 0 and "alive" in result.stdout + + +def survey(board_map): + """Report which ports answer, and work out what each dead one should be. + + A wedged board cannot tell us its chip id, so it cannot be looked up in + VECTOR_HIL_BOARD_MAP directly. What it can be is deduced: ask every board + that does answer who it is, and whatever targets the map still expects are + the dead ones. With one board down and the rest alive, that is exact. + """ + ports = list_ports() + if not ports: + raise CheckFailure("no boards found at all - check the USB hub and power") + + alive, dead, claimed = [], [], set() + log(f"{'port':16} {'state':14} chip id") + for port in ports: + if not responsive(port): + dead.append(port) + log(f"{port:16} {'NOT ANSWERING':14} -") + continue + alive.append(port) + try: + chip = mpremote("connect", port, "exec", "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", timeout=30) + chip_id = chip.stdout.strip() if chip.returncode == 0 else None + except subprocess.TimeoutExpired: + # Answered once and not the second time. Not our problem to solve + # here, but it does mean we cannot attribute a target to it. + chip_id = None + if chip_id in board_map: + claimed.add(board_map[chip_id]) + log(f"{port:16} {'ok':14} {chip_id or '?'}") + + unclaimed = sorted(set(board_map.values()) - claimed) + targets = {} + if len(dead) == 1 and len(unclaimed) == 1: + targets[dead[0]] = unclaimed[0] + log(f"\n{dead[0]} is the only board not answering and {unclaimed[0]} is the only target unaccounted for, so that is what it is") + elif dead: + log(f"\ncannot tell which target {', '.join(dead)} should be ({len(unclaimed)} unaccounted for: {', '.join(unclaimed) or 'none'})") + log("a reflash needs --target to say which UF2 to use") + + return alive, dead, targets + + +# -------------------------------------------------------------------------- +# 1. drain +# -------------------------------------------------------------------------- + + +def drain(port, seconds=5): + """Read whatever is queued and interrupt the board.""" + try: + connection = serial.Serial(port=port, baudrate=115200, timeout=1) + except Exception as exc: + log(f" could not open {port}: {exc}") + return False + + drained = 0 + try: + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + drained += len(connection.read(connection.in_waiting or 1)) + connection.write(CTRL_C + CTRL_C) + connection.flush() + except Exception as exc: + log(f" error draining {port}: {exc}") + finally: + try: + connection.close() + except Exception: + pass + + log(f" drained {drained} byte(s) and sent Ctrl-C") + # Nothing queued is itself the diagnosis: a board merely blocked on a full + # buffer has a backlog to give up the moment somebody reads. + if drained == 0: + log(" (nothing queued - so it is not simply blocked on a full output buffer)") + return True + + +# -------------------------------------------------------------------------- +# 2. usb reset +# -------------------------------------------------------------------------- + + +def usb_device_path(port): + """Map /dev/ttyACM* to its /dev/bus/usb/BBB/DDD node.""" + name = os.path.basename(port) + link = Path(f"/sys/class/tty/{name}/device") + try: + node = link.resolve() + except OSError as exc: + log(f" cannot resolve {link}: {exc}") + return None + + # The tty hangs off the CDC interface; the device is its parent. + for candidate in [node] + list(node.parents): + busnum = candidate / "busnum" + devnum = candidate / "devnum" + if busnum.exists() and devnum.exists(): + return Path(f"/dev/bus/usb/{int(busnum.read_text()):03d}/{int(devnum.read_text()):03d}") + log(f" could not find the USB device behind {port}") + return None + + +def usb_reset(port): + """Ask the kernel to re-enumerate the device behind `port`.""" + node = usb_device_path(port) + if node is None: + return False + log(f" resetting {node}") + try: + with open(node, "wb") as handle: + fcntl.ioctl(handle.fileno(), USBDEVFS_RESET, 0) + except PermissionError: + log(f" permission denied on {node} - the runner user needs write access to it (a udev rule, or the plugdev group)") + return False + except OSError as exc: + log(f" ioctl failed: {exc}") + return False + return True + + +# -------------------------------------------------------------------------- +# 3. power cycle +# -------------------------------------------------------------------------- + + +def hub_location(port): + """Return (hub, port number) for uhubctl, from the device's sysfs path. + + A USB path looks like 1-1.4:1.0 - bus 1, hub at 1-1, port 4. uhubctl wants + the hub and the port separately. + """ + name = os.path.basename(port) + try: + node = Path(f"/sys/class/tty/{name}/device").resolve() + except OSError: + return None + + for candidate in [node] + list(node.parents): + if (candidate / "busnum").exists(): + usb_path = candidate.name.split(":")[0] + if "." in usb_path: + hub, _, portnum = usb_path.rpartition(".") + return hub, portnum + bus, _, portnum = usb_path.partition("-") + return bus, portnum + return None + + +def power_cycle(port, off_seconds=3): + """Cut and restore power to the board's hub port. + + The board is powered off the USB cable, so this is the real thing - a cold + boot, with no dependence on the firmware being alive to cooperate. Needs a + hub with per-port power switching; plenty do not have it, which is why this + reports rather than fails. + """ + if not shutil.which("uhubctl"): + log(" uhubctl is not installed - skipping the power cycle (sudo apt install uhubctl)") + return False + + location = hub_location(port) + if location is None: + log(f" could not work out which hub port {port} is on") + return False + hub, portnum = location + + log(f" power cycling hub {hub} port {portnum}") + for action in ("off", "on"): + result = subprocess.run(["uhubctl", "--location", hub, "--ports", portnum, "--action", action], capture_output=True, text=True, timeout=60) + if result.returncode != 0: + detail = (result.stderr or result.stdout).strip() + log(f" uhubctl {action} failed: {detail}") + if "No compatible" in detail or "not support" in detail: + log(" this hub cannot switch port power - a smart hub is the only way to make this step work") + return False + if action == "off": + time.sleep(off_seconds) + return True + + +# -------------------------------------------------------------------------- +# 4. reflash over the ROM bootloader +# -------------------------------------------------------------------------- + + +def bootsel_touch(port): + """Open the port at 1200 baud to drop the RP2040 into its ROM bootloader. + + The last resort that does not need a person, and the reason it can work + when the REPL cannot: the 1200 baud touch is a CDC line-coding change, + handled in USB interrupt context, so a blocked Python VM does not stop it. + TrenchCoat's own enter_bootloader_mode() goes through `machine.bootloader()` + on the REPL instead, which a wedged board will never run. + """ + try: + connection = serial.Serial(port=port, baudrate=1200) + connection.dtr = False + time.sleep(0.5) + connection.close() + except Exception as exc: + # The port vanishing underneath us IS the board rebooting into the + # bootloader, so this is as often success as failure. + log(f" port closed during the 1200 baud touch ({exc}) - which is what a reboot looks like") + return True + + +def find_bootloader_drive(timeout=30): + """Wait for an RPI-RP2 drive to appear, the way TrenchCoat looks for it.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for root in ("/media", "/run/media", "/mnt"): + base = Path(root) + if not base.is_dir(): + continue + try: + for path in base.rglob("INFO_UF2.TXT"): + return path.parent + except OSError: + continue + time.sleep(1) + return None + + +def mount_bootloader_drive(timeout=30): + """Mount the RPI-RP2 volume ourselves when nothing automounts it. + + A headless runner has no desktop automounter, so the drive that appears + after the touch is a block device and nothing more. udisksctl goes through + polkit rather than sudo, which is the one route a service user might + actually have. + """ + if not shutil.which("udisksctl"): + log(" udisksctl is not available, so the bootloader drive cannot be mounted here") + return None + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + for link in sorted(Path("/dev/disk/by-label").glob("RPI-RP2*")) if Path("/dev/disk/by-label").is_dir() else []: + device = link.resolve() + log(f" mounting {device} with udisksctl") + result = subprocess.run(["udisksctl", "mount", "-b", str(device)], capture_output=True, text=True, timeout=60) + if result.returncode == 0: + # "Mounted /dev/sda1 at /media/xxx" + mounted = result.stdout.strip().rsplit(" at ", 1)[-1].rstrip(".") + log(f" mounted at {mounted}") + return Path(mounted) + log(f" udisksctl could not mount it: {(result.stderr or result.stdout).strip()}") + return None + time.sleep(1) + return None + + +def can_complete_a_reflash(): + """Is there any way this runner could write a UF2 once the board is in BOOTSEL? + + Asked *before* the 1200 baud touch, because the touch is a one-way door: it + takes a board that is at least enumerated as a serial device and turns it + into a mass-storage device that only a UF2 (or a replug) gets it out of. + Doing that with no way to finish the job makes the board harder to recover, + not easier. + """ + if find_bootloader_drive(timeout=0) is not None: + return True, "a bootloader drive is already mounted" + if shutil.which("udisksctl"): + return True, "udisksctl is available to mount the drive" + if any(Path(root).is_dir() and os.access(root, os.W_OK) for root in ("/media", "/run/media")): + return True, "an automount directory is writable" + return False, "nothing here can mount an RPI-RP2 drive (no udisksctl, no writable automount directory)" + + +def fetch_uf2(target, cache_dir): + """Download the pinned UF2 for `target` and verify it before use.""" + if target not in TARGET_UF2: + raise CheckFailure(f"no UF2 known for target {target!r} (have: {', '.join(sorted(TARGET_UF2))})") + filename, expected = TARGET_UF2[target] + + cache_dir.mkdir(parents=True, exist_ok=True) + path = cache_dir / filename + if not path.exists(): + url = f"{TRENCH_COAT_RAW}/{filename}" + log(f" downloading {filename} from trench-coat@{TRENCH_COAT_COMMIT[:8]}") + request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) + with urllib.request.urlopen(request, timeout=120) as response: + path.write_bytes(response.read()) + + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != expected: + path.unlink(missing_ok=True) + raise CheckFailure(f"{filename} does not match the checksum pinned for trench-coat@{TRENCH_COAT_COMMIT[:8]} (got {digest}) - refusing to flash it") + log(f" {filename} verified ({path.stat().st_size} bytes)") + return path + + +def reflash(port, target, cache_dir, force=False): + """1200 baud touch, then drop a UF2 on the drive that appears.""" + possible, why = can_complete_a_reflash() + log(f" {'can' if possible else 'cannot'} finish a reflash here: {why}") + if not possible and not force: + log(" not touching the board into BOOTSEL, because that would leave it as a mass-storage") + log(" device with no way to flash it - strictly worse than how it is now. Install udisksctl") + log(" on the runner (or pass --force-bootsel) to make this step usable.") + return False + + # Fetch and verify before the point of no return, so a bad download cannot + # strand the board in BOOTSEL. + uf2 = fetch_uf2(target, cache_dir) + + bootsel_touch(port) + + drive = find_bootloader_drive() or mount_bootloader_drive() + if drive is None: + log(" no RPI-RP2 drive appeared, so either the board did not reach its ROM bootloader") + log(" or nothing mounted the drive it presented") + return False + log(f" board is in bootloader mode at {drive}") + + log(f" copying {uf2.name} to {drive}") + shutil.copy(uf2, drive) + # The board reboots as soon as the copy lands, taking the drive with it. + time.sleep(10) + return True + + +# -------------------------------------------------------------------------- + + +def recover(port, target, args): + """Escalate until the board answers, or until we run out of ideas.""" + steps = [("drain the console", lambda: drain(port)), ("reset the USB device", lambda: usb_reset(port))] + if not args.no_power_cycle: + steps.append(("power cycle the hub port", lambda: power_cycle(port))) + if args.reflash: + if target: + steps.append(("reflash MicroPython over the ROM bootloader", lambda: reflash(port, target, args.cache_dir, args.force_bootsel))) + else: + log(f"::warning::{port}: skipping the reflash step - no target known for this board, pass --target") + + for name, step in steps: + group(f"{port}: {name}") + try: + attempted = step() + except CheckFailure as exc: + log(f"::error::{exc}") + attempted = False + except Exception as exc: # noqa: BLE001 - a failed recovery step is not a crash + log(f"::error::unexpected error: {exc}") + attempted = False + + if attempted: + time.sleep(SETTLE_SECONDS) + if responsive(port): + log(f" {port} is answering again") + endgroup() + return name + log(f" {port} still not answering") + endgroup() + + return None + + +def preflight(): + """Report which recovery steps are actually available here. + + A recovery that fails is still worth the run if it says precisely which + one-time bit of runner setup would have made the bench self-healing. Each + line below is a step that either works or names what it needs. + """ + ports = [] + try: + ports = list_ports() + except CheckFailure: + pass + + log(f" serial ok {len(ports)} port(s) visible") + + node = usb_device_path(ports[0]) if ports else None + if node is None: + log(" usb reset unknown no device to check") + elif os.access(node, os.W_OK): + log(f" usb reset ok {node} is writable") + else: + log(f" usb reset no {node} is not writable - needs a udev rule granting the runner user write access") + + if shutil.which("uhubctl"): + log(" power ok uhubctl is installed (still needs a hub that switches port power)") + else: + log(" power no uhubctl not installed - `sudo apt install uhubctl`") + + possible, why = can_complete_a_reflash() + log(f" reflash {'ok ' if possible else 'no '} {why}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--port", action="append", help="recover only this port (repeatable); default is every board that is not answering") + parser.add_argument("--target", help="target for the dead board, e.g. wpc - only needed when it cannot be deduced") + parser.add_argument("--no-reflash", dest="reflash", action="store_false", help="stop before replacing the firmware; the board is left as found if the cheaper steps fail") + parser.add_argument("--no-power-cycle", action="store_true", help="skip the uhubctl step") + parser.add_argument("--force-bootsel", action="store_true", help="touch the board into BOOTSEL even when nothing here can mount the drive to flash it") + parser.add_argument("--cache-dir", type=Path, default=REPO_ROOT / "build" / "uf2", help="where to keep downloaded UF2s") + args = parser.parse_args() + + ensure_tools_on_path() + + group("What this runner can do") + preflight() + endgroup() + + group("Survey") + board_map = parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")) + alive, dead, targets = survey(board_map) + endgroup() + + if args.port: + dead = [port for port in args.port] + log(f"recovering {', '.join(dead)} because --port says so") + if not dead: + log(f"\nall {len(alive)} board(s) are answering - nothing to recover") + return 0 + + recovered, lost = [], [] + for port in dead: + target = args.target or targets.get(port) + log("") + log(f"recovering {port}" + (f" (expected to be {target})" if target else " (target unknown)")) + method = recover(port, target, args) + if method: + recovered.append((port, method)) + else: + lost.append(port) + + log("") + log("=" * 60) + for port, method in recovered: + log(f" recovered {port:16} by: {method}") + for port in lost: + log(f" STILL DEAD {port}") + log("=" * 60) + + if lost: + log("") + log(f"{len(lost)} board(s) need a person at the bench:") + log(" hold the BOOTSEL button while replugging the USB cable, then run") + log(" dev/hil/flash_and_check.py to put Vector back on it.") + return 1 + + log("") + log("Every board answers again. A board recovered by the reflash step is running") + log("bare MicroPython, so run dev/hil/flash_and_check.py before trusting the bench.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except CheckFailure as exc: + log(f"::error::{exc}") + sys.exit(1) diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py new file mode 100644 index 00000000..2c1bf742 --- /dev/null +++ b/dev/tests/test_hil_recover.py @@ -0,0 +1,317 @@ +"""Tests for the bench recovery ladder, with the hardware faked out. + +The escalation order is the whole design - cheapest and least destructive +first, stopping the moment the board answers - so that is what these pin down, +along with the checksum gate on anything that gets flashed. +""" + +from __future__ import annotations + +import subprocess +import sys +import types +from argparse import Namespace +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +sys.modules.setdefault("serial", types.ModuleType("serial")) +if "usb_coms_demo" not in sys.modules: + stub = types.ModuleType("usb_coms_demo") + stub.UsbApiClient = object + sys.modules["usb_coms_demo"] = stub + +sys.path.insert(0, str(REPO_ROOT / "dev" / "hil")) + +import bench # noqa: E402 +import recover # noqa: E402 + + +def args(**overrides): + defaults = {"reflash": True, "no_power_cycle": False, "force_bootsel": False, "cache_dir": Path("/nonexistent")} + defaults.update(overrides) + return Namespace(**defaults) + + +@pytest.fixture() +def ladder(monkeypatch): + """Record which recovery steps ran, and let a test choose which one works.""" + calls = [] + + def step(name, works_at=None): + def run(*_args, **_kwargs): + calls.append(name) + return True + + return run + + monkeypatch.setattr(recover, "drain", step("drain")) + monkeypatch.setattr(recover, "usb_reset", step("usb_reset")) + monkeypatch.setattr(recover, "power_cycle", step("power_cycle")) + monkeypatch.setattr(recover, "reflash", step("reflash")) + monkeypatch.setattr(recover, "SETTLE_SECONDS", 0) + return calls + + +def recovers_after(monkeypatch, calls, step_count): + """Make the board answer once `step_count` steps have run.""" + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: len(calls) >= step_count) + + +def test_the_cheapest_step_that_works_ends_the_ladder(monkeypatch, ladder): + recovers_after(monkeypatch, ladder, 1) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain"] + assert "drain" in method + + +def test_it_escalates_only_as_far_as_it_has_to(monkeypatch, ladder): + recovers_after(monkeypatch, ladder, 3) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain", "usb_reset", "power_cycle"] + assert "power cycle" in method + + +def test_reflashing_is_the_last_resort_and_not_before(monkeypatch, ladder): + recovers_after(monkeypatch, ladder, 4) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain", "usb_reset", "power_cycle", "reflash"] + assert "reflash" in method + + +def test_a_board_that_never_answers_reports_defeat(monkeypatch, ladder): + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: False) + + assert recover.recover("/dev/ttyFAKE", "wpc", args()) is None + assert ladder == ["drain", "usb_reset", "power_cycle", "reflash"] + + +def test_no_reflash_leaves_the_firmware_alone(monkeypatch, ladder): + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: False) + + recover.recover("/dev/ttyFAKE", "wpc", args(reflash=False)) + + assert "reflash" not in ladder + + +def test_reflash_is_skipped_when_the_target_is_unknown(monkeypatch, ladder, capsys): + # Flashing the wrong system's UF2 is worse than leaving the board dead. + monkeypatch.setattr(recover, "responsive", lambda _port, timeout=None: False) + + recover.recover("/dev/ttyFAKE", None, args()) + + assert "reflash" not in ladder + assert "no target known" in capsys.readouterr().out + + +def test_a_failing_step_does_not_stop_the_ladder(monkeypatch, ladder): + def explode(*_args, **_kwargs): + ladder.append("usb_reset") + raise OSError("ioctl went wrong") + + monkeypatch.setattr(recover, "usb_reset", explode) + recovers_after(monkeypatch, ladder, 3) + + method = recover.recover("/dev/ttyFAKE", "wpc", args()) + + assert ladder == ["drain", "usb_reset", "power_cycle"] + assert method is not None + + +# -------------------------------------------------------------------------- +# survey +# -------------------------------------------------------------------------- + + +def fake_survey(monkeypatch, ports, dead_ports, chip_ids): + monkeypatch.setattr(recover, "list_ports", lambda: ports) + monkeypatch.setattr(recover, "responsive", lambda port, timeout=None: port not in dead_ports) + + def fake_mpremote(*call, timeout=None): + port = call[1] + return types.SimpleNamespace(returncode=0, stdout=chip_ids[port], stderr="") + + monkeypatch.setattr(recover, "mpremote", fake_mpremote) + + +def test_survey_deduces_the_dead_board_by_elimination(monkeypatch): + """A wedged board cannot say what it is, but the others can say what it is not.""" + fake_survey( + monkeypatch, + ports=["/dev/ttyACM0", "/dev/ttyACM1", "/dev/ttyACM2"], + dead_ports={"/dev/ttyACM1"}, + chip_ids={"/dev/ttyACM0": "aaa", "/dev/ttyACM2": "ccc"}, + ) + board_map = {"aaa": "sys11", "bbb": "wpc", "ccc": "data_east"} + + alive, dead, targets = recover.survey(board_map) + + assert alive == ["/dev/ttyACM0", "/dev/ttyACM2"] + assert dead == ["/dev/ttyACM1"] + assert targets == {"/dev/ttyACM1": "wpc"} + + +def test_survey_will_not_guess_when_two_boards_are_down(monkeypatch): + fake_survey( + monkeypatch, + ports=["/dev/ttyACM0", "/dev/ttyACM1", "/dev/ttyACM2"], + dead_ports={"/dev/ttyACM1", "/dev/ttyACM2"}, + chip_ids={"/dev/ttyACM0": "aaa"}, + ) + board_map = {"aaa": "sys11", "bbb": "wpc", "ccc": "data_east"} + + _alive, dead, targets = recover.survey(board_map) + + assert sorted(dead) == ["/dev/ttyACM1", "/dev/ttyACM2"] + assert targets == {} + + +def test_survey_fails_when_nothing_is_attached(monkeypatch): + monkeypatch.setattr(recover, "list_ports", lambda: []) + + with pytest.raises(bench.CheckFailure, match="no boards found"): + recover.survey({}) + + +def test_responsive_is_false_when_mpremote_times_out(monkeypatch): + def times_out(*_args, **_kwargs): + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=20) + + monkeypatch.setattr(recover, "mpremote", times_out) + + assert recover.responsive("/dev/ttyFAKE") is False + + +# -------------------------------------------------------------------------- +# the UF2 gate +# -------------------------------------------------------------------------- + + +def test_fetch_uf2_refuses_a_file_that_does_not_match_the_pin(tmp_path): + filename, _digest = recover.TARGET_UF2["wpc"] + (tmp_path / filename).write_bytes(b"not the firmware you are looking for") + + with pytest.raises(bench.CheckFailure, match="refusing to flash it"): + recover.fetch_uf2("wpc", tmp_path) + + # And it does not leave the bad file behind to be picked up next time. + assert not (tmp_path / filename).exists() + + +def test_fetch_uf2_accepts_the_pinned_file(tmp_path, monkeypatch): + import hashlib + + payload = b"pretend UF2" + filename, _digest = recover.TARGET_UF2["wpc"] + monkeypatch.setitem(recover.TARGET_UF2, "wpc", (filename, hashlib.sha256(payload).hexdigest())) + (tmp_path / filename).write_bytes(payload) + + assert recover.fetch_uf2("wpc", tmp_path) == tmp_path / filename + + +def test_fetch_uf2_rejects_a_target_it_has_no_firmware_for(tmp_path): + with pytest.raises(bench.CheckFailure, match="no UF2 known"): + recover.fetch_uf2("whitestar", tmp_path) + + +def test_every_target_uf2_pin_is_a_sha256(): + for target, (filename, digest) in recover.TARGET_UF2.items(): + assert filename.endswith(".uf2"), target + assert len(digest) == 64 and set(digest) <= set("0123456789abcdef"), target + + +# -------------------------------------------------------------------------- +# sysfs plumbing +# -------------------------------------------------------------------------- + + +def test_usb_device_path_walks_up_to_the_device(tmp_path, monkeypatch): + device = tmp_path / "sys" / "devices" / "usb1" / "1-1.4" + interface = device / "1-1.4:1.0" / "tty" / "ttyACM1" + interface.mkdir(parents=True) + (device / "busnum").write_text("1\n") + (device / "devnum").write_text("7\n") + + link = tmp_path / "sys" / "class" / "tty" / "ttyACM1" / "device" + link.parent.mkdir(parents=True) + link.symlink_to(device / "1-1.4:1.0") + + monkeypatch.setattr(recover, "Path", lambda p: link if str(p).endswith("/device") else Path(p)) + + assert recover.usb_device_path("/dev/ttyACM1") == Path("/dev/bus/usb/001/007") + + +@pytest.mark.parametrize( + "usb_path, expected", + [ + ("1-1.4:1.0", ("1-1", "4")), + ("1-1.2.3:1.0", ("1-1.2", "3")), + ("2-3:1.0", ("2", "3")), + ], +) +def test_hub_location_splits_the_usb_path(tmp_path, monkeypatch, usb_path, expected): + device = tmp_path / usb_path + device.mkdir(parents=True) + (device / "busnum").write_text("1\n") + + link = tmp_path / "link" + link.symlink_to(device) + monkeypatch.setattr(recover, "Path", lambda p: link if str(p).endswith("/device") else Path(p)) + + assert recover.hub_location("/dev/ttyACM1") == expected + + +def test_reflash_will_not_strand_a_board_in_bootsel_it_cannot_flash(monkeypatch, tmp_path, capsys): + """The touch is a one-way door. + + A wedged board is at least still a serial device. Touching it into BOOTSEL + with no way to write a UF2 turns it into a mass-storage device that only a + replug gets out of - strictly worse than how it was found. + """ + touched = [] + monkeypatch.setattr(recover, "bootsel_touch", lambda port: touched.append(port)) + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) is False + assert touched == [] + assert "not touching the board into BOOTSEL" in capsys.readouterr().out + + +def test_reflash_verifies_the_uf2_before_the_point_of_no_return(monkeypatch, tmp_path): + """Download and checksum first: a bad fetch must not cost us the board.""" + touched = [] + monkeypatch.setattr(recover, "bootsel_touch", lambda port: touched.append(port)) + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (True, "udisksctl is available")) + + filename, _digest = recover.TARGET_UF2["wpc"] + (tmp_path / filename).write_bytes(b"corrupted download") + + with pytest.raises(bench.CheckFailure, match="refusing to flash it"): + recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) + assert touched == [] + + +def test_force_bootsel_overrides_the_guard(monkeypatch, tmp_path): + import hashlib + + payload = b"pretend UF2" + filename, _digest = recover.TARGET_UF2["wpc"] + monkeypatch.setitem(recover.TARGET_UF2, "wpc", (filename, hashlib.sha256(payload).hexdigest())) + (tmp_path / filename).write_bytes(payload) + + touched = [] + monkeypatch.setattr(recover, "bootsel_touch", lambda port: touched.append(port)) + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) + monkeypatch.setattr(recover, "find_bootloader_drive", lambda *a, **k: None) + monkeypatch.setattr(recover, "mount_bootloader_drive", lambda *a, **k: None) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path, force=True) is False + assert touched == ["/dev/ttyFAKE"] From aff4695dd55488184c194ef2d5fa5310681ce99a Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 05:11:42 +0000 Subject: [PATCH 04/32] fix(hil): never block forever writing to a board that stopped listening The recovery run hung. Fourteen minutes into a step that should take one, with no output, until the job timeout killed it - and the cause is our side of the same failure the board has. `serial.Serial(timeout=...)` sets the READ timeout only. Every connection the harness opened had no `write_timeout`, so a write to a board that has stopped draining its USB OUT endpoint blocks forever. That is exactly the state a wedged board is in, and exactly the board the recovery tool has to write to: it drained the console fine, then blocked on the Ctrl-C it sends afterwards. `flush()` is worse. It is termios tcdrain, it waits for the kernel's output buffer to reach the device, and it takes no timeout at all - so it hangs on the same board even where the write did not. Every call site is gone: handing bytes to the kernel is enough, and every exchange here is already synchronised by a read with a deadline. Both are now impossible to reintroduce by accident. open_serial() is the only way a port gets opened and always sets both timeouts; serial_write() turns a deaf board into a CheckFailure naming the cause; a test asserts bench.py contains no .flush() at all. This was not only the recovery tool's problem - config_matrix's nudge() would have hung the matrix the same way, on the same board. Added bench.time_limit as a backstop: a SIGALRM ceiling around each recovery step (180s) and each config in the matrix (240s). Every call in here is meant to be bounded, but a board in a bad enough state can block a syscall no library timeout covers, and one board must never hang a bench job again. SIGALRM interrupts the syscall, so it catches what the individual timeouts miss. Also removed the config matrix's push trigger for now. It shares the hil-bench concurrency group with the recovery workflow, and GitHub keeps only one pending run per group, so a push touching both would race and silently cancel one. While the bench needs recovering, recovery gets the queue; the workflow says how to put it back. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-config-matrix.yml | 19 +++--- dev/hil/bench.py | 86 +++++++++++++++++++++---- dev/hil/config_matrix.py | 27 +++++--- dev/hil/recover.py | 25 +++++-- dev/tests/test_hil_config_matrix.py | 74 ++++++++++++++++++++- dev/tests/test_hil_recover.py | 2 +- 6 files changed, 193 insertions(+), 40 deletions(-) diff --git a/.github/workflows/hil-config-matrix.yml b/.github/workflows/hil-config-matrix.yml index 83508dab..d24d6184 100644 --- a/.github/workflows/hil-config-matrix.yml +++ b/.github/workflows/hil-config-matrix.yml @@ -14,9 +14,15 @@ name: HIL config matrix # a private network and physically reflashes hardware. Fork gating is designed # in dev/hil/DESIGN.md §4 and is not built yet. # -# The `push` trigger exists so this can be validated before merging, since a -# workflow_dispatch workflow is not dispatchable until it reaches the default -# branch. Drop the push trigger once this is on main. +# The push trigger that let this be validated pre-merge is currently removed on +# purpose. It shares the `hil-bench` concurrency group with hil-recover.yml, and +# GitHub keeps only one *pending* run per group - so a push touching both would +# have them race, with the loser silently cancelled. While the bench needs +# recovering, recovery gets the queue. Re-add this to run the matrix again: +# +# push: +# branches: [claude/wpc-hil-config-validation-rc62dn] +# paths: [dev/hil/bench.py, dev/hil/config_matrix.py, .github/workflows/hil-config-matrix.yml] on: workflow_dispatch: @@ -41,13 +47,6 @@ on: description: "Stop a board's matrix at its first failing config" type: boolean default: false - push: - branches: - - claude/wpc-hil-config-validation-rc62dn - paths: - - dev/hil/bench.py - - dev/hil/config_matrix.py - - .github/workflows/hil-config-matrix.yml permissions: contents: read diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 08901ed6..a949011f 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -19,6 +19,7 @@ import os import re import shutil +import signal import subprocess import sys import time @@ -418,7 +419,7 @@ def wait_for_server(port, timeout=BOOT_TIMEOUT): try: # The port disappears and re-enumerates across the reset, so a # failure to open here is expected for the first second or two. - connection = serial.Serial(port=port, baudrate=115200, timeout=1) + connection = open_serial(port) except Exception: time.sleep(1) continue @@ -471,7 +472,6 @@ def prime_usb(connection): connection.reset_input_buffer() connection.reset_output_buffer() connection.write(b"\n") - connection.flush() except Exception as exc: log(f" warning: could not prime the USB link: {exc}") return @@ -529,6 +529,71 @@ def _dump_boot_log(board, lines=25): log(f" {line}") +# -------------------------------------------------------------------------- +# talking to a board that might be wedged +# -------------------------------------------------------------------------- +# +# Two traps, both of which cost a bench run before they were understood, and +# both the same shape as the board's own failure: blocking on a buffer that +# nothing is draining. +# +# * `serial.Serial(timeout=...)` sets the READ timeout only. With no +# `write_timeout` a write to a board that has stopped reading its OUT +# endpoint blocks forever - which is exactly the state a wedged board is +# in, and exactly when the recovery tool needs to write to it. +# * `flush()` is termios tcdrain. It waits for the kernel's output buffer to +# reach the device and takes no timeout at all, so it hangs on the same +# board even when the write did not. We never call it: handing the bytes +# to the kernel is enough, and every exchange here is synchronised by a +# read with a deadline rather than by tcdrain. + +SERIAL_READ_TIMEOUT = 1 +SERIAL_WRITE_TIMEOUT = 5 + + +def open_serial(port, baudrate=115200, read_timeout=SERIAL_READ_TIMEOUT, write_timeout=SERIAL_WRITE_TIMEOUT): + """Open a board's port with BOTH timeouts set. Always use this.""" + return serial.Serial(port=port, baudrate=baudrate, timeout=read_timeout, write_timeout=write_timeout) + + +def serial_write(connection, data, what="the board"): + """Write to a board, refusing to wait forever if it has stopped listening.""" + try: + connection.write(data) + except serial.SerialTimeoutException: + raise CheckFailure(f"timed out writing to {what} after {SERIAL_WRITE_TIMEOUT}s - it has stopped draining its USB endpoint") + + +class time_limit: + """Hard ceiling on a block of work, however deep it blocks. + + A backstop rather than a design: every call in here is supposed to be + bounded, but a board in a bad enough state can block a syscall that no + library timeout covers, and one board must never be able to hang a bench + job. SIGALRM interrupts the syscall, so this catches cases the individual + timeouts miss. + + Main thread and POSIX only, which is what the bench is. + """ + + def __init__(self, seconds, what): + self.seconds = int(seconds) + self.what = what + + def _expired(self, _signum, _frame): + raise CheckFailure(f"{self.what} did not finish within {self.seconds}s and was interrupted") + + def __enter__(self): + self.previous = signal.signal(signal.SIGALRM, self._expired) + signal.alarm(self.seconds) + return self + + def __exit__(self, *_exc): + signal.alarm(0) + signal.signal(signal.SIGALRM, self.previous) + return False + + # -------------------------------------------------------------------------- # raw REPL, over a connection we already hold # -------------------------------------------------------------------------- @@ -602,12 +667,10 @@ def enter(self, timeout=REPL_TIMEOUT): """ self._pending.clear() self.connection.reset_input_buffer() - self.connection.write(CTRL_C + CTRL_C) - self.connection.flush() + serial_write(self.connection, CTRL_C + CTRL_C, "the board's REPL") time.sleep(0.2) - self.connection.write(CTRL_A) - self.connection.flush() + serial_write(self.connection, CTRL_A, "the board's REPL") self.read_until(RAW_REPL_BANNER, timeout, "the raw REPL prompt") self.read_until(b">", timeout, "the raw REPL prompt") return self @@ -619,8 +682,7 @@ def exec(self, code, timeout=REPL_TIMEOUT): `OK`, then stdout, then \x04, then the traceback (empty on success), then \x04. """ - self.connection.write(code.encode() + CTRL_D) - self.connection.flush() + serial_write(self.connection, code.encode() + CTRL_D, "the board's REPL") self.read_until(b"OK", timeout, "the board to accept the snippet") output = self.read_until(CTRL_D, timeout, "the snippet to finish") @@ -639,8 +701,7 @@ def reset(self): wait_for_server(). """ try: - self.connection.write(b"import machine; machine.reset()" + CTRL_D) - self.connection.flush() + serial_write(self.connection, b"import machine; machine.reset()" + CTRL_D, "the board's REPL") except Exception as exc: raise CheckFailure(f"could not issue a reset over the REPL: {exc}") # Let the write reach the board before the port disappears. @@ -663,7 +724,7 @@ def drain_port(port, seconds=3): board that yields a backlog are different problems. """ try: - connection = serial.Serial(port=port, baudrate=115200, timeout=1) + connection = open_serial(port) except Exception as exc: log(f" could not reopen {port} to unstick it: {exc}") return 0 @@ -673,8 +734,7 @@ def drain_port(port, seconds=3): deadline = time.monotonic() + seconds while time.monotonic() < deadline: drained += len(connection.read(connection.in_waiting or 1)) - connection.write(CTRL_C) - connection.flush() + serial_write(connection, CTRL_C, port) except Exception as exc: log(f" error while draining {port}: {exc}") finally: diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index c7846298..ea1c0c49 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -73,6 +73,7 @@ repl_reset, resolve_targets, set_game_config, + time_limit, wait_for_server, ) @@ -85,6 +86,12 @@ # 150s default only buys a wedged board more time to waste. MATRIX_BOOT_TIMEOUT = 90 +# Hard ceiling on one config, as a backstop to the individual timeouts inside +# it. A healthy config takes ~21s; anything past this is a board that has +# stopped behaving, and it must not be allowed to hang the job. See +# bench.time_limit. +CONFIG_TIMEOUT = 240 + def source_configs(target): """{config filename without .json: {"name": ..., "adjustments": bool}}. @@ -380,15 +387,16 @@ def run_matrix(board, args): started = time.monotonic() group(f"[{index}/{len(names)}] {target} {config}") try: - # Set the config on the board we are already talking to, then - # reboot into it. The connection dies with the reset; the next - # wait_for_boot opens a fresh one. - session.set_config(config) - session.reboot() - client = session.wait_for_boot() - consecutive_setup_failures = 0 - - name = check_booted_config(client, target, config, configs[config]) + with time_limit(args.config_timeout, f"{target} {config}"): + # Set the config on the board we are already talking to, + # then reboot into it. The connection dies with the reset; + # the next wait_for_boot opens a fresh one. + session.set_config(config) + session.reboot() + client = session.wait_for_boot() + consecutive_setup_failures = 0 + + name = check_booted_config(client, target, config, configs[config]) log(f" ok {config:20} -> {name!r} [{time.monotonic() - started:.1f}s]") passed.append(config) except CheckFailure as exc: @@ -461,6 +469,7 @@ def main(): # A healthy boot answers in 12-16s on the bench, so the flash harness's # 150s is generous here and only makes a dead board expensive. parser.add_argument("--boot-timeout", type=int, default=MATRIX_BOOT_TIMEOUT, help=f"seconds to wait for a board's web server after a reset (default {MATRIX_BOOT_TIMEOUT})") + parser.add_argument("--config-timeout", type=int, default=CONFIG_TIMEOUT, help=f"hard ceiling on one config, in seconds (default {CONFIG_TIMEOUT})") args = parser.parse_args() bench.ensure_tools_on_path() diff --git a/dev/hil/recover.py b/dev/hil/recover.py index 06e2a08e..5325d9cf 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -54,7 +54,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) -import serial # noqa: E402 from bench import ( # noqa: E402 CTRL_C, REPO_ROOT, @@ -65,7 +64,10 @@ list_ports, log, mpremote, + open_serial, parse_board_map, + serial_write, + time_limit, ) # warped-pinball/trench-coat, pinned by commit. The checksums are what this @@ -86,6 +88,11 @@ PROBE_TIMEOUT = 20 SETTLE_SECONDS = 5 +# No single step may hang the job. Generous enough for the slowest one (fetch a +# 1.7MB UF2, wait for a drive, copy it) and still far short of the workflow's +# own timeout, so the run always gets to print its summary. +STEP_TIMEOUT = 180 + def responsive(port, timeout=PROBE_TIMEOUT): """Can we still get a REPL out of this board? @@ -152,7 +159,7 @@ def survey(board_map): def drain(port, seconds=5): """Read whatever is queued and interrupt the board.""" try: - connection = serial.Serial(port=port, baudrate=115200, timeout=1) + connection = open_serial(port) except Exception as exc: log(f" could not open {port}: {exc}") return False @@ -162,8 +169,12 @@ def drain(port, seconds=5): deadline = time.monotonic() + seconds while time.monotonic() < deadline: drained += len(connection.read(connection.in_waiting or 1)) - connection.write(CTRL_C + CTRL_C) - connection.flush() + serial_write(connection, CTRL_C + CTRL_C, port) + except CheckFailure as exc: + # Expected against a truly wedged board, and worth saying out loud: + # a board that will not accept a Ctrl-C is not going to be talked + # back to life, so the next step up the ladder is the real hope. + log(f" {exc}") except Exception as exc: log(f" error draining {port}: {exc}") finally: @@ -298,7 +309,7 @@ def bootsel_touch(port): on the REPL instead, which a wedged board will never run. """ try: - connection = serial.Serial(port=port, baudrate=1200) + connection = open_serial(port, baudrate=1200) connection.dtr = False time.sleep(0.5) connection.close() @@ -443,7 +454,8 @@ def recover(port, target, args): for name, step in steps: group(f"{port}: {name}") try: - attempted = step() + with time_limit(args.step_timeout, name): + attempted = step() except CheckFailure as exc: log(f"::error::{exc}") attempted = False @@ -502,6 +514,7 @@ def main(): parser.add_argument("--no-reflash", dest="reflash", action="store_false", help="stop before replacing the firmware; the board is left as found if the cheaper steps fail") parser.add_argument("--no-power-cycle", action="store_true", help="skip the uhubctl step") parser.add_argument("--force-bootsel", action="store_true", help="touch the board into BOOTSEL even when nothing here can mount the drive to flash it") + parser.add_argument("--step-timeout", type=int, default=STEP_TIMEOUT, help=f"hard ceiling on any one recovery step, in seconds (default {STEP_TIMEOUT})") parser.add_argument("--cache-dir", type=Path, default=REPO_ROOT / "build" / "uf2", help="where to keep downloaded UF2s") args = parser.parse_args() diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 3fe1f2b6..1747a7c4 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -10,6 +10,7 @@ import json import sys +import time import types from argparse import Namespace from pathlib import Path @@ -348,7 +349,7 @@ def check(_client, _target, _config, expected): monkeypatch.setattr(cm, "check_booted_config", check) monkeypatch.setattr(cm, "restore_default", lambda s, _target: setattr(s, "restored", True)) - defaults = {"configs": None, "limit": None, "changed_since": None, "keep_going": True, "boot_timeout": 90} + defaults = {"configs": None, "limit": None, "changed_since": None, "keep_going": True, "boot_timeout": 90, "config_timeout": 60} defaults.update(arg_overrides) return cm.run_matrix({"port": session.port, "target": "wpc"}, Namespace(**defaults)) @@ -568,3 +569,74 @@ def test_write_step_summary_is_a_no_op_outside_actions(monkeypatch): monkeypatch.delenv("GITHUB_STEP_SUMMARY", raising=False) cm.write_step_summary([({"port": "/dev/ttyACM0", "target": "wpc"}, [], [])]) + + +# -------------------------------------------------------------------------- +# not hanging on a board that has stopped listening +# -------------------------------------------------------------------------- + + +def test_open_serial_always_sets_a_write_timeout(monkeypatch): + """The bug that hung a bench job for 30 minutes. + + `serial.Serial(timeout=...)` sets the READ timeout only. Without a + write_timeout, writing to a board that has stopped draining its USB + endpoint blocks forever - which is precisely the board the recovery tool + exists to write to. + """ + opened = {} + + def fake_serial(**kwargs): + opened.update(kwargs) + return FakeSerial() + + monkeypatch.setattr(bench.serial, "Serial", fake_serial, raising=False) + bench.open_serial("/dev/ttyFAKE") + + assert opened["timeout"] == bench.SERIAL_READ_TIMEOUT + assert opened["write_timeout"] == bench.SERIAL_WRITE_TIMEOUT + + +def test_serial_write_turns_a_stuck_board_into_an_error(monkeypatch): + monkeypatch.setattr(bench.serial, "SerialTimeoutException", RuntimeError, raising=False) + + class Deaf: + def write(self, _data): + raise RuntimeError("write timed out") + + with pytest.raises(bench.CheckFailure, match="stopped draining its USB endpoint"): + bench.serial_write(Deaf(), b"\x03", "the board") + + +def test_repl_never_calls_flush(): + """flush() is tcdrain, which takes no timeout and hangs on the same board. + + Handing the bytes to the kernel is enough; every exchange here is + synchronised by a read with a deadline instead. + """ + source = (REPO_ROOT / "dev" / "hil" / "bench.py").read_text() + + assert ".flush()" not in source + + +def test_time_limit_interrupts_work_that_overruns(): + with pytest.raises(bench.CheckFailure, match="did not finish within"): + with bench.time_limit(1, "a step that hangs"): + time.sleep(5) + + +def test_time_limit_is_invisible_when_work_finishes_in_time(): + with bench.time_limit(5, "quick work"): + result = 1 + 1 + + assert result == 2 + + +def test_time_limit_restores_the_previous_handler(): + import signal + + before = signal.getsignal(signal.SIGALRM) + with bench.time_limit(5, "quick work"): + pass + + assert signal.getsignal(signal.SIGALRM) is before diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index 2c1bf742..f1a37e2c 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -30,7 +30,7 @@ def args(**overrides): - defaults = {"reflash": True, "no_power_cycle": False, "force_bootsel": False, "cache_dir": Path("/nonexistent")} + defaults = {"reflash": True, "no_power_cycle": False, "force_bootsel": False, "step_timeout": 30, "cache_dir": Path("/nonexistent")} defaults.update(overrides) return Namespace(**defaults) From 255381c4ab751c1d184f8f05d87810768a412f9a Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 13:44:57 +0000 Subject: [PATCH 05/32] feat(hil): let TrenchCoat do the reflashing instead of reimplementing it TrenchCoat is the team's own tool for recovering a board without the BOOTSEL button, so the bench uses it rather than a second implementation of the same delicate sequence. dev/hil/trench_coat.py drives its real code. Reading it properly showed my hand-rolled step 4 was missing the part that makes this a recovery at all. TrenchCoat's flash_firmware() does: bootloader -> nuke.uf2 -> wait for the drive to cycle -> real UF2 -> wait for the board to re-enumerate The nuke.uf2 wipe is load-bearing: it erases the whole flash, so nothing from the old filesystem survives into the new firmware. Copying a UF2 over the top, which is all I had, would have carried a corrupt filesystem straight through. Its route into the bootloader is also better than mine. `machine.bootloader()` over the REPL, fire-and-forget - it never waits for a reply, so unlike mpremote's handshake it does not need the board to answer, only to execute one statement. That is tried first now, with the 1200 baud touch (a CDC line-coding change handled in USB interrupt context) kept as the fallback for when the VM cannot run anything at all. Two adaptations, both by narrowing what TrenchCoat can see rather than changing what it does: * It flashes every board it finds. On the bench that would nuke the two healthy boards along with the broken one, so Ray.find_board_ports is narrowed to the single port being recovered. There is a test for this specifically - it is the one thing here that must not go wrong. * It locates bootloader drives under /media and /Volumes, assuming a desktop automounter. A headless runner has none, so list_rpi_rp2_drives is wrapped to mount the RPI-RP2 volume with udisksctl first. Its graceful_exit() calls sys.exit after printing advice; that is replaced with an exception the harness can report and carry on from. Only src.core, src.ray, src.ui and src.util are imported, and between them they need nothing but pyserial, which the bench venv already has. Verified the import works even though this repo also has a top-level src/ package - and load() fails loudly rather than silently importing the wrong one. The checkout is pinned by commit and the workflow clones it up front, so a network problem fails the job visibly instead of halfway through recovering a board. This replaces the raw UF2 downloads and their checksums: the files now come from the pinned checkout itself. Also: `timeout 5` around the stty in all three HIL workflows. Opening a tty waits for carrier, so a wedged board can block even stty - that is what left the last run's diagnostic step running for 32 minutes after the recovery step itself finished cleanly in 2m12s. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-config-matrix.yml | 2 +- .github/workflows/hil-flash-check.yml | 2 +- .github/workflows/hil-recover.yml | 25 ++- dev/hil/recover.py | 185 +++++--------------- dev/hil/trench_coat.py | 213 ++++++++++++++++++++++ dev/tests/test_hil_recover.py | 223 ++++++++++++++++++------ 6 files changed, 448 insertions(+), 202 deletions(-) create mode 100644 dev/hil/trench_coat.py diff --git a/.github/workflows/hil-config-matrix.yml b/.github/workflows/hil-config-matrix.yml index d24d6184..9f79997c 100644 --- a/.github/workflows/hil-config-matrix.yml +++ b/.github/workflows/hil-config-matrix.yml @@ -128,6 +128,6 @@ jobs: # parsed their own log lines as USB API requests. for dev in /dev/ttyACM*; do echo "--- $dev" - stty -F "$dev" raw -echo 115200 || true + timeout 5 stty -F "$dev" raw -echo 115200 || true timeout 8 cat "$dev" || true done diff --git a/.github/workflows/hil-flash-check.yml b/.github/workflows/hil-flash-check.yml index 722472c9..103583b6 100644 --- a/.github/workflows/hil-flash-check.yml +++ b/.github/workflows/hil-flash-check.yml @@ -88,6 +88,6 @@ jobs: # it, and the board then parses its own log lines as USB API requests. for dev in /dev/ttyACM*; do echo "--- $dev" - stty -F "$dev" raw -echo 115200 || true + timeout 5 stty -F "$dev" raw -echo 115200 || true timeout 8 cat "$dev" || true done diff --git a/.github/workflows/hil-recover.yml b/.github/workflows/hil-recover.yml index 42462d51..677a98bb 100644 --- a/.github/workflows/hil-recover.yml +++ b/.github/workflows/hil-recover.yml @@ -42,6 +42,7 @@ on: - claude/wpc-hil-config-validation-rc62dn paths: - dev/hil/recover.py + - dev/hil/trench_coat.py - .github/workflows/hil-recover.yml permissions: @@ -68,6 +69,23 @@ jobs: test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + # TrenchCoat does the actual reflash (dev/hil/trench_coat.py drives it). + # Cloned here rather than at runtime so a network problem fails the job + # visibly instead of halfway through recovering a board. Pin lives in + # dev/hil/trench_coat.py; recover.py re-checks the checkout matches it. + - name: Fetch TrenchCoat + run: | + commit=$(python -c "import sys; sys.path.insert(0, 'dev/hil'); import trench_coat; print(trench_coat.TRENCH_COAT_COMMIT)") + root="$PWD/build/hil/trench-coat" + mkdir -p "$(dirname "$root")" + if [ ! -d "$root/.git" ]; then + git clone --quiet https://github.com/warped-pinball/trench-coat "$root" + fi + git -C "$root" fetch --quiet origin "$commit" + git -C "$root" checkout --quiet "$commit" + echo "trench-coat at $(git -C "$root" rev-parse --short HEAD)" + ls -l "$root/uf2" + - name: Recover env: HIL_PORT: ${{ inputs.port }} @@ -88,7 +106,10 @@ jobs: if: always() run: | # stty raw -echo first: a tty reverts to ECHO-on once every handle is - # closed, so a bare `cat` feeds the board its own output back. + # closed, so a bare `cat` feeds the board its own output back. Both + # calls are under `timeout`, because opening a tty waits for carrier + # and a wedged board can block even stty - which is what left this + # step running for 32 minutes on the last run. echo "--- serial ports" ls -l /dev/ttyACM* 2>/dev/null || echo "(none)" echo "--- bootloader drives" @@ -96,6 +117,6 @@ jobs: for dev in /dev/ttyACM*; do [ -e "$dev" ] || continue echo "--- $dev" - stty -F "$dev" raw -echo 115200 || true + timeout 5 stty -F "$dev" raw -echo 115200 || true timeout 5 cat "$dev" || true done diff --git a/dev/hil/recover.py b/dev/hil/recover.py index 5325d9cf..67029c64 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -28,32 +28,31 @@ The board is USB bus powered (Trench-Coat-Install-Guide.md), so this is a real power cycle, not a signal. Needs a hub that supports per-port power switching. - 4. reflash - 1200 baud touch to drop the RP2040 into its ROM bootloader, - then copy a MicroPython UF2 onto the RPI-RP2 drive that - appears. The touch is handled in USB interrupt context rather - than by the Python VM, so it can work when everything above - has failed. Destructive: it replaces the firmware, and the - board needs `dev/flash.py` afterwards to get Vector back. + 4. reflash - hand the board to TrenchCoat, the team's own tool for + recovering a board without the BOOTSEL button. It resets into + the ROM bootloader, wipes the whole flash with nuke.uf2, then + writes the real firmware. Destructive: the board needs + `dev/hil/flash_and_check.py` afterwards to get Vector back. -Only step 4 needs anything from outside the repo - the UF2s come from -warped-pinball/trench-coat, pinned by commit and verified by checksum. +Only step 4 needs anything from outside the repo: a checkout of +warped-pinball/trench-coat, pinned by commit. dev/hil/trench_coat.py drives +it - the sequence is not reimplemented here. If all four fail, the board needs a person: hold BOOTSEL while replugging it. """ import argparse import fcntl -import hashlib import os import shutil import subprocess import sys import time -import urllib.request from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) +import trench_coat # noqa: E402 from bench import ( # noqa: E402 CTRL_C, REPO_ROOT, @@ -70,28 +69,17 @@ time_limit, ) -# warped-pinball/trench-coat, pinned by commit. The checksums are what this -# revision ships; a mismatch means the pin moved under us and the file is not -# flashed. UF2s are large, so they are fetched rather than vendored here. -TRENCH_COAT_COMMIT = "26e6d508c362bed1f6d1323155435c18528de758" -TRENCH_COAT_RAW = f"https://raw.githubusercontent.com/warped-pinball/trench-coat/{TRENCH_COAT_COMMIT}/uf2" - -TARGET_UF2 = { - "wpc": ("Vector_WPC_v5.uf2", "3d02a60de852c11087f76ad61a1baf5921270c9a98ca9542a450aad26fac5191"), - "data_east": ("Vector_DataEast_v1.uf2", "11afc1d22f28099921e63950ba1e86832f47f2c558f8384ff04d9cf6650e7047"), - "sys11": ("vector_system_11_and_9_v4.uf2", "ba63972475f5126c1e5270c30b418510505f5859da9366eb0fac9ef35c9e7a15"), -} - # ioctl number for USBDEVFS_RESET, from : _IO('U', 20). USBDEVFS_RESET = ord("U") << 8 | 20 PROBE_TIMEOUT = 20 SETTLE_SECONDS = 5 -# No single step may hang the job. Generous enough for the slowest one (fetch a -# 1.7MB UF2, wait for a drive, copy it) and still far short of the workflow's -# own timeout, so the run always gets to print its summary. -STEP_TIMEOUT = 180 +# No single step may hang the job. Generous enough for the slowest one - the +# TrenchCoat reflash, which clones, waits for a drive, and writes two UF2s - +# and still short of the workflow's own timeout, so the run always gets to +# print its summary. +STEP_TIMEOUT = 600 def responsive(port, timeout=PROBE_TIMEOUT): @@ -295,88 +283,28 @@ def power_cycle(port, off_seconds=3): # -------------------------------------------------------------------------- -# 4. reflash over the ROM bootloader +# 4. reflash, using TrenchCoat # -------------------------------------------------------------------------- -def bootsel_touch(port): - """Open the port at 1200 baud to drop the RP2040 into its ROM bootloader. - - The last resort that does not need a person, and the reason it can work - when the REPL cannot: the 1200 baud touch is a CDC line-coding change, - handled in USB interrupt context, so a blocked Python VM does not stop it. - TrenchCoat's own enter_bootloader_mode() goes through `machine.bootloader()` - on the REPL instead, which a wedged board will never run. - """ - try: - connection = open_serial(port, baudrate=1200) - connection.dtr = False - time.sleep(0.5) - connection.close() - except Exception as exc: - # The port vanishing underneath us IS the board rebooting into the - # bootloader, so this is as often success as failure. - log(f" port closed during the 1200 baud touch ({exc}) - which is what a reboot looks like") - return True - - -def find_bootloader_drive(timeout=30): - """Wait for an RPI-RP2 drive to appear, the way TrenchCoat looks for it.""" - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - for root in ("/media", "/run/media", "/mnt"): - base = Path(root) - if not base.is_dir(): - continue - try: - for path in base.rglob("INFO_UF2.TXT"): - return path.parent - except OSError: - continue - time.sleep(1) - return None - - -def mount_bootloader_drive(timeout=30): - """Mount the RPI-RP2 volume ourselves when nothing automounts it. - - A headless runner has no desktop automounter, so the drive that appears - after the touch is a block device and nothing more. udisksctl goes through - polkit rather than sudo, which is the one route a service user might - actually have. - """ - if not shutil.which("udisksctl"): - log(" udisksctl is not available, so the bootloader drive cannot be mounted here") - return None - - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - for link in sorted(Path("/dev/disk/by-label").glob("RPI-RP2*")) if Path("/dev/disk/by-label").is_dir() else []: - device = link.resolve() - log(f" mounting {device} with udisksctl") - result = subprocess.run(["udisksctl", "mount", "-b", str(device)], capture_output=True, text=True, timeout=60) - if result.returncode == 0: - # "Mounted /dev/sda1 at /media/xxx" - mounted = result.stdout.strip().rsplit(" at ", 1)[-1].rstrip(".") - log(f" mounted at {mounted}") - return Path(mounted) - log(f" udisksctl could not mount it: {(result.stderr or result.stdout).strip()}") - return None - time.sleep(1) - return None - - def can_complete_a_reflash(): """Is there any way this runner could write a UF2 once the board is in BOOTSEL? - Asked *before* the 1200 baud touch, because the touch is a one-way door: it - takes a board that is at least enumerated as a serial device and turns it - into a mass-storage device that only a UF2 (or a replug) gets it out of. - Doing that with no way to finish the job makes the board harder to recover, - not easier. + Asked *before* anything touches the board into its bootloader, because that + is a one-way door: it takes a board that is at least enumerated as a serial + device and turns it into a mass-storage device that only a UF2 (or a + replug) gets it out of. Doing that with no way to finish the job makes the + board harder to recover, not easier. """ - if find_bootloader_drive(timeout=0) is not None: - return True, "a bootloader drive is already mounted" + for root in ("/media", "/run/media", "/mnt"): + base = Path(root) + if not base.is_dir(): + continue + try: + if any(base.rglob("INFO_UF2.TXT")): + return True, "a bootloader drive is already mounted" + except OSError: + continue if shutil.which("udisksctl"): return True, "udisksctl is available to mount the drive" if any(Path(root).is_dir() and os.access(root, os.W_OK) for root in ("/media", "/run/media")): @@ -384,31 +312,16 @@ def can_complete_a_reflash(): return False, "nothing here can mount an RPI-RP2 drive (no udisksctl, no writable automount directory)" -def fetch_uf2(target, cache_dir): - """Download the pinned UF2 for `target` and verify it before use.""" - if target not in TARGET_UF2: - raise CheckFailure(f"no UF2 known for target {target!r} (have: {', '.join(sorted(TARGET_UF2))})") - filename, expected = TARGET_UF2[target] - - cache_dir.mkdir(parents=True, exist_ok=True) - path = cache_dir / filename - if not path.exists(): - url = f"{TRENCH_COAT_RAW}/{filename}" - log(f" downloading {filename} from trench-coat@{TRENCH_COAT_COMMIT[:8]}") - request = urllib.request.Request(url, headers={"User-Agent": "vector-hil"}) - with urllib.request.urlopen(request, timeout=120) as response: - path.write_bytes(response.read()) - - digest = hashlib.sha256(path.read_bytes()).hexdigest() - if digest != expected: - path.unlink(missing_ok=True) - raise CheckFailure(f"{filename} does not match the checksum pinned for trench-coat@{TRENCH_COAT_COMMIT[:8]} (got {digest}) - refusing to flash it") - log(f" {filename} verified ({path.stat().st_size} bytes)") - return path - - def reflash(port, target, cache_dir, force=False): - """1200 baud touch, then drop a UF2 on the drive that appears.""" + """Hand the board to TrenchCoat, which is the tool for exactly this job. + + Not reimplemented here on purpose. TrenchCoat is what the team uses to + recover a board without the BOOTSEL button, and its sequence includes the + step a naive UF2 copy misses: a nuke.uf2 wipe of the whole flash before the + real firmware, so nothing from the old filesystem survives. See + dev/hil/trench_coat.py for how it is pointed at one board instead of all of + them. + """ possible, why = can_complete_a_reflash() log(f" {'can' if possible else 'cannot'} finish a reflash here: {why}") if not possible and not force: @@ -417,24 +330,7 @@ def reflash(port, target, cache_dir, force=False): log(" on the runner (or pass --force-bootsel) to make this step usable.") return False - # Fetch and verify before the point of no return, so a bad download cannot - # strand the board in BOOTSEL. - uf2 = fetch_uf2(target, cache_dir) - - bootsel_touch(port) - - drive = find_bootloader_drive() or mount_bootloader_drive() - if drive is None: - log(" no RPI-RP2 drive appeared, so either the board did not reach its ROM bootloader") - log(" or nothing mounted the drive it presented") - return False - log(f" board is in bootloader mode at {drive}") - - log(f" copying {uf2.name} to {drive}") - shutil.copy(uf2, drive) - # The board reboots as soon as the copy lands, taking the drive with it. - time.sleep(10) - return True + return trench_coat.flash(port, target, cache_dir / "trench-coat") # -------------------------------------------------------------------------- @@ -505,6 +401,7 @@ def preflight(): possible, why = can_complete_a_reflash() log(f" reflash {'ok ' if possible else 'no '} {why}") + log(f" {'':11} {'':9} TrenchCoat pinned at {trench_coat.TRENCH_COAT_COMMIT[:8]}") def main(): @@ -515,7 +412,7 @@ def main(): parser.add_argument("--no-power-cycle", action="store_true", help="skip the uhubctl step") parser.add_argument("--force-bootsel", action="store_true", help="touch the board into BOOTSEL even when nothing here can mount the drive to flash it") parser.add_argument("--step-timeout", type=int, default=STEP_TIMEOUT, help=f"hard ceiling on any one recovery step, in seconds (default {STEP_TIMEOUT})") - parser.add_argument("--cache-dir", type=Path, default=REPO_ROOT / "build" / "uf2", help="where to keep downloaded UF2s") + parser.add_argument("--cache-dir", type=Path, default=REPO_ROOT / "build" / "hil", help="where to keep the trench-coat checkout") args = parser.parse_args() ensure_tools_on_path() diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py new file mode 100644 index 00000000..74f48347 --- /dev/null +++ b/dev/hil/trench_coat.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +"""Drive warped-pinball/trench-coat to reflash a bench board. + +TrenchCoat is the tool the Warped Pinball team already uses to recover boards +without touching the BOOTSEL button, so the bench uses *it* rather than a +second implementation of the same delicate sequence. Its `flash_firmware()` +does the part that matters and that a naive "copy a UF2" misses: + + bootloader -> nuke.uf2 -> wait for the drive to cycle -> real UF2 -> wait + for the board to re-enumerate as a serial device + +The `nuke.uf2` wipe is the load-bearing step. It erases the whole flash, so +nothing from the old filesystem survives into the new firmware - which is what +makes this a recovery rather than an upgrade. + +Two things have to be adapted for the bench, and both are done by narrowing +what TrenchCoat can see rather than by changing what it does: + + * It flashes *every* board it finds. On the bench that would nuke the two + healthy boards along with the broken one, so `Ray.find_board_ports` is + narrowed to the single port being recovered. + * It finds bootloader drives by looking for INFO_UF2.TXT under /media and + /Volumes, which assumes a desktop automounter. A headless runner has none, + so `list_rpi_rp2_drives` is wrapped to mount the RPI-RP2 volume with + udisksctl first. + +Only `src.core`, `src.ray`, `src.ui` and `src.util` are imported, and between +them they need nothing but pyserial - which the bench venv already has because +mpremote ships it. `src.main` and `src.interactive` are the parts that want +InquirerPy and a human, and neither is used here. +""" + +import subprocess +import sys +import time +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from bench import CheckFailure, log, open_serial # noqa: E402 + +# Pinned, like every other third-party input to this bench. Bumping it means +# reviewing what changed in the flashing sequence first. +TRENCH_COAT_COMMIT = "26e6d508c362bed1f6d1323155435c18528de758" +TRENCH_COAT_URL = "https://github.com/warped-pinball/trench-coat" + +# Which bundled UF2 belongs to which bench target. +TARGET_UF2 = { + "wpc": "Vector_WPC_v5.uf2", + "data_east": "Vector_DataEast_v1.uf2", + "sys11": "vector_system_11_and_9_v4.uf2", + "em": "Vector_WPC_v5.uf2", # EM runs on the WPC OS (see TrenchCoat's series menu) +} + + +def clone(root, commit=TRENCH_COAT_COMMIT): + """Make sure `root` holds trench-coat at exactly `commit`.""" + root = Path(root) + if not (root / ".git").is_dir(): + root.parent.mkdir(parents=True, exist_ok=True) + log(f" cloning trench-coat into {root}") + subprocess.run(["git", "clone", "--quiet", TRENCH_COAT_URL, str(root)], check=True, timeout=600) + + subprocess.run(["git", "-C", str(root), "fetch", "--quiet", "origin", commit], check=True, timeout=600) + subprocess.run(["git", "-C", str(root), "checkout", "--quiet", commit], check=True, timeout=120) + + head = subprocess.run(["git", "-C", str(root), "rev-parse", "HEAD"], capture_output=True, text=True, timeout=60).stdout.strip() + if head != commit: + raise CheckFailure(f"trench-coat checkout is at {head}, expected the pinned {commit}") + log(f" trench-coat at {commit[:8]}") + return root + + +def load(root): + """Import TrenchCoat's modules from `root` and return (core, ray). + + Guarded because both repositories have a top-level `src` package, and + importing the wrong one would be a confusing way to fail. + """ + root = str(Path(root).resolve()) + if root in sys.path: + sys.path.remove(root) + sys.path.insert(0, root) + + for name in [module for module in sys.modules if module == "src" or module.startswith("src.")]: + del sys.modules[name] + + import src.core as core + import src.ray as ray + + if not str(Path(core.__file__).resolve()).startswith(root): + raise CheckFailure(f"imported the wrong `src` package: got {core.__file__}, expected it under {root}") + return core, ray + + +def bundled_uf2(root, target): + if target not in TARGET_UF2: + raise CheckFailure(f"no TrenchCoat UF2 known for target {target!r} (have: {', '.join(sorted(TARGET_UF2))})") + path = Path(root) / "uf2" / TARGET_UF2[target] + if not path.exists(): + raise CheckFailure(f"{path} is missing from the trench-coat checkout") + return path + + +def mount_rpi_rp2(): + """Mount an RPI-RP2 volume that nothing automounted. Returns the path or None.""" + by_label = Path("/dev/disk/by-label") + if not by_label.is_dir(): + return None + for link in sorted(by_label.glob("RPI-RP2*")): + device = link.resolve() + result = subprocess.run(["udisksctl", "mount", "-b", str(device)], capture_output=True, text=True, timeout=60) + if result.returncode == 0: + mounted = result.stdout.strip().rsplit(" at ", 1)[-1].rstrip(".") + log(f" mounted {device} at {mounted}") + return mounted + if "AlreadyMounted" in result.stderr: + continue + log(f" udisksctl could not mount {device}: {(result.stderr or result.stdout).strip()}") + return None + + +def bootsel_touch(port): + """Open the port at 1200 baud, which asks the RP2040 to reset to its ROM. + + TrenchCoat's own route into the bootloader is `machine.bootloader()` over + the REPL, fire-and-forget. That is the right first try - it is what the + team uses and it needs no privileges - but it does need the firmware alive + enough to run one statement. The 1200 baud touch is a CDC line-coding + change handled in USB interrupt context, so it can still land when the + Python VM cannot run anything at all. + """ + try: + connection = open_serial(port, baudrate=1200) + connection.dtr = False + time.sleep(0.5) + connection.close() + except Exception as exc: + # The port disappearing underneath us is what a reboot looks like. + log(f" port closed during the 1200 baud touch ({exc})") + + +def wait_for_drive(core, timeout=45): + """Wait for a bootloader drive, mounting it ourselves if nothing else does.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + drives = core.list_rpi_rp2_drives() + if drives: + return drives + if mount_rpi_rp2(): + drives = core.list_rpi_rp2_drives() + if drives: + return drives + time.sleep(1) + return [] + + +def enter_bootloader(core, ray, port): + """Get one board into its ROM bootloader, TrenchCoat's way then ours.""" + log(" asking the board to reset into the bootloader (TrenchCoat's machine.bootloader())") + try: + ray.Ray(port).enter_bootloader_mode() + except Exception as exc: + log(f" that did not go through: {exc}") + + drives = wait_for_drive(core, timeout=20) + if drives: + log(f" board is in bootloader mode: {', '.join(drives)}") + return drives + + log(" no drive yet, falling back to the 1200 baud touch") + bootsel_touch(port) + drives = wait_for_drive(core, timeout=45) + if drives: + log(f" board is in bootloader mode: {', '.join(drives)}") + return drives + + +def flash(port, target, root): + """Recover one board by running TrenchCoat's own firmware flash against it. + + Returns True if TrenchCoat reported the board back as a serial device. + """ + core, ray = load(clone(root)) + uf2 = bundled_uf2(root, target) + + drives = enter_bootloader(core, ray, port) + if not drives: + log(" the board never presented a bootloader drive, so there is nothing to flash") + return False + + # From here TrenchCoat drives, on this board only. find_board_ports is + # emptied because the board is already a drive - that makes its + # get_all_boards_into_bootloader() a no-op instead of a second attempt, and + # keeps it away from the healthy boards on the bench. + ray.Ray.find_board_ports = classmethod(lambda cls: []) + original_list_drives = core.list_rpi_rp2_drives + core.list_rpi_rp2_drives = lambda: original_list_drives() or drives + + # Their failure path prints advice and calls sys.exit; make it an exception + # this harness can report and carry on from. + def refuse_to_exit(now=False): + raise CheckFailure("TrenchCoat could not complete the flash (see its output above)") + + core.graceful_exit = refuse_to_exit + + log(f" handing over to TrenchCoat: nuke.uf2, then {uf2.name}") + core.flash_firmware(str(uf2)) + + # flash_firmware only returns cleanly once the board is back as a serial + # port, so reaching here is the success condition. + log(" TrenchCoat reports the board restarted") + return True diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index f1a37e2c..7085c09d 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -27,6 +27,7 @@ import bench # noqa: E402 import recover # noqa: E402 +import trench_coat # noqa: E402 def args(**overrides): @@ -191,41 +192,92 @@ def times_out(*_args, **_kwargs): # -------------------------------------------------------------------------- -# the UF2 gate +# handing the board to TrenchCoat # -------------------------------------------------------------------------- -def test_fetch_uf2_refuses_a_file_that_does_not_match_the_pin(tmp_path): - filename, _digest = recover.TARGET_UF2["wpc"] - (tmp_path / filename).write_bytes(b"not the firmware you are looking for") +def test_reflash_delegates_to_trench_coat(monkeypatch, tmp_path): + """The sequence is TrenchCoat's, not ours - we only point it at one board.""" + called = {} + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (True, "udisksctl is available")) + monkeypatch.setattr(recover.trench_coat, "flash", lambda port, target, root: called.update(port=port, target=target, root=root) or True) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) is True + assert called["port"] == "/dev/ttyFAKE" + assert called["target"] == "wpc" + + +def test_reflash_will_not_strand_a_board_in_bootsel_it_cannot_flash(monkeypatch, tmp_path, capsys): + """Entering BOOTSEL is a one-way door. + + A wedged board is at least still a serial device. Sending it to the ROM + bootloader with no way to write a UF2 turns it into a mass-storage device + that only a replug gets out of - strictly worse than how it was found. + """ + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) + monkeypatch.setattr(recover.trench_coat, "flash", lambda *a, **k: pytest.fail("must not touch the board")) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) is False + assert "not touching the board into BOOTSEL" in capsys.readouterr().out - with pytest.raises(bench.CheckFailure, match="refusing to flash it"): - recover.fetch_uf2("wpc", tmp_path) - # And it does not leave the bad file behind to be picked up next time. - assert not (tmp_path / filename).exists() +def test_force_bootsel_overrides_the_guard(monkeypatch, tmp_path): + monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) + monkeypatch.setattr(recover.trench_coat, "flash", lambda *a, **k: True) + + assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path, force=True) is True -def test_fetch_uf2_accepts_the_pinned_file(tmp_path, monkeypatch): - import hashlib +def test_can_complete_a_reflash_accepts_udisksctl(monkeypatch): + monkeypatch.setattr(recover.shutil, "which", lambda name: "/usr/bin/udisksctl" if name == "udisksctl" else None) + monkeypatch.setattr(recover.Path, "is_dir", lambda self: False) - payload = b"pretend UF2" - filename, _digest = recover.TARGET_UF2["wpc"] - monkeypatch.setitem(recover.TARGET_UF2, "wpc", (filename, hashlib.sha256(payload).hexdigest())) - (tmp_path / filename).write_bytes(payload) + possible, why = recover.can_complete_a_reflash() - assert recover.fetch_uf2("wpc", tmp_path) == tmp_path / filename + assert possible is True + assert "udisksctl" in why -def test_fetch_uf2_rejects_a_target_it_has_no_firmware_for(tmp_path): - with pytest.raises(bench.CheckFailure, match="no UF2 known"): - recover.fetch_uf2("whitestar", tmp_path) +def test_can_complete_a_reflash_says_no_when_nothing_can_mount(monkeypatch): + monkeypatch.setattr(recover.shutil, "which", lambda _name: None) + monkeypatch.setattr(recover.Path, "is_dir", lambda self: False) + possible, why = recover.can_complete_a_reflash() -def test_every_target_uf2_pin_is_a_sha256(): - for target, (filename, digest) in recover.TARGET_UF2.items(): + assert possible is False + assert "nothing here can mount" in why + + +def test_every_target_maps_to_a_bundled_uf2(): + for target, filename in trench_coat.TARGET_UF2.items(): assert filename.endswith(".uf2"), target - assert len(digest) == 64 and set(digest) <= set("0123456789abcdef"), target + + +def test_bundled_uf2_rejects_an_unknown_target(tmp_path): + with pytest.raises(bench.CheckFailure, match="no TrenchCoat UF2 known"): + trench_coat.bundled_uf2(tmp_path, "whitestar") + + +def test_bundled_uf2_reports_a_checkout_missing_the_file(tmp_path): + with pytest.raises(bench.CheckFailure, match="missing from the trench-coat checkout"): + trench_coat.bundled_uf2(tmp_path, "wpc") + + +def test_load_refuses_the_wrong_src_package(tmp_path, monkeypatch): + """Both repos have a top-level `src`; importing ours would fail confusingly.""" + fake = tmp_path / "src" + fake.mkdir() + (fake / "__init__.py").write_text("") + (fake / "core.py").write_text("") + (fake / "ray.py").write_text("") + + real_import = trench_coat.load + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: root) + + # Loading from the right root works; the guard only fires on a mismatch, + # which is what the assertion inside load() covers. + core, _ray = real_import(tmp_path) + assert str(Path(core.__file__).resolve()).startswith(str(tmp_path.resolve())) # -------------------------------------------------------------------------- @@ -269,49 +321,112 @@ def test_hub_location_splits_the_usb_path(tmp_path, monkeypatch, usb_path, expec assert recover.hub_location("/dev/ttyACM1") == expected -def test_reflash_will_not_strand_a_board_in_bootsel_it_cannot_flash(monkeypatch, tmp_path, capsys): - """The touch is a one-way door. +# -------------------------------------------------------------------------- +# narrowing TrenchCoat to one board +# -------------------------------------------------------------------------- - A wedged board is at least still a serial device. Touching it into BOOTSEL - with no way to write a UF2 turns it into a mass-storage device that only a - replug gets out of - strictly worse than how it was found. + +def fake_trench_coat(monkeypatch, drives=("/media/RPI-RP2",)): + """Stand in for TrenchCoat's core/ray modules and record how they are used.""" + seen = {"flashed": None, "ports_seen": None, "drives_seen": None, "bootloader": []} + + class FakeRay: + def __init__(self, port): + seen["bootloader"].append(port) + + def enter_bootloader_mode(self): + pass + + @classmethod + def find_board_ports(cls): + return ["/dev/ttyACM0", "/dev/ttyACM1", "/dev/ttyACM2"] + + ray = types.SimpleNamespace(Ray=FakeRay) + core = types.SimpleNamespace( + list_rpi_rp2_drives=lambda: list(drives), + graceful_exit=lambda now=False: None, + flash_firmware=lambda path: seen.update( + flashed=path, + ports_seen=ray.Ray.find_board_ports(), + drives_seen=core.list_rpi_rp2_drives(), + ), + ) + + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: root) + monkeypatch.setattr(trench_coat, "load", lambda root: (core, ray)) + monkeypatch.setattr(trench_coat, "bundled_uf2", lambda root, target: Path(f"/uf2/{target}.uf2")) + return seen + + +def test_flash_hides_the_other_boards_from_trench_coat(monkeypatch, tmp_path): + """The one thing that must not go wrong. + + TrenchCoat flashes every board it finds, which on the bench would nuke the + two healthy boards alongside the broken one. It only ever gets to see the + board being recovered. """ - touched = [] - monkeypatch.setattr(recover, "bootsel_touch", lambda port: touched.append(port)) - monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) + seen = fake_trench_coat(monkeypatch) - assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) is False - assert touched == [] - assert "not touching the board into BOOTSEL" in capsys.readouterr().out + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is True + assert seen["ports_seen"] == [] + assert seen["flashed"] == "/uf2/wpc.uf2" + # Only the board being recovered is asked to enter the bootloader. + assert seen["bootloader"] == ["/dev/ttyACM1"] -def test_reflash_verifies_the_uf2_before_the_point_of_no_return(monkeypatch, tmp_path): - """Download and checksum first: a bad fetch must not cost us the board.""" - touched = [] - monkeypatch.setattr(recover, "bootsel_touch", lambda port: touched.append(port)) - monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (True, "udisksctl is available")) +def test_flash_gives_trench_coat_the_drive_it_could_not_find(monkeypatch, tmp_path): + """A headless runner has no automounter, so we mount and hand the path over.""" + seen = fake_trench_coat(monkeypatch, drives=()) + monkeypatch.setattr(trench_coat, "mount_rpi_rp2", lambda: "/media/runner/RPI-RP2") + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: ["/media/runner/RPI-RP2"]) - filename, _digest = recover.TARGET_UF2["wpc"] - (tmp_path / filename).write_bytes(b"corrupted download") + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is True + assert seen["drives_seen"] == ["/media/runner/RPI-RP2"] - with pytest.raises(bench.CheckFailure, match="refusing to flash it"): - recover.reflash("/dev/ttyFAKE", "wpc", tmp_path) - assert touched == [] +def test_flash_stops_when_the_board_never_reaches_the_bootloader(monkeypatch, tmp_path, capsys): + seen = fake_trench_coat(monkeypatch, drives=()) + monkeypatch.setattr(trench_coat, "mount_rpi_rp2", lambda: None) + monkeypatch.setattr(trench_coat, "bootsel_touch", lambda port: None) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: []) -def test_force_bootsel_overrides_the_guard(monkeypatch, tmp_path): - import hashlib + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is False + assert seen["flashed"] is None + assert "never presented a bootloader drive" in capsys.readouterr().out + + +def test_flash_turns_trench_coats_exit_into_an_error(monkeypatch, tmp_path): + """Its failure path calls sys.exit; the harness needs an exception instead.""" + seen = fake_trench_coat(monkeypatch) + core, _ray = trench_coat.load(tmp_path) + + def bail(_path): + core.graceful_exit() + + core.flash_firmware = bail - payload = b"pretend UF2" - filename, _digest = recover.TARGET_UF2["wpc"] - monkeypatch.setitem(recover.TARGET_UF2, "wpc", (filename, hashlib.sha256(payload).hexdigest())) - (tmp_path / filename).write_bytes(payload) + with pytest.raises(bench.CheckFailure, match="could not complete the flash"): + trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) + assert seen["flashed"] is None + +def test_enter_bootloader_falls_back_to_the_1200_baud_touch(monkeypatch): + """TrenchCoat's route needs the VM alive enough to run one statement.""" touched = [] - monkeypatch.setattr(recover, "bootsel_touch", lambda port: touched.append(port)) - monkeypatch.setattr(recover, "can_complete_a_reflash", lambda: (False, "no udisksctl")) - monkeypatch.setattr(recover, "find_bootloader_drive", lambda *a, **k: None) - monkeypatch.setattr(recover, "mount_bootloader_drive", lambda *a, **k: None) + attempts = [] + + class FakeRay: + def __init__(self, port): + pass + + def enter_bootloader_mode(self): + attempts.append("machine.bootloader()") + + monkeypatch.setattr(trench_coat, "bootsel_touch", lambda port: touched.append(port)) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: [] if not touched else ["/media/RPI-RP2"]) + + drives = trench_coat.enter_bootloader(types.SimpleNamespace(), types.SimpleNamespace(Ray=FakeRay), "/dev/ttyACM1") - assert recover.reflash("/dev/ttyFAKE", "wpc", tmp_path, force=True) is False - assert touched == ["/dev/ttyFAKE"] + assert attempts == ["machine.bootloader()"] + assert touched == ["/dev/ttyACM1"] + assert drives == ["/media/RPI-RP2"] From 44d348ad363457e90d57e2f74473ca79ceefc0c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 14:32:05 +0000 Subject: [PATCH 06/32] fix(hil): stop the recovery job taking the runner down with it The TrenchCoat-driven run reached the bench and died 11 minutes in with the Recover step still marked in_progress - which is the runner agent dying, not a Python exit. It also explains why the last two bench runs uploaded no logs at all: the runner never got to. Two things in the recovery path could do that, both fixed here. An unbounded filesystem walk. can_complete_a_reflash() did `base.rglob("INFO_UF2.TXT")` over /media, /run/media AND /mnt - a recursive walk of whatever happens to be mounted there. On a 512MB Zero 2 W with a drive under /mnt that is minutes of I/O and enough memory pressure to take a job down; DESIGN already warns that jobs get OOM-killed on this host. Replaced with a bounded lookup of the two depths automounters actually use (/media/LABEL, /media//LABEL), and /mnt is no longer searched. The same function now backs the drive-waiting in trench_coat.py, so TrenchCoat's own os.walk of /media and /Volumes is not used either. A power cycle that cuts the whole bus. hub_location() fell back to treating a root-hub device (1-1:1.0) as bus 1 port 1, so uhubctl would have cut power to every downstream port - killing the two healthy boards to recover one, and taking the USB tree out from under the running job. It now returns None for a root-hub port and says why; only a downstream hub port is ours to switch. Also added a job summary. Findings are written to GITHUB_STEP_SUMMARY as they happen - the capability report, the board survey, and the outcome of each recovery step. Summaries are stored separately from the log archive, so the next run reports something even if the runner dies again the same way. Flying blind through three bench runs is what made this take as long as it did. Test fixtures updated: they were stubbing TrenchCoat's list_rpi_rp2_drives, which the bounded search replaced, so three tests sat through a real 45s wait each. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/recover.py | 83 ++++++++++++++++++++++++----------- dev/hil/trench_coat.py | 47 +++++++++++++++++--- dev/tests/test_hil_recover.py | 30 ++++++++++++- 3 files changed, 129 insertions(+), 31 deletions(-) diff --git a/dev/hil/recover.py b/dev/hil/recover.py index 67029c64..e1af0104 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -109,11 +109,15 @@ def survey(board_map): raise CheckFailure("no boards found at all - check the USB hub and power") alive, dead, claimed = [], [], set() - log(f"{'port':16} {'state':14} chip id") + note("") + note("### Boards") + note("") + note("```") + note(f"{'port':16} {'state':14} chip id") for port in ports: if not responsive(port): dead.append(port) - log(f"{port:16} {'NOT ANSWERING':14} -") + note(f"{port:16} {'NOT ANSWERING':14} -") continue alive.append(port) try: @@ -125,7 +129,9 @@ def survey(board_map): chip_id = None if chip_id in board_map: claimed.add(board_map[chip_id]) - log(f"{port:16} {'ok':14} {chip_id or '?'}") + note(f"{port:16} {'ok':14} {chip_id or '?'}") + + note("```") unclaimed = sorted(set(board_map.values()) - claimed) targets = {} @@ -232,6 +238,11 @@ def hub_location(port): A USB path looks like 1-1.4:1.0 - bus 1, hub at 1-1, port 4. uhubctl wants the hub and the port separately. + + Returns None for a device sitting directly on a root hub (1-1:1.0). That is + not a case worth handling: cutting power there takes down the whole bus and + every other board on it, which on this bench means killing the two healthy + boards to recover one. Only a downstream hub port is ours to switch. """ name = os.path.basename(port) try: @@ -245,8 +256,8 @@ def hub_location(port): if "." in usb_path: hub, _, portnum = usb_path.rpartition(".") return hub, portnum - bus, _, portnum = usb_path.partition("-") - return bus, portnum + log(f" {port} is on a root hub ({usb_path}); power cycling it would cut every board on the bus") + return None return None @@ -296,18 +307,11 @@ def can_complete_a_reflash(): replug) gets it out of. Doing that with no way to finish the job makes the board harder to recover, not easier. """ - for root in ("/media", "/run/media", "/mnt"): - base = Path(root) - if not base.is_dir(): - continue - try: - if any(base.rglob("INFO_UF2.TXT")): - return True, "a bootloader drive is already mounted" - except OSError: - continue + if trench_coat.find_bootloader_drives(): + return True, "a bootloader drive is already mounted" if shutil.which("udisksctl"): return True, "udisksctl is available to mount the drive" - if any(Path(root).is_dir() and os.access(root, os.W_OK) for root in ("/media", "/run/media")): + if any(Path(root).is_dir() and os.access(root, os.W_OK) for root in trench_coat.MOUNT_ROOTS): return True, "an automount directory is writable" return False, "nothing here can mount an RPI-RP2 drive (no udisksctl, no writable automount directory)" @@ -347,6 +351,10 @@ def recover(port, target, args): else: log(f"::warning::{port}: skipping the reflash step - no target known for this board, pass --target") + note("") + note(f"### Recovering {port}" + (f" ({target})" if target else "")) + note("") + for name, step in steps: group(f"{port}: {name}") try: @@ -362,15 +370,36 @@ def recover(port, target, args): if attempted: time.sleep(SETTLE_SECONDS) if responsive(port): - log(f" {port} is answering again") + note(f"- **{port}: recovered by {name}**") endgroup() return name - log(f" {port} still not answering") + note(f"- {port}: tried {name} - still not answering") + else: + note(f"- {port}: could not attempt {name}") endgroup() return None +def note(line): + """Append a line to the Actions job summary as well as the log. + + Job summaries are stored separately from the log archive, and the last two + bench runs proved why that matters: the runner died mid-job, never uploaded + its logs, and every finding went with them. Written incrementally rather + than at the end for the same reason. + """ + log(line) + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + try: + with open(path, "a") as handle: + handle.write(line.replace("::warning::", "WARNING: ").replace("::error::", "ERROR: ") + "\n") + except OSError: + pass + + def preflight(): """Report which recovery steps are actually available here. @@ -384,24 +413,28 @@ def preflight(): except CheckFailure: pass - log(f" serial ok {len(ports)} port(s) visible") + note("### What this runner can do") + note("") + note("```") + note(f" serial ok {len(ports)} port(s) visible") node = usb_device_path(ports[0]) if ports else None if node is None: - log(" usb reset unknown no device to check") + note(" usb reset unknown no device to check") elif os.access(node, os.W_OK): - log(f" usb reset ok {node} is writable") + note(f" usb reset ok {node} is writable") else: - log(f" usb reset no {node} is not writable - needs a udev rule granting the runner user write access") + note(f" usb reset no {node} is not writable - needs a udev rule granting the runner user write access") if shutil.which("uhubctl"): - log(" power ok uhubctl is installed (still needs a hub that switches port power)") + note(" power ok uhubctl is installed (still needs a hub that switches port power)") else: - log(" power no uhubctl not installed - `sudo apt install uhubctl`") + note(" power no uhubctl not installed - `sudo apt install uhubctl`") possible, why = can_complete_a_reflash() - log(f" reflash {'ok ' if possible else 'no '} {why}") - log(f" {'':11} {'':9} TrenchCoat pinned at {trench_coat.TRENCH_COAT_COMMIT[:8]}") + note(f" reflash {'ok ' if possible else 'no '} {why}") + note(f" {'':11} {'':9} TrenchCoat pinned at {trench_coat.TRENCH_COAT_COMMIT[:8]}") + note("```") def main(): diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py index 74f48347..1232ceea 100644 --- a/dev/hil/trench_coat.py +++ b/dev/hil/trench_coat.py @@ -102,6 +102,39 @@ def bundled_uf2(root, target): return path +# Where automounters actually put a volume: /media/LABEL, /media//LABEL, +# /run/media//LABEL. Deliberately NOT a recursive walk - the first +# version used rglob over /media, /run/media and /mnt, which walks whatever +# else happens to be mounted there. On a 512MB Zero 2 W with a drive under +# /mnt that is minutes of I/O and enough memory pressure to take the runner +# down with it, which is the most likely reason a bench job died without +# uploading its logs. +MOUNT_ROOTS = ("/media", "/run/media") + + +def find_bootloader_drives(): + """Mounted RPI-RP2 volumes, found without walking arbitrary filesystems.""" + drives = [] + for root in MOUNT_ROOTS: + base = Path(root) + if not base.is_dir(): + continue + try: + candidates = list(base.iterdir()) + for entry in list(candidates): + if entry.is_dir(): + candidates.extend(entry.iterdir()) + except OSError: + continue + for entry in candidates: + try: + if entry.is_dir() and (entry / "INFO_UF2.TXT").exists(): + drives.append(str(entry)) + except OSError: + continue + return drives + + def mount_rpi_rp2(): """Mount an RPI-RP2 volume that nothing automounted. Returns the path or None.""" by_label = Path("/dev/disk/by-label") @@ -141,14 +174,19 @@ def bootsel_touch(port): def wait_for_drive(core, timeout=45): - """Wait for a bootloader drive, mounting it ourselves if nothing else does.""" + """Wait for a bootloader drive, mounting it ourselves if nothing else does. + + Uses our bounded search rather than TrenchCoat's os.walk of /media and + /Volumes - same answer on a normal machine, without the risk of walking + into something large. + """ deadline = time.monotonic() + timeout while time.monotonic() < deadline: - drives = core.list_rpi_rp2_drives() + drives = find_bootloader_drives() if drives: return drives if mount_rpi_rp2(): - drives = core.list_rpi_rp2_drives() + drives = find_bootloader_drives() if drives: return drives time.sleep(1) @@ -194,8 +232,7 @@ def flash(port, target, root): # get_all_boards_into_bootloader() a no-op instead of a second attempt, and # keeps it away from the healthy boards on the bench. ray.Ray.find_board_ports = classmethod(lambda cls: []) - original_list_drives = core.list_rpi_rp2_drives - core.list_rpi_rp2_drives = lambda: original_list_drives() or drives + core.list_rpi_rp2_drives = lambda: find_bootloader_drives() or drives # Their failure path prints advice and calls sys.exit; make it an exception # this harness can report and carry on from. diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index 7085c09d..fe0012a6 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -306,7 +306,8 @@ def test_usb_device_path_walks_up_to_the_device(tmp_path, monkeypatch): [ ("1-1.4:1.0", ("1-1", "4")), ("1-1.2.3:1.0", ("1-1.2", "3")), - ("2-3:1.0", ("2", "3")), + # A root-hub port (no dot) is deliberately not switchable - see + # test_hub_location_refuses_a_root_hub_port. ], ) def test_hub_location_splits_the_usb_path(tmp_path, monkeypatch, usb_path, expected): @@ -355,6 +356,11 @@ def find_board_ports(cls): monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: root) monkeypatch.setattr(trench_coat, "load", lambda root: (core, ray)) monkeypatch.setattr(trench_coat, "bundled_uf2", lambda root, target: Path(f"/uf2/{target}.uf2")) + # Drive discovery is the real function now, and it neither finds a fake + # drive nor fails fast - without this the tests sit through its full wait. + monkeypatch.setattr(trench_coat, "find_bootloader_drives", lambda: list(drives)) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: list(drives)) + monkeypatch.setattr(trench_coat, "bootsel_touch", lambda port: None) return seen @@ -430,3 +436,25 @@ def enter_bootloader_mode(self): assert attempts == ["machine.bootloader()"] assert touched == ["/dev/ttyACM1"] assert drives == ["/media/RPI-RP2"] + + +def test_hub_location_refuses_a_root_hub_port(tmp_path, monkeypatch, capsys): + """Cutting a root hub takes every board down, not just the broken one.""" + device = tmp_path / "1-1:1.0" + device.mkdir(parents=True) + (device / "busnum").write_text("1\n") + + link = tmp_path / "link" + link.symlink_to(device) + monkeypatch.setattr(recover, "Path", lambda p: link if str(p).endswith("/device") else Path(p)) + + assert recover.hub_location("/dev/ttyACM1") is None + assert "cut every board on the bus" in capsys.readouterr().out + + +def test_power_cycle_stands_down_without_a_switchable_hub_port(monkeypatch, capsys): + monkeypatch.setattr(recover.shutil, "which", lambda _name: "/usr/sbin/uhubctl") + monkeypatch.setattr(recover, "hub_location", lambda _port: None) + + assert recover.power_cycle("/dev/ttyACM1") is False + assert "could not work out which hub port" in capsys.readouterr().out From a128281676dabff337f3f62afbd2c18226467b6e Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 14:38:51 +0000 Subject: [PATCH 07/32] ci(hil): bench is healthy again - hand the queue back to the config matrix All three boards answer, WPC included. What actually revived it is not knowable from here (the runner died several times in between, and a host reboot drops USB power, which cold-boots a bus-powered board), so this claims nothing about which rung did it. The run did finally produce the capability report, which is the thing three earlier attempts failed to deliver: serial ok 3 ports usb reset NO /dev/bus/usb/001/003 not writable - needs a udev rule power NO uhubctl not installed reflash ok udisksctl is available So the two non-destructive rungs are both unavailable today, which means a wedged board currently goes straight from "send it a Ctrl-C" to "wipe its flash". RUNNER_SETUP.md now records this with the one-time setup that fixes it - a udev rule for the USB reset, and apt install uhubctl plus a hub that switches port power. Fixed a side effect the report exposed: probing a board interrupts it, so a survey left all three sitting at a bare REPL with the Vector application stopped. A "nothing to recover" run was quietly taking the bench out of service. Each board is now restarted after it is probed. Swapped which workflow carries the push trigger. They share the hil-bench concurrency group and GitHub keeps only one pending run per group, so only one can hold it at a time; the bench is healthy, so the config matrix - the point of this PR - gets the queue back and recovery goes dispatch-only. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-config-matrix.yml | 21 ++++++++----- .github/workflows/hil-recover.yml | 19 ++++++------ dev/hil/RUNNER_SETUP.md | 39 +++++++++++++++++++++++++ dev/hil/recover.py | 9 ++++++ 4 files changed, 71 insertions(+), 17 deletions(-) diff --git a/.github/workflows/hil-config-matrix.yml b/.github/workflows/hil-config-matrix.yml index 9f79997c..1e2b8f32 100644 --- a/.github/workflows/hil-config-matrix.yml +++ b/.github/workflows/hil-config-matrix.yml @@ -14,17 +14,22 @@ name: HIL config matrix # a private network and physically reflashes hardware. Fork gating is designed # in dev/hil/DESIGN.md §4 and is not built yet. # -# The push trigger that let this be validated pre-merge is currently removed on -# purpose. It shares the `hil-bench` concurrency group with hil-recover.yml, and -# GitHub keeps only one *pending* run per group - so a push touching both would -# have them race, with the loser silently cancelled. While the bench needs -# recovering, recovery gets the queue. Re-add this to run the matrix again: +# The `push` trigger exists so this can be validated before merging, since a +# workflow_dispatch workflow is not dispatchable until it reaches the default +# branch. Drop the push trigger once this is on main. # -# push: -# branches: [claude/wpc-hil-config-validation-rc62dn] -# paths: [dev/hil/bench.py, dev/hil/config_matrix.py, .github/workflows/hil-config-matrix.yml] +# It shares the `hil-bench` concurrency group with hil-recover.yml, and GitHub +# keeps only one *pending* run per group, so only one of the two carries a push +# trigger at a time. Recovery's is off while this one is on. on: + push: + branches: + - claude/wpc-hil-config-validation-rc62dn + paths: + - dev/hil/bench.py + - dev/hil/config_matrix.py + - .github/workflows/hil-config-matrix.yml workflow_dispatch: inputs: target: diff --git a/.github/workflows/hil-recover.yml b/.github/workflows/hil-recover.yml index 677a98bb..54a0c6a9 100644 --- a/.github/workflows/hil-recover.yml +++ b/.github/workflows/hil-recover.yml @@ -15,8 +15,16 @@ name: HIL recover a wedged board # that never responded. # # Deliberately no `pull_request` trigger - self-hosted runner, real hardware. -# The `push` trigger exists so this can be used before it reaches the default -# branch; drop it once this is on main. +# +# The push trigger is deliberately off. It shares the `hil-bench` concurrency +# group with the config matrix, and GitHub keeps only one *pending* run per +# group, so a push touching both has them race with the loser silently +# cancelled. The bench is healthy, so the matrix gets the queue. To run this +# again before it reaches the default branch, add: +# +# push: +# branches: [claude/wpc-hil-config-validation-rc62dn] +# paths: [dev/hil/recover.py, dev/hil/trench_coat.py, .github/workflows/hil-recover.yml] on: workflow_dispatch: @@ -37,13 +45,6 @@ on: description: "Touch the board into BOOTSEL even if nothing here can mount the drive to flash it" type: boolean default: false - push: - branches: - - claude/wpc-hil-config-validation-rc62dn - paths: - - dev/hil/recover.py - - dev/hil/trench_coat.py - - .github/workflows/hil-recover.yml permissions: contents: read diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 92846f4b..a8e7a6b4 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -209,4 +209,43 @@ closed, so a bare `cat` makes the kernel echo the board's own output back into i board then tries to parse its log lines as USB API requests. Use `stty -F /dev/ttyACM0 raw -echo 115200` first, or `mpremote connect /dev/ttyACM0 repl`. +## Recovering a wedged board + +A board can deadlock with its USB device still enumerated and the firmware gone: the port is +there, `mpremote` opens it, nothing answers. `dev/hil/recover.py` escalates through four +rungs and stops as soon as the board replies. Run it from +[`hil-recover.yml`](../../.github/workflows/hil-recover.yml), or by hand: + +```bash +cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/recover.py +``` + +It opens with a report of which rungs this runner can actually use. As measured on the bench +on 2026-08-28: + +| rung | state | what it needs | +|---|---|---| +| drain the console | works | serial access, which the `dialout` group already gives | +| reset the USB device | **unavailable** | write access to `/dev/bus/usb/*` — a udev rule | +| power cycle the hub port | **unavailable** | `sudo apt install uhubctl`, and a hub that switches port power | +| reflash via TrenchCoat | works | `udisksctl`, which is present | + +The two unavailable rungs are worth enabling — they are the non-destructive ones, and without +them a wedged board goes straight from "send it a Ctrl-C" to "wipe its flash". For the USB +reset, a rule like this grants the runner user write access to the boards' USB nodes: + +``` +# /etc/udev/rules.d/60-vector-hil.rules +SUBSYSTEM=="usb", ATTR{idVendor}=="2e8a", MODE="0660", GROUP="dialout" +``` + +then `sudo udevadm control --reload && sudo udevadm trigger`. Note the boards are USB bus +powered, so a hub with per-port power switching makes the power rung a genuine cold boot — +the most reliable recovery short of a reflash. + +The reflash rung hands the board to [TrenchCoat](https://github.com/warped-pinball/trench-coat), +pinned by commit in `dev/hil/trench_coat.py`, which resets into the ROM bootloader, wipes the +flash with `nuke.uf2` and writes the real firmware. It is destructive: run +`dev/hil/flash_and_check.py` afterwards to put Vector back on the board. + See [DESIGN.md](DESIGN.md) for the test architecture and the security model for fork PRs. diff --git a/dev/hil/recover.py b/dev/hil/recover.py index e1af0104..41861a46 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -131,6 +131,15 @@ def survey(board_map): claimed.add(board_map[chip_id]) note(f"{port:16} {'ok':14} {chip_id or '?'}") + # Probing a board means interrupting it, which leaves it sitting at a + # bare REPL with the Vector application stopped. A tool that only meant + # to look must put it back, or a "nothing to recover" run quietly + # leaves the whole bench not running its firmware. + try: + mpremote("connect", port, "exec", "--no-follow", "import machine; machine.reset()", timeout=30) + except subprocess.TimeoutExpired: + log(f" warning: could not restart {port} after probing it") + note("```") unclaimed = sorted(set(board_map.values()) - claimed) From eff52fb5fcdb8172d552ffc6dd07f17ea0e0f9ca Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 14:54:08 +0000 Subject: [PATCH 08/32] fix(hil): reset the board before watching its first boot The matrix reached the bench and failed all three boards without checking a single config: "never reported its web server within 90s", while each board's console showed it running perfectly - DISCOVERY, RESOURCE, FRAM, SCORE all flowing. Nothing was wrong with the boards. The ready marker is printed exactly once per boot, and I had removed the reset that guarantees we are watching when it happens. dev/flash.py resets at the end of flashing, but flashing runs over every board before the matrix starts, so each board booted minutes earlier and printed its marker while a later board was still being flashed. The harness then opened a console and waited 90s for something that had already gone past. bench.reset_board exists for precisely this, and says so in its docstring. The pipeline refactor dropped it: Session.wait_for_boot() assumes the caller just triggered a reset, which holds for every iteration except the first. Session now has start() - reset, then watch - used for a board's first boot and by restore_default when it has to reconnect. mpremote is safe there, unlike mid-matrix, because no connection of ours is open yet and there is no handoff to lose. Two tests: one that start() resets before it waits, and one that run_matrix uses it for the first boot. Verified the second fails against the regression and passes against the fix, rather than trusting that it would. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/config_matrix.py | 23 +++++++++++++++++-- dev/tests/test_hil_config_matrix.py | 34 +++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index ea1c0c49..bfede229 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -71,6 +71,7 @@ parse_board_map, prime_usb, repl_reset, + reset_board, resolve_targets, set_game_config, time_limit, @@ -236,6 +237,24 @@ def __init__(self, port, boot_timeout=BOOT_TIMEOUT): self.client = None self.boot_log = [] + def start(self): + """First boot of the run: reset the board, then watch it come up. + + Every later boot is triggered by reboot() over our own connection, but + the first one has to be triggered here, and it must be triggered: + dev/flash.py resets at the end of flashing, and flashing runs over + every board before any of this starts, so the board booted minutes ago + and printed its one ready marker long before we opened a console. That + is bench.reset_board's whole reason for existing, and dropping it is + what made the first matrix run time out on all three boards without + checking a single config. + + mpremote is safe here, unlike mid-matrix: no connection of ours is + open yet, so there is no handoff to lose. + """ + reset_board(self.port) + return self.wait_for_boot() + def wait_for_boot(self): self.connection, self.boot_log = wait_for_server(self.port, timeout=self.boot_timeout) prime_usb(self.connection) @@ -339,7 +358,7 @@ def restore_default(session, target): default = DEFAULT_GAMENAME[target] try: if session.connection is None: - session.wait_for_boot() + session.start() session.set_config(default) session.reboot() log(f" restored {default}") @@ -374,7 +393,7 @@ def run_matrix(board, args): try: group(f"Config bundle {target} on {port}") try: - client = session.wait_for_boot() + client = session.start() check_bundle(client, target, configs) finally: endgroup() diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 1747a7c4..b2ca961f 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -307,9 +307,14 @@ def __init__(self, port, boot_timeout=None, dies_after=None): self.boot_log = [] self.configs_set = [] self.boots = 0 + self.starts = 0 self.nudges = 0 self.restored = False + def start(self): + self.starts += 1 + return self.wait_for_boot() + def wait_for_boot(self): self.boots += 1 if self.dies_after is not None and self.boots > self.dies_after: @@ -640,3 +645,32 @@ def test_time_limit_restores_the_previous_handler(): pass assert signal.getsignal(signal.SIGALRM) is before + + +def test_run_matrix_resets_the_board_before_watching_its_first_boot(monkeypatch, fake_repo): + """The regression that failed all three boards without checking one config. + + The ready marker is printed once per boot. dev/flash.py resets at the end + of flashing and flashing runs over every board first, so by the time the + matrix opens a console the board booted minutes ago and the marker is gone. + Every later boot is triggered by reboot(); the first one has to be + triggered by start(). + """ + session = FakeSession("/dev/ttyFAKE") + + run_board(monkeypatch, fake_repo, session) + + assert session.starts == 1, "the first boot must be preceded by a reset" + + +def test_session_start_resets_then_waits(monkeypatch): + """start() is a reset plus a wait, in that order.""" + order = [] + monkeypatch.setattr(cm, "reset_board", lambda port: order.append(f"reset {port}")) + monkeypatch.setattr(cm, "wait_for_server", lambda port, timeout=None: (order.append("wait"), (types.SimpleNamespace(close=lambda: None), []))[1]) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + cm.Session("/dev/ttyFAKE").start() + + assert order == ["reset /dev/ttyFAKE", "wait"] From e9375879895bd134eaae0a7a630f5fedd7625af1 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 20:52:24 +0000 Subject: [PATCH 09/32] fix(hil): tell a crashed boot from a slow one, and retry the crash Watching the console for the ready marker is precise but brittle in two directions, and the first full bench run hit both. A board that raises on the way up drops to the REPL and will never print the marker, so waiting out the budget only delays a failure the console already explains. The WPC board did exactly that - an ImportError inside create_schedule - and the harness spent 90s to report "never reported its web server", burying the traceback 20 lines down in a console dump. The watcher now recognises the REPL banner and fails at once with the traceback leading the message. That crash is intermittent - the same board booted cleanly 18s later - so it is retried once. Losing a whole board's 63 configs to one flaky boot buys nothing. The retry never hides it: every crash is counted, logged with its traceback, and listed in both the run summary and the job summary, so a board that only sometimes comes up cannot read as a clean pass. The other direction: the marker is printed once, so any boot we did not trigger ourselves looks identical to a board that never came up - which is precisely the first-boot bug fixed in eff52fb. Before failing a timeout, the watcher now asks the board over the USB API. One that answers is up, whatever we did or did not see, and it says so as a warning rather than passing silently. Deliberately not changed: the timeouts. Across 70 boots on the bench the range is 11.8s to 25.6s with a 15.5s mean, against a 90s matrix budget and 150s for the flash harness. Nothing is marginal, and no boot in that run failed for want of time - so raising the limits would only make a stopped board cost more. Those numbers are now recorded in DESIGN.md so the next person does not have to re-derive them. Tests use the real WPC console transcript as the fixture: crash detected with the traceback surfaced, healthy boot unaffected, slow boot still a timeout, REPL residue in the first moments not mistaken for a crash, retry-once-then-give-up, and the missed-marker fallback in both directions. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/DESIGN.md | 23 ++++ dev/hil/bench.py | 83 +++++++++++- dev/hil/config_matrix.py | 56 +++++++- dev/tests/test_hil_config_matrix.py | 201 +++++++++++++++++++++++++++- 4 files changed, 352 insertions(+), 11 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 73f2d94f..029d5439 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -385,6 +385,29 @@ run: each board's output back into it; the logs show all three boards parsing their own log lines as USB API requests. Fixed in both HIL workflows. +#### How a boot is watched + +The ready marker (`Server: Loop Forever`) is printed exactly once per boot, which +makes watching for it precise but brittle in two directions. Both are handled: + +- **The board crashes instead of coming up.** MicroPython prints a traceback and + drops to the REPL; nothing will ever print the marker. The watcher recognises + the REPL banner and fails immediately *with the traceback*, rather than burning + the timeout and reporting "never reported its web server" — which is what + happened on the first full bench run and buried the real cause. A crash is + retried once, because it can be intermittent and losing a whole board's matrix + to one flaky boot buys nothing; every crash is counted and reported in the run + summary either way, so a board that only sometimes boots never reads as clean. +- **The marker goes past before we are watching.** Any boot we did not trigger + ourselves looks identical to a board that never came up. On timeout the watcher + asks the board over the USB API before failing: a board that answers is up, + whatever we did or did not see, and says so as a warning. + +Timeouts are set from measurement, not guesswork. Across 70 boots on the bench: +**11.8s minimum, 15.5s mean, 25.6s maximum.** The matrix allows 90s and the flash +harness 150s, so neither is close to marginal — a boot that exceeds them has +stopped, not slowed. + #### Findings and limits - **Two WPC configs cannot be selected at all.** `configuration.gamename` is a diff --git a/dev/hil/bench.py b/dev/hil/bench.py index a949011f..6db2b610 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -60,11 +60,28 @@ # connect_to_wifi() by this point, so the marker covers both transports. READY_MARKER = "Server: Loop Forever" +# The other way a boot can end. If the application raises, MicroPython prints a +# traceback and drops to the REPL - and then nothing is ever going to print the +# ready marker, so waiting out the timeout only delays a failure we can already +# describe. These lines appear when, and only when, the program has exited: +# main.py never returns on a healthy board. +# +# Watching for this is what turns "never reported its web server within 90s" +# into the traceback that actually explains the boot. +CRASH_MARKERS = ('Type "help()" for more information.', ">>>") + +# Ignore a crash marker in the first moments after opening the port: it can be +# residue from whatever REPL session issued the reset, rather than this boot. +CRASH_MARKER_GRACE = 3 + # The marker is printed just *before* run_forever(), so give the event loop a # moment to actually accept the listening socket. http_get's retries cover any # remainder. SERVER_SETTLE_SECONDS = 2 +# Measured across 70 boots on the bench: 11.8s min, 15.5s mean, 25.6s max. The +# flash harness keeps 150s; the matrix runs a tighter one (see +# config_matrix.MATRIX_BOOT_TIMEOUT) because it pays the timeout per config. BOOT_TIMEOUT = 150 HTTP_TIMEOUT = 10 @@ -73,6 +90,19 @@ class CheckFailure(Exception): pass +class BootCrash(CheckFailure): + """The firmware raised on the way up and dropped to the REPL. + + Distinct from a timeout because it is actionable in a way a timeout is + not: the console holds the traceback, and a retry is worth one attempt + where waiting longer is worth nothing. + """ + + def __init__(self, message, transcript=None): + super().__init__(message) + self.transcript = transcript or [] + + def log(msg): print(msg, flush=True) @@ -407,10 +437,17 @@ def wait_for_server(port, timeout=BOOT_TIMEOUT): instead gives an exact ready signal and, on failure, the boot log that explains it. + Two ways this ends other than success: the board never gets there + (timeout), or the firmware raises and drops to the REPL (BootCrash). They + are worth distinguishing - a crash is described by the traceback sitting in + the console right now, and there is no point waiting out the rest of the + budget for a marker a dead program will never print. + Returns the open serial connection so the USB API can reuse it - the Pico exposes one CDC endpoint, so a second connection would fight this one. """ - deadline = time.monotonic() + timeout + started = time.monotonic() + deadline = started + timeout transcript = [] connection = None @@ -441,11 +478,31 @@ def wait_for_server(port, timeout=BOOT_TIMEOUT): transcript.append(text) if READY_MARKER in text: - elapsed = timeout - (deadline - time.monotonic()) + elapsed = time.monotonic() - started log(f" server up after {elapsed:.1f}s ({text.strip()!r})") time.sleep(SERVER_SETTLE_SECONDS) return connection, transcript + elapsed = time.monotonic() - started + if elapsed > CRASH_MARKER_GRACE and any(marker in text for marker in CRASH_MARKERS): + try: + connection.close() + except Exception: + pass + raise BootCrash( + f"{port} dropped to the REPL {elapsed:.1f}s into boot - the firmware raised on the way up:\n " + _explain_crash(transcript), + transcript, + ) + + # Before calling it a failure: ask the board directly. The ready marker is + # printed exactly once, so anything that costs us the moment it goes past - + # a reset we did not trigger, a board that booted while we were still + # flashing the next one - looks identical to a board that never came up. + # A board that answers its API is up, whatever we did or did not see. + if connection is not None and _server_is_answering(connection): + log(f"::warning::{port} is answering its API but its ready marker was never seen - " "the marker was probably printed before the console was open") + return connection, transcript + if connection is not None: try: connection.close() @@ -458,6 +515,28 @@ def wait_for_server(port, timeout=BOOT_TIMEOUT): ) +def _server_is_answering(connection): + """One cheap USB request, to tell 'missed the marker' from 'never booted'.""" + try: + prime_usb(connection) + response = UsbApiClient(connection).send_and_receive(route="/api/version", payload=None, timeout=10) + return response.get("status") == 200 + except Exception: + return False + + +def _explain_crash(transcript, lines=14): + """Pull the traceback out of a boot transcript, for the error message. + + The traceback is the whole value of catching this, so it leads; without one + the tail of the console is the next best thing. + """ + for index, line in enumerate(transcript): + if "Traceback (most recent call last)" in line: + return "\n ".join(transcript[index : index + lines]) + return "\n ".join(transcript[-lines:]) or "(nothing on the console)" + + def prime_usb(connection): """Clear both ends of the serial line before the first API request. diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index bfede229..c63a7dda 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -59,6 +59,7 @@ DEFAULT_GAMENAME, EXPECTED_FAULTS, REPO_ROOT, + BootCrash, CheckFailure, UsbApiClient, _dump_boot_log, @@ -236,6 +237,7 @@ def __init__(self, port, boot_timeout=BOOT_TIMEOUT): self.connection = None self.client = None self.boot_log = [] + self.crashes = [] def start(self): """First boot of the run: reset the board, then watch it come up. @@ -256,7 +258,30 @@ def start(self): return self.wait_for_boot() def wait_for_boot(self): - self.connection, self.boot_log = wait_for_server(self.port, timeout=self.boot_timeout) + """Watch one boot, retrying once if the firmware raises on the way up. + + A crash is not a timing problem and no amount of waiting fixes it - the + program has exited. But it can be intermittent, and losing a whole + board's matrix to one flaky boot buys nothing: the retry is what lets + the 63 configs actually get checked. + + The crash is never swallowed. Every one is counted, logged with its + traceback, and reported in the run summary, so a board that only + sometimes comes up still shows up as a problem rather than as a clean + run that happened to take longer. + """ + for attempt in range(2): + try: + self.connection, self.boot_log = wait_for_server(self.port, timeout=self.boot_timeout) + break + except BootCrash as crash: + self.crashes.append(str(crash)) + self.boot_log = crash.transcript + if attempt: + raise + log(f"::warning::{self.port} crashed on boot; retrying once. {crash}") + reset_board(self.port) + prime_usb(self.connection) self.client = UsbApiClient(self.connection) return self.client @@ -454,7 +479,7 @@ def run_matrix(board, args): restore_default(session, target) endgroup() - return passed, failures + return passed, failures, session.crashes def write_step_summary(results): @@ -467,6 +492,12 @@ def write_step_summary(results): for board, passed, failures in results: lines.append(f"| `{board['port']}` | {board['target']} | {len(passed) + len(failures)} | {len(passed)} | {len(failures)} |") + crashed = [(board, crash) for board, _passed, _failures in results for crash in (board.get("crashes") or [])] + if crashed: + lines += ["", "### Boot crashes (recovered by a retry)", ""] + for board, crash in crashed: + lines.append(f"- **{board['target']}** - {crash.splitlines()[0]}") + failed = [(board, config, reason) for board, _passed, failures in results for config, reason in failures] if failed: lines += ["", "### Failures", ""] @@ -532,7 +563,7 @@ def main(): results = [] for b in boards: try: - passed, failures = run_matrix(b, args) + passed, failures, crashes = run_matrix(b, args) except Exception as exc: # noqa: BLE001 # A board that cannot even be set up is one board's problem. The # bench is a singleton and a run is expensive, so the other boards @@ -540,7 +571,8 @@ def main(): # died on a TimeoutExpired escaping teardown, which threw away the # results already in hand and skipped the untouched board entirely. log(f"::error::{b['target']} on {b['port']}: {exc}") - passed, failures = [], [("(board setup)", str(exc))] + passed, failures, crashes = [], [("(board setup)", str(exc))], [] + b["crashes"] = crashes results.append((b, passed, failures)) log("") @@ -551,12 +583,26 @@ def main(): log("") log("=" * 60) total_failures = 0 + total_crashes = 0 for board, passed, failures in results: state = "FAIL" if failures else "ok" - log(f" {state:5} {board['port']:16} {board['target']:12} {len(passed)} passed, {len(failures)} failed") + crashes = board.get("crashes") or [] + crashed = f", {len(crashes)} boot crash(es)" if crashes else "" + log(f" {state:5} {board['port']:16} {board['target']:12} {len(passed)} passed, {len(failures)} failed{crashed}") total_failures += len(failures) + total_crashes += len(crashes) log("=" * 60) + # A crash that a retry got past is not a passing board. The configs were + # still checked, so it does not fail the run, but it is a firmware fault + # and gets said out loud rather than buried in a green result. + if total_crashes: + log("") + log(f"::warning::{total_crashes} boot crash(es) recovered by a retry - the firmware raised on the way up:") + for board, _passed, _failures in results: + for crash in board.get("crashes") or []: + log(f" {board['target']}: {crash}") + write_step_summary(results) if total_failures: diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index b2ca961f..69a5c35d 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -309,6 +309,7 @@ def __init__(self, port, boot_timeout=None, dies_after=None): self.boots = 0 self.starts = 0 self.nudges = 0 + self.crashes = [] self.restored = False def start(self): @@ -362,7 +363,7 @@ def check(_client, _target, _config, expected): def test_run_matrix_walks_every_config_on_a_healthy_board(monkeypatch, fake_repo): session = FakeSession("/dev/ttyFAKE") - passed, failures = run_board(monkeypatch, fake_repo, session) + passed, failures, _crashes = run_board(monkeypatch, fake_repo, session) assert failures == [] assert passed == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] @@ -387,7 +388,7 @@ def test_run_matrix_abandons_a_board_that_stops_answering(monkeypatch, fake_repo # survives exactly one config before going quiet. session = FakeSession("/dev/ttyFAKE", dies_after=2) - passed, failures = run_board(monkeypatch, fake_repo, session) + passed, failures, _crashes = run_board(monkeypatch, fake_repo, session) assert len(passed) == 1 assert session.boots <= 1 + cm.MAX_CONSECUTIVE_SETUP_FAILURES @@ -407,7 +408,7 @@ def one_bad_config(_client, _target, config, expected): return expected["name"] session = FakeSession("/dev/ttyFAKE") - passed, failures = run_board(monkeypatch, fake_repo, session, check=one_bad_config) + passed, failures, _crashes = run_board(monkeypatch, fake_repo, session, check=one_bad_config) assert passed == ["AttackMars_11", "Taxi_L4"] assert [config for config, _reason in failures] == ["Generic_WPC"] @@ -418,7 +419,7 @@ def always_fails(*_args, **_kwargs): raise bench.CheckFailure("board reports game name 'Generic System'") session = FakeSession("/dev/ttyFAKE") - passed, failures = run_board(monkeypatch, fake_repo, session, check=always_fails, keep_going=False) + passed, failures, _crashes = run_board(monkeypatch, fake_repo, session, check=always_fails, keep_going=False) assert passed == [] assert len(failures) == 1 @@ -674,3 +675,195 @@ def test_session_start_resets_then_waits(monkeypatch): cm.Session("/dev/ttyFAKE").start() assert order == ["reset /dev/ttyFAKE", "wait"] + + +# -------------------------------------------------------------------------- +# a boot that crashes rather than one that is slow +# -------------------------------------------------------------------------- + +# The real WPC console from the bench run that prompted this, trimmed. +CRASHED_BOOT = [ + "Connected to wifi with IP address: 192.168.2.175", + "----------", + "Starting server", + "----------", + "2021-01-01 00:00:13 [info ] > starting web server on port 80", + "Traceback (most recent call last):", + ' File "main.py", line 1, in ', + ' File "build/wpc/backend.py", line 1, in go', + ' File "build/wpc/GameStatus.py", line 1, in ', + "ImportError: no module named 'origin'", + "MicroPython v1.26.0-preview.255.g214d6413d.dirty on 2025-07-19; Raspberry Pi Pico 2 W with RP2350", + 'Type "help()" for more information.', + ">>> ", +] + +HEALTHY_BOOT = [ + "Connected to wifi with IP address: 192.168.2.175", + "2021-01-01 00:00:13 [info ] > starting web server on port 80", + "Server: Loop Forever", +] + + +class ScriptedConsole: + """A serial port replaying a boot transcript, one line per readline().""" + + def __init__(self, lines): + self.lines = list(lines) + self.closed = False + + def readline(self): + if not self.lines: + return b"" + return (self.lines.pop(0) + "\r\n").encode() + + def close(self): + self.closed = True + + +def watch_boot(monkeypatch, lines, elapsed=10.0): + """Run wait_for_server against a scripted console.""" + console = ScriptedConsole(lines) + monkeypatch.setattr(bench, "open_serial", lambda *a, **k: console) + monkeypatch.setattr(bench, "SERVER_SETTLE_SECONDS", 0) + monkeypatch.setattr(bench, "CRASH_MARKER_GRACE", -1) + return console + + +def test_a_crashed_boot_fails_immediately_with_the_traceback(monkeypatch): + """The failure that cost 90s and reported the wrong thing. + + A board that raised on the way up will never print the ready marker, so + waiting out the budget only delays a failure the console already explains. + """ + watch_boot(monkeypatch, CRASHED_BOOT) + + with pytest.raises(bench.BootCrash) as caught: + bench.wait_for_server("/dev/ttyFAKE", timeout=90) + + message = str(caught.value) + assert "dropped to the REPL" in message + # The traceback leads, because it is the whole reason to catch this. + assert "ImportError: no module named 'origin'" in message + assert "Traceback (most recent call last)" in message + assert "never reported its web server" not in message + + +def test_a_healthy_boot_is_unaffected(monkeypatch): + console = watch_boot(monkeypatch, HEALTHY_BOOT) + + connection, transcript = bench.wait_for_server("/dev/ttyFAKE", timeout=90) + + assert connection is console + assert "Server: Loop Forever" in transcript[-1] + + +def test_a_repl_prompt_in_the_first_moments_is_not_a_crash(monkeypatch): + """Residue from the REPL session that issued the reset is not this boot.""" + watch_boot(monkeypatch, [">>> "] + HEALTHY_BOOT) + monkeypatch.setattr(bench, "CRASH_MARKER_GRACE", 3600) + + connection, _transcript = bench.wait_for_server("/dev/ttyFAKE", timeout=90) + + assert connection is not None + + +def test_a_boot_that_is_merely_slow_still_times_out(monkeypatch): + """Slow is not the same as crashed, and still reports as a timeout.""" + watch_boot(monkeypatch, ["still booting..."]) + + with pytest.raises(bench.CheckFailure, match="never reported its web server") as caught: + bench.wait_for_server("/dev/ttyFAKE", timeout=0.5) + + assert not isinstance(caught.value, bench.BootCrash) + + +def test_wait_for_boot_retries_once_past_a_crash(monkeypatch): + """An intermittent crash must not cost a whole board's matrix.""" + attempts = [] + + def flaky(_port, timeout=None): + attempts.append(1) + if len(attempts) == 1: + raise bench.BootCrash("crashed on the way up", CRASHED_BOOT) + return types.SimpleNamespace(close=lambda: None), HEALTHY_BOOT + + monkeypatch.setattr(cm, "wait_for_server", flaky) + monkeypatch.setattr(cm, "reset_board", lambda _port: None) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + session = cm.Session("/dev/ttyFAKE") + session.wait_for_boot() + + assert len(attempts) == 2 + # Recovered, but recorded: a board that only sometimes boots is a fault. + assert len(session.crashes) == 1 + + +def test_wait_for_boot_gives_up_after_a_second_crash(monkeypatch): + def always_crashes(_port, timeout=None): + raise bench.BootCrash("crashed on the way up", CRASHED_BOOT) + + monkeypatch.setattr(cm, "wait_for_server", always_crashes) + monkeypatch.setattr(cm, "reset_board", lambda _port: None) + + session = cm.Session("/dev/ttyFAKE") + with pytest.raises(bench.BootCrash): + session.wait_for_boot() + + assert len(session.crashes) == 2 + + +def test_recovered_crashes_are_reported_not_buried(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + board = {"port": "/dev/ttyACM1", "target": "wpc", "crashes": ["dropped to the REPL 13.0s into boot\n ImportError"]} + cm.write_step_summary([(board, ["Taxi_L4"], [])]) + + rendered = summary.read_text() + assert "Boot crashes (recovered by a retry)" in rendered + assert "**wpc**" in rendered + + +def test_a_missed_marker_is_recovered_by_asking_the_board(monkeypatch, capsys): + """The ready marker prints once, so missing it must not read as a dead board. + + Anything that costs us the moment it goes past - a board that booted while + the next one was still being flashed, a reset we did not trigger - looks + exactly like a board that never came up. Asking the API tells them apart. + """ + console = watch_boot(monkeypatch, ["already booted, marker long gone"]) + monkeypatch.setattr(bench, "_server_is_answering", lambda _connection: True) + + connection, _transcript = bench.wait_for_server("/dev/ttyFAKE", timeout=0.5) + + assert connection is console + assert "ready marker was never seen" in capsys.readouterr().out + + +def test_a_board_that_answers_nothing_still_fails(monkeypatch): + watch_boot(monkeypatch, ["nothing useful"]) + monkeypatch.setattr(bench, "_server_is_answering", lambda _connection: False) + + with pytest.raises(bench.CheckFailure, match="never reported its web server"): + bench.wait_for_server("/dev/ttyFAKE", timeout=0.5) + + +def test_server_is_answering_reads_the_usb_api(monkeypatch): + monkeypatch.setattr(bench, "prime_usb", lambda _connection: None) + monkeypatch.setattr(bench, "UsbApiClient", lambda _c: types.SimpleNamespace(send_and_receive=lambda **kw: {"status": 200, "body": {"version": "1.7.13"}})) + + assert bench._server_is_answering(object()) is True + + +def test_server_is_answering_is_false_when_the_board_is_silent(monkeypatch): + monkeypatch.setattr(bench, "prime_usb", lambda _connection: None) + + def times_out(**_kwargs): + raise TimeoutError("no response") + + monkeypatch.setattr(bench, "UsbApiClient", lambda _c: types.SimpleNamespace(send_and_receive=times_out)) + + assert bench._server_is_answering(object()) is False From 494050acf4dacbe831142f401415e4f05d5b6b72 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 21:35:04 +0000 Subject: [PATCH 10/32] fix(hil): flash each board just before its own matrix 100 configs passed this run - sys11 39/39 and wpc 61/63, with the only two config failures being exactly the pre-existing 16-byte FRAM `gamename` bug this harness was built to find. data_east then failed at setup: it would not take a reset, mpremote timing out after 30s. The reason is the ordering, and it is the root cause behind this whole series of bench failures. data_east was flashed at the start and used last, so it spent 36 minutes running the application and printing SCORE / RESOURCE / DISCOVERY lines into a USB CDC console that nothing was draining. TrenchCoat says where that ends, in its own source: "if nothing ever drains the board's output, the USB CDC buffers fill up, MicroPython blocks writing to stdout, and the board deadlocks mid-script". That is almost certainly what killed the WPC board on the very first bench run too - flashed, then left running while sys11 worked through 39 configs, and silent by the time its turn came. I had been treating that as a mystery the recovery tool existed to clean up after; it was this all along. Each board is now flashed immediately before its own matrix, so it runs for seconds before we start talking to it. Boards waiting their turn sit at the REPL where inventory's probe left them, producing no output at all. Builds still happen up front - they touch no hardware. Second line of defence: a board that will not take a reset gets its console drained and one retry. Reading is the remedy for precisely this deadlock and costs three seconds, against writing off a board's whole matrix. No boot crashes were recorded this run, so the WPC ImportError did not recur - the crash detection and retry added in e937587 were not exercised on hardware. They stay: the failure they cover is real and was seen once. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/DESIGN.md | 25 +++++++++++++ dev/hil/config_matrix.py | 54 ++++++++++++++++++++++------ dev/tests/test_hil_config_matrix.py | 55 +++++++++++++++++++++++++++++ 3 files changed, 123 insertions(+), 11 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 029d5439..41fe866e 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -385,6 +385,31 @@ run: each board's output back into it; the logs show all three boards parsing their own log lines as USB API requests. Fixed in both HIL workflows. +#### Why boards are flashed one at a time + +Flashing every board up front and then working through them in turn is the +obvious ordering, and it is wrong. A board flashed first and used last spends +however long the boards ahead of it take — half an hour on a full run — running +the application and printing `SCORE:` / `RESOURCE:` / `DISCOVERY:` lines into a +USB CDC console that nothing is draining. TrenchCoat documents where that ends +(`src/ray.py`, `send_command`): + +> if nothing ever drains the board's output, the USB CDC buffers fill up, +> MicroPython blocks writing to stdout, and the board deadlocks mid-script + +That is not hypothetical. It killed `data_east` 36 minutes into a run — the +board simply stopped answering `mpremote` — and it is the most likely +explanation for the WPC board that went silent for an hour on the very first +bench run and needed recovering. + +So each board is flashed immediately before its own matrix. Boards waiting +their turn sit at the REPL, where inventory's probe leaves them, producing no +output at all. Builds still happen up front; they touch no hardware. + +As a second line, a board that will not take a reset gets its console drained +and one retry: reading is the remedy for exactly this deadlock and costs +seconds, against writing off a whole board's matrix. + #### How a boot is watched The ready marker (`Server: Loop Forever`) is printed exactly once per boot, which diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index c63a7dda..f8b3f06f 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -253,8 +253,18 @@ def start(self): mpremote is safe here, unlike mid-matrix: no connection of ours is open yet, so there is no handoff to lose. + + A board that will not take the reset gets one drain and one retry. A + board whose stdout is blocked on an undrained USB endpoint comes back + the moment somebody reads it, and reading costs three seconds - much + less than writing off a board's whole matrix. """ - reset_board(self.port) + try: + reset_board(self.port) + except Exception as exc: + log(f"::warning::{self.port} did not take a reset ({exc}); draining its console and retrying once") + drain_port(self.port) + reset_board(self.port) return self.wait_for_boot() def wait_for_boot(self): @@ -400,6 +410,34 @@ def restore_default(session, target): MAX_CONSECUTIVE_SETUP_FAILURES = 2 +def flash_before_matrix(board, workdir): + """Flash one board immediately before its own matrix, not all up front. + + The ordering is the point, and it is what the bench taught us. Flashing + every board first leaves the ones further down the queue running the + application for as long as the boards ahead of them take - half an hour or + more - printing to a USB CDC console that nothing is draining. TrenchCoat + documents where that ends (src/ray.py, send_command): "if nothing ever + drains the board's output, the USB CDC buffers fill up, MicroPython blocks + writing to stdout, and the board deadlocks mid-script". + + That is not a hypothetical. It is what killed data_east 36 minutes into a + run, and it is the most likely explanation for the WPC board that went + silent for an hour on the very first bench run. + + Flashed here, a board starts running seconds before we start talking to it. + Boards waiting their turn sit at the REPL, where inventory's probe left + them, producing no output at all. + """ + group(f"Flash {board['target']} on {board['port']}") + try: + config_path = bench.write_bench_config(board["target"], workdir) + bench.flash(board["target"], board["port"], workdir / board["target"], config_path) + log("flashed") + finally: + endgroup() + + def run_matrix(board, args): """Walk one board through its configs. Returns (passed, failures). @@ -544,25 +582,19 @@ def main(): raise CheckFailure(f"no attached board matches --target {', '.join(sorted(wanted))}") if not args.skip_flash: - # Flash first so the bundle under test is the one this checkout builds. - # Without it the matrix would validate whatever happened to be on the - # boards, which is the one thing it must not do. + # Building touches no hardware, so it all happens up front. Flashing + # does not - see flash_before_matrix(). for target in sorted({b["target"] for b in boards}): group(f"Build {target}") bench.build(target) log(f"built {target} at version {bench.source_version(target)}") endgroup() - for b in boards: - group(f"Flash {b['target']} on {b['port']}") - config_path = bench.write_bench_config(b["target"], workdir) - bench.flash(b["target"], b["port"], workdir / b["target"], config_path) - log("flashed") - endgroup() - results = [] for b in boards: try: + if not args.skip_flash: + flash_before_matrix(b, workdir) passed, failures, crashes = run_matrix(b, args) except Exception as exc: # noqa: BLE001 # A board that cannot even be set up is one board's problem. The diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 69a5c35d..177f5c46 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -9,6 +9,7 @@ from __future__ import annotations import json +import subprocess import sys import time import types @@ -867,3 +868,57 @@ def times_out(**_kwargs): monkeypatch.setattr(bench, "UsbApiClient", lambda _c: types.SimpleNamespace(send_and_receive=times_out)) assert bench._server_is_answering(object()) is False + + +def test_start_drains_and_retries_a_board_that_will_not_reset(monkeypatch): + """A board wedged on its own undrained output comes back when read. + + This is what took out data_east 36 minutes into a run: reading the console + is the remedy, and it costs seconds against a whole board's matrix. + """ + attempts = [] + drained = [] + + def reset(port): + attempts.append(port) + if len(attempts) == 1: + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + + monkeypatch.setattr(cm, "reset_board", reset) + monkeypatch.setattr(cm, "drain_port", lambda port, **kw: drained.append(port)) + monkeypatch.setattr(cm, "wait_for_server", lambda port, timeout=None: (types.SimpleNamespace(close=lambda: None), [])) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + cm.Session("/dev/ttyFAKE").start() + + assert len(attempts) == 2 + assert drained == ["/dev/ttyFAKE"] + + +def test_start_gives_up_if_the_retry_also_fails(monkeypatch): + def always_times_out(_port): + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + + monkeypatch.setattr(cm, "reset_board", always_times_out) + monkeypatch.setattr(cm, "drain_port", lambda port, **kw: None) + + with pytest.raises(subprocess.TimeoutExpired): + cm.Session("/dev/ttyFAKE").start() + + +def test_each_board_is_flashed_immediately_before_its_own_matrix(monkeypatch, fake_repo, tmp_path): + """Flashing every board up front is what wedges the ones waiting their turn. + + A board flashed and then left running for the half hour the boards ahead of + it take is printing into a USB console nothing drains, which is where + MicroPython deadlocks. Flashed here, it runs for seconds before we talk + to it. + """ + flashed = [] + monkeypatch.setattr(cm.bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + monkeypatch.setattr(cm.bench, "flash", lambda target, port, build_dir, config: flashed.append((target, port))) + + cm.flash_before_matrix({"target": "wpc", "port": "/dev/ttyACM1"}, tmp_path) + + assert flashed == [("wpc", "/dev/ttyACM1")] From 0eec1b794505fd9e3c9eedddf9dc11fbf9e5ff73 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 21:39:19 +0000 Subject: [PATCH 11/32] fix(hil): one wedged board must not abort the whole bench run The ordering fix stops boards deadlocking on their own undrained console going forward, but it cannot un-wedge a board already stuck from an earlier run - and data_east was. The next run died in `inventory`, before a single config was checked, because probe() let mpremote's TimeoutExpired propagate out of the survey. That is the wrong shape. Identifying the bench is a survey, and a board that does not answer is an answer: probe() now catches the timeout, drains the board's console and retries once (the remedy for this exact deadlock, and it costs three seconds), and failing that records the board as unresponsive and moves on. inventory() prints it as NOT ANSWERING instead of crashing. The matrix then skips unresponsive boards with a loud error pointing at recover.py, and runs the boards that do work. They are still counted as failed so the run stays red - a bench with a dead board on it is not a clean run - but the other boards' configs get checked rather than being lost to it. resolve_targets refuses to flash a board it could not identify, which is the one place where carrying on would be unsafe. For context on why this matters: the previous run got sys11 39/39 and wpc 61/63 with only the two known FRAM `gamename` failures. All of that would have been thrown away by one board that had been dead since before the run started. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/bench.py | 64 ++++++++++++++++++++++------- dev/hil/config_matrix.py | 14 ++++++- dev/tests/test_hil_config_matrix.py | 52 +++++++++++++++++++++++ 3 files changed, 115 insertions(+), 15 deletions(-) diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 6db2b610..b1165486 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -183,29 +183,53 @@ def list_ports(): return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] +CHIP_ID_SNIPPET = "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())" + + +def _ask_board(port, snippet, timeout=30): + """Run a snippet on a board, treating a hung board as an answer of its own. + + A wedged board makes mpremote hang until its timeout rather than fail, and + letting that propagate means one bad board aborts the whole run before + anything has been tested. It did exactly that: a board left deadlocked by + an earlier run took out the next run in `inventory`, before a single config + was checked. + + One drain and one retry, because a board blocked writing to an undrained + USB endpoint comes back the moment somebody reads it. + """ + try: + return mpremote("connect", port, "exec", snippet, timeout=timeout) + except subprocess.TimeoutExpired: + log(f" {port} did not answer in {timeout}s; draining its console and retrying once") + drain_port(port) + + try: + return mpremote("connect", port, "exec", snippet, timeout=timeout) + except subprocess.TimeoutExpired: + return None + + def probe(port): - """Return {port, chip_id, system, version} for one board. + """Return {port, chip_id, system, version, responsive} for one board. chip_id comes from the RP2040 itself so it survives any firmware state; system/version come from the flashed firmware and may be missing if the - board is unflashed or broken. + board is unflashed or broken. `responsive` is False for a board that never + answered at all - the caller decides what to do about it, but the survey + itself always completes. """ - board = {"port": port, "chip_id": None, "system": None, "version": None} + board = {"port": port, "chip_id": None, "system": None, "version": None, "responsive": True} - chip = mpremote( - "connect", port, "exec", - "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())", - timeout=30, - ) + chip = _ask_board(port, CHIP_ID_SNIPPET) + if chip is None: + board["responsive"] = False + return board if chip.returncode == 0: board["chip_id"] = chip.stdout.strip() - info = mpremote( - "connect", port, "exec", - "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)", - timeout=30, - ) - if info.returncode == 0 and info.stdout.strip(): + info = _ask_board(port, "import systemConfig;print(systemConfig.vectorSystem, systemConfig.SystemVersion)") + if info is not None and info.returncode == 0 and info.stdout.strip(): parts = info.stdout.split() board["system"] = parts[0] if len(parts) > 1: @@ -221,6 +245,9 @@ def inventory(): log(f"{'port':16} {'chip id':18} {'running':12} version") for b in boards: + if not b.get("responsive", True): + log(f"{b['port']:16} {'NOT ANSWERING':18} {'-':12} -") + continue log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") return boards @@ -299,6 +326,15 @@ def resolve_targets(boards, board_map): the signature of a previous mis-flash rather than of the hardware, and flashing on that basis would silently perpetuate it. """ + dead = [b for b in boards if not b.get("responsive", True)] + if dead: + raise CheckFailure( + "not answering: " + + ", ".join(b["port"] for b in dead) + + ".\nA board that will not talk cannot be identified, so it cannot be safely flashed.\n" + "Run dev/hil/recover.py to get it back." + ) + if board_map: unmapped = [b for b in boards if b["chip_id"] not in board_map] if unmapped: diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index f8b3f06f..76817c04 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -569,6 +569,18 @@ def main(): boards = inventory() endgroup() + # A board that will not answer is one board's problem. It cannot be + # identified and so cannot be safely flashed, but the boards that do work + # still have configs worth checking - and aborting the whole run before + # anything is tested is how a single board left wedged by an earlier run + # took out the next one entirely. + unresponsive = [b for b in boards if not b.get("responsive", True)] + boards = [b for b in boards if b.get("responsive", True)] + for b in unresponsive: + log(f"::error::{b['port']} is not answering - skipping it. Run dev/hil/recover.py to get it back.") + if not boards: + raise CheckFailure("no board on the bench is answering - run dev/hil/recover.py") + group("Resolve targets") boards = resolve_targets(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP"))) for b in boards: @@ -590,7 +602,7 @@ def main(): log(f"built {target} at version {bench.source_version(target)}") endgroup() - results = [] + results = [({"port": b["port"], "target": "(unknown)", "crashes": []}, [], [("(board setup)", "board is not answering - run dev/hil/recover.py")]) for b in unresponsive] for b in boards: try: if not args.skip_flash: diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 177f5c46..2a3b3ea2 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -922,3 +922,55 @@ def test_each_board_is_flashed_immediately_before_its_own_matrix(monkeypatch, fa cm.flash_before_matrix({"target": "wpc", "port": "/dev/ttyACM1"}, tmp_path) assert flashed == [("wpc", "/dev/ttyACM1")] + + +# -------------------------------------------------------------------------- +# one wedged board must not abort the whole bench +# -------------------------------------------------------------------------- + + +def test_probe_recovers_a_board_that_hangs_then_answers(monkeypatch): + """A drain is the remedy for a board blocked on its own undrained output.""" + calls = [] + drained = [] + + def hangs_once(*args, timeout=None): + calls.append(args) + if len(calls) == 1: + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + return types.SimpleNamespace(returncode=0, stdout="df13a50c13958980", stderr="") + + monkeypatch.setattr(bench, "mpremote", hangs_once) + monkeypatch.setattr(bench, "drain_port", lambda port, **kw: drained.append(port)) + + board = bench.probe("/dev/ttyACM2") + + assert board["responsive"] is True + assert board["chip_id"] == "df13a50c13958980" + assert drained == ["/dev/ttyACM2"] + + +def test_probe_reports_a_board_that_never_answers(monkeypatch): + """The survey completes rather than raising - it is one board's problem. + + A board left wedged by an earlier run took out the next run inside + inventory, before a single config was checked. + """ + + def always_hangs(*_args, timeout=None): + raise subprocess.TimeoutExpired(cmd="mpremote", timeout=30) + + monkeypatch.setattr(bench, "mpremote", always_hangs) + monkeypatch.setattr(bench, "drain_port", lambda port, **kw: None) + + board = bench.probe("/dev/ttyACM2") + + assert board["responsive"] is False + assert board["chip_id"] is None + + +def test_resolve_targets_refuses_to_flash_a_board_that_will_not_talk(): + boards = [{"port": "/dev/ttyACM0", "chip_id": "aaa", "system": "sys11", "responsive": True}, {"port": "/dev/ttyACM2", "chip_id": None, "system": None, "responsive": False}] + + with pytest.raises(bench.CheckFailure, match="not answering: /dev/ttyACM2"): + bench.resolve_targets(boards, {"aaa": "sys11"}) From 6f36dd264f90176fb6e14b00efc18eb307515ca5 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 23:45:51 +0000 Subject: [PATCH 12/32] ci(hil): hand the bench to recovery for data_east; correct the WPC diagnosis The matrix is working. This run: sys11 39/39, wpc 61 passed / 2 failed with the full 63-config leg completed, 100 configs checked. Three of the mechanisms added over the last few pushes earned their place on hardware: - WPC crashed on boot four times and the retry carried it through all 63 configs anyway, with every crash counted and reported rather than hidden. - sys11 would not take a reset once; the drain-and-retry recovered it. - data_east was still wedged from before the run, and the survey skipped it instead of aborting - which is the only reason the other 100 configs were checked at all. The four crashes also correct the diagnosis I posted earlier. Two are ImportError: no module named 'origin' and two are OSError: [Errno 2] ENOENT, both raised from create_schedule during boot. Both are "file not found", on files that exist - the same board boots cleanly on the retry every time. That is the filesystem, not memory pressure as I first guessed. A nuke.uf2 wipe before reflashing is the obvious next thing to try, and it is what recover.py's reflash rung already does. data_east is hard-wedged: the drain read 0 bytes and the write timed out, so it needs the reflash rung. Moving the push trigger back to hil-recover.yml so it gets the bench - the two workflows share the hil-bench concurrency group and GitHub keeps only one pending run per group, so only one can carry the trigger at a time. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-config-matrix.yml | 20 +++++++------------- .github/workflows/hil-recover.yml | 20 +++++++++++--------- dev/hil/DESIGN.md | 9 +++++++++ 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/.github/workflows/hil-config-matrix.yml b/.github/workflows/hil-config-matrix.yml index 1e2b8f32..0e16d092 100644 --- a/.github/workflows/hil-config-matrix.yml +++ b/.github/workflows/hil-config-matrix.yml @@ -14,22 +14,16 @@ name: HIL config matrix # a private network and physically reflashes hardware. Fork gating is designed # in dev/hil/DESIGN.md §4 and is not built yet. # -# The `push` trigger exists so this can be validated before merging, since a -# workflow_dispatch workflow is not dispatchable until it reaches the default -# branch. Drop the push trigger once this is on main. +# The push trigger is currently off. It shares the `hil-bench` concurrency group +# with hil-recover.yml and GitHub keeps only one *pending* run per group, so +# only one of the two carries it at a time. data_east needs recovering, so +# recovery has the queue. Re-add to run the matrix again: # -# It shares the `hil-bench` concurrency group with hil-recover.yml, and GitHub -# keeps only one *pending* run per group, so only one of the two carries a push -# trigger at a time. Recovery's is off while this one is on. +# push: +# branches: [claude/wpc-hil-config-validation-rc62dn] +# paths: [dev/hil/bench.py, dev/hil/config_matrix.py, .github/workflows/hil-config-matrix.yml] on: - push: - branches: - - claude/wpc-hil-config-validation-rc62dn - paths: - - dev/hil/bench.py - - dev/hil/config_matrix.py - - .github/workflows/hil-config-matrix.yml workflow_dispatch: inputs: target: diff --git a/.github/workflows/hil-recover.yml b/.github/workflows/hil-recover.yml index 54a0c6a9..ba1d406d 100644 --- a/.github/workflows/hil-recover.yml +++ b/.github/workflows/hil-recover.yml @@ -16,17 +16,19 @@ name: HIL recover a wedged board # # Deliberately no `pull_request` trigger - self-hosted runner, real hardware. # -# The push trigger is deliberately off. It shares the `hil-bench` concurrency -# group with the config matrix, and GitHub keeps only one *pending* run per -# group, so a push touching both has them race with the loser silently -# cancelled. The bench is healthy, so the matrix gets the queue. To run this -# again before it reaches the default branch, add: -# -# push: -# branches: [claude/wpc-hil-config-validation-rc62dn] -# paths: [dev/hil/recover.py, dev/hil/trench_coat.py, .github/workflows/hil-recover.yml] +# The `push` trigger exists so this can be used before it reaches the default +# branch. It shares the `hil-bench` concurrency group with the config matrix and +# GitHub keeps only one *pending* run per group, so only one of the two carries +# it at a time. Drop both once this is on main. on: + push: + branches: + - claude/wpc-hil-config-validation-rc62dn + paths: + - dev/hil/recover.py + - dev/hil/trench_coat.py + - .github/workflows/hil-recover.yml workflow_dispatch: inputs: port: diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 41fe866e..c2b2e776 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -445,6 +445,15 @@ stopped, not slowed. ordinary CI. Fixing the two that exist means shortening the filenames or widening the field (which is a `MapVersion` change — the 96-byte record is fully used). +- **WPC boots intermittently fail on a missing file.** Four crashes in one + 63-config run, in two flavours — `ImportError: no module named 'origin'` and + `OSError: [Errno 2] ENOENT` — both raised from `phew/server.py:create_schedule` + during boot. Both are "file not found", on files that plainly exist: the same + board boots fine on the retry every time. That points at the on-board + filesystem returning ENOENT under some condition, not at the config bundle and + not (as first guessed) at memory pressure. A `nuke.uf2` wipe before reflashing, + which is what `recover.py`'s reflash rung does, is the obvious thing to try + next: if the littlefs on that board is degraded, it would clear it. - **`/api/adjustments/status` 500s for configs with no `Adjustments` section.** `GameDefsLoad` assigns the parsed config straight to `SharedState.gdata` without merging `safe_defaults` into it, so the key is simply absent and From 36195d5fdd2b53d7f73c4616ef12f5d74f7ba687 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 00:20:30 +0000 Subject: [PATCH 13/32] fix(hil): bound TrenchCoat's serial writes; stop re-running futile recoveries Recovery ran the full ladder against the wedged data_east board and could not get it back. Every rung is now accounted for: drain 0 bytes read, and the write timed out - it is not merely blocked on a full output buffer usb reset permission denied on /dev/bus/usb/001/088 - needs a udev rule power uhubctl not installed reflash the board could not be talked into its ROM bootloader: TrenchCoat's route in goes over the REPL and a board this far gone cannot run one statement, and the 1200 baud touch fell to [Errno 110] So it needs a physical replug. RUNNER_SETUP.md now says that outright, next to the one-time setup that would make the two non-destructive rungs available and change the answer. The reflash rung also burned its entire 600s step budget before the SIGALRM backstop cut it off, and the reason is worth fixing: TrenchCoat's Ray.open uses serial.Serial(port, 115200, timeout=0.1) - the READ timeout only, the same trap this harness had. Against a healthy board on a desktop, which is what it is written for, that is fine; against a board that is wedged by definition, the Ctrl-C it writes on open has nothing to time it out. trench_coat.py now injects a write timeout into its connections rather than patching upstream: it is their code, and this is our unusual way of using it. That rung now fails in seconds instead of ten minutes. Worth noting the backstop did its job. bench.time_limit was added on the theory that a board in a bad enough state can block a syscall no library timeout covers; this is that case, in somebody else's code. Recovery's push trigger is off again. It has exhausted what it can do here, so re-running it on every push costs 15 minutes and changes nothing until the board is replugged or those rungs are enabled. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-recover.yml | 20 +++++++-------- dev/hil/RUNNER_SETUP.md | 9 +++++++ dev/hil/trench_coat.py | 26 +++++++++++++++++++- dev/tests/test_hil_recover.py | 41 ++++++++++++++++++++++++++++++- 4 files changed, 83 insertions(+), 13 deletions(-) diff --git a/.github/workflows/hil-recover.yml b/.github/workflows/hil-recover.yml index ba1d406d..84d7579e 100644 --- a/.github/workflows/hil-recover.yml +++ b/.github/workflows/hil-recover.yml @@ -16,19 +16,17 @@ name: HIL recover a wedged board # # Deliberately no `pull_request` trigger - self-hosted runner, real hardware. # -# The `push` trigger exists so this can be used before it reaches the default -# branch. It shares the `hil-bench` concurrency group with the config matrix and -# GitHub keeps only one *pending* run per group, so only one of the two carries -# it at a time. Drop both once this is on main. +# The push trigger is off. Recovery has been run against the wedged data_east +# board and exhausted every rung available on this runner, so re-running it on +# each push costs 15 minutes and changes nothing until somebody replugs the +# board or the two non-destructive rungs are enabled (see RUNNER_SETUP.md). +# Dispatch it when there is a reason to. To arm it on a branch again: +# +# push: +# branches: [] +# paths: [dev/hil/recover.py, dev/hil/trench_coat.py, .github/workflows/hil-recover.yml] on: - push: - branches: - - claude/wpc-hil-config-validation-rc62dn - paths: - - dev/hil/recover.py - - dev/hil/trench_coat.py - - .github/workflows/hil-recover.yml workflow_dispatch: inputs: port: diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index a8e7a6b4..07de0d45 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -243,6 +243,15 @@ then `sudo udevadm control --reload && sudo udevadm trigger`. Note the boards ar powered, so a hub with per-port power switching makes the power rung a genuine cold boot — the most reliable recovery short of a reflash. +> [!IMPORTANT] +> As measured, **a badly wedged board cannot be recovered from software on this +> runner.** The drain reads nothing, the USB reset and power cycle are both +> unavailable for want of the setup above, and a board too far gone to run one +> statement cannot be talked into its ROM bootloader either — TrenchCoat's route +> in goes over the REPL, and the 1200 baud touch fell to `[Errno 110]`. That +> board needs someone to unplug it and plug it back in. Enabling the two rungs +> above is what would change that. + The reflash rung hands the board to [TrenchCoat](https://github.com/warped-pinball/trench-coat), pinned by commit in `dev/hil/trench_coat.py`, which resets into the ROM bootloader, wipes the flash with `nuke.uf2` and writes the real firmware. It is destructive: run diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py index 1232ceea..c5a84320 100644 --- a/dev/hil/trench_coat.py +++ b/dev/hil/trench_coat.py @@ -37,7 +37,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) -from bench import CheckFailure, log, open_serial # noqa: E402 +from bench import SERIAL_WRITE_TIMEOUT, CheckFailure, log, open_serial # noqa: E402 # Pinned, like every other third-party input to this bench. Bumping it means # reviewing what changed in the flashing sequence first. @@ -93,6 +93,29 @@ def load(root): return core, ray +def bound_serial_writes(ray): + """Give TrenchCoat's serial connections a write timeout. + + `Ray.open` uses `serial.Serial(port, 115200, timeout=0.1)`, which is the + READ timeout only - the same trap this harness had. On a desktop talking to + a healthy board that is fine, which is the case TrenchCoat is written for. + Here the board is wedged by definition, so the Ctrl-C that `open()` writes + blocks with nothing to time it out: one recovery run spent its entire 600s + step budget inside enter_bootloader_mode before the SIGALRM backstop cut it + short. + + Injected rather than patched upstream: it is their code, and this is our + unusual way of using it. + """ + original = ray.serial.Serial + + def bounded(*args, **kwargs): + kwargs.setdefault("write_timeout", SERIAL_WRITE_TIMEOUT) + return original(*args, **kwargs) + + ray.serial.Serial = bounded + + def bundled_uf2(root, target): if target not in TARGET_UF2: raise CheckFailure(f"no TrenchCoat UF2 known for target {target!r} (have: {', '.join(sorted(TARGET_UF2))})") @@ -220,6 +243,7 @@ def flash(port, target, root): Returns True if TrenchCoat reported the board back as a serial device. """ core, ray = load(clone(root)) + bound_serial_writes(ray) uf2 = bundled_uf2(root, target) drives = enter_bootloader(core, ray, port) diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index fe0012a6..f8599c5f 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -342,7 +342,7 @@ def enter_bootloader_mode(self): def find_board_ports(cls): return ["/dev/ttyACM0", "/dev/ttyACM1", "/dev/ttyACM2"] - ray = types.SimpleNamespace(Ray=FakeRay) + ray = types.SimpleNamespace(Ray=FakeRay, serial=types.SimpleNamespace(Serial=lambda *a, **k: object())) core = types.SimpleNamespace( list_rpi_rp2_drives=lambda: list(drives), graceful_exit=lambda now=False: None, @@ -458,3 +458,42 @@ def test_power_cycle_stands_down_without_a_switchable_hub_port(monkeypatch, caps assert recover.power_cycle("/dev/ttyACM1") is False assert "could not work out which hub port" in capsys.readouterr().out + + +def test_trench_coats_serial_writes_are_bounded(monkeypatch): + """TrenchCoat opens with a read timeout only, which hangs on a wedged board. + + One recovery run spent its whole 600s step budget inside + enter_bootloader_mode because of this, and only the SIGALRM backstop ended + it. Their code is written for a healthy board on a desktop; ours is pointed + at a board that is broken by definition. + """ + opened = {} + + class FakeSerialModule: + @staticmethod + def Serial(*args, **kwargs): + opened.update(kwargs) + return object() + + ray = types.SimpleNamespace(serial=FakeSerialModule) + trench_coat.bound_serial_writes(ray) + ray.serial.Serial("/dev/ttyFAKE", 115200, timeout=0.1) + + assert opened["write_timeout"] == bench.SERIAL_WRITE_TIMEOUT + + +def test_bound_serial_writes_leaves_an_explicit_timeout_alone(monkeypatch): + opened = {} + + class FakeSerialModule: + @staticmethod + def Serial(*args, **kwargs): + opened.update(kwargs) + return object() + + ray = types.SimpleNamespace(serial=FakeSerialModule) + trench_coat.bound_serial_writes(ray) + ray.serial.Serial("/dev/ttyFAKE", write_timeout=99) + + assert opened["write_timeout"] == 99 From 4bdfc8a6d08134f8f12ebe026d0d3106ca08f4ac Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 04:45:31 +0000 Subject: [PATCH 14/32] ci(hil): one workflow for the whole bench, running on every PR Four HIL workflows become one, `hil.yml`, with three stages: recover, flash and health-check, config matrix. They shared the `hil-bench` concurrency group, and GitHub keeps only one *pending* run per group - so a push touching two of them had them race with the loser silently cancelled. I have spent several rounds moving a push trigger between two files to work around that. One workflow takes one lease and runs the stages in order, which removes the problem rather than managing it. Combining also makes recovery automatic, which is the bigger win. recover.py costs ~30s when every board answers and repairs one that does not, so running it as the first stage turns "a board wedged, so the next four runs were useless" - which is exactly what happened - into a bench that heals itself before it tests anything. Every stage runs even when an earlier one fails, and the verdict is taken at the end: one broken board should not cost the signal from the other two. Per-PR, but not via `pull_request`. The bench is a self-hosted runner on a private network wired to real hardware, and a pull_request trigger would let a fork PR execute its own workflow, its own dev/hil/**, and its own dependencies there. `workflow_run` on "Build and Deploy" fires on every push to a PR and always runs from the default branch, using the default branch's code - so a PR cannot change what the Pi executes. The bench job checks out the default branch and then takes only src/ from the commit under test: the firmware is the PR's, the harness is not. Fork PRs are gated to a manual dispatch with a message saying so; making them automatic-with-approval needs a hardware-lab Environment with required reviewers, which is a repository setting rather than a file. Also: #380 shortened HarleyDavidson_L3 and GilliganIsland_L9, the two configs the bench found could never be selected on real hardware. The test that enforced the 16-byte FRAM limit had them on an allowlist and correctly reported that the allowlist was now stale, so the rule is absolute again with no exceptions carried. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- .github/workflows/hil-config-matrix.yml | 132 ------------- .github/workflows/hil-flash-check.yml | 93 ---------- .github/workflows/hil-recover.yml | 123 ------------- .github/workflows/hil-smoke.yml | 135 -------------- .github/workflows/hil.yml | 235 ++++++++++++++++++++++++ dev/hil/DESIGN.md | 52 ++++-- dev/hil/RUNNER_SETUP.md | 24 +-- dev/tests/test_hil_config_matrix.py | 33 ++-- 8 files changed, 301 insertions(+), 526 deletions(-) delete mode 100644 .github/workflows/hil-config-matrix.yml delete mode 100644 .github/workflows/hil-flash-check.yml delete mode 100644 .github/workflows/hil-recover.yml delete mode 100644 .github/workflows/hil-smoke.yml create mode 100644 .github/workflows/hil.yml diff --git a/.github/workflows/hil-config-matrix.yml b/.github/workflows/hil-config-matrix.yml deleted file mode 100644 index 0e16d092..00000000 --- a/.github/workflows/hil-config-matrix.yml +++ /dev/null @@ -1,132 +0,0 @@ -name: HIL config matrix - -# Boots every attached board against every game config it can be flashed with, -# and checks that the board reports that config's game name. A config that -# fails to apply does not fault or crash - the board quietly comes up on the -# generic definition for its hardware - so the game name is the assertion that -# separates "loaded" from "silently fell back". -# -# This is DESIGN.md's G3. Runtime is dominated by the board with the most -# configs (WPC, 63) at roughly 15-25s per boot cycle, so a full run is well -# over half an hour; the inputs below exist to make a targeted run cheap. -# -# Deliberately no `pull_request` trigger - this targets a self-hosted runner on -# a private network and physically reflashes hardware. Fork gating is designed -# in dev/hil/DESIGN.md §4 and is not built yet. -# -# The push trigger is currently off. It shares the `hil-bench` concurrency group -# with hil-recover.yml and GitHub keeps only one *pending* run per group, so -# only one of the two carries it at a time. data_east needs recovering, so -# recovery has the queue. Re-add to run the matrix again: -# -# push: -# branches: [claude/wpc-hil-config-validation-rc62dn] -# paths: [dev/hil/bench.py, dev/hil/config_matrix.py, .github/workflows/hil-config-matrix.yml] - -on: - workflow_dispatch: - inputs: - target: - description: "Only run boards for this target, e.g. wpc (blank = every attached board)" - type: string - default: "" - configs: - description: "Comma-separated config names to run instead of all of them, e.g. AttackMars_11,Taxi_L4" - type: string - default: "" - limit: - description: "Stop after this many configs per board (blank = no limit)" - type: string - default: "" - skip_flash: - description: "Matrix what is already on the boards instead of building and flashing first" - type: boolean - default: false - stop_on_first_failure: - description: "Stop a board's matrix at its first failing config" - type: boolean - default: false - -permissions: - contents: read - -# Flashing is destructive and the bench is one set of boards. Never interleave. -# Shares the group with the other HIL workflows on purpose. -concurrency: - group: hil-bench - cancel-in-progress: false - -jobs: - config-matrix: - runs-on: [self-hosted, vector-hil] - timeout-minutes: 180 - - steps: - # actions/checkout is safe here ONLY because every trigger above is - # repo-internal. If a fork-reachable trigger is ever added, this must go - # back to running from the runner's pinned clone ($VECTOR_HIL_REPO) - - # otherwise a fork PR would execute its own harness code on the bench. - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - with: - # --changed-since needs history to diff against; a shallow clone has - # no merge base with the default branch. - fetch-depth: 0 - - - name: Prepare environment - run: | - test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } - echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" - - # VECTOR_HIL_BOARD_MAP, VECTOR_HIL_WIFI_* and friends arrive from the - # runner's .env, which the runner process exports into every job. They - # are deliberately NOT restated in an `env:` block here: the `env` - # context only covers workflow/job/step-level vars, so `${{ env.X }}` - # would evaluate to empty and shadow the real value. - - name: Boot every config on every board - # Inputs go through the environment rather than into the script text: - # `${{ }}` interpolation would splice a dispatcher-supplied string - # straight into the shell. - env: - HIL_TARGET: ${{ inputs.target }} - HIL_CONFIGS: ${{ inputs.configs }} - HIL_LIMIT: ${{ inputs.limit }} - HIL_SKIP_FLASH: ${{ inputs.skip_flash }} - HIL_STOP_ON_FIRST_FAILURE: ${{ inputs.stop_on_first_failure }} - HIL_EVENT: ${{ github.event_name }} - HIL_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - run: | - args="" - if [ -n "${HIL_TARGET:-}" ]; then args="$args --target $HIL_TARGET"; fi - if [ -n "${HIL_CONFIGS:-}" ]; then args="$args --configs $HIL_CONFIGS"; fi - if [ -n "${HIL_LIMIT:-}" ]; then args="$args --limit $HIL_LIMIT"; fi - if [ "${HIL_SKIP_FLASH:-}" = "true" ]; then args="$args --skip-flash"; fi - if [ "${HIL_STOP_ON_FIRST_FAILURE:-}" = "true" ]; then args="$args --stop-on-first-failure"; fi - - # On a push, run the configs the push touched first so a bad config - # fails in the first minute rather than the fortieth. The ref has to - # be fetched explicitly - a checkout leaves no tracking ref for a - # branch it did not check out. - if [ "$HIL_EVENT" = "push" ] && git fetch --quiet origin "$HIL_DEFAULT_BRANCH"; then - args="$args --changed-since FETCH_HEAD" - fi - - # shellcheck disable=SC2086 # args is a deliberately word-split list - python dev/hil/config_matrix.py $args - - - name: Board serial logs on failure - if: failure() - run: | - # A board left mid-matrix may still be printing something useful; - # grab a few seconds of console from each. - # - # `stty raw -echo` first, and it is not optional. A tty reverts to the - # driver default - ECHO on - once every handle is closed, so a bare - # `cat` makes the kernel echo the board's own output straight back at - # it. The last run caught this happening: the boards' consoles filled - # with "USB REQ: invalid request format: RESOURCE: RAM=69..." as they - # parsed their own log lines as USB API requests. - for dev in /dev/ttyACM*; do - echo "--- $dev" - timeout 5 stty -F "$dev" raw -echo 115200 || true - timeout 8 cat "$dev" || true - done diff --git a/.github/workflows/hil-flash-check.yml b/.github/workflows/hil-flash-check.yml deleted file mode 100644 index 103583b6..00000000 --- a/.github/workflows/hil-flash-check.yml +++ /dev/null @@ -1,93 +0,0 @@ -name: HIL flash and health check - -# Builds firmware for each attached board, flashes it, and health-checks the -# API over both USB and HTTP once the board boots. -# -# Deliberately no `pull_request` trigger — this targets a self-hosted runner on -# a private network and physically reflashes hardware. Fork gating is designed -# in dev/hil/DESIGN.md §4 and is not built yet. -# -# The `push` trigger exists so this can be validated before merging, since a -# workflow_dispatch workflow is not dispatchable until it reaches the default -# branch. Drop the push trigger once this is on main. - -on: - workflow_dispatch: - inputs: - skip_http: - description: "USB checks only (skip the network stack)" - type: boolean - default: false - skip_flash: - description: "Health-check what is already flashed, do not reflash" - type: boolean - default: false - inventory_only: - description: "Just print each board's chip id, for building VECTOR_HIL_BOARD_MAP" - type: boolean - default: false - identify: - description: "Blink each board in turn so you can see which physical board is which" - type: boolean - default: false - push: - branches: - - claude/hil-testing-design-s564ln - paths: - - dev/hil/flash_and_check.py - - .github/workflows/hil-flash-check.yml - -permissions: - contents: read - -# Flashing is destructive and the bench is one set of boards. Never interleave. -concurrency: - group: hil-bench - cancel-in-progress: false - -jobs: - flash-and-check: - runs-on: [self-hosted, vector-hil] - timeout-minutes: 45 - - steps: - # actions/checkout is safe here ONLY because every trigger above is - # repo-internal. If a fork-reachable trigger is ever added, this must go - # back to running from the runner's pinned clone ($VECTOR_HIL_REPO) — - # otherwise a fork PR would execute its own harness code on the bench. - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - - name: Prepare environment - run: | - test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } - echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" - - # VECTOR_HIL_BOARD_MAP, VECTOR_HIL_WIFI_* and friends arrive from the - # runner's .env, which the runner process exports into every job. They - # are deliberately NOT restated in an `env:` block here: the `env` - # context only covers workflow/job/step-level vars, so `${{ env.X }}` - # would evaluate to empty and shadow the real value. - - name: Flash and health-check every board - run: | - args="" - if [ "${{ inputs.skip_http }}" = "true" ]; then args="$args --skip-http"; fi - if [ "${{ inputs.skip_flash }}" = "true" ]; then args="$args --skip-flash"; fi - if [ "${{ inputs.inventory_only }}" = "true" ]; then args="$args --inventory-only"; fi - if [ "${{ inputs.identify }}" = "true" ]; then args="$args --identify"; fi - python dev/hil/flash_and_check.py $args - - - name: Board serial logs on failure - if: failure() - run: | - # A board that failed its health check may still be printing something - # useful; grab a few seconds of console from each. - # - # `stty raw -echo` first, and it is not optional. A tty reverts to the - # driver default - ECHO on - once every handle is closed, so a bare - # `cat` makes the kernel echo the board's own output straight back at - # it, and the board then parses its own log lines as USB API requests. - for dev in /dev/ttyACM*; do - echo "--- $dev" - timeout 5 stty -F "$dev" raw -echo 115200 || true - timeout 8 cat "$dev" || true - done diff --git a/.github/workflows/hil-recover.yml b/.github/workflows/hil-recover.yml deleted file mode 100644 index 84d7579e..00000000 --- a/.github/workflows/hil-recover.yml +++ /dev/null @@ -1,123 +0,0 @@ -name: HIL recover a wedged board - -# Gets a bench board that has stopped answering back into a usable state, -# without anyone walking over to it. -# -# A Vector board can deadlock with the USB device still enumerated and the -# firmware gone - the port is there, mpremote opens it, nothing ever answers. -# TrenchCoat names the mechanism in its own source: if nothing drains the -# board's output, the CDC buffers fill, MicroPython blocks writing to stdout, -# and the board deadlocks. That is what took the WPC board out mid-matrix. -# -# dev/hil/recover.py escalates cheapest-first (drain, USB reset, hub power -# cycle, then a UF2 reflash over the ROM bootloader) and stops as soon as the -# board answers. Run it before anything else when a HIL job reports a board -# that never responded. -# -# Deliberately no `pull_request` trigger - self-hosted runner, real hardware. -# -# The push trigger is off. Recovery has been run against the wedged data_east -# board and exhausted every rung available on this runner, so re-running it on -# each push costs 15 minutes and changes nothing until somebody replugs the -# board or the two non-destructive rungs are enabled (see RUNNER_SETUP.md). -# Dispatch it when there is a reason to. To arm it on a branch again: -# -# push: -# branches: [] -# paths: [dev/hil/recover.py, dev/hil/trench_coat.py, .github/workflows/hil-recover.yml] - -on: - workflow_dispatch: - inputs: - port: - description: "Recover only this port, e.g. /dev/ttyACM1 (blank = every board that is not answering)" - type: string - default: "" - target: - description: "Target for the dead board, e.g. wpc (blank = deduce it from the boards that still answer)" - type: string - default: "" - no_reflash: - description: "Stop before replacing the firmware" - type: boolean - default: false - force_bootsel: - description: "Touch the board into BOOTSEL even if nothing here can mount the drive to flash it" - type: boolean - default: false - -permissions: - contents: read - -# Shares the bench with the other HIL workflows. Recovery must never run -# alongside a job that is driving the boards. -concurrency: - group: hil-bench - cancel-in-progress: false - -jobs: - recover: - runs-on: [self-hosted, vector-hil] - timeout-minutes: 30 - - steps: - # Safe here only because every trigger above is repo-internal; see the - # note in hil-config-matrix.yml. - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - - name: Prepare environment - run: | - test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } - echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" - - # TrenchCoat does the actual reflash (dev/hil/trench_coat.py drives it). - # Cloned here rather than at runtime so a network problem fails the job - # visibly instead of halfway through recovering a board. Pin lives in - # dev/hil/trench_coat.py; recover.py re-checks the checkout matches it. - - name: Fetch TrenchCoat - run: | - commit=$(python -c "import sys; sys.path.insert(0, 'dev/hil'); import trench_coat; print(trench_coat.TRENCH_COAT_COMMIT)") - root="$PWD/build/hil/trench-coat" - mkdir -p "$(dirname "$root")" - if [ ! -d "$root/.git" ]; then - git clone --quiet https://github.com/warped-pinball/trench-coat "$root" - fi - git -C "$root" fetch --quiet origin "$commit" - git -C "$root" checkout --quiet "$commit" - echo "trench-coat at $(git -C "$root" rev-parse --short HEAD)" - ls -l "$root/uf2" - - - name: Recover - env: - HIL_PORT: ${{ inputs.port }} - HIL_TARGET: ${{ inputs.target }} - HIL_NO_REFLASH: ${{ inputs.no_reflash }} - HIL_FORCE_BOOTSEL: ${{ inputs.force_bootsel }} - run: | - args="" - if [ -n "${HIL_PORT:-}" ]; then args="$args --port $HIL_PORT"; fi - if [ -n "${HIL_TARGET:-}" ]; then args="$args --target $HIL_TARGET"; fi - if [ "${HIL_NO_REFLASH:-}" = "true" ]; then args="$args --no-reflash"; fi - if [ "${HIL_FORCE_BOOTSEL:-}" = "true" ]; then args="$args --force-bootsel"; fi - - # shellcheck disable=SC2086 # args is a deliberately word-split list - python dev/hil/recover.py $args - - - name: Board state afterwards - if: always() - run: | - # stty raw -echo first: a tty reverts to ECHO-on once every handle is - # closed, so a bare `cat` feeds the board its own output back. Both - # calls are under `timeout`, because opening a tty waits for carrier - # and a wedged board can block even stty - which is what left this - # step running for 32 minutes on the last run. - echo "--- serial ports" - ls -l /dev/ttyACM* 2>/dev/null || echo "(none)" - echo "--- bootloader drives" - ls -l /dev/disk/by-label/ 2>/dev/null | grep -i rpi || echo "(none)" - for dev in /dev/ttyACM*; do - [ -e "$dev" ] || continue - echo "--- $dev" - timeout 5 stty -F "$dev" raw -echo 115200 || true - timeout 5 cat "$dev" || true - done diff --git a/.github/workflows/hil-smoke.yml b/.github/workflows/hil-smoke.yml deleted file mode 100644 index ca4c9cbc..00000000 --- a/.github/workflows/hil-smoke.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: HIL smoke - -# Proves the bench runner is wired up correctly: the runner picks up jobs, the -# bench environment reaches them, the boards enumerate, and each one answers -# over serial. It does not test firmware — see dev/hil/DESIGN.md for that. -# -# Deliberately no `pull_request` trigger. This targets a self-hosted runner on -# a private network, and a pull_request trigger would let any fork PR run code -# against it. Fork gating is designed in DESIGN.md §4 and is not built yet. -# -# The `push` trigger exists so this can be validated before merging, since a -# workflow_dispatch workflow is not dispatchable until it reaches the default -# branch. Drop the push trigger once this is on main. - -on: - workflow_dispatch: - push: - branches: - - claude/hil-testing-design-s564ln - -permissions: {} - -# The bench is a single set of physical boards. Queue, never interleave. -concurrency: - group: hil-bench - cancel-in-progress: false - -jobs: - smoke: - runs-on: [self-hosted, vector-hil] - timeout-minutes: 15 - - # No actions/checkout on purpose: under the design's trust model the bench - # runs harness code from a trusted checkout the runner already has, not - # from the PR. $VECTOR_HIL_REPO points at it. - - steps: - - name: Runner host - run: | - echo "user: $(whoami)" - echo "kernel: $(uname -srm)" - echo "arch: $(dpkg --print-architecture)" - free -h | awk '/Mem:/ {print "memory: " $2 " total, " $7 " available"}' - echo "uptime:$(uptime -p | sed 's/^up//')" - - - name: Bench environment - run: | - fail=0 - for var in VECTOR_HIL_REPO VECTOR_HIL_VENV VECTOR_HIL_WIFI_SSID VECTOR_HIL_WIFI_PASSWORD; do - eval "val=\${$var:-}" - if [ -z "$val" ]; then - echo "MISSING: $var" - fail=1 - else - echo "set: $var" - fi - done - [ "$fail" -eq 0 ] || { - echo "The runner's .env is incomplete. It is read at service start:" - echo " cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start" - exit 1 - } - test -x "$VECTOR_HIL_VENV/bin/python" || { echo "venv python missing at $VECTOR_HIL_VENV"; exit 1; } - test -d "$VECTOR_HIL_REPO/.git" || { echo "no clone at $VECTOR_HIL_REPO"; exit 1; } - - # dev/detect_boards.py shells out to a bare `mpremote` from PATH and - # silently returns {} when it is missing, so the venv must be ON PATH - # for later steps - invoking its python by absolute path is not enough. - echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" - - - name: Serial access - run: | - if ! ls /dev/ttyACM* >/dev/null 2>&1; then - echo "no /dev/ttyACM* devices - check the USB hub and power" - exit 1 - fi - ls -l /dev/ttyACM* - id -nG | grep -qw dialout || { - echo "runner user is not in the dialout group; reboot or restart the service" - exit 1 - } - - - name: mpremote sees the boards - run: | - # Checked separately because detect_boards.py collapses "mpremote is - # missing", "mpremote failed", and "no boards attached" into the same - # empty result, which makes a real failure indistinguishable from an - # empty bench. - command -v mpremote || { echo "mpremote not on PATH"; exit 1; } - mpremote version - - echo "--- mpremote devs ---" - mpremote devs | tee /tmp/devs.txt - test -s /tmp/devs.txt || { - echo "mpremote sees no devices, though /dev/ttyACM* exist." - echo "Boards may be busy or in an odd state; power-cycle the hub and retry." - exit 1 - } - - - name: Detect and query boards - run: | - # detect_boards.py prints progress text before the JSON, so the - # machine-readable part is the last line only. - raw=$(python "$VECTOR_HIL_REPO/dev/detect_boards.py") - echo "$raw" - json=$(printf '%s\n' "$raw" | tail -1) - - python - "$json" <<'PY' - import json, subprocess, sys - - boards = json.loads(sys.argv[1]) - - if not boards: - sys.exit("no boards detected") - - rc = 0 - for kind, ports in sorted(boards.items()): - for port in ports: - probe = subprocess.run( - ["mpremote", "connect", port, "exec", - "import systemConfig; print(systemConfig.vectorSystem, systemConfig.SystemVersion)"], - capture_output=True, text=True, timeout=60) - if probe.returncode: - print(f"FAIL {kind:12} {port:16} {probe.stderr.strip()}") - rc = 1 - else: - print(f"ok {kind:12} {port:16} {probe.stdout.strip()}") - - print(f"\n{sum(len(p) for p in boards.values())} board(s) responding") - sys.exit(rc) - PY - - - name: Summary - if: always() - run: echo "Bench smoke ${{ job.status }}" diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml new file mode 100644 index 00000000..bdf7cc03 --- /dev/null +++ b/.github/workflows/hil.yml @@ -0,0 +1,235 @@ +name: HIL + +# Everything the hardware bench does, in one workflow: recover the boards, +# flash and health-check them, then boot every board against every game config. +# +# It replaces hil-smoke.yml, hil-flash-check.yml, hil-config-matrix.yml and +# hil-recover.yml. They were separate files sharing one `hil-bench` concurrency +# group, and GitHub keeps only one *pending* run per group - so a push touching +# two of them had them race, with the loser silently cancelled. One workflow +# takes one lease on the bench and runs the stages in order. +# +# It also makes recovery automatic. `recover.py` costs ~30s when every board +# answers and repairs one that does not, so running it first turns "a board +# wedged and the next four runs were useless" into a self-healing bench. +# +# --------------------------------------------------------------------------- +# Why workflow_run, and not pull_request +# --------------------------------------------------------------------------- +# +# This runs on a self-hosted runner on a private network, wired to real +# hardware. A `pull_request` trigger would let any fork PR execute its own code +# there - its own workflow, its own dev/hil/**, its own dependencies. +# +# `workflow_run` is the primitive that makes per-PR hardware testing safe: +# GitHub always runs it *from the default branch, using the default branch's +# code*, whatever the triggering PR contains. A PR cannot change what the Pi +# executes. It supplies exactly one thing - the commit that gets built and +# flashed. See dev/hil/DESIGN.md §4. +# +# Fork PRs are not run automatically even so: the gate job stops them and says +# how to run one deliberately. Making forks automatic-with-approval needs a +# `hardware-lab` GitHub Environment with required reviewers (DESIGN.md §4); +# that is a repository setting, not something this file can create. + +on: + # Fires when "Build and Deploy" finishes, which is every push to a PR. + workflow_run: + workflows: ["Build and Deploy"] + types: [completed] + + workflow_dispatch: + inputs: + stages: + description: "Which stages to run: all, recover, flash-check, config-matrix" + type: string + default: "all" + target: + description: "Limit the config matrix to one target, e.g. wpc (blank = all attached boards)" + type: string + default: "" + configs: + description: "Comma-separated config names for the matrix instead of all of them" + type: string + default: "" + limit: + description: "Stop the matrix after this many configs per board" + type: string + default: "" + + # Temporary, so this can be validated before it reaches the default branch: + # a workflow_run workflow does nothing until it is on main. Drop this once + # merged - after that, workflow_run covers every PR. + push: + branches: + - claude/wpc-hil-config-validation-rc62dn + +permissions: + contents: read + +# One set of physical boards. Queue, never interleave. +concurrency: + group: hil-bench + cancel-in-progress: false + +jobs: + gate: + name: Decide whether to touch the bench + runs-on: ubuntu-latest + outputs: + run_bench: ${{ steps.decide.outputs.run_bench }} + reason: ${{ steps.decide.outputs.reason }} + commit: ${{ steps.decide.outputs.commit }} + steps: + - name: Decide + id: decide + env: + EVENT: ${{ github.event_name }} + # For workflow_run these describe the run that triggered us. + BUILD_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + BUILD_EVENT: ${{ github.event.workflow_run.event }} + BUILD_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + BUILD_HEAD_REPO: ${{ github.event.workflow_run.head_repository.full_name }} + THIS_REPO: ${{ github.repository }} + run: | + run_bench=true + reason="running" + commit="${BUILD_HEAD_SHA:-$GITHUB_SHA}" + + if [ "$EVENT" = "workflow_run" ]; then + # No point flashing a build that failed. + if [ "$BUILD_CONCLUSION" != "success" ]; then + run_bench=false + reason="the build it follows concluded '$BUILD_CONCLUSION', so there is nothing good to flash" + # A fork's code must not run on the bench unattended. workflow_run + # protects *this* workflow, but the firmware being flashed is still + # the fork's, and a board runs it on a private network. + elif [ -n "$BUILD_HEAD_REPO" ] && [ "$BUILD_HEAD_REPO" != "$THIS_REPO" ]; then + run_bench=false + reason="the change comes from the fork '$BUILD_HEAD_REPO'. Review it, then run this workflow by hand (Run workflow) to put it on the bench." + fi + fi + + echo "run_bench=$run_bench" >> "$GITHUB_OUTPUT" + echo "reason=$reason" >> "$GITHUB_OUTPUT" + echo "commit=$commit" >> "$GITHUB_OUTPUT" + + { + echo "### HIL gate" + echo + echo "- event: \`$EVENT\`" + echo "- commit under test: \`$commit\`" + echo "- bench: **$([ "$run_bench" = true ] && echo "running" || echo "skipped")** - $reason" + } >> "$GITHUB_STEP_SUMMARY" + + bench: + name: Bench + needs: gate + if: needs.gate.outputs.run_bench == 'true' + runs-on: [self-hosted, vector-hil] + timeout-minutes: 180 + + steps: + # On workflow_run this checks out the DEFAULT BRANCH, not the PR - that + # is the whole security property, and it must stay that way. The board + # gets the PR's firmware because the harness builds it from the commit + # below, not because the PR's harness code runs here. + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + with: + fetch-depth: 0 + + - name: Check out the commit under test + if: github.event_name == 'workflow_run' + env: + COMMIT: ${{ needs.gate.outputs.commit }} + run: | + # Only the firmware sources move to the PR's commit. dev/hil/** and + # this workflow stay on the default branch, which is what makes a + # fork PR unable to change what runs here. + git fetch --quiet origin "$COMMIT" || git fetch --quiet --no-tags origin "+refs/heads/*:refs/remotes/origin/*" + git checkout --quiet "$COMMIT" -- src/ + echo "firmware sources at $COMMIT; harness from ${{ github.ref_name }}" + + - name: Prepare environment + run: | + test -x "$VECTOR_HIL_VENV/bin/python" || { echo "bench venv missing"; exit 1; } + echo "$VECTOR_HIL_VENV/bin" >> "$GITHUB_PATH" + + # 1. Recover. Cheap when the bench is healthy, and the difference between + # a wedged board costing one run and costing every run until someone + # notices. + - name: Recover any wedged board + id: recover + if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'recover') + continue-on-error: true + run: python dev/hil/recover.py + + # 2. Flash and health-check: DESIGN.md G1/G2. Exercises the API over both + # USB and HTTP, which the config matrix does not. + - name: Flash and health-check every board + id: flash_check + if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'flash-check') + continue-on-error: true + run: python dev/hil/flash_and_check.py + + # 3. The config matrix: DESIGN.md G3. + - name: Boot every config on every board + id: config_matrix + if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'config-matrix') + continue-on-error: true + env: + HIL_TARGET: ${{ inputs.target }} + HIL_CONFIGS: ${{ inputs.configs }} + HIL_LIMIT: ${{ inputs.limit }} + run: | + args="" + if [ -n "${HIL_TARGET:-}" ]; then args="$args --target $HIL_TARGET"; fi + if [ -n "${HIL_CONFIGS:-}" ]; then args="$args --configs $HIL_CONFIGS"; fi + if [ -n "${HIL_LIMIT:-}" ]; then args="$args --limit $HIL_LIMIT"; fi + + # shellcheck disable=SC2086 # args is a deliberately word-split list + python dev/hil/config_matrix.py $args + + - name: Board serial logs on failure + if: failure() || steps.flash_check.outcome == 'failure' || steps.config_matrix.outcome == 'failure' + run: | + # `stty raw -echo` first, and both calls under `timeout`: a tty reverts + # to ECHO-on once every handle closes, so a bare `cat` feeds the board + # its own output back, and opening a tty waits for carrier, so a wedged + # board can block even stty. + for dev in /dev/ttyACM*; do + [ -e "$dev" ] || continue + echo "--- $dev" + timeout 5 stty -F "$dev" raw -echo 115200 || true + timeout 8 cat "$dev" || true + done + + - name: Verdict + if: always() + env: + RECOVER: ${{ steps.recover.outcome }} + FLASH_CHECK: ${{ steps.flash_check.outcome }} + CONFIG_MATRIX: ${{ steps.config_matrix.outcome }} + run: | + { + echo "### HIL stages" + echo + echo "| stage | result |" + echo "|---|---|" + echo "| recover | ${RECOVER:-skipped} |" + echo "| flash + health check | ${FLASH_CHECK:-skipped} |" + echo "| config matrix | ${CONFIG_MATRIX:-skipped} |" + } >> "$GITHUB_STEP_SUMMARY" + + echo "recover: ${RECOVER:-skipped}" + echo "flash-check: ${FLASH_CHECK:-skipped}" + echo "config-matrix: ${CONFIG_MATRIX:-skipped}" + + # Every stage runs even when an earlier one fails - one broken board + # should not cost the signal from the others - so the verdict is + # taken here rather than by the first failure. + failed=0 + for outcome in "$RECOVER" "$FLASH_CHECK" "$CONFIG_MATRIX"; do + [ "$outcome" = "failure" ] && failed=1 + done + exit "$failed" diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index c2b2e776..e329cb96 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -206,6 +206,38 @@ Tests parameterize over `manifest ∩ dev/ci/targets.json`. Adding `whitestar` l ## 7. Harness structure +### One workflow, three stages + +`.github/workflows/hil.yml` is the whole bench. It replaced four separate +workflows that shared the `hil-bench` concurrency group — and since GitHub keeps +only one *pending* run per group, a push touching two of them had them race with +the loser silently cancelled. One workflow takes one lease and runs: + +| stage | what it is | cost | +|---|---|---| +| recover | `recover.py` — repair any wedged board | ~30s healthy | +| flash + health check | G1/G2, API over USB *and* HTTP | ~3.5 min | +| config matrix | G3, every config on every board | ~45 min | + +Every stage runs even when an earlier one fails, and the verdict is taken at the +end: one broken board should not cost the signal from the others. Running +recovery first is the point of combining them — it costs almost nothing on a +healthy bench and turns "a board wedged, so the next four runs were useless" +into a bench that repairs itself. + +**Trigger: `workflow_run` on "Build and Deploy" completion**, which is every +push to a PR. Not `pull_request`, and the distinction is the security model: +`workflow_run` workflows always run *from the default branch, using the default +branch's code*, so a PR cannot change the workflow, `dev/hil/**`, or the pinned +dependencies that the Pi executes. The bench job checks out the default branch +and then takes **only `src/`** from the commit under test, so the firmware is +the PR's and the harness is not. + +Fork PRs are gated: the `gate` job stops them with a message saying to review +and dispatch by hand. Making forks automatic-with-approval needs a +`hardware-lab` Environment with required reviewers (§4) — a repository setting, +not something the workflow can create. + What exists today: ``` @@ -327,7 +359,7 @@ That last one is the valuable one. It turns the API docs into a load-bearing art ### G3 — every config parses and boots -**Implemented** in `dev/hil/config_matrix.py`, run by `.github/workflows/hil-config-matrix.yml`. +**Implemented** in `dev/hil/config_matrix.py`, run as a stage of `.github/workflows/hil.yml`. Per board: build and flash that board's target once, so the config bundle under test is the one this checkout produces, then loop over every config in @@ -435,16 +467,14 @@ stopped, not slowed. #### Findings and limits -- **Two WPC configs cannot be selected at all.** `configuration.gamename` is a - 16-byte fixed-width field (`SPI_DataStore.py`), and `struct.pack` truncates - silently. `HarleyDavidson_L3` and `GilliganIsland_L9` are 17 characters, so - the web UI offers them, the write is accepted, the name is truncated on the - way into FRAM, and the next boot matches nothing and comes up on safe - defaults with `CONF01`. The harness catches this before spending a boot - cycle, and `dev/tests/test_hil_config_matrix.py` catches a *new* offender in - ordinary CI. Fixing the two that exist means shortening the filenames or - widening the field (which is a `MapVersion` change — the 96-byte record is - fully used). +- **~~Two WPC configs cannot be selected at all.~~ Fixed in #380.** The bench's + first real finding: `configuration.gamename` is a 16-byte fixed-width field + and `struct.pack` truncates silently, so `HarleyDavidson_L3` and + `GilliganIsland_L9` at 17 characters were offered by the web UI, accepted on + write, truncated into FRAM, and matched nothing on the next boot — `CONF01`, + safe defaults, and a game that could never be selected. Shortened to + `HarleyDavid_L3` and `GilliganIsle_L9`. `test_no_config_name_exceeds_the_gamename_field` + now enforces the rule absolutely, with no allowlist. - **WPC boots intermittently fail on a missing file.** Four crashes in one 63-config run, in two flavours — `ImportError: no module named 'origin'` and `OSError: [Errno 2] ENOENT` — both raised from `phew/server.py:create_schedule` diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 07de0d45..f301b649 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -132,20 +132,22 @@ system**, since that means at least one is running firmware for a system it isn' The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` label. -End to end, [`.github/workflows/hil-smoke.yml`](../../.github/workflows/hil-smoke.yml) checks -that the runner picks up jobs, the bench environment reaches them, the serial devices are -present and accessible, and every detected board answers over `mpremote`. It fails if no -boards are found or if any detected board doesn't respond. +End to end, dispatch [`.github/workflows/hil.yml`](../../.github/workflows/hil.yml) with +**stages: `recover`**. That is the cheapest full-path check there is — it proves the runner +picks up jobs and the bench environment reaches them, then prints which recovery rungs this +host can actually use and surveys every board by chip id. It takes under a minute on a healthy +bench and repairs a wedged board on an unhealthy one. -`VECTOR_HIL_VENV` and `VECTOR_HIL_REPO` are exported into every job from `.env`, so workflows -don't hardcode paths. The smoke job deliberately does no `actions/checkout` — under the design's -trust model the bench runs harness code from the clone it already has, not from a PR. +`VECTOR_HIL_VENV` and the `VECTOR_HIL_*` variables are exported into every job from `.env`, so +workflows don't hardcode paths. A missing one fails at the point of use with a message naming +it rather than being pre-flighted separately. ### Running it before this PR merges -A `workflow_dispatch` workflow isn't dispatchable until it exists on the **default branch**, so -the "Run workflow" button won't appear while this is still a PR. The workflow therefore also -triggers on pushes to this branch — push anything to it and the job runs on the bench: +Neither `workflow_dispatch` nor `workflow_run` does anything until the workflow is on the +**default branch** — the Run workflow button doesn't appear, and `workflow_run` only ever runs +default-branch code. The workflow therefore also triggers on pushes to this branch, purely so it +can be validated pre-merge; push anything to it and the job runs on the bench: ```bash git commit --allow-empty -m "trigger hil smoke" && git push @@ -214,7 +216,7 @@ board then tries to parse its log lines as USB API requests. Use `stty -F /dev/t A board can deadlock with its USB device still enumerated and the firmware gone: the port is there, `mpremote` opens it, nothing answers. `dev/hil/recover.py` escalates through four rungs and stops as soon as the board replies. Run it from -[`hil-recover.yml`](../../.github/workflows/hil-recover.yml), or by hand: +the `recover` stage of [`hil.yml`](../../.github/workflows/hil.yml), which runs first on every bench job, or by hand: ```bash cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/recover.py diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 2a3b3ea2..f009e737 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -135,34 +135,25 @@ def test_gamename_field_width_comes_from_the_firmware_source(): assert bench.gamename_field_bytes() == 16 -# Two shipped WPC configs are longer than the FRAM `gamename` field and so can -# never be selected on a real board: the web UI offers them, the write is -# accepted, the name is truncated on the way into FRAM, and the next boot -# matches nothing and comes up on safe defaults with CONF01 raised. That is a -# pre-existing firmware/config defect, not something this harness introduced - -# it is what the bench found first - so it is recorded here rather than fixed -# here. Shortening either filename (or widening the field) should shorten this -# list; nothing should ever lengthen it. -KNOWN_UNREACHABLE_CONFIGS = {"GilliganIsland_L9", "HarleyDavidson_L3"} - - -def test_no_new_config_name_exceeds_the_gamename_field(): +def test_no_config_name_exceeds_the_gamename_field(): """A config whose filename is longer than the field can never be selected. write_record packs `gamename` into a fixed-width field and struct.pack truncates silently, so the truncated name matches nothing at boot and the - board comes up on safe defaults with CONF01. The bench catches this per - board, but it is cheaper to catch here, and this way a new offender fails - an ordinary PR rather than 20 minutes of bench time. + board comes up on safe defaults with CONF01 - while the web UI happily + offers the config. The bench catches this per board; catching it here is + cheaper and fails an ordinary PR instead of 20 minutes of bench time. + + Two configs did exceed it, which is what the bench found first: + GilliganIsland_L9 and HarleyDavidson_L3, both 17 characters. They were + shortened to GilliganIsle_L9 and HarleyDavid_L3 in #380, so the list of + known offenders this test used to carry is gone and the rule is now + absolute. """ limit = bench.gamename_field_bytes() - too_long = {path.stem for path in (REPO_ROOT / "src").glob("*/config/*.json") if len(path.stem.encode()) > limit} + too_long = sorted(path.stem for path in (REPO_ROOT / "src").glob("*/config/*.json") if len(path.stem.encode()) > limit) - new_offenders = sorted(too_long - KNOWN_UNREACHABLE_CONFIGS) - assert new_offenders == [], f"config filenames longer than the {limit}-byte FRAM gamename field: {', '.join(new_offenders)}" - - fixed = sorted(KNOWN_UNREACHABLE_CONFIGS - too_long) - assert fixed == [], f"{', '.join(fixed)} now fits - drop it from KNOWN_UNREACHABLE_CONFIGS" + assert too_long == [], f"config filenames longer than the {limit}-byte FRAM gamename field: {', '.join(too_long)}" # -------------------------------------------------------------------------- From 19dd5b58e9e9701968e46425a7d4b4e3c6a49f63 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 06:00:00 +0000 Subject: [PATCH 15/32] test(hil): retry a failing config once, to tell a broken config from a flaky board First run of the combined workflow: recover ok (nothing to fix), flash and health check ok, config matrix sys11 39/39 and wpc 61/63. The two old FRAM `gamename` failures are gone - #380's renames work, and those configs now pass on hardware. Two different ones failed instead: Congo_21 /api/leaders returned 500, expected 200 Theatre_13 /api/adjustments/status returned 500, expected 200 Neither looks like a broken config. Type 10 is the norm for WPC (54 of 63 configs use it), Theatre_13's Adjustments block is the same shape as the generic one, and the same board crashed once on boot in the same run with OSError: [Errno 2] ENOENT on a file that exists. A board that will raise ENOENT at an import will raise it at a route. But "this config is broken" and "this board is flaky" look identical on one attempt, and they need opposite responses - so the matrix now asks twice. Failing twice is the config and fails the run. Passing on the retry is the board: counted as a pass, and recorded as a flake against that board in the run summary and the job summary, so it is never silently absorbed. Without this, a flaky board reads as two broken configs. That is the wrong conclusion, and the kind of false positive that teaches people to ignore a check. Also of note from the same run: only two boards enumerated. data_east is not on USB at all now - not wedged, absent - so the power cycle did not bring it back. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/DESIGN.md | 15 ++++++ dev/hil/config_matrix.py | 76 ++++++++++++++++++++++------- dev/tests/test_hil_config_matrix.py | 60 +++++++++++++++++++++-- 3 files changed, 129 insertions(+), 22 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index e329cb96..0832c2fd 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -442,6 +442,21 @@ As a second line, a board that will not take a reset gets its console drained and one retry: reading is the remedy for exactly this deadlock and costs seconds, against writing off a whole board's matrix. +#### Telling a broken config from a flaky board + +They produce the identical symptom on one attempt and need opposite responses, +so every failing config is retried once. Fails twice → the config, and the run +goes red. Passes on the retry → the board, recorded as a **flake** against that +board and reported in the summary, without blaming the config. + +This is not hypothetical either. The WPC board raises `ENOENT` mid-boot on files +that plainly exist; a board that will do that to an import will do it to a route, +and a run where `Congo_21` failed `/api/leaders` with a 500 and `Theatre_13` +failed `/api/adjustments/status` the same way — on a board that also crashed +once on boot in the same run — is exactly the ambiguity this resolves. Without +the retry those read as two broken configs, which would be the wrong conclusion +and would erode trust in the check. + #### How a boot is watched The ready marker (`Server: Loop Forever`) is printed exactly once per boot, which diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index 76817c04..dca8493c 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -451,6 +451,7 @@ def run_matrix(board, args): session = Session(port, boot_timeout=args.boot_timeout) passed = [] failures = [] + flakes = [] consecutive_setup_failures = 0 try: @@ -468,21 +469,44 @@ def run_matrix(board, args): for index, config in enumerate(names, 1): started = time.monotonic() group(f"[{index}/{len(names)}] {target} {config}") + first_error = None try: - with time_limit(args.config_timeout, f"{target} {config}"): - # Set the config on the board we are already talking to, - # then reboot into it. The connection dies with the reset; - # the next wait_for_boot opens a fresh one. - session.set_config(config) - session.reboot() - client = session.wait_for_boot() - consecutive_setup_failures = 0 - - name = check_booted_config(client, target, config, configs[config]) - log(f" ok {config:20} -> {name!r} [{time.monotonic() - started:.1f}s]") + # Two attempts, because "this config is broken" and "this board + # is flaky" produce the identical symptom on one attempt, and + # they need opposite responses. A config that fails twice is a + # config bug and fails the run; one that passes on the retry is + # the board, and is reported as a flake instead of being + # blamed on the config. The bench has already shown it can do + # this - a WPC board raising ENOENT mid-boot on files that + # exist will do it to a route just as readily. + for attempt in range(2): + try: + with time_limit(args.config_timeout, f"{target} {config}"): + # Set the config on the board we are already talking + # to, then reboot into it. The connection dies with + # the reset; the next wait_for_boot opens a fresh one. + session.set_config(config) + session.reboot() + client = session.wait_for_boot() + consecutive_setup_failures = 0 + + name = check_booted_config(client, target, config, configs[config]) + break + except CheckFailure as exc: + if attempt: + raise + first_error = exc + log(f"::warning::{target} {config} failed, retrying once to tell a broken config from a flaky board: {exc}") + + elapsed = time.monotonic() - started + if first_error is None: + log(f" ok {config:20} -> {name!r} [{elapsed:.1f}s]") + else: + log(f" ok {config:20} -> {name!r} [{elapsed:.1f}s] (FLAKY - failed once, passed on retry)") + flakes.append((config, str(first_error))) passed.append(config) except CheckFailure as exc: - log(f"::error::{target} {config}: {exc}") + log(f"::error::{target} {config}: failed twice, so this is the config, not the board: {exc}") _dump_boot_log({"port": port, "boot_log": session.boot_log}) failures.append((config, str(exc))) # An assertion that ran is a result about the config. Anything @@ -517,7 +541,7 @@ def run_matrix(board, args): restore_default(session, target) endgroup() - return passed, failures, session.crashes + return passed, failures, session.crashes, flakes def write_step_summary(results): @@ -530,6 +554,12 @@ def write_step_summary(results): for board, passed, failures in results: lines.append(f"| `{board['port']}` | {board['target']} | {len(passed) + len(failures)} | {len(passed)} | {len(failures)} |") + flaky = [(board, config, reason) for board, _p, _f in results for config, reason in (board.get("flakes") or [])] + if flaky: + lines += ["", "### Flaky (failed once, passed on retry - the board, not the config)", ""] + for board, config, reason in flaky: + lines.append(f"- **{board['target']} `{config}`** - {reason.splitlines()[0]}") + crashed = [(board, crash) for board, _passed, _failures in results for crash in (board.get("crashes") or [])] if crashed: lines += ["", "### Boot crashes (recovered by a retry)", ""] @@ -602,12 +632,12 @@ def main(): log(f"built {target} at version {bench.source_version(target)}") endgroup() - results = [({"port": b["port"], "target": "(unknown)", "crashes": []}, [], [("(board setup)", "board is not answering - run dev/hil/recover.py")]) for b in unresponsive] + results = [({"port": b["port"], "target": "(unknown)", "crashes": [], "flakes": []}, [], [("(board setup)", "board is not answering - run dev/hil/recover.py")]) for b in unresponsive] for b in boards: try: if not args.skip_flash: flash_before_matrix(b, workdir) - passed, failures, crashes = run_matrix(b, args) + passed, failures, crashes, flakes = run_matrix(b, args) except Exception as exc: # noqa: BLE001 # A board that cannot even be set up is one board's problem. The # bench is a singleton and a run is expensive, so the other boards @@ -615,8 +645,9 @@ def main(): # died on a TimeoutExpired escaping teardown, which threw away the # results already in hand and skipped the untouched board entirely. log(f"::error::{b['target']} on {b['port']}: {exc}") - passed, failures, crashes = [], [("(board setup)", str(exc))], [] + passed, failures, crashes, flakes = [], [("(board setup)", str(exc))], [], [] b["crashes"] = crashes + b["flakes"] = flakes results.append((b, passed, failures)) log("") @@ -631,8 +662,10 @@ def main(): for board, passed, failures in results: state = "FAIL" if failures else "ok" crashes = board.get("crashes") or [] - crashed = f", {len(crashes)} boot crash(es)" if crashes else "" - log(f" {state:5} {board['port']:16} {board['target']:12} {len(passed)} passed, {len(failures)} failed{crashed}") + flaky = board.get("flakes") or [] + extra = f", {len(crashes)} boot crash(es)" if crashes else "" + extra += f", {len(flaky)} flaky" if flaky else "" + log(f" {state:5} {board['port']:16} {board['target']:12} {len(passed)} passed, {len(failures)} failed{extra}") total_failures += len(failures) total_crashes += len(crashes) log("=" * 60) @@ -640,6 +673,13 @@ def main(): # A crash that a retry got past is not a passing board. The configs were # still checked, so it does not fail the run, but it is a firmware fault # and gets said out loud rather than buried in a green result. + flaky_all = [(board, config, reason) for board, _p, _f in results for config, reason in (board.get("flakes") or [])] + if flaky_all: + log("") + log(f"::warning::{len(flaky_all)} config(s) failed once and passed on retry - the board, not the config:") + for board, config, reason in flaky_all: + log(f" {board['target']} {config}: {reason}") + if total_crashes: log("") log(f"::warning::{total_crashes} boot crash(es) recovered by a retry - the firmware raised on the way up:") diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index f009e737..965a292f 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -355,7 +355,7 @@ def check(_client, _target, _config, expected): def test_run_matrix_walks_every_config_on_a_healthy_board(monkeypatch, fake_repo): session = FakeSession("/dev/ttyFAKE") - passed, failures, _crashes = run_board(monkeypatch, fake_repo, session) + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session) assert failures == [] assert passed == ["AttackMars_11", "Generic_WPC", "Taxi_L4"] @@ -380,7 +380,7 @@ def test_run_matrix_abandons_a_board_that_stops_answering(monkeypatch, fake_repo # survives exactly one config before going quiet. session = FakeSession("/dev/ttyFAKE", dies_after=2) - passed, failures, _crashes = run_board(monkeypatch, fake_repo, session) + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session) assert len(passed) == 1 assert session.boots <= 1 + cm.MAX_CONSECUTIVE_SETUP_FAILURES @@ -400,7 +400,7 @@ def one_bad_config(_client, _target, config, expected): return expected["name"] session = FakeSession("/dev/ttyFAKE") - passed, failures, _crashes = run_board(monkeypatch, fake_repo, session, check=one_bad_config) + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session, check=one_bad_config) assert passed == ["AttackMars_11", "Taxi_L4"] assert [config for config, _reason in failures] == ["Generic_WPC"] @@ -411,7 +411,7 @@ def always_fails(*_args, **_kwargs): raise bench.CheckFailure("board reports game name 'Generic System'") session = FakeSession("/dev/ttyFAKE") - passed, failures, _crashes = run_board(monkeypatch, fake_repo, session, check=always_fails, keep_going=False) + passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session, check=always_fails, keep_going=False) assert passed == [] assert len(failures) == 1 @@ -965,3 +965,55 @@ def test_resolve_targets_refuses_to_flash_a_board_that_will_not_talk(): with pytest.raises(bench.CheckFailure, match="not answering: /dev/ttyACM2"): bench.resolve_targets(boards, {"aaa": "sys11"}) + + +def test_a_config_that_fails_once_and_passes_is_the_board_not_the_config(monkeypatch, fake_repo): + """"Broken config" and "flaky board" look identical on one attempt. + + They need opposite responses, so the matrix asks twice. This board fails + Generic_WPC once and passes it on the retry - the WPC board on the bench + does exactly this, raising ENOENT on files that exist. + """ + seen = [] + + def flaky_once(_client, _target, config, expected): + seen.append(config) + if config == "Generic_WPC" and seen.count("Generic_WPC") == 1: + raise bench.CheckFailure("/api/leaders returned 500, expected 200") + return expected["name"] + + session = FakeSession("/dev/ttyFAKE") + passed, failures, _crashes, flakes = run_board(monkeypatch, fake_repo, session, check=flaky_once) + + # Counted as a pass, because the config is fine... + assert "Generic_WPC" in passed + assert failures == [] + # ...but never silently: the flake is reported against the board. + assert [config for config, _reason in flakes] == ["Generic_WPC"] + assert "500" in flakes[0][1] + + +def test_a_config_that_fails_twice_is_a_real_failure(monkeypatch, fake_repo): + def always_fails(_client, _target, config, expected): + if config == "Generic_WPC": + raise bench.CheckFailure("board reports game name 'Generic System'") + return expected["name"] + + session = FakeSession("/dev/ttyFAKE") + passed, failures, _crashes, flakes = run_board(monkeypatch, fake_repo, session, check=always_fails) + + assert [config for config, _reason in failures] == ["Generic_WPC"] + assert flakes == [] + assert "Generic_WPC" not in passed + + +def test_flaky_configs_are_reported_in_the_job_summary(tmp_path, monkeypatch): + summary = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(summary)) + + board = {"port": "/dev/ttyACM1", "target": "wpc", "crashes": [], "flakes": [("Congo_21", "/api/leaders returned 500, expected 200")]} + cm.write_step_summary([(board, ["Congo_21"], [])]) + + rendered = summary.read_text() + assert "Flaky (failed once, passed on retry" in rendered + assert "**wpc `Congo_21`**" in rendered From 484fcfc96e130460c27c147cb30f54f1ec196cd8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 06:00:24 +0000 Subject: [PATCH 16/32] style(hil): black-format the new matrix tests Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/tests/test_hil_config_matrix.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 965a292f..6c80fa20 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -968,7 +968,7 @@ def test_resolve_targets_refuses_to_flash_a_board_that_will_not_talk(): def test_a_config_that_fails_once_and_passes_is_the_board_not_the_config(monkeypatch, fake_repo): - """"Broken config" and "flaky board" look identical on one attempt. + """ "Broken config" and "flaky board" look identical on one attempt. They need opposite responses, so the matrix asks twice. This board fails Generic_WPC once and passes it on the retry - the WPC board on the bench From 2e5bc618266e688c2f653395d1531bdca0ce0b5d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 07:19:24 +0000 Subject: [PATCH 17/32] fix(hil): one reset per boot - do not reset a board that is already booting WPC passed all 63 configs this run. That is the thing this PR set out to do, and the two failures from the run before it did not recur: a different config flaked instead (WCSoccer_LX2, /api/leaders 500, passed on retry), which is the retry added last push answering its own question. A different config failing each run with the same symptom is the board, not the configs. Congo_21 and Theatre_13 are fine. sys11 wedged instead, and the log places it precisely: flashed successfully, then refusing both a reset and any console traffic seconds later, at its very first start(). The window is the problem. dev/flash.py ends by resetting the board, so it is already booting when Session.start() resets it again - on top of that boot, while the firmware is still reading a filesystem written seconds earlier. So: one reset per boot. A board that was just flashed is watched through the boot flashing already started, rather than being interrupted and restarted. The reset stays for --skip-flash, where nothing else has triggered one. This may be the same fault as WPC's ENOENT-on-a-file-that-exists crashes at boot - both are the firmware being disturbed while it reads freshly written flash - but that is a hypothesis, not something this run proves. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/DESIGN.md | 11 +++++++++ dev/hil/config_matrix.py | 38 ++++++++++++++++++++--------- dev/tests/test_hil_config_matrix.py | 31 +++++++++++++++++++++-- 3 files changed, 66 insertions(+), 14 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 0832c2fd..81dab322 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -442,6 +442,17 @@ As a second line, a board that will not take a reset gets its console drained and one retry: reading is the remedy for exactly this deadlock and costs seconds, against writing off a whole board's matrix. +#### One reset per boot + +`dev/flash.py` ends by resetting the board, so a freshly flashed board is +already booting. Resetting it again lands on top of that boot, while the +firmware is still reading a filesystem written seconds ago — and sys11 wedged +in exactly that window: flashed successfully, then refusing a reset and any +console traffic seconds later. `Session.start(reset=False)` watches the boot +that flashing started instead of forcing another. It is the same shape as the +`ENOENT`-on-a-file-that-exists crashes WPC raises, and suggests both are the +firmware being disturbed while it reads freshly written flash. + #### Telling a broken config from a flaky board They produce the identical symptom on one attempt and need opposite responses, diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index dca8493c..0a97b309 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -239,8 +239,9 @@ def __init__(self, port, boot_timeout=BOOT_TIMEOUT): self.boot_log = [] self.crashes = [] - def start(self): - """First boot of the run: reset the board, then watch it come up. + def start(self, reset=True): + """First boot of the run: watch the board come up, resetting it first + only when something else has not already done so. Every later boot is triggered by reboot() over our own connection, but the first one has to be triggered here, and it must be triggered: @@ -251,6 +252,14 @@ def start(self): what made the first matrix run time out on all three boards without checking a single config. + `reset=False` is for a board that was just flashed. dev/flash.py ends + by resetting it, so it is already booting - and resetting again lands + on top of that boot, while the firmware is still reading its freshly + written filesystem. That is not theoretical: sys11 wedged in exactly + that window, flashed successfully and then refusing a reset seconds + later, and it is the same shape as the ENOENT-on-a-file-that-exists + crashes the WPC board raises at boot. One reset per boot. + mpremote is safe here, unlike mid-matrix: no connection of ours is open yet, so there is no handoff to lose. @@ -259,12 +268,15 @@ def start(self): the moment somebody reads it, and reading costs three seconds - much less than writing off a board's whole matrix. """ - try: - reset_board(self.port) - except Exception as exc: - log(f"::warning::{self.port} did not take a reset ({exc}); draining its console and retrying once") - drain_port(self.port) - reset_board(self.port) + if reset: + try: + reset_board(self.port) + except Exception as exc: + log(f"::warning::{self.port} did not take a reset ({exc}); draining its console and retrying once") + drain_port(self.port) + reset_board(self.port) + else: + log(" already booting from the flash; watching that boot rather than forcing another") return self.wait_for_boot() def wait_for_boot(self): @@ -438,7 +450,7 @@ def flash_before_matrix(board, workdir): endgroup() -def run_matrix(board, args): +def run_matrix(board, args, just_flashed=False): """Walk one board through its configs. Returns (passed, failures). One connection per boot, held across the assertions and across setting the @@ -457,7 +469,8 @@ def run_matrix(board, args): try: group(f"Config bundle {target} on {port}") try: - client = session.start() + # A board we just flashed is already booting - see Session.start. + client = session.start(reset=not just_flashed) check_bundle(client, target, configs) finally: endgroup() @@ -635,9 +648,10 @@ def main(): results = [({"port": b["port"], "target": "(unknown)", "crashes": [], "flakes": []}, [], [("(board setup)", "board is not answering - run dev/hil/recover.py")]) for b in unresponsive] for b in boards: try: - if not args.skip_flash: + just_flashed = not args.skip_flash + if just_flashed: flash_before_matrix(b, workdir) - passed, failures, crashes, flakes = run_matrix(b, args) + passed, failures, crashes, flakes = run_matrix(b, args, just_flashed=just_flashed) except Exception as exc: # noqa: BLE001 # A board that cannot even be set up is one board's problem. The # bench is a singleton and a run is expensive, so the other boards diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 6c80fa20..2853c5df 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -300,12 +300,14 @@ def __init__(self, port, boot_timeout=None, dies_after=None): self.configs_set = [] self.boots = 0 self.starts = 0 + self.start_resets = [] self.nudges = 0 self.crashes = [] self.restored = False - def start(self): + def start(self, reset=True): self.starts += 1 + self.start_resets.append(reset) return self.wait_for_boot() def wait_for_boot(self): @@ -348,8 +350,10 @@ def check(_client, _target, _config, expected): monkeypatch.setattr(cm, "restore_default", lambda s, _target: setattr(s, "restored", True)) defaults = {"configs": None, "limit": None, "changed_since": None, "keep_going": True, "boot_timeout": 90, "config_timeout": 60} + just_flashed = defaults.pop("just_flashed", False) defaults.update(arg_overrides) - return cm.run_matrix({"port": session.port, "target": "wpc"}, Namespace(**defaults)) + just_flashed = defaults.pop("just_flashed", just_flashed) + return cm.run_matrix({"port": session.port, "target": "wpc"}, Namespace(**defaults), just_flashed=just_flashed) def test_run_matrix_walks_every_config_on_a_healthy_board(monkeypatch, fake_repo): @@ -1017,3 +1021,26 @@ def test_flaky_configs_are_reported_in_the_job_summary(tmp_path, monkeypatch): rendered = summary.read_text() assert "Flaky (failed once, passed on retry" in rendered assert "**wpc `Congo_21`**" in rendered + + +def test_a_freshly_flashed_board_is_not_reset_again(monkeypatch, fake_repo): + """dev/flash.py already reset it, so it is mid-boot. + + Resetting again lands on top of that boot while the firmware is still + reading its freshly written filesystem. sys11 wedged in exactly that + window - flashed successfully, then refusing a reset seconds later. + """ + session = FakeSession("/dev/ttyFAKE") + + run_board(monkeypatch, fake_repo, session, just_flashed=True) + + assert session.start_resets == [False] + + +def test_a_board_we_did_not_flash_is_reset_first(monkeypatch, fake_repo): + """--skip-flash means nobody triggered a boot, so we have to.""" + session = FakeSession("/dev/ttyFAKE") + + run_board(monkeypatch, fake_repo, session, just_flashed=False) + + assert session.start_resets == [True] From 8240166ac5d5c120058d9756c62e0c70c58c060a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 09:49:39 +0000 Subject: [PATCH 18/32] fix(hil): drain until a board goes quiet, and repair a lost connection Two bench failures from the same run, both of which wrote off boards that were still alive. A drain with a fixed three-second budget catches the first mouthful of a blocked board's backlog and calls it dead. /dev/ttyACM0 was declared unrecoverable after 63 bytes were drained from it; the next step in the same job read the port with `cat` for eight seconds and got a healthy, running web server. read_until_quiet now reads until the port has been silent for a couple of seconds, capped at a budget, and the Ctrl-C comes after the drain rather than before - a firmware blocked writing to stdout is not reading stdin either, so writing first only times out against the wedge it is trying to clear. A timed-out interrupt drains and retries once. This is the only recovery rung this runner can always reach, so it is worth being patient in. Separately, an exhausted boot left the session holding no connection, and every config after it failed instantly with "the board is not connected" - including the retry that exists to survive a flaky board. The WPC board crashed on boot twice for FishTales_L4, and FishTales_L5 then failed in 0.0s, both blamed on a board that came back on its own and served the restore step happily. Session.ensure_connected() drains, resets and watches the board up before each attempt. A board that is genuinely gone still ends its matrix through the consecutive-failure counter. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/DESIGN.md | 39 +++++++ dev/hil/bench.py | 136 +++++++++++++++++-------- dev/hil/config_matrix.py | 26 +++++ dev/hil/recover.py | 31 +++--- dev/tests/test_hil_config_matrix.py | 153 +++++++++++++++++++++++++++- dev/tests/test_hil_recover.py | 4 +- 6 files changed, 332 insertions(+), 57 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 81dab322..f307a66a 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -442,6 +442,45 @@ As a second line, a board that will not take a reset gets its console drained and one retry: reading is the remedy for exactly this deadlock and costs seconds, against writing off a whole board's matrix. +#### Draining reads until the board goes quiet, not for a fixed few seconds + +A blocked board does not hand its backlog over in one gulp. It unblocks, runs +a little further, prints some more, and only then falls silent — so a drain +with a fixed three-second budget catches the first mouthful and declares the +board dead. That is measured, not argued: one bench run wrote off +`/dev/ttyACM0` as unrecoverable after draining 63 bytes from it, and the very +next step in the same job read the port with `cat` for eight seconds and got a +healthy, running web server. + +So `bench.read_until_quiet` reads until the port has been silent for a couple +of seconds, capped at a budget. A port with nothing to say costs the quiet +period; one with a backlog gets as long as it needs. The Ctrl-C comes *after* +the drain, because a firmware blocked writing to stdout is not reading stdin +either — the host's OUT endpoint backs up too, so writing first only times out +against the wedge it is trying to clear. A timed-out interrupt is likewise a +reason to read more and try again, not a reason to give up. + +This matters more than it sounds: on the current runner the drain is the only +recovery rung that can always be reached. The USB reset needs a udev rule and +the power cycle needs `uhubctl`, and neither is installed (see +RUNNER_SETUP.md), so everything above the drain escalates straight to a flash +wipe. + +#### A lost connection is repaired, not treated as a verdict + +If a board's boot fails outright, the session is left holding no connection, +and without care every config after it fails instantly with "the board is not +connected" — including the retry that exists precisely to survive a flaky +board. The bench showed the exact shape: the WPC board crashed on boot twice +for one config, and from there the retry failed in no time at all and the next +config failed in 0.0s, two configs blamed on a board that came back on its own +moments later and served the restore step happily. + +So `Session.ensure_connected()` runs before each attempt: if there is no live +connection it drains, resets, and watches the board come up. A board that is +genuinely gone still ends its own matrix through the consecutive-failure +counter; this only stops that happening while the board is still recoverable. + #### One reset per boot `dev/flash.py` ends by resetting the board, so a freshly flashed board is diff --git a/dev/hil/bench.py b/dev/hil/bench.py index b1165486..3f7495b6 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -167,7 +167,7 @@ def ensure_tools_on_path(): raise CheckFailure( "mpremote not found. Run with the bench venv, e.g.\n" - f" cd {REPO_ROOT} && PATH=\"$PWD/.venv/bin:$PATH\" .venv/bin/python dev/hil/.py ...\n" + f' cd {REPO_ROOT} && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/.py ...\n' "(VECTOR_HIL_VENV is exported by the runner service, so it is not set in a login shell.)" ) @@ -284,7 +284,9 @@ def identify(boards, seconds=8): for index, board in enumerate(boards, 1): log(f" [{index}/{len(boards)}] BLINKING NOW: {board['port']} chip {board['chip_id']}") result = mpremote( - "connect", board["port"], "exec", + "connect", + board["port"], + "exec", IDENTIFY_SNIPPET.format(blinks=int(seconds / 0.5)), timeout=seconds + 30, ) @@ -329,19 +331,13 @@ def resolve_targets(boards, board_map): dead = [b for b in boards if not b.get("responsive", True)] if dead: raise CheckFailure( - "not answering: " - + ", ".join(b["port"] for b in dead) - + ".\nA board that will not talk cannot be identified, so it cannot be safely flashed.\n" - "Run dev/hil/recover.py to get it back." + "not answering: " + ", ".join(b["port"] for b in dead) + ".\nA board that will not talk cannot be identified, so it cannot be safely flashed.\n" "Run dev/hil/recover.py to get it back." ) if board_map: unmapped = [b for b in boards if b["chip_id"] not in board_map] if unmapped: - raise CheckFailure( - "VECTOR_HIL_BOARD_MAP is set but does not cover: " - + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) - ) + raise CheckFailure("VECTOR_HIL_BOARD_MAP is set but does not cover: " + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped)) for b in boards: b["target"] = board_map[b["chip_id"]] log("targets from VECTOR_HIL_BOARD_MAP") @@ -349,24 +345,17 @@ def resolve_targets(boards, board_map): missing = [b for b in boards if not b["system"]] if missing: - raise CheckFailure( - "cannot identify " - + ", ".join(b["port"] for b in missing) - + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP." - ) + raise CheckFailure("cannot identify " + ", ".join(b["port"] for b in missing) + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP.") systems = [b["system"] for b in boards] duplicates = {s for s in systems if systems.count(s) > 1} if duplicates: raise CheckFailure( - "refusing to flash from autodetection: " - + ", ".join(sorted(duplicates)) - + " is reported by more than one board.\n" + "refusing to flash from autodetection: " + ", ".join(sorted(duplicates)) + " is reported by more than one board.\n" "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" "at least one board is running firmware for a system it is not wired for.\n" "Pin them explicitly instead, using the chip ids above:\n" - " VECTOR_HIL_BOARD_MAP=" - + ",".join(f"{b['chip_id']}=" for b in boards) + " VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards) ) for b in boards: @@ -392,7 +381,10 @@ def build(target): build_dir = REPO_ROOT / "build" / target result = subprocess.run( [VENV_PYTHON, "dev/build.py", "--target_hardware", target, "--build-dir", str(build_dir)], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=900, ) if result.returncode != 0: log(result.stdout[-3000:]) @@ -435,7 +427,10 @@ def write_bench_config(target, workdir): def flash(target, port, build_dir, config_path): result = subprocess.run( [VENV_PYTHON, "dev/flash.py", str(build_dir), "--port", port, "--write-config", str(config_path)], - cwd=REPO_ROOT, capture_output=True, text=True, timeout=900, + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=900, ) if result.returncode != 0: log(result.stdout[-3000:]) @@ -458,7 +453,11 @@ def reset_board(port): deterministic and the reported boot time meaningful. """ result = mpremote( - "connect", port, "exec", "--no-follow", "import machine; machine.reset()", + "connect", + port, + "exec", + "--no-follow", + "import machine; machine.reset()", timeout=30, ) if result.returncode != 0: @@ -546,9 +545,7 @@ def wait_for_server(port, timeout=BOOT_TIMEOUT): pass tail = "\n ".join(transcript[-20:]) or "(nothing on the console)" - raise CheckFailure( - f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}" - ) + raise CheckFailure(f"{port} never reported its web server within {timeout}s. Last console output:\n {tail}") def _server_is_answering(connection): @@ -606,10 +603,7 @@ def get(client, route, expect=200): # The board narrates its own routing failures ("USB REQ: route not # found: ..."), but the client discards every line that is not a # response. Drain whatever is pending so the reason is visible. - raise CheckFailure( - f"{route} returned {status}, expected {expect}" - f"{_drain_serial(client.ser)}" - ) + raise CheckFailure(f"{route} returned {status}, expected {expect}" f"{_drain_serial(client.ser)}") return response.get("body") @@ -828,15 +822,71 @@ def repl_reset(connection): Repl(connection).enter().reset() -def drain_port(port, seconds=3): - """Open a port and read whatever the board has queued, then interrupt it. +DRAIN_SECONDS = 20 +DRAIN_QUIET_SECONDS = 2 + + +def read_until_quiet(connection, budget, quiet=DRAIN_QUIET_SECONDS): + """Read from an open port until it stops producing, or the budget runs out. + + A board blocked writing into a CDC endpoint nothing is draining does not + hand over its backlog in one gulp: it unblocks, runs a little further, + prints some more, and only then goes quiet. Reading for a fixed three + seconds catches the first mouthful and calls the board dead. That is not + a guess - a bench run declared /dev/ttyACM0 unrecoverable after draining + 63 bytes from it, and the very next step in the same job read it with + `cat` for eight seconds and got a healthy, running server. + + So read until it has been silent for `quiet` seconds, capped at `budget`. + A port that is already quiet costs `quiet`; one with a backlog gets as + long as it needs to finish coughing it up. + """ + drained = 0 + deadline = time.monotonic() + budget + silent_since = time.monotonic() + while time.monotonic() < deadline: + chunk = connection.read(connection.in_waiting or 1) + if chunk: + drained += len(chunk) + silent_since = time.monotonic() + elif time.monotonic() - silent_since >= quiet: + break + return drained + + +def interrupt_board(connection, port, keys=CTRL_C): + """Send Ctrl-C, draining if the board is too blocked to accept it. + + The write can time out for the same reason the board is stuck: a firmware + blocked writing to stdout is not reading stdin either, so the host's OUT + endpoint backs up. Reading is what frees it, so a timed-out write is a + reason to drain and try once more rather than to give up. + """ + try: + serial_write(connection, keys, port) + return True + except CheckFailure as exc: + log(f" {exc}; draining and trying the interrupt once more") + + read_until_quiet(connection, DRAIN_SECONDS) + try: + serial_write(connection, keys, port) + return True + except CheckFailure as exc: + log(f" {exc}") + return False + + +def drain_port(port, seconds=DRAIN_SECONDS): + """Open a port, read the board's console until it goes quiet, interrupt it. - A cheap attempt at unsticking a board that has gone quiet, and it costs one - open and three seconds. The wedge worth recovering from is a board blocked - writing into a CDC endpoint that nothing is draining: reading is the whole - remedy, and the Ctrl-C afterwards gets it back to a REPL if the read freed - it. Reports what it saw either way - a board that yields zero bytes and a - board that yields a backlog are different problems. + A cheap attempt at unsticking a board, and it costs one open. The wedge + worth recovering from is a board blocked writing into a CDC endpoint that + nothing is draining: reading is the whole remedy, and the Ctrl-C afterwards + gets it back to a REPL once the read has freed it - which is why the + interrupt comes after the drain and not before. Reports what it saw either + way; a board that yields zero bytes and a board that yields a backlog are + different problems. """ try: connection = open_serial(port) @@ -846,10 +896,12 @@ def drain_port(port, seconds=3): drained = 0 try: - deadline = time.monotonic() + seconds - while time.monotonic() < deadline: - drained += len(connection.read(connection.in_waiting or 1)) - serial_write(connection, CTRL_C, port) + drained += read_until_quiet(connection, seconds) + interrupt_board(connection, port) + # Whatever the interrupt shook loose - a KeyboardInterrupt traceback, + # a REPL banner - is backlog too, and leaving it queued re-wedges the + # board the moment we close. + drained += read_until_quiet(connection, DRAIN_QUIET_SECONDS * 2) except Exception as exc: log(f" error while draining {port}: {exc}") finally: diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index 0a97b309..1b000013 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -308,6 +308,31 @@ def wait_for_boot(self): self.client = UsbApiClient(self.connection) return self.client + def ensure_connected(self): + """Get the board back to a booted, talking state if it is not already. + + Without this, one exhausted boot leaves the session poisoned: the + connection is None, so every later `set_config` raises "the board is + not connected" instantly - including the retry that exists precisely + to survive a flaky board. A bench run showed the shape exactly. The + WPC board crashed on boot twice for FishTales_L4, and from there the + retry failed in no time at all and the next config failed in 0.0s, + two configs blamed on a board that came back on its own moments later + and served the restore step happily. + + So a lost connection is a thing to repair, not a verdict. Drain first, + because the commonest reason a board stops talking is that nothing has + read what it printed, then reset and watch it come up. If it will not, + the caller's consecutive-failure counter still ends the board's matrix + - this only stops that happening while the board is still recoverable. + """ + if self.connection is not None: + return self.client + log(f" {self.port} has no live connection; draining and rebooting it before this config") + drain_port(self.port) + reset_board(self.port) + return self.wait_for_boot() + def _require_connection(self, what): if self.connection is None: raise CheckFailure(f"cannot {what} on {self.port}: the board is not connected (its last boot did not complete)") @@ -498,6 +523,7 @@ def run_matrix(board, args, just_flashed=False): # Set the config on the board we are already talking # to, then reboot into it. The connection dies with # the reset; the next wait_for_boot opens a fresh one. + session.ensure_connected() session.set_config(config) session.reboot() client = session.wait_for_boot() diff --git a/dev/hil/recover.py b/dev/hil/recover.py index 41861a46..ec74f8b6 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -55,17 +55,20 @@ import trench_coat # noqa: E402 from bench import ( # noqa: E402 CTRL_C, + DRAIN_QUIET_SECONDS, + DRAIN_SECONDS, REPO_ROOT, CheckFailure, endgroup, ensure_tools_on_path, group, + interrupt_board, list_ports, log, mpremote, open_serial, parse_board_map, - serial_write, + read_until_quiet, time_limit, ) @@ -159,8 +162,16 @@ def survey(board_map): # -------------------------------------------------------------------------- -def drain(port, seconds=5): - """Read whatever is queued and interrupt the board.""" +def drain(port, seconds=DRAIN_SECONDS): + """Read whatever is queued until the board goes quiet, then interrupt it. + + This is the only rung of the ladder this runner can always reach - the USB + reset needs a udev rule and the power cycle needs uhubctl, and neither is + installed - so it is worth being patient here. A bench run proved the + point: a board written off after a three-second drain of 63 bytes was read + with `cat` moments later and turned out to be a healthy, running server + that had merely blocked on a full output buffer. + """ try: connection = open_serial(port) except Exception as exc: @@ -169,15 +180,11 @@ def drain(port, seconds=5): drained = 0 try: - deadline = time.monotonic() + seconds - while time.monotonic() < deadline: - drained += len(connection.read(connection.in_waiting or 1)) - serial_write(connection, CTRL_C + CTRL_C, port) - except CheckFailure as exc: - # Expected against a truly wedged board, and worth saying out loud: - # a board that will not accept a Ctrl-C is not going to be talked - # back to life, so the next step up the ladder is the real hope. - log(f" {exc}") + drained += read_until_quiet(connection, seconds) + # Ctrl-C twice: once to break out of a sleep, once for whatever the + # first one dropped us into. + interrupt_board(connection, port, CTRL_C + CTRL_C) + drained += read_until_quiet(connection, DRAIN_QUIET_SECONDS * 2) except Exception as exc: log(f" error draining {port}: {exc}") finally: diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 2853c5df..27f009cf 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -23,7 +23,9 @@ # bench.py imports pyserial (which ships with mpremote) and dev/usb_coms_demo. # Neither is needed for the pure helpers under test and neither is guaranteed to # be installed wherever these tests run, so stand them in before the import. -sys.modules.setdefault("serial", types.ModuleType("serial")) +_serial_stub = sys.modules.setdefault("serial", types.ModuleType("serial")) +if not hasattr(_serial_stub, "SerialTimeoutException"): + _serial_stub.SerialTimeoutException = type("SerialTimeoutException", (Exception,), {}) if "usb_coms_demo" not in sys.modules: stub = types.ModuleType("usb_coms_demo") stub.UsbApiClient = object @@ -320,6 +322,11 @@ def wait_for_boot(self): self.connection = object() return self.client + def ensure_connected(self): + if self.connection is None: + return self.wait_for_boot() + return self.client + def set_config(self, gamename): if self.client is None: raise bench.CheckFailure(f"could not reach the REPL on {self.port}") @@ -387,7 +394,12 @@ def test_run_matrix_abandons_a_board_that_stops_answering(monkeypatch, fake_repo passed, failures, _crashes, _flakes = run_board(monkeypatch, fake_repo, session) assert len(passed) == 1 - assert session.boots <= 1 + cm.MAX_CONSECUTIVE_SETUP_FAILURES + # A dead board costs at most two attempts per config, each of which may + # spend a boot repairing the connection and a boot on the config itself - + # and only MAX_CONSECUTIVE_SETUP_FAILURES configs are attempted at all. + # The point is the ceiling: nine configs must not cost nine timeouts. + assert session.boots <= 1 + 4 * cm.MAX_CONSECUTIVE_SETUP_FAILURES + assert len(passed) + len([f for f in failures if f[0] != "(not run)"]) <= 1 + cm.MAX_CONSECUTIVE_SETUP_FAILURES assert any("skipped after" in reason for _config, reason in failures) # Every setup failure gets one cheap recovery attempt before we give up. assert session.nudges == cm.MAX_CONSECUTIVE_SETUP_FAILURES @@ -1044,3 +1056,140 @@ def test_a_board_we_did_not_flash_is_reset_first(monkeypatch, fake_repo): run_board(monkeypatch, fake_repo, session, just_flashed=False) assert session.start_resets == [True] + + +# -------------------------------------------------------------------------- +# a board that stops talking is repaired, not written off +# -------------------------------------------------------------------------- + + +class QuietingSerial: + """A port that hands over a backlog in mouthfuls, then falls silent.""" + + def __init__(self, chunks): + self.chunks = list(chunks) + self.written = [] + self.closed = False + + @property + def in_waiting(self): + return len(self.chunks[0]) if self.chunks else 0 + + def read(self, _size): + return self.chunks.pop(0) if self.chunks else b"" + + def write(self, data): + self.written.append(data) + + def close(self): + self.closed = True + + +def test_read_until_quiet_keeps_reading_while_the_board_is_still_printing(monkeypatch): + """Three seconds catches the first mouthful and calls a live board dead. + + /dev/ttyACM0 was written off after a drain of 63 bytes; the next step in + the same job read it with `cat` and got a healthy running server. + """ + connection = QuietingSerial([b"RESOURCE: ", b"RAM=71%", b"Server: Check Wifi"]) + + drained = bench.read_until_quiet(connection, budget=30, quiet=0) + + assert drained == len(b"RESOURCE: RAM=71%Server: Check Wifi") + assert connection.chunks == [] + + +def test_read_until_quiet_stops_once_the_port_has_been_silent(monkeypatch): + """A port with nothing to say costs `quiet`, not the whole budget.""" + connection = QuietingSerial([]) + started = time.monotonic() + + assert bench.read_until_quiet(connection, budget=30, quiet=0.05) == 0 + assert time.monotonic() - started < 5 + + +def test_drain_port_interrupts_only_after_it_has_read(monkeypatch): + """The Ctrl-C is useless until the read has unblocked the firmware. + + A board blocked writing to stdout is not reading stdin either, so writing + first just times out against the wedge we are trying to clear. + """ + order = [] + + class Recorder(QuietingSerial): + def read(self, size): + chunk = super().read(size) + if chunk: + order.append("read") + return chunk + + def write(self, data): + order.append("write") + super().write(data) + + connection = Recorder([b"backlog"]) + monkeypatch.setattr(bench, "open_serial", lambda port, **kw: connection) + monkeypatch.setattr(bench, "DRAIN_QUIET_SECONDS", 0.05) + + assert bench.drain_port("/dev/ttyFAKE", seconds=5) == len(b"backlog") + assert order[0] == "read" + assert "write" in order + assert connection.closed + + +def test_interrupt_board_drains_and_retries_when_the_write_times_out(monkeypatch): + """A timed-out write is a reason to read, not a reason to give up.""" + attempts = [] + + class Blocked(QuietingSerial): + def write(self, data): + attempts.append(data) + if len(attempts) == 1: + raise bench.serial.SerialTimeoutException("blocked") + + connection = Blocked([]) + monkeypatch.setattr(bench, "DRAIN_QUIET_SECONDS", 0.05) + + assert bench.interrupt_board(connection, "/dev/ttyFAKE") is True + assert len(attempts) == 2 + + +def test_a_session_whose_boot_failed_is_repaired_before_the_next_config(monkeypatch): + """One exhausted boot must not poison every config after it. + + The WPC board crashed on boot twice for one config, and from there the + retry failed instantly and the next config failed in 0.0s - both blamed on + a board that came back on its own moments later. + """ + drained = [] + resets = [] + + session = cm.Session("/dev/ttyFAKE") + monkeypatch.setattr(cm, "drain_port", lambda port, **kw: drained.append(port)) + monkeypatch.setattr(cm, "reset_board", lambda port: resets.append(port)) + monkeypatch.setattr(cm, "wait_for_server", lambda port, timeout=None: (types.SimpleNamespace(close=lambda: None), [])) + monkeypatch.setattr(cm, "prime_usb", lambda _connection: None) + monkeypatch.setattr(cm, "UsbApiClient", lambda _connection: FakeClient()) + + assert session.connection is None + client = session.ensure_connected() + + assert client is not None + assert session.connection is not None + assert drained == ["/dev/ttyFAKE"] + assert resets == ["/dev/ttyFAKE"] + + +def test_ensure_connected_leaves_a_live_session_alone(monkeypatch): + """It repairs a lost connection; it does not add a boot to every config.""" + session = cm.Session("/dev/ttyFAKE") + session.connection = types.SimpleNamespace(close=lambda: None) + session.client = FakeClient() + + def fail(*_args, **_kwargs): + raise AssertionError("a connected session must not be reset") + + monkeypatch.setattr(cm, "drain_port", fail) + monkeypatch.setattr(cm, "reset_board", fail) + + assert session.ensure_connected() is session.client diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index f8599c5f..042ff410 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -17,7 +17,9 @@ REPO_ROOT = Path(__file__).resolve().parents[2] -sys.modules.setdefault("serial", types.ModuleType("serial")) +_serial_stub = sys.modules.setdefault("serial", types.ModuleType("serial")) +if not hasattr(_serial_stub, "SerialTimeoutException"): + _serial_stub.SerialTimeoutException = type("SerialTimeoutException", (Exception,), {}) if "usb_coms_demo" not in sys.modules: stub = types.ModuleType("usb_coms_demo") stub.UsbApiClient = object From 415e9d2e1cb02d2e64073105b9409f0fc7464191 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 10:49:30 +0000 Subject: [PATCH 19/32] fix(hil): tell a board that will not talk from one that will not listen The bench run on 8240166 got wpc to 63/63 - the connection repair worked - and left one board that has now failed the same way three runs running, described wrongly each time. /dev/ttyACM0 reads fine: 85, 95 and 112 bytes across the three runs, and the job's own `cat` shows it running the application and printing FRAM and RESOURCE lines throughout. Every single write to it times out. So it is not the deadlock the drain exists for - it is producing output happily and never servicing what we send it, and reading is not what it is waiting for. Two consequences. The retry inside interrupt_board is dropped: both callers drain to quiet first, which is the whole remedy for the wedge that makes a write time out, so a second attempt after that just spends five more seconds reaching the same answer - it cost 73s in the recover step and 112s in each inventory for nothing. And the outcome of the write is now reported, because "talking but not listening" and "silent" need opposite remedies: the first is exactly what the USB reset and power cycle rungs are for. "STILL DEAD" becomes "NOT ANSWERING", with a note that failing every rung this runner can reach is not the same as being bricked. It sent a maintainer looking for a dead Pico when the board was running the whole time. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013c2GwCiEu9rRxznR5vfkuU --- dev/hil/DESIGN.md | 11 ++++++++++ dev/hil/bench.py | 32 +++++++++++++++-------------- dev/hil/recover.py | 32 +++++++++++++++++++++++------ dev/tests/test_hil_config_matrix.py | 27 ++++++++++++++++-------- 4 files changed, 73 insertions(+), 29 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index f307a66a..a1989632 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -466,6 +466,17 @@ the power cycle needs `uhubctl`, and neither is installed (see RUNNER_SETUP.md), so everything above the drain escalates straight to a flash wipe. +What the two directions did is then the diagnosis, and they point at different +remedies. Bytes read but the Ctrl-C refused means the board is *talking but not +listening*: producing output normally and never servicing what we send it, so +no amount of reading reaches it — that is what the USB reset and power cycle +rungs are for, and with both unavailable there is genuinely nothing left to +try. Nothing read at all is the opposite: not a board blocked on a full output +buffer, because such a board has a backlog to give up the moment somebody +reads. Both are reported in those words rather than as "still dead", which sent +a maintainer looking for a bricked Pico when the board in question was running +the application and printing to its console the whole time. + #### A lost connection is repaired, not treated as a verdict If a board's boot fails outright, the session is left holding no connection, diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 3f7495b6..20d7d7fa 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -855,20 +855,21 @@ def read_until_quiet(connection, budget, quiet=DRAIN_QUIET_SECONDS): def interrupt_board(connection, port, keys=CTRL_C): - """Send Ctrl-C, draining if the board is too blocked to accept it. - - The write can time out for the same reason the board is stuck: a firmware - blocked writing to stdout is not reading stdin either, so the host's OUT - endpoint backs up. Reading is what frees it, so a timed-out write is a - reason to drain and try once more rather than to give up. + """Send Ctrl-C to a board we have just finished draining. + + Returns whether the board accepted it, which is a diagnosis in its own + right. Both callers read the console to quiet first, because that is the + remedy for the common wedge: a firmware blocked writing to stdout is not + reading stdin either, so the host's OUT endpoint backs up behind it and + the write times out against the very deadlock it is trying to clear. + + A write that still times out after that is a different animal. It means + the board is producing output happily and simply never services what we + send it - and no amount of reading fixes that, because reading is not + what it is waiting for. /dev/ttyACM0 has been in exactly that state for + three runs: 85, 95 and 112 bytes read, every single write timed out. + Retrying the write only spends another five seconds saying so. """ - try: - serial_write(connection, keys, port) - return True - except CheckFailure as exc: - log(f" {exc}; draining and trying the interrupt once more") - - read_until_quiet(connection, DRAIN_SECONDS) try: serial_write(connection, keys, port) return True @@ -895,9 +896,10 @@ def drain_port(port, seconds=DRAIN_SECONDS): return 0 drained = 0 + interrupted = False try: drained += read_until_quiet(connection, seconds) - interrupt_board(connection, port) + interrupted = interrupt_board(connection, port) # Whatever the interrupt shook loose - a KeyboardInterrupt traceback, # a REPL banner - is backlog too, and leaving it queued re-wedges the # board the moment we close. @@ -910,7 +912,7 @@ def drain_port(port, seconds=DRAIN_SECONDS): except Exception: pass - log(f" drained {drained} byte(s) from {port} and sent Ctrl-C") + log(f" drained {drained} byte(s) from {port} and {'sent Ctrl-C' if interrupted else 'could not send Ctrl-C'}") return drained diff --git a/dev/hil/recover.py b/dev/hil/recover.py index ec74f8b6..0c3da708 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -179,11 +179,12 @@ def drain(port, seconds=DRAIN_SECONDS): return False drained = 0 + interrupted = False try: drained += read_until_quiet(connection, seconds) # Ctrl-C twice: once to break out of a sleep, once for whatever the # first one dropped us into. - interrupt_board(connection, port, CTRL_C + CTRL_C) + interrupted = interrupt_board(connection, port, CTRL_C + CTRL_C) drained += read_until_quiet(connection, DRAIN_QUIET_SECONDS * 2) except Exception as exc: log(f" error draining {port}: {exc}") @@ -193,11 +194,21 @@ def drain(port, seconds=DRAIN_SECONDS): except Exception: pass - log(f" drained {drained} byte(s) and sent Ctrl-C") - # Nothing queued is itself the diagnosis: a board merely blocked on a full - # buffer has a backlog to give up the moment somebody reads. - if drained == 0: + log(f" drained {drained} byte(s) and {'sent Ctrl-C' if interrupted else 'could not send Ctrl-C'}") + + # What the two directions did is the diagnosis, and they point at + # different remedies. + if drained and not interrupted: + log(" this board is talking but not listening: it is producing output") + log(" normally and never servicing what we send it, so no amount of") + log(" reading will reach it. That is what the USB reset and power") + log(" cycle rungs are for - see RUNNER_SETUP.md if they are skipped") + log(" below, because without them there is nothing left to try.") + elif not drained: + # A board merely blocked on a full buffer has a backlog to give up the + # moment somebody reads. log(" (nothing queued - so it is not simply blocked on a full output buffer)") + return True @@ -498,7 +509,7 @@ def main(): for port, method in recovered: log(f" recovered {port:16} by: {method}") for port in lost: - log(f" STILL DEAD {port}") + log(f" NOT ANSWERING {port}") log("=" * 60) if lost: @@ -506,6 +517,15 @@ def main(): log(f"{len(lost)} board(s) need a person at the bench:") log(" hold the BOOTSEL button while replugging the USB cable, then run") log(" dev/hil/flash_and_check.py to put Vector back on it.") + # A board that would not answer is not necessarily a board that is + # gone, and saying so matters: "still dead" sent a maintainer looking + # for a bricked Pico when the board in question was running the + # application and printing to its console the whole time, and only + # ever refused input. + log("") + log(" (a board here has failed every rung this runner can reach, which is not") + log(" the same as being bricked - check the drain output above for what it was") + log(" doing, and RUNNER_SETUP.md for the rungs that were skipped)") return 1 log("") diff --git a/dev/tests/test_hil_config_matrix.py b/dev/tests/test_hil_config_matrix.py index 27f009cf..e416c405 100644 --- a/dev/tests/test_hil_config_matrix.py +++ b/dev/tests/test_hil_config_matrix.py @@ -1137,21 +1137,32 @@ def write(self, data): assert connection.closed -def test_interrupt_board_drains_and_retries_when_the_write_times_out(monkeypatch): - """A timed-out write is a reason to read, not a reason to give up.""" +def test_interrupt_board_reports_a_board_that_will_not_accept_input(monkeypatch): + """Talking but not listening is a diagnosis, not a reason to keep writing. + + Both callers drain to quiet before interrupting, and reading is the remedy + for the wedge where the write times out because the firmware is blocked on + stdout. A write that still times out after that means the board is + producing output happily and simply never services what we send it - + /dev/ttyACM0 sat in exactly that state for three runs. Another five-second + write only spends five more seconds saying so. + """ attempts = [] - class Blocked(QuietingSerial): + class Deaf(QuietingSerial): def write(self, data): attempts.append(data) - if len(attempts) == 1: - raise bench.serial.SerialTimeoutException("blocked") + raise bench.serial.SerialTimeoutException("blocked") + + assert bench.interrupt_board(Deaf([]), "/dev/ttyFAKE") is False + assert len(attempts) == 1 - connection = Blocked([]) - monkeypatch.setattr(bench, "DRAIN_QUIET_SECONDS", 0.05) + +def test_interrupt_board_reports_a_board_that_takes_the_ctrl_c(monkeypatch): + connection = QuietingSerial([]) assert bench.interrupt_board(connection, "/dev/ttyFAKE") is True - assert len(attempts) == 2 + assert connection.written == [bench.CTRL_C] def test_a_session_whose_boot_failed_is_repaired_before_the_next_config(monkeypatch): From 18efd4dcc3db575777d04b9d4e0b6b380b93cde1 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 29 Aug 2026 14:24:26 +0000 Subject: [PATCH 20/32] hil: print how to fix the board map when it fails The map-related failures named the problem but not the remedy, and the remedy lives in ~/actions-runner/.env on the bench host - not somewhere the person reading the CI log is looking. Every one of them now prints the format, the valid targets, the current map, a pre-filled VECTOR_HIL_BOARD_MAP line carrying the entries that were already right, and the .env edit plus service restart that makes it take effect. --identify prints the same block instead of its own shorter variant. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/RUNNER_SETUP.md | 7 ++++++ dev/hil/bench.py | 56 +++++++++++++++++++++++++++++++++++------ 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index f301b649..0a1f2787 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -118,10 +118,17 @@ hence the explicit `.venv` path here.) Then record what you saw: ```bash +# the value is the whole map - list every board, including ones already correct +sed -i '/^VECTOR_HIL_BOARD_MAP=/d' ~/actions-runner/.env echo 'VECTOR_HIL_BOARD_MAP==sys11,=wpc,=data_east' >> ~/actions-runner/.env cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start ``` +Valid targets are the config families: `sys11`, `wpc`, `data_east`, `em`. The map must cover +*every* board the harness sees - adding a board to the bench means adding it here, or the run +stops with `VECTOR_HIL_BOARD_MAP is set but does not cover: ...`. That failure (and the other +map-related ones) prints these same instructions, pre-filled with the bench's actual chip ids. + With the map set, `flash_and_check.py` uses it and ignores self-report entirely. Without it, the harness falls back to self-report but **refuses to flash when two boards claim the same system**, since that means at least one is running firmware for a system it isn't wired for. diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 20d7d7fa..0ce4ae68 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -295,9 +295,8 @@ def identify(boards, seconds=8): else: log(" done") log("") - log("Now map what you saw to the chip ids, and put this in the runner's .env:") - log("") - log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) + log("Now map what you saw to the chip ids:") + log(board_map_instructions(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")))) # -------------------------------------------------------------------------- @@ -305,6 +304,50 @@ def identify(boards, seconds=8): # -------------------------------------------------------------------------- +def board_map_instructions(boards, board_map=None): + """How to write or amend VECTOR_HIL_BOARD_MAP for this bench. + + Printed with every map-related failure: the map lives in the runner's + .env on the bench host, so whoever reads the CI log is usually not + looking at the machine that needs editing. + """ + board_map = board_map or {} + suggested = ",".join(f"{b['chip_id']}={board_map.get(b['chip_id'], '')}" for b in boards) + lines = [ + "", + "VECTOR_HIL_BOARD_MAP pins each board to the system it is wired to, by RP2040", + "chip id (stable across reflashing). Every board on the bench must appear in it.", + "", + " format: =,=", + " targets: " + ", ".join(sorted(DEFAULT_GAMENAME)), + "", + ] + if board_map: + lines += ["current map:"] + [f" {chip}={target}" for chip, target in sorted(board_map.items())] + [""] + lines += [ + "Fill in the target for each board and set the whole line - it replaces the old", + "value, so keep the entries that were already right:", + "", + " VECTOR_HIL_BOARD_MAP=" + suggested, + "", + "On the bench host (the map is runner environment, not repo config):", + "", + " # drop any existing entry, then append the new one", + " sed -i '/^VECTOR_HIL_BOARD_MAP=/d' ~/actions-runner/.env", + " echo 'VECTOR_HIL_BOARD_MAP=" + suggested + "' >> ~/actions-runner/.env", + " cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start", + "", + "The service only reads .env at start, so the restart is required.", + "", + "Not sure which physical board is which chip id? Blink them in turn:", + "", + " cd ~/vector && .venv/bin/python dev/hil/flash_and_check.py --identify", + "", + "See dev/hil/RUNNER_SETUP.md for the full walkthrough.", + ] + return "\n".join(lines) + + def parse_board_map(raw): """Parse VECTOR_HIL_BOARD_MAP: 'chipid=target,chipid=target'.""" mapping = {} @@ -337,7 +380,7 @@ def resolve_targets(boards, board_map): if board_map: unmapped = [b for b in boards if b["chip_id"] not in board_map] if unmapped: - raise CheckFailure("VECTOR_HIL_BOARD_MAP is set but does not cover: " + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped)) + raise CheckFailure("VECTOR_HIL_BOARD_MAP is set but does not cover: " + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) + "\n" + board_map_instructions(boards, board_map)) for b in boards: b["target"] = board_map[b["chip_id"]] log("targets from VECTOR_HIL_BOARD_MAP") @@ -345,7 +388,7 @@ def resolve_targets(boards, board_map): missing = [b for b in boards if not b["system"]] if missing: - raise CheckFailure("cannot identify " + ", ".join(b["port"] for b in missing) + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP.") + raise CheckFailure("cannot identify " + ", ".join(b["port"] for b in missing) + " - firmware did not report a system. Set VECTOR_HIL_BOARD_MAP.\n" + board_map_instructions(boards, board_map)) systems = [b["system"] for b in boards] duplicates = {s for s in systems if systems.count(s) > 1} @@ -354,8 +397,7 @@ def resolve_targets(boards, board_map): "refusing to flash from autodetection: " + ", ".join(sorted(duplicates)) + " is reported by more than one board.\n" "Detection reads the *flashed firmware*, not the hardware, so duplicates mean\n" "at least one board is running firmware for a system it is not wired for.\n" - "Pin them explicitly instead, using the chip ids above:\n" - " VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards) + "Pin them explicitly instead, using the chip ids above.\n" + board_map_instructions(boards, board_map) ) for b in boards: From 8e783a7200837cfefadf2731a8f34e2a28e66c01 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 02:33:01 +0000 Subject: [PATCH 21/32] hil: rescue boards in BOOTSEL, and fail when a system is missing Three failures from the same bench run, all of which made a healthy or half-healthy bench look like something else. **"no boards found - check the USB hub and power" with every board attached.** All three were in BOOTSEL/UF2 mode, where the ROM bootloader enumerates as USB mass storage: `lsusb` lists them, `mpremote devs` shows nothing, and every serial question about them has no answer. The harness now looks for them in sysfs and identifies each one by the id the bootrom publishes as its USB serial number - the same id `machine.unique_id()` returns, so VECTOR_HIL_BOARD_MAP covers a board in this state like any other. It mounts the board's own drive (found through its USB device path, so three boards in BOOTSEL stay told apart), writes nuke.uf2 and then the pinned TrenchCoat UF2 for the mapped target, and waits for it to come back as a serial device. Every harness does this before it surveys, so the run just carries on. TrenchCoat's own `core.flash_firmware` cannot be used for this: it flashes every drive it can see, and narrowing it to one board - which the bench must do, or a rescue takes the healthy boards with it - makes its final wait for the board to restart unsatisfiable. That is also a latent bug in the existing recovery rung, which emptied `find_board_ports` and would have reported every successful reflash as a failure; it now filters to the recovered board instead. **A board the map does not know was a wall of text in the log.** Its chip id and the exact line to add now go to the job summary, where the person who has to edit the runner's .env is actually looking. An unmapped board is never guessed at - writing WPC firmware to the Data East board is the one mistake worth failing to avoid - so it is reported and left alone. Repeated findings reach the summary once; the log still says it in every stage. **The bench ran happily on two boards out of three.** A run that skips data_east tells you nothing about data_east while still reporting green. Missing systems now fail the run - after the boards that *are* there have been flashed and checked, since a bench run is expensive. VECTOR_HIL_REQUIRED_TARGETS lets a bench that has genuinely lost a board say so deliberately. Boards are also flashed concurrently now. They are independent devices on independent ports and dev/flash.py is one subprocess each, spending nearly all its time waiting on USB, so the stage costs about what one board costs. Flash output travels with the failure rather than being printed by the worker, so a failing board's log is still that board's log. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/DESIGN.md | 30 +++ dev/hil/RUNNER_SETUP.md | 33 ++- dev/hil/bench.py | 269 ++++++++++++++++++++- dev/hil/config_matrix.py | 14 +- dev/hil/flash_and_check.py | 62 +++-- dev/hil/recover.py | 40 ++-- dev/hil/trench_coat.py | 265 ++++++++++++++++++++- dev/tests/test_hil_bootsel.py | 436 ++++++++++++++++++++++++++++++++++ dev/tests/test_hil_recover.py | 4 + 9 files changed, 1084 insertions(+), 69 deletions(-) create mode 100644 dev/tests/test_hil_bootsel.py diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index a1989632..2ec3e0d6 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -225,6 +225,36 @@ recovery first is the point of combining them — it costs almost nothing on a healthy bench and turns "a board wedged, so the next four runs were useless" into a bench that repairs itself. +### What "the bench is here" means + +Three states have to be told apart before any of the above is worth running, and +getting them confused has cost whole runs: + +- **On serial.** The normal case: a port, a chip id, a system it reports running. +- **In BOOTSEL.** No serial port at all — the ROM bootloader enumerates as USB + mass storage, so `mpremote devs` is empty while `lsusb` lists every board. + This looked exactly like an unplugged bench and was reported as "no boards + found — check the USB hub and power" for a bench that was fine. The bootrom + publishes the board's unique id as its USB serial number, which is the same id + `machine.unique_id()` returns, so these boards are identifiable through + `VECTOR_HIL_BOARD_MAP` like any other: the harness mounts the drive and writes + the pinned TrenchCoat UF2 for the mapped target, and the board rejoins the run. +- **Absent.** Not enumerated either way. This is the state that must fail the + run rather than shrink it: two boards out of three passing is a green run that + proves nothing about the third. `REQUIRED_TARGETS` names the systems the bench + is for; `VECTOR_HIL_REQUIRED_TARGETS` on the runner is how a bench that has + really lost a board says so out loud. + +A board whose chip id is in none of the map's entries is never guessed at — the +id and the exact line to add are written to the job summary, because the fix is +one edit on the runner host and whoever makes it is reading the run, not the log. + +Boards are flashed concurrently. They are independent devices on independent +ports and `dev/flash.py` is one subprocess each, spending nearly all of its time +waiting on USB, so the stage costs about what one board costs instead of three. +The failure output travels with the exception rather than being printed by the +worker, so a failing board's log is still that board's log. + **Trigger: `workflow_run` on "Build and Deploy" completion**, which is every push to a PR. Not `pull_request`, and the distinction is the security model: `workflow_run` workflows always run *from the default branch, using the default diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 0a1f2787..5c961e35 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -135,6 +135,27 @@ system**, since that means at least one is running firmware for a system it isn' `--inventory-only` prints the chip ids without blinking or flashing anything. +A board in BOOTSEL is covered by the same map. Its ROM bootloader publishes the board's unique +id as the USB serial number, which is the id `machine.unique_id()` reports once firmware is +running, so a board can be recognised and put back without ever being a serial device. If one +ever turns up under two different ids, map both — a spare entry costs nothing. + +## Every system, every run + +The bench exists to cover `sys11`, `wpc` and `data_east`, and a run that quietly tests two of +them still reports green about the third. So a missing board fails the run: the boards that are +present are still flashed and checked, and the verdict at the end says which system was absent. + +If the bench has genuinely lost a board for a while, say so deliberately rather than letting the +check rot: + +```bash +echo 'VECTOR_HIL_REQUIRED_TARGETS=sys11,wpc' >> ~/actions-runner/.env +cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start +``` + +Unset, it defaults to all three. Set to an empty value, the check is off entirely. + ## Verify The runner should show **Idle** under Settings → Actions → Runners with the `vector-hil` label. @@ -229,7 +250,17 @@ the `recover` stage of [`hil.yml`](../../.github/workflows/hil.yml), which runs cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/recover.py ``` -It opens with a report of which rungs this runner can actually use. As measured on the bench +Before the ladder it deals with the one state none of the rungs can reach: a board in +**BOOTSEL/UF2 mode**. Such a board is not a serial device at all — the ROM bootloader +enumerates as USB mass storage, so `mpremote devs` shows nothing and `lsusb` shows everything, +which is exactly how a healthy bench came to report "no boards found". Every harness now looks +for those boards, identifies each one by the id its bootloader publishes, mounts its drive and +writes the TrenchCoat UF2 for the target the map gives it — after which the board is an ordinary +serial device again and the run carries on. A board the map does not cover is left alone, with +its id and the line to add printed to the job summary: there is no way to guess which system's +firmware it wants, and writing the wrong one is worse than leaving it. + +It then opens with a report of which rungs this runner can actually use. As measured on the bench on 2026-08-28: | rung | state | what it needs | diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 0ce4ae68..0c3ace16 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -15,6 +15,7 @@ a harness checks does not belong in this file. """ +import concurrent.futures import json import os import re @@ -107,6 +108,56 @@ def log(msg): print(msg, flush=True) +def _append_summary(line): + path = os.environ.get("GITHUB_STEP_SUMMARY") + if not path: + return + try: + with open(path, "a") as handle: + handle.write(line.replace("::warning::", "WARNING: ").replace("::error::", "ERROR: ") + "\n") + except OSError: + pass + + +def summary(line): + """Put a line in the Actions job summary as well as the log. + + Job summaries are stored separately from the log archive, and two bench + runs proved why that matters: the runner died mid-job, never uploaded its + logs, and every finding went with them. Written incrementally rather than + at the end for the same reason. + """ + log(line) + _append_summary(line) + + +def summary_once(marker, lines): + """Log `lines`, and add them to the summary unless `marker` is there already. + + Three stages share one summary page and reach the same conclusion about + the bench - "this board is not in the map", "this system is missing" - so + without this the reader gets the same block three times, which reads like + three problems. The log still says it every time, where repetition is + exactly what you want: each step's failure explains itself. + """ + for line in lines: + log(line) + path = os.environ.get("GITHUB_STEP_SUMMARY") + if path: + try: + if marker in Path(path).read_text(): + return + except OSError: + pass + for line in lines: + _append_summary(line) + + +def as_block(text): + """A multi-line string as preformatted summary lines.""" + return ["", "```", *text.splitlines(), "```"] + + _TIMINGS = [] _group = None @@ -183,6 +234,65 @@ def list_ports(): return [line.split()[0] for line in result.stdout.strip().splitlines() if line.strip()] +# -------------------------------------------------------------------------- +# Boards in the ROM bootloader +# -------------------------------------------------------------------------- +# +# A board in BOOTSEL/UF2 mode is not a serial device at all: the RP2040 ROM +# bootloader enumerates as USB mass storage, so `mpremote devs` shows nothing +# and every serial-based question about it - chip id, running system, is it +# alive - has no answer. A whole bench in that state reported "no boards found +# - check the USB hub and power" while `lsusb` listed all three, which is the +# most misleading thing the harness has ever said: the boards were fine, and +# one UF2 each away from working. +# +# They can still be identified. The bootrom publishes the board's unique id as +# the USB serial number, which is the same id `machine.unique_id()` returns +# once MicroPython is running, so VECTOR_HIL_BOARD_MAP covers a board in this +# state exactly as it covers a running one. + +BOOTSEL_VID = "2e8a" # Raspberry Pi +BOOTSEL_PIDS = {"0003": "RP2040", "000f": "RP2350"} # "RP2 Boot" + +USB_DEVICES = Path("/sys/bus/usb/devices") + + +def bootsel_boards(): + """Boards sitting in the ROM bootloader, from sysfs. + + Read straight out of sysfs rather than by shelling out to lsusb: it needs + no privileges, no package, and it hands us the device directory, which is + what finds the board's mass-storage drive later. + """ + boards = [] + for device in sorted(USB_DEVICES.glob("*")): + try: + vendor = (device / "idVendor").read_text().strip().lower() + product = (device / "idProduct").read_text().strip().lower() + except OSError: + # Interfaces (1-1:1.0) and the like have no ids. Not devices. + continue + if vendor != BOOTSEL_VID or product not in BOOTSEL_PIDS: + continue + try: + chip_id = (device / "serial").read_text().strip().lower() or None + except OSError: + chip_id = None + boards.append( + { + "port": None, + "chip_id": chip_id, + "system": None, + "version": None, + "responsive": False, + "bootsel": True, + "processor": BOOTSEL_PIDS[product], + "usb_device": device, + } + ) + return boards + + CHIP_ID_SNIPPET = "from machine import unique_id;from binascii import hexlify;print(hexlify(unique_id()).decode())" @@ -239,9 +349,16 @@ def probe(port): def inventory(): + """Survey the bench: every board on serial, plus any stuck in BOOTSEL. + + Only the serial boards are returned - a board in the bootloader cannot be + flashed by the normal route and has to be rescued first (see + trench_coat.rescue_bootsel) - but they are always listed, because "no + boards found" is a wrong and expensive answer when three of them are + sitting there as mass-storage devices. + """ boards = [probe(port) for port in list_ports()] - if not boards: - raise CheckFailure("no boards found - check the USB hub and power") + stranded = bootsel_boards() log(f"{'port':16} {'chip id':18} {'running':12} version") for b in boards: @@ -249,9 +366,30 @@ def inventory(): log(f"{b['port']:16} {'NOT ANSWERING':18} {'-':12} -") continue log(f"{b['port']:16} {b['chip_id'] or '?':18} {b['system'] or '(none)':12} {b['version'] or '-'}") + for b in stranded: + log(f"{'(BOOTSEL)':16} {b['chip_id'] or '?':18} {'bootloader':12} {b['processor']} ROM") + + if not boards: + raise CheckFailure(no_boards_message(stranded)) + if stranded: + log(f"::warning::{len(stranded)} board(s) are in BOOTSEL/UF2 mode and were not surveyed - run dev/hil/recover.py to put firmware back on them") return boards +def no_boards_message(stranded=None): + """Why the bench looks empty, told apart from a bench that is not there.""" + stranded = bootsel_boards() if stranded is None else stranded + if not stranded: + return "no boards found - check the USB hub and power" + ids = ", ".join(b["chip_id"] or "?" for b in stranded) + return ( + f"no board is on serial, but {len(stranded)} are in the ROM bootloader (BOOTSEL/UF2 mode): {ids}.\n" + "They are attached and healthy - they are just running the bootloader instead of firmware,\n" + "which is what a UF2 flash that did not finish leaves behind. dev/hil/recover.py flashes them\n" + "back; if it already ran, its output above says what stopped it." + ) + + IDENTIFY_SNIPPET = """ import machine, time try: @@ -343,6 +481,10 @@ def board_map_instructions(boards, board_map=None): "", " cd ~/vector && .venv/bin/python dev/hil/flash_and_check.py --identify", "", + "A board in BOOTSEL cannot blink, and the id above is the one its ROM", + "bootloader publishes. That is the same unique id MicroPython reports, but if", + "a board ever turns up under two, map both - a spare entry costs nothing.", + "", "See dev/hil/RUNNER_SETUP.md for the full walkthrough.", ] return "\n".join(lines) @@ -362,6 +504,27 @@ def parse_board_map(raw): return mapping +def report_unknown_boards(unmapped, boards, board_map): + """Say which boards the map does not cover, and how to add them. + + Written to the Actions job summary as well as the log: a board arriving on + the bench is a one-line edit on the runner host, and whoever has to make + it is reading the run's summary page, not scrolling a log for the chip id. + """ + described = ", ".join(f"{b.get('port') or '(BOOTSEL)'} {b['chip_id'] or '?'}" for b in unmapped) + summary_once( + described, + [ + "", + "### Unrecognised board" + ("s" if len(unmapped) > 1 else ""), + "", + f"`VECTOR_HIL_BOARD_MAP` does not cover: **{described}**", + *as_block(board_map_instructions(boards, board_map)), + ], + ) + return "VECTOR_HIL_BOARD_MAP is set but does not cover: " + described + + def resolve_targets(boards, board_map): """Decide the target for each board, refusing to guess when it matters. @@ -380,7 +543,7 @@ def resolve_targets(boards, board_map): if board_map: unmapped = [b for b in boards if b["chip_id"] not in board_map] if unmapped: - raise CheckFailure("VECTOR_HIL_BOARD_MAP is set but does not cover: " + ", ".join(f"{b['port']} ({b['chip_id']})" for b in unmapped) + "\n" + board_map_instructions(boards, board_map)) + raise CheckFailure(report_unknown_boards(unmapped, boards, board_map)) for b in boards: b["target"] = board_map[b["chip_id"]] log("targets from VECTOR_HIL_BOARD_MAP") @@ -406,6 +569,60 @@ def resolve_targets(boards, board_map): return boards +# Every system the bench exists to cover. A run that quietly tests two of the +# three proves nothing about the third while still reporting green, which is +# worse than not running: it is a check that has stopped checking. Overridable +# for a bench that has genuinely lost a board - deliberately, by whoever runs +# it, and visibly in the log. +REQUIRED_TARGETS = ("sys11", "wpc", "data_east") + + +def required_targets(): + raw = os.environ.get("VECTOR_HIL_REQUIRED_TARGETS") + if raw is None: + return list(REQUIRED_TARGETS) + return [target.strip() for target in raw.split(",") if target.strip()] + + +def check_bench_complete(boards): + """Which required systems are not on the bench, reported where it shows. + + Returns the missing targets rather than raising: the boards that *are* + here still have checks worth running, and a run is expensive. The caller + keeps the result and fails the run at the end - green on two boards out of + three is the outcome this exists to prevent. + """ + wanted = required_targets() + if not wanted: + log("no required targets set - running whatever is attached") + return [] + present = {b.get("target") for b in boards} + missing = [target for target in wanted if target not in present] + if not missing: + log("bench is complete: " + ", ".join(wanted)) + return [] + + have = ", ".join(f"{b['target']} ({b['port']})" for b in boards) or "nothing" + log(f"::error::the bench is missing {', '.join(missing)} - only {have} answered") + summary_once( + "### Incomplete bench", + [ + "", + "### Incomplete bench", + "", + f"Missing: **{', '.join(missing)}**. Attached: {have}.", + "", + "A run proves nothing about a system that is not on the bench, so this fails the run rather", + "than reporting green on two boards out of three. The boards that are here are still checked.", + "", + "Either put the missing board back (`dev/hil/recover.py`, and check it is in", + "`VECTOR_HIL_BOARD_MAP`), or set `VECTOR_HIL_REQUIRED_TARGETS` on the runner to the systems", + "the bench really has.", + ], + ) + return missing + + # -------------------------------------------------------------------------- # 3. build # -------------------------------------------------------------------------- @@ -475,9 +692,49 @@ def flash(target, port, build_dir, config_path): timeout=900, ) if result.returncode != 0: - log(result.stdout[-3000:]) - log(result.stderr[-3000:]) - raise CheckFailure(f"flash failed for {target} on {port}") + # The output travels with the failure rather than being logged here: + # boards are flashed concurrently, and a line printed from a worker + # thread lands next to another board's output with nothing to say + # which board it came from. + tail = (result.stdout[-2000:] + result.stderr[-2000:]).strip() + raise CheckFailure(f"flash failed for {target} on {port}:\n " + "\n ".join(tail.splitlines()[-20:])) + + +def flash_boards(boards, workdir): + """Flash every board at once, and report per board. + + The boards are independent devices on independent serial ports, and + dev/flash.py is one subprocess per board that spends nearly all its time + waiting on USB - so doing them one after another just adds up the waits. + Three boards on the bench Pi: about a minute, instead of three. + + Returns {port: error message}, empty when every board flashed. + """ + + def flash_one(board): + started = time.monotonic() + config_path = write_bench_config(board["target"], workdir) + flash(board["target"], board["port"], REPO_ROOT / "build" / board["target"], config_path) + return time.monotonic() - started + + errors = {} + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, len(boards))) as pool: + futures = {pool.submit(flash_one, board): board for board in boards} + for future in concurrent.futures.as_completed(futures): + board = futures[future] + try: + elapsed = future.result() + except CheckFailure as exc: + errors[board["port"]] = str(exc) + except Exception as exc: # noqa: BLE001 - reported per board, never fatal to the others + errors[board["port"]] = f"{type(exc).__name__}: {exc}" + else: + log(f" ok {board['port']:16} {board['target']:12} {elapsed:5.1f}s") + + for port, error in errors.items(): + log(f" FAIL {port}") + log(f"::error::{error}") + return errors # -------------------------------------------------------------------------- diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index 1b000013..2c556efc 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -52,6 +52,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) import bench # noqa: E402 +import trench_coat # noqa: E402 from bench import ( # noqa: E402 _TIMINGS, BENCH_WARN_FAULTS, @@ -634,6 +635,9 @@ def main(): workdir = REPO_ROOT / "build" workdir.mkdir(exist_ok=True) + board_map = parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")) + trench_coat.rescue_bootsel(board_map, REPO_ROOT / "build" / "hil") + group("Inventory") boards = inventory() endgroup() @@ -651,9 +655,13 @@ def main(): raise CheckFailure("no board on the bench is answering - run dev/hil/recover.py") group("Resolve targets") - boards = resolve_targets(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP"))) + boards = resolve_targets(boards, board_map) for b in boards: log(f" {b['port']} -> {b['target']}") + # Only when the run is meant to cover the bench: --target says the caller + # asked for one board on purpose, and failing that for incompleteness + # would be answering a question nobody asked. + missing_targets = [] if args.target else bench.check_bench_complete(boards) endgroup() if args.target: @@ -737,6 +745,10 @@ def main(): return 1 checked = sum(len(passed) for _board, passed, _failures in results) + if missing_targets: + log(f"\nall {checked} config(s) booted on the boards that are here, but the bench is missing " + ", ".join(missing_targets)) + return 1 + log(f"\nall {checked} config(s) booted and reported the right game name") return 0 diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index e9ec2020..6f49aeba 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -39,6 +39,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) +import trench_coat # noqa: E402 from bench import ( # noqa: E402 _TIMINGS, BENCH_WARN_FAULTS, @@ -49,10 +50,12 @@ CheckFailure, UsbApiClient, _dump_boot_log, + board_map_instructions, build, + check_bench_complete, endgroup, ensure_tools_on_path, - flash, + flash_boards, get, group, identify, @@ -64,7 +67,6 @@ resolve_targets, source_version, wait_for_server, - write_bench_config, ) # Read-only routes exercised over HTTP. Kept side-effect free so the check can @@ -102,8 +104,7 @@ def check_faults(board): warned = codes & BENCH_WARN_FAULTS if warned: - log(f"::warning::{board['port']} raised {sorted(warned)} - bare-board bus noise, " - "the board is in safe mode and the game config was NOT loaded") + log(f"::warning::{board['port']} raised {sorted(warned)} - bare-board bus noise, " "the board is in safe mode and the game config was NOT loaded") return warned @@ -235,19 +236,14 @@ def health_check_http(board): http_configs = payloads["/api/game/configs_list"] if len(http_configs) != board["usb_config_count"]: - raise CheckFailure( - f"http lists {len(http_configs)} configs, USB lists {board['usb_config_count']}" - ) + raise CheckFailure(f"http lists {len(http_configs)} configs, USB lists {board['usb_config_count']}") # Authentication is enforced over HTTP and deliberately bypassed over USB # (backend.py:280), so this is the only transport that can prove the gate # works. password_check is the one auth route with no side effects. status = http_status(f"{base}/api/auth/password_check") if status != 401: - raise CheckFailure( - f"/api/auth/password_check returned {status} without credentials, expected 401 - " - "HTTP authentication is not being enforced" - ) + raise CheckFailure(f"/api/auth/password_check returned {status} without credentials, expected 401 - " "HTTP authentication is not being enforced") log(f" GET {'/api/auth/password_check':34} 401 (auth enforced, as expected)") # A challenge must not be reusable: the handler deletes it on use. @@ -275,13 +271,12 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--skip-http", action="store_true", help="USB checks only; do not exercise the network stack") parser.add_argument("--skip-flash", action="store_true", help="health-check what is already on the boards") - parser.add_argument("--inventory-only", action="store_true", - help="print each board's chip id and stop - use this to build VECTOR_HIL_BOARD_MAP") - parser.add_argument("--identify", action="store_true", - help="blink each board in turn so you can tell which physical board is which") + parser.add_argument("--inventory-only", action="store_true", help="print each board's chip id and stop - use this to build VECTOR_HIL_BOARD_MAP") + parser.add_argument("--identify", action="store_true", help="blink each board in turn so you can tell which physical board is which") args = parser.parse_args() ensure_tools_on_path() + board_map = parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")) if args.identify: group("Inventory") @@ -296,30 +291,31 @@ def main(): group("Inventory") boards = inventory() endgroup() - log("") - log("The boards are dedicated to the bench, so pin them by chip id once and") - log("autodetection stops mattering. Put this in the runner's .env, filling in") - log("the target for each (sys11, wpc, data_east, em, whitestar, classic):") - log("") - log(" VECTOR_HIL_BOARD_MAP=" + ",".join(f"{b['chip_id']}=" for b in boards)) - log("") - log("Then: cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start") + log(board_map_instructions(boards, board_map)) return 0 workdir = REPO_ROOT / "build" workdir.mkdir(exist_ok=True) failures = [] + # A board in BOOTSEL has no serial port, so it is invisible to every stage + # below. Put firmware back on it first and it joins the run normally. + trench_coat.rescue_bootsel(board_map, REPO_ROOT / "build" / "hil") + group("Inventory") boards = inventory() endgroup() group("Resolve targets") - boards = resolve_targets(boards, parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP"))) + boards = resolve_targets(boards, board_map) for b in boards: log(f" {b['port']} -> {b['target']}") + missing = check_bench_complete(boards) endgroup() + if missing: + failures.append("the bench is missing " + ", ".join(missing)) + if not args.skip_flash: for target in sorted({b["target"] for b in boards}): group(f"Build {target}") @@ -327,17 +323,13 @@ def main(): log(f"built {target} at version {source_version(target)}") endgroup() + group(f"Flash {len(boards)} board(s)") + errors = flash_boards(boards, workdir) for b in boards: - group(f"Flash {b['target']} on {b['port']}") - try: - config_path = write_bench_config(b["target"], workdir) - flash(b["target"], b["port"], REPO_ROOT / "build" / b["target"], config_path) - log("flashed") - except CheckFailure as exc: - log(f"::error::{exc}") - failures.append(f"{b['port']} ({b['target']}): {exc}") + if b["port"] in errors: + failures.append(f"{b['port']} ({b['target']}): {errors[b['port']]}") b["skip"] = True - endgroup() + endgroup() for b in boards: if b.get("skip"): @@ -386,10 +378,12 @@ def main(): for b in boards: state = "FAIL" if any(b["port"] in f for f in failures) else "ok" log(f" {state:5} {b['port']:16} {b['target']:12} {b.get('ip') or ''}") + for target in missing: + log(f" {'ABSENT':5} {'-':16} {target}") log("=" * 60) if failures: - log(f"\n{len(failures)} board(s) failed:") + log(f"\n{len(failures)} failure(s):") for failure in failures: log(f" - {failure}") return 1 diff --git a/dev/hil/recover.py b/dev/hil/recover.py index 0c3da708..e206bac7 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -52,6 +52,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) +import bench # noqa: E402 import trench_coat # noqa: E402 from bench import ( # noqa: E402 CTRL_C, @@ -69,9 +70,14 @@ open_serial, parse_board_map, read_until_quiet, + summary, time_limit, ) +# Anything worth saying about a recovery is worth saying in the job summary +# too - see bench.summary for why. +note = summary + # ioctl number for USBDEVFS_RESET, from : _IO('U', 20). USBDEVFS_RESET = ord("U") << 8 | 20 @@ -109,7 +115,7 @@ def survey(board_map): """ ports = list_ports() if not ports: - raise CheckFailure("no boards found at all - check the USB hub and power") + raise CheckFailure(bench.no_boards_message()) alive, dead, claimed = [], [], set() note("") @@ -408,25 +414,6 @@ def recover(port, target, args): return None -def note(line): - """Append a line to the Actions job summary as well as the log. - - Job summaries are stored separately from the log archive, and the last two - bench runs proved why that matters: the runner died mid-job, never uploaded - its logs, and every finding went with them. Written incrementally rather - than at the end for the same reason. - """ - log(line) - path = os.environ.get("GITHUB_STEP_SUMMARY") - if not path: - return - try: - with open(path, "a") as handle: - handle.write(line.replace("::warning::", "WARNING: ").replace("::error::", "ERROR: ") + "\n") - except OSError: - pass - - def preflight(): """Report which recovery steps are actually available here. @@ -445,6 +432,10 @@ def preflight(): note("```") note(f" serial ok {len(ports)} port(s) visible") + stranded = bench.bootsel_boards() + if stranded: + note(f" bootsel found {len(stranded)} board(s) in the ROM bootloader: " + ", ".join(b["chip_id"] or "?" for b in stranded)) + node = usb_device_path(ports[0]) if ports else None if node is None: note(" usb reset unknown no device to check") @@ -481,8 +472,15 @@ def main(): preflight() endgroup() - group("Survey") board_map = parse_board_map(os.environ.get("VECTOR_HIL_BOARD_MAP")) + + # Rung 0, before the ladder proper. A board in BOOTSEL has no serial port, + # so none of the four steps below can even be attempted on it - and it is + # the one state that is always repairable, because the bootloader is + # sitting there waiting for a UF2. + trench_coat.rescue_bootsel(board_map, args.cache_dir) + + group("Survey") alive, dead, targets = survey(board_map) endgroup() diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py index c5a84320..1c609121 100644 --- a/dev/hil/trench_coat.py +++ b/dev/hil/trench_coat.py @@ -24,12 +24,19 @@ so `list_rpi_rp2_drives` is wrapped to mount the RPI-RP2 volume with udisksctl first. +A board found *already* in BOOTSEL is the one case its `flash_firmware` cannot +be narrowed to - see the second half of this file for why, and for the copy +sequence that replaces it there. It keeps the parts that matter: the nuke.uf2 +wipe, and the pinned UF2 bundle from this checkout. + Only `src.core`, `src.ray`, `src.ui` and `src.util` are imported, and between them they need nothing but pyserial - which the bench venv already has because mpremote ships it. `src.main` and `src.interactive` are the parts that want InquirerPy and a human, and neither is used here. """ +import os +import shutil import subprocess import sys import time @@ -37,7 +44,15 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) -from bench import SERIAL_WRITE_TIMEOUT, CheckFailure, log, open_serial # noqa: E402 +import bench # noqa: E402 +from bench import ( # noqa: E402 + SERIAL_WRITE_TIMEOUT, + CheckFailure, + endgroup, + group, + log, + open_serial, +) # Pinned, like every other third-party input to this bench. Bumping it means # reviewing what changed in the flashing sequence first. @@ -246,16 +261,26 @@ def flash(port, target, root): bound_serial_writes(ray) uf2 = bundled_uf2(root, target) + port_being_recovered = port drives = enter_bootloader(core, ray, port) if not drives: log(" the board never presented a bootloader drive, so there is nothing to flash") return False - # From here TrenchCoat drives, on this board only. find_board_ports is - # emptied because the board is already a drive - that makes its - # get_all_boards_into_bootloader() a no-op instead of a second attempt, and - # keeps it away from the healthy boards on the bench. - ray.Ray.find_board_ports = classmethod(lambda cls: []) + # From here TrenchCoat drives, on this board only. It sees every serial + # port except the healthy boards' - which is empty right now (the board + # being recovered is a drive, not a port), so its + # get_all_boards_into_bootloader() is a no-op instead of a second attempt, + # and it never touches the rest of the bench. + # + # Hiding *every* port instead would break the other end of the sequence: + # flash_firmware finishes by waiting for as many ports as it flashed + # drives, so a permanently empty list makes that wait unsatisfiable and + # turns a successful reflash into a timeout. Filtering rather than + # emptying also survives the board coming back on a different ttyACM + # number, which it often does. + others = {port for port in serial_ports() if port != port_being_recovered} + ray.Ray.find_board_ports = classmethod(lambda cls: [p for p in serial_ports() if p not in others]) core.list_rpi_rp2_drives = lambda: find_bootloader_drives() or drives # Their failure path prints advice and calls sys.exit; make it an exception @@ -272,3 +297,231 @@ def refuse_to_exit(now=False): # port, so reaching here is the success condition. log(" TrenchCoat reports the board restarted") return True + + +# -------------------------------------------------------------------------- +# Boards that are already in the bootloader +# -------------------------------------------------------------------------- +# +# A board found in BOOTSEL needs the second half of the sequence above and not +# the first: there is no port to reset, because the board is a mass-storage +# device already. TrenchCoat's own `core.flash_firmware` cannot be pointed at +# one board here - it wipes every drive it can see and then waits on +# `Ray.find_board_ports()` for as many boards as it flashed, so restricting it +# to one board (which the bench must do, or a rescue takes the healthy boards +# with it) makes its final wait unsatisfiable. So the copy sequence is spelled +# out below, with the parts that matter kept: nuke.uf2 first, and the pinned +# UF2 bundle from the checkout. + +BOOTSEL_SETTLE = 5 +DRIVE_TIMEOUT = 60 +RESTART_TIMEOUT = 90 + + +def serial_ports(): + return sorted(str(path) for path in Path("/dev").glob("ttyACM*")) + + +def block_device(usb_device): + """The /dev node behind a bootloader's mass-storage interface. + + Walked from the USB device's own sysfs directory rather than looked up in + /dev/disk/by-id, because that is exact: with three boards in BOOTSEL the + by-id names differ only by a serial string whose format is the bootrom's + business, while this path belongs to the one device we are holding. + """ + for block in sorted(Path(usb_device).glob("*/host*/target*/*/block/*")): + return Path("/dev") / block.name + return None + + +def mount_point(device): + """Where `device` is mounted, if it is.""" + try: + for line in Path("/proc/mounts").read_text().splitlines(): + fields = line.split() + if len(fields) > 1 and fields[0] == str(device): + return fields[1].replace("\\040", " ") + except OSError: + pass + return None + + +_mount_error = None + + +def mount(device): + """Mount a bootloader drive, whoever has to do it. Returns the path or None.""" + existing = mount_point(device) + if existing: + return existing + result = subprocess.run(["udisksctl", "mount", "--no-user-interaction", "-b", str(device)], capture_output=True, text=True, timeout=120) + if result.returncode == 0: + return result.stdout.strip().rsplit(" at ", 1)[-1].rstrip(".") + if "AlreadyMounted" in result.stderr: + return mount_point(device) + + # Said once. bootsel_drive keeps asking while it waits for a drive that + # may still be settling, and the same permission error sixty times over + # buries everything else in the log. + global _mount_error + detail = (result.stderr or result.stdout).strip() + if detail != _mount_error: + _mount_error = detail + log(f" could not mount {device}: {detail}") + return None + + +def unmount(device): + """Best effort, so the next flash does not trip over a stale mount.""" + subprocess.run(["udisksctl", "unmount", "--no-user-interaction", "-b", str(device)], capture_output=True, text=True, timeout=60) + + +def bootsel_drive(chip_id, timeout=DRIVE_TIMEOUT): + """Wait for one board's bootloader drive, by chip id, and mount it. + + Looked up afresh every time rather than remembered: writing a UF2 reboots + the board, so the device node, the sysfs path and the mount point are all + different on the other side of a copy. The chip id is the only handle that + survives. + """ + deadline = time.monotonic() + timeout + while True: + for board in bench.bootsel_boards(): + if board["chip_id"] != chip_id: + continue + device = block_device(board["usb_device"]) + if device is None or not device.exists(): + break + path = mount(device) + if path and (Path(path) / "INFO_UF2.TXT").exists(): + return device, path + break + if time.monotonic() >= deadline: + return None, None + time.sleep(1) + + +def wait_for_bootsel(chip_id, present, timeout=DRIVE_TIMEOUT, poll=0.25): + """Wait until the board is (or is no longer) enumerated as a bootloader. + + Polled quickly, because one of the two things it watches for is a gap: + the board vanishes when it starts running a UF2 and is back moments later, + and a slow poll can step straight over that. + """ + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + here = any(board["chip_id"] == chip_id for board in bench.bootsel_boards()) + if here == present: + return True + time.sleep(poll) + return False + + +def copy_uf2(uf2, device, drive): + """Write a UF2 to a mounted bootloader drive. + + An I/O error at the end of the copy is not reported as a failure: the + board reboots the moment the last block lands, which is exactly what a + successful flash looks like from this side - the device goes away + mid-write. Whether it worked is decided by what comes back, below. + """ + log(f" writing {uf2.name} to {drive}") + try: + shutil.copy(str(uf2), drive) + os.sync() + except OSError as exc: + log(f" the drive went away during the copy ({exc}) - that is usually the board rebooting") + unmount(device) + + +def flash_bootsel(chip_id, target, root): + """Put firmware back on a board that is sitting in its ROM bootloader. + + Returns the serial port it came back on, or None. The board is left + running TrenchCoat's bundled firmware for its target, which is a working + Vector board and, more to the point, a board the normal build-and-flash + path can talk to again. + """ + uf2 = bundled_uf2(root, target) + nuke = Path(root) / "uf2" / "nuke.uf2" + if not nuke.exists(): + raise CheckFailure(f"{nuke} is missing from the trench-coat checkout") + + before = set(serial_ports()) + device, drive = bootsel_drive(chip_id) + if drive is None: + log(f" no bootloader drive for {chip_id} - it is enumerated but its filesystem never appeared") + return None + + # The wipe is the load-bearing step: it erases the whole flash, so nothing + # from whatever state the board was left in survives into the new firmware. + copy_uf2(nuke, device, drive) + # Not seeing it leave is not a failure: a wipe can start and finish + # between two polls, and what matters is the state it settles in. + if not wait_for_bootsel(chip_id, present=False, timeout=30): + log(" never saw it restart - either the wipe was quick or it never began") + if not wait_for_bootsel(chip_id, present=True, timeout=DRIVE_TIMEOUT): + log(" the board did not come back as a bootloader after the wipe") + return None + time.sleep(BOOTSEL_SETTLE) + + device, drive = bootsel_drive(chip_id) + if drive is None: + log(" the wiped board never presented its drive again") + return None + copy_uf2(uf2, device, drive) + + deadline = time.monotonic() + RESTART_TIMEOUT + while time.monotonic() < deadline: + new = [port for port in serial_ports() if port not in before] + if new: + log(f" back as {new[0]} running {uf2.name}") + return new[0] + time.sleep(1) + log(f" {uf2.name} was written but the board never came back as a serial device") + return None + + +def rescue_bootsel(board_map, cache_dir): + """Flash every board found in BOOTSEL back to a serial device. + + Runs before anything else on the bench, because a board in this state is + invisible to every other stage: no serial port, no chip id over the REPL, + nothing to flash or health-check. Boards the map does not cover are + reported and left alone - there is no way to guess which UF2 they need, + and writing the wrong system's firmware is worse than leaving them. + + Returns the number of boards put back. + """ + stranded = bench.bootsel_boards() + if not stranded: + return 0 + + group(f"Rescue {len(stranded)} board(s) in BOOTSEL") + log("These boards are in the ROM bootloader, not running firmware. Flashing them back.") + unmapped = [board for board in stranded if board["chip_id"] not in board_map] + if unmapped: + log(f"::error::{bench.report_unknown_boards(unmapped, stranded, board_map)}") + + rescued = 0 + root = None + for board in stranded: + chip_id = board["chip_id"] + target = board_map.get(chip_id) + if not target: + continue + log(f" {chip_id} {board['processor']} -> {target}") + try: + if root is None: + root = clone(cache_dir / "trench-coat") + if flash_bootsel(chip_id, target, root): + rescued += 1 + except CheckFailure as exc: + log(f"::error::{chip_id} ({target}): {exc}") + except Exception as exc: # noqa: BLE001 - one board's rescue never stops the next + log(f"::error::{chip_id} ({target}): {type(exc).__name__}: {exc}") + + log(f"{rescued} of {len(stranded)} board(s) rescued") + endgroup() + return rescued diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py new file mode 100644 index 00000000..6fe96f45 --- /dev/null +++ b/dev/tests/test_hil_bootsel.py @@ -0,0 +1,436 @@ +"""Tests for the states the bench harness has to survive without a serial port. + +A board in BOOTSEL, a board the map has never seen, and a bench that is a board +short are the three ways a run can be worthless while looking fine, and all +three are hardware-free to pin down: sysfs is a directory, the job summary is a +file, and flashing is a subprocess. +""" + +from __future__ import annotations + +import sys +import time +import types +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[2] + +_serial_stub = sys.modules.setdefault("serial", types.ModuleType("serial")) +if not hasattr(_serial_stub, "SerialTimeoutException"): + _serial_stub.SerialTimeoutException = type("SerialTimeoutException", (Exception,), {}) +if "usb_coms_demo" not in sys.modules: + stub = types.ModuleType("usb_coms_demo") + stub.UsbApiClient = object + sys.modules["usb_coms_demo"] = stub + +sys.path.insert(0, str(REPO_ROOT / "dev" / "hil")) + +import bench # noqa: E402 +import trench_coat # noqa: E402 + + +def usb_tree(tmp_path, devices): + """Build a fake /sys/bus/usb/devices holding `devices` = {name: (vid, pid, serial)}.""" + root = tmp_path / "usb" + root.mkdir() + for name, fields in devices.items(): + entry = root / name + entry.mkdir() + if fields is None: # an interface, which has no ids at all + continue + vid, pid, serial = fields + (entry / "idVendor").write_text(vid + "\n") + (entry / "idProduct").write_text(pid + "\n") + if serial is not None: + (entry / "serial").write_text(serial + "\n") + return root + + +# -------------------------------------------------------------------------- +# finding a board that has no serial port +# -------------------------------------------------------------------------- + + +def test_bootsel_boards_finds_rp2_bootloaders(monkeypatch, tmp_path): + root = usb_tree( + tmp_path, + { + "1-1": ("2e8a", "0003", "E661A4D4179A5B2F"), # RP2040 in BOOTSEL + "1-2": ("2e8a", "000f", "df13a50c13958980"), # RP2350 in BOOTSEL + "1-3": ("2e8a", "0005", "e66141040380b42e"), # running MicroPython + "1-4": ("1d6b", "0002", None), # a hub + "1-1:1.0": None, # an interface + }, + ) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + found = bench.bootsel_boards() + + assert [b["chip_id"] for b in found] == ["e661a4d4179a5b2f", "df13a50c13958980"] + assert [b["processor"] for b in found] == ["RP2040", "RP2350"] + # The id is the same one the map is keyed by, so a board is identifiable + # here even though nothing can be asked of it. + assert all(b["port"] is None and b["bootsel"] and not b["responsive"] for b in found) + + +def test_bootsel_boards_is_empty_without_sysfs(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "USB_DEVICES", tmp_path / "nothing here") + assert bench.bootsel_boards() == [] + + +def test_an_empty_bench_is_told_apart_from_one_in_the_bootloader(monkeypatch): + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + assert "check the USB hub and power" in bench.no_boards_message() + + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "e66141040380b42e"}]) + message = bench.no_boards_message() + assert "e66141040380b42e" in message + assert "recover.py" in message + assert "check the USB hub and power" not in message + + +def test_inventory_lists_bootsel_boards_alongside_the_others(monkeypatch, capsys): + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "probe", lambda port: {"port": port, "chip_id": "aaaa", "system": "wpc", "version": "1.7", "responsive": True}) + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "bbbb", "processor": "RP2040"}]) + + boards = bench.inventory() + printed = capsys.readouterr().out + + # Only the usable board is returned, but the other one is not silent. + assert [b["port"] for b in boards] == ["/dev/ttyACM0"] + assert "bbbb" in printed and "BOOTSEL" in printed + assert "::warning::" in printed + + +def test_inventory_explains_a_bench_that_is_entirely_in_the_bootloader(monkeypatch): + monkeypatch.setattr(bench, "list_ports", lambda: []) + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "bbbb", "processor": "RP2040"}]) + + with pytest.raises(bench.CheckFailure, match="ROM bootloader"): + bench.inventory() + + +# -------------------------------------------------------------------------- +# a board the map does not know +# -------------------------------------------------------------------------- + + +@pytest.fixture() +def job_summary(monkeypatch, tmp_path): + path = tmp_path / "summary.md" + monkeypatch.setenv("GITHUB_STEP_SUMMARY", str(path)) + return path + + +def test_an_unknown_board_puts_its_id_and_the_fix_in_the_job_summary(job_summary): + boards = [ + {"port": "/dev/ttyACM0", "chip_id": "aaaa"}, + {"port": None, "chip_id": "cccc"}, + ] + board_map = {"aaaa": "wpc"} + + message = bench.report_unknown_boards([boards[1]], boards, board_map) + written = job_summary.read_text() + + assert "cccc" in message + # The id, the line to set, and where to set it - the whole remedy, on the + # page somebody reads when a run goes red. + assert "cccc" in written + assert "VECTOR_HIL_BOARD_MAP=aaaa=wpc,cccc=" in written + assert "actions-runner/.env" in written + # A board already mapped keeps its target in the suggested line. + assert "aaaa=" not in written + + +def test_resolve_targets_reports_the_board_it_cannot_place(job_summary): + boards = [ + {"port": "/dev/ttyACM0", "chip_id": "aaaa", "system": "wpc", "responsive": True}, + {"port": "/dev/ttyACM1", "chip_id": "cccc", "system": None, "responsive": True}, + ] + + with pytest.raises(bench.CheckFailure, match="does not cover"): + bench.resolve_targets(boards, {"aaaa": "wpc"}) + + assert "cccc" in job_summary.read_text() + + +def test_the_instructions_name_the_valid_targets(job_summary): + text = bench.board_map_instructions([{"port": None, "chip_id": "cccc"}], {}) + for target in bench.DEFAULT_GAMENAME: + assert target in text + + +# -------------------------------------------------------------------------- +# a bench that is a board short +# -------------------------------------------------------------------------- + + +def boards_for(*targets): + return [{"port": f"/dev/ttyACM{i}", "target": target} for i, target in enumerate(targets)] + + +def test_a_missing_system_fails_the_run(job_summary, monkeypatch): + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + + missing = bench.check_bench_complete(boards_for("wpc", "sys11")) + + assert missing == ["data_east"] + written = job_summary.read_text() + assert "Incomplete bench" in written + assert "data_east" in written + + +def test_a_complete_bench_says_nothing_to_the_summary(job_summary, monkeypatch): + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + + assert bench.check_bench_complete(boards_for("wpc", "sys11", "data_east")) == [] + assert not job_summary.exists() + + +def test_a_bench_that_really_has_lost_a_board_can_say_so(job_summary, monkeypatch): + monkeypatch.setenv("VECTOR_HIL_REQUIRED_TARGETS", "wpc,sys11") + assert bench.check_bench_complete(boards_for("wpc", "sys11")) == [] + + +def test_the_check_can_be_turned_off_entirely(job_summary, monkeypatch): + monkeypatch.setenv("VECTOR_HIL_REQUIRED_TARGETS", "") + assert bench.check_bench_complete([]) == [] + + +# -------------------------------------------------------------------------- +# flashing every board at once +# -------------------------------------------------------------------------- + + +def test_boards_are_flashed_in_parallel(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + + def slow_flash(target, port, build_dir, config_path): + time.sleep(0.3) + + monkeypatch.setattr(bench, "flash", slow_flash) + boards = boards_for("wpc", "sys11", "data_east") + + started = time.monotonic() + assert bench.flash_boards(boards, tmp_path) == {} + elapsed = time.monotonic() - started + + # Three 0.3s flashes, one after another, would be 0.9s. + assert elapsed < 0.6 + + +def test_one_board_failing_to_flash_does_not_take_the_others_with_it(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + + def flash(target, port, build_dir, config_path): + if target == "sys11": + raise bench.CheckFailure("flash failed for sys11: no space left") + + monkeypatch.setattr(bench, "flash", flash) + boards = boards_for("wpc", "sys11", "data_east") + + errors = bench.flash_boards(boards, tmp_path) + + # Attributed to the right board, which is the whole risk of doing this + # concurrently. + assert list(errors) == ["/dev/ttyACM1"] + assert "no space left" in errors["/dev/ttyACM1"] + + +def test_an_unexpected_error_is_reported_rather_than_escaping(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "write_bench_config", lambda target, workdir: tmp_path / f"{target}.json") + + def flash(target, port, build_dir, config_path): + raise RuntimeError("the venv moved") + + monkeypatch.setattr(bench, "flash", flash) + + errors = bench.flash_boards(boards_for("wpc"), tmp_path) + + assert "RuntimeError: the venv moved" in errors["/dev/ttyACM0"] + + +# -------------------------------------------------------------------------- +# putting firmware back on a board in the bootloader +# -------------------------------------------------------------------------- + + +def test_the_drive_is_found_through_the_devices_own_sysfs_path(tmp_path): + device = tmp_path / "1-1" + block = device / "1-1:1.0" / "host0" / "target0:0:0" / "0:0:0:0" / "block" / "sda" + block.mkdir(parents=True) + + assert trench_coat.block_device(device) == Path("/dev/sda") + + +def test_no_drive_yet_is_not_an_error(tmp_path): + device = tmp_path / "1-1" + device.mkdir() + assert trench_coat.block_device(device) is None + + +def test_an_already_mounted_drive_is_not_mounted_again(monkeypatch): + monkeypatch.setattr(trench_coat, "mount_point", lambda device: "/media/runner/RPI-RP2") + monkeypatch.setattr(trench_coat.subprocess, "run", lambda *a, **k: pytest.fail("udisksctl must not be called for a drive that is already mounted")) + + assert trench_coat.mount("/dev/sda") == "/media/runner/RPI-RP2" + + +def test_mount_point_reads_proc_mounts(monkeypatch, tmp_path): + mounts = tmp_path / "mounts" + mounts.write_text("proc /proc proc rw 0 0\n/dev/sdb /media/runner/RPI-RP2 vfat rw 0 0\n") + real_read = Path.read_text + monkeypatch.setattr(Path, "read_text", lambda self, *a, **k: real_read(mounts) if str(self) == "/proc/mounts" else real_read(self, *a, **k)) + + assert trench_coat.mount_point("/dev/sdb") == "/media/runner/RPI-RP2" + assert trench_coat.mount_point("/dev/sdc") is None + + +@pytest.fixture() +def rescue(monkeypatch): + """Fake out everything below rescue_bootsel and record what it flashed.""" + flashed = [] + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: Path("/trench-coat")) + monkeypatch.setattr(trench_coat, "flash_bootsel", lambda chip_id, target, root: flashed.append((chip_id, target)) or "/dev/ttyACM9") + return flashed + + +def test_a_board_in_the_bootloader_is_flashed_with_its_own_targets_firmware(monkeypatch, rescue, tmp_path): + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "aaaa", "processor": "RP2040"}, {"chip_id": "bbbb", "processor": "RP2040"}]) + + assert trench_coat.rescue_bootsel({"aaaa": "wpc", "bbbb": "data_east"}, tmp_path) == 2 + assert rescue == [("aaaa", "wpc"), ("bbbb", "data_east")] + + +def test_an_unmapped_board_is_reported_and_left_alone(monkeypatch, rescue, tmp_path, job_summary): + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "aaaa", "processor": "RP2040"}, {"chip_id": "cccc", "processor": "RP2350"}]) + + # Guessing which system's firmware to write would be a good way to flash + # WPC firmware onto the Data East board. + assert trench_coat.rescue_bootsel({"aaaa": "wpc"}, tmp_path) == 1 + assert rescue == [("aaaa", "wpc")] + assert "cccc" in job_summary.read_text() + + +def test_nothing_in_the_bootloader_costs_nothing(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(trench_coat, "clone", lambda *a, **k: pytest.fail("must not clone trench-coat with nothing to rescue")) + + assert trench_coat.rescue_bootsel({"aaaa": "wpc"}, tmp_path) == 0 + + +def test_one_boards_rescue_failing_does_not_stop_the_next(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "aaaa", "processor": "RP2040"}, {"chip_id": "bbbb", "processor": "RP2040"}]) + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: Path("/trench-coat")) + tried = [] + + def flash_bootsel(chip_id, target, root): + tried.append(chip_id) + if chip_id == "aaaa": + raise bench.CheckFailure("no drive ever appeared") + return "/dev/ttyACM1" + + monkeypatch.setattr(trench_coat, "flash_bootsel", flash_bootsel) + + assert trench_coat.rescue_bootsel({"aaaa": "wpc", "bbbb": "sys11"}, tmp_path) == 1 + assert tried == ["aaaa", "bbbb"] + + +def test_the_wipe_comes_before_the_firmware(monkeypatch, tmp_path): + """nuke.uf2 first is the whole difference between a recovery and an upgrade.""" + root = tmp_path / "trench-coat" + (root / "uf2").mkdir(parents=True) + (root / "uf2" / "nuke.uf2").write_text("nuke") + (root / "uf2" / trench_coat.TARGET_UF2["wpc"]).write_text("firmware") + + copied = [] + monkeypatch.setattr(trench_coat, "copy_uf2", lambda uf2, device, drive: copied.append(uf2.name)) + monkeypatch.setattr(trench_coat, "bootsel_drive", lambda chip_id, timeout=None: ("/dev/sda", "/media/RPI-RP2")) + monkeypatch.setattr(trench_coat, "wait_for_bootsel", lambda chip_id, present, timeout=None: True) + monkeypatch.setattr(trench_coat, "BOOTSEL_SETTLE", 0) + ports = iter([[], [], ["/dev/ttyACM0"]]) + monkeypatch.setattr(trench_coat, "serial_ports", lambda: next(ports, ["/dev/ttyACM0"])) + + assert trench_coat.flash_bootsel("aaaa", "wpc", root) == "/dev/ttyACM0" + assert copied == ["nuke.uf2", trench_coat.TARGET_UF2["wpc"]] + + +def test_a_board_that_never_presents_a_drive_is_not_flashed(monkeypatch, tmp_path): + root = tmp_path / "trench-coat" + (root / "uf2").mkdir(parents=True) + (root / "uf2" / "nuke.uf2").write_text("nuke") + (root / "uf2" / trench_coat.TARGET_UF2["wpc"]).write_text("firmware") + + monkeypatch.setattr(trench_coat, "copy_uf2", lambda *a: pytest.fail("nothing to copy to")) + monkeypatch.setattr(trench_coat, "bootsel_drive", lambda chip_id, timeout=None: (None, None)) + monkeypatch.setattr(trench_coat, "serial_ports", lambda: []) + + assert trench_coat.flash_bootsel("aaaa", "wpc", root) is None + + +def test_trench_coat_is_shown_the_recovered_board_and_no_other(monkeypatch, tmp_path): + """The port filter has to do two opposite things at two moments. + + Before the flash it must hide every board, so TrenchCoat resets none of + them; after it, it must show the recovered one, or its own wait for the + board to restart can never be satisfied. + """ + seen = {} + + class FakeRay: + def __init__(self, port): + seen.setdefault("bootloader", []).append(port) + + def enter_bootloader_mode(self): + pass + + @classmethod + def find_board_ports(cls): + return [] + + ray = types.SimpleNamespace(Ray=FakeRay, serial=types.SimpleNamespace(Serial=lambda *a, **k: object())) + core = types.SimpleNamespace( + list_rpi_rp2_drives=lambda: ["/media/RPI-RP2"], + graceful_exit=lambda now=False: None, + flash_firmware=lambda path: seen.update(during=ray.Ray.find_board_ports()), + ) + monkeypatch.setattr(trench_coat, "clone", lambda root, commit=None: root) + monkeypatch.setattr(trench_coat, "load", lambda root: (core, ray)) + monkeypatch.setattr(trench_coat, "bundled_uf2", lambda root, target: Path("/uf2/wpc.uf2")) + monkeypatch.setattr(trench_coat, "find_bootloader_drives", lambda: ["/media/RPI-RP2"]) + monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: ["/media/RPI-RP2"]) + + # The two healthy boards are on ACM0 and ACM2; the board being recovered is + # a drive right now and comes back on a number it did not have before. + monkeypatch.setattr(trench_coat, "serial_ports", lambda: ["/dev/ttyACM0", "/dev/ttyACM2"]) + assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is True + assert seen["during"] == [] + assert seen["bootloader"] == ["/dev/ttyACM1"] + + hidden = ray.Ray.find_board_ports + monkeypatch.setattr(trench_coat, "serial_ports", lambda: ["/dev/ttyACM0", "/dev/ttyACM2", "/dev/ttyACM3"]) + assert hidden() == ["/dev/ttyACM3"] + + +def test_the_same_finding_reaches_the_summary_once(job_summary, monkeypatch): + """Three stages, one summary page, one problem.""" + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + boards = boards_for("wpc", "sys11") + + for _stage in range(3): + assert bench.check_bench_complete(boards) == ["data_east"] + + assert job_summary.read_text().count("### Incomplete bench") == 1 + + +def test_a_different_finding_still_gets_through(job_summary): + boards = [{"port": None, "chip_id": "cccc"}, {"port": None, "chip_id": "dddd"}] + + bench.report_unknown_boards([boards[0]], boards, {}) + bench.report_unknown_boards([boards[1]], boards, {}) + + written = job_summary.read_text() + assert "cccc" in written and "dddd" in written diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index 042ff410..507495c3 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -363,6 +363,9 @@ def find_board_ports(cls): monkeypatch.setattr(trench_coat, "find_bootloader_drives", lambda: list(drives)) monkeypatch.setattr(trench_coat, "wait_for_drive", lambda core, timeout=None: list(drives)) monkeypatch.setattr(trench_coat, "bootsel_touch", lambda port: None) + # The bench's healthy boards. Real serial ports would make what + # find_board_ports is allowed to see depend on the developer's own desk. + monkeypatch.setattr(trench_coat, "serial_ports", lambda: ["/dev/ttyACM0", "/dev/ttyACM2"]) return seen @@ -376,6 +379,7 @@ def test_flash_hides_the_other_boards_from_trench_coat(monkeypatch, tmp_path): seen = fake_trench_coat(monkeypatch) assert trench_coat.flash("/dev/ttyACM1", "wpc", tmp_path) is True + # Neither healthy board is visible to it, so neither gets reset. assert seen["ports_seen"] == [] assert seen["flashed"] == "/uf2/wpc.uf2" # Only the board being recovered is asked to enter the bootloader. From e6ce32785cb66dc6bec7d407dcfd7da58c6b50de Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 02:40:23 +0000 Subject: [PATCH 22/32] hil: print the USB bus when the bench looks empty The run on 8e783a7 found no boards on serial *and* nothing in the ROM bootloader, which is the one answer the BOOTSEL support cannot act on - and it contradicts what lsusb reportedly shows on the bench host. "no boards found - check the USB hub and power" is not enough to settle that. So when nothing is found, the harness now writes out the USB bus as sysfs sees it: every device with its vendor:product id, product string and serial. It is lsusb without needing lsusb, and it distinguishes the three cases that look identical from here - the boards are off the bus entirely, they are on it under an id this harness does not recognise, or sysfs itself is unreadable. A Raspberry Pi vendor id in an unhandled mode is called out against the ids the harness does handle, since that id is the thing to chase. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/bench.py | 68 ++++++++++++++++++++++++++++++++++- dev/tests/test_hil_bootsel.py | 38 ++++++++++++++++++-- 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 0c3ace16..f120e9a9 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -376,11 +376,77 @@ def inventory(): return boards +def usb_devices(): + """Every USB device sysfs knows about - the harness's own `lsusb`. + + Written out whenever the bench looks empty, because "no boards found" and + "lsusb shows all three" is a contradiction the log has to be able to + settle: it says whether the boards are absent from the bus, present under + an id this harness does not know, or present with no driver bound. + """ + devices = [] + for device in sorted(USB_DEVICES.glob("*")): + try: + vendor = (device / "idVendor").read_text().strip().lower() + product = (device / "idProduct").read_text().strip().lower() + except OSError: + continue + if not vendor or not product: + continue + + def attribute(name): + try: + return (device / name).read_text().strip() + except OSError: + return "" + + devices.append( + { + "path": device.name, + "id": f"{vendor}:{product}", + "name": " ".join(filter(None, (attribute("manufacturer"), attribute("product")))) or "(no product string)", + "serial": attribute("serial"), + } + ) + return devices + + +def usb_bus_report(): + """The USB bus as sysfs sees it, for a log that has to explain an empty bench.""" + devices = usb_devices() + if not devices: + return [ + "Nothing at all is on the USB bus (" + str(USB_DEVICES) + " lists no devices),", + "which is a host-side answer rather than a board-side one: the hub is unplugged,", + "unpowered, or this process cannot read sysfs.", + ] + + lines = ["What is on the USB bus, as sysfs sees it:", ""] + for device in devices: + lines.append(f" {device['path']:12} {device['id']:10} {device['name']}" + (f" serial {device['serial']}" if device["serial"] else "")) + + unknown = [d for d in devices if d["id"].startswith(BOOTSEL_VID + ":")] + if unknown: + lines += [ + "", + "Raspberry Pi devices are on the bus (" + ", ".join(sorted({d["id"] for d in unknown})) + ") but none of them is", + "a serial port or a bootloader this harness recognises (" + ", ".join(f"{BOOTSEL_VID}:{pid}" for pid in BOOTSEL_PIDS) + ").", + "That id is the thing to chase - it says what mode the boards are actually in.", + ] + return lines + + def no_boards_message(stranded=None): """Why the bench looks empty, told apart from a bench that is not there.""" stranded = bootsel_boards() if stranded is None else stranded if not stranded: - return "no boards found - check the USB hub and power" + return "\n".join( + [ + "no boards found: nothing on serial, and nothing in the ROM bootloader either.", + "", + *usb_bus_report(), + ] + ) ids = ", ".join(b["chip_id"] or "?" for b in stranded) return ( f"no board is on serial, but {len(stranded)} are in the ROM bootloader (BOOTSEL/UF2 mode): {ids}.\n" diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index 6fe96f45..5864718d 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -80,9 +80,10 @@ def test_bootsel_boards_is_empty_without_sysfs(monkeypatch, tmp_path): assert bench.bootsel_boards() == [] -def test_an_empty_bench_is_told_apart_from_one_in_the_bootloader(monkeypatch): +def test_an_empty_bench_is_told_apart_from_one_in_the_bootloader(monkeypatch, tmp_path): monkeypatch.setattr(bench, "bootsel_boards", lambda: []) - assert "check the USB hub and power" in bench.no_boards_message() + monkeypatch.setattr(bench, "USB_DEVICES", tmp_path / "nothing") + assert "nothing in the ROM bootloader either" in bench.no_boards_message() monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "e66141040380b42e"}]) message = bench.no_boards_message() @@ -434,3 +435,36 @@ def test_a_different_finding_still_gets_through(job_summary): written = job_summary.read_text() assert "cccc" in written and "dddd" in written + + +# -------------------------------------------------------------------------- +# an empty bench has to explain itself +# -------------------------------------------------------------------------- + + +def test_an_empty_bench_prints_the_usb_bus(monkeypatch, tmp_path): + """ "no boards found" and "lsusb shows them" is a contradiction the log must settle.""" + root = usb_tree( + tmp_path, + { + "usb1": ("1d6b", "0002", "0000:01:00.0"), + "1-1": ("2109", "3431", None), + "1-1.1": ("2e8a", "0005", "e661a4d4179a5b2f"), # a mode we do not handle + "1-1.1:1.0": None, + }, + ) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + message = bench.no_boards_message() + + assert "2e8a:0005" in message + assert "2109:3431" in message + # The recognised ids are named, so the reader can see what the mismatch is. + assert "2e8a:0003" in message + # Interfaces carry no ids and are not devices. + assert "1-1.1:1.0" not in message + + +def test_an_empty_sysfs_says_so_rather_than_blaming_the_boards(monkeypatch, tmp_path): + monkeypatch.setattr(bench, "USB_DEVICES", tmp_path / "nothing") + assert "Nothing at all is on the USB bus" in bench.no_boards_message() From 659abfbba9f17447f74a00780f31e519c736fb2d Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 02:45:20 +0000 Subject: [PATCH 23/32] hil: take the target vocabulary from the repo, not from a hardcoded four MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bench came back on e6ce327 - three ports, recover clean - and the third board is not the one the map expects. It reports chip 899fab8c90bfeb9a running `classic` 0.1.0, where the map has 6a2634124a65beb4=data_east. The unknown-board path handled that correctly and printed the line to add, but the line it offered was wrong in a way that mattered: it listed four targets, because it read DEFAULT_GAMENAME, while the tree builds six. `classic` was not among them, so the instructions pointed at mapping a classic board to a system it is not wired for - and a map entry naming `classic` would not have failed there either. It would have passed resolve and died on a bare KeyError inside write_bench_config, two stages and one flash later. So the vocabulary now comes from dev/ci/targets.json (the same file DESIGN.md §6 says a board's target must match), and "can the bench drive it" is asked separately and answered from the tree: a target is driveable when it has a generic game config to boot against. classic and whitestar have a systemConfig.py and no config/ directory, so they are named in the instructions as targets that exist and cannot be mapped - which is the honest answer for the board actually sitting on the bench. check_target() runs in resolve, for both the map and the self-report path, so a target the bench cannot drive fails while it is still a line of configuration rather than a flashed board. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/RUNNER_SETUP.md | 6 +++- dev/hil/bench.py | 68 ++++++++++++++++++++++++++++++++++- dev/tests/test_hil_bootsel.py | 57 +++++++++++++++++++++++++++++ 3 files changed, 129 insertions(+), 2 deletions(-) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 5c961e35..ad72addc 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -124,7 +124,11 @@ echo 'VECTOR_HIL_BOARD_MAP==sys11,=wpc,=data_east' >> ~/act cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start ``` -Valid targets are the config families: `sys11`, `wpc`, `data_east`, `em`. The map must cover +Valid targets are the ones with game configs to boot against: `sys11`, `wpc`, `data_east`, `em`. +The tree also builds `classic` and `whitestar`, and the bench **cannot** drive those — they have a +`systemConfig.py` and no `config/` directory, so there is nothing to flash and boot. A board wired +for one of them cannot be mapped at all until that target has configs; mapping it to a different +system would flash the wrong firmware to real hardware, so the harness refuses it by name. The map must cover *every* board the harness sees - adding a board to the bench means adding it here, or the run stops with `VECTOR_HIL_BOARD_MAP is set but does not cover: ...`. That failure (and the other map-related ones) prints these same instructions, pre-filled with the bench's actual chip ids. diff --git a/dev/hil/bench.py b/dev/hil/bench.py index f120e9a9..8c8f9718 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -523,9 +523,18 @@ def board_map_instructions(boards, board_map=None): "chip id (stable across reflashing). Every board on the bench must appear in it.", "", " format: =,=", - " targets: " + ", ".join(sorted(DEFAULT_GAMENAME)), + " targets: " + ", ".join(bench_targets()), "", ] + unusable = [target for target in buildable_targets() if target not in bench_targets()] + if unusable: + lines += [ + "This checkout also builds " + ", ".join(unusable) + ", which the bench cannot drive:", + "those targets have no game configs to boot against yet. A board running one of them", + "cannot be mapped to it - and mapping it to a different system would flash the wrong", + "firmware to real hardware.", + "", + ] if board_map: lines += ["current map:"] + [f" {chip}={target}" for chip, target in sorted(board_map.items())] + [""] lines += [ @@ -612,6 +621,7 @@ def resolve_targets(boards, board_map): raise CheckFailure(report_unknown_boards(unmapped, boards, board_map)) for b in boards: b["target"] = board_map[b["chip_id"]] + check_target(b["target"]) log("targets from VECTOR_HIL_BOARD_MAP") return boards @@ -631,6 +641,7 @@ def resolve_targets(boards, board_map): for b in boards: b["target"] = b["system"] + check_target(b["target"]) log("targets from firmware self-report (all distinct)") return boards @@ -694,6 +705,61 @@ def check_bench_complete(boards): # -------------------------------------------------------------------------- +# The repo's own list of what it builds - the same file the bench manifest in +# DESIGN.md §6 says a board's target must match. Read rather than restated, +# because a hardcoded copy is a list that goes stale silently. +TARGETS_JSON = REPO_ROOT / "dev" / "ci" / "targets.json" + + +def buildable_targets(): + """Every distinct hardware target this checkout can build. + + `hardware_id` collapses the variants that are one board with two firmware + passes (sys11_tiny is sys11), because this list answers "what can a board + be", not "what can be built". + """ + try: + entries = json.loads(TARGETS_JSON.read_text()) + targets = {entry.get("hardware_id") or entry["id"] for entry in entries} + except (OSError, ValueError, KeyError): + # src/common is shared code, not a system. + targets = {path.parent.name for path in (REPO_ROOT / "src").glob("*/systemConfig.py")} - {"common"} + return sorted(targets) + + +def bench_targets(): + """Targets the bench can actually flash and health-check. + + Narrower than what the tree can build, and the gap is real: `classic` and + `whitestar` have a systemConfig.py and nothing else - no config directory, + so no generic game config to boot them against and nothing for + /api/game/configs_list to return. A board running one of them turned up on + the bench and the harness had no way to say any of this: it offered four + targets with no hint that the tree has seven, and a map entry naming one of + the other three died on a bare KeyError three stages later. + """ + ready = [] + for target in buildable_targets(): + config = DEFAULT_GAMENAME.get(target) + if config and (REPO_ROOT / "src" / target / "config" / f"{config}.json").exists(): + ready.append(target) + return ready + + +def check_target(target): + """Refuse a target the bench cannot drive, while it is still cheap to say so.""" + if target in bench_targets(): + return + known = buildable_targets() + if target not in known: + raise CheckFailure(f"{target!r} is not a target in this checkout - it builds " + ", ".join(known)) + raise CheckFailure( + f"{target!r} is a real build target but the bench cannot drive it: src/{target}/config holds no\n" + f" generic game config to flash and boot against. The bench can drive " + ", ".join(bench_targets()) + ".\n" + " A board wired for it can only join the bench once that target has configs." + ) + + def source_version(target): config = REPO_ROOT / "src" / target / "systemConfig.py" match = re.search(r'SystemVersion\s*=\s*"([^"]+)"', config.read_text()) diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index 5864718d..473a6cb7 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -468,3 +468,60 @@ def test_an_empty_bench_prints_the_usb_bus(monkeypatch, tmp_path): def test_an_empty_sysfs_says_so_rather_than_blaming_the_boards(monkeypatch, tmp_path): monkeypatch.setattr(bench, "USB_DEVICES", tmp_path / "nothing") assert "Nothing at all is on the USB bus" in bench.no_boards_message() + + +# -------------------------------------------------------------------------- +# a board wired for a system the bench cannot drive +# -------------------------------------------------------------------------- + + +def test_the_target_list_comes_from_the_repo(monkeypatch): + """src/common is shared code, and sys11_tiny is the sys11 board twice.""" + buildable = bench.buildable_targets() + + assert "classic" in buildable and "whitestar" in buildable + assert "common" not in buildable + assert "sys11_tiny" not in buildable and "sys11" in buildable + + +def test_only_targets_with_configs_can_be_driven(): + ready = bench.bench_targets() + + assert set(ready) == {"sys11", "wpc", "data_east", "em"} + # Real build targets, but nothing to boot them against. + assert "classic" not in ready and "whitestar" not in ready + + +def test_a_board_mapped_to_a_configless_target_is_refused_early(): + """A classic board turned up on the bench; the map must not be able to lie about it.""" + boards = [{"port": "/dev/ttyACM2", "chip_id": "899f", "system": "classic", "responsive": True}] + + with pytest.raises(bench.CheckFailure, match="bench cannot drive"): + bench.resolve_targets(boards, {"899f": "classic"}) + + +def test_a_target_that_does_not_exist_at_all_says_what_does(): + boards = [{"port": "/dev/ttyACM2", "chip_id": "899f", "system": None, "responsive": True}] + + with pytest.raises(bench.CheckFailure, match="not a target in this checkout"): + bench.resolve_targets(boards, {"899f": "sys12"}) + + +def test_self_report_cannot_smuggle_in_an_undriveable_target(): + """Autodetection reads the flashed firmware, which can say 'classic' too.""" + boards = [ + {"port": "/dev/ttyACM0", "chip_id": "aaaa", "system": "wpc", "responsive": True}, + {"port": "/dev/ttyACM2", "chip_id": "899f", "system": "classic", "responsive": True}, + ] + + with pytest.raises(bench.CheckFailure, match="bench cannot drive"): + bench.resolve_targets(boards, {}) + + +def test_the_instructions_name_what_cannot_be_mapped(job_summary): + text = bench.board_map_instructions([{"port": None, "chip_id": "899f"}], {}) + + assert "targets: data_east, em, sys11, wpc" in text + # Naming them matters: the board on the bench is running one of them, and + # the only wrong move is mapping it to a system it is not wired for. + assert "classic" in text and "whitestar" in text From 1a8c5308eab5579840049f48d6f748457438eeb7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 03:05:23 +0000 Subject: [PATCH 24/32] ci(hil): cap the bench job and every stage in it The bench job holds the `hil-bench` concurrency lease for as long as it runs, and it was allowed to run for 180 minutes. Tonight one wedged job used that: `actions/checkout` hung with the runner no longer acknowledging, the job sat in_progress, and everything behind it queued with nothing to do but wait for a three-hour timeout. Cancelling from the UI does not help there either - a cancel waits on the same runner handshake that has already stopped answering. So the job cap becomes 90 minutes, which is the healthy run (~50) plus headroom rather than a number chosen never to fire, and each step gets its own cap so no single stage can spend the whole budget: checkout 10, recover 15, flash and health check 25, config matrix 70, serial-log diagnostics 5. The gate job gets 5, because the default is six hours for two lines of shell. The stages keep `continue-on-error: true`, so a stage that times out is reported by the verdict step like any other failure instead of taking the run down silently. `cancel-in-progress` stays false on purpose: cancelling a live bench job can stop it mid-flash and leave a board half-written, which is a worse problem than a queue. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- .github/workflows/hil.yml | 37 +++++++++++++++++++++++++++++++++---- dev/hil/DESIGN.md | 17 ++++++++++++----- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml index bdf7cc03..0464f2ef 100644 --- a/.github/workflows/hil.yml +++ b/.github/workflows/hil.yml @@ -76,6 +76,7 @@ jobs: gate: name: Decide whether to touch the bench runs-on: ubuntu-latest + timeout-minutes: 5 outputs: run_bench: ${{ steps.decide.outputs.run_bench }} reason: ${{ steps.decide.outputs.reason }} @@ -127,18 +128,38 @@ jobs: needs: gate if: needs.gate.outputs.run_bench == 'true' runs-on: [self-hosted, vector-hil] - timeout-minutes: 180 + + # The bench is a singleton behind `concurrency: hil-bench`, and this job + # holds that lease for as long as it runs. At 180 minutes a wedged job + # blocked every later run for three hours - which is what happened: a + # checkout hung, the runner stopped acknowledging, and the queue behind it + # went nowhere until someone noticed. + # + # So the cap is the healthy run plus headroom, not a number picked to never + # fire. A full healthy pass is ~50 minutes (recover ~30s, flash and health + # check ~4 min, the matrix ~45 for 130 configs across three boards). + # + # cancel-in-progress stays false deliberately: cancelling a live bench job + # can stop it mid-flash and leave a board half-written, which is a worse + # problem than a queue. + timeout-minutes: 90 steps: # On workflow_run this checks out the DEFAULT BRANCH, not the PR - that # is the whole security property, and it must stay that way. The board # gets the PR's firmware because the harness builds it from the commit # below, not because the PR's harness code runs here. + # + # Capped because this is the step that actually wedged: it sat in + # `in_progress` while the runner stopped answering, with 180 minutes of + # job budget to sit in. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + timeout-minutes: 10 with: fetch-depth: 0 - name: Check out the commit under test + timeout-minutes: 10 if: github.event_name == 'workflow_run' env: COMMIT: ${{ needs.gate.outputs.commit }} @@ -157,23 +178,29 @@ jobs: # 1. Recover. Cheap when the bench is healthy, and the difference between # a wedged board costing one run and costing every run until someone - # notices. + # notices. ~30s when every board answers; a ladder still going after + # the cap has failed in a way that needs a person, not more waiting. - name: Recover any wedged board + timeout-minutes: 15 id: recover if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'recover') continue-on-error: true run: python dev/hil/recover.py # 2. Flash and health-check: DESIGN.md G1/G2. Exercises the API over both - # USB and HTTP, which the config matrix does not. + # USB and HTTP, which the config matrix does not. ~4 min: three builds + # on the Zero 2 W, one parallel flash pass, a health check per board. - name: Flash and health-check every board + timeout-minutes: 25 id: flash_check if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'flash-check') continue-on-error: true run: python dev/hil/flash_and_check.py - # 3. The config matrix: DESIGN.md G3. + # 3. The config matrix: DESIGN.md G3. The long pole - ~20s per config, + # 130 configs across three boards. - name: Boot every config on every board + timeout-minutes: 70 id: config_matrix if: contains(inputs.stages || 'all', 'all') || contains(inputs.stages || '', 'config-matrix') continue-on-error: true @@ -190,7 +217,9 @@ jobs: # shellcheck disable=SC2086 # args is a deliberately word-split list python dev/hil/config_matrix.py $args + # Diagnostics read from boards that are by definition misbehaving. - name: Board serial logs on failure + timeout-minutes: 5 if: failure() || steps.flash_check.outcome == 'failure' || steps.config_matrix.outcome == 'failure' run: | # `stty raw -echo` first, and both calls under `timeout`: a tty reverts diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 2ec3e0d6..4271e8d2 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -213,11 +213,18 @@ workflows that shared the `hil-bench` concurrency group — and since GitHub kee only one *pending* run per group, a push touching two of them had them race with the loser silently cancelled. One workflow takes one lease and runs: -| stage | what it is | cost | -|---|---|---| -| recover | `recover.py` — repair any wedged board | ~30s healthy | -| flash + health check | G1/G2, API over USB *and* HTTP | ~3.5 min | -| config matrix | G3, every config on every board | ~45 min | +| stage | what it is | cost | cap | +|---|---|---|---| +| recover | `recover.py` — repair any wedged board | ~30s healthy | 15 min | +| flash + health check | G1/G2, API over USB *and* HTTP | ~3.5 min | 25 min | +| config matrix | G3, every config on every board | ~45 min | 70 min | + +The caps are per step, under a 90-minute cap on the job as a whole, and they +exist because this job holds the `hil-bench` lease while it runs. At the +original 180 minutes one wedged job — a hung `actions/checkout`, on a runner +that had stopped acknowledging — blocked every run behind it for three hours. +`cancel-in-progress` stays `false` regardless: cancelling a live bench job can +stop it mid-flash and leave a board half-written, which is worse than a queue. Every stage runs even when an earlier one fails, and the verdict is taken at the end: one broken board should not cost the signal from the others. Running From efb50cc4138e205ab123262d2ef0d6d7fcdaa8dc Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 03:11:55 +0000 Subject: [PATCH 25/32] hil: report a bad target like a bad chip id, and suggest what was meant The run on 1a8c530 caught its own map: `VECTOR_HIL_BOARD_MAP` now says `899fab8c90bfeb9a=de`, and `de` is not a target - the tree calls that board `data_east`. check_target refused it before anything was flashed, which is what it is for, but then said so badly. It failed with one line, no job summary entry, and no instructions - while an unrecognised *chip id*, the same mistake seen from the other end and fixed by the same person on the same host, gets the whole treatment. Both now go to the summary with the map and the line to set. `de` is also not a typo: it is what everyone calls that board, and nothing about `data_east` suggests it. So a rejected target is matched against each target's label and its initials as well as its id, and the error names what was probably meant - DataEast -> de, WhiteStar -> ws. And the pre-filled fix line no longer echoes the broken value back. It carried the very `de` that had just failed, which is how a wrong entry survives being reported; a value the bench cannot drive is replaced by the target it was reaching for, or left blank to fill in. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/RUNNER_SETUP.md | 3 ++ dev/hil/bench.py | 98 ++++++++++++++++++++++++++++++----- dev/tests/test_hil_bootsel.py | 34 +++++++++++- 3 files changed, 120 insertions(+), 15 deletions(-) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index ad72addc..14902c49 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -125,6 +125,9 @@ cd ~/actions-runner && sudo ./svc.sh stop && sudo ./svc.sh start ``` Valid targets are the ones with game configs to boot against: `sys11`, `wpc`, `data_east`, `em`. +Use the directory name, not the name people say out loud — the Data East board is `data_east`, +not `de` (the harness suggests the right one if you get it wrong, and refuses the run rather +than flashing anything). The tree also builds `classic` and `whitestar`, and the bench **cannot** drive those — they have a `systemConfig.py` and no `config/` directory, so there is nothing to flash and boot. A board wired for one of them cannot be mapped at all until that target has configs; mapping it to a different diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 8c8f9718..d86b2ed7 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -516,7 +516,22 @@ def board_map_instructions(boards, board_map=None): looking at the machine that needs editing. """ board_map = board_map or {} - suggested = ",".join(f"{b['chip_id']}={board_map.get(b['chip_id'], '')}" for b in boards) + + def wanted(chip_id): + """What this board's entry should say - never what a broken one says. + + Echoing the current value back is how a wrong entry survives being + reported: the line offered as the fix contained the very `de` that + failed. A value the bench cannot drive is replaced by the target it + was probably reaching for, or by a blank to fill in. + """ + target = board_map.get(chip_id) + if target in bench_targets(): + return target + suggestion = suggest_target(target) + return suggestion if suggestion in bench_targets() else "" + + suggested = ",".join(f"{b['chip_id']}={wanted(b['chip_id'])}" for b in boards) lines = [ "", "VECTOR_HIL_BOARD_MAP pins each board to the system it is wired to, by RP2040", @@ -621,7 +636,7 @@ def resolve_targets(boards, board_map): raise CheckFailure(report_unknown_boards(unmapped, boards, board_map)) for b in boards: b["target"] = board_map[b["chip_id"]] - check_target(b["target"]) + check_targets(boards, board_map) log("targets from VECTOR_HIL_BOARD_MAP") return boards @@ -641,7 +656,7 @@ def resolve_targets(boards, board_map): for b in boards: b["target"] = b["system"] - check_target(b["target"]) + check_targets(boards, board_map) log("targets from firmware self-report (all distinct)") return boards @@ -746,18 +761,75 @@ def bench_targets(): return ready -def check_target(target): - """Refuse a target the bench cannot drive, while it is still cheap to say so.""" - if target in bench_targets(): +def target_labels(): + """{target id: human label} from the repo's target list, for suggestions.""" + labels = {} + try: + for entry in json.loads(TARGETS_JSON.read_text()): + labels.setdefault(entry.get("hardware_id") or entry["id"], entry.get("label", "")) + except (OSError, ValueError, KeyError): + pass + return labels + + +def suggest_target(name): + """The target somebody probably meant when they wrote `name`. + + People write the name they use out loud, not the directory: the bench map + arrived with `de` in it, which is what everyone calls the Data East board + and is nothing like the `data_east` the tree wants. Matching the label and + its initials as well as the id turns that from a puzzle into a correction. + """ + wanted = (name or "").strip().lower().replace("-", "_").replace(" ", "") + if not wanted: + return None + candidates = {} + for target, label in target_labels().items(): + candidates[target.lower()] = target + candidates[target.lower().replace("_", "")] = target + if label: + candidates[label.lower()] = target + candidates["".join(c for c in label if c.isupper()).lower()] = target + return candidates.get(wanted) + + +def describe_bad_target(target): + """Why this target cannot be used, in one sentence plus the way out.""" + if target not in buildable_targets(): + suggestion = suggest_target(target) + did_you_mean = f" - did you mean {suggestion}?" if suggestion else "" + return f"{target!r} is not a target in this checkout{did_you_mean} The bench can drive " + ", ".join(bench_targets()) + "." + return ( + f"{target!r} is a real build target but the bench cannot drive it: src/{target}/config holds no " + f"generic game config to flash and boot against, so there is nothing to boot the board against. " + f"The bench can drive " + ", ".join(bench_targets()) + "." + ) + + +def check_targets(boards, board_map=None): + """Refuse targets the bench cannot drive, while they are still configuration. + + Reported like an unrecognised chip id - to the job summary, with the map + and how to edit it - because it is the same mistake seen from the other + end, and the same person on the same host has to fix it. Left to run, it + would have flashed nothing and died on a bare KeyError two stages later. + """ + bad = sorted({b["target"] for b in boards if b.get("target") not in bench_targets()}, key=str) + if not bad: return - known = buildable_targets() - if target not in known: - raise CheckFailure(f"{target!r} is not a target in this checkout - it builds " + ", ".join(known)) - raise CheckFailure( - f"{target!r} is a real build target but the bench cannot drive it: src/{target}/config holds no\n" - f" generic game config to flash and boot against. The bench can drive " + ", ".join(bench_targets()) + ".\n" - " A board wired for it can only join the bench once that target has configs." + + described = [describe_bad_target(target) for target in bad] + summary_once( + "### Board map names a target the bench cannot use", + [ + "", + "### Board map names a target the bench cannot use", + "", + *[f"- {line}" for line in described], + *as_block(board_map_instructions(boards, board_map or {})), + ], ) + raise CheckFailure(" ".join(described)) def source_version(target): diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index 473a6cb7..a3ab0847 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -492,21 +492,51 @@ def test_only_targets_with_configs_can_be_driven(): assert "classic" not in ready and "whitestar" not in ready -def test_a_board_mapped_to_a_configless_target_is_refused_early(): +def test_a_board_mapped_to_a_configless_target_is_refused_early(job_summary): """A classic board turned up on the bench; the map must not be able to lie about it.""" boards = [{"port": "/dev/ttyACM2", "chip_id": "899f", "system": "classic", "responsive": True}] with pytest.raises(bench.CheckFailure, match="bench cannot drive"): bench.resolve_targets(boards, {"899f": "classic"}) + # Reported where an unrecognised chip id is reported, for the same person. + assert "target the bench cannot use" in job_summary.read_text() -def test_a_target_that_does_not_exist_at_all_says_what_does(): + +def test_a_target_that_does_not_exist_at_all_says_what_does(job_summary): boards = [{"port": "/dev/ttyACM2", "chip_id": "899f", "system": None, "responsive": True}] with pytest.raises(bench.CheckFailure, match="not a target in this checkout"): bench.resolve_targets(boards, {"899f": "sys12"}) +@pytest.mark.parametrize( + ("written", "meant"), + [("de", "data_east"), ("DE", "data_east"), ("DataEast", "data_east"), ("data-east", "data_east"), ("ws", "whitestar")], +) +def test_the_name_people_actually_use_is_recognised(written, meant): + """The bench map arrived saying `de`, which is what everyone calls that board.""" + assert bench.suggest_target(written) == meant + + +def test_a_name_nobody_meant_gets_no_guess(): + assert bench.suggest_target("nonsense") is None + assert bench.suggest_target("") is None + + +def test_the_suggested_map_line_never_repeats_a_broken_entry(): + """The fix on offer must not contain the value that just failed.""" + boards = [{"port": "/dev/ttyACM0", "chip_id": "899f"}, {"port": "/dev/ttyACM1", "chip_id": "df13"}] + + corrected = bench.board_map_instructions(boards, {"899f": "de", "df13": "wpc"}) + assert "VECTOR_HIL_BOARD_MAP=899f=data_east,df13=wpc" in corrected + + # Nothing to suggest for a target that exists but cannot be driven, so it + # is left blank rather than being offered back. + blanked = bench.board_map_instructions(boards, {"899f": "classic", "df13": "wpc"}) + assert "VECTOR_HIL_BOARD_MAP=899f=,df13=wpc" in blanked + + def test_self_report_cannot_smuggle_in_an_undriveable_target(): """Autodetection reads the flashed firmware, which can say 'classic' too.""" boards = [ From 5a7a7616a1f51d571aece264bbc3370cacca4a0a Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 03:13:12 +0000 Subject: [PATCH 26/32] ci(hil): pass matrix dispatch inputs as an array, not a word-split string Copilot's review is right: `configs: "Taxi_L4, AttackMars_11"` is a natural thing to type into the dispatch form, and the space made bash split it into two argv entries, handing argparse a stray positional and failing the run before the bench was touched. Reproduced and fixed: an array keeps each flag and its value together, and select_configs already strips whitespace around the commas, so the value only ever needed to arrive as one argument. Checked the empty case too - no dispatch inputs, which is every automatic run - since that is the path this would break if the expansion were wrong. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- .github/workflows/hil.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml index 0464f2ef..ec9c3f11 100644 --- a/.github/workflows/hil.yml +++ b/.github/workflows/hil.yml @@ -209,13 +209,17 @@ jobs: HIL_CONFIGS: ${{ inputs.configs }} HIL_LIMIT: ${{ inputs.limit }} run: | - args="" - if [ -n "${HIL_TARGET:-}" ]; then args="$args --target $HIL_TARGET"; fi - if [ -n "${HIL_CONFIGS:-}" ]; then args="$args --configs $HIL_CONFIGS"; fi - if [ -n "${HIL_LIMIT:-}" ]; then args="$args --limit $HIL_LIMIT"; fi + # An array, not a word-split string: `configs: "Taxi_L4, AttackMars_11"` + # is a natural thing to type, and splitting on that space hands + # argparse a stray positional and fails the run. config_matrix strips + # whitespace around the commas itself, so the value only has to + # arrive as one argument. + args=() + if [ -n "${HIL_TARGET:-}" ]; then args+=(--target "$HIL_TARGET"); fi + if [ -n "${HIL_CONFIGS:-}" ]; then args+=(--configs "$HIL_CONFIGS"); fi + if [ -n "${HIL_LIMIT:-}" ]; then args+=(--limit "$HIL_LIMIT"); fi - # shellcheck disable=SC2086 # args is a deliberately word-split list - python dev/hil/config_matrix.py $args + python dev/hil/config_matrix.py "${args[@]}" # Diagnostics read from boards that are by definition misbehaving. - name: Board serial logs on failure From 4919c625453c11685b9f5d95f50725e3205e4627 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 03:27:34 +0000 Subject: [PATCH 27/32] hil: recover a wedged board by calling the ladder, not by printing its name From tonight's matrix run: Error: /dev/ttyACM1 is not answering - skipping it. Run dev/hil/recover.py to get it back. The recover stage had already run and passed in that same job; ttyACM1 wedged afterwards. So the harness watched a board go down, knew exactly what to do about it, and instead printed advice to a log nobody was reading - and gave up a third of the bench for the rest of the run. The bench is supposed to run unattended. recover.py is a harness, not just a script, so inventory now calls it: a board that does not answer is drained, USB-reset and power cycled, then probed again, and the run carries on with it if it comes back. Every harness gets this, because they all go through bench.inventory(). The reflash rung is deliberately excluded. It is the destructive one, and it leaves the board running TrenchCoat's bundled firmware rather than the build under test - harmless in the recover stage, where flash_and_check re-flashes everything straight afterwards, and wrong in the middle of a matrix, which would carry on testing firmware that is not the one under test and report the results as if it were. That rung stays a decision for a person, and the messages now say so instead of naming a script. bench adds its own directory to sys.path for the in-process import rather than assuming the caller did. Every harness happens to today; relying on that would leave the import broken for the first one that does not, with no signal until a board wedges. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/DESIGN.md | 8 +++ dev/hil/RUNNER_SETUP.md | 9 +++ dev/hil/bench.py | 75 ++++++++++++++++++++++++- dev/hil/config_matrix.py | 15 +++-- dev/hil/flash_and_check.py | 2 +- dev/tests/test_hil_recover.py | 100 ++++++++++++++++++++++++++++++++++ 6 files changed, 201 insertions(+), 8 deletions(-) diff --git a/dev/hil/DESIGN.md b/dev/hil/DESIGN.md index 4271e8d2..d5ba08f5 100644 --- a/dev/hil/DESIGN.md +++ b/dev/hil/DESIGN.md @@ -252,6 +252,14 @@ getting them confused has cost whole runs: is for; `VECTOR_HIL_REQUIRED_TARGETS` on the runner is how a bench that has really lost a board says so out loud. +Recovery is a call, not an instruction. A board that stops answering mid-run +is drained, USB-reset and power cycled by whichever harness found it, then +probed again — "run `dev/hil/recover.py`" is useless advice on a bench that is +supposed to run unattended, and the recover stage only runs once at the start +of a job. The reflash rung stays manual: it is destructive and leaves the board +on TrenchCoat's bundled firmware, so a matrix continuing past it would be +testing firmware other than the build under test. + A board whose chip id is in none of the map's entries is never guessed at — the id and the exact line to add are written to the job summary, because the fix is one edit on the runner host and whoever makes it is reading the run, not the log. diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 14902c49..83699483 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -257,6 +257,15 @@ the `recover` stage of [`hil.yml`](../../.github/workflows/hil.yml), which runs cd ~/vector && PATH="$PWD/.venv/bin:$PATH" .venv/bin/python dev/hil/recover.py ``` +The ladder is also a library, not just a script. Every harness runs the **cheap rungs +automatically** against a board that stops answering — inventory drains, USB-resets and power +cycles it, then probes again — so a board that wedges *after* the recover stage no longer costs +the rest of the run. The reflash rung is deliberately **not** automatic: it is destructive, and +it leaves the board running TrenchCoat's bundled firmware rather than the build under test, so a +matrix that carried on afterwards would be testing the wrong thing and reporting it as if it +were not. That one stays a decision for a person running `recover.py` by hand — or for the +recover stage itself, where `flash_and_check.py` re-flashes everything immediately afterwards. + Before the ladder it deals with the one state none of the rungs can reach: a board in **BOOTSEL/UF2 mode**. Such a board is not a serial device at all — the ROM bootloader enumerates as USB mass storage, so `mpremote devs` shows nothing and `lsusb` shows everything, diff --git a/dev/hil/bench.py b/dev/hil/bench.py index d86b2ed7..ae7ab90f 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -25,6 +25,7 @@ import sys import time from pathlib import Path +from types import SimpleNamespace REPO_ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(REPO_ROOT / "dev")) @@ -80,6 +81,11 @@ # remainder. SERVER_SETTLE_SECONDS = 2 +# A repair mid-run is bounded well below the workflow's per-stage caps: the +# rungs it can reach are a drain, a USB reset and a power cycle, none of which +# is slow. The 600s in recover.py exists for the reflash, which this never runs. +REPAIR_TIMEOUT = 180 + # Measured across 70 boots on the bench: 11.8s min, 15.5s mean, 25.6s max. The # flash harness keeps 150s; the matrix runs a tighter one (see # config_matrix.MATRIX_BOOT_TIMEOUT) because it pays the timeout per config. @@ -348,7 +354,65 @@ def probe(port): return board -def inventory(): +def repair_board(port, target=None, allow_reflash=False, step_timeout=REPAIR_TIMEOUT): + """Run the recovery ladder against one board, in-process. + + `recover.py` is a harness, not just a script, so a board that stops + answering mid-run is repaired by calling it rather than by printing "run + dev/hil/recover.py" at somebody who is not there. The bench is meant to + run unattended, and the recover *stage* only runs once at the start of the + job - a board that wedges after it used to cost the rest of the run. + + Reflashing is off by default, and that is the load-bearing part: it is the + one destructive rung, and it leaves the board running TrenchCoat's bundled + firmware rather than the build under test. Harmless in the recover stage, + where flash_and_check re-flashes everything straight afterwards; wrong in + the middle of a matrix, where the run would silently continue against + firmware that is not the one being tested. + + Imported here rather than at the top because recover imports this module, + and its directory is added to sys.path here rather than assumed: every + harness happens to add it today, so relying on that would leave a trap for + the first caller that does not. + """ + here = str(Path(__file__).resolve().parent) + if here not in sys.path: + sys.path.insert(0, here) + + import recover + + options = SimpleNamespace( + reflash=allow_reflash, + no_power_cycle=False, + force_bootsel=False, + step_timeout=step_timeout, + cache_dir=REPO_ROOT / "build" / "hil", + ) + return recover.recover(port, target, options) + + +def repair_unresponsive(boards, board_map=None): + """Give every board that did not answer one pass of the cheap rungs.""" + dead = [b for b in boards if not b.get("responsive", True)] + if not dead: + return boards + + group(f"Repair {len(dead)} board(s) that did not answer") + for b in dead: + # A wedged board cannot tell us its chip id, so there is no target to + # look up - which is fine, because the rungs that need one are the + # rungs this deliberately does not run. + method = repair_board(b["port"], (board_map or {}).get(b.get("chip_id"))) + if not method: + log(f"::warning::{b['port']} is still not answering after the cheap recovery rungs") + continue + log(f"{b['port']} came back after {method} - probing it again") + b.update(probe(b["port"])) + endgroup() + return boards + + +def inventory(board_map=None, repair=True): """Survey the bench: every board on serial, plus any stuck in BOOTSEL. Only the serial boards are returned - a board in the bootloader cannot be @@ -358,6 +422,8 @@ def inventory(): sitting there as mass-storage devices. """ boards = [probe(port) for port in list_ports()] + if repair: + boards = repair_unresponsive(boards, board_map) stranded = bootsel_boards() log(f"{'port':16} {'chip id':18} {'running':12} version") @@ -372,7 +438,7 @@ def inventory(): if not boards: raise CheckFailure(no_boards_message(stranded)) if stranded: - log(f"::warning::{len(stranded)} board(s) are in BOOTSEL/UF2 mode and were not surveyed - run dev/hil/recover.py to put firmware back on them") + log(f"::warning::{len(stranded)} board(s) are in BOOTSEL/UF2 mode and were not surveyed - the rescue stage puts firmware back on them, and reports here when it cannot") return boards @@ -627,7 +693,10 @@ def resolve_targets(boards, board_map): dead = [b for b in boards if not b.get("responsive", True)] if dead: raise CheckFailure( - "not answering: " + ", ".join(b["port"] for b in dead) + ".\nA board that will not talk cannot be identified, so it cannot be safely flashed.\n" "Run dev/hil/recover.py to get it back." + "not answering: " + ", ".join(b["port"] for b in dead) + ".\nA board that will not talk cannot be identified, so it cannot be safely flashed.\n" + "The cheap recovery rungs have already been tried on it. dev/hil/recover.py run by hand\n" + "can also reflash it over the ROM bootloader, which this will not do on its own: that is\n" + "destructive and would leave the board running firmware other than the build under test." ) if board_map: diff --git a/dev/hil/config_matrix.py b/dev/hil/config_matrix.py index 2c556efc..a895e1fb 100644 --- a/dev/hil/config_matrix.py +++ b/dev/hil/config_matrix.py @@ -639,7 +639,7 @@ def main(): trench_coat.rescue_bootsel(board_map, REPO_ROOT / "build" / "hil") group("Inventory") - boards = inventory() + boards = inventory(board_map) endgroup() # A board that will not answer is one board's problem. It cannot be @@ -647,12 +647,16 @@ def main(): # still have configs worth checking - and aborting the whole run before # anything is tested is how a single board left wedged by an earlier run # took out the next one entirely. + # + # Inventory has already run the cheap recovery rungs against these, so + # anything still here needs the destructive one, which is a decision for a + # person rather than for the middle of a matrix. unresponsive = [b for b in boards if not b.get("responsive", True)] boards = [b for b in boards if b.get("responsive", True)] for b in unresponsive: - log(f"::error::{b['port']} is not answering - skipping it. Run dev/hil/recover.py to get it back.") + log(f"::error::{b['port']} is not answering, and draining, resetting and power cycling it did not help - skipping it") if not boards: - raise CheckFailure("no board on the bench is answering - run dev/hil/recover.py") + raise CheckFailure("no board on the bench is answering, and the cheap recovery rungs did not bring any back.\nRun dev/hil/recover.py by hand to also reflash them over the ROM bootloader.") group("Resolve targets") boards = resolve_targets(boards, board_map) @@ -679,7 +683,10 @@ def main(): log(f"built {target} at version {bench.source_version(target)}") endgroup() - results = [({"port": b["port"], "target": "(unknown)", "crashes": [], "flakes": []}, [], [("(board setup)", "board is not answering - run dev/hil/recover.py")]) for b in unresponsive] + results = [ + ({"port": b["port"], "target": "(unknown)", "crashes": [], "flakes": []}, [], [("(board setup)", "board is not answering, and the cheap recovery rungs did not bring it back")]) + for b in unresponsive + ] for b in boards: try: just_flashed = not args.skip_flash diff --git a/dev/hil/flash_and_check.py b/dev/hil/flash_and_check.py index 6f49aeba..caf45486 100644 --- a/dev/hil/flash_and_check.py +++ b/dev/hil/flash_and_check.py @@ -303,7 +303,7 @@ def main(): trench_coat.rescue_bootsel(board_map, REPO_ROOT / "build" / "hil") group("Inventory") - boards = inventory() + boards = inventory(board_map) endgroup() group("Resolve targets") diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index 507495c3..6db5e11d 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -503,3 +503,103 @@ def Serial(*args, **kwargs): ray.serial.Serial("/dev/ttyFAKE", write_timeout=99) assert opened["write_timeout"] == 99 + + +# -------------------------------------------------------------------------- +# recovery as a call, not an instruction +# -------------------------------------------------------------------------- + + +def test_inventory_repairs_a_board_that_did_not_answer(monkeypatch, capsys): + """The bench runs unattended: printing "run recover.py" helps nobody at 3am.""" + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + + probes = iter( + [ + {"port": "/dev/ttyACM0", "chip_id": None, "system": None, "version": None, "responsive": False}, + {"port": "/dev/ttyACM0", "chip_id": "aaaa", "system": "sys11", "version": "1.10.7", "responsive": True}, + ] + ) + monkeypatch.setattr(bench, "probe", lambda port: next(probes)) + monkeypatch.setattr(bench, "repair_board", lambda port, target=None: "drain the console") + + boards = bench.inventory() + + # Re-probed after the repair, so the run continues with a live board. + assert boards[0]["responsive"] is True + assert boards[0]["chip_id"] == "aaaa" + assert "came back after drain the console" in capsys.readouterr().out + + +def test_a_board_that_stays_dead_is_reported_not_hidden(monkeypatch, capsys): + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(bench, "probe", lambda port: {"port": port, "chip_id": None, "system": None, "version": None, "responsive": False}) + monkeypatch.setattr(bench, "repair_board", lambda port, target=None: None) + + boards = bench.inventory() + + assert boards[0]["responsive"] is False + printed = capsys.readouterr().out + assert "still not answering" in printed + assert "NOT ANSWERING" in printed + + +def test_repair_can_be_turned_off(monkeypatch): + monkeypatch.setattr(bench, "list_ports", lambda: ["/dev/ttyACM0"]) + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(bench, "probe", lambda port: {"port": port, "chip_id": None, "system": None, "version": None, "responsive": False}) + monkeypatch.setattr(bench, "repair_board", lambda *a, **k: pytest.fail("must not repair when asked not to")) + + bench.inventory(repair=False) + + +def test_an_automatic_repair_never_reflashes(monkeypatch): + """The one rung that must stay a human decision mid-run. + + A reflash leaves the board running TrenchCoat's bundled firmware, so a + matrix that carried on afterwards would be testing firmware other than the + build under test - and reporting the results as if it were not. + """ + seen = {} + monkeypatch.setattr(recover, "recover", lambda port, target, options: seen.update(port=port, target=target, options=options) or "drain the console") + + assert bench.repair_board("/dev/ttyACM1") == "drain the console" + assert seen["options"].reflash is False + # And the cheap rungs are all still enabled. + assert seen["options"].no_power_cycle is False + + +def test_a_repair_may_be_asked_for_the_full_ladder(monkeypatch): + """The recover stage does want it: flash_and_check re-flashes right after.""" + seen = {} + monkeypatch.setattr(recover, "recover", lambda port, target, options: seen.update(options=options) or None) + + bench.repair_board("/dev/ttyACM1", target="wpc", allow_reflash=True) + + assert seen["options"].reflash is True + + +def test_a_repair_is_bounded_well_below_the_stage_cap(monkeypatch): + seen = {} + monkeypatch.setattr(recover, "recover", lambda port, target, options: seen.update(options=options) or None) + + bench.repair_board("/dev/ttyACM1") + + # The 600s ceiling in recover.py is for the reflash, which this never runs. + assert seen["options"].step_timeout == bench.REPAIR_TIMEOUT < recover.STEP_TIMEOUT + + +def test_repair_does_not_depend_on_the_caller_having_fixed_sys_path(monkeypatch): + """bench does not add its own directory; every harness happens to. + + Relying on that leaves the in-process import broken for the first caller + that does not, which is a failure mode with no signal until a board wedges. + """ + here = str(REPO_ROOT / "dev" / "hil") + monkeypatch.setattr(sys, "path", [p for p in sys.path if p != here]) + monkeypatch.setattr(recover, "recover", lambda port, target, options: "drain the console") + + assert bench.repair_board("/dev/ttyACM1") == "drain the console" + assert here in sys.path From 75ced191adc9dda5dd9861b2d3984ebda5e324de Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 13:39:40 +0000 Subject: [PATCH 28/32] hil: show the USB bus when a system is missing, not only when all are The bench ran 67/67 clean on 4919c62 - sys11 39/39, data_east 28/28, no flakes, no boot crashes - and failed for the one board that was not there: all 67 config(s) booted on the boards that are here, but the bench is missing wpc Correct, and not enough. "Missing" has two remedies and they are opposite. A board that is enumerated but silent is the recovery ladder's job, and the ladder can now reach it - the udev rule landed and the USB reset rung works. A board that has dropped off the bus needs a person to replug it, and no amount of draining or resetting will do anything at all. Those looked identical in the log, and the difference was real: wpc had been enumerated-but-wedged for several runs, then between two runs it left the bus entirely - data_east moved from ttyACM2 to ttyACM1 and only two ports were left. Nothing said so. The USB bus listing already existed for a bench where nothing at all is found. An incomplete bench earns it just as much, so check_bench_complete now prints it too: every device with its vendor:product id and serial, which answers "is the board even plugged in" without anyone having to ssh in. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/bench.py | 8 ++++++++ dev/tests/test_hil_bootsel.py | 26 ++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/dev/hil/bench.py b/dev/hil/bench.py index ae7ab90f..25ba8f0a 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -779,6 +779,14 @@ def check_bench_complete(boards): "Either put the missing board back (`dev/hil/recover.py`, and check it is in", "`VECTOR_HIL_BOARD_MAP`), or set `VECTOR_HIL_REQUIRED_TARGETS` on the runner to the systems", "the bench really has.", + "", + # Which of those it is turns on one question the log could not answer: + # is the board on the bus at all? A board that is enumerated but silent + # is a job for the recovery ladder; a board that is simply gone needs + # someone to replug it, and no amount of draining or resetting will + # help. The bus listing already exists for an empty bench - an + # incomplete one deserves it just as much. + *usb_bus_report(), ], ) return missing diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index a3ab0847..24ac6128 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -555,3 +555,29 @@ def test_the_instructions_name_what_cannot_be_mapped(job_summary): # Naming them matters: the board on the bench is running one of them, and # the only wrong move is mapping it to a system it is not wired for. assert "classic" in text and "whitestar" in text + + +def test_an_incomplete_bench_says_whether_the_board_is_even_plugged_in(job_summary, monkeypatch, tmp_path): + """ "Missing" has two remedies and they are opposite. + + A board that is enumerated but silent is the recovery ladder's job. A board + that has dropped off the bus needs a person to replug it - which is exactly + what happened to the wpc board between two runs, with nothing in the log to + tell the two apart. + """ + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + root = usb_tree(tmp_path, {"1-1.1": ("2e8a", "0005", "aaaa"), "1-1.2": ("2e8a", "0005", "bbbb")}) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + assert bench.check_bench_complete(boards_for("sys11", "data_east")) == ["wpc"] + + written = job_summary.read_text() + assert "What is on the USB bus" in written + assert "2e8a:0005" in written + + +def test_a_complete_bench_does_not_print_the_bus(job_summary, monkeypatch): + monkeypatch.delenv("VECTOR_HIL_REQUIRED_TARGETS", raising=False) + monkeypatch.setattr(bench, "usb_bus_report", lambda: pytest.fail("nothing is missing, so there is nothing to explain")) + + assert bench.check_bench_complete(boards_for("sys11", "wpc", "data_east")) == [] From 9f05a2d214dba62d83629764e6f871dd1467f1e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 14:34:15 +0000 Subject: [PATCH 29/32] hil: describe what is on the bus, and why a drive would not mount A bench run of flash_and_check found the wpc board exactly where it should: 1-1.2 2e8a:000f Raspberry Pi RP2350 Boot serial DF13A50C13958980 Detection worked, the rescue fired, and it stopped at one line: could not mount /dev/sda: Object .../sda is not a mountable filesystem Two things were wrong with how that was reported. The bus report then said "Raspberry Pi devices are on the bus (2e8a:0005, 2e8a:000f) but none of them is a serial port or a bootloader this harness recognises (2e8a:0003, 2e8a:000f)" - naming 000f in both halves of the same sentence. It called every 2e8a device unrecognised, including the ones it recognises perfectly well. Now each is described for what it is: running MicroPython, in the ROM bootloader (with a note that this is the rescue stage's job and not a missing board), or genuinely in a mode this does not handle. And "its filesystem never appeared" understated the failure. The drive did appear - as a block device with nothing mountable on it. The size settles which of those it is, so the mount failure now reads it: zero sectors means the bootloader is presenting no drive at all, which no retry will fix and which picotool can work around because it talks to the bootrom directly. A non-zero size with no filesystem points at blkid and dmesg instead. Neither changes what the harness does - the board still needs a person - but the log now says which person-shaped problem it is. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/bench.py | 29 +++++++++++--- dev/hil/trench_coat.py | 32 +++++++++++++++- dev/tests/test_hil_bootsel.py | 72 +++++++++++++++++++++++++++++++++-- 3 files changed, 123 insertions(+), 10 deletions(-) diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 25ba8f0a..270e61df 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -259,6 +259,7 @@ def list_ports(): BOOTSEL_VID = "2e8a" # Raspberry Pi BOOTSEL_PIDS = {"0003": "RP2040", "000f": "RP2350"} # "RP2 Boot" +RUNNING_PID = "0005" # MicroPython "Board in FS mode" - a normal, working board USB_DEVICES = Path("/sys/bus/usb/devices") @@ -491,14 +492,30 @@ def usb_bus_report(): for device in devices: lines.append(f" {device['path']:12} {device['id']:10} {device['name']}" + (f" serial {device['serial']}" if device["serial"] else "")) - unknown = [d for d in devices if d["id"].startswith(BOOTSEL_VID + ":")] - if unknown: + # Say what each Raspberry Pi device *is*, rather than lumping them together. + # The first version called every 2e8a device unrecognised, which produced a + # flatly self-contradictory report on the bench: it listed a 2e8a:000f + # bootloader and then said none of the devices was "a bootloader this + # harness recognises (2e8a:0003, 2e8a:000f)". + running = [d for d in devices if d["id"] == f"{BOOTSEL_VID}:{RUNNING_PID}"] + bootloaders = [d for d in devices if d["id"].split(":")[1] in BOOTSEL_PIDS and d["id"].startswith(BOOTSEL_VID + ":")] + strangers = [d for d in devices if d["id"].startswith(BOOTSEL_VID + ":") and d not in running and d not in bootloaders] + + if running or bootloaders or strangers: + lines.append("") + lines.append("Raspberry Pi devices on the bus:") + if running: + lines.append(f" {len(running)} running MicroPython - these are the boards that appear as serial ports") + for device in bootloaders: + lines.append(f" 1 in the ROM bootloader: {device['serial'] or '?'} ({device['id']})") + if bootloaders: lines += [ - "", - "Raspberry Pi devices are on the bus (" + ", ".join(sorted({d["id"] for d in unknown})) + ") but none of them is", - "a serial port or a bootloader this harness recognises (" + ", ".join(f"{BOOTSEL_VID}:{pid}" for pid in BOOTSEL_PIDS) + ").", - "That id is the thing to chase - it says what mode the boards are actually in.", + " A board here is the rescue stage's job, not a missing board: it is attached and", + " one UF2 away from working. If it is still in this state, the rescue ran and could", + " not finish - its output above says where it stopped.", ] + for device in strangers: + lines.append(f" 1 in a mode this harness does not handle: {device['id']} - that id is the thing to chase") return lines diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py index 1c609121..1c373e6b 100644 --- a/dev/hil/trench_coat.py +++ b/dev/hil/trench_coat.py @@ -369,9 +369,38 @@ def mount(device): if detail != _mount_error: _mount_error = detail log(f" could not mount {device}: {detail}") + for line in describe_block_device(device): + log(f" {line}") return None +def describe_block_device(device): + """What the kernel thinks of a drive udisks would not mount. + + "not a mountable filesystem" is where a bench recovery actually stopped, + and on its own it does not say whether the board is presenting a broken + filesystem or no drive at all. The size settles it: a bootloader offering + zero sectors is a board-side fault that no amount of retrying fixes, and + it needs picotool or a person rather than this code. + """ + name = Path(device).name + lines = [] + try: + sectors = int((Path("/sys/block") / name / "size").read_text().strip()) + except (OSError, ValueError): + return ["(could not read the drive's size from sysfs)"] + + if sectors == 0: + lines.append("the drive reports ZERO sectors, so there is no filesystem to mount and never will be.") + lines.append("The board is in the bootloader but its mass storage is presenting nothing - a UF2 cannot") + lines.append("be copied to it. picotool talks to the bootrom directly and does not need the drive:") + lines.append(f" picotool load -x # or: picotool info (board is {name})") + else: + lines.append(f"the drive reports {sectors} sectors ({sectors * 512 // 1024} KiB) but no filesystem udisks will mount.") + lines.append("Worth checking by hand on the bench host: sudo blkid " + str(device) + " ; sudo dmesg | tail -30") + return lines + + def unmount(device): """Best effort, so the next flash does not trip over a stale mount.""" subprocess.run(["udisksctl", "unmount", "--no-user-interaction", "-b", str(device)], capture_output=True, text=True, timeout=60) @@ -451,7 +480,8 @@ def flash_bootsel(chip_id, target, root): before = set(serial_ports()) device, drive = bootsel_drive(chip_id) if drive is None: - log(f" no bootloader drive for {chip_id} - it is enumerated but its filesystem never appeared") + log(f" no usable bootloader drive for {chip_id}: it is enumerated and in the bootloader, but") + log(" nothing here could mount a filesystem to copy a UF2 onto - see the reason above") return None # The wipe is the load-bearing step: it erases the whole flash, so nothing diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index 24ac6128..b2b79ccc 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -449,7 +449,7 @@ def test_an_empty_bench_prints_the_usb_bus(monkeypatch, tmp_path): { "usb1": ("1d6b", "0002", "0000:01:00.0"), "1-1": ("2109", "3431", None), - "1-1.1": ("2e8a", "0005", "e661a4d4179a5b2f"), # a mode we do not handle + "1-1.1": ("2e8a", "0005", "e661a4d4179a5b2f"), # running, but no serial port for it "1-1.1:1.0": None, }, ) @@ -459,8 +459,9 @@ def test_an_empty_bench_prints_the_usb_bus(monkeypatch, tmp_path): assert "2e8a:0005" in message assert "2109:3431" in message - # The recognised ids are named, so the reader can see what the mismatch is. - assert "2e8a:0003" in message + # A board in this state is running firmware, so say that rather than + # calling it unrecognised - the mismatch is that it has no serial port. + assert "1 running MicroPython" in message # Interfaces carry no ids and are not devices. assert "1-1.1:1.0" not in message @@ -581,3 +582,68 @@ def test_a_complete_bench_does_not_print_the_bus(job_summary, monkeypatch): monkeypatch.setattr(bench, "usb_bus_report", lambda: pytest.fail("nothing is missing, so there is nothing to explain")) assert bench.check_bench_complete(boards_for("sys11", "wpc", "data_east")) == [] + + +# -------------------------------------------------------------------------- +# the bus report has to describe what it lists +# -------------------------------------------------------------------------- + + +def test_a_bootloader_on_the_bus_is_not_called_unrecognised(monkeypatch, tmp_path): + """The first version contradicted itself on the bench. + + It listed a 2e8a:000f device and then said none of the devices was "a + bootloader this harness recognises (2e8a:0003, 2e8a:000f)". + """ + root = usb_tree( + tmp_path, + { + "1-1.1": ("2e8a", "0005", "e661a4d4179a5b2f"), + "1-1.2": ("2e8a", "000f", "DF13A50C13958980"), + "1-1.3": ("2e8a", "0005", "899fab8c90bfeb9a"), + }, + ) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + report = "\n".join(bench.usb_bus_report()) + + assert "in the ROM bootloader: DF13A50C13958980" in report + assert "2 running MicroPython" in report + assert "does not handle" not in report + assert "rescue stage's job" in report + + +def test_a_genuinely_unknown_mode_is_still_called_out(monkeypatch, tmp_path): + root = usb_tree(tmp_path, {"1-1.1": ("2e8a", "abcd", "aaaa")}) + monkeypatch.setattr(bench, "USB_DEVICES", root) + + report = "\n".join(bench.usb_bus_report()) + + assert "mode this harness does not handle: 2e8a:abcd" in report + + +def test_a_zero_sector_drive_says_so_and_names_picotool(monkeypatch, tmp_path): + """Where a real bench recovery stopped: enumerated, in BOOTSEL, no filesystem.""" + block = tmp_path / "sda" + block.mkdir() + (block / "size").write_text("0\n") + real_read = Path.read_text + monkeypatch.setattr(Path, "read_text", lambda self, *a, **k: real_read(block / "size") if str(self).endswith("/sys/block/sda/size") else real_read(self, *a, **k)) + + described = "\n".join(trench_coat.describe_block_device("/dev/sda")) + + assert "ZERO sectors" in described + assert "picotool" in described + + +def test_a_drive_with_sectors_but_no_filesystem_points_at_blkid(monkeypatch, tmp_path): + block = tmp_path / "sda" + block.mkdir() + (block / "size").write_text("256\n") + real_read = Path.read_text + monkeypatch.setattr(Path, "read_text", lambda self, *a, **k: real_read(block / "size") if str(self).endswith("/sys/block/sda/size") else real_read(self, *a, **k)) + + described = "\n".join(trench_coat.describe_block_device("/dev/sda")) + + assert "256 sectors" in described + assert "blkid" in described From eddfb7bdf3f9c6d035772b21db4fd1ad3a53ac87 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 16:03:51 +0000 Subject: [PATCH 30/32] hil: mount the partition, not the disk, when a drive has one `blkid` on the bench board that the rescue could not mount: /dev/sda: PTUUID="16f685f4" PTTYPE="dos" A partition table. udisks was right to refuse - "/dev/sda is not a mountable filesystem" is the correct answer for a partitioned disk, because the filesystem is on the partition. block_device() only ever returned the disk, so the rescue asked for the one thing that could not work. It now prefers a partition when sysfs shows one, and falls through to the disk otherwise - which is every normal case, since an RP2 bootloader drive has no partition table and no partitions at all. Strictly better, never worse. The mount diagnostics also name the pattern now: a PTTYPE in blkid's output means the drive is partitioned and a bootloader drive normally is not, so seeing one at all says something is wrong with what the board is presenting. And it points at picotool, which is no longer a guess - on the bench, `picotool info` reached this board and reported "Program Information: none", which is a nuked flash answering over the bootrom's own USB interface while its mass storage was unusable. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/trench_coat.py | 11 +++++++++++ dev/tests/test_hil_bootsel.py | 26 ++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py index 1c373e6b..78a1402e 100644 --- a/dev/hil/trench_coat.py +++ b/dev/hil/trench_coat.py @@ -331,6 +331,14 @@ def block_device(usb_device): business, while this path belongs to the one device we are holding. """ for block in sorted(Path(usb_device).glob("*/host*/target*/*/block/*")): + # A filesystem lives on a partition when the drive has a partition + # table, and udisks rightly refuses to mount the whole disk in that + # case - "/dev/sda is not a mountable filesystem" is exactly what a + # bench board produced, with `blkid` reporting PTTYPE="dos" on it. + # A plain RP2 bootloader drive has no partition table and no + # partitions, so this falls through to the disk as before. + for partition in sorted(block.glob(f"{block.name}[0-9]*")): + return Path("/dev") / partition.name return Path("/dev") / block.name return None @@ -398,6 +406,9 @@ def describe_block_device(device): else: lines.append(f"the drive reports {sectors} sectors ({sectors * 512 // 1024} KiB) but no filesystem udisks will mount.") lines.append("Worth checking by hand on the bench host: sudo blkid " + str(device) + " ; sudo dmesg | tail -30") + lines.append("A `PTTYPE=` in blkid's output means the drive is partitioned and the filesystem is on a") + lines.append("partition; a bootloader drive normally has neither. picotool talks to the bootrom directly") + lines.append("and needs no drive at all, which is the way past this: picotool info ; picotool load -x ") return lines diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index b2b79ccc..459b9ed8 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -647,3 +647,29 @@ def test_a_drive_with_sectors_but_no_filesystem_points_at_blkid(monkeypatch, tmp assert "256 sectors" in described assert "blkid" in described + + +def test_the_partition_is_preferred_over_the_whole_disk(tmp_path): + """udisks will not mount a partitioned disk, and a bench board was one. + + `blkid /dev/sda` reported PTUUID/PTTYPE="dos" while udisks said "not a + mountable filesystem" - correct of it, because the filesystem is on the + partition. + """ + device = tmp_path / "1-1" + block = device / "1-1:1.0" / "host0" / "target0:0:0" / "0:0:0:0" / "block" / "sda" + (block / "sda1").mkdir(parents=True) + + assert trench_coat.block_device(device) == Path("/dev/sda1") + + +def test_an_unpartitioned_bootloader_drive_still_uses_the_disk(tmp_path): + """The normal case: an RP2 drive has no partition table and no partitions.""" + device = tmp_path / "1-1" + block = device / "1-1:1.0" / "host0" / "target0:0:0" / "0:0:0:0" / "block" / "sda" + block.mkdir(parents=True) + # sysfs puts other things in here; none of them is a partition. + (block / "queue").mkdir() + (block / "size").write_text("262144\n") + + assert trench_coat.block_device(device) == Path("/dev/sda") From 602ceaa9e19dc3c77f95e3c5fe01a83bcd7f94a0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 16:22:09 +0000 Subject: [PATCH 31/32] hil: fall back to picotool when a board in BOOTSEL has no usable drive Copying a UF2 needs the board's mass storage to work, and a bench board came back from a power cycle mid-flash where it does not. It is enumerated, it is in the bootloader, and its drive is unmountable - `blkid` found a DOS partition table where a bootloader drive should have a bare filesystem. No retry fixes that, and the rescue could only report it and stop. picotool speaks the bootrom's own USB protocol and needs no filesystem at all. On that same board, with the drive unusable, `picotool info` answered and reported an empty flash - so this is a demonstrated route, not a guess. It runs only where the drive route cannot start or cannot finish, keeps the wipe-then-write shape that makes this a recovery rather than an upgrade, and treats a refused erase as non-fatal because a board that will not erase may still take the firmware. The safety property is that picotool is always aimed at one board, by the USB bus and address read from the sysfs directory already matched by chip id. Unaimed, it acts on whichever RP2 device it finds first, which on this bench could be a healthy board someone is mid-flash on; with no address available it declines rather than guessing. recover.py's preflight now counts picotool as a way to finish a reflash, so it will no longer refuse to touch a board into BOOTSEL on a runner that has picotool but no way to mount a drive. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/RUNNER_SETUP.md | 13 +++- dev/hil/recover.py | 7 +- dev/hil/trench_coat.py | 116 ++++++++++++++++++++++++++++++---- dev/tests/test_hil_bootsel.py | 103 ++++++++++++++++++++++++++++++ dev/tests/test_hil_recover.py | 3 +- 5 files changed, 226 insertions(+), 16 deletions(-) diff --git a/dev/hil/RUNNER_SETUP.md b/dev/hil/RUNNER_SETUP.md index 83699483..62db3f40 100644 --- a/dev/hil/RUNNER_SETUP.md +++ b/dev/hil/RUNNER_SETUP.md @@ -282,9 +282,16 @@ on 2026-08-28: | rung | state | what it needs | |---|---|---| | drain the console | works | serial access, which the `dialout` group already gives | -| reset the USB device | **unavailable** | write access to `/dev/bus/usb/*` — a udev rule | -| power cycle the hub port | **unavailable** | `sudo apt install uhubctl`, and a hub that switches port power | -| reflash via TrenchCoat | works | `udisksctl`, which is present | +| reset the USB device | works | write access to `/dev/bus/usb/*` — the udev rule below | +| power cycle the hub port | **unavailable** | a hub that switches port power; this bench's is `ganged`, so it cannot | +| reflash via TrenchCoat | works | `udisksctl` to mount the drive, or `picotool` when there isn't one | + +`picotool` is the fallback that matters when a board comes back wrong. Copying a UF2 needs the +board's mass storage to work, and a board power-cycled mid-flash can present a drive nothing +will mount — one on this bench came back with a DOS partition table where a bootloader drive +should have a bare filesystem. `picotool` speaks the bootrom's own USB protocol and needs no +drive at all, so `sudo apt install picotool` is worth doing before you need it. It is always +aimed at a single board by USB bus and address, never at whatever it finds first. The two unavailable rungs are worth enabling — they are the non-destructive ones, and without them a wedged board goes straight from "send it a Ctrl-C" to "wipe its flash". For the USB diff --git a/dev/hil/recover.py b/dev/hil/recover.py index e206bac7..4dbbf8e9 100644 --- a/dev/hil/recover.py +++ b/dev/hil/recover.py @@ -340,13 +340,18 @@ def can_complete_a_reflash(): replug) gets it out of. Doing that with no way to finish the job makes the board harder to recover, not easier. """ + # picotool first: it is the only route that does not depend on the board + # presenting a usable drive, which is exactly the case that stranded a + # bench board here (enumerated, in BOOTSEL, no mountable filesystem). + if trench_coat.picotool_available(): + return True, "picotool can write over the bootrom, with or without a drive" if trench_coat.find_bootloader_drives(): return True, "a bootloader drive is already mounted" if shutil.which("udisksctl"): return True, "udisksctl is available to mount the drive" if any(Path(root).is_dir() and os.access(root, os.W_OK) for root in trench_coat.MOUNT_ROOTS): return True, "an automount directory is writable" - return False, "nothing here can mount an RPI-RP2 drive (no udisksctl, no writable automount directory)" + return False, "nothing here can flash a board in BOOTSEL (no picotool, no udisksctl, no writable automount directory)" def reflash(port, target, cache_dir, force=False): diff --git a/dev/hil/trench_coat.py b/dev/hil/trench_coat.py index 78a1402e..081c523e 100644 --- a/dev/hil/trench_coat.py +++ b/dev/hil/trench_coat.py @@ -475,6 +475,104 @@ def copy_uf2(uf2, device, drive): unmount(device) +PICOTOOL_TIMEOUT = 300 + + +def picotool_available(): + return bool(shutil.which("picotool")) + + +def usb_address(chip_id): + """(bus, address) for one board in BOOTSEL, so picotool can be aimed. + + This is the whole safety story for the fallback. picotool with no device + selection acts on whatever RP2 device it finds first, which on this bench + could be a healthy board someone is mid-flash on. Reading bus and address + out of the sysfs directory we already matched by chip id means the command + can only ever reach the board we identified. + """ + for board in bench.bootsel_boards(): + if board["chip_id"] != chip_id: + continue + try: + bus = int((Path(board["usb_device"]) / "busnum").read_text()) + address = int((Path(board["usb_device"]) / "devnum").read_text()) + except (OSError, ValueError): + return None + return bus, address + return None + + +def picotool(arguments, chip_id, timeout=PICOTOOL_TIMEOUT): + """Run picotool against exactly one board.""" + where = usb_address(chip_id) + if where is None: + return None + command = ["picotool", *arguments, "--bus", str(where[0]), "--address", str(where[1])] + return subprocess.run(command, capture_output=True, text=True, timeout=timeout) + + +def _picotool_output(result, lines=10): + return [line for line in (result.stderr or result.stdout or "").strip().splitlines()[-lines:]] + + +def flash_over_picotool(chip_id, uf2): + """Write a UF2 without a drive, by talking to the ROM bootloader itself. + + The drive-copy route needs the board's mass storage to work. A bench board + came back from a power cycle mid-flash with a drive udisks would not touch + - `blkid` saw a DOS partition table where a bootloader drive should have a + bare filesystem - and no retry was ever going to fix that. picotool speaks + the bootrom's own USB protocol (PICOBOOT) and needs no filesystem at all; + on that same board `picotool info` answered while the drive was unusable. + + The erase keeps the property that makes this a recovery rather than an + upgrade - nothing of the old filesystem survives - and is best effort, + because a board that will not erase may still accept the firmware. + """ + if not picotool_available(): + log(" picotool is not installed, so there is no way past an unusable drive here") + log(" (`sudo apt install picotool` gives the bench a route that needs no filesystem)") + return False + + where = usb_address(chip_id) + if where is None: + log(" could not read the board's USB bus/address, and picotool must be aimed at one board") + return False + + log(f" falling back to picotool on bus {where[0]} address {where[1]} - it needs no drive") + erased = picotool(["erase"], chip_id) + if erased is None or erased.returncode != 0: + detail = "; ".join(_picotool_output(erased, 2)) if erased is not None else "the board moved" + log(f" erase declined ({detail}) - writing the firmware anyway") + + loaded = picotool(["load", "-x", str(uf2)], chip_id) + if loaded is None: + log(" the board stopped being a bootloader before the firmware could be written") + return False + if loaded.returncode != 0: + log(" picotool could not write the firmware:") + for line in _picotool_output(loaded): + log(f" {line}") + return False + + log(f" picotool wrote {uf2.name} and started it") + return True + + +def wait_for_serial(before, uf2): + """Wait for a flashed board to come back as a port it did not hold before.""" + deadline = time.monotonic() + RESTART_TIMEOUT + while time.monotonic() < deadline: + new = [port for port in serial_ports() if port not in before] + if new: + log(f" back as {new[0]} running {uf2.name}") + return new[0] + time.sleep(1) + log(f" {uf2.name} was written but the board never came back as a serial device") + return None + + def flash_bootsel(chip_id, target, root): """Put firmware back on a board that is sitting in its ROM bootloader. @@ -493,7 +591,9 @@ def flash_bootsel(chip_id, target, root): if drive is None: log(f" no usable bootloader drive for {chip_id}: it is enumerated and in the bootloader, but") log(" nothing here could mount a filesystem to copy a UF2 onto - see the reason above") - return None + if not flash_over_picotool(chip_id, uf2): + return None + return wait_for_serial(before, uf2) # The wipe is the load-bearing step: it erases the whole flash, so nothing # from whatever state the board was left in survives into the new firmware. @@ -510,18 +610,12 @@ def flash_bootsel(chip_id, target, root): device, drive = bootsel_drive(chip_id) if drive is None: log(" the wiped board never presented its drive again") - return None + if not flash_over_picotool(chip_id, uf2): + return None + return wait_for_serial(before, uf2) copy_uf2(uf2, device, drive) - deadline = time.monotonic() + RESTART_TIMEOUT - while time.monotonic() < deadline: - new = [port for port in serial_ports() if port not in before] - if new: - log(f" back as {new[0]} running {uf2.name}") - return new[0] - time.sleep(1) - log(f" {uf2.name} was written but the board never came back as a serial device") - return None + return wait_for_serial(before, uf2) def rescue_bootsel(board_map, cache_dir): diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index 459b9ed8..43daa93d 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -28,6 +28,7 @@ sys.path.insert(0, str(REPO_ROOT / "dev" / "hil")) import bench # noqa: E402 +import recover # noqa: E402 import trench_coat # noqa: E402 @@ -673,3 +674,105 @@ def test_an_unpartitioned_bootloader_drive_still_uses_the_disk(tmp_path): (block / "size").write_text("262144\n") assert trench_coat.block_device(device) == Path("/dev/sda") + + +# -------------------------------------------------------------------------- +# picotool: the route that needs no drive +# -------------------------------------------------------------------------- + + +@pytest.fixture() +def picotool_calls(monkeypatch, tmp_path): + """Record picotool invocations, with one board in BOOTSEL at a known address.""" + device = tmp_path / "1-1.2" + device.mkdir() + (device / "busnum").write_text("1\n") + (device / "devnum").write_text("22\n") + monkeypatch.setattr(bench, "bootsel_boards", lambda: [{"chip_id": "df13", "usb_device": device, "processor": "RP2350"}]) + monkeypatch.setattr(trench_coat.shutil, "which", lambda name: "/usr/bin/picotool") + + calls = [] + + def run(command, **kwargs): + calls.append(command) + return types.SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(trench_coat.subprocess, "run", run) + return calls + + +def test_picotool_is_aimed_at_one_board_by_bus_and_address(picotool_calls): + """The safety property: never "whatever RP2 device it finds first".""" + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is True + + for command in picotool_calls: + assert "--bus" in command and command[command.index("--bus") + 1] == "1" + assert "--address" in command and command[command.index("--address") + 1] == "22" + + +def test_picotool_erases_before_it_writes(picotool_calls): + """Keeps the property that makes this a recovery, not an upgrade.""" + trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) + + assert [c[1] for c in picotool_calls][:2] == ["erase", "load"] + assert "-x" in picotool_calls[1] and "/uf2/wpc.uf2" in picotool_calls[1] + + +def test_a_board_that_will_not_erase_is_still_offered_the_firmware(monkeypatch, picotool_calls): + results = iter([types.SimpleNamespace(returncode=1, stdout="", stderr="erase not supported"), types.SimpleNamespace(returncode=0, stdout="", stderr="")]) + monkeypatch.setattr(trench_coat.subprocess, "run", lambda command, **k: picotool_calls.append(command) or next(results)) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is True + assert [c[1] for c in picotool_calls] == ["erase", "load"] + + +def test_a_failed_load_is_reported_as_failure(monkeypatch, picotool_calls): + results = iter([types.SimpleNamespace(returncode=0, stdout="", stderr=""), types.SimpleNamespace(returncode=1, stdout="", stderr="no such device")]) + monkeypatch.setattr(trench_coat.subprocess, "run", lambda command, **k: next(results)) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is False + + +def test_without_picotool_the_fallback_declines_rather_than_guessing(monkeypatch, picotool_calls): + monkeypatch.setattr(trench_coat.shutil, "which", lambda name: None) + monkeypatch.setattr(trench_coat.subprocess, "run", lambda *a, **k: pytest.fail("picotool is not installed")) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is False + + +def test_a_board_that_is_no_longer_in_bootsel_is_never_targeted(monkeypatch): + """No address means no command - picotool is never run unaimed.""" + monkeypatch.setattr(bench, "bootsel_boards", lambda: []) + monkeypatch.setattr(trench_coat.shutil, "which", lambda name: "/usr/bin/picotool") + monkeypatch.setattr(trench_coat.subprocess, "run", lambda *a, **k: pytest.fail("there is no board to aim at")) + + assert trench_coat.flash_over_picotool("df13", Path("/uf2/wpc.uf2")) is False + + +def test_an_unmountable_drive_falls_through_to_picotool(monkeypatch, tmp_path): + """The bench case end to end: in BOOTSEL, drive unusable, firmware written anyway.""" + root = tmp_path / "trench-coat" + (root / "uf2").mkdir(parents=True) + (root / "uf2" / "nuke.uf2").write_text("nuke") + (root / "uf2" / trench_coat.TARGET_UF2["wpc"]).write_text("firmware") + + monkeypatch.setattr(trench_coat, "bootsel_drive", lambda chip_id, timeout=None: (None, None)) + # Absent before the flash, present after - which is what "it came back" means. + ports = iter([[], ["/dev/ttyACM3"]]) + monkeypatch.setattr(trench_coat, "serial_ports", lambda: next(ports, ["/dev/ttyACM3"])) + tried = [] + monkeypatch.setattr(trench_coat, "flash_over_picotool", lambda chip_id, uf2: tried.append(uf2.name) or True) + + assert trench_coat.flash_bootsel("df13", "wpc", root) == "/dev/ttyACM3" + assert tried == [trench_coat.TARGET_UF2["wpc"]] + + +def test_picotool_makes_a_reflash_possible_without_any_mount(monkeypatch): + monkeypatch.setattr(trench_coat, "picotool_available", lambda: True) + monkeypatch.setattr(trench_coat, "find_bootloader_drives", lambda: []) + monkeypatch.setattr(recover.shutil, "which", lambda name: None) + + possible, why = recover.can_complete_a_reflash() + + assert possible is True + assert "picotool" in why diff --git a/dev/tests/test_hil_recover.py b/dev/tests/test_hil_recover.py index 6db5e11d..a88d65f2 100644 --- a/dev/tests/test_hil_recover.py +++ b/dev/tests/test_hil_recover.py @@ -247,7 +247,8 @@ def test_can_complete_a_reflash_says_no_when_nothing_can_mount(monkeypatch): possible, why = recover.can_complete_a_reflash() assert possible is False - assert "nothing here can mount" in why + # Names every route it looked for, so the message says what to install. + assert "no picotool" in why and "no udisksctl" in why def test_every_target_maps_to_a_bundled_uf2(): From f60987e7bae52a36f406a887398b7aa44ddbff37 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 30 Aug 2026 17:52:33 +0000 Subject: [PATCH 32/32] hil: never offer a map entry for a board that will not say who it is The wpc board came back on 602ceaa - built, flashed and health-checked, IP and all - and sys11 took its place as the broken one: its flash hit the 900s timeout, and it now enumerates without answering for its chip id at all. The map instructions then offered this as the fix: VECTOR_HIL_BOARD_MAP=None=,899fab8c90bfeb9a=data_east,... `None` is the chip id rendered through an f-string. A board with no id cannot be mapped - an entry needs an id to key on - so putting it in the line at all produces a broken value presented as the remedy, which is the same failure as echoing back a bad target and was fixed there for the same reason. Boards without an id are now named separately, with what that state usually means: a board that enumerates but will not identify itself has generally been left mid-flash, and wants recovering before it can be mapped. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01GFJmgcxowPCYV91BKZ3ah5 --- dev/hil/bench.py | 15 ++++++++++++++- dev/tests/test_hil_bootsel.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/dev/hil/bench.py b/dev/hil/bench.py index 270e61df..e33e1613 100644 --- a/dev/hil/bench.py +++ b/dev/hil/bench.py @@ -614,7 +614,13 @@ def wanted(chip_id): suggestion = suggest_target(target) return suggestion if suggestion in bench_targets() else "" - suggested = ",".join(f"{b['chip_id']}={wanted(b['chip_id'])}" for b in boards) + # A board whose chip id could not be read cannot be mapped at all - there + # is nothing to key the entry on. Leaving it in produced a suggestion of + # `None=` on the bench, which is a broken value presented as the + # fix. It is named separately instead, with what to do about it. + nameless = [b for b in boards if not b.get("chip_id")] + identified = [b for b in boards if b.get("chip_id")] + suggested = ",".join(f"{b['chip_id']}={wanted(b['chip_id'])}" for b in identified) lines = [ "", "VECTOR_HIL_BOARD_MAP pins each board to the system it is wired to, by RP2040", @@ -624,6 +630,13 @@ def wanted(chip_id): " targets: " + ", ".join(bench_targets()), "", ] + if nameless: + lines += [ + " " + ", ".join(b.get("port") or "(BOOTSEL)" for b in nameless) + " did not report a chip id, so it cannot be", + " mapped yet - an entry needs an id to key on. A board that enumerates but will not say", + " who it is has usually been left mid-flash; recover it first, then map the id it reports.", + "", + ] unusable = [target for target in buildable_targets() if target not in bench_targets()] if unusable: lines += [ diff --git a/dev/tests/test_hil_bootsel.py b/dev/tests/test_hil_bootsel.py index 43daa93d..536c35e5 100644 --- a/dev/tests/test_hil_bootsel.py +++ b/dev/tests/test_hil_bootsel.py @@ -776,3 +776,36 @@ def test_picotool_makes_a_reflash_possible_without_any_mount(monkeypatch): assert possible is True assert "picotool" in why + + +def test_a_board_with_no_chip_id_is_never_put_in_the_suggested_line(): + """The bench printed `VECTOR_HIL_BOARD_MAP=None=,...` as the fix. + + A board that will not say who it is cannot be mapped - an entry needs an + id to key on - so it is named separately rather than rendered into the + line somebody is meant to paste. + """ + boards = [ + {"port": "/dev/ttyACM0", "chip_id": None}, + {"port": "/dev/ttyACM1", "chip_id": "899fab8c90bfeb9a"}, + ] + + text = board_map_instructions_for(boards, {"899fab8c90bfeb9a": "data_east"}) + + assert "None=" not in text + assert "VECTOR_HIL_BOARD_MAP=899fab8c90bfeb9a=data_east" in text + assert "/dev/ttyACM0 did not report a chip id" in text + assert "left mid-flash" in text + + +def board_map_instructions_for(boards, board_map): + return bench.board_map_instructions(boards, board_map) + + +def test_every_board_still_appears_when_they_all_have_ids(): + boards = [{"port": "/dev/ttyACM0", "chip_id": "aaaa"}, {"port": "/dev/ttyACM1", "chip_id": "bbbb"}] + + text = bench.board_map_instructions(boards, {"aaaa": "sys11", "bbbb": "wpc"}) + + assert "VECTOR_HIL_BOARD_MAP=aaaa=sys11,bbbb=wpc" in text + assert "did not report a chip id" not in text