From 5639f2d0b1d99d98c67f39f130662bd1263c216e Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Tue, 20 Oct 2026 20:17:01 +0700 Subject: [PATCH] HIL: run the farm-shared mechanics from agentrc's hil skill (#583) The battery lock, DFU and esptool children, report publication and the htpc-to-ci.lan remote runner now come from agentrc's hil skill, pinned at hathach/agentrc#165's merge. --- .claude/skills/build/scripts/check_build.py | 28 +- .claude/skills/hil-contract/SKILL.md | 78 +- .../hil-contract/scripts/esp_session.py | 88 -- .claude/skills/hil-contract/scripts/hil_ci.py | 255 +++-- .../skills/hil-contract/scripts/hil_remote.py | 507 ++++++++++ .../hil-contract/scripts/hil_remote_driver.py | 221 +++++ .../hil-contract/scripts/hil_run_report.py | 129 +++ .../skills/hil-contract/scripts/hil_test.py | 410 ++++++-- .../skills/hil-contract/scripts/usb_dfu.py | 65 -- .claude/test/test_hil.py | 830 ++++++++++++---- .claude/test/test_hil_ci.py | 496 +++++++++- .claude/test/test_hil_remote.py | 909 ++++++++++++++++++ .github/workflows/build_util.yml | 2 +- .github/workflows/hil.yml | 17 +- .gitignore | 1 + lib/agentrc | 2 +- 16 files changed, 3491 insertions(+), 547 deletions(-) delete mode 100644 .claude/skills/hil-contract/scripts/esp_session.py create mode 100644 .claude/skills/hil-contract/scripts/hil_remote.py create mode 100644 .claude/skills/hil-contract/scripts/hil_remote_driver.py create mode 100644 .claude/skills/hil-contract/scripts/hil_run_report.py delete mode 100644 .claude/skills/hil-contract/scripts/usb_dfu.py create mode 100644 .claude/test/test_hil_remote.py diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index 2420c2cb6..0f12a8505 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -16,7 +16,6 @@ import argparse import contextlib -import fcntl import functools import hashlib import json @@ -31,6 +30,11 @@ import uuid from pathlib import Path +try: + import fcntl +except ImportError: # Windows, where hil_remote.py's wait and collect load this module for its pin check only + fcntl = None + ROOT = Path(__file__).resolve().parents[4] BUILD_YML = ROOT / '.github' / 'workflows' / 'build.yml' BUILD_DIR = ROOT / 'cmake-build' @@ -446,6 +450,28 @@ def dirty_inputs(paths): return [p for p in paths if git_in(p, 'status', '--porcelain', '--untracked-files=all').strip()] +AGENTRC = 'lib/agentrc' + + +def check_agentrc_pin(): + """Raise Fail unless lib/agentrc is a checkout of its own, clean (untracked files included) at the commit HEAD + pins: the one check before any tinyuf2 script runs agentrc code from it.""" + init = f'git submodule update --init {AGENTRC}' + if not own_checkout(AGENTRC): + raise Fail(f'{AGENTRC} is not checked out: {init}') + want = git('rev-parse', f'HEAD:{AGENTRC}').strip() + have = git_in(AGENTRC, 'rev-parse', 'HEAD').strip() + if have != want: + # submodule update checks out the index's gitlink, so a staged pin change would defeat the hint + staged = git('rev-parse', f':{AGENTRC}').strip() + if staged != want: + raise Fail(f'{AGENTRC} is at {have}; HEAD pins {want} but {staged} is staged: commit or unstage that ' + f'{AGENTRC} pin change') + raise Fail(f'{AGENTRC} is at {have}; HEAD pins {want}: {init}') + if dirty_inputs([AGENTRC]): + raise Fail(f'{AGENTRC} checkout is dirty at the pinned {want}: discard its local changes') + + def identity(ports=()): """What a result was built from: HEAD, whether the tree differs from it, the submodule commits, the commits of the get_deps dependencies `ports` use, and which of those inputs cannot be trusted: diff --git a/.claude/skills/hil-contract/SKILL.md b/.claude/skills/hil-contract/SKILL.md index f44bb9009..b04ee81f3 100644 --- a/.claude/skills/hil-contract/SKILL.md +++ b/.claude/skills/hil-contract/SKILL.md @@ -10,6 +10,7 @@ description: The TinyUF2 HIL contract: its roster, selection, build record, batt ## Hosts/rosters - One roster per host, `` in the commands below: `test/hil/local.json` on htpc's bench and `test/hil/ci.json` on the ci.lan rig (`hostname` `ci`, where the CI job below runs); `hostname` says which you are on. Elsewhere a J-Link or ST-LINK run fails at `install`, where no probe with the roster's serial is found, before anything is written: `cleanup.state` is `untouched` and nothing stays held. An Espressif board without exactly one tty with its route's serial stops at the runner's probe check, after the lock: a `ran: false` row with `detail` `probe check: ...`, nothing written and nothing held. Both hosts are shared with tinyusb's rosters (its contract's Hosts/rosters: `test/hil/local.json` on htpc, `test/hil/tinyusb.json` on ci.lan): a rig-wide operation holds both rosters' lock names (`hil` Board locks), tinyusb's under the names and in the sequence its contract's Reservation gives. +- From htpc, `scripts/hil_remote.py` runs ci.lan's batteries there (Execution, Remote) over agentrc's remote core (`lib/agentrc/skills/hil/scripts/hil_remote_core.py`, loaded from the pin): the rig builds and runs with its own roster, `test/hil/ci.json`, under the same board locks and beside CI's state root, `~/actions-runner-tinyuf2/_work/tinyuf2/hil-state` (hil.yml's `$(dirname $GITHUB_WORKSPACE)/hil-state` for the runner's account; the runner service runs as the ssh user, with no PrivateTmp, so both see one filesystem and one `/tmp/tinyusb-hil-locks`). Each remote run is an attempt directory on ci.lan, `~/hil-remote/tinyuf2/`. A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. A J-Link entry has `args` exactly `-device `; the runner uses SWD at 4000 kHz, so pass the same to those scripts, `--interface swd --speed 4000`. An ST-LINK entry has `flasher.name` `stlink`, `flasher.uid` the ST-LINK's USB serial and `args` exactly `-device `, a name the runner holds limits and a chip identity for (`FLASH_LIMITS`, `STLINK_CHIPS`); STM32CubeProgrammer takes no device name, so every session checks the Device ID and NVM size it prints. An Espressif entry has `flasher.name` `esptool`, `flasher.uid` the USB-UART bridge's serial (`/dev/serial/by-id/*_-if*`), `flasher.args` exactly `--chip `, and the top-level `uid` the chip's factory MAC, which is TinyUF2's USB serial. The board's battery is its kind, Espressif, whatever route installs it; the route is the transport. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: - the board lock name (`lock`) when tinyusb names the board differently, as `mimxrt1060_evk` for `imxrt1060_evk`; both runners take that one lock, and the helper's `status`, holder records and the report rows' `lock` show it. It defaults to the board name; @@ -36,6 +37,8 @@ python3 .claude/skills/build/scripts/check_build.py --board --backend cm The build record is this contract's build receipt: build it on the clean, pinned HEAD with `--hil-config` and pass it with `--build-record`. `--hil-config` binds the build's own flash target to the roster's probe or bridge (build contract), a build-time compatibility check, and binds each route in `tinyuf2.routes` to the segments that binding hashed (`flashRoutes`); the run refuses a record whose binding names another flasher or route, and checks the files the install reads against the record. The runner never runs the target: it passes the bound files to agentrc's flash provider (Execution). The build dir's lock covers the install's inputs only: the files the runner reads itself (the BIN it verifies, the segments it reads back, `apps/blinky.uf2` and `apps/dfu_runtime.uf2`) are copies in the run dir, taken at admission from bytes that hash as recorded, so a rebuild after the target cannot change them. For a J-Link or ST-LINK board it must be a cmake build (`--backend cmake`), whatever backend CI builds the port with; board facts are read from the port's CMake files too. For an espressif board it must be an IDF build from CI's `espressif/idf` image; the run binds its generated `sdkconfig`, `flash_args`, the four segments (bootloader, partition table, otadata, `tinyuf2.bin` in the factory partition) and `apps/dfu_runtime.uf2`, and checks every segment against the generated partition table and the candidate's partition CSV. The run refuses a build record that is not clean, does not match the checkout now (HEAD, submodules, `get_deps` dependencies at their pins), or whose artifacts changed since. Commit first; ESP-IDF's records also go stale at the next commit (build contract). +A remote run (Execution, Remote) builds on the rig from the committed identity, never here: the attempt's rebuilt checkout (`/src`) builds each board's record with the command above (`hil_ci.py`'s, `--backend cmake`) as `/out/.build.json`, its build dir inside that checkout (`buildDir` is checkout-relative), and the record stays there for recovery. + The Ninja query that inspects a flash target at the build (`ninja -t commands`) is bounded to 30 seconds. A timed-out or interrupted query's process is killed (only that process, which stays in check_build's process group, so a kill of the group reaches it too) and its ESP-IDF container (`tinyuf2-query--<32 hex digits>`, unique per query) removed until docker has listed none for 2 seconds; a timeout fails the build naming the build dir, the query and any container that may be left, and an interruption prints that container on stderr. The install runs no Ninja query and no container. ## Execution @@ -58,7 +61,7 @@ On an ST-LINK board the steps are the J-Link board's, each probe session one STM A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; on a J-Link board a run triggered by the upload alone is unverified (an Espressif board's app-ran observes the copied app itself over USB). "reenter" by `retained-register` is the forced-entry path, not a physical double tap. -On an Espressif board, each esptool connection is a child of the runner (`esp_session.py`), and so is each USB control request (`usb_dfu.py`, pyusb): +On an Espressif board, each esptool connection is a child of the runner, and so is each USB control request (pyusb): the `hil` provider's `esptool_session.py` and `usb_dfu.py` beside `hil_lock.py` at the pinned `lib/agentrc` (Reservation), which admission requires before any board access: 1. **install:** a first connection takes the MD5s of every partition the battery never writes (nvs, ota_1, ffat on the 8 MB layout) as the run's snapshot, kept in the row's `snapshot` however the run ends; that session is the first to touch the board, which only a missing tty with the route's serial leaves untouched. Then the provider's `flash_esptool` runs the runner's own esptool, the 5.2.0 `esptool` command of the runner's Python (refused before any board access otherwise), as `NO_COLOR=1 esptool --chip -p --before=default_reset --after=hard_reset write_flash ` from the build dir under its lock, with the same hash checks as a J-Link install. esptool 5.2.0 writes every segment, as `IDF_FLASH_FULL=1` made the IDF target do, and copies none to `*_flashed.bin`. Its transcript, every CSI sequence and the underscore spellings' deprecation warnings stripped, must show every segment written at its offset and hash-verified, then the hard reset. A runner session then reads each segment back and takes those MD5s again, then resets once; the runner compares the read-back bytes with the build's segment files and the MD5s with the snapshot, and any difference fails the step. esptool rewrites the bootloader header from `flash_args`, which the build already holds (dio, 80 MHz, 8 MB): on the bench (2026-10-13, esptool 5.3.1 in espressif/idf:v6.1) all four segments read back identical to the files. The roster's bridge selects the board (it is wired to that board only), so the chip is not identified, and its security eFuses and flash size are commissioning's to check. The evidence lists each write's span and the 4 KiB-aligned span covering it, and the esptool version. Blank otadata boots the factory partition, TinyUF2. On the `openocd_esp` route (Hosts/rosters) the install runs under the same one board lock, each step only once the previous one's cleanup completed: an esptool session over the USJ's tty reads the chip's security state on the ROM loader, before the stub, from esptool's GET_SECURITY_INFO and from the eFuse registers (on the P4 SPI_BOOT_CRYPT_CNT and SECURE_BOOT_EN), takes the snapshot and hard-resets the chip, since one left in the ROM loader faults openocd's flash stub after `reset init` (PMP Load access fault at `flash probe`, bench 2026-10-20); the install is refused, nothing programmed, unless both sources read flash encryption and secure boot off, which is the evidence the provider requires and records (`security`). After a one-second wait (bench-history.md, espressif_p4_function_ev JTAG route), the provider's `flash_openocd_esp` programs the record's segment copies in one openocd-esp32 session pinned to the USJ by serial and bus-port (the build `$HIL_OPENOCD_ESP` names, else `openocd` on PATH; the provider refuses any but v0.12.0-esp32-20260703), after a `reset init` whose event hooks all passed, each written and checked with `esp verify_bank_hash` without a reset between, reads each back byte for byte and ends with `reset run` and a confirmed shutdown; it writes the files as built, without esptool's header rewrite, so the read-back compares the built bytes. Then the esptool read-back and preservation check as above. A session the provider leaves not ended is cleanup pending (Results, Recovery): the runner never signals or reaps it, and no later session, restore or retry runs. @@ -73,12 +76,29 @@ On an Espressif board, each esptool connection is a child of the runner (`esp_se ### CI -The `HIL` workflow (`.github/workflows/hil.yml`) runs the battery on the ci.lan rig for every pull request from this repository (a fork's never reaches the rig). A hosted job selects the boards the change reaches (Selection, with `test/hil/ci.json`) and prints what is uncovered in its summary; the rig job, on the runner labelled `tinyuf2`, checks out the merge commit, fetches the submodules and the roster ports' dependencies, and runs `scripts/hil_ci.py`: per board a record built there (Build) and one battery whose run directory (`run --run-root`) lies under the rig's `hil-state/` with the record and the battery's output, uploaded as the job's artifact. The battery keeps trying a held board every 15 s within `run --lock-wait` (10 min from CI), then reports it locked, which the job reads as not-run, never a pass; a cleanup holder is never waited for. The job's only pre-flight is its markers and the runner's own probe check: a surviving `mcopy` or USB helper of a killed run is seen through the marker, not by a host-wide process scan. Each row maps onto the Verdicts: the job exits 0 only when every selected board passed, 1 on a fail or not-run, 2 on a blocker. A battery that ended without a report, or kept a cleanup holder, leaves its `.started` marker without a `.done`: that board is a blocker on every later run until recovery (Recovery) has run and the marker is removed by hand; the workflow is never cancelled by a newer push for the same reason. `workflow_dispatch` runs named roster boards, with `fail_at` for `--fail-at` (acceptance only). +The `HIL` workflow (`.github/workflows/hil.yml`) runs the battery on the ci.lan rig for every pull request from this repository (a fork's never reaches the rig). A hosted job selects the boards the change reaches (Selection, with `test/hil/ci.json`) and prints what is uncovered in its summary; the rig job, on the runner labelled `tinyuf2`, checks out the merge commit, fetches the submodules and the roster ports' dependencies, and runs `scripts/hil_ci.py`: per board a record built there (Build) and one battery whose run directory (`run --run-root`) lies under the rig's `hil-state/` with the record and the battery's output, uploaded as the job's artifact. The battery keeps trying a held board every 15 s within `run --lock-wait` (10 min from CI), then reports it locked, which the job reads as not-run, never a pass; a cleanup holder is never waited for. The job's only pre-flight is its markers and the runner's own probe check: a surviving `mcopy` or USB helper of a killed run is seen through the marker, not by a host-wide process scan. Each row maps onto the Verdicts: the job exits 0 only when every selected board passed, 1 on a fail or not-run, 2 on a blocker. Before the upload each run directory's report pair is prepared through the pinned report core (its Markdown regenerated from the JSON, or an explicit unavailable notice where the JSON is missing or unreadable, never a stale view; when the preparation itself fails, each Markdown is replaced by that notice with the run's view kept below it only as labelled text, or removed, under the core's writer lock; one that can be neither, or whose lock another writer holds, is listed in `summary.json` `unsafeArtifacts`, as is a run root that cannot be listed, and then the upload excludes every `hil_report.md`, as it does when `hil_ci.py` did not finish: its decision, `upload-views`, is written last), and `summary.json` gives each board, beside its `verdict`, the `hil` shared run rules' `sharedVerdict` and `sharedDetail` over that JSON: the same rows, with a run aborted by a signal or a non-empty caveat a blocker and an unreadable or unprepared report a blocker. The job summary labels the native verdict and shows the shared one where it differs or the report was not prepared; it never changes the exit. A battery touches its `.started` only once it holds the board, before any hardware access (`run --started-marker`, `hil_ci.py`'s only; a marker it cannot write refuses the run and releases the board), so a run that takes the lock while it still waits never sees it started, and a battery that never held the board leaves no marker. A battery that is no blocker touches its `.done` before it releases the board (`run --done-marker`, `hil_ci.py`'s only), so a run that takes the lock next never sees it unfinished; `hil_ci.py` then keeps that marker only where its own verdict is no blocker and the battery marked its start. A battery that ended without a report, or kept a cleanup holder, leaves its `.started` marker without a `.done`: that board is a blocker on every later run until recovery (Recovery) has run and the marker is removed by hand; the workflow is never cancelled by a newer push for the same reason. `workflow_dispatch` runs named roster boards, with `fail_at` for `--fail-at` (acceptance only). + +### Remote + +From htpc, for ci.lan's roster boards: + +```bash +python3 .claude/skills/hil-contract/scripts/hil_remote.py run -b [-b ...] [--run-id ] [--lock-wait ] +python3 .claude/skills/hil-contract/scripts/hil_remote.py wait [--timeout ] +python3 .claude/skills/hil-contract/scripts/hil_remote.py collect +``` + +`run` freezes HEAD and every repository it pins (the submodules and the get_deps dependencies of the boards' ports, unpushed commits and a candidate `lib/agentrc` pin included) as self-contained git bundles, bound by a manifest of the commits, shallow boundaries, roster digest, boards and bundle and driver digests. Before anything reaches the rig it refuses a checkout that is not clean, a submodule off the commit HEAD pins, a dependency missing or off its pin, an input that moved while frozen, a board not in `test/hil/ci.json` or an Espressif one, and a run id used before. It then creates the create-only attempt directory, stages and verifies the bundles there and launches `scripts/hil_remote_driver.py` detached: its own session, its stdio on the attempt's files, agentrc's launch and completion records in its `.attempt/`. The driver rebuilds detached checkouts with no remote, worktree pointer file or link to htpc, checks each repository's commit and clean state, the pins, the roster digest and its own, and refuses (`refused`, nothing built) on any difference or an unreadable CI state root. Then per board, one after another: CI's markers (a board with an unfinished CI battery is a blocker, not built; the runner checks again under its lock), the record (Build), and `hil_test.py run -b --build-record --run-root /out/ --lock-wait --ci-state-root test/hil/ci.json` through `hil_ci.py`'s `run_board()`, CI's markers read, never written: a runner that stays the in-process cleanup holder is left running, not waited for, and is a blocker. Each verdict is `hil_run_report.py`'s, and each run dir's report pair is prepared as CI prepares it, before the copy-back. `result.json` is the driver's outcome, `progress.json` its rows so far. + +`--lock-wait` defaults to 0 and is one budget for all of the attempt's batteries, counted from the first one's start (at most 3600 s); a recovery blocker is never waited for. `--fail-at` is acceptance only. `install`, `restore`, `clear-pending`, `--under-hold`, `--recover-holder`, `--recover-takeover`, `--try-reentry`, `--route`, `--build-record`, `--run-root`, `--ci-state-root`, `--done-marker` and `--started-marker` are refused before staging. No `HIL_*` variable is forwarded, the driver drops the rig's own `HIL_*` and `GIT_*` before the rebuild, `HIL_NO_BOARD_LOCK` set here refuses the run, and a caller's `GIT_*` (a hook's `GIT_DIR`) never reaches the identity check or the bundles. + +`run` waits for the attempt's completion record and collects it; `wait` reconnects and does the same within `--timeout` (at most 570 s); `collect` fetches the attempt whatever its state. Neither launches it again, and the end of `run` or `wait` (a timeout, a kill, a dropped connection) ends nothing on the rig: the attempt runs to its completion record. A launch whose acknowledgement was lost is found by its records and waited for, never launched twice. ## Reservation Locks are taken through agentrc's `hil` helper, `lib/agentrc/skills/hil/scripts/hil_lock.py`, at the commit the `lib/agentrc` submodule pins (`git submodule update --init lib/agentrc`), never an installed agentrc's copy. `hil_test.py`'s `run`, `install` and `restore` refuse before any hardware access while the submodule is missing, checked out at another commit or locally modified; the manual `hold`, `release` and `status` below run the helper directly, with no such check. The lock is the board's `` (Hosts/rosters): `hold` and `release` take it and `status` (no board argument) shows it, while `hil_test.py -b` and the probe scripts take the board name. -- **The battery locks itself,** as `hil_test.py`, which the helper's `release` refuses to kill. +- **The battery locks itself,** as `hil_test.py`, which the helper's `release` refuses to kill. Its record is protected and names its run dir (`runDir`): every exit clears it, a clean one or one before the board was touched, and a failed cleanup hands it to the cleanup holder (Recovery), so a battery that died holding the board (SIGKILL, a crash, the host's OOM killer) leaves it on the free lock as a stale record. Every project's lock helper then refuses the board as a recovery blocker until Recovery's stale-record route clears it, whatever `--lock-wait`; it is never waited for. +- **A remote run takes no lock of its own** (Execution, Remote): each of its batteries is `hil_test.py run` on the rig and locks its board as above, then, holding it and before any hardware access, checks CI's state root (`--ci-state-root`, Results). Attempts on different boards run side by side, one on a held board waits within `--lock-wait` or is not run, and an attempt id is used once. CI's markers stay CI's: a remote run never writes them. - **The lock fails closed.** An unusable lock dir refuses the run (exit 2); it never runs unlocked. `HIL_NO_BOARD_LOCK=1` is the bypass (`hil` Board locks); the report then says `lockBypassed: true`. - **Manual sessions** (hw-debugger, hw-validator): 1. Take your own hold: `python3 lib/agentrc/skills/hil/scripts/hil_lock.py hold --reason ""`. @@ -94,15 +114,15 @@ Locks are taken through agentrc's `hil` helper, `lib/agentrc/skills/hil/scripts/ Manual exits: 0 the operation succeeded (for `install`, DFU came up; for `restore`, DFU left after the blinky copy and stayed away); 1 it failed before writing; 2 refused before any hardware access; 4 a child survived SIGKILL, an `openocd_esp` session is not ended (cleanup pending, its receipt in `survivors`), or an `install` or `restore` failed after it began writing: keep the hold. `buildLocksKept` names a build dir whose lock a keeper holds until the listed survivors end. - On a refused hold, `python3 lib/agentrc/skills/hil/scripts/hil_lock.py status` (no board argument; it prints every holder) gives the holder to report. A `role: cleanup-holder` holder is this contract's exit 4 (Recovery); any other `hil_test.py` holder is a battery run (`hil` Board locks: wait within the budget). + On a refused hold, `python3 lib/agentrc/skills/hil/scripts/hil_lock.py status` (no board argument; it prints every holder) gives the holder to report. A `role: cleanup-holder` holder is this contract's exit 4 (Recovery); any other live `hil_test.py` holder is a battery run (`hil` Board locks: wait within the budget). A `stale` record or a `takeover journal` that `status` marks as a recovery blocker is Recovery's stale-record route. - **Cleanup state** for a hw-validator or hw-debugger dispatch: `restore --under-hold` leaves the board on the record's TinyUF2 with `apps/blinky` copied through it, and its `cleanup.evidence` (DFU left after the copy and stayed away) is the observation that Cleanup records. Blinky running is not checked. Whoever uses the board next flashes it first; tinyusb's HIL does. -- **Lock-wait budget:** htpc has no CI runner, so it is the dev-bench row of chief's budget unless the dispatch names another; ci.lan is the CI-runner row. `run --lock-wait ` keeps trying a held board that long, never a cleanup holder; the CI job passes its own (Execution, CI). +- **Lock-wait budget:** htpc has no CI runner, so it is the dev-bench row of chief's budget unless the dispatch names another; ci.lan is the CI-runner row. `run --lock-wait ` keeps trying a held board that long, never a cleanup holder or a recovery blocker; the CI job passes its own (Execution, CI). ## Results -The last stdout line is JSON, also written to its `reportPath`. Its fields: -- `pass`, `caveat`; -- per board: `ran`, `pass`, `locked`, `recoveryBlocker` (a `locked` row whose holder is a cleanup holder), `lockBypassed`, `probeUsers` (pids left on the board's probe, refused after the lock), `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`, and `receipts` when `state` is `pending`, `markErrors` when the pending mark could not be written), `jlinkRetried` (J-Link boards only: sessions run again after a crash before the target connect, each `{exit, commands}`; Limits), `route` (a run over a `tinyuf2.routes` route), `buildLocksKept` (each `{buildDir, survivors, keeper}`: a build dir whose lock a keeper holds until those children end), and `detail`; +The last stdout line is JSON. It is also published, through the pinned agentrc report core (`hil` Reporting; `scripts/hil_run_report.py` is this runner's adapter), as the run directory's `hil_report.json`, `reportPath`: the same fields plus the reserved `_hil` (attempt id = the run directory's name, termination `completed` or `aborted` by a signal, invocation, the board, host, the roster's file name and sha256 (the same in a checkout and on a rig), and the admitted record's `firmware` as build provenance, omitted when none was admitted), with `hil_report.md` beside it, a view of each board's steps rendered from that JSON. A report the core could not publish adds `reportError` to the stdout JSON, and `reportPath` is `null` unless its JSON was written; the run's exit, row and any hold are unchanged. Its fields: +- `pass`, `caveat`, `exitCode` (the runner's exit) and `heldInProcess` (the runner stays the cleanup holder, so it never exits), from which the Verdicts below follow from the file alone; +- per board: `ran`, `pass`, `locked`, `recoveryBlocker` (the board carries an unresolved recovery: a `locked` row whose holder is a cleanup holder, or whose lock holds a dead battery's stale record or a takeover journal; or, with `run --ci-state-root `, a transition check a driver passes CI's state directory for, an `unfinishedAttempts` row: a CI run under it started a battery on the board and has no `.done`, its liveness never guessed, and this run, refused after its own lock and before any hardware access, leaves its record as a stale one naming those runs; an unreadable state directory refuses, exit 2), `lockBypassed`, `probeUsers` (pids left on the board's probe, refused after the lock), `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`, and `receipts` when `state` is `pending`, `markErrors` when the pending mark could not be written), `jlinkRetried` (J-Link boards only: sessions run again after a crash before the target connect, each `{exit, commands}`; Limits), `route` (a run over a `tinyuf2.routes` route), `buildLocksKept` (each `{buildDir, survivors, keeper}`: a build dir whose lock a keeper holds until those children end), and `detail`; - in an install's evidence (`steps[0]`, `cleanup.evidence`, a manual `install`): `operation`, the provider's record of the flash, `{attempts: [{kind, exit}], cleanup, receipts}`, each attempt's `kind` `exited`, `timed_out`, `interrupted` or `error` and the operation's `cleanup` `done`, `pending` or `unknown` (`hil` Flashers), never the transcripts joined (they are in the run dir's `hil.log`); on an Espressif board also `esptool` (its version), and on the `openocd_esp` route `route`, `security` (the evidence passed to the provider) and `jtag` (openocd build, USJ bus-port, session log). A failed flash's step evidence carries `operation` and `beforeTarget`; - `cleanup.state` is `blinky copied`, `untouched`, `unknown` (touched, not restored), or `pending`: an `openocd_esp` session is not ended (Recovery, Cleanup pending), its receipt in `receipts` and in `survivors` (`{cmd, receipt}`, with `pid` and `start` when the receipt was readable; `receipt` is the provider's record itself when it could not be written), the board stays held and the build lock with a keeper; - `error` on a refusal or an operation failure. @@ -115,11 +135,26 @@ The last stdout line is JSON, also written to its `reportPath`. Its fields: | 4 | cleanup failed or is pending; the board is held when `cleanup.heldAfter` (not under `HIL_NO_BOARD_LOCK`). From `install` or `restore`: a child survived, an `openocd_esp` session is not ended, or the operation failed after it began writing; keep your hold | A delegated run (`hil-operator`) returns `{pass, results, caveat, wedged}` and, beside them, `reportPaths`: -- `reportPaths` maps each board to its JSON's `reportPath`, which a recovery dispatch needs. +- `reportPaths` maps each board to its JSON's `reportPath` (`null` with `reportError` when it was not published), which a recovery dispatch needs. - Across several boards (Execution: one battery per board), `results` holds every board's row, `pass` is true only when every run's is, and `caveat` joins the runs' non-empty caveats. - A launched run that ended without printing its JSON (killed, crashed) may have programmed the board and left it running a candidate. The operator writes `pass: false`, `caveat: "run ended without a report; board state unknown"` and one row per board, exactly `{board, ran: null, pass: false, locked: false, detail}`, with `detail` naming the exit status, the output tail and `python3 lib/agentrc/skills/hil/scripts/hil_lock.py status` at that moment. - The operator runs the battery once and returns. It never waits on a lock and never retries: a `locked` row comes back as is, and its caller retries under `hil`'s Caller rules, marking a flaky pass `flaky: true`. -- A recovery dispatch runs the `restore` Recovery names for the state (`--recover-holder` for a cleanup holder, `--under-hold` after a run with no report) in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Recovery). +- A recovery dispatch runs the `restore` Recovery names for the state (`--recover-holder` for a cleanup holder, `--recover-takeover` for a stale record or takeover journal, `--under-hold` after a run with no report that left none) in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Recovery). + +### Remote results + +`hil_remote.py` (Execution, Remote) prints one JSON line, also kept as `.hil-remote/.outcome.json`: `outcome` (`pass`, `fail`, `blocker`, `refused`, `incomplete`, or for an attempt not done `running`, `dead`, `not-launched`, `inconclusive`), `pass`, `collection` and `remoteState`, `remote` (`host`, the attempt's `dir` and `checkout` on the rig, and `local`), and per board `phase` (`preflight`, `build`, `battery` or `driver`), `verdict` and `detail` (Verdicts; a build refusal is `not-run`), `exit`, `heldInProcess`, `holder`, `reportError`, `unfinishedAttempts`, `sharedVerdict` and `sharedDetail` (as CI's, never the exit), `unsafeArtifacts` (views the rig could neither prepare nor withhold: removed from the local copy, never evidence, as is every view of a collection without a readable `result.json` and of a board whose row has no prepared report pair), and `paths`: `buildRecord`, `runRoot` and each run dir's `reports`, every one as `{remote, local}`; `driverFailure` when the driver itself failed or exited other than its rows say, which makes the outcome a blocker whatever the rows. The attempt comes home into `.hil-remote//`: `manifest.json`, `result.json`, `progress.json`, `out/` (each board's record and run root: `stdout.txt`, `stderr.txt` and the run dir with its report pair and `hil.log`), a failed build's `src/cmake-build/cmake-build-[-cmake]/check_build.log`, `.attempt/` (agentrc's launch and completion records and the driver's stdout and stderr) and agentrc's `collection.json`. Only the attempt's own paths are fetched, never one a report names. A file the driver's inventory lists, or the board's phase requires (a build's record; a battery's record, `stdout.txt`, `stderr.txt` and its published `hil_report.json`), that did not come home makes the board's `copyBack` `incomplete` and the outcome `incomplete`, even after a pass, the board's own verdict kept; a later `collect` fetches such an attempt again and keeps the new copy only once it is whole. Evidence collected before the attempt is done is `incomplete`, never a pass, and never replaces a complete collection. + +| Exit | Meaning | +|---|---| +| 0 | every board passed and came home whole | +| 1 | a board failed or was not run, a build refusal included | +| 2 | a blocker: failed cleanup, a cleanup holder, a run without a report, a recovery blocker, an unfinished CI attempt | +| 3 | still running at `--timeout`; wait or collect again | +| 4 | the attempt ended without its completion record (`dead`), or was never launched | +| 5 | inconclusive: the rig unreachable or its evidence incomplete, a transport failure after the launch included | +| 6 | incomplete copy-back | +| 7 | refused: here before staging, or on the rig before any build (`refused`); `error:` on stderr says why | ### Verdicts @@ -127,21 +162,32 @@ Each board's row maps onto the `hil` verdicts, except a commissioning run's (`co - `pass`: `ran`, `pass` and `cleanup.pass` all true; - `fail`: `ran` and not `pass`, with a failing step, `cleanup.state` other than `untouched`, and that step's `detail`; - `not-run`: `ran` false (`locked`, a refusal, or a failed probe check: not exactly one tty with the route's serial), or a failing step with `cleanup.state` `untouched`, with its `detail`. Nothing was written, so the failure is the rig's: no probe with that serial on this machine, or one unplugged or dead; -- a blocker, never a `fail` row: exit 4, `ran: null` (no report), a `locked` row with `recoveryBlocker`, a `probeUsers` row, and an abort, which is `ran` and not `pass` with no failing step (an interruption or an unexpected error, named in the board's `detail`). +- a blocker, never a `fail` row: exit 4, `ran: null` (no report), a `recoveryBlocker` row, a `probeUsers` row, and an abort, which is `ran` and not `pass` with no failing step (an interruption or an unexpected error, named in the board's `detail`). `testedHead` is the build record's `input.head`. `regression` is `unknown` unless the battery also ran on the merge base. ## Recovery -- A board whose run ended without a report gets a manual `restore --under-hold` (a recovery dispatch carrying the board and a record, as for exit 4) before its next run, and only once no such process is left on the host. Killing the runner frees its lock but not its children, which `restore` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: +- A board whose run ended without a report gets a manual `restore --under-hold` (a recovery dispatch carrying the board and a record, as for exit 4) before its next run, and only once no such process is left on the host; while its lock carries the dead battery's stale record, a fresh hold is refused and the stale-record route below runs instead. Killing the runner frees its lock but not its children, which `restore` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: - `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'; pgrep -af '[e]sp_session\.py'; pgrep -af '[u]sb_dfu\.py'` + `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'; pgrep -af '[e]sptool_session\.py'; pgrep -af '[u]sb_dfu\.py'` The `hil_test.py` check also covers the runner's `fat-read` subcommand. While a process is left, or this cannot be established, the blocker stays. +Stale record (a `locked` row with `recoveryBlocker` naming a stale record or a takeover journal, or a refused hold on the same): a battery, or a cleanup holder, died holding the board, or a run refused on an unfinished CI attempt. It is a separate recovery dispatch under `hil`'s Board locks scope, carrying the board and a record; after the host-wide check above prints nothing, it runs: + +1. `python3 lib/agentrc/skills/hil/scripts/hil_lock.py hold --take-over --reason ""`, which journals the stale record and holds the board as a cleanup holder, printing its token. +2. `python3 .claude/skills/hil-contract/scripts/hil_test.py restore -b --build-record --recover-takeover [--report ] `. It proves the token's live takeover holder covers this board alone and reads the record it replaced from the journal chain, which must be this runner's: a battery's protected record or a cleanup holder's of this board. What that run left must be known, none of it alive and every `openocd_esp` session ended, as for `--recover-holder`: + - A dead cleanup holder's run printed its report: `--report` is required, its row must name this board and that holder's pid, and its `cleanup.survivors` join the record's. On an Espressif board the restore checks the preserved regions against that report's snapshot, refused without one for a touched run or with another layout, exactly as `--recover-holder` does. + - A battery that died holding the board printed none, and `--report` is refused. Its record's `runDir` is listed for `openocd_esp` session dirs, each with its receipt; a run dir or session dir that cannot be listed, or a session dir without its receipt, refuses. On an Espressif board the restore takes a fresh snapshot, as after a run with no report: the preserved regions are proven unchanged only across the recovery itself, never across the failed run, so a region the dead run changed goes unseen; the recovery dispatch reports that limit with the board. + + It then restores the board under the takeover, as `restore --under-hold` does, and only then releases the token, verifying its records and journal are gone and the lock is free (exit 0, `cleanup.recovered`). + +A refusal (exit 2: another token, a takeover over other boards, a replaced record that is not this runner's or names neither survivors nor a run dir, a missing or mismatched report or snapshot, a run dir that cannot be listed, a child alive or a session not ended) and a failed restore (exit 4) leave the takeover holder, its journal and so the blocker in place: missing recovery evidence stays a blocker for the lead or the human, and `hil_lock.py release --token` is never run by hand to clear it. A session that is not ended, or a pending mark, is ended as Cleanup pending below says; after a separately reviewed recovery, `clear-pending -b --after-reviewed-recovery "" --under-hold "" ` proves the takeover holder as the board's hold (its reason is the takeover's), records it with the note and removes the mark, and step 2 then counts the mark's sessions as reviewed. A run refused on an unfinished CI attempt also leaves that attempt's marker for CI's own recovery (Execution, CI). + Exit 4, a cleanup holder: -- The board stays held by a detached holder forked from the run: reason `hil_test.py`, `role: cleanup-holder`, with its pid in `cleanup.holder`. It ignores SIGTERM and SIGINT, so a timeout kill cannot free the board; only recovery releases it. It re-executes itself without `RUNNER_TRACKING_ID`, the variable under which a GitHub Actions runner kills every process still alive when its job ends; a holder whose re-execution failed, or the in-process holder below, keeps it and dies with the job: the board is then unheld and not cleaned up, which `hil_ci.py`'s marker records (Execution, CI), and recovery is a `restore --under-hold` under a fresh hold, as for a run without a report. +- The board stays held by a detached holder forked from the run: reason `hil_test.py`, `role: cleanup-holder`, with its pid in `cleanup.holder`. It ignores SIGTERM and SIGINT, so a timeout kill cannot free the board; only recovery releases it. It re-executes itself without `RUNNER_TRACKING_ID`, the variable under which a GitHub Actions runner kills every process still alive when its job ends; a holder whose re-execution failed, or the in-process holder below, keeps it and dies with the job: the board is then unheld and not cleaned up, which `hil_ci.py`'s marker records (Execution, CI), and its record stays on the free lock as a stale one: recovery is the stale-record route above. - If no holder could be forked, `cleanup.holder.inProcess` is set and the runner itself is the holder: it prints the report, closes its stdio and does not exit until recovered, so no completion notice arrives. When a run's notice is overdue, read the task's output: a final JSON line with `cleanup.holder.inProcess` true is this case. Do not wait for its exit. - `cleanup.survivors` lists children that outlived SIGKILL (a J-Link, mtools or FAT read stuck on the device), each by pid and start time, and any `openocd_esp` session not ended (cleanup pending), with its receipt; recovery refuses while any child is still alive or any session is not ended. A session's receipt is reread on every check, never remembered: an unreadable one blocks until it reads `ended`. Each child runs in a session of its own and the kill reaches the whole session. No runner creates a container any more; a report from an older runner may still name one (`{"container": "tinyuf2-hil-"}` or `...-query`), which recovery counts as gone only once docker has listed it no more for 2 seconds, `docker rm -f` before each look, and never while a process of that report lives. - Cleanup pending: the runner never signals an openocd-esp32 server (a killed one can leave the chip in its Breakpoint boot loop), so a pending receipt is the lead's to resolve, never the recovery dispatch's. The dispatch reports the receipt and stops. A session's state, read from its receipt on every check, is `ended` (closed with cleanup `done` by the provider's end), `running` (open, its process alive by pid and start time), `unreadable`, or `unresolved` (closed without that end, or its process gone without closing it). The build lock's keeper exits once none is `running` or `unreadable`; everything else waits for `ended`. The lead runs `python3 lib/agentrc/skills/hil/scripts/hil_openocd_esp.py end --receipt [--timeout S]`, which ends the session only after proving its last command ran and its init was `ok`, by `reset run` then `shutdown`; never SIGTERM or SIGKILL the server. Exit 0 (`ended` or `alreadyEnded`) leaves the receipt closed with cleanup `done`, which discharges it from `survivors`, the keeper and the pending mark; then the recovery `restore` below runs, with the route's `--route`, and releases the holder, or, after a manual `install`, `restore --under-hold` and `hil_lock.py release` (Manual use). Exit 1 (refused or failed) keeps the receipt and the board held; 2 is a usage error. A receipt whose init failed or is unproven, one closed without the reviewed end, one whose process is gone without closing it, and a session whose receipt the provider could not write (a record with no path in `survivors`) need a separately reviewed recovery, the lead's or the human's to decide; none is defined here. A pending mark that could not be written (`markErrors`, in the row's `cleanup` or the manual report, exit 4) leaves the session unrecorded for later invocations: the board stays a blocker, held, until the lead ends or recovers it as above. @@ -151,7 +197,8 @@ Exit 4, a cleanup holder: `python3 .claude/skills/hil-contract/scripts/hil_test.py restore -b --build-record --recover-holder --report ` - `` is the failed report's `firmware.buildRecord`, or a record rebuilt on the clean HEAD when it no longer verifies. The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, restores the board while the lock stays held, and only then releases the holder. On an Espressif board the restore checks the preserved regions against the snapshot in the failed report, and recovery is refused for a touched run without one or one whose regions differ from the checkout's. A candidate whose drive cannot take the blinky copy fails the restore; the board stays held until a record that can restore it is given. A second recovery of the same board while one runs is refused (exit 2). + `` is the failed report's `firmware.buildRecord`, or a record rebuilt on the clean HEAD when it no longer verifies. `--report`, here and with `--recover-takeover`, is read through the report core before the recovery lock or any probe: a report without `_hil` (an earlier runner's) is read as it is, and one that is missing or corrupt or carries an unknown `_hil` or body version is refused (exit 2). The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, restores the board while the lock stays held, and only then releases the holder. On an Espressif board the restore checks the preserved regions against the snapshot in the failed report, and recovery is refused for a touched run without one or one whose regions differ from the checkout's. A candidate whose drive cannot take the blinky copy fails the restore; the board stays held until a record that can restore it is given. A second recovery of the same board while one runs is refused (exit 2). +- A remote attempt's board (Execution, Remote) is recovered by the same dispatches, run on ci.lan from the attempt, never through `hil_remote.py`: in the outcome's `remote.checkout`, with `paths.buildRecord.remote` as ``, a `paths.reports[].remote` as `` and `holder.pid` as ``, and `test/hil/ci.json` as ``; a stale record's `runDir` is that attempt's run dir. A later run never wipes an attempt; one is removed by hand only once its collection is complete and none of its boards is a blocker. - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide), and on imxrt1064_evk the same switch (SW7, MIMXRT1064-EVK schematic sheet 16), on the ci.lan rig, where tinyusb's roster also gives the board an OpenOCD recovery flasher; on espressif_esp32s3_devkitc_1, ROM download mode: hold Boot (SW1, GPIO0 low), press and release RST (SW2, CHIP_PU), then release Boot (DevKitC-1 schematic V1.0; the TRM's strap table also needs GPIO46 low, its default pull-down); on espressif_p4_function_ev, the same with BOOT (SW2, GPIO35 low) and RST (SW1, CHIP_PU) (Function-EV-Board schematic rev 1.4); on stm32h563_nucleo, a power cycle: unplug and replug its STLINK USB (CN1), which powers the board (bench-history.md). ## Limits @@ -162,6 +209,7 @@ Measurements and dated bench observations behind these limits, loader footprints - On ci.lan, imxrt1064_evk's probe (1-1.1) and its USB (1-1.2) share one hub, the farm's cabling, where commissioning.md asks for separate hubs: a device hung mid-write there could stall the probe connect too (Shared hubs below). Its silicon revision was not read from the marking (the rig is remote); the A and B errata were both checked. Whether its FlexSPI configuration observes ERR011377 (a delay after the DLL lock before the first flash access, IMXRT1064CE_A Rev 1.3 and IMXRT1064CE_B Rev 1.1) is unchecked: pending, not a found defect. Every install leaves [0x7000C000, 0x70010000) erased, an inference from one sample (bench-history.md, imxrt1064_evk), as on the RT1060 boards. - The `usbPower` port cycle (Hosts/rosters) runs under the board's own lock, a limitation of `hil`'s rig-wide rule, not an exemption. On the Port Authority hub (5-3) a cycle re-enumerated only its own port's device when measured; neither a VBUS drop nor an enumeration storm was measured (bench-history.md: Shared hubs). On a ganged hub (5-1, 5-2) a port cycle removes no power while another port in the gang is powered, and can power the whole gang off when it is the last, so `usbPower` must not name a port there. On the Port Authority hub's USB2514B a full- or low-speed device that detaches and re-attaches within 100 µs of SE0 may not register as a detach (USB251xB errata, module 4); stm32f303disco's missing disconnect is instead its board's fixed D+ pull-up, by design (bench-history.md: stm32f303disco). Never cycle a rig hub port with `uhubctl` or a raw hub request: a cycle that met an enumeration in progress killed an xHCI controller. Another board's enumeration errors on the same hub can reset a neighbour mid-read, and `install`'s FAT read then times out (exit 4) (bench-history.md: stm32h563_nucleo). A device hung mid-write stalled a J-Link connect on the same hub. - Espressif boards: the runner needs esptool 5.2.0, its `esptool` command an entry script of the runner's own Python, and pyusb with libusb-1.0 for that Python. The install runs that esptool, not esptool 5.3.1 in CI's image: the dialect differs (`default_reset`, `write_flash`, flash_args inline), and the S3 and P4 UART routes are hardware-pending until their bridges are attached. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The route's tty carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. The host log shows a USB reset and block-device read errors at the end of a UF2 copy, and on the P4 a full-speed device that never answers while an esptool session holds the chip; the battery passed with the latter. +- Remote runs (Execution, Remote): J-Link and ST-LINK boards only, since the rig has no ESP-IDF image selection yet; the Windows client is pending; attempts are never pruned (each holds the bundles, the rebuilt checkout and its build). The detached attempt outliving a real ssh session and the wrapper, and the imxrt1064_evk run on ci.lan, are tested only against a local stand-in rig until a hardware run. Older CI drivers of other projects gain no awareness of tinyuf2's CI markers. - Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. - cmake builds pass `TUF2_LOG=0` unless `LOG` is set, as make does, so the HIL image does not log on the board UART. - `install` from a blank board, or one whose ROM found no boot image, brings up DFU on imxrt1060_evk and metro_m7_1011; until a `loadbin` has run, a new J-Link session reads such a board's flash wrong. diff --git a/.claude/skills/hil-contract/scripts/esp_session.py b/.claude/skills/hil-contract/scripts/esp_session.py deleted file mode 100644 index 8bafc2826..000000000 --- a/.claude/skills/hil-contract/scripts/esp_session.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""One esptool connection for hil_test.py, run as its own child so the runner can time it out, kill it and -see it survive: connect into download mode, run the ops in order, then reset or not. - - esp_session.py --port DEV --chip CHIP --baud N --after hard-reset|no-reset OPS.json - -OPS.json is a list of {"op": "read", "addr", "size", "file"} and {"op": "md5", "addr", "size"}: no write, which -is the provider's flash. A {"op": "security"} first reads the chip's flash encryption and secure boot state on the -ROM loader, before the stub runs. -The last stdout line is JSON: {"ok": true, "results": [...]} or {"ok": false, "error", "results"}. -""" -import argparse -import json -import sys - -sys.dont_write_bytecode = True - - -def security(esp): - """The chip's flash encryption and secure boot state from two sources, both on the ROM loader: esptool 5.2.0's - GET_SECURITY_INFO (loader.py get_security_info: flags bit 0 SECURE_BOOT_EN, flash_crypt_cnt enabled when its count - of 1 bits is odd) and its eFuse register reads (targets/*.py get_flash_encryption_enabled, get_secure_boot_enabled; - on the P4 EFUSE_RD_REPEAT_DATA1_REG [20:18] SPI_BOOT_CRYPT_CNT and EFUSE_RD_REPEAT_DATA2_REG [20] SECURE_BOOT_EN).""" - if esp.IS_STUB: - raise RuntimeError('the chip already runs the flasher stub; its security state is read from the ROM loader') - si = esp.get_security_info(cache=False) - - def state(on): - return 'on' if on else 'off' - return {'flags': si['flags'], 'flashCryptCnt': si['flash_crypt_cnt'], - 'rom': {'flashEncryption': state(bin(si['flash_crypt_cnt']).count('1') % 2), - 'secureBoot': state(si['parsed_flags']['SECURE_BOOT_EN'])}, - 'efuse': {'flashEncryption': state(esp.get_flash_encryption_enabled()), - 'secureBoot': state(esp.get_secure_boot_enabled())}} - - -def run(esp, op): - kind = op['op'] - if kind == 'read': - data = esp.read_flash(op['addr'], op['size']) - with open(op['file'], 'wb') as f: - f.write(data) - return {'size': len(data)} - if kind == 'md5': - return {'md5': esp.flash_md5sum(op['addr'], op['size'])} - raise ValueError(f'unknown op {kind!r}') - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument('--port', required=True) - ap.add_argument('--chip', required=True) - ap.add_argument('--baud', type=int, required=True) - ap.add_argument('--after', choices=('hard-reset', 'no-reset'), required=True) - ap.add_argument('ops') - args = ap.parse_args() - results, esp = [], None - try: - import esptool - from esptool import cmds - ops = json.load(open(args.ops)) - if any(op['op'] == 'security' for op in ops[1:]): - raise ValueError('a security op comes first, before the stub runs') - esp = cmds.detect_chip(args.port, connect_mode='default-reset') - if esp.CHIP_NAME.lower().replace('-', '') != args.chip: - raise RuntimeError(f'the port reaches a {esp.CHIP_NAME}, not an {args.chip}') - if ops and ops[0]['op'] == 'security': - results.append(dict(security(esp), op='security')) - ops = ops[1:] - esp = cmds.run_stub(esp) - esp.change_baud(args.baud) - cmds.attach_flash(esp) - for op in ops: - results.append(dict(run(esp, op), op=op['op'])) - if args.after == 'hard-reset': - cmds.reset_chip(esp, 'hard-reset') - out = {'ok': True, 'esptool': esptool.__version__, 'results': results} - except Exception as e: # every failure is reported, never raised past the JSON line - out = {'ok': False, 'error': f'{type(e).__name__}: {e}', 'results': results} - finally: - if esp is not None: - esp._port.close() - print(json.dumps(out), flush=True) - return 0 if out['ok'] else 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/.claude/skills/hil-contract/scripts/hil_ci.py b/.claude/skills/hil-contract/scripts/hil_ci.py index d842035fd..1a52932c6 100644 --- a/.claude/skills/hil-contract/scripts/hil_ci.py +++ b/.claude/skills/hil-contract/scripts/hil_ci.py @@ -7,14 +7,18 @@ `--out` is this run's directory under a state directory kept across runs (`/`): the build record, each board's run directory (`hil_test.py run --run-root`: its report, hil.log and output) and the -`.started` / `.done` markers go there. A `.started` without its `.done` in any earlier run is a -run that ended without a report or kept a cleanup holder (SKILL.md Recovery): that board is a blocker here -until recovery removes the marker. Runs older than PRUNE_DAYS with every marker done are removed; one with a -marker left is kept, whatever its age. +`.started` / `.done` markers go there; the battery touches `.started` once it holds the board +(`run --started-marker`) and `.done` before it releases it (`run --done-marker`), so whoever takes the lock meanwhile +never sees a waiting battery started, and whoever takes it next sees it finished. A `.started` without its `.done` in +any earlier run is a run that ended without a report or kept a cleanup holder (SKILL.md Recovery): that board is a +blocker here until recovery removes the marker. Runs older than PRUNE_DAYS with every marker done are removed; +one with a marker left is kept, whatever its age. Each battery's report pair is prepared for the upload (prepare()), and +summary.json carries the shared run rules' `sharedVerdict` beside each native `verdict`, which alone decides the exit. +`upload-views`, written last, is the upload's decision on the report views (UPLOAD_VIEWS). Exit 0: every board passed. 1: a board failed or was not run (held past `--lock-wait`, its build or admission -refused). 2: a blocker (a run without a report, a failed cleanup, a cleanup holder or a stale probe session on -the board). +refused). 2: a blocker (a run without a report, a failed cleanup, a cleanup holder, a dead battery's record or a +takeover journal, or a stale probe session on the board). """ import argparse import json @@ -25,9 +29,19 @@ import time from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.dont_write_bytecode = True # as hil_test.py: nothing CI imports writes into the checkout it builds +import hil_run_report as run_report # noqa: E402 + ROOT = Path(__file__).resolve().parents[4] CHECK_BUILD = ROOT / '.claude' / 'skills' / 'build' / 'scripts' / 'check_build.py' HIL_TEST = Path(__file__).resolve().with_name('hil_test.py') +# the pinned report core, loaded only once check_pin() passed +REPORT_CORE = ROOT / 'lib' / 'agentrc' / 'skills' / 'hil' / 'scripts' / 'hil_report_core.py' +UNPREPARED = 'report not prepared' +NOTICE = '**HIL report unavailable' # the core's NOTICE, which this must write when the core itself failed +# `upload` when no view is unsafe; anything else, or no file, and the workflow uploads no hil_report.md at all +UPLOAD_VIEWS = 'upload-views' STEPS = ('install', 'dfu', 'copy', 'app-ran', 'reenter', 'recopy', 'cleanup') # hil_test.py --fail-at LOCK_WAIT = 600 PRUNE_DAYS = 30 @@ -45,8 +59,20 @@ def roster_boards(roster): def unfinished(state, board): - """Earlier runs under `state` whose battery on `board` started and never finished.""" - return sorted(p.parent.name for p in state.glob(f'*/{board}.started') if not p.with_suffix('.done').exists()) + """The runs under `state` (`//.started`) whose battery on `board` started and has no `.done`, + its liveness never guessed; a linked run dir is skipped. OSError when `state` or a run in it cannot be listed.""" + found = [] + with os.scandir(state) as runs: + for run in runs: + if not run.is_dir(follow_symlinks=False): + continue + try: + names = set(os.listdir(run.path)) + except FileNotFoundError: # pruned meanwhile: prune() removes only runs whose every battery finished + continue + if f'{board}.started' in names and f'{board}.done' not in names: + found.append(run.name) + return sorted(found) def prune(state, keep, days=PRUNE_DAYS): @@ -72,19 +98,15 @@ def build_record(board, roster, out): return record, ok, why -def battery(board, roster, record, where, lock_wait, fail_at=None): - """One `hil_test.py run` with its run directory under `where`, its output streamed to `where/stdout.txt` as - it comes, so a killed run keeps what it printed; its last JSON line is the report (SKILL.md Results). A runner - left holding the board in process (`cleanup.holder.inProcess`, Recovery) never exits: it is not waited for. - Returns (exit or None, report or None).""" - where.mkdir(parents=True) - cmd = [sys.executable, HIL_TEST, 'run', '-b', board, '--build-record', record, '--run-root', where, - '--lock-wait', lock_wait] - if fail_at: - cmd += ['--fail-at', fail_at] +def launch(cmd, where, board): + """One `hil_test.py run` of `board` (`cmd`, its argv), its output streamed to `where/stdout.txt` as it comes, so a + killed run keeps what it printed, and its stderr to `where/stderr.txt`; its last JSON line is the report (SKILL.md + Results). A runner that became the in-process cleanup holder (`cleanup.holder.inProcess`, Recovery) printed its + report, closed its stdio and never exits: it is left running, not waited for. Returns (report or None, exit or None, + held), `held` true for that live holder, whose exit is None.""" report = None with open(where / 'stderr.txt', 'w') as err, open(where / 'stdout.txt', 'w') as log: - p = subprocess.Popen([str(c) for c in cmd + [roster]], cwd=ROOT, stdout=subprocess.PIPE, stderr=err, text=True) + p = subprocess.Popen([str(c) for c in cmd], cwd=ROOT, stdout=subprocess.PIPE, stderr=err, text=True) for line in p.stdout: log.write(line) log.flush() @@ -94,56 +116,150 @@ def battery(board, roster, record, where, lock_wait, fail_at=None): except ValueError: pass p.stdout.close() - row = next((r for r in (report or {}).get('results', []) if r.get('board') == board), {}) - in_process = ((row.get('cleanup') or {}).get('holder') or {}).get('inProcess') - return (None if in_process else p.wait()), report - - -def verdict(code, report, board): - """The `hil` verdict of one run from the report's row (SKILL.md Verdicts): (verdict, detail).""" - if report is None: - return 'blocker', f'run ended without a report (exit {code})' - row = next((r for r in report.get('results', []) if r.get('board') == board), None) - if row is None: - return 'blocker', f'report has no row for {board} (exit {code}): {report.get("caveat") or report.get("error")}' - cleanup = row.get('cleanup') or {} - if code in (4, None) or row.get('ran') is None or cleanup.get('heldAfter'): - return 'blocker', f'cleanup failed or no report; board held: {row.get("detail")} {cleanup}'[:400] - if row.get('probeUsers'): - return 'blocker', str(row.get('detail')) - if row.get('locked'): - return ('blocker' if row.get('recoveryBlocker') else 'not-run'), f'locked: {row.get("detail")}' - if not row.get('ran'): - return 'not-run', str(row.get('detail') or report.get('error')) - failed = next((s for s in row.get('steps', []) if not s.get('pass')), None) - if row.get('pass') and cleanup.get('pass'): - return 'pass', '' - if failed is None: - return 'blocker', f'aborted: {row.get("detail")}' - if cleanup.get('state') == 'untouched': - return 'not-run', f'{failed["step"]}: {failed.get("detail")} (nothing written)' - return 'fail', f'{failed["step"]}: {failed.get("detail")}' - - -def run_board(board, roster, out, state, lock_wait, fail_at=None): - earlier = unfinished(state, board) + row = run_report.row_of(report, board) or {} + if ((row.get('cleanup') or {}).get('holder') or {}).get('inProcess') and p.poll() is None: + return report, None, True + return report, p.wait(), False + + +def battery(board, roster, record, where, lock_wait, fail_at=None, ci_state_root=None, done=None, started=None): + """One `hil_test.py run` with its run directory under `where`, `ci_state_root` a remote driver's (its + --ci-state-root), `done` and `started` CI's markers (its --done-marker, --started-marker). Returns launch()'s (report + or None, exit or None, held).""" + where.mkdir(parents=True) + cmd = [sys.executable, HIL_TEST, 'run', '-b', board, '--build-record', record, '--run-root', where, + '--lock-wait', lock_wait] + if ci_state_root is not None: + cmd += ['--ci-state-root', ci_state_root] + if done is not None: + cmd += ['--done-marker', done] + if started is not None: + cmd += ['--started-marker', started] + if fail_at: + cmd += ['--fail-at', fail_at] + return launch(cmd + [roster], where, board) + + +def check_pin(): + """check_build's lib/agentrc pin check, raising its Fail: a battery that ran never proved the pin (its own check + may have refused it).""" + run_report.load(CHECK_BUILD, 'tinyuf2_check_build').check_agentrc_pin() + + +def run_dirs(where): + """The `run-*` entries under `where`, sorted; OSError when it cannot be listed, which pathlib's glob would hide.""" + with os.scandir(where) as entries: + return sorted(Path(e.path) for e in entries if e.name.startswith('run-')) + + +def prepare(code, where, board): + """Each run dir's report pair under `where` made consistent before the artifact exposes it (the core's + prepare_pair: the Markdown regenerated from the JSON, else an explicit unavailable notice, never a stale view), and + the shared run rules' verdict of its JSON (run_report.shared_run), kept beside the native one.""" + try: + check_pin() # before any provider code runs + core = run_report.load(REPORT_CORE) + schema = run_report.schema(core) + readings = [core.prepare_pair(d, schema) for d in run_dirs(where) if d.is_dir()] + reading = readings[-1] if readings else core.Reading('missing', None, None, f'no run directory under {where}') + shared = run_report.shared_run(core, code, reading, [board])['rows'][0] + return {'sharedVerdict': shared['verdict'], 'sharedDetail': shared['detail']} + except Exception as e: # the battery's exit, markers and any holder stand: only this board's shared view is lost + why = f'{UNPREPARED}: {type(e).__name__}: {e}' + unsafe = withhold(where, why) + return {'sharedVerdict': 'blocker', 'sharedDetail': why, **({'unsafeArtifacts': unsafe} if unsafe else {})} + + +def withhold(where, why): + """Each run dir's Markdown under `where` replaced, under the core's writer lock, by an unavailable notice, the view + the run left kept below it only as labelled text, or removed when that cannot be written. Returns the views that + could be neither, or whose lock was not taken (never touched outside it), which no upload or copy-back may expose + (summary.json `unsafeArtifacts`): `where` itself when it cannot be listed.""" + import fcntl # here only: hil_remote.py imports this module on clients without it + try: + runs = run_dirs(where) + except OSError: + return [str(where)] + unsafe = [] + for d in runs: + md = d / f'{run_report.STEM}.md' + try: + fd = os.open(d / f'.{run_report.STEM}.lock', os.O_RDWR | os.O_CREAT, 0o666) + except OSError: + unsafe.append(str(md)) + continue + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + old = md.read_text(encoding='utf-8', errors='replace') if md.is_file() else '' + text = f'{NOTICE}: {run_report.STEM}.json was not checked ({why}).** No verdict is read from this view.\n' + if old.strip(): + text += '\n_Legacy text: the view the run left, never checked against its JSON._\n\n' + old + tmp = d / f'.{run_report.STEM}.md.withheld' + tmp.write_text(text, encoding='utf-8') + os.replace(tmp, md) + except OSError: + try: + md.unlink(missing_ok=True) + except OSError: + unsafe.append(str(md)) + except OSError: # another writer holds it + unsafe.append(str(md)) + finally: + os.close(fd) + return unsafe + + +def run_board(board, roster, out, state, lock_wait, fail_at=None, markers=True): + """One board's row: CI's markers under `state`, the record, the battery, its verdict and its prepared report pair. + `markers` false is a remote driver's: `state` is CI's state directory, read, never written, and checked again by + the runner under its lock (--ci-state-root). `lock_wait` is seconds, or a function giving them as the battery + starts.""" + try: + earlier = unfinished(state, board) + except OSError as e: + return {'board': board, 'phase': 'preflight', 'verdict': 'blocker', + 'detail': f'state directory {state} cannot be read: {e}'} if earlier: - return {'board': board, 'verdict': 'blocker', - 'detail': f'run {", ".join(earlier)} ended without finishing: recover the board, then remove its marker'} + return {'board': board, 'phase': 'preflight', 'verdict': 'blocker', 'unfinishedAttempts': earlier, + 'detail': f'run {", ".join(earlier)} under {state} started a battery on {board} and never finished: ' + 'recover the board, then remove its marker'} record, ok, why = build_record(board, roster, out) if not ok: - return {'board': board, 'verdict': 'not-run', 'detail': why} - (out / f'{board}.started').touch() - code, report = battery(board, roster, record, out / board, lock_wait, fail_at) - v, detail = verdict(code, report, board) - if v != 'blocker': - (out / f'{board}.done').touch() - return {'board': board, 'verdict': v, 'detail': detail, 'run': str(out / board)} + return {'board': board, 'phase': 'build', 'verdict': 'not-run', 'detail': why} + done, started = (out / f'{board}.done', out / f'{board}.started') if markers else (None, None) + wait = lock_wait() if callable(lock_wait) else lock_wait + report, code, held = battery(board, roster, record, out / board, wait, fail_at, None if markers else state, done, + started) + v, detail = run_report.verdict(code, report, board) + if done is not None and v == 'blocker': # the battery's own touch stands only where this verdict agrees + done.unlink(missing_ok=True) + elif done is not None and started.exists(): # a battery that never held the board leaves no marker + done.touch() + row = {'board': board, 'phase': 'battery', 'verdict': v, 'detail': detail, 'run': str(out / board), 'exit': code, + 'heldInProcess': held, 'report': report is not None} + holder = ((run_report.row_of(report, board) or {}).get('cleanup') or {}).get('holder') + if holder: + row['holder'] = holder + if (report or {}).get('reportError'): # its run dir's report is missing or incomplete; stdout.txt keeps the JSON + row['reportError'] = report['reportError'] + row.update(prepare(code, out / board, board)) + return row def summary(rows, roster): - lines = [f'HIL on `{roster}`', '', '| board | verdict | detail |', '|---|---|---|'] - lines += [f'| {r["board"]} | {r["verdict"]} | {r["detail"].replace("|", "/")} |' for r in rows] + """The rows as a table: the native verdict, which decides the exit, and the shared run rules' verdict where it + differs or the report could not be prepared.""" + def cell(text): + return ' '.join(text.replace('|', '/').splitlines()) + + def shared(r): + if 'sharedVerdict' not in r or (r['sharedVerdict'] == r['verdict'] and not r['sharedDetail'].startswith(UNPREPARED)): + return '' + kept = f' (not uploaded: {", ".join(r["unsafeArtifacts"])})' if r.get('unsafeArtifacts') else '' + return cell(f'{r["sharedVerdict"]}: {r["sharedDetail"]}{kept}') + lines = [f'HIL on `{roster}`', '', '| board | native verdict | detail | shared run rules |', '|---|---|---|---|'] + lines += [f'| {r["board"]} | {r["verdict"]} | {cell(r["detail"])} | {shared(r)} |' for r in rows] return '\n'.join(lines) + '\n' @@ -160,6 +276,7 @@ def main(argv=None): if unknown: ap.error(f'not in {args.roster}: {unknown}') args.out.mkdir(parents=True, exist_ok=True) + (args.out / UPLOAD_VIEWS).unlink(missing_ok=True) state = args.out.parent prune(state, keep=args.out) rows = [run_board(b, args.roster, args.out, state, args.lock_wait, args.fail_at) for b in boards] @@ -169,8 +286,16 @@ def main(argv=None): with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as f: f.write(text) (args.out / 'summary.json').write_text(json.dumps({'boards': rows}, indent=1)) - verdicts = {r['verdict'] for r in rows} - return 2 if 'blocker' in verdicts else 1 if verdicts & {'fail', 'not-run'} else 0 + decide_views(args.out, rows) + return run_report.fold_exit(r['verdict'] for r in rows) + + +def decide_views(out, rows): + """The upload's decision, written last and atomically: any unsafe view withholds every one; the JSON and logs go + regardless.""" + tmp = out / f'.{UPLOAD_VIEWS}.tmp' + tmp.write_text('withhold\n' if any(r.get('unsafeArtifacts') for r in rows) else 'upload\n') + os.replace(tmp, out / UPLOAD_VIEWS) if __name__ == '__main__': diff --git a/.claude/skills/hil-contract/scripts/hil_remote.py b/.claude/skills/hil-contract/scripts/hil_remote.py new file mode 100644 index 000000000..7fb227cc8 --- /dev/null +++ b/.claude/skills/hil-contract/scripts/hil_remote.py @@ -0,0 +1,507 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Run TinyUF2's HIL battery on the ci.lan rig from a dev PC, built there from this checkout's committed identity +(SKILL.md Execution, Remote). + + hil_remote.py run -b BOARD [-b BOARD ...] [--run-id ID] [--host HOST] [--lock-wait S] [--fail-at STEP] + hil_remote.py wait RUN_ID [--timeout S] + hil_remote.py collect RUN_ID + +`run` freezes HEAD and every repository it pins -- the submodules and the get_deps dependencies of the boards' ports, +unpushed commits and a candidate lib/agentrc pin included -- as self-contained git bundles bound by a manifest +(commits, roster digest, boards, bundle digests), refusing a dirty, off-pin or missing input or one that moved while +frozen. It then creates the create-only attempt / on the rig, stages and verifies the bundles there, +launches hil_remote_driver.py detached in it (the rig rebuilds and checks the checkout, builds each board's record +with check_build.py --hil-config and runs `hil_test.py run` per board, one after another), waits for the attempt's +completion record and collects it into .hil-remote//. `wait` reconnects and does the same within --timeout; +`collect` fetches the attempt whatever its state. Neither ever launches it: an attempt runs once. Ending this process +ends nothing on the rig. + +The transport is agentrc's hil_remote_core, loaded from this checkout's lib/agentrc at its pin. stdout is one JSON +line; progress goes to stderr. + +Exit 0 every board passed and came home whole; 1 a board failed or was not run; 2 a blocker; 3 still running at the +timeout; 4 the attempt ended without completing, or was never launched; 5 inconclusive (the rig unreachable or its +evidence incomplete); 6 an incomplete copy-back; 7 refused (nothing launched, unless the message says it may have). +""" +import argparse +import contextlib +import hashlib +import json +import os +import re +import secrets +import shutil +import subprocess +import sys +import time +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +sys.dont_write_bytecode = True +import hil_ci # noqa: E402 STEPS, the battery's --fail-at steps +import hil_remote_driver as driver # noqa: E402 the rig side's names: manifest schema, result, checkout, out +import hil_run_report as run_report # noqa: E402 the loader, the verdict fold + +ROOT = HERE.parents[3] +AGENTRC = 'lib/agentrc' +CHECK_BUILD = HERE.parents[1] / 'build' / 'scripts' / 'check_build.py' +DRIVER = Path(driver.__file__).resolve().relative_to(ROOT).as_posix() +RUNS = '.hil-remote' +# the rigs a remote run may use: the roster that host's battery takes, the attempts' base dir, and CI's state root +# for the runner's transition check (hil.yml: $(dirname $GITHUB_WORKSPACE)/hil-state, the runner's account on ci.lan) +HOSTS = {'ci.lan': {'hostname': 'ci', 'roster': 'test/hil/ci.json', 'base': '~/hil-remote/tinyuf2', + 'ciStateRoot': '~/actions-runner-tinyuf2/_work/tinyuf2/hil-state'}} +LOCK_WAIT_MAX = 3600 +INCOMPLETE, REFUSED = 6, 7 +MANUAL = ('install', 'restore', 'clear-pending', 'fat-read', 'keep-held', 'keep-build') +# hil_test.py's manual, recovery and commissioning options and the overrides a remote run never takes +REFUSED_FLAGS = ('--under-hold', '--recover-holder', '--recover-takeover', '--report', '--try-reentry', '--route', + '--build-record', '--run-root', '--ci-state-root', '--done-marker', '--started-marker', + '--after-reviewed-recovery') + +core = None # the provider, loaded by provider() only once check_pin() passed + + +def fail(why): + """core.fail once the core is loaded (RemoteError, a SystemExit), a plain SystemExit before.""" + raise SystemExit(f'error: {why}') if core is None else core.RemoteError(why) + + +def git(where, *argv, binary=False): + r = subprocess.run(['git', '-C', str(where), *argv], capture_output=True, text=not binary) + if r.returncode: + err = r.stderr.decode(errors='replace') if binary else r.stderr + fail(f'git -C {where} {" ".join(argv)}: {err.strip()}') + return r.stdout if binary else r.stdout.strip() + + +def load_core(): + """hil_remote_core from this checkout's lib/agentrc, never an installed agentrc; main() checks the pin first.""" + path = HERE.parents[3] / AGENTRC / 'skills' / 'hil' / 'scripts' / 'hil_remote_core.py' + if not path.is_file(): + fail(f'{path} is missing: git submodule update --init {AGENTRC}') + return run_report.load(path, 'tinyuf2_hil_remote_core') + + +def check_pin(): + """Refuse unless lib/agentrc is its own checkout, clean at the commit HEAD pins (check_build's check): never + another provider.""" + cb = check_build() + try: + cb.check_agentrc_pin() + except cb.Fail as e: + fail(str(e)) + + +POLL = None # seconds between status polls, the core's own when None + + +def provider(): + """The remote core, loaded once: no provider code runs before main() has checked its pin.""" + global core + if core is None: + core = load_core() + return core + + +_cb = None + + +def check_build(): + """The build contract's module, for the checkout identity its build records and the runner check.""" + global _cb + if _cb is None: + _cb = run_report.load(CHECK_BUILD, 'tinyuf2_check_build') + return _cb + + +# --- inputs -------------------------------------------------------------------------------------------------------- +def select(boards, roster, roster_path): + entries = {b.get('name'): b for b in roster.get('boards', [])} + unknown = sorted(set(boards) - set(entries)) + if unknown: + fail(f'not in {roster_path} at HEAD: {unknown}') + esp = sorted(b for b in boards if entries[b].get('flasher', {}).get('name') == 'esptool') + if esp: + fail(f'{esp}: Espressif boards are not run remotely yet (no ESP-IDF image selection on the rig)') + return sorted(set(boards)) + + +def identity(boards): + """check_build's identity of this checkout for the boards' ports, refused unless clean, every submodule at the + commit HEAD pins and every dependency its own checkout at its get_deps pin.""" + cb = check_build() + try: + info = cb.identity({cb.port_of(b) for b in boards}) + except (cb.Fail, OSError) as e: + fail(f'cannot establish the checkout identity: {e}') + bad = [f'{k}: {info[k]}' for k in ('dirtyInputs', 'missingInputs', 'depsOffPin') if info[k]] + if info['dirty']: + bad.insert(0, f'changed or untracked: {git(ROOT, "status", "--porcelain").splitlines()[:5]}') + bad += [f'{p} is at {sha[:12]}, HEAD pins {pin[:12]}' for p, sha in info['submodules'].items() + if (pin := git(ROOT, 'rev-parse', f'HEAD:{p}')) != sha] + if bad: + fail(f'the inputs are not HEAD\'s committed identity ({"; ".join(bad)}): commit, or check out the pins, first') + return info + + +def shallow_marks(where, commit): + """The shallow boundaries under commit in a shallow repository: what the rig's fetch needs to accept its bundle.""" + path = Path(where) / git(where, 'rev-parse', '--git-path', 'shallow') + if not path.is_file(): + return [] + return sorted(set(path.read_text().split()) & set(git(where, 'rev-list', commit).split())) + + +def freeze(stage, run_id, host, cfg, boards, args): + """Bundle every repository into stage with the driver and a manifest binding them; refused when an input moved + meanwhile. Returns the manifest.""" + info = identity(boards) + (stage / 'bundles').mkdir(parents=True) + repos = [] + for i, (path, commit) in enumerate([('.', info['head']), *sorted(info['submodules'].items()), + *sorted(info['deps'].items())]): + where, bundle = ROOT / path, f'bundles/{i}.bundle' + git(where, 'bundle', 'create', '-q', str(stage / bundle), 'HEAD') + if git(where, 'bundle', 'list-heads', str(stage / bundle)) != f'{commit} HEAD': + fail(f'the bundle of {path} is not {commit} alone: it moved while it was frozen') + repos.append({'path': path, 'commit': commit, 'bundle': bundle, 'sha256': core.digest(stage / bundle), + 'shallow': shallow_marks(where, commit)}) + (stage / 'driver').mkdir() + (stage / 'driver' / Path(DRIVER).name).write_bytes(git(ROOT, 'cat-file', 'blob', f'HEAD:{DRIVER}', binary=True)) + roster = git(ROOT, 'cat-file', 'blob', f'HEAD:{cfg["roster"]}', binary=True) + manifest = {'schema': driver.SCHEMA, 'version': driver.VERSION, 'runId': run_id, 'host': host, + 'hostname': cfg['hostname'], 'head': info['head'], 'repos': repos, 'submodules': info['submodules'], + 'deps': info['deps'], 'roster': {'path': cfg['roster'], 'sha256': hashlib.sha256(roster).hexdigest()}, + 'boards': boards, 'lockWait': args.lock_wait, 'failAt': args.fail_at, 'ciStateRoot': cfg['ciStateRoot'], + 'driver': {'path': DRIVER, 'sha256': core.digest(stage / 'driver' / Path(DRIVER).name)}, + 'createdAt': time.time()} + core.write_json(stage / 'manifest.json', manifest) + if identity(boards) != info: + fail('the checkout moved while it was frozen; nothing was sent: run again') + return manifest + + +# --- the attempt --------------------------------------------------------------------------------------------------- +def members(manifest): + """The copy-back: one group (the manifest, the driver's result and every board's records, run dirs, reports, logs + and output under out/), and optional evidence: the driver's progress and a failed build's log, at the build dir + check_build.py names for either backend.""" + logs = [f'{driver.CHECKOUT}/cmake-build/cmake-build-{b}{s}/check_build.log' for b in manifest['boards'] for s in ('', '-cmake')] + return [['manifest.json', driver.RESULT, f'{driver.OUT}/']], [driver.PROGRESS, *logs] + + +def local_manifest(run_id): + return core.attempt_record(ROOT / RUNS, run_id).with_name(f'{run_id}.manifest.json') + + +def load_manifest(run_id): + try: + return json.loads(local_manifest(run_id).read_text()) + except (OSError, ValueError) as e: + fail(f'the manifest of run {run_id} is unreadable: {e}') + + +def emit(data, code): + print(json.dumps(data), flush=True) + return code + + +def where(tree, manifest): + return {'host': tree.host, 'dir': tree.dir, 'checkout': f'{tree.dir}/{driver.CHECKOUT}', + 'local': f'{RUNS}/{manifest["runId"]}'} + + +def wait_for(tree, manifest, timeout=None): + """Poll until the attempt completes and collect it (report()); with no timeout keep polling while it runs.""" + while True: + groups, optional = members(manifest) + code, st = core.wait_attempt(tree, core.WAIT_SECS if timeout is None else timeout, poll=POLL or core.POLL_SECS, + dest=ROOT / RUNS / manifest['runId'], groups=groups, optional=optional, + accept=acceptance(tree, manifest)) + if code == 0: + return report(tree, manifest, st['collection']) + if code != core.RUNNING or timeout is not None: + outcome = {core.RUNNING: 'running', core.DEAD: st['state'], core.INCONCLUSIVE: 'inconclusive'}[code] + return emit({'runId': manifest['runId'], 'outcome': outcome, 'pass': False, 'state': st['state'], + 'remote': where(tree, manifest), **({'detail': st['detail']} if st.get('detail') else {})}, code) + print(f'==> {manifest["runId"]} still running on {tree.host}', file=sys.stderr) + + +def canonical(rel): + """Whether `rel` is a relative path the core's check_relative takes as it is spelt.""" + try: + return isinstance(rel, str) and core.check_relative(rel, 'an inventory path') == rel + except core.RemoteError: + return False + + +def unsafe_views(row, board): + """The board's report views the rig could neither prepare nor withhold (`unsafeArtifacts`), None when that list + names anything else.""" + views = row.get('unsafeArtifacts', []) + view = re.compile(rf'{re.escape(driver.OUT)}/{re.escape(board)}/run-[^/]+/{run_report.STEM}\.md') + ok = isinstance(views, list) and all(canonical(v) and view.fullmatch(v) for v in views) + return views if ok else None + + +def artifacts_missing(row, board, dest): + """What the board's phase requires, and the rig's inventory of what it left, that did not come home. An inventory + path is checked to lie under the board's own out/ entries, never followed elsewhere; an unsafe view is not + required (withhold_unchecked() removes it).""" + need = {'build': [f'{driver.OUT}/{board}.build.json'], + 'battery': [f'{driver.OUT}/{board}.build.json', f'{driver.OUT}/{board}/stdout.txt', + f'{driver.OUT}/{board}/stderr.txt']}.get(row.get('phase'), []) + listed = row.get('artifacts') + if not isinstance(listed, list): + return ['(no artifact inventory)'] + own = (f'{driver.OUT}/{board}.build.json', f'{driver.OUT}/{board}/') + bad = [rel for rel in listed if not canonical(rel) or not (rel == own[0] or rel.startswith(own[1]))] + if bad: + return [f'(inventory entries outside {board}\'s artifacts: {bad[:3]!r})'] + unsafe = unsafe_views(row, board) + if unsafe is None: + return [f'(unsafeArtifacts outside {board}\'s report views: {row.get("unsafeArtifacts")!r:.200})'] + lost = [p for p in dict.fromkeys([*need, *listed]) + if p not in unsafe and (not (dest / p).is_file() or (dest / p).is_symlink())] + report_json = re.compile(rf'{re.escape(own[1])}run-[^/]+/hil_report\.json') + if row.get('phase') == 'battery' and not any(report_json.fullmatch(p) for p in listed): + lost.append(f'{own[1]}run-*/hil_report.json (never published{": " + row["reportError"] if row.get("reportError") else ""})') + return lost + + +def read_result(dest, run_id): + """The driver's result.json in dest, None when missing, unreadable or another run's.""" + try: + result = json.loads((dest / driver.RESULT).read_text()) + except (OSError, ValueError): + return None + return result if isinstance(result, dict) and result.get('runId') == run_id else None + + +def acceptance(tree, manifest): + """The core's completeness predicate over a collected tree, run under its collector lock before it publishes a + fetch and on a cached collection: the tree's unchecked views removed first (withhold_unchecked()), then each board's + required artifacts the copy-back lacks, so a collection short of them is never final and a later collect fetches + it again.""" + def missing(path): + path = Path(path) + result = read_result(path, manifest['runId']) + boards = [] if result is None or result.get('refused') else board_rows(tree, manifest, path, result) + withhold_unchecked(path, boards) + if result is None: # never a final collection: a later collect fetches it again + return [f'{driver.RESULT} missing, unreadable or not run {manifest["runId"]}\'s'] + return [f'{r["board"]}: {m}' for r in boards for m in r.get('missing', [])] + return missing + + +def withhold_unchecked(path, boards): + """Each report view under a collected tree's out/ that the rig did not check removed before the tree is published, + never here as evidence: a board's `unsafeArtifacts`, and every view of a board whose row has no prepared report pair + (no `sharedVerdict`: no row, or no result at all) or an unreadable `unsafeArtifacts`, as CI's upload excludes every + view when hil_ci.py did not finish. A removal that fails refuses the publication, the cached collection kept as it + was.""" + checked = {r['board'] for r in boards if 'sharedVerdict' in r and unsafe_views(r, r['board']) is not None} + out = path / driver.OUT + views = {p.relative_to(path).as_posix(): p.relative_to(out).parts[0] + for p in out.glob(f'*/run-*/{run_report.STEM}.md')} + drop = {*(rel for r in boards for rel in unsafe_views(r, r['board']) or []), + *(rel for rel, board in views.items() if board not in checked)} + for rel in sorted(drop): + try: + (path / rel).unlink(missing_ok=True) + except OSError as e: + fail(f'cannot remove {rel}, a report view the rig could not check, from {path}: {e}') + + +def board_rows(tree, manifest, dest, result): + """Each requested board's row of the result, with its copy-back state and paths.""" + rows = {r.get('board'): r for r in result.get('boards', []) if isinstance(r, dict)} + boards = [] + for b in manifest['boards']: + row = rows.get(b) + if row is None: # the driver ended before this board: an error after the checkout may have left a battery + boards.append({'board': b, 'verdict': 'blocker' if result.get('error') else 'incomplete', + 'detail': result.get('error') or 'no row in the result'}) + continue + got = {k: row.get(k) for k in ('board', 'phase', 'verdict', 'detail', 'exit', 'heldInProcess', 'holder', + 'reportError', 'unfinishedAttempts', 'sharedVerdict', 'sharedDetail', + 'unsafeArtifacts') if row.get(k) is not None} + lost = artifacts_missing(row, b, dest) + if lost: + got.update(copyBack='incomplete', missing=lost) + boards.append(dict(got, paths=board_paths(tree, b, dest))) + return boards + + +def board_paths(tree, board, dest): + """Each of a board's evidence paths on the rig, as recovery names them, beside its local copy.""" + def pair(rel): + return {'remote': f'{tree.dir}/{rel}', 'local': f'{dest.relative_to(ROOT).as_posix()}/{rel}'} + paths = {'buildRecord': pair(f'{driver.OUT}/{board}.build.json'), 'runRoot': pair(f'{driver.OUT}/{board}')} + reports = sorted(p.relative_to(dest).as_posix() for p in (dest / driver.OUT / board).glob('run-*/hil_report.json') + if not p.is_symlink()) + if reports: + paths['reports'] = [pair(r) for r in reports] + return paths + + +def report(tree, manifest, collection): + """The attempt's outcome from its collected copy: each requested board's verdict, the paths recovery needs, and + an incomplete copy-back named as such whatever the battery said.""" + run_id = manifest['runId'] + dest = ROOT / RUNS / run_id + out = {'runId': run_id, 'remote': where(tree, manifest), 'collection': collection['state'], + 'remoteState': collection['remoteState']} + result = read_result(dest, run_id) + if collection['remoteState'] != 'done' or result is None: + missing = collection.get('missing') or [[driver.RESULT]] + code = {'running': core.RUNNING, 'dead': core.DEAD, 'not-launched': core.DEAD, + 'inconclusive': core.INCONCLUSIVE}.get(collection['remoteState'], INCOMPLETE) + return finish(dest, {**out, 'outcome': 'incomplete', 'pass': False, 'missing': missing}, code) + if result.get('refused'): + return finish(dest, {**out, 'outcome': 'refused', 'pass': False, 'detail': result['refused']}, REFUSED) + boards = board_rows(tree, manifest, dest, result) + verdicts = {r['verdict'] for r in boards} + ran = run_report.fold_exit(verdicts) + # the driver's own failure, or an exit its rows do not explain, is never a success + failed = bool(result.get('error')) or result.get('exit') != ran + # a collection the predicate found short (acceptance()) keeps each board's verdict beside its missing evidence + lost = collection['state'] != 'complete' or any(r.get('copyBack') for r in boards) or 'incomplete' in verdicts + code = 2 if ran == 2 or failed else INCOMPLETE if lost else ran + outcome = {0: 'pass', 1: 'fail', 2: 'blocker', INCOMPLETE: 'incomplete'}[code] + return finish(dest, {**out, 'outcome': outcome, 'pass': code == 0, 'boards': boards, + **({'driverFailure': result.get('error') or f'the driver exited {result.get("exit")!r}'} + if failed else {})}, code) + + +def finish(dest, data, code): + core.write_json(dest.with_name(f'{dest.name}.outcome.json'), data) + return emit(data, code) + + +# --- commands ------------------------------------------------------------------------------------------------------ +def refuse_unsupported(argv): + if argv[:1] and argv[0] in MANUAL: + fail(f'{argv[0]} is a manual or recovery operation: it runs on the rig, from the attempt\'s retained checkout ' + f'(SKILL.md Recovery), never through hil_remote.py') + hit = [a for a in argv for f in REFUSED_FLAGS if a == f or a.startswith(f + '=')] + if hit: + fail(f'{", ".join(hit)}: a remote run takes no manual, recovery, commissioning or override option') + + +def lock_wait(text): + secs = int(text) + if not 0 <= secs <= LOCK_WAIT_MAX: + raise argparse.ArgumentTypeError(f'0 to {LOCK_WAIT_MAX} s') + return secs + + +def run(args): + if os.name == 'nt': + fail('the Windows client is pending (SKILL.md Limits): run from a POSIX host') + if 'HIL_NO_BOARD_LOCK' in os.environ: + fail('HIL_NO_BOARD_LOCK is set: a remote run is always locked and forwards no HIL_* variable; unset it') + cfg = HOSTS.get(args.host) or fail(f'no rig {args.host}: one of {sorted(HOSTS)}') + run_id = args.run_id or f'{time.strftime("%Y%m%d-%H%M%S")}-{secrets.token_hex(3)}' + runs = ROOT / RUNS + if core.attempt_record(runs, run_id).exists() or local_manifest(run_id).exists(): + fail(f'run id {run_id} was used before ({RUNS}/{run_id}.*); pick a new one') + try: + roster = json.loads(git(ROOT, 'cat-file', 'blob', f'HEAD:{cfg["roster"]}')) + except ValueError as e: + fail(f'{cfg["roster"]} at HEAD is unreadable: {e}') + boards = select(args.board, roster, cfg['roster']) + runs.mkdir(exist_ok=True) + stage = runs / f'.{run_id}.stage' # inside the checkout, where rsync can name it on every client + try: + stage.mkdir() + except FileExistsError: + fail(f'{stage} exists: another run with id {run_id} is staging') + try: + print(f'==> Freezing HEAD and its pinned repositories for {", ".join(boards)}', file=sys.stderr) + manifest = freeze(stage, run_id, args.host, cfg, boards, args) + with open(local_manifest(run_id), 'x') as f: + f.write(json.dumps(manifest) + '\n') + digests = core.file_digests(stage, [p for p in stage.rglob('*') if p.is_file()]) + tree = core.create_attempt(ROOT, runs, run_id, args.host, cfg['base'], (driver.OUT,)) + print(f'==> Staging attempt {tree.spec()}', file=sys.stderr) + core.stage_tree(tree, stage) + bad = core.verify_remote(tree, digests) + if bad: + fail(f'staged files differ on the rig: {", ".join(bad)}; attempt {run_id} stays unlaunched') + try: + core.launch(tree, ['python3', f'driver/{Path(DRIVER).name}', 'manifest.json']) + print(f'==> Launched {run_id}: `hil_remote.py wait {run_id}` reconnects', file=sys.stderr) + except core.RemoteError as e: # it may have started: the wait below finds out, and nothing launches it again + print(e.code, file=sys.stderr) + finally: + shutil.rmtree(stage, ignore_errors=True) + return after_launch(tree, manifest, lambda: wait_for(tree, manifest)) + + +def after_launch(tree, manifest, op): + """op() on a launched attempt: a transport failure is inconclusive (it says nothing of the attempt), never a + refusal.""" + try: + return op() + except core.RemoteError as e: + print(e.code, file=sys.stderr) + return emit({'runId': manifest['runId'], 'outcome': 'inconclusive', 'pass': False, 'remote': where(tree, manifest), + 'detail': str(e.code)}, core.INCONCLUSIVE) + + +@contextlib.contextmanager +def without_git_overrides(): + """A hook's GIT_DIR, GIT_WORK_TREE or GIT_INDEX_FILE would aim every git call, check_build's included, at another + repository: none reaches them, and the caller's are restored.""" + saved = {k: os.environ.pop(k) for k in [k for k in os.environ if k.startswith('GIT_')]} + try: + yield + finally: + os.environ.update(saved) + + +@without_git_overrides() +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + try: + check_pin() + except SystemExit as e: # raised by fail() before the provider is loaded + print(e.code, file=sys.stderr) + return REFUSED + provider() + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + sub = ap.add_subparsers(dest='cmd', required=True) + r = sub.add_parser('run', help='freeze, stage, launch, wait and collect one attempt') + r.add_argument('-b', '--board', action='append', required=True, help='a roster board (repeatable; run one after another)') + r.add_argument('--run-id', help='the attempt id, used once (default: a time stamp)') + r.add_argument('--host', default='ci.lan', help=f'the rig, one of {sorted(HOSTS)}') + r.add_argument('--lock-wait', type=lock_wait, default=0, + help='seconds the batteries together keep trying a held board (default 0); a blocker is never waited for') + r.add_argument('--fail-at', choices=hil_ci.STEPS, help='acceptance only (SKILL.md Execution): passed to every battery') + w = sub.add_parser('wait', help='reconnect, wait for the attempt and collect it') + w.add_argument('run_id') + w.add_argument('--timeout', type=core.wait_secs, default=core.WAIT_SECS, help=f'seconds, at most {core.WAIT_SECS}') + c = sub.add_parser('collect', help="fetch the attempt's evidence whatever its state") + c.add_argument('run_id') + try: + refuse_unsupported(argv) + args = ap.parse_args(argv) + if args.cmd == 'run': + return run(args) + tree, manifest = core.load_attempt(ROOT, ROOT / RUNS, args.run_id), load_manifest(args.run_id) + if args.cmd == 'wait': + return after_launch(tree, manifest, lambda: wait_for(tree, manifest, args.timeout)) + groups, optional = members(manifest) + return after_launch(tree, manifest, lambda: report(tree, manifest, core.collect( + tree, ROOT / RUNS / args.run_id, groups, optional, accept=acceptance(tree, manifest)))) + except core.RemoteError as e: + print(e.code, file=sys.stderr) + return REFUSED + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/skills/hil-contract/scripts/hil_remote_driver.py b/.claude/skills/hil-contract/scripts/hil_remote_driver.py new file mode 100644 index 000000000..1638f71cf --- /dev/null +++ b/.claude/skills/hil-contract/scripts/hil_remote_driver.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""The rig side of hil_remote.py (SKILL.md Execution, Remote): launched detached in its attempt directory by +agentrc's remote core, it rebuilds the committed checkout from the staged git bundles, verifies it against the +manifest, then per board builds the HIL record and runs one battery as hil_ci.py does (its report pair prepared +before the copy-back), and writes result.json, the attempt's outcome. + + python3 driver/hil_remote_driver.py manifest.json (cwd: the attempt directory) + +Stdlib only until the checkout is rebuilt; from then on the checkout's own hil_ci.py (run_board(markers=False): CI's +markers read, never written, the build record, the battery, the verdict and the report pair). The driver takes no +board lock: each `hil_test.py run` locks its board itself, protected, and checks CI's state directory +(`--ci-state-root`) once it holds the lock. + +Exit 0 every board passed, 1 a board failed or was not run, 2 a blocker, 3 refused before any build. +""" +import hashlib +import json +import os +import re +import socket +import subprocess +import sys +import time +from pathlib import Path + +SCHEMA, VERSION = 'tinyuf2-hil-remote', 1 +RESULT = 'result.json' # written once, when the driver ends +PROGRESS = 'progress.json' # the rows so far, after each board: what a driver that died had reached +CHECKOUT = 'src' +OUT = 'out' +REFUSED = 3 +SCRIPTS = '.claude/skills/hil-contract/scripts' +REPO_PATH = re.compile(r'[A-Za-z0-9_.-]+(/[A-Za-z0-9_.-]+)*') +SHA = re.compile(r'[0-9a-f]{40}') +FIELDS = ('runId', 'head', 'hostname', 'repos', 'submodules', 'roster', 'driver', 'boards', 'lockWait', 'failAt', + 'ciStateRoot') + + +class Refused(Exception): + """Nothing was built and no board was touched.""" + + +def git(where, *argv): + r = subprocess.run(['git', '-C', str(where), *argv], capture_output=True, text=True) + if r.returncode: + raise Refused(f'git -C {where} {" ".join(argv)}: {r.stderr.strip()}') + return r.stdout.strip() + + +def sha256(path): + try: + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + except OSError: + return None + + +def write_json(path, data): + tmp = path.with_name(f'.{path.name}.{os.getpid()}.tmp') + with open(tmp, 'w') as f: + f.write(json.dumps(data, indent=1) + '\n') + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, path) + + +def load_manifest(path): + try: + m = json.loads(Path(path).read_text()) + except (OSError, ValueError) as e: + raise Refused(f'manifest {path} unreadable: {e}') + if not isinstance(m, dict) or (m.get('schema'), m.get('version')) != (SCHEMA, VERSION): + raise Refused(f'manifest {path} is not {SCHEMA} version {VERSION}') + missing = [k for k in FIELDS if k not in m] + if missing: + raise Refused(f'manifest {path} lacks {missing}') + try: + for r in m['repos']: + if not (r['path'] == '.' or REPO_PATH.fullmatch(r['path']) and '..' not in r['path'].split('/')) \ + or not SHA.fullmatch(r['commit']) or not all(SHA.fullmatch(s) for s in r['shallow']): + raise Refused(f'manifest repo entry {r} is malformed') + except (KeyError, TypeError) as e: + raise Refused(f'manifest {path}: a repo entry lacks {e}') + return m + + +def check_packages(attempt, m): + """The staged bundles hash as the manifest froze them (the transfer was verified too; this is the driver's own + check, before any of them is read).""" + bad = [r['bundle'] for r in m['repos'] if sha256(attempt / r['bundle']) != r['sha256']] + if bad: + raise Refused(f'staged packages missing or changed since the manifest froze them: {", ".join(bad)}') + + +def rebuild(attempt, m): + """Detached checkouts of every repository from its bundle, the superproject first: no remote, no worktree pointer + file, no link to the dev PC. A shallow repository gets its shallow boundary before the fetch, which then checks + the bundle's objects are complete above it.""" + src = attempt / CHECKOUT + if src.exists(): + raise Refused(f'{src} exists: an attempt rebuilds its checkout once') + for r in sorted(m['repos'], key=lambda r: (r['path'] != '.', r['path'].count('/'), r['path'])): + where = src if r['path'] == '.' else src / r['path'] + if where.exists() and any(where.iterdir()): + raise Refused(f'{where} is not empty before its checkout') + where.mkdir(parents=True, exist_ok=True) + git(where, 'init', '-q', '--template=') + if r['shallow']: + Path(git(where, 'rev-parse', '--absolute-git-dir'), 'shallow').write_text(''.join(f'{s}\n' for s in r['shallow'])) + git(where, 'fetch', '-q', '--no-tags', str(attempt / r['bundle']), 'HEAD') + got = git(where, 'rev-parse', 'FETCH_HEAD') + if got != r['commit']: + raise Refused(f'{r["bundle"]} holds {got} for {r["path"]}, not {r["commit"]}') + git(where, '-c', 'advice.detachedHead=false', 'checkout', '-q', '--detach', r['commit']) + return src + + +def verify(src, m, driver): + """Why the rebuilt checkout is not the manifest's committed identity on a clean tree; empty when it is.""" + why = [] + for r in m['repos']: + where = src if r['path'] == '.' else src / r['path'] + if os.path.realpath(git(where, 'rev-parse', '--show-toplevel')) != os.path.realpath(where): + why.append(f'{r["path"]} is not a checkout of its own') + continue + if git(where, 'rev-parse', 'HEAD') != r['commit']: + why.append(f'{r["path"]} is not at {r["commit"]}') + status = git(where, 'status', '--porcelain', *(() if r['path'] == '.' else ('--untracked-files=all',))) + if status: + why.append(f'{r["path"]} is not clean: {status.splitlines()[:5]}') + for path, commit in m['submodules'].items(): + if git(src, 'rev-parse', f'HEAD:{path}') != commit: + why.append(f'HEAD does not pin {path} at {commit}') + if sha256(src / m['roster']['path']) != m['roster']['sha256']: + why.append(f'{m["roster"]["path"]} is not the roster the manifest names') + if sha256(src / m['driver']['path']) != sha256(driver) or sha256(driver) != m['driver']['sha256']: + why.append('the running driver is not the checkout\'s committed one') + return why + + +def state_root(m): + root = Path(os.path.expanduser(m['ciStateRoot'])) + try: + os.listdir(root) + except OSError as e: + raise Refused(f'CI state root {root} cannot be read: {e}') + return root + + +def inventory(attempt, board): + """The attempt-relative regular files a board left under out/, links never followed: what the copy-back must + bring home.""" + out = attempt / OUT + found = [f'{OUT}/{board}.build.json'] if (out / f'{board}.build.json').is_file() else [] + for d, dirs, files in os.walk(out / board): + dirs[:] = [x for x in dirs if not os.path.islink(os.path.join(d, x))] + found += [Path(d, f).relative_to(attempt).as_posix() for f in files if os.path.isfile(os.path.join(d, f)) + and not os.path.islink(os.path.join(d, f))] + return sorted(found) + + +def run_board(hil_ci, board, m, attempt, ci_root, budget): + def lock_wait(): # one budget for every battery, counted from the first one's start, after its build + deadline = budget.setdefault('deadline', time.monotonic() + m['lockWait']) + return max(0, round(deadline - time.monotonic())) + row = hil_ci.run_board(board, m['roster']['path'], attempt / OUT, ci_root, lock_wait, m['failAt'], markers=False) + if row.get('unsafeArtifacts'): # attempt-relative, for the copy-back to withhold + row['unsafeArtifacts'] = [Path(p).relative_to(attempt).as_posix() for p in row['unsafeArtifacts']] + return row + + +def drive(attempt, m, driver, result): + stripped = sorted(k for k in os.environ if k.startswith(('HIL_', 'GIT_'))) + for k in stripped: # no lock bypass or override reaches the runner, and no GIT_DIR aims a rebuild elsewhere + del os.environ[k] + result['strippedEnv'] = stripped + if socket.gethostname() != m['hostname']: + raise Refused(f'this rig is {socket.gethostname()}, not {m["hostname"]}, the host {m["roster"]["path"]} is for') + check_packages(attempt, m) + src = rebuild(attempt, m) + why = verify(src, m, driver) + if why: + raise Refused(f'the rebuilt checkout is not the committed identity: {"; ".join(why)}') + result['checkout'] = str(src) + ci_root = state_root(m) + sys.dont_write_bytecode = True # nothing imported from the checkout may write into it + sys.path.insert(0, str(src / SCRIPTS)) + import hil_ci # noqa: E402 the checkout's own: its ROOT is the rebuilt checkout + budget = {} + for board in m['boards']: + try: + row = run_board(hil_ci, board, m, attempt, ci_root, budget) + except Exception as e: # a battery may have started: never a pass, never a not-run + row = {'board': board, 'phase': 'driver', 'verdict': 'blocker', 'detail': f'driver error: {type(e).__name__}: {e}'} + row['artifacts'] = inventory(attempt, board) + result['boards'].append(row) + write_json(attempt / PROGRESS, result) + return hil_ci.run_report.fold_exit(r['verdict'] for r in result['boards']) + + +def main(argv=None): + argv = sys.argv[1:] if argv is None else argv + attempt, driver = Path.cwd(), Path(__file__).resolve() + result = {'schema': SCHEMA, 'version': VERSION, 'boards': [], 'startedAt': time.time()} + code = REFUSED + try: + m = load_manifest(attempt / argv[0]) + result.update(runId=m['runId'], head=m['head']) + code = drive(attempt, m, driver, result) + except Refused as e: + result['refused'] = str(e) + except Exception as e: # after the checkout: whatever ran is in boards[]; a board never reached is no pass + result['error'] = f'{type(e).__name__}: {e}' + code = 2 + result.update(exit=code, endedAt=time.time()) + write_json(attempt / RESULT, result) + return code + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/skills/hil-contract/scripts/hil_run_report.py b/.claude/skills/hil-contract/scripts/hil_run_report.py new file mode 100644 index 000000000..1e74f59b5 --- /dev/null +++ b/.claude/skills/hil-contract/scripts/hil_run_report.py @@ -0,0 +1,129 @@ +# SPDX-License-Identifier: MIT +"""tinyuf2's run report as agentrc's report core (`hil` Reporting) reads and writes it: the body schema, its Markdown +view of the steps, the native `hil` verdict of a board's row (SKILL.md Results, Verdicts) and its rows normalised for +the core's shared run rules. The core module is passed in: hil_test.py loads it through its pin check, hil_ci.py by +load() after the same check (check_build.check_agentrc_pin); verdict() needs no core. Stdlib and portable: hil_remote.py +imports it on any client.""" +import importlib.util + +NAME, VERSION, STEM = 'tinyuf2-hil-run', 1, 'hil_report' +STEP_MARK = {True: '✅', False: '❌'} +# native fields typed when present: a legacy report never recorded exitCode or heldInProcess +TYPES = {'pass': bool, 'caveat': str, 'reportPath': str, 'error': str, 'exitCode': int, 'heldInProcess': bool} + + +def check(body): + if not isinstance(body.get('results'), list) or not all(isinstance(r, dict) for r in body['results']): + raise ValueError('results must be a list of objects') + for k, t in TYPES.items(): + if k in body and type(body[k]) is not t: + raise ValueError(f'{k} must be {t.__name__}') + + +def legacy(raw): + """A report without `_hil`, from a runner before this one: its native fields are this version's, read as they are.""" + return raw + + +def _cell(v): + return ' '.join(str(v).splitlines()).replace('|', '\\|') + + +def render(core, body): + lines = [f'HIL run: {"pass" if body.get("pass") else "fail"}'] + for k in ('error', 'caveat'): + if body.get(k): + lines.append(f'{k.capitalize()}: {_cell(body[k])}') + for r in body['results']: + steps = [[_cell(s.get('step', '?')), STEP_MARK[bool(s.get('pass'))], _cell(s.get('detail', ''))] + for s in r.get('steps') or [] if isinstance(s, dict)] + cleanup = r.get('cleanup') + if isinstance(cleanup, dict) and cleanup: + steps.append(['cleanup', STEP_MARK[bool(cleanup.get('pass'))], _cell(cleanup.get('state', ''))]) + lines += ['', f'### {_cell(r.get("board", "?"))}', ''] + if r.get('detail'): + lines += [_cell(r['detail']), ''] + lines.append(core.table(['Step', 'Result', 'Detail'], steps, center={1}) if steps else '_No steps ran._') + return '\n'.join(lines) + '\n' + + +def schema(core): + return core.BodySchema(name=NAME, version=VERSION, stem=STEM, evidence='firmware-validation', check=check, + empty=lambda b: not b['results'], render=lambda b: render(core, b), legacy=legacy) + + +def exit_of(body): + """verdict()'s `code` from a report alone: None for a holder kept in process, and for a legacy report, which never + recorded its exit.""" + if body.get('heldInProcess') or 'exitCode' not in body: + return None + return body['exitCode'] + + +def row_of(report, board): + """`board`'s row of a runner report, None when it has none or there is no report.""" + return next((r for r in (report or {}).get('results', []) if r.get('board') == board), None) + + +def fold_exit(verdicts): + """The one exit of a set of `hil` verdicts (SKILL.md Verdicts): 2 a blocker, 1 a fail or not-run, else 0.""" + verdicts = set(verdicts) + return 2 if 'blocker' in verdicts else 1 if verdicts & {'fail', 'not-run'} else 0 + + +def verdict(code, report, board): + """The `hil` verdict of one run from the report's row (SKILL.md Verdicts): (verdict, detail). `code` is the runner's + exit, None for a runner that stayed the in-process cleanup holder.""" + if report is None: + return 'blocker', f'run ended without a report (exit {code})' + row = row_of(report, board) + if row is None: + return 'blocker', f'report has no row for {board} (exit {code}): {report.get("caveat") or report.get("error")}' + cleanup = row.get('cleanup') or {} + if code in (4, None) or row.get('ran') is None or cleanup.get('heldAfter'): + return 'blocker', f'cleanup failed or no report; board held: {row.get("detail")} {cleanup}'[:400] + if row.get('probeUsers') or row.get('recoveryBlocker') and not row.get('locked'): + return 'blocker', str(row.get('detail')) + if row.get('locked'): + return ('blocker' if row.get('recoveryBlocker') else 'not-run'), f'locked: {row.get("detail")}' + if not row.get('ran'): + return 'not-run', str(row.get('detail') or report.get('error')) + failed = next((s for s in row.get('steps', []) if not s.get('pass')), None) + if row.get('pass') and cleanup.get('pass'): + return 'pass', '' + if failed is None: + return 'blocker', f'aborted: {row.get("detail")}' + if cleanup.get('state') == 'untouched': + return 'not-run', f'{failed["step"]}: {failed.get("detail")} (nothing written)' + return 'fail', f'{failed["step"]}: {failed.get("detail")}' + + +def load(path, name='agentrc_hil_report_core'): + """The module at `path` as `name`, loaded by path, never an installed one: every tinyuf2 script's loader of + lib/agentrc code (after its pin check, check_build.check_agentrc_pin) and of check_build.py.""" + spec = importlib.util.spec_from_file_location(name, path) + if spec is None: + raise ImportError(f'{path} not found') + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +def run_rows(code, report, requested): + """One normalised {board, verdict, detail, evidence} row per requested board, from the native verdict().""" + evidence = [report['reportPath']] if (report or {}).get('reportPath') else [] + return [dict(zip(('verdict', 'detail'), verdict(code, report, b)), board=b, evidence=evidence) for b in requested] + + +def shared_run(core, code, reading, requested): + """The core's apply_run_rules over the rows of `reading` (read_document or prepare_pair of a run's report), apart + from the native verdict, which it can only tighten: `_hil.termination` aborted makes every board a blocker and a + caveat denies a pass. A report that cannot be read is a blocker for every board; a legacy one never recorded its + termination and is read as completed.""" + if reading.outcome not in core.READABLE: + rows = [{'board': b, 'verdict': 'blocker', 'detail': f'report {reading.outcome}: {reading.detail}', 'evidence': []} + for b in requested] + return core.apply_run_rules(schema(core), requested, rows, 'completed', '') + termination = reading.meta['termination'] if reading.meta else 'completed' + return core.apply_run_rules(schema(core), requested, run_rows(code, reading.body, requested), termination, + reading.body.get('caveat', '')) diff --git a/.claude/skills/hil-contract/scripts/hil_test.py b/.claude/skills/hil-contract/scripts/hil_test.py index 323aa1ff8..ed238818a 100755 --- a/.claude/skills/hil-contract/scripts/hil_test.py +++ b/.claude/skills/hil-contract/scripts/hil_test.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 """TinyUF2 HIL battery and the probe operations debug sessions share with it. - hil_test.py run -b BOARD --build-record FILE CONFIG [--fail-at STEP] [--try-reentry METHOD] + hil_test.py run -b BOARD --build-record FILE CONFIG [--fail-at STEP] [--try-reentry METHOD] [--ci-state-root DIR] hil_test.py install -b BOARD --build-record FILE --under-hold REASON [--try-reentry METHOD] CONFIG - hil_test.py restore -b BOARD --build-record FILE (--under-hold REASON [--try-reentry METHOD] | --recover-holder PID --report FILE) CONFIG + hil_test.py restore -b BOARD --build-record FILE (--under-hold REASON [--try-reentry METHOD] | --recover-holder PID --report FILE + | --recover-takeover TOKEN [--report FILE]) CONFIG The last stdout line is JSON. Exit 0 every board passed and was cleaned up, 1 a board failed or did not run (cleaned up or never touched), 2 refused before any hardware access, 4 cleanup failed (the board is @@ -13,10 +14,11 @@ """ import argparse import contextlib +import errno import fcntl +import fnmatch import glob import hashlib -import importlib.util import json import os import re @@ -24,6 +26,7 @@ import shlex import shutil import signal +import socket import struct import subprocess import sys @@ -35,7 +38,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.dont_write_bytecode = True # nothing the runner imports may write into a checkout it binds import check_build as cb # noqa: E402 +import hil_ci # noqa: E402 CI's state directory: unfinished() import hil_esp as esp # noqa: E402 +import hil_run_report as run_report # noqa: E402 # [floor, ceiling) the runner ever programs, per J-Link device, independent of roster and source: # LPC55S69 PFR starts at 0x9DE00 and a bad CMPA write can lock the part (UM11126 Table 173, Table 1064); @@ -58,7 +63,7 @@ STEPS['esp'] = STEPS['jlink'] + ('recopy',) REENTRY = {'jlink': ('retained-register',), 'stlink': ('retained-register',), 'esp': ('dfu-detach',)} REENTRY_METHODS = sorted({m for v in REENTRY.values() for m in v}) -PROTECTED = {'hil_test.py', 'pool_check'} # hil_lock.PROTECTED_REASONS, checked at load +PROTECTED = {'hil_test.py', 'pool_check', 'farm_check'} # hil_lock.PROTECTED_REASONS, checked at load # the board-lock helper is the submodule's at its pinned commit, never an operator's installed agentrc AGENTRC = 'lib/agentrc' LOCK_HELPER = ROOT / AGENTRC / 'skills' / 'hil' / 'scripts' / 'hil_lock.py' @@ -79,8 +84,10 @@ JLINK_RETRY_DELAY = 2 ESPTOOL_VERSION = '5.2.0' # the CLI and API the esptool session helper is written and bench-checked against ESP_BAUD, ESP_TIMEOUT = 921600, 300 -ESP_SESSION = Path(__file__).resolve().parent / 'esp_session.py' -USB_DFU = Path(__file__).resolve().parent / 'usb_dfu.py' +# the provider's children an Espressif battery spawns, at the pin (agentrc `hil`); admission refuses without them +ESP_SESSION = LOCK_HELPER.with_name('esptool_session.py') +USB_DFU = LOCK_HELPER.with_name('usb_dfu.py') +REPORT_CORE = LOCK_HELPER.with_name('hil_report_core.py') # the report document (agentrc `hil` Reporting) NONCE_MAGIC, NONCE_BLOCK = b'TUF2NONC', 256 # apps/dfu_runtime/src/dfu_runtime.h JLINK_IF, JLINK_SPEED = 'SWD', '4000' STM32_PROGRAMMER_CLI = 'STM32_Programmer_CLI' @@ -942,18 +949,17 @@ def refuse_pending(workdir, mark=None): 'sysresetreq_stlink', 'Operation', 'Attempt') OPENOCD = [] # and its openocd_esp module (openocd_esp()) OPENOCD_API = ('admit', 'find_openocd', 'usj_location', 'flash_openocd_esp', 'ci_refusal') +CORE = [] # and the report core (report_core()) +CORE_API = ('BodySchema', 'READABLE', 'ReportError', 'read_document', 'start_attempt', 'make_meta', 'publish', 'table', + 'roster_identity') def load_pinned(path, name, api, what='flash provider'): """A lib/agentrc module at the commit HEAD pins, loaded by path, never an installed agentrc's; Refused when the submodule is off its pin or the module lacks what the runner calls.""" check_lock_pin() - spec = importlib.util.spec_from_file_location(name, path) - if not spec: - raise Refused(f'{what} {path} not found') - mod = importlib.util.module_from_spec(spec) try: - spec.loader.exec_module(mod) + mod = run_report.load(path, name) except Exception as e: raise Refused(f'{what} {path} failed to load: {e}') missing = [n for n in api if not hasattr(mod, n)] @@ -979,6 +985,13 @@ def openocd_esp(): return OPENOCD[0] +def report_core(): + """hil_report_core.py, once per process: every report this runner writes or a recovery reads goes through it.""" + if not CORE: + CORE.append(load_pinned(REPORT_CORE, 'agentrc_hil_report_core', CORE_API, 'report core')) + return CORE[0] + + def command_argv(cmd): """(argv, environment) of a provider command: an argv as is, a shell string split into words with its leading NAME=value words as the environment they set. Any other shell syntax is refused: no shell runs it.""" @@ -1369,7 +1382,7 @@ def esp_port(uid): def child_json(cmd, log, timeout, what): - """The JSON last line of a helper child (esp_session.py, usb_dfu.py, fat-read); a missing line, a non-zero + """The JSON last line of a helper child (esptool_session.py, usb_dfu.py, fat-read); a missing line, a non-zero exit or an `ok` other than true fails the step with the helper's `reason` (else `error`) and the output tail.""" code, out = spawn(cmd, log, timeout) tail = out.strip().splitlines()[-5:] @@ -1385,7 +1398,7 @@ def child_json(cmd, log, timeout, what): class Esptool: - """esptool sessions on one board, each one connection in its own child (esp_session.py).""" + """esptool sessions on one board, each one connection in its own child (esptool_session.py).""" def __init__(self, board, workdir, log): self.b, self.dir, self.log = board, workdir, log @@ -1774,42 +1787,28 @@ def fat_read_main(args): # ------------------------------------------------------------------ locks def check_lock_pin(): - """Refuse unless lib/agentrc is checked out, clean, at the commit HEAD pins.""" - init = f'git submodule update --init {AGENTRC}' - if not LOCK_HELPER.is_file() or not cb.own_checkout(AGENTRC): - raise Refused(f'lock helper {LOCK_HELPER} is not checked out: {init}') - try: - want = cb.git('rev-parse', f'HEAD:{AGENTRC}').strip() - have = cb.git_in(AGENTRC, 'rev-parse', 'HEAD').strip() - except cb.Fail as e: - raise Refused(f'cannot read the {AGENTRC} pin: {e}') - if have != want: - # submodule update checks out the index's gitlink, so a staged pin change would defeat the hint - try: - staged = cb.git('rev-parse', f':{AGENTRC}').strip() - except cb.Fail as e: - raise Refused(f'cannot read the staged {AGENTRC} pin: {e}') - if staged != want: - raise Refused(f'{AGENTRC} is at {have}; HEAD pins {want} but {staged} is staged: commit or unstage that ' - f'{AGENTRC} pin change') - raise Refused(f'{AGENTRC} is at {have}; HEAD pins {want}: {init}') + """Refuse unless lib/agentrc is checked out, clean, at the commit HEAD pins (check_build's check).""" + if not LOCK_HELPER.is_file(): + raise Refused(f'lock helper {LOCK_HELPER} is not checked out: git submodule update --init {AGENTRC}') try: - dirty = cb.dirty_inputs([AGENTRC]) + cb.check_agentrc_pin() except cb.Fail as e: - raise Refused(f'cannot read the {AGENTRC} checkout state: {e}') - if dirty: - raise Refused(f'{AGENTRC} checkout is dirty at the pinned {want}: discard its local changes') + raise Refused(str(e)) def load_helper(): mod = load_pinned(LOCK_HELPER, 'hil_lock', (), 'lock helper') - lu = getattr(mod, 'LockUnavailable', None) + errors = [getattr(mod, e, None) for e in ('LockUnavailable', 'RecoveryBlocked', 'ReleaseRefused')] if getattr(mod, 'PROTECTED_REASONS', None) != PROTECTED or getattr(mod, 'CI_REASON', None) != 'hil_test.py' or \ - getattr(mod, 'CLEANUP_ROLE', None) != HOLDER_ROLE or not (isinstance(lu, type) and issubclass(lu, RuntimeError)) \ - or not all(callable(getattr(mod, f, None)) for f in ('_open_dir', '_open_file', '_read_file')): + getattr(mod, 'CLEANUP_ROLE', None) != HOLDER_ROLE or not isinstance(getattr(mod, 'HOLD_KIND', None), str) or \ + not all(isinstance(e, type) and issubclass(e, RuntimeError) for e in errors) or \ + not all(callable(getattr(mod, f, None)) for f in ('_open_dir', '_open_file', '_read_file', 'journal_state', + 'release_token', 'write_record', 'proc_start', + 'is_recovery_blocker')): raise Refused(f'lock helper {LOCK_HELPER}: protected reasons are not {sorted(PROTECTED)}, its CI reason is not ' - f"'hil_test.py', its cleanup role is not {HOLDER_ROLE!r}, it has no LockUnavailable or no checked file opens; " - 'recheck this contract') + f"'hil_test.py', its cleanup role is not {HOLDER_ROLE!r}, it has no LockUnavailable, RecoveryBlocked or " + 'ReleaseRefused, no checked file opens, no takeover journal, hold kind and token release, or no ' + 'recovery-blocker check; recheck this contract') return mod @@ -1904,6 +1903,106 @@ def manual_holder(helper, board, reason): prove_hold(helper, board, pid, reason=reason) +def takeover_predecessor(helper, board, token): + """The battery record that `token`'s live takeover holder replaced (`hil_lock.py hold --take-over`), from the + board's journal chain: refused unless the holder is proven and that record is this runner's, a battery's + protected record or its cleanup holder's, left on this board alone.""" + rec = helper.read_record(board) + if not isinstance(rec, dict) or rec.get('token') != token or rec.get('holder') != helper.HOLD_KIND: + raise Refused(f'{board}: lock record {rec} is not the takeover holder of token {token}') + if rec.get('locks') != [board]: + raise Refused(f'{board}: takeover {token} covers {rec.get("locks")}, not this board alone') + prove_hold(helper, board, rec.get('pid'), role=HOLDER_ROLE) + if helper.proc_start(rec['pid']) != rec.get('start'): + raise Refused(f'{board}: pid {rec["pid"]} is not the takeover holder that wrote the record') + chain, journal = [], helper.journal_state(board) + while journal is not None: # a takeover whose holder died before this one chains the original as `prior` + if not isinstance(journal, dict) or not journal: + raise Refused(f'{board}: a takeover journal in its chain cannot be read: the record it replaced is unknown') + chain.append(journal) + journal = journal.get('prior') + if not chain or chain[0].get('token') != token: + raise Refused(f'{board}: no takeover journal of token {token}: the record it replaced is unknown') + try: + pred = json.loads(chain[-1].get('predecessor') or '') + except (TypeError, ValueError): + pred = None + if not isinstance(pred, dict) or pred.get('reason') != 'hil_test.py' or \ + not helper.is_recovery_blocker(pred) or \ + pred.get('role') == HOLDER_ROLE and pred.get('board') != board: + raise Refused(f'{board}: the takeover replaced {pred!r}, not a battery record of this runner on {board}: recover ' + 'it by hand') + return pred + + +def run_dir_sessions(run_dir): + """The openocd_esp sessions a dead battery's run dir holds (OPENOCD_SESSIONS), every listing's error raised: a + run dir or session dir that cannot be listed, a session dir without its receipt or a receipt that cannot be read + hides what the battery left.""" + pattern, receipt = OPENOCD_SESSIONS.split('/') + found = [] + for name in sorted(os.listdir(run_dir)): + if fnmatch.fnmatchcase(name, pattern): + path = Path(run_dir) / name + if receipt not in os.listdir(path): + raise OSError(errno.ENOENT, f'{path} holds no {receipt}') + found.append(session_entry(path / receipt)) + return found + + +def predecessor_evidence(pred, reported, reviewed): + """What a battery that died left on the host: a cleanup holder's survivors, from its record and its report's row + (`reported`), or a protected record's run dir with its openocd_esp receipts. Refused while any lives or is not + ended, and when the record names neither or its run dir cannot be read: missing evidence keeps the blocker.""" + if pred.get('role') == HOLDER_ROLE: + survivors = pred.get('survivors') + survivors = survivors + reported if isinstance(survivors, list) else None + elif isinstance(pred.get('runDir'), str): + try: + survivors = run_dir_sessions(pred['runDir']) + except OSError as e: + raise Refused(f"the dead battery's run dir {pred['runDir']} cannot be read: {e}; what it left is unknown") + else: + survivors = None + if not isinstance(survivors, list): + raise Refused(f'the replaced record {pred} names neither survivors nor a run dir: what the battery left is unknown') + refuse_live(survivors, reviewed, "the dead battery's") + return survivors + + +def refuse_live(survivors, reviewed, whose): + """Refused while any of `whose` openocd_esp sessions is not ended, outside the `reviewed` receipts, or any of its + children lives: recovery never overlaps them.""" + pending = [s['receipt'] for s in survivors + if 'receipt' in s and session_state(s) != 'ended' and s['receipt'] not in reviewed] + if pending: + raise Refused(f'{whose} openocd_esp sessions {pending} are not ended: end each with ' + '`hil_openocd_esp.py end --receipt `, or clear-pending after a separately reviewed ' + 'recovery (SKILL.md Recovery)') + alive = still_alive(survivors) + if alive: + raise Refused(f'{whose} children {alive} are still alive; recovery may not overlap them') + + +def read_report(report): + """A failed run's hil_report.json, read through the report core before any recovery lock or probe: a legacy one + (no `_hil`) by this runner's adapter; a missing or corrupt one, or an unknown `_hil` or body version, is refused.""" + core = report_core() + reading = core.read_document(report, run_report.schema(core)) + if reading.outcome not in core.READABLE: + raise Refused(f'cannot read {report}: {reading.outcome}: {reading.detail}') + return reading.body + + +def failed_row(body, report, board, holder): + """The row of `body` (read_report of `report`) whose cleanup holder is pid `holder` on `board`.""" + failed = [x for x in body['results'] if isinstance(x.get('cleanup'), dict) and + ((x['cleanup'].get('holder') or {}).get('pid') == holder and x.get('board') == board['name'])] + if len(failed) != 1: + raise Refused(f'{report} names no cleanup holder {holder} of {board["name"]}') + return failed[0] + + def probe_users(board): """Processes using this board's probe: for J-Link one opened by serial (a JLinkExe or GDB server from an earlier session), for ST-LINK any STM32CubeProgrammer CLI (its script names the probe, not its command line), for @@ -1958,12 +2057,10 @@ def readlink(fd): return None -def holder_record(fh, board, detail, survivors): - fh.truncate(0) - fh.seek(0) - json.dump({'pid': os.getpid(), 'reason': 'hil_test.py', 'role': HOLDER_ROLE, 'board': board, - 'detail': detail, 'survivors': survivors, 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) - fh.flush() +def holder_record(helper, fh, board, detail, survivors): + """The cleanup holder's record over the battery's, through the helper's interruption-safe rewrite: a death + midway leaves the protected record or an unreadable one, a recovery blocker either way, never an empty one.""" + helper.write_record(fh, 'hil_test.py', role=HOLDER_ROLE, board=board, detail=detail, survivors=survivors) HELD_IN_PROCESS = [] @@ -2007,7 +2104,7 @@ def keep_held(fh): os._exit(0) -def hand_off(fh, board, detail, survivors=()): +def hand_off(fh, board, detail, survivors=(), *, helper): """Keep the board held after a failed cleanup, by a forked holder that shares the open file description, so the flock is never dropped, and keeps none of the caller's stdio, so the caller's wait ends with this process. When no holder starts, this process keeps the lock itself and main() @@ -2025,7 +2122,7 @@ def hand_off(fh, board, detail, survivors=()): signal.signal(sig, signal.SIG_IGN) os.close(r) os.setsid() - holder_record(fh, board, detail, survivors) + holder_record(helper, fh, board, detail, survivors) exec_holder(fh, w) # the replacement acknowledges once it runs; a failed exec acknowledges here os.write(w, b'ok') os.close(w) @@ -2044,7 +2141,7 @@ def hand_off(fh, board, detail, survivors=()): os.kill(pid, signal.SIGKILL) with contextlib.suppress(ChildProcessError): os.waitpid(pid, 0) - holder_record(fh, board, detail, survivors) + holder_record(helper, fh, board, detail, survivors) HELD_IN_PROCESS.append(fh) return {'pid': os.getpid(), 'reason': 'hil_test.py', 'ready': True, 'inProcess': True} @@ -2544,7 +2641,7 @@ def esp_app_chunks(uf2_path, facts, board): def security_evidence(res, tty): - """The `security` the provider requires, from the esp_session security op: off only when both the ROM's security + """The `security` the provider requires, from the esptool_session security op: off only when both the ROM's security info and the eFuse registers read off; anything else refuses the JTAG flash before it starts.""" off = {'flashEncryption': 'off', 'secureBoot': 'off'} if res.get('op') != 'security' or res.get('rom') != off or res.get('efuse') != off: @@ -2731,6 +2828,7 @@ def run(self, files, version): TERM_SIGNALS = {signal.SIGTERM, signal.SIGINT} +INTERRUPTS = [] # every signal on_signal took: the run ended `aborted` (report `_hil.termination`) @contextlib.contextmanager @@ -2745,6 +2843,7 @@ def signals(how): def on_signal(signum, _): # blocked again before raising: no later signal can break into the unwinding or the cleanup signal.pthread_sigmask(signal.SIG_BLOCK, TERM_SIGNALS) + INTERRUPTS.append(signum) if REGISTERING['on']: REGISTERING['signum'] = REGISTERING['signum'] or signum return @@ -2762,7 +2861,7 @@ def check_stlink(): def check_esptool(): - """The runner's own esptool ESPTOOL_VERSION: its package for this Python (esp_session.py), and the `esptool` command + """The runner's own esptool ESPTOOL_VERSION: its package for this Python (esptool_session.py), and the `esptool` command the provider runs, an entry script of this same Python, so the flash and the sessions run one esptool.""" import importlib.metadata try: @@ -2804,7 +2903,16 @@ def check_pyusb(): raise Refused('pyusb finds no libusb-1.0; usb_dfu.py needs it') +def check_children(): + missing = [p.name for p in (ESP_SESSION, USB_DFU) if not p.is_file()] + if missing: + raise Refused(f'{AGENTRC} at its pin has no {", ".join(missing)} in {LOCK_HELPER.parent}: an Espressif battery ' + "needs the `hil` provider's children") + + def check_flasher(board): + if board['kind'] == 'esp': + check_children() {'esptool': check_esptool, 'stlink': check_stlink, 'openocd_esp': check_openocd_esp}.get(board['flasher'], check_jlink)() if board.get('usbPower'): try: @@ -2861,6 +2969,31 @@ def trial_reentry(board, method): return dict(board, reentry=method) +def keep_blocker(helper, fh, workdir, detail): + """Leave this run's protected record on the board as a recovery blocker naming why: closed, never cleared. A + failed rewrite leaves the old record or an unreadable one, a blocker either way.""" + with contextlib.suppress(OSError): + helper.write_record(fh, 'hil_test.py', protected=True, runDir=str(workdir), detail=detail) + fh.close() + + +def release(helper, fh): + if fh is not None: + helper.clear_record(fh) + fh.close() + + +def release_finished(helper, fh, row, code, done): + """Release the board, touching CI's `done` marker (hil_ci.py's `.done`) first when the row is no blocker: + a run that takes the lock next never sees CI's finished battery as unfinished. hil_ci.py touches it again after + its own verdict.""" + if done is not None and not MARK_ERRORS and \ + run_report.verdict(code, {'results': [row]}, row['board'])[0] != 'blocker': + with contextlib.suppress(OSError): + done.touch() + release(helper, fh) + + def run_board(args, board, facts, helper, workdir): row = {'board': board['name'], 'lock': board['lock'], 'ran': False, 'pass': False, 'locked': False, 'lockBypassed': False, 'steps': [], 'cleanup': None, 'detail': ''} @@ -2883,10 +3016,14 @@ def run_board(args, board, facts, helper, workdir): deadline = time.monotonic() + args.lock_wait while True: try: - fh = helper.acquire_board_lock(board['lock']) + # protected: a battery that dies holding the board leaves this record as a recovery blocker (SKILL.md Recovery) + fh = helper.acquire_board_lock(board['lock'], protected=True, runDir=str(workdir)) break except helper.LockUnavailable as e: raise Refused(f'{board["name"]}: {e}; refusing to run unlocked') + except helper.RecoveryBlocked as e: # a stale record or a takeover journal: never waited for + row.update(locked=True, recoveryBlocker=True, detail=str(e)) + return deferred(row), 1 except RuntimeError as e: # a cleanup holder never frees on its own (SKILL.md Recovery): a blocker, never waited for blocker, held = helper.is_cleanup_holder(helper.read_record(board['lock'])), str(e) @@ -2905,20 +3042,38 @@ def run_board(args, board, facts, helper, workdir): return deferred(row), 1 row['lockBypassed'] = fh is None try: - users = probe_users(board) - except StepFail as e: # an Espressif board without exactly one tty: not run, nothing touched - row['detail'] = f'probe check: {e}' - if fh is not None: - helper.clear_record(fh) - fh.close() - return deferred(row), 1 - if users: # a session from an earlier run, left on this probe: nothing may overlap it (SKILL.md Recovery) - row.update(probeUsers=users, detail=f'probe {board["probe"]} is still in use by pids {users}; no session may overlap them') - if fh is not None: - helper.clear_record(fh) - fh.close() - return deferred(row), 2 - bat = BATTERIES[board['kind']](board, facts, workdir, args.fail_at, row.get('commissioning', False)) + if args.started_marker is not None: # under the lock: a run waiting on the board is never seen started + try: + args.started_marker.touch() + except OSError as e: + raise Refused(f'started marker {args.started_marker} cannot be written: {e}') + # a transition check for the remote driver: CI's markers name a battery an older driver started without this lock + try: + earlier = [] if args.ci_state_root is None else hil_ci.unfinished(args.ci_state_root, board['name']) + except OSError as e: + raise Refused(f'CI state root {args.ci_state_root} cannot be read: {e}') + if earlier: + detail = (f'CI run {", ".join(earlier)} under {args.ci_state_root} started a battery on {board["name"]} and has ' + 'no .done: an unfinished attempt') + if fh is not None: + keep_blocker(helper, fh, workdir, detail) + row.update(recoveryBlocker=True, unfinishedAttempts=earlier, detail=detail) + return deferred(row), 1 + try: + users = probe_users(board) + except StepFail as e: # an Espressif board without exactly one tty: not run, nothing touched + row['detail'] = f'probe check: {e}' + release_finished(helper, fh, row, 1, args.done_marker) + return deferred(row), 1 + if users: # a session from an earlier run, left on this probe: nothing may overlap it (SKILL.md Recovery) + row.update(probeUsers=users, detail=f'probe {board["probe"]} is still in use by pids {users}; no session may overlap them') + release(helper, fh) + return deferred(row), 2 + bat = BATTERIES[board['kind']](board, facts, workdir, args.fail_at, row.get('commissioning', False)) + except BaseException: # nothing touched yet: no recovery blocker + if fh is not None and not fh.closed: + release(helper, fh) + raise code = 0 try: row['ran'] = True @@ -2929,7 +3084,7 @@ def run_board(args, board, facts, helper, workdir): row['detail'] = f'{bat.at}: {e}' if isinstance(e, StepFail) else f'{bat.at}: {type(e).__name__}: {e}' code = 1 row['steps'] = bat.steps - code = clean_up(row, code, bat, fh, board, helper, files) + code = clean_up(row, code, bat, fh, board, helper, files, args.done_marker) row.update(bat.report_fields()) # after the cleanup, whose install may take the snapshot a recovery needs if row.get('commissioning'): row['reentryDecision'] = reentry_decision(row) @@ -2943,9 +3098,10 @@ def deferred(row): return row -def clean_up(row, code, bat, fh, board, helper, files): - """Restore a touched board to the record's TinyUF2 with blinky copied, then release it. A child that survived - SIGKILL, or any unsuccessful cleanup, leaves the board held (exit 4), touched or not.""" +def clean_up(row, code, bat, fh, board, helper, files, done=None): + """Restore a touched board to the record's TinyUF2 with blinky copied, then release it (release_finished(), `done` + CI's marker). A child that survived SIGKILL, or any unsuccessful cleanup, leaves the board held (exit 4), touched or + not.""" stuck = [] try: refuse_pending(bat.dir, board.get('pendingMark')) @@ -2973,12 +3129,10 @@ def clean_up(row, code, bat, fh, board, helper, files): **({'markErrors': MARK_ERRORS[:]} if MARK_ERRORS else {})} row['pass'] = False if fh is not None: - row['cleanup']['holder'] = hand_off(fh, board['lock'], row['cleanup']['detail'], stuck) + row['cleanup']['holder'] = hand_off(fh, board['lock'], row['cleanup']['detail'], stuck, helper=helper) fh = None return 4 - if fh is not None: - helper.clear_record(fh) - fh.close() + release_finished(helper, fh, row, code, done) return code @@ -2990,6 +3144,38 @@ def settle_manual(result, bat, board, run_dir): result.update(bat.kept and {'buildLocksKept': bat.kept} or {}) +def build_provenance(result): + """The admitted record's firmware, as the report's rows name it, or {} when no row does: never filled in.""" + for r in result['results']: + fw = r.get('firmware') or r.get('installed') + if isinstance(fw, dict): + return {k: fw[k] for k in ('buildRecord', 'path', 'sha256', 'version') if fw.get(k) is not None} + return {} + + +def publish_report(result, run_dir, args, argv): + """`result` as the run dir's hil_report.json and .md through the report core (`hil` Reporting): `_hil` beside + the native fields, attempt id the run dir's own unique name. A failure never raises: it sets `reportError`, and + `reportPath` to None unless the JSON was written, for the stdout report and the holder that follow.""" + try: + core = report_core() + schema = run_report.schema(core) + attempt = core.start_attempt(run_dir, schema, False, run_dir.name) + try: # {name, sha256}: the same in a checkout and on a rig, where the report chain compares it whole + roster = core.roster_identity(args.config) + except OSError: + roster = {'name': Path(args.config).name} + meta = core.make_meta(schema, attempt, termination='aborted' if INTERRUPTS else 'completed', + invocation=['hil_test.py', *map(str, argv)], selection=[args.board], host=socket.gethostname(), + roster=roster, + build=build_provenance(result) or None) + core.publish(run_dir, schema, result, meta) + except Exception as e: # the run's own outcome, its stdout report and any hold stand whatever failed here + if result['reportPath'] not in getattr(e, 'published', ()): + result['reportPath'] = None + result['reportError'] = f'report not published: {type(e).__name__}: {e}' + + def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest='cmd', required=True) @@ -3000,9 +3186,16 @@ def main(argv=None): r.add_argument('--fail-at', choices=(*STEPS['esp'], 'cleanup'), help='acceptance only: fail this step, or the cleanup, ' 'to exercise cleanup and its holder') r.add_argument('--try-reentry', choices=REENTRY_METHODS, help="commissioning only, while the roster's reentry is unset") - r.add_argument('--lock-wait', type=int, default=0, help='seconds to keep trying a held board; a cleanup holder is never waited for') + r.add_argument('--lock-wait', type=int, default=0, + help='seconds to keep trying a held board; a cleanup holder or a recovery blocker is never waited for') r.add_argument('--run-root', type=Path, help="where this run's directory goes (default cmake-build/hil)") r.add_argument('--route', help="an install route from the board's tinyuf2.routes (default its roster flasher)") + r.add_argument('--ci-state-root', type=Path, help="transition only, from a driver: CI's state directory, whose " + 'unfinished battery on the board refuses the run and keeps it a recovery blocker') + r.add_argument('--done-marker', type=Path, help="hil_ci.py's only: its .done, touched before the board is " + 'released when the run is no blocker') + r.add_argument('--started-marker', type=Path, help="hil_ci.py's only: its .started, touched once the board " + 'is locked, before any hardware access; refused, the board released, when it cannot be written') r.add_argument('config') i = sub.add_parser('install', help='program a bootloader under your own manual hold') i.add_argument('-b', '--board', required=True) @@ -3019,7 +3212,9 @@ def main(argv=None): g = s.add_mutually_exclusive_group(required=True) g.add_argument('--under-hold', metavar='REASON') g.add_argument('--recover-holder', type=int, metavar='PID') - s.add_argument('--report', help="the failed run's hil_report.json (with --recover-holder)") + g.add_argument('--recover-takeover', metavar='TOKEN', help='the `hil_lock.py hold --take-over` of a stale battery ' + 'record: restore, then release that token') + s.add_argument('--report', help="the failed run's hil_report.json (with --recover-holder, or --recover-takeover of a dead cleanup holder)") s.add_argument('--try-reentry', choices=REENTRY_METHODS, help="commissioning only, under your hold, while the roster's reentry is unset") s.add_argument('--route', help="an install route from the board's tinyuf2.routes (default its roster flasher)") @@ -3062,6 +3257,7 @@ def main(argv=None): keep_build(args.survivors) for sig in TERM_SIGNALS: signal.signal(sig, on_signal) + INTERRUPTS.clear() # absolute: a provider command file is written in it while the working directory is run_dir = ((getattr(args, 'run_root', None) or cb.BUILD_DIR / 'hil') / f'run-{time.strftime("%Y%m%d-%H%M%S")}-{os.getpid()}').resolve() @@ -3098,35 +3294,35 @@ def main(argv=None): else: files, fw = admit_record(args.build_record, board, facts, run_dir) rec = snapshot = None + if args.try_reentry and (args.recover_holder is not None or args.recover_takeover): + raise Refused('--try-reentry is for a restore under your own hold, not a recovery') if args.recover_holder is not None: - if args.try_reentry: - raise Refused('--try-reentry is for a restore under your own hold, not a recovery') if not args.report: raise Refused("--recover-holder needs --report, the failed run's report naming that holder") - try: - rows = json.loads(Path(args.report).read_text())['results'] - except (OSError, ValueError, KeyError, TypeError) as e: - raise Refused(f'cannot read {args.report}: {e}') - failed = [x for x in rows if ((x.get('cleanup') or {}).get('holder') or {}).get('pid') - == args.recover_holder and x.get('board') == board['name']] - if len(failed) != 1: - raise Refused(f'{args.report} names no cleanup holder {args.recover_holder} of {board["name"]}') - held = [failed[0]['cleanup']] + failed = failed_row(read_report(args.report), args.report, board, args.recover_holder) + held = [failed['cleanup']] if board['kind'] == 'esp': - snapshot = report_snapshot(failed[0], board['regions']['preserved']) + snapshot = report_snapshot(failed, board['regions']['preserved']) recovery = recovery_lock(helper, board['lock']) rec = prove_cleanup_holder(helper, board['lock'], args.recover_holder) - survivors = held[0].get('survivors', []) + rec.get('survivors', []) - reviewed = reviewed_receipts(board['pendingMark']) - pending = [s['receipt'] for s in survivors - if 'receipt' in s and session_state(s) != 'ended' and s['receipt'] not in reviewed] - if pending: # the pending mark may be missing (markErrors); the report still names the sessions - raise Refused(f"the failed run's openocd_esp sessions {pending} are not ended: end each with " - '`hil_openocd_esp.py end --receipt `, or clear-pending after a separately reviewed ' - 'recovery (SKILL.md Recovery)') - alive = still_alive(survivors) - if alive: - raise Refused(f'the failed run\'s children {alive} are still alive; recovery may not overlap them') + # the pending mark may be missing (markErrors); the report still names the sessions + refuse_live(held[0].get('survivors', []) + rec.get('survivors', []), reviewed_receipts(board['pendingMark']), + "the failed run's") + elif args.recover_takeover: + body = read_report(args.report) if args.report else None + recovery = recovery_lock(helper, board['lock']) + rec = takeover_predecessor(helper, board['lock'], args.recover_takeover) + reported = [] + if rec.get('role') == HOLDER_ROLE: # its run's report names it: that run's evidence, snapshot included + if not args.report: + raise Refused("a dead cleanup holder's recovery needs --report, the failed run's report naming it") + failed = failed_row(body, args.report, board, rec.get('pid')) + reported = failed['cleanup'].get('survivors') or [] + if board['kind'] == 'esp': + snapshot = report_snapshot(failed, board['regions']['preserved']) + elif args.report: + raise Refused('a battery that died holding the board printed no report: --report is for a dead cleanup holder') + predecessor_evidence(rec, reported, reviewed_receipts(board['pendingMark'])) else: board = trial_reentry(board, args.try_reentry) manual_holder(helper, board['lock'], args.under_hold) @@ -3147,6 +3343,14 @@ def main(argv=None): wait_for(lambda: flock_free(helper.lock_path(board['lock'])), 10, f'cleanup holder {args.recover_holder} releasing {board["name"]}') cleanup['recovered'] = {'holder': args.recover_holder, 'record': rec} + elif args.recover_takeover: + try: # its records and journal gone and its flock dropped, or ReleaseRefused + pid, _ = helper.release_token(args.recover_takeover) + except helper.ReleaseRefused as e: + raise StepFail(f'restored, but the takeover holder was not released: {e}') + wait_for(lambda: flock_free(helper.lock_path(board['lock'])), 10, + f'takeover holder {pid} releasing {board["name"]}') + cleanup['recovered'] = {'takeover': args.recover_takeover, 'holder': pid, 'record': rec} result['results'].append({'board': board['name'], 'firmware': fw, 'cleanup': cleanup}) result['pass'], code = True, 0 except Refused as e: @@ -3165,10 +3369,8 @@ def main(argv=None): if MARK_ERRORS: # the board stays a blocker however the operation went: no later invocation sees its sessions result.update(markErrors=MARK_ERRORS[:], **{'pass': False}) code = max(code, 4) - try: - Path(result['reportPath']).write_text(json.dumps(result, indent=1)) - except OSError: - pass + result.update(exitCode=code, heldInProcess=bool(HELD_IN_PROCESS)) + publish_report(result, run_dir, args, sys.argv[1:] if argv is None else argv) print(json.dumps(result), flush=True) if HELD_IN_PROCESS: keep_held(HELD_IN_PROCESS[0]) diff --git a/.claude/skills/hil-contract/scripts/usb_dfu.py b/.claude/skills/hil-contract/scripts/usb_dfu.py deleted file mode 100644 index 4d7ee5b7b..000000000 --- a/.claude/skills/hil-contract/scripts/usb_dfu.py +++ /dev/null @@ -1,65 +0,0 @@ -#!/usr/bin/env python3 -"""USB control requests to a DFU runtime interface for hil_test.py, run as its own child so the runner can -time it out, kill it and see it survive. - - usb_dfu.py --vid V --pid P --serial S --path BUS-PORT[.PORT...] string|detach - -The device must be the only one with that VID, PID, serial and sysfs path, with exactly one DFU runtime -interface (class 0xFE, subclass 1, protocol 1). `string` reads that interface's string from the device -itself, not from a cache; `detach` sends DFU_DETACH (USB DFU 1.1 section 5.1) to it. -The last stdout line is JSON: {"ok": true, ...} or {"ok": false, "error"}. -""" -import argparse -import json -import sys - -sys.dont_write_bytecode = True - -DFU_CLASS = (0xFE, 0x01, 0x01) -DFU_DETACH = 0x00 -DETACH_TIMEOUT_MS = 1000 # wDetachTimeOut; a bitWillDetach device leaves the bus by itself -CONTROL_TIMEOUT_MS = 1000 - - -def find(vid, pid, serial, path): - import usb.core - hits = [] - for dev in usb.core.find(find_all=True, idVendor=vid, idProduct=pid): - where = f'{dev.bus}-{".".join(map(str, dev.port_numbers or ()))}' - if where == path and dev.serial_number == serial: - hits.append(dev) - if len(hits) != 1: - raise RuntimeError(f'{len(hits)} devices {vid:04x}:{pid:04x} serial {serial} at {path}') - dev = hits[0] - itfs = [i for i in dev.get_active_configuration() - if (i.bInterfaceClass, i.bInterfaceSubClass, i.bInterfaceProtocol) == DFU_CLASS] - if len(itfs) != 1: - raise RuntimeError(f'{len(itfs)} DFU runtime interfaces') - return dev, itfs[0] - - -def main(): - ap = argparse.ArgumentParser() - ap.add_argument('--vid', type=lambda s: int(s, 0), required=True) - ap.add_argument('--pid', type=lambda s: int(s, 0), required=True) - ap.add_argument('--serial', required=True) - ap.add_argument('--path', required=True) - ap.add_argument('op', choices=('string', 'detach')) - args = ap.parse_args() - try: - import usb.util - dev, itf = find(args.vid, args.pid, args.serial, args.path) - if args.op == 'string': - out = {'ok': True, 'interface': itf.bInterfaceNumber, - 'string': usb.util.get_string(dev, itf.iInterface) if itf.iInterface else None} - else: - dev.ctrl_transfer(0x21, DFU_DETACH, DETACH_TIMEOUT_MS, itf.bInterfaceNumber, None, CONTROL_TIMEOUT_MS) - out = {'ok': True, 'interface': itf.bInterfaceNumber} - except Exception as e: # every failure is reported, never raised past the JSON line - out = {'ok': False, 'error': f'{type(e).__name__}: {e}'} - print(json.dumps(out), flush=True) - return 0 if out['ok'] else 1 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/.claude/test/test_hil.py b/.claude/test/test_hil.py index 5a5068197..d5f361cc4 100644 --- a/.claude/test/test_hil.py +++ b/.claude/test/test_hil.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """Tests of the HIL contract's runner. Stubs prove the plumbing only: every probe and USB path here is unverified until a real run on the rig (commissioning.md).""" +import argparse import contextlib import importlib.metadata import importlib.util @@ -11,6 +12,7 @@ import select import shutil import signal +import socket import struct import subprocess import sys @@ -29,9 +31,10 @@ spec = importlib.util.spec_from_file_location('hil_test', SCRIPT) ht = importlib.util.module_from_spec(spec) spec.loader.exec_module(ht) -# lib/agentrc's flash provider, loaded once as main() loads it; its pin check is Roster's to test, against the real -# checkout and a git hook's environment +# lib/agentrc's report core and flash provider, loaded once as main() loads them; the pin check is Roster's to test, +# against the real checkout and a git hook's environment with mock.patch.object(ht, 'check_lock_pin'), contextlib.suppress(ht.Refused): + ht.report_core() ht.provider() ht.openocd_esp() BOARD = 'lpcxpresso55s69' @@ -292,22 +295,28 @@ def test_the_pinned_lock_helper_checks_its_pin_and_passes_the_contract(self): helper = pinned_helper(self, self.tmp / 'locks') pin.assert_called_once_with() self.assertEqual((helper.PROTECTED_REASONS, helper.CI_REASON, helper.CLEANUP_ROLE), - (ht.PROTECTED, 'hil_test.py', ht.HOLDER_ROLE)) - self.assertTrue(issubclass(helper.LockUnavailable, RuntimeError)) + ({'hil_test.py', 'pool_check', 'farm_check'}, 'hil_test.py', ht.HOLDER_ROLE)) + self.assertTrue(all(issubclass(e, RuntimeError) for e in (helper.LockUnavailable, helper.RecoveryBlocked, + helper.ReleaseRefused))) def test_a_lock_helper_off_the_contract_is_refused(self): if not ht.LOCK_HELPER.is_file(): self.fail('lib/agentrc not checked out: git submodule update --init lib/agentrc') text = ht.LOCK_HELPER.read_text() + reasons = re.search(r'^PROTECTED_REASONS = .*$', text, re.M).group(0) for what, old, new in (('cleanup role', "CLEANUP_ROLE = 'cleanup-holder'", "CLEANUP_ROLE = 'other'"), - ('protected reasons', "{CI_REASON, 'pool_check'}", '{CI_REASON}'), + ('protected reasons without farm_check', reasons, "PROTECTED_REASONS = {CI_REASON, 'pool_check'}"), + ('protected reasons with another', reasons, + "PROTECTED_REASONS = {CI_REASON, 'pool_check', 'farm_check', 'other'}"), ('LockUnavailable not a RuntimeError', 'class LockUnavailable(RuntimeError)', 'class LockUnavailable(OSError)'), ('LockUnavailable not a class', 'class LockUnavailable(RuntimeError):', 'LockUnavailable = 1\nclass _LockUnavailable(RuntimeError):'), - ('protected reasons not iterable', "PROTECTED_REASONS = {CI_REASON, 'pool_check'}", - 'PROTECTED_REASONS = 1'), - ('protected reasons unhashable', "PROTECTED_REASONS = {CI_REASON, 'pool_check'}", - 'PROTECTED_REASONS = [[CI_REASON]]')): + ('no RecoveryBlocked', 'class RecoveryBlocked(BoardLocked):', 'class _RecoveryBlocked(BoardLocked):'), + ('no token release', 'def release_token(', 'def _release_token('), + ('no hold kind', "HOLD_KIND = 'hil_lock-hold'", "_HOLD_KIND = 'hil_lock-hold'"), + ('no recovery-blocker check', 'def is_recovery_blocker(', 'def _is_recovery_blocker('), + ('protected reasons not iterable', reasons, 'PROTECTED_REASONS = 1'), + ('protected reasons unhashable', reasons, 'PROTECTED_REASONS = [[CI_REASON]]')): self.assertIn(old, text) off = self.tmp / 'off.py' off.write_text(text.replace(old, new)) @@ -762,6 +771,18 @@ def verify(self, rec): class Esp(EspCase): """The S3 DevKitC entry: esptool by the CP2102N's serial, facts from the IDF port's own files.""" + def test_the_children_are_the_pinned_providers_and_admission_refuses_without_one(self): + self.assertEqual({p.parent for p in (ht.ESP_SESSION, ht.USB_DFU)}, {ht.LOCK_HELPER.parent}) + self.assertTrue(ht.ESP_SESSION.is_file() and ht.USB_DFU.is_file()) + checked = [] + with mock.patch.object(ht, 'check_esptool', lambda: checked.append(1)): + ht.check_flasher(self.board) + for child in ('ESP_SESSION', 'USB_DFU'): + with self.subTest(child), mock.patch.object(ht, child, self.tmp / 'gone' / f'{child}.py'), \ + self.assertRaisesRegex(ht.Refused, f"no {child}.py .*the `hil` provider's children"): + ht.check_flasher(self.board) + self.assertEqual(checked, [1]) # refused before the flasher's own checks, let alone the board + def test_the_facts_come_from_the_idf_port(self): f = self.facts self.assertEqual((f['flashSize'], f['partitionCsv'], f['partitions']), (0x800000, 'partitions-8MB.csv', S3_PARTS)) @@ -1572,39 +1593,6 @@ def signal_at_registration(pid): # inside the provider's hold, between the spa self.assertEqual([k['survivors'] for k in self.bat.kept], [left]) -STUB_ESPTOOL = { - 'esptool/__init__.py': "__version__ = '5.2.0'\n", - 'esptool/cmds.py': textwrap.dedent(''' - import json, os - LOG = os.environ['STUB_LOG'] - def log(*a): - open(LOG, 'a').write(json.dumps(a) + '\\n') - class Port: - def close(self): log('close') - class Esp: - CHIP_NAME = os.environ.get('STUB_CHIP', 'ESP32-S3') - _port = Port() - def change_baud(self, b): log('baud', b) - def read_flash(self, addr, size): - if os.environ.get('STUB_FAIL_READ'): raise IOError('timed out') - return bytes([addr & 0xff]) * size - def flash_md5sum(self, addr, size): return f'md5-{addr:x}-{size:x}' - IS_STUB = bool(os.environ.get('STUB_IS_STUB')) - SEC = json.loads(os.environ.get('STUB_SECURITY', '[0, 0, 0, 0]')) # flags, flash_crypt_cnt, eFuse FE, eFuse SB - def get_security_info(self, cache=True): - log('security', cache) - return {'flags': self.SEC[0], 'flash_crypt_cnt': self.SEC[1], 'parsed_flags': {'SECURE_BOOT_EN': bool(self.SEC[0] & 1)}} - def get_flash_encryption_enabled(self): log('efuse'); return bin(self.SEC[2]).count('1') % 2 == 1 - def get_secure_boot_enabled(self): return self.SEC[3] << 20 - def detect_chip(port, connect_mode): - log('connect', port, connect_mode); return Esp() - def run_stub(esp): log('stub'); return esp - def attach_flash(esp): log('attach') - def reset_chip(esp, mode): log('reset', mode) - '''), -} - - class EspRig: """TinyUF2 and apps/dfu_runtime as the host sees them: who is on the bus, the app partition TinyUF2 writes from a copied UF2, and the string and DETACH the app answers.""" @@ -1820,144 +1808,6 @@ def test_the_nonce_block_ends_the_app_partition_inside_one_cache_window(self): self.assertEqual(ht.esp_app_chunks(uf2, self.facts, board)['nonceOffset'], off) -class EspSession(Tmp): - """esp_session.py itself, over stub esptool and pyserial packages: its dispatch, arguments, guard and reset - order.""" - - def setUp(self): - super().setUp() - stub = self.tmp / 'stub' - for rel, text in STUB_ESPTOOL.items(): - (stub / rel).parent.mkdir(parents=True, exist_ok=True) - (stub / rel).write_text(text) - self.log = self.tmp / 'stub.log' - self.env = dict(os.environ, PYTHONPATH=str(stub), STUB_LOG=str(self.log)) - - def session(self, ops, after='hard-reset', **env): - spec = self.tmp / 'ops.json' - spec.write_text(json.dumps(ops)) - r = subprocess.run([sys.executable, '-B', ht.ESP_SESSION, '--port', '/dev/x', '--chip', 'esp32s3', '--baud', '921600', - '--after', after, spec], capture_output=True, text=True, env=dict(self.env, **env)) - calls = [json.loads(l) for l in self.log.read_text().splitlines()] if self.log.exists() else [] - self.log.unlink(missing_ok=True) - return r.returncode, json.loads(r.stdout.strip().splitlines()[-1]), calls - - def test_ops_run_in_order_in_one_connection_then_one_reset(self): - code, out, calls = self.session([{'op': 'read', 'addr': 0x10, 'size': 4, 'file': str(self.tmp / 'r.bin')}, - {'op': 'md5', 'addr': 0x9000, 'size': 0x5000}]) - self.assertEqual(code, 0, out) - self.assertEqual([r['op'] for r in out['results']], ['read', 'md5']) - self.assertEqual(out['results'][1]['md5'], 'md5-9000-5000') - self.assertEqual((self.tmp / 'r.bin').read_bytes(), b'\x10' * 4) - self.assertEqual([c[0] for c in calls], ['connect', 'stub', 'baud', 'attach', 'reset', 'close']) - self.assertEqual(calls[0][2], 'default-reset') - self.assertEqual(calls[-2], ['reset', 'hard-reset']) - - def test_a_write_is_refused(self): - code, out, calls = self.session([{'op': 'write', 'addr': 0x8000, 'file': str(self.tmp / 'seg.bin')}]) - self.assertEqual((code, out['ok']), (1, False)) - self.assertIn("unknown op 'write'", out['error']) - - def test_no_reset_leaves_the_chip_alone(self): - code, out, calls = self.session([{'op': 'md5', 'addr': 0, 'size': 0x1000}], after='no-reset') - self.assertEqual(code, 0) - self.assertNotIn('reset', [c[0] for c in calls]) - - def test_another_chip_and_a_failing_op_are_reported_and_the_port_closed(self): - code, out, calls = self.session([{'op': 'md5', 'addr': 0, 'size': 0x1000}], STUB_CHIP='ESP32-S2') - self.assertEqual(code, 1) - self.assertIn('not an esp32s3', out['error']) - code, out, calls = self.session([{'op': 'md5', 'addr': 0, 'size': 0x1000}, {'op': 'read', 'addr': 0, 'size': 1, 'file': str(self.tmp / 'x')}], - STUB_FAIL_READ='1') - self.assertEqual((code, [r['op'] for r in out['results']]), (1, ['md5'])) - self.assertIn('timed out', out['error']) - self.assertEqual(calls[-1], ['close']) - self.assertNotIn('reset', [c[0] for c in calls]) - - def test_the_security_state_is_read_first_on_the_rom_loader_from_two_sources(self): - off = {'flashEncryption': 'off', 'secureBoot': 'off'} - code, out, calls = self.session([{'op': 'security'}, {'op': 'md5', 'addr': 0, 'size': 0x1000}], after='no-reset') - self.assertEqual(code, 0, out) - self.assertEqual(out['results'][0], {'op': 'security', 'flags': 0, 'flashCryptCnt': 0, 'rom': off, 'efuse': off}) - self.assertEqual([c[0] for c in calls][:4], ['connect', 'security', 'efuse', 'stub']) # before the stub - self.assertEqual(calls[1], ['security', False]) # never esptool's cached connect-time answer - for sec, rom, efuse in (([1, 0, 0, 0], {'secureBoot': 'on'}, {}), ([0, 3, 7, 0], {}, {'flashEncryption': 'on'}), - ([0, 1, 1, 1], {'flashEncryption': 'on'}, {'flashEncryption': 'on', 'secureBoot': 'on'})): - code, out, _ = self.session([{'op': 'security'}], after='no-reset', STUB_SECURITY=json.dumps(sec)) - self.assertEqual((out['results'][0]['rom'], out['results'][0]['efuse']), (dict(off, **rom), dict(off, **efuse))) - code, out, _ = self.session([{'op': 'security'}], after='no-reset', STUB_IS_STUB='1') - self.assertEqual(code, 1) - self.assertIn('ROM loader', out['error']) - code, out, calls = self.session([{'op': 'md5', 'addr': 0, 'size': 0x1000}, {'op': 'security'}], after='no-reset') - self.assertEqual((code, calls), (1, [])) - self.assertIn('comes first', out['error']) - - -STUB_USB = { - 'usb/__init__.py': '', - 'usb/core.py': """import json, os - -class Itf: - def __init__(self, c, sub, proto, num, istr): - self.bInterfaceClass, self.bInterfaceSubClass, self.bInterfaceProtocol = c, sub, proto - self.bInterfaceNumber, self.iInterface = num, istr - -class Dev: - def __init__(self, d): - self.d, self.bus, self.port_numbers, self.serial_number = d, d['bus'], tuple(d['ports']), d['serial'] - def get_active_configuration(self): - return [Itf(*i) for i in self.d['itfs']] - def ctrl_transfer(self, *a): - with open(os.environ['STUB_LOG'], 'a') as f: - f.write(json.dumps(list(a)) + '\\n') - -def find(find_all, idVendor, idProduct): - return [Dev(d) for d in json.loads(os.environ['STUB_USB']) if (d['vid'], d['pid']) == (idVendor, idProduct)] -""", - 'usb/util.py': """def get_string(dev, index): - return dev.d['strings'][str(index)] -""", -} - - -class UsbDfu(Tmp): - """usb_dfu.py over a stub pyusb: device selection, the interface it picks and the request it sends. - The DETACH on a real device is proven only on the rig.""" - - APP = {'vid': 0xCAFE, 'pid': 0x4000, 'bus': 3, 'ports': [1, 2], 'serial': '7CDFA1E06FD0', - 'itfs': [[0xFE, 1, 1, 2, 4]], 'strings': {'4': 'HIL 00ff'}} - - def call(self, op, devs, path='3-1.2'): - stub = self.tmp / 'stub' - for rel, text in STUB_USB.items(): - (stub / rel).parent.mkdir(parents=True, exist_ok=True) - (stub / rel).write_text(text) - log = self.tmp / 'usb.log' - log.unlink(missing_ok=True) - r = subprocess.run([sys.executable, '-B', ht.USB_DFU, '--vid', '0xcafe', '--pid', '0x4000', '--serial', '7CDFA1E06FD0', - '--path', path, op], capture_output=True, text=True, - env=dict(os.environ, PYTHONPATH=str(stub), STUB_USB=json.dumps(devs), STUB_LOG=str(log))) - calls = [json.loads(l) for l in log.read_text().splitlines()] if log.exists() else [] - return r.returncode, json.loads(r.stdout.strip().splitlines()[-1]), calls - - def test_string_and_detach_go_to_the_dfu_runtime_interface(self): - other = dict(self.APP, ports=[1, 3], strings={'4': 'HIL other'}) - code, out, _ = self.call('string', [other, self.APP]) - self.assertEqual((code, out['string'], out['interface']), (0, 'HIL 00ff', 2)) - code, out, calls = self.call('detach', [self.APP]) - self.assertEqual(code, 0, out) - self.assertEqual(calls, [[0x21, 0, 1000, 2, None, 1000]]) - - def test_no_match_two_matches_or_no_dfu_interface_is_an_error_with_no_request(self): - cases = {'another path': ([self.APP], '3-1.3', '0 devices'), 'two at one path': ([self.APP, self.APP], '3-1.2', '2 devices'), - 'no dfu interface': ([dict(self.APP, itfs=[[0xFF, 0, 0, 0, 0]])], '3-1.2', '0 DFU runtime interfaces')} - for what, (devs, path, err) in cases.items(): - with self.subTest(what): - code, out, calls = self.call('detach', devs, path) - self.assertEqual((code, out['ok'], calls), (1, False, [])) - self.assertIn(err, out['error']) - - real_esp_port = ht.esp_port @@ -2384,6 +2234,15 @@ def test_the_pinned_provider_loads_once_and_a_missing_or_incomplete_one_is_refus self.assertRaisesRegex(ht.Refused, 'HEAD pins'): ht.provider() + def test_a_report_core_without_what_the_runner_calls_is_refused_at_load(self): + text = ht.REPORT_CORE.read_text() + self.assertIn('def roster_identity(', text) + off = self.tmp / 'core.py' + off.write_text(text.replace('def roster_identity(', 'def _roster_identity(')) + with mock.patch.object(ht, 'CORE', []), mock.patch.object(ht, 'REPORT_CORE', off), \ + mock.patch.object(ht, 'check_lock_pin'), self.assertRaisesRegex(ht.Refused, "lacks \\['roster_identity'\\]"): + ht.report_core() + def test_a_shell_string_runs_as_argv_with_its_assignments_as_environment(self): self.assertEqual(ht.command_argv("NO_COLOR=1 esptool --chip esp32s3 -p /dev/ttyACM0 write_flash 0x0 'a b.bin'"), (['esptool', '--chip', 'esp32s3', '-p', '/dev/ttyACM0', 'write_flash', '0x0', 'a b.bin'], @@ -2646,7 +2505,7 @@ def test_another_reason_or_a_protected_one_is_refused(self): def test_cleanup_handoff_keeps_the_board_held_across_exec_and_recovery_can_end_it(self): fh = self.helper.acquire_board_lock(BOARD) with mock.patch.dict(os.environ, {ht.JOB_TRACKING: 'job-7'}): - holder = ht.hand_off(fh, BOARD, 'cleanup failed', [{'pid': 99, 'start': 1, 'cmd': ['mcopy']}]) + holder = ht.hand_off(fh, BOARD, 'cleanup failed', [{'pid': 99, 'start': 1, 'cmd': ['mcopy']}], helper=self.helper) self.addCleanup(lambda: end_child(holder['pid'])) self.assertTrue(holder['ready']) self.assertNotIn('inProcess', holder) @@ -2669,6 +2528,29 @@ def test_cleanup_handoff_keeps_the_board_held_across_exec_and_recovery_can_end_i self.assertIsNone(self.helper.read_record(BOARD)) + def test_a_handoff_cut_short_anywhere_leaves_a_blocker_the_helper_refuses(self): + real = {f: getattr(os, f) for f in ('pwrite', 'ftruncate')} + for cut in range(5): # write_record's '#', body, length and first byte, then none cut + with self.subTest(cut=cut): + fh = self.helper.take_board_lock(BOARD, 'hil_test.py', protected=True) + done = [] + + def op(name): + def call(*a): + if len(done) == cut: + raise OSError(f'holder killed before {name}') + done.append(name) + return real[name](*a) + return call + with mock.patch.object(os, 'pwrite', op('pwrite')), mock.patch.object(os, 'ftruncate', op('ftruncate')), \ + self.assertRaises(OSError) if cut < 4 else contextlib.nullcontext(): + ht.holder_record(self.helper, fh, BOARD, 'cleanup failed', []) + fh.close() # the holder's death: its flock goes, its record stays + self.assertIn(self.helper.record_state(BOARD)[0], ('record', 'unreadable')) + with self.assertRaises(self.helper.RecoveryBlocked): + self.helper.take_board_lock(BOARD, 'next') + self.helper.release_board_lock(self.helper.flock_nb(BOARD)) + def test_a_holder_that_never_starts_leaves_the_lock_held_in_process(self): parent = os.getpid() real = ht.holder_record @@ -2685,7 +2567,7 @@ def dies_in_the_child(*a): with self.subTest(what): fh = self.helper.acquire_board_lock(BOARD) with patch: - holder = ht.hand_off(fh, BOARD, 'cleanup failed') + holder = ht.hand_off(fh, BOARD, 'cleanup failed', helper=self.helper) self.assertEqual((holder['pid'], holder.get('inProcess')), (parent, True)) with self.assertRaises(OSError): self.helper.flock_nb(BOARD) @@ -2712,7 +2594,7 @@ def test_with_no_descriptor_left_the_runner_itself_holds_and_releases_only_on_si hog.append(os.open(os.devnull, os.O_RDONLY)) except OSError: pass - holder = ht.hand_off(fh, {BOARD!r}, 'cleanup failed') + holder = ht.hand_off(fh, {BOARD!r}, 'cleanup failed', helper=helper) print(json.dumps(holder), flush=True) ht.keep_held(ht.HELD_IN_PROCESS[0]) ''') @@ -2747,7 +2629,7 @@ def test_with_both_handoffs_failed_the_holding_runner_releases_the_build_lock_on signal.pthread_sigmask(signal.SIG_BLOCK, ht.TERM_SIGNALS) survivors = {json.dumps([ht.survivor(late.pid)])} keeper = ht.keep_build_lock(ht.take_build_lock(Path({str(root)!r})), survivors) - holder = ht.hand_off(fh, {BOARD!r}, 'cleanup failed', survivors) + holder = ht.hand_off(fh, {BOARD!r}, 'cleanup failed', survivors, helper=helper) print(json.dumps([keeper, holder]), flush=True) ht.keep_held(ht.HELD_IN_PROCESS[0]) ''') @@ -2770,7 +2652,7 @@ def test_with_both_handoffs_failed_the_holding_runner_releases_the_build_lock_on def test_a_holder_handed_off_during_cleanup_still_ends_on_sigusr1(self): fh = self.helper.acquire_board_lock(BOARD) with ht.signals(signal.SIG_BLOCK): # cleanup blocks SIGTERM and SIGINT; the holder must not care - holder = ht.hand_off(fh, BOARD, 'cleanup failed') + holder = ht.hand_off(fh, BOARD, 'cleanup failed', helper=self.helper) self.addCleanup(lambda: end_child(holder['pid'])) os.kill(holder['pid'], signal.SIGUSR1) ht.wait_for(lambda: ht.flock_free(self.lock), 5, 'release') @@ -2790,7 +2672,7 @@ def test_survivors_are_identified_against_pid_reuse(self): def test_the_holder_keeps_none_of_the_callers_stdio(self): fh = self.helper.acquire_board_lock(BOARD) - holder = ht.hand_off(fh, BOARD, 'x') + holder = ht.hand_off(fh, BOARD, 'x', helper=self.helper) self.addCleanup(lambda: end_child(holder['pid'])) ht.wait_for(lambda: not any(Path(f'/proc/{holder["pid"]}/fd/{std}').exists() for std in (0, 1, 2)), 5, 'the holder closing its stdio') @@ -2815,7 +2697,8 @@ def unblock(): # run_board leaves the signals blocked until exit; drop what is self.helper = pinned_helper(self, self.tmp / 'locks') self.board = self.load(roster(self.tmp)) self.facts = ht.source_facts(self.board) - self.args = mock.Mock(build_record='rec.json', fail_at=None, try_reentry=None, lock_wait=0) + self.args = mock.Mock(build_record='rec.json', fail_at=None, try_reentry=None, lock_wait=0, ci_state_root=None, + done_marker=None, started_marker=None) for p in (mock.patch.object(ht, 'verify_record', lambda *a: ({'hex': 'x.hex', 'bin': 'x.bin'}, {'version': 'v'})), mock.patch.object(ht, 'parse_hex', lambda p: {0: 1}), mock.patch.object(ht, 'hex_matches_bin', lambda d, b, base: True), @@ -2866,8 +2749,10 @@ def fail(bat, files, version): bat.touched = True raise ht.StepFail('x') for err in (ht.StepFail('verify failed'), ht.Refused('range'), ValueError('parse'), ht.Interrupted('TERM')): + # each holder recovered before the next run, its record cleared: the protected record would block that run with self.subTest(type(err).__name__), \ - mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (self.helper.clear_record(fh), fh.close(), + {'pid': 1, 'reason': 'hil_test.py'})[2]): row, code, _ = self.run_board(fail, cleanup_error=err) self.assertEqual((code, row['pass'], row['cleanup']['heldAfter'], row['cleanup']['holder']['pid']), (4, False, True, 1)) @@ -2882,7 +2767,7 @@ def snapshot_then_fail(bat, *a): raise ht.StepFail('read-back failed') with mock.patch.object(ht.Battery, 'report_fields', lambda bat: {'snapshot': getattr(bat, 'snap', None)}), \ mock.patch.object(ht.Battery, 'run', no_snapshot), mock.patch.object(ht.Battery, 'restore', snapshot_then_fail), \ - mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) self.assertEqual((code, row['cleanup']['pass'], row['snapshot']), (4, False, {'nvs': '0' * 32})) @@ -2895,7 +2780,7 @@ def test_fail_at_cleanup_skips_the_restore_and_hands_the_board_to_a_holder(self) def ok(bat, files, version): bat.touched = True self.args.fail_at = 'cleanup' - with mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): + with mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): row, code, cleaned = self.run_board(ok) self.assertEqual((code, cleaned, row['cleanup']['heldAfter']), (4, [], True)) self.assertIn('--fail-at cleanup', row['cleanup']['detail']) @@ -2941,9 +2826,9 @@ def test_a_signal_pending_at_a_locked_board_still_gives_its_row(self): signal.signal(signal.SIGTERM, ht.on_signal) real = self.helper.acquire_board_lock - def signalled_while_locking(board): + def signalled_while_locking(board, **kw): os.kill(os.getpid(), signal.SIGTERM) - return real(board) + return real(board, **kw) with mock.patch.object(self.helper, 'acquire_board_lock', signalled_while_locking): row, code, _ = self.run_board(lambda *a: self.fail('ran')) self.assertEqual((code, row['locked'], row['ran']), (1, True, False)) @@ -2955,7 +2840,7 @@ def fail(bat, files, version): raise ht.StepFail('x') stuck = [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}] with mock.patch.object(ht, 'settle_children', lambda started=(): list(stuck)), \ - mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'survivors': s})[1]): + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (fh.close(), {'pid': 1, 'survivors': s})[1]): row, code, cleaned = self.run_board(fail) self.assertEqual((code, cleaned, row['cleanup']['survivors']), (4, [], stuck)) self.assertEqual(row['cleanup']['detail'], 'StepFail: children survived SIGKILL; no flash may overlap them') @@ -2982,7 +2867,7 @@ def flashed_with_a_survivor(bat, files, version): # its keeper never started: raise ht.StepFail('children survived SIGKILL') with mock.patch.object(ht.Battery, 'run', flashed_with_a_survivor), \ mock.patch.object(ht.Battery, 'restore', restore), mock.patch.object(ht, 'BUILD_LOCK_WAIT', 0.3), \ - mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1})[1]): + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (fh.close(), {'pid': 1})[1]): row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) if ended: self.assertEqual((code, row['cleanup']['pass'], ht.HELD_BUILD), (1, True, [])) @@ -3004,7 +2889,7 @@ def fail(bat, files, version): # as flash_openocd_esp leaves it: its receipt o 'argv': ['openocd'], 'ports': [None, None, 1], 'token': 't', 'log': 'l'})) raise ht.CleanupPending('the openocd_esp session is still running: cleanup pending') handed = [] - with mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (handed.append(s), fh.close(), {'pid': 1})[2]): + with mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (handed.append(s), fh.close(), {'pid': 1})[2]): row, code, cleaned = self.run_board(fail) want = [{'pid': server.pid, 'start': ht.proc_start(server.pid), 'cmd': 'openocd', 'receipt': str(receipt)}] self.assertEqual((code, cleaned, row['cleanup']['state'], row['cleanup']['receipts'], handed), @@ -3032,7 +2917,7 @@ def fail(bat, files, version): def enospc(*a): raise OSError(28, 'No space left on device') with mock.patch.object(ht, 'write_mark', enospc), mock.patch.object(ht, 'MARK_ERRORS', []), \ - mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (handed.append(s), fh.close(), {'pid': 1})[2]): + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (handed.append(s), fh.close(), {'pid': 1})[2]): row, code, cleaned = self.run_board(fail) self.assertEqual((code, row['cleanup']['state'], row['cleanup']['receipts'], len(handed)), (4, 'pending', [str(receipt)], 1)) # held by its holder, the session reported @@ -3043,7 +2928,7 @@ def test_a_survivor_on_an_untouched_board_still_keeps_it_held(self): def fail(bat, files, version): raise ht.StepFail('install: probe hung') with mock.patch.object(ht, 'settle_children', lambda started=(): [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}]), \ - mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1})[1]): + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (fh.close(), {'pid': 1})[1]): row, code, cleaned = self.run_board(fail) self.assertEqual((code, cleaned, row['cleanup']['state'], row['cleanup']['heldAfter']), (4, [], 'untouched', True)) @@ -3175,6 +3060,362 @@ def test_fail_at_injects_a_criterion_failure(self): with self.assertRaisesRegex(ht.StepFail, 'injected'): bat.step('dfu', lambda: {}) + def test_the_battery_holds_a_protected_record_naming_its_run_dir_and_a_clean_exit_releases_it(self): + seen = [] + + def ok(bat, files, version): + seen.append(self.helper.read_record(BOARD)) + bat.touched = True + row, code, _ = self.run_board(ok) + self.assertEqual(code, 0) + self.assertEqual({k: seen[0].get(k) for k in ('reason', 'protected', 'runDir')}, + {'reason': 'hil_test.py', 'protected': True, 'runDir': str(self.tmp)}) + self.assertTrue(self.helper.is_protected(seen[0])) + self.assertEqual(self.helper.record_state(BOARD)[0], 'empty') + self.helper.release_board_lock(self.helper.take_board_lock(BOARD, 'next')) # no recovery blocker left + + def test_every_release_before_the_battery_clears_the_protected_record(self): + def no_bridge(board): + raise ht.StepFail('0 serial ports for bridge cp2102n: []') + + def unexpected(*a, **k): + raise ValueError('battery setup') + for what, patch in (('probe check', mock.patch.object(ht, 'probe_users', no_bridge)), + ('probe users', mock.patch.object(ht, 'probe_users', lambda b: [101])), + ('battery setup', mock.patch.dict(ht.BATTERIES, {'jlink': unexpected}))): + with self.subTest(what), patch, contextlib.suppress(ValueError): + self.run_board(lambda *a: self.fail('ran')) + self.assertEqual(self.helper.record_state(BOARD)[0], 'empty', what) + self.helper.release_board_lock(self.helper.take_board_lock(BOARD, 'next')) + + def test_a_dead_batterys_record_or_a_takeover_journal_is_a_blocker_never_waited_for(self): + self.args.lock_wait = 100 + for what in ('stale record', 'takeover journal'): + with self.subTest(what): + lockdir = self.tmp / what.replace(' ', '-') + lockdir.mkdir() + with mock.patch.object(self.helper, 'BOARD_LOCK_DIR', str(lockdir)): + if what == 'stale record': # as a battery killed by SIGKILL leaves it: closed, never cleared + self.helper.take_board_lock(BOARD, 'hil_test.py', protected=True).close() + else: + (lockdir / f'{BOARD}.takeover').write_text('{}') + t0 = time.monotonic() + row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) + self.assertLess(time.monotonic() - t0, 5) + self.assertEqual((code, row['ran'], row['locked'], row['recoveryBlocker'], cleaned), (1, False, True, True, [])) + self.assertIn(what, row['detail']) + with self.assertRaises(self.helper.RecoveryBlocked): # still blocked: nothing cleared it + self.helper.take_board_lock(BOARD, 'next') + + def ci_state(self): + root = self.tmp / 'hil-state' + for run, done in (('run-1', True), ('run-2', False)): + (root / run).mkdir(parents=True) + (root / run / 'other_board.started').touch() + if done: + (root / run / f'{BOARD}.started').touch() + (root / run / f'{BOARD}.done').touch() + return root + + def test_finished_ci_attempts_let_the_run_through(self): + self.args.ci_state_root = self.ci_state() + row, code, cleaned = self.run_board(lambda bat, f, v: setattr(bat, 'touched', True)) + self.assertEqual((code, row['pass'], cleaned), (0, True, [True])) + self.assertEqual(self.helper.record_state(BOARD)[0], 'empty') + + def test_an_old_ci_attempt_that_died_between_a_preflight_and_the_lock_keeps_a_blocker(self): + root = self.args.ci_state_root = self.ci_state() + self.assertEqual(ht.hil_ci.unfinished(root, BOARD), []) # the driver's preflight: nothing unfinished + real = self.helper.acquire_board_lock + + def old_ci_started_and_died(board, **kw): # its unprotected lock dropped with it; its marker stays + (root / 'run-3').mkdir() + (root / 'run-3' / f'{BOARD}.started').touch() + return real(board, **kw) + with mock.patch.object(self.helper, 'acquire_board_lock', old_ci_started_and_died): + row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['ran'], row['recoveryBlocker'], row['unfinishedAttempts'], cleaned), + (1, False, True, ['run-3'], [])) + self.assertIn('unfinished attempt', row['detail']) + rec = self.helper.read_record(BOARD) + self.assertTrue(self.helper.is_recovery_blocker(rec)) # kept, not cleared as untouched + self.assertIn('run-3', rec['detail']) + with self.assertRaises(self.helper.RecoveryBlocked): + self.helper.take_board_lock(BOARD, 'next') + self.assertEqual(sorted(p.name for p in (root / 'run-3').iterdir()), [f'{BOARD}.started']) # CI's state unwritten + + def test_a_ci_batterys_done_marker_is_visible_before_its_lock_is_free(self): + root = self.tmp / 'hil-state' + (root / 'ci-1').mkdir(parents=True) + (root / 'ci-1' / f'{BOARD}.started').touch() + done = self.args.done_marker = root / 'ci-1' / f'{BOARD}.done' + seen, real = [], ht.release + + def release(helper, fh): # a remote runner taking the lock right after this release sees nothing unfinished + seen.append((done.exists(), ht.hil_ci.unfinished(root, BOARD))) + real(helper, fh) + with mock.patch.object(ht, 'release', release): + row, code, cleaned = self.run_board(lambda bat, f, v: setattr(bat, 'touched', True)) + self.assertEqual((code, row['pass'], cleaned, seen), (0, True, [True], [(True, [])])) + self.args.done_marker, self.args.ci_state_root = None, root + row, code, cleaned = self.run_board(lambda bat, f, v: setattr(bat, 'touched', True)) + self.assertEqual((code, row['pass'], cleaned, 'unfinishedAttempts' in row), (0, True, [True], False)) + + def test_only_a_battery_that_is_no_blocker_touches_its_done_marker(self): + def touched_then_failed(bat, files, version): + bat.touched = True + bat.step('app-ran', lambda: (_ for _ in ()).throw(ht.StepFail('pc not in the loop'))) + + def aborted(bat, files, version): # no failed step: an abort, a blocker + raise ValueError('boom') + holder = mock.patch.object(ht, 'hand_off', lambda fh, b, d, s, **_: (self.helper.clear_record(fh), fh.close(), + {'pid': 1, 'reason': 'hil_test.py'})[2]) + for name, battery, cleanup_error, want in (('fail', touched_then_failed, None, True), + ('aborted', aborted, None, False), + ('cleanup holder', touched_then_failed, ht.StepFail('verify'), False)): + with self.subTest(name), holder: + done = self.args.done_marker = self.tmp / f'{name}.done' + row, code, _ = self.run_board(battery, cleanup_error=cleanup_error) + self.assertEqual((done.exists(), ht.run_report.verdict(code, {'results': [row]}, BOARD)[0] != 'blocker'), + (want, want)) + + def test_a_ci_batterys_started_marker_is_touched_only_once_it_holds_the_board(self): + (self.tmp / 'ci-1').mkdir() + started = self.args.started_marker = self.tmp / 'ci-1' / f'{BOARD}.started' + seen, real = [], self.helper.acquire_board_lock + + def acquire(board, **kw): # while the board may still be another run's, no run may see CI's battery started + seen.append(('acquire', started.exists())) + return real(board, **kw) + + def ran(bat, files, version): + seen.append(('battery', started.exists())) + bat.touched = True + with mock.patch.object(self.helper, 'acquire_board_lock', acquire): + row, code, cleaned = self.run_board(ran) + self.assertEqual((code, row['pass'], cleaned, seen), (0, True, [True], [('acquire', False), ('battery', True)])) + + def test_a_started_marker_that_cannot_be_written_refuses_and_releases(self): + self.args.started_marker = self.tmp / 'missing' / f'{BOARD}.started' + with self.assertRaisesRegex(ht.Refused, 'started marker'): + self.run_board(lambda *a: self.fail('ran')) + self.assertEqual(self.helper.record_state(BOARD)[0], 'empty') + self.helper.release_board_lock(self.helper.take_board_lock(BOARD, 'next')) + + def test_an_unreadable_ci_state_root_refuses_and_releases(self): + unreadable = self.tmp / 'unreadable' + unreadable.mkdir(mode=0) + self.addCleanup(unreadable.chmod, 0o700) + roots = [self.tmp / 'missing'] + ([unreadable] if os.geteuid() else []) + for root in roots: + with self.subTest(root.name): + self.args.ci_state_root = root + with self.assertRaisesRegex(ht.Refused, 'cannot be read'): + self.run_board(lambda *a: self.fail('ran')) + self.assertEqual(self.helper.record_state(BOARD)[0], 'empty') + self.helper.release_board_lock(self.helper.take_board_lock(BOARD, 'next')) + + +TAKE_OVER = ''' +import importlib.util, sys +spec = importlib.util.spec_from_file_location('hil_lock', sys.argv[1]) +helper = importlib.util.module_from_spec(spec) +spec.loader.exec_module(helper) +helper.BOARD_LOCK_DIR = sys.argv[2] +sys.exit(helper.cmd_hold([], 'recovery 7', take_over_board=sys.argv[3])) +''' + + +class StaleRecovery(Tmp): + """The stale-record route: `hil_lock.py hold --take-over`, then `restore --recover-takeover`, which proves the + takeover and what the dead battery left, restores and releases the token.""" + + def setUp(self): + super().setUp() + mask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) + self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) + lockdir = self.tmp / 'locks' + lockdir.mkdir() + with mock.patch.object(ht, 'check_lock_pin'): + self.helper = pinned_helper(self, lockdir) + self.cfg = roster(self.tmp) + self.run_dir = self.tmp / 'run-dead' + self.run_dir.mkdir() + self.restored = [] + for p in (mock.patch.object(ht, 'load_helper', lambda: self.helper), mock.patch.object(ht, 'probe_users', lambda b: []), + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), + mock.patch.object(ht.Battery, 'restore', lambda bat, f, v: self.restored.append(1) or + {'pass': True, 'state': 'blinky copied', 'evidence': {}})): + p.start() + self.addCleanup(p.stop) + + def stale(self, lock=BOARD, **rec): + """A record left on a free flock, as a battery killed by SIGKILL leaves its protected one.""" + fh = self.helper.flock_nb(lock) + self.helper.write_record(fh, rec.pop('reason', 'hil_test.py'), **rec) + fh.close() + return self.helper.read_record(lock) + + def take_over(self, board=BOARD): + r = subprocess.run([sys.executable, '-c', TAKE_OVER, str(ht.LOCK_HELPER), self.helper.BOARD_LOCK_DIR, board], + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + holder = self.helper.read_record(board) + + def end(): + if self.helper.proc_start(holder['pid']) == holder['start']: + os.kill(holder['pid'], signal.SIGKILL) + self.addCleanup(end) + return re.search(r'token (\w+)', r.stdout).group(1), holder + + def main(self, *argv): + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', Path(tempfile.mkdtemp(dir=self.tmp))): + code = ht.main(list(argv)) + return code, json.loads(out.getvalue().splitlines()[-1]) + + def restore(self, token, *extra, board=BOARD, cfg=None): + return self.main('restore', '-b', board, '--build-record', 'r.json', '--recover-takeover', token, *extra, + str(cfg or self.cfg)) + + def report(self, pid, board=BOARD, **row): + path = self.tmp / f'report-{pid}.json' + path.write_text(json.dumps({'results': [dict({'board': board, 'cleanup': {'holder': {'pid': pid}, 'state': 'unknown', + 'survivors': []}}, **row)]})) + return str(path) + + def assert_released(self, board=BOARD): + self.assertIn(self.helper.record_state(board)[0], ('absent', 'empty')) + self.assertIsNone(self.helper.journal_state(board)) + self.helper.release_board_lock(self.helper.take_board_lock(board, 'next')) + + def assert_kept(self, token, holder, board=BOARD): + self.assertEqual(self.helper.read_record(board), holder) # the takeover stays, and its journal + self.assertEqual(self.helper.journal_state(board)['token'], token) + + def test_a_fresh_hold_is_refused_and_the_takeover_route_restores_and_releases(self): + for what, rec in (('a battery that died', {'protected': True, 'runDir': str(self.run_dir)}), + ('a cleanup holder that died', {'role': ht.HOLDER_ROLE, 'board': BOARD, 'survivors': []})): + with self.subTest(what): + self.restored.clear() + dead = self.stale(**rec) + with self.assertRaises(self.helper.RecoveryBlocked): + self.helper.take_board_lock(BOARD, 'debug session 7') + token, holder = self.take_over() + extra = ['--report', self.report(dead['pid'])] if 'role' in rec else [] + code, res = self.restore(token, *extra) + self.assertEqual((code, self.restored), (0, [1]), res) + recovered = res['results'][0]['cleanup']['recovered'] + self.assertEqual((recovered['takeover'], recovered['holder'], recovered['record']), (token, holder['pid'], dead)) + self.assert_released() + + def test_missing_recovery_evidence_keeps_the_takeover_and_the_blocker(self): + ended = {'tool': 'hil_openocd_esp', 'closed': True, 'cleanup': 'done'} + sessions = {'pending': self.tmp / 'pending', 'no receipt': self.tmp / 'no-receipt', 'unlistable': self.tmp / 'unlistable', + 'unlistable session': self.tmp / 'unlistable-session'} + for name, d in sessions.items(): + (d / 'hil_openocd_esp_x').mkdir(parents=True) + if name != 'no receipt': + (d / 'hil_openocd_esp_x' / 'receipt.json').write_text(json.dumps(dict(ended, cleanup='pending') if name == 'pending' + else ended)) + for d, at in ((sessions['unlistable'], sessions['unlistable']), + (sessions['unlistable session'], sessions['unlistable session'] / 'hil_openocd_esp_x')): + at.chmod(0) + self.addCleanup(at.chmod, 0o700) + holder_rec = {'role': ht.HOLDER_ROLE, 'board': BOARD, 'survivors': []} + cases = [('not a battery record', {'reason': 'farm_check', 'protected': True}, (), 'not a battery record'), + ('a cleanup holder of another board', dict(holder_rec, board='other'), (), 'not a battery record'), + ('no run dir', {'protected': True}, (), 'names neither'), + ('a run dir gone', {'protected': True, 'runDir': str(self.tmp / 'gone')}, (), 'cannot be read'), + ('a session not ended', {'protected': True, 'runDir': str(sessions['pending'])}, (), 'not ended'), + ('a session without its receipt', {'protected': True, 'runDir': str(sessions['no receipt'])}, (), + 'holds no receipt.json'), + ('a report for a reportless battery', {'protected': True, 'runDir': str(self.run_dir)}, + ('--report', self.report(1)), 'printed no report'), + ('a dead cleanup holder without its report', holder_rec, (), 'needs --report'), + ("a report naming another holder", holder_rec, ('--report', self.report(4242)), 'names no cleanup holder')] + if os.geteuid(): # root lists any dir + cases += [('an unlistable run dir', {'protected': True, 'runDir': str(sessions['unlistable'])}, (), 'cannot be read'), + ('an unlistable session dir', {'protected': True, 'runDir': str(sessions['unlistable session'])}, (), + 'cannot be read')] + for what, rec, extra, want in cases: + with self.subTest(what): + lockdir = self.tmp / what.replace(' ', '-') + lockdir.mkdir() + with mock.patch.object(self.helper, 'BOARD_LOCK_DIR', str(lockdir)): + self.stale(**rec) + token, holder = self.take_over() + code, res = self.restore(token, *extra) + self.assertEqual((code, self.restored), (2, []), res) + self.assertIn(want, res['error']) + self.assert_kept(token, holder) + + def test_another_token_or_no_takeover_is_refused(self): + self.stale(protected=True, runDir=str(self.run_dir)) + code, res = self.restore('feedface') + self.assertEqual((code, self.restored), (2, [])) + self.assertIn('is not the takeover holder of token feedface', res['error']) + token, holder = self.take_over() + code, res = self.restore('feedface') + self.assertEqual((code, self.restored), (2, [])) + self.assert_kept(token, holder) + + def test_an_esptool_takeover_checks_the_snapshot_in_the_dead_holders_report(self): + cfg = s3_roster(self.tmp) + keep = self.load(cfg, S3)['regions']['preserved'] + lock = ht.load_board(cfg, S3)['lock'] + md5 = {n: 'ab' * 16 for n in keep} + + def snapshot(regions, sums): + return {'snapshot': {'preserved': sums, 'preservedRegions': {n: list(r) for n, r in regions.items()}}} + seen = [] + + def restore(bat, f, v): # the restore's own check against the snapshot it was given, the device as `found` + seen.append(bat.preserved) + bat.touched = True + ht.check_preserved(bat.preserved, found, 'restore') + return {'pass': True, 'state': 'blinky copied', 'evidence': {}} + cases = (('a matching snapshot', snapshot(keep, md5), md5, 0, ''), + ('a changed region', snapshot(keep, md5), dict(md5, nvs='cd' * 16), 4, 'preserved region changed'), + ('no snapshot', {}, md5, 2, 'no preserved-region snapshot'), + ('another layout', snapshot({'nvs': (0x9000, 0xe000)}, {'nvs': 'ab' * 16}), md5, 2, 'this checkout preserves')) + for what, row, found, want, err in cases: + with self.subTest(what), mock.patch.object(ht.EspBattery, 'restore', restore): + seen.clear() + lockdir = self.tmp / what.replace(' ', '-') + lockdir.mkdir() + with mock.patch.object(self.helper, 'BOARD_LOCK_DIR', str(lockdir)): + dead = self.stale(lock, role=ht.HOLDER_ROLE, board=lock, survivors=[]) + token, holder = self.take_over(lock) + code, res = self.restore(token, '--report', self.report(dead['pid'], S3, **row), board=S3, cfg=cfg) + self.assertEqual(code, want, res) + self.assertEqual(seen, [md5] if want != 2 else []) # the dead run's snapshot, never a fresh one + if want: + self.assertIn(err, res['error']) + self.assert_kept(token, holder, lock) + else: + self.assert_released(lock) + + def test_a_reviewed_recovery_under_the_takeover_clears_the_pending_mark_and_the_route_then_completes(self): + receipt = self.run_dir / 'hil_openocd_esp_x' / 'receipt.json' + receipt.parent.mkdir() + receipt.write_text(json.dumps({'tool': 'hil_openocd_esp', 'closed': True, 'cleanup': 'pending'})) # unresolved + session = {'cmd': 'openocd', 'receipt': str(receipt)} + ht.mark_pending(ht.PendingMark(None, BOARD), [session]) # as the failed cleanup left it + dead = self.stale(role=ht.HOLDER_ROLE, board=BOARD, survivors=[session]) + token, holder = self.take_over() + report = self.report(dead['pid'], cleanup={'holder': {'pid': dead['pid']}, 'state': 'pending', 'survivors': [session]}) + code, res = self.restore(token, '--report', report) + self.assertEqual((code, self.restored), (2, [])) # the mark refuses every invocation first + code, res = self.main('clear-pending', '-b', BOARD, '--after-reviewed-recovery', 'server ended by the reviewed recovery', + '--under-hold', 'recovery 7', str(self.cfg)) + self.assertEqual(code, 0, res) + self.assertEqual(res['cleared']['hold'], holder) # the takeover holder proved as the hold + self.assertFalse(Path(ht.PendingMark(None, BOARD)).exists()) + code, res = self.restore(token, '--report', report) + self.assertEqual((code, self.restored), (0, [1]), res) + self.assert_released() + def one_attempt(result): """A provider Operation of the one run `result`, for a test's flash op.""" @@ -4091,6 +4332,201 @@ def test_a_lock_pin_refusal_stops_a_run_before_its_record_and_battery(self): battery.assert_not_called() +class RunReport(Tmp): + """The run dir's report through agentrc's report core: `_hil` beside the native fields, a Markdown view of the steps, + the runner's exit kept natively, and a publication failure that changes neither the stdout report nor the hold.""" + + FW = {'path': '/b/tinyuf2.bin', 'sha256': 'ab' * 32, 'buildRecord': 'r.json', 'defines': [], 'version': 'v'} + ROW = {'board': BOARD, 'lock': BOARD, 'ran': True, 'pass': True, 'locked': False, 'lockBypassed': False, + 'steps': [{'step': 'install', 'pass': True, 'evidence': {}}], 'detail': '', 'firmware': FW, + 'cleanup': {'pass': True, 'state': 'blinky copied', 'evidence': {}, 'heldAfter': False}} + HELD = {'pass': False, 'state': 'pending', 'evidence': {}, 'heldAfter': True, 'receipts': ['/r/receipt.json'], + 'survivors': [{'cmd': 'openocd', 'receipt': '/r/receipt.json'}], 'holder': {'pid': 9, 'reason': 'hil_test.py'}} + + def setUp(self): + super().setUp() + self.cfg = roster(self.tmp) + self.core = ht.report_core() + self.kept = [] + for p in (mock.patch.object(ht, 'load_helper', lambda: None), mock.patch.object(ht, 'HELD_IN_PROCESS', []), + mock.patch.object(ht, 'keep_held', self.kept.append)): + p.start() + self.addCleanup(p.stop) + + def run_main(self, row, code, in_process=False, cfg=None): + """main() over a battery that returns (row, code); `in_process` makes this process the cleanup holder.""" + def run_board(args, board, facts, helper, workdir): + if in_process: + ht.HELD_IN_PROCESS.append('fh') + return json.loads(json.dumps(row)), code + ht.HELD_IN_PROCESS.clear() + root = Path(tempfile.mkdtemp(dir=self.tmp)) + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht, 'run_board', run_board): + got = ht.main(['run', '-b', BOARD, '--build-record', 'r.json', '--run-root', str(root), str(cfg or self.cfg)]) + run_dir, = root.iterdir() + return got, json.loads(out.getvalue().splitlines()[-1]), run_dir + + def test_the_report_carries_its_metadata_beside_the_native_fields_and_a_markdown_view(self): + code, out, run_dir = self.run_main(self.ROW, 0) + doc = json.loads((run_dir / 'hil_report.json').read_text()) + meta = doc.pop('_hil') + self.assertEqual(code, 0) + self.assertEqual(doc, out) # the stdout report is the file's native body + self.assertEqual((doc['exitCode'], doc['heldInProcess'], doc['reportPath']), + (0, False, str(run_dir / 'hil_report.json'))) + self.assertEqual({k: v for k, v in meta.items() if k != 'version'}, { + 'body_schema': 'tinyuf2-hil-run', 'body_version': 1, 'attempt_id': run_dir.name, 'previous_attempt': None, + 'termination': 'completed', 'selection': [BOARD], 'host': socket.gethostname(), + 'invocation': ['hil_test.py', 'run', '-b', BOARD, '--build-record', 'r.json', '--run-root', str(run_dir.parent), + str(self.cfg)], + 'roster': {'name': self.cfg.name, 'sha256': ht.sha256(self.cfg)}, + 'build': {k: self.FW[k] for k in ('buildRecord', 'path', 'sha256', 'version')}}) + self.assertEqual((run_dir / 'hil_report.md').read_text(encoding='utf-8'), ht.run_report.render(self.core, doc)) + + def test_a_refused_run_names_no_build_it_never_admitted(self): + cfg = roster(self.tmp, flasher={'name': 'jlink', 'uid': '1', 'args': '-device NOPE'}) + code, out, run_dir = self.run_main(self.ROW, 0, cfg=cfg) + meta = json.loads((run_dir / 'hil_report.json').read_text())['_hil'] + self.assertEqual((code, out['exitCode'], out['results'][0]['ran']), (2, 2, False)) + self.assertNotIn('build', meta) + self.assertEqual(meta['termination'], 'completed') + + def test_the_markdown_renders_each_boards_steps_and_cleanup(self): + body = {'pass': False, 'caveat': '', 'error': 'one board failed', 'reportPath': 'x', 'results': [ + {'board': 'metro_m7_1011', 'steps': [{'step': 'install', 'pass': True}, {'step': 'copy', 'pass': False, + 'detail': 'drive | vanished\nhere'}], + 'cleanup': {'pass': True, 'state': 'blinky copied'}}, + {'board': 'imxrt1064_evk', 'ran': False, 'locked': True, 'detail': 'held by pid 7', 'steps': []}]} + self.assertEqual(ht.run_report.render(self.core, body), textwrap.dedent('''\ + HIL run: fail + Error: one board failed + + ### metro_m7_1011 + + | Step | Result | Detail | + | ------- | :----: | ---------------------- | + | install | ✅ | | + | copy | ❌ | drive \\| vanished here | + | cleanup | ✅ | blinky copied | + + ### imxrt1064_evk + + held by pid 7 + + _No steps ran._ + ''')) + + def test_the_runners_exit_survives_the_file_round_trip(self): + failed = dict(self.ROW, **{'pass': False, 'steps': [{'step': 'copy', 'pass': False, 'detail': 'drive vanished'}]}) + for what, row, code, in_process, want in ( + ('a pass', self.ROW, 0, False, 'pass'), + ('a fail', failed, 1, False, 'fail'), + ('a passing row whose run exits 4', self.ROW, 4, False, 'blocker'), # e.g. a pending mark not written + ('an in-process holder', dict(failed, cleanup=dict(self.HELD, holder={'pid': 9, 'inProcess': True})), 4, True, + 'blocker')): + with self.subTest(what): + got, out, run_dir = self.run_main(row, code, in_process) + body = ht.read_report(run_dir / 'hil_report.json') + seen = None if in_process else got # hil_ci.launch gets no exit from a runner that stays the holder + self.assertEqual(ht.run_report.verdict(ht.run_report.exit_of(body), body, BOARD)[0], want) + self.assertEqual(ht.run_report.verdict(ht.run_report.exit_of(body), body, BOARD), + ht.run_report.verdict(seen, out, BOARD)) + + def test_a_report_that_cannot_be_published_still_prints_and_keeps_the_hold(self): + def refuse(*a, **k): + raise self.core.ReportError('replace', 'No space left on device') + failures = (('the core refuses', lambda: mock.patch.object(self.core, 'publish', refuse), 'No space left'), + ('the core does not load', lambda: mock.patch.object(ht, 'report_core', mock.Mock( + side_effect=ht.Refused('report core x failed to load'))), 'failed to load')) + for what, failure, err in failures: + for in_process in (True, False): + row = dict(self.ROW, **{'pass': False, 'cleanup': dict(self.HELD, holder=dict(self.HELD['holder'], **( + {'inProcess': True} if in_process else {})))}) + self.kept.clear() + with self.subTest(what, in_process=in_process), failure(): + code, out, run_dir = self.run_main(row, 4, in_process) + self.assertEqual((code, out['exitCode'], out['reportPath']), (4, 4, None)) + self.assertIn(err, out['reportError']) + self.assertEqual(out['results'][0]['cleanup'], row['cleanup']) + self.assertEqual(self.kept, ['fh'] if in_process else []) # the holder lifecycle runs after the report + self.assertEqual(sorted(p.name for p in run_dir.iterdir() if p.name.startswith('hil_report')), []) + + def test_a_markdown_that_cannot_be_replaced_leaves_the_json_named_and_the_error_explicit(self): + real = os.replace + + def replace(src, dst): + if str(dst).endswith('hil_report.md'): + raise OSError(28, 'No space left on device') + return real(src, dst) + with mock.patch.object(self.core.os, 'replace', replace): + code, out, run_dir = self.run_main(self.ROW, 0) + self.assertEqual((code, out['reportPath']), (0, str(run_dir / 'hil_report.json'))) + self.assertIn('markdown', out['reportError']) + self.assertEqual(ht.read_report(out['reportPath'])['results'], out['results']) + self.assertFalse((run_dir / 'hil_report.md').exists()) + + +class ReportAdmission(Tmp): + """A recovery's --report goes through the report core's metadata and schema check before any recovery lock, then + this runner's own evidence checks.""" + + ROW = {'board': BOARD, 'ran': True, 'pass': False, 'steps': [], + 'cleanup': {'holder': {'pid': 4242}, 'state': 'unknown', 'survivors': [], 'heldAfter': True}} + + def meta(self, **over): + core = ht.report_core() + return dict(core.make_meta(ht.run_report.schema(core), core.Attempt('run-1', None, None, None, None), + termination='completed', invocation=['hil_test.py', 'run'], selection=[BOARD], host='h', + roster={'path': 'r.json'}), **over) + + def test_an_unreadable_or_unsupported_report_is_refused_before_the_recovery_lock(self): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + body = {'pass': False, 'caveat': '', 'results': [self.ROW], 'reportPath': 'x'} + cases = (('not JSON', '{', 'corrupt'), + ('a legacy report without results', {'pass': False}, 'corrupt'), + ('rows that are not objects', dict(body, results=['x']), 'corrupt'), + ('a _hil that is no object', dict(body, _hil='x'), 'corrupt'), + ('a _hil without its attempt id', dict(body, _hil={k: v for k, v in self.meta().items() if k != 'attempt_id'}), + 'corrupt'), + ('an unknown _hil version', dict(body, _hil=self.meta(version=2)), 'unsupported'), + ('another body schema', dict(body, _hil=self.meta(body_schema='tinyusb-hil-report')), 'unsupported'), + ('an unknown body version', dict(body, _hil=self.meta(body_version=2)), 'unsupported')) + for flag, value in (('--recover-holder', '4242'), ('--recover-takeover', 'feedface')): + for what, doc, outcome in cases: + report = self.tmp / 'failed.json' + report.write_text(doc if isinstance(doc, str) else json.dumps(doc)) + lock, out = mock.Mock(side_effect=AssertionError('recovery lock taken')), io.StringIO() + with self.subTest(what, flag=flag), contextlib.redirect_stdout(out), \ + mock.patch.object(ht.cb, 'BUILD_DIR', Path(tempfile.mkdtemp(dir=self.tmp))), \ + mock.patch.object(ht, 'load_helper', mock.Mock), mock.patch.object(ht, 'recovery_lock', lock), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ + mock.patch.object(ht, 'prove_hold', side_effect=AssertionError('checked the holder')), \ + mock.patch.object(ht, 'takeover_predecessor', side_effect=AssertionError('checked the takeover')), \ + mock.patch.object(ht.Battery, 'restore', side_effect=AssertionError('probed')): + code = ht.main(['restore', '-b', BOARD, '--build-record', 'r.json', flag, value, '--report', str(report), + str(cfg)]) + self.assertEqual(code, 2) + self.assertTrue(json.loads(out.getvalue().splitlines()[-1])['error'].startswith( + f'cannot read {report}: {outcome}: '), out.getvalue()) + lock.assert_not_called() + + def test_a_legacy_report_and_one_this_runner_published_are_both_admitted(self): + cfg = roster(self.tmp) + legacy = {'pass': False, 'caveat': '', 'results': [self.ROW], 'reportPath': str(self.tmp / 'old' / 'hil_report.json'), + 'error': 'cleanup failed'} + old = self.tmp / 'legacy.json' + old.write_text(json.dumps(legacy, indent=1)) # as every runner before `_hil` wrote it + run_dir = self.tmp / 'run-20261020-120000-7' + run_dir.mkdir() + published = dict(legacy, reportPath=str(run_dir / 'hil_report.json'), exitCode=4, heldInProcess=False) + ht.publish_report(published, run_dir, argparse.Namespace(config=cfg, board=BOARD), ['run', '-b', BOARD]) + self.assertNotIn('reportError', published) + for report in (old, run_dir / 'hil_report.json'): + with self.subTest(report.name): + self.assertEqual(ht.failed_row(ht.read_report(report), report, {'name': BOARD}, 4242), self.ROW) + + H563 = {'port': 'stm32h5', 'name': 'stm32h563_nucleo', 'device': 'stm32h563zi', 'flasher': 'stlink'} CUBE_CONNECT = ('ST-LINK SN : 0023\nST-LINK FW : V3J16M9\nBoard : NUCLEO-H563ZI\nVoltage : 3.26V\n' 'SWD freq : 8000 KHz\nConnect mode: Hot Plug\nReset mode : Software reset\nDevice ID : {dev}\n' diff --git a/.claude/test/test_hil_ci.py b/.claude/test/test_hil_ci.py index 065d873d7..c79c6bd32 100644 --- a/.claude/test/test_hil_ci.py +++ b/.claude/test/test_hil_ci.py @@ -3,11 +3,17 @@ # The CI job's orchestration of the battery (hil_ci.py): record, one run per board, verdicts, markers, pruning # and the exit. check_build is stubbed; the battery is a stand-in script that prints a canned report into the run # root it is given, so the streaming, the run directory and the in-process holder path run for real. +import contextlib import io import json import os +import re +import signal +import subprocess import sys import tempfile +import textwrap +import threading import time import unittest from pathlib import Path @@ -16,19 +22,32 @@ ROOT = Path(__file__).resolve().parents[2] sys.path.insert(0, str(ROOT / '.claude' / 'skills' / 'hil-contract' / 'scripts')) import hil_ci # noqa: E402 +import hil_run_report as run_report # noqa: E402 import hil_test as ht # noqa: E402 BOARD = 'imxrt1064_evk' +CHECK_PIN = hil_ci.check_pin # the real one: the tests patch it, as a commit's git env would fail it STAND_IN = ''' import json, os, sys, time one = json.load(open(os.environ["HIL_STAND_IN"])) +if "--started-marker" in sys.argv and not one["row"].get("locked"): # the battery's own touch once it holds the board + open(sys.argv[sys.argv.index("--started-marker") + 1], "w").close() run_dir = os.path.join(sys.argv[sys.argv.index("--run-root") + 1], "run-1") os.makedirs(run_dir) open(os.path.join(run_dir, "hil.log"), "w").write("log\\n") report = {"pass": one["row"].get("pass", False), "caveat": "", "results": [dict(board=sys.argv[sys.argv.index("-b") + 1], **one["row"])], "reportPath": os.path.join(run_dir, "hil_report.json")} -open(report["reportPath"], "w").write(json.dumps(report)) +report.update(one.get("report", {})) +doc = json.dumps(dict(report, _hil=one["meta"]) if "meta" in one else report) +# a run interrupted between its JSON and Markdown replacement: the JSON cut short, or an earlier Markdown left +open(report["reportPath"], "w").write(doc[:one["cut"]] if "cut" in one else doc) +if "md" in one: + open(os.path.join(run_dir, "hil_report.md"), "w").write(one["md"]) +if one.get("touch_done"): # the battery's own touch before it released the board + open(sys.argv[sys.argv.index("--done-marker") + 1], "w").close() +if one.get("unlistable"): # its run root left unlistable: names still open, a listing is refused + os.chmod(os.path.dirname(run_dir), 0o300) print("noise", " ".join(sys.argv[1:])) print(json.dumps(report), flush=True) if one.get("hang"): @@ -38,6 +57,49 @@ ''' +# run_report.verdict() before it moved into the report adapter (tinyuf2 dbe43d3), over the report core's tinyuf2 inputs +# (lib/agentrc tests/data/hil_report_core/inputs.json) and phase A's unfinished-attempt row: (exit, report, verdict) +HELD_ROW = {'board': 'B', 'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': True}]} +GOLDEN = { + 'no_report': (137, None, ('blocker', 'run ended without a report (exit 137)')), + 'no_row': (1, {'results': [], 'error': 'boom'}, ('blocker', 'report has no row for B (exit 1): boom')), + 'ran_none': (1, {'results': [{'board': 'B', 'ran': None, 'pass': False}]}, + ('blocker', 'cleanup failed or no report; board held: None {}')), + 'refused_before_hardware': (2, {'results': [{'board': 'B', 'ran': False, 'detail': 'no probe'}]}, ('not-run', 'no probe')), + 'probe_users': (2, {'results': [{'board': 'B', 'ran': False, 'probeUsers': [5], 'detail': 'p'}]}, ('blocker', 'p')), + 'untouched_failure': (1, {'results': [{'board': 'B', 'ran': True, 'pass': False, + 'steps': [{'step': 'install', 'pass': False, 'detail': 'x'}], + 'cleanup': {'pass': True, 'state': 'untouched'}}]}, + ('not-run', 'install: x (nothing written)')), + 'abort_without_failed_step': (1, {'results': [{'board': 'B', 'ran': True, 'pass': False, 'steps': [], + 'cleanup': {'pass': True, 'state': 'untouched'}}]}, + ('blocker', 'aborted: None')), + 'held_after_cleanup': (4, {'results': [dict(HELD_ROW, cleanup={'pass': False, 'state': 'blinky copied', 'heldAfter': True, + 'holder': {'pid': 9}})]}, + ('blocker', "cleanup failed or no report; board held: None {'pass': False, 'state': 'blinky " + "copied', 'heldAfter': True, 'holder': {'pid': 9}}")), + 'in_process_holder': (None, {'results': [dict(HELD_ROW, cleanup={'pass': False, 'heldAfter': True, + 'holder': {'pid': 9, 'inProcess': True}})]}, + ('blocker', "cleanup failed or no report; board held: None {'pass': False, 'heldAfter': True, " + "'holder': {'pid': 9, 'inProcess': True}}")), + 'locked': (1, {'results': [{'board': 'B', 'ran': False, 'pass': False, 'locked': True, 'recoveryBlocker': False, + 'detail': 'held by pid 7'}]}, ('not-run', 'locked: held by pid 7')), + 'locked_recovery_blocker': (1, {'results': [{'board': 'B', 'ran': False, 'pass': False, 'locked': True, + 'recoveryBlocker': True, 'detail': 'cleanup holder 8'}]}, + ('blocker', 'locked: cleanup holder 8')), + 'pass': (0, {'results': [{'board': 'B', 'ran': True, 'pass': True, 'steps': [{'step': 'install', 'pass': True}], + 'cleanup': {'pass': True, 'state': 'blinky copied', 'heldAfter': False}}]}, ('pass', '')), + 'fail_after_write': (1, {'results': [{'board': 'B', 'ran': True, 'pass': False, + 'steps': [{'step': 'install', 'pass': True}, + {'step': 'copy', 'pass': False, 'detail': 'drive vanished'}], + 'cleanup': {'pass': True, 'state': 'blinky copied', 'heldAfter': False}}]}, + ('fail', 'copy: drive vanished')), + 'unfinished_attempt': (1, {'results': [{'board': 'B', 'ran': False, 'pass': False, 'locked': False, 'recoveryBlocker': True, + 'unfinishedAttempts': ['r1'], 'detail': 'CI run r1 has no .done'}]}, + ('blocker', 'CI run r1 has no .done')), +} + + def completed(stdout='', code=0, stderr=''): return mock.Mock(returncode=code, stdout=stdout, stderr=stderr) @@ -59,6 +121,7 @@ def setUp(self): mock.patch.object(hil_ci, 'HIL_TEST', self.stand_in).start() mock.patch.object(hil_ci, 'ROOT', self.tmp).start() mock.patch.object(hil_ci, 'run', self.fake_run).start() + mock.patch.object(hil_ci, 'check_pin').start() mock.patch.dict(os.environ, {'HIL_STAND_IN': str(self.spec)}).start() self.addCleanup(mock.patch.stopall) self.addCleanup(self.reap_hung_stand_in) @@ -113,7 +176,20 @@ def test_a_board_held_past_the_wait_is_not_run_and_a_cleanup_holder_a_blocker(se locked = {'row': {'ran': False, 'locked': True, 'recoveryBlocker': blocker, 'detail': 'board locked: pid 7'}} self.assertEqual(self.main('--board', BOARD, battery=locked, out=out), 1 if want == 'not-run' else 2) self.assertEqual(self.rows(out)[0]['verdict'], want) - self.assertEqual((out / f'{BOARD}.done').exists(), want == 'not-run') + # a battery that never held the board leaves no marker: no run sees it started + self.assertEqual(((out / f'{BOARD}.started').exists(), (out / f'{BOARD}.done').exists()), (False, False)) + + def test_the_battery_gets_the_done_marker_and_a_blocker_verdict_keeps_none(self): + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}, 'touch_done': True} + self.assertEqual(self.main('--board', BOARD, battery=ok), 0) + argv = (self.out / BOARD / 'stdout.txt').read_text().splitlines()[0].split() + self.assertEqual(argv[argv.index('--done-marker') + 1], str(self.out / f'{BOARD}.done')) + self.assertEqual(argv[argv.index('--started-marker') + 1], str(self.out / f'{BOARD}.started')) + self.assertTrue((self.out / f'{BOARD}.done').exists()) + out = self.state / 'run-2' + probe = {'row': {'ran': False, 'probeUsers': [5], 'detail': 'probe in use'}, 'exit': 2, 'touch_done': True} + self.assertEqual(self.main('--board', BOARD, battery=probe, out=out), 2) + self.assertFalse((out / f'{BOARD}.done').exists()) def test_a_failed_cleanup_is_a_blocker_that_keeps_its_marker_and_blocks_the_next_run(self): held = {'row': {'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': True}], @@ -139,21 +215,231 @@ def test_a_holder_kept_in_process_is_not_waited_for(self): self.assertEqual(self.rows()[0]['verdict'], 'blocker') self.assertFalse((self.out / f'{BOARD}.done').exists()) + def test_the_verdicts_are_the_report_adapters_and_unchanged(self): + for name, (code, report, want) in GOLDEN.items(): + with self.subTest(name): + self.assertEqual(run_report.verdict(code, report, 'B'), want) + + def test_the_exit_folds_the_verdicts(self): + for verdicts, want in (([], 0), (['pass'], 0), (['pass', 'not-run'], 1), (['fail', 'pass'], 1), + (['fail', 'blocker'], 2), (['incomplete'], 0)): + self.assertEqual(run_report.fold_exit(verdicts), want, verdicts) + + def test_a_report_the_battery_could_not_publish_is_named_in_the_summary(self): + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}, + 'report': {'reportError': 'report not published: ReportError: replace: disk full'}} + self.assertEqual(self.main('--board', BOARD, battery=ok), 0) # the run's own outcome stands + r = self.rows()[0] + self.assertEqual((r['verdict'], r['reportError']), ('pass', ok['report']['reportError'])) + + def meta(self, termination='completed'): + core = run_report.load(hil_ci.REPORT_CORE) + return core.make_meta(run_report.schema(core), core.Attempt('run-1', None, None, None, None), termination=termination, + invocation=['hil_test.py', 'run'], selection=[BOARD], host='h', roster={'path': 'ci.json'}) + + def test_the_shared_run_rules_tighten_the_native_verdict_and_never_decide_the_exit(self): + passed = {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}} + waited = {'ran': False, 'pass': False, 'locked': True, 'recoveryBlocker': False, + 'detail': 'board locked: pid 7; interrupted by SIGTERM while waiting for the lock'} + for what, battery, exit_, native, shared, why in ( + ('a pass', {'row': passed, 'meta': self.meta()}, 0, 'pass', 'pass', ''), + ('a lock wait a signal aborted', {'row': waited, 'meta': self.meta('aborted')}, 1, 'not-run', 'blocker', + 'run aborted'), + ('a pass under a run caveat', {'row': passed, 'report': {'caveat': 'probe firmware unverified'}, + 'meta': self.meta()}, 0, 'pass', 'blocker', 'run caveat denies a pass'), + ('a legacy report', {'row': passed}, 0, 'pass', 'pass', '')): + with self.subTest(what): + out = self.state / what.replace(' ', '-') + self.assertEqual(self.main('--board', BOARD, battery=battery, out=out), exit_) + r = self.rows(out)[0] + self.assertEqual((r['verdict'], r['sharedVerdict']), (native, shared)) + self.assertIn(why, r['sharedDetail']) + + def test_the_report_pair_is_prepared_before_the_upload_and_never_left_stale(self): + passed = {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}} + core = run_report.load(hil_ci.REPORT_CORE) + for what, battery, md_starts, shared in ( + ('the Markdown never replaced', {'row': passed, 'meta': self.meta()}, 'HIL run: pass\n', 'pass'), + ('an earlier Markdown left', {'row': passed, 'meta': self.meta(), 'md': 'HIL run: fail\n'}, 'HIL run: pass\n', + 'pass'), + ('the JSON cut short', {'row': passed, 'meta': self.meta(), 'cut': 40, 'md': 'HIL run: pass\n'}, core.NOTICE, + 'blocker')): + with self.subTest(what): + out = self.state / what.replace(' ', '-') + self.assertEqual(self.main('--board', BOARD, battery=battery, out=out), 0) # the battery's own exit stands + md = (out / BOARD / 'run-1' / 'hil_report.md').read_text(encoding='utf-8') + self.assertTrue(md.startswith(md_starts), md) + if 'md' in battery and 'cut' in battery: # the earlier view is kept only as labelled legacy text + self.assertIn('_Legacy text:', md) + r = self.rows(out)[0] + self.assertEqual((r['verdict'], r['sharedVerdict']), ('pass', shared)) + self.assertTrue((out / f'{BOARD}.done').exists()) + + def test_a_battery_that_left_no_run_dir_gets_a_blocker(self): + self.assertEqual(hil_ci.prepare(0, self.tmp / 'nowhere', BOARD)['sharedVerdict'], 'blocker') + + def test_a_failed_preparation_withholds_the_runs_markdown_and_keeps_the_native_outcome(self): + core = run_report.load(hil_ci.REPORT_CORE) + self.assertEqual(hil_ci.NOTICE, core.NOTICE) + broken = run_report.load(hil_ci.REPORT_CORE) # a module of its own, so the patch stays here + + def boom(*a): + raise OSError(5, 'Input/output error') + broken.prepare_pair = boom + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}, 'md': 'HIL run: pass\n'} + for what, failure, err in ( + ('the core does not load', mock.patch.object(hil_ci, 'REPORT_CORE', self.tmp / 'missing.py'), 'missing.py'), + ('the preparation raises', mock.patch.object(hil_ci.run_report, 'load', lambda p: broken), 'Input/output')): + with self.subTest(what): + out = self.state / what.replace(' ', '-') + with failure: + self.assertEqual(self.main('--board', BOARD, battery=ok, out=out), 0) + r = self.rows(out)[0] + self.assertEqual((r['verdict'], r['sharedVerdict']), ('pass', 'blocker')) + self.assertIn('report not prepared', r['sharedDetail']) + self.assertIn(err, r['sharedDetail']) + self.assertTrue((out / f'{BOARD}.done').exists()) + md = (out / BOARD / 'run-1' / 'hil_report.md').read_text(encoding='utf-8') + head, _, kept = md.partition('_Legacy text:') + self.assertTrue(head.startswith(core.NOTICE) and 'HIL run: pass' not in head, md) # the pass view, unchecked, + self.assertIn('HIL run: pass', kept) # only as labelled text + + def test_a_rejected_pin_never_runs_the_provider(self): + ran = self.tmp / 'provider-ran' + provider = self.tmp / 'provider.py' + provider.write_text(f'open({str(ran)!r}, "w").close()\n') + build = self.tmp / 'check_build.py' # check_build's contract: Fail, and the pin check raising it + build.write_text('class Fail(Exception):\n pass\n\n\ndef check_agentrc_pin():\n' + ' raise Fail("lib/agentrc checkout is dirty at the pinned x: discard its local changes")\n') + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}, 'md': 'HIL run: pass\n'} + with mock.patch.object(hil_ci, 'check_pin', CHECK_PIN), mock.patch.object(hil_ci, 'CHECK_BUILD', build), \ + mock.patch.object(hil_ci, 'REPORT_CORE', provider): + self.assertEqual(self.main('--board', BOARD, battery=ok), 0) # the battery's own exit stands + self.assertFalse(ran.exists(), 'the provider ran') + r = self.rows()[0] + self.assertEqual((r['verdict'], r['sharedVerdict']), ('pass', 'blocker')) + self.assertIn('report not prepared: Fail: lib/agentrc checkout is dirty', r['sharedDetail']) + md = (self.out / BOARD / 'run-1' / 'hil_report.md').read_text(encoding='utf-8') + self.assertTrue(md.startswith(hil_ci.NOTICE), md) + + def test_a_view_that_can_be_neither_replaced_nor_removed_is_named_unsafe_for_the_upload(self): + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}, 'md': 'HIL run: pass\n'} + write_text, unlink = Path.write_text, Path.unlink + + def denied(real): + def op(path, *a, **k): + if path.name in ('hil_report.md', '.hil_report.md.withheld'): # a run dir turned read-only + raise PermissionError(13, 'Permission denied', str(path)) + return real(path, *a, **k) + return op + with mock.patch.object(hil_ci, 'REPORT_CORE', self.tmp / 'missing.py'), \ + mock.patch.object(Path, 'write_text', denied(write_text)), mock.patch.object(Path, 'unlink', denied(unlink)): + self.assertEqual(self.main('--board', BOARD, battery=ok), 0) # the battery's exit and markers stand + md = self.out / BOARD / 'run-1' / 'hil_report.md' + self.assertEqual(md.read_text(), 'HIL run: pass\n') + r = self.rows()[0] + self.assertEqual((r['verdict'], r['sharedVerdict'], r['unsafeArtifacts']), ('pass', 'blocker', [str(md)])) + self.assertTrue((self.out / f'{BOARD}.done').exists()) + self.assertIn(f'(not uploaded: {md})', hil_ci.summary([r], 'ci.json')) + self.assertEqual(self.upload_step(self.out), f'!{self.out}/**/hil_report.md') # every view, not only that one + + def test_a_view_another_writer_holds_is_never_touched_and_never_uploaded(self): + import fcntl + where = self.tmp / 'b' + md = where / 'run-1' / 'hil_report.md' + md.parent.mkdir(parents=True) + md.write_text('HIL run: pass\n') + with open(where / 'run-1' / '.hil_report.lock', 'w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX) # the core's writer, mid-publication + self.assertEqual(hil_ci.withhold(where, 'why'), [str(md)]) + self.assertEqual(md.read_text(), 'HIL run: pass\n') + self.assertEqual(hil_ci.withhold(where, 'why'), []) # once it let go + self.assertTrue(md.read_text().startswith(hil_ci.NOTICE)) + + def upload_step(self, out): + """The workflow's decision step run as the runner runs it: the exclusion it hands the upload.""" + text = (ROOT / '.github' / 'workflows' / 'hil.yml').read_text() + body = re.search(r'\n id: views\n(?: .*\n)*? run: \|\n((?: .*\n)+)', text).group(1) + output = self.tmp / 'github-output' + output.write_text('') + subprocess.run(['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', textwrap.dedent(body)], check=True, + env=dict(os.environ, HIL_OUT=str(out), GITHUB_OUTPUT=str(output))) + return dict(ln.split('=', 1) for ln in output.read_text().splitlines())['exclude'] + + def test_the_views_are_uploaded_only_on_hil_cis_last_word(self): + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}} + self.assertEqual(self.main('--board', BOARD, battery=ok), 0) + self.assertEqual(((self.out / 'upload-views').read_text(), self.upload_step(self.out)), ('upload\n', '')) + for what, change in (('no decision: hil_ci.py did not finish', lambda d: d.unlink()), + ('an unreadable decision', lambda d: d.write_bytes(b'\xff')), + ('another decision', lambda d: d.write_text('withhold\n'))): + with self.subTest(what): + change(self.out / 'upload-views') + self.assertEqual(self.upload_step(self.out), f'!{self.out}/**/hil_report.md') + later = self.state / 'run-2' + later.mkdir() + (later / 'upload-views').write_text('upload\n') # an earlier attempt's word never stands for this one + with mock.patch.object(hil_ci, 'summary', side_effect=OSError('killed')), self.assertRaises(OSError): + self.main('--board', BOARD, battery=ok, out=later) + self.assertEqual(self.upload_step(later), f'!{later}/**/hil_report.md') + + @unittest.skipUnless(os.geteuid(), 'root lists an unlistable dir') + def test_a_run_root_that_cannot_be_listed_withholds_every_view(self): + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}, + 'meta': self.meta(), 'md': 'HIL run: pass\n', 'unlistable': True} + self.addCleanup(lambda: (self.out / BOARD).exists() and (self.out / BOARD).chmod(0o755)) + self.assertEqual(self.main('--board', BOARD, battery=ok), 0) # the battery's own exit stands + r = self.rows()[0] + (self.out / BOARD).chmod(0o755) # readable again by the upload, its view never checked + self.assertEqual((r['sharedVerdict'], r.get('unsafeArtifacts'), (self.out / 'upload-views').read_text(), + self.upload_step(self.out)), + ('blocker', [str(self.out / BOARD)], 'withhold\n', f'!{self.out}/**/hil_report.md')) + self.assertIn(hil_ci.UNPREPARED, r['sharedDetail']) + + def test_the_summary_labels_the_native_verdict_and_shows_the_shared_one_where_it_differs(self): + passed = {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}} + step_summary = self.tmp / 'step-summary.md' + for what, battery, patch, shared in ( + ('a pass', {'row': passed, 'meta': self.meta()}, contextlib.nullcontext(), ''), + ('a pass under a run caveat', {'row': passed, 'report': {'caveat': 'probe | unverified'}, 'meta': self.meta()}, + contextlib.nullcontext(), 'blocker: run caveat denies a pass: probe / unverified'), + ('an unprepared report', {'row': passed}, mock.patch.object(hil_ci, 'REPORT_CORE', self.tmp / 'missing.py'), + 'blocker: report not prepared')): + with self.subTest(what), patch, mock.patch.dict(os.environ, {'GITHUB_STEP_SUMMARY': str(step_summary)}): + step_summary.write_text('') + out = self.state / what.replace(' ', '-') + stdout = io.StringIO() + self.spec.write_text(json.dumps(battery)) + with mock.patch.object(sys, 'stdout', stdout): + self.assertEqual(hil_ci.main(['--roster', 'ci.json', '--out', str(out), '--board', BOARD]), 0) + text = stdout.getvalue() + self.assertEqual(step_summary.read_text(), text) + self.assertIn('| board | native verdict | detail | shared run rules |', text) + line = text.splitlines()[-1] + self.assertTrue(line.startswith(f'| {BOARD} | pass | | {shared}'), line) + if not shared: + self.assertTrue(line.endswith('| |'), line) + blocked = {'board': BOARD, 'verdict': 'blocker', 'detail': 'held', 'sharedVerdict': 'blocker'} + lines = hil_ci.summary([dict(blocked, sharedDetail='run aborted: held'), + dict(blocked, sharedDetail='report not prepared: OSError: x')], 'ci.json').splitlines() + self.assertEqual(lines[-2:], [f'| {BOARD} | blocker | held | |', # the same verdict: nothing more to show + f'| {BOARD} | blocker | held | blocker: report not prepared: OSError: x |']) + def test_a_run_without_a_report_is_a_blocker(self): - self.assertEqual(hil_ci.verdict(137, None, BOARD)[0], 'blocker') - self.assertEqual(hil_ci.verdict(1, {'results': []}, BOARD)[0], 'blocker') - self.assertEqual(hil_ci.verdict(1, {'results': [{'board': BOARD, 'ran': None, 'pass': False}]}, BOARD)[0], 'blocker') + self.assertEqual(run_report.verdict(137, None, BOARD)[0], 'blocker') + self.assertEqual(run_report.verdict(1, {'results': []}, BOARD)[0], 'blocker') + self.assertEqual(run_report.verdict(1, {'results': [{'board': BOARD, 'ran': None, 'pass': False}]}, BOARD)[0], 'blocker') def test_a_refusal_before_hardware_and_an_untouched_failure_are_not_run_and_a_used_probe_a_blocker(self): - self.assertEqual(hil_ci.verdict(2, {'results': [{'board': BOARD, 'ran': False, 'detail': 'no probe'}]}, BOARD), + self.assertEqual(run_report.verdict(2, {'results': [{'board': BOARD, 'ran': False, 'detail': 'no probe'}]}, BOARD), ('not-run', 'no probe')) - self.assertEqual(hil_ci.verdict(2, {'results': [{'board': BOARD, 'ran': False, 'probeUsers': [5], 'detail': 'p'}]}, BOARD), + self.assertEqual(run_report.verdict(2, {'results': [{'board': BOARD, 'ran': False, 'probeUsers': [5], 'detail': 'p'}]}, BOARD), ('blocker', 'p')) row = {'board': BOARD, 'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': False, 'detail': 'x'}], 'cleanup': {'pass': True, 'state': 'untouched'}} - self.assertEqual(hil_ci.verdict(1, {'results': [row]}, BOARD)[0], 'not-run') + self.assertEqual(run_report.verdict(1, {'results': [row]}, BOARD)[0], 'not-run') row['steps'] = [] - self.assertEqual(hil_ci.verdict(1, {'results': [row]}, BOARD)[0], 'blocker') # an abort + self.assertEqual(run_report.verdict(1, {'results': [row]}, BOARD)[0], 'blocker') # an abort def test_a_failed_build_is_not_run_and_touches_nothing(self): self.build_ok = False @@ -178,6 +464,36 @@ def test_fail_at_reaches_the_battery(self): self.assertEqual(self.main('--board', BOARD, '--fail-at', 'cleanup', battery=held), 2) self.assertIn('--fail-at cleanup', (self.out / BOARD / 'stdout.txt').read_text()) + def test_launch_returns_the_report_the_exit_and_whether_the_runner_stayed_a_holder(self): + def cmd(where): + where.mkdir() + return [sys.executable, self.stand_in, 'run', '-b', BOARD, '--run-root', where] + self.spec.write_text(json.dumps({'row': {'ran': True, 'pass': True}})) + report, code, held = hil_ci.launch(cmd(self.tmp / 'a'), self.tmp / 'a', BOARD) + self.assertEqual((report['results'][0]['board'], code, held), (BOARD, 0, False)) + self.spec.write_text(json.dumps({'row': {'ran': True, 'cleanup': {'holder': {'pid': 9, 'inProcess': True}}}, + 'hang': True})) + report, code, held = hil_ci.launch(cmd(self.tmp / 'b'), self.tmp / 'b', BOARD) + self.assertEqual((code, held), (None, True)) + self.assertTrue((self.tmp / 'b' / 'stdout.txt').read_text().endswith(json.dumps(report) + '\n')) + + def test_a_recovery_blocker_row_is_a_blocker_locked_or_not(self): + for row in ({'ran': False, 'locked': True, 'recoveryBlocker': True, 'detail': 'stale record'}, + {'ran': False, 'locked': False, 'recoveryBlocker': True, 'detail': 'an unfinished attempt'}): + self.assertEqual(run_report.verdict(1, {'results': [dict(row, board=BOARD)]}, BOARD)[0], 'blocker') + + def test_the_marker_scan_skips_links_and_fails_closed(self): + (self.state / 'real').mkdir(parents=True) + (self.state / 'real' / f'{BOARD}.started').touch() + (self.state / 'linked').symlink_to(self.state / 'real') + self.assertEqual(hil_ci.unfinished(self.state, BOARD), ['real']) + with self.assertRaises(OSError): + hil_ci.unfinished(self.tmp / 'missing', BOARD) + row = hil_ci.run_board(BOARD, 'ci.json', self.out, self.tmp / 'missing', 0) + self.assertEqual((row['verdict'], row['phase']), ('blocker', 'preflight')) + self.assertIn('cannot be read', row['detail']) + self.assertEqual(self.commands, [], 'nothing built') + def test_old_finished_runs_are_pruned_and_an_unfinished_one_is_kept_whatever_its_age(self): old = time.time() - 40 * 86400 for name, done in (('run-old-done', True), ('run-old-held', False), ('run-new-held', False)): @@ -192,5 +508,167 @@ def test_old_finished_runs_are_pruned_and_an_unfinished_one_is_kept_whatever_its self.assertEqual(sorted(p.name for p in self.state.iterdir()), ['run-new-held', 'run-old-held']) +LPC = 'lpcxpresso55s69' +LPC_FACTS = {'flashBase': 0, 'binBase': 0, 'appStart': 0x10000, 'flashSize': 0x98000, 'vid': 0x1fc9, 'pid': 0x0094, + 'boardId': 'LPC55S69-Xpresso-revA', 'family': 0x2abc77ec, 'vectors': 76, 'ram': 0x20000000, + 'dblTapDelayMs': 500, 'dblTapReg': 0x4002C05C, 'dblTapMagic': 0xF01669EF} +# the real runner's main(), its lock taken through the pinned helper in a test lock dir; only admission and the probe +# steps are stand-ins, and HIL_BATTERY=die leaves the battery to be SIGKILLed mid-run +REAL_RUNNER = """ +import importlib.util, json, os, sys, time +spec = importlib.util.spec_from_file_location('hil_test', os.environ['HIL_RUNNER']) +ht = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ht) +ht.check_lock_pin = lambda: None +ht.LOCK_POLL = float(os.environ.get('HIL_LOCK_POLL', ht.LOCK_POLL)) +load = ht.load_helper + + +def load_helper(): + helper = load() + helper.BOARD_LOCK_DIR = os.environ['HIL_LOCKS'] + acquire = helper.acquire_board_lock + + def acquire_noted(lock, *a, **k): # each try at the board, for a test to order a contender after it + if os.environ.get('HIL_TRIES'): + with open(os.environ['HIL_TRIES'], 'a') as f: + f.write('try\\n') + return acquire(lock, *a, **k) + helper.acquire_board_lock = acquire_noted + return helper + + +def run(bat, files, version): + bat.touched = True + if os.environ['HIL_BATTERY'] == 'die': + open(os.environ['HIL_PID'], 'w').write(str(os.getpid())) + time.sleep(60) + + +ht.load_helper = load_helper +ht.source_facts = lambda board: json.loads(os.environ['HIL_FACTS']) +ht.admit_record = lambda *a: ({}, {'version': 'v'}) +ht.probe_users = lambda board: [] +ht.Battery.run = run +ht.Battery.restore = lambda bat, files, version: {'pass': True, 'state': 'blinky copied', 'evidence': {}} +sys.exit(ht.main(sys.argv[1:])) +""" + + +class RealRunner(unittest.TestCase): + """hil_ci.py over the real `hil_test.py run` and the pinned lock helper: the protected battery lock as CI sees it.""" + + def setUp(self): + t = tempfile.TemporaryDirectory() + self.addCleanup(t.cleanup) + self.tmp = Path(t.name) + self.state = self.tmp / 'state' + self.out = self.state / 'run-1' + (self.tmp / 'ci.json').write_text(json.dumps({'boards': [ + {'name': LPC, 'uid': 'UID', 'flasher': {'name': 'jlink', 'uid': '1', 'args': '-device LPC55S69_M33_0'}, + 'tinyuf2': {'reentry': 'retained-register'}}]})) + runner = self.tmp / 'hil_test.py' + runner.write_text(REAL_RUNNER) + locks = self.tmp / 'locks' + locks.mkdir() + self.pid_file = self.tmp / 'battery.pid' + if not ht.LOCK_HELPER.is_file(): + self.fail('lib/agentrc not checked out: git submodule update --init lib/agentrc') + with mock.patch.object(ht, 'check_lock_pin'): + self.helper = ht.load_helper() + self.helper.BOARD_LOCK_DIR = str(locks) + self.env = {'HIL_RUNNER': str(Path(ht.__file__).resolve()), 'HIL_LOCKS': str(locks), 'HIL_PID': str(self.pid_file), + 'HIL_FACTS': json.dumps(LPC_FACTS)} + mock.patch.object(hil_ci, 'HIL_TEST', runner).start() + mock.patch.object(hil_ci, 'ROOT', self.tmp).start() + mock.patch.object(hil_ci, 'check_pin').start() + mock.patch.object(hil_ci, 'run', lambda cmd: completed(json.dumps( + {'pass': True, 'boards': [{'board': LPC, 'status': 'ok', 'firstError': ''}]}))).start() + self.addCleanup(mock.patch.stopall) + + def main(self, battery, out, lock_wait): + with mock.patch.dict(os.environ, dict(self.env, HIL_BATTERY=battery)), mock.patch.object(sys, 'stdout', io.StringIO()): + return hil_ci.main(['--roster', 'ci.json', '--out', str(out), '--board', LPC, '--lock-wait', str(lock_wait)]) + + def row(self, out): + return json.loads((out / 'summary.json').read_text())['boards'][0] + + def test_a_clean_exit_releases_the_protected_lock(self): + self.assertEqual(self.main('pass', self.out, 0), 0) + self.assertEqual(self.row(self.out)['verdict'], 'pass') + # the run dir's report pair, published through the pinned report core, gives the same verdict from the file alone + run_dir, = (self.out / LPC).glob('run-*') + with mock.patch.object(ht, 'check_lock_pin'): + core = ht.report_core() + reading = core.read_document(run_dir / 'hil_report.json', run_report.schema(core)) + self.assertEqual((reading.outcome, reading.meta['attempt_id'], reading.body['exitCode']), ('valid', run_dir.name, 0)) + self.assertEqual(run_report.verdict(run_report.exit_of(reading.body), reading.body, LPC), ('pass', '')) + self.assertTrue((run_dir / 'hil_report.md').read_text(encoding='utf-8').startswith('HIL run: pass\n')) + self.assertEqual(self.row(self.out)['sharedVerdict'], 'pass') + self.assertTrue((self.out / f'{LPC}.done').exists()) + self.assertEqual(self.helper.record_state(LPC)[0], 'empty') + self.helper.release_board_lock(self.helper.take_board_lock(LPC, 'next')) + + def test_a_battery_killed_mid_run_leaves_a_blocker_the_next_run_reports_without_waiting(self): + def kill_the_battery(): + end = time.monotonic() + 30 + while not self.pid_file.exists() and time.monotonic() < end: + time.sleep(0.02) + os.kill(int(self.pid_file.read_text()), signal.SIGKILL) + killer = threading.Thread(target=kill_the_battery) + killer.start() + self.assertEqual(self.main('die', self.out, 0), 2) + killer.join() + row = self.row(self.out) + self.assertEqual(row['verdict'], 'blocker') + self.assertIn('without a report', row['detail']) + # it died holding the board: its start stands without a .done, a blocker for every later run + self.assertEqual(((self.out / f'{LPC}.started').exists(), (self.out / f'{LPC}.done').exists()), (True, False)) + self.assertEqual(hil_ci.unfinished(self.state, LPC), ['run-1']) + run_dir, = (self.out / LPC).glob('run-*') # the dead battery's dir uploads an explicit notice, no view + self.assertTrue((run_dir / 'hil_report.md').read_text(encoding='utf-8').startswith( + run_report.load(hil_ci.REPORT_CORE).NOTICE)) + self.assertIn('report missing', row['sharedDetail']) + rec = self.helper.read_record(LPC) + self.assertTrue(self.helper.is_recovery_blocker(rec)) # the shared helper's view: every project refuses the board + self.assertEqual((rec['reason'], rec['protected']), ('hil_test.py', True)) + self.assertEqual(Path(rec['runDir']).parent, self.out / LPC) + # a later CI run with no marker of its own (another state dir) meets the record: a blocker at once, never waited for + later = self.tmp / 'state-2' / 'run-1' + t0 = time.monotonic() + self.assertEqual(self.main('pass', later, 600), 2) + self.assertLess(time.monotonic() - t0, 20) + row = self.row(later) + self.assertEqual(row['verdict'], 'blocker') + self.assertIn('stale record', row['detail']) + self.assertFalse((later / f'{LPC}.done').exists()) + self.assertEqual(self.helper.read_record(LPC), rec) + + def test_a_ci_battery_waiting_on_the_board_is_not_unfinished_to_a_runner_that_takes_it_meanwhile(self): + tries, ended = self.tmp / 'tries', {} + third = self.helper.take_board_lock(LPC, 'a third holder') + env = dict(self.env, HIL_BATTERY='pass', HIL_LOCK_POLL='4', HIL_TRIES=str(tries)) + with mock.patch.dict(os.environ, env), mock.patch.object(sys, 'stdout', io.StringIO()): + ci = threading.Thread(target=lambda: ended.update(code=hil_ci.main( + ['--roster', 'ci.json', '--out', str(self.out), '--board', LPC, '--lock-wait', '4']))) + ci.start() + end = time.monotonic() + 20 + while not tries.exists() and time.monotonic() < end: # CI's battery tried the board and sleeps on it + time.sleep(0.02) + waiting = ((self.out / f'{LPC}.started').exists(), hil_ci.unfinished(self.state, LPC)) + self.helper.release_board_lock(third) + # a remote driver's runner takes the board before CI's battery tries again + report, code, _ = hil_ci.battery(LPC, 'ci.json', self.out / f'{LPC}.build.json', self.tmp / 'remote' / LPC, 0, + ci_state_root=self.state) + ci.join() + remote = run_report.row_of(report, LPC) or {} + self.assertEqual((waiting, run_report.verdict(code, report, LPC), remote.get('unfinishedAttempts'), + ended['code'], self.row(self.out)['verdict'], + (self.out / f'{LPC}.started').exists(), (self.out / f'{LPC}.done').exists(), + self.helper.record_state(LPC)[0]), + ((False, []), ('pass', ''), None, 1, 'not-run', False, False, 'empty')) + + + if __name__ == '__main__': unittest.main() diff --git a/.claude/test/test_hil_remote.py b/.claude/test/test_hil_remote.py new file mode 100644 index 000000000..8f7b7c514 --- /dev/null +++ b/.claude/test/test_hil_remote.py @@ -0,0 +1,909 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# The remote adapter (hil_remote.py) and its rig-side driver (hil_remote_driver.py) over agentrc's remote core, on a +# local "rig": the core's own fake ssh and rsync (lib/agentrc tests/test_hil_remote_core.py) run every remote command +# here under a clean environment, so the attempt, the detached launch, the driver's rebuild of the checkout from git +# bundles, hil_ci.py's build record and battery launch and the copy-back all run for real. The project is a temp +# superproject in a relocated worktree with real submodules, an unpushed candidate lib/agentrc pin and a shallow +# get_deps dependency; its check_build.py and hil_test.py are stand-ins, or the real runner with only admission and +# the probe steps stubbed. Plumbing only: a real ssh session's end and the rig are proved by a run on ci.lan. +import contextlib +import hashlib +import importlib.util +import io +import json +import os +import shutil +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / '.claude' / 'skills' / 'hil-contract' / 'scripts' +sys.path.insert(0, str(SCRIPTS)) +import hil_remote as hr # noqa: E402 +import hil_remote_driver as drv # noqa: E402 +import hil_test as ht # noqa: E402 + +CHECK_PIN = hr.check_pin # before Project patches it +hr.provider() # the tests' own core: main() checks the pin before it loads one + + +def load_core_tests(): + spec = importlib.util.spec_from_file_location('agentrc_test_hil_remote_core', + ROOT / 'lib' / 'agentrc' / 'tests' / 'test_hil_remote_core.py') + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +rig = load_core_tests() # its FAKE_SSH (RIG_PATH /usr/bin:/bin) and FAKE_RSYNC +needs_rig = unittest.skipUnless(os.name != 'nt' and rig.REAL_RSYNC and shutil.which('flock', path=rig.RIG_PATH) and + shutil.which('python3', path=rig.RIG_PATH) and shutil.which('git', path=rig.RIG_PATH), + 'the fake rig needs POSIX, rsync, and flock, python3 and git on /usr/bin:/bin') + +LPC = 'lpcxpresso55s69' +OTHER = 'lpcxpresso55s28' +ESP = 'espressif_esp32s3_devkitc_1' +LPC_FACTS = {'flashBase': 0, 'binBase': 0, 'appStart': 0x10000, 'flashSize': 0x98000, 'vid': 0x1fc9, 'pid': 0x0094, + 'boardId': 'LPC55S69-Xpresso-revA', 'family': 0x2abc77ec, 'vectors': 76, 'ram': 0x20000000, + 'dblTapDelayMs': 500, 'dblTapReg': 0x4002C05C, 'dblTapMagic': 0xF01669EF} +ROSTER = {'boards': [ + {'name': LPC, 'uid': 'UID', 'flasher': {'name': 'jlink', 'uid': '1', 'args': '-device LPC55S69_M33_0'}, + 'tinyuf2': {'reentry': 'retained-register'}}, + {'name': OTHER, 'uid': 'UID2', 'flasher': {'name': 'jlink', 'uid': '2', 'args': '-device LPC55S69_M33_0'}, + 'tinyuf2': {'reentry': 'retained-register'}}, + {'name': ESP, 'uid': 'MAC', 'flasher': {'name': 'esptool', 'uid': 'X', 'args': '--chip esp32s3'}, + 'tinyuf2': {'reentry': 'dfu-detach'}}]} + +# the build contract on the rig: run, a record of the board, ok unless FAKE_BUILD=fail, and its build dir's log; +# loaded, the real check_build's lib/agentrc pin check (FAKE_CHECK_BUILD) over this checkout +CHECK_BUILD = r''' +import importlib.util, json, os, sys +from pathlib import Path +spec = importlib.util.spec_from_file_location('real_check_build', os.environ['FAKE_CHECK_BUILD']) +real = importlib.util.module_from_spec(spec) +spec.loader.exec_module(real) +real.ROOT = Path(__file__).resolve().parents[4] +Fail, check_agentrc_pin = real.Fail, real.check_agentrc_pin +if __name__ == '__main__': + board = sys.argv[sys.argv.index('--board') + 1] + ok = os.environ.get('FAKE_BUILD', 'ok') == 'ok' + d = Path('cmake-build') / f'cmake-build-{board}' + d.mkdir(parents=True, exist_ok=True) + (d / 'check_build.log').write_text('the build log\n') + print(json.dumps({'pass': ok, 'boards': [{'board': board, 'status': 'ok' if ok else 'failed', + 'firstError': '' if ok else 'boom', 'buildDir': str(d)}]})) + sys.exit(0 if ok else 1) +''' + +# `hil_test.py run` on the rig. It appends its argv and HIL_* environment to FAKE_ARGV, then either runs the real runner (from +# FAKE_REAL_RUNNER, its lock in FAKE_LOCKS, admission and the probe steps stubbed; FAKE_BATTERY=die leaves the battery +# to be killed, FAKE_CI_RACE starts a CI battery's marker just before the lock is taken) or prints a canned report: +# FAKE_BATTERY pass, fail (after a write), die (no report), holder (stays the in-process cleanup holder) or unpublished +# (a pass whose report the core could not publish: reportPath null, reportError). Before its report it sleeps +# FAKE_SLEEP s (only on board FAKE_SLEEP_BOARD when set) and waits up to 30 s for the file FAKE_GATE; FAKE_MD is a +# Markdown view left beside the JSON, FAKE_READONLY leaves the run dir read-only and FAKE_UNLISTABLE its run root +# unlistable. +HIL_TEST = r''' +import json, os, sys, time +from pathlib import Path +sys.dont_write_bytecode = True +argv = sys.argv[1:] +with open(os.environ['FAKE_ARGV'], 'a') as f: + f.write(json.dumps({'argv': argv, 'hil': sorted(k for k in os.environ if k.startswith('HIL_'))}) + '\n') +mode = os.environ.get('FAKE_BATTERY', 'pass') +if os.environ.get('FAKE_REAL_RUNNER'): + import importlib.util + spec = importlib.util.spec_from_file_location('hil_test', os.environ['FAKE_REAL_RUNNER']) + ht = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ht) + ht.check_lock_pin = lambda: None + load = ht.load_helper + + def load_helper(): + helper = load() + helper.BOARD_LOCK_DIR = os.environ['FAKE_LOCKS'] + race, acquire = os.environ.get('FAKE_CI_RACE'), helper.acquire_board_lock + + def acquire_after_a_ci_start(lock, *a, **k): + if race: + Path(race).mkdir(parents=True, exist_ok=True) + (Path(race) / f'{lock}.started').touch() + return acquire(lock, *a, **k) + helper.acquire_board_lock = acquire_after_a_ci_start + return helper + + def run(bat, files, version): + bat.touched = True + if mode == 'die': + Path(os.environ['FAKE_PID']).write_text(str(os.getpid())) + time.sleep(60) + ht.load_helper = load_helper + ht.source_facts = lambda board: json.loads(os.environ['FAKE_FACTS']) + ht.admit_record = lambda *a: ({}, {'version': 'v'}) + ht.probe_users = lambda board: [] + ht.Battery.run = run + ht.Battery.restore = lambda bat, files, version: {'pass': True, 'state': 'blinky copied', 'evidence': {}} + sys.exit(ht.main(argv)) +board = argv[argv.index('-b') + 1] +run_dir = Path(argv[argv.index('--run-root') + 1]) / f'run-20261020-000000-{os.getpid()}' +run_dir.mkdir(parents=True) +(run_dir / 'hil.log').write_text('log\n') +if os.environ.get('FAKE_SLEEP_BOARD', board) == board: + time.sleep(float(os.environ.get('FAKE_SLEEP', '0'))) +gate, end = os.environ.get('FAKE_GATE'), time.monotonic() + 30 +while gate and not os.path.exists(gate) and time.monotonic() < end: + time.sleep(0.02) +if mode == 'die': + sys.exit(137) +steps = [{'step': 'install', 'pass': True}, {'step': 'dfu', 'pass': mode != 'fail', 'detail': 'no DFU'}] +cleanup = {'pass': True, 'state': 'blinky copied', 'heldAfter': False} +if mode == 'holder': + cleanup = {'pass': False, 'state': 'unknown', 'heldAfter': True, 'holder': {'pid': os.getpid(), 'inProcess': True}} +code = {'pass': 0, 'unpublished': 0, 'fail': 1, 'holder': 4}[mode] +passed = mode in ('pass', 'unpublished') +report = {'pass': passed, 'caveat': '', 'reportPath': str(run_dir / 'hil_report.json'), 'exitCode': code, + 'heldInProcess': mode == 'holder', 'results': [{'board': board, 'ran': True, 'pass': passed, + 'locked': False, 'steps': steps, 'cleanup': cleanup, 'detail': ''}]} +if mode == 'unpublished': + report.update(reportPath=None, reportError='report not published: OSError: disk full') +else: + (run_dir / 'hil_report.json').write_text(json.dumps(report)) +if os.environ.get('FAKE_MD'): + (run_dir / 'hil_report.md').write_text(os.environ['FAKE_MD']) +if os.environ.get('FAKE_READONLY'): + run_dir.chmod(0o555) +if os.environ.get('FAKE_UNLISTABLE'): + run_dir.parent.chmod(0o300) +print('noise') +print(json.dumps(report), flush=True) +if mode == 'holder': + Path(os.environ['FAKE_PID']).write_text(str(os.getpid())) + os.close(1) + time.sleep(60) +sys.exit(code) +''' + +GET_DEPS = '''deps_all = {'lib/mcu/vendor/dep': ['https://example.invalid/dep.git', %r, 'lpc55']}\n''' + + +# a git hook's GIT_DIR/GIT_INDEX_FILE would aim the fixtures' git at the checkout under test +LOCAL_GIT_VARS = subprocess.run(['git', 'rev-parse', '--local-env-vars'], capture_output=True, text=True).stdout.split() + + +def fixture_env(): + return {k: v for k, v in os.environ.items() if k not in LOCAL_GIT_VARS} + + +def git(cwd, *argv): + return subprocess.run(['git', '-C', str(cwd), *argv], check=True, capture_output=True, text=True, + env=fixture_env()).stdout.strip() + + +def commit(repo, files, msg): + for rel, text in files.items(): + (repo / rel).parent.mkdir(parents=True, exist_ok=True) + (repo / rel).write_text(text) + git(repo, 'add', '-A') + git(repo, 'commit', '-q', '-m', msg) + return git(repo, 'rev-parse', 'HEAD') + + +def origin(path, *texts): + path.mkdir(parents=True) + git(path, 'init', '-q') + for i, t in enumerate(texts): + sha = commit(path, {'README': t}, f'c{i}') + return sha + + +def run_main(*argv): + out = io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()): + code = hr.main(list(argv)) + lines = out.getvalue().strip().splitlines() + return code, json.loads(lines[-1]) if lines else None + + +class Project(unittest.TestCase): + """A superproject in a relocated worktree, its submodules at a candidate pin, on a local fake rig.""" + + def setUp(self): + t = tempfile.TemporaryDirectory() + self.addCleanup(t.cleanup) + self.tmp = Path(t.name) + gitconfig = self.tmp / 'gitconfig' + gitconfig.write_text('[user]\n name = t\n email = t@t\n[protocol "file"]\n allow = always\n' + '[init]\n defaultBranch = main\n[commit]\n gpgsign = false\n[advice]\n detachedHead = false\n') + self.home = self.tmp / 'home' + (self.home / '.local' / 'bin').mkdir(parents=True) + bindir = self.tmp / 'bin' + bindir.mkdir() + rig.fake(bindir / 'ssh', rig.FAKE_SSH) + rig.fake(bindir / 'rsync', rig.FAKE_RSYNC) + self.state = self.tmp / 'state' + self.state.mkdir() + self.locks = self.tmp / 'locks' + self.locks.mkdir() + env = {k: v for k, v in os.environ.items() if not k.startswith(('HIL_', 'FAKE_', 'GIT_'))} + env.update(PATH=f'{bindir}{os.pathsep}{os.environ["PATH"]}', GIT_CONFIG_GLOBAL=str(gitconfig), GIT_CONFIG_NOSYSTEM='1', + FAKE_LOG=str(self.tmp / 'log.jsonl'), FAKE_HOME=str(self.home), FAKE_ARGV=str(self.tmp / 'argv.json'), + FAKE_PID=str(self.tmp / 'battery.pid'), + FAKE_CHECK_BUILD=str(ROOT / '.claude' / 'skills' / 'build' / 'scripts' / 'check_build.py')) + mock.patch.dict(os.environ, env, clear=True).start() + self.addCleanup(mock.patch.stopall) + self.make_project() + cb = hr.check_build() + cb.get_deps_table.cache_clear() + self.addCleanup(cb.get_deps_table.cache_clear) + mock.patch.object(hr, 'ROOT', self.wt).start() + mock.patch.object(cb, 'ROOT', self.wt).start() + mock.patch.object(hr, 'POLL', 0.05).start() + mock.patch.object(hr, 'check_pin').start() # the real checkout's pin; check_pin is tested on the project's + mock.patch.dict(hr.HOSTS, {'rig': {'hostname': socket.gethostname(), 'roster': 'test/hil/ci.json', + 'base': str(self.tmp / 'attempts'), 'ciStateRoot': str(self.state)}}).start() + self.addCleanup(self.kill_left_sessions) + + def make_project(self): + o = self.tmp / 'origins' + self.dep = origin(o / 'dep', 'dep 1', 'dep 2') + for name in ('agentrc', 'tinyusb', 'uf2'): + origin(o / name, name) + core = 'skills/hil/scripts/hil_report_core.py' # the driver prepares each report pair through it, as CI does + commit(o / 'agentrc', {core: (ROOT / 'lib' / 'agentrc' / core).read_text()}, 'report core') + primary = self.tmp / 'primary' + primary.mkdir() + git(primary, 'init', '-q') + scripts = '.claude/skills/hil-contract/scripts' + files = {'.gitignore': 'lib/mcu/\ncmake-build/\n__pycache__/\n.hil-remote/\n', + f'{scripts}/hil_test.py': HIL_TEST, '.claude/skills/build/scripts/check_build.py': CHECK_BUILD, + 'test/hil/ci.json': json.dumps(ROSTER), f'ports/lpc55/boards/{LPC}/board.h': 'lpc\n', f'ports/lpc55/boards/{OTHER}/board.h': 'lpc\n', + 'tools/get_deps.py': GET_DEPS % self.dep} + files.update({f'{scripts}/{n}': (SCRIPTS / n).read_text() for n in ('hil_ci.py', 'hil_run_report.py', 'hil_remote_driver.py')}) + commit(primary, files, 'project') + for name in ('agentrc', 'tinyusb', 'uf2'): + git(primary, 'submodule', 'add', '-q', str(o / name), f'lib/{name}') + git(primary, 'commit', '-q', '-m', 'submodules') + self.wt = self.tmp / 'wt' + git(primary, 'worktree', 'add', '-q', '-b', 'feature', str(self.wt)) + git(self.wt, 'submodule', 'update', '--init', '-q') + subprocess.run(['git', 'clone', '-q', '--depth', '1', f'file://{o / "dep"}', str(self.wt / 'lib/mcu/vendor/dep')], + check=True, capture_output=True, env=fixture_env()) + self.pin = commit(self.wt / 'lib/agentrc', {'NEW': 'unpushed'}, 'candidate') + git(self.wt, 'add', 'lib/agentrc') + git(self.wt, 'commit', '-q', '-m', 'pin the candidate') + self.head = git(self.wt, 'rev-parse', 'HEAD') + + def kill_left_sessions(self): + for launch in (self.tmp / 'attempts').glob('*/.attempt/launch.json'): + with contextlib.suppress(OSError, ValueError, KeyError): + os.killpg(json.loads(launch.read_text())['sid'], signal.SIGKILL) + pid = self.tmp / 'battery.pid' + if pid.exists(): + with contextlib.suppress(OSError, ValueError): + os.kill(int(pid.read_text()), signal.SIGKILL) + + def ssh_calls(self): + log = self.tmp / 'log.jsonl' + return [json.loads(ln) for ln in log.read_text().splitlines()] if log.exists() else [] + + def attempt(self, run_id='r1'): + return self.tmp / 'attempts' / run_id + + def run_(self, *extra, run_id='r1', board=LPC): + return run_main('run', '-b', board, '--run-id', run_id, '--host', 'rig', *extra) + + def batteries(self): + argv = self.tmp / 'argv.json' + return [json.loads(ln) for ln in argv.read_text().splitlines()] if argv.exists() else [] + + +class Refusals(Project): + """Every refusal comes before the rig is touched: no ssh, no attempt.""" + + def refused(self, *argv, text): + argv = argv or ('run', '-b', LPC, '--run-id', 'r1', '--host', 'rig') + with contextlib.redirect_stderr(io.StringIO()) as err, contextlib.redirect_stdout(io.StringIO()): + code = hr.main(list(argv)) + self.assertEqual(code, hr.REFUSED) + self.assertIn(text, err.getvalue()) + self.assertEqual(self.ssh_calls(), []) + self.assertFalse((self.tmp / 'attempts').exists()) + self.assertEqual(list((self.wt / hr.RUNS).glob('.*.stage')) if (self.wt / hr.RUNS).exists() else [], []) + + def test_manual_recovery_and_override_operations(self): + for argv, text in ((('install', '-b', LPC), 'manual or recovery operation'), + (('restore', '-b', LPC), 'manual or recovery operation'), + (('clear-pending', '-b', LPC), 'manual or recovery operation')): + self.refused(*argv, text=text) + for flag in ('--under-hold=x', '--recover-holder', '--recover-takeover', '--try-reentry', '--route', '--run-root', + '--ci-state-root', '--done-marker', '--started-marker', '--build-record'): + self.refused('run', '-b', LPC, flag, 'v', text='takes no manual, recovery, commissioning or override option') + + def test_boards_and_run_ids(self): + self.refused('run', '-b', 'nope', '--host', 'rig', text='not in test/hil/ci.json at HEAD') + self.refused('run', '-b', ESP, '--host', 'rig', text='Espressif boards are not run remotely') + self.refused('run', '-b', LPC, '--host', 'elsewhere', text='no rig elsewhere') + self.refused('run', '-b', LPC, '--host', 'rig', '--run-id', '../x', text='a run id is') + with mock.patch.dict(os.environ, {'HIL_NO_BOARD_LOCK': '1'}): + self.refused(text='HIL_NO_BOARD_LOCK is set') + with mock.patch.object(hr.os, 'name', 'nt'): + self.refused(text='Windows client is pending') + + def test_lib_agentrc_off_its_pin_or_changed_is_refused(self): + check = CHECK_PIN # check_build's check, on the project (cb.ROOT) + check() + for what, change, undo in ( + ('a changed file', lambda: (self.wt / 'lib/agentrc/NEW').write_text('changed'), + lambda: git(self.wt / 'lib/agentrc', 'checkout', '-q', '--', 'NEW')), + ('an untracked file', lambda: (self.wt / 'lib/agentrc/stray').write_text('x'), + lambda: (self.wt / 'lib/agentrc/stray').unlink())): + with self.subTest(what): + change() + with self.assertRaisesRegex(hr.core.RemoteError, 'dirty at the pinned'): + check() + undo() + git(self.wt / 'lib/agentrc', 'checkout', '-q', 'HEAD~1') + with self.assertRaises(hr.core.RemoteError) as e: + check() + self.assertIn(f'HEAD pins {self.pin}: git submodule update --init lib/agentrc', e.exception.code) + git(self.wt, 'add', 'lib/agentrc') # the pin change staged: submodule update would check that out + with self.assertRaisesRegex(hr.core.RemoteError, 'is staged: commit or unstage'): + check() + + def test_wait_and_collect_load_nothing_that_needs_fcntl(self): + # Windows has no fcntl: the pin check's check_build and every module the client imports must load without it + r = subprocess.run([sys.executable, '-c', "import sys; sys.modules['fcntl'] = None; import hil_remote; " + "hil_remote.check_build(); print('ok')"], cwd=SCRIPTS, capture_output=True, text=True) + self.assertEqual(r.stdout.strip(), 'ok', r.stderr) + + def test_no_provider_code_runs_before_its_pin_is_checked(self): + r = subprocess.run([sys.executable, '-c', 'import hil_remote; print(hil_remote.core)'], cwd=SCRIPTS, + capture_output=True, text=True) + self.assertEqual(r.stdout.strip(), 'None', r.stderr) + with mock.patch.object(hr, 'core', None), mock.patch.object(hr, 'load_core') as load, \ + mock.patch.object(hr, 'check_pin', side_effect=lambda: hr.fail('lib/agentrc is at 0 or locally changed')): + self.refused('collect', 'r1', text='locally changed') + load.assert_not_called() + + def test_inputs_that_are_not_heads_committed_identity(self): + roster = self.wt / 'test/hil/ci.json' + cases = { + 'a changed roster': lambda: roster.write_text(roster.read_text() + ' '), + 'an untracked file': lambda: (self.wt / 'stray').write_text('x'), + 'a dirty submodule': lambda: (self.wt / 'lib/uf2/README').write_text('changed'), + 'a submodule off its pin': lambda: git(self.wt / 'lib/agentrc', 'checkout', '-q', 'HEAD~1'), + 'a missing dependency': lambda: shutil.rmtree(self.wt / 'lib/mcu'), + 'a dependency off its pin': lambda: commit(self.wt, {'tools/get_deps.py': GET_DEPS % ('0' * 40)}, 'off pin'), + } + for what, change in cases.items(): + with self.subTest(what): + snapshot = self.tmp / 'snap' + shutil.copytree(self.wt / 'lib', snapshot) + head = git(self.wt, 'rev-parse', 'HEAD') + change() + self.refused(text="not HEAD's committed identity") + git(self.wt, 'reset', '-q', '--hard', head) + git(self.wt, 'clean', '-qfd', '-e', 'lib/') + shutil.rmtree(self.wt / 'lib') + shutil.move(snapshot, self.wt / 'lib') + + +@needs_rig +class Attempt(Project): + def test_a_pass_rebuilds_the_committed_identity_on_the_rig_and_comes_home(self): + code, out = self.run_() + self.assertEqual((code, out['outcome'], out['pass'], out['collection']), (0, 'pass', True, 'complete')) + src = self.attempt() / 'src' + self.assertEqual(git(src, 'rev-parse', 'HEAD'), self.head) + self.assertEqual(git(src / 'lib/agentrc', 'rev-parse', 'HEAD'), self.pin, 'the unpushed candidate pin') + dep = src / 'lib/mcu/vendor/dep' + self.assertEqual((git(dep, 'rev-parse', 'HEAD'), git(dep, 'rev-parse', '--is-shallow-repository')), (self.dep, 'true')) + git(dep, 'fsck', '--no-dangling') + for repo in (src, *(src / f'lib/{n}' for n in ('agentrc', 'tinyusb', 'uf2')), dep): + self.assertTrue((repo / '.git').is_dir(), f'{repo}: a checkout of its own, no pointer file') + self.assertEqual(subprocess.run(['git', '-C', str(repo), 'config', '--local', '--get-regexp', 'remote'], + capture_output=True, env=fixture_env()).stdout, b'') + self.assertFalse([p for p in src.rglob('*') if p.is_symlink()]) + ran, = self.batteries() + argv = ran['argv'] + self.assertEqual((argv[0], argv[argv.index('--ci-state-root') + 1], argv[argv.index('--lock-wait') + 1], argv[-1]), + ('run', str(self.state), '0', 'test/hil/ci.json')) + self.assertNotIn('--done-marker', argv) # CI's markers stay CI's + self.assertNotIn('--started-marker', argv) + self.assertEqual(ran['hil'], []) + # home: the records, run dir, report and output, the paths recovery needs on the rig beside their local copies + local = self.wt / hr.RUNS / 'r1' + row, = out['boards'] + self.assertEqual((row['verdict'], row['phase']), ('pass', 'battery')) + report, = row['paths']['reports'] + self.assertTrue((self.wt / report['local']).is_file()) + self.assertEqual(report['remote'], str(self.attempt() / Path(report['local']).relative_to(f'{hr.RUNS}/r1'))) + self.assertEqual(out['remote']['checkout'], str(src)) + for rel in (f'out/{LPC}.build.json', f'out/{LPC}/stdout.txt', f'out/{LPC}/stderr.txt', 'manifest.json', 'result.json'): + self.assertTrue((local / rel).is_file(), rel) + self.assertEqual(json.loads((self.wt / hr.RUNS / 'r1.outcome.json').read_text()), out) + self.assertFalse((self.wt / hr.RUNS / '.r1.stage').exists()) + # reconnecting answers from the complete collection, and nothing launches the attempt again + launch = (self.attempt() / '.attempt/launch.json').read_text() + self.assertEqual(run_main('wait', 'r1', '--timeout', '1'), (0, out)) + self.assertEqual(run_main('collect', 'r1'), (0, out)) + self.assertEqual((self.attempt() / '.attempt/launch.json').read_text(), launch) + self.assertEqual(self.run_()[0], hr.REFUSED, 'a run id is used once') + + def test_boards_run_one_after_another_within_one_lock_wait(self): + os.environ.update(FAKE_SLEEP='2', FAKE_SLEEP_BOARD=OTHER) # the first battery uses up 2 s of the budget + code, out = self.run_('-b', OTHER, '--lock-wait', '100') + self.assertEqual((code, [r['board'] for r in out['boards']]), (0, [OTHER, LPC])) + waits = [int(b['argv'][b['argv'].index('--lock-wait') + 1]) for b in self.batteries()] + self.assertEqual(waits[0], 100) + self.assertLessEqual(waits[1], 98) + + def test_a_git_override_in_the_callers_environment_is_ignored(self): + victim = self.tmp / 'victim' + sha = origin(victim, 'victim') + os.environ.update(GIT_DIR=str(victim / '.git'), GIT_WORK_TREE=str(victim)) + code, out = self.run_() + self.assertEqual((code, out['outcome']), (0, 'pass')) + self.assertEqual(os.environ['GIT_DIR'], str(victim / '.git'), 'restored for the caller') + del os.environ['GIT_DIR'], os.environ['GIT_WORK_TREE'] + self.assertEqual(git(self.attempt() / 'src', 'rev-parse', 'HEAD'), self.head) + self.assertEqual(git(victim, 'rev-parse', 'HEAD'), sha) + + def test_a_driver_failure_after_a_passing_row_is_no_success(self): + stage = hr.core.stage_tree + + def obstructed(tree, src, dest='.'): + stage(tree, src, dest) + (Path(tree.dir) / 'progress.json').mkdir() # the driver's publication of its rows fails + with mock.patch.object(hr.core, 'stage_tree', obstructed), mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), \ + self.assertRaises(KeyboardInterrupt): + self.run_() + self.await_done() + (self.attempt() / 'progress.json').rmdir() + code, out = run_main('wait', 'r1', '--timeout', '5') + self.assertEqual((code, out['outcome'], out['pass'], out['boards'][0]['verdict']), (2, 'blocker', False, 'pass')) + self.assertIn('progress.json', out['driverFailure']) + + def test_an_unpublished_report_is_an_incomplete_copy_back_beside_the_native_verdict(self): + os.environ['FAKE_BATTERY'] = 'unpublished' + code, out = self.run_() + row, = out['boards'] + self.assertEqual((code, out['outcome'], row['verdict'], row['copyBack']), (hr.INCOMPLETE, 'incomplete', 'pass', 'incomplete')) + self.assertIn('never published: report not published: OSError: disk full', row['missing'][0]) + self.assertNotIn('reports', row['paths']) + + def test_each_report_pair_is_prepared_on_the_rig_before_it_comes_home(self): + os.environ['FAKE_MD'] = 'HIL run: fail\n' # a stale view beside a passing JSON + code, out = self.run_() + row, = out['boards'] + self.assertEqual((code, out['outcome'], row['verdict'], row['sharedVerdict']), (0, 'pass', 'pass', 'pass')) + md, = (self.wt / hr.RUNS / 'r1' / 'out' / LPC).glob('run-*/hil_report.md') + self.assertTrue(md.read_text(encoding='utf-8').startswith('HIL run: pass\n'), md.read_text()) + self.assertNotIn('HIL run: fail', md.read_text(encoding='utf-8')) + + @unittest.skipUnless(os.name != 'nt' and os.geteuid(), 'root writes a read-only dir') + def test_a_view_the_rig_could_not_check_never_comes_home(self): + os.environ.update(FAKE_MD='HIL run: pass\n', FAKE_READONLY='1') # neither prepared nor withheld on the rig + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + self.await_done() + run_dir, = (self.attempt() / 'out' / LPC).glob('run-*') + run_dir.chmod(0o755) + rel = (run_dir / 'hil_report.md').relative_to(self.attempt()).as_posix() + self.assertTrue((self.attempt() / rel).is_file(), 'still on the rig') + local = self.wt / hr.RUNS / 'r1' + unlink = Path.unlink + + def stuck(path, *a, **k): + if path.name == 'hil_report.md': + raise PermissionError(13, 'Permission denied', str(path)) + return unlink(path, *a, **k) + with mock.patch.object(Path, 'unlink', stuck): # a view that cannot be removed is never published + self.assertEqual(run_main('collect', 'r1')[0], hr.core.INCONCLUSIVE) + self.assertFalse(local.exists()) + with mock.patch.object(hr, 'report', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + run_main('collect', 'r1') # interrupted once the core published the collection + self.assertEqual(json.loads((local / hr.core.COLLECTION).read_text())['state'], 'complete') + self.assertFalse((local / rel).exists(), 'never published as evidence') + self.assertTrue((local / rel).with_name('hil_report.json').is_file()) + code, out = run_main('collect', 'r1') + row, = out['boards'] + self.assertEqual((code, out['outcome'], out['collection'], row['verdict'], row['sharedVerdict'], row['unsafeArtifacts']), + (0, 'pass', 'complete', 'pass', 'blocker', [rel])) + self.assertFalse((local / rel).exists()) + + @unittest.skipUnless(os.name != 'nt' and os.geteuid(), 'root lists an unlistable dir') + def test_a_view_under_a_run_root_the_rig_could_not_list_never_comes_home(self): + os.environ.update(FAKE_MD='HIL run: pass\n', FAKE_UNLISTABLE='1') + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + self.await_done() + board = self.attempt() / 'out' / LPC + board.chmod(0o755) # readable again by the copy-back + md, = board.glob('run-*/hil_report.md') + self.assertEqual(md.read_text(encoding='utf-8'), 'HIL run: pass\n', 'never checked on the rig') + code, out = run_main('collect', 'r1') + row, = out['boards'] + local = self.wt / hr.RUNS / 'r1' / md.relative_to(self.attempt()) + self.assertEqual((row['verdict'], row['sharedVerdict'], local.with_name('hil_report.json').is_file(), + local.exists()), ('pass', 'blocker', True, False)) + # the rig's inventory of that board is unverified: never a final collection + self.assertEqual((code, out['outcome'], row['unsafeArtifacts']), (hr.INCOMPLETE, 'incomplete', [f'out/{LPC}'])) + + def test_an_incomplete_collection_brings_home_no_unchecked_view(self): + os.environ['FAKE_MD'] = 'HIL run: pass\n' + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + self.await_done() + result = self.attempt() / 'result.json' + good = result.read_text() + result.write_text('{') # the driver's outcome unreadable: no view is known prepared + local = self.wt / hr.RUNS / 'r1' / 'out' / LPC + unlink = Path.unlink + + def stuck(path, *a, **k): + if path.name == 'hil_report.md': + raise PermissionError(13, 'Permission denied', str(path)) + return unlink(path, *a, **k) + with mock.patch.object(Path, 'unlink', stuck): # an unchecked view that cannot be removed is never published + self.assertEqual(run_main('collect', 'r1')[0], hr.core.INCONCLUSIVE) + self.assertFalse(local.exists()) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['collection']), (hr.INCOMPLETE, 'incomplete', 'incomplete')) + self.assertTrue(list(local.glob('run-*/hil_report.json'))) + self.assertEqual(list(local.glob('run-*/hil_report.md')), [], 'an unchecked view came home') + result.write_text(good) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['collection']), (0, 'pass', 'complete')) + md, = local.glob('run-*/hil_report.md') # prepared on the rig: it comes home with its result + self.assertTrue(md.read_text(encoding='utf-8').startswith('HIL run: pass\n')) + + def test_a_board_without_a_prepared_row_brings_home_no_view(self): + os.environ['FAKE_MD'] = 'HIL run: pass\n' + for run_id, edit in (('r1', lambda rows: []), # the driver ended before the board's row + ('r2', lambda rows: [{k: v for k, v in r.items() if k not in ('sharedVerdict', 'sharedDetail')} + for r in rows])): # a row whose report pair was never prepared + with self.subTest(run_id): + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_(run_id=run_id) + self.await_done(run_id) + result = self.attempt(run_id) / 'result.json' + data = json.loads(result.read_text()) + result.write_text(json.dumps(dict(data, boards=edit(data['boards'])))) + code, out = run_main('collect', run_id) + self.assertNotEqual(out['outcome'], 'pass') + local = self.wt / hr.RUNS / run_id / 'out' / LPC + self.assertTrue(list(local.glob('run-*/hil_report.json'))) + self.assertEqual(list(local.glob('run-*/hil_report.md')), [], 'an unchecked view came home') + + def test_a_short_collection_recovers_once_the_rig_has_the_artifact_again(self): + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + self.await_done() + log, = (self.attempt() / 'out' / LPC).glob('run-*/hil.log') + log.rename(self.tmp / 'hil.log') + self.assertEqual(run_main('collect', 'r1')[0], hr.INCOMPLETE) + (self.tmp / 'hil.log').rename(log) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['collection']), (0, 'pass', 'complete')) + self.assertTrue((self.wt / hr.RUNS / 'r1' / log.relative_to(self.attempt())).is_file()) + self.assertFalse([p for p in (self.wt / hr.RUNS).glob('.r1.*') if p.is_dir()], 'no staging dir left') + + def test_an_unreadable_result_is_never_a_final_collection(self): + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + self.await_done() + result = self.attempt() / 'result.json' + good = result.read_text() + for corrupt in ('{', json.dumps(dict(json.loads(good), runId='another'))): + with self.subTest(corrupt[:12]): + result.write_text(corrupt) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['collection']), (hr.INCOMPLETE, 'incomplete', 'incomplete')) + self.assertTrue(json.loads((self.wt / hr.RUNS / 'r1' / hr.core.COLLECTION).read_text())['unaccepted']) + result.write_text(good) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['collection']), (0, 'pass', 'complete')) + + def test_concurrent_collectors_never_remove_a_published_collection(self): + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + self.await_done() + tree, manifest = hr.core.load_attempt(self.wt, self.wt / hr.RUNS, 'r1'), hr.load_manifest('r1') + groups, optional = hr.members(manifest) + inside, release, got = threading.Event(), threading.Event(), [] + + def held(path): # the first collector pauses inside its predicate, under the collector lock + inside.set() + release.wait(30) + return hr.acceptance(tree, manifest)(path) + first = threading.Thread(target=lambda: got.append(hr.core.collect(tree, self.wt / hr.RUNS / 'r1', groups, optional, accept=held))) + first.start() + self.assertTrue(inside.wait(30)) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome']), (hr.core.INCONCLUSIVE, 'inconclusive')) + self.assertIn('another collector holds', out['detail']) + release.set() + first.join(30) + self.assertEqual(got[0]['state'], 'complete') + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['collection']), (0, 'pass', 'complete')) + + def test_a_transport_failure_after_the_launch_is_inconclusive_not_a_refusal(self): + code, out = self.run_() + self.assertEqual(code, 0) + shutil.rmtree(self.wt / hr.RUNS / 'r1') # nothing cached: the next collect must reach the rig + os.environ['FAKE_SSH_DOWN'] = "' status " + for argv in (('collect', 'r1'), ('wait', 'r1', '--timeout', '0')): + code, out = run_main(*argv) + self.assertEqual((code, out['outcome'], out['pass']), (hr.core.INCONCLUSIVE, 'inconclusive', False), argv) + self.assertEqual(run_main('collect', 'nope')[0], hr.REFUSED, 'an unknown run is still a refusal') + + def await_done(self, run_id='r1'): + done = self.attempt(run_id) / '.attempt/done.json' + deadline = time.monotonic() + 30 + while not done.exists() and time.monotonic() < deadline: + time.sleep(0.05) + self.assertTrue(done.exists()) + + def test_a_build_refusal_brings_its_diagnostics_and_no_battery(self): + os.environ['FAKE_BUILD'] = 'fail' + code, out = self.run_() + row, = out['boards'] + self.assertEqual((code, row['verdict'], row['phase'], row['detail']), (1, 'not-run', 'build', 'build failed: boom')) + local = self.wt / hr.RUNS / 'r1' + self.assertFalse((local / 'out' / LPC).exists()) + self.assertTrue((local / f'out/{LPC}.build.json').is_file()) + self.assertTrue((local / f'src/cmake-build/cmake-build-{LPC}/check_build.log').is_file()) + self.assertFalse((self.tmp / 'argv.json').exists(), 'no battery ran') + + def test_a_battery_without_a_report_is_a_blocker(self): + os.environ['FAKE_BATTERY'] = 'die' + code, out = self.run_() + self.assertEqual((code, out['outcome'], out['boards'][0]['verdict']), (2, 'blocker', 'blocker')) + self.assertIn('without a report', out['boards'][0]['detail']) + + def test_an_in_process_holder_is_left_running_and_reported_as_a_blocker(self): + os.environ['FAKE_BATTERY'] = 'holder' + code, out = self.run_() + row, = out['boards'] + self.assertEqual((code, row['verdict'], row['heldInProcess']), (2, 'blocker', True)) + pid = int((self.tmp / 'battery.pid').read_text()) + self.assertEqual(row['holder']['pid'], pid) + os.kill(pid, 0) # the attempt completed; the holder keeps the board, untouched by the driver's end + + def test_an_abandoned_ci_marker_is_a_blocker_before_any_build(self): + (self.state / 'ci-run').mkdir() + (self.state / 'ci-run' / f'{LPC}.started').touch() + code, out = self.run_() + row, = out['boards'] + self.assertEqual((code, row['verdict'], row['phase'], row['unfinishedAttempts']), (2, 'blocker', 'preflight', ['ci-run'])) + self.assertFalse((self.attempt() / f'out/{LPC}.build.json').exists()) + + def test_an_unreadable_ci_state_root_refuses_on_the_rig(self): + shutil.rmtree(self.state) + code, out = self.run_() + self.assertEqual((code, out['outcome']), (hr.REFUSED, 'refused')) + self.assertIn('cannot be read', out['detail']) + + def test_corrupt_packages_are_refused_before_the_launch(self): + stage = hr.core.stage_tree + + def corrupting(tree, src, dest='.'): + stage(tree, src, dest) + (Path(tree.dir) / 'bundles' / '0.bundle').write_bytes(b'corrupt') + with mock.patch.object(hr.core, 'stage_tree', corrupting): + code, out = self.run_() + self.assertEqual((code, out), (hr.REFUSED, None)) + self.assertFalse((self.attempt() / '.attempt/launch.claim').exists()) + self.assertEqual(run_main('wait', 'r1', '--timeout', '0')[0], hr.core.DEAD) + self.assertFalse((self.attempt() / '.attempt/launch.claim').exists(), 'waiting never launches') + + def test_a_replaced_attempt_token_refuses_the_staging(self): + create = hr.core.create_attempt + + def replaced(*a, **k): + tree = create(*a, **k) + (Path(tree.dir) / hr.core.TOKEN_FILE).write_text('another session\n') + return tree + with mock.patch.object(hr.core, 'create_attempt', replaced), rig.captured_fd2(): + self.assertEqual(self.run_()[0], hr.REFUSED) + self.assertFalse((self.attempt() / 'manifest.json').exists()) + + def test_a_wrapper_dead_while_staging_leaves_an_attempt_never_launched(self): + def killed(*a, **k): + raise KeyboardInterrupt + with mock.patch.object(hr.core, 'stage_tree', killed), self.assertRaises(KeyboardInterrupt): + self.run_() + self.assertFalse((self.wt / hr.RUNS / '.r1.stage').exists()) + code, out = run_main('wait', 'r1', '--timeout', '0') + self.assertEqual((code, out['outcome']), (hr.core.DEAD, 'not-launched')) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['remoteState']), (hr.core.DEAD, 'incomplete', 'not-launched')) + self.assertFalse((self.attempt() / '.attempt/launch.claim').exists()) + + def test_a_lost_launch_acknowledgement_finds_the_attempt_and_never_launches_another(self): + os.environ['FAKE_SSH_DROP'] = "' launch " + code, out = self.run_() + self.assertEqual((code, out['outcome']), (0, 'pass')) + launches = [c for c in self.ssh_calls() if "' launch " in ' '.join(c['argv'])] + self.assertEqual(len(launches), 1) + + def test_the_attempt_outlives_its_wrapper_and_partial_evidence_never_passes(self): + gate = self.tmp / 'gate' + os.environ['FAKE_GATE'] = str(gate) # the battery holds until the gate opens, at most 30 s + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + code, out = run_main('wait', 'r1', '--timeout', '0.2') + self.assertEqual((code, out['outcome']), (hr.core.RUNNING, 'running')) + code, out = run_main('collect', 'r1') + self.assertEqual((code, out['outcome'], out['remoteState'], out['pass']), (hr.core.RUNNING, 'incomplete', 'running', False)) + gate.touch() + code, out = run_main('wait', 'r1', '--timeout', '30') + self.assertEqual((code, out['outcome'], out['collection']), (0, 'pass', 'complete')) + self.assertEqual(run_main('collect', 'r1'), (0, out)) + + def test_a_driver_dead_before_completion_is_dead_never_done(self): + os.environ['FAKE_SLEEP'] = '30' + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + launch = self.attempt() / '.attempt/launch.json' + deadline = time.monotonic() + 10 + while not (self.tmp / 'argv.json').exists() and time.monotonic() < deadline: + time.sleep(0.05) + os.killpg(json.loads(launch.read_text())['sid'], signal.SIGKILL) + deadline = time.monotonic() + 10 + while run_main('wait', 'r1', '--timeout', '0')[0] != hr.core.DEAD and time.monotonic() < deadline: + time.sleep(0.1) + code, out = run_main('wait', 'r1', '--timeout', '0') + self.assertEqual((code, out['outcome'], out['pass']), (hr.core.DEAD, 'dead', False)) + self.assertEqual(run_main('collect', 'r1')[1]['outcome'], 'incomplete') + self.assertFalse((self.attempt() / '.attempt/done.json').exists(), 'nothing inferred a completion') + + def test_a_missing_artifact_is_an_incomplete_copy_back_even_when_the_battery_passed(self): + with mock.patch.object(hr, 'wait_for', side_effect=KeyboardInterrupt), self.assertRaises(KeyboardInterrupt): + self.run_() + done = self.attempt() / '.attempt/done.json' + deadline = time.monotonic() + 30 + while not done.exists() and time.monotonic() < deadline: + time.sleep(0.05) + log, = (self.attempt() / 'out' / LPC).glob('run-*/hil.log') + log.unlink() # gone from the rig after the driver listed it + code, out = run_main('wait', 'r1', '--timeout', '5') + row, = out['boards'] + self.assertEqual((code, out['outcome'], out['collection'], row['verdict'], row['copyBack']), + (hr.INCOMPLETE, 'incomplete', 'incomplete', 'pass', 'incomplete')) + self.assertEqual(row['missing'], [log.relative_to(self.attempt()).as_posix()]) + col = json.loads((self.wt / hr.RUNS / 'r1' / hr.core.COLLECTION).read_text()) + self.assertEqual(col['unaccepted'], [f'{LPC}: {row["missing"][0]}'], 'the core keeps it unaccepted, never final') + + +@needs_rig +class RealRunner(Project): + """The driver over the real `hil_test.py run` and the pinned lock helper: the protected battery lock as every project + sees it.""" + + def setUp(self): + super().setUp() + os.environ.update(FAKE_REAL_RUNNER=str(SCRIPTS / 'hil_test.py'), FAKE_LOCKS=str(self.locks), + FAKE_FACTS=json.dumps(LPC_FACTS)) + with mock.patch.object(ht, 'check_lock_pin'): + self.helper = ht.load_helper() + mock.patch.object(self.helper, 'BOARD_LOCK_DIR', str(self.locks)).start() + + def test_a_clean_battery_releases_the_board(self): + code, out = self.run_() + self.assertEqual((code, out['boards'][0]['verdict']), (0, 'pass')) + self.assertEqual(self.helper.record_state(LPC)[0], 'empty') + + def test_another_attempt_on_the_same_board_meets_its_lock(self): + fh = self.helper.take_board_lock(LPC, 'another attempt') + self.addCleanup(self.helper.release_board_lock, fh) + code, out = self.run_() + row, = out['boards'] + self.assertEqual((code, row['verdict']), (1, 'not-run')) + self.assertIn('locked', row['detail']) + + def test_a_ci_battery_started_between_the_preflight_and_the_lock_keeps_the_board_a_blocker(self): + os.environ['FAKE_CI_RACE'] = str(self.state / 'ci-old') + code, out = self.run_() + row, = out['boards'] + self.assertEqual((code, row['verdict'], row['phase']), (2, 'blocker', 'battery')) + self.assertIn('ci-old', row['detail']) + rec = self.helper.read_record(LPC) + self.assertTrue(self.helper.is_recovery_blocker(rec)) # every project's helper refuses the board + self.assertEqual((rec['reason'], rec['protected']), ('hil_test.py', True)) + self.assertEqual(Path(rec['runDir']).parent, self.attempt() / 'out' / LPC) + + def test_a_battery_killed_mid_run_leaves_a_stale_protected_record(self): + os.environ['FAKE_BATTERY'] = 'die' + pid = self.tmp / 'battery.pid' + + def kill_the_battery(): + end = time.monotonic() + 30 + while not pid.exists() and time.monotonic() < end: + time.sleep(0.02) + os.kill(int(pid.read_text()), signal.SIGKILL) + killer = threading.Thread(target=kill_the_battery) + killer.start() + code, out = self.run_() + killer.join() + row, = out['boards'] + self.assertEqual((code, row['verdict']), (2, 'blocker')) + self.assertIn('without a report', row['detail']) + rec = self.helper.read_record(LPC) + self.assertTrue(self.helper.is_recovery_blocker(rec)) + self.assertEqual((rec['reason'], rec['protected']), ('hil_test.py', True)) + + +class Driver(unittest.TestCase): + def test_the_rigs_hil_environment_never_reaches_a_runner(self): + env = {k: v for k, v in os.environ.items() if not k.startswith(('HIL_', 'GIT_'))} + env.update(HIL_NO_BOARD_LOCK='1', HIL_X='y', GIT_DIR='/elsewhere/.git') + with tempfile.TemporaryDirectory() as d, mock.patch.dict(os.environ, env, clear=True): + m = {'schema': drv.SCHEMA, 'version': drv.VERSION, 'runId': 'r', 'head': '0' * 40, 'hostname': 'not-this-host', + 'repos': [], 'submodules': {}, 'roster': {'path': 'r.json'}, 'driver': {'path': 'd.py'}, 'boards': [], + 'lockWait': 0, 'failAt': None, 'ciStateRoot': d} + (Path(d) / 'manifest.json').write_text(json.dumps(m)) + with mock.patch.object(drv.Path, 'cwd', return_value=Path(d)): + self.assertEqual(drv.main(['manifest.json']), drv.REFUSED) + self.assertFalse([k for k in os.environ if k.startswith(('HIL_', 'GIT_'))]) + result = json.loads((Path(d) / drv.RESULT).read_text()) + self.assertEqual(result['strippedEnv'], ['GIT_DIR', 'HIL_NO_BOARD_LOCK', 'HIL_X']) + self.assertIn('not not-this-host', result['refused']) + + def test_a_git_override_on_the_rig_never_aims_the_rebuild_at_another_repository(self): + with tempfile.TemporaryDirectory() as d: + d = Path(d) + env = {k: v for k, v in os.environ.items() if not k.startswith(('HIL_', 'GIT_'))} + config = d / 'gitconfig' + config.write_text('[user]\n name = t\n email = t@t\n[commit]\n gpgsign = false\n') + with mock.patch.dict(os.environ, dict(env, GIT_CONFIG_GLOBAL=str(config), GIT_CONFIG_NOSYSTEM='1'), clear=True): + head = origin(d / 'repo', 'candidate') + victim = origin(d / 'victim', 'victim') + attempt = d / 'attempt' + (attempt / 'bundles').mkdir(parents=True) + git(d / 'repo', 'bundle', 'create', '-q', str(attempt / 'bundles/0.bundle'), 'HEAD') + m = {'schema': drv.SCHEMA, 'version': drv.VERSION, 'runId': 'r', 'head': head, 'hostname': socket.gethostname(), + 'repos': [{'path': '.', 'commit': head, 'bundle': 'bundles/0.bundle', 'shallow': [], + 'sha256': hashlib.sha256((attempt / 'bundles/0.bundle').read_bytes()).hexdigest()}], + 'submodules': {}, 'roster': {'path': 'r.json', 'sha256': ''}, 'driver': {'path': 'd.py', 'sha256': ''}, + 'boards': [], 'lockWait': 0, 'failAt': None, 'ciStateRoot': str(d)} + (attempt / 'manifest.json').write_text(json.dumps(m)) + with mock.patch.dict(os.environ, dict(env, GIT_DIR=str(d / 'victim/.git'), GIT_WORK_TREE=str(d / 'victim')), clear=True), \ + mock.patch.object(drv.Path, 'cwd', return_value=attempt): + drv.main(['manifest.json']) + self.assertEqual((git(d / 'victim', 'rev-parse', 'HEAD'), (d / 'victim/README').read_text()), (victim, 'victim')) + self.assertEqual(git(attempt / 'src', 'rev-parse', '--absolute-git-dir'), str(attempt / 'src/.git')) + self.assertEqual(git(attempt / 'src', 'rev-parse', 'HEAD'), head) + + def test_a_malformed_manifest_is_refused(self): + with tempfile.TemporaryDirectory() as d: + bad = {'schema': drv.SCHEMA, 'version': drv.VERSION} + for m in (bad, dict(bad, version=2), {**bad, **{k: [] for k in drv.FIELDS}, 'repos': [{'path': '../x', 'commit': '0' * 40, 'shallow': []}]}): + (Path(d) / 'manifest.json').write_text(json.dumps(m)) + with self.assertRaises(drv.Refused): + drv.load_manifest(Path(d) / 'manifest.json') + + def test_inventory_entries_outside_the_boards_artifacts_are_an_incomplete_copy_back(self): + with tempfile.TemporaryDirectory() as d: + for listed in (['out/other.build.json'], ['out/b/../x'], ['out/b//x'], ['out/b/./x'], ['/etc/passwd'], [3], 'x'): + with self.subTest(listed): + self.assertTrue(hr.artifacts_missing({'phase': 'battery', 'artifacts': listed}, 'b', Path(d))[0].startswith('(')) + for unsafe in (['out/b/stdout.txt'], ['out/b/run-1/../hil_report.md'], ['out/c/run-1/hil_report.md'], 'x'): + with self.subTest(unsafe): + row = {'phase': 'build', 'artifacts': [], 'unsafeArtifacts': unsafe} + self.assertTrue(hr.artifacts_missing(row, 'b', Path(d))[0].startswith('(unsafeArtifacts')) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index b8467ceaa..1583df659 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -36,7 +36,7 @@ jobs: submodules: true - name: Fetch tags - run: git fetch --tags + run: git fetch --tags --no-recurse-submodules - name: Setup Toolchain id: setup-toolchain diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml index 012c0f4e6..c095490b0 100644 --- a/.github/workflows/hil.yml +++ b/.github/workflows/hil.yml @@ -93,10 +93,25 @@ jobs: python3 .claude/skills/hil-contract/scripts/hil_ci.py --board $BOARDS ${FAIL_AT:+--fail-at "$FAIL_AT"} \ --roster test/hil/ci.json --out "$HIL_OUT" + # hil_ci.py's decision, written last (upload-views): `upload` only when no report view is unsafe; any other + # content, no file (hil_ci.py did not finish) or a failed step uploads no hil_report.md at all + - name: Read the report views' upload decision + id: views + if: always() + run: | + if [ "$(cat "$HIL_OUT/upload-views" 2>/dev/null)" = upload ]; then + echo "exclude=" >> "$GITHUB_OUTPUT" + else + echo "exclude=!$HIL_OUT/**/hil_report.md" >> "$GITHUB_OUTPUT" + fi + - name: Upload the records, reports and logs if: always() uses: actions/upload-artifact@v4 with: name: hil-ci-lan-${{ github.run_attempt }} - path: ${{ env.HIL_OUT }} + path: | + ${{ env.HIL_OUT }} + ${{ steps.views.outputs.exclude }} + ${{ steps.views.outcome != 'success' && format('!{0}/**/hil_report.md', env.HIL_OUT) || '' }} if-no-files-found: ignore diff --git a/.gitignore b/.gitignore index 8217d1dd7..0b1bc5175 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ lib/mcu/ lib/sct_neopixel __pycache__/ *.pyc +.hil-remote/ diff --git a/lib/agentrc b/lib/agentrc index 330f58da3..9e41f6fc8 160000 --- a/lib/agentrc +++ b/lib/agentrc @@ -1 +1 @@ -Subproject commit 330f58da350dbaf00fee7b5875ad695c94fef81c +Subproject commit 9e41f6fc89375df73d453c45e4c5a97103830fcd