diff --git a/.agents b/.agents new file mode 120000 index 000000000..c8161850a --- /dev/null +++ b/.agents @@ -0,0 +1 @@ +.claude \ No newline at end of file diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md new file mode 100644 index 000000000..f2c6219e8 --- /dev/null +++ b/.claude/skills/build/SKILL.md @@ -0,0 +1,49 @@ +--- +name: build +description: The TinyUF2 build contract. Use when a change must be verified by building, when an agent needs the boards a diff affects, or when a workflow asks for "the project's build". Resolves scope to boards, builds each the way CI does in a private dir, runs the ghostfat self-test when its inputs change, reports pass/fail with the first error. +--- + +# Build contract + +`scripts/check_build.py` is the entry point for verifying a change; every agent and workflow that does so calls it rather than composing make, cmake or idf.py lines. A debug build for a probe session is not verification and uses the port's own build directly (below). + +```bash +B=.claude/skills/build/scripts/check_build.py +python3 $B --scope # boards a change affects +python3 $B --base master # same, from the branch diff +python3 $B --base master --worktree # same, plus uncommitted and untracked changes +python3 $B --select-only (--scope ... | --base REF [--worktree] | --diff-file FILE) [--hil-config ROSTER] # the selection alone +python3 $B --board feather_stm32f405_express [--board ...] +python3 $B --board lpcxpresso55s69 -D LOG=2 # a define, passed to the board's backend +python3 $B ... --shared # keep cmake-build/cmake-build-, the dir a flash or HIL run uses +python3 $B --board imxrt1060_evk --backend cmake --shared --hil-config test/hil/local.json # the HIL's receipt +python3 $B ... --idf local # exported ESP-IDF instead of CI's docker image +``` + +The last stdout line is JSON. An agent acts on `pass`; `firstError`, and `error` on exit 2, naming the cause or the remedy; `boards[]` (`status` `ok`, `failed` or `error`, `log`, `artifacts`, `notImplemented`); `built`, the artifacts of `--shared` roots, the only ones kept after a pass, with each one's hash in `boards[].sha256`; `nothingToBuild` and `uncovered`, each path with its reason; and `idf.matchesCi`. `input` and `defines` record what a build used (HEAD, `dirty`, submodule and `get_deps` dependency commits, `dirtyInputs` for any of those with local changes, `inputsDirtiedByBuild` for any that changed or moved while building, `.` for the checkout itself): the HIL contract binds firmware to this record. `paths` gives every changed path's `rule` and `reason`, the answer to why a path selected what it did. Exit 0 pass, 1 a build, self-test or test failed, 2 usage, resolution or a missing prerequisite, 3 an uncovered path; 2 outranks 1, and 1 outranks 3. `-v` streams the build to stderr; each build's log is `/check_build.log`. + +`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths,ci}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`) but `dfu_runtime`'s, which the battery never runs. `dfu_runtime` selects the esptool roster boards of the ports that build it, and is uncovered once for each port that builds it without one. + +## Reporting + +An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the JSON this way: `buildOk` is `pass`. A pass with no board, self-test or test run is "nothing to build": quote each `nothingToBuild` reason in `notes`, so nobody mistakes it for a build that ran. Exit 3 is `buildOk` false with each `uncovered` reason quoted verbatim: that path is firmware no CI build compiles, and a human adds coverage or accepts the gap. Exit 1 or 2 quotes `firstError` (and `error`). A pass on `--idf local` whose `idf.matchesCi` is false is supplemental evidence only: say so in `notes`; acceptance needs CI's IDF. + +## Judgment + +- **Selection is the script's**, first match per path, and `paths[].reason` says why. A mixed change takes the union, and an uncovered path fails it whatever else built. The policy behind the rules: + - A change to a port, board, app or dependency builds the boards that compile it: a board's own files that board; other port files the port's `REPRESENTATIVES` (rig boards first, every ESP32 target the port's boards use, both i.MX RT10xx and RT1170); an app or dependency the representatives of the ports whose CI backend compiles it. + - Shared code (`src/`, `lib/`, the shared build files) and anything unclassified builds every port's representatives, plus the ghostfat self-test when a ghostfat input changed. + - A change to how boards build (this script, `build.yml`, `build_util.yml`, `ci_set_matrix.py`, `.github/actions/`) runs the contract's own tests and one smoke build per backend, and a change to this script the HIL battery on every roster board, since the HIL runner imports it; no firmware build verifies workflow syntax or orchestration. A change to the contract's tests runs the tests only. + - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads; `ports/make.mk` and `ports/rules.mk` select the make ports' representatives and that build. + - Firmware no CI build compiles is uncovered: `ports/template_port/`, `ports/ch32v20x/` (`--board ch32v203_r0_1v0` builds it on request with make, with a note), a build file no CI build of the port reads (a make port's CMake files, a cmake port's make files but CodeQL's, Espressif's `Makefile`), a stub app. + - Docs and agent, editor and repo metadata build nothing. + - `build.ci` is CI's selection from the same rules (`.github/workflows/ci_set_matrix.py` reads it on a pull request): every board a change reaches, not the representatives. A board's own files select that board; a port file, dependency, shared app or `make.mk`/`rules.mk` every board of the ports it reaches; a port app the boards that enable it. Shared code and a change to how boards build select `full`, every board and the ghostfat self-test; a ghostfat input the self-test alone. Tests, docs, CodeQL's build and uncovered paths select nothing. +- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`, which also builds each app the port's `CMakeLists.txt` adds with `family_add_app()` as its own project; Espressif `idf.py build`. A port moves from make to cmake in `build.yml` once a board of it passes the HIL battery with cmake. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds (cmake: its `family_add_app()` calls enabled for the board). A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. +- **`--backend cmake`** is for the HIL's receipt only, with `--board` and `--shared`: on a make port it is no CI evidence, and it is how such a port's board gets the HIL run that moves the port to cmake. Espressif is refused. `-D` takes the cmake spellings: `LOGGER=RTT`, not `rtt`; `DEBUG` is ignored, the build stays MinSizeRel; `LOG` (default 0, as in make) sets `TUF2_LOG` and `CFG_TUSB_DEBUG`; raw flags go through `CFLAGS_CLI`. +- **A HIL record** (`--board --shared --hil-config `) binds the build's own flash target to the board's roster flasher: a J-Link board is configured with `JLINK_OPTION=-USB -ExitOnError 1 -NoGui 1` (so `tinyuf2-jlink` selects that probe and fails non-zero), an esptool board's ESP-IDF `flash` target takes the bridge's tty at flash time. `boards[].flash` holds the target, the roster flasher, the target's generated command (checked to name that probe or chip and the build's own image), the configure defines that bound it and the build-tree files it reads (in `sha256` too); a rebuild and a HIL flash hold the build dir's lock (`.lock`). Another flasher, a roster `args` other than `-device ` or `--chip `, or a `-D` of `JLINK_OPTION`, `OPENOCD_SERIAL`, `OPENOCD_OPTION` or `ESPPORT` is refused. Before flashing, the HIL dry-runs that target under the lock and refuses any command but the flasher and, on ESP-IDF, the generated ones that write no flash input (its bootloader subproject dry-run too); ESP-IDF re-runs CMake after any commit (its `git describe` dependency), so rebuild the record on the commit under test. +- **ESP-IDF** builds in CI's image, `espressif/idf:`, through docker by default; a missing image is exit 2 naming the `docker pull`. `--idf local` uses the exported IDF and records its version and `matchesCi`: evidence from a different IDF is supplemental, not acceptance. +- **ghostfat** builds each `ports/test_ghostfat/boards/*` config natively with cmake and runs its `selftest` target, which compares the generated image with the config's known-good one, in the private dir, one config at a time: `huge` needs about 3 GiB of disk while it runs. A passing config's dir is removed; a failing one keeps its log. +- **Private dirs for parallel agents.** Each build starts from an empty `cmake-build/cmake-build-agent--` (with `-` when a board builds with another backend than its port's) and is removed when it passes (`removed`; its `artifacts` say what it produced, nothing is left to flash); a failed one is kept for its log. Nothing is written into the source tree: make takes `BUILD_ROOT=`, cmake and idf `-DARTIFACT_PATH=/_bin`. `--shared` rebuilds `cmake-build/cmake-build-` from clean and keeps it for flashing; it needs exclusive ownership of that board. +- **Dependencies are never fetched here.** `lib/tinyusb` and `lib/uf2` are submodules (`git submodule update --init lib/tinyusb lib/uf2`); a port's `tools/get_deps.py` rows must exist. In a worktree, symlink them from the primary checkout (CLAUDE.md), never run `get_deps.py` through a symlink: it runs `git reset --hard` in each dependency. +- **Debug builds** for a probe session use the port's backend directly: make `make -C ports/ BOARD= DEBUG=1 LOG=2 LOGGER=rtt all` (`LOGGER=swo` too); cmake `-DCMAKE_BUILD_TYPE=Debug -DLOG=2 -DLOGGER=RTT` (only `RTT` adds the RTT sources). Flash targets: make `flash-jlink`, `flash-pyocd`, `flash-stlink`, `flash-uf2`; cmake `tinyuf2-jlink`, `-stlink`, `-openocd`, `-pyocd` (lpc55, kinetis_k32l2), `-dfu-util` (stm32l4), and `-uf2` for each app; Espressif `idf.py -C ports/espressif -DBOARD= flash`. +- **Printing a command is not verification.** A verdict is the JSON of a run that happened; an agent that cannot run the script reports the build as not run. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py new file mode 100755 index 000000000..e11328dd2 --- /dev/null +++ b/.claude/skills/build/scripts/check_build.py @@ -0,0 +1,1115 @@ +#!/usr/bin/env python3 +"""Build TinyUF2 for the boards a change affects, or for named boards: the project's build contract. + + check_build.py (--scope PATH... | --base REF [--worktree] | --board B...) [-D VAR=VAL]... [--shared] [--idf docker|local] [-v] + check_build.py --board B --shared --hil-config ROSTER [--backend cmake] (a HIL record: binds the flash target) + check_build.py --select-only (--scope PATH... | --base REF [--worktree] | --diff-file FILE) [--hil-config ROSTER] + +Each board builds the way .github/workflows/build.yml builds its port (make, cmake or ESP-IDF) +into its own build root, cmake-build/cmake-build-agent-- (--shared: +cmake-build/cmake-build-), and must produce its expected artifacts. A ghostfat input +also builds and runs the native ghostfat self-test of every ports/test_ghostfat config. + +stdout ends with one JSON line (SKILL.md). Exit 0 pass, 1 a build or test failed, 2 usage, +resolution or a missing prerequisite ("error" names the remedy), 3 uncovered paths. +""" + +import argparse +import contextlib +import fcntl +import functools +import hashlib +import json +import os +import re +import runpy +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +BUILD_YML = ROOT / '.github' / 'workflows' / 'build.yml' +BUILD_DIR = ROOT / 'cmake-build' + +# Backend of every port CI builds, as build.yml's jobs build it; test_build.py holds the two equal. +BACKENDS = { + 'espressif': 'idf', + 'kinetis_k32l2': 'make', + 'lpc55': 'cmake', + 'maxim': 'cmake', + 'mimxrt10xx': 'cmake', + 'stm32f3': 'cmake', + 'stm32f4': 'cmake', + 'stm32h5': 'cmake', + 'stm32h7': 'make', + 'stm32l4': 'make', +} +# Boards that stand in for their port: rig boards first; espressif covers every IDF target, +# mimxrt10xx both the RT10xx and RT1170 boot layouts. +REPRESENTATIVES = { + 'espressif': ['espressif_esp32s3_devkitm_1', 'espressif_esp32s2_devkitc_1', 'espressif_p4_function_ev', + 'espressif_esp32s31_func_core'], + 'kinetis_k32l2': ['frdm_k32l2b'], + 'lpc55': ['lpcxpresso55s69'], + 'maxim': ['max32666fthr'], + 'mimxrt10xx': ['imxrt1064_evk', 'imxrt1170_evk'], + 'stm32f3': ['stm32f303disco'], + 'stm32f4': ['stm32f411ve_discovery', 'feather_stm32f405_express'], + 'stm32h5': ['stm32h563_nucleo'], + 'stm32h7': ['daisy_seed'], + 'stm32l4': ['swan_r5'], +} +# Ports no CI job builds: a change there is uncovered; --board may still build one explicitly. +UNBUILT = { + 'ch32v20x': ('make', 'its riscv job is disabled in build.yml'), + 'template_port': (None, 'a template no build compiles'), +} +# compiler a port's make/cmake build needs, when not arm-none-eabi-gcc +COMPILER = {'ch32v20x': 'riscv-none-elf-gcc'} +# ports whose make self-update target only echoes "not implemented yet" +NO_SELF_UPDATE = {'ch32v20x', 'kinetis_k32l2', 'stm32h7', 'stm32l4'} +GHOSTFAT = 'test_ghostfat' +GHOSTFAT_INPUTS = ('src/ghostfat.c', 'src/favicon/', 'src/board_api.h', 'src/compile_date.h', 'src/uf2.h', + 'lib/uf2', 'ports/family_support.cmake', 'ports/test_ghostfat/', + '.github/workflows/build_ghostfat.yml') +CORE = ('src/', 'lib/', 'ports/family_support.cmake', 'CMakeLists.txt', 'cmake/', 'tools/get_deps.py') +# make files every Makefile port includes: read by the make ports' builds and CodeQL's +MAKE_SHARED = ('ports/make.mk', 'ports/rules.mk') +CONTRACT_TESTS = ('.claude/test/test_build', '.claude/test/test_hil') +# the HIL contract's runner and rosters: a change there re-runs the battery on every roster board +HIL_CONTROL = ('.claude/skills/hil/scripts/', 'test/hil/') +# hil_test.py imports these: the identity, artifacts and build dirs the battery trusts +HIL_IMPORTS = '.claude/skills/build/scripts/' +BUILD_CONTROL = (HIL_IMPORTS, '.github/workflows/build.yml', + '.github/workflows/build_util.yml', '.github/workflows/ci_set_matrix.py', '.github/actions/') +NOTHING = ('.claude/', '.github/', '.idea/', 'tools/codespell/', 'tools/gen_boards.py', 'tools/touch1200.py', + 'ports/build_all.py', 'CLAUDE.md', 'AGENTS.md', '.agents', 'LICENSE', '.gitignore', '.gitmodules', + '.pre-commit-config.yaml', '.codespellrc', '.clang-format') +CODEQL = '.github/workflows/codeql-buildscript.sh' +# one board per backend for a change to how boards build +SMOKE = {'make': 'frdm_k32l2b', 'cmake': 'feather_stm32f405_express', 'idf': 'espressif_esp32s3_devkitm_1'} + + +# What a roster flasher runs to install TinyUF2: the build's own flash target, and the build-tree files that +# target reads, which a HIL record pins so the runner can check them before it flashes. +FLASH = {'jlink': ('tinyuf2-jlink', ('tinyuf2.jlink', 'tinyuf2.hex')), + 'esptool': ('flash', ('flash_args', 'bootloader/bootloader.bin', 'partition_table/partition-table.bin', + 'ota_data_initial.bin', 'tinyuf2.bin'))} +# a roster flasher's args, whose one group is the device or chip its flash target must name +FLASHER_ARGS = {'jlink': r'-device (\S+)', 'esptool': r'--chip (\w+)'} +NINJA_STEP = re.compile(r'\[\d+/\d+\] ') # ninja's progress prefix +# configure variables that select a probe or port: a HIL record sets them from the roster, never from -D +SELECTION = ('JLINK_OPTION', 'OPENOCD_SERIAL', 'OPENOCD_OPTION', 'ESPPORT') + + +class Fail(Exception): + pass + + +def git(*args): + r = subprocess.run(['git', *args], capture_output=True, text=True, cwd=ROOT) + if r.returncode != 0: + raise Fail(f'git {" ".join(args)} failed: {r.stderr.strip()}') + return r.stdout + + +def ports_dir(port): + return ROOT / 'ports' / port + + +def boards_of(port): + d = ports_dir(port) / 'boards' + return sorted(p.name for p in d.iterdir() if p.is_dir()) if d.is_dir() else [] + + +def port_of(board): + hits = sorted(p.parent.parent.name for p in (ROOT / 'ports').glob(f'*/boards/{board}') if p.is_dir()) + hits = [h for h in hits if h != GHOSTFAT] + if len(hits) != 1: + raise Fail(f'board {board!r} matches {hits or "no"} port') + return hits[0] + + +def ci_idf_version(): + m = re.search(r"toolchain:\s*'esp-idf'\s*\n\s*toolchain_version:\s*'([^']+)'", BUILD_YML.read_text()) + if not m: + raise Fail(f'no esp-idf toolchain_version in {BUILD_YML.relative_to(ROOT)}') + return m.group(1) + + +def app_boards(app_dir): + """Boards an app's Makefile builds for: None for every board, [] for a stub that builds + nothing (no make.mk or app.mk include, only an echo).""" + mk = app_dir / 'Makefile' + if not mk.is_file(): + return None + text = mk.read_text() + if not re.search(r'^\s*include\s+\S*(make|app)\.mk', text, re.M): + return [] + return re.findall(r'ifeq\s*\(\$\(BOARD\),\s*(\w+)\)', text) or None + + +@functools.cache +def codeql_build(): + """(port, board) codeql-buildscript.sh builds with make `all` on every pull request.""" + text = (ROOT / CODEQL).read_text() + ports = set(re.findall(r'(?:cd|-C)\s+ports/(\w+)', text)) + boards = set(re.findall(r'^\s*make\b[^\n]*\bBOARD=(\w+)[^\n]*\ball\b', text, re.M)) + if len(ports) != 1 or len(boards) != 1: + raise Fail(f'cannot read one port and one `make ... all` board from {CODEQL}: update codeql_build()') + return ports.pop(), boards.pop() + + +def build_file_kind(name): + """'make' for a Makefile or *.mk, 'cmake' for a CMakeLists.txt or *.cmake, else None.""" + if name == 'Makefile' or name.endswith('.mk'): + return 'make' + if name == 'CMakeLists.txt' or name.endswith('.cmake'): + return 'cmake' + return None + + +def read_by_backend(path, backend): + """Whether CI's backend for the port reads this build file: a make port never reads CMake + files, a cmake or idf port never reads Makefiles. Sources and headers are read by both.""" + kind = build_file_kind(path.rsplit('/', 1)[-1]) + return kind is None or (kind == 'make') == (backend == 'make') + + +def code_lines(f): + return '\n'.join(l for l in f.read_text(errors='replace').splitlines() if not l.lstrip().startswith('#')) + + +def espressif_apps(): + """Apps ports/espressif/CMakeLists.txt builds as ExternalProjects.""" + return re.findall(r'SOURCE_DIR\s+\$\{CMAKE_CURRENT_LIST_DIR\}/apps/(\w+)', code_lines(ports_dir('espressif') / 'CMakeLists.txt')) + + +def cmake_app_calls(port): + """The family_add_app() calls of a cmake port's CMakeLists.txt as (name, source dir, boards): boards + None for every board, else those its BOARDS argument names.""" + calls = [] + for m in re.finditer(r'^\s*family_add_app\(\s*([\w-]+)\s+([^\s)]+)([^)]*)\)', + code_lines(ports_dir(port) / 'CMakeLists.txt'), re.M): + boards = re.search(r'\bBOARDS\s+([^)]*)', m.group(3)) + calls.append((m.group(1), m.group(2), boards.group(1).split() if boards else None)) + return calls + + +def cmake_apps(port, board): + """Apps a cmake port builds for the board, each named after the app's target and so its .uf2.""" + return [name for name, _, boards in cmake_app_calls(port) if boards is None or board in boards] + + +def port_app_selection(port, path): + """Boards that compile a change under ports//apps/: the app's own enabled boards, or for + a file shared by the port's apps (app.mk) those of every app that includes it; and CI's scope.""" + parts = path.split('/') + apps_dir = ports_dir(port) / 'apps' + if BACKENDS[port] == 'idf': + if len(parts) >= 4 and parts[3] not in espressif_apps(): + return None, f'ports/espressif/CMakeLists.txt builds no ExternalProject for apps/{parts[3]}', {} + return (rep_selection([port], f'builds {"/".join(parts[2:4])}'), f'{port} {"/".join(parts[2:4])}', + whole_ports([port])) + if BACKENDS[port] == 'cmake': + calls = [c for c in cmake_app_calls(port) if c[1].endswith(f'/apps/{parts[3]}')] if len(parts) >= 5 else [] + if not calls: + return None, f'ports/{port}/CMakeLists.txt adds no app that reads {path}', {} + boards, ci = {}, {} + for name, _, enabled in calls: + boards.update(rep_selection([port], f'builds {name}') if enabled is None else + {(b, 'cmake'): f'enables {name}' for b in enabled}) + ci = merge_ci(ci, whole_ports([port]) if enabled is None else {port: enabled}) + return boards, f'{port} apps: {", ".join(sorted(c[0] for c in calls))}', ci + if len(parts) >= 5 and (apps_dir / parts[3]).is_dir(): + apps = [apps_dir / parts[3]] + else: + apps = [m.parent for m in apps_dir.glob('*/Makefile') if re.search(rf'^\s*include\s+\S*\b{re.escape(parts[-1])}\b', + m.read_text(), re.M)] + boards, ci = {}, {} + for app in apps: + enabled = app_boards(app) + if enabled is None: + boards.update(rep_selection([port], f'builds apps/{app.name}')) + for b in enabled or []: + boards[(b, BACKENDS[port])] = f'enables apps/{app.name}' + ci = merge_ci(ci, whole_ports([port]) if enabled is None else {port: enabled}) + if not boards: + return None, f'no {port} board builds an app that reads {path}', {} + return boards, f'{port} apps: {", ".join(sorted(a.name for a in apps))}', ci + + +def ports_using_app(app): + """CI-built ports whose build, in the backend CI uses, compiles sources of the top-level apps/.""" + hits = [] + for port, backend in sorted(BACKENDS.items()): + kind = 'make' if backend == 'make' else 'cmake' + ref = re.compile(rf'(?= 2 and parts[0] == 'ports' and parts[1] in UNBUILT: + return out('unbuilt-port', 'gap', f'ports/{parts[1]}: {UNBUILT[parts[1]][1]}') + if len(parts) >= 3 and parts[0] == 'ports' and parts[1] in BACKENDS: + port = parts[1] + if not read_by_backend(path, BACKENDS[port]): + if port == codeql_port and parts[2] != 'apps' and (parts[2] != 'boards' or parts[3:4] == [codeql_board]): + return out('codeql', 'select', f'CodeQL builds {codeql_board} with make on every pull request', + {(codeql_board, 'make'): f'reads {path}'}) + return out('other-backend', 'gap', f'CI builds {port} with {BACKENDS[port]}, which never reads {parts[-1]}') + if parts[2] == 'boards' and len(parts) >= 4 and (ports_dir(port) / 'boards' / parts[3]).is_dir(): + return out('board', 'select', f'board {parts[3]} changed', + {(parts[3], BACKENDS[port]): f'{path} is its own'}, ghostfat, ci={port: [parts[3]]}) + if parts[2] == 'apps': + boards, reason, ci = port_app_selection(port, path) + return out('port-app', 'select' if boards else 'gap', reason, boards, ci=ci) + return out('port', 'select', f'{port} port file', rep_selection([port], f'{port} representative'), ghostfat, + ci=whole_ports([port])) + if len(parts) >= 2 and parts[0] == 'ports' and len(parts) >= 3 and (ROOT / 'ports' / parts[1]).is_dir(): + raise Fail(f'ports/{parts[1]} has no backend in check_build.py: add it to BACKENDS or UNBUILT') + if parts[0] == 'apps' and len(parts) >= 2: + users = ports_using_app(parts[1]) + if not users: + return out('app', 'gap', f'no CI-built port builds apps/{parts[1]}') + return out('app', 'select', f'apps/{parts[1]} is built by {", ".join(users)}', + rep_selection(users, f'builds apps/{parts[1]}'), ci=whole_ports(users)) + if path.startswith(('lib/mcu/', 'lib/sct_neopixel')): + families = deps_families(path) + if families is not None: + if not families: + return out('dep', 'gap', f'{path}: no CI-built port uses this dependency') + return out('dep', 'select', f'dependency of {", ".join(families)}', rep_selection(families, f'uses {path}'), + ci=whole_ports(families)) + if path in MAKE_SHARED: + make_ports = [p for p, be in BACKENDS.items() if be == 'make'] + return out('make-shared', 'select', 'every make port and the CodeQL build read it', + {**rep_selection(make_ports, f'make reads {path}'), + (codeql_board, 'make'): f'codeql build reads {path}'}, ci=whole_ports(make_ports)) + if path.startswith(NOTHING): + return out('nothing', 'none', 'no firmware build reads this path') + if path.startswith(CORE): + return out('core', 'core', 'shared by every port', ghostfat=ghostfat, ci=FULL) + return out('unclassified', 'core', 'unclassified path: every port', ghostfat=ghostfat, ci=FULL) + + +def repo_path(p): + """A path as git spells it, relative to the checkout: ./src/x.c and /abs/.../src/x.c are src/x.c.""" + # lexical, not resolve(): a worktree's dependency symlinks point into the primary checkout + p = os.path.relpath(os.path.abspath(p), ROOT) if os.path.isabs(p) else os.path.normpath(p) + if p.startswith('..'): + raise Fail(f'{p} is outside the checkout') + return p + + +def changed_paths(base=None, worktree=False, scope=None, diff_file=None): + if diff_file: + try: + lines = Path(diff_file).read_text().splitlines() + except OSError as e: + raise Fail(f'cannot read --diff-file: {e}') + return [repo_path(l.strip()) for l in lines if l.strip()], {'mode': 'diff-file'} + if scope: + paths = [] + for p in scope: + p = repo_path(p) + if (ROOT / p).is_dir() and not (ROOT / p / '.git').exists(): + files = [f for f in git('ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', p).split('\0') if f] + if not files: + raise Fail(f'{p} is a directory with no files git reports') + paths += files + elif (ROOT / p).exists() or subprocess.run(['git', 'cat-file', '-e', f'HEAD:{p}'], cwd=ROOT, + capture_output=True).returncode == 0: + paths.append(p) + else: + raise Fail(f'{p} does not exist and is not tracked at HEAD') + return paths, {'mode': 'paths'} + merge_base = git('merge-base', base, 'HEAD').strip() + paths = git('diff', '--name-only', '--no-renames', merge_base, 'HEAD').split() + info = {'mode': 'worktree' if worktree else 'base', 'base': base, 'merge_base': merge_base, + 'head': git('rev-parse', 'HEAD').strip()} + if worktree: + paths += git('diff', '--name-only', '--no-renames', 'HEAD').split() + paths += git('ls-files', '--others', '--exclude-standard').split() + return sorted(set(paths)), info + + +def own_checkout(path): + """Whether `path` is a git checkout of its own: git -C on a plain or empty directory (an + uninitialised submodule, an unpacked dependency) silently answers for the parent repository.""" + r = subprocess.run(['git', '-C', str(ROOT / path), 'rev-parse', '--show-toplevel'], capture_output=True, text=True) + return r.returncode == 0 and os.path.realpath(r.stdout.strip()) == os.path.realpath(ROOT / path) + + +def git_in(path, *args): + r = subprocess.run(['git', '-C', str(ROOT / path), *args], capture_output=True, text=True) + if r.returncode != 0: + raise Fail(f'git -C {path} {" ".join(args)} failed: {r.stderr.strip()}') + return r.stdout + + +def dirty_inputs(paths): + return [p for p in paths if git_in(p, 'status', '--porcelain', '--untracked-files=all').strip()] + + +def identity(ports=()): + """What a result was built from: HEAD, whether the tree differs from it, the submodule commits, the + commits of the get_deps dependencies `ports` use, and which of those inputs cannot be trusted: + local changes, not a checkout of its own, or a dependency off its get_deps pin.""" + subs = {} + for line in git('submodule', 'status').splitlines(): + sha, path = line[1:].split()[:2] + subs[path] = sha + used = [dep for dep, families in sorted(deps_rows().items()) if any(p in families for p in ports)] + # worktree deps are symlinks into the primary checkout, invisible to this checkout's status + missing = [p for p in [*subs, *used] if not own_checkout(p)] + deps = {dep: git_in(dep, 'rev-parse', 'HEAD').strip() for dep in used if dep not in missing} + pins = deps_pins() + info = {'head': git('rev-parse', 'HEAD').strip(), 'dirty': bool(git('status', '--porcelain').strip()), + 'submodules': subs, 'deps': deps, 'missingInputs': missing, + 'depsOffPin': [d for d, sha in deps.items() if sha != pins[d]]} + info['dirtyInputs'] = dirty_inputs(own_inputs(info)) + return info + + +def own_inputs(info): + """The submodules and dependencies `identity` could check, i.e. those that are checkouts of their own.""" + return [p for p in [*info['submodules'], *info['deps']] if p not in info['missingInputs']] + + +def moved_inputs(before, after): + """Inputs whose identity differs between two `identity` records: '.' for the checkout's HEAD or dirty + state, else each submodule or dependency whose commit or checkout changed.""" + moved = {'.'} if (before['head'], before['dirty']) != (after['head'], after['dirty']) else set() + commits = [{**i['submodules'], **i['deps']} for i in (before, after)] + moved |= {p for p in commits[0].keys() | commits[1].keys() if commits[0].get(p) != commits[1].get(p)} + return moved | (set(before['missingInputs']) ^ set(after['missingInputs'])) + + +def select(paths): + records = [classify(p) for p in paths] + full = any(r['effect'] == 'core' for r in records) + boards = rep_selection(sorted(REPRESENTATIVES), 'core: shared by every port') if full else {} + for r in records: + for k, why in r['boards'].items(): + boards.setdefault(k, why) + return { + 'full': full, + 'boards': boards, + 'ghostfat': any(r['ghostfat'] for r in records), + 'tests': any(r['tests'] for r in records), + 'records': records, + } + + +def ci_selection(sel): + """What CI builds: every board of every port when full (ghostfat too), else each reached board.""" + ci = {} + for r in sel['records']: + ci = merge_ci(ci, r['ci']) + if ci == FULL: + return {'full': True, 'ports': {}, 'ghostfat': True} + return {'full': False, 'ports': {p: boards_of(p) if b is None else b for p, b in sorted(ci.items())}, + 'ghostfat': sel['ghostfat']} + + +def path_view(records, effect=None, keys=('path', 'rule', 'effect', 'reason')): + return [{k: r[k] for k in keys} for r in records if effect in (None, r['effect'])] + + +HIL_APP = 'dfu_runtime' # the app hil_test.py's esptool battery copies through TinyUF2 + + +def hil_selection(sel, config): + """Roster boards the selection puts under the HIL battery. A board's own change reaches that board, + an app change none, any other change every roster board of the ports it selects, a core or HIL-control + change the whole roster; what no roster board on this host can run is uncovered, whatever else was + selected.""" + try: + entries = json.loads((ROOT / config).read_text())['boards'] + roster = sorted({b['name'] for b in entries}) # TinyUF2 names + esptool = {b['name'] for b in entries if (b.get('flasher') or {}).get('name') == 'esptool'} + except (OSError, ValueError, KeyError, TypeError, AttributeError) as e: + raise Fail(f'cannot read boards from {config}: {e}') + by_port = {} + for name in roster: + by_port.setdefault(port_of(name), []).append(name) + def kind(parts): # the HIL builds with cmake whatever CI uses: its build files decide, not CI's + if parts[0] == 'ports' and (len(parts) == 2 or parts[1] in BACKENDS): + return build_file_kind(parts[-1]) + return None + records = [(r, parts, kind(parts)) for r in sel['records'] for parts in [r['path'].split('/')]] + full = any(r['effect'] == 'core' and k != 'make' for r, _, k in records) + boards, uncovered = set(), {} + if full: + boards.update(roster) + for p in sorted(set(BACKENDS) - set(by_port)): + uncovered[p] = f'core change: no {config} board for {p}' + for r, parts, k in records: + if r['hil']: + boards.update(roster) + elif k == 'make': + uncovered[r['path']] = f'the HIL builds with cmake, which never reads {r["path"]}' + # the battery builds no port app, and a core file reaches every roster board above + elif k == 'cmake' and len(parts) >= 3 and r['effect'] != 'core' and parts[2] != 'apps': + if parts[2] == 'boards' and len(parts) >= 5: + own = [parts[3]] if parts[3] in roster else [] + else: + own = by_port.get(parts[1], []) + if own: + boards.update(own) + else: + uncovered[r['path']] = f'no {config} board builds {r["path"]}' + elif r['effect'] == 'gap': # firmware no build covers is no HIL board's either + uncovered[r['path']] = r['reason'] + elif r['rule'] in ('app', 'port-app'): + app = parts[1] if r['rule'] == 'app' else parts[3] if len(parts) > 3 else None + ports = {port_of(b) for b, _ in r['boards']} if r['rule'] == 'app' else {parts[1]} + if app != HIL_APP: + uncovered[r['path']] = f'the battery runs the bootloader and {HIL_APP} on esptool boards, never {r["path"]}' + continue + # a port that builds the app but has no esptool roster board: its battery never runs it + for p in sorted(ports): + own = [b for b in esptool if port_of(b) == p] + if own: + boards.update(own) + else: + uncovered[f'{r["path"]} on {p}'] = f'no {config} esptool board of {p} runs {HIL_APP}' + elif r['rule'] == 'board': + (b, _), = r['boards'] + if b in roster: + boards.add(b) + else: + uncovered[b] = f'{r["path"]}: {b} is not in {config}' + else: + for p in sorted({port_of(b) for b, _ in r['boards']}): + if p in by_port: + boards.update(by_port[p]) + else: + uncovered[p] = f'{r["path"]}: no {config} board for {p}' + return {'config': config, 'full': full or any(r['hil'] for r in sel['records']), 'boards': sorted(boards), + 'uncovered': [{'target': t, 'reason': why} for t, why in sorted(uncovered.items())]} + + +def manifest(sel, info, hil_config=None): + ports = {} + for (b, backend) in sel['boards']: + ports.setdefault(port_of(b), []).append(b) + out = { + 'version': 1, + 'input': info, + 'build': { + 'full': sel['full'], + 'needed': bool(sel['boards'] or sel['ghostfat'] or sel['tests']), + 'ports': {p: sorted(set(v)) for p, v in sorted(ports.items())}, + 'builds': [{'board': b, 'backend': backend} for (b, backend) in sel['boards']], + 'ghostfat': sel['ghostfat'], + 'tests': sel['tests'], + 'paths': path_view(sel['records']), + 'ci': ci_selection(sel), + }, + } + if hil_config: + out['hil'] = hil_selection(sel, hil_config) + return out + + +# ------------------------------------------------------------------ building + +def first_error(log): + lines = log.splitlines() + # a compiler, linker or CMake diagnostic names the cause; ninja's FAILED line only the object + for pattern in (r':\d+(:\d+)?: (fatal )?error:|undefined reference|region .* overflowed|multiple definition|' + r'^CMake Error\b', + r'\berror\b|Error \d|FAILED|No rule to make|not found'): + for i, line in enumerate(lines): + if NINJA_STEP.match(line): # e.g. "Building ... error.c.obj" + continue + if re.search(pattern, line, re.I) and 'warning' not in line: + if re.match(r'CMake Error\b.*:$', line): # its message is on the next indented line + nxt = next((l for l in lines[i + 1:] if l.strip()), '') + if nxt[:1].isspace(): + line = f'{line} {nxt.strip()}' + return line.strip()[:400] + tail = [l for l in lines if l.strip()] + return tail[-1].strip()[:400] if tail else '' + + +def run(cmd, log, verbose, cwd=ROOT): + with open(log, 'a') as f: + f.write('$ ' + ' '.join(map(str, cmd)) + '\n') + f.flush() + p = subprocess.Popen([str(c) for c in cmd], cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors='replace') + for line in p.stdout: + f.write(line) + if verbose: + sys.stderr.write(line) + return p.wait() + + +def check_deps(port): + """Remedy for a missing submodule or family dependency, None when all are present.""" + missing = [s for s in ('lib/tinyusb', 'lib/uf2') if not (ROOT / s).is_dir() or not any((ROOT / s).iterdir())] + if missing: + return f'empty submodule {", ".join(missing)}: git submodule update --init {" ".join(missing)}' + for dep, families in deps_rows().items(): + if port in families: + d = ROOT / dep + if not d.is_dir() or not any(d.iterdir()): + return (f'{dep} missing: in a worktree symlink it from the primary checkout (CLAUDE.md), ' + f'else python3 tools/get_deps.py {port}') + return None + + +def check_tools(port, backend, idf_mode): + cc = COMPILER.get(port, 'arm-none-eabi-gcc') + need = {'make': ['make', cc], 'cmake': ['cmake', 'ninja', cc], + 'idf': ['docker'] if idf_mode == 'docker' else ['idf.py']}[backend] + missing = [t for t in need if not shutil.which(t)] + if missing: + return f'{", ".join(missing)} not on PATH' + (' (export ESP-IDF first)' if 'idf.py' in missing else '') + return None + + +def expected_artifacts(port, board, backend, root): + bin_dir = root / '_bin' + if backend == 'make': + exp = [bin_dir / f'tinyuf2-{board}.bin', bin_dir / f'tinyuf2-{board}.hex'] + if port not in NO_SELF_UPDATE: + exp.append(bin_dir / 'apps' / f'update-tinyuf2-{board}.uf2') + return exp + if backend == 'cmake': + exp = [bin_dir / 'tinyuf2.bin', bin_dir / 'tinyuf2.hex'] + cml = (ports_dir(port) / 'CMakeLists.txt').read_text() + exp += [bin_dir / 'apps' / f'{app}.uf2' for app in cmake_apps(port, board)] + exp += [bin_dir / a for a in re.findall(r'\$\{ARTIFACT_PATH\}/(apps/[\w.-]+\.uf2)', cml)] + return exp + return [bin_dir / f for f in ('tinyuf2.bin', 'bootloader.bin', 'partition-table.bin', 'ota_data_initial.bin', + 'flash_args', 'combined.bin', 'apps/update-tinyuf2.uf2', 'apps/blinky.uf2', + f'apps/{HIL_APP}.uf2')] + + +# what an IDF build's flash layout also depends on: the HIL admits its segments only against these +IDF_CONFIG = ('sdkconfig',) + + +def build_make(port, board, root, defines, log, verbose, rec): + base = ['make', '-C', ports_dir(port), f'BOARD={board}', f'BUILD_ROOT={root}', *defines] + # copy-artifact runs after self-update, never beside it under -j + if run(base + [f'-j{os.cpu_count()}', 'all', 'self-update'], log, verbose) or run(base + ['copy-artifact'], log, verbose): + return False, [] + apps = [] + for app in sorted((ports_dir(port) / 'apps').glob('*/Makefile')): + if app.parent.name == 'self_update': + continue + enabled = app_boards(app.parent) + if run(['make', '-C', app.parent, f'BOARD={board}', f'BUILD_ROOT={root}', f'-j{os.cpu_count()}', *defines, + 'all'], log, verbose): + return False, apps + if enabled is None or board in enabled: + apps.append(root / 'ports' / port / 'apps' / app.parent.name) + elif enabled == []: + rec.setdefault('notImplemented', []).append(f'apps/{app.parent.name}') + return True, apps + + +def build_cmake(port, board, root, defines, log, verbose): + cfg = ['cmake', '-B', root, '-G', 'Ninja', '-DCMAKE_BUILD_TYPE=MinSizeRel', f'-DBOARD={board}', + f'-DARTIFACT_PATH={root / "_bin"}', *[f'-D{d}' for d in defines], ports_dir(port)] + return not (run(cfg, log, verbose) or run(['cmake', '--build', root], log, verbose)), [] + + +def build_idf(board, root, defines, log, verbose, idf_mode, image): + idf = ['idf.py', '-C', 'ports/espressif', '-B', str(root), f'-DBOARD={board}', + f'-DARTIFACT_PATH={root / "_bin"}', *[f'-D{d}' for d in defines], 'build'] + if idf_mode == 'local': + return not run(idf, log, verbose), [] + inner = "git config --global --add safe.directory '*' && " + shlex.join(idf) + return not run(idf_docker(root) + [image, 'bash', '-c', inner], log, verbose), [] + + +def idf_docker(root, verb='run'): + """`docker run` (or `create`) with the checkout, its git dir and the build dir mounted at their own paths, image + last.""" + common = Path(git('rev-parse', '--path-format=absolute', '--git-common-dir').strip()) + cmd = ['docker', verb, '--rm', '--user', f'{os.getuid()}:{os.getgid()}', '-e', 'HOME=/tmp', '-w', str(ROOT)] + for m in sorted({ROOT, common, root.parent}): + cmd += ['-v', f'{m}:{m}'] + return cmd + + +def ninja_in(backend, idf_mode, image, root, args): + """Runs ninja on `root` with the ninja that built it: the host's, or the IDF image's (whose build log and + generator a host ninja would rewrite). Callers bind the build's own backend, mode and image.""" + cmd = ['ninja', '-C', str(root), *args] + if backend == 'idf' and idf_mode == 'docker': + cmd = idf_docker(root) + [image, *cmd] # the image's entrypoint sources export.sh, then runs cmd + return subprocess.run(cmd, capture_output=True, text=True, cwd=ROOT) + + +def backend_of(port): + """The backend CI builds a port with, or an unbuilt port's on-request backend (None for none).""" + if port not in BACKENDS and port not in UNBUILT: + raise Fail(f'ports/{port} has no backend in check_build.py: add it to BACKENDS or UNBUILT') + return BACKENDS.get(port) or UNBUILT[port][0] + + +def roster_flasher(config, board): + """The board's roster flasher, and the configure defines that bind the build's flash target to its probe. + ESP-IDF takes the bridge's tty at flash time (ESPPORT), which the runner resolves from the serial.""" + try: + entry = next((b for b in json.loads((ROOT / config).read_text())['boards'] if b['name'] == board), None) + if entry is None: + raise Fail(f'{board} is not on {config}') + name, uid, fargs = (entry['flasher'][k] for k in ('name', 'uid', 'args')) + except (OSError, ValueError, KeyError, TypeError) as e: + raise Fail(f'cannot read {board} from {config}: {e}') + form = FLASHER_ARGS.get(name) + if form is None: + raise Fail(f'{board}: flasher {name!r} has no HIL flash target (jlink or esptool)') + m = re.fullmatch(form, fargs) + if not m or not re.fullmatch(r'[\w-]+', uid): + raise Fail(f'{board}: flasher args {fargs!r} or uid {uid!r} is not the roster form') + flasher = {'name': name, 'uid': uid, 'device': m[1]} + return flasher, [f'JLINK_OPTION=-USB {uid} -ExitOnError 1 -NoGui 1'] if name == 'jlink' else [] + + +def flash_binding(flasher, root, ninja): + """The generated flash target's own command, checked token for token against the roster flasher: the probe, + device and image it names, not CMakeCache, which need not hold JLINK_DEVICE. `ninja(dir, args)` runs ninja in the + build's environment.""" + target, inputs = FLASH[flasher['name']] + r = ninja(root, ['-t', 'commands', target]) + tool = 'JLinkExe' if flasher['name'] == 'jlink' else 'run_serial_tool.cmake' + cmds = [l for l in r.stdout.splitlines() if tool in l] if r.returncode == 0 else [] + ok = len(cmds) == 1 + if ok and flasher['name'] == 'jlink': + cd, _, cmd = cmds[0].partition(' && ') + argv = shlex.split(cmd) + ok = cd == f'cd {root}' and Path(argv[0]).name == 'JLinkExe' and argv[1:] == [ + '-device', flasher['device'], '-USB', flasher['uid'], '-ExitOnError', '1', '-NoGui', '1', '-if', 'swd', + '-JTAGConf', '-1,-1', '-speed', 'auto', '-CommandFile', str(root / inputs[0])] and \ + (root / inputs[0]).is_file() and (root / inputs[0]).read_text() == f'halt\nloadfile {root / inputs[1]}\nr\ngo\nexit' + elif ok: + argv = shlex.split(cmds[0].partition(' && ')[2]) + defs = dict(v.split('=', 1) for k, v in zip(argv[1:-2], argv[2:-2]) if k == '-D' and '=' in v) + tool_argv = defs.get('SERIAL_TOOL', '').split(';') + ok = len(argv) == 1 + 2 * len(defs) + 2 and argv[-2] == '-P' and Path(argv[-1]).name == 'run_serial_tool.cmake' and \ + defs.keys() == {'IDF_PATH', 'SERIAL_TOOL', 'SERIAL_TOOL_ARGS', 'SERIAL_TOOL_IS_WRITE_FLASH', + 'SERIAL_TOOL_FLASH_ARGS_FILE', 'WORKING_DIRECTORY'} and \ + tool_argv[1:] == ['-m', 'esptool', '--chip', flasher['device']] and \ + defs['SERIAL_TOOL_ARGS'].split(';') == ['--before=default-reset', '--after=hard-reset', 'write-flash', '@flash_args'] and \ + defs['SERIAL_TOOL_IS_WRITE_FLASH'] == '1' and defs['SERIAL_TOOL_FLASH_ARGS_FILE'] == 'flash_args' and \ + defs['WORKING_DIRECTORY'] == str(root) + # flash_args: one settings line, then " " per segment + segs = (root / 'flash_args').read_text().splitlines()[1:] if (root / 'flash_args').is_file() else [] + ok = ok and sorted(l.split()[1] for l in segs if l.strip()) == sorted(inputs[1:]) + if not ok: + raise Fail(f'{root.name}: target {target} does not flash {flasher} with the build\'s image: ' + f'{cmds if cmds else r.stderr.strip()[-200:]}') + return {'target': target, 'flasher': flasher, 'command': cmds[0], 'inputs': list(inputs)} + + +def dry_run(ninja, root, target=None): + """The commands `ninja -n -v` would run; whatever precedes ninja's own output (the IDF image's entrypoint + banner) is skipped. None when ninja failed or its output is not ninja's.""" + r = ninja(root, ['-n', '-v'] + ([target] if target else [])) + lines = r.stdout.splitlines() + start = next((i for i, l in enumerate(lines) if l.startswith('ninja: Entering directory')), None) + if r.returncode or start is None: + return None + steps = [l for l in lines[start + 1:] if l.strip() and l != 'ninja: no work to do.'] + return None if any(not NINJA_STEP.match(l) for l in steps) else [l.split('] ', 1)[1] for l in steps] + + +def idf_step_writes_no_input(cmd, root, chip): + """True for a command ESP-IDF's flash target always runs before its flasher, segment by `&&` segment in one + of the forms TinyUF2's port and IDF generate: the bootloader subproject's build (its own tree is dry-run too), + stamps, read-only size checks, the port's post-build (combined.bin, copies into _bin, the update app's C + array). None of them writes a flash input; anything else, a shell operator included, is refused.""" + cwd, py, cm = None, r'\S*/python[\d.]*', r'\S*/cmake' + # a path resolved, against a boundary resolved only up to the build root, so no symlink inside the tree moves + # either; a write never lands on a flash input + real = Path(os.path.realpath(root)) + inputs = {Path(os.path.realpath(root / i)) for i in FLASH['esptool'][1]} + under = lambda p, d='': Path(os.path.realpath(p)).is_relative_to(real / d) # noqa: E731 + writes = lambda p, d='': under(p, d) and Path(os.path.realpath(p)) not in inputs # noqa: E731 + postbuild = root / 'tinyuf2.postbuild.sh' + for seg in cmd.split(' && '): + if re.search(r'[;|<>&$`(){}\\]', seg): + return False + argv = shlex.split(seg) + head, tail = ' '.join(argv[:3]), argv[3:] + ok = argv[:1] == ['cd'] and len(argv) == 2 and under(argv[1]) or \ + re.fullmatch(cm, argv[0]) and ( + argv[1:] == ['--build', '.'] and cwd == root / 'bootloader' or + argv[1:] == ['-E', 'echo_append'] or argv[1:3] == ['-E', 'echo'] and len(argv) == 4 or + argv[1:3] in (['-E', 'make_directory'], ['-E', 'touch']) and len(argv) == 4 and writes(argv[3]) or + argv[1:3] == ['-E', 'copy'] and len(argv) == 5 and writes(argv[4], '_bin')) or \ + re.fullmatch(py + r' \S*/partition_table/check_sizes\.py --offset', head) and ( + tail[1:] in ( + ['partition', '--type', 'app', str(root / 'partition_table/partition-table.bin'), str(root / 'tinyuf2.bin')], + ['partition', '--type', 'app', '--subtype', 'factory', str(root / 'partition_table/partition-table.bin'), + str(root / 'tinyuf2.bin')]) or + len(tail) == 4 and tail[1] == 'bootloader' and re.fullmatch(r'0x[0-9a-f]+', tail[2]) and + tail[3] == str(root / 'bootloader/bootloader.bin')) or \ + re.fullmatch(py + r' \S*/lib/uf2/utils/uf2conv\.py --carray', head) and \ + tail == ['-o', str(root / 'apps/update_tinyuf2/bootloader_bin.c'), str(root / 'tinyuf2.bin')] and writes(tail[1]) or \ + argv == ['bash', str(postbuild)] and cwd == root and writes(root / 'combined.bin') and postbuild.is_file() and \ + postbuild.read_text() == ( + '#!/bin/bash\necho "Creating combined.bin"\n' + f"esptool --chip {chip} merge-bin --output combined.bin $(tr '\\n' ' ' < {root / 'flash_args'})\n") + if not ok: + return False + if argv[0] == 'cd': + cwd = Path(argv[1]) + return True + + +def flash_ready(root, flash, ninja): + """Before a HIL flash, under the build lock and with its inputs hash-checked: a dry run shows the target would run + the recorded command once and nothing else that could write a flash input: no rebuild, relink or CMake re-run + (ESP-IDF re-runs CMake after any commit, through its git describe dependency). `ninja(dir, args)` runs ninja in + the build's environment.""" + steps = dry_run(ninja, root, flash['target']) + extra = None + if steps is not None: + others = [c for c in steps if c != flash['command']] + if flash['flasher']['name'] == 'esptool': + nested = dry_run(ninja, root / 'bootloader') # always: the outer build step runs whatever it needs + others += ['bootloader: ninja -n failed'] if nested is None else nested + others = [c for c in others if not idf_step_writes_no_input(c, root, flash['flasher']['device'])] + extra = others if steps.count(flash['command']) == 1 else others + ['no flash command'] + if extra is None or extra: + raise Fail(f'{root.name}: target {flash["target"]} would not run only its flasher: ' + f'{"ninja -n failed" if extra is None else extra[:3]}; rebuild the record') + + +@contextlib.contextmanager +def build_lock(root, wait=True): + """Excludes a rebuild of a shared build dir while another build or a HIL flash uses it; without `wait`, a held + lock raises BlockingIOError.""" + root.parent.mkdir(parents=True, exist_ok=True) + with open(root.with_name(root.name + '.lock'), 'a') as f: + fcntl.flock(f, fcntl.LOCK_EX | (0 if wait else fcntl.LOCK_NB)) + yield + + +def build_board(board, backend, args, idf, flasher=None): + port = port_of(board) + # a board built with another backend than its port's (codeql's make) gets its own root + name = board if backend == backend_of(port) else f'{board}-{backend}' + root = BUILD_DIR / (f'cmake-build-{name}' if args.shared else f'cmake-build-agent-{os.getpid()}-{name}') + rec = {'board': board, 'port': port, 'backend': backend, 'buildDir': str(root.relative_to(ROOT)), + 'status': 'ok', 'firstError': '', 'artifacts': []} + if port in UNBUILT: + rec['note'] = f'ports/{port}: {UNBUILT[port][1]}' + if backend == 'make' and port in NO_SELF_UPDATE or backend == 'cmake' and \ + 'update-tinyuf2.uf2' not in {p.name for p in expected_artifacts(port, board, backend, root)}: + rec['notImplemented'] = ['self-update'] + problem = check_tools(port, backend, args.idf) or check_deps(port) + if backend == 'idf' and args.idf == 'docker' and not problem: + if subprocess.run(['docker', 'image', 'inspect', idf['image']], capture_output=True).returncode: + problem = f'docker image {idf["image"]} missing: docker pull {idf["image"]}' + if problem: + rec.update(status='error', firstError=problem) + return rec + with build_lock(root) if args.shared else contextlib.nullcontext(): + return build_root(board, port, backend, root, rec, args, idf, flasher) + + +def build_root(board, port, backend, root, rec, args, idf, flasher): + defines = args.define + if flasher: + flasher, extra = flasher + defines = defines + extra + # always from clean: make cannot see a changed define or flag, so a reused root could pass stale firmware + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True) + log = root / 'check_build.log' + try: + if backend == 'make': + ok, apps = build_make(port, board, root, defines, log, args.verbose, rec) + elif backend == 'cmake': + ok, apps = build_cmake(port, board, root, defines, log, args.verbose) + else: + ok, apps = build_idf(board, root, defines, log, args.verbose, args.idf, idf['image']) + except OSError as e: + rec.update(status='error', firstError=f'{type(e).__name__}: {e}', log=str(log.relative_to(ROOT))) + return rec + if not ok: + rec.update(status='failed', firstError=first_error(log.read_text(errors='replace')), log=str(log.relative_to(ROOT))) + return rec + missing = [p for p in expected_artifacts(port, board, backend, root) if not p.is_file()] + if backend == 'idf': + missing += [root / c for c in IDF_CONFIG if not (root / c).is_file()] + missing += [a for a in apps if not list(a.glob('*.elf'))] + if missing: + rec.update(status='failed', log=str(log.relative_to(ROOT)), firstError='missing expected artifact(s): ' + + ', '.join(str(p.relative_to(root)) for p in missing)) + return rec + rec['artifacts'] = sorted(str(p.relative_to(root)) for p in (root / '_bin').rglob('*') if p.is_file()) + rec['artifacts'] += sorted(str(e.relative_to(root)) for a in apps for e in a.glob('*.elf')) + if backend == 'idf': + rec['artifacts'] += list(IDF_CONFIG) + if flasher: + try: + rec['flash'] = flash_binding(flasher, root, functools.partial(ninja_in, backend, args.idf, idf.get('image'))) + except (Fail, OSError) as e: + rec.update(status='failed', firstError=str(e), log=str(log.relative_to(ROOT))) + return rec + rec['flash']['defines'] = extra + rec['artifacts'] += [a for a in rec['flash']['inputs'] if a not in rec['artifacts']] + if args.shared: + rec['sha256'] = {a: hashlib.sha256((root / a).read_bytes()).hexdigest() for a in rec['artifacts']} + else: + shutil.rmtree(root) + rec['removed'] = True + return rec + + +def keep_only_log(root, log): + for p in root.iterdir(): + if p != log: + shutil.rmtree(p) if p.is_dir() else p.unlink() + + +def run_ghostfat(verbose): + """Build and run the native self-test (its CMake `selftest` target) of every ports/test_ghostfat config, + one at a time: the `huge` image is about 1.5 GiB, twice over with its known-good copy.""" + missing = [t for t in ('cmake', 'ninja', 'gcc', 'gzip') if not shutil.which(t)] + if missing: + return [{'config': cfg, 'status': 'error', 'firstError': f'{", ".join(missing)} not on PATH'} + for cfg in boards_of(GHOSTFAT)] + results = [] + for cfg in boards_of(GHOSTFAT): + root = BUILD_DIR / f'cmake-build-agent-{os.getpid()}-ghostfat-{cfg}' + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True) + log = root / 'check_build.log' + rec = {'config': cfg, 'status': 'ok', 'firstError': ''} + try: + if run(['cmake', '-B', root, '-G', 'Ninja', f'-DBOARD={cfg}', ports_dir(GHOSTFAT)], log, verbose) or \ + run(['cmake', '--build', root, '--target', 'selftest'], log, verbose): + out = log.read_text(errors='replace') + rec.update(status='failed', firstError=(re.findall(r'^FAIL.*$', out, re.M) or [first_error(out)])[-1]) + except OSError as e: + rec.update(status='error', firstError=f'{type(e).__name__}: {e}') + finally: + if rec['status'] == 'ok': + shutil.rmtree(root, ignore_errors=True) + else: + keep_only_log(root, log) + rec['log'] = str(log.relative_to(ROOT)) + results.append(rec) + return results + + +def run_tests(): + """The .claude script tests, for a change to how boards build or to the HIL contract.""" + r = subprocess.run([sys.executable, '-m', 'unittest', 'discover', '-s', '.claude/test', '-p', 'test_*.py'], + capture_output=True, text=True, cwd=ROOT) + last = [l for l in r.stderr.splitlines() if l.strip()] + return {'status': 'ok' if r.returncode == 0 else 'failed', 'summary': last[-1] if last else '', + 'firstError': '' if r.returncode == 0 else first_error(r.stderr)} + + +class Parser(argparse.ArgumentParser): + def error(self, message): + print(json.dumps({'pass': False, 'error': f'usage: {message}'})) + sys.exit(2) + + +def main(argv=None): + ap = Parser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument('--scope', nargs='+', metavar='PATH', help='changed paths or directories') + src.add_argument('--base', metavar='REF', help='the branch diff against REF') + src.add_argument('--board', action='append', metavar='B', help='build named boards (repeatable)') + src.add_argument('--diff-file', metavar='FILE', help='changed paths, one per line (--select-only)') + ap.add_argument('--worktree', action='store_true', help='with --base: add uncommitted and untracked files') + ap.add_argument('--select-only', action='store_true', help='print the selection manifest and stop') + ap.add_argument('--hil-config', metavar='ROSTER', help="the host's HIL roster: with --select-only adds `hil`; with --board --shared binds the flash target (a HIL record)") + ap.add_argument('-D', '--define', action='append', default=[], metavar='VAR=VAL', + help="passed to the board's backend: make VAR=VAL, cmake/idf -DVAR=VAL") + ap.add_argument('--shared', action='store_true', help='build into cmake-build/cmake-build- and keep it') + ap.add_argument('--backend', choices=('cmake',), + help="with --board and --shared: build with cmake whatever CI uses (the HIL's receipt)") + ap.add_argument('--idf', choices=('docker', 'local'), default='docker', + help="ESP-IDF: CI's pinned docker image (default) or the exported local IDF") + ap.add_argument('-v', '--verbose', action='store_true', help='stream build output to stderr') + args = ap.parse_args(argv) + if args.worktree and not args.base: + ap.error('--worktree needs --base') + if args.diff_file and not args.select_only: + ap.error('--diff-file is for --select-only') + if args.hil_config and not (args.select_only or (args.board and args.shared)): + ap.error('--hil-config is for --select-only, or a HIL record with --board and --shared') + if args.hil_config and args.board and any(d.split('=', 1)[0].split(':', 1)[0] in SELECTION for d in args.define): + ap.error(f'a HIL record takes {", ".join(SELECTION)} from the roster, not -D') + if args.backend and not (args.board and args.shared): + ap.error('--backend needs --board and --shared') + if args.select_only and args.board: + ap.error('--select-only takes --scope, --base or --diff-file') + + result = {'pass': False} + try: + if args.board: + boards = {} + for b in args.board: + port = port_of(b) + backend = args.backend or backend_of(port) + if not backend: + raise Fail(f'ports/{port}: {UNBUILT[port][1]}') + if backend == 'cmake' and (backend_of(port) == 'idf' or not (ports_dir(port) / 'CMakeLists.txt').is_file()): + raise Fail(f'ports/{port} has no cmake build') + boards[(b, backend)] = 'named' + sel = {'full': False, 'boards': boards, 'ghostfat': False, 'tests': False, 'records': []} + info = {'mode': 'boards'} + else: + paths, info = changed_paths(args.base, args.worktree, args.scope, args.diff_file) + sel = select(paths) + if args.select_only: + print(json.dumps(manifest(sel, info, args.hil_config))) + return 0 + ports = {port_of(b) for b, _ in sel['boards']} + info.update(identity(ports)) + result['input'] = info + idf = {} + if any(backend == 'idf' for _, backend in sel['boards']): + ci = ci_idf_version() + idf = {'mode': args.idf, 'ciVersion': ci, 'image': f'espressif/idf:{ci}'} + if args.idf == 'local' and shutil.which('idf.py'): + v = subprocess.run(['idf.py', '--version'], capture_output=True, text=True) + idf['version'] = (v.stdout or v.stderr).strip() + idf['matchesCi'] = ci in idf['version'] + flashers = {b: roster_flasher(args.hil_config, b) for b, _ in sel['boards']} if args.hil_config else {} + records = [build_board(b, backend, args, idf, flashers.get(b)) for (b, backend) in sel['boards']] + for r in records: + r['selectedBy'] = sel['boards'][(r['board'], r['backend'])] + # an input that moved while building, even if restored before the HIL checks it, leaves firmware + # the record cannot vouch for; hil_test.py refuses any entry here + now = identity(ports) + info['inputsDirtiedByBuild'] = sorted((set(now['dirtyInputs']) - set(info['dirtyInputs'])) | moved_inputs(info, now)) + ghostfat = run_ghostfat(args.verbose) if sel['ghostfat'] else [] + tests = run_tests() if sel['tests'] else None + nothing = path_view(sel['records'], 'none', ('path', 'reason')) + uncovered = path_view(sel['records'], 'gap', ('path', 'reason')) + checks = records + ghostfat + ([tests] if tests else []) + statuses = [c['status'] for c in checks] + code = 2 if 'error' in statuses else 1 if 'failed' in statuses else 3 if uncovered else 0 + result.update({ + 'pass': code == 0, + 'boards': records, + 'ghostfat': ghostfat, + 'built': [f"{r['buildDir']}/{a}" for r in records if not r.get('removed') for a in r['artifacts']], + 'resolution': 'full' if sel['full'] else 'boards' if args.board else 'selected', + 'defines': args.define, + 'paths': path_view(sel['records']), + 'nothingToBuild': nothing, + 'uncovered': uncovered, + }) + if tests: + result['tests'] = tests + if idf: + result['idf'] = idf + first = next((c['firstError'] for c in checks if c['status'] != 'ok'), '') + if first: + result['firstError'] = first + print(json.dumps(result)) + return code + except (Fail, OSError) as e: + result['error'] = str(e) + print(json.dumps(result)) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md new file mode 100644 index 000000000..8f7d9b651 --- /dev/null +++ b/.claude/skills/hil/SKILL.md @@ -0,0 +1,197 @@ +--- +name: hil +description: The TinyUF2 HIL contract. Use when a change needs hardware evidence on a rig board, when a debug or validation session must take, flash, clean up or release a TinyUF2 rig board, or when a workflow asks for "the project's HIL contract". Runs the bootloader battery on the roster's boards, owns the board lock, and reports pass/fail per board with target-side evidence. +--- + +# HIL contract + +`scripts/hil_test.py` holds the mechanics: roster and source checks, firmware binding, locking, the probe and USB steps, cleanup and the JSON report. Agents call it instead of composing J-Link, mtools or lock commands. This page is the judgment. + +## Roster + +- The roster is `test/hil/local.json`, the boards on htpc's bench. On a machine without a board, its run fails at `install`, where J-Link finds no probe with that serial, before anything is written: `cleanup.state` is `untouched` and nothing stays held. ci.lan has no TinyUF2 rig yet (a follow-up). +- `TINYUSB_DIR` must name the tinyusb checkout whose `test/hil/helper/hil_lock.py` owns the board locks; every command refuses without it. Set it in the login profile, which headless sessions (chief, `hil-operator`) inherit. + +A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. A J-Link entry has `args` exactly `-device `; the runner uses SWD at 4000 kHz, so pass the same to those scripts, `--interface swd --speed 4000`. An espressif entry has `flasher.name` `esptool`, `flasher.uid` the USB-UART bridge's serial (`/dev/serial/by-id/*_-if*`), `flasher.args` exactly `--chip `, and the top-level `uid` the chip's factory MAC, which is TinyUF2's USB serial. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: +- the board lock name (`lock`) when tinyusb names the board differently, as `mimxrt1060_evk` for `imxrt1060_evk`; both runners take that one lock, and the helper's `status`, holder records and the report rows' `lock` show it. It defaults to the board name; +- the commissioned re-entry path (`reentry`); +- for a board whose TinyUF2 signals no disconnect when it comes up after a reset, the per-port-power hub port its USB is on (`usbPower`: `{"hub": "", "port": N}`; the serial, not the bus location, so a replugged hub is still found). J-Link boards only. While `dfu` waits, the runner power-cycles that port once it has sat connected but unaddressed (sysfs state `default`) for 2 s of that wait, at most once per 10 s, and at no other time: it writes the port's sysfs `disable` 1, then 0 a second later, which the kernel runs under the hub's lock, after any enumeration in progress, disconnecting the child before VBUS goes off. Each write and read-back runs in a bounded child; a cycle that fails or is interrupted after the off write still writes 0, and says in `hil.log` whether that restored power. The report lists the cycles (`portCycles`). + +The battery runs a board only after commissioning sets `reentry` and confirms `uid`; `install` and `restore` work without `reentry`, for commissioning. Only commissioned roster boards can be offered for a run. Adding a board is commissioning, the lead's (below), never a review-time or dispatch-time action: the runner refuses a build record from a dirty tree, an uncommitted roster entry included. + +## Locks + +Locks are tinyusb's: one kernel flock per board in `/tmp/tinyusb-hil-locks/`, taken through `$TINYUSB_DIR/test/hil/helper/hil_lock.py`. A board's lock is ``: its roster `tinyuf2.lock`, else the board name. It is tinyusb's name for the board, so TinyUF2 and tinyusb runs exclude each other; the helper's `hold` and `release` take it and `status` (no board argument) shows it, while `hil_test.py -b` and the probe scripts take the board name. +- **The battery locks itself,** as `hil_test.py`, which tinyusb's `release` refuses to kill. Never pre-hold a board you are about to run the battery on. +- **The lock fails closed.** An unusable lock dir refuses the run (exit 2); it never runs unlocked. `HIL_NO_BOARD_LOCK=1` bypasses the lock only when the task scope names forcing that board. The report then says `lockBypassed: true`. +- **Manual sessions** (hw-debugger, hw-validator): + 1. Take your own hold: `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py hold --reason ""`. + 2. Install a build record's TinyUF2, and restore the board when done: + + ```bash + python3 .claude/skills/hil/scripts/hil_test.py install -b --build-record --under-hold "" test/hil/local.json + python3 .claude/skills/hil/scripts/hil_test.py restore -b --build-record --under-hold "" test/hil/local.json + ``` + + `install` is the battery's install step from the record (Run the battery: a J-Link board runs the record's `tinyuf2-jlink` target, an esptool board takes the four IDF segments), then waits for DFU; it takes the record the battery takes, admitted the same way, so it needs the automount exclusion rule (Commissioning) and a clean build of HEAD. `--try-reentry `, while the roster's `reentry` is unset, gives DFU's fallback (Run the battery, dfu) for commissioning; `restore` takes it too. `restore` is the cleanup (below), from a clean build record of HEAD. Both refuse unless the hold is provably yours: the record's reason, a live pid, a FLOCK on the lock file in that pid's own fds, and a failing probe of the flock. + 3. Release with `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py release `, only after a `restore` that exited 0 with no `survivors`. Otherwise the board is not cleaned up: keep the hold and report it as a blocker. + + Manual exits: 0 the operation succeeded (for `install`, DFU came up; for `restore`, DFU left after the blinky copy and stayed away); 1 it failed before writing; 2 refused before any hardware access; 4 a child survived SIGKILL, or an `install` or `restore` failed after it began writing: keep the hold. + + On a refused hold, report the holder from `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` (no board argument; it prints every holder); never kill it. A `hil_test.py` holder whose record has `role: cleanup-holder` never frees on its own: report it as a blocker that needs the recovery dispatch (Exit 4), without waiting. Any other `hil_test.py` holder is a battery run; wait for it within the lock-wait budget. +- **Cleanup state** for a hw-validator or hw-debugger dispatch: `restore --under-hold` leaves the board on the record's TinyUF2 with `apps/blinky` copied through it, and its `cleanup.evidence` (DFU left after the copy and stayed away) is the observation that Cleanup records. Blinky running is not checked. Whoever uses the board next flashes it first; tinyusb's HIL does. +- **Lock-wait budget:** htpc has no CI runner, so it is the dev-bench row of chief's budget, 2 min, unless the dispatch names another. + +## Run the battery + +```bash +mkdir -p cmake-build/hil +python3 .claude/skills/build/scripts/check_build.py --board --backend cmake --shared --hil-config test/hil/local.json > cmake-build/hil/.build.json +python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json test/hil/local.json +``` + +`` is the roster name, the TinyUF2 board name. An espressif board builds without `--backend cmake`: ESP-IDF in CI's docker image, as CI does. + +- **Admission.** Before taking the lock the run checks the automount exclusion rule, and every span it could write (the HEX the flash target loads, inside the install span and the device's flash limits and laid out as the BIN; the BIN's board id), with the install and app spans taken from the candidate's source. +- **Build record.** It is this contract's build receipt: build it on the clean, pinned HEAD with `--hil-config` and pass it with `--build-record`. `--hil-config` binds the build's own flash target to the roster's probe or bridge (build contract); the run refuses a record whose binding names another flasher, and checks the files that target reads against the record. The build dir's lock covers that target only: the files the runner reads itself (the BIN it verifies, the segments it reads back, `apps/blinky.uf2` and `apps/dfu_runtime.uf2`) are copies in the run dir, taken at admission from bytes that hash as recorded, so a rebuild after the target cannot change them. For a J-Link board it must be a cmake build (`--backend cmake`), whatever backend CI builds the port with; board facts are read from the port's CMake files too. For an espressif board it must be an IDF build from CI's `espressif/idf` image; the run binds its generated `sdkconfig`, `flash_args`, the four segments (bootloader, partition table, otadata, `tinyuf2.bin` in the factory partition) and `apps/dfu_runtime.uf2`, and checks every segment against the generated partition table and the candidate's partition CSV. The run refuses a build record that is not clean, does not match the checkout now (HEAD, submodules, `get_deps` dependencies at their pins), or whose artifacts changed since. Commit first; ESP-IDF's records also go stale at the next commit (build contract). +- **Background.** Run it in the background; one board takes about 2 minutes. Use one board per run. +- **Selection.** There are no examples or test names to select: a run is the whole battery on one board, and the board is the only selection. + +The battery's steps on one board, all under one lock. The roster's probe serial selects the board (each probe is wired to one target), so no step identifies the chip: +1. **install:** the record's `tinyuf2-jlink` target runs (`cmake --build --target tinyuf2-jlink`) under its build dir's lock, once the files it reads hash as recorded and its dry run shows nothing but the flasher; JLinkExe can exit 0 with every command failed, so the transcript must show the record's HEX downloaded (programmed, or skipped as already matching), `O.K.` and the script completed, with no failure line. The files are hashed again after it, and a session of the runner's verifies the BIN at its flash address (`verifybin`), then resets. The board is touched from the target's start; only a transcript that ends at "Connecting to J-Link ...FAILED", before any target connect, leaves it untouched. +2. **dfu:** the UF2 drive with this board's serial is found. On the RT boards the install has left the app start erased on the bench (Limits), so DFU comes up; if it never enumerates, as when a valid app boots (an install on stm32f303disco or stm32f411ve_discovery keeps the app, Limits), the commissioned `retained-register` re-entry brings DFU up (its claim says so); any other DFU failure fails the step. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. +3. **copy:** a generated app is copied to the drive. +4. **app-ran:** the probe shows the app counting in thread mode with VTOR at the app start. +5. **reenter:** the bootloader is re-entered by the commissioned path, and the drive's `CURRENT.UF2` reads the app back. +6. **cleanup** (reported as `cleanup`, not a step, state `blinky copied`, or `untouched` with nothing to do when no session reached the board): the record's TinyUF2 is installed again as in install, DFU is found, the record's `apps/blinky.uf2` (admitted with the record: one family, every block in the app region) is copied to the drive, and the DFU device must leave and stay away for a quiet period. Any failure keeps the board held (Exit 4). + +A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; on a J-Link board a run triggered by the upload alone is unverified (an esptool board's app-ran observes the copied app itself over USB). "reenter" by `retained-register` is the forced-entry path, not a physical double tap. + +On an esptool board, each esptool connection is a child of the runner (`esp_session.py`), and so is each USB control request (`usb_dfu.py`, pyusb): +1. **install:** a first connection takes the MD5s of every partition the battery never writes (nvs, ota_1, ffat on the 8 MB layout) as the run's snapshot, kept in the row's `snapshot` however the run ends; that session is the first to touch the board, which only a missing bridge (no tty with its serial), a container left by an earlier flash, or a failed or interrupted `docker create` of the flash container leaves untouched. Then ESP-IDF's `flash` target runs in CI's image (`cmake --build --target flash`), with the same lock, hash and dry-run checks as a J-Link install: the bridge's tty is the only device passed in, as `ESPPORT`, with `IDF_FLASH_FULL=1` so ESP-IDF v6 writes segments already in flash too, and the container is named `tinyuf2-hil-` so a survivor is found and ended by name. Its transcript must show every segment written at its offset and hash-verified, then the hard reset. A runner session then reads each segment back and takes those MD5s again, then resets once; the runner compares the read-back bytes with the build's segment files and the MD5s with the snapshot, and any difference fails the step. esptool rewrites the bootloader header from `flash_args`, which the build already holds (dio, 80 MHz, 8 MB): on the bench (2026-10-13, esptool 5.3.1 in espressif/idf:v6.1) all four segments read back identical to the files. The roster's bridge selects the board (it is wired to that board only), so the chip is not identified, and its security eFuses and flash size are commissioning's to check. The evidence lists each write's span and the 4 KiB-aligned span covering it. Blank otadata boots the factory partition, TinyUF2. +2. **dfu:** as above; `INFO_UF2.TXT` reports the app partition (ota_0) as its flash, and no register is written. +3. **copy:** `apps/dfu_runtime` from the record, plus a block carrying a fresh nonce in ota_0's last 256 bytes, where the app reads it, renumbered under one `numBlocks`; the image itself is never patched. The record is refused when the image reaches that block or TinyUF2's 64 KiB flash cache window over it (`FLASH_CACHE_SIZE`) leaves ota_0. The evidence lists the image and nonce spans and the cache windows covering them. +4. **app-ran:** the app enumerates with its own VID:PID and the board's serial on the DFU's USB path, its DFU runtime interface string read from the device is `HIL `, and TinyUF2 stays away for a quiet period. +5. **reenter** (`dfu-detach`): DFU_DETACH to the app is acknowledged, the app leaves, TinyUF2 comes back on the same path, and `CURRENT.UF2` reads back the image and the nonce block. +6. **recopy:** a copy with a fresh nonce boots the app again. +7. **cleanup:** as on a J-Link board, with the install as in step 1: the preserved regions' MD5s after the writes, and again after the blinky copy, must equal the run's snapshot (a recovery takes it from the failed report's `snapshot`; a manual `restore` takes its own first). A mismatch fails the cleanup and keeps the hold. + +`--fail-at ` exists only for acceptance: it fails that step to exercise cleanup. + +## Reporting + +The last stdout line is JSON, also written to its `reportPath`; read that path, never an older report. Its fields: +- `pass`, `caveat`; +- per board: `ran`, `pass`, `locked`, `lockBypassed`, `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`), `jlinkRetried` (J-Link boards only: sessions run again after a crash before the target connect; Limits), and `detail`; +- `error` on a refusal or an operation failure. + +| Exit | Meaning | +|---|---| +| 0 | every board passed and was cleaned up | +| 1 | a board failed or did not run; it was cleaned up or never touched | +| 2 | refused before any hardware access (usage, roster, source, build record, lock helper, lock dir) | +| 4 | cleanup failed; the board is held when `cleanup.heldAfter` (not under `HIL_NO_BOARD_LOCK`). From `install` or `restore`: a child survived, or the operation failed after it began writing; keep your hold | + +A delegated run (`hil-operator`) returns `{pass, results, caveat, wedged}` and, beside them, `reportPaths`: +- `pass`, `results` and `caveat` are copied verbatim from the JSON; +- `wedged` lists the boards the operator saw stop responding; +- `reportPaths` maps each board to its JSON's `reportPath`, which a recovery dispatch needs. +- With several boards, the operator runs one battery per board, one after another: `results` holds every board's row, `pass` is true only when every run's is, and `caveat` joins the runs' non-empty caveats. +- A launched run that ended without printing its JSON (killed, crashed) may have programmed the board and left it running a candidate. The operator writes `pass: false`, `caveat: "run ended without a report; board state unknown"` and one row per board, exactly `{board, ran: null, pass: false, locked: false, detail}`, with `detail` naming the exit status, the output tail and `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` at that moment. +- The operator runs the battery once and returns. It never waits on a lock and never retries: a `locked` row comes back as is, and the caller rules below are the caller's (chief's). +- A recovery dispatch names the board, the holder pid and the failed run's `reportPath`. It runs `restore --recover-holder` in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Exit 4). + +### Review mapping + +A review (`pr-review`) maps each board's row, except a commissioning run's (`commissioning: true`, from `--try-reentry`), which is never validation. A blocker takes precedence over every other mapping: +- `pass`: `ran`, `pass` and `cleanup.pass` all true; +- `fail`: `ran` and not `pass`, with a failing step, `cleanup.state` other than `untouched`, and that step's `detail`; +- `not-run`: `ran` false (`locked` or a refusal), or a failing step with `cleanup.state` `untouched`, with its `detail`. Nothing was written, so the failure is the rig's: no probe with that serial on this machine, one unplugged or dead, or no esptool bridge with that serial; +- a blocker, never a `fail` row: exit 4, `ran: null` (no report), and an abort, which is `ran` and not `pass` with no failing step (an interruption or an unexpected error, named in the board's `detail`). + +`testedHead` is the build record's `input.head`. `regression` is `unknown` unless the battery also ran on the merge base. + +### Caller rules +- **Retry.** A failed test may be retried once, and only when no board is `locked`, `wedged` or `heldAfter`, `caveat` is empty, and every board was cleaned up. + - Keep both reports. A fail-then-pass is a pass with `flaky: true`, quoting both. + - An abort (a failed run with no failing step), a cleanup failure, `heldAfter`, `wedged` or a missing report is never retried; it is a blocker until diagnosed. + - A board whose run ended without a report gets a manual `restore --under-hold` before its next run, and only once no such process is left on the host. Killing the runner frees its lock but not its children, which `restore` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: + + `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'; pgrep -af '[e]sp_session\.py'; pgrep -af '[u]sb_dfu\.py'` + + The `hil_test.py` check also covers the runner's `fat-read` subcommand. While a process is left, or this cannot be established, the blocker stays. +- **Locked.** `locked` is an admission refusal, not a test failure. The caller admits again after the holder releases, within the lock-wait budget; the operator does not wait. Otherwise report the board not covered, naming the holder. +- **Exit 4.** + - The board stays held by a detached holder forked from the run: reason `hil_test.py`, `role: cleanup-holder`, with its pid in `cleanup.holder`. It ignores SIGTERM and SIGINT, so a timeout kill cannot free the board; only recovery releases it. + - If no holder could be forked, `cleanup.holder.inProcess` is set and the runner itself is the holder: it prints the report, closes its stdio and does not exit until recovered, so no completion notice arrives. When a run's notice is overdue, read the task's output: a final JSON line with `cleanup.holder.inProcess` true is this case. Do not wait for its exit. + - `cleanup.survivors` lists children that outlived SIGKILL (a J-Link, mtools or FAT read stuck on the device), and a flash container that outlived `docker rm -f` (`{"container": "tinyuf2-hil-"}`); recovery refuses while any of them is still alive. A runner killed outright leaves its container to run out; the next install refuses while `docker ps -a` lists it. Each child runs in a session of its own and the kill reaches the whole session, so a JLinkExe that ninja runs in a process group of its own under the flash target is killed and reported too. + - tinyusb sees that holder as a battery run and waits, which is safe but means the board is out of service until recovered. + - Do not release it by hand, and do not treat it as safe because tinyusb reflashes. The holder may stand for a stuck J-Link or mtools process. + - Recovery is a separate `hil-operator` dispatch whose scope names forcing that board. After process, probe and host readiness are established, it runs: + + `python3 .claude/skills/hil/scripts/hil_test.py restore -b --build-record --recover-holder --report test/hil/local.json` + + `` is the failed report's `firmware.buildRecord`, or a record rebuilt on the clean HEAD when it no longer verifies. The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, restores the board while the lock stays held, and only then releases the holder. On an esptool board the restore checks the preserved regions against the snapshot in the failed report, and recovery is refused for a touched run without one or one whose regions differ from the checkout's. A candidate whose drive cannot take the blinky copy fails the restore; the board stays held until a record that can restore it is given. A second recovery of the same board while one runs is refused (exit 2). + - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide); on espressif_esp32s3_devkitc_1, ROM download mode: hold Boot (SW1, GPIO0 low), press and release RST (SW2, CHIP_PU), then release Boot (DevKitC-1 schematic V1.0; the TRM's strap table also needs GPIO46 low, its default pull-down); on espressif_p4_function_ev, the same with BOOT (SW2, GPIO35 low) and RST (SW1, CHIP_PU) (Function-EV-Board schematic rev 1.4). + +## Selection + +`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config test/hil/local.json` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered, except `dfu_runtime`, which selects the esptool boards of the ports that build it, and leaves uncovered each port that builds it without one (mimxrt10xx today): the battery runs the bootloader with its generated fixture on J-Link boards and with `dfu_runtime` on esptool boards. The runner, the build scripts it imports and the roster (`.claude/skills/hil/scripts/`, `.claude/skills/build/scripts/`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. + +## Commissioning a board + +Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under a manual hold, except for the re-entry run, which locks itself. Record each result in the board's commissioning note (agentrc `target-debug` project notes), and `uid`, `lock` and `reentry` in the roster. In this order: +- **Cabling:** cable the USB port TinyUF2 enumerates on (`BOARD_TUD_RHPORT`). It can differ from the one tinyusb examples use: USB OTG1 on imxrt1060_evk; USB USER (CN3) on stm32f303disco; the USB OTG FS Micro-AB connector, not the ST-LINK one, on stm32f411ve_discovery (UM1842). A DFU device "not seen" with no new device in `dmesg` is a cabling problem first. Keep the probe on a hub apart from the target's (Limits). +- **Silicon revision,** before any other result: read it from the chip marking or registers, and check that revision's errata through read-doc. RT1060's DIGPROG does not tell A from B, so the marking decides. +- **Port power** (a board with `usbPower`): the hub's port `disable` files writable by the runner's user, through `/etc/udev/rules.d/52-tinyuf2-hil-uhubctl.rules`, one line per hub: on `ACTION=="add|bind"` of its `DRIVER=="hub"` interface, matched by the hub's VID, PID and serial, `RUN` `chgrp plugdev` and `chmod g+w` on `/sys$devpath/*-port*/disable`. The runner refuses the board while that file is not writable. Check once that writing `disable` 1 switches the port off (`uhubctl -l -p ` reads `off`), then write 0. +- **Automount exclusion:** `/etc/udev/rules.d/99-tinyuf2-hil.rules`, one line per board, with the VID and PID from the board's source, in exactly this form (the runner accepts only this form, with exact `==` matches): + + ``` + SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ATTRS{serial}=="", ENV{UDISKS_IGNORE}="1" + ``` + + Write it with the roster's `uid`, before the serial can be confirmed: `install` and `run` refuse without the rule, and a fresh board runs whatever tinyusb last flashed, not TinyUF2. A serial a running tinyusb example reports (`lsusb -v`) is an early cross-check, not a confirmation. Until the serial is confirmed, also inhibit automount for every board of this VID:PID with a temporary `/etc/udev/rules.d/98-tinyuf2-hil-commissioning.rules`: + + ``` + SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ENV{UDISKS_IGNORE}="1" + ``` + + Then `sudo udevadm control --reload`. +- **Bootloader:** `install` a clean HIL record of HEAD under the hold and observe DFU (add `--try-reentry retained-register` if a valid app could keep DFU away). +- **USB serial:** confirm the enumerated serial equals `uid` (sysfs or `lsusb -v`), and record the USB port paths. If it differs, fix `uid` and the rule together, reload, and `install` again; its reset re-enumerates the board and udev applies the reloaded rules, so no replug should be needed (by mechanism, not yet observed on a uid fix). +- **Cleanup:** `restore` under the hold and observe DFU leave after the blinky copy. Record the reset notes J-Link prints on install (`BENIGN`). +- **Exclusion proof:** delete the temporary rule and reload before the re-entry run below; its `dfu` step passes only when `udevadm info` on the enumerated block device shows `UDISKS_IGNORE=1` (`check_exclusive`). A host rule that ignores a whole VID, such as tinyusb's `88-tinyusb.rules` for 239a, sets the same property, so the proof cannot tell it from the serial rule; the runner still requires the serial line. +- **Loader footprint:** for the flash target's `loadfile`, place sentinels just past the written range and elsewhere in its erase block. Confirm them from a fresh session before the operation under test, and again from another fresh session after it: a verify in the session that wrote them cannot show what a later operation destroys. Make the written data differ from what is on the flash, since a download whose data already matches is skipped ("Contents already match") and erases nothing. Record the result in Limits. +- **Fixture prerequisites,** before anything executes the battery's fixture: + - the app start meets VTOR's alignment for the fixture's vector table (`fixture()`); + - its stack top, the `m_data` origin the runner reads (`ram_origin`) + 0x1000, is in RAM enabled when the bootloader hands off: the ROM default per read-doc, which TinyUF2 does not reconfigure (DTCM on imxrt1060_evk); + - under a hold, a word just below that stack top takes a written value and reads it back, and then its saved value (0x20000FFC on imxrt1060_evk). +- **Re-entry:** commit the roster entry without `reentry`, build the record, release the manual hold, and run the battery in commissioning mode; it locks itself and ends with the cleanup, so it also proves the restore: + + ```bash + python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json --try-reentry retained-register test/hil/local.json + ``` + + `--try-reentry` is the lead's only, and is refused once `reentry` is set. The row's `reentryDecision` names the method when the run passed or failed only at `reenter`'s `CURRENT.UF2` readback, every observation of the tried method is true (`controlBootedApp`, `dfuCameUp` and `magicConsumed` for `retained-register`), and the board was cleaned up and not held. Set `reentry` to it and commit. A null decision leaves `reentry` unset: file a `followup` issue with the observations for diagnosis, not as a diagnosed bug. When the drive copy itself fails, a fixture installed by hand with J-Link `loadbin` can isolate re-entry from the copy for diagnosis; it never sets `reentry`. +- **On an esptool board,** in place of the J-Link items: cable the native USB connector as well as the bridge's; check once by hand that the chip has no secure boot, secure download or flash encryption and that its flash holds the candidate's layout (`esptool get-security-info`, `esptool flash-id`), since the runner never checks them; record the bridge serial and the chip's MAC (`uid`, TinyUF2's USB serial); `install` from the record and observe DFU, then `restore` and observe DFU leave. Re-entry is `--try-reentry dfu-detach`, whose decision needs `detachAcked`, `appLeft` and `dfuCameUp`; before setting it, repeat DFU_DETACH ten times under the hold and check the host log shows no USB reset on the DETACH transitions (USB DFU 1.1 bitWillDetach). +- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid; on MIMXRT1011DAE5A the same note on every reset of a TinyUF2 image; on stm32f303vc, "Active write protection detected.", "Device will be unsecured now.", "Failed to unlock device." and "Can not attach to CPU. Trying connect under reset.", the connect that removes TinyUF2's write protection (Limits). On stm32f411ve the same connect prints "Active write protection detected.", "Device will be unsecured now." and "Device unlocked successfully.", which match no failure pattern and need no exemption; only the flash target connects as that device (Limits). A reset session (`w4` to AIRCR) passes only once J-Link reports the write issued ("Writing VALUE -> ADDR") with no failure before it. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. + +## Limits + +- Covered: imxrt1060_evk, metro_m7_1011, stm32f303disco, stm32f411ve_discovery, espressif_esp32s3_devkitc_1 and espressif_p4_function_ev on htpc. Not covered: ports without source facts in the runner, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69), espressif chips other than the ESP32-S3 and ESP32-P4, self-update, an interrupted update, malformed UF2 input and the physical double tap. +- esptool boards: the runner needs esptool 5.2.0 and pyusb with libusb-1.0 for its own Python. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The bridge's UART carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. At the end of a UF2 copy TinyUF2 restarts while the host's usb-storage is still talking to it: the host log shows a reset of that device and read errors on its block device. Seen on the S3 and the P4; other ports are unchecked. On the P4, while an esptool session holds the chip, the host log shows a full-speed device on TinyUF2's USB path that never answers its descriptor request (htpc 2026-10-14); the battery passed with it. +- Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. +- cmake builds pass `TUF2_LOG=0` unless `LOG` is set, as make does, so the HIL image does not log on the board UART. +- `install` from a blank board, or one whose ROM found no boot image, brings up DFU (measured on imxrt1060_evk 2026-10-12 and metro_m7_1011 2026-10-13). With no boot image a new J-Link session reads the flash wrong (imxrt1060_evk: zeros; metro_m7_1011: bytes shifted) until a `loadbin` has run; a `loadbin` compare reads it right. On metro_m7_1011, J-Link then prints "Invalid flash header detected." at the install's first reset. +- Of a port's CMake flash targets the runner runs only `tinyuf2-jlink` and ESP-IDF's `flash`, bound by the record to the roster's probe or chip (the bridge's tty is passed at flash time). Never use the erase targets on a rig board (`tinyuf2-erase-external-jlink` erases [FLASH_BASE, FLASH_BASE + 64 KiB), the bootloader with it), nor ESP-IDF's `combined-flash`, which has no port selection and writes nvs and both OTA slots blank. +- On metro_m7_1011 (2026-10-13, J-Link V9.84), `tinyuf2-jlink`'s `loadfile` of the cmake HEX (0x60000400 on) reported "1 range affected (65536 bytes)" and, from a fresh session after it, 0x60000000, 0x6000BF00, 0x6000C000-0x6000C00F, 0x6000C200 and 0x6000FF00 read 0xFF while 0x60010000 kept its data. Its compare covered the whole block: over blinky it programmed, and right after a program it skipped ("Contents already match"). DFU came up within 2 s in each of these three runs. That every install by the target leaves [0x6000C000, 0x60010000), the app's first 16 KiB, erased is an inference from these samples and this HEX, not a measurement of every case; the re-entry fallback (Run the battery, dfu) covers an install that leaves a valid app. On imxrt1060_evk (2026-10-13), the same target over blinky also reported "1 range affected (65536 bytes)"; from a fresh session after it, blinky's vectors at 0x6000C000-0x6000C00F and 0x6000C200 read 0xFF, while the FCB at 0x60000000 and 0x60010000 kept their data. +- On stm32f303disco (2026-10-14, J-Link V9.84 on the board's J-Link OB), `tinyuf2-jlink`'s `loadfile` compared [0x08000000, 0x08004000) and reprogrammed only the 2 KiB pages that differed, erasing the rest of such a page past the image (a sentinel at 0x08003F00 read 0xFF from a fresh session after it), while 0x08004000 (the app start) and 0x08004800 kept their data. So an install keeps a valid app. DFU still came up without the fallback in the battery of 2026-10-14: the runner's verify session halted the core in TinyUF2's double-tap wait (`src/main.c`, magic armed), so its reset entered DFU as a double tap, an inference from that trace. When that reset misses the window the app boots, and the retained-register fallback brings DFU up after `dfu`'s 20 s wait. +- On stm32f303vc, TinyUF2 write-protects its pages 0-7 at boot through the option bytes unless RCC_CSR's OBLRSTF is set, and it never clears that flag, so after any option-byte reload it skips protection until a power-on reset. A J-Link connect to a chip with that protection set prints the `BENIGN` unlock note and, read from a fresh `-device Cortex-M4` session after it, leaves the option bytes at RDP 0xAA with no WRP and flash intact (three samples, 2026-10-14). Its double-tap word is the last SRAM word, 0x20009FFC; that it survives SYSRESETREQ is undocumented (RM0316 Rev 10, DS9118, PM0214, ES0204) and inferred from the commissioning run (2026-10-14: the reset without the magic booted the app, the one with it entered DFU and consumed it). A battery therefore runs unprotected after its install; the protection itself is not covered. +- On stm32f411ve_discovery (2026-10-15, J-Link V9.84 on the board's J-Link OB, rev A silicon: DBGMCU REV_ID 0x1000, ES0287 Rev 6), `tinyuf2-jlink`'s `loadfile` of the HEX ([0x08000000, 0x08003E18)) reported "1 range affected (16384 bytes)": from a fresh session after it, a sentinel at 0x08003F00 read 0xFF while sentinels at 0x08004000, 0x08007C00, 0x08008000, 0x0800C000, 0x08010000 (the app start) and 0x08010800 kept their data. So an install keeps a valid app, and DFU comes up through the retained-register fallback. TinyUF2 write-protects sectors 0-1 at every boot (`src/main.c`; on the F4 the option bytes take effect when OPTSTRT completes, with no reset, RM0383 Rev 4 3.6.2). A `-device stm32f411ve` connect to a chip with that protection prints the unlock note (J-Link transcript shape) and removes it by resetting the chip: with RCC_CSR cleared and TinyUF2 booted, the next such connect read RCC_CSR 0x14000000 (SFTRSTF, PINRSTF) and nWRP 0xFF, while a `-device Cortex-M4` connect read RCC_CSR 0 and nWRP 0xFC, the core running the app. A runner session that reset the chip that way after a retained-register re-entry booted the app and dropped DFU, so runner sessions on this device connect as `Cortex-M4` (`SESSION_DEVICE`); the unlock is not in UM08001 Rev 9.78. Right after the flash target's unlock, a `Cortex-M4` session's halt failed ("CPU could not be halted", flash read back wrong) with no delay (three runs) and succeeded after a 200, 400, 600, 800 or 1500 ms `sleep`; TinyUF2's boot rewriting the option bytes is the inferred cause, and no rewrite time is documented. So the install's verify session waits 1 s first (`INSTALL_SETTLE_MS`, empirical). A battery therefore leaves the protection set; the protection itself is not covered. Its double-tap word is 0x2000FFFC, the last word of the boot script's 64 KiB RAM; that it survives SYSRESETREQ is undocumented (RM0383 Rev 4 6.1.1) and inferred from the commissioning run (the reset without the magic booted the app, the one with it entered DFU and consumed it). DFU re-enumerated after each tested reset into DFU without port power control (OTG FS soft disconnect), so it has no `usbPower`. +- stm32f303disco does not signal a disconnect when TinyUF2 comes up after a reset (the F303 has no internal D+ pull-up, DS9118 Rev 14 Table 66 note 2; the MB1035 schematic is not in the library, and on the bench the host kept the old device), so the host does not re-enumerate it unless it was polling the device (TinyUF2 as an MSC disk) when the reset came. For the same reason the `stays_away` checks (the control trial's, the copy's) cannot see a DFU entry there; `observe_app` shows the fixture running when probed, not that no transient DFU entry came before. Its roster `usbPower` names its USB USER port on the Port Authority hub (PA40004, port 3), which the runner cycles while `dfu` waits (Roster). A bench `uhubctl` cycle, a raw hub request, that met an enumeration in progress on hub 1-1 of an AMD 500-series xHCI controller killed that controller (2026-10-14: "HC died", every device on its buses lost, recovered by unbinding and binding `xhci_hcd`; the Port Authority hub needed a replug); hence the kernel's serialized `disable` and the 2 s in state `default`. On this hub `disable` 1 reads `off` from `uhubctl` (2026-10-14). +- A `JLinkExe` crash inside the target is not retried (the crash retry below covers the runner's own sessions only): it fails the install, touched. +- J-Link's QSPI loader is undocumented (SEGGER UM08001 has nothing on it). Historically, at sampled sentinel points on both RT boards, the runner's former `loadbin` install (no longer issued) lost the rest of its 64 KiB block past the image while a sentinel at 0x60010000, in the next block, survived; a sample, not a guarantee for every image. +- `JLinkExe` sometimes crashes in `libjlinkarm` at the probe connect on htpc (SIGSEGV; once a glibc double-free abort); the cause is unknown. A session that crashed this way, before the target connect, runs once more after 2 s (`jlinkRetried`); a second crash fails the run. Seen in 2 of 94 runner sessions on 2026-10-12. +- A device hung mid-write makes the host controller retry its enumeration, which stalled a J-Link connect on the same hub for ~80 s (lpcxpresso55s69); that the hub is the cause is a hypothesis. +- If no cleanup holder can be forked (exit 4 with `cleanup.holder.inProcess`), the runner never exits until recovered. +- No reproducer for an instrumented build: the battery and `install` take only a clean build record. A claim that needs one is reported pending. diff --git a/.claude/skills/hil/scripts/esp_session.py b/.claude/skills/hil/scripts/esp_session.py new file mode 100644 index 000000000..a4e1ff019 --- /dev/null +++ b/.claude/skills/hil/scripts/esp_session.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""One esptool connection for hil_test.py, run as its own child so the runner can time it out, kill it and +see it survive: connect into download mode, run the ops in order, then reset or not. + + esp_session.py --port DEV --chip CHIP --baud N --after hard-reset|no-reset OPS.json + +OPS.json is a list of {"op": "read", "addr", "size", "file"} and {"op": "md5", "addr", "size"}: no write, which +is the build's flash target's. +The last stdout line is JSON: {"ok": true, "results": [...]} or {"ok": false, "error", "results"}. +""" +import argparse +import json +import sys + +sys.dont_write_bytecode = True + + +def run(esp, op): + kind = op['op'] + if kind == 'read': + data = esp.read_flash(op['addr'], op['size']) + with open(op['file'], 'wb') as f: + f.write(data) + return {'size': len(data)} + if kind == 'md5': + return {'md5': esp.flash_md5sum(op['addr'], op['size'])} + raise ValueError(f'unknown op {kind!r}') + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--port', required=True) + ap.add_argument('--chip', required=True) + ap.add_argument('--baud', type=int, required=True) + ap.add_argument('--after', choices=('hard-reset', 'no-reset'), required=True) + ap.add_argument('ops') + args = ap.parse_args() + results, esp = [], None + try: + import esptool + from esptool import cmds + ops = json.load(open(args.ops)) + esp = cmds.detect_chip(args.port, connect_mode='default-reset') + if esp.CHIP_NAME.lower().replace('-', '') != args.chip: + raise RuntimeError(f'the port reaches a {esp.CHIP_NAME}, not an {args.chip}') + esp = cmds.run_stub(esp) + esp.change_baud(args.baud) + cmds.attach_flash(esp) + for op in ops: + results.append(dict(run(esp, op), op=op['op'])) + if args.after == 'hard-reset': + cmds.reset_chip(esp, 'hard-reset') + out = {'ok': True, 'esptool': esptool.__version__, 'results': results} + except Exception as e: # every failure is reported, never raised past the JSON line + out = {'ok': False, 'error': f'{type(e).__name__}: {e}', 'results': results} + finally: + if esp is not None: + esp._port.close() + print(json.dumps(out), flush=True) + return 0 if out['ok'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/skills/hil/scripts/hil_esp.py b/.claude/skills/hil/scripts/hil_esp.py new file mode 100644 index 000000000..cb97bea2c --- /dev/null +++ b/.claude/skills/hil/scripts/hil_esp.py @@ -0,0 +1,178 @@ +"""Espressif facts and build-record admission for hil_test.py: what an IDF build of a TinyUF2 board +writes where, read from the candidate's source and checked against the generated build files.""" +import csv +import hashlib +import io +import re +import struct + +from pathlib import Path + +from check_build import FLASH, HIL_APP, IDF_CONFIG + +# 2nd-stage bootloader offset per chip, independent of the build: the ROM loads it from here (IDF v6.1 +# esp_rom//esp_rom_caps.h ESP_ROM_BOOTLOADER_OFFSET_FLASH, esptool 5.2.0 targets/.py; no checked TRM states it) +BOOTLOADER_OFFSET = {'esp32s3': 0x0, 'esp32p4': 0x2000} +PARTITION_TABLE = (0x8000, 0x1000) # ports/espressif/sdkconfig.defaults CONFIG_PARTITION_TABLE_OFFSET; one sector +APP, DATA = 0x00, 0x01 +SUBTYPE = {(APP, 'factory'): 0x00, (APP, 'ota_0'): 0x10, (APP, 'ota_1'): 0x11, (DATA, 'ota'): 0x00, + (DATA, 'nvs'): 0x02, (DATA, 'fat'): 0x81} +TYPE = {'app': APP, 'data': DATA} +# the IDF build's own segments, the flash target's inputs after flash_args: build-dir path -> the segment's name +# (its roster region) +SEGMENT_PATHS = {p: Path(p).name for p in FLASH['esptool'][1][1:]} +SEGMENTS = tuple(SEGMENT_PATHS.values()) +NATIVE = {v: k for k, v in SEGMENT_PATHS.items()} +APP_UF2 = f'_bin/apps/{HIL_APP}.uf2' # the battery's app, copied through TinyUF2 +BLINKY_UF2 = '_bin/apps/blinky.uf2' # what cleanup leaves running +RECORD_FILES = (APP_UF2, BLINKY_UF2, *IDF_CONFIG) # besides the flash target's inputs, which admit_flash checks +IMAGE_MAGIC = 0xE9 + + +class Refused(Exception): + pass + + +def size_of(text): + m = re.fullmatch(r'(\d+)([KM]?)', text.strip()) + if not m: + raise Refused(f'{text!r} is not a partition size') + return int(m.group(1)) * {'': 1, 'K': 1024, 'M': 1024 * 1024}[m.group(2)] + + +def parse_csv(text, what): + """[(label, type, subtype, offset, size)] of an IDF partition CSV with explicit offsets only.""" + rows = [] + for row in csv.reader(io.StringIO(text)): + row = [c.strip() for c in row] + if not row or not row[0] or row[0].startswith('#'): + continue + if len(row) < 5 or row[1] not in TYPE or (TYPE[row[1]], row[2]) not in SUBTYPE or not row[3]: + raise Refused(f'{what}: unsupported row {row}') + rows.append((row[0], TYPE[row[1]], SUBTYPE[TYPE[row[1]], row[2]], int(row[3], 0), size_of(row[4]))) + return rows + + +def parse_table(blob, what): + """[(label, type, subtype, offset, size)] of a generated partition-table.bin, its MD5 entry checked.""" + rows, md5 = [], None + for i in range(0, len(blob), 32): + e = blob[i:i + 32] + if e[:2] == b'\xaa\x50': + t, st, off, size = struct.unpack_from('MB=y and a partitions-*.csv') + return {'flashSize': sizes[0] * 1024 * 1024, 'partitionCsv': name, + 'partitions': parse_csv((root / 'ports/espressif' / name).read_text(), name)} + + +def layout(f, chip): + """Where the runner may write, from the partitions: each install segment's region, the app (ota_0, which + only TinyUF2 writes, through copy), and every other partition, preserved.""" + parts = {p[0]: p for p in f['partitions']} + by_kind = {(p[1], p[2]): p for p in f['partitions']} + need = {'otadata': (DATA, 0x00), 'app': (APP, 0x10), 'factory': (APP, 0x00)} + missing = [k for k, v in need.items() if v not in by_kind] + if missing: + raise Refused(f'{f["partitionCsv"]}: no {", ".join(missing)} partition') + span = lambda p: (p[3], p[3] + p[4]) + install = {'bootloader.bin': (BOOTLOADER_OFFSET[chip], PARTITION_TABLE[0]), + 'partition-table.bin': (PARTITION_TABLE[0], sum(PARTITION_TABLE)), + 'ota_data_initial.bin': span(by_kind[need['otadata']]), 'tinyuf2.bin': span(by_kind[need['factory']])} + spans = sorted([(*install['bootloader.bin'], 'bootloader'), (*install['partition-table.bin'], 'partition table')] + + [(*span(p), p[0]) for p in f['partitions']]) + for (s0, e0, a), (s1, e1, b) in zip(spans, spans[1:]): + if e0 > s1: + raise Refused(f'{f["partitionCsv"]}: {a} [{s0:#x},{e0:#x}) overlaps {b} at {s1:#x}') + if spans[-1][1] > f['flashSize']: + raise Refused(f'{f["partitionCsv"]}: {spans[-1][2]} ends at {spans[-1][1]:#x}, past the {f["flashSize"]:#x} flash') + owned = {by_kind[v][0] for v in need.values()} + return {'install': install, 'app': span(by_kind[need['app']]), + 'preserved': {n: span(p) for n, p in parts.items() if n not in owned}} + + +def read_flash_args(text, what): + """(settings, {offset: basename}) of an IDF flash_args file.""" + lines = [l.split() for l in text.splitlines() if l.strip()] + if not lines: + raise Refused(f'{what}: empty') + opts = lines[0] + if len(opts) % 2 or not all(o.startswith('--') for o in opts[::2]): + raise Refused(f'{what}: first line is not --option value pairs: {opts}') + settings = {o[2:]: v for o, v in zip(opts[::2], opts[1::2])} + segs = {} + for l in lines[1:]: + if len(l) != 2 or not re.fullmatch(r'0x[0-9a-fA-F]+', l[0]): + raise Refused(f'{what}: {" ".join(l)!r} is not "OFFSET FILE"') + off = int(l[0], 16) + if off in segs: + raise Refused(f'{what}: two segments at {off:#x}') + if l[1] not in SEGMENT_PATHS: + raise Refused(f'{what}: {l[1]!r} is not one of the IDF build\'s segments {sorted(SEGMENT_PATHS)}') + segs[off] = SEGMENT_PATHS[l[1]] + return settings, segs + + +def admit(root, board, f, lay): + """The segments an install writes, [(offset, path)], from an IDF build dir already hash-checked against + its record: flash_args, the generated sdkconfig and partition table must agree with the candidate's + source, and every segment must sit inside its own region. Paths are the build tree's own files, which + ESP-IDF's flash target writes, not the _bin copies.""" + gen = sdkconfig((root / 'sdkconfig').read_text()) + want = {'CONFIG_IDF_TARGET': f'"{board["device"]}"', 'CONFIG_PARTITION_TABLE_OFFSET': hex(PARTITION_TABLE[0]), + 'CONFIG_PARTITION_TABLE_CUSTOM_FILENAME': f'"{f["partitionCsv"]}"', + 'CONFIG_ESPTOOLPY_FLASHSIZE': f'"{f["flashSize"] // (1024 * 1024)}MB"'} + bad = {k: gen.get(k) for k, v in want.items() if gen.get(k) != v} + if bad: + raise Refused(f'generated sdkconfig differs from the candidate source: {bad}') + settings, segs = read_flash_args((root / 'flash_args').read_text(), 'flash_args') + for k, v in (('flash-mode', gen.get('CONFIG_ESPTOOLPY_FLASHMODE')), ('flash-freq', gen.get('CONFIG_ESPTOOLPY_FLASHFREQ')), + ('flash-size', gen.get('CONFIG_ESPTOOLPY_FLASHSIZE'))): + if f'"{settings.get(k)}"' != v: + raise Refused(f'flash_args --{k} {settings.get(k)} differs from the generated sdkconfig {v}') + if sorted(segs.values()) != sorted(SEGMENTS): + raise Refused(f'flash_args names {sorted(segs.values())}, not exactly {sorted(SEGMENTS)}') + table = parse_table((root / NATIVE['partition-table.bin']).read_bytes(), 'partition-table.bin') + if table != f['partitions']: + raise Refused(f'partition-table.bin {table} differs from {f["partitionCsv"]} {f["partitions"]}') + out = [] + for off, name in sorted(segs.items()): + path = root / NATIVE[name] + lo, hi = lay['install'][name] + size = path.stat().st_size + if off != lo or off + size > hi: + raise Refused(f'{name} [{off:#x},{off + size:#x}) is not at the start of its region [{lo:#x},{hi:#x})') + with open(path, 'rb') as fh: + magic = fh.read(1) + if name in ('bootloader.bin', 'tinyuf2.bin') and magic != bytes([IMAGE_MAGIC]): + raise Refused(f'{name} is not an ESP image (no 0xE9 magic)') + out.append((off, path)) + return out diff --git a/.claude/skills/hil/scripts/hil_test.py b/.claude/skills/hil/scripts/hil_test.py new file mode 100755 index 000000000..9e709093c --- /dev/null +++ b/.claude/skills/hil/scripts/hil_test.py @@ -0,0 +1,2161 @@ +#!/usr/bin/env python3 +"""TinyUF2 HIL battery and the probe operations debug sessions share with it. + + hil_test.py run -b BOARD --build-record FILE CONFIG [--fail-at STEP] [--try-reentry METHOD] + hil_test.py install -b BOARD --build-record FILE --under-hold REASON [--try-reentry METHOD] CONFIG + hil_test.py restore -b BOARD --build-record FILE (--under-hold REASON [--try-reentry METHOD] | --recover-holder PID --report FILE) CONFIG + +The last stdout line is JSON. Exit 0 every board passed and was cleaned up, 1 a board failed or did not +run (cleaned up or never touched), 2 refused before any hardware access, 4 cleanup failed (the board is +held when `cleanup.heldAfter`; not under HIL_NO_BOARD_LOCK), a manual operation left `survivors`, or a +`restore` failed after it began writing. +SKILL.md beside this script is the contract; this file is its mechanics. +""" +import argparse +import contextlib +import fcntl +import functools +import glob +import hashlib +import importlib.util +import json +import os +import re +import select +import shutil +import signal +import struct +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +sys.path.insert(0, str(ROOT / '.claude' / 'skills' / 'build' / 'scripts')) +sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.dont_write_bytecode = True # nothing the runner imports may write into a checkout it binds +import check_build as cb # noqa: E402 +import hil_esp as esp # noqa: E402 + +# [floor, ceiling) the runner ever programs, per J-Link device, independent of roster and source: +# LPC55S69 PFR starts at 0x9DE00 and a bad CMPA write can lock the part (UM11126 Table 173, Table 1064); +# 0x9D800-0x9DDFF is reserved and undocumented. MIMXRT1062: FlexSPI XIP at 0x60000000 (IMXRT1060RM +# Table 3-1) holding the EVK's 8 MiB IS25WP064A (EVK hardware user's guide 2.7). MIMXRT1011: FlexSPI XIP at +# 0x60000000 (IMXRT1010RM Table 3-1) holding the Metro M7 1011's W25Q64JV (Metro M7 RT1011 SD revA schematic), 8 MiB. +# STM32F303VC: 256 KiB main flash at 0x08000000 (RM0316 Rev 10 Table 2); option bytes at 0x1FFFF800 stay outside. +# STM32F411VE: 512 KiB main flash at 0x08000000 (RM0383 Rev 4 Table 4); option bytes at 0x1FFFC000 stay outside. +FLASH_LIMITS = {'LPC55S69_M33_0': (0, 0x9D800), 'MIMXRT1062xxx6A': (0x60000000, 0x60800000), + 'MIMXRT1011DAE5A': (0x60000000, 0x60800000), 'stm32f303vc': (0x08000000, 0x08040000), + 'stm32f411ve': (0x08000000, 0x08080000)} +STEPS = {'jlink': ('install', 'dfu', 'copy', 'app-ran', 'reenter')} +STEPS['esptool'] = STEPS['jlink'] + ('recopy',) +REENTRY = {'jlink': ('retained-register',), 'esptool': ('dfu-detach',)} +REENTRY_METHODS = sorted({m for v in REENTRY.values() for m in v}) +PROTECTED = {'hil_test.py', 'pool_check'} # tinyusb hil_lock.PROTECTED_REASONS, checked at load +HOLDER_ROLE = 'cleanup-holder' +SCB_VTOR, SCB_AIRCR = 0xE000ED08, 0xE000ED0C +AIRCR_SYSRESETREQ = 0x05FA0004 +UF2_MAGIC = (0x0A324655, 0x9E5D5157, 0x0AB16F30) +UF2_FLAG_FAMILY = 0x2000 +PROBE_TIMEOUT, USB_TIMEOUT, QUIET_TIME, KILL_WAIT, UDEV_TIMEOUT = 120, 20, 3, 5, 5 +JLINK_RETRY_DELAY = 2 +ESPTOOL_VERSION = '5.2.0' # the CLI and API the esptool session helper is written and bench-checked against +ESP_BAUD, ESP_TIMEOUT = 921600, 300 +ESP_SESSION = Path(__file__).resolve().parent / 'esp_session.py' +USB_DFU = Path(__file__).resolve().parent / 'usb_dfu.py' +NONCE_MAGIC, NONCE_BLOCK = b'TUF2NONC', 256 # apps/dfu_runtime/src/dfu_runtime.h +JLINK_IF, JLINK_SPEED = 'SWD', '4000' +CMAKE_RECORD_FILES = ('_bin/tinyuf2.bin', '_bin/apps/blinky.uf2') # besides the flash target's inputs +JLINK_ERRORS = (r'Verify failed|Could not|Cannot connect|ERROR|ResetTarget failed|Failed to halt|CPU is not halted|' + r'Failed to unlock|Can not attach') +UDEV_RULES = ('/etc/udev/rules.d', '/run/udev/rules.d') +# J-Link notes that are not failures, per device, each with the line that shows the operation completed +# (bench: every MIMXRT1062 reset with an invalid app vector, then the reset halts as usual; every MIMXRT1011 reset +# of a TinyUF2 image, which then boots; STM32F303VC: the connect that removes TinyUF2's write protection, SKILL.md +# Limits) +_RESET_VECTOR_NOTE = r'(?m)^Could not find valid reset vector\.\nTarget halted on flash header read\.$' +_F3_UNPROTECT_NOTE = (r'(?m)^Active write protection detected\.\nDevice will be unsecured now\.\nFailed to unlock device\.\n' + r'Can not attach to CPU\. Trying connect under reset\.$') +BENIGN = {'MIMXRT1062xxx6A': _RESET_VECTOR_NOTE, 'MIMXRT1011DAE5A': _RESET_VECTOR_NOTE, 'stm32f303vc': _F3_UNPROTECT_NOTE} +# runner sessions connect as the core: J-Link's stm32f411ve connect resets the chip to unprotect it (SKILL.md Limits) +SESSION_DEVICE = {'stm32f411ve': 'Cortex-M4'} +# ms the install's verify session waits before its halt after the flash target's unlock, empirical (SKILL.md Limits) +INSTALL_SETTLE_MS = {'stm32f411ve': 1000} + + +class Refused(Exception): + """Exit 2: nothing on the board was touched.""" + + +class StepFail(Exception): + def __init__(self, msg, evidence=None): + super().__init__(msg) + self.evidence = evidence or {} + + +class NotEnumerated(StepFail): + """The DFU device never appeared: the only failure after an install that re-entry may follow.""" + + +class Interrupted(Exception): + pass + + +# ------------------------------------------------------------------ roster and source facts + +def load_board(config, name): + try: + cfg = json.loads(Path(config).read_text()) + boards = [b for b in cfg['boards'] if b.get('name') == name] + except (OSError, ValueError, KeyError, TypeError, AttributeError) as e: + raise Refused(f'cannot read {config}: {e}') + if len(boards) != 1: + raise Refused(f'{config} has {len(boards)} entries named {name!r}') + b = boards[0] + tinyusb = os.environ.get('TINYUSB_DIR') + if not (tinyusb and os.path.isabs(tinyusb)): + raise Refused('set TINYUSB_DIR to the absolute path of the tinyusb checkout whose test/hil/helper/hil_lock.py ' + 'owns the board locks') + fl, t = b.get('flasher') or {}, b.get('tinyuf2') or {} + m = re.fullmatch(cb.FLASHER_ARGS.get(fl.get('name'), r'(?!)'), fl.get('args') or '') + if not isinstance(fl.get('uid'), str) or not m: + raise Refused(f'{name}: flasher must be jlink with args exactly "-device NAME", or esptool with args exactly ' + f'"--chip CHIP", with a string uid') + if not isinstance(b.get('uid'), str) or not b['uid']: + raise Refused(f'{name}: uid (the bootloader USB serial) is required') + lock = t.get('lock', name) # tinyusb's name for the board when it differs: both runners take one lock + if not (isinstance(lock, str) and re.fullmatch(r'[\w.-]+', lock)): + raise Refused(f'{name}: tinyuf2.lock must be the board lock name') + try: + port = cb.port_of(name) + except cb.Fail as e: + raise Refused(f'{name}: the roster names a board by its TinyUF2 name: {e}') + if fl['name'] == 'esptool': + if m.group(1) not in esp.BOOTLOADER_OFFSET or port != 'espressif': + raise Refused(f'{name}: esptool is supported for espressif chips {sorted(esp.BOOTLOADER_OFFSET)} only') + elif m.group(1) not in FLASH_LIMITS: + raise Refused(f'{name}: no programming limits known for J-Link device {m.group(1)}') + power = t.get('usbPower') + if power is not None: + if not (isinstance(power, dict) and set(power) == {'hub', 'port'} and isinstance(power['hub'], str) and + re.fullmatch(r'[\w-]+', power['hub']) and type(power['port']) is int and 1 <= power['port'] <= 15): + raise Refused(f'{name}: tinyuf2.usbPower must be {{"hub": "", "port": <1-15>}}') + if fl['name'] != 'jlink': + raise Refused(f'{name}: tinyuf2.usbPower is for J-Link boards only') + return {'name': name, 'lock': lock, 'uid': b['uid'], 'flasher': fl['name'], 'probe': fl['uid'], 'device': m.group(1), + 'if': JLINK_IF, 'speed': JLINK_SPEED, 'port': port, + 'helper': os.path.join(tinyusb, 'test', 'hil', 'helper', 'hil_lock.py'), + 'reentry': t.get('reentry'), 'usbPower': power} + + +def has_define(path, name): + return bool(re.search(rf'^\s*#\s*define\s+{name}\b', (ROOT / path).read_text(), re.M)) + + +def define(path, name): + m = re.search(rf'^\s*#\s*define\s+{name}\s+(.+?)\s*(//.*|/\*.*)?$', (ROOT / path).read_text(), re.M) + if not m: + raise Refused(f'{path}: no #define {name}') + return m.group(1) + + +LITERAL = r'(0x[0-9a-fA-F]+|\d+)[uUlL]*' + + +def num(expr, what): + m = re.fullmatch(rf'\(?\s*{LITERAL}\s*\)?', expr) + if not m: + raise Refused(f'{what}: {expr!r} is not a literal') + return int(m.group(1), 0) + + +def product(expr, what): + """A literal or a product of literals, e.g. (8*1024*1024); nothing else.""" + inner = expr.strip() + if inner.startswith('(') and inner.endswith(')'): + inner = inner[1:-1] + terms = inner.split('*') + if not all(re.fullmatch(rf'\s*{LITERAL}\s*', t) for t in terms): + raise Refused(f'{what}: {expr!r} is not a literal or a product of literals') + v = 1 + for t in terms: + v *= num(t, what) + return v + + +def flash_addr(expr, flash_base, what): + """A literal, or FLASH_BASE + literal where the port defines FLASH_BASE; nothing else.""" + m = re.fullmatch(rf'\(?\s*FLASH_BASE\s*\+\s*{LITERAL}\s*\)?', expr) + if not m: + return num(expr, what) + if flash_base is None: + raise Refused(f'{what}: {expr!r} uses FLASH_BASE, which this port does not define') + return flash_base + int(m.group(1), 0) + + +def cmake_var(path, name): + """The value of the one `set(NAME value)` in a CMake file, ${TOP}/ read as the checkout; nothing else.""" + hits = re.findall(rf'\bset\s*\(\s*{name}\b([^)]*)\)', cb.code_lines(ROOT / path), re.I) + if len(hits) != 1: + raise Refused(f'{path}: expected one set({name} ...), found {len(hits)}') + v = hits[0].strip().removeprefix('${TOP}/') + if not v or re.search(r'[\s$"]', v): + raise Refused(f'{path}: set({name} {hits[0].strip()}) is not one plain value') + return v + + +def port_sources(port, bdir): + """(CMSIS device header, the bootloader's linker script, FLASH_BASE, where the BIN starts in flash, RAM origin) of + a port the HIL supports, read from the CMake files the HIL's build uses.""" + if port in ('stm32f3', 'stm32f4'): + ld = cmake_script(f'ports/{port}/CMakeLists.txt') + series = port.upper() + mcus = set(re.findall(rf'\b{series}\d\dx[A-Z]\b', cb.code_lines(ROOT / bdir / 'board.cmake'))) + if len(mcus) != 1: + raise Refused(f'{bdir}/board.cmake: expected one {series} device define, found {sorted(mcus)}') + header = f'{cmake_var(f"ports/{port}/family.cmake", "ST_CMSIS")}/Include/{mcus.pop().lower()}.h' + flash_base, ram = ld_memory(ld, 'FLASH')[0], ld_memory(ld, 'RAM')[0] + return header, ld, flash_base, flash_base, ram + if port != 'mimxrt10xx': + raise Refused(f'ports/{port}: the HIL runner has no source facts for this port') + sdk = cmake_var(f'ports/{port}/family.cmake', 'SDK_DIR') + mcu = cmake_var(f'{bdir}/board.cmake', 'MCU_VARIANT') + ld = f'ports/{port}/linker/{mcu}_ram.ld' # where family.cmake takes FLASH_BASE and FCFB_ORIGIN from + vals = {} + for name in ('_flash_base', '_fcfb_origin'): + m = re.search(rf'^\s*{name}\s*=\s*(0x[0-9a-fA-F]+)\s*;', (ROOT / ld).read_text(), re.M) + if not m: + raise Refused(f'{ld}: no {name}') + vals[name] = int(m.group(1), 16) + flash_base = vals['_flash_base'] + # family.cmake FLASH_FCFB_ADDR: the BIN starts at the FCB, 0x400 into flash on RT1010 (IMXRT1010RM Table 9-8) + bin_base = flash_base + (vals['_fcfb_origin'] & 0xFFF) + return (f'{sdk}/devices/{mcu}/{mcu}.h', ld, flash_base, bin_base, + ram_origin(f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld')) + + +def cmake_script(path): + """The one linker script a port's CMakeLists.txt passes with --script=${CMAKE_CURRENT_LIST_DIR}/...""" + hits = re.findall(r'--script=\$\{CMAKE_CURRENT_LIST_DIR\}/([\w./-]+\.ld)(?=["\s)])', cb.code_lines(ROOT / path)) + if len(hits) != 1: + raise Refused(f'{path}: expected one --script=${{CMAKE_CURRENT_LIST_DIR}}/