diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index f2c6219e8..12b13d718 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -22,7 +22,9 @@ python3 $B ... --idf local # exported ESP-IDF instead of CI The last stdout line is JSON. An agent acts on `pass`; `firstError`, and `error` on exit 2, naming the cause or the remedy; `boards[]` (`status` `ok`, `failed` or `error`, `log`, `artifacts`, `notImplemented`); `built`, the artifacts of `--shared` roots, the only ones kept after a pass, with each one's hash in `boards[].sha256`; `nothingToBuild` and `uncovered`, each path with its reason; and `idf.matchesCi`. `input` and `defines` record what a build used (HEAD, `dirty`, submodule and `get_deps` dependency commits, `dirtyInputs` for any of those with local changes, `inputsDirtiedByBuild` for any that changed or moved while building, `.` for the checkout itself): the HIL contract binds firmware to this record. `paths` gives every changed path's `rule` and `reason`, the answer to why a path selected what it did. Exit 0 pass, 1 a build, self-test or test failed, 2 usage, resolution or a missing prerequisite, 3 an uncovered path; 2 outranks 1, and 1 outranks 3. `-v` streams the build to stderr; each build's log is `/check_build.log`. -`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths,ci}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`) but `dfu_runtime`'s, which the battery never runs. `dfu_runtime` selects the esptool roster boards of the ports that build it, and is uncovered once for each port that builds it without one. +`--shared` also keeps the bootloader ELF for debugging. Make places it at `cmake-build/cmake-build-/ports//tinyuf2-.elf`; CMake and ESP-IDF place it at `cmake-build/cmake-build-/tinyuf2.elf`. With `--backend cmake` on a make port, the root is `cmake-build/cmake-build--cmake` and the ELF its `tinyuf2.elf`. + +`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths,ci}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`) but `dfu_runtime`'s, which the battery never runs. `dfu_runtime` selects the Espressif roster boards of the ports that build it, and is uncovered once for each port that builds it without one. ## Reporting @@ -40,10 +42,10 @@ An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the J - `build.ci` is CI's selection from the same rules (`.github/workflows/ci_set_matrix.py` reads it on a pull request): every board a change reaches, not the representatives. A board's own files select that board; a port file, dependency, shared app or `make.mk`/`rules.mk` every board of the ports it reaches; a port app the boards that enable it. Shared code and a change to how boards build select `full`, every board and the ghostfat self-test; a ghostfat input the self-test alone. Tests, docs, CodeQL's build and uncovered paths select nothing. - **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`, which also builds each app the port's `CMakeLists.txt` adds with `family_add_app()` as its own project; Espressif `idf.py build`. A port moves from make to cmake in `build.yml` once a board of it passes the HIL battery with cmake. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds (cmake: its `family_add_app()` calls enabled for the board). A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. - **`--backend cmake`** is for the HIL's receipt only, with `--board` and `--shared`: on a make port it is no CI evidence, and it is how such a port's board gets the HIL run that moves the port to cmake. Espressif is refused. `-D` takes the cmake spellings: `LOGGER=RTT`, not `rtt`; `DEBUG` is ignored, the build stays MinSizeRel; `LOG` (default 0, as in make) sets `TUF2_LOG` and `CFG_TUSB_DEBUG`; raw flags go through `CFLAGS_CLI`. -- **A HIL record** (`--board --shared --hil-config `) binds the build's own flash target to the board's roster flasher: a J-Link board is configured with `JLINK_OPTION=-USB -ExitOnError 1 -NoGui 1` (so `tinyuf2-jlink` selects that probe and fails non-zero), an esptool board's ESP-IDF `flash` target takes the bridge's tty at flash time. `boards[].flash` holds the target, the roster flasher, the target's generated command (checked to name that probe or chip and the build's own image), the configure defines that bound it and the build-tree files it reads (in `sha256` too); a rebuild and a HIL flash hold the build dir's lock (`.lock`). Another flasher, a roster `args` other than `-device ` or `--chip `, or a `-D` of `JLINK_OPTION`, `OPENOCD_SERIAL`, `OPENOCD_OPTION` or `ESPPORT` is refused. Before flashing, the HIL dry-runs that target under the lock and refuses any command but the flasher and, on ESP-IDF, the generated ones that write no flash input (its bootloader subproject dry-run too); ESP-IDF re-runs CMake after any commit (its `git describe` dependency), so rebuild the record on the commit under test. +- **A HIL record** (`--board --shared --hil-config `) binds the build's own flash target to the board's roster flasher: a J-Link board is configured with `JLINK_OPTION=-USB -ExitOnError 1 -NoGui 1` (so `tinyuf2-jlink` selects that probe and fails non-zero), an ST-LINK board with `STLINK_SERIAL=` (so `tinyuf2-stlink` connects to that probe and writes the HEX), an esptool board's ESP-IDF `flash` target takes the bridge's tty at flash time. The HIL never runs these targets: it passes the bound files to agentrc's flash provider, so the binding is a build-time check that the target and the roster agree. `boards[].flash` holds the target, the roster flasher, the target's generated command (checked to name that probe or chip and the build's own image), the configure defines that bound it and the build-tree files it reads (in `sha256` too); a rebuild and a HIL flash hold the build dir's lock (`.lock`). `boards[].flashRoutes` binds each of the board's `tinyuf2.routes` (`openocd_esp`, `-f board/-builtin.cfg`, the USJ serial as `uid`) to the files the esptool binding hashed, for the same chip. Another flasher, a roster `args` other than `-device ` or `--chip `, or a `-D` of `JLINK_OPTION`, `STLINK_SERIAL`, `OPENOCD_SERIAL`, `OPENOCD_OPTION` or `ESPPORT` is refused. ESP-IDF's records go stale at the next commit (its `git describe` dependency re-runs CMake), so rebuild the record on the commit under test. - **ESP-IDF** builds in CI's image, `espressif/idf:`, through docker by default; a missing image is exit 2 naming the `docker pull`. `--idf local` uses the exported IDF and records its version and `matchesCi`: evidence from a different IDF is supplemental, not acceptance. - **ghostfat** builds each `ports/test_ghostfat/boards/*` config natively with cmake and runs its `selftest` target, which compares the generated image with the config's known-good one, in the private dir, one config at a time: `huge` needs about 3 GiB of disk while it runs. A passing config's dir is removed; a failing one keeps its log. - **Private dirs for parallel agents.** Each build starts from an empty `cmake-build/cmake-build-agent--` (with `-` when a board builds with another backend than its port's) and is removed when it passes (`removed`; its `artifacts` say what it produced, nothing is left to flash); a failed one is kept for its log. Nothing is written into the source tree: make takes `BUILD_ROOT=`, cmake and idf `-DARTIFACT_PATH=/_bin`. `--shared` rebuilds `cmake-build/cmake-build-` from clean and keeps it for flashing; it needs exclusive ownership of that board. -- **Dependencies are never fetched here.** `lib/tinyusb` and `lib/uf2` are submodules (`git submodule update --init lib/tinyusb lib/uf2`); a port's `tools/get_deps.py` rows must exist. In a worktree, symlink them from the primary checkout (CLAUDE.md), never run `get_deps.py` through a symlink: it runs `git reset --hard` in each dependency. -- **Debug builds** for a probe session use the port's backend directly: make `make -C ports/ BOARD= DEBUG=1 LOG=2 LOGGER=rtt all` (`LOGGER=swo` too); cmake `-DCMAKE_BUILD_TYPE=Debug -DLOG=2 -DLOGGER=RTT` (only `RTT` adds the RTT sources). Flash targets: make `flash-jlink`, `flash-pyocd`, `flash-stlink`, `flash-uf2`; cmake `tinyuf2-jlink`, `-stlink`, `-openocd`, `-pyocd` (lpc55, kinetis_k32l2), `-dfu-util` (stm32l4), and `-uf2` for each app; Espressif `idf.py -C ports/espressif -DBOARD= flash`. +- **Dependencies are never fetched here.** `lib/tinyusb`, `lib/uf2` and `lib/agentrc` (the HIL lock helper the contract tests load) are submodules (`git submodule update --init lib/tinyusb lib/uf2 lib/agentrc`); a port's `tools/get_deps.py` rows must exist. In a worktree, symlink them from the primary checkout (CLAUDE.md); `get_deps.py` refuses to update a symlinked dependency path. +- **Debug builds** for a probe session use the port's backend directly: make `make -C ports/ BOARD= DEBUG=1 LOG=2 LOGGER=rtt all` (`LOGGER=swo` too); cmake `-DCMAKE_BUILD_TYPE=Debug -DLOG=2 -DLOGGER=RTT` (only `RTT` adds the RTT sources). Flash targets: make `flash-jlink`, `flash-pyocd`, `flash-stlink`, `flash-uf2`; cmake `tinyuf2-jlink`, `-stlink`, `-openocd`, `-pyocd` (lpc55, kinetis_k32l2), `-dfu-util` (stm32l4), and `-uf2` for each app; Espressif `idf.py -C ports/espressif -DBOARD= flash`. On ports built with `-flto`, a `static` function may be inlined and absent as a standalone symbol; debug its caller, or use a local instrumented build that marks the function `__attribute__((noinline))`. - **Printing a command is not verification.** A verdict is the JSON of a run that happened; an agent that cannot run the script reports the build as not run. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index e11328dd2..0b9d56f9f 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -27,6 +27,8 @@ import shutil import subprocess import sys +import time +import uuid from pathlib import Path ROOT = Path(__file__).resolve().parents[4] @@ -40,6 +42,7 @@ 'lpc55': 'cmake', 'maxim': 'cmake', 'mimxrt10xx': 'cmake', + 'samd51': 'cmake', 'stm32f3': 'cmake', 'stm32f4': 'cmake', 'stm32h5': 'cmake', @@ -55,6 +58,7 @@ 'lpc55': ['lpcxpresso55s69'], 'maxim': ['max32666fthr'], 'mimxrt10xx': ['imxrt1064_evk', 'imxrt1170_evk'], + 'samd51': ['intech_grid'], 'stm32f3': ['stm32f303disco'], 'stm32f4': ['stm32f411ve_discovery', 'feather_stm32f405_express'], 'stm32h5': ['stm32h563_nucleo'], @@ -78,8 +82,8 @@ # make files every Makefile port includes: read by the make ports' builds and CodeQL's MAKE_SHARED = ('ports/make.mk', 'ports/rules.mk') CONTRACT_TESTS = ('.claude/test/test_build', '.claude/test/test_hil') -# the HIL contract's runner and rosters: a change there re-runs the battery on every roster board -HIL_CONTROL = ('.claude/skills/hil/scripts/', 'test/hil/') +# the HIL contract's runner, its lock helper's pin and rosters: a change there re-runs the battery on every roster board +HIL_CONTROL = ('.claude/skills/hil-contract/scripts/', 'test/hil/', 'lib/agentrc', '.github/workflows/hil.yml') # hil_test.py imports these: the identity, artifacts and build dirs the battery trusts HIL_IMPORTS = '.claude/skills/build/scripts/' BUILD_CONTROL = (HIL_IMPORTS, '.github/workflows/build.yml', @@ -94,14 +98,20 @@ # What a roster flasher runs to install TinyUF2: the build's own flash target, and the build-tree files that # target reads, which a HIL record pins so the runner can check them before it flashes. -FLASH = {'jlink': ('tinyuf2-jlink', ('tinyuf2.jlink', 'tinyuf2.hex')), +FLASH = {'jlink': ('tinyuf2-jlink', ('tinyuf2.jlink', 'tinyuf2.hex')), 'stlink': ('tinyuf2-stlink', ('tinyuf2.hex',)), 'esptool': ('flash', ('flash_args', 'bootloader/bootloader.bin', 'partition_table/partition-table.bin', 'ota_data_initial.bin', 'tinyuf2.bin'))} # a roster flasher's args, whose one group is the device or chip its flash target must name -FLASHER_ARGS = {'jlink': r'-device (\S+)', 'esptool': r'--chip (\w+)'} +FLASHER_ARGS = {'jlink': r'-device (\S+)', 'stlink': r'-device (\S+)', 'esptool': r'--chip (\w+)'} +# a further install route of an Espressif board (tinyuf2.routes): the chip's built-in USB-Serial/JTAG through agentrc's +# openocd_esp, which programs the esptool binding's segments; uid is the USJ serial +ROUTE_ARGS = {'openocd_esp': r'-f board/(esp32[a-z0-9]*)-builtin\.cfg'} NINJA_STEP = re.compile(r'\[\d+/\d+\] ') # ninja's progress prefix +NINJA_QUERY_TIMEOUT = 30 # s for one ninja query +DOCKER_TIMEOUT, CONTAINER_GONE_WAIT = 30, 10 # s for one docker call; s a killed query's container may linger +CONTAINER_SETTLE = 2 # s docker must list no container before a killed client's pending create counts as never landing # configure variables that select a probe or port: a HIL record sets them from the roster, never from -D -SELECTION = ('JLINK_OPTION', 'OPENOCD_SERIAL', 'OPENOCD_OPTION', 'ESPPORT') +SELECTION = ('JLINK_OPTION', 'STLINK_SERIAL', 'OPENOCD_SERIAL', 'OPENOCD_OPTION', 'ESPPORT') class Fail(Exception): @@ -503,7 +513,7 @@ def path_view(records, effect=None, keys=('path', 'rule', 'effect', 'reason')): return [{k: r[k] for k in keys} for r in records if effect in (None, r['effect'])] -HIL_APP = 'dfu_runtime' # the app hil_test.py's esptool battery copies through TinyUF2 +HIL_APP = 'dfu_runtime' # the app hil_test.py's Espressif battery copies through TinyUF2 def hil_selection(sel, config): @@ -514,7 +524,7 @@ def hil_selection(sel, config): try: entries = json.loads((ROOT / config).read_text())['boards'] roster = sorted({b['name'] for b in entries}) # TinyUF2 names - esptool = {b['name'] for b in entries if (b.get('flasher') or {}).get('name') == 'esptool'} + espressif = {b['name'] for b in entries if b['name'] in boards_of('espressif')} # whatever installs them except (OSError, ValueError, KeyError, TypeError, AttributeError) as e: raise Fail(f'cannot read boards from {config}: {e}') by_port = {} @@ -552,15 +562,15 @@ def kind(parts): # the HIL builds with cmake whatever CI uses: its build files app = parts[1] if r['rule'] == 'app' else parts[3] if len(parts) > 3 else None ports = {port_of(b) for b, _ in r['boards']} if r['rule'] == 'app' else {parts[1]} if app != HIL_APP: - uncovered[r['path']] = f'the battery runs the bootloader and {HIL_APP} on esptool boards, never {r["path"]}' + uncovered[r['path']] = f'the battery runs the bootloader and {HIL_APP} on Espressif boards, never {r["path"]}' continue - # a port that builds the app but has no esptool roster board: its battery never runs it + # a port that builds the app but has no Espressif roster board: its battery never runs it for p in sorted(ports): - own = [b for b in esptool if port_of(b) == p] + own = [b for b in espressif if port_of(b) == p] if own: boards.update(own) else: - uncovered[f'{r["path"]} on {p}'] = f'no {config} esptool board of {p} runs {HIL_APP}' + uncovered[f'{r["path"]} on {p}'] = f'no {config} Espressif board of {p} runs {HIL_APP}' elif r['rule'] == 'board': (b, _), = r['boards'] if b in roster: @@ -636,7 +646,7 @@ def run(cmd, log, verbose, cwd=ROOT): def check_deps(port): """Remedy for a missing submodule or family dependency, None when all are present.""" - missing = [s for s in ('lib/tinyusb', 'lib/uf2') if not (ROOT / s).is_dir() or not any((ROOT / s).iterdir())] + missing = [s for s in ('lib/tinyusb', 'lib/uf2', 'lib/agentrc') if not (ROOT / s).is_dir() or not any((ROOT / s).iterdir())] if missing: return f'empty submodule {", ".join(missing)}: git submodule update --init {" ".join(missing)}' for dep, families in deps_rows().items(): @@ -715,11 +725,12 @@ def build_idf(board, root, defines, log, verbose, idf_mode, image): return not run(idf_docker(root) + [image, 'bash', '-c', inner], log, verbose), [] -def idf_docker(root, verb='run'): - """`docker run` (or `create`) with the checkout, its git dir and the build dir mounted at their own paths, image - last.""" +def idf_docker(root, name=None): + """`docker run` with the checkout, its git dir and the build dir mounted at their own paths, image last.""" common = Path(git('rev-parse', '--path-format=absolute', '--git-common-dir').strip()) - cmd = ['docker', verb, '--rm', '--user', f'{os.getuid()}:{os.getgid()}', '-e', 'HOME=/tmp', '-w', str(ROOT)] + cmd = ['docker', 'run', '--rm', '--user', f'{os.getuid()}:{os.getgid()}', '-e', 'HOME=/tmp', '-w', str(ROOT)] + if name: + cmd += ['--name', name] for m in sorted({ROOT, common, root.parent}): cmd += ['-v', f'{m}:{m}'] return cmd @@ -727,11 +738,50 @@ def idf_docker(root, verb='run'): def ninja_in(backend, idf_mode, image, root, args): """Runs ninja on `root` with the ninja that built it: the host's, or the IDF image's (whose build log and - generator a host ninja would rewrite). Callers bind the build's own backend, mode and image.""" + generator a host ninja would rewrite). Callers bind the build's own backend, mode and image. The query runs here, + its process killed past NINJA_QUERY_TIMEOUT or on an interruption (only that process: `ninja -t commands` starts + nothing, though a docker client may), and it stays in this process group, so a kill of the group reaches it too. + stdout and stderr come back apart.""" cmd = ['ninja', '-C', str(root), *args] + container = None if backend == 'idf' and idf_mode == 'docker': - cmd = idf_docker(root) + [image, *cmd] # the image's entrypoint sources export.sh, then runs cmd - return subprocess.run(cmd, capture_output=True, text=True, cwd=ROOT) + container = f'tinyuf2-query-{os.getpid()}-{uuid.uuid4().hex}' + cmd = idf_docker(root, name=container) + [image, *cmd] # the entrypoint sources export.sh, then runs cmd + try: + return subprocess.run(cmd, capture_output=True, text=True, cwd=ROOT, timeout=NINJA_QUERY_TIMEOUT) + except BaseException as e: # subprocess.run has killed the query + left = container_left(container) if container else '' + if isinstance(e, subprocess.TimeoutExpired): + raise Fail(f'{root.name}: ninja {shlex.join(args)} timed out after {NINJA_QUERY_TIMEOUT}s{left}') from None + if left: + print(f'{root.name}: interrupted ninja {shlex.join(args)}{left}', file=sys.stderr) + raise + + +def container_left(name): + """'' once docker has listed no `name` for CONTAINER_SETTLE, `docker rm -f` before each look, so a create still + pending when the client was killed is removed when it lands; else why, naming it, for an error message.""" + end, absent_since = time.monotonic() + CONTAINER_GONE_WAIT, None + try: + while True: + subprocess.run(['docker', 'rm', '-f', name], capture_output=True, text=True, timeout=DOCKER_TIMEOUT) + r = subprocess.run(['docker', 'ps', '-aq', '-f', f'name=^{name}$'], capture_output=True, text=True, + timeout=DOCKER_TIMEOUT) + if r.returncode: + return f'; docker ps failed ({r.stderr.strip()[-200:]}), so container {name} may be left: docker rm -f {name}' + now = time.monotonic() + if r.stdout.strip(): + absent_since = None + elif absent_since is None: + absent_since = now + if absent_since is not None and now - absent_since >= CONTAINER_SETTLE: + return '' + if now > end: + return (f'; container {name} still exists: docker rm -f {name}' if absent_since is None else + f'; container {name} was listed within {CONTAINER_SETTLE}s, so it may be left: docker rm -f {name}') + time.sleep(0.2) + except (OSError, subprocess.SubprocessError) as e: + return f'; docker failed ({e}), so container {name} may be left: docker rm -f {name}' def backend_of(port): @@ -753,12 +803,40 @@ def roster_flasher(config, board): raise Fail(f'cannot read {board} from {config}: {e}') form = FLASHER_ARGS.get(name) if form is None: - raise Fail(f'{board}: flasher {name!r} has no HIL flash target (jlink or esptool)') + raise Fail(f'{board}: flasher {name!r} has no HIL flash target (jlink, stlink or esptool)') m = re.fullmatch(form, fargs) if not m or not re.fullmatch(r'[\w-]+', uid): raise Fail(f'{board}: flasher args {fargs!r} or uid {uid!r} is not the roster form') flasher = {'name': name, 'uid': uid, 'device': m[1]} - return flasher, [f'JLINK_OPTION=-USB {uid} -ExitOnError 1 -NoGui 1'] if name == 'jlink' else [] + defines = {'jlink': [f'JLINK_OPTION=-USB {uid} -ExitOnError 1 -NoGui 1'], 'stlink': [f'STLINK_SERIAL={uid}']} + return flasher, defines.get(name, []) + + +def roster_routes(config, board): + """The board's further install routes (tinyuf2.routes), each {name, uid, device} as a record binds it.""" + try: + entry = next(b for b in json.loads((ROOT / config).read_text())['boards'] if b['name'] == board) + routes = [(r['flasher']['name'], r['flasher']['uid'], r['flasher']['args']) + for r in (entry.get('tinyuf2') or {}).get('routes', [])] + except (OSError, ValueError, KeyError, TypeError, StopIteration, AttributeError) as e: + raise Fail(f'cannot read {board}\'s routes from {config}: {e}') + out = [] + for name, uid, fargs in routes: + m = re.fullmatch(ROUTE_ARGS.get(name, r'(?!)'), fargs) if isinstance(fargs, str) else None + if not m or not isinstance(uid, str) or not re.fullmatch(r'[\w.:+-]+', uid): + raise Fail(f'{board}: route {name!r} args {fargs!r} or uid {uid!r} is not the roster form') + out.append({'name': name, 'uid': uid, 'device': m[1]}) + return out + + +def flash_routes(rec, routes): + """A passing IDF build's further routes, each bound to the files its esptool binding hashed: the segments + openocd_esp programs as they are, without esptool's header rewrite.""" + fl = rec.get('flash') or {} + for r in routes: + if (fl.get('flasher') or {}).get('name') != 'esptool' or fl['flasher']['device'] != r['device']: + raise Fail(f'{rec["board"]}: route {r["name"]} for {r["device"]} needs the esptool binding of that chip') + return {r['name']: {'flasher': r, 'inputs': list(fl['inputs'])} for r in routes} def flash_binding(flasher, root, ninja): @@ -767,7 +845,7 @@ def flash_binding(flasher, root, ninja): build's environment.""" target, inputs = FLASH[flasher['name']] r = ninja(root, ['-t', 'commands', target]) - tool = 'JLinkExe' if flasher['name'] == 'jlink' else 'run_serial_tool.cmake' + tool = {'jlink': 'JLinkExe', 'stlink': 'STM32_Programmer_CLI'}.get(flasher['name'], 'run_serial_tool.cmake') cmds = [l for l in r.stdout.splitlines() if tool in l] if r.returncode == 0 else [] ok = len(cmds) == 1 if ok and flasher['name'] == 'jlink': @@ -777,6 +855,12 @@ def flash_binding(flasher, root, ninja): '-device', flasher['device'], '-USB', flasher['uid'], '-ExitOnError', '1', '-NoGui', '1', '-if', 'swd', '-JTAGConf', '-1,-1', '-speed', 'auto', '-CommandFile', str(root / inputs[0])] and \ (root / inputs[0]).is_file() and (root / inputs[0]).read_text() == f'halt\nloadfile {root / inputs[1]}\nr\ngo\nexit' + elif ok and flasher['name'] == 'stlink': + # the device is the roster's to name: the CLI identifies the chip itself, which the runner checks every session + cd, _, cmd = cmds[0].partition(' && ') + argv = shlex.split(cmd) + ok = cd == f'cd {root}' and Path(argv[0]).name == 'STM32_Programmer_CLI' and argv[1:] == [ + '--connect', 'port=swd', f'sn={flasher["uid"]}', '--write', str(root / inputs[0]), '--go'] elif ok: argv = shlex.split(cmds[0].partition(' && ')[2]) defs = dict(v.split('=', 1) for k, v in zip(argv[1:-2], argv[2:-2]) if k == '-D' and '=' in v) @@ -793,83 +877,12 @@ def flash_binding(flasher, root, ninja): ok = ok and sorted(l.split()[1] for l in segs if l.strip()) == sorted(inputs[1:]) if not ok: raise Fail(f'{root.name}: target {target} does not flash {flasher} with the build\'s image: ' - f'{cmds if cmds else r.stderr.strip()[-200:]}') + f'{cmds if cmds else (r.stderr or r.stdout).strip()[-200:]}') return {'target': target, 'flasher': flasher, 'command': cmds[0], 'inputs': list(inputs)} -def dry_run(ninja, root, target=None): - """The commands `ninja -n -v` would run; whatever precedes ninja's own output (the IDF image's entrypoint - banner) is skipped. None when ninja failed or its output is not ninja's.""" - r = ninja(root, ['-n', '-v'] + ([target] if target else [])) - lines = r.stdout.splitlines() - start = next((i for i, l in enumerate(lines) if l.startswith('ninja: Entering directory')), None) - if r.returncode or start is None: - return None - steps = [l for l in lines[start + 1:] if l.strip() and l != 'ninja: no work to do.'] - return None if any(not NINJA_STEP.match(l) for l in steps) else [l.split('] ', 1)[1] for l in steps] - - -def idf_step_writes_no_input(cmd, root, chip): - """True for a command ESP-IDF's flash target always runs before its flasher, segment by `&&` segment in one - of the forms TinyUF2's port and IDF generate: the bootloader subproject's build (its own tree is dry-run too), - stamps, read-only size checks, the port's post-build (combined.bin, copies into _bin, the update app's C - array). None of them writes a flash input; anything else, a shell operator included, is refused.""" - cwd, py, cm = None, r'\S*/python[\d.]*', r'\S*/cmake' - # a path resolved, against a boundary resolved only up to the build root, so no symlink inside the tree moves - # either; a write never lands on a flash input - real = Path(os.path.realpath(root)) - inputs = {Path(os.path.realpath(root / i)) for i in FLASH['esptool'][1]} - under = lambda p, d='': Path(os.path.realpath(p)).is_relative_to(real / d) # noqa: E731 - writes = lambda p, d='': under(p, d) and Path(os.path.realpath(p)) not in inputs # noqa: E731 - postbuild = root / 'tinyuf2.postbuild.sh' - for seg in cmd.split(' && '): - if re.search(r'[;|<>&$`(){}\\]', seg): - return False - argv = shlex.split(seg) - head, tail = ' '.join(argv[:3]), argv[3:] - ok = argv[:1] == ['cd'] and len(argv) == 2 and under(argv[1]) or \ - re.fullmatch(cm, argv[0]) and ( - argv[1:] == ['--build', '.'] and cwd == root / 'bootloader' or - argv[1:] == ['-E', 'echo_append'] or argv[1:3] == ['-E', 'echo'] and len(argv) == 4 or - argv[1:3] in (['-E', 'make_directory'], ['-E', 'touch']) and len(argv) == 4 and writes(argv[3]) or - argv[1:3] == ['-E', 'copy'] and len(argv) == 5 and writes(argv[4], '_bin')) or \ - re.fullmatch(py + r' \S*/partition_table/check_sizes\.py --offset', head) and ( - tail[1:] in ( - ['partition', '--type', 'app', str(root / 'partition_table/partition-table.bin'), str(root / 'tinyuf2.bin')], - ['partition', '--type', 'app', '--subtype', 'factory', str(root / 'partition_table/partition-table.bin'), - str(root / 'tinyuf2.bin')]) or - len(tail) == 4 and tail[1] == 'bootloader' and re.fullmatch(r'0x[0-9a-f]+', tail[2]) and - tail[3] == str(root / 'bootloader/bootloader.bin')) or \ - re.fullmatch(py + r' \S*/lib/uf2/utils/uf2conv\.py --carray', head) and \ - tail == ['-o', str(root / 'apps/update_tinyuf2/bootloader_bin.c'), str(root / 'tinyuf2.bin')] and writes(tail[1]) or \ - argv == ['bash', str(postbuild)] and cwd == root and writes(root / 'combined.bin') and postbuild.is_file() and \ - postbuild.read_text() == ( - '#!/bin/bash\necho "Creating combined.bin"\n' - f"esptool --chip {chip} merge-bin --output combined.bin $(tr '\\n' ' ' < {root / 'flash_args'})\n") - if not ok: - return False - if argv[0] == 'cd': - cwd = Path(argv[1]) - return True - - -def flash_ready(root, flash, ninja): - """Before a HIL flash, under the build lock and with its inputs hash-checked: a dry run shows the target would run - the recorded command once and nothing else that could write a flash input: no rebuild, relink or CMake re-run - (ESP-IDF re-runs CMake after any commit, through its git describe dependency). `ninja(dir, args)` runs ninja in - the build's environment.""" - steps = dry_run(ninja, root, flash['target']) - extra = None - if steps is not None: - others = [c for c in steps if c != flash['command']] - if flash['flasher']['name'] == 'esptool': - nested = dry_run(ninja, root / 'bootloader') # always: the outer build step runs whatever it needs - others += ['bootloader: ninja -n failed'] if nested is None else nested - others = [c for c in others if not idf_step_writes_no_input(c, root, flash['flasher']['device'])] - extra = others if steps.count(flash['command']) == 1 else others + ['no flash command'] - if extra is None or extra: - raise Fail(f'{root.name}: target {flash["target"]} would not run only its flasher: ' - f'{"ninja -n failed" if extra is None else extra[:3]}; rebuild the record') +def build_lock_file(root): + return root.with_name(root.name + '.lock') @contextlib.contextmanager @@ -877,7 +890,7 @@ def build_lock(root, wait=True): """Excludes a rebuild of a shared build dir while another build or a HIL flash uses it; without `wait`, a held lock raises BlockingIOError.""" root.parent.mkdir(parents=True, exist_ok=True) - with open(root.with_name(root.name + '.lock'), 'a') as f: + with open(build_lock_file(root), 'a') as f: fcntl.flock(f, fcntl.LOCK_EX | (0 if wait else fcntl.LOCK_NB)) yield @@ -998,7 +1011,20 @@ def run_tests(): capture_output=True, text=True, cwd=ROOT) last = [l for l in r.stderr.splitlines() if l.strip()] return {'status': 'ok' if r.returncode == 0 else 'failed', 'summary': last[-1] if last else '', - 'firstError': '' if r.returncode == 0 else first_error(r.stderr)} + 'firstError': '' if r.returncode == 0 else unittest_first_error(r.stderr)} + + +def unittest_first_error(log): + """The first failed test's id (without the docstring line unittest may print under it) and its exception's first + line: the first unindented line after the last traceback header, since an assertion's message (a diff) or a chained + exception may follow or precede it.""" + failure = re.search(r'^={70}\n((?:FAIL|ERROR): [^\n]+)\n(?:[^\n]*\n)?-{70}\n(.*?)(?=^={70}\n|^-{70}\nRan \d+ test)', + log, re.M | re.S) + if not failure: + return first_error(log) + frames = failure.group(2).split('Traceback (most recent call last):\n')[-1].splitlines() + exc = next((line for line in frames if line.strip() and not line[0].isspace()), '') + return (failure.group(1) + (f': {exc}' if exc else ''))[:400] class Parser(argparse.ArgumentParser): @@ -1071,7 +1097,11 @@ def main(argv=None): idf['version'] = (v.stdout or v.stderr).strip() idf['matchesCi'] = ci in idf['version'] flashers = {b: roster_flasher(args.hil_config, b) for b, _ in sel['boards']} if args.hil_config else {} + routes = {b: roster_routes(args.hil_config, b) for b, _ in sel['boards']} if args.hil_config else {} records = [build_board(b, backend, args, idf, flashers.get(b)) for (b, backend) in sel['boards']] + for r in records: + if routes.get(r['board']) and r['status'] == 'ok': + r['flashRoutes'] = flash_routes(r, routes[r['board']]) for r in records: r['selectedBy'] = sel['boards'][(r['board'], r['backend'])] # an input that moved while building, even if restored before the HIL checks it, leaves firmware diff --git a/.claude/skills/hil-contract/SKILL.md b/.claude/skills/hil-contract/SKILL.md new file mode 100644 index 000000000..f44bb9009 --- /dev/null +++ b/.claude/skills/hil-contract/SKILL.md @@ -0,0 +1,180 @@ +--- +name: hil-contract +description: The TinyUF2 HIL contract: its roster, selection, build record, battery, locks, report and recovery under agentrc's `hil` rules. Use when a change needs hardware evidence on a rig board, when a debug or validation session must take, flash, clean up or release a TinyUF2 rig board, or when a workflow asks for "the project's HIL contract". +--- + +# HIL contract + +`scripts/hil_test.py` holds the mechanics: roster and source checks, firmware binding, locking, the probe and USB steps, cleanup and the JSON report. Agents call it instead of composing J-Link, mtools or lock commands. This page is TinyUF2's contract under agentrc's `hil` skill, which is read first. + +## Hosts/rosters + +- One roster per host, `` in the commands below: `test/hil/local.json` on htpc's bench and `test/hil/ci.json` on the ci.lan rig (`hostname` `ci`, where the CI job below runs); `hostname` says which you are on. Elsewhere a J-Link or ST-LINK run fails at `install`, where no probe with the roster's serial is found, before anything is written: `cleanup.state` is `untouched` and nothing stays held. An Espressif board without exactly one tty with its route's serial stops at the runner's probe check, after the lock: a `ran: false` row with `detail` `probe check: ...`, nothing written and nothing held. Both hosts are shared with tinyusb's rosters (its contract's Hosts/rosters: `test/hil/local.json` on htpc, `test/hil/tinyusb.json` on ci.lan): a rig-wide operation holds both rosters' lock names (`hil` Board locks), tinyusb's under the names and in the sequence its contract's Reservation gives. + +A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. A J-Link entry has `args` exactly `-device `; the runner uses SWD at 4000 kHz, so pass the same to those scripts, `--interface swd --speed 4000`. An ST-LINK entry has `flasher.name` `stlink`, `flasher.uid` the ST-LINK's USB serial and `args` exactly `-device `, a name the runner holds limits and a chip identity for (`FLASH_LIMITS`, `STLINK_CHIPS`); STM32CubeProgrammer takes no device name, so every session checks the Device ID and NVM size it prints. An Espressif entry has `flasher.name` `esptool`, `flasher.uid` the USB-UART bridge's serial (`/dev/serial/by-id/*_-if*`), `flasher.args` exactly `--chip `, and the top-level `uid` the chip's factory MAC, which is TinyUF2's USB serial. The board's battery is its kind, Espressif, whatever route installs it; the route is the transport. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: +- the board lock name (`lock`) when tinyusb names the board differently, as `mimxrt1060_evk` for `imxrt1060_evk`; both runners take that one lock, and the helper's `status`, holder records and the report rows' `lock` show it. It defaults to the board name; +- the commissioned re-entry path (`reentry`); +- an Espressif board's further install routes (`routes`), each `{"flasher": {"name": "openocd_esp", "uid": "", "args": "-f board/-builtin.cfg"}}` for the chip `flasher.args` names, with `"commissioning": "pending"` until commissioning clears it (commissioning.md, On an esptool board). The route reaches the chip's built-in USB-Serial/JTAG (USJ) through agentrc's `openocd_esp` (`hil` Flashers), and its USJ serial also selects the tty of the route's esptool sessions (`/dev/serial/by-id/*_-if*`). It shares the board's one lock and entry: `run`, `install` and `restore` take `--route openocd_esp`, chosen before any board access and never switched after a failure, and `run` refuses a route pending commissioning, which installs and restores only. Under a CI job (`RUNNER_TRACKING_ID` or `GITHUB_ACTIONS` set, even empty) the `openocd_esp` route is refused (exit 2) by the provider's `ci_refusal()`, at route selection before any board access and again before the route's first esptool session; `hil_openocd_esp.py end` is not refused. espressif_p4_function_ev's route (`30:ED:A0:E0:C9:02`) is commissioned (bench-history.md); +- for a board whose TinyUF2 signals no disconnect when it comes up after a reset, the per-port-power hub port its USB is on (`usbPower`: `{"hub": "", "port": N}`; the serial, not the bus location, so a replugged hub is still found). J-Link boards only. The hub must switch that port's power on its own. `uhubctl` showing `ppps` is a required preliminary check, not proof: a hub with several gangs, or with no power switches, may report per-port switching too (USB 2.0 11.11, 11.11.1). On a ganged hub a cycle removes no power while another port in the gang is powered, and can power the whole gang off when it is the last (USB 2.0 11.11). The board-only lock is relied on only once commissioning has shown VBUS on that port drop and the cycle reach no other port (commissioning.md, Port power). While `dfu` waits, the runner power-cycles that port, under the board's own lock (Limits), once it has sat connected but unaddressed (sysfs state `default`) for 2 s of that wait, at most once per 10 s, and at no other time: it writes the port's sysfs `disable` 1, then 0 a second later, which the kernel runs under the hub's lock, after any enumeration in progress, disconnecting the child before VBUS goes off. Each write and read-back runs in a bounded child; a cycle that fails or is interrupted from the off write on still writes 0, unless one of its children survives SIGKILL and so could still write after it, and says in `hil.log` whether that restored power or was not attempted. The report lists the cycles (`portCycles`). + +The battery runs a board only after commissioning sets `reentry` and confirms `uid`; `install` and `restore` work without `reentry`, for commissioning. Only commissioned roster boards can be offered for a run. Adding a board is commissioning, the lead's (`commissioning.md`), never a review-time or dispatch-time action: the runner refuses a build record from a dirty tree, an uncommitted roster entry included. + +## Selection + +`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config ` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered, except `dfu_runtime`, which selects the Espressif boards of the ports that build it, and leaves uncovered each port that builds it without one (mimxrt10xx today): the battery runs the bootloader with its generated fixture on J-Link and ST-LINK boards and with `dfu_runtime` on Espressif boards. The runner, the build scripts it imports, its lock helper's pin and the roster (`.claude/skills/hil-contract/scripts/`, `.claude/skills/build/scripts/`, `lib/agentrc`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. + +There are no examples or test names to select: a run is the whole battery on one board, and the board is the only selection. + +## Build + +```bash +mkdir -p cmake-build/hil +python3 .claude/skills/build/scripts/check_build.py --board --backend cmake --shared --hil-config > cmake-build/hil/.build.json +``` + +`` is the roster name, the TinyUF2 board name. An espressif board builds without `--backend cmake`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. + +The build record is this contract's build receipt: build it on the clean, pinned HEAD with `--hil-config` and pass it with `--build-record`. `--hil-config` binds the build's own flash target to the roster's probe or bridge (build contract), a build-time compatibility check, and binds each route in `tinyuf2.routes` to the segments that binding hashed (`flashRoutes`); the run refuses a record whose binding names another flasher or route, and checks the files the install reads against the record. The runner never runs the target: it passes the bound files to agentrc's flash provider (Execution). The build dir's lock covers the install's inputs only: the files the runner reads itself (the BIN it verifies, the segments it reads back, `apps/blinky.uf2` and `apps/dfu_runtime.uf2`) are copies in the run dir, taken at admission from bytes that hash as recorded, so a rebuild after the target cannot change them. For a J-Link or ST-LINK board it must be a cmake build (`--backend cmake`), whatever backend CI builds the port with; board facts are read from the port's CMake files too. For an espressif board it must be an IDF build from CI's `espressif/idf` image; the run binds its generated `sdkconfig`, `flash_args`, the four segments (bootloader, partition table, otadata, `tinyuf2.bin` in the factory partition) and `apps/dfu_runtime.uf2`, and checks every segment against the generated partition table and the candidate's partition CSV. The run refuses a build record that is not clean, does not match the checkout now (HEAD, submodules, `get_deps` dependencies at their pins), or whose artifacts changed since. Commit first; ESP-IDF's records also go stale at the next commit (build contract). + +The Ninja query that inspects a flash target at the build (`ninja -t commands`) is bounded to 30 seconds. A timed-out or interrupted query's process is killed (only that process, which stays in check_build's process group, so a kill of the group reaches it too) and its ESP-IDF container (`tinyuf2-query--<32 hex digits>`, unique per query) removed until docker has listed none for 2 seconds; a timeout fails the build naming the build dir, the query and any container that may be left, and an interruption prints that container on stderr. The install runs no Ninja query and no container. + +## Execution + +```bash +python3 .claude/skills/hil-contract/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json +``` + +Before taking the lock, the run checks the automount exclusion rule, and every span it could write (the HEX the flash target loads, inside the install span and the device's flash limits and laid out as the BIN; the BIN's board id), with the install and app spans taken from the candidate's source. Run it in the background; one board takes about 2 minutes, and the run is complete when it exits, or when it printed `cleanup.holder.inProcess` true and will not (Recovery): its last stdout line is the report JSON (Results). With several boards, one battery per board, one after another. + +The battery's steps on one board, all under one lock. The roster's probe serial selects the board (each probe is wired to one target), so no step identifies the chip: +1. **install:** agentrc's flash provider (`hil` Flashers, `flash_jlink`) runs JLinkExe on the record's HEX with the script the record's `tinyuf2-jlink` target runs, `halt`, `loadfile `, `r`, `go`, `exit` (a test keeps the two equal), from a command file in the run dir, under the build dir's lock once the files it reads hash as recorded. JLinkExe can exit 0 with every command failed, so the transcript must show the record's HEX downloaded (programmed, or skipped as already matching), `O.K.` and the script completed, with no failure line. The files are hashed again after it, and a session of the runner's, through the provider's `run_jlink_commands` with this contract's connection profile, verifies the BIN at its flash address (`verifybin`), then resets. The install is never retried. The board is touched from JLinkExe's start; only attempts that all end before any target connect leave it untouched, either at "Connecting to J-Link ...FAILED" or in the `libjlinkarm` crash at the probe connect (Limits), and a timed-out one never does. The build dir's lock is released only once every child the flash started has ended: one that outlives SIGKILL keeps it through a keeper that exits once the child does (`buildLocksKept`), so a rebuild waits. +2. **dfu:** the UF2 drive with this board's serial is found. On the RT boards the install has left the app start erased on the bench (Limits), so DFU comes up; if it never enumerates, as when a valid app boots (an install on stm32f303disco, stm32f411ve_discovery or stm32h563_nucleo keeps the app; per-board evidence: bench-history.md), the commissioned `retained-register` re-entry brings DFU up (its claim says so); any other DFU failure fails the step. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. +3. **copy:** a generated app is copied to the drive. +4. **app-ran:** the probe shows the app counting in thread mode with VTOR at the app start. +5. **reenter:** the bootloader is re-entered by the commissioned path, and the drive's `CURRENT.UF2` reads the app back. +6. **cleanup** (reported as `cleanup`, not a step, state `blinky copied`, or `untouched` with nothing to do when no session reached the board): the record's TinyUF2 is installed again as in install, DFU is found, the record's `apps/blinky.uf2` (admitted with the record: one family, every block in the app region) is copied to the drive, and the DFU device must leave and stay away for a quiet period. Any failure keeps the board held (Recovery, exit 4). + +On an ST-LINK board the steps are the J-Link board's, each probe session one STM32CubeProgrammer CLI script (`-script`) that connects to the roster's serial in `HOTPLUG` mode, without a reset. A session passes only with exit 0, the roster's Device ID and NVM size, every operation echoed in order with its success line, no `Error` line and the script's end; the CLI stops a script at its first failed operation. `install` runs the provider's `flash_stlink` on the record's HEX, the command the record's `tinyuf2-stlink` runs (`--connect port=swd sn= --write tinyuf2.hex --go`), whose transcript must show the HEX parsed, the sectors erased, "File download complete" and "Start operation achieved successfully"; a session of the runner's then reads the BIN back from its flash address (`-u`), compares it, and resets (`-rst`). `app-ran` takes both samples, the delay and the image read-back in one script. The SYSRESETREQ write is the provider's `sysresetreq_stlink` (`-w32 … -nv`: with the CLI's read-back the write fails across the reset), and on a J-Link board its `sysresetreq_jlink`; this contract judges both. Only "Error: Serial number not found" before any device line leaves the board untouched. + +A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; on a J-Link board a run triggered by the upload alone is unverified (an Espressif board's app-ran observes the copied app itself over USB). "reenter" by `retained-register` is the forced-entry path, not a physical double tap. + +On an Espressif board, each esptool connection is a child of the runner (`esp_session.py`), and so is each USB control request (`usb_dfu.py`, pyusb): +1. **install:** a first connection takes the MD5s of every partition the battery never writes (nvs, ota_1, ffat on the 8 MB layout) as the run's snapshot, kept in the row's `snapshot` however the run ends; that session is the first to touch the board, which only a missing tty with the route's serial leaves untouched. Then the provider's `flash_esptool` runs the runner's own esptool, the 5.2.0 `esptool` command of the runner's Python (refused before any board access otherwise), as `NO_COLOR=1 esptool --chip -p --before=default_reset --after=hard_reset write_flash ` from the build dir under its lock, with the same hash checks as a J-Link install. esptool 5.2.0 writes every segment, as `IDF_FLASH_FULL=1` made the IDF target do, and copies none to `*_flashed.bin`. Its transcript, every CSI sequence and the underscore spellings' deprecation warnings stripped, must show every segment written at its offset and hash-verified, then the hard reset. A runner session then reads each segment back and takes those MD5s again, then resets once; the runner compares the read-back bytes with the build's segment files and the MD5s with the snapshot, and any difference fails the step. esptool rewrites the bootloader header from `flash_args`, which the build already holds (dio, 80 MHz, 8 MB): on the bench (2026-10-13, esptool 5.3.1 in espressif/idf:v6.1) all four segments read back identical to the files. The roster's bridge selects the board (it is wired to that board only), so the chip is not identified, and its security eFuses and flash size are commissioning's to check. The evidence lists each write's span and the 4 KiB-aligned span covering it, and the esptool version. Blank otadata boots the factory partition, TinyUF2. + + On the `openocd_esp` route (Hosts/rosters) the install runs under the same one board lock, each step only once the previous one's cleanup completed: an esptool session over the USJ's tty reads the chip's security state on the ROM loader, before the stub, from esptool's GET_SECURITY_INFO and from the eFuse registers (on the P4 SPI_BOOT_CRYPT_CNT and SECURE_BOOT_EN), takes the snapshot and hard-resets the chip, since one left in the ROM loader faults openocd's flash stub after `reset init` (PMP Load access fault at `flash probe`, bench 2026-10-20); the install is refused, nothing programmed, unless both sources read flash encryption and secure boot off, which is the evidence the provider requires and records (`security`). After a one-second wait (bench-history.md, espressif_p4_function_ev JTAG route), the provider's `flash_openocd_esp` programs the record's segment copies in one openocd-esp32 session pinned to the USJ by serial and bus-port (the build `$HIL_OPENOCD_ESP` names, else `openocd` on PATH; the provider refuses any but v0.12.0-esp32-20260703), after a `reset init` whose event hooks all passed, each written and checked with `esp verify_bank_hash` without a reset between, reads each back byte for byte and ends with `reset run` and a confirmed shutdown; it writes the files as built, without esptool's header rewrite, so the read-back compares the built bytes. Then the esptool read-back and preservation check as above. A session the provider leaves not ended is cleanup pending (Results, Recovery): the runner never signals or reaps it, and no later session, restore or retry runs. +2. **dfu:** as above; `INFO_UF2.TXT` reports the app partition (ota_0) as its flash, and no register is written. +3. **copy:** `apps/dfu_runtime` from the record, plus a block carrying a fresh nonce in ota_0's last 256 bytes, where the app reads it, renumbered under one `numBlocks`; the image itself is never patched. The record is refused when the image reaches that block or TinyUF2's 64 KiB flash cache window over it (`FLASH_CACHE_SIZE`) leaves ota_0. The evidence lists the image and nonce spans and the cache windows covering them. +4. **app-ran:** the app enumerates with its own VID:PID and the board's serial on the DFU's USB path, its DFU runtime interface string read from the device is `HIL `, and TinyUF2 stays away for a quiet period. +5. **reenter** (`dfu-detach`): DFU_DETACH to the app is acknowledged, the app leaves, TinyUF2 comes back on the same path, and `CURRENT.UF2` reads back the image and the nonce block. +6. **recopy:** a copy with a fresh nonce boots the app again. +7. **cleanup:** as on a J-Link board, with the install as in step 1: the preserved regions' MD5s after the writes, and again after the blinky copy, must equal the run's snapshot (a recovery takes it from the failed report's `snapshot`; a manual `restore` takes its own first). A mismatch fails the cleanup and keeps the hold. + +`--fail-at ` exists only for acceptance: it fails that step to exercise cleanup. + +### CI + +The `HIL` workflow (`.github/workflows/hil.yml`) runs the battery on the ci.lan rig for every pull request from this repository (a fork's never reaches the rig). A hosted job selects the boards the change reaches (Selection, with `test/hil/ci.json`) and prints what is uncovered in its summary; the rig job, on the runner labelled `tinyuf2`, checks out the merge commit, fetches the submodules and the roster ports' dependencies, and runs `scripts/hil_ci.py`: per board a record built there (Build) and one battery whose run directory (`run --run-root`) lies under the rig's `hil-state/` with the record and the battery's output, uploaded as the job's artifact. The battery keeps trying a held board every 15 s within `run --lock-wait` (10 min from CI), then reports it locked, which the job reads as not-run, never a pass; a cleanup holder is never waited for. The job's only pre-flight is its markers and the runner's own probe check: a surviving `mcopy` or USB helper of a killed run is seen through the marker, not by a host-wide process scan. Each row maps onto the Verdicts: the job exits 0 only when every selected board passed, 1 on a fail or not-run, 2 on a blocker. A battery that ended without a report, or kept a cleanup holder, leaves its `.started` marker without a `.done`: that board is a blocker on every later run until recovery (Recovery) has run and the marker is removed by hand; the workflow is never cancelled by a newer push for the same reason. `workflow_dispatch` runs named roster boards, with `fail_at` for `--fail-at` (acceptance only). + +## Reservation + +Locks are taken through agentrc's `hil` helper, `lib/agentrc/skills/hil/scripts/hil_lock.py`, at the commit the `lib/agentrc` submodule pins (`git submodule update --init lib/agentrc`), never an installed agentrc's copy. `hil_test.py`'s `run`, `install` and `restore` refuse before any hardware access while the submodule is missing, checked out at another commit or locally modified; the manual `hold`, `release` and `status` below run the helper directly, with no such check. The lock is the board's `` (Hosts/rosters): `hold` and `release` take it and `status` (no board argument) shows it, while `hil_test.py -b` and the probe scripts take the board name. +- **The battery locks itself,** as `hil_test.py`, which the helper's `release` refuses to kill. +- **The lock fails closed.** An unusable lock dir refuses the run (exit 2); it never runs unlocked. `HIL_NO_BOARD_LOCK=1` is the bypass (`hil` Board locks); the report then says `lockBypassed: true`. +- **Manual sessions** (hw-debugger, hw-validator): + 1. Take your own hold: `python3 lib/agentrc/skills/hil/scripts/hil_lock.py hold --reason ""`. + 2. Install a build record's TinyUF2, and restore the board when done: + + ```bash + python3 .claude/skills/hil-contract/scripts/hil_test.py install -b --build-record --under-hold "" + python3 .claude/skills/hil-contract/scripts/hil_test.py restore -b --build-record --under-hold "" + ``` + + `install` is the battery's install step from the record through the flash provider (Execution: a J-Link or ST-LINK board gets the record's HEX, an Espressif board the four IDF segments, over `--route openocd_esp` through its route), then waits for DFU; it takes the record the battery takes, admitted the same way, so it needs the automount exclusion rule (`commissioning.md`) and a clean build of HEAD. `--try-reentry `, while the roster's `reentry` is unset, gives DFU's fallback (Execution, dfu) for commissioning; `restore` takes it too. `restore` is the cleanup (Execution, step 6), from a clean build record of HEAD. Both refuse unless the hold is provably yours: the record's reason, a live pid, a FLOCK on the lock file in that pid's own fds, and a failing probe of the flock. Both also refuse before any session while a process still uses the board's probe, or while the board's pending mark lists an `openocd_esp` session not ended (Recovery, Pending mark). + 3. Release with `python3 lib/agentrc/skills/hil/scripts/hil_lock.py release `, only after a `restore` that exited 0 with no `survivors`. Otherwise the board is not cleaned up: keep the hold and report it as a blocker. + + Manual exits: 0 the operation succeeded (for `install`, DFU came up; for `restore`, DFU left after the blinky copy and stayed away); 1 it failed before writing; 2 refused before any hardware access; 4 a child survived SIGKILL, an `openocd_esp` session is not ended (cleanup pending, its receipt in `survivors`), or an `install` or `restore` failed after it began writing: keep the hold. `buildLocksKept` names a build dir whose lock a keeper holds until the listed survivors end. + + On a refused hold, `python3 lib/agentrc/skills/hil/scripts/hil_lock.py status` (no board argument; it prints every holder) gives the holder to report. A `role: cleanup-holder` holder is this contract's exit 4 (Recovery); any other `hil_test.py` holder is a battery run (`hil` Board locks: wait within the budget). +- **Cleanup state** for a hw-validator or hw-debugger dispatch: `restore --under-hold` leaves the board on the record's TinyUF2 with `apps/blinky` copied through it, and its `cleanup.evidence` (DFU left after the copy and stayed away) is the observation that Cleanup records. Blinky running is not checked. Whoever uses the board next flashes it first; tinyusb's HIL does. +- **Lock-wait budget:** htpc has no CI runner, so it is the dev-bench row of chief's budget unless the dispatch names another; ci.lan is the CI-runner row. `run --lock-wait ` keeps trying a held board that long, never a cleanup holder; the CI job passes its own (Execution, CI). + +## Results + +The last stdout line is JSON, also written to its `reportPath`. Its fields: +- `pass`, `caveat`; +- per board: `ran`, `pass`, `locked`, `recoveryBlocker` (a `locked` row whose holder is a cleanup holder), `lockBypassed`, `probeUsers` (pids left on the board's probe, refused after the lock), `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`, and `receipts` when `state` is `pending`, `markErrors` when the pending mark could not be written), `jlinkRetried` (J-Link boards only: sessions run again after a crash before the target connect, each `{exit, commands}`; Limits), `route` (a run over a `tinyuf2.routes` route), `buildLocksKept` (each `{buildDir, survivors, keeper}`: a build dir whose lock a keeper holds until those children end), and `detail`; +- in an install's evidence (`steps[0]`, `cleanup.evidence`, a manual `install`): `operation`, the provider's record of the flash, `{attempts: [{kind, exit}], cleanup, receipts}`, each attempt's `kind` `exited`, `timed_out`, `interrupted` or `error` and the operation's `cleanup` `done`, `pending` or `unknown` (`hil` Flashers), never the transcripts joined (they are in the run dir's `hil.log`); on an Espressif board also `esptool` (its version), and on the `openocd_esp` route `route`, `security` (the evidence passed to the provider) and `jtag` (openocd build, USJ bus-port, session log). A failed flash's step evidence carries `operation` and `beforeTarget`; +- `cleanup.state` is `blinky copied`, `untouched`, `unknown` (touched, not restored), or `pending`: an `openocd_esp` session is not ended (Recovery, Cleanup pending), its receipt in `receipts` and in `survivors` (`{cmd, receipt}`, with `pid` and `start` when the receipt was readable; `receipt` is the provider's record itself when it could not be written), the board stays held and the build lock with a keeper; +- `error` on a refusal or an operation failure. + +| Exit | Meaning | +|---|---| +| 0 | every board passed and was cleaned up | +| 1 | a board failed or did not run; it was cleaned up or never touched | +| 2 | refused before any hardware access (usage, roster, source, build record, lock helper, lock dir) | +| 4 | cleanup failed or is pending; the board is held when `cleanup.heldAfter` (not under `HIL_NO_BOARD_LOCK`). From `install` or `restore`: a child survived, an `openocd_esp` session is not ended, or the operation failed after it began writing; keep your hold | + +A delegated run (`hil-operator`) returns `{pass, results, caveat, wedged}` and, beside them, `reportPaths`: +- `reportPaths` maps each board to its JSON's `reportPath`, which a recovery dispatch needs. +- Across several boards (Execution: one battery per board), `results` holds every board's row, `pass` is true only when every run's is, and `caveat` joins the runs' non-empty caveats. +- A launched run that ended without printing its JSON (killed, crashed) may have programmed the board and left it running a candidate. The operator writes `pass: false`, `caveat: "run ended without a report; board state unknown"` and one row per board, exactly `{board, ran: null, pass: false, locked: false, detail}`, with `detail` naming the exit status, the output tail and `python3 lib/agentrc/skills/hil/scripts/hil_lock.py status` at that moment. +- The operator runs the battery once and returns. It never waits on a lock and never retries: a `locked` row comes back as is, and its caller retries under `hil`'s Caller rules, marking a flaky pass `flaky: true`. +- A recovery dispatch runs the `restore` Recovery names for the state (`--recover-holder` for a cleanup holder, `--under-hold` after a run with no report) in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Recovery). + +### Verdicts + +Each board's row maps onto the `hil` verdicts, except a commissioning run's (`commissioning: true`, from `--try-reentry`), which is never validation. +- `pass`: `ran`, `pass` and `cleanup.pass` all true; +- `fail`: `ran` and not `pass`, with a failing step, `cleanup.state` other than `untouched`, and that step's `detail`; +- `not-run`: `ran` false (`locked`, a refusal, or a failed probe check: not exactly one tty with the route's serial), or a failing step with `cleanup.state` `untouched`, with its `detail`. Nothing was written, so the failure is the rig's: no probe with that serial on this machine, or one unplugged or dead; +- a blocker, never a `fail` row: exit 4, `ran: null` (no report), a `locked` row with `recoveryBlocker`, a `probeUsers` row, and an abort, which is `ran` and not `pass` with no failing step (an interruption or an unexpected error, named in the board's `detail`). + +`testedHead` is the build record's `input.head`. `regression` is `unknown` unless the battery also ran on the merge base. + +## Recovery + +- A board whose run ended without a report gets a manual `restore --under-hold` (a recovery dispatch carrying the board and a record, as for exit 4) before its next run, and only once no such process is left on the host. Killing the runner frees its lock but not its children, which `restore` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: + + `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'; pgrep -af '[e]sp_session\.py'; pgrep -af '[u]sb_dfu\.py'` + + The `hil_test.py` check also covers the runner's `fat-read` subcommand. While a process is left, or this cannot be established, the blocker stays. + +Exit 4, a cleanup holder: + +- The board stays held by a detached holder forked from the run: reason `hil_test.py`, `role: cleanup-holder`, with its pid in `cleanup.holder`. It ignores SIGTERM and SIGINT, so a timeout kill cannot free the board; only recovery releases it. It re-executes itself without `RUNNER_TRACKING_ID`, the variable under which a GitHub Actions runner kills every process still alive when its job ends; a holder whose re-execution failed, or the in-process holder below, keeps it and dies with the job: the board is then unheld and not cleaned up, which `hil_ci.py`'s marker records (Execution, CI), and recovery is a `restore --under-hold` under a fresh hold, as for a run without a report. +- If no holder could be forked, `cleanup.holder.inProcess` is set and the runner itself is the holder: it prints the report, closes its stdio and does not exit until recovered, so no completion notice arrives. When a run's notice is overdue, read the task's output: a final JSON line with `cleanup.holder.inProcess` true is this case. Do not wait for its exit. +- `cleanup.survivors` lists children that outlived SIGKILL (a J-Link, mtools or FAT read stuck on the device), each by pid and start time, and any `openocd_esp` session not ended (cleanup pending), with its receipt; recovery refuses while any child is still alive or any session is not ended. A session's receipt is reread on every check, never remembered: an unreadable one blocks until it reads `ended`. Each child runs in a session of its own and the kill reaches the whole session. No runner creates a container any more; a report from an older runner may still name one (`{"container": "tinyuf2-hil-"}` or `...-query`), which recovery counts as gone only once docker has listed it no more for 2 seconds, `docker rm -f` before each look, and never while a process of that report lives. +- Cleanup pending: the runner never signals an openocd-esp32 server (a killed one can leave the chip in its Breakpoint boot loop), so a pending receipt is the lead's to resolve, never the recovery dispatch's. The dispatch reports the receipt and stops. A session's state, read from its receipt on every check, is `ended` (closed with cleanup `done` by the provider's end), `running` (open, its process alive by pid and start time), `unreadable`, or `unresolved` (closed without that end, or its process gone without closing it). The build lock's keeper exits once none is `running` or `unreadable`; everything else waits for `ended`. The lead runs `python3 lib/agentrc/skills/hil/scripts/hil_openocd_esp.py end --receipt [--timeout S]`, which ends the session only after proving its last command ran and its init was `ok`, by `reset run` then `shutdown`; never SIGTERM or SIGKILL the server. Exit 0 (`ended` or `alreadyEnded`) leaves the receipt closed with cleanup `done`, which discharges it from `survivors`, the keeper and the pending mark; then the recovery `restore` below runs, with the route's `--route`, and releases the holder, or, after a manual `install`, `restore --under-hold` and `hil_lock.py release` (Manual use). Exit 1 (refused or failed) keeps the receipt and the board held; 2 is a usage error. A receipt whose init failed or is unproven, one closed without the reviewed end, one whose process is gone without closing it, and a session whose receipt the provider could not write (a record with no path in `survivors`) need a separately reviewed recovery, the lead's or the human's to decide; none is defined here. A pending mark that could not be written (`markErrors`, in the row's `cleanup` or the manual report, exit 4) leaves the session unrecorded for later invocations: the board stays a blocker, held, until the lead ends or recovers it as above. +- Pending mark: each session not ended is also recorded beside the board's lock file, as `.tinyuf2-openocd-pending.json` in `hil_lock.py`'s lock dir, so any later `run`, `install` or `restore`, whatever its run dir, is refused (exit 2) until every listed receipt is `ended`; the runner then removes the mark. Every read, write and removal of it holds an flock on `.mutex` beside it, and every file of it is opened through `hil_lock.py`'s own checked dir and file opens (the dir created on first use; the mutex 0666 and the mark 0644 whatever the umask). A mark that cannot be parsed, or lists a malformed session, is unreadable: it is kept as it is, refuses every later invocation, and a run that meets it reports `markErrors` and still hands off. To remove it without a run: `python3 .claude/skills/hil-contract/scripts/hil_test.py clear-pending -b ` removes it only when every listed receipt reads `ended` and otherwise refuses, naming them (exit 2); it needs no board hold, the mutex suffices. After a separately reviewed recovery that ended the sessions some other way, `clear-pending -b --after-reviewed-recovery "" (--under-hold "" | --recover-holder ) ` proves the board's existing manual or cleanup hold as `restore` does, never taking the board's flock, holds the board's recovery lock, appends the note, the hold's record and the mark's content to `.cleared.jsonl`, then removes the mark (exit 0); without that proof it refuses (exit 2). Never remove the mark any other way. +- The holder may stand for a stuck J-Link or mtools process; tinyusb's reflash does not make the board safe, and it is out of service until recovered. +- Recovery is a separate `hil-operator` dispatch under `hil`'s Board locks scope, carrying the board, its holder pid, the failed run's `reportPath` and the release. After process, probe and host readiness are established, it runs: + + `python3 .claude/skills/hil-contract/scripts/hil_test.py restore -b --build-record --recover-holder --report ` + + `` is the failed report's `firmware.buildRecord`, or a record rebuilt on the clean HEAD when it no longer verifies. The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, restores the board while the lock stays held, and only then releases the holder. On an Espressif board the restore checks the preserved regions against the snapshot in the failed report, and recovery is refused for a touched run without one or one whose regions differ from the checkout's. A candidate whose drive cannot take the blinky copy fails the restore; the board stays held until a record that can restore it is given. A second recovery of the same board while one runs is refused (exit 2). +- If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide), and on imxrt1064_evk the same switch (SW7, MIMXRT1064-EVK schematic sheet 16), on the ci.lan rig, where tinyusb's roster also gives the board an OpenOCD recovery flasher; on espressif_esp32s3_devkitc_1, ROM download mode: hold Boot (SW1, GPIO0 low), press and release RST (SW2, CHIP_PU), then release Boot (DevKitC-1 schematic V1.0; the TRM's strap table also needs GPIO46 low, its default pull-down); on espressif_p4_function_ev, the same with BOOT (SW2, GPIO35 low) and RST (SW1, CHIP_PU) (Function-EV-Board schematic rev 1.4); on stm32h563_nucleo, a power cycle: unplug and replug its STLINK USB (CN1), which powers the board (bench-history.md). + +## Limits + +Measurements and dated bench observations behind these limits, loader footprints included, live in `bench-history.md` beside this page. + +- Covered: imxrt1060_evk, metro_m7_1011, stm32f303disco, stm32f411ve_discovery, stm32h563_nucleo, espressif_esp32s3_devkitc_1 and espressif_p4_function_ev on htpc; imxrt1064_evk on ci.lan. Not covered: ports without source facts in the runner, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69), espressif chips other than the ESP32-S3 and ESP32-P4, self-update, an interrupted update, malformed UF2 input, the physical double tap, and TinyUF2's write protection on the STM32 boards: a battery runs F303 unprotected after any option-byte reload until a power-on reset, leaves F411's protection set, and the H5 sets none today; on the H5 the runner never removes protection and an install that meets it fails. +- On ci.lan, imxrt1064_evk's probe (1-1.1) and its USB (1-1.2) share one hub, the farm's cabling, where commissioning.md asks for separate hubs: a device hung mid-write there could stall the probe connect too (Shared hubs below). Its silicon revision was not read from the marking (the rig is remote); the A and B errata were both checked. Whether its FlexSPI configuration observes ERR011377 (a delay after the DLL lock before the first flash access, IMXRT1064CE_A Rev 1.3 and IMXRT1064CE_B Rev 1.1) is unchecked: pending, not a found defect. Every install leaves [0x7000C000, 0x70010000) erased, an inference from one sample (bench-history.md, imxrt1064_evk), as on the RT1060 boards. +- The `usbPower` port cycle (Hosts/rosters) runs under the board's own lock, a limitation of `hil`'s rig-wide rule, not an exemption. On the Port Authority hub (5-3) a cycle re-enumerated only its own port's device when measured; neither a VBUS drop nor an enumeration storm was measured (bench-history.md: Shared hubs). On a ganged hub (5-1, 5-2) a port cycle removes no power while another port in the gang is powered, and can power the whole gang off when it is the last, so `usbPower` must not name a port there. On the Port Authority hub's USB2514B a full- or low-speed device that detaches and re-attaches within 100 µs of SE0 may not register as a detach (USB251xB errata, module 4); stm32f303disco's missing disconnect is instead its board's fixed D+ pull-up, by design (bench-history.md: stm32f303disco). Never cycle a rig hub port with `uhubctl` or a raw hub request: a cycle that met an enumeration in progress killed an xHCI controller. Another board's enumeration errors on the same hub can reset a neighbour mid-read, and `install`'s FAT read then times out (exit 4) (bench-history.md: stm32h563_nucleo). A device hung mid-write stalled a J-Link connect on the same hub. +- Espressif boards: the runner needs esptool 5.2.0, its `esptool` command an entry script of the runner's own Python, and pyusb with libusb-1.0 for that Python. The install runs that esptool, not esptool 5.3.1 in CI's image: the dialect differs (`default_reset`, `write_flash`, flash_args inline), and the S3 and P4 UART routes are hardware-pending until their bridges are attached. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The route's tty carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. The host log shows a USB reset and block-device read errors at the end of a UF2 copy, and on the P4 a full-speed device that never answers while an esptool session holds the chip; the battery passed with the latter. +- Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. +- cmake builds pass `TUF2_LOG=0` unless `LOG` is set, as make does, so the HIL image does not log on the board UART. +- `install` from a blank board, or one whose ROM found no boot image, brings up DFU on imxrt1060_evk and metro_m7_1011; until a `loadbin` has run, a new J-Link session reads such a board's flash wrong. +- That every install on the RT boards leaves the app start erased is an inference from samples, not a measurement of every image. +- The runner runs no CMake flash target: the provider runs the commands the record's targets run, with the same files, so the targets' own execution is no longer covered end to end; the record still checks each target's command against the roster at the build, and a test keeps the provider's J-Link script equal to the target's. Of the J-Link and ST-LINK boards, imxrt1060_evk, stm32f303disco and stm32f411ve_discovery are not covered by a hardware run of this direct install. agentrc's pyOCD backend is unproven on hardware and no roster board uses it. +- `openocd_esp`: never convoy-safe (`hil` Flashers), so another 303a:1001 on the host may be opened on a transfer error. openocd itself never detects flash encryption or secure boot: the runner's esptool reads them, and the chip revision's eFuse layout is from the P4 TRMs for revisions v1.3 and v3.x (the bench P4 is v1.0). The route writes the built bytes without esptool's header rewrite. espressif_p4_function_ev's route was commissioned on 2026-10-20 (bench-history.md). +- Never use the erase targets on a rig board (`tinyuf2-erase-external-jlink` erases [FLASH_BASE, FLASH_BASE + 64 KiB), the bootloader with it), nor ESP-IDF's `combined-flash`, which has no port selection and writes nvs and both OTA slots blank. +- stm32f411ve: runner sessions connect as `Cortex-M4` (`SESSION_DEVICE`) because a `-device stm32f411ve` connect resets the chip and drops DFU, and the install's verify session waits 1 s before its halt (`INSTALL_SETTLE_MS`). +- Double-tap retention: on stm32h563_nucleo the word survives a system reset only with option SRAM1_3_RST=1 (set on this chip); on stm32f303disco and stm32f411ve_discovery its survival across SYSRESETREQ is undocumented and inferred from commissioning. +- stm32f303disco signals no disconnect when TinyUF2 comes up after a reset, so `stays_away` (the control trial's, the copy's) and `observe_app` cannot see a transient DFU entry there; its roster `usbPower` covers DFU (Hosts/rosters). +- stm32h563_nucleo: STM32CubeProgrammer behaviour the runner relies on is bench-observed on v2.23.0 only (UM2237 not in the library). +- A `JLinkExe` crash in the install is not retried (the crash retry below covers the runner's own sessions only): it fails the install, touched unless it is the crash at the probe connect below. +- `JLinkExe` sometimes crashes in `libjlinkarm` at the probe connect on htpc, cause unknown. A session that crashed this way, before the target connect, runs once more after 2 s (`jlinkRetried`); a second crash fails the run. +- No reproducer for an instrumented build: the battery and `install` take only a clean build record. A claim that needs one is reported pending. + +Commissioning a board follows `commissioning.md` beside this page. diff --git a/.claude/skills/hil-contract/bench-history.md b/.claude/skills/hil-contract/bench-history.md new file mode 100644 index 000000000..4af25e94f --- /dev/null +++ b/.claude/skills/hil-contract/bench-history.md @@ -0,0 +1,85 @@ +# HIL bench history + +Part of the HIL contract (`SKILL.md` beside this file): the measurements and dated bench observations behind its Limits. + +## RT boards + +metro_m7_1011 and imxrt1060_evk. + +`install` from a blank board, or one whose ROM found no boot image, brings up DFU (measured on imxrt1060_evk 2026-10-12 and metro_m7_1011 2026-10-13). With no boot image a new J-Link session reads the flash wrong (imxrt1060_evk: zeros; metro_m7_1011: bytes shifted) until a `loadbin` has run; a `loadbin` compare reads it right. On metro_m7_1011, J-Link then prints "Invalid flash header detected." at the install's first reset. + +On metro_m7_1011 (2026-10-13, J-Link V9.84), `tinyuf2-jlink`'s `loadfile` of the cmake HEX (0x60000400 on) reported "1 range affected (65536 bytes)" and, from a fresh session after it, 0x60000000, 0x6000BF00, 0x6000C000-0x6000C00F, 0x6000C200 and 0x6000FF00 read 0xFF while 0x60010000 kept its data. Its compare covered the whole block: over blinky it programmed, and right after a program it skipped ("Contents already match"). DFU came up within 2 s in each of these three runs. That every install by the target leaves [0x6000C000, 0x60010000), the app's first 16 KiB, erased is an inference from these samples and this HEX, not a measurement of every case. + +On imxrt1060_evk (2026-10-13), the same target over blinky also reported "1 range affected (65536 bytes)"; from a fresh session after it, blinky's vectors at 0x6000C000-0x6000C00F and 0x6000C200 read 0xFF, while the FCB at 0x60000000 and 0x60010000 kept their data. + +## imxrt1064_evk + +On imxrt1064_evk (2026-10-18, the ci.lan rig, J-Link V9.78 on an external J-Link, S/N 000725299165, with "an old firmware +module: V1"; DIGPROG 0x006C0000, the marking that decides the silicon revision not read, the rig being remote, so both +IMXRT1064CE_A Rev 1.3 and IMXRT1064CE_B Rev 1.1 were checked), `tinyuf2-jlink`'s `loadfile` of the cmake HEX +([0x70000000, 0x70007134)) reported "1 range affected (65536 bytes)": from a fresh session after it, sentinels at 0x7000BF00 +and 0x7000C000 (the app start) read 0xFF while 0x70010000 kept its data. So every install leaves [0x7000C000, 0x70010000), +the app's first 16 KiB, erased, as on the RT1060 (an inference from this sample and this HEX). The sentinels had been +programmed by `loadbin` over [0x7000BF00, 0x70010004) ("1 range affected (131072 bytes)"), which erased the FCB with block +0: a fresh session then read every flash word, the sentinels included, as 0xFF (no boot image, as on the RT1060), while +a session whose `loadbin` compare had run ("Contents already match") read them right. The install's reset after the +download printed "Could not find valid reset vector." then "Target halted on flash header read." (`BENIGN`, the RT1062 +note), and "Invalid flash header detected." at the implicit reset before the download while the FCB was erased. DFU came +up with serial BAE96FB95AFA6DBB8F00005002001200 (volume RT1064BOOT) at 1-1.2, on the same hub (1-1) as the probe at +1-1.1: the farm's cabling, which this contract's Limits records. The word at 0x20000FFC (the fixture's stack top, less +4, in DTCM) took a written value and read it back, then its saved value 0, so DTCM is enabled at the ROM default; the +FlexRAM fuse itself was not read. + +## stm32f303disco + +On stm32f303disco (2026-10-14, J-Link V9.84 on the board's J-Link OB), `tinyuf2-jlink`'s `loadfile` compared [0x08000000, 0x08004000) and reprogrammed only the 2 KiB pages that differed, erasing the rest of such a page past the image (a sentinel at 0x08003F00 read 0xFF from a fresh session after it), while 0x08004000 (the app start) and 0x08004800 kept their data. So an install keeps a valid app. DFU still came up without the fallback in the battery of 2026-10-14: the runner's verify session halted the core in TinyUF2's double-tap wait (`src/main.c`, magic armed), so its reset entered DFU as a double tap, an inference from that trace. When that reset misses the window the app boots, and the retained-register fallback brings DFU up after `dfu`'s 20 s wait. + +stm32f303disco does not signal a disconnect when TinyUF2 comes up after a reset (by design, MB1035 Rev D.1 sheet 4, book 6582, the bench board's revision not read: USB USER's D+ has a fixed 1.5 kΩ pull-up, R16, to the 3V rail the ST-LINK USB keeps powered, so D+ stays high through an MCU reset; the F303 has no internal D+ pull-up, DS9118 Rev 14 Table 66 note 2; on the bench the host kept the old device), so the host does not re-enumerate it unless it was polling the device (TinyUF2 as an MSC disk) when the reset came. Its roster `usbPower` names its USB USER port on the Port Authority hub (PA40004, port 3). + +A bench `uhubctl` cycle, a raw hub request, that met an enumeration in progress on hub 1-1 of an AMD 500-series xHCI controller killed that controller (2026-10-14: "HC died", every device on its buses lost, recovered by unbinding and binding `xhci_hcd`; the Port Authority hub needed a replug); hence the kernel's serialized `disable` and the 2 s in state `default`. On this hub `disable` 1 reads `off` from `uhubctl` (2026-10-14). + +## stm32f303vc + +On stm32f303vc, TinyUF2 write-protects its pages 0-7 at boot through the option bytes unless RCC_CSR's OBLRSTF is set, and it never clears that flag, so after any option-byte reload it skips protection until a power-on reset. A J-Link connect to a chip with that protection set prints the `BENIGN` unlock note and, read from a fresh `-device Cortex-M4` session after it, leaves the option bytes at RDP 0xAA with no WRP and flash intact (three samples, 2026-10-14). Its double-tap word is the last SRAM word, 0x20009FFC; that it survives SYSRESETREQ is undocumented (RM0316 Rev 10, DS9118, PM0214, ES0204) and inferred from the commissioning run (2026-10-14: the reset without the magic booted the app, the one with it entered DFU and consumed it). + +## stm32f411ve_discovery + +On stm32f411ve_discovery (2026-10-15, J-Link V9.84 on the board's J-Link OB, rev A silicon: DBGMCU REV_ID 0x1000, ES0287 Rev 6), `tinyuf2-jlink`'s `loadfile` of the HEX ([0x08000000, 0x08003E18)) reported "1 range affected (16384 bytes)": from a fresh session after it, a sentinel at 0x08003F00 read 0xFF while sentinels at 0x08004000, 0x08007C00, 0x08008000, 0x0800C000, 0x08010000 (the app start) and 0x08010800 kept their data. So an install keeps a valid app. + +TinyUF2 write-protects sectors 0-1 at every boot (`src/main.c`; on the F4 the option bytes take effect when OPTSTRT completes, with no reset, RM0383 Rev 4 3.6.2). A `-device stm32f411ve` connect to a chip with that protection prints the unlock note (J-Link transcript shape) and removes it by resetting the chip: with RCC_CSR cleared and TinyUF2 booted, the next such connect read RCC_CSR 0x14000000 (SFTRSTF, PINRSTF) and nWRP 0xFF, while a `-device Cortex-M4` connect read RCC_CSR 0 and nWRP 0xFC, the core running the app. A runner session that reset the chip that way after a retained-register re-entry booted the app and dropped DFU; the unlock is not in UM08001 Rev 9.78. Right after the flash target's unlock, a `Cortex-M4` session's halt failed ("CPU could not be halted", flash read back wrong) with no delay (three runs) and succeeded after a 200, 400, 600, 800 or 1500 ms `sleep`; TinyUF2's boot rewriting the option bytes is the inferred cause, and no rewrite time is documented. + +Its double-tap word is 0x2000FFFC, the last word of the boot script's 64 KiB RAM; that it survives SYSRESETREQ is undocumented (RM0383 Rev 4 6.1.1) and inferred from the commissioning run (the reset without the magic booted the app, the one with it entered DFU and consumed it). DFU re-enumerated after each tested reset into DFU without port power control (OTG FS soft disconnect). + +## stm32h563_nucleo + +On stm32h563_nucleo (2026-10-15, STM32CubeProgrammer v2.23.0 on the board's STLINK-V3, STM32H563ZI rev Z), `tinyuf2-stlink` erased exactly the 8 KiB sectors holding HEX data (sectors 0 and 1 for a 15 KiB image): from a fresh session after it, a sentinel at 0x08003F00 read 0xFF while 0x08004000 and 0x08008000 (the app start) kept theirs. So an install keeps a valid app. + +TinyUF2 sets no write protection on the H5 today: WRPSGn1 stayed 0xFFFFFFFF after it booted, since its boot-time call reads the option bytes with no bank selected. The H5's D+ pull-up is internal (RM0481 Rev 5 USB_BCDR.DPPU_DPD, reset value 0), so a reset shows the host a disconnect; its hub, 5-2 on htpc, has ganged power. The double-tap word is SRAM3's last, 0x2009FFFC, which a system reset keeps with option SRAM1_3_RST=1 (RM0481 Table 43; set on this chip). + +The H5's debug AP stops answering after a killed ETM trace session until a power cycle (agentrc `etm-trace` notes; seen again 2026-10-15 before commissioning). Another board's run on the same hub can reset this one: on 2026-10-15 a P4 battery's ROM-mode enumeration errors on 5-2.3 made the hub power-cycle that port and reset 5-2.7 mid-read, and `install`'s FAT read timed out (exit 4). + +## J-Link + +J-Link's QSPI loader is undocumented (SEGGER UM08001 has nothing on it). Historically, at sampled sentinel points on both RT boards, the runner's former `loadbin` install (no longer issued) lost the rest of its 64 KiB block past the image while a sentinel at 0x60010000, in the next block, survived; a sample, not a guarantee for every image. + +`JLinkExe` sometimes crashes in `libjlinkarm` at the probe connect on htpc (SIGSEGV; once a glibc double-free abort); the cause is unknown. Seen in 2 of 94 runner sessions on 2026-10-12. + +## Shared hubs + +A device hung mid-write makes the host controller retry its enumeration, which stalled a J-Link connect on the same hub for ~80 s (lpcxpresso55s69); that the hub is the cause is a hypothesis. + +Port cycle reach (enumeration-reach evidence only: physical power removal on 5-3 was not measured), measured 2026-10-17 on htpc (kernel 6.12) on hub 5-3 (Adafruit Port Authority PA40004, Microchip USB2514B, `uhubctl` reports `ppps`), under a hold of every board on it (metro_m7_1011, espressif_p4_function_ev, bench_esp32s3 and espressif_esp32s3_devkitc_1): writing `disable` 1 then 0 on an empty port (5-3.1) produced no kernel log line and no device-number change on 5-3.2 or 5-3.4; the same on the occupied port 5-3.4 (the P4's USB-Serial-JTAG unit) disconnected and re-enumerated only that device (device number 69 to 14) while 5-3.2 kept device number 73. Not measured: whether VBUS on the cycled port actually dropped (no meter or bus-powered indicator was used, and a hub with no power switch would show the same disconnect and re-enumeration), an enumeration storm (no safe way to manufacture one), and the ganged hubs 5-1 and 5-2 (Terminus FE 2.1, 1a40:0201, `uhubctl` reports `ganged`), whose other devices are not rig boards. + +Documented (read-doc, 2026-10-17): USB Specification 2.0 (book 775), section 11.11, p363: with per-port power switching a port's power is off in the Powered-off state; with ganged switching "the power to a gang is not turned off unless all ports in a gang are in the Powered-off or Not Configured states", so a `usbPower` cycle of one port on a ganged hub removes no power while another port in the gang is powered, and can switch the whole gang off when it is the last. Section 11.11 also lets a hub with no power switches report per-port switching, and 11.11.1 requires a hub with several gangs to report it, so `ppps` alone does not prove a port's power is switched on its own. Microchip DS00001692 USB251xB data sheet (book 6533), p19 and p22: individual port power switching is the default strap (`PORT_PWR`), and the ability to switch per port is "hardware implementation dependent" (p22). Microchip DS80000627D USB251xB errata (book 6531), module 4, p4: a full-/low-speed device that self-detaches and re-attaches within 100 µs of SE0 may not be seen as a detach, and "may result in a failure to establish a connection after re-attach"; workaround 2 is a port reset. No Terminus FE 2.1 document is in the library (searches FE2.1, Terminus, 1a40: no match). The kernel's `disable` attribute (drivers/usb/core/port.c `disable_store`, v6.12.48) clears PORT_POWER through `usb_hub_set_port_power`, which sends ClearPortFeature(PORT_POWER) for that port only. + +Board design (read-pcb, 2026-10-18): the Port Authority is inferred to be Adafruit's Smart HIL Hub (adafruit/MBAdafruitBoards `Development/HIL Bench/Smart HIL Hub`, @ a2003e8687c1), the only four-port USB2514B design in that repository; its strings and configuration live in its EEPROM (U14), which was not read, and its revision is not known. Revisions A, B and C wire it the same way: each `PRTPWRn` drives the EN pin of its own Diodes AP22653 load switch (U2, U6, U8, U10 for ports 1-4), and each switch's output feeds only its own port: USB-A VBUS, JST-XH header, 100 µF, and a green LED to ground through 2.2 kΩ, so no port's power is ganged. The PRTPWR nets carry no battery-charging strap, but the EEPROM can still enable battery charging (USB251xB data sheet, book 6533, p18-19). Documented (read-doc): the hub asserts PRTPWR to enable VBUS through the switch's EN (book 6533, p17, §4.1.1); the AP22653's enable is active-high, and a disabled switch discharges its output through 600 Ω typical "for limited time", a duration the data sheet does not give (Diodes DS41186 Rev 5-2, book 6580, p1, p5 Note 9, p11). If that discharge lasts, an unloaded port's 100 µF falls with τ ≈ 60 ms, below 1 % of 5 V in about 0.3 s, inside the runner's 1 s off (`PortPower.OFF`); if it stops early, the LED branch drains the port only to about the LED's forward voltage. This is a design estimate, not a measurement. It does not cover a self-powered board that back-drives VBUS. stm32f303disco, the one `usbPower` board (port 3), is powered from its ST-LINK USB, but by design its USB USER VBUS goes only to two series BAT60J Schottky diodes pointing into the board's 5V (D4, D5, MB1035 Rev D.1 sheet 4, book 6582), so it does not back-drive the port beyond their reverse leakage; the bench board's revision was not read, and a cycle of port 3 itself was never measured. The INA3221 (IC2) senses ports 1-3's switched VBUS and is reachable only through the hub's STEMMA QT I2C header, so an I2C adapter there could measure the drop on port 3; that has not been tried. + +## espressif_p4_function_ev JTAG route + +Commissioned 2026-10-20 on htpc (ESP32-P4 rev v1.0, openocd-esp32 v0.12.0-esp32-20260703, agentrc 4f6a028). Documented (read-doc): the USB Serial/JTAG controller is a full-speed CDC-ACM and JTAG device on an internal PHY (ESP32-P4 TRM Rev 0.7, book 6567, p3187-3188); `EFUSE_USB_PHY_SEL` 0 gives it the GPIO24/25 PHY and OTG11 the GPIO26/27 one (p942; the bench chip's value was not read). RTS=1 with DTR=0 on its CDC resets the SoC (Table 54.3-2, p3190), a "USB (UART) reset", classified as an HP Core Reset (Table 11.1-1, p1082-1083); the TRM does not say whether the controller's USB connection survives it, and Deep-sleep powers it down (p3195). The errata (book 6568 v1.4) list nothing for it. Measured: the USJ (5-3.4) kept its device number through esptool hard resets, polled every 2 ms for 3 s after three sessions, and stayed available through two battery runs over the route (`run --route openocd_esp`, 04:30 and 04:33), whose esptool sessions follow the install; then `install` and `restore` under a hold passed, as did three restores before them. Each provider session logged no failed event and only "Reset cause (24)". + +The runner waits one second after its esptool session's hard reset before the provider starts. Without it (openocd started ~4 ms after the session ended), 6 of 6 route restores failed every `reset init` of the session: `reset-assert-post` could not write 0x500c2064 (sysbus access failed, target not halted) and the reset cause read 3, software system reset. The provider stopped before writing each time. Plain openocd sessions after the same esptool session passed 17 of 17 (driven by `-c` or over Tcl, the provider's argv and wrapper, 0.15-2 s gaps); the provider called ~150 ms after it passed 4 of 4 and called immediately failed 2 of 2. The cause is unknown (agentrc #160). + +## esptool boards + +At the end of a UF2 copy TinyUF2 restarts while the host's usb-storage is still talking to it: the host log shows a reset of that device and read errors on its block device. Seen on the S3 and the P4; other ports are unchecked. On the P4, while an esptool session holds the chip, the host log shows a full-speed device on TinyUF2's USB path that never answers its descriptor request (htpc 2026-10-14); the battery passed with it. diff --git a/.claude/skills/hil-contract/commissioning.md b/.claude/skills/hil-contract/commissioning.md new file mode 100644 index 000000000..b0f96ca1a --- /dev/null +++ b/.claude/skills/hil-contract/commissioning.md @@ -0,0 +1,40 @@ +# Commissioning a TinyUF2 rig board + +Part of the HIL contract (`SKILL.md` beside this file). + +Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under a manual hold, except for the re-entry run, which locks itself. Record each result in the board's commissioning note (agentrc `target-debug` project notes), and `uid`, `lock` and `reentry` in the roster. In this order: +- **Cabling:** cable the USB port TinyUF2 enumerates on (`BOARD_TUD_RHPORT`). It can differ from the one tinyusb examples use: USB OTG1 on imxrt1060_evk; USB USER (CN3) on stm32f303disco; the USB OTG FS Micro-AB connector, not the ST-LINK one, on stm32f411ve_discovery (UM1842). A DFU device "not seen" with no new device in `dmesg` is a cabling problem first. Keep the probe on a hub apart from the target's (SKILL.md Limits). +- **Silicon revision,** before any other result: read it from the chip marking or registers, and check that revision's errata through read-doc. RT1060's DIGPROG does not tell A from B, so the marking decides. +- **Port power** (a board with `usbPower`): before setting `usbPower`, check that `uhubctl` reports the hub's power switching as `ppps`, not `ganged`; a ganged hub keeps a gang's power on while another of its ports is powered and can switch the whole gang off with its last port (USB 2.0 11.11). `ppps` is a preliminary check only, since a hub with several gangs or no switches may report it too (USB 2.0 11.11, 11.11.1; the USB251xB's switching depends on its board, DS00001692 p22). The hub's port `disable` files writable by the runner's user, through `/etc/udev/rules.d/52-tinyuf2-hil-uhubctl.rules`, one line per hub: on `ACTION=="add|bind"` of its `DRIVER=="hub"` interface, matched by the hub's VID, PID and serial, `RUN` `chgrp plugdev` and `chmod g+w` on `/sys$devpath/*-port*/disable`. The runner refuses the board while that file is not writable. Check once that writing `disable` 1 switches the port off (`uhubctl -l -p ` reads `off`), then write 0. Under a hold of every board on the hub, show before `usbPower` is relied on that VBUS on that port actually drops during the cycle, by evidence independent of the hub's own report (a meter on the port, or a power indicator on its switched VBUS going out, such as a bus-powered device's or one the hub's confirmed schematic places there), and that the cycle reaches no other port: the board on that port disconnects and re-enumerates with a new device number, and every other device on the hub keeps its device number (as measured on 5-3, bench-history.md: Shared hubs). `uhubctl` reading `off`, a disconnect and a new device number show only the port's logical state and a re-enumeration, which a hub with no power switch also gives (USB 2.0 11.11; the kernel's `disable` disconnects the device before it clears PORT_POWER). +- **Automount exclusion:** `/etc/udev/rules.d/99-tinyuf2-hil.rules`, one line per board, with the VID and PID from the board's source, in exactly this form (the runner accepts only this form, with exact `==` matches): + + ``` + SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ATTRS{serial}=="", ENV{UDISKS_IGNORE}="1" + ``` + + Write it with the roster's `uid`, before the serial can be confirmed: `install` and `run` refuse without the rule, and a fresh board runs whatever tinyusb last flashed, not TinyUF2. A serial a running tinyusb example reports (`lsusb -v`) is an early cross-check, not a confirmation. Until the serial is confirmed, also inhibit automount for every board of this VID:PID with a temporary `/etc/udev/rules.d/98-tinyuf2-hil-commissioning.rules`: + + ``` + SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ENV{UDISKS_IGNORE}="1" + ``` + + Then `sudo udevadm control --reload`. +- **Bootloader:** `install` a clean HIL record of HEAD under the hold and observe DFU (add `--try-reentry retained-register` if a valid app could keep DFU away). +- **USB serial:** confirm the enumerated serial equals `uid` (sysfs or `lsusb -v`), and record the USB port paths. If it differs, fix `uid` and the rule together, reload, and `install` again; its reset re-enumerates the board and udev applies the reloaded rules, so no replug should be needed (by mechanism, not yet observed on a uid fix). +- **Cleanup:** `restore` under the hold and observe DFU leave after the blinky copy. Record the reset notes J-Link prints on install (`BENIGN`). +- **Exclusion proof:** delete the temporary rule and reload before the re-entry run below; its `dfu` step passes only when `udevadm info` on the enumerated block device shows `UDISKS_IGNORE=1` (`check_exclusive`). A host rule that ignores a whole VID, such as tinyusb's `88-tinyusb.rules` for 239a, sets the same property, so the proof cannot tell it from the serial rule; the runner still requires the serial line. +- **Loader footprint:** for the flash target's `loadfile`, place sentinels just past the written range and elsewhere in its erase block. Confirm them from a fresh session before the operation under test, and again from another fresh session after it: a verify in the session that wrote them cannot show what a later operation destroys. Make the written data differ from what is on the flash, since a download whose data already matches is skipped ("Contents already match") and erases nothing. Record the result in bench-history.md. +- **Fixture prerequisites,** before anything executes the battery's fixture: + - the app start meets VTOR's alignment for the fixture's vector table (`fixture()`); + - its stack top, the `m_data` origin the runner reads (`ram_origin`) + 0x1000, is in RAM enabled when the bootloader hands off: the ROM default per read-doc, which TinyUF2 does not reconfigure (DTCM on imxrt1060_evk); + - under a hold, a word just below that stack top takes a written value and reads it back, and then its saved value (0x20000FFC on imxrt1060_evk). +- **Re-entry:** commit the roster entry without `reentry`, build the record, release the manual hold, and run the battery in commissioning mode; it locks itself and ends with the cleanup, so it also proves the restore: + + ```bash + python3 .claude/skills/hil-contract/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json --try-reentry retained-register test/hil/local.json + ``` + + `--try-reentry` is the lead's only, and is refused once `reentry` is set. The row's `reentryDecision` names the method when the run passed or failed only at `reenter`'s `CURRENT.UF2` readback, every observation of the tried method is true (`controlBootedApp`, `dfuCameUp` and `magicConsumed` for `retained-register`), and the board was cleaned up and not held. Set `reentry` to it and commit. A null decision leaves `reentry` unset: file a `followup` issue with the observations for diagnosis, not as a diagnosed bug. When the drive copy itself fails, a fixture installed by hand with J-Link `loadbin` can isolate re-entry from the copy for diagnosis; it never sets `reentry`. +- **On an esptool board,** in place of the J-Link items: cable the native USB connector as well as the bridge's; check once by hand that the chip has no secure boot, secure download or flash encryption and that its flash holds the candidate's layout (`esptool get-security-info`, `esptool flash-id`), since the runner never checks them; record the bridge serial and the chip's MAC (`uid`, TinyUF2's USB serial); `install` from the record and observe DFU, then `restore` and observe DFU leave. Re-entry is `--try-reentry dfu-detach`, whose decision needs `detachAcked`, `appLeft` and `dfuCameUp`; before setting it, repeat DFU_DETACH ten times under the hold and check the host log shows no USB reset on the DETACH transitions (USB DFU 1.1 bitWillDetach). +- **An `openocd_esp` route** (Hosts/rosters) stays `"commissioning": "pending"` until the lead has checked, with read-doc, that the chip has a USJ, how it is routed against OTG and what its resets do, and has measured, with the battery's exact firmware and operations, that the USJ stays available, then run the route's `install` and `restore` under a hold. Clear the key and commit. +- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid; on MIMXRT1011DAE5A the same note on every reset of a TinyUF2 image; on stm32f303vc, "Active write protection detected.", "Device will be unsecured now.", "Failed to unlock device." and "Can not attach to CPU. Trying connect under reset.", the connect that removes TinyUF2's write protection (SKILL.md Limits). On stm32f411ve the same connect prints "Active write protection detected.", "Device will be unsecured now." and "Device unlocked successfully.", which match no failure pattern and need no exemption; only the flash target connects as that device (SKILL.md Limits). A reset session (`w4` to AIRCR) passes only once J-Link reports the write issued ("Writing VALUE -> ADDR") with no failure before it. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. diff --git a/.claude/skills/hil/scripts/esp_session.py b/.claude/skills/hil-contract/scripts/esp_session.py similarity index 59% rename from .claude/skills/hil/scripts/esp_session.py rename to .claude/skills/hil-contract/scripts/esp_session.py index a4e1ff019..8bafc2826 100644 --- a/.claude/skills/hil/scripts/esp_session.py +++ b/.claude/skills/hil-contract/scripts/esp_session.py @@ -5,7 +5,8 @@ esp_session.py --port DEV --chip CHIP --baud N --after hard-reset|no-reset OPS.json OPS.json is a list of {"op": "read", "addr", "size", "file"} and {"op": "md5", "addr", "size"}: no write, which -is the build's flash target's. +is the provider's flash. A {"op": "security"} first reads the chip's flash encryption and secure boot state on the +ROM loader, before the stub runs. The last stdout line is JSON: {"ok": true, "results": [...]} or {"ok": false, "error", "results"}. """ import argparse @@ -15,6 +16,24 @@ sys.dont_write_bytecode = True +def security(esp): + """The chip's flash encryption and secure boot state from two sources, both on the ROM loader: esptool 5.2.0's + GET_SECURITY_INFO (loader.py get_security_info: flags bit 0 SECURE_BOOT_EN, flash_crypt_cnt enabled when its count + of 1 bits is odd) and its eFuse register reads (targets/*.py get_flash_encryption_enabled, get_secure_boot_enabled; + on the P4 EFUSE_RD_REPEAT_DATA1_REG [20:18] SPI_BOOT_CRYPT_CNT and EFUSE_RD_REPEAT_DATA2_REG [20] SECURE_BOOT_EN).""" + if esp.IS_STUB: + raise RuntimeError('the chip already runs the flasher stub; its security state is read from the ROM loader') + si = esp.get_security_info(cache=False) + + def state(on): + return 'on' if on else 'off' + return {'flags': si['flags'], 'flashCryptCnt': si['flash_crypt_cnt'], + 'rom': {'flashEncryption': state(bin(si['flash_crypt_cnt']).count('1') % 2), + 'secureBoot': state(si['parsed_flags']['SECURE_BOOT_EN'])}, + 'efuse': {'flashEncryption': state(esp.get_flash_encryption_enabled()), + 'secureBoot': state(esp.get_secure_boot_enabled())}} + + def run(esp, op): kind = op['op'] if kind == 'read': @@ -40,9 +59,14 @@ def main(): import esptool from esptool import cmds ops = json.load(open(args.ops)) + if any(op['op'] == 'security' for op in ops[1:]): + raise ValueError('a security op comes first, before the stub runs') esp = cmds.detect_chip(args.port, connect_mode='default-reset') if esp.CHIP_NAME.lower().replace('-', '') != args.chip: raise RuntimeError(f'the port reaches a {esp.CHIP_NAME}, not an {args.chip}') + if ops and ops[0]['op'] == 'security': + results.append(dict(security(esp), op='security')) + ops = ops[1:] esp = cmds.run_stub(esp) esp.change_baud(args.baud) cmds.attach_flash(esp) diff --git a/.claude/skills/hil-contract/scripts/hil_ci.py b/.claude/skills/hil-contract/scripts/hil_ci.py new file mode 100644 index 000000000..d842035fd --- /dev/null +++ b/.claude/skills/hil-contract/scripts/hil_ci.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""The CI job's HIL run on this host's roster: one battery per named board from a record built here, every +run kept, a `hil` verdict per board and one exit for the job. + + hil_ci.py --board B [B ...] --roster test/hil/ci.json --out DIR [--lock-wait S] [--fail-at STEP] + +`--out` is this run's directory under a state directory kept across runs (`/`): the build +record, each board's run directory (`hil_test.py run --run-root`: its report, hil.log and output) and the +`.started` / `.done` markers go there. A `.started` without its `.done` in any earlier run is a +run that ended without a report or kept a cleanup holder (SKILL.md Recovery): that board is a blocker here +until recovery removes the marker. Runs older than PRUNE_DAYS with every marker done are removed; one with a +marker left is kept, whatever its age. + +Exit 0: every board passed. 1: a board failed or was not run (held past `--lock-wait`, its build or admission +refused). 2: a blocker (a run without a report, a failed cleanup, a cleanup holder or a stale probe session on +the board). +""" +import argparse +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +CHECK_BUILD = ROOT / '.claude' / 'skills' / 'build' / 'scripts' / 'check_build.py' +HIL_TEST = Path(__file__).resolve().with_name('hil_test.py') +STEPS = ('install', 'dfu', 'copy', 'app-ran', 'reenter', 'recopy', 'cleanup') # hil_test.py --fail-at +LOCK_WAIT = 600 +PRUNE_DAYS = 30 + + +def run(cmd): + return subprocess.run([str(c) for c in cmd], cwd=ROOT, capture_output=True, text=True) + + +def roster_boards(roster): + try: + return sorted(b['name'] for b in json.loads((ROOT / roster).read_text())['boards']) + except (OSError, ValueError, KeyError, TypeError) as e: + raise SystemExit(f'cannot read boards from {roster}: {e}') + + +def unfinished(state, board): + """Earlier runs under `state` whose battery on `board` started and never finished.""" + return sorted(p.parent.name for p in state.glob(f'*/{board}.started') if not p.with_suffix('.done').exists()) + + +def prune(state, keep, days=PRUNE_DAYS): + """Remove the runs under `state` older than `days` whose every started board finished; never `keep`.""" + cutoff = time.time() - days * 86400 + for d in state.iterdir(): + if d.is_dir() and d != keep and d.stat().st_mtime < cutoff and \ + all(p.with_suffix('.done').exists() for p in d.glob('*.started')): + shutil.rmtree(d, ignore_errors=True) + + +def build_record(board, roster, out): + record = out / f'{board}.build.json' + r = run([sys.executable, CHECK_BUILD, '--board', board, '--backend', 'cmake', '--shared', '--hil-config', roster]) + record.write_text(r.stdout) + try: + rec = json.loads(r.stdout) + row = next(b for b in rec['boards'] if b['board'] == board) + ok = rec['pass'] and row['status'] == 'ok' + why = '' if ok else f'build {row["status"]}: {row.get("firstError", "")}'[:400] + except (ValueError, KeyError, StopIteration): + ok, why = False, f'build record unreadable (exit {r.returncode}): {(r.stderr or r.stdout).strip()[-300:]}' + return record, ok, why + + +def battery(board, roster, record, where, lock_wait, fail_at=None): + """One `hil_test.py run` with its run directory under `where`, its output streamed to `where/stdout.txt` as + it comes, so a killed run keeps what it printed; its last JSON line is the report (SKILL.md Results). A runner + left holding the board in process (`cleanup.holder.inProcess`, Recovery) never exits: it is not waited for. + Returns (exit or None, report or None).""" + where.mkdir(parents=True) + cmd = [sys.executable, HIL_TEST, 'run', '-b', board, '--build-record', record, '--run-root', where, + '--lock-wait', lock_wait] + if fail_at: + cmd += ['--fail-at', fail_at] + report = None + with open(where / 'stderr.txt', 'w') as err, open(where / 'stdout.txt', 'w') as log: + p = subprocess.Popen([str(c) for c in cmd + [roster]], cwd=ROOT, stdout=subprocess.PIPE, stderr=err, text=True) + for line in p.stdout: + log.write(line) + log.flush() + if line.startswith('{'): + try: + report = json.loads(line) + except ValueError: + pass + p.stdout.close() + row = next((r for r in (report or {}).get('results', []) if r.get('board') == board), {}) + in_process = ((row.get('cleanup') or {}).get('holder') or {}).get('inProcess') + return (None if in_process else p.wait()), report + + +def verdict(code, report, board): + """The `hil` verdict of one run from the report's row (SKILL.md Verdicts): (verdict, detail).""" + if report is None: + return 'blocker', f'run ended without a report (exit {code})' + row = next((r for r in report.get('results', []) if r.get('board') == board), None) + if row is None: + return 'blocker', f'report has no row for {board} (exit {code}): {report.get("caveat") or report.get("error")}' + cleanup = row.get('cleanup') or {} + if code in (4, None) or row.get('ran') is None or cleanup.get('heldAfter'): + return 'blocker', f'cleanup failed or no report; board held: {row.get("detail")} {cleanup}'[:400] + if row.get('probeUsers'): + return 'blocker', str(row.get('detail')) + if row.get('locked'): + return ('blocker' if row.get('recoveryBlocker') else 'not-run'), f'locked: {row.get("detail")}' + if not row.get('ran'): + return 'not-run', str(row.get('detail') or report.get('error')) + failed = next((s for s in row.get('steps', []) if not s.get('pass')), None) + if row.get('pass') and cleanup.get('pass'): + return 'pass', '' + if failed is None: + return 'blocker', f'aborted: {row.get("detail")}' + if cleanup.get('state') == 'untouched': + return 'not-run', f'{failed["step"]}: {failed.get("detail")} (nothing written)' + return 'fail', f'{failed["step"]}: {failed.get("detail")}' + + +def run_board(board, roster, out, state, lock_wait, fail_at=None): + earlier = unfinished(state, board) + if earlier: + return {'board': board, 'verdict': 'blocker', + 'detail': f'run {", ".join(earlier)} ended without finishing: recover the board, then remove its marker'} + record, ok, why = build_record(board, roster, out) + if not ok: + return {'board': board, 'verdict': 'not-run', 'detail': why} + (out / f'{board}.started').touch() + code, report = battery(board, roster, record, out / board, lock_wait, fail_at) + v, detail = verdict(code, report, board) + if v != 'blocker': + (out / f'{board}.done').touch() + return {'board': board, 'verdict': v, 'detail': detail, 'run': str(out / board)} + + +def summary(rows, roster): + lines = [f'HIL on `{roster}`', '', '| board | verdict | detail |', '|---|---|---|'] + lines += [f'| {r["board"]} | {r["verdict"]} | {r["detail"].replace("|", "/")} |' for r in rows] + return '\n'.join(lines) + '\n' + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split('\n\n')[0]) + ap.add_argument('--board', nargs='+', required=True, help='roster boards to run, whatever changed') + ap.add_argument('--roster', required=True) + ap.add_argument('--out', required=True, type=Path, help="this run's directory under the state directory") + ap.add_argument('--lock-wait', type=int, default=LOCK_WAIT, help='seconds each battery keeps trying a held board') + ap.add_argument('--fail-at', choices=STEPS, help='acceptance only (SKILL.md Execution): passed to every battery') + args = ap.parse_args(argv) + boards = sorted(set(args.board)) + unknown = sorted(set(boards) - set(roster_boards(args.roster))) + if unknown: + ap.error(f'not in {args.roster}: {unknown}') + args.out.mkdir(parents=True, exist_ok=True) + state = args.out.parent + prune(state, keep=args.out) + rows = [run_board(b, args.roster, args.out, state, args.lock_wait, args.fail_at) for b in boards] + text = summary(rows, args.roster) + sys.stdout.write(text) + if os.environ.get('GITHUB_STEP_SUMMARY'): + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as f: + f.write(text) + (args.out / 'summary.json').write_text(json.dumps({'boards': rows}, indent=1)) + verdicts = {r['verdict'] for r in rows} + return 2 if 'blocker' in verdicts else 1 if verdicts & {'fail', 'not-run'} else 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/skills/hil/scripts/hil_esp.py b/.claude/skills/hil-contract/scripts/hil_esp.py similarity index 97% rename from .claude/skills/hil/scripts/hil_esp.py rename to .claude/skills/hil-contract/scripts/hil_esp.py index cb97bea2c..426553d60 100644 --- a/.claude/skills/hil/scripts/hil_esp.py +++ b/.claude/skills/hil-contract/scripts/hil_esp.py @@ -18,14 +18,14 @@ SUBTYPE = {(APP, 'factory'): 0x00, (APP, 'ota_0'): 0x10, (APP, 'ota_1'): 0x11, (DATA, 'ota'): 0x00, (DATA, 'nvs'): 0x02, (DATA, 'fat'): 0x81} TYPE = {'app': APP, 'data': DATA} -# the IDF build's own segments, the flash target's inputs after flash_args: build-dir path -> the segment's name +# the IDF build's own segments, the flash's inputs after flash_args: build-dir path -> the segment's name # (its roster region) SEGMENT_PATHS = {p: Path(p).name for p in FLASH['esptool'][1][1:]} SEGMENTS = tuple(SEGMENT_PATHS.values()) NATIVE = {v: k for k, v in SEGMENT_PATHS.items()} APP_UF2 = f'_bin/apps/{HIL_APP}.uf2' # the battery's app, copied through TinyUF2 BLINKY_UF2 = '_bin/apps/blinky.uf2' # what cleanup leaves running -RECORD_FILES = (APP_UF2, BLINKY_UF2, *IDF_CONFIG) # besides the flash target's inputs, which admit_flash checks +RECORD_FILES = (APP_UF2, BLINKY_UF2, *IDF_CONFIG) # besides the flash's inputs, which admit_flash checks IMAGE_MAGIC = 0xE9 @@ -145,7 +145,7 @@ def admit(root, board, f, lay): """The segments an install writes, [(offset, path)], from an IDF build dir already hash-checked against its record: flash_args, the generated sdkconfig and partition table must agree with the candidate's source, and every segment must sit inside its own region. Paths are the build tree's own files, which - ESP-IDF's flash target writes, not the _bin copies.""" + the flash writes, not the _bin copies.""" gen = sdkconfig((root / 'sdkconfig').read_text()) want = {'CONFIG_IDF_TARGET': f'"{board["device"]}"', 'CONFIG_PARTITION_TABLE_OFFSET': hex(PARTITION_TABLE[0]), 'CONFIG_PARTITION_TABLE_CUSTOM_FILENAME': f'"{f["partitionCsv"]}"', diff --git a/.claude/skills/hil/scripts/hil_test.py b/.claude/skills/hil-contract/scripts/hil_test.py similarity index 55% rename from .claude/skills/hil/scripts/hil_test.py rename to .claude/skills/hil-contract/scripts/hil_test.py index 9e709093c..323aa1ff8 100755 --- a/.claude/skills/hil/scripts/hil_test.py +++ b/.claude/skills/hil-contract/scripts/hil_test.py @@ -14,7 +14,6 @@ import argparse import contextlib import fcntl -import functools import glob import hashlib import importlib.util @@ -22,6 +21,7 @@ import os import re import select +import shlex import shutil import signal import struct @@ -42,17 +42,35 @@ # 0x9D800-0x9DDFF is reserved and undocumented. MIMXRT1062: FlexSPI XIP at 0x60000000 (IMXRT1060RM # Table 3-1) holding the EVK's 8 MiB IS25WP064A (EVK hardware user's guide 2.7). MIMXRT1011: FlexSPI XIP at # 0x60000000 (IMXRT1010RM Table 3-1) holding the Metro M7 1011's W25Q64JV (Metro M7 RT1011 SD revA schematic), 8 MiB. +# MIMXRT1064: FlexSPI2 at 0x70000000 (IMXRT1064RM Rev 2 Table 3-1) holding the 4 MB on-chip flash (IMXRT1064IEC Rev 3, 1.1). # STM32F303VC: 256 KiB main flash at 0x08000000 (RM0316 Rev 10 Table 2); option bytes at 0x1FFFF800 stay outside. # STM32F411VE: 512 KiB main flash at 0x08000000 (RM0383 Rev 4 Table 4); option bytes at 0x1FFFC000 stay outside. +# STM32H563ZI: 2 MiB main flash, banks at 0x08000000 and 0x08100000 (RM0481 Rev 5 Table 51); OTP at 0x08FFF000 and +# system memory at 0x0BF80000 stay outside, and its option bytes have no memory address (RM0481 7.4.1). FLASH_LIMITS = {'LPC55S69_M33_0': (0, 0x9D800), 'MIMXRT1062xxx6A': (0x60000000, 0x60800000), - 'MIMXRT1011DAE5A': (0x60000000, 0x60800000), 'stm32f303vc': (0x08000000, 0x08040000), - 'stm32f411ve': (0x08000000, 0x08080000)} + 'MIMXRT1011DAE5A': (0x60000000, 0x60800000), 'MIMXRT1064xxx6A': (0x70000000, 0x70400000), + 'stm32f303vc': (0x08000000, 0x08040000), + 'stm32f411ve': (0x08000000, 0x08080000), 'stm32h563zi': (0x08000000, 0x08200000)} +# by a board's kind, its firmware and battery: a J-Link or ST-LINK Cortex-M board, or an Espressif one whatever transport +# installs it (board['flasher']) STEPS = {'jlink': ('install', 'dfu', 'copy', 'app-ran', 'reenter')} -STEPS['esptool'] = STEPS['jlink'] + ('recopy',) -REENTRY = {'jlink': ('retained-register',), 'esptool': ('dfu-detach',)} +STEPS['stlink'] = STEPS['jlink'] +STEPS['esp'] = STEPS['jlink'] + ('recopy',) +REENTRY = {'jlink': ('retained-register',), 'stlink': ('retained-register',), 'esp': ('dfu-detach',)} REENTRY_METHODS = sorted({m for v in REENTRY.values() for m in v}) -PROTECTED = {'hil_test.py', 'pool_check'} # tinyusb hil_lock.PROTECTED_REASONS, checked at load +PROTECTED = {'hil_test.py', 'pool_check'} # hil_lock.PROTECTED_REASONS, checked at load +# the board-lock helper is the submodule's at its pinned commit, never an operator's installed agentrc +AGENTRC = 'lib/agentrc' +LOCK_HELPER = ROOT / AGENTRC / 'skills' / 'hil' / 'scripts' / 'hil_lock.py' +# the flash provider beside it (agentrc `hil` Flashers): every install, probe session and reset runs through it +FLASH_PROVIDER = LOCK_HELPER.with_name('hil_flash.py') +OPENOCD_ESP = LOCK_HELPER.with_name('hil_openocd_esp.py') # an Espressif board's JTAG route (tinyuf2.routes) +OPENOCD_TIMEOUT = 300 # s for one openocd_esp flash: every segment programmed, verified and read back +# s from the esptool session's hard reset to the provider's start: on the P4 rev v1.0 a start within ms failed every +# reset init of its session (reset-assert-post); with this wait it passed (bench 2026-10-20, agentrc #160) +OPENOCD_BOOT_SETTLE = 1 HOLDER_ROLE = 'cleanup-holder' +LOCK_POLL = 15 # s between tries at a held board within run's --lock-wait SCB_VTOR, SCB_AIRCR = 0xE000ED08, 0xE000ED0C AIRCR_SYSRESETREQ = 0x05FA0004 UF2_MAGIC = (0x0A324655, 0x9E5D5157, 0x0AB16F30) @@ -65,21 +83,33 @@ USB_DFU = Path(__file__).resolve().parent / 'usb_dfu.py' NONCE_MAGIC, NONCE_BLOCK = b'TUF2NONC', 256 # apps/dfu_runtime/src/dfu_runtime.h JLINK_IF, JLINK_SPEED = 'SWD', '4000' -CMAKE_RECORD_FILES = ('_bin/tinyuf2.bin', '_bin/apps/blinky.uf2') # besides the flash target's inputs +STM32_PROGRAMMER_CLI = 'STM32_Programmer_CLI' +# what STM32CubeProgrammer prints at every connect to the roster's device: its Device ID and NVM size (bench, v2.23.0); +# 0x484 names the H56x/573 family only, so the size pins the part +STLINK_CHIPS = {'stm32h563zi': ('0x484', '2 MBytes')} +ANSI = re.compile(r'\x1b\[[0-?]*[ -/]*[@-~]') # any CSI sequence: esptool 5.2.0 moves the cursor off a TTY too +# each script operation's success line, by its first word (bench, v2.23.0) +STLINK_DONE = {'-halt': 'Core halted', '-run': 'Core run', '-rst': 'Software reset is performed', + '-coreReg': 'XPSR = 0x', '-r32': 'Reading 32-bit memory content', '-w32': '32-bit data download complete', + '-u': 'Data read successfully', '#Delay': 'The system go to sleep for'} +CMAKE_RECORD_FILES = ('_bin/tinyuf2.bin', '_bin/apps/blinky.uf2') # besides the flash's inputs JLINK_ERRORS = (r'Verify failed|Could not|Cannot connect|ERROR|ResetTarget failed|Failed to halt|CPU is not halted|' r'Failed to unlock|Can not attach') UDEV_RULES = ('/etc/udev/rules.d', '/run/udev/rules.d') # J-Link notes that are not failures, per device, each with the line that shows the operation completed # (bench: every MIMXRT1062 reset with an invalid app vector, then the reset halts as usual; every MIMXRT1011 reset -# of a TinyUF2 image, which then boots; STM32F303VC: the connect that removes TinyUF2's write protection, SKILL.md -# Limits) +# of a TinyUF2 image, which then boots; MIMXRT1064 the same as MIMXRT1062 (bench-history.md, imxrt1064_evk); +# STM32F303VC: the connect that removes TinyUF2's write protection, bench-history.md, stm32f303vc) _RESET_VECTOR_NOTE = r'(?m)^Could not find valid reset vector\.\nTarget halted on flash header read\.$' _F3_UNPROTECT_NOTE = (r'(?m)^Active write protection detected\.\nDevice will be unsecured now\.\nFailed to unlock device\.\n' r'Can not attach to CPU\. Trying connect under reset\.$') -BENIGN = {'MIMXRT1062xxx6A': _RESET_VECTOR_NOTE, 'MIMXRT1011DAE5A': _RESET_VECTOR_NOTE, 'stm32f303vc': _F3_UNPROTECT_NOTE} -# runner sessions connect as the core: J-Link's stm32f411ve connect resets the chip to unprotect it (SKILL.md Limits) +BENIGN = {'MIMXRT1062xxx6A': _RESET_VECTOR_NOTE, 'MIMXRT1011DAE5A': _RESET_VECTOR_NOTE, 'MIMXRT1064xxx6A': _RESET_VECTOR_NOTE, + 'stm32f303vc': _F3_UNPROTECT_NOTE} +# runner sessions connect as the core: J-Link's stm32f411ve connect resets the chip to unprotect it +# (bench-history.md, stm32f411ve_discovery) SESSION_DEVICE = {'stm32f411ve': 'Cortex-M4'} -# ms the install's verify session waits before its halt after the flash target's unlock, empirical (SKILL.md Limits) +# ms the install's verify session waits before its halt after the flash's unlock, empirical +# (bench-history.md, stm32f411ve_discovery) INSTALL_SETTLE_MS = {'stm32f411ve': 1000} @@ -98,7 +128,14 @@ class NotEnumerated(StepFail): class Interrupted(Exception): - pass + interrupted = True # what the flash provider reads to record an attempt as interrupted, not an error + + +class SpawnTimeout(StepFail): + """A spawned child killed on its bound: its output so far and what of its session outlived SIGKILL.""" + def __init__(self, msg, output, survivors): + super().__init__(msg) + self.output, self.survivors = output, survivors # ------------------------------------------------------------------ roster and source facts @@ -112,15 +149,11 @@ def load_board(config, name): if len(boards) != 1: raise Refused(f'{config} has {len(boards)} entries named {name!r}') b = boards[0] - tinyusb = os.environ.get('TINYUSB_DIR') - if not (tinyusb and os.path.isabs(tinyusb)): - raise Refused('set TINYUSB_DIR to the absolute path of the tinyusb checkout whose test/hil/helper/hil_lock.py ' - 'owns the board locks') fl, t = b.get('flasher') or {}, b.get('tinyuf2') or {} m = re.fullmatch(cb.FLASHER_ARGS.get(fl.get('name'), r'(?!)'), fl.get('args') or '') if not isinstance(fl.get('uid'), str) or not m: - raise Refused(f'{name}: flasher must be jlink with args exactly "-device NAME", or esptool with args exactly ' - f'"--chip CHIP", with a string uid') + raise Refused(f'{name}: flasher must be jlink or stlink with args exactly "-device NAME", or esptool with args ' + f'exactly "--chip CHIP", with a string uid') if not isinstance(b.get('uid'), str) or not b['uid']: raise Refused(f'{name}: uid (the bootloader USB serial) is required') lock = t.get('lock', name) # tinyusb's name for the board when it differs: both runners take one lock @@ -135,6 +168,8 @@ def load_board(config, name): raise Refused(f'{name}: esptool is supported for espressif chips {sorted(esp.BOOTLOADER_OFFSET)} only') elif m.group(1) not in FLASH_LIMITS: raise Refused(f'{name}: no programming limits known for J-Link device {m.group(1)}') + elif fl['name'] == 'stlink' and m.group(1) not in STLINK_CHIPS: + raise Refused(f'{name}: no chip identity known for ST-LINK device {m.group(1)}') power = t.get('usbPower') if power is not None: if not (isinstance(power, dict) and set(power) == {'hub', 'port'} and isinstance(power['hub'], str) and @@ -143,9 +178,56 @@ def load_board(config, name): if fl['name'] != 'jlink': raise Refused(f'{name}: tinyuf2.usbPower is for J-Link boards only') return {'name': name, 'lock': lock, 'uid': b['uid'], 'flasher': fl['name'], 'probe': fl['uid'], 'device': m.group(1), - 'if': JLINK_IF, 'speed': JLINK_SPEED, 'port': port, - 'helper': os.path.join(tinyusb, 'test', 'hil', 'helper', 'hil_lock.py'), - 'reentry': t.get('reentry'), 'usbPower': power} + 'kind': 'esp' if port == 'espressif' else fl['name'], 'if': JLINK_IF, 'speed': JLINK_SPEED, 'port': port, + 'reentry': t.get('reentry'), 'usbPower': power, 'routes': board_routes(name, port, m.group(1), t), 'route': None} + + +def board_routes(name, port, chip, t): + """tinyuf2.routes: an Espressif board's further install routes, [{"flasher": {name, uid, args}, "commissioning": + "pending"}], each under the board's own lock. Only openocd_esp, for the chip the roster flasher names; a route + still pending commissioning installs and restores only.""" + routes = t.get('routes', []) + if not isinstance(routes, list): + raise Refused(f'{name}: tinyuf2.routes must be a list') + out = {} + for r in routes: + fl = r.get('flasher') if isinstance(r, dict) else None + rname = fl.get('name') if isinstance(fl, dict) else None + if port != 'espressif' or rname not in cb.ROUTE_ARGS or rname in out or set(r) - {'flasher', 'commissioning'} or \ + r.get('commissioning', 'pending') != 'pending' or not re.fullmatch(cb.ROUTE_ARGS[rname], fl.get('args') or ''): + raise Refused(f'{name}: a tinyuf2.routes entry must be {{"flasher": {{"name": "openocd_esp", "uid": "", ' + f'"args": "-f board/-builtin.cfg"}}}}, optionally "commissioning": "pending", on an Espressif ' + f'board, one per route: {r!r}') + try: + route_chip, _ = openocd_esp().admit(fl) + except ValueError as e: + raise Refused(f'{name}: route {rname}: {e}') + if route_chip != chip: + raise Refused(f'{name}: route {rname} reaches an {route_chip}; the roster flasher names {chip}') + out[rname] = {'flasher': dict(fl), 'pending': r.get('commissioning') == 'pending'} + return out + + +def select_route(board, route, run=False): + """The board as `route` installs it, chosen before any board access and never switched after a failure: its + roster flasher, or one of tinyuf2.routes, whose USJ serial then also selects the esptool sessions' tty. A route + pending commissioning installs and restores only.""" + if route in (None, board['flasher']): + return board + r = board['routes'].get(route) + if r is None: + raise Refused(f'{board["name"]}: no install route {route} (roster flasher {board["flasher"]}, tinyuf2.routes ' + f'{sorted(board["routes"])})') + if run and r['pending']: + raise Refused(f'{board["name"]}: route {route} is pending commissioning: install and restore only') + refuse_ci_job(route) + return dict(board, flasher=route, probe=r['flasher']['uid'], route=r['flasher']) + + +def refuse_ci_job(route): + """The provider's own refusal of an openocd_esp session under a CI job, before any session of the route's.""" + if route == 'openocd_esp' and (why := openocd_esp().ci_refusal()): + raise Refused(why) def has_define(path, name): @@ -204,18 +286,23 @@ def cmake_var(path, name): return v +STM32_DEVICE = {'stm32f3': ('STM32F3', r'STM32F3\d\dx[A-Z]'), 'stm32f4': ('STM32F4', r'STM32F4\d\dx[A-Z]'), + 'stm32h5': ('STM32H5', r'STM32H5\d\dxx')} + + def port_sources(port, bdir): - """(CMSIS device header, the bootloader's linker script, FLASH_BASE, where the BIN starts in flash, RAM origin) of - a port the HIL supports, read from the CMake files the HIL's build uses.""" - if port in ('stm32f3', 'stm32f4'): + """(CMSIS device header, the bootloader's linker script, FLASH_BASE, where the BIN starts in flash, RAM origin, the + link's --defsym values) of a port the HIL supports, read from the CMake files the HIL's build uses.""" + if port in STM32_DEVICE: ld = cmake_script(f'ports/{port}/CMakeLists.txt') - series = port.upper() - mcus = set(re.findall(rf'\b{series}\d\dx[A-Z]\b', cb.code_lines(ROOT / bdir / 'board.cmake'))) + series, form = STM32_DEVICE[port] + mcus = set(re.findall(rf'\b{form}\b', cb.code_lines(ROOT / bdir / 'board.cmake'))) if len(mcus) != 1: raise Refused(f'{bdir}/board.cmake: expected one {series} device define, found {sorted(mcus)}') header = f'{cmake_var(f"ports/{port}/family.cmake", "ST_CMSIS")}/Include/{mcus.pop().lower()}.h' - flash_base, ram = ld_memory(ld, 'FLASH')[0], ld_memory(ld, 'RAM')[0] - return header, ld, flash_base, flash_base, ram + syms = defsyms(port, bdir, ld) if port == 'stm32h5' else {} + flash_base, ram = ld_memory(ld, 'FLASH', syms)[0], ld_memory(ld, 'RAM', syms)[0] + return header, ld, flash_base, flash_base, ram, syms if port != 'mimxrt10xx': raise Refused(f'ports/{port}: the HIL runner has no source facts for this port') sdk = cmake_var(f'ports/{port}/family.cmake', 'SDK_DIR') @@ -231,7 +318,30 @@ def port_sources(port, bdir): # family.cmake FLASH_FCFB_ADDR: the BIN starts at the FCB, 0x400 into flash on RT1010 (IMXRT1010RM Table 9-8) bin_base = flash_base + (vals['_fcfb_origin'] & 0xFFF) return (f'{sdk}/devices/{mcu}/{mcu}.h', ld, flash_base, bin_base, - ram_origin(f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld')) + ram_origin(f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld'), {}) + + +# the stm32h5 link's --defsym symbols, each from the board's set() of this variable +DEFSYMS = {'__flash_boot_size': 'LD_FLASH_BOOT_SIZE', '__ram_size': 'LD_RAM_SIZE'} + + +def defsyms(port, bdir, ld): + """The values family.cmake passes the linker for DEFSYMS, from the board.cmake's one set() each, in the exact + forms the port uses: one `-Wl,--defsym=SYM=${VAR}` each, the script's one `SYM = DEFINED(SYM) ? SYM : ;`, + and family.cmake setting a variable only for Debug builds, which a HIL record (MinSizeRel) is not.""" + fam = cb.code_lines(ROOT / f'ports/{port}/family.cmake') + text, syms = ld_text(ld), {} + for sym, var in DEFSYMS.items(): + sets = re.findall(rf'\bset\s*\(\s*{var}\b', fam, re.I) + debug = re.findall(rf'if\s*\(\s*CMAKE_BUILD_TYPE\s+STREQUAL\s+"Debug"\s*\)\s*set\s*\(\s*{var}\s+\w+\s*\)\s*endif\s*\(\s*\)', + fam, re.I) + if fam.count('--defsym=') != len(DEFSYMS) or fam.count(f'-Wl,--defsym={sym}=${{{var}}}') != 1 or \ + len(sets) != len(debug) or \ + len(re.findall(rf'(?>|[-+*/|&]?)=(?!=)', text)) != 1 or \ + len(re.findall(rf'(?>|[-+*/|&]?)=(?!=)\s*([^;]+);', ld_text(ld)) + hits = re.findall(rf'(?>|[-+*/|&]?)=(?!=)\s*([^;]+);', ld_text(ld, syms)) if len(hits) != 1 or hits[0][0]: raise Refused(f'{ld}: expected one plain {name} = ...;, found {[op + "=" for op, _ in hits]}') rhs = hits[0][1].strip() if re.fullmatch(r'[A-Za-z_]\w*', rhs) and depth < 2: - return ld_symbol(ld, rhs, depth + 1) + return ld_symbol(ld, rhs, depth + 1, syms) m = re.fullmatch(r'ORIGIN\((\w+)\)\s*\+\s*LENGTH\(\1\)', rhs) if m: - return sum(ld_memory(ld, m.group(1))) + return sum(ld_memory(ld, m.group(1), syms)) return ld_expr(rhs, f'{ld} {name}') @@ -299,17 +416,19 @@ def source_facts(board): bdir = f'ports/{port}/boards/{board["name"]}' if not (ROOT / bdir).is_dir(): raise Refused(f'{bdir} does not exist') - if board.get('flasher') == 'esptool': + if board.get('kind') == 'esp': return esp_source_facts(board, bdir) - header, ld, flash_base, bin_base, ram = port_sources(port, bdir) + header, ld, flash_base, bin_base, ram, syms = port_sources(port, bdir) if has_define(f'{bdir}/board.h', 'BOARD_FLASH_APP_START'): raise Refused(f'{bdir}/board.h overrides BOARD_FLASH_APP_START; the runner reads the port default only') + size = define(f'{bdir}/board.h', 'BOARD_FLASH_SIZE') + floor, ceiling = FLASH_LIMITS.get(board.get('device'), (0, 0)) facts = { 'flashBase': flash_base, 'binBase': bin_base, - 'appStart': flash_addr(define(f'ports/{port}/boards.h', 'BOARD_FLASH_APP_START'), flash_base, - 'BOARD_FLASH_APP_START'), - 'flashSize': product(define(f'{bdir}/board.h', 'BOARD_FLASH_SIZE'), 'BOARD_FLASH_SIZE'), + 'appStart': app_start(port, flash_base, syms), + # FLASH_SIZE is CMSIS's read of the chip's size register: the roster device's flash, which INFO_UF2.TXT reports + 'flashSize': ceiling - floor if size == 'FLASH_SIZE' and ceiling else product(size, 'BOARD_FLASH_SIZE'), 'vid': num(define(f'{bdir}/board.h', 'USB_VID'), 'USB_VID'), 'pid': num(define(f'{bdir}/board.h', 'USB_PID'), 'USB_PID'), 'boardId': define(f'{bdir}/board.h', 'UF2_BOARD_ID').strip('"'), @@ -326,13 +445,26 @@ def source_facts(board): base, offset, step = register_layout(header, m.group(1), m.group(2)) facts['dblTapReg'] = base + offset + step * int(m.group(3)) else: # board_api.h's default: the word the bootloader's linker script places - facts['dblTapReg'] = ld_symbol(ld, '_board_dfu_dbl_tap') + facts['dblTapReg'] = ld_symbol(ld, '_board_dfu_dbl_tap', syms=syms) facts['dblTapMagic'] = 0xf01669ef # board_api.h DBL_TAP_MAGIC at TINYUF2_DBL_TAP_REG_SIZE 32 if 'TINYUF2_DBL_TAP_REG_SIZE' in (ROOT / f'ports/{port}/boards.h').read_text(): raise Refused(f'ports/{port} overrides TINYUF2_DBL_TAP_REG_SIZE: the magic would need its shift') return facts +def app_start(port, flash_base, syms): + """BOARD_FLASH_APP_START: a literal or FLASH_BASE + literal, or the stm32h5 form, the flash base plus the link's + __flash_boot_size.""" + expr = define(f'ports/{port}/boards.h', 'BOARD_FLASH_APP_START') + if not re.fullmatch(r'\(\s*BOARD_FLASH_ADDR_ZERO\s*\+\s*\(uint32_t\)\s*__flash_boot_size\s*\)', expr): + return flash_addr(expr, flash_base, 'BOARD_FLASH_APP_START') + zero = num(define(f'ports/{port}/boards.h', 'BOARD_FLASH_ADDR_ZERO'), 'BOARD_FLASH_ADDR_ZERO') + if '__flash_boot_size' not in syms or zero != flash_base: + raise Refused(f'ports/{port}: BOARD_FLASH_APP_START {expr!r} needs the link\'s __flash_boot_size and ' + f'BOARD_FLASH_ADDR_ZERO at the flash base') + return zero + syms['__flash_boot_size'] + + def esp_source_facts(board, bdir): try: facts = esp.facts(ROOT, board) @@ -380,7 +512,7 @@ def register_layout(header, periph, member): def apply_layout(board, facts): """The install and app spans, from the candidate's source: the bootloader from the flash base up to the app start, the app from there to the end of the board's flash, all inside the device's programming limits.""" - if board.get('flasher') == 'esptool': + if board.get('kind') == 'esp': try: lay = esp.layout(facts, board['device']) except esp.Refused as e: @@ -497,7 +629,6 @@ def make_uf2(img, base, family, workdir): # ------------------------------------------------------------------ processes CHILDREN = [] -CONTAINERS = [] # docker containers a flash started, by name: outside every child's session REGISTERING = {'on': False, 'signum': None} @@ -515,24 +646,34 @@ def registering(): raise Interrupted(f'interrupted by {signal.Signals(signum).name}') -def spawn(cmd, log, timeout, cwd=None): +def spawn(cmd, log, timeout, cwd=None, stderr_to_log=False, env=None): """Run a child in its own process group with a bound; a child that survives SIGKILL (D state on a stuck USB device) is left in CHILDREN for cleanup to see. Children spawned during cleanup inherit its - blocked SIGTERM and SIGINT, so only SIGKILL (reap) may be relied on to end them.""" + blocked SIGTERM and SIGINT, so only SIGKILL (reap) may be relied on to end them. With `stderr_to_log`, the + child's stderr goes to the log only and the output returned is its stdout. `env` adds to the environment. + A timeout raises SpawnTimeout and an interruption Interrupted, each carrying the output so far.""" with open(log, 'a') as f: - f.write('$ ' + ' '.join(map(str, cmd)) + '\n') + f.write('$ ' + ' '.join([f'{k}={v}' for k, v in (env or {}).items()] + list(map(str, cmd))) + '\n') f.flush() with registering(): - p = subprocess.Popen([str(c) for c in cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, - errors='replace', start_new_session=True, cwd=cwd) + p = subprocess.Popen([str(c) for c in cmd], stdout=subprocess.PIPE, + stderr=f if stderr_to_log else subprocess.STDOUT, text=True, errors='replace', + start_new_session=True, cwd=cwd, env=env and {**os.environ, **env}) CHILDREN.append(p) try: out, _ = p.communicate(timeout=timeout) - except subprocess.TimeoutExpired: - reap(p) - raise StepFail(f'{Path(str(cmd[0])).name} timed out after {timeout}s') + except subprocess.TimeoutExpired as e: + left = reap(p) + partial = text(e.output, p.stdout.encoding) + f.write(partial) + raise SpawnTimeout(f'{Path(str(cmd[0])).name} timed out after {timeout}s', partial, left) + except Interrupted as e: # left for cleanup's reap; what communicate read so far is only in its own buffer + e.stdout = text(b''.join(getattr(p, '_fileobj2output', {}).get(p.stdout, [])), p.stdout.encoding) + f.write(e.stdout) + raise f.write(out) - CHILDREN.remove(p) + if not session(p.pid): # else a process it started lives on, tracked for cleanup's reap + CHILDREN.remove(p) return p.returncode, out @@ -565,6 +706,9 @@ def reap(p): time.sleep(0.1) if not left and p in CHILDREN: # tracked while any of its session lives, for cleanup to see CHILDREN.remove(p) + if p.stdout: # nothing more is read from a killed child + with contextlib.suppress(OSError): + p.stdout.close() return left @@ -574,49 +718,435 @@ def proc_start(pid): return int(st[19]) if st and len(st) > 19 else None -def container_present(name): - r = subprocess.run(['docker', 'ps', '-aq', '-f', f'name=^{name}$'], capture_output=True, text=True, timeout=30) - if r.returncode: - raise StepFail(f'docker ps failed: {r.stderr.strip()[-200:]}') - return bool(r.stdout.strip()) +def still_alive(survivors): + """The survivors still alive: a process by its pid and start time; a container, which only a report from a runner + that still ran the IDF flash target or its dry runs names, unless docker has listed it no more for check_build's + settle (cb.container_left), and never while a process of the same report lives, which may still create it (#580).""" + sessions = [s for s in survivors if 'receipt' in s and session_state(s) in ('running', 'unreadable')] + procs = [s for s in survivors if 'container' not in s and 'receipt' not in s and s.get('start') is not None + and proc_start(s['pid']) == s['start']] + return procs + sessions + [s for s in survivors if 'container' in s and (procs or cb.container_left(s['container']))] + + +class CleanupPending(StepFail): + """A flash left a child or an openocd_esp session running: no session may follow or overlap it, and the board and + build locks stay held until it ends (a session: until recovery ends it by its receipt).""" + + +OPENOCD_SESSIONS = 'hil_openocd_esp_*/receipt.json' # flash_openocd_esp's session dirs under the run dir +PROTECTED_SESSIONS = [] # every receipt an openocd_esp operation of this process returned, by path or record +PENDING_MARK = '.tinyuf2-openocd-pending.json' # beside the board's lock: sessions no later invocation may overlap +CLEARED_LOG = '.cleared.jsonl' # .cleared.jsonl: every mark cleared after a reviewed recovery, with its note +MARK_MUTEX = '.mutex' # .mutex, never replaced or removed; not '.lock', which hil_lock lists as a board +MARK_ERRORS = [] # every failure to write the mark this process met, for the report + + +def session_entry(receipt): + """A survivor entry for one openocd_esp receipt: its path, or the record itself when the provider could not write + it; its pid and start time for the report.""" + rec = receipt if isinstance(receipt, dict) else read_receipt(receipt) or {} + e = {'cmd': 'openocd', 'receipt': receipt if isinstance(receipt, dict) else str(receipt)} + with contextlib.suppress(KeyError, TypeError, ValueError): + e.update(pid=rec['pid'], start=int(rec['startTime'])) + return e + + +def read_receipt(path): + try: + rec = json.loads(Path(path).read_text()) + except (OSError, ValueError): + return None + return rec if isinstance(rec, dict) else None + + +def session_state(e): + """An openocd_esp session's state from its receipt, reread on every call: 'ended' (closed after the reviewed end, + cleanup done), 'running' (open, its process alive), 'unreadable', or 'unresolved' (closed without the reviewed + end, or its process gone without closing it), which a separately reviewed recovery decides.""" + rec = e['receipt'] if isinstance(e['receipt'], dict) else read_receipt(e['receipt']) + try: + if rec['closed']: + return 'ended' if rec.get('cleanup') == 'done' else 'unresolved' + return 'running' if proc_start(rec['pid']) == int(rec['startTime']) else 'unresolved' + except (KeyError, TypeError, ValueError): + return 'unreadable' + + +def merge_sessions(old, new): + """`old`, then each entry of `new` whose receipt no earlier entry has; compared by equality, as a receipt may be + the provider's record itself.""" + merged = list(old) + for e in new: + if all(e['receipt'] != m['receipt'] for m in merged): + merged.append(e) + return merged + + +def protect(op): + """Keep every receipt an openocd_esp operation reports, on disk or in memory, whatever the run dir holds.""" + PROTECTED_SESSIONS[:] = merge_sessions(PROTECTED_SESSIONS, [session_entry(a.receipt) for a in + getattr(op, 'attempts', ()) if a.receipt is not None]) + + +def outstanding_sessions(workdir): + """The openocd_esp sessions of this run not ended by the reviewed end: the receipts under `workdir`, which the + provider writes before an interrupt can be acted on, and every receipt an operation returned (protect()), the + record itself when the provider could not write it. Never children to reap: a killed openocd can leave the chip + in its Breakpoint boot loop.""" + found = merge_sessions([session_entry(r) for r in sorted(Path(workdir).glob(OPENOCD_SESSIONS))], PROTECTED_SESSIONS) + return [e for e in found if session_state(e) != 'ended'] + + +class PendingMark(os.PathLike): + """A board's pending mark: .tinyuf2-openocd-pending.json in the lock helper's dir, every file of it opened + through the helper's own checked dir and file opens (created umask-proof, readable across users).""" + + def __init__(self, helper, lock): + self.helper, self.lock, self.name = helper, lock, lock + PENDING_MARK + self.path = os.path.join(helper.BOARD_LOCK_DIR, self.name) + + def __fspath__(self): + return self.path + + def __str__(self): + return self.path + + +@contextlib.contextmanager +def mark_locked(mark): + """Serialize every read, write and removal of a pending mark across processes, yielding the lock dir's fd: an + flock on a file beside it that is never replaced, so the board lock, taken only later, need not.""" + dfd = mark.helper._open_dir(create=True) + try: + fd = mark.helper._open_file(dfd, mark.name + MARK_MUTEX, os.O_RDWR, create=True) + try: + fcntl.flock(fd, fcntl.LOCK_EX) + yield dfd + finally: + os.close(fd) + finally: + os.close(dfd) + + +def read_mark(mark): + """The sessions a mark lists, [] without one; None when it is unreadable or any entry is malformed.""" + try: + data = mark.helper._read_file(mark.lock, PENDING_MARK) + rec = data is not None and json.loads(data) + except (OSError, ValueError): + return None + if data is None: + return [] + sessions = rec.get('sessions') if isinstance(rec, dict) else None + ok = isinstance(sessions, list) and all(isinstance(e, dict) and isinstance(e.get('receipt'), (str, dict)) + for e in sessions) + return sessions if ok else None + + +def write_mark(mark, dfd, sessions): + tmp = f'{mark.name}.{os.getpid()}.{os.urandom(4).hex()}.tmp' # one per write: no writer truncates another's + fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, 0o644, dir_fd=dfd) + try: + with os.fdopen(fd, 'w') as f: + os.fchmod(f.fileno(), 0o644) # umask-proof: another user's invocation reads it and refuses + json.dump({'sessions': sessions}, f, indent=1) + f.flush() + os.fsync(f.fileno()) + os.replace(tmp, mark.name, src_dir_fd=dfd, dst_dir_fd=dfd) + os.fsync(dfd) # the rename itself durable, as hil_lock's journal writer makes its own + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp, dir_fd=dfd) + raise + + +def mark_pending(mark, sessions): + """Record sessions not ended beside the board's lock, so another invocation, in another run dir, refuses too. + Never raises: a failure, an unreadable or malformed mark included (kept as it is), is returned and kept in + MARK_ERRORS for the report, and the caller's survivor reporting and lock ownership go on as without a mark.""" + if not mark or not sessions: + return None + try: + with mark_locked(mark) as dfd: + old = read_mark(mark) + if old is None: + raise ValueError(f'{mark} is unreadable or malformed; it is left as it is') + write_mark(mark, dfd, merge_sessions(old, sessions)) + return None + except Exception as e: + err = f'the pending mark {mark} could not be written: {type(e).__name__}: {e}' + MARK_ERRORS.append(err) + return err + + +def clear_pending_mark(mark, note=None, hold=None): + """Under the mark's mutex, remove the mark once every session it lists reads ended (as `hil_openocd_esp.py end` + leaves its receipt), or, given the note of a separately reviewed recovery, whatever it holds. Refuses otherwise, + naming the receipts. Returns what was cleared.""" + try: + with mark_locked(mark) as dfd: + sessions = read_mark(mark) + if sessions is None and note is None: + raise Refused(f'{mark} is unreadable or malformed: an earlier openocd_esp session may still be pending; clear ' + 'it only after a separately reviewed recovery (SKILL.md Recovery)') + left = [e['receipt'] for e in sessions or [] if session_state(e) != 'ended'] + if left and note is None: + raise Refused(f'openocd_esp sessions {left} of an earlier run are not ended: end each with `hil_openocd_esp.py ' + 'end --receipt `, or clear the mark after a separately reviewed recovery (SKILL.md ' + 'Recovery)') + cleared = {'mark': str(mark), 'sessions': sessions, 'notEnded': left} + if note is not None: + raw = mark.helper._read_file(mark.lock, PENDING_MARK) + cleared.update(note=note, hold=hold, **({'unreadable': raw.decode(errors='replace')} if sessions is None and raw else {})) + fd = mark.helper._open_file(dfd, mark.name + CLEARED_LOG, os.O_WRONLY | os.O_APPEND, create=True) + with os.fdopen(fd, 'a') as f: + f.write(json.dumps(dict(cleared, at=time.strftime('%Y-%m-%dT%H:%M:%S%z'))) + '\n') + f.flush() + os.fsync(f.fileno()) + with contextlib.suppress(FileNotFoundError): + os.unlink(mark.name, dir_fd=dfd) + return cleared + except OSError as e: + raise Refused(f'the pending mark {mark} could not be checked: {e}') from None + + +def reviewed_receipts(mark): + """Every receipt a reviewed recovery's clear-pending recorded in .cleared.jsonl: the mark's sessions and the + survivors of the cleanup holder it proved.""" + try: + with mark_locked(mark): + data = mark.helper._read_file(mark.lock, PENDING_MARK + CLEARED_LOG) or b'' + except OSError as e: + raise Refused(f'the cleared log of {mark} could not be read: {e}') from None + found = [] + for line in data.splitlines(): + with contextlib.suppress(ValueError, TypeError, AttributeError): + x = json.loads(line) + found += [e['receipt'] for e in (x.get('sessions') or []) + ((x.get('hold') or {}).get('survivors') or []) + if isinstance(e, dict) and 'receipt' in e] + return found + + +def refuse_pending(workdir, mark=None): + left = outstanding_sessions(workdir) + err = mark_pending(mark, left) + if left: + raise CleanupPending(f'openocd_esp session {[s["receipt"] for s in left]} is still running: cleanup pending, no ' + 'session may overlap it', {'survivors': left, **({'markError': err} if err else {})}) + + +# ------------------------------------------------------------------ the flash provider + +PROVIDER = [] # the one hil_flash module this process loads (provider()) +PROVIDER_API = ('flash_jlink', 'flash_stlink', 'run_jlink_commands', 'run_stlink_commands', 'sysresetreq_jlink', + 'sysresetreq_stlink', 'Operation', 'Attempt') +OPENOCD = [] # and its openocd_esp module (openocd_esp()) +OPENOCD_API = ('admit', 'find_openocd', 'usj_location', 'flash_openocd_esp', 'ci_refusal') + + +def load_pinned(path, name, api, what='flash provider'): + """A lib/agentrc module at the commit HEAD pins, loaded by path, never an installed agentrc's; Refused when the + submodule is off its pin or the module lacks what the runner calls.""" + check_lock_pin() + spec = importlib.util.spec_from_file_location(name, path) + if not spec: + raise Refused(f'{what} {path} not found') + mod = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(mod) + except Exception as e: + raise Refused(f'{what} {path} failed to load: {e}') + missing = [n for n in api if not hasattr(mod, n)] + if missing: + raise Refused(f'{what} {path}: lacks {missing}; recheck this contract') + return mod + + +def provider(): + """hil_flash.py, once per process.""" + if not PROVIDER: + mod = load_pinned(FLASH_PROVIDER, 'agentrc_hil_flash', PROVIDER_API) + if (getattr(mod, 'SCB_AIRCR', None), getattr(mod, 'AIRCR_SYSRESETREQ', None)) != (SCB_AIRCR, AIRCR_SYSRESETREQ): + raise Refused(f'flash provider {FLASH_PROVIDER} writes another SYSRESETREQ; recheck this contract') + PROVIDER.append(mod) + return PROVIDER[0] + + +def openocd_esp(): + """hil_openocd_esp.py, once per process, only for a board with a route through it.""" + if not OPENOCD: + OPENOCD.append(load_pinned(OPENOCD_ESP, 'agentrc_hil_openocd_esp', OPENOCD_API)) + return OPENOCD[0] + + +def command_argv(cmd): + """(argv, environment) of a provider command: an argv as is, a shell string split into words with its leading + NAME=value words as the environment they set. Any other shell syntax is refused: no shell runs it.""" + if not isinstance(cmd, str): + return [str(c) for c in cmd], None + if re.search(r'[|&;<>()$`\\*?\n]', cmd): + raise ValueError(f'a provider command this runner cannot run without a shell: {cmd!r}') + words, env = shlex.split(cmd), {} + while words and re.fullmatch(r'[A-Za-z_]\w*=\S*', words[0]): + k, v = words.pop(0).split('=', 1) + env[k] = v + return words, env or None + + +def tool_runner(log): + """The flash provider's `run=`: each command one tracked spawn child, its transcript in `log`, its session killed on + the bound and a survivor kept in CHILDREN for cleanup. A timeout returns rc 124 with the output so far (the + provider records the attempt as timed out); an interruption raises Interrupted carrying it. Cleanup is 'done' only + when nothing of the child's session is left, else 'unknown' (cleanup reaps it). Executed directly, a tool that dies + by a signal reports -N.""" + def run(cmd, cwd=None, *, timeout): + argv, env = command_argv(cmd) + tracked = len(CHILDREN) + try: + code, out = spawn(argv, log, timeout, cwd=cwd, env=env) + except SpawnTimeout as e: + code, out = 124, e.output + r = subprocess.CompletedProcess(argv, code, out, None) + r.cleanup = 'done' if len(CHILDREN) == tracked else 'unknown' # 'done': nothing of its session is left + return r + return run + + +def text(out, encoding='utf-8'): + return out.decode(encoding, 'replace') if isinstance(out, bytes) else out or '' + + +def operation_record(op): + """What the report keeps of a provider operation: each attempt's kind and exit, never their output joined (the + transcripts are in hil.log), its cleanup and any outstanding session's receipt.""" + return {'attempts': [{'kind': a.kind, 'exit': a.result.returncode} for a in op.attempts], 'cleanup': op.cleanup, + **({'receipts': [str(r) for r in op.receipts]} if op.receipts else {})} + + +def before_target(attempts, before): + """Every attempt ended before any target connect (before(code, out)); a timed-out one never did, since its output + so far does not show how far it got.""" + return all(not a.timed_out and before(a.result.returncode, text(a.result.stdout)) for a in attempts) -def container_gone(name): - """`docker rm -f` (a running, created or exited container alike), then whether docker no longer lists it; a - docker error is not an answer, so it counts as still there.""" +def judged(op, judge, before, what, timeout): + """judge(code, out) of the operation's last attempt. A failure's `beforeTarget` holds only when every attempt ended + before any target connect (before(code, out)); a timed-out one never does, since its output so far does not show + how far it got.""" + attempts = op.attempts + out = text(attempts[-1].result.stdout) + timed_out = any(a.timed_out for a in attempts) try: - subprocess.run(['docker', 'rm', '-f', name], capture_output=True, timeout=30) - end = time.monotonic() + 2 * KILL_WAIT - while (left := container_present(name)) and time.monotonic() < end: + if timed_out: + raise StepFail(f'{what} timed out after {timeout}s', {'tail': out.strip().splitlines()[-5:]}) + ev = judge(attempts[-1].result.returncode, out) + except StepFail as e: + e.evidence.update(beforeTarget=before_target(attempts, before), operation=operation_record(op)) + raise + return dict(ev, operation=operation_record(op)) + + +# ------------------------------------------------------------------ build locks + +BUILD_LOCK_WAIT = 2 # s a flash waits for a build dir's lock: a keeper (keep_build_lock) polls every KEEPER_POLL +KEEPER_POLL = 1 +HELD_BUILD = [] # (lock file, survivors) this process keeps itself when no keeper could start + + +def take_build_lock(root): + """check_build's lock of the build dir `root`, as an open file held until it is closed or kept; StepFail when a + build, another flash or a keeper still holds it after BUILD_LOCK_WAIT.""" + path = cb.build_lock_file(root) + path.parent.mkdir(parents=True, exist_ok=True) + fh = open(path, 'a') + end = time.monotonic() + BUILD_LOCK_WAIT + while True: + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + return fh + except BlockingIOError: + if time.monotonic() >= end: + fh.close() + raise StepFail(f'{root.name} is in use by a build or another flash; nothing was flashed') time.sleep(0.2) - return not left - except (StepFail, OSError, subprocess.SubprocessError): - return False -def stuck_children(): - """Processes of the children's sessions, and containers a flash created, that survived SIGKILL, identified so - a later recovery can tell they are still alive. The launchers go first, so no `docker` child can create or start - a container after its name is checked.""" - stuck = [] - for p in list(CHILDREN): - for pid in reap(p): - try: - cmd = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0')[:3] - except OSError: - cmd = [] - stuck.append({'pid': pid, 'start': proc_start(pid), 'cmd': [c.decode(errors='replace') for c in cmd]}) - for name in list(CONTAINERS): - if container_gone(name): - CONTAINERS.remove(name) - else: - stuck.append({'container': name}) - return stuck +def survivor(pid): + try: + cmd = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0')[:3] + except OSError: + cmd = [] + return {'pid': pid, 'start': proc_start(pid), 'cmd': [c.decode(errors='replace') for c in cmd]} -def still_alive(survivors): - return [s for s in survivors if (not container_gone(s['container']) if 'container' in s else - s.get('start') is not None and proc_start(s['pid']) == s['start'])] +def settle_children(started=()): + """Reap every child started since `started` (CHILDREN then; by default every one) that is still tracked: an + interrupted one, or what of a timed-out one outlived its kill. Returns the processes still alive after SIGKILL, + identified so a later recovery can tell they are still alive.""" + left = [] + for p in [p for p in CHILDREN if p not in started]: + left += [survivor(pid) for pid in reap(p)] + return left + + +def keep_build_lock(fh, survivors): + """Hand a build dir's lock to a detached keeper while processes that may still read the tree survive (#579): it + shares the open file description, re-executes itself as `keep-build` without the job tracking variable (as + exec_holder), ignores SIGTERM and SIGINT, and exits, releasing the lock, once none of `survivors` lives. Its + acknowledgement comes only once it runs; without one this process keeps the lock and waits for them before it + exits (main).""" + try: + r, w = os.pipe() + pid = os.fork() + except OSError: + pid = None + if pid == 0: + try: + for sig in TERM_SIGNALS: + signal.signal(sig, signal.SIG_IGN) + os.close(r) + os.setsid() + for fd in (fh.fileno(), w): + os.set_inheritable(fd, True) + env = {k: v for k, v in os.environ.items() if k != JOB_TRACKING} + with contextlib.suppress(OSError): + os.execve(sys.executable, [sys.executable, '-B', str(HOLDER_SCRIPT), 'keep-build', str(fh.fileno()), str(w), + json.dumps(survivors)], env) + finally: + os._exit(1) + if pid: + os.close(w) + ready, _, _ = select.select([r], [], [], 10) + ack = os.read(r, 16) if ready else b'' + os.close(r) + if ack == b'ok': + fh.close() # the keeper's description now; never LOCK_UN + return {'pid': pid} + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(ChildProcessError): + os.waitpid(pid, 0) + HELD_BUILD.append((fh, survivors)) + return {'pid': os.getpid(), 'inProcess': True} + + +def release_ended_builds(): + """Close each build lock this process keeps (HELD_BUILD) once none of its survivors lives; the ones still kept.""" + for kept in [k for k in HELD_BUILD if not still_alive(k[1])]: + kept[0].close() + HELD_BUILD.remove(kept) + return HELD_BUILD + + +def keep_build(survivors): + """The keeper's wait (keep_build_lock): its stdio closed, it polls until no survivor lives, then exits, which + releases the lock it inherited.""" + for std in (0, 1, 2): + with contextlib.suppress(OSError): + os.close(std) + while still_alive(survivors): + time.sleep(KEEPER_POLL) + os._exit(0) # ------------------------------------------------------------------ probe @@ -644,59 +1174,82 @@ def jlink_failure(code, out, probe): def judge_jlink_target(code, out, board, hex_path): - """The tinyuf2-jlink transcript, through CMake and Ninja: JLinkExe can exit 0 with every command FAILED (no probe - with the serial), so success is read from the transcript: this HEX downloaded, programmed or already matching, - then O.K. and the script completed, with no failure line anywhere.""" + """The provider's J-Link flash transcript (the record's tinyuf2-jlink script): JLinkExe can exit 0 with every command + FAILED (no probe with the serial), so success is read from the transcript: this HEX downloaded, programmed or + already matching, then O.K. and the script completed, with no failure line anywhere.""" judged = re.sub(BENIGN[board['device']], '', out) if board['device'] in BENIGN else out m = re.search(rf'^Downloading file \[{re.escape(str(hex_path))}\]\.\.\.$(?:\n.*)*?\nJ-Link: Flash download: Bank \d+ @ ' r'0x[0-9A-F]+: (Skipped\. Contents already match|\d+ ranges? affected \(\d+ bytes\))$(?:\n.*)*?\nO\.K\.$', judged, re.M) if code or not m or 'Script processing completed.' not in judged or re.search(JLINK_ERRORS + '|FAILED', judged): - raise StepFail(f'tinyuf2-jlink failed (exit {code})', jlink_failure(code, out, board['probe'])) + raise StepFail(f'the J-Link flash failed (exit {code})', jlink_failure(code, out, board['probe'])) return {'flashDownload': m[1]} def judge_esp_target(code, out, segments): - """ESP-IDF's flash target, through CMake and Ninja in CI's image: esptool's write of every segment at its offset, - each hash-verified, then the hard reset (bench transcript, esptool v5.3.1 in espressif/idf:v6.1).""" + """The provider's esptool flash: esptool's write of every segment at its offset, each hash-verified, then the hard + reset (esptool 5.2.0 cmds.py write_flash and loader.py hard_reset, which the bench transcripts of 5.2.0 on the P4's + USB-Serial/JTAG and 5.3.1 in espressif/idf:v6.1 match), with any CSI sequence and the underscore spellings' + deprecation warnings around them ignored.""" + out = ANSI.sub('', out) wrote = sorted(int(a, 16) for a in re.findall(r'^Wrote \d+ bytes \(\d+ compressed\) at 0x([0-9a-f]{8}) in ', out, re.M)) if code or wrote != sorted(off for off, _ in segments) or out.count('Hash of data verified.') != len(segments) or \ 'Hard resetting via RTS pin' not in out: - raise StepFail(f'the flash target failed (exit {code})', {'tail': out.strip().splitlines()[-5:]}) + raise StepFail(f'the esptool flash failed (exit {code})', {'tail': out.strip().splitlines()[-5:]}) return {'wrote': [hex(a) for a in wrote]} class Probe: + """J-Link sessions of the runner's, each through the flash provider's run_jlink_commands with tinyuf2's session + profile, judged here.""" def __init__(self, board, workdir, log): self.b, self.dir, self.log = board, workdir, log self.retried = [] + self.flasher = {'name': 'jlink', 'uid': board['probe'], 'args': f'-device {board["device"]}'} + # SWD at 4000 kHz, autoconnect, and the core for a device whose own connect resets the chip (SESSION_DEVICE) + self.profile = {'interface': board['if'], 'speed': board['speed'], 'autoconnect': True, + 'device': SESSION_DEVICE.get(board['device'], board['device']), 'timeout': PROBE_TIMEOUT, + 'workdir': workdir} def run(self, commands, judge=True): """One JLinkExe session. judge=False is for a last `w4` that resets the target, which may drop the connection mid-transaction: only its issue on a connected target is judged (J-Link's "Writing VALUE -> ADDR" with no failure before it), and the caller's observation is the evidence.""" - script = self.dir / f'jlink-{time.monotonic_ns()}.jlink' - script.write_text('\n'.join(list(commands) + ['exit']) + '\n') - device = SESSION_DEVICE.get(self.b['device'], self.b['device']) - cmd = ['JLinkExe', '-USB', self.b['probe'], '-device', device, '-if', self.b['if'], '-speed', - self.b['speed'], '-autoconnect', '1', '-NoGui', '1', '-ExitOnError', '1', '-CommandFile', script] - code, out = spawn(cmd, self.log, PROBE_TIMEOUT) + return self.session(lambda run: provider().run_jlink_commands(self.flasher, list(commands), run=run, + **self.profile), commands, judge) + + def sysreset(self): + """The provider's SYSRESETREQ session (the AIRCR write), judged on its issue.""" + return self.session(lambda run: provider().sysresetreq_jlink(self.flasher, run=run, **self.profile), + [f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) + + def failure(self, attempts): + """jlink_failure over every attempt: before the target only when each one ended there.""" + return {'tail': text(attempts[-1].result.stdout).strip().splitlines()[-5:], + 'beforeTarget': before_target(attempts, lambda code, out: stopped_before_target(code, out, self.b['probe']))} + + def session(self, op, commands, judge): + run = tool_runner(self.log) + attempts = list(op(run).attempts) + code, out = attempts[-1].result.returncode, text(attempts[-1].result.stdout) if died_at_probe_connect(code, out, self.b['probe']): - self.retried.append({'exit': code, 'script': script.name}) + self.retried.append({'exit': code, 'commands': list(commands)}) with open(self.log, 'a') as f: f.write(f'# JLinkExe died (exit {code}) at the probe connect, before the target; retrying once\n') time.sleep(JLINK_RETRY_DELAY) - code, out = spawn(cmd, self.log, PROBE_TIMEOUT) - judged = re.sub(BENIGN[self.b['device']], '', out) if self.b['device'] in BENIGN else out + attempts += op(run).attempts + code, out = attempts[-1].result.returncode, text(attempts[-1].result.stdout) + if attempts[-1].timed_out: + raise StepFail(f'JLinkExe timed out after {PROBE_TIMEOUT}s', self.failure(attempts)) + seen = re.sub(BENIGN[self.b['device']], '', out) if self.b['device'] in BENIGN else out # a crash fails even a reset session (judge=False): its commands may not have run - if code < 0 or judge and (code or re.search(JLINK_ERRORS, judged)): - raise StepFail(f'JLinkExe failed (exit {code})', jlink_failure(code, out, self.b['probe'])) + if code < 0 or judge and (code or re.search(JLINK_ERRORS, seen)): + raise StepFail(f'JLinkExe failed (exit {code})', self.failure(attempts)) if not judge and commands[-1].startswith('w4 '): _, addr, value = commands[-1].split() - head, issued, _ = judged.partition(f'Writing {value} -> {addr}') + head, issued, _ = seen.partition(f'Writing {value} -> {addr}') if not issued or re.search(JLINK_ERRORS, head): - raise StepFail(f'the reset write was not issued on a connected target (exit {code})', - jlink_failure(code, out, self.b['probe'])) + raise StepFail(f'the reset write was not issued on a connected target (exit {code})', self.failure(attempts)) return out def mem32(self, addrs): @@ -717,6 +1270,96 @@ def write32(self, addr, value): return hex(value) +def chip_reported(out, board): + dev, nvm = STLINK_CHIPS[board['device']] + return bool(re.search(rf'^Device ID\s*: {dev}$', out, re.M) and re.search(rf'^NVM size\s*: {nvm}$', out, re.M)) + + +def stlink_failure(out): + """A session recognised as ending before any target connect: no ST-LINK with the serial (bench: "Error: Serial + number not found", before any device line). Any other failure may have reached the target.""" + before = re.search(r'^Error: Serial number not found', out, re.M) and not re.search(r'^Device ID', out, re.M) + return {'tail': out.strip().splitlines()[-5:], 'beforeTarget': bool(before)} + + +def judge_stlink_target(code, out, board): + """The provider's ST-LINK flash transcript, the record's tinyuf2-stlink command (bench, STM32CubeProgrammer + v2.23.0): the roster's chip, the HEX downloaded and the start command's success, with no error line.""" + out = ANSI.sub('', out) + m = re.search(r'^Opening and parsing file: tinyuf2\.hex$(?:\n.*)*?\nErasing internal memory sectors? \[?([\d ]+)\]?$' + r'(?:\n.*)*?\nFile download complete$(?:\n.*)*?\nStart operation achieved successfully$', out, re.M) + if code or not m or not chip_reported(out, board) or re.search(r'^Error', out, re.M): + raise StepFail(f'the ST-LINK flash failed (exit {code})', stlink_failure(out)) + return {'flashDownload': 'File download complete', 'erasedSectors': m[1].split()} + + +class StlinkProbe: + """One STM32CubeProgrammer CLI connection per session, through the flash provider's run_stlink_commands: a script + that connects to the roster's ST-LINK without a reset (HOTPLUG), then runs `ops` in order. The CLI stops at the + first failed operation and exits 1 (bench, v2.23.0); success is read from the transcript too: the roster's chip, + every script line echoed in order with its success line, no error line, the script's end.""" + + def __init__(self, board, workdir, log): + self.b, self.dir, self.log = board, workdir, log + self.retried = [] # the report's jlinkRetried: a CLI session is never retried + self.flasher = {'name': 'stlink', 'uid': board['probe'], 'args': f'-device {board["device"]}'} + self.profile = {'connect_mode': 'HOTPLUG', 'timeout': PROBE_TIMEOUT, 'workdir': workdir} + + def run(self, ops): + """Each operation's output, in order.""" + return self.judge(provider().run_stlink_commands(self.flasher, list(ops), run=tool_runner(self.log), + **self.profile), ops) + + def sysreset(self): + """The provider's SYSRESETREQ session: `-w32` with `-nv`.""" + op = provider().sysresetreq_stlink(self.flasher, run=tool_runner(self.log), **self.profile) + return self.judge(op, op.attempts[-1].script.splitlines()[1:]) + + def judge(self, op, ops): + a = op.attempts[-1] + code, out, lines = a.result.returncode, ANSI.sub('', text(a.result.stdout)), a.script.splitlines() + if a.timed_out: + raise StepFail(f'STM32_Programmer_CLI timed out after {PROBE_TIMEOUT}s', dict(stlink_failure(out), beforeTarget=False)) + parts = re.split(r'(?m)^Operation \[(\d+)\]: (.*?) *$', out) + if code or list(zip(parts[1::3], parts[2::3])) != [(str(i), x) for i, x in enumerate(lines, 1)] or \ + re.search(r'^Error', out, re.M) or not chip_reported(parts[3] if len(parts) > 3 else '', self.b) or \ + '=== Script Manager END ===' not in parts[-1] or \ + any(STLINK_DONE[re.split(r'[ (]', op)[0]] not in body for op, body in zip(ops, parts[6::3])): + raise StepFail(f'STM32_Programmer_CLI failed (exit {code})', stlink_failure(out)) + return parts[6::3] + + def mem32(self, addrs): + return {a: r32_value(body, a) for a, body in zip(addrs, self.run([f'-r32 0x{a:08X} 1' for a in addrs]))} + + def write32(self, addr, value): + """Halt, write, read back, resume: only the read-back proves the write.""" + got = r32_value(self.run(['-halt', f'-w32 0x{addr:08X} 0x{value:08X}', f'-r32 0x{addr:08X} 1', '-run'])[2], addr) + if got != value: + raise StepFail(f'{addr:#010x} reads back {got:08X}, not {value:#010x}') + return hex(value) + + +def r32_value(body, addr): + m = re.findall(rf'^0x{addr:08X} : ([0-9A-F]{{8}})$', body, re.M) + if len(m) != 1: + raise StepFail(f'-r32 gave no single value for {addr:#x}') + return int(m[0], 16) + + +def uploaded(path, want): + """Whether an `-u` operation's file holds exactly `want`.""" + return path.is_file() and path.read_bytes() == want + + +def core_regs(body): + def one(name): + m = re.findall(rf'^{name}\s*= 0x([0-9A-F]{{8}})$', body, re.M) + if len(m) != 1: + raise StepFail(f'no single {name} in the -coreReg output') + return int(m[0], 16) + return {'pc': one('PC'), 'r0': one('R0'), 'msp': one('MSP'), 'ipsr': one('XPSR') & 0x1FF} + + def esp_port(uid): """The tty of the USB-UART bridge with this serial: exactly one, never a guessed /dev/ttyUSBn.""" hits = sorted(glob.glob(f'/dev/serial/by-id/*_{glob.escape(uid)}-if*')) @@ -747,6 +1390,7 @@ def __init__(self, board, workdir, log): self.b, self.dir, self.log = board, workdir, log def run(self, ops, after): + refuse_pending(self.dir, self.b.get('pendingMark')) spec = self.dir / f'esp-{time.monotonic_ns()}.json' spec.write_text(json.dumps(ops)) return child_json([sys.executable, '-B', ESP_SESSION, '--port', esp_port(self.b['probe']), '--chip', self.b['device'], @@ -768,7 +1412,7 @@ def check_preserved(before, after, by): def esp_verify(tool, board, segments, workdir, snapshot): - """After the flash target: every segment read back byte for byte in one connection of the runner's, then one + """After the flash: every segment read back byte for byte in one connection of the runner's, then one reset. The preserved regions' MD5s must equal `snapshot`, taken before the install.""" reads = [(off, path, workdir / f'readback-{off:x}.bin') for off, path in segments] keep = board['regions']['preserved'] @@ -868,8 +1512,8 @@ def port_dir(cfg): class PortPower: - """The roster's usbPower (SKILL.md Roster): while DFU is awaited, the hub port's sysfs `disable` is written 1 then 0 - once the port sat in state "default" for STUCK s of this wait, at most once per GAP s.""" + """The roster's usbPower (SKILL.md Hosts/rosters): while DFU is awaited, the hub port's sysfs `disable` is written 1 + then 0 once the port sat in state "default" for STUCK s of this wait, at most once per GAP s.""" STUCK, GAP, OFF, TIMEOUT = 2.0, 10.0, 1, 15 RESTORED = {True: 'restored', False: 'NOT restored', None: 'not attempted: a cycle child survived'} # each write and its read-back in a bounded child: both take the hub's lock and talk to the hub @@ -906,7 +1550,7 @@ def poll(self): done = off and on finally: self.last = time.monotonic() - if not done: # an interruption or failure past the off write must not leave the port without VBUS + if not done: # an interruption or failure from the off write on must not leave the port without VBUS restored = self.restore(disable) with open(self.log, 'a') as f: f.write(f'# power-cycle of {port} failed; power-on {self.RESTORED[restored]}\n') @@ -951,7 +1595,7 @@ def stays_away(fn, seconds, what, ev): def exclusion_configured(facts, serial): - """A udev rule in exactly SKILL.md's form sets UDISKS_IGNORE=1 on this bootloader's VID, PID and + """A udev rule in exactly commissioning.md's form sets UDISKS_IGNORE=1 on this bootloader's VID, PID and serial, so desktop automount cannot race the raw I/O: any other predicate could keep it from applying when the device appears. Checked before any hardware access; the property on the enumerated device (check_exclusive) is the proof.""" @@ -968,8 +1612,9 @@ def exclusion_configured(facts, serial): continue if rule == want: return str(f) - raise Refused(f'no udev rule in SKILL.md\'s form sets ENV{{UDISKS_IGNORE}}="1" on idVendor ' - f'{want[("ATTRS{idVendor}", "==")]}, idProduct {want[("ATTRS{idProduct}", "==")]} and serial {serial}') + raise Refused(f'no udev rule in the required form sets ENV{{UDISKS_IGNORE}}="1" on idVendor ' + f'{want[("ATTRS{idVendor}", "==")]}, idProduct {want[("ATTRS{idProduct}", "==")]} and serial {serial} ' + '(commissioning.md: 99-tinyuf2-hil.rules)') def udisks_ignored(dev): @@ -984,7 +1629,8 @@ def check_exclusive(dev): try: wait_for(lambda: udisks_ignored(dev), UDEV_TIMEOUT, f'UDISKS_IGNORE=1 on {dev}') except StepFail: - raise StepFail(f'{dev} has no UDISKS_IGNORE=1: desktop automount could race the raw I/O (SKILL.md, Commissioning)') + raise StepFail(f'{dev} has no UDISKS_IGNORE=1: desktop automount could race the raw I/O ' + '(commissioning.md: 99-tinyuf2-hil.rules)') mounted = [l.split()[0] for l in Path('/proc/mounts').read_text().splitlines()] if any(os.path.realpath(m) == dev for m in mounted): raise StepFail(f'{dev} is mounted; refusing raw I/O') @@ -1081,6 +1727,37 @@ def fat_read(dev, name, length, log, offset=0): return {'entries': r['entries'], 'data': bytes.fromhex(r['data']), 'size': r['size']} +def clear_pending_main(args): + """`clear-pending`: remove a board's pending mark once each session it lists is ended, or after a separately + reviewed recovery whose note is recorded (SKILL.md Recovery). Exit 0 cleared or none, 2 refused.""" + note, recovery = args.after_reviewed_recovery, None + try: + if note is not None and not note.strip(): + raise Refused('--after-reviewed-recovery needs the note of the reviewed recovery') + if note is not None and (args.under_hold is None) == (args.recover_holder is None): + raise Refused("--after-reviewed-recovery needs proof of the board's hold: --under-hold or " + '--recover-holder ') + lock = load_board(args.config, args.board)['lock'] + helper = load_helper() + hold = None + if note is not None: # the note alone proves nothing: the board's existing hold does, never a new flock + recovery = recovery_lock(helper, lock) + if args.recover_holder is not None: + hold = prove_cleanup_holder(helper, lock, args.recover_holder) + else: + manual_holder(helper, lock, args.under_hold) + hold = helper.read_record(lock) + cleared = clear_pending_mark(PendingMark(helper, lock), note, hold) + except Refused as e: + print(json.dumps({'pass': False, 'error': str(e)}), flush=True) + return 2 + finally: + if recovery: + recovery.close() + print(json.dumps({'pass': True, 'cleared': cleared}), flush=True) + return 0 + + def fat_read_main(args): try: with Fat(args.dev) as fat: @@ -1096,17 +1773,43 @@ def fat_read_main(args): # ------------------------------------------------------------------ locks -def load_helper(path): - spec = importlib.util.spec_from_file_location('tinyusb_hil_lock', path) - if not spec or not Path(path).is_file(): - raise Refused(f'lock helper {path} not found') - mod = importlib.util.module_from_spec(spec) +def check_lock_pin(): + """Refuse unless lib/agentrc is checked out, clean, at the commit HEAD pins.""" + init = f'git submodule update --init {AGENTRC}' + if not LOCK_HELPER.is_file() or not cb.own_checkout(AGENTRC): + raise Refused(f'lock helper {LOCK_HELPER} is not checked out: {init}') try: - spec.loader.exec_module(mod) - except Exception as e: - raise Refused(f'lock helper {path} failed to load: {e}') - if set(getattr(mod, 'PROTECTED_REASONS', ())) != PROTECTED or getattr(mod, 'CI_REASON', None) != 'hil_test.py': - raise Refused(f'lock helper {path}: protected reasons are not {sorted(PROTECTED)}; recheck this contract') + want = cb.git('rev-parse', f'HEAD:{AGENTRC}').strip() + have = cb.git_in(AGENTRC, 'rev-parse', 'HEAD').strip() + except cb.Fail as e: + raise Refused(f'cannot read the {AGENTRC} pin: {e}') + if have != want: + # submodule update checks out the index's gitlink, so a staged pin change would defeat the hint + try: + staged = cb.git('rev-parse', f':{AGENTRC}').strip() + except cb.Fail as e: + raise Refused(f'cannot read the staged {AGENTRC} pin: {e}') + if staged != want: + raise Refused(f'{AGENTRC} is at {have}; HEAD pins {want} but {staged} is staged: commit or unstage that ' + f'{AGENTRC} pin change') + raise Refused(f'{AGENTRC} is at {have}; HEAD pins {want}: {init}') + try: + dirty = cb.dirty_inputs([AGENTRC]) + except cb.Fail as e: + raise Refused(f'cannot read the {AGENTRC} checkout state: {e}') + if dirty: + raise Refused(f'{AGENTRC} checkout is dirty at the pinned {want}: discard its local changes') + + +def load_helper(): + mod = load_pinned(LOCK_HELPER, 'hil_lock', (), 'lock helper') + lu = getattr(mod, 'LockUnavailable', None) + if getattr(mod, 'PROTECTED_REASONS', None) != PROTECTED or getattr(mod, 'CI_REASON', None) != 'hil_test.py' or \ + getattr(mod, 'CLEANUP_ROLE', None) != HOLDER_ROLE or not (isinstance(lu, type) and issubclass(lu, RuntimeError)) \ + or not all(callable(getattr(mod, f, None)) for f in ('_open_dir', '_open_file', '_read_file')): + raise Refused(f'lock helper {LOCK_HELPER}: protected reasons are not {sorted(PROTECTED)}, its CI reason is not ' + f"'hil_test.py', its cleanup role is not {HOLDER_ROLE!r}, it has no LockUnavailable or no checked file opens; " + 'recheck this contract') return mod @@ -1165,7 +1868,7 @@ def prove_hold(helper, board, pid, reason=None, role=None): def recovery_lock(helper, board): """An exclusive flock on `.recover`, kept until the caller drops it: one recovery per board, so - none releases the holder while another is still between its probe sessions. tinyusb's helper lists `*.lock` + none releases the holder while another is still between its probe sessions. The lock helper lists `*.lock` only, so it never reports this file as a holder.""" path = helper.lock_path(board) + '.recover' try: @@ -1180,6 +1883,17 @@ def recovery_lock(helper, board): return fh +def prove_cleanup_holder(helper, board, pid): + rec = prove_hold(helper, board, pid, reason='hil_test.py', role=HOLDER_ROLE) + try: + cmdline = Path(f'/proc/{pid}/cmdline').read_bytes() + except OSError as e: + raise Refused(f'pid {pid}: {e}') from None + if b'hil_test.py' not in cmdline: + raise Refused(f'pid {pid} is not a hil_test.py cleanup holder') + return rec + + def manual_holder(helper, board, reason): if reason in PROTECTED: raise Refused(f'{reason!r} is a protected holder; --under-hold takes the reason of your own manual hold') @@ -1192,22 +1906,37 @@ def manual_holder(helper, board, reason): def probe_users(board): """Processes using this board's probe: for J-Link one opened by serial (a JLinkExe or GDB server from an - earlier session), for esptool any process holding the bridge's tty open.""" - if board['flasher'] == 'esptool': - return tty_users(esp_port(board['probe'])) - probe, users = board['probe'], [] + earlier session), for ST-LINK any STM32CubeProgrammer CLI (its script names the probe, not its command line), for + esptool any process holding the bridge's tty open, for openocd_esp also any holding the USJ's usbfs node.""" + if board['flasher'] in ('esptool', 'openocd_esp'): # both devices resolved first, then one /proc walk + devs = [esp_port(board['probe'])] + ([usj_node(board['probe'])] if board['flasher'] == 'openocd_esp' else []) + return tty_users(*devs) + probe, users = board['probe'].encode(), [] for d in Path('/proc').iterdir(): try: cmd = (d / 'cmdline').read_bytes().split(b'\0') except OSError: continue - if d.name.isdigit() and int(d.name) != os.getpid() and any(b'JLink' in c for c in cmd[:1]) and \ - any(probe.encode() in c for c in cmd): + if not d.name.isdigit() or int(d.name) == os.getpid(): + continue + if board['flasher'] == 'stlink' and Path(cmd[0].decode(errors='replace')).name == STM32_PROGRAMMER_CLI or \ + board['flasher'] == 'jlink' and any(b'JLink' in c for c in cmd[:1]) and any(probe in c for c in cmd): users.append(int(d.name)) return users -def tty_users(dev): +def usj_node(serial): + """The USJ's usbfs node, which an openocd from an earlier session holds open, the device found as the provider + pins it.""" + try: + d = Path(openocd_esp().usb_observe.SYS_USB_DEVICES) / openocd_esp().usj_location(serial) + return f'/dev/bus/usb/{int((d / "busnum").read_text()):03d}/{int((d / "devnum").read_text()):03d}' + except (RuntimeError, OSError, ValueError) as e: + raise StepFail(f'USJ {serial}: {e}') + + +def tty_users(*devs): + """Processes holding any of `devs` open, each once.""" users = [] for d in Path('/proc').iterdir(): if not d.name.isdigit() or int(d.name) == os.getpid(): @@ -1216,7 +1945,7 @@ def tty_users(dev): fds = list((d / 'fd').iterdir()) except OSError: continue - if any(readlink(fd) == dev for fd in fds): + if any(readlink(fd) in devs for fd in fds): users.append(int(d.name)) return users @@ -1238,19 +1967,42 @@ def holder_record(fh, board, detail, survivors): HELD_IN_PROCESS = [] +# the GitHub Actions runner kills every process whose environment carries the job's tracking id when the job +# ends (actions/runner JobExtension.FinalizeJob), so a holder forked under it would free a board that was +# never cleaned up: the holder re-executes itself without that variable +JOB_TRACKING = 'RUNNER_TRACKING_ID' +HOLDER_SCRIPT = Path(__file__).resolve() + + +def exec_holder(fh, ack): + """Replace this process with `hil_test.py keep-held `: the same open file description (and its + flock) and the acknowledgement pipe passed on, the job tracking variable dropped. The replacement + acknowledges only once it runs, so a holder that dies at startup is never acknowledged. Returns only + when the exec itself fails, to keep holding in this process.""" + for fd in (fh.fileno(), ack): + os.set_inheritable(fd, True) + env = {k: v for k, v in os.environ.items() if k != JOB_TRACKING} + with contextlib.suppress(OSError): + os.execve(sys.executable, [sys.executable, str(HOLDER_SCRIPT), 'keep-held', str(fh.fileno()), str(ack)], env) def keep_held(fh): """Hold until recovery's SIGUSR1, taken by sigwait with SIGUSR1 blocked since hand_off: a handler and pause() lose a signal that lands between the interpreter's check and the syscall. SIGTERM and SIGINT - are ignored: an operator's timeout kill must not free a board that was never cleaned up.""" + are ignored: an operator's timeout kill must not free a board that was never cleaned up. Meanwhile each build lock + this process kept (HELD_BUILD) is released once its survivors end, as restore() needs it before recovery signals.""" for sig in (signal.SIGTERM, signal.SIGINT): signal.signal(sig, signal.SIG_IGN) signal.pthread_sigmask(signal.SIG_SETMASK, {signal.SIGUSR1}) for std in (0, 1, 2): # closed, not reopened on /dev/null: holding must not need a free descriptor with contextlib.suppress(OSError): os.close(std) - signal.sigwait({signal.SIGUSR1}) + signalled = False + with contextlib.suppress(Exception): # a failed look keeps the build locks, never frees the board + while not signalled and release_ended_builds(): + signalled = signal.sigtimedwait({signal.SIGUSR1}, KEEPER_POLL) is not None + if not signalled: + signal.sigwait({signal.SIGUSR1}) fh.truncate(0) os._exit(0) @@ -1274,6 +2026,7 @@ def hand_off(fh, board, detail, survivors=()): os.close(r) os.setsid() holder_record(fh, board, detail, survivors) + exec_holder(fh, w) # the replacement acknowledges once it runs; a failed exec acknowledges here os.write(w, b'ok') os.close(w) keep_held(fh) @@ -1307,7 +2060,7 @@ def verify_record(record_path, board, facts, workdir): if not rec.get('pass') or len(rows) != 1 or rows[0].get('status') != 'ok' or rows[0].get('removed'): raise Refused(f'{record_path}: not a passing --shared build of {board["name"]}') row, info = rows[0], rec.get('input', {}) - want = 'idf' if board.get('flasher') == 'esptool' else 'cmake' + want = 'idf' if board.get('kind') == 'esp' else 'cmake' if row.get('backend') != want: raise Refused(f'{record_path}: a {row.get("backend")} build; the HIL takes {want} builds only for this board ' f'(check_build.py --board {board["name"]}{" --backend cmake" if want == "cmake" else ""} --shared ' @@ -1333,8 +2086,8 @@ def verify_record(record_path, board, facts, workdir): if bad: raise Refused(f'build inputs are not clean: {bad} (commit, or rebuild from a clean checkout)') root = cb.ROOT / row['buildDir'] - flash = admit_flash(record_path, rec, row, root, board) - # the runner reads its own copies of the BIN, segments and blinky: the build dir's lock covers the flash target + flash = admit_flash(record_path, row, root, board) + # the runner reads its own copies of the BIN, segments and blinky: the build dir's lock covers the flash # only, and a rebuild may replace the dir after it copy = workdir / 'record' if want == 'idf': @@ -1350,18 +2103,21 @@ def verify_record(record_path, board, facts, workdir): 'defines': rec.get('defines', []), 'version': embedded_version(files['bin'])} -def admit_flash(record_path, rec, row, root, board): - """The record's flash target, bound by check_build --hil-config to this roster flasher, with every file it reads - hash-checked; the install runs that target, under the build dir's lock.""" - fl = row.get('flash') or {} - target, inputs = cb.FLASH[board['flasher']] +def admit_flash(record_path, row, root, board): + """The record's flash binding (check_build --hil-config: its flash target checked against this roster flasher at + the build), with every file the install reads hash-checked; the install reads them under the build dir's lock, + through the provider, never the target.""" want = {'name': board['flasher'], 'uid': board['probe'], 'device': board['device']} - if (fl.get('flasher'), fl.get('target'), fl.get('inputs')) != (want, target, list(inputs)): + if board['flasher'] in cb.ROUTE_ARGS: # a further route: the esptool binding's segments, bound to this route + fl, (target, inputs) = (row.get('flashRoutes') or {}).get(board['flasher']) or {}, cb.FLASH['esptool'] + bound = (fl.get('flasher'), fl.get('inputs')) == (want, list(inputs)) + else: + fl, (target, inputs) = row.get('flash') or {}, cb.FLASH[board['flasher']] + bound = (fl.get('flasher'), fl.get('target'), fl.get('inputs')) == (want, target, list(inputs)) + if not bound: raise Refused(f'{record_path}: its flash target is not bound to this roster flasher {want} (rebuild it with ' f'check_build.py --board {board["name"]} ... --shared --hil-config )') - idf = rec.get('idf') or {} - return {'root': root, 'binding': fl, 'sha256': recorded(root, inputs, row), 'image': idf.get('image'), - 'ninja': functools.partial(cb.ninja_in, row['backend'], idf.get('mode'), idf.get('image'))} + return {'root': root, 'binding': fl, 'sha256': recorded(root, inputs, row)} def recorded(root, rels, row): @@ -1441,7 +2197,7 @@ def __init__(self, board, facts, workdir, fail_at=None, commissioning=False): self.log = workdir / 'hil.log' self.probe = Probe(board, workdir, self.log) self.power = PortPower(board.get('usbPower'), self.log) - self.steps, self.usb_path, self.touched, self.at = [], None, False, 'setup' + self.steps, self.usb_path, self.touched, self.at, self.kept = [], None, False, 'setup', [] # where CURRENT.UF2 starts and the flash size INFO_UF2.TXT reports self.app_start, self.dfu_flash = facts.get('appStart'), facts['flashSize'] self.dfu_claim = 'DFU observed after the install' @@ -1458,31 +2214,38 @@ def step(self, name, fn): raise self.steps.append({'step': name, 'pass': True, 'evidence': ev, 'detail': ''}) - def flash_target(self, flash, cmd, timeout, judge): - """The record's flash target, under its build dir's lock: the inputs still hash as recorded and a dry run shows - the target runs nothing but its flasher (cb.flash_ready); after it, the inputs are hashed again. Touched from the - target's start; only a transcript recognised as ending before any target connect leaves the board as it was.""" - root, fl = flash['root'], flash['binding'] - with contextlib.ExitStack() as held: - try: - held.enter_context(cb.build_lock(root, wait=False)) - except BlockingIOError: - raise StepFail(f'{root.name} is in use by a build or another flash; nothing was flashed') + def flash_target(self, flash, op, judge, before, what, timeout): + """The record's image through the flash provider, `op(run)` with tool_runner's run, under its build dir's lock: + the inputs still hash as recorded before it, and again after it; judged(). Touched from + the tool's start; only attempts that all ended before any target connect leave the board as it was. The lock + is released only once every child the operation started has ended, reaped here when interrupted; one that + survives SIGKILL keeps it through a keeper (#579), and the board through cleanup's; so does an openocd_esp + session left running. The flash then fails however it went, so nothing follows it while either lives.""" + refuse_pending(self.dir, self.b.get('pendingMark')) + started, lock = list(CHILDREN), take_build_lock(flash['root']) + flashed = False + try: self.check_inputs(flash, 'before the flash') - try: - cb.flash_ready(root, fl, flash['ninja']) - except cb.Fail as e: - raise StepFail(str(e)) was, self.touched = self.touched, True - code, out = spawn(cmd, self.log, timeout) try: - ev = judge(code, out) + ev = judged(op(tool_runner(self.log)), judge, before, what, timeout) except StepFail as e: if e.evidence.get('beforeTarget'): self.touched = was raise self.check_inputs(flash, 'while flashing', ev) - return ev + flashed = True + return ev + finally: + with signals(signal.SIG_BLOCK): + left = settle_children(started) + outstanding_sessions(self.dir) + if left: + self.kept.append({'buildDir': flash['root'].name, 'survivors': left, 'keeper': keep_build_lock(lock, left)}) + else: + lock.close() + if left and flashed: + raise CleanupPending(f'{what}: children survived SIGKILL or a session still runs; the build lock stays with a ' + 'keeper and the board with cleanup', {'survivors': left}) @staticmethod def check_inputs(flash, when, ev=None): @@ -1492,10 +2255,10 @@ def check_inputs(flash, when, ev=None): def dfu_after_install(self, version): """DFU after an install: on the RT boards J-Link's download erased the bootloader's whole 64 KiB block on the - bench, the app start with it (SKILL.md Limits), so DFU comes up; should enumeration time out instead, as when a - valid app boots (the F303's install keeps the app; there the verify session's reset may land in TinyUF2's - double-tap window instead, SKILL.md Limits), the board's commissioned retained-register re-entry brings DFU up. - Any other failure fails as it is.""" + bench, the app start with it (bench-history.md, RT boards), so DFU comes up; should enumeration time out + instead, as when a valid app boots (the F303's install keeps the app; there the verify session's reset may land + in TinyUF2's double-tap window instead, bench-history.md, stm32f303disco), the board's commissioned + retained-register re-entry brings DFU up. Any other failure fails as it is.""" try: return self.dfu(version) except NotEnumerated as e: @@ -1507,9 +2270,8 @@ def dfu_after_install(self, version): def retained_reentry(self, version, claim): f = self.f self.probe.write32(f['dblTapReg'], f['dblTapMagic']) - # SYSRESETREQ may drop the debug connection mid-write: its exit is not judged, enumeration is; # no probe session opens again until DFU is observed - tail = self.probe.run([f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) + tail = self.sysreset() ev = dict(self.dfu(version), dfuCameUp=True, resetTail=tail.strip().splitlines()[-3:], claim=claim) try: left = self.probe.mem32([f['dblTapReg']])[f['dblTapReg']] @@ -1521,6 +2283,11 @@ def retained_reentry(self, version, claim): raise StepFail('the double-tap register was not consumed', ev) return ev + def sysreset(self): + """SYSRESETREQ from the probe. It may drop the debug connection mid-write: its exit is not judged, its issue is; + the caller's observation is the evidence.""" + return self.probe.sysreset() + def dfu(self, version): f = self.f self.power.start() @@ -1633,13 +2400,17 @@ def restore(self, files, version): def report_fields(self): # the cleanup's retries and cycles land here too: the same lists - return {'jlinkRetried': self.probe.retried, **({'portCycles': self.power.cycles} if self.power.cfg else {})} + return {'jlinkRetried': self.probe.retried, **({'portCycles': self.power.cycles} if self.power.cfg else {}), + **({'buildLocksKept': self.kept} if self.kept else {})} def install(self, files): - """The record's tinyuf2-jlink target, then the BIN verified at its flash address in a session of the runner's.""" - fl = files['flash'] - ev = self.flash_target(fl, ['cmake', '--build', fl['root'], '--target', fl['binding']['target']], PROBE_TIMEOUT, - lambda code, out: judge_jlink_target(code, out, self.b, files['hex'])) + """The record's HEX through the provider's J-Link flash (the script the record's tinyuf2-jlink target runs), then + the BIN verified at its flash address in a session of the runner's.""" + def flash(run): + return provider().flash_jlink(self.probe.flasher, str(files['hex']), timeout=PROBE_TIMEOUT, run=run, + workdir=str(self.dir)) + ev = self.flash_target(files['flash'], flash, lambda code, out: judge_jlink_target(code, out, self.b, files['hex']), + lambda code, out: stopped_before_target(code, out, self.b['probe']), 'JLinkExe', PROBE_TIMEOUT) settle = INSTALL_SETTLE_MS.get(self.b['device']) verify = ['h', f'verifybin {files["bin"]},{self.f["binBase"]:#x}', 'r', 'g'] self.probe.run(([f'sleep {settle}'] if settle else []) + verify) @@ -1662,7 +2433,12 @@ def observe_app(self, app_bin): m = re.search(rf'^{SCB_VTOR:08X} = ([0-9A-F]{{8}})', out, re.M) if not m: raise StepFail('no VTOR read in the transcript') - vtor = int(m.group(1), 16) + return self.app_evidence(first, second, int(m.group(1), 16)) + + def app_evidence(self, first, second, vtor): + """The fixture's two samples: the counter moving, PC at its handler in thread mode, VTOR and MSP from its vector + table; its image was verified in the same session.""" + f = self.f ev = {'pc': hex(first['pc']), 'ipsr': first['ipsr'], 'vtor': hex(vtor), 'msp': hex(first['msp']), 'r0': [hex(first['r0']), hex(second['r0'])], 'verifybin': True, 'scope': 'executes through bootloader handoff after the observed reset; upload-triggered run unverified'} @@ -1678,13 +2454,62 @@ def control(self, app_bin): arms the magic itself for its double-tap window (src/main.c), so a session that reset twice would re-enter DFU with no magic written; USB is watched past that window before any probe reconnects.""" f = self.f - self.probe.run([f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) + self.sysreset() watch = 2 * f['dblTapDelayMs'] / 1000 + USB_TIMEOUT stays_away(lambda: usb_device(f['vid'], f['pid'], self.b['uid']), watch, 'the reset without the magic entered DFU', {'controlBootedApp': False}) return {'control': self.observe_app(app_bin), 'controlWatchedS': watch, 'controlBootedApp': True} +class StlinkBattery(Battery): + """The J-Link battery's steps on an ST-LINK board, each probe session one STM32CubeProgrammer script.""" + + def __init__(self, board, facts, workdir, fail_at=None, commissioning=False): + super().__init__(board, facts, workdir, fail_at, commissioning) + self.probe = StlinkProbe(board, workdir, self.log) + + def readback(self, addr, want): + return self.dir / f'readback-{time.monotonic_ns()}.bin', f'-u 0x{addr:08X} {len(want):#x}' + + def install(self, files): + """The record's HEX through the provider's ST-LINK flash (the command the record's tinyuf2-stlink target runs, + without the CLI's verify), then the BIN read back from its flash address in a session of the runner's, which ends + in a system reset (bench: the core runs after `-rst`).""" + ev = self.flash_target(files['flash'], lambda run: provider().flash_stlink(self.probe.flasher, str(files['hex']), + timeout=PROBE_TIMEOUT, run=run), + lambda code, out: judge_stlink_target(code, out, self.b), + lambda code, out: stlink_failure(ANSI.sub('', out))['beforeTarget'], 'STM32_Programmer_CLI', + PROBE_TIMEOUT) + want = Path(files['bin']).read_bytes() + path, op = self.readback(self.f['binBase'], want) + self.probe.run(['-halt', f'{op} {path}', '-rst']) + if not uploaded(path, want): + raise StepFail('the BIN does not read back from its flash address', ev) + return dict(ev, verified=True) + + def sysreset(self): + """SYSRESETREQ without the CLI's read-back, which the reset would fail (bench: `-nv` writes and exits 0, the host + sees the device re-enumerate); issued only once the write is reported done.""" + body = self.probe.sysreset()[0] + if f'w @0x{SCB_AIRCR:08x}: 0x{AIRCR_SYSRESETREQ:08x}' not in body: + raise StepFail('the reset write was not reported done', {'tail': body.strip().splitlines()[-3:]}) + return body + + def observe_app(self, app_bin): + """observe_app's session as one script: two halts in one connection, the image read back in it.""" + f = self.f + want = Path(app_bin).read_bytes() + path, op = self.readback(f['appStart'], want) + vtor = f'-r32 0x{SCB_VTOR:08X} 1' + ops = ['-halt', '-run', f'#Delay({2 * f["dblTapDelayMs"] + 200})', '-halt', '-coreReg', vtor, '-run', '#Delay(10)', + '-halt', '-coreReg', f'{op} {path}', '-run'] + out = self.probe.run(ops) + if not uploaded(path, want): + raise StepFail('the fixture image does not read back from the app start') + first, second = (core_regs(body) for o, body in zip(ops, out) if o == '-coreReg') + return self.app_evidence(first, second, r32_value(out[ops.index(vtor)], SCB_VTOR)) + + def nonce_block(nonce): """apps/dfu_runtime's nonce block (dfu_runtime.h): magic, LE length, the nonce, then each byte its offset.""" head = NONCE_MAGIC + len(nonce).to_bytes(4, 'little') + nonce @@ -1718,8 +2543,30 @@ def esp_app_chunks(uf2_path, facts, board): return {'appChunks': [bl['data'] for bl in blocks], 'nonceOffset': off} +def security_evidence(res, tty): + """The `security` the provider requires, from the esp_session security op: off only when both the ROM's security + info and the eFuse registers read off; anything else refuses the JTAG flash before it starts.""" + off = {'flashEncryption': 'off', 'secureBoot': 'off'} + if res.get('op') != 'security' or res.get('rom') != off or res.get('efuse') != off: + raise StepFail(f'flash encryption or secure boot is not read off on this chip: {res}; openocd_esp writes plaintext, ' + 'so nothing was programmed', {'security': res}) + return dict(off, source=f'esptool {ESPTOOL_VERSION} on {tty}, ROM loader: GET_SECURITY_INFO flags {res["flags"]:#010x}, ' + f'flash_crypt_cnt {res["flashCryptCnt"]:#x}; eFuse SPI_BOOT_CRYPT_CNT and SECURE_BOOT_EN off') + + +def judge_openocd_esp(op): + """flash_openocd_esp's own record: every segment written, its bank hash verified and read back byte for byte, + then the reviewed end (reset run, shutdown, exit confirmed), so cleanup done.""" + segs = [{'offset': hex(s['offset']), 'verified': bool((s.get('program') or {}).get('verified')), + 'readBack': bool((s.get('readBack') or {}).get('match'))} for s in getattr(op, 'segments', ())] + if op.returncode != 0 or op.cleanup != 'done' or not segs or not all(s['verified'] and s['readBack'] for s in segs): + raise StepFail(f'the openocd_esp flash failed (exit {op.returncode}): {op.session.get("failure")}', + {'segments': segs, 'cleanup': op.cleanup, 'log': op.session.get('log')}) + return {'wrote': [s['offset'] for s in segs], 'jtag': {k: op.session.get(k) for k in ('openocd', 'location', 'log')}} + + class EspBattery(Battery): - """The battery on an esptool board. TinyUF2 copies apps/dfu_runtime with this run's nonce block past the + """The battery on an Espressif board. TinyUF2 copies apps/dfu_runtime with this run's nonce block past the image; the app reports the nonce in its DFU runtime interface string and returns to DFU on DFU_DETACH.""" def __init__(self, board, facts, workdir, fail_at=None, commissioning=False): super().__init__(board, facts, workdir, fail_at, commissioning) @@ -1736,38 +2583,82 @@ def after_copy(self): def install(self, files): """The preserved regions' snapshot first, in a session of its own, which the report keeps however the run ends: - a recovery's restore checks against it. A restore reuses the run's snapshot, or a recovery's. Then ESP-IDF's - flash target in CI's image, given only the bridge's tty, in a container named for the board so a survivor is - found and ended by name (cleanup, recovery); then the runner's own read-back.""" - tty = esp_port(self.b['probe']) # no bridge fails here, untouched; any session may leave the chip in its ROM loader - name = f'tinyuf2-hil-{self.b["name"]}' - if container_present(name): - raise StepFail(f'container {name} from an earlier flash still exists: check what it runs, then docker rm -f {name}') - fl = files['flash'] - # created, not run: the container exists, and its name is tracked, before anything can start it; cleanup - # removes it by name however the run ends. IDF_FLASH_FULL: v6's fast reflash skips segments already in - # flash, and judge_esp_target needs every segment written and verified - CONTAINERS.append(name) - code, out = spawn(cb.idf_docker(fl['root'], 'create') + [ - '--name', name, '--device', tty, '--group-add', str(os.stat(tty).st_gid), '-e', f'ESPPORT={tty}', - '-e', 'IDF_FLASH_FULL=1', fl['image'], - 'cmake', '--build', str(fl['root']), '--target', fl['binding']['target']], self.log, 60) - if code: - raise StepFail(f'docker create {name} failed (exit {code})', {'tail': out.strip().splitlines()[-3:]}) + a recovery's restore checks against it. A restore reuses the run's snapshot, or a recovery's. Then the record's + segments through the provider's esptool flash, the runner's own esptool on the transport's tty, from the build + dir under its lock as flash_args names them; then the runner's own read-back. The openocd_esp route: jtag().""" + if self.b['flasher'] == 'openocd_esp': + return self.jtag(files) + tty = esp_port(self.b['probe']) # no tty fails here, untouched; any session may leave the chip in its ROM loader self.touched = True if self.preserved is None: keep = self.b['regions']['preserved'] self.preserved = preserved_md5(keep, self.tool.run(preserved_ops(keep), 'no-reset')) - ev = self.flash_target(fl, ['docker', 'start', '-a', name], ESP_TIMEOUT, - lambda code, out: judge_esp_target(code, out, files['segments'])) - if container_gone(name): - CONTAINERS.remove(name) + fl = files['flash'] + + def flash(run): # under the build dir's lock, its inputs checked: flash_args from bytes that hash as recorded + raw = (fl['root'] / 'flash_args').read_bytes() + if hashlib.sha256(raw).hexdigest() != fl['sha256']['flash_args']: + raise StepFail('flash_args changed after the inputs were checked; nothing was flashed') + # the settings line, then one "OFFSET FILE" per segment, relative to the build dir + return provider().flash_esptool({'name': 'esptool', 'uid': self.b['probe'], 'args': ''}, chip=self.b['device'], + port=tty, flash_args=' '.join(raw.decode().split()), cwd=str(fl['root']), + timeout=ESP_TIMEOUT, run=run) + ev = self.flash_target(fl, flash, + lambda code, out: judge_esp_target(code, out, files['segments']), lambda code, out: False, + 'esptool', ESP_TIMEOUT) + ev.update(esp_verify(self.tool, self.b, files['segments'], self.dir, self.preserved)) + return {'wrote': ev['wrote'], 'writes': ev['writes'], 'readBack': True, 'esptool': ESPTOOL_VERSION, + 'operation': ev['operation']} + + def jtag(self, files): + """The JTAG route under the one board lock, each step only after the previous one's cleanup completed: an esptool + session over the USJ's tty reads the security state on the ROM loader, takes the snapshot (a restore reuses + the run's) and hard-resets the chip out of the loader; agentrc's flash_openocd_esp programs the record's segments with verify, reads each back and ends + with reset run and a confirmed shutdown; then the runner's esptool read-back and preservation check. A session + left running is cleanup pending, and nothing follows it.""" + refuse_ci_job(self.b['flasher']) # before the security and snapshot session, as the provider before its own + tty = esp_port(self.b['probe']) + self.touched = True + keep = self.b['regions']['preserved'] + # hard-reset: a chip left in the ROM loader faults openocd's flash stub after its reset init (PMP Load access fault) + res = self.tool.run([{'op': 'security'}] + (preserved_ops(keep) if self.preserved is None else []), 'hard-reset') + security = security_evidence(res[0], tty) + if self.preserved is None: + self.preserved = preserved_md5(keep, res[1:]) + segments = [{'offset': off, 'file': str(p), 'size': p.stat().st_size} for off, p in files['segments']] + seen = [] + time.sleep(OPENOCD_BOOT_SETTLE) + + def flash(run): # the provider runs its own server, never a tracked child: run is unused + op = None + try: + op = openocd_esp().flash_openocd_esp(self.b['route'], segments, security=security, workdir=str(self.dir), + timeout=OPENOCD_TIMEOUT) + except BaseException as e: + op = getattr(e, 'operation', None) + if isinstance(e, Interrupted) or not isinstance(e, Exception): + raise + if op is None: + raise StepFail(f'openocd_esp refused the flash before starting: {e}') from None + finally: # before anything propagates: every receipt kept and marked + protect(op) + left = outstanding_sessions(self.dir) + mark_pending(self.b.get('pendingMark'), left) + seen.append(op) + if op.cleanup == 'pending' or left: + raise CleanupPending(f'the openocd_esp session is not ended ({op.session.get("failure")}): cleanup pending, ' + f'receipt {op.session["receipt"]}', {'operation': operation_record(op)}) + return op + ev = self.flash_target(files['flash'], flash, lambda code, out: judge_openocd_esp(seen[-1]), lambda code, out: False, + 'openocd_esp', OPENOCD_TIMEOUT) ev.update(esp_verify(self.tool, self.b, files['segments'], self.dir, self.preserved)) - return {'wrote': ev['wrote'], 'writes': ev['writes'], 'readBack': True} + return {'route': 'openocd_esp', 'wrote': ev['wrote'], 'writes': ev['writes'], 'readBack': True, 'security': security, + 'jtag': ev['jtag'], 'esptool': ESPTOOL_VERSION, 'operation': ev['operation']} def report_fields(self): keep = self.b['regions']['preserved'] - return {'snapshot': self.preserved and {'preserved': self.preserved, + return {**({'buildLocksKept': self.kept} if self.kept else {}), + 'snapshot': self.preserved and {'preserved': self.preserved, 'preservedRegions': {n: list(r) for n, r in keep.items()}}} def usb_dfu(self, op): @@ -1836,7 +2727,7 @@ def run(self, files, version): claim='a fresh copy boots the app after re-entry')) -BATTERIES = {'jlink': Battery, 'esptool': EspBattery} +BATTERIES = {'jlink': Battery, 'stlink': StlinkBattery, 'esp': EspBattery} TERM_SIGNALS = {signal.SIGTERM, signal.SIGINT} @@ -1865,7 +2756,14 @@ def check_jlink(): raise Refused('JLinkExe must be on PATH') +def check_stlink(): + if shutil.which(STM32_PROGRAMMER_CLI) is None: + raise Refused(f'{STM32_PROGRAMMER_CLI} must be on PATH') + + def check_esptool(): + """The runner's own esptool ESPTOOL_VERSION: its package for this Python (esp_session.py), and the `esptool` command + the provider runs, an entry script of this same Python, so the flash and the sessions run one esptool.""" import importlib.metadata try: v = importlib.metadata.version('esptool') @@ -1873,6 +2771,28 @@ def check_esptool(): v = None if v != ESPTOOL_VERSION: raise Refused(f'esptool {v} is installed for {sys.executable}; the runner needs esptool {ESPTOOL_VERSION}') + exe = shutil.which('esptool') + try: + with open(exe or '', 'rb') as f: + line = f.readline().decode(errors='replace').strip() + except OSError: + line = '' + py = line[2:].split()[0] if line.startswith('#!') and line[2:].split() else '' + # the same directory, as a venv's python links out of it, and the same interpreter, as one bin dir holds several + bindir = [os.path.realpath(os.path.dirname(a)) for a in (py, sys.executable)] + interpreter = [os.path.realpath(a) for a in (py, sys.executable)] + if not py or bindir[0] != bindir[1] or interpreter[0] != interpreter[1]: + raise Refused(f'the esptool on PATH ({exe}) does not run under {sys.executable}, whose esptool {ESPTOOL_VERSION} the ' + f'runner checked: put that installation\'s esptool first on PATH') + + +def check_openocd_esp(): + """The verified openocd-esp32 the provider runs, and the runner's esptool for the sessions over the USJ's tty.""" + check_esptool() + try: + openocd_esp().find_openocd() + except RuntimeError as e: + raise Refused(str(e)) def check_pyusb(): @@ -1885,18 +2805,19 @@ def check_pyusb(): def check_flasher(board): - check_esptool() if board['flasher'] == 'esptool' else check_jlink() + {'esptool': check_esptool, 'stlink': check_stlink, 'openocd_esp': check_openocd_esp}.get(board['flasher'], check_jlink)() if board.get('usbPower'): try: disable = port_dir(board['usbPower']) / 'disable' except StepFail as e: raise Refused(f'{board["name"]}: usbPower: {e}') if not os.access(disable, os.W_OK): - raise Refused(f'{board["name"]}: usbPower: {disable} is not writable (SKILL.md, Commissioning: Port power)') + raise Refused(f'{board["name"]}: usbPower: {disable} is not writable ' + '(commissioning.md: 52-tinyuf2-hil-uhubctl.rules)') def reentry_decision(row): - """The re-entry method a commissioning run establishes (SKILL.md, Commissioning), or None: every + """The re-entry method a commissioning run establishes (commissioning.md, Re-entry), or None: every observation of the tried method held, the board was cleaned up and released, and the run passed or failed only at reenter's CURRENT.UF2 readback.""" cleanup = row.get('cleanup') or {} @@ -1916,7 +2837,7 @@ def reentry_decision(row): def admit_record(record_path, board, facts, workdir): """A build record the battery and restore may install, with what they need on this host.""" files, fw = verify_record(record_path, board, facts, workdir) - if board['flasher'] == 'jlink': + if board['kind'] != 'esp': data = parse_hex(files['hex']) check_ranges(data, [board['install']], FLASH_LIMITS[board['device']], files['hex']) if not hex_matches_bin(data, files['bin'], facts['binBase']): @@ -1935,38 +2856,69 @@ def trial_reentry(board, method): return board if board['reentry'] is not None: raise Refused(f'{board["name"]}: reentry is already commissioned; --try-reentry is for commissioning only') - if method not in REENTRY[board['flasher']]: - raise Refused(f'{board["name"]}: a {board["flasher"]} board re-enters by one of {REENTRY[board["flasher"]]}') + if method not in REENTRY[board['kind']]: + raise Refused(f'{board["name"]}: its battery re-enters by one of {REENTRY[board["kind"]]}') return dict(board, reentry=method) def run_board(args, board, facts, helper, workdir): row = {'board': board['name'], 'lock': board['lock'], 'ran': False, 'pass': False, 'locked': False, 'lockBypassed': False, 'steps': [], 'cleanup': None, 'detail': ''} - methods = REENTRY[board['flasher']] - if args.fail_at and args.fail_at not in (*STEPS[board['flasher']], 'cleanup'): - raise Refused(f'{board["name"]}: the {board["flasher"]} battery has no step {args.fail_at}') + if board['route']: + row['route'] = board['flasher'] + methods = REENTRY[board['kind']] + if args.fail_at and args.fail_at not in (*STEPS[board['kind']], 'cleanup'): + raise Refused(f'{board["name"]}: its battery has no step {args.fail_at}') if args.try_reentry: board = trial_reentry(board, args.try_reentry) row.update(commissioning=True, tried=args.try_reentry) elif board['reentry'] not in methods: raise Refused(f'{board["name"]}: tinyuf2.reentry must be one of {methods} (set from commissioning)') files, row['firmware'] = admit_record(args.build_record, board, facts, workdir) - if board['flasher'] == 'esptool': + if board['kind'] == 'esp': check_pyusb() - bypass = os.environ.get('HIL_NO_BOARD_LOCK') == '1' # blocked from here until the process exits or becomes the holder (keep_held): a signal acts only # inside the battery, and the report and any hold are settled first signal.pthread_sigmask(signal.SIG_BLOCK, TERM_SIGNALS) + deadline = time.monotonic() + args.lock_wait + while True: + try: + fh = helper.acquire_board_lock(board['lock']) + break + except helper.LockUnavailable as e: + raise Refused(f'{board["name"]}: {e}; refusing to run unlocked') + except RuntimeError as e: + # a cleanup holder never frees on its own (SKILL.md Recovery): a blocker, never waited for + blocker, held = helper.is_cleanup_holder(helper.read_record(board['lock'])), str(e) + left = deadline - time.monotonic() + if blocker or left <= 0: + row.update(locked=True, recoveryBlocker=blocker, detail=held) + return deferred(row), 1 + try: + with signals(signal.SIG_UNBLOCK): + time.sleep(min(LOCK_POLL, left)) + except Interrupted as e: # nothing taken yet: the board's row is a locked one + row.update(locked=True, recoveryBlocker=False, detail=f'{held}; {e} while waiting for the lock') + return deferred(row), 1 + if time.monotonic() >= deadline: # no try past the budget, whatever the lock does meanwhile + row.update(locked=True, recoveryBlocker=False, detail=held) + return deferred(row), 1 + row['lockBypassed'] = fh is None try: - fh = helper.acquire_board_lock(board['lock']) - except RuntimeError as e: - row.update(locked=True, detail=str(e)) + users = probe_users(board) + except StepFail as e: # an Espressif board without exactly one tty: not run, nothing touched + row['detail'] = f'probe check: {e}' + if fh is not None: + helper.clear_record(fh) + fh.close() return deferred(row), 1 - if fh is None and not bypass: - raise Refused(f'{board["name"]}: the lock dir is unusable (tinyusb helper fails open); refusing to run unlocked') - row['lockBypassed'] = fh is None - bat = BATTERIES[board['flasher']](board, facts, workdir, args.fail_at, row.get('commissioning', False)) + if users: # a session from an earlier run, left on this probe: nothing may overlap it (SKILL.md Recovery) + row.update(probeUsers=users, detail=f'probe {board["probe"]} is still in use by pids {users}; no session may overlap them') + if fh is not None: + helper.clear_record(fh) + fh.close() + return deferred(row), 2 + bat = BATTERIES[board['kind']](board, facts, workdir, args.fail_at, row.get('commissioning', False)) code = 0 try: row['ran'] = True @@ -1996,20 +2948,29 @@ def clean_up(row, code, bat, fh, board, helper, files): SIGKILL, or any unsuccessful cleanup, leaves the board held (exit 4), touched or not.""" stuck = [] try: - stuck = stuck_children() + refuse_pending(bat.dir, board.get('pendingMark')) + stuck = settle_children() if stuck: raise StepFail('children survived SIGKILL; no flash may overlap them') # their pids are in survivors if bat.touched: if bat.fail_at == 'cleanup': raise StepFail('--fail-at cleanup: injected cleanup failure (acceptance only)') + release_ended_builds() # the restore's flash would block on a build lock this process still keeps row['cleanup'] = dict(bat.restore(files, row['firmware']['version']), heldAfter=False) + stuck = settle_children() # the restore's own children, before the lock is released + if stuck: + raise StepFail('the restore left children that survived SIGKILL') else: row['cleanup'] = {'pass': True, 'state': 'untouched', 'evidence': {}, 'heldAfter': False} except BaseException as e: - stuck += [x for x in stuck_children() if x not in stuck] - row['cleanup'] = {'pass': False, 'state': 'unknown' if bat.touched else 'untouched', + pending = outstanding_sessions(bat.dir) # kept, never reaped: recovery ends each by its receipt + mark_pending(board.get('pendingMark'), pending) # contained: a failure must not cost the hand-off below + stuck += [x for x in settle_children() + pending if x not in stuck] + row['cleanup'] = {'pass': False, 'state': 'pending' if pending else 'unknown' if bat.touched else 'untouched', 'evidence': getattr(e, 'evidence', {}), 'detail': f'{type(e).__name__}: {e}', - 'survivors': stuck, 'heldAfter': fh is not None} + 'survivors': stuck, 'heldAfter': fh is not None, + **({'receipts': [s['receipt'] for s in pending]} if pending else {}), + **({'markErrors': MARK_ERRORS[:]} if MARK_ERRORS else {})} row['pass'] = False if fh is not None: row['cleanup']['holder'] = hand_off(fh, board['lock'], row['cleanup']['detail'], stuck) @@ -2021,6 +2982,14 @@ def clean_up(row, code, bat, fh, board, helper, files): return code +def settle_manual(result, bat, board, run_dir): + """A manual install's or restore's survivors however it went: children left, sessions not ended (marked for later + invocations) and the build locks a keeper holds.""" + result['survivors'] = settle_children() + outstanding_sessions(run_dir) + mark_pending(board['pendingMark'], [x for x in result['survivors'] if 'receipt' in x]) + result.update(bat.kept and {'buildLocksKept': bat.kept} or {}) + + def main(argv=None): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest='cmd', required=True) @@ -2028,9 +2997,12 @@ def main(argv=None): r.add_argument('-b', '--board', required=True) r.add_argument('--build-record', required=True, help='check_build.py --board B [--backend cmake] --shared --hil-config ROSTER ' 'JSON, under cmake-build/') - r.add_argument('--fail-at', choices=(*STEPS['esptool'], 'cleanup'), help='acceptance only: fail this step, or the cleanup, ' + r.add_argument('--fail-at', choices=(*STEPS['esp'], 'cleanup'), help='acceptance only: fail this step, or the cleanup, ' 'to exercise cleanup and its holder') r.add_argument('--try-reentry', choices=REENTRY_METHODS, help="commissioning only, while the roster's reentry is unset") + r.add_argument('--lock-wait', type=int, default=0, help='seconds to keep trying a held board; a cleanup holder is never waited for') + r.add_argument('--run-root', type=Path, help="where this run's directory goes (default cmake-build/hil)") + r.add_argument('--route', help="an install route from the board's tinyuf2.routes (default its roster flasher)") r.add_argument('config') i = sub.add_parser('install', help='program a bootloader under your own manual hold') i.add_argument('-b', '--board', required=True) @@ -2038,6 +3010,7 @@ def main(argv=None): i.add_argument('--under-hold', required=True, metavar='REASON') i.add_argument('--try-reentry', choices=REENTRY_METHODS, help="commissioning only, while the roster's reentry is unset: DFU's fallback after the install") + i.add_argument('--route', help="an install route from the board's tinyuf2.routes (default its roster flasher)") i.add_argument('config') s = sub.add_parser('restore', help="install the record's TinyUF2 and copy its blinky, under your hold or to " 'recover a holder') @@ -2049,26 +3022,60 @@ def main(argv=None): s.add_argument('--report', help="the failed run's hil_report.json (with --recover-holder)") s.add_argument('--try-reentry', choices=REENTRY_METHODS, help="commissioning only, under your hold, while the roster's reentry is unset") + s.add_argument('--route', help="an install route from the board's tinyuf2.routes (default its roster flasher)") s.add_argument('config') + cp = sub.add_parser('clear-pending', help="remove a board's pending mark: every session it lists ended, or after a " + 'separately reviewed recovery') + cp.add_argument('-b', '--board', required=True) + cp.add_argument('--after-reviewed-recovery', metavar='NOTE', + help='the reviewed recovery that ended the sessions some other way; recorded, then the mark removed') + ch = cp.add_mutually_exclusive_group() + ch.add_argument('--under-hold', metavar='REASON', help='with --after-reviewed-recovery: your manual hold of the board') + ch.add_argument('--recover-holder', type=int, metavar='PID', + help="with --after-reviewed-recovery: the failed run's cleanup holder") + cp.add_argument('config') fr = sub.add_parser('fat-read') fr.add_argument('--dev', required=True) fr.add_argument('--file', required=True) fr.add_argument('--length', type=int, required=True) fr.add_argument('--offset', type=int, default=0) + kh = sub.add_parser('keep-held', help='internal: the re-executed cleanup holder (exec_holder)') + kh.add_argument('fd', type=int) + kh.add_argument('ack', type=int) + kb = sub.add_parser('keep-build', help="internal: the re-executed keeper of a build dir's lock (keep_build_lock)") + kb.add_argument('fd', type=int) + kb.add_argument('ack', type=int) + kb.add_argument('survivors', type=json.loads) args = ap.parse_args(argv) if args.cmd == 'fat-read': return fat_read_main(args) + if args.cmd == 'clear-pending': + return clear_pending_main(args) + if args.cmd == 'keep-held': + fh = os.fdopen(args.fd, 'r+') + os.write(args.ack, b'ok') + os.close(args.ack) + keep_held(fh) + if args.cmd == 'keep-build': + os.write(args.ack, b'ok') + os.close(args.ack) + keep_build(args.survivors) for sig in TERM_SIGNALS: signal.signal(sig, on_signal) - run_dir = cb.BUILD_DIR / 'hil' / f'run-{time.strftime("%Y%m%d-%H%M%S")}-{os.getpid()}' + # absolute: a provider command file is written in it while the working directory is + run_dir = ((getattr(args, 'run_root', None) or cb.BUILD_DIR / 'hil') / + f'run-{time.strftime("%Y%m%d-%H%M%S")}-{os.getpid()}').resolve() result = {'pass': False, 'caveat': '', 'results': [], 'reportPath': str(run_dir / 'hil_report.json')} code, recovery, bat = 2, None, None try: run_dir.mkdir(parents=True) - board = load_board(args.config, args.board) + board = select_route(load_board(args.config, args.board), args.route, run=args.cmd == 'run') facts = source_facts(board) apply_layout(board, facts) - helper = load_helper(board['helper']) + helper = load_helper() + provider() + board['pendingMark'] = PendingMark(helper, board['lock']) + clear_pending_mark(board['pendingMark']) # refused while an earlier invocation's session, any run dir, is not ended if args.cmd == 'run': row, code = run_board(args, board, facts, helper, run_dir) result['results'].append(row) @@ -2077,12 +3084,15 @@ def main(argv=None): board = trial_reentry(board, args.try_reentry) manual_holder(helper, board['lock'], args.under_hold) files, fw = admit_record(args.build_record, board, facts, run_dir) - bat = BATTERIES[board['flasher']](board, facts, run_dir) + users = probe_users(board) + if users: + raise Refused(f'probe {board["probe"]} is still in use by pids {users}; no session may overlap them') + bat = BATTERIES[board['kind']](board, facts, run_dir) try: ev = bat.install(files) ev['dfu'] = bat.dfu_after_install(fw['version']) finally: - result['survivors'] = stuck_children() + settle_manual(result, bat, board, run_dir) result['results'].append({'board': board['name'], 'installed': fw, 'evidence': ev}) result['pass'], code = True, 0 else: @@ -2102,14 +3112,19 @@ def main(argv=None): if len(failed) != 1: raise Refused(f'{args.report} names no cleanup holder {args.recover_holder} of {board["name"]}') held = [failed[0]['cleanup']] - if board['flasher'] == 'esptool': + if board['kind'] == 'esp': snapshot = report_snapshot(failed[0], board['regions']['preserved']) recovery = recovery_lock(helper, board['lock']) - rec = prove_hold(helper, board['lock'], args.recover_holder, reason='hil_test.py', role=HOLDER_ROLE) - cmdline = Path(f'/proc/{args.recover_holder}/cmdline').read_bytes() - if b'hil_test.py' not in cmdline: - raise Refused(f'pid {args.recover_holder} is not a hil_test.py cleanup holder') - alive = still_alive(held[0].get('survivors', []) + rec.get('survivors', [])) + rec = prove_cleanup_holder(helper, board['lock'], args.recover_holder) + survivors = held[0].get('survivors', []) + rec.get('survivors', []) + reviewed = reviewed_receipts(board['pendingMark']) + pending = [s['receipt'] for s in survivors + if 'receipt' in s and session_state(s) != 'ended' and s['receipt'] not in reviewed] + if pending: # the pending mark may be missing (markErrors); the report still names the sessions + raise Refused(f"the failed run's openocd_esp sessions {pending} are not ended: end each with " + '`hil_openocd_esp.py end --receipt `, or clear-pending after a separately reviewed ' + 'recovery (SKILL.md Recovery)') + alive = still_alive(survivors) if alive: raise Refused(f'the failed run\'s children {alive} are still alive; recovery may not overlap them') else: @@ -2118,13 +3133,13 @@ def main(argv=None): users = probe_users(board) if users: raise Refused(f'probe {board["probe"]} is still in use by pids {users}; recovery may not overlap them') - bat = BATTERIES[board['flasher']](board, facts, run_dir) + bat = BATTERIES[board['kind']](board, facts, run_dir) try: if snapshot: bat.preserved = snapshot cleanup = bat.restore(files, fw['version']) finally: - result['survivors'] = stuck_children() + settle_manual(result, bat, board, run_dir) if result['survivors']: raise StepFail('restore left children that survived SIGKILL') if args.recover_holder is not None: @@ -2147,6 +3162,9 @@ def main(argv=None): recovery.close() if result.get('survivors'): # a manual operation's survivor: keep your hold until it is gone result['pass'], code = False, 4 + if MARK_ERRORS: # the board stays a blocker however the operation went: no later invocation sees its sessions + result.update(markErrors=MARK_ERRORS[:], **{'pass': False}) + code = max(code, 4) try: Path(result['reportPath']).write_text(json.dumps(result, indent=1)) except OSError: @@ -2154,6 +3172,8 @@ def main(argv=None): print(json.dumps(result), flush=True) if HELD_IN_PROCESS: keep_held(HELD_IN_PROCESS[0]) + while release_ended_builds(): # no keeper started: keep them here + time.sleep(KEEPER_POLL) return code diff --git a/.claude/skills/hil/scripts/usb_dfu.py b/.claude/skills/hil-contract/scripts/usb_dfu.py similarity index 100% rename from .claude/skills/hil/scripts/usb_dfu.py rename to .claude/skills/hil-contract/scripts/usb_dfu.py diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md deleted file mode 100644 index 8f7d9b651..000000000 --- a/.claude/skills/hil/SKILL.md +++ /dev/null @@ -1,197 +0,0 @@ ---- -name: hil -description: The TinyUF2 HIL contract. Use when a change needs hardware evidence on a rig board, when a debug or validation session must take, flash, clean up or release a TinyUF2 rig board, or when a workflow asks for "the project's HIL contract". Runs the bootloader battery on the roster's boards, owns the board lock, and reports pass/fail per board with target-side evidence. ---- - -# HIL contract - -`scripts/hil_test.py` holds the mechanics: roster and source checks, firmware binding, locking, the probe and USB steps, cleanup and the JSON report. Agents call it instead of composing J-Link, mtools or lock commands. This page is the judgment. - -## Roster - -- The roster is `test/hil/local.json`, the boards on htpc's bench. On a machine without a board, its run fails at `install`, where J-Link finds no probe with that serial, before anything is written: `cleanup.state` is `untouched` and nothing stays held. ci.lan has no TinyUF2 rig yet (a follow-up). -- `TINYUSB_DIR` must name the tinyusb checkout whose `test/hil/helper/hil_lock.py` owns the board locks; every command refuses without it. Set it in the login profile, which headless sessions (chief, `hil-operator`) inherit. - -A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. A J-Link entry has `args` exactly `-device `; the runner uses SWD at 4000 kHz, so pass the same to those scripts, `--interface swd --speed 4000`. An espressif entry has `flasher.name` `esptool`, `flasher.uid` the USB-UART bridge's serial (`/dev/serial/by-id/*_-if*`), `flasher.args` exactly `--chip `, and the top-level `uid` the chip's factory MAC, which is TinyUF2's USB serial. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: -- the board lock name (`lock`) when tinyusb names the board differently, as `mimxrt1060_evk` for `imxrt1060_evk`; both runners take that one lock, and the helper's `status`, holder records and the report rows' `lock` show it. It defaults to the board name; -- the commissioned re-entry path (`reentry`); -- for a board whose TinyUF2 signals no disconnect when it comes up after a reset, the per-port-power hub port its USB is on (`usbPower`: `{"hub": "", "port": N}`; the serial, not the bus location, so a replugged hub is still found). J-Link boards only. While `dfu` waits, the runner power-cycles that port once it has sat connected but unaddressed (sysfs state `default`) for 2 s of that wait, at most once per 10 s, and at no other time: it writes the port's sysfs `disable` 1, then 0 a second later, which the kernel runs under the hub's lock, after any enumeration in progress, disconnecting the child before VBUS goes off. Each write and read-back runs in a bounded child; a cycle that fails or is interrupted after the off write still writes 0, and says in `hil.log` whether that restored power. The report lists the cycles (`portCycles`). - -The battery runs a board only after commissioning sets `reentry` and confirms `uid`; `install` and `restore` work without `reentry`, for commissioning. Only commissioned roster boards can be offered for a run. Adding a board is commissioning, the lead's (below), never a review-time or dispatch-time action: the runner refuses a build record from a dirty tree, an uncommitted roster entry included. - -## Locks - -Locks are tinyusb's: one kernel flock per board in `/tmp/tinyusb-hil-locks/`, taken through `$TINYUSB_DIR/test/hil/helper/hil_lock.py`. A board's lock is ``: its roster `tinyuf2.lock`, else the board name. It is tinyusb's name for the board, so TinyUF2 and tinyusb runs exclude each other; the helper's `hold` and `release` take it and `status` (no board argument) shows it, while `hil_test.py -b` and the probe scripts take the board name. -- **The battery locks itself,** as `hil_test.py`, which tinyusb's `release` refuses to kill. Never pre-hold a board you are about to run the battery on. -- **The lock fails closed.** An unusable lock dir refuses the run (exit 2); it never runs unlocked. `HIL_NO_BOARD_LOCK=1` bypasses the lock only when the task scope names forcing that board. The report then says `lockBypassed: true`. -- **Manual sessions** (hw-debugger, hw-validator): - 1. Take your own hold: `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py hold --reason ""`. - 2. Install a build record's TinyUF2, and restore the board when done: - - ```bash - python3 .claude/skills/hil/scripts/hil_test.py install -b --build-record --under-hold "" test/hil/local.json - python3 .claude/skills/hil/scripts/hil_test.py restore -b --build-record --under-hold "" test/hil/local.json - ``` - - `install` is the battery's install step from the record (Run the battery: a J-Link board runs the record's `tinyuf2-jlink` target, an esptool board takes the four IDF segments), then waits for DFU; it takes the record the battery takes, admitted the same way, so it needs the automount exclusion rule (Commissioning) and a clean build of HEAD. `--try-reentry `, while the roster's `reentry` is unset, gives DFU's fallback (Run the battery, dfu) for commissioning; `restore` takes it too. `restore` is the cleanup (below), from a clean build record of HEAD. Both refuse unless the hold is provably yours: the record's reason, a live pid, a FLOCK on the lock file in that pid's own fds, and a failing probe of the flock. - 3. Release with `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py release `, only after a `restore` that exited 0 with no `survivors`. Otherwise the board is not cleaned up: keep the hold and report it as a blocker. - - Manual exits: 0 the operation succeeded (for `install`, DFU came up; for `restore`, DFU left after the blinky copy and stayed away); 1 it failed before writing; 2 refused before any hardware access; 4 a child survived SIGKILL, or an `install` or `restore` failed after it began writing: keep the hold. - - On a refused hold, report the holder from `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` (no board argument; it prints every holder); never kill it. A `hil_test.py` holder whose record has `role: cleanup-holder` never frees on its own: report it as a blocker that needs the recovery dispatch (Exit 4), without waiting. Any other `hil_test.py` holder is a battery run; wait for it within the lock-wait budget. -- **Cleanup state** for a hw-validator or hw-debugger dispatch: `restore --under-hold` leaves the board on the record's TinyUF2 with `apps/blinky` copied through it, and its `cleanup.evidence` (DFU left after the copy and stayed away) is the observation that Cleanup records. Blinky running is not checked. Whoever uses the board next flashes it first; tinyusb's HIL does. -- **Lock-wait budget:** htpc has no CI runner, so it is the dev-bench row of chief's budget, 2 min, unless the dispatch names another. - -## Run the battery - -```bash -mkdir -p cmake-build/hil -python3 .claude/skills/build/scripts/check_build.py --board --backend cmake --shared --hil-config test/hil/local.json > cmake-build/hil/.build.json -python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json test/hil/local.json -``` - -`` is the roster name, the TinyUF2 board name. An espressif board builds without `--backend cmake`: ESP-IDF in CI's docker image, as CI does. - -- **Admission.** Before taking the lock the run checks the automount exclusion rule, and every span it could write (the HEX the flash target loads, inside the install span and the device's flash limits and laid out as the BIN; the BIN's board id), with the install and app spans taken from the candidate's source. -- **Build record.** It is this contract's build receipt: build it on the clean, pinned HEAD with `--hil-config` and pass it with `--build-record`. `--hil-config` binds the build's own flash target to the roster's probe or bridge (build contract); the run refuses a record whose binding names another flasher, and checks the files that target reads against the record. The build dir's lock covers that target only: the files the runner reads itself (the BIN it verifies, the segments it reads back, `apps/blinky.uf2` and `apps/dfu_runtime.uf2`) are copies in the run dir, taken at admission from bytes that hash as recorded, so a rebuild after the target cannot change them. For a J-Link board it must be a cmake build (`--backend cmake`), whatever backend CI builds the port with; board facts are read from the port's CMake files too. For an espressif board it must be an IDF build from CI's `espressif/idf` image; the run binds its generated `sdkconfig`, `flash_args`, the four segments (bootloader, partition table, otadata, `tinyuf2.bin` in the factory partition) and `apps/dfu_runtime.uf2`, and checks every segment against the generated partition table and the candidate's partition CSV. The run refuses a build record that is not clean, does not match the checkout now (HEAD, submodules, `get_deps` dependencies at their pins), or whose artifacts changed since. Commit first; ESP-IDF's records also go stale at the next commit (build contract). -- **Background.** Run it in the background; one board takes about 2 minutes. Use one board per run. -- **Selection.** There are no examples or test names to select: a run is the whole battery on one board, and the board is the only selection. - -The battery's steps on one board, all under one lock. The roster's probe serial selects the board (each probe is wired to one target), so no step identifies the chip: -1. **install:** the record's `tinyuf2-jlink` target runs (`cmake --build --target tinyuf2-jlink`) under its build dir's lock, once the files it reads hash as recorded and its dry run shows nothing but the flasher; JLinkExe can exit 0 with every command failed, so the transcript must show the record's HEX downloaded (programmed, or skipped as already matching), `O.K.` and the script completed, with no failure line. The files are hashed again after it, and a session of the runner's verifies the BIN at its flash address (`verifybin`), then resets. The board is touched from the target's start; only a transcript that ends at "Connecting to J-Link ...FAILED", before any target connect, leaves it untouched. -2. **dfu:** the UF2 drive with this board's serial is found. On the RT boards the install has left the app start erased on the bench (Limits), so DFU comes up; if it never enumerates, as when a valid app boots (an install on stm32f303disco or stm32f411ve_discovery keeps the app, Limits), the commissioned `retained-register` re-entry brings DFU up (its claim says so); any other DFU failure fails the step. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. -3. **copy:** a generated app is copied to the drive. -4. **app-ran:** the probe shows the app counting in thread mode with VTOR at the app start. -5. **reenter:** the bootloader is re-entered by the commissioned path, and the drive's `CURRENT.UF2` reads the app back. -6. **cleanup** (reported as `cleanup`, not a step, state `blinky copied`, or `untouched` with nothing to do when no session reached the board): the record's TinyUF2 is installed again as in install, DFU is found, the record's `apps/blinky.uf2` (admitted with the record: one family, every block in the app region) is copied to the drive, and the DFU device must leave and stay away for a quiet period. Any failure keeps the board held (Exit 4). - -A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; on a J-Link board a run triggered by the upload alone is unverified (an esptool board's app-ran observes the copied app itself over USB). "reenter" by `retained-register` is the forced-entry path, not a physical double tap. - -On an esptool board, each esptool connection is a child of the runner (`esp_session.py`), and so is each USB control request (`usb_dfu.py`, pyusb): -1. **install:** a first connection takes the MD5s of every partition the battery never writes (nvs, ota_1, ffat on the 8 MB layout) as the run's snapshot, kept in the row's `snapshot` however the run ends; that session is the first to touch the board, which only a missing bridge (no tty with its serial), a container left by an earlier flash, or a failed or interrupted `docker create` of the flash container leaves untouched. Then ESP-IDF's `flash` target runs in CI's image (`cmake --build --target flash`), with the same lock, hash and dry-run checks as a J-Link install: the bridge's tty is the only device passed in, as `ESPPORT`, with `IDF_FLASH_FULL=1` so ESP-IDF v6 writes segments already in flash too, and the container is named `tinyuf2-hil-` so a survivor is found and ended by name. Its transcript must show every segment written at its offset and hash-verified, then the hard reset. A runner session then reads each segment back and takes those MD5s again, then resets once; the runner compares the read-back bytes with the build's segment files and the MD5s with the snapshot, and any difference fails the step. esptool rewrites the bootloader header from `flash_args`, which the build already holds (dio, 80 MHz, 8 MB): on the bench (2026-10-13, esptool 5.3.1 in espressif/idf:v6.1) all four segments read back identical to the files. The roster's bridge selects the board (it is wired to that board only), so the chip is not identified, and its security eFuses and flash size are commissioning's to check. The evidence lists each write's span and the 4 KiB-aligned span covering it. Blank otadata boots the factory partition, TinyUF2. -2. **dfu:** as above; `INFO_UF2.TXT` reports the app partition (ota_0) as its flash, and no register is written. -3. **copy:** `apps/dfu_runtime` from the record, plus a block carrying a fresh nonce in ota_0's last 256 bytes, where the app reads it, renumbered under one `numBlocks`; the image itself is never patched. The record is refused when the image reaches that block or TinyUF2's 64 KiB flash cache window over it (`FLASH_CACHE_SIZE`) leaves ota_0. The evidence lists the image and nonce spans and the cache windows covering them. -4. **app-ran:** the app enumerates with its own VID:PID and the board's serial on the DFU's USB path, its DFU runtime interface string read from the device is `HIL `, and TinyUF2 stays away for a quiet period. -5. **reenter** (`dfu-detach`): DFU_DETACH to the app is acknowledged, the app leaves, TinyUF2 comes back on the same path, and `CURRENT.UF2` reads back the image and the nonce block. -6. **recopy:** a copy with a fresh nonce boots the app again. -7. **cleanup:** as on a J-Link board, with the install as in step 1: the preserved regions' MD5s after the writes, and again after the blinky copy, must equal the run's snapshot (a recovery takes it from the failed report's `snapshot`; a manual `restore` takes its own first). A mismatch fails the cleanup and keeps the hold. - -`--fail-at ` exists only for acceptance: it fails that step to exercise cleanup. - -## Reporting - -The last stdout line is JSON, also written to its `reportPath`; read that path, never an older report. Its fields: -- `pass`, `caveat`; -- per board: `ran`, `pass`, `locked`, `lockBypassed`, `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`), `jlinkRetried` (J-Link boards only: sessions run again after a crash before the target connect; Limits), and `detail`; -- `error` on a refusal or an operation failure. - -| Exit | Meaning | -|---|---| -| 0 | every board passed and was cleaned up | -| 1 | a board failed or did not run; it was cleaned up or never touched | -| 2 | refused before any hardware access (usage, roster, source, build record, lock helper, lock dir) | -| 4 | cleanup failed; the board is held when `cleanup.heldAfter` (not under `HIL_NO_BOARD_LOCK`). From `install` or `restore`: a child survived, or the operation failed after it began writing; keep your hold | - -A delegated run (`hil-operator`) returns `{pass, results, caveat, wedged}` and, beside them, `reportPaths`: -- `pass`, `results` and `caveat` are copied verbatim from the JSON; -- `wedged` lists the boards the operator saw stop responding; -- `reportPaths` maps each board to its JSON's `reportPath`, which a recovery dispatch needs. -- With several boards, the operator runs one battery per board, one after another: `results` holds every board's row, `pass` is true only when every run's is, and `caveat` joins the runs' non-empty caveats. -- A launched run that ended without printing its JSON (killed, crashed) may have programmed the board and left it running a candidate. The operator writes `pass: false`, `caveat: "run ended without a report; board state unknown"` and one row per board, exactly `{board, ran: null, pass: false, locked: false, detail}`, with `detail` naming the exit status, the output tail and `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` at that moment. -- The operator runs the battery once and returns. It never waits on a lock and never retries: a `locked` row comes back as is, and the caller rules below are the caller's (chief's). -- A recovery dispatch names the board, the holder pid and the failed run's `reportPath`. It runs `restore --recover-holder` in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Exit 4). - -### Review mapping - -A review (`pr-review`) maps each board's row, except a commissioning run's (`commissioning: true`, from `--try-reentry`), which is never validation. A blocker takes precedence over every other mapping: -- `pass`: `ran`, `pass` and `cleanup.pass` all true; -- `fail`: `ran` and not `pass`, with a failing step, `cleanup.state` other than `untouched`, and that step's `detail`; -- `not-run`: `ran` false (`locked` or a refusal), or a failing step with `cleanup.state` `untouched`, with its `detail`. Nothing was written, so the failure is the rig's: no probe with that serial on this machine, one unplugged or dead, or no esptool bridge with that serial; -- a blocker, never a `fail` row: exit 4, `ran: null` (no report), and an abort, which is `ran` and not `pass` with no failing step (an interruption or an unexpected error, named in the board's `detail`). - -`testedHead` is the build record's `input.head`. `regression` is `unknown` unless the battery also ran on the merge base. - -### Caller rules -- **Retry.** A failed test may be retried once, and only when no board is `locked`, `wedged` or `heldAfter`, `caveat` is empty, and every board was cleaned up. - - Keep both reports. A fail-then-pass is a pass with `flaky: true`, quoting both. - - An abort (a failed run with no failing step), a cleanup failure, `heldAfter`, `wedged` or a missing report is never retried; it is a blocker until diagnosed. - - A board whose run ended without a report gets a manual `restore --under-hold` before its next run, and only once no such process is left on the host. Killing the runner frees its lock but not its children, which `restore` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: - - `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'; pgrep -af '[e]sp_session\.py'; pgrep -af '[u]sb_dfu\.py'` - - The `hil_test.py` check also covers the runner's `fat-read` subcommand. While a process is left, or this cannot be established, the blocker stays. -- **Locked.** `locked` is an admission refusal, not a test failure. The caller admits again after the holder releases, within the lock-wait budget; the operator does not wait. Otherwise report the board not covered, naming the holder. -- **Exit 4.** - - The board stays held by a detached holder forked from the run: reason `hil_test.py`, `role: cleanup-holder`, with its pid in `cleanup.holder`. It ignores SIGTERM and SIGINT, so a timeout kill cannot free the board; only recovery releases it. - - If no holder could be forked, `cleanup.holder.inProcess` is set and the runner itself is the holder: it prints the report, closes its stdio and does not exit until recovered, so no completion notice arrives. When a run's notice is overdue, read the task's output: a final JSON line with `cleanup.holder.inProcess` true is this case. Do not wait for its exit. - - `cleanup.survivors` lists children that outlived SIGKILL (a J-Link, mtools or FAT read stuck on the device), and a flash container that outlived `docker rm -f` (`{"container": "tinyuf2-hil-"}`); recovery refuses while any of them is still alive. A runner killed outright leaves its container to run out; the next install refuses while `docker ps -a` lists it. Each child runs in a session of its own and the kill reaches the whole session, so a JLinkExe that ninja runs in a process group of its own under the flash target is killed and reported too. - - tinyusb sees that holder as a battery run and waits, which is safe but means the board is out of service until recovered. - - Do not release it by hand, and do not treat it as safe because tinyusb reflashes. The holder may stand for a stuck J-Link or mtools process. - - Recovery is a separate `hil-operator` dispatch whose scope names forcing that board. After process, probe and host readiness are established, it runs: - - `python3 .claude/skills/hil/scripts/hil_test.py restore -b --build-record --recover-holder --report test/hil/local.json` - - `` is the failed report's `firmware.buildRecord`, or a record rebuilt on the clean HEAD when it no longer verifies. The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, restores the board while the lock stays held, and only then releases the holder. On an esptool board the restore checks the preserved regions against the snapshot in the failed report, and recovery is refused for a touched run without one or one whose regions differ from the checkout's. A candidate whose drive cannot take the blinky copy fails the restore; the board stays held until a record that can restore it is given. A second recovery of the same board while one runs is refused (exit 2). - - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide); on espressif_esp32s3_devkitc_1, ROM download mode: hold Boot (SW1, GPIO0 low), press and release RST (SW2, CHIP_PU), then release Boot (DevKitC-1 schematic V1.0; the TRM's strap table also needs GPIO46 low, its default pull-down); on espressif_p4_function_ev, the same with BOOT (SW2, GPIO35 low) and RST (SW1, CHIP_PU) (Function-EV-Board schematic rev 1.4). - -## Selection - -`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config test/hil/local.json` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered, except `dfu_runtime`, which selects the esptool boards of the ports that build it, and leaves uncovered each port that builds it without one (mimxrt10xx today): the battery runs the bootloader with its generated fixture on J-Link boards and with `dfu_runtime` on esptool boards. The runner, the build scripts it imports and the roster (`.claude/skills/hil/scripts/`, `.claude/skills/build/scripts/`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. - -## Commissioning a board - -Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under a manual hold, except for the re-entry run, which locks itself. Record each result in the board's commissioning note (agentrc `target-debug` project notes), and `uid`, `lock` and `reentry` in the roster. In this order: -- **Cabling:** cable the USB port TinyUF2 enumerates on (`BOARD_TUD_RHPORT`). It can differ from the one tinyusb examples use: USB OTG1 on imxrt1060_evk; USB USER (CN3) on stm32f303disco; the USB OTG FS Micro-AB connector, not the ST-LINK one, on stm32f411ve_discovery (UM1842). A DFU device "not seen" with no new device in `dmesg` is a cabling problem first. Keep the probe on a hub apart from the target's (Limits). -- **Silicon revision,** before any other result: read it from the chip marking or registers, and check that revision's errata through read-doc. RT1060's DIGPROG does not tell A from B, so the marking decides. -- **Port power** (a board with `usbPower`): the hub's port `disable` files writable by the runner's user, through `/etc/udev/rules.d/52-tinyuf2-hil-uhubctl.rules`, one line per hub: on `ACTION=="add|bind"` of its `DRIVER=="hub"` interface, matched by the hub's VID, PID and serial, `RUN` `chgrp plugdev` and `chmod g+w` on `/sys$devpath/*-port*/disable`. The runner refuses the board while that file is not writable. Check once that writing `disable` 1 switches the port off (`uhubctl -l -p ` reads `off`), then write 0. -- **Automount exclusion:** `/etc/udev/rules.d/99-tinyuf2-hil.rules`, one line per board, with the VID and PID from the board's source, in exactly this form (the runner accepts only this form, with exact `==` matches): - - ``` - SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ATTRS{serial}=="", ENV{UDISKS_IGNORE}="1" - ``` - - Write it with the roster's `uid`, before the serial can be confirmed: `install` and `run` refuse without the rule, and a fresh board runs whatever tinyusb last flashed, not TinyUF2. A serial a running tinyusb example reports (`lsusb -v`) is an early cross-check, not a confirmation. Until the serial is confirmed, also inhibit automount for every board of this VID:PID with a temporary `/etc/udev/rules.d/98-tinyuf2-hil-commissioning.rules`: - - ``` - SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ENV{UDISKS_IGNORE}="1" - ``` - - Then `sudo udevadm control --reload`. -- **Bootloader:** `install` a clean HIL record of HEAD under the hold and observe DFU (add `--try-reentry retained-register` if a valid app could keep DFU away). -- **USB serial:** confirm the enumerated serial equals `uid` (sysfs or `lsusb -v`), and record the USB port paths. If it differs, fix `uid` and the rule together, reload, and `install` again; its reset re-enumerates the board and udev applies the reloaded rules, so no replug should be needed (by mechanism, not yet observed on a uid fix). -- **Cleanup:** `restore` under the hold and observe DFU leave after the blinky copy. Record the reset notes J-Link prints on install (`BENIGN`). -- **Exclusion proof:** delete the temporary rule and reload before the re-entry run below; its `dfu` step passes only when `udevadm info` on the enumerated block device shows `UDISKS_IGNORE=1` (`check_exclusive`). A host rule that ignores a whole VID, such as tinyusb's `88-tinyusb.rules` for 239a, sets the same property, so the proof cannot tell it from the serial rule; the runner still requires the serial line. -- **Loader footprint:** for the flash target's `loadfile`, place sentinels just past the written range and elsewhere in its erase block. Confirm them from a fresh session before the operation under test, and again from another fresh session after it: a verify in the session that wrote them cannot show what a later operation destroys. Make the written data differ from what is on the flash, since a download whose data already matches is skipped ("Contents already match") and erases nothing. Record the result in Limits. -- **Fixture prerequisites,** before anything executes the battery's fixture: - - the app start meets VTOR's alignment for the fixture's vector table (`fixture()`); - - its stack top, the `m_data` origin the runner reads (`ram_origin`) + 0x1000, is in RAM enabled when the bootloader hands off: the ROM default per read-doc, which TinyUF2 does not reconfigure (DTCM on imxrt1060_evk); - - under a hold, a word just below that stack top takes a written value and reads it back, and then its saved value (0x20000FFC on imxrt1060_evk). -- **Re-entry:** commit the roster entry without `reentry`, build the record, release the manual hold, and run the battery in commissioning mode; it locks itself and ends with the cleanup, so it also proves the restore: - - ```bash - python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json --try-reentry retained-register test/hil/local.json - ``` - - `--try-reentry` is the lead's only, and is refused once `reentry` is set. The row's `reentryDecision` names the method when the run passed or failed only at `reenter`'s `CURRENT.UF2` readback, every observation of the tried method is true (`controlBootedApp`, `dfuCameUp` and `magicConsumed` for `retained-register`), and the board was cleaned up and not held. Set `reentry` to it and commit. A null decision leaves `reentry` unset: file a `followup` issue with the observations for diagnosis, not as a diagnosed bug. When the drive copy itself fails, a fixture installed by hand with J-Link `loadbin` can isolate re-entry from the copy for diagnosis; it never sets `reentry`. -- **On an esptool board,** in place of the J-Link items: cable the native USB connector as well as the bridge's; check once by hand that the chip has no secure boot, secure download or flash encryption and that its flash holds the candidate's layout (`esptool get-security-info`, `esptool flash-id`), since the runner never checks them; record the bridge serial and the chip's MAC (`uid`, TinyUF2's USB serial); `install` from the record and observe DFU, then `restore` and observe DFU leave. Re-entry is `--try-reentry dfu-detach`, whose decision needs `detachAcked`, `appLeft` and `dfuCameUp`; before setting it, repeat DFU_DETACH ten times under the hold and check the host log shows no USB reset on the DETACH transitions (USB DFU 1.1 bitWillDetach). -- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid; on MIMXRT1011DAE5A the same note on every reset of a TinyUF2 image; on stm32f303vc, "Active write protection detected.", "Device will be unsecured now.", "Failed to unlock device." and "Can not attach to CPU. Trying connect under reset.", the connect that removes TinyUF2's write protection (Limits). On stm32f411ve the same connect prints "Active write protection detected.", "Device will be unsecured now." and "Device unlocked successfully.", which match no failure pattern and need no exemption; only the flash target connects as that device (Limits). A reset session (`w4` to AIRCR) passes only once J-Link reports the write issued ("Writing VALUE -> ADDR") with no failure before it. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. - -## Limits - -- Covered: imxrt1060_evk, metro_m7_1011, stm32f303disco, stm32f411ve_discovery, espressif_esp32s3_devkitc_1 and espressif_p4_function_ev on htpc. Not covered: ports without source facts in the runner, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69), espressif chips other than the ESP32-S3 and ESP32-P4, self-update, an interrupted update, malformed UF2 input and the physical double tap. -- esptool boards: the runner needs esptool 5.2.0 and pyusb with libusb-1.0 for its own Python. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The bridge's UART carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. At the end of a UF2 copy TinyUF2 restarts while the host's usb-storage is still talking to it: the host log shows a reset of that device and read errors on its block device. Seen on the S3 and the P4; other ports are unchecked. On the P4, while an esptool session holds the chip, the host log shows a full-speed device on TinyUF2's USB path that never answers its descriptor request (htpc 2026-10-14); the battery passed with it. -- Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. -- cmake builds pass `TUF2_LOG=0` unless `LOG` is set, as make does, so the HIL image does not log on the board UART. -- `install` from a blank board, or one whose ROM found no boot image, brings up DFU (measured on imxrt1060_evk 2026-10-12 and metro_m7_1011 2026-10-13). With no boot image a new J-Link session reads the flash wrong (imxrt1060_evk: zeros; metro_m7_1011: bytes shifted) until a `loadbin` has run; a `loadbin` compare reads it right. On metro_m7_1011, J-Link then prints "Invalid flash header detected." at the install's first reset. -- Of a port's CMake flash targets the runner runs only `tinyuf2-jlink` and ESP-IDF's `flash`, bound by the record to the roster's probe or chip (the bridge's tty is passed at flash time). Never use the erase targets on a rig board (`tinyuf2-erase-external-jlink` erases [FLASH_BASE, FLASH_BASE + 64 KiB), the bootloader with it), nor ESP-IDF's `combined-flash`, which has no port selection and writes nvs and both OTA slots blank. -- On metro_m7_1011 (2026-10-13, J-Link V9.84), `tinyuf2-jlink`'s `loadfile` of the cmake HEX (0x60000400 on) reported "1 range affected (65536 bytes)" and, from a fresh session after it, 0x60000000, 0x6000BF00, 0x6000C000-0x6000C00F, 0x6000C200 and 0x6000FF00 read 0xFF while 0x60010000 kept its data. Its compare covered the whole block: over blinky it programmed, and right after a program it skipped ("Contents already match"). DFU came up within 2 s in each of these three runs. That every install by the target leaves [0x6000C000, 0x60010000), the app's first 16 KiB, erased is an inference from these samples and this HEX, not a measurement of every case; the re-entry fallback (Run the battery, dfu) covers an install that leaves a valid app. On imxrt1060_evk (2026-10-13), the same target over blinky also reported "1 range affected (65536 bytes)"; from a fresh session after it, blinky's vectors at 0x6000C000-0x6000C00F and 0x6000C200 read 0xFF, while the FCB at 0x60000000 and 0x60010000 kept their data. -- On stm32f303disco (2026-10-14, J-Link V9.84 on the board's J-Link OB), `tinyuf2-jlink`'s `loadfile` compared [0x08000000, 0x08004000) and reprogrammed only the 2 KiB pages that differed, erasing the rest of such a page past the image (a sentinel at 0x08003F00 read 0xFF from a fresh session after it), while 0x08004000 (the app start) and 0x08004800 kept their data. So an install keeps a valid app. DFU still came up without the fallback in the battery of 2026-10-14: the runner's verify session halted the core in TinyUF2's double-tap wait (`src/main.c`, magic armed), so its reset entered DFU as a double tap, an inference from that trace. When that reset misses the window the app boots, and the retained-register fallback brings DFU up after `dfu`'s 20 s wait. -- On stm32f303vc, TinyUF2 write-protects its pages 0-7 at boot through the option bytes unless RCC_CSR's OBLRSTF is set, and it never clears that flag, so after any option-byte reload it skips protection until a power-on reset. A J-Link connect to a chip with that protection set prints the `BENIGN` unlock note and, read from a fresh `-device Cortex-M4` session after it, leaves the option bytes at RDP 0xAA with no WRP and flash intact (three samples, 2026-10-14). Its double-tap word is the last SRAM word, 0x20009FFC; that it survives SYSRESETREQ is undocumented (RM0316 Rev 10, DS9118, PM0214, ES0204) and inferred from the commissioning run (2026-10-14: the reset without the magic booted the app, the one with it entered DFU and consumed it). A battery therefore runs unprotected after its install; the protection itself is not covered. -- On stm32f411ve_discovery (2026-10-15, J-Link V9.84 on the board's J-Link OB, rev A silicon: DBGMCU REV_ID 0x1000, ES0287 Rev 6), `tinyuf2-jlink`'s `loadfile` of the HEX ([0x08000000, 0x08003E18)) reported "1 range affected (16384 bytes)": from a fresh session after it, a sentinel at 0x08003F00 read 0xFF while sentinels at 0x08004000, 0x08007C00, 0x08008000, 0x0800C000, 0x08010000 (the app start) and 0x08010800 kept their data. So an install keeps a valid app, and DFU comes up through the retained-register fallback. TinyUF2 write-protects sectors 0-1 at every boot (`src/main.c`; on the F4 the option bytes take effect when OPTSTRT completes, with no reset, RM0383 Rev 4 3.6.2). A `-device stm32f411ve` connect to a chip with that protection prints the unlock note (J-Link transcript shape) and removes it by resetting the chip: with RCC_CSR cleared and TinyUF2 booted, the next such connect read RCC_CSR 0x14000000 (SFTRSTF, PINRSTF) and nWRP 0xFF, while a `-device Cortex-M4` connect read RCC_CSR 0 and nWRP 0xFC, the core running the app. A runner session that reset the chip that way after a retained-register re-entry booted the app and dropped DFU, so runner sessions on this device connect as `Cortex-M4` (`SESSION_DEVICE`); the unlock is not in UM08001 Rev 9.78. Right after the flash target's unlock, a `Cortex-M4` session's halt failed ("CPU could not be halted", flash read back wrong) with no delay (three runs) and succeeded after a 200, 400, 600, 800 or 1500 ms `sleep`; TinyUF2's boot rewriting the option bytes is the inferred cause, and no rewrite time is documented. So the install's verify session waits 1 s first (`INSTALL_SETTLE_MS`, empirical). A battery therefore leaves the protection set; the protection itself is not covered. Its double-tap word is 0x2000FFFC, the last word of the boot script's 64 KiB RAM; that it survives SYSRESETREQ is undocumented (RM0383 Rev 4 6.1.1) and inferred from the commissioning run (the reset without the magic booted the app, the one with it entered DFU and consumed it). DFU re-enumerated after each tested reset into DFU without port power control (OTG FS soft disconnect), so it has no `usbPower`. -- stm32f303disco does not signal a disconnect when TinyUF2 comes up after a reset (the F303 has no internal D+ pull-up, DS9118 Rev 14 Table 66 note 2; the MB1035 schematic is not in the library, and on the bench the host kept the old device), so the host does not re-enumerate it unless it was polling the device (TinyUF2 as an MSC disk) when the reset came. For the same reason the `stays_away` checks (the control trial's, the copy's) cannot see a DFU entry there; `observe_app` shows the fixture running when probed, not that no transient DFU entry came before. Its roster `usbPower` names its USB USER port on the Port Authority hub (PA40004, port 3), which the runner cycles while `dfu` waits (Roster). A bench `uhubctl` cycle, a raw hub request, that met an enumeration in progress on hub 1-1 of an AMD 500-series xHCI controller killed that controller (2026-10-14: "HC died", every device on its buses lost, recovered by unbinding and binding `xhci_hcd`; the Port Authority hub needed a replug); hence the kernel's serialized `disable` and the 2 s in state `default`. On this hub `disable` 1 reads `off` from `uhubctl` (2026-10-14). -- A `JLinkExe` crash inside the target is not retried (the crash retry below covers the runner's own sessions only): it fails the install, touched. -- J-Link's QSPI loader is undocumented (SEGGER UM08001 has nothing on it). Historically, at sampled sentinel points on both RT boards, the runner's former `loadbin` install (no longer issued) lost the rest of its 64 KiB block past the image while a sentinel at 0x60010000, in the next block, survived; a sample, not a guarantee for every image. -- `JLinkExe` sometimes crashes in `libjlinkarm` at the probe connect on htpc (SIGSEGV; once a glibc double-free abort); the cause is unknown. A session that crashed this way, before the target connect, runs once more after 2 s (`jlinkRetried`); a second crash fails the run. Seen in 2 of 94 runner sessions on 2026-10-12. -- A device hung mid-write makes the host controller retry its enumeration, which stalled a J-Link connect on the same hub for ~80 s (lpcxpresso55s69); that the hub is the cause is a hypothesis. -- If no cleanup holder can be forked (exit 4 with `cleanup.holder.inProcess`), the runner never exits until recovered. -- No reproducer for an instrumented build: the battery and `install` take only a clean build record. A claim that needs one is reported pending. diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index 86eb63190..dbe9ba6e1 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -2,13 +2,17 @@ that compile a change, and the JSON and exit code follow SKILL.md. Builds are stubbed here; a real build is evidence only from a run of the script itself.""" import contextlib +import functools import importlib.util import io import json import os import re +import signal import subprocess +import sys import tempfile +import time import unittest from pathlib import Path from unittest import mock @@ -19,6 +23,53 @@ '_bin/apps/update-tinyuf2-lpcxpresso55s69.uf2'] +FAKE_TOOL = '''#!{python} +import os, subprocess, sys, time +from pathlib import Path +log = Path(os.environ['TOOLS_LOG']) +container = log.with_name('container') # the one container the fake daemon holds +with open(log, 'a') as f: + f.write(' '.join([Path(sys.argv[0]).name] + sys.argv[1:]) + '\\n') +if sys.argv[1] == 'rm': + if os.environ.get('RM_EXIT'): + sys.exit(int(os.environ['RM_EXIT'])) + container.unlink(missing_ok=True) + sys.exit(0) +if sys.argv[1] == 'ps': + print('c0ffee' if container.exists() else '') + sys.exit(0) +if sys.argv[1] == 'run' and os.environ.get('CREATE_AFTER'): # the daemon's create, landing after its client is gone + subprocess.Popen([sys.executable, '-c', 'import pathlib, sys, time; time.sleep(float(sys.argv[1])); pathlib.Path(sys.argv[2]).touch(); ' + 'open(sys.argv[3], "a").write("daemon created")', os.environ['CREATE_AFTER'], str(container), str(log)], + start_new_session=True) +elif sys.argv[1] == 'run': + container.touch() +if os.environ.get('HANG'): + with open(os.environ['PIDS'], 'a') as f: + f.write(f'{{os.getpid()}} {{subprocess.Popen(["sleep", "60"]).pid}}\\n') + time.sleep(60) +if os.environ.get('UNKNOWN_TARGET'): # as ninja 1.13 reports it: stderr only, nothing on stdout + print(f"ninja: error: unknown target '{{sys.argv[-1]}}'", file=sys.stderr) + sys.exit(1) +print(os.environ.get('STDOUT', "ninja: Entering directory `x'")) +print('ninja: warning: on stderr', file=sys.stderr) +''' + + +def alive(pid, wait=2): + """Whether `pid` is still a live process (not gone or a zombie) after up to `wait` s.""" + end = time.monotonic() + wait + while True: + try: + if Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[0] == 'Z': + return False + except OSError: + return False + if time.monotonic() >= end: + return True + time.sleep(0.05) + + def build_tmpdir(): """A scratch build root inside the checkout's ignored cmake-build/, where the script reports from.""" (ROOT / 'cmake-build').mkdir(exist_ok=True) @@ -127,7 +178,7 @@ def test_app_nothing_builds_is_uncovered(self): def test_top_level_app_selects_ports_whose_backend_compiles_it(self): ports = {cb.port_of(b) for b, _ in classify('apps/blinky/src/blinky.c')[2]} - self.assertEqual(ports, {'espressif', 'maxim', 'mimxrt10xx', 'stm32f3', 'stm32f4', 'stm32h5'}) + self.assertEqual(ports, {'espressif', 'maxim', 'mimxrt10xx', 'samd51', 'stm32f3', 'stm32f4', 'stm32h5'}) def test_dependency_selects_the_ports_get_deps_names(self): self.assertEqual(classify('lib/mcu/st/cmsis_device_f4/x.h')[2], @@ -169,10 +220,11 @@ def test_nothing_to_build(self): self.assertEqual(classify(path)[1], 'none', path) def test_hil_runner_and_roster_select_tests_and_every_roster_board(self): - for path in ('.claude/skills/hil/scripts/hil_test.py', 'test/hil/local.json'): + for path in ('.claude/skills/hil-contract/scripts/hil_test.py', 'test/hil/local.json', 'test/hil/ci.json', 'lib/agentrc', + '.github/workflows/hil.yml'): r = cb.classify(path) self.assertEqual((r['rule'], r['tests'], r['hil'], r['boards']), ('hil-control', True, True, {}), path) - self.assertEqual(cb.classify('.claude/skills/hil/SKILL.md')['effect'], 'none') # wording costs no hardware + self.assertEqual(cb.classify('.claude/skills/hil-contract/SKILL.md')['effect'], 'none') # wording costs no hardware self.assertEqual(classify('.claude/test/test_hil.py'), ('contract-tests', 'select', [], False, True)) def test_core_and_unclassified_select_every_representative(self): @@ -232,7 +284,7 @@ def test_ghostfat_input_selects_ghostfat_only(self): def test_paths_no_build_job_compiles_select_nothing(self): for path in ('README.md', '.claude/test/test_build.py', 'test/hil/local.json', '.github/workflows/codeql.yml', '.github/workflows/codeql-buildscript.sh', 'ports/stm32f4/Makefile', 'ports/ch32v20x/boards.c', - 'ports/lpc55/apps/erase_firmware/Makefile'): + 'ports/lpc55/apps/erase_firmware/Makefile', 'lib/agentrc'): self.assertEqual(ci(path), {'full': False, 'ports': {}, 'ghostfat': False}, path) def test_selections_union_and_a_whole_port_absorbs_its_boards(self): @@ -355,7 +407,7 @@ def test_docs_select_no_board(self): self.assertEqual((h['boards'], h['uncovered']), ([], [])) def test_a_runner_change_selects_every_roster_board(self): - h = self.hil('.claude/skills/hil/scripts/hil_test.py') + h = self.hil('.claude/skills/hil-contract/scripts/hil_test.py') self.assertEqual((h['full'], h['boards'], h['uncovered']), (True, ['lpcxpresso55s69'], [])) def test_a_change_to_the_build_script_the_runner_imports_selects_every_roster_board(self): @@ -369,12 +421,12 @@ def test_a_core_change_selects_every_roster_board_and_names_ports_without_one(se self.assertNotIn('lpc55', [u['target'] for u in h['uncovered']]) def test_a_port_without_a_roster_board_is_uncovered_even_beside_a_runner_change(self): - h = self.hil('ports/stm32f4/board_flash.c', 'ports/lpc55/boards.c', '.claude/skills/hil/scripts/hil_test.py') + h = self.hil('ports/stm32f4/board_flash.c', 'ports/lpc55/boards.c', '.claude/skills/hil-contract/scripts/hil_test.py') self.assertEqual(h['boards'], ['lpcxpresso55s69']) self.assertEqual([u['target'] for u in h['uncovered']], ['stm32f4']) def test_a_build_gap_stays_uncovered_beside_a_runner_change(self): - h = self.hil('ports/ch32v20x/board_flash.c', '.claude/skills/hil/scripts/hil_test.py') + h = self.hil('ports/ch32v20x/board_flash.c', '.claude/skills/hil-contract/scripts/hil_test.py') self.assertEqual(h['boards'], ['lpcxpresso55s69']) self.assertEqual([u['target'] for u in h['uncovered']], ['ports/ch32v20x/board_flash.c']) @@ -397,7 +449,7 @@ def test_the_battery_app_selects_the_esptool_roster_boards_and_other_apps_none(s self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk', 'flasher': {'name': 'jlink'}}, {'name': 'espressif_esp32s3_devkitc_1', 'flasher': {'name': 'esptool'}}]})) rt_gap = {'target': 'apps/dfu_runtime/src/nonce.c on mimxrt10xx', - 'reason': f'no {self.config} esptool board of mimxrt10xx runs dfu_runtime'} + 'reason': f'no {self.config} Espressif board of mimxrt10xx runs dfu_runtime'} h = self.hil('apps/dfu_runtime/src/nonce.c') # espressif and mimxrt10xx build it; only the S3 runs it self.assertEqual((h['boards'], h['uncovered']), (['espressif_esp32s3_devkitc_1'], [rt_gap])) h = self.hil('apps/dfu_runtime/src/nonce.c', 'src/main.c') # the whole roster's battery still never runs it on RT @@ -407,7 +459,7 @@ def test_the_battery_app_selects_the_esptool_roster_boards_and_other_apps_none(s self.assertEqual((h['boards'], h['uncovered']), (['espressif_esp32s3_devkitc_1'], [])) h = self.hil('ports/mimxrt10xx/apps/dfu_runtime/board_dfu_runtime.c') self.assertEqual((h['boards'], [u['reason'] for u in h['uncovered']]), - ([], [f'no {self.config} esptool board of mimxrt10xx runs dfu_runtime'])) + ([], [f'no {self.config} Espressif board of mimxrt10xx runs dfu_runtime'])) h = self.hil('apps/blinky/src/blinky.c', 'ports/espressif/apps/blinky/main/CMakeLists.txt') self.assertEqual((h['boards'], len(h['uncovered'])), ([], 2)) self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk', 'flasher': {'name': 'jlink'}}]})) @@ -736,6 +788,7 @@ def setUp(self): self.roster.write_text(json.dumps({'boards': [ {'name': 'metro_m7_1011', 'uid': 'U', 'flasher': {'name': 'jlink', 'uid': '000801012283', 'args': '-device MIMXRT1011DAE5A'}}, {'name': 'espressif_esp32s3_devkitc_1', 'uid': 'M', 'flasher': {'name': 'esptool', 'uid': 'b1', 'args': '--chip esp32s3'}}, + {'name': 'stm32h563_nucleo', 'uid': 'S', 'flasher': {'name': 'stlink', 'uid': '0023ab', 'args': '-device stm32h563zi'}}, {'name': 'lpcxpresso55s69', 'uid': 'L', 'flasher': {'name': 'openocd', 'uid': 'x', 'args': '-f a.cfg'}}]})) self.config = str(self.roster.relative_to(ROOT)) @@ -746,6 +799,11 @@ def jlink_tree(self, root, script=None, options='-USB 000801012283 -ExitOnError f'-CommandFile {root / "tinyuf2.jlink"}') return lambda d, args: subprocess.CompletedProcess(args, 0, f'arm-none-eabi-gcc ...\n{line}\n', '') + def stlink_tree(self, root, connect='port=swd sn=0023ab', image=None, go='--go'): + root.mkdir(parents=True, exist_ok=True) + line = f'cd {root} && /home/u/bin/STM32_Programmer_CLI --connect {connect} --write {image or root / "tinyuf2.hex"} {go}' + return lambda d, args: subprocess.CompletedProcess(args, 0, f'arm-none-eabi-gcc ...\n{line}\n', '') + def idf_tree(self, root, chip='esp32s3', extra='', args_file='flash_args'): root.mkdir(parents=True, exist_ok=True) (root / 'flash_args').write_text('--flash-mode dio --flash-freq 80m --flash-size 8MB\n0x0 bootloader/bootloader.bin\n' @@ -762,12 +820,172 @@ def test_the_roster_flasher_selects_the_probe_through_configure_defines(self): self.assertEqual(cb.roster_flasher(self.config, 'metro_m7_1011'), ({'name': 'jlink', 'uid': '000801012283', 'device': 'MIMXRT1011DAE5A'}, ['JLINK_OPTION=-USB 000801012283 -ExitOnError 1 -NoGui 1'])) + self.assertEqual(cb.roster_flasher(self.config, 'stm32h563_nucleo'), + ({'name': 'stlink', 'uid': '0023ab', 'device': 'stm32h563zi'}, ['STLINK_SERIAL=0023ab'])) self.assertEqual(cb.roster_flasher(self.config, 'espressif_esp32s3_devkitc_1'), ({'name': 'esptool', 'uid': 'b1', 'device': 'esp32s3'}, [])) # the tty is chosen at flash time for board, err in (('lpcxpresso55s69', 'no HIL flash target'), ('imxrt1060_evk', 'not on')): with self.subTest(board), self.assertRaisesRegex(cb.Fail, err): cb.roster_flasher(self.config, board) + def test_a_route_binds_the_esptool_bindings_segments(self): + p4 = 'espressif_p4_function_ev' + route = {'name': 'openocd_esp', 'uid': '30:ED:A0:E0:C9:02', 'device': 'esp32p4'} + self.assertEqual(cb.roster_routes('test/hil/local.json', p4), [route]) + self.assertEqual(cb.roster_routes('test/hil/local.json', 'metro_m7_1011'), []) + inputs = list(cb.FLASH['esptool'][1]) + rec = {'board': p4, 'flash': {'flasher': {'name': 'esptool', 'uid': 'b', 'device': 'esp32p4'}, 'inputs': inputs}} + self.assertEqual(cb.flash_routes(rec, [route]), {'openocd_esp': {'flasher': route, 'inputs': inputs}}) + with self.assertRaisesRegex(cb.Fail, 'needs the esptool binding of that chip'): + cb.flash_routes(rec, [dict(route, device='esp32s3')]) + with tempfile.TemporaryDirectory() as d: + for args in ('-f board/esp32p4-builtin.cfg -c init', '-f interface/esp_usb_jtag.cfg -f target/esp32p4.cfg'): + cfg = Path(d) / 'r.json' + cfg.write_text(json.dumps({'boards': [{'name': p4, 'tinyuf2': {'routes': [{'flasher': dict(route, args=args)}]}}]})) + with self.subTest(args), self.assertRaisesRegex(cb.Fail, 'not the roster form'): + cb.roster_routes(str(cfg), p4) + + def fake_tools(self, **env): + """`ninja` and `docker` on PATH that log their argv; as the query, each prints ninja's dry run with a warning on + stderr, or with HANG starts a grandchild, adds both pids to `self.pids` and hangs. `docker run` creates the + container, CREATE_AFTER s later if set; `docker rm` removes it, or exits RM_EXIT; `docker ps` lists it.""" + tools = self.tmp / 'bin' + tools.mkdir(exist_ok=True) + for name in ('ninja', 'docker'): + (tools / name).write_text(FAKE_TOOL.format(python=sys.executable)) + (tools / name).chmod(0o755) + self.tools_log, self.pids = self.tmp / 'tools.log', self.tmp / 'pids' + self.tools_log.unlink(missing_ok=True) + (self.tmp / 'container').unlink(missing_ok=True) + self.addCleanup(self.kill_hung) + return mock.patch.dict(os.environ, {'PATH': f'{tools}:{os.environ["PATH"]}', 'TOOLS_LOG': str(self.tools_log), + 'PIDS': str(self.pids), **env}) + + def hung(self): + return [int(pid) for pid in self.pids.read_text().split()] if self.pids.exists() else [] + + def kill_hung(self): + for pid in self.hung(): + if alive(pid, wait=0): + with contextlib.suppress(ProcessLookupError): # it may exit between the check and the kill + os.kill(pid, signal.SIGKILL) + + def tool_calls(self, tool): + return [l.split()[1:] for l in self.tools_log.read_text().splitlines() if l.split()[0] == tool] + + def test_an_unbound_query_returns_ninja_stdout_alone_and_a_timeout_kills_it(self): + root = self.tmp / 'b' + with self.fake_tools(): + r = cb.ninja_in('cmake', None, None, root, ['-n', '-v']) + self.assertEqual(r.stdout, "ninja: Entering directory `x'\n") # the stderr warning is not in it + self.assertEqual((r.returncode, r.stderr), (0, 'ninja: warning: on stderr\n')) + with self.fake_tools(HANG='1'), mock.patch.object(cb, 'NINJA_QUERY_TIMEOUT', 0.5), \ + self.assertRaisesRegex(cb.Fail, '^' + re.escape(f'b: ninja -n -v timed out after {cb.NINJA_QUERY_TIMEOUT}s') + '$'): + cb.ninja_in('cmake', None, None, root, ['-n', '-v']) + self.assertFalse(alive(self.hung()[0])) + self.assertEqual(self.tool_calls('docker'), []) + + def test_a_kill_of_check_builds_process_group_also_ends_its_query(self): + # as `timeout` ends what it runs: the whole process group, which the query and what it starts never leave + code = ('import importlib.util, sys; from pathlib import Path; ' + 'spec = importlib.util.spec_from_file_location("check_build", sys.argv[1]); ' + 'cb = importlib.util.module_from_spec(spec); spec.loader.exec_module(cb); ' + 'cb.ninja_in("cmake", None, None, Path(sys.argv[2]), ["-n", "-v"])') + with self.fake_tools(HANG='1'): + outer = subprocess.Popen([sys.executable, '-c', code, str(SCRIPT), str(self.tmp / 'b')], cwd=ROOT, + start_new_session=True) + end = time.monotonic() + 10 + while len(self.hung()) < 2 and time.monotonic() < end: + time.sleep(0.05) + os.killpg(outer.pid, signal.SIGKILL) + outer.wait() + self.assertEqual(len(self.hung()), 2) + self.assertEqual([pid for pid in self.hung() if alive(pid)], []) + + def test_a_timed_out_idf_query_removes_its_named_container_or_names_the_one_left(self): + root, name, bound = self.tmp / 'b', f'tinyuf2-query-{os.getpid()}-deadbeef', 0.5 + timed_out = re.escape(f'b: ninja -n -v timed out after {bound}s') + cases = (('removed', {}, '$'), ('created after the kill', {'CREATE_AFTER': str(bound + 0.1)}, '$'), + ('still listed', {'RM_EXIT': '1'}, re.escape(f'; container {name} still exists: docker rm -f {name}') + '$')) + for what, env, tail in cases: + with self.subTest(what), self.fake_tools(HANG='1', **env), mock.patch.object(cb, 'git', return_value=str(self.tmp)), \ + mock.patch.object(cb.uuid, 'uuid4', return_value=mock.Mock(hex='deadbeef')), \ + mock.patch.object(cb, 'NINJA_QUERY_TIMEOUT', bound), mock.patch.object(cb, 'CONTAINER_SETTLE', 1), \ + mock.patch.object(cb, 'CONTAINER_GONE_WAIT', 3), self.assertRaisesRegex(cb.Fail, '^' + timed_out + tail): + try: + cb.ninja_in('idf', 'docker', 'idf:image', root, ['-n', '-v']) + finally: + query = next(c for c in self.tool_calls('docker') if c[0] == 'run') + self.assertEqual((query[query.index('--name') + 1], query[-6:]), (name, ['idf:image', 'ninja', '-C', str(root), '-n', '-v'])) + self.assertEqual([c for c in self.tool_calls('docker') if c[0] == 'rm'][0], ['rm', '-f', name]) + # absent once is not gone: a create pending at the kill lands later and is removed too + end = time.monotonic() + 5 + while 'CREATE_AFTER' in env and 'daemon created' not in self.tools_log.read_text() and time.monotonic() < end: + time.sleep(0.05) + self.assertEqual((self.tmp / 'container').exists(), bool(env.get('RM_EXIT'))) + calls = [] + + def no_docker(cmd, **k): # the query times out, then docker is gone + calls.append(cmd) + raise subprocess.TimeoutExpired(cmd, bound) if len(calls) == 1 else FileNotFoundError('docker') + with mock.patch.object(cb, 'git', return_value=str(self.tmp)), mock.patch.object(cb, 'NINJA_QUERY_TIMEOUT', bound), \ + mock.patch.object(cb.uuid, 'uuid4', return_value=mock.Mock(hex='deadbeef')), \ + mock.patch.object(cb.subprocess, 'run', no_docker), \ + self.assertRaisesRegex(cb.Fail, '^' + timed_out + re.escape(f'; docker failed (docker), so container {name} may be left')): + cb.ninja_in('idf', 'docker', 'idf:image', root, ['-n', '-v']) + + def test_an_interrupted_idf_query_is_killed_and_its_container_removed_and_the_interruption_kept(self): + root, name, communicate = self.tmp / 'b', f'tinyuf2-query-{os.getpid()}-deadbeef', subprocess.Popen.communicate + for what, env, err in (('removed', {}, ''), ('still listed', {'RM_EXIT': '1'}, + f'b: interrupted ninja -n -v; container {name} still exists: docker rm -f {name}\n')): + interrupted = [] + + def ctrl_c(p, *a, **k): # the query's own communicate only: container_left's docker calls run as usual + if not interrupted: + interrupted.append(p.pid) + raise KeyboardInterrupt + return communicate(p, *a, **k) + stderr = io.StringIO() + with self.subTest(what), self.fake_tools(HANG='1', **env), mock.patch.object(cb, 'git', return_value=str(self.tmp)), \ + mock.patch.object(cb.uuid, 'uuid4', return_value=mock.Mock(hex='deadbeef')), \ + mock.patch.object(subprocess.Popen, 'communicate', ctrl_c), mock.patch.object(cb, 'CONTAINER_SETTLE', 0.2), \ + mock.patch.object(cb, 'CONTAINER_GONE_WAIT', 0.5), contextlib.redirect_stderr(stderr), \ + self.assertRaises(KeyboardInterrupt): + (self.tmp / 'container').touch() # created before the interruption + cb.ninja_in('idf', 'docker', 'idf:image', root, ['-n', '-v']) + self.assertEqual((stderr.getvalue(), (self.tmp / 'container').exists()), (err, bool(env))) + self.assertIn(['rm', '-f', name], self.tool_calls('docker')) + self.assertFalse(alive(interrupted[0])) + + def test_each_unbound_idf_query_gets_a_distinct_container_name(self): + root = self.tmp / 'b' + ids = [mock.Mock(hex='first'), mock.Mock(hex='second')] + with self.fake_tools(), mock.patch.object(cb, 'git', return_value=str(self.tmp)), \ + mock.patch.object(cb.uuid, 'uuid4', side_effect=ids): + cb.ninja_in('idf', 'docker', 'idf:image', root, ['-n', '-v']) + cb.ninja_in('idf', 'docker', 'idf:image', root, ['-t', 'commands']) + names = [c[c.index('--name') + 1] for c in self.tool_calls('docker') if c[0] == 'run'] + self.assertEqual(names, [f'tinyuf2-query-{os.getpid()}-first', f'tinyuf2-query-{os.getpid()}-second']) + + def test_an_unbound_idf_querys_container_is_named_for_the_pid_and_a_uuid4(self): # as hil-contract documents it + with self.fake_tools(), mock.patch.object(cb, 'git', return_value=str(self.tmp)): + cb.ninja_in('idf', 'docker', 'idf:image', self.tmp / 'b', ['-n', '-v']) + [name] = [c[c.index('--name') + 1] for c in self.tool_calls('docker') if c[0] == 'run'] + self.assertRegex(name, '^' + re.escape(f'tinyuf2-query-{os.getpid()}-') + '[0-9a-f]{32}$') + + def test_the_build_time_commands_query_reads_ninjas_stdout_and_reports_its_stderr(self): + # a real query through the unbound ninja_in: a failure's diagnostic is on stderr only, a warning there is never + # parsed as a command (#572 B1) + flasher = {'name': 'jlink', 'uid': '000801012283', 'device': 'MIMXRT1011DAE5A'} + root, query = self.tmp / 'b', functools.partial(cb.ninja_in, 'cmake', None, None) + with self.fake_tools(UNKNOWN_TARGET='1'), \ + self.assertRaisesRegex(cb.Fail, re.escape("ninja: error: unknown target 'tinyuf2-jlink'")): + cb.flash_binding(flasher, root, query) + line = self.jlink_tree(root)(root, []) + with self.fake_tools(STDOUT=line.stdout.strip()): + self.assertEqual(cb.flash_binding(flasher, root, query)['command'], line.stdout.splitlines()[1]) + self.assertEqual(self.tool_calls('ninja'), [['-C', str(root), '-t', 'commands', 'tinyuf2-jlink']]) + def test_the_generated_jlink_target_must_name_the_roster_probe_and_the_build_image(self): flasher = {'name': 'jlink', 'uid': '000801012283', 'device': 'MIMXRT1011DAE5A'} root = self.tmp / 'b' @@ -782,6 +1000,21 @@ def test_the_generated_jlink_target_must_name_the_roster_probe_and_the_build_ima with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'does not flash'): cb.flash_binding(flasher, root, self.jlink_tree(root, **tree)) + def test_the_generated_stlink_target_must_name_the_roster_probe_and_the_build_hex(self): + flasher = {'name': 'stlink', 'uid': '0023ab', 'device': 'stm32h563zi'} + root = self.tmp / 's' + out = cb.flash_binding(flasher, root, self.stlink_tree(root)) + self.assertEqual((out['target'], out['inputs']), ('tinyuf2-stlink', ['tinyuf2.hex'])) + for what, tree in (('no probe serial', dict(connect='port=swd')), + ('another probe', dict(connect='port=swd sn=1')), + ('another connect mode', dict(connect='port=swd sn=0023ab mode=UR')), + ('the ELF', dict(image=root / 'tinyuf2.elf')), + ('another image', dict(image=self.tmp / 'other.hex')), + ('no start', dict(go='')), + ('a second write', dict(go=f'--go --write {self.tmp / "o.hex"}'))): + with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'does not flash'): + cb.flash_binding(flasher, root, self.stlink_tree(root, **tree)) + def test_the_idf_flash_target_must_write_flash_args_for_the_roster_chip(self): flasher = {'name': 'esptool', 'uid': 'b1', 'device': 'esp32s3'} root = self.tmp / 'e' @@ -794,109 +1027,6 @@ def test_the_idf_flash_target_must_write_flash_args_for_the_roster_chip(self): with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'does not flash'): cb.flash_binding(flasher, root, self.idf_tree(root, **tree)) - def ready(self, flasher, root, tree, steps, nested=(), code=0, banner=True): - """flash_ready on `root` whose target dry-runs `steps` and whose bootloader subproject dry-runs `nested`; - `banner` False drops ninja's banner from both, 'nested' from the subproject's only.""" - commands = tree(root) - command = cb.flash_binding(flasher, root, commands)['command'] - flash = {'target': cb.FLASH[flasher['name']][0], 'flasher': flasher, 'command': command} - - def ninja(d, args): - self.assertEqual(args[:2], ['-n', '-v']) - out = [command if x == 'FLASH' else x for x in (steps if d == root else nested)] - return subprocess.CompletedProcess(args, code, 'entrypoint banner\n' + ("ninja: Entering directory `x'\n" if banner is True or banner == 'nested' and d == root else '') + - ''.join(f'[{i}/9] {x}\n' for i, x in enumerate(out, 1)), '') - cb.flash_ready(root, flash, ninja) - - def test_a_flash_target_is_ready_only_when_its_dry_run_shows_just_the_flasher(self): - jlink = {'name': 'jlink', 'uid': '000801012283', 'device': 'MIMXRT1011DAE5A'} - root = self.tmp / 'b' - self.ready(jlink, root, self.jlink_tree, ['FLASH']) - rebuild = [f'/usr/bin/arm-none-eabi-gcc -c {ROOT}/src/main.c -o main.c.obj', ': && arm-none-eabi-gcc -o tinyuf2.elf', 'FLASH'] - for what, steps, kw in (('a rebuild', rebuild, {}), ('a CMake re-run', ['/usr/bin/cmake --regenerate-during-build -S.. -B.'], {}), - ('ninja fails', ['FLASH'], {'code': 1}), ('nothing', [], {}), - ('no ninja banner', ['FLASH'], {'banner': False}), - ('another command', ['JLinkExe -USB 1'], {}), ('the flasher twice', ['FLASH', 'FLASH'], {})): - with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): - self.ready(jlink, root, self.jlink_tree, steps, **kw) - - def test_an_idf_flash_target_may_also_run_steps_that_write_no_flash_input(self): - esp = {'name': 'esptool', 'uid': 'b1', 'device': 'esp32s3'} - root = self.tmp / 'e' - cmake, py = '/opt/esp/tools/cmake/3.30.2/bin/cmake', '/opt/esp/python_env/idf5.3_py3.10_env/bin/python' - sizes = f'{py} /opt/esp/idf/components/partition_table/check_sizes.py --offset 0x8000' - size = (f'cd {root}/esp-idf/esptool_py && {sizes} partition --type app {root}/partition_table/partition-table.bin ' - f'{root}/tinyuf2.bin') - boot_size = f'cd {root}/bootloader/esp-idf/esptool_py && {sizes} bootloader 0x0 {root}/bootloader/bootloader.bin' - uf2_size = f'cd {root} && {sizes} partition --type app --subtype factory {root}/partition_table/partition-table.bin {root}/tinyuf2.bin' - copies = ' && '.join(f'{cmake} -E copy {root}/{f} {root}/_bin/{Path(f).name}' for f in cb.FLASH['esptool'][1]) - post = (f'cd {root} && bash {root}/tinyuf2.postbuild.sh && cd {root} && {cmake} -E echo "Copy binaries for artifact" && ' - f'{copies} && cd {root} && {py}3.10 {ROOT}/lib/uf2/utils/uf2conv.py --carray -o ' - f'{root}/apps/update_tinyuf2/bootloader_bin.c {root}/tinyuf2.bin') - stamps = (f'cd {root} && {cmake} -E make_directory {root}/CMakeFiles && {cmake} -E touch {root}/CMakeFiles/bootloader-complete ' - f'&& {cmake} -E touch {root}/bootloader-prefix/src/bootloader-stamp/bootloader-done') - fresh = [f'cd {root}/bootloader && {cmake} --build .', size, uf2_size, f'cd {root}/bootloader && {cmake} -E echo_append', post, - stamps, 'FLASH'] - script = root / 'tinyuf2.postbuild.sh' - good = ('#!/bin/bash\necho "Creating combined.bin"\n' - f"esptool --chip esp32s3 merge-bin --output combined.bin $(tr '\\n' ' ' < {root}/flash_args)\n") - self.idf_tree(root) - script.write_text(good) - self.ready(esp, root, self.idf_tree, fresh, [boot_size]) - cases = (('a CMake re-run', [f'cd {root} && {cmake} --regenerate-during-build -S/p -B{root}'], [boot_size], good), - ('an app rebuild', [f'{root}/xtensa-esp32s3-elf-g++ -o tinyuf2.elf'] + fresh, [boot_size], good), - ('a bootloader rebuild', fresh, ['xtensa-esp32s3-elf-gcc -c bootloader_start.c', boot_size], good), - ('a size check and a copy', [size + f' && cp /tmp/o.bin {root}/tinyuf2.bin', 'FLASH'], [], good), - ('a copy out of _bin', [f'{cmake} -E copy /tmp/o.bin {root}/_bin/../tinyuf2.bin', 'FLASH'], [], good), - ('a redirect', [f'{cmake} -E echo x > {root}/tinyuf2.bin', 'FLASH'], [], good), - ('a changed post-build script', fresh, [boot_size], good + f'cp /tmp/o.bin {root}/tinyuf2.bin\n'), - ('a step after the bootloader build', [f'cd {root}/bootloader && {cmake} --build . && {cmake} -E echo_append', 'FLASH'], - ['xtensa-esp32s3-elf-gcc -c bootloader_start.c'], good), - ('a copy through a symlink', [f'{cmake} -E copy /tmp/o.bin {root}/_bin/alias/tinyuf2.bin', 'FLASH'], [], good), - ('a touch of a flash input', [f'{cmake} -E touch {root}/flash_args', 'FLASH'], [], good)) - (root / '_bin').mkdir() - (root / '_bin/alias').symlink_to('..') - for what, steps, nested, text in cases: - with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): - script.write_text(text) - self.ready(esp, root, self.idf_tree, steps, nested) - script.write_text(good) - (root / '_bin/alias').unlink() - (root / '_bin').rmdir() - (root / '_bin').symlink_to('.') # a copy into _bin would land on the input itself - with self.subTest('a symlinked _bin'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): - self.ready(esp, root, self.idf_tree, [f'{cmake} -E copy /tmp/o.bin {root}/_bin/tinyuf2.bin', 'FLASH'], [boot_size]) - (root / 'apps').symlink_to('.') - (root / 'update_tinyuf2').symlink_to('.') - carray = 'cd ' + post.split(' && cd ')[-1] - self.ready(esp, root, self.idf_tree, [carray, 'FLASH'], [boot_size]) # before the alias - (root / 'bootloader_bin.c').symlink_to('tinyuf2.bin') - with self.subTest('a C array onto the image'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): - self.ready(esp, root, self.idf_tree, [carray, 'FLASH'], [boot_size]) - (root / 'combined.bin').symlink_to('tinyuf2.bin') - with self.subTest('a post-build onto the image'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): - self.ready(esp, root, self.idf_tree, [post.split(' && cd ')[0], 'FLASH'], [boot_size]) - (root / 'combined.bin').unlink() - self.ready(esp, root, self.idf_tree, [post.split(' && cd ')[0], 'FLASH'], [boot_size]) - (root / '_bin').unlink() - (root / '_bin').mkdir() - (root / '_bin/image.bin').write_text('x') - (root / 'ota_data_initial.bin').unlink(missing_ok=True) - (root / 'ota_data_initial.bin').symlink_to('_bin/image.bin') - with self.subTest('a copy onto a symlinked input'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): - self.ready(esp, root, self.idf_tree, [f'{cmake} -E copy /tmp/o.bin {root}/_bin/image.bin', 'FLASH'], [boot_size]) - with self.assertRaisesRegex(cb.Fail, 'bootloader: ninja -n failed'): - self.ready(esp, root, self.idf_tree, fresh, [boot_size], banner='nested') - - def test_an_idf_bootloader_size_check_is_read_only_at_any_offset(self): - root = self.tmp / 'p4' - sizes = '/opt/esp/python_env/idf6.1_py3.12_env/bin/python /opt/esp/idf/components/partition_table/check_sizes.py --offset 0x8000' - boot_size = lambda arg: f'cd {root}/bootloader && {sizes} bootloader {arg} {root}/bootloader/bootloader.bin' - self.assertTrue(cb.idf_step_writes_no_input(boot_size('0x2000'), root, 'esp32p4')) - for arg in ('2000', '0x2000 x', '0x20g0'): - with self.subTest(arg): - self.assertFalse(cb.idf_step_writes_no_input(boot_size(arg), root, 'esp32p4')) - def test_a_hil_record_pins_the_target_inputs_and_holds_the_build_lock(self): args = mock.Mock(shared=True, define=[], verbose=False, idf='docker') built = [] @@ -1060,6 +1190,36 @@ def test_fallback_skips_progress_lines(self): log = '[5/9] Building C object x/error.c.obj\nninja: error: loading build.ninja\n' self.assertEqual(cb.first_error(log), 'ninja: error: loading build.ninja') + def test_a_failed_script_test_is_named_with_its_exception_line(self): + """From unittest's own report: a multi-line assertion message, a chained exception, and a docstring that unittest + prints under the test's id.""" + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, 'test_contract.py').write_text( + 'import unittest\n' + 'class Contract(unittest.TestCase):\n' + ' def test_diff(self):\n' + ' self.assertEqual({"a": 1, "b": [1, 2]}, {"a": 2, "b": [1, 3]})\n' + ' def test_chained(self):\n' + ' try:\n' + ' {}["k"]\n' + ' except KeyError as e:\n' + ' raise RuntimeError("wrapped") from e\n' + ' def test_documented(self):\n' + ' """A docstring line."""\n' + ' self.fail("documented")\n') + for test, verdict, exc in ( + ('test_diff', 'FAIL', "AssertionError: {'a': 1, 'b': [1, 2]} != {'a': 2, 'b': [1, 3]}"), + ('test_chained', 'ERROR', 'RuntimeError: wrapped'), + ('test_documented', 'FAIL', 'AssertionError: documented')): + with self.subTest(test): + real = subprocess.run([sys.executable, '-m', 'unittest', f'test_contract.Contract.{test}'], + capture_output=True, text=True, cwd=tmp) + with mock.patch.object(cb.subprocess, 'run', return_value=real): + rec = cb.run_tests() + self.assertEqual(rec['status'], 'failed') + pattern = rf'^{verdict}: {test} \(test_contract\.Contract[^)]*\): {re.escape(exc)}$' + self.assertRegex(rec['firstError'], pattern) + if __name__ == '__main__': unittest.main() diff --git a/.claude/test/test_dfu_runtime.py b/.claude/test/test_dfu_runtime.py index 69692909c..dbf071570 100644 --- a/.claude/test/test_dfu_runtime.py +++ b/.claude/test/test_dfu_runtime.py @@ -9,7 +9,7 @@ ROOT = Path(__file__).resolve().parents[2] SRC = ROOT / 'apps' / 'dfu_runtime' / 'src' -spec = importlib.util.spec_from_file_location('hil_test', ROOT / '.claude' / 'skills' / 'hil' / 'scripts' / 'hil_test.py') +spec = importlib.util.spec_from_file_location('hil_test', ROOT / '.claude' / 'skills' / 'hil-contract' / 'scripts' / 'hil_test.py') ht = importlib.util.module_from_spec(spec) spec.loader.exec_module(ht) STRING_SIZE = len('HIL ') + 1 + 2 * 16 diff --git a/.claude/test/test_get_deps.py b/.claude/test/test_get_deps.py new file mode 100644 index 000000000..2541833c0 --- /dev/null +++ b/.claude/test/test_get_deps.py @@ -0,0 +1,70 @@ +"""tools/get_deps.py refuses to reset a dependency through a symlink.""" +import contextlib +import importlib.util +import io +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('get_deps', ROOT / 'tools' / 'get_deps.py') +gd = importlib.util.module_from_spec(spec) +spec.loader.exec_module(gd) + + +class SymlinkGuard(unittest.TestCase): + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.tmp = Path(tmp.name) + self.root = self.tmp / 'checkout' + self.root.mkdir() + self.dep = 'lib/mcu/vendor/sdk' + self.pin = '0123456789abcdef' + mock.patch.object(gd, 'TOP', self.root).start() + mock.patch.object(gd, 'deps_all', {self.dep: ['https://example.invalid/sdk.git', self.pin, 'port']}).start() + self.run_cmd = mock.patch.object(gd, 'run_cmd').start() + self.addCleanup(mock.patch.stopall) + + def run_dep(self): + output = io.StringIO() + with contextlib.redirect_stdout(output): + status = gd.get_a_dep(self.dep) + return status, output.getvalue() + + def assert_refused(self, component): + status, output = self.run_dep() + self.assertEqual(status, 1) + self.assertIn(f'{component} is a symlink', output) + self.assertIn(self.pin, output) + self.assertIn('checkout that owns the dependency', output) + self.run_cmd.assert_not_called() + + def test_dependency_symlink_is_refused_before_git_runs(self): + target = self.tmp / 'primary-sdk' + path = self.root / self.dep + path.parent.mkdir(parents=True) + path.symlink_to(target, target_is_directory=True) + + self.assert_refused(self.dep) + + def test_symlinked_ancestor_is_refused_before_git_runs(self): + target = self.tmp / 'primary-mcu' + target.mkdir() + path = self.root / 'lib' / 'mcu' + path.parent.mkdir(parents=True) + path.symlink_to(target, target_is_directory=True) + + self.assert_refused('lib/mcu') + + def test_regular_dependency_path_continues_to_git(self): + status, output = self.run_dep() + self.assertEqual(status, 0) + self.assertIn(f'cloning {self.dep}', output) + self.assertEqual(self.run_cmd.call_count, 4) + self.assertTrue((self.root / self.dep).is_dir()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.claude/test/test_hil.py b/.claude/test/test_hil.py index 5722585df..5a5068197 100644 --- a/.claude/test/test_hil.py +++ b/.claude/test/test_hil.py @@ -1,11 +1,14 @@ #!/usr/bin/env python3 """Tests of the HIL contract's runner. Stubs prove the plumbing only: every probe and USB path here is -unverified until a real run on the rig (SKILL.md, commissioning).""" +unverified until a real run on the rig (commissioning.md).""" import contextlib +import importlib.metadata import importlib.util import io import json import os +import re +import select import shutil import signal import struct @@ -13,57 +16,27 @@ import sys import tempfile import textwrap +import threading import time import unittest +import warnings from pathlib import Path from unittest import mock ROOT = Path(__file__).resolve().parents[2] -SCRIPT = ROOT / '.claude' / 'skills' / 'hil' / 'scripts' / 'hil_test.py' +SCRIPT = ROOT / '.claude' / 'skills' / 'hil-contract' / 'scripts' / 'hil_test.py' sys.dont_write_bytecode = True spec = importlib.util.spec_from_file_location('hil_test', SCRIPT) ht = importlib.util.module_from_spec(spec) spec.loader.exec_module(ht) +# lib/agentrc's flash provider, loaded once as main() loads it; its pin check is Roster's to test, against the real +# checkout and a git hook's environment +with mock.patch.object(ht, 'check_lock_pin'), contextlib.suppress(ht.Refused): + ht.provider() + ht.openocd_esp() BOARD = 'lpcxpresso55s69' RT = {'port': 'mimxrt10xx', 'name': 'imxrt1060_evk', 'lock': 'mimxrt1060_evk'} -# the API hil_test.py uses from tinyusb's test/hil/helper/hil_lock.py, with the same flock and record -# semantics; set TINYUSB_HIL_LOCK to the real helper to run the lock tests against it instead -FAKE_HELPER = ''' -import fcntl, json, os, time -BOARD_LOCK_DIR = {dir!r} -CI_REASON = 'hil_test.py' -PROTECTED_REASONS = {{CI_REASON, 'pool_check'}} -def lock_path(board): return os.path.join(BOARD_LOCK_DIR, f'{{board}}.lock') -def flock_nb(board): - fh = os.fdopen(os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666), 'r+') - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - fh.close(); raise - return fh -def write_record(fh, reason): - fh.truncate(0); fh.seek(0) - json.dump({{'pid': os.getpid(), 'reason': reason, 'since': ''}}, fh); fh.flush(); return True -def clear_record(fh): fh.truncate(0) -def read_record(board): - try: - with open(lock_path(board)) as f: return json.load(f) - except (OSError, ValueError): return None -def acquire_board_lock(board, reason=CI_REASON): - if os.environ.get('HIL_NO_BOARD_LOCK') == '1': return None - try: - os.makedirs(BOARD_LOCK_DIR, exist_ok=True) - fh = os.fdopen(os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666), 'r+') - except OSError: - return None - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - info = fh.read(500).strip(); fh.close(); raise RuntimeError(f'board locked: {{info}}') - write_record(fh, reason); return fh -''' - HOLDER = ''' import fcntl, json, os, sys, time fh = os.fdopen(os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o666), 'r+') @@ -75,6 +48,88 @@ def acquire_board_lock(board, reason=CI_REASON): ''' +def pinned_helper(test, lockdir): + """lib/agentrc's hil_lock.py, the helper the runner takes its locks through, with its locks under `lockdir`.""" + if not ht.LOCK_HELPER.is_file(): + test.fail('lib/agentrc not checked out: git submodule update --init lib/agentrc') + helper = ht.load_helper() + helper.BOARD_LOCK_DIR = str(lockdir) + return helper + + +MARK_HELPER = [] +PENDING_MARK = ht.PendingMark # the class, which Tmp patches with one over the test's helper + + +def stat_mode(path): + return os.stat(path).st_mode & 0o777 + + +def mark_helper(test, lockdir): + """One pinned hil_lock for every test's pending marks (the pin is checked once), its dir `lockdir` for this test.""" + if not MARK_HELPER: + with mock.patch.object(ht, 'check_lock_pin'): # the pin's own tests check it, outside a commit's git env + MARK_HELPER.append(pinned_helper(test, lockdir)) + p = mock.patch.object(MARK_HELPER[0], 'BOARD_LOCK_DIR', str(lockdir)) + p.start() + test.addCleanup(p.stop) + return MARK_HELPER[0] + + +def end_child(pid): + """SIGKILL and reap `pid` only while it is still this process's unreaped child, so its number cannot have been + reused; one already reaped, or not a child, is skipped. True when it was still running.""" + try: + if os.waitpid(pid, os.WNOHANG)[0]: # it had exited: now reaped + return False + except ChildProcessError: + return False + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + return True + + +def reap_detached(test): + """End every build-lock keeper and cleanup holder the test forks, before its temp dirs go: a keeper over a pending + session's receipt waits for ever once that receipt is deleted (unreadable keeps the lock, by design).""" + forked, keep, hand = [], ht.keep_build_lock, ht.hand_off + + def recorded(fn): + def call(*a, **k): + r = fn(*a, **k) + if isinstance(r, dict) and not r.get('inProcess'): + forked.append(r['pid']) + return r + return call + test.addCleanup(lambda: [end_child(pid) for pid in forked]) # after the patches stop, before the temp dirs go + for p in (mock.patch.object(ht, 'keep_build_lock', recorded(keep)), mock.patch.object(ht, 'hand_off', recorded(hand))): + p.start() + test.addCleanup(p.stop) + + +def tearDownModule(): + """No `hil_test.py keep-build` or `keep-held` child of this process outlives the suite.""" + mine = [] + for d in Path('/proc').iterdir(): + try: + ppid = re.search(r'^PPid:\s*(\d+)', (d / 'status').read_text(), re.M) + cmd = (d / 'cmdline').read_bytes().split(b'\0') + except (OSError, ValueError): + continue + if ppid and int(ppid.group(1)) == os.getpid() and any(c.endswith(b'hil_test.py') for c in cmd) and \ + {b'keep-build', b'keep-held'} & set(cmd): + mine.append(int(d.name)) + left = [pid for pid in mine if end_child(pid)] + if left: + raise AssertionError(f'keep-build or keep-held children {left} outlived the suite') + + +def no_ci_job(test): + """This process outside a CI job, as the openocd_esp route requires (ci_refusal): the suite may run in one.""" + env = {k: v for k, v in os.environ.items() if k not in ht.openocd_esp().CI_JOB_ENV} + test.enterContext(mock.patch.dict(os.environ, env, clear=True)) + + def roster(tmp, **over): t = {'reentry': 'retained-register'} t.update(over.pop('tinyuf2', {})) @@ -86,18 +141,21 @@ def roster(tmp, **over): return cfg -HELPER = Path('test/hil/helper/hil_lock.py') # under TINYUSB_DIR - - # the runner reads no source facts for lpc55; most tests keep its layout as synthetic facts LPC_FACTS = {'flashBase': 0, 'binBase': 0, 'appStart': 0x10000, 'flashSize': 0x98000, 'vid': 0x1fc9, 'pid': 0x0094, 'boardId': 'LPC55S69-Xpresso-revA', 'family': 0x2abc77ec, 'vectors': 76, 'ram': 0x20000000, 'dblTapDelayMs': 500, 'dblTapReg': 0x4002C05C, 'dblTapMagic': 0xF01669EF} +# stm32h563_nucleo's source facts, which StlinkSourceFacts checks against the source: the pre-commit job does not fetch +# the stm32h5 CMSIS header its vector count needs +H563_FACTS = {'flashBase': 0x08000000, 'binBase': 0x08000000, 'appStart': 0x08008000, 'flashSize': 0x200000, + 'vid': 0x239a, 'pid': 0x0163, 'boardId': 'STM32H563-Nucleo', 'family': 0x4e8f1c5d, 'vectors': 147, + 'ram': 0x20000000, 'dblTapDelayMs': 500, 'dblTapReg': 0x2009FFFC, 'dblTapMagic': 0xF01669EF} +SYNTHETIC_FACTS = {'lpc55': LPC_FACTS, 'stm32h5': H563_FACTS} real_source_facts = ht.source_facts def source_facts(board): - return dict(LPC_FACTS) if board['port'] == 'lpc55' else real_source_facts(board) + return dict(SYNTHETIC_FACTS[board['port']]) if board['port'] in SYNTHETIC_FACTS else real_source_facts(board) class Tmp(unittest.TestCase): @@ -105,9 +163,10 @@ def setUp(self): t = tempfile.TemporaryDirectory() self.addCleanup(t.cleanup) self.tmp = Path(t.name) - for p in (mock.patch.object(ht, 'source_facts', source_facts), - mock.patch.object(ht, 'CONTAINERS', []), # a test's tracked container never reaches real docker later - mock.patch.dict(os.environ, {'TINYUSB_DIR': str(self.tmp)})): + reap_detached(self) + self.mark_helper = mark_helper(self, self.tmp / 'marks') + for p in (mock.patch.object(ht, 'source_facts', source_facts), mock.patch.object(ht, 'PROTECTED_SESSIONS', []), + mock.patch.object(ht, 'PendingMark', lambda helper, lock: PENDING_MARK(self.mark_helper, lock))): p.start() self.addCleanup(p.stop) @@ -123,7 +182,6 @@ def test_a_valid_entry_loads_with_its_layout_from_the_source(self): b = self.load(roster(self.tmp)) self.assertEqual((b['install'], b['app'], b['device'], b['if'], b['speed']), ((0, 0x10000), (0x10000, 0x98000), 'LPC55S69_M33_0', 'SWD', '4000')) - self.assertEqual(b['helper'], str(self.tmp / HELPER)) def test_refusals(self): cases = { @@ -145,14 +203,117 @@ def test_usb_power_is_for_jlink_boards_only(self): with self.assertRaisesRegex(ht.Refused, 'usbPower is for J-Link boards only'): ht.load_board(s3_roster(self.tmp, tinyuf2={'reentry': 'dfu-detach', 'usbPower': {'hub': 'PA40004', 'port': 3}}), S3) - def test_the_lock_helper_comes_from_an_absolute_tinyusb_dir(self): - for value in ('', 'tinyusb'): - with self.subTest(value), mock.patch.dict(os.environ, {'TINYUSB_DIR': value}), \ - self.assertRaisesRegex(ht.Refused, 'TINYUSB_DIR'): - ht.load_board(roster(self.tmp), BOARD) - with mock.patch.dict(os.environ), self.assertRaisesRegex(ht.Refused, 'TINYUSB_DIR'): - del os.environ['TINYUSB_DIR'] - ht.load_board(roster(self.tmp), BOARD) + def test_a_missing_lock_helper_submodule_is_refused(self): + for what, file, checkout in (('no file', False, True), ('not a checkout', True, False)): + helper = self.tmp / 'hil_lock.py' + helper.unlink(missing_ok=True) + if file: + helper.write_text('') + with self.subTest(what), mock.patch.object(ht, 'LOCK_HELPER', helper), \ + mock.patch.object(ht.cb, 'own_checkout', lambda p: checkout), \ + self.assertRaisesRegex(ht.Refused, 'git submodule update --init lib/agentrc'): + ht.check_lock_pin() + + def test_a_lock_helper_off_its_pin_is_refused(self): + helper = self.tmp / 'hil_lock.py' + helper.write_text('') + pin, other = 'a' * 40, 'b' * 40 + cases = {'off its pin': (other, pin, [], f'lib/agentrc is at {other}; HEAD pins {pin}: git submodule update'), + 'off its pin with the pin change staged': (other, other, [], f'but {other} is staged: commit or unstage'), + 'dirty at its pin': (pin, pin, ['lib/agentrc'], f'lib/agentrc checkout is dirty at the pinned {pin}'), + 'clean at its pin': (pin, pin, [], None)} + for what, (have, staged, dirty, refusal) in cases.items(): + pinned = {('rev-parse', 'HEAD:lib/agentrc'): pin + '\n', ('rev-parse', ':lib/agentrc'): staged + '\n'} + checked_out = {('lib/agentrc', 'rev-parse', 'HEAD'): have + '\n'} + with self.subTest(what), mock.patch.object(ht, 'LOCK_HELPER', helper), \ + mock.patch.object(ht.cb, 'own_checkout', lambda p: p == 'lib/agentrc'), \ + mock.patch.object(ht.cb, 'git', lambda *a: pinned[a]), \ + mock.patch.object(ht.cb, 'git_in', lambda *a: checked_out[a]), \ + mock.patch.object(ht.cb, 'dirty_inputs', lambda paths: [p for p in paths if p in dirty]): + if refusal: + with self.assertRaisesRegex(ht.Refused, refusal): + ht.check_lock_pin() + else: + ht.check_lock_pin() + + def test_a_git_failure_reading_the_pin_is_refused(self): + helper = self.tmp / 'hil_lock.py' + helper.write_text('') + pin, other = 'a' * 40, 'b' * 40 + + def fail(*a): + raise ht.cb.Fail('git failed') + + def git_staged_fails(*a): + return fail() if a == ('rev-parse', ':lib/agentrc') else pin + '\n' + + cases = {'git': (fail, lambda *a: pin + '\n', lambda p: []), + 'git_in': (lambda *a: pin + '\n', fail, lambda p: []), + 'git reading the staged pin': (git_staged_fails, lambda *a: other + '\n', lambda p: []), + 'dirty_inputs': (lambda *a: pin + '\n', lambda *a: pin + '\n', fail)} + for what, (git, git_in, dirty_inputs) in cases.items(): + with self.subTest(what), mock.patch.object(ht, 'LOCK_HELPER', helper), \ + mock.patch.object(ht.cb, 'own_checkout', lambda p: True), mock.patch.object(ht.cb, 'git', git), \ + mock.patch.object(ht.cb, 'git_in', git_in), mock.patch.object(ht.cb, 'dirty_inputs', dirty_inputs), \ + self.assertRaisesRegex(ht.Refused, 'git failed'): + ht.check_lock_pin() + + def test_this_checkout_passes_the_real_pin_check(self): + if not ht.LOCK_HELPER.is_file(): + self.fail('lib/agentrc not checked out: git submodule update --init lib/agentrc') + # a git hook exports these, which would point git -C lib/agentrc at the superproject + env = {k: v for k, v in os.environ.items() if k not in ('GIT_DIR', 'GIT_WORK_TREE')} + if 'GIT_INDEX_FILE' in env: + # commit -a and commit run the hook on a temporary index that holds the pin being committed + env['GIT_INDEX_FILE'] = str(ht.ROOT / env['GIT_INDEX_FILE']) + self.enterContext(mock.patch.dict(os.environ, env, clear=True)) + git_in = ht.cb.git_in + + def git_in_own_index(*a): + with mock.patch.dict(os.environ): + os.environ.pop('GIT_INDEX_FILE', None) + return git_in(*a) + + self.enterContext(mock.patch.object(ht.cb, 'git_in', git_in_own_index)) + want, staged = (ht.cb.git('rev-parse', ref).strip() for ref in ('HEAD:lib/agentrc', ':lib/agentrc')) + if staged != want: + # a commit bumping the pin runs this hook with the new pin staged over HEAD's: the other tests load + # whatever is checked out, so that must be the staged pin, clean, or they test the wrong helper + have = ht.cb.git_in('lib/agentrc', 'rev-parse', 'HEAD').strip() + self.assertEqual((have, ht.cb.dirty_inputs(['lib/agentrc'])), (staged, []), + f'lib/agentrc is not checked out clean at the staged pin {staged}: git submodule update --init lib/agentrc') + with self.assertRaisesRegex(ht.Refused, 'is staged: commit or unstage'): + ht.check_lock_pin() + else: + ht.check_lock_pin() + + def test_the_pinned_lock_helper_checks_its_pin_and_passes_the_contract(self): + with mock.patch.object(ht, 'check_lock_pin') as pin: + helper = pinned_helper(self, self.tmp / 'locks') + pin.assert_called_once_with() + self.assertEqual((helper.PROTECTED_REASONS, helper.CI_REASON, helper.CLEANUP_ROLE), + (ht.PROTECTED, 'hil_test.py', ht.HOLDER_ROLE)) + self.assertTrue(issubclass(helper.LockUnavailable, RuntimeError)) + + def test_a_lock_helper_off_the_contract_is_refused(self): + if not ht.LOCK_HELPER.is_file(): + self.fail('lib/agentrc not checked out: git submodule update --init lib/agentrc') + text = ht.LOCK_HELPER.read_text() + for what, old, new in (('cleanup role', "CLEANUP_ROLE = 'cleanup-holder'", "CLEANUP_ROLE = 'other'"), + ('protected reasons', "{CI_REASON, 'pool_check'}", '{CI_REASON}'), + ('LockUnavailable not a RuntimeError', 'class LockUnavailable(RuntimeError)', 'class LockUnavailable(OSError)'), + ('LockUnavailable not a class', 'class LockUnavailable(RuntimeError):', + 'LockUnavailable = 1\nclass _LockUnavailable(RuntimeError):'), + ('protected reasons not iterable', "PROTECTED_REASONS = {CI_REASON, 'pool_check'}", + 'PROTECTED_REASONS = 1'), + ('protected reasons unhashable', "PROTECTED_REASONS = {CI_REASON, 'pool_check'}", + 'PROTECTED_REASONS = [[CI_REASON]]')): + self.assertIn(old, text) + off = self.tmp / 'off.py' + off.write_text(text.replace(old, new)) + with self.subTest(what), mock.patch.object(ht, 'LOCK_HELPER', off), mock.patch.object(ht, 'check_lock_pin'), \ + self.assertRaisesRegex(ht.Refused, 'recheck this contract'): + ht.load_helper() def test_the_lock_defaults_to_the_board_name_and_the_port_comes_from_it(self): b = ht.load_board(roster(self.tmp), BOARD) @@ -181,6 +342,9 @@ def test_the_committed_roster_is_well_formed_and_commissioned(self): f4 = self.load(ROOT / 'test' / 'hil' / 'local.json', 'stm32f411ve_discovery') self.assertEqual((f4['lock'], f4['reentry'], f4['usbPower'], f4['install'], f4['app']), ('stm32f411disco', 'retained-register', None, (0x08000000, 0x08010000), (0x08010000, 0x08080000))) + ci = self.load(ROOT / 'test' / 'hil' / 'ci.json', 'imxrt1064_evk') + self.assertEqual((ci['lock'], ci['reentry'], ci['usbPower'], ci['install'], ci['app']), + ('mimxrt1064_evk', 'retained-register', None, (0x70000000, 0x7000C000), (0x7000C000, 0x70400000))) facts = ht.source_facts(f4) for what, over in (('flash past the F411VE\'s 512 KiB', {'flashSize': 0x100000}), ('flash below 0x08000000', {'flashBase': 0x07FF0000})): @@ -205,7 +369,7 @@ def setUp(self): self.clock, self.writes, self.fail = [0.0], [], {} real_path = ht.Path - def spawn(cmd, log, timeout, cwd=None): + def spawn(cmd, log, timeout, cwd=None, **k): self.assertEqual((cmd[:3], timeout), (['sh', '-c', ht.PortPower.SET], ht.PortPower.TIMEOUT)) value = cmd[-1] self.writes.append((self.clock[0], value)) @@ -366,6 +530,7 @@ def test_the_build_record_is_read_by_board_name_and_must_be_cmake(self): (work / 'record/_bin/tinyuf2.bin', str(build / '_bin/tinyuf2.bin'), '1.0')) self.assertEqual((files['hex'], files['flash']['root'], files['flash']['sha256']['tinyuf2.hex']), (build / 'tinyuf2.hex', build, rows['tinyuf2.hex'])) # the HEX the target loads, not a copy + self.assertNotIn('ninja', files['flash']) # the install runs the provider's flash, never the target or a dry run for what, flash in (('another probe', dict(row['flash'], flasher={'name': 'jlink', 'uid': '9', 'device': 'MIMXRT1062xxx6A'})), ('no binding', None)): with self.subTest(what), self.assertRaisesRegex(ht.Refused, 'not bound to this roster flasher'): @@ -397,6 +562,19 @@ def test_the_build_record_is_read_by_board_name_and_must_be_cmake(self): ht.verify_record(rec, self.board, self.facts, work) +class EndChild(unittest.TestCase): + def test_only_a_still_unreaped_child_is_signalled(self): + gone = subprocess.Popen(['sleep', '30']) + gone.kill() + gone.wait() # reaped, as a test's own cleanup may already have done: its number may be reused + with mock.patch.object(os, 'kill') as kill: + self.assertEqual((end_child(gone.pid), end_child(1)), (False, False)) # reaped; not a child + kill.assert_not_called() + live = subprocess.Popen(['sleep', '30']) + self.assertTrue(end_child(live.pid)) + live.returncode = -signal.SIGKILL # reaped by end_child + + class SourceFacts(unittest.TestCase): def test_a_port_the_runner_has_no_facts_for_is_refused(self): with self.assertRaisesRegex(ht.Refused, 'ports/lpc55: the HIL runner has no source facts'): @@ -532,7 +710,7 @@ def partition_table(rows): def idf_build(root, parts=S3_PARTS, flash_args=None, sdk=None, board=S3, device='esp32s3', - board_id='ESP32S3-DevKitC-v1.0', family=0xc47e5767): + board_id='ESP32S3-DevKitC-v1.1', family=0xc47e5767): """A synthetic IDF build of the S3 DevKitC (or another board) as the port's CMakeLists lays it out, and its record row.""" (root / '_bin').mkdir(parents=True) @@ -588,7 +766,7 @@ def test_the_facts_come_from_the_idf_port(self): f = self.facts self.assertEqual((f['flashSize'], f['partitionCsv'], f['partitions']), (0x800000, 'partitions-8MB.csv', S3_PARTS)) self.assertEqual((f['vid'], f['pid'], f['boardId'], f['family'], f['detectionDelayMs']), - (0x239a, 0x00a5, 'ESP32S3-DevKitC-v1.0', 0xc47e5767, 500)) + (0x239a, 0x00a5, 'ESP32S3-DevKitC-v1.1', 0xc47e5767, 500)) self.assertEqual(f['flashCache'], 0x10000) self.assertEqual((self.board['flasher'], self.board['probe'], self.board['device']), ('esptool', 'cp2102n', 'esp32s3')) @@ -737,6 +915,9 @@ def test_a_bootloader_at_the_s3_offset_is_refused(self): k, r = op['op'], {'op': op['op']} if k == 'read': open(op['file'], 'wb').write(bytes(data[op['addr']:op['addr'] + op['size']])) elif k == 'md5': r['md5'] = hashlib.md5(bytes(data[op['addr']:op['addr'] + op['size']])).hexdigest() + elif k == 'security': r.update(json.loads(os.environ.get('FAKE_ESP_SECURITY', json.dumps( + {'flags': 0, 'flashCryptCnt': 0, 'rom': {'flashEncryption': 'off', 'secureBoot': 'off'}, + 'efuse': {'flashEncryption': 'off', 'secureBoot': 'off'}})))) out.append(r) open(img, 'wb').write(data) with open(img + '.sessions', 'a') as f: f.write(json.dumps({'args': args, 'ops': [o['op'] for o in ops]}) + '\\n') @@ -762,10 +943,8 @@ def setUp(self): real = ht.spawn self.targets = [] for p in (mock.patch.object(ht, 'ESP_SESSION', fake), mock.patch.object(ht, 'esp_port', lambda uid: str(tty)), - mock.patch.object(ht, 'container_present', lambda name: False), - mock.patch.object(ht, 'container_gone', lambda name: True), # never the real docker rm -f - mock.patch.object(ht.cb, 'flash_ready', lambda *a: None), - mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: self.docker(cmd) if cmd[0] == 'docker' else real(cmd, *a, **k)), + mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: self.esptool(cmd, k) if cmd[0] == 'esptool' + else real(cmd, *a, **k)), mock.patch.dict(os.environ, {'FAKE_ESP_FLASH': str(self.flash)})): p.start() self.addCleanup(p.stop) @@ -775,14 +954,16 @@ def setUp(self): def sessions(self): return [json.loads(l) for l in (self.tmp / 'flash.img.sessions').read_text().splitlines()] - def docker(self, cmd): - self.targets.append([str(c) for c in cmd]) - return (0, 'c0ffee\n') if cmd[1] == 'create' else self.target(cmd) + def esptool(self, cmd, k): + self.targets.append(([str(c) for c in cmd], k.get('cwd'), k.get('env'))) + return self.target(cmd) def target(self, cmd): - """ESP-IDF's flash target in its container: every segment written, FAKE_ESP_CORRUPT corrupting the segment at - that offset and FAKE_ESP_SPILL a preserved byte, and the transcript's shape from the bench.""" - img, out = bytearray(self.flash.read_bytes()), '' + """The provider's `esptool write_flash`: every segment written, FAKE_ESP_CORRUPT corrupting the segment at that + offset and FAKE_ESP_SPILL a preserved byte, and the transcript's shape from the bench, CSI codes and the underscore + spellings' warnings included (esptool 5.2.0 logger.py, cli_util.py).""" + img, out = bytearray(self.flash.read_bytes()), ("\x1b[0;33mWarning:\x1b[0m Deprecated: Command 'write_flash' is " + "deprecated. Use 'write-flash' instead.\n") for off, path in self.segments: blob = path.read_bytes() if os.environ.get('FAKE_ESP_CORRUPT') == hex(off): @@ -790,14 +971,15 @@ def target(self, cmd): img[off:off + len(blob)] = blob if os.environ.get('FAKE_ESP_SPILL') and off == 0: img[0x9000] ^= 0xff - out += f'Writing at 0x{off:08x}... (100 %)\nWrote {len(blob)} bytes (9 compressed) at 0x{off:08x} in 0.1 seconds\nHash of data verified.\n' + out += (f'Writing at 0x{off:08x} [====] 100.0% \x1b[1A\x1b[2K\nWrote {len(blob)} bytes (9 compressed) at ' + f'0x{off:08x} in 0.1 seconds (100.0 kbit/s).\nHash of data verified.\n') self.flash.write_bytes(img) - return 0, out + '\nLeaving...\nHard resetting via RTS pin...\n' + return 0, out + '\nHard resetting via RTS pin...\n' def files(self): root = self.tmp / 'b' return {'segments': self.segments, - 'flash': {'root': root, 'binding': {'target': 'flash'}, 'image': 'espressif/idf:v6.1', 'ninja': None, + 'flash': {'root': root, 'binding': {'target': 'flash'}, 'sha256': {rel: ht.sha256(root / rel) for rel in ht.cb.FLASH['esptool'][1]}}} @@ -810,7 +992,7 @@ def install(self): def test_install_writes_the_segments_reads_them_back_and_keeps_the_preserved_regions(self): before = self.flash.read_bytes() - self.install() + ev = self.install() img = self.flash.read_bytes() for off, path in self.segments: self.assertEqual(img[off:off + path.stat().st_size], path.read_bytes()) @@ -821,20 +1003,55 @@ def test_install_writes_the_segments_reads_them_back_and_keeps_the_preserved_reg self.assertEqual((snap['ops'], snap['args'][-1]), (['md5'] * 3, 'no-reset')) self.assertEqual((check['ops'], check['args'][-1]), (['read'] * 4 + ['md5'] * 3, 'hard-reset')) self.assertIn('--chip', check['args']) - docker, start = self.targets # the bridge's tty only, in a container named for the board, created first - tty = str(self.tmp / 'ttyUSB0') - self.assertEqual((docker[:2], start), (['docker', 'create'], ['docker', 'start', '-a', f'tinyuf2-hil-{S3}'])) - self.assertEqual(docker[docker.index('--name') + 1], f'tinyuf2-hil-{S3}') - self.assertEqual((docker[docker.index('--device') + 1], f'ESPPORT={tty}' in docker), (tty, True)) - self.assertIn('IDF_FLASH_FULL=1', docker) # every segment written, as judge_esp_target requires - self.assertEqual(docker[-5:], ['cmake', '--build', str(self.tmp / 'b'), '--target', 'flash']) + # the runner's own esptool on the bridge's tty, from the build dir as flash_args names the segments; no container + (argv, cwd, env), = self.targets + root, tty = self.tmp / 'b', str(self.tmp / 'ttyUSB0') + self.assertEqual(argv, ['esptool', '--chip', 'esp32s3', '-p', tty, '--before=default_reset', '--after=hard_reset', + 'write_flash', *(root / 'flash_args').read_text().split()]) + self.assertEqual((cwd, env), (str(root), {'NO_COLOR': '1'})) + self.assertEqual((ev['wrote'], ev['operation']), (['0x0', '0x8000', '0xe000', '0x410000'], + {'attempts': [{'kind': 'exited', 'exit': 0}], 'cleanup': 'done'})) + self.assertEqual(ev['esptool'], ht.ESPTOOL_VERSION) + + def test_flash_args_are_read_under_the_build_lock_from_bytes_that_hash_as_recorded(self): + fa, files = self.tmp / 'b' / 'flash_args', self.files() + good = fa.read_text() + fa.write_text(good.replace('0x410000', '0x400000')) # a writer that finishes as the lock is taken + take = ht.take_build_lock + + def taken(root): + fa.write_text(good) + return take(root) + with mock.patch.object(ht, 'take_build_lock', taken): + ht.EspBattery(self.board, self.facts, self.work).install(files) + self.assertEqual(self.targets[0][0][8:], good.split()) + + def test_flash_args_changed_after_the_hash_check_are_not_flashed(self): + fa = self.tmp / 'b' / 'flash_args' + check = ht.Battery.check_inputs + + def changed(flash, when, ev=None): + check(flash, when, ev) + if when == 'before the flash': + fa.write_text(fa.read_text().replace('0x410000', '0x400000')) + with mock.patch.object(ht.Battery, 'check_inputs', staticmethod(changed)), \ + self.assertRaisesRegex(ht.StepFail, 'flash_args changed'): + self.install() + self.assertEqual(self.targets, []) def test_a_session_that_fails_to_connect_writes_nothing(self): before = self.flash.read_bytes() with mock.patch.dict(os.environ, {'FAKE_ESP_FAIL': 'No serial data received.'}), \ self.assertRaisesRegex(ht.StepFail, 'No serial data'): self.install() - self.assertEqual(self.flash.read_bytes(), before) + self.assertEqual((self.flash.read_bytes(), self.targets), (before, [])) + + def test_no_tty_fails_before_any_session_and_leaves_the_board_untouched(self): + def gone(uid): + raise ht.StepFail('0 serial ports') + with mock.patch.object(ht, 'esp_port', gone), self.assertRaisesRegex(ht.StepFail, '0 serial ports'): + self.install() + self.assertEqual((self.bat.touched, (self.tmp / 'flash.img.sessions').exists(), self.targets), (False, False, [])) def test_a_bad_read_back_or_a_touched_preserved_region_fails_the_install(self): for env, err in (({'FAKE_ESP_CORRUPT': '0x410000'}, 'tinyuf2.bin does not read back as written'), @@ -842,10 +1059,26 @@ def test_a_bad_read_back_or_a_touched_preserved_region_fails_the_install(self): with self.subTest(err), mock.patch.dict(os.environ, env), self.assertRaisesRegex(ht.StepFail, err): self.install() + def test_a_failed_or_timed_out_flash_fails_touched_with_its_operation(self): + for what, result, err in (('exit 2', (2, 'A fatal error occurred: Failed to connect\n'), 'the esptool flash failed'), + ('timeout', ht.SpawnTimeout('esptool timed out', 'Writing at 0x00000000', []), + f'esptool timed out after {ht.ESP_TIMEOUT}s')): + def esptool(cmd, k): + self.targets.append(cmd) + if isinstance(result, Exception): + raise result + return result + with self.subTest(what), mock.patch.object(self, 'esptool', esptool), \ + self.assertRaisesRegex(ht.StepFail, err) as cm: + self.install() + self.assertTrue(self.bat.touched) + self.assertEqual((cm.exception.evidence['beforeTarget'], cm.exception.evidence['operation']['attempts'][0]['kind']), + (False, 'exited' if what == 'exit 2' else 'timed_out')) + def test_the_read_back_compares_the_preserved_regions_with_the_snapshot(self): tool, keep = ht.Esptool(self.board, self.work, self.work / 'hil.log'), self.board['regions']['preserved'] snap = ht.preserved_md5(keep, tool.run(ht.preserved_ops(keep), 'no-reset')) - self.target(['docker']) + self.target(['esptool']) ev = ht.esp_verify(tool, self.board, self.segments, self.work, snapshot=snap) self.assertEqual(ev['preserved'], snap) self.assertEqual(ev['writes'][-1], {'what': 'tinyuf2.bin', 'span': ['0x410000', hex(0x410000 + self.segments[-1][1].stat().st_size)], @@ -854,42 +1087,39 @@ def test_the_read_back_compares_the_preserved_regions_with_the_snapshot(self): with self.assertRaisesRegex(ht.StepFail, 'preserved region changed by the end of the install'): ht.esp_verify(tool, self.board, self.segments, self.work, snapshot=dict(snap, nvs='x')) - def test_the_target_transcript_must_show_every_segment_written_and_verified(self): - out = self.target(['docker'])[1] + def test_the_flash_transcript_must_show_every_segment_written_and_verified(self): + out = self.target(['esptool'])[1] + self.assertIn('\x1b[1A', out) self.assertEqual(ht.judge_esp_target(0, out, self.segments)['wrote'], ['0x0', '0x8000', '0xe000', '0x410000']) + # a CSI code at a line's start would hide a write from a judge that strips only colour codes + csi = out.replace('Wrote', '\x1b[2K\x1b[1AWrote') + self.assertEqual(ht.judge_esp_target(0, csi, self.segments)['wrote'], ['0x0', '0x8000', '0xe000', '0x410000']) cases = (('a nonzero exit', 1, out), ('a segment missing', 0, out.replace('at 0x00008000 in', 'at 0x00009000 in')), ('a hash not verified', 0, out.replace('Hash of data verified.\n', '', 1)), ('no reset', 0, out.replace('Hard resetting via RTS pin...', '')), ('nothing', 0, '')) for what, code, text in cases: - with self.subTest(what), self.assertRaisesRegex(ht.StepFail, 'the flash target failed'): + with self.subTest(what), self.assertRaisesRegex(ht.StepFail, 'the esptool flash failed'): ht.judge_esp_target(code, text, self.segments) - def test_an_admitted_record_installs(self): + def admitted(self): now = {'head': 'h', 'submodules': {}, 'deps': {}, 'dirty': False} rec = self.tmp / 'rec.json' rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [self.row], 'idf': {'mode': 'docker', 'image': 'espressif/idf:v6.1'}})) with mock.patch.object(ht.cb, 'identity', lambda ports: now), mock.patch.object(ht.cb, 'ci_idf_version', lambda: 'v6.1'): - files, _ = ht.verify_record(rec, self.board, self.facts, self.work) - self.bat = ht.EspBattery(self.board, self.facts, self.work) - self.assertTrue(self.bat.install(files)['readBack']) - self.assertEqual(self.targets[0][self.targets[0].index('--device') - 1], f'tinyuf2-hil-{S3}') - self.assertIn('espressif/idf:v6.1', self.targets[0]) - - def test_an_interrupted_create_leaves_the_name_for_cleanup_and_the_board_untouched(self): - def interrupted(cmd): - raise ht.Interrupted('interrupted by SIGTERM') - self.docker = interrupted # the patched spawn looks it up per call - with mock.patch.object(ht, 'CONTAINERS', []) as tracked, self.assertRaises(ht.Interrupted): - self.install() - self.assertEqual(tracked, [f'tinyuf2-hil-{S3}']) - self.assertEqual((self.bat.touched, (self.tmp / 'flash.img.sessions').exists()), (False, False)) + return ht.verify_record(rec, self.board, self.facts, self.work)[0] - def test_a_container_left_by_an_earlier_flash_refuses_before_any_session(self): - with mock.patch.object(ht, 'container_present', lambda name: True), \ - self.assertRaisesRegex(ht.StepFail, f'container tinyuf2-hil-{S3} from an earlier flash'): - self.install() - self.assertEqual((self.bat.touched, (self.tmp / 'flash.img.sessions').exists(), self.targets), (False, False, [])) + def test_an_admitted_record_installs_from_its_build_dir_without_a_container_or_a_dry_run(self): + files = self.admitted() + self.assertEqual(set(files['flash']), {'root', 'binding', 'sha256'}) + spawned = [] + real = ht.spawn + self.bat = ht.EspBattery(self.board, self.facts, self.work) + with mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: (spawned.append(cmd[0]), real(cmd, *a, **k))[1]): + self.assertTrue(self.bat.install(files)['readBack']) + self.assertNotIn('docker', spawned) + self.assertNotIn('ninja', spawned) + self.assertEqual(self.targets[0][1], str(files['flash']['root'])) def test_the_install_snapshot_round_trips_through_the_report(self): keep = self.board['regions']['preserved'] @@ -935,6 +1165,7 @@ def test_tty_users_sees_a_process_holding_the_device_open(self): p = subprocess.Popen([sys.executable, '-c', f'f = open({str(dev)!r}); import time; time.sleep(30)']) self.addCleanup(lambda: (p.kill(), p.wait())) ht.wait_for(lambda: p.pid in ht.tty_users(str(dev)), 5, 'the holder') + self.assertEqual(ht.tty_users(str(self.tmp / 'other'), str(dev)).count(p.pid), 1) # either device, listed once def test_a_closing_fd_does_not_hide_the_process_others(self): dev = self.tmp / 'tty' @@ -952,7 +1183,7 @@ def closing(fd): # stdin closes under the scan def main(self, *argv): out = io.StringIO() - with mock.patch.object(ht, 'load_helper', lambda path: mock.Mock()), mock.patch.object(ht, 'manual_holder', mock.Mock()), \ + with mock.patch.object(ht, 'load_helper', mock.Mock), mock.patch.object(ht, 'manual_holder', mock.Mock()), \ mock.patch.object(ht.cb, 'BUILD_DIR', Path(tempfile.mkdtemp(dir=self.tmp))), contextlib.redirect_stdout(out): code = ht.main(list(argv)) return code, json.loads(out.getvalue().strip().splitlines()[-1]) @@ -967,9 +1198,378 @@ def test_an_esptool_install_takes_a_build_record_and_a_run_only_esptool_steps_an self.assertEqual(code, 2) self.assertIn("re-enters by one of ('dfu-detach',)", res['error']) - def test_the_esptool_version_is_gated(self): - with mock.patch('importlib.metadata.version', lambda name: '4.8.1'), self.assertRaisesRegex(ht.Refused, '5.2.0'): - ht.check_esptool() + def test_a_run_refuses_a_route_pending_commissioning_or_unknown_before_any_board_access(self): + cfg = str(s3_roster(self.tmp, tinyuf2={'reentry': 'dfu-detach', 'routes': [{'flasher': { + 'name': 'openocd_esp', 'uid': 'AA:BB:CC:DD:EE:FF', 'args': '-f board/esp32s3-builtin.cfg'}, 'commissioning': 'pending'}]})) + for route, err in (('openocd_esp', 'pending commissioning'), ('pyocd', 'no install route pyocd')): + with self.subTest(route), mock.patch.object(ht, 'run_board', side_effect=AssertionError('board access')): + code, res = self.main('run', '-b', S3, '--build-record', 'rec.json', '--route', route, cfg) + self.assertEqual(code, 2) + self.assertIn(err, res['error']) + + def test_the_esptool_version_and_the_command_the_provider_runs_are_gated(self): + def missing(name): + raise importlib.metadata.PackageNotFoundError(name) + for what, version in (('wrong version', lambda name: '4.8.1'), ('absent', missing)): + with self.subTest(what), mock.patch('importlib.metadata.version', version), \ + self.assertRaisesRegex(ht.Refused, 'the runner needs esptool 5.2.0'): + ht.check_esptool() + bindir = self.tmp / 'bin' + bindir.mkdir() + venv = self.tmp / 'venv' / 'bin' + venv.mkdir(parents=True) + (venv / 'python3').symlink_to(sys.executable) # a venv's python links out of the venv + script = bindir / 'esptool' + cases = (('this python', f'#!{sys.executable}\n', None), ('another python', f'#!{venv / "python3"}\n', 'does not run under'), + ('another python in its directory', f'#!{Path(sys.executable).with_name("python-other")}\n', 'does not run under'), + ('no shebang', 'import esptool\n', 'does not run under'), ('not on PATH', None, 'does not run under')) + for what, head, err in cases: + if head is None: + script.unlink(missing_ok=True) + else: + script.write_text(head) + script.chmod(0o755) + with self.subTest(what), mock.patch('importlib.metadata.version', lambda name: '5.2.0'), \ + mock.patch.object(ht.shutil, 'which', lambda n: str(script) if script.exists() else None): + if err: + with self.assertRaisesRegex(ht.Refused, err): + ht.check_esptool() + else: + ht.check_esptool() + + +USJ = 'AA:BB:CC:DD:EE:FF' +ROUTE = {'flasher': {'name': 'openocd_esp', 'uid': USJ, 'args': '-f board/esp32s3-builtin.cfg'}, 'commissioning': 'pending'} +# agentrc's own openocd-esp32 stand-in and fake sysfs bus, from the pinned checkout's tests of the provider +_ag = importlib.util.spec_from_file_location('agentrc_test_openocd_esp', ROOT / 'lib/agentrc/tests/test_hil_openocd_esp.py') +AG = importlib.util.module_from_spec(_ag) +_ag.loader.exec_module(AG) + + +def route_roster(tmp, **route): + return s3_roster(tmp, tinyuf2={'reentry': 'dfu-detach', 'routes': [dict(ROUTE, **route)]}) + + +class RouteRoster(Tmp): + """tinyuf2.routes: an Espressif board's openocd_esp route, under the board's own lock, chosen before board access.""" + + def setUp(self): + super().setUp() + no_ci_job(self) + + def test_the_route_is_refused_under_a_ci_job_before_any_board_access(self): + b = self.load(route_roster(self.tmp), S3) + for var, value in (('RUNNER_TRACKING_ID', ''), ('GITHUB_ACTIONS', 'true')): + with self.subTest(var), mock.patch.dict(os.environ, {var: value}): + with self.assertRaisesRegex(ht.Refused, rf'under a CI job \({var} set\)'): + ht.select_route(b, 'openocd_esp') + self.assertIs(ht.select_route(b, 'esptool'), b) # not an openocd_esp session + self.assertEqual(ht.select_route(b, 'openocd_esp')['flasher'], 'openocd_esp') + + def test_a_route_loads_and_a_chosen_one_selects_the_flash_and_the_esptool_tty(self): + b = self.load(route_roster(self.tmp), S3) + self.assertEqual(b['routes'], {'openocd_esp': {'flasher': ROUTE['flasher'], 'pending': True}}) + self.assertIs(ht.select_route(b, None), b) + self.assertIs(ht.select_route(b, 'esptool'), b) + r = ht.select_route(b, 'openocd_esp') + self.assertEqual((r['flasher'], r['probe'], r['route'], r['kind'], r['lock']), + ('openocd_esp', USJ, ROUTE['flasher'], 'esp', b['lock'])) # one lock for both routes + with self.assertRaisesRegex(ht.Refused, 'pending commissioning: install and restore only'): + ht.select_route(b, 'openocd_esp', run=True) + commissioned = self.load(s3_roster(self.tmp, tinyuf2={'reentry': 'dfu-detach', 'routes': [{'flasher': ROUTE['flasher']}]}), S3) + self.assertEqual(ht.select_route(commissioned, 'openocd_esp', run=True)['flasher'], 'openocd_esp') + with self.assertRaisesRegex(ht.Refused, 'no install route jlink'): + ht.select_route(b, 'jlink') + + def test_route_refusals(self): + fl = ROUTE['flasher'] + cases = {'not espressif': (roster(self.tmp, tinyuf2={'reentry': 'retained-register', 'routes': [ROUTE]}), BOARD), + 'another chip': (route_roster(self.tmp, flasher=dict(fl, args='-f board/esp32p4-builtin.cfg')), S3), + 'a selector in args': (route_roster(self.tmp, flasher=dict(fl, args='-f board/esp32s3-builtin.cfg -c init')), S3), + 'another backend': (route_roster(self.tmp, flasher=dict(fl, name='openocd')), S3), + 'a vid_pid': (route_roster(self.tmp, flasher=dict(fl, vid_pid='0x303a 0x1001')), S3), + 'an odd uid': (route_roster(self.tmp, flasher=dict(fl, uid='x;shutdown')), S3), + 'another state': (route_roster(self.tmp, commissioning='done'), S3), + 'an unknown key': (route_roster(self.tmp, lock='x'), S3), + 'twice': (s3_roster(self.tmp, tinyuf2={'reentry': 'dfu-detach', 'routes': [ROUTE, ROUTE]}), S3), + 'not a list': (s3_roster(self.tmp, tinyuf2={'reentry': 'dfu-detach', 'routes': ROUTE}), S3)} + for what, (cfg, name) in cases.items(): + with self.subTest(what), self.assertRaises(ht.Refused): + ht.load_board(cfg, name) + + def test_the_p4_roster_entry_is_a_commissioned_openocd_esp_route_of_its_board(self): + b = ht.load_board(ROOT / 'test/hil/local.json', 'espressif_p4_function_ev') + self.assertEqual(b['routes'], {'openocd_esp': {'flasher': {'name': 'openocd_esp', 'uid': '30:ED:A0:E0:C9:02', + 'args': '-f board/esp32p4-builtin.cfg'}, 'pending': False}}) + + def test_the_route_needs_the_verified_openocd_and_the_runners_esptool(self): + b = ht.select_route(self.load(route_roster(self.tmp), S3), 'openocd_esp') + with mock.patch.object(ht, 'check_esptool'), \ + mock.patch.object(ht.openocd_esp(), 'find_openocd', side_effect=RuntimeError('openocd not found')), \ + self.assertRaisesRegex(ht.Refused, 'openocd not found'): + ht.check_flasher(b) + + def test_the_probe_check_sees_the_usj_tty_and_usbfs_node_users(self): + with mock.patch.object(AG, 'oe', ht.openocd_esp()): + AG.FakeBus(self).add('5-3.4', USJ, bus=5, dev=14) + b = ht.select_route(self.load(route_roster(self.tmp), S3), 'openocd_esp') + seen = [] + with mock.patch.object(ht, 'esp_port', lambda uid: f'/dev/tty-{uid}'), \ + mock.patch.object(ht, 'tty_users', lambda *devs: seen.append(devs) or [7]): + self.assertEqual(ht.probe_users(b), [7]) + self.assertEqual(seen, [(f'/dev/tty-{USJ}', '/dev/bus/usb/005/014')]) # one walk for both + + +class JtagAdmission(EspCase): + """A record binds the openocd_esp route to the segments its esptool binding hashed (check_build flashRoutes).""" + roster = {'tinyuf2': {'reentry': 'dfu-detach', 'routes': [ROUTE]}} + + def setUp(self): + super().setUp() + no_ci_job(self) + self.board = ht.select_route(self.board, 'openocd_esp') + + def row(self, root, **binding): + row = idf_build(root) + row['flashRoutes'] = {'openocd_esp': dict({'flasher': {'name': 'openocd_esp', 'uid': USJ, 'device': 'esp32s3'}, + 'inputs': list(ht.cb.FLASH['esptool'][1])}, **binding)} + return row + + def test_a_bound_record_gives_the_hashed_segments(self): + files, _ = self.verify(self.record(self.row(self.tmp / 'b'))) + self.assertEqual(files['flash']['binding']['flasher']['name'], 'openocd_esp') + self.assertEqual(sorted(files['flash']['sha256']), sorted(ht.cb.FLASH['esptool'][1])) + self.assertEqual([off for off, _ in files['segments']], [0, 0x8000, 0xe000, 0x410000]) + + def test_a_record_not_bound_to_this_route_is_refused(self): + fl = {'name': 'openocd_esp', 'uid': USJ, 'device': 'esp32s3'} + for i, (what, over) in enumerate((('another USJ', {'flasher': dict(fl, uid='00:11:22:33:44:55')}), + ('another chip', {'flasher': dict(fl, device='esp32p4')}), + ('a segment missing', {'inputs': list(ht.cb.FLASH['esptool'][1])[:-1]}), + ('no route', None))): + with self.subTest(what), self.assertRaisesRegex(ht.Refused, 'not bound to this roster flasher'): + row = self.row(self.tmp / f'b{i}', **(over or {})) + if over is None: + del row['flashRoutes'] # an esptool binding alone + self.verify(self.record(row)) + + def test_a_stale_segment_is_refused(self): + row = self.row(self.tmp / 'b') + (self.tmp / 'b' / 'tinyuf2.bin').write_bytes(b'rebuilt') + with self.assertRaisesRegex(ht.Refused, 'tinyuf2.bin: missing, unreadable or changed'): + self.verify(self.record(row)) + + +class JtagRoute(EspFake): + """The JTAG route's install over agentrc's flash_openocd_esp and its openocd-esp32 stand-in, the esptool sessions + over the fake session helper, both on one flash image and one event log: plumbing only, never a live session.""" + + def setUp(self): + super().setUp() + no_ci_job(self) + self.board = ht.select_route(self.load(route_roster(self.tmp), S3), 'openocd_esp') + self.mark = self.board['pendingMark'] = ht.PendingMark(None, 'b') + exe, _ = AG.fake_install(self.tmp / 'oocd', chips=('esp32s3',), tool=AG.FAKE_OPENOCD) + with mock.patch.object(AG, 'oe', ht.openocd_esp()): + AG.FakeBus(self).add('5-3.4', USJ) + for p in (mock.patch.dict(os.environ, {'HIL_OPENOCD_ESP': str(exe), 'FAKE_BANNER': AG.BANNER, + 'FAKE_EVENTS': str(self.flash) + '.sessions', 'FAKE_FLASH': str(self.flash), + 'FAKE_BANK_SIZE': hex(0x800000)}), + mock.patch.object(ht, 'OPENOCD_TIMEOUT', 3), mock.patch.object(ht, 'OPENOCD_BOOT_SETTLE', 0), + mock.patch.object(ht.openocd_esp(), 'CLEANUP_TIMEOUT', 3)): + p.start() + self.addCleanup(p.stop) + self.addCleanup(self.kill_fakes) + # the provider leaves a pending session's Popen unwaited, by design; the stand-in is ended at cleanup + self.enterContext(warnings.catch_warnings()) + warnings.simplefilter('ignore', ResourceWarning) + + def kill_fakes(self): # the stand-in ignores SIGTERM, as the runner must never signal openocd; the test ends it + for e in self.events(): + if 'pid' in e: + with contextlib.suppress(ProcessLookupError): + os.kill(e['pid'], signal.SIGKILL) + + def events(self): + path = self.tmp / 'flash.img.sessions' + return [json.loads(l) for l in path.read_text().splitlines()] if path.exists() else [] + + def install(self): + self.bat = ht.EspBattery(self.board, self.facts, self.work) + return self.bat.install(self.files()) + + def test_security_and_snapshot_then_jtag_program_and_read_back_then_esptool_preservation_check(self): + before = self.flash.read_bytes() + ev = self.install() + log = self.events() + order = ['esptool' if 'ops' in e else e['tcl'].split()[0] for e in log if 'ops' in e or e.get('tcl', 'token?') != 'token?'] + cpus = ['esp32p4.hp.cpu0'] * 2 + ['esp32p4.hp.cpu1'] * 2 # each -work-area-backup saved, then cleared + self.assertEqual(order, ['esptool', 'init', 'reset', 'flash', 'flash', 'target', *cpus, 'esp', 'esp', 'esp', + *['check_image_revision', 'flash', 'esp'] * 4, 'esp32p4.hp.cpu0', 'esp32p4.hp.cpu1', + 'flash', 'flash', 'flash', 'flash', 'reset', 'shutdown', 'esptool']) + snap, check = [e for e in log if 'ops' in e] + self.assertEqual((snap['ops'], snap['args'][-1]), (['security'] + ['md5'] * 3, 'hard-reset')) # one ROM-loader session + self.assertEqual((check['ops'], check['args'][-1]), (['read'] * 4 + ['md5'] * 3, 'hard-reset')) + self.assertFalse([e for e in log if 'signal' in e]) + img = self.flash.read_bytes() + for off, path in self.segments: + self.assertEqual(img[off:off + path.stat().st_size], path.read_bytes()) + for lo, hi in self.board['regions']['preserved'].values(): + self.assertEqual(img[lo:hi], before[lo:hi]) + self.assertEqual((ev['route'], ev['wrote'], ev['readBack'], ev['operation']['cleanup']), + ('openocd_esp', ['0x0', '0x8000', '0xe000', '0x410000'], True, 'done')) + self.assertEqual((ev['security']['flashEncryption'], ev['security']['secureBoot']), ('off', 'off')) + self.assertIn('ROM loader', ev['security']['source']) + self.assertEqual(ht.outstanding_sessions(self.work), []) + + def test_security_not_read_off_or_a_failed_snapshot_programs_nothing(self): + on = {'flags': 1, 'flashCryptCnt': 1, 'rom': {'flashEncryption': 'on', 'secureBoot': 'on'}, + 'efuse': {'flashEncryption': 'on', 'secureBoot': 'on'}} + disagree = dict(on, flags=0, flashCryptCnt=0, rom={'flashEncryption': 'off', 'secureBoot': 'off'}, + efuse={'flashEncryption': 'off', 'secureBoot': 'on'}) + for what, env, err in (('on', {'FAKE_ESP_SECURITY': json.dumps(on)}, 'not read off'), + ('sources disagree', {'FAKE_ESP_SECURITY': json.dumps(disagree)}, 'not read off'), + ('snapshot fails', {'FAKE_ESP_FAIL': 'No serial data received.'}, 'No serial data')): + before = self.flash.read_bytes() + with self.subTest(what), mock.patch.dict(os.environ, env), self.assertRaisesRegex(ht.StepFail, err): + self.install() + self.assertEqual((self.flash.read_bytes(), [e for e in self.events() if 'argv' in e]), (before, [])) + self.assertTrue(self.bat.touched) + + def pending(self, err, **env): + root = self.files()['flash']['root'] + with mock.patch.dict(os.environ, env), self.assertRaisesRegex(ht.CleanupPending, err) as cm: + self.install() + left = ht.outstanding_sessions(self.work) + self.assertEqual(len(left), 1) + self.assertIsNotNone(ht.proc_stat(left[0]['pid'])) # never signalled, never reaped + self.assertEqual(ht.settle_children(), []) + self.assertIsNotNone(ht.proc_stat(left[0]['pid'])) + self.assertFalse([e for e in self.events() if 'signal' in e]) + # the build dir stays locked, by a keeper, while the session runs (#579) + self.assertEqual([k['survivors'] for k in self.bat.kept], [left]) + with self.assertRaises(BlockingIOError), ht.cb.build_lock(root, wait=False): + pass + # no later session: read-back, restore and retry are refused + for later in (lambda: self.bat.tool.run(ht.preserved_ops(self.board['regions']['preserved']), 'hard-reset'), + lambda: self.bat.install(self.files())): + with self.assertRaisesRegex(ht.CleanupPending, 'no session may overlap it'): + later() + # another invocation, in a run dir of its own, refuses before any session (#2) + with self.assertRaisesRegex(ht.Refused, 'not ended'): + ht.clear_pending_mark(self.mark) + os.kill(left[0]['pid'], signal.SIGKILL) # stands for the server exiting + ht.wait_for(lambda: ht.flock_free(Path(str(root) + '.lock')), 10, 'the keeper releasing the build lock') + # gone without the reviewed end: still not ended, for a separately reviewed recovery + self.assertEqual([ht.session_state(e) for e in ht.outstanding_sessions(self.work)], ['unresolved']) + with self.assertRaisesRegex(ht.Refused, 'not ended'): + ht.clear_pending_mark(self.mark) + if isinstance(left[0]['receipt'], str): # what `hil_openocd_esp.py end` leaves: closed after the reviewed end + rec = json.loads(Path(left[0]['receipt']).read_text()) + Path(left[0]['receipt']).write_text(json.dumps(dict(rec, closed=True, cleanup='done'))) + self.assertEqual(ht.outstanding_sessions(self.work), []) + ht.clear_pending_mark(self.mark) + self.assertFalse(Path(self.mark).exists()) + return cm.exception + + def test_the_provider_starts_at_least_the_settle_time_after_the_esptool_session(self): + ended, started = [], [] + run, flash = ht.Esptool.run, ht.openocd_esp().flash_openocd_esp + with mock.patch.object(ht.Esptool, 'run', lambda *a: (run(*a), ended.append(time.monotonic()))[0]), \ + mock.patch.object(ht.openocd_esp(), 'flash_openocd_esp', + lambda *a, **k: (started.append(time.monotonic()), flash(*a, **k))[1]), \ + mock.patch.object(ht, 'OPENOCD_BOOT_SETTLE', 0.2): + self.install() + self.assertGreaterEqual(started[0] - ended[0], ht.OPENOCD_BOOT_SETTLE) + + def test_a_ci_job_is_refused_before_the_security_and_snapshot_session(self): + with mock.patch.dict(os.environ, {'GITHUB_ACTIONS': ''}), mock.patch.object(ht.Esptool, 'run') as run, \ + self.assertRaisesRegex(ht.Refused, 'under a CI job'): + self.install() + self.assertEqual((run.call_count, self.bat.touched, self.events()), (0, False, [])) + + def test_a_timeout_while_programming_leaves_cleanup_pending(self): + with mock.patch.object(ht, 'OPENOCD_TIMEOUT', 0.7), mock.patch.object(ht.openocd_esp(), 'CLEANUP_TIMEOUT', 0.7): + e = self.pending('cleanup pending', FAKE_SCRIPT=json.dumps({'flash#3': 'hang'})) # the first write; each bound expires + self.assertEqual(e.evidence['operation']['cleanup'], 'pending') + + def test_a_server_answering_another_token_is_never_driven_and_is_left_pending(self): + self.pending('cleanup pending', FAKE_OTHER_TOKEN='f00f') + self.assertFalse([e for e in self.events() if e.get('tcl') not in (None, 'token?')]) + + def test_a_session_whose_receipt_could_not_be_written_is_still_protected(self): + es = ht.openocd_esp().esp_session # the provider writes every receipt through it + write = es.write_receipt + + def failing(path, rec): + if not rec.get('closed') and rec.get('submitted') == 0: + raise OSError(28, 'No space left on device') + return write(path, rec) + with mock.patch.object(es, 'write_receipt', failing): + e = self.pending('cleanup pending') + self.assertEqual(ht.outstanding_sessions(self.work)[0]['receipt']['tool'], 'hil_openocd_esp') # the record + self.assertFalse(list(self.work.glob(ht.OPENOCD_SESSIONS))) + self.assertEqual(e.evidence['operation']['cleanup'], 'pending') + + def test_a_failed_shutdown_leaves_cleanup_pending(self): + with mock.patch.object(ht.openocd_esp(), 'CLEANUP_TIMEOUT', 0.7): # the bound the ignored shutdown expires + self.pending('cleanup pending', FAKE_IGNORE_SHUTDOWN='1') + + def test_a_failed_program_ends_the_session_in_order_and_fails_the_install(self): + # esp#5: after the three esp setup commands and the first verify, the second segment's verify + with mock.patch.dict(os.environ, {'FAKE_SCRIPT': json.dumps({'esp#5': 'err:hash mismatch'})}), \ + self.assertRaisesRegex(ht.StepFail, 'the openocd_esp flash failed') as cm: + self.install() + self.assertNotIsInstance(cm.exception, ht.CleanupPending) + self.assertEqual([e['tcl'] for e in self.events() if e.get('tcl') in ('reset run', 'shutdown')], ['reset run', 'shutdown']) + self.assertEqual((ht.outstanding_sessions(self.work), self.bat.kept), ([], [])) + + def test_only_a_clean_exit_with_every_segment_verified_and_read_back_passes(self): + seg = {'offset': 0, 'program': {'verified': True}, 'readBack': {'match': True}} + op = mock.Mock(returncode=0, cleanup='done', segments=(seg,), session={'openocd': 'o', 'location': '5-3.4', 'log': 'l'}) + self.assertEqual(ht.judge_openocd_esp(op)['wrote'], ['0x0']) + for what, over in (('raised after the start', {'returncode': None}), ('cleanup unknown', {'cleanup': 'unknown'}), + ('no segments', {'segments': ()}), + ('a read-back mismatch', {'segments': (dict(seg, readBack={'match': False}),)})): + with self.subTest(what), self.assertRaisesRegex(ht.StepFail, 'the openocd_esp flash failed'): + ht.judge_openocd_esp(mock.Mock(**dict(dict(returncode=0, cleanup='done', segments=(seg,), session={}), **over))) + + def test_an_interrupt_while_programming_unwinds_with_the_session_kept_for_cleanup(self): + handler = signal.signal(signal.SIGTERM, ht.on_signal) + mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, ht.TERM_SIGNALS) + self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) + self.addCleanup(signal.signal, signal.SIGTERM, handler) + real = ht.openocd_esp().flash_openocd_esp + + def signalled(*a, **k): + sessions = self.tmp / 'flash.img.sessions' + sigterm_when(lambda: sessions.exists() and 'write_image' in sessions.read_text(), 'the hanging write') + return real(*a, **k) + with mock.patch.dict(os.environ, {'FAKE_SCRIPT': json.dumps({'flash#3': 'hang'})}), \ + mock.patch.object(ht, 'OPENOCD_TIMEOUT', 30), mock.patch.object(ht.openocd_esp(), 'flash_openocd_esp', signalled), \ + self.assertRaises(ht.Interrupted) as cm: + self.install() + self.assertEqual(cm.exception.operation.cleanup, 'pending') + left = ht.outstanding_sessions(self.work) + self.assertEqual(([k['survivors'] for k in self.bat.kept], len(left)), ([left], 1)) + self.assertFalse([e for e in self.events() if 'signal' in e]) + + def test_an_interrupt_while_the_server_registers_still_leaves_its_receipt(self): + handler = signal.signal(signal.SIGTERM, ht.on_signal) + mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, ht.TERM_SIGNALS) + self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) + self.addCleanup(signal.signal, signal.SIGTERM, handler) + oe = ht.openocd_esp() + start_time = oe.esp_session.start_time + + def signal_at_registration(pid): # inside the provider's hold, between the spawn and the receipt + os.kill(os.getpid(), signal.SIGTERM) + return start_time(pid) + with mock.patch.object(oe.esp_session, 'start_time', signal_at_registration), self.assertRaises(ht.Interrupted): + self.install() + left = ht.outstanding_sessions(self.work) + self.assertEqual(len(left), 1) + self.assertEqual([k['survivors'] for k in self.bat.kept], [left]) STUB_ESPTOOL = { @@ -989,6 +1589,13 @@ def read_flash(self, addr, size): if os.environ.get('STUB_FAIL_READ'): raise IOError('timed out') return bytes([addr & 0xff]) * size def flash_md5sum(self, addr, size): return f'md5-{addr:x}-{size:x}' + IS_STUB = bool(os.environ.get('STUB_IS_STUB')) + SEC = json.loads(os.environ.get('STUB_SECURITY', '[0, 0, 0, 0]')) # flags, flash_crypt_cnt, eFuse FE, eFuse SB + def get_security_info(self, cache=True): + log('security', cache) + return {'flags': self.SEC[0], 'flash_crypt_cnt': self.SEC[1], 'parsed_flags': {'SECURE_BOOT_EN': bool(self.SEC[0] & 1)}} + def get_flash_encryption_enabled(self): log('efuse'); return bin(self.SEC[2]).count('1') % 2 == 1 + def get_secure_boot_enabled(self): return self.SEC[3] << 20 def detect_chip(port, connect_mode): log('connect', port, connect_mode); return Esp() def run_stub(esp): log('stub'); return esp @@ -1010,7 +1617,7 @@ def usb_device(self, vid, pid, serial): want = {'dfu': (f['vid'], f['pid']), 'app': (f['appVid'], f['appPid'])}.get(self.on) return '1-2' if (vid, pid) == want and serial == self.bat.b['uid'] else None - def spawn(self, cmd, log, timeout, cwd=None): + def spawn(self, cmd, log, timeout, cwd=None, **k): if cmd[0] == 'mcopy': blocks = ht.parse_uf2(Path(cmd[4]).read_bytes()) self.copied.append(blocks) @@ -1028,9 +1635,9 @@ def spawn(self, cmd, log, timeout, cwd=None): return 0, json.dumps({'ok': True, 'interface': 0}) blk = self.stale or self.app.get(self.bat.nonce_off, b'') return 0, json.dumps({'ok': True, 'interface': 0, 'string': f'HIL {blk[12:28].hex()}'}) - code, out = self.real_spawn(cmd, log, timeout, cwd) - if cmd[:2] == ['docker', 'start']: - self.on = 'dfu' # the target's hard reset; blank otadata boots the factory app: TinyUF2 + code, out = self.real_spawn(cmd, log, timeout, cwd, **k) + if cmd[0] == 'esptool': + self.on = 'dfu' # esptool's hard reset; blank otadata boots the factory app: TinyUF2 return code, out def fat_read(self, dev, name, length, log, offset=0): @@ -1069,7 +1676,7 @@ def battery(self): def test_every_step_passes_with_a_fresh_nonce_per_copy(self): bat, rig, files = self.battery() bat.run(files, 'v1') - self.assertEqual([x['step'] for x in bat.steps if x['pass']], list(ht.STEPS['esptool'])) + self.assertEqual([x['step'] for x in bat.steps if x['pass']], list(ht.STEPS['esp'])) ev = {x['step']: x['evidence'] for x in bat.steps} self.assertEqual(ev['app-ran']['string'], f'HIL {ev["copy"]["nonce"]}') self.assertEqual(ev['recopy']['string'], f'HIL {ev["recopy"]["nonce"]}') @@ -1267,6 +1874,25 @@ def test_another_chip_and_a_failing_op_are_reported_and_the_port_closed(self): self.assertEqual(calls[-1], ['close']) self.assertNotIn('reset', [c[0] for c in calls]) + def test_the_security_state_is_read_first_on_the_rom_loader_from_two_sources(self): + off = {'flashEncryption': 'off', 'secureBoot': 'off'} + code, out, calls = self.session([{'op': 'security'}, {'op': 'md5', 'addr': 0, 'size': 0x1000}], after='no-reset') + self.assertEqual(code, 0, out) + self.assertEqual(out['results'][0], {'op': 'security', 'flags': 0, 'flashCryptCnt': 0, 'rom': off, 'efuse': off}) + self.assertEqual([c[0] for c in calls][:4], ['connect', 'security', 'efuse', 'stub']) # before the stub + self.assertEqual(calls[1], ['security', False]) # never esptool's cached connect-time answer + for sec, rom, efuse in (([1, 0, 0, 0], {'secureBoot': 'on'}, {}), ([0, 3, 7, 0], {}, {'flashEncryption': 'on'}), + ([0, 1, 1, 1], {'flashEncryption': 'on'}, {'flashEncryption': 'on', 'secureBoot': 'on'})): + code, out, _ = self.session([{'op': 'security'}], after='no-reset', STUB_SECURITY=json.dumps(sec)) + self.assertEqual((out['results'][0]['rom'], out['results'][0]['efuse']), (dict(off, **rom), dict(off, **efuse))) + code, out, _ = self.session([{'op': 'security'}], after='no-reset', STUB_IS_STUB='1') + self.assertEqual(code, 1) + self.assertIn('ROM loader', out['error']) + code, out, calls = self.session([{'op': 'md5', 'addr': 0, 'size': 0x1000}, {'op': 'security'}], after='no-reset') + self.assertEqual((code, calls), (1, [])) + self.assertIn('comes first', out['error']) + + STUB_USB = { 'usb/__init__.py': '', 'usb/core.py': """import json, os @@ -1382,6 +2008,31 @@ def test_embedded_version(self): self.assertEqual(ht.embedded_version(b), '0.35.0-1-gabc - tinyusb (x)') +class Citations(unittest.TestCase): + CONTRACT = SCRIPT.parent.parent + CITE = re.compile(r'\b(SKILL|commissioning|bench-history)\.md(?:[,:]\s+([^,;:)\n]+?)\s*[),;:]| ([A-Z][\w/-]*))') + + def unresolved(self, text): + """Citations in text whose label is no heading, bold item label or file name of the page they name.""" + bad = [] + for m in self.CITE.finditer(text): + page, label = f'{m[1]}.md', (m[2] or m[3]).strip() + doc = (self.CONTRACT / page).read_text() + labels = {h.strip() for h in re.findall(r'(?m)^#{2,}\s+(.+)$', doc)} + labels |= {b.strip().rstrip(':,') for b in re.findall(r'(?m)^\s*(?:[-*]|\d+\.)\s+\*\*(.+?)\*\*', doc)} + if label not in labels and not ('.' in label and label in doc): + bad.append(f'{page}: {label}') + return bad + + def test_every_citation_of_a_contract_page_names_a_section_it_has(self): + self.assertEqual(self.unresolved('(SKILL.md, Commissioning) (SKILL.md, Commissioning: Port power) (SKILL.md Roster)'), + ['SKILL.md: Commissioning', 'SKILL.md: Commissioning', 'SKILL.md: Roster']) + for f in sorted(self.CONTRACT.rglob('*')): + if f.suffix in ('.py', '.md'): + with self.subTest(f.relative_to(self.CONTRACT)): + self.assertEqual(self.unresolved(f.read_text()), []) + + class Exclusive(unittest.TestCase): def test_the_udev_property_gets_a_bounded_wait(self): seen = iter([False, False, True]) @@ -1577,7 +2228,7 @@ def setUp(self): def test_a_signal_right_after_the_child_starts_still_leaves_it_tracked_for_cleanup(self): real, launched = subprocess.Popen, [] - self.addCleanup(lambda: [(ht.reap(p), p.communicate()) for p in launched]) + self.addCleanup(lambda: [(ht.reap(p), p.wait()) for p in launched]) # reap closes our end of its pipe def popen_then_signalled(*a, **k): launched.append(real(*a, **k)) @@ -1587,10 +2238,18 @@ def popen_then_signalled(*a, **k): self.assertRaisesRegex(ht.Interrupted, 'SIGTERM'): ht.spawn([sys.executable, '-c', 'import time; time.sleep(30)'], self.tmp / 'log', 5) self.assertEqual(ht.CHILDREN, launched) - self.assertEqual(ht.stuck_children(), []) # reaped by SIGKILL, not left running + self.assertEqual(ht.settle_children(), []) # reaped by SIGKILL, not left running self.assertIsNotNone(launched[0].poll()) self.assertEqual(ht.CHILDREN, []) + def test_a_spawned_childs_stderr_can_go_to_the_log_only(self): + log = self.tmp / 'log' + code, out = ht.spawn([sys.executable, '-c', 'import sys; print("out"); print("err", file=sys.stderr)'], log, 5, + stderr_to_log=True) + self.assertEqual((code, out), (0, 'out\n')) + self.assertIn('err\n', log.read_text()) + self.assertIn('out\n', log.read_text()) + def test_a_kill_reaches_a_grandchild_in_a_process_group_of_its_own(self): # as ninja runs JLinkExe: setpgid moves it out of the child's group, not out of its session pidfile = self.tmp / 'grandchild' @@ -1620,11 +2279,11 @@ def end(): kill(grandchild, signal.SIGKILL) self.addCleanup(end) with mock.patch.object(ht.os, 'kill', spared), mock.patch.object(ht, 'KILL_WAIT', 0.5): - stuck = ht.stuck_children() # as cleanup asks after the failed step + stuck = ht.settle_children() # as cleanup asks after the failed step self.assertEqual([x['pid'] for x in stuck], [grandchild]) self.assertEqual(ht.still_alive(stuck), stuck) kill(grandchild, signal.SIGKILL) - self.assertEqual((ht.stuck_children(), ht.CHILDREN), ([], [])) + self.assertEqual((ht.settle_children(), ht.CHILDREN), ([], [])) def test_a_zombie_is_not_a_live_survivor(self): p = subprocess.Popen([sys.executable, '-c', 'pass']) @@ -1635,38 +2294,62 @@ def test_a_zombie_is_not_a_live_survivor(self): self.assertTrue(Path(f'/proc/{p.pid}').exists()) self.assertEqual(ht.still_alive([{'pid': p.pid, 'start': start}]), []) - def test_a_flash_container_is_removed_by_name_after_its_launcher_and_a_survivor_blocks_recovery(self): - order, present = [], {'tinyuf2-hil-b': True} + def test_a_legacy_reports_container_is_gone_only_once_docker_settles_on_none_and_its_launchers_are_gone(self): + """#580: no runner makes a container any more, but a report from one that did may name one. A create still in + flight when that runner died may land after the first empty listing; the name is gone only once docker has listed + it no more for CONTAINER_SETTLE, and never while a process of the same report lives.""" + name, listings, calls = 'tinyuf2-hil-b', [], [] + def run(cmd, **k): - order.append(cmd) - return subprocess.CompletedProcess(cmd, 0, '', '') - with mock.patch.object(ht, 'CONTAINERS', ['tinyuf2-hil-b']), mock.patch.object(ht.subprocess, 'run', run), \ - mock.patch.object(ht, 'container_present', lambda name: present[name]), mock.patch.object(ht, 'KILL_WAIT', 0.1): - stuck = ht.stuck_children() - self.assertEqual((stuck, order[0], ht.CONTAINERS), ([{'container': 'tinyuf2-hil-b'}], ['docker', 'rm', '-f', 'tinyuf2-hil-b'], - ['tinyuf2-hil-b'])) - self.assertEqual(ht.still_alive(stuck), stuck) - present['tinyuf2-hil-b'] = False - self.assertEqual((ht.still_alive(stuck), ht.stuck_children(), ht.CONTAINERS), ([], [], [])) - - def test_a_docker_error_in_cleanup_is_a_survivor_not_an_escape(self): - for what, run, present in (('docker ps fails', None, mock.Mock(side_effect=ht.StepFail('docker ps failed: x'))), - ('docker rm times out', mock.Mock(side_effect=subprocess.TimeoutExpired('docker', 30)), None), - ('no docker', mock.Mock(side_effect=FileNotFoundError('docker')), None)): - with self.subTest(what), mock.patch.object(ht, 'CONTAINERS', ['tinyuf2-hil-b']), \ - mock.patch.object(ht.subprocess, 'run', run or (lambda cmd, **k: subprocess.CompletedProcess(cmd, 0, '', ''))), \ - mock.patch.object(ht, 'container_present', present or (lambda name: True)), mock.patch.object(ht, 'KILL_WAIT', 0.1): - self.assertEqual(ht.stuck_children(), [{'container': 'tinyuf2-hil-b'}]) - self.assertEqual(ht.still_alive([{'container': 'tinyuf2-hil-b'}]), [{'container': 'tinyuf2-hil-b'}]) - - def test_cleanup_reaps_the_launchers_before_it_checks_a_container(self): - order = [] - p = mock.Mock(pid=1, args=['docker', 'create']) - with mock.patch.object(ht, 'CHILDREN', [p]), mock.patch.object(ht, 'CONTAINERS', ['tinyuf2-hil-b']), \ - mock.patch.object(ht, 'reap', lambda q: order.append('reap') or ht.CHILDREN.remove(q) or []), \ - mock.patch.object(ht, 'container_gone', lambda name: order.append('container') or True): - self.assertEqual(ht.stuck_children(), []) - self.assertEqual(order, ['reap', 'container']) + calls.append(cmd[:2]) + out = listings.pop(0) if cmd[1] == 'ps' and listings else '' + return subprocess.CompletedProcess(cmd, 0, out, '') + launcher = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(30)']) + self.addCleanup(lambda: (launcher.kill(), launcher.wait())) + live = {'pid': launcher.pid, 'start': ht.proc_start(launcher.pid), 'cmd': 'docker start -a ' + name} + with mock.patch.object(ht.cb.subprocess, 'run', run), mock.patch.object(ht.cb, 'CONTAINER_SETTLE', 0.3), \ + mock.patch.object(ht.cb, 'CONTAINER_GONE_WAIT', 5): + listings[:] = ['', 'c0ffee\n'] # absent once, then the delayed create lands + self.assertEqual(ht.still_alive([{'container': name}]), []) + self.assertGreaterEqual(calls.count(['docker', 'rm']), 3) # removed again once it landed, then settled + self.assertEqual(listings, []) + calls.clear() + self.assertEqual(ht.still_alive([live, {'container': name}]), [live, {'container': name}]) + self.assertEqual(calls, []) # not even looked at while its launcher lives + with mock.patch.object(ht.cb, 'container_left', lambda n: f'; container {n} still exists: docker rm -f {n}'): + self.assertEqual(ht.still_alive([{'container': name}]), [{'container': name}]) + + def test_cleanup_tracks_no_container(self): + p = mock.Mock(pid=1, args=['esptool']) + with mock.patch.object(ht, 'CHILDREN', [p]), mock.patch.object(ht, 'reap', lambda q: ht.CHILDREN.remove(q) or []), \ + mock.patch.object(ht.subprocess, 'run', mock.Mock(side_effect=AssertionError('no docker in cleanup'))): + self.assertEqual(ht.settle_children(), []) + self.assertFalse(hasattr(ht, 'CONTAINERS')) + + def test_a_timed_out_childs_output_so_far_is_in_the_log(self): + log = self.tmp / 'log' + with self.assertRaisesRegex(ht.StepFail, f'^{re.escape(Path(sys.executable).name)} timed out after 2s$'): + ht.spawn([sys.executable, '-c', 'import time; print("partial", flush=True); time.sleep(30)'], log, 2) + self.assertIn('partial\n', log.read_text()) + + def test_an_interrupted_childs_output_so_far_is_in_the_log(self): + log, ready = self.tmp / 'log', self.tmp / 'ready' + self.addCleanup(lambda: [ht.reap(p) for p in list(ht.CHILDREN)]) + + def signal_once_printed(): + end = time.monotonic() + 10 + while not ready.exists() and time.monotonic() < end: + time.sleep(0.01) + time.sleep(0.5) # for communicate to read it + os.kill(os.getpid(), signal.SIGTERM) + signaller = threading.Thread(target=signal_once_printed) + signaller.start() + self.addCleanup(signaller.join) + with self.assertRaisesRegex(ht.Interrupted, 'SIGTERM'): + ht.spawn([sys.executable, '-c', f'import time; print("partial", flush=True); open({str(ready)!r}, "w"); ' + 'time.sleep(30)'], log, 20) + self.assertEqual(len(ht.CHILDREN), 1) # left for cleanup's reap, as before + self.assertIn('partial\n', log.read_text()) def test_registering_leaves_the_childs_signal_mask_alone(self): code, out = ht.spawn(['grep', 'SigBlk', '/proc/self/status'], self.tmp / 'log', 5) @@ -1674,22 +2357,266 @@ def test_registering_leaves_the_childs_signal_mask_alone(self): self.assertFalse(int(out.split()[1], 16) & (1 << (signal.SIGTERM - 1)), out) +def sigterm_when(ready, what): + """SIGTERM this process from a thread once ready() holds: the interruption lands where the test means it to, with + no fixed delay to outwait.""" + def go(): + with contextlib.suppress(ht.StepFail): + ht.wait_for(ready, 10, what) + os.kill(os.getpid(), signal.SIGTERM) + threading.Thread(target=go, daemon=True).start() + + +class FlashProvider(Tmp): + """The runner's use of agentrc's flash provider: its loading, the run= adapter over spawn, the one J-Link flash + script, and sessions and resets through its primitives.""" + + def test_the_pinned_provider_loads_once_and_a_missing_or_incomplete_one_is_refused(self): + self.assertIs(ht.provider(), ht.provider()) + self.assertEqual(Path(ht.provider().__file__), ht.FLASH_PROVIDER) + for what, path, api in (('missing', self.tmp / 'none.py', ht.PROVIDER_API), + ('incomplete', ht.FLASH_PROVIDER, ht.PROVIDER_API + ('flash_everything',))): + with self.subTest(what), mock.patch.object(ht, 'PROVIDER', []), mock.patch.object(ht, 'FLASH_PROVIDER', path), \ + mock.patch.object(ht, 'PROVIDER_API', api), self.assertRaises(ht.Refused): + ht.provider() + with mock.patch.object(ht, 'PROVIDER', []), \ + mock.patch.object(ht, 'check_lock_pin', side_effect=ht.Refused('lib/agentrc is at x; HEAD pins y')), \ + self.assertRaisesRegex(ht.Refused, 'HEAD pins'): + ht.provider() + + def test_a_shell_string_runs_as_argv_with_its_assignments_as_environment(self): + self.assertEqual(ht.command_argv("NO_COLOR=1 esptool --chip esp32s3 -p /dev/ttyACM0 write_flash 0x0 'a b.bin'"), + (['esptool', '--chip', 'esp32s3', '-p', '/dev/ttyACM0', 'write_flash', '0x0', 'a b.bin'], + {'NO_COLOR': '1'})) + self.assertEqual(ht.command_argv(['JLinkExe', '-USB', 1]), (['JLinkExe', '-USB', '1'], None)) + for cmd in ('a | b', 'a; b', 'a && b', 'echo $HOME', 'a > f', 'echo `x`'): + with self.subTest(cmd), self.assertRaises(ValueError): + ht.command_argv(cmd) + + def test_the_adapter_returns_exit_output_and_a_timeouts_transcript_so_far(self): + run, log = ht.tool_runner(self.tmp / 'hil.log'), self.tmp / 'hil.log' + r = run('FOO=bar printenv FOO', timeout=10) + self.assertEqual((r.returncode, r.stdout), (0, 'bar\n')) + self.assertEqual(run([sys.executable, '-c', 'import sys; sys.exit(3)'], timeout=10).returncode, 3) + r = run([sys.executable, '-c', 'import os, signal; os.kill(os.getpid(), signal.SIGSEGV)'], timeout=10) + self.assertEqual(r.returncode, -signal.SIGSEGV) # executed directly: a crash keeps its signal + r = run([sys.executable, '-c', 'import time; print("so far", flush=True); time.sleep(30)'], timeout=1) + self.assertEqual((r.returncode, r.stdout, ht.CHILDREN), (124, 'so far\n', [])) + op = ht.provider().Operation([ht.provider().Attempt(r.args, None, r, None, 'unknown', None)]) + self.assertEqual(op.attempts[0].kind, 'timed_out') + self.assertIn('so far', log.read_text()) + + def test_the_adapter_reports_cleanup_done_only_once_nothing_of_the_tools_session_is_left(self): + run = ht.tool_runner(self.tmp / 'hil.log') + self.assertEqual(run([sys.executable, '-c', 'pass'], timeout=10).cleanup, 'done') + self.assertEqual(run([sys.executable, '-c', 'import time; time.sleep(30)'], timeout=1).cleanup, 'done') # reaped + # a process the tool started, its stdout closed, outlives the tool: tracked for cleanup, never called done + bg = 'import subprocess, sys; subprocess.Popen(["sleep", "30"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)' + self.addCleanup(ht.settle_children) + r = run([sys.executable, '-c', bg], timeout=10) + self.assertEqual((r.returncode, r.cleanup, len(ht.CHILDREN)), (0, 'unknown', 1)) + self.assertEqual(ht.settle_children(), []) + self.assertEqual(ht.CHILDREN, []) + op = ht.provider().flash_esptool({'args': ''}, chip='esp32s3', port='/dev/null', flash_args='0x0 a.bin', cwd=str(self.tmp), + timeout=5, run=lambda cmd, cwd, timeout: run([sys.executable, '-c', 'pass'], timeout=5)) + self.assertEqual(op.cleanup, 'done') + + def test_an_interrupted_tool_is_recorded_interrupted_with_its_output_and_left_for_cleanup(self): + handler = signal.signal(signal.SIGTERM, ht.on_signal) + mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, ht.TERM_SIGNALS) + self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) + self.addCleanup(signal.signal, signal.SIGTERM, handler) + self.addCleanup(ht.settle_children) + # ready once the tracked child's output reached communicate's own buffer, not once spawn logged its command line + sigterm_when(lambda: any(b'hi\n' in b''.join(getattr(p, '_fileobj2output', {}).get(p.stdout, [])) + for p in list(ht.CHILDREN)), "the tool's output captured") + with self.assertRaises(ht.Interrupted) as e: + ht.provider().run_jlink_commands({'uid': '1'}, ['h'], interface='SWD', speed='4000', autoconnect=True, device='d', + timeout=10, workdir=self.tmp, run=lambda cmd, timeout: ht.tool_runner( + self.tmp / 'log')([sys.executable, '-c', 'import time; print("hi", flush=True); ' + 'time.sleep(30)'], timeout=timeout)) + a = e.exception.operation.attempts[-1] + self.assertEqual((a.kind, a.result.stdout, len(ht.CHILDREN)), ('interrupted', 'hi\n', 1)) + + def test_the_providers_jlink_flash_script_is_the_record_targets(self): + # option B: one script for both projects; the CMake target's (check_build's flash_binding fixture) must not drift + hexp, seen = '/b/tinyuf2.hex', [] + op = ht.provider().flash_jlink({'uid': '000801012283', 'args': '-device MIMXRT1011DAE5A'}, hexp, timeout=5, + run=lambda cmd, timeout: seen.append(cmd) or subprocess.CompletedProcess(cmd, 0, '', '')) + self.assertEqual(op.attempts[0].script, JLINK_TARGET_SCRIPT.format(hex=hexp) + '\n') + argv = ht.command_argv(seen[0])[0] + pairs = lambda a: sorted(zip(a[1:-2:2], a[2:-2:2])) # noqa: E731 flag, value + self.assertEqual((argv[0], argv[-2], pairs(argv)), ('JLinkExe', '-CommandFile', pairs(JLINK_TARGET_ARGV))) + + def test_sessions_and_resets_go_through_the_providers_primitives_with_tinyuf2s_profile(self): + seen = [] + + def spawn(cmd, log, timeout, cwd=None, **k): + seen.append((list(cmd), Path(cmd[-1]).read_text(), timeout)) + return 0, f'J-Link>w4 E000ED0C 05FA0004\nWriting 05FA0004 -> E000ED0C\n' + probe = ht.Probe({'probe': '7', 'device': 'stm32f411ve', 'if': 'SWD', 'speed': '4000'}, self.tmp, self.tmp / 'log') + with mock.patch.object(ht, 'spawn', spawn): + probe.sysreset() + self.assertEqual(seen[0][0][:-1], ['JLinkExe', '-USB', '7', '-device', 'Cortex-M4', '-if', 'SWD', '-speed', '4000', + '-autoconnect', '1', '-NoGui', '1', '-ExitOnError', '1', '-CommandFile']) + self.assertEqual(seen[0][1:], ('w4 E000ED0C 05FA0004\nexit\n', ht.PROBE_TIMEOUT)) + self.assertEqual(Path(seen[0][0][-1]).parent, self.tmp) + cube, scripts = FakeCube(), [] + cube.on_reset = lambda: None + stlink = ht.StlinkProbe({'probe': '0023', 'device': 'stm32h563zi'}, self.tmp, self.tmp / 'log') + with mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: scripts.append(Path(cmd[2]).read_text()) or cube.run(cmd[2])): + stlink.sysreset() + self.assertEqual(scripts, ['-c port=SWD sn=0023 mode=HOTPLUG\n-w32 0xE000ED0C 0x05FA0004 -nv\n']) + + def test_before_target_holds_only_when_every_attempt_ended_before_the_target(self): + probe = ht.Probe({'probe': '000726129165', 'device': 'd', 'if': 'SWD', 'speed': '4000'}, self.tmp, self.tmp / 'hil.log') + crash = (-11, 'Connecting to J-Link ...O.K.\nS/N: 726129165\n') + for what, second, before in (('no probe on the rerun', (1, JLINK_NO_PROBE), True), + ('the rerun reached the target', (1, 'Connecting to target via SWD\nCannot connect to target.\n'), + False)): + calls = iter([crash, second]) + with self.subTest(what), mock.patch.object(ht, 'JLINK_RETRY_DELAY', 0), \ + mock.patch.object(ht, 'spawn', lambda *a, **k: next(calls)), self.assertRaises(ht.StepFail) as e: + probe.run(['h']) + self.assertEqual(e.exception.evidence['beforeTarget'], before) + self.assertEqual(probe.retried[-1], {'exit': -11, 'commands': ['h']}) + + +# the CMake target's J-Link command and script, as check_build's flash_binding requires them (test_build.jlink_tree) +JLINK_TARGET_ARGV = ['JLinkExe', '-device', 'MIMXRT1011DAE5A', '-USB', '000801012283', '-ExitOnError', '1', '-NoGui', '1', + '-if', 'swd', '-JTAGConf', '-1,-1', '-speed', 'auto', '-CommandFile', '/b/tinyuf2.jlink'] +JLINK_TARGET_SCRIPT = 'halt\nloadfile {hex}\nr\ngo\nexit' + +CONTENDER = """ +import fcntl, json, os, sys, time +with open(sys.argv[1], 'a') as f: + print('waiting', flush=True) + fcntl.flock(f, fcntl.LOCK_EX) + pid = int(open(sys.argv[2]).read()) if os.path.exists(sys.argv[2]) else None + alive = pid is not None and os.path.exists(f'/proc/{pid}') and open(f'/proc/{pid}/stat').read().rsplit(')', 1)[1].split()[0] != 'Z' + print(json.dumps({'at': time.monotonic(), 'childAlive': alive}), flush=True) +""" + + +class BuildLockLifetime(Tmp): + """#579: a flash keeps its build dir's lock until every child it started has ended, against a real contender.""" + + def setUp(self): + super().setUp() + board = self.load(roster(self.tmp)) + self.bat = ht.Battery(board, ht.source_facts(board), self.tmp) + self.root = self.tmp / 'build' + self.root.mkdir() + self.flash = {'root': self.root, 'sha256': {}} + self.pidfile = self.tmp / 'child.pid' + self.addCleanup(ht.settle_children) + + def child(self, body): + return [sys.executable, '-c', f'import os, sys, time; open({str(self.pidfile)!r}, "w").write(str(os.getpid())); {body}'] + + def contender(self): + c = subprocess.Popen([sys.executable, '-c', CONTENDER, str(ht.cb.build_lock_file(self.root)), str(self.pidfile)], + stdout=subprocess.PIPE, text=True) + self.addCleanup(lambda: (c.kill(), c.wait(), c.stdout.close())) + self.assertEqual(c.stdout.readline().strip(), 'waiting') + return c + + def flash_with(self, body, timeout=10): + def op(run): + self.contended = self.contender() # started while the tool runs, the lock already held + return one_attempt(run(self.child(body), timeout=timeout)) + return self.bat.flash_target(self.flash, op, lambda code, out: {'exit': code}, lambda code, out: False, 'tool', timeout) + + def acquired(self): + got = json.loads(self.contended.stdout.readline()) + return got, time.monotonic() + + def test_success_releases_the_lock_once_the_tool_has_exited(self): + self.assertEqual(self.flash_with('time.sleep(0.5)')['exit'], 0) + got, _ = self.acquired() + self.assertFalse(got['childAlive']) + + def test_a_timeout_kills_the_tool_before_the_lock_goes(self): + with self.assertRaisesRegex(ht.StepFail, 'timed out after 1s'): + self.flash_with('time.sleep(30)', timeout=1) + self.assertFalse(self.acquired()[0]['childAlive']) + + def test_an_interruption_reaps_the_tool_inside_the_lock(self): + handler = signal.signal(signal.SIGTERM, ht.on_signal) + mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, ht.TERM_SIGNALS) + self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) + self.addCleanup(signal.signal, signal.SIGTERM, handler) + sigterm_when(lambda: self.pidfile.exists() and self.pidfile.read_text(), 'the tool running under the lock') + with self.assertRaises(ht.Interrupted): + self.flash_with('time.sleep(30)') + self.assertEqual(ht.CHILDREN, []) + self.assertFalse(self.acquired()[0]['childAlive']) + + def outlived(self, late): + """reap as if `late` were a member of the child's session that outlived SIGKILL: the child stays tracked.""" + real = ht.reap + + def reap(p): + left = real(p) + [late.pid] + if p not in ht.CHILDREN: + ht.CHILDREN.append(p) + return left + return mock.patch.object(ht, 'reap', reap) + + def test_a_survivor_keeps_the_lock_through_a_keeper_until_it_exits(self): + late = subprocess.Popen(['sleep', '30'], start_new_session=True) # stands for a child that outlived SIGKILL + self.addCleanup(lambda: (late.kill(), late.wait())) + with self.outlived(late), \ + self.assertRaisesRegex(ht.StepFail, 'timed out'): + self.flash_with('time.sleep(30)', timeout=1) + kept, = self.bat.kept + self.assertEqual(([s['pid'] for s in kept['survivors']], kept['buildDir']), ([late.pid], 'build')) + self.assertNotIn('inProcess', kept['keeper']) + keeper = kept['keeper']['pid'] + self.addCleanup(lambda: end_child(keeper)) + with open(ht.cb.build_lock_file(self.root), 'a') as f, self.assertRaises(OSError): + ht.fcntl.flock(f, ht.fcntl.LOCK_EX | ht.fcntl.LOCK_NB) # the keeper holds it after this process let go + ready, _, _ = select.select([self.contended.stdout], [], [], 0.3) + self.assertFalse(ready, 'the contender got the lock while the survivor lived') + late.kill() + late.wait() + self.acquired() # once the survivor is gone + ht.wait_for(lambda: ht.proc_stat(keeper) is None, 5, 'the keeper exiting') + + def test_a_flash_that_succeeded_but_left_a_survivor_fails_after_handing_the_lock_to_its_keeper(self): + late = subprocess.Popen(['sleep', '3'], start_new_session=True) + self.addCleanup(lambda: (late.kill(), late.wait())) + left = [ht.survivor(late.pid)] + with mock.patch.object(ht, 'settle_children', lambda started: list(left)), \ + self.assertRaisesRegex(ht.StepFail, 'children survived SIGKILL') as cm: + self.flash_with('pass') + self.assertEqual((cm.exception.evidence['survivors'], self.bat.kept[0]['survivors']), (left, left)) + self.addCleanup(lambda: end_child(self.bat.kept[0]['keeper']['pid'])) + with open(ht.cb.build_lock_file(self.root), 'a') as f, self.assertRaises(OSError): + ht.fcntl.flock(f, ht.fcntl.LOCK_EX | ht.fcntl.LOCK_NB) + + def test_a_keeper_that_never_starts_leaves_the_lock_with_this_process(self): + late = subprocess.Popen(['sleep', '30'], start_new_session=True) + self.addCleanup(lambda: (late.kill(), late.wait())) + self.addCleanup(lambda: ([fh.close() for fh, _ in ht.HELD_BUILD], ht.HELD_BUILD.clear())) + with self.outlived(late), \ + mock.patch.object(ht, 'HOLDER_SCRIPT', self.tmp / 'missing.py'), \ + self.assertRaisesRegex(ht.StepFail, 'timed out'): + self.flash_with('time.sleep(30)', timeout=1) + self.assertEqual((self.bat.kept[0]['keeper'], len(ht.HELD_BUILD)), ({'pid': os.getpid(), 'inProcess': True}, 1)) + with open(ht.cb.build_lock_file(self.root), 'a') as f, self.assertRaises(OSError): + ht.fcntl.flock(f, ht.fcntl.LOCK_EX | ht.fcntl.LOCK_NB) + + class Locks(Tmp): def setUp(self): super().setUp() mask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) # hand_off blocks SIGUSR1 in this process self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) - real = os.environ.get('TINYUSB_HIL_LOCK') lockdir = self.tmp / 'locks' lockdir.mkdir() - if real: - self.helper = ht.load_helper(real) - patch = mock.patch.object(self.helper, 'BOARD_LOCK_DIR', str(lockdir)) - patch.start() - self.addCleanup(patch.stop) - else: - (self.tmp / 'hil_lock.py').write_text(FAKE_HELPER.format(dir=str(lockdir))) - self.helper = ht.load_helper(self.tmp / 'hil_lock.py') + with mock.patch.object(ht, 'check_lock_pin'): + self.helper = pinned_helper(self, lockdir) self.lock = self.helper.lock_path(BOARD) def holder(self, reason, lock=True): @@ -1718,10 +2645,15 @@ def test_another_reason_or_a_protected_one_is_refused(self): def test_cleanup_handoff_keeps_the_board_held_across_exec_and_recovery_can_end_it(self): fh = self.helper.acquire_board_lock(BOARD) - holder = ht.hand_off(fh, BOARD, 'cleanup failed', [{'pid': 99, 'start': 1, 'cmd': ['mcopy']}]) - self.addCleanup(lambda: _kill(holder['pid'])) + with mock.patch.dict(os.environ, {ht.JOB_TRACKING: 'job-7'}): + holder = ht.hand_off(fh, BOARD, 'cleanup failed', [{'pid': 99, 'start': 1, 'cmd': ['mcopy']}]) + self.addCleanup(lambda: end_child(holder['pid'])) self.assertTrue(holder['ready']) self.assertNotIn('inProcess', holder) + # the holder is the re-executed `keep-held`, outside the Actions job's process tracking + proc = Path('/proc') / str(holder['pid']) + ht.wait_for(lambda: b'keep-held' in (proc / 'cmdline').read_bytes(), 5, 'holder exec') + self.assertNotIn(ht.JOB_TRACKING.encode(), (proc / 'environ').read_bytes()) with self.assertRaises(OSError): self.helper.flock_nb(BOARD) rec = ht.prove_hold(self.helper, BOARD, holder['pid'], reason='hil_test.py', role=ht.HOLDER_ROLE) @@ -1747,7 +2679,9 @@ def dies_in_the_child(*a): return real(*a) for what, patch in (('fork fails', mock.patch.object(ht.os, 'fork', side_effect=OSError(11, 'EAGAIN'))), ('pipe fails', mock.patch.object(ht.os, 'pipe', side_effect=OSError(24, 'EMFILE'))), - ('no ack', mock.patch.object(ht, 'holder_record', dies_in_the_child))): + ('no ack', mock.patch.object(ht, 'holder_record', dies_in_the_child)), + # the exec succeeds but the replacement dies at startup: never acknowledged + ('exec dies', mock.patch.object(ht, 'HOLDER_SCRIPT', self.tmp / 'missing.py'))): with self.subTest(what): fh = self.helper.acquire_board_lock(BOARD) with patch: @@ -1757,14 +2691,17 @@ def dies_in_the_child(*a): self.helper.flock_nb(BOARD) rec = ht.prove_hold(self.helper, BOARD, parent, reason='hil_test.py', role=ht.HOLDER_ROLE) self.assertEqual(rec['detail'], 'cleanup failed') - ht.HELD_IN_PROCESS.pop().close() + held = ht.HELD_IN_PROCESS.pop() + self.helper.clear_record(held) # a cleanup holder's record left on a free flock is a recovery blocker + held.close() def test_with_no_descriptor_left_the_runner_itself_holds_and_releases_only_on_sigusr1(self): script = textwrap.dedent(f''' import importlib.util, json, os, resource, signal, sys spec = importlib.util.spec_from_file_location('ht', {str(SCRIPT)!r}); ht = importlib.util.module_from_spec(spec) spec.loader.exec_module(ht) - helper = ht.load_helper({str(self.helper.__file__)!r}) + ht.check_lock_pin = lambda: None + helper = ht.load_helper() helper.BOARD_LOCK_DIR = {str(Path(self.lock).parent)!r} fh = helper.acquire_board_lock({BOARD!r}) signal.pthread_sigmask(signal.SIG_BLOCK, ht.TERM_SIGNALS) @@ -1793,15 +2730,52 @@ def test_with_no_descriptor_left_the_runner_itself_holds_and_releases_only_on_si self.assertEqual(p.wait(5), 0) self.helper.flock_nb(BOARD).close() + def test_with_both_handoffs_failed_the_holding_runner_releases_the_build_lock_once_its_survivor_ends(self): + late = subprocess.Popen(['sleep', '30'], start_new_session=True) + self.addCleanup(lambda: (late.kill(), late.wait())) + root = self.tmp / 'build' + script = textwrap.dedent(f''' + import importlib.util, json, signal + from pathlib import Path + spec = importlib.util.spec_from_file_location('ht', {str(SCRIPT)!r}); ht = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ht) + ht.check_lock_pin = lambda: None + ht.HOLDER_SCRIPT = Path({str(self.tmp / 'missing.py')!r}) # neither the keeper nor the holder starts + helper = ht.load_helper() + helper.BOARD_LOCK_DIR = {str(Path(self.lock).parent)!r} + fh = helper.acquire_board_lock({BOARD!r}) + signal.pthread_sigmask(signal.SIG_BLOCK, ht.TERM_SIGNALS) + survivors = {json.dumps([ht.survivor(late.pid)])} + keeper = ht.keep_build_lock(ht.take_build_lock(Path({str(root)!r})), survivors) + holder = ht.hand_off(fh, {BOARD!r}, 'cleanup failed', survivors) + print(json.dumps([keeper, holder]), flush=True) + ht.keep_held(ht.HELD_IN_PROCESS[0]) + ''') + p = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE, text=True) + self.addCleanup(lambda: (p.kill(), p.wait(), p.stdout.close())) + keeper, holder = json.loads(p.stdout.readline()) + self.assertEqual([(r['pid'], r['inProcess']) for r in (keeper, holder)], [(p.pid, True)] * 2) + build_lock = ht.cb.build_lock_file(root) + time.sleep(0.3) + self.assertFalse(ht.flock_free(build_lock), 'the build lock went while its survivor lived') + late.kill() + late.wait() + ht.wait_for(lambda: ht.flock_free(build_lock), 5, 'the build lock released') + self.assertIsNone(p.poll()) + ht.prove_hold(self.helper, BOARD, p.pid, reason='hil_test.py', role=ht.HOLDER_ROLE) # the board stays held + p.send_signal(signal.SIGUSR1) + self.assertEqual(p.wait(5), 0) + self.helper.flock_nb(BOARD).close() + def test_a_holder_handed_off_during_cleanup_still_ends_on_sigusr1(self): fh = self.helper.acquire_board_lock(BOARD) with ht.signals(signal.SIG_BLOCK): # cleanup blocks SIGTERM and SIGINT; the holder must not care holder = ht.hand_off(fh, BOARD, 'cleanup failed') - self.addCleanup(lambda: _kill(holder['pid'])) + self.addCleanup(lambda: end_child(holder['pid'])) os.kill(holder['pid'], signal.SIGUSR1) ht.wait_for(lambda: ht.flock_free(self.lock), 5, 'release') - def test_one_recovery_per_board_and_tinyusb_never_lists_it_as_a_holder(self): + def test_one_recovery_per_board_and_the_helper_never_lists_it_as_a_holder(self): first = ht.recovery_lock(self.helper, BOARD) with self.assertRaisesRegex(ht.Refused, 'another recovery holds'): ht.recovery_lock(self.helper, BOARD) @@ -1817,19 +2791,11 @@ def test_survivors_are_identified_against_pid_reuse(self): def test_the_holder_keeps_none_of_the_callers_stdio(self): fh = self.helper.acquire_board_lock(BOARD) holder = ht.hand_off(fh, BOARD, 'x') - self.addCleanup(lambda: _kill(holder['pid'])) + self.addCleanup(lambda: end_child(holder['pid'])) ht.wait_for(lambda: not any(Path(f'/proc/{holder["pid"]}/fd/{std}').exists() for std in (0, 1, 2)), 5, 'the holder closing its stdio') -def _kill(pid): - try: - os.kill(pid, signal.SIGKILL) - os.waitpid(pid, 0) - except (ProcessLookupError, ChildProcessError): - pass - - class RunBoard(Tmp): """The state machine around the battery: who cleans up, and what is left held.""" @@ -1845,17 +2811,18 @@ def unblock(): # run_board leaves the signals blocked until exit; drop what is for sig, h in handlers.items(): signal.signal(sig, h) self.addCleanup(unblock) - (self.tmp / 'hil_lock.py').write_text(FAKE_HELPER.format(dir=str(self.tmp / 'locks'))) - self.helper = ht.load_helper(self.tmp / 'hil_lock.py') + with mock.patch.object(ht, 'check_lock_pin'): + self.helper = pinned_helper(self, self.tmp / 'locks') self.board = self.load(roster(self.tmp)) self.facts = ht.source_facts(self.board) - self.args = mock.Mock(build_record='rec.json', fail_at=None, try_reentry=None) + self.args = mock.Mock(build_record='rec.json', fail_at=None, try_reentry=None, lock_wait=0) for p in (mock.patch.object(ht, 'verify_record', lambda *a: ({'hex': 'x.hex', 'bin': 'x.bin'}, {'version': 'v'})), mock.patch.object(ht, 'parse_hex', lambda p: {0: 1}), mock.patch.object(ht, 'hex_matches_bin', lambda d, b, base: True), mock.patch.object(ht, 'check_jlink', lambda: None), mock.patch.object(ht, 'check_bootloader_bin', lambda *a: None), mock.patch.object(ht, 'exclusion_configured', lambda *a: None), + mock.patch.object(ht, 'probe_users', lambda b: []), # not the host's J-Link tools: tests patch their own mock.patch.object(ht.shutil, 'which', lambda t: '/usr/bin/' + t)): p.start() self.addCleanup(p.stop) @@ -1987,16 +2954,95 @@ def fail(bat, files, version): bat.touched = True raise ht.StepFail('x') stuck = [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}] - with mock.patch.object(ht, 'stuck_children', lambda: list(stuck)), \ + with mock.patch.object(ht, 'settle_children', lambda started=(): list(stuck)), \ mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'survivors': s})[1]): row, code, cleaned = self.run_board(fail) self.assertEqual((code, cleaned, row['cleanup']['survivors']), (4, [], stuck)) self.assertEqual(row['cleanup']['detail'], 'StepFail: children survived SIGKILL; no flash may overlap them') + def test_the_restore_reflashes_through_a_build_lock_this_process_kept_once_its_survivor_ended(self): + root = self.tmp / 'build' + self.addCleanup(lambda: ([fh.close() for fh, _ in ht.HELD_BUILD], ht.HELD_BUILD.clear())) + + def restore(*a, **k): # as flash_target, on the build dir the run's flash used + ht.take_build_lock(root).close() + return {'pass': True, 'state': 'blinky copied', 'evidence': {}} + for ended in (True, False): + with self.subTest(ended=ended): + late = subprocess.Popen(['sleep', '30'], start_new_session=True) + self.addCleanup(lambda p=late: (p.kill(), p.wait())) + + def flashed_with_a_survivor(bat, files, version): # its keeper never started: the lock stays in process + bat.touched = True + with mock.patch.object(ht, 'HOLDER_SCRIPT', self.tmp / 'missing.py'): + ht.keep_build_lock(ht.take_build_lock(root), [ht.survivor(late.pid)]) + if ended: + late.kill() + late.wait() + raise ht.StepFail('children survived SIGKILL') + with mock.patch.object(ht.Battery, 'run', flashed_with_a_survivor), \ + mock.patch.object(ht.Battery, 'restore', restore), mock.patch.object(ht, 'BUILD_LOCK_WAIT', 0.3), \ + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1})[1]): + row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) + if ended: + self.assertEqual((code, row['cleanup']['pass'], ht.HELD_BUILD), (1, True, [])) + self.helper.flock_nb(BOARD).close() + else: # its survivor alive: the lock stays kept, the restore cannot flash and the board stays held + self.assertEqual((code, row['cleanup']['heldAfter'], len(ht.HELD_BUILD)), (4, True, 1)) + self.assertIn('is in use by a build or another flash', row['cleanup']['detail']) + self.assertFalse(ht.flock_free(ht.cb.build_lock_file(root))) + + def test_a_session_left_running_is_cleanup_pending_never_reaped_and_keeps_the_board(self): + server = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], start_new_session=True) + self.addCleanup(lambda: (server.kill(), server.wait())) + receipt = self.tmp / 'hil_openocd_esp_x' / 'receipt.json' + + def fail(bat, files, version): # as flash_openocd_esp leaves it: its receipt open, its server alive + bat.touched = True + receipt.parent.mkdir() + receipt.write_text(json.dumps({'pid': server.pid, 'startTime': str(ht.proc_start(server.pid)), 'closed': False, + 'argv': ['openocd'], 'ports': [None, None, 1], 'token': 't', 'log': 'l'})) + raise ht.CleanupPending('the openocd_esp session is still running: cleanup pending') + handed = [] + with mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (handed.append(s), fh.close(), {'pid': 1})[2]): + row, code, cleaned = self.run_board(fail) + want = [{'pid': server.pid, 'start': ht.proc_start(server.pid), 'cmd': 'openocd', 'receipt': str(receipt)}] + self.assertEqual((code, cleaned, row['cleanup']['state'], row['cleanup']['receipts'], handed), + (4, [], 'pending', [str(receipt)], [want])) # no restore, the board held + self.assertEqual((server.poll(), ht.still_alive(want)), (None, want)) # recovery refuses while it runs + good = receipt.read_text() + receipt.write_text('{') # an unreadable receipt blocks recovery and keeps a build lock's keeper, reread each check + self.assertEqual(ht.still_alive(want), want) + receipt.write_text(json.dumps(dict(json.loads(good), closed=True, cleanup='done'))) # ended by `end` + self.assertEqual((ht.still_alive(want), ht.outstanding_sessions(self.tmp)), ([], [])) + + def test_a_pending_mark_that_cannot_be_written_never_costs_the_hand_off_or_the_report(self): + server = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'], start_new_session=True) + self.addCleanup(lambda: (server.kill(), server.wait())) + receipt = self.tmp / 'hil_openocd_esp_x' / 'receipt.json' + self.board['pendingMark'] = ht.PendingMark(None, 'b') + + def fail(bat, files, version): + bat.touched = True + receipt.parent.mkdir() + receipt.write_text(json.dumps({'pid': server.pid, 'startTime': str(ht.proc_start(server.pid)), 'closed': False})) + raise ht.CleanupPending('the openocd_esp session is still running: cleanup pending') + handed = [] + + def enospc(*a): + raise OSError(28, 'No space left on device') + with mock.patch.object(ht, 'write_mark', enospc), mock.patch.object(ht, 'MARK_ERRORS', []), \ + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (handed.append(s), fh.close(), {'pid': 1})[2]): + row, code, cleaned = self.run_board(fail) + self.assertEqual((code, row['cleanup']['state'], row['cleanup']['receipts'], len(handed)), + (4, 'pending', [str(receipt)], 1)) # held by its holder, the session reported + self.assertIn('could not be written', row['cleanup']['markErrors'][0]) + self.assertFalse(Path(self.board['pendingMark']).exists()) + def test_a_survivor_on_an_untouched_board_still_keeps_it_held(self): def fail(bat, files, version): raise ht.StepFail('install: probe hung') - with mock.patch.object(ht, 'stuck_children', lambda: [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}]), \ + with mock.patch.object(ht, 'settle_children', lambda started=(): [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}]), \ mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1})[1]): row, code, cleaned = self.run_board(fail) self.assertEqual((code, cleaned, row['cleanup']['state'], row['cleanup']['heldAfter']), @@ -2036,6 +3082,79 @@ def test_a_locked_board_is_not_touched(self): row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) self.assertEqual((code, row['locked'], row['ran'], cleaned), (1, True, False, [])) + def test_a_held_board_is_tried_again_within_lock_wait(self): + held = self.helper.acquire_board_lock(BOARD, reason='someone') + self.args.lock_wait = 5 + threading.Timer(0.1, lambda: (self.helper.clear_record(held), held.close())).start() + with mock.patch.object(ht, 'LOCK_POLL', 0.02): + row, code, cleaned = self.run_board(lambda bat, f, v: setattr(bat, 'touched', True)) + self.assertEqual((code, row['locked'], row['pass'], cleaned), (0, False, True, [True])) + + def test_a_signal_during_the_lock_wait_still_gives_a_locked_row(self): + held = self.helper.acquire_board_lock(BOARD, reason='someone') + self.addCleanup(held.close) + self.args.lock_wait = 100 + signal.signal(signal.SIGTERM, ht.on_signal) + real_sleep = time.sleep + + def signalled(s): + os.kill(os.getpid(), signal.SIGTERM) + real_sleep(1) + self.fail('the signal did not interrupt the wait') + with mock.patch.object(ht.time, 'sleep', signalled): + row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['ran'], row['locked'], row['recoveryBlocker'], cleaned), (1, False, True, False, [])) + self.assertIn('interrupted by SIGTERM while waiting for the lock', row['detail']) + self.assertTrue(self.helper.is_locked(BOARD)) + self.assertEqual(self.helper.read_record(BOARD)['reason'], 'someone') + + def test_a_failed_probe_check_is_not_run_and_released(self): + def no_bridge(board): + raise ht.StepFail('0 serial ports for bridge cp2102n: []') + with mock.patch.object(ht, 'probe_users', no_bridge): + row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['ran'], row['locked'], cleaned), (1, False, False, [])) + self.assertEqual(row['detail'], 'probe check: 0 serial ports for bridge cp2102n: []') + self.assertFalse(self.helper.is_locked(BOARD)) + self.helper.flock_nb(BOARD).close() + + def test_a_failed_probe_check_without_the_board_lock_is_not_run(self): + def no_bridge(board): + raise ht.StepFail('0 serial ports for bridge cp2102n: []') + with mock.patch.object(ht, 'probe_users', no_bridge), mock.patch.dict(os.environ, {'HIL_NO_BOARD_LOCK': '1'}): + row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['ran'], row['lockBypassed'], cleaned), (1, False, True, [])) + self.assertEqual(row['detail'], 'probe check: 0 serial ports for bridge cp2102n: []') + + def test_a_cleanup_holder_is_not_waited_for(self): + held = self.helper.acquire_board_lock(BOARD, reason='someone') + self.addCleanup(held.close) + rec = json.loads(Path(self.helper.lock_path(BOARD)).read_text()) + Path(self.helper.lock_path(BOARD)).write_text(json.dumps(dict(rec, role=ht.HOLDER_ROLE))) + self.args.lock_wait = 100 + t0 = time.monotonic() + row, code, _ = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['locked'], row['recoveryBlocker']), (1, True, True)) + self.assertIn(ht.HOLDER_ROLE, row['detail']) + self.assertLess(time.monotonic() - t0, 5) + + def test_the_lock_wait_ends_at_its_budget_even_when_the_board_frees_meanwhile(self): + held = self.helper.acquire_board_lock(BOARD, reason='someone') + self.args.lock_wait = 1 + threading.Timer(0.5, lambda: (self.helper.clear_record(held), held.close())).start() + t0 = time.monotonic() + with mock.patch.object(ht, 'LOCK_POLL', 5): # one sleep, capped to the budget, waking after the release + row, code, _ = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['locked'], row['recoveryBlocker']), (1, True, False)) + self.assertLess(time.monotonic() - t0, 3) + + def test_a_probe_still_in_use_is_refused_after_the_lock_and_released(self): + with mock.patch.object(ht, 'probe_users', lambda b: [101]): + row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['ran'], row['probeUsers'], cleaned), (2, False, [101], [])) + self.assertFalse(self.helper.is_locked(BOARD)) + self.helper.flock_nb(BOARD).close() + def test_the_board_is_locked_and_reported_under_its_lock_name(self): self.board = self.load(roster(self.tmp, tinyuf2={'lock': 'tinyusb_name'})) held = self.helper.acquire_board_lock('tinyusb_name', reason='someone') @@ -2044,9 +3163,10 @@ def test_the_board_is_locked_and_reported_under_its_lock_name(self): self.assertEqual((code, row['board'], row['lock'], row['locked']), (1, BOARD, 'tinyusb_name', True)) def test_an_unusable_lock_dir_is_refused_not_run_unlocked(self): - with mock.patch.object(self.helper, 'acquire_board_lock', lambda b: None), \ + (self.tmp / 'file').write_text('') + with mock.patch.object(self.helper, 'BOARD_LOCK_DIR', str(self.tmp / 'file' / 'locks')), \ mock.patch.dict(os.environ, {'HIL_NO_BOARD_LOCK': ''}): - with self.assertRaises(ht.Refused): + with self.assertRaisesRegex(ht.Refused, 'board lock unavailable.*refusing to run unlocked'): self.run_board(lambda *a: self.fail('ran')) def test_fail_at_injects_a_criterion_failure(self): @@ -2056,19 +3176,26 @@ def test_fail_at_injects_a_criterion_failure(self): bat.step('dfu', lambda: {}) +def one_attempt(result): + """A provider Operation of the one run `result`, for a test's flash op.""" + return ht.provider().Operation([ht.provider().Attempt(result.args, None, result, None, 'unknown', None)]) + + +def FLASHED(hex_path): # noqa: N802 FakeRig's record of the provider's J-Link flash script + return ['flash', 'halt', f'loadfile {hex_path}', 'r', 'go', 'exit'] + + def jlink_target(hex_path, download='1 range affected (65536 bytes)', tail='O.K.\n'): - """tinyuf2-jlink's transcript through CMake and Ninja, in the shape the bench recorded on metro_m7_1011.""" - return (f'[1/1] cd /b && JLinkExe -device D -USB 1 -ExitOnError 1 -NoGui 1 -if swd -JTAGConf -1,-1 -speed auto ' - f'-CommandFile /b/tinyuf2.jlink\nJ-Link>halt\nConnecting to J-Link ...O.K.\nS/N: 1\nConnecting to target via SWD\n' + """The J-Link flash script's transcript, in the shape the bench recorded on metro_m7_1011.""" + return (f'J-Link>halt\nConnecting to J-Link ...O.K.\nS/N: 1\nConnecting to target via SWD\n' f'Cortex-M7 identified.\nJ-Link>loadfile {hex_path}\nResetTarget() start\n' f'ResetTarget() end - Took 117ms\nDownloading file [{hex_path}]...\n' f'Comparing flash [000%]100%] Done.\nJ-Link: Flash download: Bank 0 @ 0x60000000: {download}\n{tail}' f'J-Link>r\nJ-Link>go\nJ-Link>exit\n\nScript processing completed.\n') -JLINK_NO_PROBE = ('[1/1] cd /b && JLinkExe -USB 999999999 -ExitOnError 1\nJ-Link>halt\nJ-Link connection not established yet but ' - 'required for command.\nConnecting to J-Link ...FAILED: Cannot connect to J-Link.\nFAILED: CMakeFiles/tinyuf2-jlink\n' - 'ninja: build stopped: subcommand failed.\n') +JLINK_NO_PROBE = ('J-Link>halt\nJ-Link connection not established yet but required for command.\n' + 'Connecting to J-Link ...FAILED: Cannot connect to J-Link.\n') # J-Link's connect to an STM32F303VC with TinyUF2's write protection set, as the bench recorded it @@ -2104,11 +3231,13 @@ def probe(self, commands, judge=True): self.mem[int(a, 16)] = int(v, 16) return out - def spawn(self, cmd, log, timeout, cwd=None): + def spawn(self, cmd, log, timeout, cwd=None, **k): if cmd[0] == 'mcopy': self.present = False - if cmd[0] == 'cmake': - self.scripts.append(['target ' + cmd[-1]]) + if cmd[0] == 'JLinkExe': + script = Path(cmd[cmd.index('-CommandFile') + 1]).read_text().splitlines() + assert any(c.startswith('loadfile ') for c in script), script # sessions go through FakeRig.probe + self.scripts.append(['flash'] + script) self.present = not self.app_survives return 0, jlink_target(self.hex) return 0, '' @@ -2126,9 +3255,11 @@ def fat_read(self, dev, name, length, log, offset=0): class BatteryRun(Tmp): """Battery.run end to end over a fake probe transcript and a fake drive: the plumbing, not the rig.""" - def battery(self, reentry='retained-register', corrupt=None, commissioning=False): + def battery(self, reentry='retained-register', corrupt=None, commissioning=False, device=None): board = self.load(roster(self.tmp, tinyuf2={'reentry': reentry})) facts = ht.source_facts(board) + if device: + board['device'] = device bat = ht.Battery(board, facts, self.tmp, commissioning=commissioning) bootloader_bin(self.tmp / 'c.bin') rig = FakeRig(bat) @@ -2137,10 +3268,11 @@ def battery(self, reentry='retained-register', corrupt=None, commissioning=False rig.hex = build / 'tinyuf2.hex' rig.hex.write_text(':00000001FF\n') self.files = {'bin': self.tmp / 'c.bin', 'hex': rig.hex, - 'flash': {'root': build, 'binding': {'target': 'tinyuf2-jlink'}, 'sha256': {'tinyuf2.hex': ht.sha256(rig.hex)}, - 'ninja': None}} + 'flash': {'root': build, 'binding': {'target': 'tinyuf2-jlink'}, + 'sha256': {'tinyuf2.hex': ht.sha256(rig.hex)}}} (self.tmp / 'cur').mkdir() bat.probe.run = rig.probe + bat.probe.sysreset = lambda: rig.probe([f'w4 {ht.SCB_AIRCR:08X} {ht.AIRCR_SYSRESETREQ:08X}'], judge=False) clock = [0.0] # time passes only when the battery sleeps def fat_read(*a): @@ -2149,8 +3281,7 @@ def fat_read(*a): if corrupt: corrupt(rig.current) return rig.fat_read(*a) - for p in (mock.patch.object(ht.cb, 'flash_ready', lambda *a: None), - mock.patch.object(ht, 'usb_device', lambda *a: '1-2' if rig.present else None), + for p in (mock.patch.object(ht, 'usb_device', lambda *a: '1-2' if rig.present else None), mock.patch.object(ht, 'block_device', lambda p: '/dev/sdz'), mock.patch.object(ht, 'check_exclusive', lambda d: None), mock.patch.object(ht, 'fat_read', fat_read), @@ -2162,19 +3293,45 @@ def fat_read(*a): return bat, rig def test_the_f411_install_verify_waits_out_the_option_byte_rewrite_before_its_halt(self): - bat, rig = self.battery() - bat.b['device'] = 'stm32f411ve' + bat, rig = self.battery(device='stm32f411ve') bat.install(self.files) self.assertEqual(rig.scripts[1][:2], ['sleep 1000', 'h']) + def test_the_install_is_the_providers_jlink_flash_of_the_record_hex_and_runs_no_ninja(self): + bat, rig = self.battery(device='stm32f411ve') # the flash connects as the roster device; only sessions connect as the core + calls, spawn = [], rig.spawn + with mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: calls.append((list(cmd), k.get('cwd'))) or spawn(cmd, *a, **k)): + ev = bat.install(self.files) + (argv, cwd), = calls + path = argv[-1] + self.assertEqual((argv[:-1], cwd), (['JLinkExe', '-USB', '1', '-device', 'stm32f411ve', '-if', 'swd', '-JTAGConf', + '-1,-1', '-speed', 'auto', '-NoGui', '1', '-ExitOnError', '1', '-CommandFile'], None)) + self.assertEqual((Path(path).parent, Path(path).exists()), (self.tmp, False)) # in the run dir, removed after + self.assertEqual(rig.scripts[0], FLASHED(self.files['hex'])) + self.assertEqual(ev['operation'], {'attempts': [{'kind': 'exited', 'exit': 0}], 'cleanup': 'done'}) + + def test_a_timed_out_flash_fails_touched_with_its_transcript_so_far(self): + bat, rig = self.battery() + + def hangs(cmd, log, timeout, **k): + raise ht.SpawnTimeout(f'JLinkExe timed out after {timeout}s', 'Connecting to J-Link ...FAILED: x\n', []) + with mock.patch.object(ht, 'spawn', hangs), \ + self.assertRaisesRegex(ht.StepFail, f'^JLinkExe timed out after {ht.PROBE_TIMEOUT}s$') as e: + bat.install(self.files) + ev = e.exception.evidence + # what a killed tool printed so far is no proof it never reached the target + self.assertEqual((ev['tail'], ev['beforeTarget'], bat.touched), (['Connecting to J-Link ...FAILED: x'], False, True)) + self.assertEqual(ev['operation']['attempts'], [{'kind': 'timed_out', 'exit': 124}]) + def test_every_step_passes_and_resumes_only_after_a_halt(self): bat, rig = self.battery() bat.run(self.files, 'v1') self.assertEqual([x['step'] for x in bat.steps if x['pass']], list(ht.STEPS['jlink'])) - self.assertEqual(rig.scripts[:2], [['target tinyuf2-jlink'], ['h', f'verifybin {self.tmp / "c.bin"},0x0', 'r', 'g']]) + self.assertEqual(rig.scripts[:2], [FLASHED(self.files['hex']), ['h', f'verifybin {self.tmp / "c.bin"},0x0', 'r', 'g']]) self.assertEqual(rig.scripts[2][:2], ['h', f'w4 {bat.f["dblTapReg"]:08X} 00000000']) # cleared in dfu self.assertIn(f'sleep {2 * bat.f["dblTapDelayMs"] + 200}', rig.scripts[3]) - self.assertEqual(bat.steps[0]['evidence'], {'flashDownload': '1 range affected (65536 bytes)', 'verified': True}) + self.assertEqual(bat.steps[0]['evidence'], {'flashDownload': '1 range affected (65536 bytes)', 'verified': True, + 'operation': {'attempts': [{'kind': 'exited', 'exit': 0}], 'cleanup': 'done'}}) for script in rig.scripts: if 'g' in script: self.assertEqual(script[0], 'h', script) @@ -2249,7 +3406,7 @@ def test_restore_reinstalls_the_record_and_leaves_dfu_after_blinky(self): with mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: copies.append(cmd) or rig.spawn(cmd, *a, **k)): out = bat.restore(dict(self.files, blinky=blinky, blinkyBlocks=1), 'v1') self.assertEqual((out['state'], out['evidence']['copy']), ('blinky copied', {'blocks': 1})) - self.assertEqual(rig.scripts, [['target tinyuf2-jlink'], ['h', f'verifybin {self.tmp / "c.bin"},0x0', 'r', 'g']]) + self.assertEqual(rig.scripts, [FLASHED(self.files['hex']), ['h', f'verifybin {self.tmp / "c.bin"},0x0', 'r', 'g']]) self.assertEqual(copies[1][4], blinky) # after the target's cmake rig.present, polls = True, [] def blip(*a): # DFU shows once, mid-way through the quiet period, then is gone again @@ -2267,14 +3424,14 @@ def test_the_target_transcript_is_judged_not_its_exit(self): {'flashDownload': '1 range affected (65536 bytes)'}) skip = 'Skipped. Contents already match' # an already-matching image is a success self.assertEqual(ht.judge_jlink_target(0, jlink_target(hexp, skip), bat.b, hexp)['flashDownload'], skip) - cases = (('every command FAILED, exit 0', 0, JLINK_NO_PROBE, True), ('ninja failed', 1, JLINK_NO_PROBE, True), + cases = (('every command FAILED, exit 0', 0, JLINK_NO_PROBE, True), ('no probe, exit 1', 1, JLINK_NO_PROBE, True), ('another image', 0, jlink_target(self.tmp / 'other.hex'), False), ('no O.K.', 0, jlink_target(hexp, tail=''), False), ('a failure line', 0, jlink_target(hexp, tail='O.K.\nCannot connect to target.\n'), False), ('cut short', 0, jlink_target(hexp).replace('Script processing completed.\n', ''), False), ('a nonzero exit', 2, jlink_target(hexp), False), ('nothing', 0, '', False)) for what, code, out, before in cases: - with self.subTest(what), self.assertRaisesRegex(ht.StepFail, 'tinyuf2-jlink failed') as e: + with self.subTest(what), self.assertRaisesRegex(ht.StepFail, 'the J-Link flash failed') as e: ht.judge_jlink_target(code, out, bat.b, hexp) self.assertEqual(e.exception.evidence['beforeTarget'], before, what) @@ -2297,37 +3454,37 @@ def test_the_f3_unprotect_note_passes_on_the_f303_only_and_only_whole(self): with self.subTest(what), self.assertRaises(ht.StepFail): ht.judge_jlink_target(0, text, board, hexp) - def test_the_target_runs_under_the_build_lock_between_two_input_checks(self): + def test_the_flash_runs_under_the_build_lock_between_two_input_checks(self): bat, rig = self.battery() fl, hexp = self.files['flash'], self.files['hex'] judge = lambda code, out: ht.judge_jlink_target(code, out, bat.b, hexp) # noqa: E731 - cmake = ['cmake', '--build', fl['root'], '--target', 'tinyuf2-jlink'] + before = lambda code, out: ht.stopped_before_target(code, out, bat.b['probe']) # noqa: E731 + + def flash(run): + return one_attempt(run(['JLinkExe', '-CommandFile', str(self.tmp / 'x.jlink')], timeout=5)) + (self.tmp / 'x.jlink').write_text(f'halt\nloadfile {hexp}\nr\ngo\nexit\n') with self.subTest('a build holds the dir'), ht.cb.build_lock(fl['root']), \ self.assertRaisesRegex(ht.StepFail, 'in use by a build or another flash'): - bat.flash_target(fl, cmake, 5, judge) + bat.flash_target(fl, flash, judge, before, 'JLinkExe', 5) good = hexp.read_text() hexp.write_text('changed') with self.subTest('an input changed'), self.assertRaisesRegex(ht.StepFail, 'changed before the flash'): - bat.flash_target(fl, cmake, 5, judge) + bat.flash_target(fl, flash, judge, before, 'JLinkExe', 5) hexp.write_text(good) - with self.subTest('a dry run with more than the flasher'), \ - mock.patch.object(ht.cb, 'flash_ready', side_effect=ht.cb.Fail('would not run only its flasher')), \ - self.assertRaisesRegex(ht.StepFail, 'only its flasher'): - bat.flash_target(fl, cmake, 5, judge) - self.assertEqual((rig.scripts, bat.touched), ([], False)) # none of these ran the target + self.assertEqual((rig.scripts, bat.touched), ([], False)) # none of these ran the tool for what, result, touched in (('no probe', (1, JLINK_NO_PROBE), False), ('the target reached', (1, jlink_target(hexp, tail='Cannot connect to target.\n')), True)): bat.touched = False with self.subTest(what), mock.patch.object(ht, 'spawn', lambda *a, r=result, **k: r), \ - self.assertRaisesRegex(ht.StepFail, 'tinyuf2-jlink failed'): - bat.flash_target(fl, cmake, 5, judge) + self.assertRaisesRegex(ht.StepFail, 'the J-Link flash failed'): + bat.flash_target(fl, flash, judge, before, 'JLinkExe', 5) self.assertEqual(bat.touched, touched, what) bat.touched = False spawn = rig.spawn with self.subTest('rebuilt while flashing'), \ mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: (hexp.write_text('rebuilt'), spawn(cmd, *a, **k))[1]), \ self.assertRaisesRegex(ht.StepFail, 'changed while flashing') as e: - bat.flash_target(fl, cmake, 5, judge) + bat.flash_target(fl, flash, judge, before, 'JLinkExe', 5) self.assertEqual((bat.touched, e.exception.evidence['flashDownload']), (True, '1 range affected (65536 bytes)')) def test_dfu_after_an_install_reenters_only_when_dfu_never_enumerates(self): @@ -2511,7 +3668,7 @@ def row(self, tried='retained-register', passed=True, failed_at=None, cleaned=Tr 'cleanup': {'pass': cleaned, 'heldAfter': held}} def test_the_commissioning_decision(self): - for method in (*ht.REENTRY['jlink'], *ht.REENTRY['esptool']): + for method in (*ht.REENTRY['jlink'], *ht.REENTRY['esp']): with self.subTest(method): self.assertEqual(ht.reentry_decision(self.row(method)), method) self.assertEqual(ht.reentry_decision(self.row(method, passed=False, failed_at='readback')), method) @@ -2527,7 +3684,169 @@ def test_the_commissioning_decision(self): self.assertIsNone(ht.reentry_decision(dict(self.row(), **{'pass': False}))) # aborted after every step passed +class PendingMark(Tmp): + def setUp(self): + super().setUp() + self.mark = ht.PendingMark(None, 'b') + self.ended = self.receipt('ended', closed=True, cleanup='done') + self.unresolved = self.receipt('unresolved', closed=True, cleanup='pending') + + def receipt(self, name, **rec): + p = self.tmp / name / 'receipt.json' + p.parent.mkdir() + p.write_text(json.dumps({'tool': 'hil_openocd_esp', **rec})) + return {'cmd': 'openocd', 'receipt': str(p)} + + def racing(self, name, other): + """Run `other` in a thread the first time `name` runs, inside the caller's critical section, and give it 0.5 s + to finish there; without the mutex it does.""" + real, done = getattr(ht, name), [] + + def hooked(*a): + if not done: + t = threading.Thread(target=other) + t.start() + t.join(0.5) + done.append(t) + return real(*a) + return mock.patch.object(ht, name, hooked), done + + def test_two_writers_never_lose_a_session(self): + patch, done = self.racing('read_mark', lambda: ht.mark_pending(self.mark, [self.unresolved])) + with patch: + ht.mark_pending(self.mark, [self.ended]) + done[0].join(5) + self.assertEqual(sorted(e['receipt'] for e in ht.read_mark(self.mark)), + sorted([self.ended['receipt'], self.unresolved['receipt']])) + + def test_a_check_never_removes_a_session_written_while_it_checked(self): + ht.mark_pending(self.mark, [self.ended]) + patch, done = self.racing('session_state', lambda: ht.mark_pending(self.mark, [self.unresolved])) + with patch: + ht.clear_pending_mark(self.mark) # it saw only the ended session + done[0].join(5) + with self.assertRaisesRegex(ht.Refused, 'not ended'): + ht.clear_pending_mark(self.mark) + self.assertEqual([e['receipt'] for e in ht.read_mark(self.mark)], [self.unresolved['receipt']]) + + def test_a_write_makes_its_rename_durable_and_reports_a_failure_to(self): + real, dirs = os.fsync, [] + + def fsync(fd): + if os.path.isdir(f'/proc/self/fd/{fd}'): + dirs.append(fd) + raise OSError(5, 'Input/output error') + return real(fd) + with mock.patch.object(ht.os, 'fsync', fsync), mock.patch.object(ht, 'MARK_ERRORS', []): + self.assertIn('Input/output error', ht.mark_pending(self.mark, [self.ended])) + self.assertEqual(len(ht.MARK_ERRORS), 1) + self.assertEqual(len(dirs), 1) + + def test_each_write_has_a_temp_file_of_its_own(self): + seen, real = [], os.replace + with mock.patch.object(ht.os, 'replace', lambda a, b, **k: (seen.append(a), real(a, b, **k))[1]): + ht.mark_pending(self.mark, [self.ended]) + ht.mark_pending(self.mark, [self.unresolved]) + self.assertEqual(len(set(seen)), 2) + self.assertNotIn(f'{self.mark.name}.tmp', seen) + self.assertEqual(sorted(p.name for p in (self.tmp / 'marks').iterdir()), + [self.mark.name, self.mark.name + ht.MARK_MUTEX]) + + def clear(self, *extra): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht, 'load_helper', lambda: self.mark_helper), \ + mock.patch.object(ht, 'PendingMark', lambda helper, lock: self.mark): + code = ht.main(['clear-pending', '-b', BOARD, *extra, str(cfg)]) + return code, json.loads(out.getvalue()) + + def hold(self): + """This process's own manual hold of the board, as `hil_lock.py hold` leaves it.""" + lock = ht.load_board(roster(self.tmp, tinyuf2={'reentry': None}), BOARD)['lock'] + fh = self.mark_helper.flock_nb(lock) + self.addCleanup(fh.close) + self.mark_helper.write_record(fh, 'mine') + return lock + + def test_a_first_run_creates_the_lock_dir_through_the_helper_and_finds_no_mark(self): + self.assertFalse((self.tmp / 'marks').exists()) + ht.clear_pending_mark(self.mark) + self.assertEqual(stat_mode(self.tmp / 'marks'), self.mark_helper.DIR_MODE) + + def test_the_mark_and_its_mutex_are_umask_proof(self): + old = os.umask(0o077) + try: + ht.mark_pending(self.mark, [self.ended]) + finally: + os.umask(old) + self.assertEqual((stat_mode(str(self.mark) + ht.MARK_MUTEX), stat_mode(self.mark)), (0o666, 0o644)) + + def test_a_malformed_mark_is_kept_reported_and_cleared_only_after_a_reviewed_recovery(self): + (self.tmp / 'marks').mkdir() + Path(self.mark).write_text('{"sessions": [{}]}') + with mock.patch.object(ht, 'MARK_ERRORS', []): + self.assertIn('malformed', ht.mark_pending(self.mark, [self.unresolved])) + self.assertEqual(Path(self.mark).read_text(), '{"sessions": [{}]}') + with self.assertRaisesRegex(ht.Refused, 'malformed'): + ht.clear_pending_mark(self.mark) + self.assertEqual(self.clear()[0], 2) + self.hold() + code, out = self.clear('--after-reviewed-recovery', 'reviewed in #9', '--under-hold', 'mine') + self.assertEqual((code, out['cleared']['unreadable']), (0, '{"sessions": [{}]}')) + self.assertFalse(Path(self.mark).exists()) + + def test_clear_pending_refuses_while_a_session_is_not_ended_naming_its_receipt(self): + ht.mark_pending(self.mark, [self.ended, self.unresolved]) + self.assertIn(self.unresolved['receipt'], self.clear()[1]['error']) + self.hold() + for extra, why in (([], 'not ended'), (['--after-reviewed-recovery', ' ', '--under-hold', 'mine'], 'needs the note'), + (['--after-reviewed-recovery', 'n'], 'needs proof'), + (['--after-reviewed-recovery', 'n', '--under-hold', 'theirs'], "not 'theirs'"), + (['--after-reviewed-recovery', 'n', '--recover-holder', str(os.getpid())], "not 'hil_test.py'")): + with self.subTest(extra): + code, out = self.clear(*extra) + self.assertEqual((code, out['pass']), (2, False)) + self.assertIn(why, out['error']) + self.assertTrue(Path(self.mark).exists()) + + def test_a_forced_clear_waits_for_no_recovery_already_running(self): + ht.mark_pending(self.mark, [self.unresolved]) + lock = self.hold() + busy = ht.recovery_lock(self.mark_helper, lock) + self.addCleanup(busy.close) + code, out = self.clear('--after-reviewed-recovery', 'n', '--under-hold', 'mine') + self.assertEqual(code, 2) + self.assertIn('another recovery', out['error']) + self.assertTrue(Path(self.mark).exists()) + + def test_clear_pending_removes_a_mark_once_every_session_ended(self): + ht.mark_pending(self.mark, [self.ended]) + code, out = self.clear() # no hold needed: the mark's mutex suffices + self.assertEqual((code, out['cleared']['notEnded']), (0, [])) + self.assertFalse(Path(self.mark).exists()) + self.assertFalse(Path(str(self.mark) + ht.CLEARED_LOG).exists()) + self.assertEqual(self.clear()[0], 0) # no mark: nothing to clear + + def test_clear_pending_after_a_reviewed_recovery_records_its_note_and_hold_then_removes_the_mark(self): + ht.mark_pending(self.mark, [self.ended, self.unresolved]) + self.hold() + code, out = self.clear('--after-reviewed-recovery', 'server ended by the reviewed recovery in #123', + '--under-hold', 'mine') + self.assertEqual((code, out['cleared']['notEnded']), (0, [self.unresolved['receipt']])) + self.assertFalse(Path(self.mark).exists()) + log = [json.loads(x) for x in Path(str(self.mark) + ht.CLEARED_LOG).read_text().splitlines()] + self.assertEqual([(x['note'], x['notEnded'], x['hold']['reason'], x['hold']['pid']) for x in log], + [('server ended by the reviewed recovery in #123', [self.unresolved['receipt']], 'mine', + os.getpid())]) + + class Main(Tmp): + def setUp(self): + super().setUp() + p = mock.patch.object(ht, 'probe_users', lambda b: []) # not the host's J-Link tools + p.start() + self.addCleanup(p.stop) + def test_manual_install_runs_the_record_and_brings_up_dfu_with_reentry_only_on_trial(self): for what, reentry, extra, want, calls in ( ('uncommissioned', None, [], 0, [('install', None), ('dfu', 'v')]), @@ -2537,21 +3856,91 @@ def test_manual_install_runs_the_record_and_brings_up_dfu_with_reentry_only_on_t cfg = roster(self.tmp, tinyuf2={'reentry': reentry}) with self.subTest(what), contextlib.redirect_stdout(io.StringIO()), \ mock.patch.object(ht.cb, 'BUILD_DIR', Path(tempfile.mkdtemp(dir=self.tmp))), \ - mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ + mock.patch.object(ht, 'load_helper', lambda: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ mock.patch.object(ht, 'admit_record', lambda *a: ({'bin': 'b'}, {'version': 'v'})), \ mock.patch.object(ht.Battery, 'install', lambda bat, f: seen.append(('install', bat.b['reentry'])) or {}), \ mock.patch.object(ht.Battery, 'dfu_after_install', lambda bat, v: seen.append(('dfu', v)) or {}): code = ht.main(['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'debug 7', *extra, str(cfg)]) self.assertEqual((code, seen), (want, calls)) + def test_manual_install_and_restore_flash_the_record_hex_through_the_provider(self): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + build = self.tmp / 'build' + build.mkdir() + hexp = build / 'tinyuf2.hex' + hexp.write_text(':00000001FF\n') + blinky = self.tmp / 'blinky.uf2' + blinky.write_bytes(ht.pack_uf2([(0x10000, bytes(256))], LPC_FACTS['family'])) + files = {'bin': self.tmp / 'c.bin', 'hex': hexp, 'blinky': blinky, 'blinkyBlocks': 1, + 'flash': {'root': build, 'binding': {'target': 'tinyuf2-jlink'}, 'sha256': {'tinyuf2.hex': ht.sha256(hexp)}}} + flashed, clock = [], [0.0] + + def spawn(cmd, log, timeout, cwd=None, **k): + if cmd[0] == 'JLinkExe': + flashed.append(Path(cmd[-1]).read_text()) + return 0, jlink_target(hexp) + return 0, '' # mcopy, sync + for argv in (['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x'], + ['restore', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x']): + out = io.StringIO() + with self.subTest(argv[0]), contextlib.redirect_stdout(out), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / argv[0]), mock.patch.object(ht, 'load_helper', lambda: None), \ + mock.patch.object(ht, 'manual_holder', lambda *a: None), \ + mock.patch.object(ht, 'admit_record', lambda *a: (files, {'version': 'v'})), mock.patch.object(ht, 'spawn', spawn), \ + mock.patch.object(ht.Probe, 'run', lambda *a, **k: ''), \ + mock.patch.object(ht.Battery, 'dfu_after_install', lambda bat, v: {'usb': '1-2'}), \ + mock.patch.object(ht, 'block_device', lambda p: '/dev/sdz'), mock.patch.object(ht, 'usb_device', lambda *a: None), \ + mock.patch.object(ht.time, 'sleep', lambda t: clock.__setitem__(0, clock[0] + t)), \ + mock.patch.object(ht.time, 'monotonic', lambda: clock[0]): + self.assertEqual(ht.main(argv + [str(cfg)]), 0, out.getvalue()) + self.assertEqual(flashed, [f'halt\nloadfile {hexp}\nr\ngo\nexit\n'] * 2) + + def test_a_manual_install_overlaps_neither_a_probe_user_nor_an_earlier_invocations_pending_session(self): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + receipt = self.tmp / 'earlier' / 'receipt.json' + receipt.parent.mkdir() + receipt.write_text(json.dumps({'tool': 'hil_openocd_esp', 'closed': False, 'init': 'ok'})) + for what, users, pending, want in (('a probe user', [101], [], 'still in use by pids'), + ('a pending session', [], [{'cmd': 'openocd', 'receipt': str(receipt)}], + 'not ended')): + installed, out = [], io.StringIO() + ht.mark_pending(ht.PendingMark(None, ht.load_board(cfg, BOARD)['lock']), pending) + with self.subTest(what), contextlib.redirect_stdout(out), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / what), mock.patch.object(ht, 'load_helper', lambda: None), \ + mock.patch.object(ht, 'manual_holder', lambda *a: None), mock.patch.object(ht, 'probe_users', lambda b: users), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ + mock.patch.object(ht.Battery, 'install', lambda bat, f: installed.append(1) or {}): + code = ht.main(['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x', str(cfg)]) + self.assertEqual((code, installed), (2, [])) + self.assertIn(want, out.getvalue()) + + def test_a_manual_install_whose_pending_mark_cannot_be_written_still_reports_and_keeps_the_hold(self): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + left = [{'cmd': 'openocd', 'receipt': str(self.tmp / 'r' / 'receipt.json')}] + + def enospc(*a): + raise OSError(28, 'No space left on device') + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / 'b'), \ + mock.patch.object(ht, 'load_helper', lambda: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ + mock.patch.object(ht, 'write_mark', enospc), \ + mock.patch.object(ht, 'MARK_ERRORS', []), mock.patch.object(ht, 'outstanding_sessions', lambda d: left), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ + mock.patch.object(ht.Battery, 'install', lambda bat, f: {}), \ + mock.patch.object(ht.Battery, 'dfu_after_install', lambda bat, v: {}): + code = ht.main(['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x', str(cfg)]) + report = json.loads(out.getvalue().splitlines()[-1]) + self.assertEqual((code, report['pass'], report['survivors']), (4, False, left)) + self.assertIn('could not be written', report['markErrors'][0]) + def test_an_install_that_fails_after_touching_the_board_is_exit_4(self): cfg = roster(self.tmp, tinyuf2={'reentry': None}) for touched, want in ((True, 4), (False, 1)): def fail(bat, files, touched=touched): bat.touched = touched - raise ht.StepFail('tinyuf2-jlink failed (exit 1)') + raise ht.StepFail('the J-Link flash failed (exit 1)') with self.subTest(touched), contextlib.redirect_stdout(io.StringIO()), \ - mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / f'i{want}'), mock.patch.object(ht, 'load_helper', lambda p: None), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / f'i{want}'), mock.patch.object(ht, 'load_helper', lambda: None), \ mock.patch.object(ht, 'manual_holder', lambda *a: None), \ mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), mock.patch.object(ht.Battery, 'install', fail): self.assertEqual(ht.main(['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x', str(cfg)]), want) @@ -2563,8 +3952,8 @@ def fail(bat, files, version, touched=touched): bat.touched = touched raise ht.StepFail('the DFU device came back after the blinky copy') with self.subTest(touched), contextlib.redirect_stdout(io.StringIO()), \ - mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / str(want)), mock.patch.object(ht, 'load_helper', lambda p: None), \ - mock.patch.object(ht, 'manual_holder', lambda *a: None), mock.patch.object(ht, 'probe_users', lambda b: []), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / str(want)), mock.patch.object(ht, 'load_helper', lambda: None), \ + mock.patch.object(ht, 'manual_holder', lambda *a: None), \ mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), mock.patch.object(ht.Battery, 'restore', fail): self.assertEqual(ht.main(['restore', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x', str(cfg)]), want) @@ -2575,7 +3964,7 @@ def test_a_recovery_while_another_runs_is_refused_before_it_checks_the_holder(se busy = mock.Mock(side_effect=ht.Refused(f'{BOARD}: another recovery holds x.lock.recover')) out = io.StringIO() with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp), \ - mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'check_jlink', lambda: None), \ + mock.patch.object(ht, 'load_helper', lambda: None), mock.patch.object(ht, 'check_jlink', lambda: None), \ mock.patch.object(ht, 'recovery_lock', busy), \ mock.patch.object(ht, 'prove_hold', side_effect=AssertionError('checked the holder')), \ mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ @@ -2586,6 +3975,40 @@ def test_a_recovery_while_another_runs_is_refused_before_it_checks_the_holder(se (2, f'{BOARD}: another recovery holds x.lock.recover')) self.assertEqual(busy.call_count, 1) + def test_a_recovery_is_refused_while_a_session_of_the_failed_run_is_unresolved_without_a_pending_mark(self): + holder = self.tmp / 'hil_test.py' + holder.write_text('import time; time.sleep(30)') + p = subprocess.Popen([sys.executable, str(holder)]) + self.addCleanup(lambda: (p.kill(), p.wait())) + end = time.monotonic() + 10 + while b'hil_test.py' not in Path(f'/proc/{p.pid}/cmdline').read_bytes() and time.monotonic() < end: + time.sleep(0.01) + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + receipt = self.tmp / 'oocd' / 'receipt.json' + receipt.parent.mkdir() + receipt.write_text(json.dumps({'tool': 'hil_openocd_esp', 'closed': True, 'cleanup': 'pending'})) + report = self.tmp / 'failed.json' + report.write_text(json.dumps({'results': [{'board': BOARD, 'cleanup': { + 'holder': {'pid': p.pid}, 'survivors': [{'cmd': 'openocd', 'receipt': str(receipt)}]}}]})) + mark = ht.PendingMark(None, ht.load_board(cfg, BOARD)['lock']) + for what, reviewed, want in (('unreviewed', False, 2), ('cleared after a reviewed recovery', True, 0)): + if reviewed: + ht.mark_pending(mark, [{'cmd': 'openocd', 'receipt': str(receipt)}]) + ht.clear_pending_mark(mark, 'server ended by the reviewed recovery in #9', {}) + self.assertFalse(Path(mark).exists()) + killed, out = [], io.StringIO() + with self.subTest(what), contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / str(want)), \ + mock.patch.object(ht, 'load_helper', mock.Mock), mock.patch.object(ht, 'check_jlink', lambda: None), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ + mock.patch.object(ht, 'recovery_lock', lambda *a: mock.Mock()), mock.patch.object(ht, 'prove_hold', lambda *a, **k: {}), \ + mock.patch.object(ht.os, 'kill', lambda *a: killed.append(a)), mock.patch.object(ht, 'wait_for', lambda *a: True), \ + mock.patch.object(ht.Battery, 'restore', lambda bat, f, v: {'pass': True}): + code = ht.main(['restore', '-b', BOARD, '--build-record', 'r.json', '--recover-holder', str(p.pid), '--report', + str(report), str(cfg)]) + self.assertEqual((code, killed), (want, [] if want else [(p.pid, signal.SIGUSR1)])) + if want: + self.assertIn(str(receipt), json.loads(out.getvalue().splitlines()[-1])['error']) + def test_an_esptool_recovery_checks_the_snapshot_in_the_failed_report(self): holder = self.tmp / 'hil_test.py' holder.write_text('import time; time.sleep(30)') @@ -2605,10 +4028,10 @@ def test_an_esptool_recovery_checks_the_snapshot_in_the_failed_report(self): 'preservedRegions': {n: list(r) for n, r in regions.items()}}}]})) seen = [] with self.subTest(what), contextlib.redirect_stdout(io.StringIO()), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / str(want)), \ - mock.patch.object(ht, 'load_helper', lambda p: mock.Mock()), \ + mock.patch.object(ht, 'load_helper', mock.Mock), \ mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ mock.patch.object(ht, 'recovery_lock', lambda *a: mock.Mock()), mock.patch.object(ht, 'prove_hold', lambda *a, **k: {}), \ - mock.patch.object(ht, 'probe_users', lambda b: []), mock.patch.object(ht.os, 'kill', lambda *a: None), \ + mock.patch.object(ht.os, 'kill', lambda *a: None), \ mock.patch.object(ht, 'wait_for', lambda *a: True), \ mock.patch.object(ht.EspBattery, 'restore', lambda bat, f, v: seen.append(bat.preserved) or {'pass': True}): code = ht.main(['restore', '-b', S3, '--build-record', 'r.json', '--recover-holder', str(p.pid), '--report', @@ -2625,7 +4048,7 @@ def test_no_jlinkexe_programs_nothing_on_any_path(self): with self.subTest(argv[0]), contextlib.redirect_stdout(io.StringIO()), \ mock.patch.object(ht, 'verify_record', lambda *a: ({'hex': 'x.hex', 'bin': fw}, {'version': 'v'})), \ mock.patch.object(ht, 'parse_hex', lambda p: {0: 1}), mock.patch.object(ht, 'hex_matches_bin', lambda *a: True), \ - mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / argv[0]), mock.patch.object(ht, 'load_helper', lambda p: None), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / argv[0]), mock.patch.object(ht, 'load_helper', lambda: None), \ mock.patch.object(ht, 'manual_holder', lambda *a: None), mock.patch.object(ht, 'exclusion_configured', lambda *a: None), \ mock.patch.object(ht.shutil, 'which', lambda t: None if t == 'JLinkExe' else '/usr/bin/' + t), \ mock.patch.object(ht.Probe, 'run', lambda *a, **k: probed.append(a)): @@ -2642,6 +4065,349 @@ def test_a_refusal_is_a_ran_false_row_and_exit_2(self): self.assertEqual((code, r['pass'], r['results'][0]['ran']), (2, False, False)) self.assertTrue(Path(r['reportPath']).is_file()) + def test_run_root_places_the_run_directory(self): + cfg = roster(self.tmp, flasher={'name': 'jlink', 'uid': '1', 'args': '-device NOPE'}) + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / 'unused'): + ht.main(['run', '-b', BOARD, '--build-record', 'r.json', '--run-root', str(self.tmp / 'here'), str(cfg)]) + r = json.loads(out.getvalue().splitlines()[-1]) + self.assertEqual(Path(r['reportPath']).parent.parent, self.tmp / 'here') + self.assertFalse((self.tmp / 'unused').exists()) + + def test_a_lock_pin_refusal_stops_a_run_before_its_record_and_battery(self): + cfg = roster(self.tmp) + admit, battery = mock.Mock(), mock.Mock() + pin = mock.Mock(side_effect=ht.Refused('lib/agentrc is at b; HEAD pins a')) + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp), \ + mock.patch.object(ht, 'check_lock_pin', pin), \ + mock.patch.object(ht, 'admit_record', admit), mock.patch.object(ht.Battery, 'run', battery): + code = ht.main(['run', '-b', BOARD, '--build-record', 'r.json', str(cfg)]) + res = json.loads(out.getvalue().splitlines()[-1]) + self.assertEqual((code, res['error'], [{k: r[k] for k in ('ran', 'pass', 'locked')} for r in res['results']]), + (2, 'lib/agentrc is at b; HEAD pins a', [{'ran': False, 'pass': False, 'locked': False}])) + pin.assert_called_once_with() + admit.assert_not_called() + battery.assert_not_called() + + +H563 = {'port': 'stm32h5', 'name': 'stm32h563_nucleo', 'device': 'stm32h563zi', 'flasher': 'stlink'} +CUBE_CONNECT = ('ST-LINK SN : 0023\nST-LINK FW : V3J16M9\nBoard : NUCLEO-H563ZI\nVoltage : 3.26V\n' + 'SWD freq : 8000 KHz\nConnect mode: Hot Plug\nReset mode : Software reset\nDevice ID : {dev}\n' + 'Revision ID : Rev Z\nDevice name : STM32H56x/573\nNVM size : {nvm}\nDevice type : MCU\n' + 'Device CPU : Cortex-M33\nBL Version : 0xE3\nSFSP Version: v2.2.0\nDebug in Low Power mode enabled\n') + + +def stlink_target(error='', start='Start operation achieved successfully\n', nvm='2 MBytes'): + """tinyuf2-stlink's transcript through CMake and Ninja, as the bench recorded it on stm32h563_nucleo (v2.23.0).""" + return ('[1/1] cd /b && STM32_Programmer_CLI --connect port=swd sn=0023 --write /b/tinyuf2.hex --go\n' + ' -------------------------------------------------------------------\n' + ' \x1b[01;34mSTM32CubeProgrammer v2.23.0\x1b[0m\n' + + CUBE_CONNECT.format(dev='0x484', nvm=nvm).replace('Hot Plug', 'Normal') + + 'Opening and parsing file: tinyuf2.hex\nMemory Programming ...\n File : tinyuf2.hex\n' + ' Size : 15.17 KB \n Address : 0x08000000\nErasing memory corresponding to segment 0:\n' + 'Erasing internal memory sectors [0 1]\nDownload in Progress:\n[==================================================] 100% \n' + f'{error}File download complete\nTime elapsed during download operation: 00:00:00.120\nRUNNING Program ... \n' + f' Address: : 0x8000000\nApplication is running, Please Hold on...\n{start}') + + +class FakeCube: + """STM32_Programmer_CLI -script, answering each operation in the shape the bench recorded (v2.23.0).""" + + def __init__(self, dev='0x484', nvm='2 MBytes'): + self.dev, self.nvm, self.scripts, self.mem, self.flash, self.connects = dev, nvm, [], {}, {}, [] + self.r0, self.pc, self.sp, self.vtor = 5, 0x08008400, 0x20001000, 0x08008000 + self.on_reset, self.break_op = None, None + + def run(self, script): + lines = Path(script).read_text().splitlines() + self.scripts.append(lines[1:]) + self.connects.append(lines[0]) + out = '=== Script Manager BEGIN ===\n' + for i, line in enumerate(lines, 1): + out += f'Operation [{i}]: {line} \n' + if self.break_op and self.break_op(line): + return 1, out + 'Error: failed to read the requested memory content\n' + if i == 1: + out += CUBE_CONNECT.format(dev=self.dev, nvm=self.nvm) + elif line == '-halt': + out += 'Core halted\n' + elif line == '-run': + out += 'Core run\n' + elif line.startswith('#Delay('): + out += f'The system go to sleep for {line[7:-1]} ms.\n' + elif line == '-coreReg': + self.r0 += 7 + out += (f'R0 = 0x{self.r0:08X}\nR1 = 0x00000001\nSP = 0x{self.sp:08X}\nLR = 0x08002DA5\n' + f'PC = 0x{self.pc:08X}\nXPSR = 0x01000000\nMSP = 0x{self.sp:08X}\nPSP = 0x00000000\n') + elif line.startswith('-r32 '): + a = int(line.split()[1], 16) + v = self.vtor if a == ht.SCB_VTOR else self.mem.get(a, 0) + out += f'Reading 32-bit memory content\n Size : 4 Bytes\n Address: : 0x{a:08X}\n0x{a:08X} : {v:08X}\n' + elif line.startswith('-w32 '): + _, a, v, *nv = line.split() + out += f'Write 32-bit data\nw @{a.lower()}: {v.lower()}\nDownloading...\n32-bit data download complete\n' + if int(a, 16) == ht.SCB_AIRCR and self.on_reset: + self.on_reset() + else: + self.mem[int(a, 16)] = int(v, 16) + elif line.startswith('-u '): + _, a, n, path = line.split() + Path(path).write_bytes(self.flash.get(int(a, 16), b'')[:int(n, 16)]) + out += f'UPLOADING ...\n File : {Path(path).name}\n Size : {int(n, 16)} Bytes\nData read successfully\n' + elif line == '-rst': + out += 'MCU Reset\nSoftware reset is performed\n' + return 0, out + 'Device is disconnected\n=== Script Manager END === \n' + + +class Stlink(Tmp): + """The ST-LINK probe sessions over a fake CLI: the plumbing, not the rig.""" + + def setUp(self): + super().setUp() + self.cube = FakeCube() + self.board = {'name': 'stm32h563_nucleo', 'probe': '0023', 'device': 'stm32h563zi', 'flasher': 'stlink'} + self.probe = ht.StlinkProbe(self.board, self.tmp, self.tmp / 'log') + self.answer = None + p = mock.patch.object(ht, 'spawn', lambda cmd, log, timeout, **k: self.answer(cmd) if self.answer else + self.cube.run(cmd[2])) + p.start() + self.addCleanup(p.stop) + + def test_a_session_connects_hotplug_to_the_roster_probe_and_returns_each_operation(self): + self.cube.mem[0x2009FFFC] = 0xF01669EF + self.assertEqual(self.probe.mem32([0x2009FFFC, ht.SCB_VTOR]), {0x2009FFFC: 0xF01669EF, ht.SCB_VTOR: 0x08008000}) + self.assertEqual((self.cube.connects[0], self.cube.scripts[0]), + ('-c port=SWD sn=0023 mode=HOTPLUG', ['-r32 0x2009FFFC 1', f'-r32 0x{ht.SCB_VTOR:08X} 1'])) + self.assertEqual(list(self.tmp.glob('*.prg')), []) # the provider's script file, in the run dir, is gone + self.assertEqual(self.probe.write32(0x20000FFC, 0x12345678), '0x12345678') + self.assertEqual(self.cube.scripts[-1], ['-halt', '-w32 0x20000FFC 0x12345678', '-r32 0x20000FFC 1', '-run']) + + def test_a_session_fails_unless_the_transcript_shows_the_roster_chip_and_every_operation(self): + ok, broken = self.cube.run, FakeCube() + broken.break_op = lambda line: line == '-run' + for what, answer in ( + ('another family', lambda cmd: FakeCube(dev='0x480').run(cmd[2])), + ('another size', lambda cmd: FakeCube(nvm='1 MBytes').run(cmd[2])), + ('exit 0 with an error line', lambda cmd: (0, ok(cmd[2])[1].replace('Core run', 'Error: x'))), + ('a nonzero exit', lambda cmd: (1, ok(cmd[2])[1])), + ('no script end', lambda cmd: (0, ok(cmd[2])[1].replace('=== Script Manager END ===', ''))), + ('an operation missing', lambda cmd: (0, ok(cmd[2])[1].replace('Operation [3]: -run', 'x'))), + ('no halt result', lambda cmd: (0, ok(cmd[2])[1].replace('Core halted', ''))), + ('no run result', lambda cmd: (0, ok(cmd[2])[1].replace('Core run', ''))), + ('a failed operation', lambda cmd: broken.run(cmd[2]))): + with self.subTest(what): + self.answer = answer + with self.assertRaisesRegex(ht.StepFail, 'STM32_Programmer_CLI failed') as e: + self.probe.run(['-halt', '-run']) + self.assertFalse(e.exception.evidence['beforeTarget']) + + def test_every_operation_kind_needs_its_success_line(self): + ok = self.cube.run + ops = ['-halt', '-coreReg', '-r32 0x2009FFFC 1', '-w32 0x20000FFC 0x00000001', f'-u 0x08000000 0x10 {self.tmp / "u.bin"}', + '#Delay(10)', '-rst', '-run'] + self.assertEqual(len(self.probe.run(ops)), len(ops)) + for line in ('Core halted', 'XPSR = 0x', 'Reading 32-bit memory content', '32-bit data download complete', + 'Data read successfully', 'The system go to sleep for', 'Software reset is performed', 'Core run'): + with self.subTest(line): + self.answer = lambda cmd, line=line: (0, ok(cmd[2])[1].replace(line, '')) + with self.assertRaisesRegex(ht.StepFail, 'STM32_Programmer_CLI failed'): + self.probe.run(ops) + + def test_only_an_unknown_probe_serial_is_recognised_as_before_the_target(self): + self.answer = lambda cmd: (1, '=== Script Manager BEGIN ===\nOperation [1]: -c port=SWD sn=0023 mode=HOTPLUG \n' + 'Error: Serial number not found. Wrong parameter: sn=0023\n') + with self.assertRaises(ht.StepFail) as e: + self.probe.run(['-halt']) + self.assertTrue(e.exception.evidence['beforeTarget']) + + def test_reads_must_give_exactly_one_value_and_writes_must_read_back(self): + ok = self.cube.run + self.answer = lambda cmd: (0, ok(cmd[2])[1].replace('0x2009FFFC : 00000000', 'x')) + with self.assertRaisesRegex(ht.StepFail, 'no single value'): + self.probe.mem32([0x2009FFFC]) + self.answer = lambda cmd: (0, ok(cmd[2])[1].replace('0x2009FFFC : 00000000', '0x2009FFFC : 00000000\n0x2009FFFC : 00000001')) + with self.assertRaisesRegex(ht.StepFail, 'no single value'): + self.probe.mem32([0x2009FFFC]) + self.answer = lambda cmd: (0, ok(cmd[2])[1].replace('0x20000FFC : 00000001', '0x20000FFC : 00000000')) + with self.assertRaisesRegex(ht.StepFail, 'reads back 00000000'): + self.probe.write32(0x20000FFC, 1) + + def test_the_flash_target_is_judged_from_its_transcript(self): + self.assertEqual(ht.judge_stlink_target(0, stlink_target(), self.board), + {'flashDownload': 'File download complete', 'erasedSectors': ['0', '1']}) + for what, out, code in (('no start', stlink_target(start=''), 0), ('an error', stlink_target(error='Error: x\n'), 0), + ('another part', stlink_target(nvm='1 MBytes'), 0), ('a nonzero exit', stlink_target(), 1)): + with self.subTest(what), self.assertRaisesRegex(ht.StepFail, 'the ST-LINK flash failed'): + ht.judge_stlink_target(code, out, self.board) + + def test_any_programmer_cli_is_a_probe_user(self): + proc = self.tmp / 'proc' + for pid, argv in (('101', b'/home/u/bin/STM32_Programmer_CLI\0-script\0x.prg\0'), ('102', b'JLinkExe\0-USB\0000023\0'), + ('103', b'python3\0STM32_Programmer_CLI.txt\0')): + (proc / pid).mkdir(parents=True) + (proc / pid / 'cmdline').write_bytes(argv) + real = ht.Path + with mock.patch.object(ht, 'Path', lambda *a: real(proc) if a == ('/proc',) else real(*a)): + self.assertEqual(ht.probe_users(self.board), [101]) + + +class StlinkBatteryRun(Tmp): + """StlinkBattery.run end to end over the fake CLI and a fake drive.""" + + def setUp(self): + super().setUp() + cfg = self.tmp / 'local.json' + cfg.write_text(json.dumps({'boards': [{'name': 'stm32h563_nucleo', 'uid': 'UID', 'flasher': { + 'name': 'stlink', 'uid': '0023', 'args': '-device stm32h563zi'}, 'tinyuf2': {'reentry': 'retained-register'}}]})) + board = self.load(cfg, 'stm32h563_nucleo') + self.bat = bat = ht.StlinkBattery(board, ht.source_facts(board), self.tmp) + cube, f = FakeCube(), bat.f + self.cube, self.present = cube, [True] + bin_path = bootloader_bin(self.tmp / 'c.bin', board_id=b'STM32H563-Nucleo') + cube.flash[f['binBase']] = bin_path.read_bytes() + + def reset(): # DFU only with the magic retained + self.present[0] = cube.mem.pop(f['dblTapReg'], None) == f['dblTapMagic'] + cube.on_reset = reset + build = self.tmp / 'build' + build.mkdir() + (build / 'tinyuf2.hex').write_text(':00000001FF\n') + self.files = {'bin': bin_path, 'hex': build / 'tinyuf2.hex', + 'flash': {'root': build, 'binding': {'target': 'tinyuf2-stlink'}, + 'sha256': {'tinyuf2.hex': ht.sha256(build / 'tinyuf2.hex')}}} + + def spawn(cmd, log, timeout, cwd=None, **k): + if cmd[:2] == [ht.STM32_PROGRAMMER_CLI, '--connect']: # the provider's flash + cube.scripts.append(cmd[1:]) + return 0, stlink_target() + if cmd[0] == 'mcopy': + self.present[0] = False + cube.flash[f['appStart']] = bat.fix['image'] + cube.pc, cube.vtor = bat.fix['handler'], f['appStart'] + return 0, '' + return cube.run(cmd[2]) if cmd[0] == ht.STM32_PROGRAMMER_CLI else (0, '') + + def fat_read(dev, name, length, log, offset=0): + if name == 'INFO_UF2.TXT': + data = (f'TinyUF2 Bootloader v1\r\nModel: x\r\nBoard-ID: {f["boardId"]}\r\nDate: d\r\n' + f'Flash Size: 0x{f["flashSize"]:08X} bytes').encode() + else: + data = ht.make_uf2(bat.fix['image'], f['appStart'], f['family'], self.tmp / 'cur')[1].read_bytes()[:length] + return {'entries': {'CURRENT.UF2': [9, f['flashSize'] // 256 * 512]}, 'data': data, 'size': len(data)} + (self.tmp / 'cur').mkdir() + clock = [0.0] + for p in (mock.patch.object(ht, 'usb_device', lambda *a: '1-2' if self.present[0] else None), + mock.patch.object(ht, 'block_device', lambda p: '/dev/sdz'), + mock.patch.object(ht, 'check_exclusive', lambda d: None), + mock.patch.object(ht, 'fat_read', fat_read), + mock.patch.object(ht, 'spawn', spawn), + mock.patch.object(ht.time, 'sleep', lambda s: clock.__setitem__(0, clock[0] + s)), + mock.patch.object(ht.time, 'monotonic', lambda: clock[0] + 1e-3)): + p.start() + self.addCleanup(p.stop) + + def test_every_step_passes_with_each_probe_session_one_script(self): + self.bat.run(self.files, 'v1') + self.assertEqual([x['step'] for x in self.bat.steps if x['pass']], list(ht.STEPS['stlink'])) + s = self.cube.scripts + self.assertEqual(s[0], ['--connect', 'port=swd', 'sn=0023', '--write', str(self.files['hex']), '--go']) # no --verify + self.assertEqual((s[1][0], s[1][2]), ('-halt', '-rst')) + self.assertRegex(s[1][1], r'^-u 0x08000000 0x5824 \S+readback-\d+\.bin$') + self.assertEqual(self.bat.steps[0]['evidence'], + {'flashDownload': 'File download complete', 'erasedSectors': ['0', '1'], 'verified': True, + 'operation': {'attempts': [{'kind': 'exited', 'exit': 0}], 'cleanup': 'done'}}) + observe = next(x for x in s if '-coreReg' in x) + self.assertEqual(observe[:5], ['-halt', '-run', f'#Delay({2 * self.bat.f["dblTapDelayMs"] + 200})', '-halt', '-coreReg']) + self.assertEqual(observe.count('-coreReg'), 2) + self.assertIn([f'-w32 0x{ht.SCB_AIRCR:08X} 0x{ht.AIRCR_SYSRESETREQ:08X} -nv'], s) + ev = self.bat.steps[-1]['evidence'] + self.assertTrue(ev['dfuCameUp'] and ev['magicConsumed'] and ev['appIntact']) + + def test_an_install_whose_bin_does_not_read_back_fails(self): + self.cube.flash[self.bat.f['binBase']] = b'\xff' * 0x5824 + with self.assertRaisesRegex(ht.StepFail, 'does not read back'): + self.bat.run(self.files, 'v1') + self.assertEqual(self.bat.steps[-1]['step'], 'install') + + def test_an_app_image_that_does_not_read_back_fails_app_ran(self): + spawn = ht.spawn + + def corrupt(cmd, *a, **k): + out = spawn(cmd, *a, **k) + if cmd[0] == 'mcopy': + self.cube.flash[self.bat.f['appStart']] = bytes(len(self.bat.fix['image'])) + return out + with mock.patch.object(ht, 'spawn', corrupt), self.assertRaisesRegex(ht.StepFail, 'fixture image does not read back'): + self.bat.run(self.files, 'v1') + self.assertEqual(self.bat.steps[-1]['step'], 'app-ran') + + def test_a_reset_write_not_reported_done_fails(self): + run = self.cube.run + self.cube.run = lambda script: (lambda r: (r[0], r[1].replace('w @0xe000ed0c: 0x05fa0004', 'x') + if '-nv' in Path(script).read_text() else r[1]))(run(script)) + with self.assertRaisesRegex(ht.StepFail, 'reset write was not reported done'): + self.bat.sysreset() + + +H563_HEADER = 'lib/mcu/st/cmsis_device_h5/Include/stm32h563xx.h' + + +class StlinkSourceFacts(unittest.TestCase): + def test_stm32h563_nucleo_facts_come_from_the_source_and_the_link_defsyms(self): + with mock.patch.object(ht, 'vector_count', return_value=H563_FACTS['vectors']) as count: + f = ht.source_facts(H563) + count.assert_called_once_with(H563_HEADER) + # __ram_size 640K: _board_dfu_dbl_tap = _estack, the last word of SRAM3 + self.assertEqual(f, H563_FACTS) + + @unittest.skipUnless((ROOT / H563_HEADER).is_file(), f'{H563_HEADER} not fetched (tools/get_deps.py stm32h5)') + def test_stm32h563_vector_count_comes_from_its_cmsis_header(self): + self.assertEqual(ht.vector_count(H563_HEADER), H563_FACTS['vectors']) + + def test_a_change_to_how_the_defsyms_reach_the_link_is_refused(self): + files = {k: (ROOT / k).read_text() for k in ('ports/stm32h5/family.cmake', 'ports/stm32h5/linker/stm32h5_boot.ld', + 'ports/stm32h5/boards/stm32h563_nucleo/board.cmake')} + fam, ld, _ = files + + def defsyms(**edits): + t = tempfile.TemporaryDirectory() + self.addCleanup(t.cleanup) + for rel, text in files.items(): + (Path(t.name) / rel).parent.mkdir(parents=True, exist_ok=True) + (Path(t.name) / rel).write_text(edits.get(rel, lambda x: x)(text)) + with mock.patch.object(ht, 'ROOT', Path(t.name)), mock.patch.object(ht.cb, 'ROOT', Path(t.name)): + return ht.defsyms('stm32h5', 'ports/stm32h5/boards/stm32h563_nucleo', 'ports/stm32h5/linker/stm32h5_boot.ld') + self.assertEqual(defsyms(), {'__flash_boot_size': 32 * 1024, '__ram_size': 640 * 1024}) + ram = '-Wl,--defsym=__ram_size=${LD_RAM_SIZE}' + for what, rel, edit in ( + ('another defsym', fam, lambda x: x.replace(ram, '-Wl,--defsym=__ram_size=64K')), + ('an extra defsym', fam, lambda x: x.replace(ram, ram + ' -Wl,--defsym=__x=1')), + ('an unconditional set', fam, lambda x: x + '\nset(LD_RAM_SIZE 64K)\n'), + ('no script default', ld, lambda x: x.replace('__ram_size = DEFINED(__ram_size) ? __ram_size : 32K;', '__ram_size = 32K;')), + ('a second assignment', ld, lambda x: x + '\n__flash_boot_size += 1K;\n')): + with self.subTest(what), self.assertRaisesRegex(ht.Refused, 'does not reach'): + defsyms(**{rel: edit}) + + def test_flash_size_from_the_cmsis_register_read_is_the_roster_device_flash(self): + with self.assertRaisesRegex(ht.Refused, 'BOARD_FLASH_SIZE'): + ht.source_facts(dict(H563, device='unknown')) + + +class StlinkRoster(Tmp): + def test_an_stlink_entry_needs_a_device_with_limits_and_a_chip_identity(self): + def entry(device): + cfg = self.tmp / 'local.json' + cfg.write_text(json.dumps({'boards': [{'name': 'stm32h563_nucleo', 'uid': 'U', 'flasher': { + 'name': 'stlink', 'uid': '0023', 'args': f'-device {device}'}}]})) + return cfg + b = self.load(entry('stm32h563zi'), 'stm32h563_nucleo') + self.assertEqual((b['flasher'], b['install'], b['app']), ('stlink', (0x08000000, 0x08008000), (0x08008000, 0x08200000))) + with mock.patch.dict(ht.FLASH_LIMITS, {'stm32h503rb': (0x08000000, 0x08020000)}): + for device, err in (('stm32h503rb', 'no chip identity'), ('stm32h573ii', 'no programming limits')): + with self.subTest(device), self.assertRaisesRegex(ht.Refused, err): + ht.load_board(entry(device), 'stm32h563_nucleo') + if __name__ == '__main__': unittest.main() diff --git a/.claude/test/test_hil_ci.py b/.claude/test/test_hil_ci.py new file mode 100644 index 000000000..065d873d7 --- /dev/null +++ b/.claude/test/test_hil_ci.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# The CI job's orchestration of the battery (hil_ci.py): record, one run per board, verdicts, markers, pruning +# and the exit. check_build is stubbed; the battery is a stand-in script that prints a canned report into the run +# root it is given, so the streaming, the run directory and the in-process holder path run for real. +import io +import json +import os +import sys +import tempfile +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / '.claude' / 'skills' / 'hil-contract' / 'scripts')) +import hil_ci # noqa: E402 +import hil_test as ht # noqa: E402 + +BOARD = 'imxrt1064_evk' + +STAND_IN = ''' +import json, os, sys, time +one = json.load(open(os.environ["HIL_STAND_IN"])) +run_dir = os.path.join(sys.argv[sys.argv.index("--run-root") + 1], "run-1") +os.makedirs(run_dir) +open(os.path.join(run_dir, "hil.log"), "w").write("log\\n") +report = {"pass": one["row"].get("pass", False), "caveat": "", "results": [dict(board=sys.argv[sys.argv.index("-b") + 1], **one["row"])], + "reportPath": os.path.join(run_dir, "hil_report.json")} +open(report["reportPath"], "w").write(json.dumps(report)) +print("noise", " ".join(sys.argv[1:])) +print(json.dumps(report), flush=True) +if one.get("hang"): + open(os.environ["HIL_STAND_IN"] + ".pid", "w").write(str(os.getpid())) + os.close(1); time.sleep(30) +sys.exit(one.get("exit", 0 if report["pass"] else 1)) +''' + + +def completed(stdout='', code=0, stderr=''): + return mock.Mock(returncode=code, stdout=stdout, stderr=stderr) + + +class Stubbed(unittest.TestCase): + def setUp(self): + t = tempfile.TemporaryDirectory() + self.addCleanup(t.cleanup) + self.tmp = Path(t.name) + self.state = self.tmp / 'state' + self.out = self.state / 'run-1' + self.roster = self.tmp / 'ci.json' + self.roster.write_text(json.dumps({'boards': [{'name': BOARD}]})) + self.stand_in = self.tmp / 'hil_test.py' + self.stand_in.write_text(STAND_IN) + self.spec = self.tmp / 'spec.json' + self.build_ok = True + self.commands = [] + mock.patch.object(hil_ci, 'HIL_TEST', self.stand_in).start() + mock.patch.object(hil_ci, 'ROOT', self.tmp).start() + mock.patch.object(hil_ci, 'run', self.fake_run).start() + mock.patch.dict(os.environ, {'HIL_STAND_IN': str(self.spec)}).start() + self.addCleanup(mock.patch.stopall) + self.addCleanup(self.reap_hung_stand_in) + + def reap_hung_stand_in(self): + """A stand-in left hanging as the in-process holder would be a stale hil_test.py on the host.""" + pid_file = self.spec.with_suffix('.json.pid') + if pid_file.exists(): + pid = int(pid_file.read_text()) + os.kill(pid, 9) + os.waitpid(pid, 0) + + def fake_run(self, cmd): + cmd = [str(c) for c in cmd] + self.commands.append(cmd) + assert cmd[1].endswith('check_build.py'), cmd + row = {'board': BOARD, 'status': 'ok' if self.build_ok else 'failed', 'firstError': '' if self.build_ok else 'boom'} + return completed(json.dumps({'pass': self.build_ok, 'boards': [row]})) + + def main(self, *argv, battery=None, out=None): + self.spec.write_text(json.dumps(battery or {})) + with mock.patch.object(sys, 'stdout', io.StringIO()): + return hil_ci.main(['--roster', 'ci.json', '--out', str(out or self.out), *argv]) + + def rows(self, out=None): + return json.loads(((out or self.out) / 'summary.json').read_text())['boards'] + + def test_a_passing_battery_is_a_pass_with_its_run_directory_and_output_kept(self): + ok = {'row': {'ran': True, 'pass': True, 'cleanup': {'pass': True, 'state': 'blinky copied'}}} + self.assertEqual(self.main('--board', BOARD, '--lock-wait', '7', battery=ok), 0) + self.assertEqual(self.rows()[0]['verdict'], 'pass') + where = self.out / BOARD + self.assertTrue(all((where / f).is_file() for f in ('stdout.txt', 'stderr.txt', 'run-1/hil_report.json', 'run-1/hil.log'))) + argv = (where / 'stdout.txt').read_text().splitlines()[0].split() + self.assertEqual(argv[argv.index('--lock-wait') + 1], '7') + self.assertTrue((self.out / f'{BOARD}.started').exists() and (self.out / f'{BOARD}.done').exists()) + self.assertTrue((self.out / f'{BOARD}.build.json').is_file()) + + def test_a_failed_step_is_a_fail(self): + failed = {'row': {'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': True}, + {'step': 'dfu', 'pass': False, 'detail': 'not seen'}], + 'cleanup': {'pass': True, 'state': 'blinky copied'}}} + self.assertEqual(self.main('--board', BOARD, battery=failed), 1) + r = self.rows()[0] + self.assertEqual((r['verdict'], r['detail']), ('fail', 'dfu: not seen')) + self.assertTrue((self.out / f'{BOARD}.done').exists()) # finished, with a report and no holder + + def test_a_board_held_past_the_wait_is_not_run_and_a_cleanup_holder_a_blocker(self): + for blocker, want in ((False, 'not-run'), (True, 'blocker')): + with self.subTest(want): + out = self.state / want + locked = {'row': {'ran': False, 'locked': True, 'recoveryBlocker': blocker, 'detail': 'board locked: pid 7'}} + self.assertEqual(self.main('--board', BOARD, battery=locked, out=out), 1 if want == 'not-run' else 2) + self.assertEqual(self.rows(out)[0]['verdict'], want) + self.assertEqual((out / f'{BOARD}.done').exists(), want == 'not-run') + + def test_a_failed_cleanup_is_a_blocker_that_keeps_its_marker_and_blocks_the_next_run(self): + held = {'row': {'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': True}], + 'cleanup': {'pass': False, 'state': 'blinky copied', 'heldAfter': True, 'holder': {'pid': 9}}}, + 'exit': 4} + self.assertEqual(self.main('--board', BOARD, battery=held), 2) + self.assertTrue((self.out / f'{BOARD}.started').exists()) + self.assertFalse((self.out / f'{BOARD}.done').exists()) + later = self.state / 'run-2' + self.assertEqual(self.main('--board', BOARD, battery=held, out=later), 2) + r = self.rows(later)[0] + self.assertIn('run-1', r['detail']) + self.assertFalse((later / BOARD).exists()) # no battery ran + + def test_a_holder_kept_in_process_is_not_waited_for(self): + held = {'row': {'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': True}], + 'cleanup': {'pass': False, 'state': 'blinky copied', 'heldAfter': True, + 'holder': {'pid': 9, 'inProcess': True}}}, + 'hang': True} + t0 = time.monotonic() + self.assertEqual(self.main('--board', BOARD, battery=held), 2) + self.assertLess(time.monotonic() - t0, 20) + self.assertEqual(self.rows()[0]['verdict'], 'blocker') + self.assertFalse((self.out / f'{BOARD}.done').exists()) + + def test_a_run_without_a_report_is_a_blocker(self): + self.assertEqual(hil_ci.verdict(137, None, BOARD)[0], 'blocker') + self.assertEqual(hil_ci.verdict(1, {'results': []}, BOARD)[0], 'blocker') + self.assertEqual(hil_ci.verdict(1, {'results': [{'board': BOARD, 'ran': None, 'pass': False}]}, BOARD)[0], 'blocker') + + def test_a_refusal_before_hardware_and_an_untouched_failure_are_not_run_and_a_used_probe_a_blocker(self): + self.assertEqual(hil_ci.verdict(2, {'results': [{'board': BOARD, 'ran': False, 'detail': 'no probe'}]}, BOARD), + ('not-run', 'no probe')) + self.assertEqual(hil_ci.verdict(2, {'results': [{'board': BOARD, 'ran': False, 'probeUsers': [5], 'detail': 'p'}]}, BOARD), + ('blocker', 'p')) + row = {'board': BOARD, 'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': False, 'detail': 'x'}], + 'cleanup': {'pass': True, 'state': 'untouched'}} + self.assertEqual(hil_ci.verdict(1, {'results': [row]}, BOARD)[0], 'not-run') + row['steps'] = [] + self.assertEqual(hil_ci.verdict(1, {'results': [row]}, BOARD)[0], 'blocker') # an abort + + def test_a_failed_build_is_not_run_and_touches_nothing(self): + self.build_ok = False + self.assertEqual(self.main('--board', BOARD), 1) + r = self.rows()[0] + self.assertEqual(r['verdict'], 'not-run') + self.assertIn('boom', r['detail']) + self.assertFalse((self.out / f'{BOARD}.started').exists()) + + def test_a_board_outside_the_roster_and_a_bad_step_are_usage_errors(self): + with self.assertRaises(SystemExit), mock.patch.object(sys, 'stderr', io.StringIO()): + self.main('--board', BOARD, 'frdm_k64f') + with self.assertRaises(SystemExit), mock.patch.object(sys, 'stderr', io.StringIO()): + self.main('--board', BOARD, '--fail-at', 'rm-rf') + + def test_the_fail_at_steps_are_the_runners(self): + self.assertEqual(hil_ci.STEPS, (*ht.STEPS['esp'], 'cleanup')) + + def test_fail_at_reaches_the_battery(self): + held = {'row': {'ran': True, 'pass': False, 'steps': [{'step': 'install', 'pass': True}], + 'cleanup': {'pass': False, 'state': 'blinky copied', 'heldAfter': True}}, 'exit': 4} + self.assertEqual(self.main('--board', BOARD, '--fail-at', 'cleanup', battery=held), 2) + self.assertIn('--fail-at cleanup', (self.out / BOARD / 'stdout.txt').read_text()) + + def test_old_finished_runs_are_pruned_and_an_unfinished_one_is_kept_whatever_its_age(self): + old = time.time() - 40 * 86400 + for name, done in (('run-old-done', True), ('run-old-held', False), ('run-new-held', False)): + d = self.state / name + d.mkdir(parents=True) + (d / f'{BOARD}.started').touch() + if done: + (d / f'{BOARD}.done').touch() + if 'old' in name: + os.utime(d, (old, old)) + hil_ci.prune(self.state, keep=self.out) + self.assertEqual(sorted(p.name for p in self.state.iterdir()), ['run-new-held', 'run-old-held']) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1aaf2a473..8c7d947e3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,10 +22,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 - # the claude-tests hook reads lib/uf2's uf2conv.py, the RT1062 SDK source and the STM32F303 and STM32F411 CMSIS headers + # the claude-tests hook reads lib/uf2's uf2conv.py, lib/agentrc's board-lock helper, the RT1062 SDK source and the + # STM32F303 and STM32F411 CMSIS headers - name: Fetch test inputs run: | - git submodule update --init lib/uf2 + git submodule update --init --depth 1 lib/uf2 lib/agentrc python3 tools/get_deps.py mimxrt10xx stm32f3 stm32f4 - name: Run pre-commit @@ -93,6 +94,7 @@ jobs: - 'lpc55' - 'maxim' - 'mimxrt10xx' + - 'samd51' - 'stm32f3' - 'stm32f4' - 'stm32h5' diff --git a/.github/workflows/codeql-buildscript.sh b/.github/workflows/codeql-buildscript.sh index eb1acde15..636f957fa 100644 --- a/.github/workflows/codeql-buildscript.sh +++ b/.github/workflows/codeql-buildscript.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +set -e cd ports/stm32f4 make BOARD=feather_stm32f405_express get-deps diff --git a/.github/workflows/hil.yml b/.github/workflows/hil.yml new file mode 100644 index 000000000..012c0f4e6 --- /dev/null +++ b/.github/workflows/hil.yml @@ -0,0 +1,102 @@ +name: HIL + +# The battery on the ci.lan rig (.claude/skills/hil-contract/SKILL.md, roster test/hil/ci.json) for the boards a +# pull request's change reaches, built on the rig from its clean checkout. A pull request from a fork never +# reaches the rig. A dispatch runs named boards, with fail_at for the contract's acceptance only. +on: + pull_request: + workflow_dispatch: + inputs: + board: + description: roster board(s), space separated + required: true + fail_at: + description: 'acceptance only: fail this battery step or the cleanup (the contract, Execution)' + required: false + default: '' + +# never cancelled: a battery killed mid-run leaves the board for recovery, so a newer push waits its turn +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: false + +jobs: + select: + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || github.event.pull_request.head.repo.full_name == github.repository + outputs: + boards: ${{ steps.select.outputs.boards }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + + # HEAD is the merge commit, HEAD^1 the base it merges into; what no ci.json board can run is reported here, + # so a skipped rig job never hides it. A dispatch names boards, each checked against the roster. + - name: Select the boards the change reaches + id: select + env: + BOARD_INPUT: ${{ github.event.inputs.board }} + shell: bash # -o pipefail: a failed selection must not become an empty, skipped rig job through tee + run: | + python3 - <<'EOF' | tee -a "$GITHUB_STEP_SUMMARY" + import json, os, subprocess, sys + roster = {b['name'] for b in json.load(open('test/hil/ci.json'))['boards']} + if os.environ.get('GITHUB_EVENT_NAME') == 'workflow_dispatch': + boards = sorted(set(os.environ['BOARD_INPUT'].split())) + if not boards or set(boards) - roster: + sys.exit(f'not in test/hil/ci.json: {sorted(set(boards) - roster) or "no board"}') + uncovered = [] + else: + sel = subprocess.run([sys.executable, '.claude/skills/build/scripts/check_build.py', '--select-only', + '--base', 'HEAD^1', '--hil-config', 'test/hil/ci.json'], capture_output=True, text=True) + if sel.returncode: + sys.exit(f'selection failed (exit {sel.returncode}): {sel.stdout or sel.stderr}') + hil = json.loads(sel.stdout)['hil'] + boards, uncovered = hil['boards'], hil['uncovered'] + print('boards:', ' '.join(boards) or 'none') + for u in uncovered: + print(f'- `{u["target"]}`: {u["reason"]}') + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write(f'boards={" ".join(boards)}\n') + EOF + + ci-lan: + needs: select + if: needs.select.outputs.boards != '' + runs-on: [ self-hosted, Linux, X64, tinyuf2 ] + # one board is a ~2 min build and a ~2 min battery; a locked board waits up to hil_ci.py's --lock-wait + timeout-minutes: 45 + env: + BOARDS: ${{ needs.select.outputs.boards }} + FAIL_AT: ${{ github.event.inputs.fail_at }} + steps: + # kept across runs and attempts, outside the workspace: records, every battery's run directory with its report, + # log and output, and the markers that make a run which ended without a report a blocker for the next (hil_ci.py) + - name: Set the run directory + run: echo "HIL_OUT=$(dirname "$GITHUB_WORKSPACE")/hil-state/${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_ENV" + + - name: Clean workspace + run: | + rm -rf "${{ github.workspace }}" + mkdir -p "${{ github.workspace }}" + + - uses: actions/checkout@v4 + + - name: Fetch the build inputs + run: | + git submodule update --init --depth 1 lib/tinyusb lib/uf2 lib/agentrc + for b in $BOARDS; do python3 tools/get_deps.py -b "$b"; done + + - name: Battery on the rig + run: | + python3 .claude/skills/hil-contract/scripts/hil_ci.py --board $BOARDS ${FAIL_AT:+--fail-at "$FAIL_AT"} \ + --roster test/hil/ci.json --out "$HIL_OUT" + + - name: Upload the records, reports and logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: hil-ci-lan-${{ github.run_attempt }} + path: ${{ env.HIL_OUT }} + if-no-files-found: ignore diff --git a/.gitmodules b/.gitmodules index 2c8e665f3..f3ed7d534 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "lib/uf2"] path = lib/uf2 url = https://github.com/microsoft/uf2.git +[submodule "lib/agentrc"] + path = lib/agentrc + url = https://github.com/hathach/agentrc.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 052046b56..859588686 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,4 +37,7 @@ repos: entry: python3 -m unittest discover -s .claude/test -p 'test_*.py' language: system pass_filenames: false - files: ^(\.claude/|\.github/workflows/|ports/|test/hil/|tools/get_deps\.py$) + # a gitlink is tagged a directory, which the default types: [file] drops + files: ^(\.claude/|\.github/workflows/|ports/|test/hil/|tools/get_deps\.py$|lib/agentrc$) + types: [] + types_or: [file, directory] diff --git a/CLAUDE.md b/CLAUDE.md index ac7a94ef7..2861482fa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ ## Working Rules -- Worktrees — `git submodule update --init lib/tinyusb lib/uf2`, then symlink the `deps_all` paths from `tools/get_deps.py` (`lib/mcu/...`, `lib/sct_neopixel`) to the primary checkout when it holds the pinned revisions. Never run `get_deps.py` through a symlink: it `git reset --hard`s each dependency in place. Where the primary's copy has local changes, use a detached `git worktree` of that dependency at its pin instead. +- Worktrees — `git submodule update --init lib/tinyusb lib/uf2 lib/agentrc`, then symlink the `deps_all` paths from `tools/get_deps.py` (`lib/mcu/...`, `lib/sct_neopixel`) to the primary checkout when it holds the pinned revisions. `get_deps.py` refuses to update through a symlink; fetch the pinned revision in the primary checkout before linking it. Where the primary's copy has local changes or another revision, use a detached `git worktree` of that dependency at its pin instead, outside the checkout and symlinked in. - Code references — ports and boards: `ports//` (`boards//`, `board_flash.c`, `boards.c`), `supported_boards.md`; bootloader: `src/` (`main.c`, `ghostfat.c`, `msc.c`, `board_api.h`, the contract every port implements); apps: `apps/`, `ports//apps/`; build: `ports/make.mk`, `ports/rules.mk`, `ports/family_support.cmake`, `.github/workflows/build.yml` (which backend builds each port); deps: `tools/get_deps.py`. ## Code Rules @@ -21,7 +21,7 @@ ## Build and Validate - Build contract: `.claude/skills/build/SKILL.md`. It resolves a change to boards and builds each the way CI does, outside the source tree. -- HIL contract: `.claude/skills/hil/SKILL.md`. Hardware validation comes from its report, never from a build; what no roster board on the host covers is reported pending. +- HIL contract: `.claude/skills/hil-contract/SKILL.md`. Hardware validation comes from its report, never from a build; what no roster board on the host covers is reported pending. - ESP-IDF: the build contract uses CI's docker image; for interactive work, `. "$IDF_PATH/export.sh"` and `idf.py -C ports/espressif -DBOARD= flash monitor`. - Before submitting, and again after a rebase: `pre-commit run --all-files`. - For code changes: `python3 .claude/skills/build/scripts/check_build.py --base master --worktree`. diff --git a/README.md b/README.md index b16c6afa2..845cef384 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ Not all features are implemented for all MCUs, following is supported MCUs and i | K32L2 | ✔ | ✔ | | | | | | LPC55 | ✔ | ✔ | | | ✔ | | | iMXRT | ✔ | ✔ | ✔ | | ✔ | | +| SAMD51 | ✔ | | ✔ | ✔ | ✔ | | | STM32F3 | ✔ | ✔ | ✔ | ✔ | ✔ | | | STM32F4 | ✔ | ✔ | ✔ | ✔ | ✔ | | | STM32H5 | ✔ | ✔ | ✔ | ✔ | | | diff --git a/apps/dfu_runtime/src/tusb_config.h b/apps/dfu_runtime/src/tusb_config.h index a516f9e58..bfe135a65 100644 --- a/apps/dfu_runtime/src/tusb_config.h +++ b/apps/dfu_runtime/src/tusb_config.h @@ -43,6 +43,11 @@ #define CFG_TUD_ENABLED 1 +#if CFG_TUSB_MCU == OPT_MCU_ESP32P4 && BOARD_TUD_RHPORT == 0 +// rhport 0 is the P4's OTG FS controller +#define CFG_TUD_MAX_SPEED OPT_MODE_FULL_SPEED +#endif + #ifndef CFG_TUSB_MEM_SECTION #define CFG_TUSB_MEM_SECTION #endif diff --git a/lib/agentrc b/lib/agentrc new file mode 160000 index 000000000..330f58da3 --- /dev/null +++ b/lib/agentrc @@ -0,0 +1 @@ +Subproject commit 330f58da350dbaf00fee7b5875ad695c94fef81c diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index b65267440..587a43a60 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -34,6 +34,9 @@ cmake_print_variables(GIT_VERSION GIT_SUBMODULE_VERSIONS) project(tinyuf2) +# The post-build merge and self-update source need every flash image first. +add_dependencies(app bootloader partition_table_bin blank_ota_data) + # Create post-build script for combined.bin / combined-ota.bin file(WRITE ${CMAKE_BINARY_DIR}/tinyuf2.postbuild.sh "#!/bin/bash\n" diff --git a/ports/espressif/README.md b/ports/espressif/README.md index be110af5f..16c326d19 100644 --- a/ports/espressif/README.md +++ b/ports/espressif/README.md @@ -8,6 +8,8 @@ The project is composed of customizing the 2nd stage bootloader from IDF and UF2 See the board list for this family in [supported_boards.md](../../supported_boards.md#espressif). +See [Metro ESP32-P4](boards/adafruit_metro_esp32p4/README.md) for its USB routing and ADC double-reset hardware requirements. + ## Build & Flash ### Build @@ -43,6 +45,7 @@ There are a few ways to enter UF2 mode: - Holding `GPIO0` then reset -> ROM bootloader - Press reset, see indicator on (purple RGB) then press `GPIO0` -> UF2 bootloader - `PIN_DOUBLE_RESET_RC` GPIO is attached to an 100K resistor and 1uF Capacitor to serve as 1-bit memory, which hold the pin value long enough for double reset detection. Simply press double reset to enter UF2 +- `PIN_DOUBLE_RESET_ADC` uses a calibrated ADC reading of an RC marker. Metro ESP32-P4 shares this with its battery divider; it requires the capacitor change described in the board notes above. - Request by application using [system reset reason](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/system/system.html?highlight=esp_reset_reason#reset-reason) with hint of `0x11F2`. Reset reason hint is different than hardware reset source, it is written to RTC's store6 register and hold value through a software reset. Since Espressif only uses an dozen of value in `esp_reset_reason_t`, it is safe to hijack and use *0x11F2* as reset reason to enter UF2 using following snippet. ``` #include "esp_private/system_internal.h" @@ -60,7 +63,7 @@ There are a few ways to enter UF2 mode: ## Convert Binary to UF2 -To create your own UF2 file, simply use the [Python conversion script](https://github.com/Microsoft/uf2/blob/master/utils/uf2conv.py) on a .bin file, specifying the family id as `ESP32S2`, `ESP32S3` or their magic number as follows. Note you must specify application address of 0x00 with the -b switch, the bootloader will use it as offset to write to ota partition. +To create your own UF2 file, simply use the [Python conversion script](https://github.com/Microsoft/uf2/blob/master/utils/uf2conv.py) on a .bin file, specifying the family id as `ESP32S2`, `ESP32S3`, `ESP32P4` or their magic number as follows. Note you must specify application address of 0x00 with the -b switch, the bootloader will use it as offset to write to ota partition. ``` uf2conv.py firmware.bin -c -b 0x00 -f ESP32S2 diff --git a/ports/espressif/apps/blinky/sdkconfig.defaults b/ports/espressif/apps/blinky/sdkconfig.defaults index fb415e80c..2971bbf80 100644 --- a/ports/espressif/apps/blinky/sdkconfig.defaults +++ b/ports/espressif/apps/blinky/sdkconfig.defaults @@ -1,5 +1,4 @@ CONFIG_IDF_CMAKE=y -CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y CONFIG_FREERTOS_SUPPORT_STATIC_ALLOCATION=y # Compiler options diff --git a/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt b/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt index 10837a626..b48648057 100644 --- a/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt +++ b/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt @@ -16,7 +16,7 @@ idf_component_register(SRCS ${tusb_src}/portable/synopsys/dwc2/dcd_dwc2.c ${tusb_src}/portable/synopsys/dwc2/dwc2_common.c INCLUDE_DIRS ${app_src} ${tusb_src} ${TOP}/src ${FAMILY_PATH}/boards ${BOARD_PATH} - PRIV_REQUIRES esp_timer esp_partition app_update) + PRIV_REQUIRES esp_timer esp_partition app_update esp_hal_usb) target_compile_definitions(${COMPONENT_LIB} PRIVATE CFG_TUSB_MCU=OPT_MCU_${target_upper} diff --git a/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c index 345e68275..5420f1a95 100644 --- a/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c +++ b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c @@ -30,11 +30,18 @@ #include "esp_private/system_internal.h" #include "esp_private/usb_phy.h" #include "esp_rom_sys.h" +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" #include "board_api.h" #include "dfu_runtime.h" #include "tusb.h" +#if BOARD_USB_FS_PHY_SWAP +#include "hal/usb_serial_jtag_ll.h" +#include "hal/usb_wrap_ll.h" +#endif + // Espressif glue for the DFU runtime example, without the bootloader's boards component // ports/espressif/bootloader_components/main/bootloader_start.c APP_REQUEST_UF2_RESET_HINT @@ -70,6 +77,10 @@ void board_timer_stop(void) { void board_usb_init(void) { // same PHY setup as TinyUF2's boards.c board_dfu_init() +#if BOARD_USB_FS_PHY_SWAP + usb_serial_jtag_ll_phy_enable_pad(false); + vTaskDelay(pdMS_TO_TICKS(100)); +#endif usb_phy_config_t const phy_conf = { .controller = USB_PHY_CTRL_OTG, #if BOARD_TUD_RHPORT == 1 || CONFIG_IDF_TARGET_ESP32S31 @@ -78,10 +89,17 @@ void board_usb_init(void) { .target = USB_PHY_TARGET_INT, #endif .otg_mode = USB_OTG_MODE_DEVICE, +#if CONFIG_IDF_TARGET_ESP32P4 && BOARD_TUD_RHPORT == 0 + .otg_speed = USB_PHY_SPEED_FULL, +#else // https://github.com/hathach/tinyusb/issues/2943#issuecomment-2601888322 .otg_speed = USB_PHY_SPEED_UNDEFINED, +#endif }; ESP_ERROR_CHECK(usb_new_phy(&phy_conf, &phy_hdl)); +#if BOARD_USB_FS_PHY_SWAP + usb_wrap_ll_phy_select(&USB_WRAP, 0); +#endif } uint8_t board_usb_get_serial(uint8_t serial_id[16]) { diff --git a/ports/espressif/apps/dfu_runtime/sdkconfig.defaults b/ports/espressif/apps/dfu_runtime/sdkconfig.defaults index fb415e80c..2971bbf80 100644 --- a/ports/espressif/apps/dfu_runtime/sdkconfig.defaults +++ b/ports/espressif/apps/dfu_runtime/sdkconfig.defaults @@ -1,5 +1,4 @@ CONFIG_IDF_CMAKE=y -CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y CONFIG_FREERTOS_SUPPORT_STATIC_ALLOCATION=y # Compiler options diff --git a/ports/espressif/apps/erase_firmware/sdkconfig.defaults b/ports/espressif/apps/erase_firmware/sdkconfig.defaults index fb415e80c..2971bbf80 100644 --- a/ports/espressif/apps/erase_firmware/sdkconfig.defaults +++ b/ports/espressif/apps/erase_firmware/sdkconfig.defaults @@ -1,5 +1,4 @@ CONFIG_IDF_CMAKE=y -CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y CONFIG_FREERTOS_SUPPORT_STATIC_ALLOCATION=y # Compiler options diff --git a/ports/espressif/apps/update_tinyuf2/sdkconfig.defaults b/ports/espressif/apps/update_tinyuf2/sdkconfig.defaults index fb415e80c..2971bbf80 100644 --- a/ports/espressif/apps/update_tinyuf2/sdkconfig.defaults +++ b/ports/espressif/apps/update_tinyuf2/sdkconfig.defaults @@ -1,5 +1,4 @@ CONFIG_IDF_CMAKE=y -CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y CONFIG_FREERTOS_SUPPORT_STATIC_ALLOCATION=y # Compiler options diff --git a/ports/espressif/boards/adafruit_metro_esp32p4/README.md b/ports/espressif/boards/adafruit_metro_esp32p4/README.md new file mode 100644 index 000000000..1d4091561 --- /dev/null +++ b/ports/espressif/boards/adafruit_metro_esp32p4/README.md @@ -0,0 +1,109 @@ +# Adafruit Metro ESP32-P4 + +This port uses the Metro's 16 MB external flash and USB-C connector. TinyUSB's +full-speed OTG controller is routed to GPIO24/25, normally used by Serial/JTAG. +The USB-A high-speed port is not used. The bootloader drive is `METROP4BOOT` +(VID:PID `239A:017B`). Logs use UART0 so USB logging cannot stall the bootloader. + +## Double reset hardware + +Replace **CBATRAW with 2.2 µF**. Keep the existing 1 MΩ / 1 MΩ battery divider +on **GPIO18 / ADC1 channel 2**. The original 0.1 µF capacitor is too small for +the intended reset timing. This firmware does not change the PCB files. + +On an external reset, the second-stage bootloader reads the node using eFuse +ADC calibration. A voltage above 2.5 V is a reset marker, distinct from the +normal battery reading of at most 2.1 V. Otherwise it drives the node HIGH +during the existing 500 ms detection window. A second reset in that window +leaves charge for the next boot to detect. Afterward it discharges the marker +for 5 ms and releases the pin with both pulls disabled. + +The divider's nominal time constant is 1.1 seconds. From 3.3 V, the capacitor +stays above 2.5 V for approximately 0.31 seconds with VBAT at 0 V, or 0.65–1.21 +seconds with VBAT at 3.0–4.2 V. These are estimates: capacitor tolerance, +effective capacitance, leakage, reset-button hold time, and boot latency need +bench verification. Applications should allow about **5.5 seconds** after +marker discharge before relying on a battery reading (about 1% settling). + +If calibration is unavailable or a conversion times out, the bootloader skips +ADC detection and still supports the BOOT button. The ADC helper preserves +the bootloader's ADC entropy configuration and does not use the application +ADC driver. + +## Entry and recovery + +- Double-tap RESET with the modified capacitor fitted. +- Or tap RESET, then press BOOT while the NeoPixel is purple. +- Hold BOOT while resetting to enter the ROM downloader, independently of + TinyUF2. BOOT is GPIO35 on this board. +- A 1200-baud touch on TinyUF2's CDC interface requests the ROM downloader. +- An application can request TinyUF2 with the existing `0x11F2` reset hint. + +### Bench verification — 2026-10-10 + +Tested with ESP-IDF v5.4.3 on a rev v0.1 P4 with USB power, no LiPo, and a +hand-reworked CBATRAW capacitor. The installed capacitance was not independently +confirmed. + +- An Arduino preflight measured 2.138 V at idle, 3.283 V with the weak pull-up, + and 0.073 V with the weak pull-down. After a 500 ms HIGH drive, the node + remained above 2.5 V for approximately 1.43 seconds. +- TinyUF2 enumerated as `METROP4BOOT` with CDC on Windows. +- Copying the generated blinky UF2 started the application and restored the + Serial/JTAG interface. All 621 UF2 payload blocks subsequently matched + readback through `CURRENT.UF2`. +- A single physical reset left the board in application mode. A quick physical + double-reset returned to `METROP4BOOT`; the user confirmed the green ready + indicator. +- A 1200-baud CDC touch entered the ROM downloader. Esptool connected without + issuing a reset, read the MAC, and requested a normal reset afterward. + +The initial bit-banged reset indicator appeared excessively bright. Replacing +the P4 bootloader transmitter with hardware RMT pulses fixed it: the user +confirmed a dim purple reset window followed by normal orange application +blinking. A subsequent double-reset on the updated build again enumerated +`METROP4BOOT`. Brightness settings were unchanged. The transmitter uses the same +WS2812 timings as the working application and TinyUF2 indicators, with a bounded +timeout and LOW output on completion or failure. It currently supports one +NeoPixel. Other ESP32 targets keep their existing transmitter. + +The full P4 build and a Metro S3 bootloader compile passed. The P4 second-stage +bootloader uses 24,208 bytes of its 24,576-byte allocation. Battery operation +(including VBAT at 3.0 V), cold-start behavior, and detection timing limits remain +unverified. The full original flash image was backed up privately before +installation. + +The subsequent ESP-IDF v6.1 build passes for TinyUF2, blinky, and self-update. +Its second-stage bootloader is 23,872 / 24,576 bytes with error-only boot logs; +TinyUF2 is 254,224 / 262,144 bytes. A Metro S3 bootloader regression compile +passes with the installed v5.4.3 toolchain. + +The v6.1 hardware retest also passed: MSC/CDC enumeration after flashing, UF2 +upload and application startup with Serial/JTAG restored, a dim purple +single-reset window followed by orange blinking, physical double-reset entry, +and 1200-baud ROM recovery verified by an esptool connection without reset. +All 814 blocks of the corrected blinky UF2 matched `CURRENT.UF2` readback. +The demo and self-update applications inherit the board defaults so their +image headers also target rev 0.1; IDF 6.1's default rev 3.1 images cannot run +on this board. Both rebuilt application headers were checked. + +## Build and install + +Use ESP-IDF **v6.1**. The board configuration selects the original P4 silicon +(minimum rev 0.1), not IDF 6.1's default ECO3 target. From `ports/espressif`: + +```sh +idf.py -B build-metro-p4 -DBOARD=adafruit_metro_esp32p4 build +idf.py -B build-metro-p4 -DBOARD=adafruit_metro_esp32p4 -p PORT flash +``` + +The partition layout is `partitions-16MB-ota4MB.csv`: two 4 MB application +slots, TinyUF2 at `0x810000`, and FAT storage at `0x850000`. Initial installation +writes the bootloader and partition table; preserve any existing firmware and +data first. The P4 second-stage bootloader starts at `0x2000`, unlike S2/S3. +Use the generated flash arguments, not another board's offsets. + +Convert an application binary (not a merged flash image) using family +`ESP32P4` / `0x3d308e94` and base address `0`. Applications must use the same +partition layout. The build also produces the standard blinky and TinyUF2 +self-update applications. diff --git a/ports/espressif/boards/adafruit_metro_esp32p4/board.cmake b/ports/espressif/boards/adafruit_metro_esp32p4/board.cmake new file mode 100644 index 000000000..3deaaff03 --- /dev/null +++ b/ports/espressif/boards/adafruit_metro_esp32p4/board.cmake @@ -0,0 +1,3 @@ +set(IDF_TARGET "esp32p4") +# USB-C is on OTG FS +set(BOARD_TUD_RHPORT 0) diff --git a/ports/espressif/boards/adafruit_metro_esp32p4/board.h b/ports/espressif/boards/adafruit_metro_esp32p4/board.h new file mode 100644 index 000000000..5a2484fc5 --- /dev/null +++ b/ports/espressif/boards/adafruit_metro_esp32p4/board.h @@ -0,0 +1,35 @@ +/* + * SPDX-FileCopyrightText: 2026 Adafruit Industries LLC + * SPDX-License-Identifier: MIT + */ +#ifndef ADAFRUIT_METRO_ESP32P4_H_ +#define ADAFRUIT_METRO_ESP32P4_H_ + +#define PIN_BUTTON_UF2 35 + +// BAT_SENSE: 1 Mohm / 1 Mohm divider, with CBATRAW changed to 2.2 uF. +// The bootloader temporarily charges this capacitor above VBAT/2. +#define PIN_DOUBLE_RESET_ADC 18 +#define BOARD_DOUBLE_RESET_ADC_CHANNEL 2 +#define BOARD_DOUBLE_RESET_ADC_THRESHOLD_MV 2500 + +#define NEOPIXEL_PIN 23 +#define NEOPIXEL_BRIGHTNESS 0x10 +#define NEOPIXEL_NUMBER 1 +#define LED_PIN 13 +#define LED_STATE_ON 1 + +// USB-C uses the Serial/JTAG pins, GPIO24/25, with the FS OTG PHY swapped in. +#define BOARD_USB_FS_PHY_SWAP 1 + +#define USB_VID 0x239A +#define USB_PID 0x017B +#define USB_MANUFACTURER "Adafruit" +#define USB_PRODUCT "Metro ESP32-P4" +#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT +#define UF2_BOARD_ID "ESP32P4-Metro-revA" +#define UF2_VOLUME_LABEL "METROP4BOOT" +#define UF2_INDEX_URL "https://www.adafruit.com" +#define TINYUF2_FAVICON_HEADER "favicon_adafruit_256.h" + +#endif diff --git a/ports/espressif/boards/adafruit_metro_esp32p4/sdkconfig b/ports/espressif/boards/adafruit_metro_esp32p4/sdkconfig new file mode 100644 index 000000000..256edb5e8 --- /dev/null +++ b/ports/espressif/boards/adafruit_metro_esp32p4/sdkconfig @@ -0,0 +1,12 @@ +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-16MB-ota4MB.csv" +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_ESPTOOLPY_FLASHMODE_DIO=y +CONFIG_ESP_CONSOLE_UART_DEFAULT=y +CONFIG_ESP_CONSOLE_SECONDARY_NONE=y + +# This Metro revision uses the original P4 silicon, not ECO3. +CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y +CONFIG_ESP32P4_REV_MIN_1=y + +# Keep room for ADC double-reset detection in P4's 24 KB bootloader partition. +CONFIG_BOOTLOADER_LOG_LEVEL_ERROR=y diff --git a/ports/espressif/boards/boards.c b/ports/espressif/boards/boards.c index 66423d7f6..941bd189d 100644 --- a/ports/espressif/boards/boards.c +++ b/ports/espressif/boards/boards.c @@ -50,6 +50,15 @@ #include "esp_rom_sys.h" #endif +#if BOARD_USB_FS_PHY_SWAP +#include "hal/usb_serial_jtag_ll.h" +#include "hal/usb_wrap_ll.h" +#endif + +#if BOARD_USB_FS_PHY_SWAP && !(CONFIG_IDF_TARGET_ESP32P4 && BOARD_TUD_RHPORT == 0) +#error "BOARD_USB_FS_PHY_SWAP needs ESP32-P4 OTG FS (BOARD_TUD_RHPORT 0)" +#endif + #ifndef BUILD_NO_TINYUSB #include "tusb.h" #endif @@ -242,7 +251,7 @@ void board_init(void) { led_strip_config_t strip_config = { .strip_gpio_num = NEOPIXEL_PIN, // The GPIO that connected to the LED strip's data line .max_leds = NEOPIXEL_NUMBER, // The number of LEDs in the strip, - .led_pixel_format = LED_PIXEL_FORMAT_GRB, // Pixel format of your LED strip + .color_component_format = LED_STRIP_COLOR_COMPONENT_FMT_GRB, // Pixel format of your LED strip .led_model = LED_MODEL_WS2812, // LED strip model .flags.invert_out = false, // whether to invert the output signal }; @@ -275,6 +284,11 @@ static void usb_disconnect_shutdown_handler(void) { static usb_phy_handle_t phy_hdl; void board_dfu_init(void) { +#if BOARD_USB_FS_PHY_SWAP + // Disconnect Serial/JTAG before moving USB-C's GPIO24/25 to the FS OTG controller. + usb_serial_jtag_ll_phy_enable_pad(false); + vTaskDelay(pdMS_TO_TICKS(100)); +#endif // Configure USB PHY usb_phy_config_t phy_conf = { .controller = USB_PHY_CTRL_OTG, @@ -284,26 +298,52 @@ void board_dfu_init(void) { .target = USB_PHY_TARGET_INT, #endif .otg_mode = USB_OTG_MODE_DEVICE, +#if CONFIG_IDF_TARGET_ESP32P4 && BOARD_TUD_RHPORT == 0 + // usb_new_phy() moves a P4 device with undefined speed onto the HS UTMI PHY + .otg_speed = USB_PHY_SPEED_FULL, +#else // https://github.com/hathach/tinyusb/issues/2943#issuecomment-2601888322 // Set speed to undefined (auto-detect) to avoid timinng/racing issue with S3 with host such as macOS .otg_speed = USB_PHY_SPEED_UNDEFINED, +#endif }; ESP_ERROR_CHECK(usb_new_phy(&phy_conf, &phy_hdl)); +#if BOARD_USB_FS_PHY_SWAP + // usb_new_phy() leaves OTG FS on FSLS1 (GPIO26/27); FSLS0 is GPIO24/25 + usb_wrap_ll_phy_select(&USB_WRAP, 0); +#endif + #if (CONFIG_IDF_TARGET_ESP32P4 || CONFIG_IDF_TARGET_ESP32S31) && !defined(BUILD_NO_TINYUSB) esp_register_shutdown_handler(usb_disconnect_shutdown_handler); #endif } +#if BOARD_USB_FS_PHY_SWAP && !defined(BUILD_NO_TINYUSB) +static void usb_restore_serial_jtag(void) { + tud_disconnect(); + vTaskDelay(pdMS_TO_TICKS(100)); + usb_del_phy(phy_hdl); + usb_serial_jtag_ll_phy_select(0); + usb_serial_jtag_ll_phy_enable_pad(true); +} +#endif + void board_reset(void) { +#if BOARD_USB_FS_PHY_SWAP && !defined(BUILD_NO_TINYUSB) + // Restore ROM's USB routing before starting an application with Serial/JTAG. + usb_restore_serial_jtag(); + esp_rom_software_reset_system(); +#else esp_restart(); +#endif } void board_dfu_complete(void) { // Set partition OTA0 as bootable and reset esp_ota_set_boot_partition(esp_partition_find_first(ESP_PARTITION_TYPE_APP, ESP_PARTITION_SUBTYPE_APP_OTA_0, NULL)); - esp_restart(); + board_reset(); } bool board_app_valid(void) { @@ -482,6 +522,12 @@ void tud_cdc_line_state_cb(uint8_t instance, bool dtr, bool rts) { if (coding.bit_rate == 1200) { printf("Touch1200: Reset to bootrom\n"); +#if BOARD_USB_FS_PHY_SWAP + // Unlike esp_restart()'s CPU-only reset, this lets ROM reinitialize Serial/JTAG for uploads. + usb_restore_serial_jtag(); + usb_persist_shutdown_handler(); + esp_rom_software_reset_system(); +#else // copied from Arduino's usb_persist_restart() esp_register_shutdown_handler(usb_persist_shutdown_handler); #if CONFIG_IDF_TARGET_ESP32S3 @@ -490,6 +536,7 @@ void tud_cdc_line_state_cb(uint8_t instance, bool dtr, bool rts) { usb_switch_to_cdc_jtag(); #endif esp_restart(); +#endif } } } diff --git a/ports/espressif/boards/espressif_esp32s3_devkitc_1/board.h b/ports/espressif/boards/espressif_esp32s3_devkitc_1/board.h index f63e07c4f..2427f4131 100644 --- a/ports/espressif/boards/espressif_esp32s3_devkitc_1/board.h +++ b/ports/espressif/boards/espressif_esp32s3_devkitc_1/board.h @@ -42,7 +42,7 @@ // LED //--------------------------------------------------------------------+ -// GPIO connected to Neopixel data +// GPIO connected to Neopixel data: GPIO38 on v1.1, GPIO48 on the initial v1.0 board #define NEOPIXEL_PIN 38 // Brightness percentage from 1 to 255 @@ -67,7 +67,7 @@ #define USB_PRODUCT "ESP32S3 DevKitC 1" #define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT -#define UF2_BOARD_ID "ESP32S3-DevKitC-v1.0" +#define UF2_BOARD_ID "ESP32S3-DevKitC-v1.1" #define UF2_VOLUME_LABEL "S3DKC1BOOT" #define UF2_INDEX_URL "https://docs.espressif.com/projects/esp-idf/en/latest/esp32s3/hw-reference/esp32s3/user-guide-devkitc-1.html" diff --git a/ports/espressif/boards/intechstudio_grid/board.cmake b/ports/espressif/boards/intechstudio_grid/board.cmake deleted file mode 100644 index c1426c950..000000000 --- a/ports/espressif/boards/intechstudio_grid/board.cmake +++ /dev/null @@ -1,2 +0,0 @@ -# Apply board specific content here -set(IDF_TARGET "esp32s3") diff --git a/ports/espressif/boards/intechstudio_grid/board.h b/ports/espressif/boards/intechstudio_grid/board.h deleted file mode 100644 index 12745e721..000000000 --- a/ports/espressif/boards/intechstudio_grid/board.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2020 Ha Thach (tinyusb.org) for Adafruit Industries - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef INTECHSTUDIO_GRID_H_ -#define INTECHSTUDIO_GRID_H_ - -//--------------------------------------------------------------------+ -// Button -//--------------------------------------------------------------------+ - -// Enter UF2 mode if GPIO is pressed while 2nd stage bootloader indicator -// is on e.g RGB = Purple. If it is GPIO0, user should not hold this while -// reset since that will instead run the 1st stage ROM bootloader -#define PIN_BUTTON_UF2 38 -#define UF2_DETECTION_DELAY_MS 0 - -// GPIO that implement 1-bit memory with RC components which hold the -// pin value long enough for double reset detection. -// #define PIN_DOUBLE_RESET_RC 47 - -//--------------------------------------------------------------------+ -// LED -//--------------------------------------------------------------------+ - -// GPIO connected to Neopixel data -#define NEOPIXEL_PIN 21 - -// Brightness percentage from 1 to 255 -#define NEOPIXEL_BRIGHTNESS 0x64 - -// Number of neopixels -#define NEOPIXEL_NUMBER 21 - -//--------------------------------------------------------------------+ -// USB UF2 -//--------------------------------------------------------------------+ - -#define USB_VID 0x303A -#define USB_PID 0x8122 -#define USB_MANUFACTURER "Intech Studio" -#define USB_PRODUCT "Grid" - -#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT -#define UF2_BOARD_ID "ESP32S3-Grid" -#define UF2_VOLUME_LABEL "GRID-S3" -#define UF2_INDEX_URL "https://github.com/intechstudio/grid-fw" - -#endif diff --git a/ports/espressif/boards/intechstudio_grid/sdkconfig b/ports/espressif/boards/intechstudio_grid/sdkconfig deleted file mode 100644 index 329b9a9f2..000000000 --- a/ports/espressif/boards/intechstudio_grid/sdkconfig +++ /dev/null @@ -1,7 +0,0 @@ -# Board Specific Config - -# Partition Table -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-8MB.csv" - -# Serial flasher config -CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y diff --git a/ports/espressif/boards/intechstudio_knot/board.cmake b/ports/espressif/boards/intechstudio_knot/board.cmake deleted file mode 100644 index c1426c950..000000000 --- a/ports/espressif/boards/intechstudio_knot/board.cmake +++ /dev/null @@ -1,2 +0,0 @@ -# Apply board specific content here -set(IDF_TARGET "esp32s3") diff --git a/ports/espressif/boards/intechstudio_knot/board.h b/ports/espressif/boards/intechstudio_knot/board.h deleted file mode 100644 index ffabb5a97..000000000 --- a/ports/espressif/boards/intechstudio_knot/board.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2020 Ha Thach (tinyusb.org) for Adafruit Industries - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef INTECHSTUDIO_KNOT_H_ -#define INTECHSTUDIO_KNOT_H_ - -//--------------------------------------------------------------------+ -// Button -//--------------------------------------------------------------------+ - -// Enter UF2 mode if GPIO is pressed while 2nd stage bootloader indicator -// is on e.g RGB = Purple. If it is GPIO0, user should not hold this while -// reset since that will instead run the 1st stage ROM bootloader -#define PIN_BUTTON_UF2 36 -#define UF2_DETECTION_DELAY_MS 0 - -// GPIO that implement 1-bit memory with RC components which hold the -// pin value long enough for double reset detection. -// #define PIN_DOUBLE_RESET_RC 47 - -//--------------------------------------------------------------------+ -// LED -//--------------------------------------------------------------------+ - -// GPIO connected to Neopixel data -#define NEOPIXEL_PIN 1 - -// Brightness percentage from 1 to 255 -#define NEOPIXEL_BRIGHTNESS 0x64 - -// Number of neopixels -#define NEOPIXEL_NUMBER 3 - -//--------------------------------------------------------------------+ -// USB UF2 -//--------------------------------------------------------------------+ - -#define USB_VID 0x303A -#define USB_PID 0x8124 -#define USB_MANUFACTURER "Intech Studio" -#define USB_PRODUCT "Knot" - -#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT -#define UF2_BOARD_ID "ESP32S3-Knot" -#define UF2_VOLUME_LABEL "KNOT-S3" -#define UF2_INDEX_URL "https://github.com/intechstudio/knot" - -#endif diff --git a/ports/espressif/boards/intechstudio_knot/sdkconfig b/ports/espressif/boards/intechstudio_knot/sdkconfig deleted file mode 100644 index 329b9a9f2..000000000 --- a/ports/espressif/boards/intechstudio_knot/sdkconfig +++ /dev/null @@ -1,7 +0,0 @@ -# Board Specific Config - -# Partition Table -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-8MB.csv" - -# Serial flasher config -CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y diff --git a/ports/espressif/bootloader_components/main/CMakeLists.txt b/ports/espressif/bootloader_components/main/CMakeLists.txt index 12cbce2de..8ead28489 100644 --- a/ports/espressif/bootloader_components/main/CMakeLists.txt +++ b/ports/espressif/bootloader_components/main/CMakeLists.txt @@ -1,3 +1,10 @@ -idf_component_register(SRCS "bootloader_start.c" +set(bootloader_sources "bootloader_start.c") +set(bootloader_requires bootloader bootloader_support hal) +if(IDF_TARGET STREQUAL "esp32p4") + list(APPEND bootloader_sources "double_reset_adc.c" "neopixel_p4.c") + list(APPEND bootloader_requires efuse esp_hal_gpio esp_hal_ana_conv esp_hal_rmt) +endif() + +idf_component_register(SRCS ${bootloader_sources} INCLUDE_DIRS "../../boards/${BOARD}" - REQUIRES bootloader bootloader_support hal) + REQUIRES ${bootloader_requires}) diff --git a/ports/espressif/bootloader_components/main/bootloader_start.c b/ports/espressif/bootloader_components/main/bootloader_start.c index 210f5ab48..384c8bef6 100644 --- a/ports/espressif/bootloader_components/main/bootloader_start.c +++ b/ports/espressif/bootloader_components/main/bootloader_start.c @@ -21,6 +21,14 @@ // Specific board header specified with -DBOARD= #include "board.h" +#ifdef PIN_DOUBLE_RESET_ADC +#include "double_reset_adc.h" +#endif + +#if CONFIG_IDF_TARGET_ESP32P4 && defined(NEOPIXEL_PIN) +#include "neopixel_p4.h" +#endif + #ifdef TCA9554_ADDR // Using GPIO expander requires long reset delay (somehow) #define NEOPIXEL_RESET_DELAY ns2cycle(1000*1000) @@ -202,6 +210,15 @@ static int selected_boot_partition(const bootloader_state_t *bs) { // TinyUF2: when reset by EN/nRST pin: check if GPIO0 is pressed and/or 1-bit RC on specific GPIO detect double reset // during this time. If yes then to load uf2 "bootloader". if (boot_index != FACTORY_INDEX && reset_reason == RESET_REASON_CHIP_POWER_ON) { + #ifdef PIN_DOUBLE_RESET_ADC + if (double_reset_adc_detect()) { + ESP_LOGI(TAG, "Detect double reset using ADC on GPIO %d", PIN_DOUBLE_RESET_ADC); + boot_index = FACTORY_INDEX; + } else { + double_reset_adc_arm(); + } + #endif + #ifdef PIN_DOUBLE_RESET_RC // Double reset detect if board implements 1-bit memory with RC components esp_rom_gpio_pad_select_gpio(PIN_DOUBLE_RESET_RC); @@ -236,7 +253,6 @@ static int selected_boot_partition(const bootloader_state_t *bs) { esp_rom_gpio_pad_pullup_only(PIN_BUTTON_UF2); // run the GPIO detection at least once even if UF2_DETECTION_DELAY_MS is set to zero - uint32_t const detection_delay_ms = UF2_DETECTION_DELAY_MS; uint32_t tm_start = esp_log_early_timestamp(); do { if ( gpio_ll_get_level(&GPIO, PIN_BUTTON_UF2) == 0 ) { @@ -244,7 +260,7 @@ static int selected_boot_partition(const bootloader_state_t *bs) { boot_index = FACTORY_INDEX; break; } - } while (detection_delay_ms > (esp_log_early_timestamp() - tm_start) ); + } while (UF2_DETECTION_DELAY_MS > (esp_log_early_timestamp() - tm_start) ); if (UF2_DETECTION_DELAY_MS > 0){ board_led_off(); @@ -261,6 +277,9 @@ static int selected_boot_partition(const bootloader_state_t *bs) { gpio_ll_set_level(&GPIO, PIN_DOUBLE_RESET_RC, 0); gpio_ll_output_disable(&GPIO, PIN_DOUBLE_RESET_RC); #endif + #ifdef PIN_DOUBLE_RESET_ADC + double_reset_adc_clear(); + #endif } // Customer implementation. @@ -288,6 +307,8 @@ static inline uint32_t ns2cycle(uint32_t ns) { #if CONFIG_IDF_TARGET_ESP32S3 || CONFIG_IDF_TARGET_ESP32S31 tick_per_us = ets_get_cpu_frequency(); +#elif CONFIG_IDF_TARGET_ESP32P4 + tick_per_us = esp_rom_get_cpu_ticks_per_us(); #else // ESP32S2 extern uint32_t g_ticks_per_us_pro; tick_per_us = g_ticks_per_us_pro; @@ -348,7 +369,7 @@ static inline __attribute__((always_inline)) void neopixel_pin_write(uint32_t nu RV_CLEAR_CSR(CSR_GPIO_OUT_USER, 1); } } -#else +#elif !CONFIG_IDF_TARGET_ESP32P4 #define NEOPIXEL_T0H_NS 400 static void neopixel_pin_begin(uint32_t num_pin) { @@ -366,11 +387,6 @@ static inline __attribute__((always_inline)) void neopixel_pin_write(uint32_t nu #endif static void board_neopixel_set(uint32_t num_pin, uint8_t const rgb[]) { - // WS2812B should be - uint32_t const time0 = ns2cycle(NEOPIXEL_T0H_NS); - uint32_t const time1 = ns2cycle(800); - uint32_t const period = ns2cycle(1250); - uint8_t pixels[3*NEOPIXEL_NUMBER]; for(uint32_t i=0; i +#include "esp_efuse_rtc_calib.h" +#include "esp_log.h" +#include "esp_private/regi2c_ctrl.h" +#include "esp_rom_gpio.h" +#include "esp_rom_sys.h" +#include "hal/adc_ll.h" +#include "hal/gpio_ll.h" +#include "soc/gpio_sig_map.h" +#include "soc/pmu_reg.h" + +#define ADC_SAMPLE_COUNT 8 +#define ADC_TIMEOUT_US 1000 +#define ADC_DISCHARGE_US 5000 + +_Static_assert(BOARD_DOUBLE_RESET_ADC_CHANNEL >= 0 && + BOARD_DOUBLE_RESET_ADC_CHANNEL < SOC_ADC_CHANNEL_NUM(ADC_UNIT_1), + "Double-reset pin must use ADC1"); +_Static_assert(PIN_DOUBLE_RESET_ADC == 16 + BOARD_DOUBLE_RESET_ADC_CHANNEL, + "Double-reset GPIO does not match its ESP32-P4 ADC1 channel"); + +static const char *TAG = "double_reset_adc"; + +static void pin_float(void) { + gpio_ll_output_disable(&GPIO, PIN_DOUBLE_RESET_ADC); + gpio_ll_input_disable(&GPIO, PIN_DOUBLE_RESET_ADC); + gpio_ll_pullup_dis(&GPIO, PIN_DOUBLE_RESET_ADC); + gpio_ll_pulldown_dis(&GPIO, PIN_DOUBLE_RESET_ADC); + gpio_ll_sleep_sel_dis(&GPIO, PIN_DOUBLE_RESET_ADC); + gpio_ll_func_sel(&GPIO, PIN_DOUBLE_RESET_ADC, PIN_FUNC_GPIO); +} + +static bool adc_read(uint32_t *raw) { + adc_oneshot_ll_start(ADC_UNIT_1); + // Let the new start reach the RTC controller before checking its DONE bit. + esp_rom_delay_us(10); + for (uint32_t waited = 0; waited < ADC_TIMEOUT_US; waited++) { + if (adc_oneshot_ll_get_event(ADC_LL_EVENT_ADC1_ONESHOT_DONE)) { + *raw = adc_oneshot_ll_get_raw_result(ADC_UNIT_1); + return true; + } + esp_rom_delay_us(1); + } + return false; +} + +bool double_reset_adc_detect(void) { + pin_float(); + + int version = esp_efuse_rtc_calib_get_ver(); + if (version < ESP_EFUSE_ADC_CALIB_VER_MIN || version > ESP_EFUSE_ADC_CALIB_VER_MAX) { + ESP_LOGW(TAG, "ADC calibration unavailable; use the BOOT button"); + return false; + } + + uint32_t reference_raw = 0; + uint32_t reference_mv = 0; + if (esp_efuse_rtc_calib_get_cal_voltage(version, ADC_UNIT_1, ADC_ATTEN_DB_12, + &reference_raw, &reference_mv) != ESP_OK || + reference_raw == 0 || reference_raw > 4095 || reference_mv != 2300) { + ESP_LOGW(TAG, "Invalid ADC voltage calibration; use the BOOT button"); + return false; + } + uint32_t init_code = esp_efuse_rtc_calib_get_init_code(version, ADC_UNIT_1, ADC_ATTEN_DB_12); + if (init_code == 0 || init_code > 4095) { + ESP_LOGW(TAG, "Invalid ADC offset calibration; use the BOOT button"); + return false; + } + int compensation = esp_efuse_rtc_calib_get_chan_compens( + version, ADC_UNIT_1, BOARD_DOUBLE_RESET_ADC_CHANNEL, ADC_ATTEN_DB_12); + + // The second-stage bootloader already uses ADC1 for entropy. Preserve that + // configuration while temporarily disconnecting its internal noise source. + uint32_t saved_adc_ctrl = ADC.ctrl_reg.val; + uint32_t saved_adc_ctrl2 = ADC.ctrl2.val; + uint32_t saved_lp_clock = LPPERI.clk_en.val; + uint32_t saved_lp_power = LP_ADC.force_wpd_sar.val; + uint32_t saved_reader = LP_ADC.reader1_ctrl.val; + uint32_t saved_mux = LP_ADC.meas1_mux.val; + uint32_t saved_measurement = LP_ADC.meas1_ctrl2.val; + uint32_t saved_atten = LP_ADC.atten1.val; + uint32_t saved_pmu = REG_READ(PMU_RF_PWC_REG); + + // Pause the digital controller before taking ownership of ADC1. + adc_ll_digi_trigger_disable(); + esp_rom_delay_us(100); + LPPERI.clk_en.ck_en_lp_i2cmst = 1; + SET_PERI_REG_MASK(PMU_RF_PWC_REG, PMU_XPD_PERIF_I2C); + + uint8_t saved_init_low = REGI2C_READ(I2C_SAR_ADC, ADC_SAR1_INITIAL_CODE_LOW_ADDR); + uint8_t saved_init_high = REGI2C_READ(I2C_SAR_ADC, ADC_SAR1_INITIAL_CODE_HIGH_ADDR); + uint8_t saved_reference = REGI2C_READ(I2C_SAR_ADC, ADC_SAR1_DREF_ADDR); + uint8_t saved_calibration = REGI2C_READ(I2C_SAR_ADC, ADC_SAR1_ENCAL_GND_ADDR); + uint8_t saved_entropy = REGI2C_READ(I2C_SAR_ADC, I2C_SAR_ADC_ENT_VDD_GRP1); + + // IDF 6's RNG disable also changes ADC2, pattern and clock state. Only clear + // these saved analog fields, leaving that unrelated configuration untouched. + REGI2C_WRITE_MASK(I2C_SAR_ADC, I2C_SAR_ADC_ENT_VDD_GRP1, 0); + REGI2C_WRITE_MASK(I2C_SAR_ADC, I2C_SAR_ADC_DTEST_VDD_GRP1, 0); + adc_ll_set_power_manage(ADC_UNIT_1, ADC_LL_POWER_SW_ON); + adc_ll_set_sar_clk_div(ADC_UNIT_1, ADC_LL_SAR_CLK_DIV_DEFAULT(ADC_UNIT_1)); + adc_oneshot_ll_output_invert(ADC_UNIT_1, false); + adc_oneshot_ll_set_atten(ADC_UNIT_1, BOARD_DOUBLE_RESET_ADC_CHANNEL, ADC_ATTEN_DB_12); + adc_oneshot_ll_set_output_bits(ADC_UNIT_1, ADC_BITWIDTH_12); + adc_oneshot_ll_set_channel(ADC_UNIT_1, BOARD_DOUBLE_RESET_ADC_CHANNEL); + adc_ll_set_controller(ADC_UNIT_1, ADC_LL_CTRL_RTC); + adc_ll_calibration_init(ADC_UNIT_1); + adc_ll_calibration_finish(ADC_UNIT_1); + adc_ll_set_calibration_param(ADC_UNIT_1, init_code); + esp_rom_delay_us(100); + + uint32_t raw = 0; + uint32_t total = 0; + bool valid = adc_read(&raw); // Discard the first conversion after switching. + for (unsigned int i = 0; valid && i < ADC_SAMPLE_COUNT; i++) { + valid = adc_read(&raw); + total += raw; + } + + // P4's adc_oneshot_ll_disable_all_unit() is a no-op. Stop the RTC start + // explicitly, then restore the entropy source, with its timer restored last. + LP_ADC.meas1_ctrl2.meas1_start_sar = 0; + adc_oneshot_ll_disable_channel(ADC_UNIT_1); + REGI2C_WRITE(I2C_SAR_ADC, ADC_SAR1_INITIAL_CODE_LOW_ADDR, saved_init_low); + REGI2C_WRITE(I2C_SAR_ADC, ADC_SAR1_INITIAL_CODE_HIGH_ADDR, saved_init_high); + REGI2C_WRITE(I2C_SAR_ADC, ADC_SAR1_DREF_ADDR, saved_reference); + REGI2C_WRITE(I2C_SAR_ADC, ADC_SAR1_ENCAL_GND_ADDR, saved_calibration); + REGI2C_WRITE(I2C_SAR_ADC, I2C_SAR_ADC_ENT_VDD_GRP1, saved_entropy); + LP_ADC.meas1_mux.val = saved_mux; + LP_ADC.meas1_ctrl2.val = saved_measurement; + LP_ADC.reader1_ctrl.val = saved_reader; + LP_ADC.atten1.val = saved_atten; + LP_ADC.force_wpd_sar.val = saved_lp_power; + ADC.ctrl_reg.val = saved_adc_ctrl; + REG_WRITE(PMU_RF_PWC_REG, saved_pmu); + LPPERI.clk_en.val = saved_lp_clock; + ADC.ctrl2.val = saved_adc_ctrl2; + + if (!valid) { + ESP_LOGW(TAG, "ADC conversion timed out; use the BOOT button"); + return false; + } + + // First-stage eFuse voltage calibration, including channel compensation. + // P4 ADC1/12 dB's further curve correction is below 1 mV near 2500 mV; + // this marker comparator does not require the full application ADC driver. + int64_t corrected_total = (int64_t)total - (int64_t)compensation * ADC_SAMPLE_COUNT; + return corrected_total * reference_mv > + (int64_t)BOARD_DOUBLE_RESET_ADC_THRESHOLD_MV * reference_raw * ADC_SAMPLE_COUNT; +} + +static void pin_drive(bool high) { + pin_float(); + gpio_ll_od_disable(&GPIO, PIN_DOUBLE_RESET_ADC); + esp_rom_gpio_connect_out_signal(PIN_DOUBLE_RESET_ADC, SIG_GPIO_OUT_IDX, false, false); + gpio_ll_set_level(&GPIO, PIN_DOUBLE_RESET_ADC, high); + gpio_ll_output_enable(&GPIO, PIN_DOUBLE_RESET_ADC); +} + +void double_reset_adc_arm(void) { + pin_drive(true); +} + +void double_reset_adc_clear(void) { + pin_drive(false); + esp_rom_delay_us(ADC_DISCHARGE_US); + pin_float(); +} + +#endif // PIN_DOUBLE_RESET_ADC diff --git a/ports/espressif/bootloader_components/main/double_reset_adc.h b/ports/espressif/bootloader_components/main/double_reset_adc.h new file mode 100644 index 000000000..6b2bdd721 --- /dev/null +++ b/ports/espressif/bootloader_components/main/double_reset_adc.h @@ -0,0 +1,12 @@ +/* + * SPDX-FileCopyrightText: 2026 Adafruit Industries + * SPDX-License-Identifier: MIT + */ +#pragma once + +#include + +// Call after bootloader_init(), before charging the marker capacitor. +bool double_reset_adc_detect(void); +void double_reset_adc_arm(void); +void double_reset_adc_clear(void); diff --git a/ports/espressif/bootloader_components/main/neopixel_p4.c b/ports/espressif/bootloader_components/main/neopixel_p4.c new file mode 100644 index 000000000..b71a9409b --- /dev/null +++ b/ports/espressif/bootloader_components/main/neopixel_p4.c @@ -0,0 +1,106 @@ +/* + * SPDX-FileCopyrightText: 2019-2025 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2026 Adafruit Industries + * SPDX-License-Identifier: Apache-2.0 + * + * Based on ESP-IDF v5.4.3 hal/rmt_hal.c and esp_driver_rmt/src/rmt_private.h. + * WS2812 pulse durations from led_strip 2.x led_strip_rmt_encoder.c. + */ + +#include "neopixel_p4.h" +#include "esp_rom_gpio.h" +#include "esp_rom_sys.h" +#include "hal/gpio_ll.h" +#include "hal/rmt_ll.h" +#include "soc/gpio_sig_map.h" +#include "soc/soc_caps.h" + +// The hardware memory symbol is supplied by esp32p4.peripherals.ld. +typedef struct { + struct { + rmt_symbol_word_t symbols[SOC_RMT_MEM_WORDS_PER_CHANNEL]; + } channels[RMT_LL_GET(CHANS_PER_INST)]; +} bootloader_rmt_mem_t; +extern bootloader_rmt_mem_t RMTMEM; + +bool bootloader_neopixel_write(uint32_t pin, const uint8_t *grb, size_t count) { + // One pixel plus the terminating symbol fits entirely in channel 0 memory. + if (!grb || count != 3 || pin >= SOC_GPIO_PIN_COUNT) { + return false; + } + + gpio_ll_set_level(&GPIO, pin, 0); + gpio_ll_func_sel(&GPIO, pin, PIN_FUNC_GPIO); + gpio_ll_input_disable(&GPIO, pin); + gpio_ll_od_disable(&GPIO, pin); + esp_rom_gpio_connect_out_signal(pin, SIG_GPIO_OUT_IDX, false, false); + gpio_ll_output_enable(&GPIO, pin); + esp_rom_delay_us(300); + + // No scheduler or interrupt-driven RMT users exist in the second stage. + const int __DECLARE_RCC_ATOMIC_ENV = 0; + bool saved_bus_clock = HP_SYS_CLKRST.soc_clk_ctrl2.reg_rmt_sys_clk_en; + uint32_t saved_clock = HP_SYS_CLKRST.peri_clk_ctrl22.val; + rmt_ll_enable_bus_clock(0, true); + rmt_ll_reset_register(0); + // IDF 6 removed rmt_ll_enable_periph_clock; retain its hardware settings. + RMT.sys_conf.clk_en = 1; + RMT.sys_conf.mem_clk_force_on = 1; + rmt_ll_mem_force_power_on(&RMT); + rmt_ll_enable_mem_access_nonfifo(&RMT, true); + rmt_ll_enable_interrupt(&RMT, UINT32_MAX, false); + rmt_ll_clear_interrupt_status(&RMT, UINT32_MAX); + + // 40 MHz XTAL / 1 group divider / 4 channel divider = 10 MHz. + rmt_ll_set_group_clock_src(&RMT, 0, RMT_CLK_SRC_XTAL, 1, 0, 0); + rmt_ll_enable_group_clock(&RMT, true); + rmt_ll_tx_set_channel_clock_div(&RMT, 0, 4); + rmt_ll_tx_set_mem_blocks(&RMT, 0, 1); + rmt_ll_tx_enable_carrier_modulation(&RMT, 0, false); + rmt_ll_tx_enable_wrap(&RMT, 0, false); + rmt_ll_tx_enable_loop(&RMT, 0, false); + rmt_ll_tx_fix_idle_level(&RMT, 0, 0, true); + rmt_ll_tx_reset_channels_clock_div(&RMT, 1); + rmt_ll_tx_reset_pointer(&RMT, 0); + + volatile rmt_symbol_word_t *symbols = RMTMEM.channels[0].symbols; + size_t index = 0; + for (size_t byte = 0; byte < count; byte++) { + for (uint8_t mask = 0x80; mask; mask >>= 1) { + // HIGH then LOW: 0 = 0.3/0.9 us; 1 = 0.9/0.3 us, MSB first. + symbols[index++].val = (grb[byte] & mask) ? + (9 | (1 << 15) | (3 << 16)) : + (3 | (1 << 15) | (9 << 16)); + } + } + symbols[index].val = 0; // A zero duration terminates transmission. + + esp_rom_gpio_connect_out_signal(pin, RMT_SIG_PAD_OUT0_IDX, false, false); + rmt_ll_tx_start(&RMT, 0); + bool done = false; + for (unsigned int waited = 0; waited < 1000; waited++) { + uint32_t status = rmt_ll_tx_get_interrupt_status_raw(&RMT, 0); + if (status & RMT_LL_EVENT_TX_ERROR(0)) { + break; + } + if (status & RMT_LL_EVENT_TX_DONE(0)) { + done = true; + break; + } + esp_rom_delay_us(1); + } + + rmt_ll_tx_stop(&RMT, 0); + // Disconnect the peripheral before resetting it; latch the pixel at LOW. + esp_rom_gpio_connect_out_signal(pin, SIG_GPIO_OUT_IDX, false, false); + gpio_ll_set_level(&GPIO, pin, 0); + gpio_ll_output_enable(&GPIO, pin); + esp_rom_delay_us(300); + rmt_ll_reset_register(0); + rmt_ll_mem_power_by_pmu(&RMT); + RMT.sys_conf.clk_en = 0; + RMT.sys_conf.mem_clk_force_on = 0; + HP_SYS_CLKRST.peri_clk_ctrl22.val = saved_clock; + rmt_ll_enable_bus_clock(0, saved_bus_clock); + return done; +} diff --git a/ports/espressif/bootloader_components/main/neopixel_p4.h b/ports/espressif/bootloader_components/main/neopixel_p4.h new file mode 100644 index 000000000..4aa82dc8f --- /dev/null +++ b/ports/espressif/bootloader_components/main/neopixel_p4.h @@ -0,0 +1,13 @@ +/* + * SPDX-FileCopyrightText: 2026 Adafruit Industries + * SPDX-License-Identifier: MIT + */ +#pragma once + +#include +#include +#include + +// Early boot only, before any other RMT user. Accepts one scaled GRB pixel. +// Leaves the GPIO driven LOW, including after a transmission timeout. +bool bootloader_neopixel_write(uint32_t pin, const uint8_t *grb, size_t count); diff --git a/ports/espressif/components/led_strip/CHANGELOG.md b/ports/espressif/components/led_strip/CHANGELOG.md index 51c0cd30c..777d3ff45 100644 --- a/ports/espressif/components/led_strip/CHANGELOG.md +++ b/ports/espressif/components/led_strip/CHANGELOG.md @@ -1,3 +1,42 @@ +## 3.1.0 + +- Add async refresh API: `led_strip_refresh_async()` and `led_strip_refresh_async_done()`. Note that the driver has no double buffer yet, so the pixel buffer still has to be swapped manually between frames; the async API mainly allows the wait to be deferred. +- Support to use custom timing config in RMT backend +- Support to switch GPIO at runtime in RMT backend +- SPI backend uses official `SPICOMMON_BUSFLAG_DATA_OUT_INV`, older IDF keeps the GPIO-matrix fallback. +- Fix IDF 6.1 build by explicitly including `freertos/FreeRTOS.h` and `freertos/task.h` in the SPI backend (espressif/idf-extra-components#870) +- Fix WS2812 and SK6812 default timings to comply with their datasheet minimum T1H/T1L requirements. (espressif/idf-extra-components#816) +- Drop support for ESP-IDF v5.0 and v5.1 + +## 3.0.3 + +- Support WS2816 with 16-bit color + +## 3.0.1 + +- Support WS2811 bit timing + +## 3.0.0 + +- Discontinued support for ESP-IDF v4.x +- Added configuration for user-defined color component format + +## 2.5.5 + +- Simplified the led_strip component dependency, the time of full build with ESP-IDF v5.3 can now be shorter. + +## 2.5.4 + +- Inserted extra delay when initialize the SPI LED device, to ensure all LEDs are in the reset state correctly + +## 2.5.3 + +- Extend reset time (280us) to support WS2812B-V5 + +## 2.5.2 + +- Added API reference doc (api.md) + ## 2.5.0 - Enabled support for IDF4.4 and above diff --git a/ports/espressif/components/led_strip/CMakeLists.txt b/ports/espressif/components/led_strip/CMakeLists.txt index 4738556a2..1e595de9d 100644 --- a/ports/espressif/components/led_strip/CMakeLists.txt +++ b/ports/espressif/components/led_strip/CMakeLists.txt @@ -1,22 +1,23 @@ include($ENV{IDF_PATH}/tools/cmake/version.cmake) set(srcs "src/led_strip_api.c") +set(public_requires) -if("${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}" VERSION_GREATER_EQUAL "5.0") - if(CONFIG_SOC_RMT_SUPPORTED) - list(APPEND srcs "src/led_strip_rmt_dev.c" "src/led_strip_rmt_encoder.c") - endif() -else() - list(APPEND srcs "src/led_strip_rmt_dev_idf4.c") +if(CONFIG_SOC_RMT_SUPPORTED) + list(APPEND srcs "src/led_strip_rmt_dev.c" "src/led_strip_rmt_encoder.c") +endif() + +if(CONFIG_SOC_GPSPI_SUPPORTED) + list(APPEND srcs "src/led_strip_spi_dev.c") endif() -# the SPI backend driver relies on something that was added in IDF 5.1 -if("${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}" VERSION_GREATER_EQUAL "5.1") - if(CONFIG_SOC_GPSPI_SUPPORTED) - list(APPEND srcs "src/led_strip_spi_dev.c") - endif() +# Starting from esp-idf v5.3, the RMT and SPI drivers are moved to separate components +if("${IDF_VERSION_MAJOR}.${IDF_VERSION_MINOR}" VERSION_GREATER_EQUAL "5.3") + list(APPEND public_requires "esp_driver_rmt" "esp_driver_spi") +else() + list(APPEND public_requires "driver") endif() idf_component_register(SRCS ${srcs} INCLUDE_DIRS "include" "interface" - REQUIRES "esp_driver_rmt" "esp_driver_spi") + REQUIRES ${public_requires}) diff --git a/ports/espressif/components/led_strip/LICENSE b/ports/espressif/components/led_strip/LICENSE index d64569567..261eeb9e9 100644 --- a/ports/espressif/components/led_strip/LICENSE +++ b/ports/espressif/components/led_strip/LICENSE @@ -1,4 +1,3 @@ - Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ diff --git a/ports/espressif/components/led_strip/README.md b/ports/espressif/components/led_strip/README.md index 543d29642..2e97fb390 100644 --- a/ports/espressif/components/led_strip/README.md +++ b/ports/espressif/components/led_strip/README.md @@ -2,96 +2,22 @@ [![Component Registry](https://components.espressif.com/components/espressif/led_strip/badge.svg)](https://components.espressif.com/components/espressif/led_strip) -This driver is designed for addressable LEDs like [WS2812](http://www.world-semi.com/Certifications/WS2812B.html), where each LED is controlled by a single data line. +This driver is designed for addressable LEDs like [WS2812](http://world-semi.com/ws2812-family/), where each LED is controlled by a single data line. -## Backend Controllers +## Supported Backend Peripherals + +The LED strip driver supports two different backend peripherals to generate the timing signals required by addressable LEDs: ### The [RMT](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/rmt.html) Peripheral This is the most economical way to drive the LEDs because it only consumes one RMT channel, leaving other channels free to use. However, the memory usage increases dramatically with the number of LEDs. If the RMT hardware can't be assist by DMA, the driver will going into interrupt very frequently, thus result in a high CPU usage. What's worse, if the RMT interrupt is delayed or not serviced in time (e.g. if Wi-Fi interrupt happens on the same CPU core), the RMT transaction will be corrupted and the LEDs will display incorrect colors. If you want to use RMT to drive a large number of LEDs, you'd better to enable the DMA feature if possible [^1]. -#### Allocate LED Strip Object with RMT Backend - -```c -#define BLINK_GPIO 0 - -led_strip_handle_t led_strip; - -/* LED strip initialization with the GPIO and pixels number*/ -led_strip_config_t strip_config = { - .strip_gpio_num = BLINK_GPIO, // The GPIO that connected to the LED strip's data line - .max_leds = 1, // The number of LEDs in the strip, - .led_pixel_format = LED_PIXEL_FORMAT_GRB, // Pixel format of your LED strip - .led_model = LED_MODEL_WS2812, // LED strip model - .flags.invert_out = false, // whether to invert the output signal (useful when your hardware has a level inverter) -}; - -led_strip_rmt_config_t rmt_config = { -#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 0, 0) - .rmt_channel = 0, -#else - .clk_src = RMT_CLK_SRC_DEFAULT, // different clock source can lead to different power consumption - .resolution_hz = 10 * 1000 * 1000, // 10MHz - .flags.with_dma = false, // whether to enable the DMA feature -#endif -}; -ESP_ERROR_CHECK(led_strip_new_rmt_device(&strip_config, &rmt_config, &led_strip)); -``` - -You can create multiple LED strip objects with different GPIOs and pixel numbers. The backend driver will automatically allocate the RMT channel for you if there is more available. - ### The [SPI](https://docs.espressif.com/projects/esp-idf/en/latest/esp32/api-reference/peripherals/spi_master.html) Peripheral -SPI peripheral can also be used to generate the timing required by the LED strip. However this backend is not as economical as the RMT one, because it will take up the whole **bus**, unlike the RMT just takes one **channel**. You **CAN'T** connect other devices to the same SPI bus if it's been used by the led_strip, because the led_strip doesn't have the concept of "Chip Select". - -Please note, the SPI backend has a dependency of **ESP-IDF >= 5.1** - -#### Allocate LED Strip Object with SPI Backend - -```c -#define BLINK_GPIO 0 - -led_strip_handle_t led_strip; - -/* LED strip initialization with the GPIO and pixels number*/ -led_strip_config_t strip_config = { - .strip_gpio_num = BLINK_GPIO, // The GPIO that connected to the LED strip's data line - .max_leds = 1, // The number of LEDs in the strip, - .led_pixel_format = LED_PIXEL_FORMAT_GRB, // Pixel format of your LED strip - .led_model = LED_MODEL_WS2812, // LED strip model - .flags.invert_out = false, // whether to invert the output signal (useful when your hardware has a level inverter) -}; - -led_strip_spi_config_t spi_config = { - .clk_src = SPI_CLK_SRC_DEFAULT, // different clock source can lead to different power consumption - .flags.with_dma = true, // Using DMA can improve performance and help drive more LEDs - .spi_bus = SPI2_HOST, // SPI bus ID -}; -ESP_ERROR_CHECK(led_strip_new_spi_device(&strip_config, &spi_config, &led_strip)); -``` - -The number of LED strip objects can be created depends on how many free SPI buses are free to use in your project. - -## FAQ - -* Which led_strip backend should I choose? - * It depends on your application requirement and target chip's ability. +SPI peripheral can also be used to generate the timing required by the LED strip, in a so-called "Clock-less" mode. However this backend is not as economical as the RMT one, because it will take up the whole **bus**. You **CANNOT** connect other devices to the same SPI bus if it's been used by the led_strip, because the led_strip doesn't have the concept of "Chip Select". - ```mermaid - flowchart LR - A{Is RMT supported?} - A --> |No| B[SPI backend] - B --> C{Does the led strip has \n a larger number of LEDs?} - C --> |No| D[Don't have to enable the DMA of the backend] - C --> |Yes| E[Enable the DMA of the backend] - A --> |Yes| F{Does the led strip has \n a larger number of LEDs?} - F --> |Yes| G{Does RMT support DMA?} - G --> |Yes| E - G --> |No| B - F --> |No| H[RMT backend] --> D - ``` +## Documentation -* How to set the brightness of the LED strip? - * You can tune the brightness by scaling the value of each R-G-B element with a **same** factor. But pay attention to the overflow of the value. +For detailed information about the LED Strip component, including API reference and user guides, please visit: -[^1]: The RMT DMA feature is not available on all ESP chips. Please check the data sheet before using it. +- **Programming Guide & API Reference**: [LED Strip Documentation](https://espressif.github.io/idf-extra-components/latest/led_strip/index.html) diff --git a/ports/espressif/components/led_strip/api.md b/ports/espressif/components/led_strip/api.md deleted file mode 100644 index 6581d6604..000000000 --- a/ports/espressif/components/led_strip/api.md +++ /dev/null @@ -1,454 +0,0 @@ -# API Reference - -## Header files - -- [include/led_strip.h](#file-includeled_striph) -- [include/led_strip_rmt.h](#file-includeled_strip_rmth) -- [include/led_strip_spi.h](#file-includeled_strip_spih) -- [include/led_strip_types.h](#file-includeled_strip_typesh) -- [interface/led_strip_interface.h](#file-interfaceled_strip_interfaceh) - -## File include/led_strip.h - -## Functions - -| Type | Name | -| ---: | :--- | -| esp\_err\_t | [**led\_strip\_clear**](#function-led_strip_clear) ([**led\_strip\_handle\_t**](#struct-led_strip_t) strip)
_Clear LED strip (turn off all LEDs)_ | -| esp\_err\_t | [**led\_strip\_del**](#function-led_strip_del) ([**led\_strip\_handle\_t**](#struct-led_strip_t) strip)
_Free LED strip resources._ | -| esp\_err\_t | [**led\_strip\_refresh**](#function-led_strip_refresh) ([**led\_strip\_handle\_t**](#struct-led_strip_t) strip)
_Refresh memory colors to LEDs._ | -| esp\_err\_t | [**led\_strip\_set\_pixel**](#function-led_strip_set_pixel) ([**led\_strip\_handle\_t**](#struct-led_strip_t) strip, uint32\_t index, uint32\_t red, uint32\_t green, uint32\_t blue)
_Set RGB for a specific pixel._ | -| esp\_err\_t | [**led\_strip\_set\_pixel\_hsv**](#function-led_strip_set_pixel_hsv) ([**led\_strip\_handle\_t**](#struct-led_strip_t) strip, uint32\_t index, uint16\_t hue, uint8\_t saturation, uint8\_t value)
_Set HSV for a specific pixel._ | -| esp\_err\_t | [**led\_strip\_set\_pixel\_rgbw**](#function-led_strip_set_pixel_rgbw) ([**led\_strip\_handle\_t**](#struct-led_strip_t) strip, uint32\_t index, uint32\_t red, uint32\_t green, uint32\_t blue, uint32\_t white)
_Set RGBW for a specific pixel._ | - -## Functions Documentation - -### function `led_strip_clear` - -_Clear LED strip (turn off all LEDs)_ - -```c -esp_err_t led_strip_clear ( - led_strip_handle_t strip -) -``` - -**Parameters:** - -- `strip` LED strip - -**Returns:** - -- ESP\_OK: Clear LEDs successfully -- ESP\_FAIL: Clear LEDs failed because some other error occurred - -### function `led_strip_del` - -_Free LED strip resources._ - -```c -esp_err_t led_strip_del ( - led_strip_handle_t strip -) -``` - -**Parameters:** - -- `strip` LED strip - -**Returns:** - -- ESP\_OK: Free resources successfully -- ESP\_FAIL: Free resources failed because error occurred - -### function `led_strip_refresh` - -_Refresh memory colors to LEDs._ - -```c -esp_err_t led_strip_refresh ( - led_strip_handle_t strip -) -``` - -**Parameters:** - -- `strip` LED strip - -**Returns:** - -- ESP\_OK: Refresh successfully -- ESP\_FAIL: Refresh failed because some other error occurred - -**Note:** - -: After updating the LED colors in the memory, a following invocation of this API is needed to flush colors to strip. - -### function `led_strip_set_pixel` - -_Set RGB for a specific pixel._ - -```c -esp_err_t led_strip_set_pixel ( - led_strip_handle_t strip, - uint32_t index, - uint32_t red, - uint32_t green, - uint32_t blue -) -``` - -**Parameters:** - -- `strip` LED strip -- `index` index of pixel to set -- `red` red part of color -- `green` green part of color -- `blue` blue part of color - -**Returns:** - -- ESP\_OK: Set RGB for a specific pixel successfully -- ESP\_ERR\_INVALID\_ARG: Set RGB for a specific pixel failed because of invalid parameters -- ESP\_FAIL: Set RGB for a specific pixel failed because other error occurred - -### function `led_strip_set_pixel_hsv` - -_Set HSV for a specific pixel._ - -```c -esp_err_t led_strip_set_pixel_hsv ( - led_strip_handle_t strip, - uint32_t index, - uint16_t hue, - uint8_t saturation, - uint8_t value -) -``` - -**Parameters:** - -- `strip` LED strip -- `index` index of pixel to set -- `hue` hue part of color (0 - 360) -- `saturation` saturation part of color (0 - 255) -- `value` value part of color (0 - 255) - -**Returns:** - -- ESP\_OK: Set HSV color for a specific pixel successfully -- ESP\_ERR\_INVALID\_ARG: Set HSV color for a specific pixel failed because of an invalid argument -- ESP\_FAIL: Set HSV color for a specific pixel failed because other error occurred - -### function `led_strip_set_pixel_rgbw` - -_Set RGBW for a specific pixel._ - -```c -esp_err_t led_strip_set_pixel_rgbw ( - led_strip_handle_t strip, - uint32_t index, - uint32_t red, - uint32_t green, - uint32_t blue, - uint32_t white -) -``` - -**Note:** - -Only call this function if your led strip does have the white component (e.g. SK6812-RGBW) - -**Note:** - -Also see `led_strip_set_pixel` if you only want to specify the RGB part of the color and bypass the white component - -**Parameters:** - -- `strip` LED strip -- `index` index of pixel to set -- `red` red part of color -- `green` green part of color -- `blue` blue part of color -- `white` separate white component - -**Returns:** - -- ESP\_OK: Set RGBW color for a specific pixel successfully -- ESP\_ERR\_INVALID\_ARG: Set RGBW color for a specific pixel failed because of an invalid argument -- ESP\_FAIL: Set RGBW color for a specific pixel failed because other error occurred - -## File include/led_strip_rmt.h - -## Structures and Types - -| Type | Name | -| ---: | :--- | -| struct | [**led\_strip\_rmt\_config\_t**](#struct-led_strip_rmt_config_t)
_LED Strip RMT specific configuration._ | - -## Functions - -| Type | Name | -| ---: | :--- | -| esp\_err\_t | [**led\_strip\_new\_rmt\_device**](#function-led_strip_new_rmt_device) (const [**led\_strip\_config\_t**](#struct-led_strip_config_t) \*led\_config, const [**led\_strip\_rmt\_config\_t**](#struct-led_strip_rmt_config_t) \*rmt\_config, [**led\_strip\_handle\_t**](#struct-led_strip_t) \*ret\_strip)
_Create LED strip based on RMT TX channel._ | - -## Structures and Types Documentation - -### struct `led_strip_rmt_config_t` - -_LED Strip RMT specific configuration._ - -Variables: - -- rmt\_clock\_source\_t clk_src
RMT clock source - -- struct [**led\_strip\_rmt\_config\_t**](#struct-led_strip_rmt_config_t) flags
Extra driver flags - -- size\_t mem_block_symbols
How many RMT symbols can one RMT channel hold at one time. Set to 0 will fallback to use the default size. - -- uint32\_t resolution_hz
RMT tick resolution, if set to zero, a default resolution (10MHz) will be applied - -- uint32\_t with_dma
Use DMA to transmit data - -## Functions Documentation - -### function `led_strip_new_rmt_device` - -_Create LED strip based on RMT TX channel._ - -```c -esp_err_t led_strip_new_rmt_device ( - const led_strip_config_t *led_config, - const led_strip_rmt_config_t *rmt_config, - led_strip_handle_t *ret_strip -) -``` - -**Parameters:** - -- `led_config` LED strip configuration -- `rmt_config` RMT specific configuration -- `ret_strip` Returned LED strip handle - -**Returns:** - -- ESP\_OK: create LED strip handle successfully -- ESP\_ERR\_INVALID\_ARG: create LED strip handle failed because of invalid argument -- ESP\_ERR\_NO\_MEM: create LED strip handle failed because of out of memory -- ESP\_FAIL: create LED strip handle failed because some other error - -## File include/led_strip_spi.h - -## Structures and Types - -| Type | Name | -| ---: | :--- | -| struct | [**led\_strip\_spi\_config\_t**](#struct-led_strip_spi_config_t)
_LED Strip SPI specific configuration._ | - -## Functions - -| Type | Name | -| ---: | :--- | -| esp\_err\_t | [**led\_strip\_new\_spi\_device**](#function-led_strip_new_spi_device) (const [**led\_strip\_config\_t**](#struct-led_strip_config_t) \*led\_config, const [**led\_strip\_spi\_config\_t**](#struct-led_strip_spi_config_t) \*spi\_config, [**led\_strip\_handle\_t**](#struct-led_strip_t) \*ret\_strip)
_Create LED strip based on SPI MOSI channel._ | - -## Structures and Types Documentation - -### struct `led_strip_spi_config_t` - -_LED Strip SPI specific configuration._ - -Variables: - -- spi\_clock\_source\_t clk_src
SPI clock source - -- struct [**led\_strip\_spi\_config\_t**](#struct-led_strip_spi_config_t) flags
Extra driver flags - -- spi\_host\_device\_t spi_bus
SPI bus ID. Which buses are available depends on the specific chip - -- uint32\_t with_dma
Use DMA to transmit data - -## Functions Documentation - -### function `led_strip_new_spi_device` - -_Create LED strip based on SPI MOSI channel._ - -```c -esp_err_t led_strip_new_spi_device ( - const led_strip_config_t *led_config, - const led_strip_spi_config_t *spi_config, - led_strip_handle_t *ret_strip -) -``` - -**Note:** - -Although only the MOSI line is used for generating the signal, the whole SPI bus can't be used for other purposes. - -**Parameters:** - -- `led_config` LED strip configuration -- `spi_config` SPI specific configuration -- `ret_strip` Returned LED strip handle - -**Returns:** - -- ESP\_OK: create LED strip handle successfully -- ESP\_ERR\_INVALID\_ARG: create LED strip handle failed because of invalid argument -- ESP\_ERR\_NOT\_SUPPORTED: create LED strip handle failed because of unsupported configuration -- ESP\_ERR\_NO\_MEM: create LED strip handle failed because of out of memory -- ESP\_FAIL: create LED strip handle failed because some other error - -## File include/led_strip_types.h - -## Structures and Types - -| Type | Name | -| ---: | :--- | -| enum | [**led\_model\_t**](#enum-led_model_t)
_LED strip model._ | -| enum | [**led\_pixel\_format\_t**](#enum-led_pixel_format_t)
_LED strip pixel format._ | -| struct | [**led\_strip\_config\_t**](#struct-led_strip_config_t)
_LED Strip Configuration._ | -| typedef struct [**led\_strip\_t**](#struct-led_strip_t) \* | [**led\_strip\_handle\_t**](#typedef-led_strip_handle_t)
_LED strip handle._ | - -## Structures and Types Documentation - -### enum `led_model_t` - -_LED strip model._ - -```c -enum led_model_t { - LED_MODEL_WS2812, - LED_MODEL_SK6812, - LED_MODEL_INVALID -}; -``` - -**Note:** - -Different led model may have different timing parameters, so we need to distinguish them. - -### enum `led_pixel_format_t` - -_LED strip pixel format._ - -```c -enum led_pixel_format_t { - LED_PIXEL_FORMAT_GRB, - LED_PIXEL_FORMAT_GRBW, - LED_PIXEL_FORMAT_INVALID -}; -``` - -### struct `led_strip_config_t` - -_LED Strip Configuration._ - -Variables: - -- struct [**led\_strip\_config\_t**](#struct-led_strip_config_t) flags
Extra driver flags - -- uint32\_t invert_out
Invert output signal - -- led\_model\_t led_model
LED model - -- led\_pixel\_format\_t led_pixel_format
LED pixel format - -- uint32\_t max_leds
Maximum LEDs in a single strip - -- int strip_gpio_num
GPIO number that used by LED strip - -### typedef `led_strip_handle_t` - -_LED strip handle._ - -```c -typedef struct led_strip_t* led_strip_handle_t; -``` - -## File interface/led_strip_interface.h - -## Structures and Types - -| Type | Name | -| ---: | :--- | -| struct | [**led\_strip\_t**](#struct-led_strip_t)
_LED strip interface definition._ | -| typedef struct [**led\_strip\_t**](#struct-led_strip_t) | [**led\_strip\_t**](#typedef-led_strip_t)
| - -## Structures and Types Documentation - -### struct `led_strip_t` - -_LED strip interface definition._ - -Variables: - -- esp\_err\_t(\* clear
_Clear LED strip (turn off all LEDs)_
**Parameters:** - -- `strip` LED strip -- `timeout_ms` timeout value for clearing task - -**Returns:** - -- ESP\_OK: Clear LEDs successfully -- ESP\_FAIL: Clear LEDs failed because some other error occurred - -- esp\_err\_t(\* del
_Free LED strip resources._
**Parameters:** - -- `strip` LED strip - -**Returns:** - -- ESP\_OK: Free resources successfully -- ESP\_FAIL: Free resources failed because error occurred - -- esp\_err\_t(\* refresh
_Refresh memory colors to LEDs._
**Parameters:** - -- `strip` LED strip -- `timeout_ms` timeout value for refreshing task - -**Returns:** - -- ESP\_OK: Refresh successfully -- ESP\_FAIL: Refresh failed because some other error occurred - -**Note:** - -: After updating the LED colors in the memory, a following invocation of this API is needed to flush colors to strip. - -- esp\_err\_t(\* set_pixel
_Set RGB for a specific pixel._
**Parameters:** - -- `strip` LED strip -- `index` index of pixel to set -- `red` red part of color -- `green` green part of color -- `blue` blue part of color - -**Returns:** - -- ESP\_OK: Set RGB for a specific pixel successfully -- ESP\_ERR\_INVALID\_ARG: Set RGB for a specific pixel failed because of invalid parameters -- ESP\_FAIL: Set RGB for a specific pixel failed because other error occurred - -- esp\_err\_t(\* set_pixel_rgbw
_Set RGBW for a specific pixel. Similar to_ `set_pixel`_but also set the white component._
**Parameters:** - -- `strip` LED strip -- `index` index of pixel to set -- `red` red part of color -- `green` green part of color -- `blue` blue part of color -- `white` separate white component - -**Returns:** - -- ESP\_OK: Set RGBW color for a specific pixel successfully -- ESP\_ERR\_INVALID\_ARG: Set RGBW color for a specific pixel failed because of an invalid argument -- ESP\_FAIL: Set RGBW color for a specific pixel failed because other error occurred - -### typedef `led_strip_t` - -```c -typedef struct led_strip_t led_strip_t; -``` - -Type of LED strip diff --git a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/CMakeLists.txt b/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/CMakeLists.txt deleted file mode 100644 index 923c46310..000000000 --- a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# For more information about build system see -# https://docs.espressif.com/projects/esp-idf/en/latest/api-guides/build-system.html -# The following five lines of boilerplate have to be in your project's -# CMakeLists in this exact order for cmake to work correctly -cmake_minimum_required(VERSION 3.16) - -set(IDF_TARGET "esp32s3") -include($ENV{IDF_PATH}/tools/cmake/project.cmake) -project(led_strip_rmt_ws2812) diff --git a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/README.md b/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/README.md deleted file mode 100644 index ad52235d5..000000000 --- a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# LED Strip Example (RMT backend + WS2812) - -This example demonstrates how to blink the WS2812 LED using the [led_strip](https://components.espressif.com/component/espressif/led_strip) component. - -## How to Use Example - -### Hardware Required - -* A development board with Espressif SoC -* A USB cable for Power supply and programming -* WS2812 LED strip - -### Configure the Example - -Before project configuration and build, be sure to set the correct chip target using `idf.py set-target `. Then assign the proper GPIO in the [source file](main/led_strip_rmt_ws2812_main.c). If your led strip has multiple LEDs, don't forget update the number. - -### Build and Flash - -Run `idf.py -p PORT build flash monitor` to build, flash and monitor the project. - -(To exit the serial monitor, type ``Ctrl-]``.) - -See the [Getting Started Guide](https://docs.espressif.com/projects/esp-idf/en/latest/get-started/index.html) for full steps to configure and use ESP-IDF to build projects. - -## Example Output - -```text -I (299) gpio: GPIO[8]| InputEn: 0| OutputEn: 1| OpenDrain: 0| Pullup: 1| Pulldown: 0| Intr:0 -I (309) example: Created LED strip object with RMT backend -I (309) example: Start blinking LED strip -``` diff --git a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/dependencies.lock b/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/dependencies.lock deleted file mode 100644 index 97840a153..000000000 --- a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/dependencies.lock +++ /dev/null @@ -1,15 +0,0 @@ -dependencies: - espressif/led_strip: - component_hash: null - source: - path: /home/hathach/code/idf-extra-components/led_strip - type: local - version: 2.5.2 - idf: - component_hash: null - source: - type: idf - version: 5.1.1 -manifest_hash: 47d47762be26168b388cb0e6cbfee6b22c68d630ebf4b27a49c47c4c54191590 -target: esp32s3 -version: 1.0.0 diff --git a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/CMakeLists.txt b/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/CMakeLists.txt deleted file mode 100644 index 37b9c1458..000000000 --- a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -idf_component_register(SRCS "led_strip_rmt_ws2812_main.c" - INCLUDE_DIRS ".") diff --git a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/idf_component.yml b/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/idf_component.yml deleted file mode 100644 index 916c366c7..000000000 --- a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/idf_component.yml +++ /dev/null @@ -1,5 +0,0 @@ -## IDF Component Manager Manifest File -dependencies: - espressif/led_strip: - version: '^2' - override_path: '../../../' diff --git a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/led_strip_rmt_ws2812_main.c b/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/led_strip_rmt_ws2812_main.c deleted file mode 100644 index 4b20a5958..000000000 --- a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/main/led_strip_rmt_ws2812_main.c +++ /dev/null @@ -1,75 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD - * - * SPDX-License-Identifier: Unlicense OR CC0-1.0 - */ -#include -#include "freertos/FreeRTOS.h" -#include "freertos/task.h" -#include "led_strip.h" -#include "esp_log.h" -#include "esp_err.h" - -// GPIO assignment -#define LED_STRIP_BLINK_GPIO 48 -// Numbers of the LED in the strip -#define LED_STRIP_LED_NUMBERS 1 -// 10MHz resolution, 1 tick = 0.1us (led strip needs a high resolution) -#define LED_STRIP_RMT_RES_HZ (10 * 1000 * 1000) - -static const char *TAG = "example"; - -led_strip_handle_t configure_led(void) -{ - // LED strip general initialization, according to your led board design - led_strip_config_t strip_config = { - .strip_gpio_num = LED_STRIP_BLINK_GPIO, // The GPIO that connected to the LED strip's data line - .max_leds = LED_STRIP_LED_NUMBERS, // The number of LEDs in the strip, - .led_pixel_format = LED_PIXEL_FORMAT_GRB, // Pixel format of your LED strip - .led_model = LED_MODEL_WS2812, // LED strip model - .flags.invert_out = false, // whether to invert the output signal - }; - - // LED strip backend configuration: RMT - led_strip_rmt_config_t rmt_config = { -#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 0, 0) - .rmt_channel = 0, -#else - .clk_src = RMT_CLK_SRC_DEFAULT, // different clock source can lead to different power consumption - .resolution_hz = LED_STRIP_RMT_RES_HZ, // RMT counter clock frequency - .flags.with_dma = false, // DMA feature is available on ESP target like ESP32-S3 -#endif - }; - - // LED Strip object handle - led_strip_handle_t led_strip; - ESP_ERROR_CHECK(led_strip_new_rmt_device(&strip_config, &rmt_config, &led_strip)); - ESP_LOGI(TAG, "Created LED strip object with RMT backend"); - return led_strip; -} - -void app_main(void) -{ - led_strip_handle_t led_strip = configure_led(); - bool led_on_off = false; - - ESP_LOGI(TAG, "Start blinking LED strip"); - while (1) { - if (led_on_off) { - /* Set the LED pixel using RGB from 0 (0%) to 255 (100%) for each color */ - for (int i = 0; i < LED_STRIP_LED_NUMBERS; i++) { - ESP_ERROR_CHECK(led_strip_set_pixel(led_strip, i, 5, 5, 5)); - } - /* Refresh the strip to send data */ - ESP_ERROR_CHECK(led_strip_refresh(led_strip)); - ESP_LOGI(TAG, "LED ON!"); - } else { - /* Set all LED off to clear all pixels */ - ESP_ERROR_CHECK(led_strip_clear(led_strip)); - ESP_LOGI(TAG, "LED OFF!"); - } - - led_on_off = !led_on_off; - vTaskDelay(pdMS_TO_TICKS(500)); - } -} diff --git a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/sdkconfig b/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/sdkconfig deleted file mode 100644 index be88ab852..000000000 --- a/ports/espressif/components/led_strip/examples/led_strip_rmt_ws2812/sdkconfig +++ /dev/null @@ -1,1936 +0,0 @@ -# -# Automatically generated file. DO NOT EDIT. -# Espressif IoT Development Framework (ESP-IDF) 5.1.1 Project Configuration -# -CONFIG_SOC_MPU_MIN_REGION_SIZE=0x20000000 -CONFIG_SOC_MPU_REGIONS_MAX_NUM=8 -CONFIG_SOC_ADC_SUPPORTED=y -CONFIG_SOC_UART_SUPPORTED=y -CONFIG_SOC_PCNT_SUPPORTED=y -CONFIG_SOC_WIFI_SUPPORTED=y -CONFIG_SOC_TWAI_SUPPORTED=y -CONFIG_SOC_GDMA_SUPPORTED=y -CONFIG_SOC_GPTIMER_SUPPORTED=y -CONFIG_SOC_LCDCAM_SUPPORTED=y -CONFIG_SOC_MCPWM_SUPPORTED=y -CONFIG_SOC_DEDICATED_GPIO_SUPPORTED=y -CONFIG_SOC_CACHE_SUPPORT_WRAP=y -CONFIG_SOC_ULP_SUPPORTED=y -CONFIG_SOC_ULP_FSM_SUPPORTED=y -CONFIG_SOC_RISCV_COPROC_SUPPORTED=y -CONFIG_SOC_BT_SUPPORTED=y -CONFIG_SOC_USB_OTG_SUPPORTED=y -CONFIG_SOC_USB_SERIAL_JTAG_SUPPORTED=y -CONFIG_SOC_CCOMP_TIMER_SUPPORTED=y -CONFIG_SOC_ASYNC_MEMCPY_SUPPORTED=y -CONFIG_SOC_SUPPORTS_SECURE_DL_MODE=y -CONFIG_SOC_EFUSE_KEY_PURPOSE_FIELD=y -CONFIG_SOC_SDMMC_HOST_SUPPORTED=y -CONFIG_SOC_RTC_FAST_MEM_SUPPORTED=y -CONFIG_SOC_RTC_SLOW_MEM_SUPPORTED=y -CONFIG_SOC_RTC_MEM_SUPPORTED=y -CONFIG_SOC_PSRAM_DMA_CAPABLE=y -CONFIG_SOC_XT_WDT_SUPPORTED=y -CONFIG_SOC_I2S_SUPPORTED=y -CONFIG_SOC_RMT_SUPPORTED=y -CONFIG_SOC_SDM_SUPPORTED=y -CONFIG_SOC_GPSPI_SUPPORTED=y -CONFIG_SOC_LEDC_SUPPORTED=y -CONFIG_SOC_I2C_SUPPORTED=y -CONFIG_SOC_SYSTIMER_SUPPORTED=y -CONFIG_SOC_SUPPORT_COEXISTENCE=y -CONFIG_SOC_TEMP_SENSOR_SUPPORTED=y -CONFIG_SOC_AES_SUPPORTED=y -CONFIG_SOC_MPI_SUPPORTED=y -CONFIG_SOC_SHA_SUPPORTED=y -CONFIG_SOC_HMAC_SUPPORTED=y -CONFIG_SOC_DIG_SIGN_SUPPORTED=y -CONFIG_SOC_FLASH_ENC_SUPPORTED=y -CONFIG_SOC_SECURE_BOOT_SUPPORTED=y -CONFIG_SOC_MEMPROT_SUPPORTED=y -CONFIG_SOC_TOUCH_SENSOR_SUPPORTED=y -CONFIG_SOC_BOD_SUPPORTED=y -CONFIG_SOC_XTAL_SUPPORT_40M=y -CONFIG_SOC_APPCPU_HAS_CLOCK_GATING_BUG=y -CONFIG_SOC_ADC_RTC_CTRL_SUPPORTED=y -CONFIG_SOC_ADC_DIG_CTRL_SUPPORTED=y -CONFIG_SOC_ADC_ARBITER_SUPPORTED=y -CONFIG_SOC_ADC_DIG_IIR_FILTER_SUPPORTED=y -CONFIG_SOC_ADC_MONITOR_SUPPORTED=y -CONFIG_SOC_ADC_DMA_SUPPORTED=y -CONFIG_SOC_ADC_PERIPH_NUM=2 -CONFIG_SOC_ADC_MAX_CHANNEL_NUM=10 -CONFIG_SOC_ADC_ATTEN_NUM=4 -CONFIG_SOC_ADC_DIGI_CONTROLLER_NUM=2 -CONFIG_SOC_ADC_PATT_LEN_MAX=24 -CONFIG_SOC_ADC_DIGI_MIN_BITWIDTH=12 -CONFIG_SOC_ADC_DIGI_MAX_BITWIDTH=12 -CONFIG_SOC_ADC_DIGI_RESULT_BYTES=4 -CONFIG_SOC_ADC_DIGI_DATA_BYTES_PER_CONV=4 -CONFIG_SOC_ADC_DIGI_IIR_FILTER_NUM=2 -CONFIG_SOC_ADC_SAMPLE_FREQ_THRES_HIGH=83333 -CONFIG_SOC_ADC_SAMPLE_FREQ_THRES_LOW=611 -CONFIG_SOC_ADC_RTC_MIN_BITWIDTH=12 -CONFIG_SOC_ADC_RTC_MAX_BITWIDTH=12 -CONFIG_SOC_ADC_CALIBRATION_V1_SUPPORTED=y -CONFIG_SOC_ADC_SELF_HW_CALI_SUPPORTED=y -CONFIG_SOC_APB_BACKUP_DMA=y -CONFIG_SOC_BROWNOUT_RESET_SUPPORTED=y -CONFIG_SOC_CACHE_WRITEBACK_SUPPORTED=y -CONFIG_SOC_CACHE_FREEZE_SUPPORTED=y -CONFIG_SOC_CPU_CORES_NUM=2 -CONFIG_SOC_CPU_INTR_NUM=32 -CONFIG_SOC_CPU_HAS_FPU=y -CONFIG_SOC_CPU_BREAKPOINTS_NUM=2 -CONFIG_SOC_CPU_WATCHPOINTS_NUM=2 -CONFIG_SOC_CPU_WATCHPOINT_SIZE=64 -CONFIG_SOC_DS_SIGNATURE_MAX_BIT_LEN=4096 -CONFIG_SOC_DS_KEY_PARAM_MD_IV_LENGTH=16 -CONFIG_SOC_DS_KEY_CHECK_MAX_WAIT_US=1100 -CONFIG_SOC_GDMA_GROUPS=y -CONFIG_SOC_GDMA_PAIRS_PER_GROUP=5 -CONFIG_SOC_GDMA_SUPPORT_PSRAM=y -CONFIG_SOC_GPIO_PORT=1 -CONFIG_SOC_GPIO_PIN_COUNT=49 -CONFIG_SOC_GPIO_SUPPORT_PIN_GLITCH_FILTER=y -CONFIG_SOC_GPIO_FILTER_CLK_SUPPORT_APB=y -CONFIG_SOC_GPIO_SUPPORT_RTC_INDEPENDENT=y -CONFIG_SOC_GPIO_SUPPORT_FORCE_HOLD=y -CONFIG_SOC_GPIO_VALID_GPIO_MASK=0x1FFFFFFFFFFFF -CONFIG_SOC_GPIO_VALID_DIGITAL_IO_PAD_MASK=0x0001FFFFFC000000 -CONFIG_SOC_DEDIC_GPIO_OUT_CHANNELS_NUM=8 -CONFIG_SOC_DEDIC_GPIO_IN_CHANNELS_NUM=8 -CONFIG_SOC_DEDIC_GPIO_OUT_AUTO_ENABLE=y -CONFIG_SOC_I2C_NUM=2 -CONFIG_SOC_I2C_FIFO_LEN=32 -CONFIG_SOC_I2C_CMD_REG_NUM=8 -CONFIG_SOC_I2C_SUPPORT_SLAVE=y -CONFIG_SOC_I2C_SUPPORT_HW_CLR_BUS=y -CONFIG_SOC_I2C_SUPPORT_XTAL=y -CONFIG_SOC_I2C_SUPPORT_RTC=y -CONFIG_SOC_I2S_NUM=2 -CONFIG_SOC_I2S_HW_VERSION_2=y -CONFIG_SOC_I2S_SUPPORTS_XTAL=y -CONFIG_SOC_I2S_SUPPORTS_PLL_F160M=y -CONFIG_SOC_I2S_SUPPORTS_PCM=y -CONFIG_SOC_I2S_SUPPORTS_PDM=y -CONFIG_SOC_I2S_SUPPORTS_PDM_TX=y -CONFIG_SOC_I2S_PDM_MAX_TX_LINES=2 -CONFIG_SOC_I2S_SUPPORTS_PDM_RX=y -CONFIG_SOC_I2S_PDM_MAX_RX_LINES=4 -CONFIG_SOC_I2S_SUPPORTS_TDM=y -CONFIG_SOC_LEDC_SUPPORT_APB_CLOCK=y -CONFIG_SOC_LEDC_SUPPORT_XTAL_CLOCK=y -CONFIG_SOC_LEDC_CHANNEL_NUM=8 -CONFIG_SOC_LEDC_TIMER_BIT_WIDTH=14 -CONFIG_SOC_LEDC_SUPPORT_FADE_STOP=y -CONFIG_SOC_MCPWM_GROUPS=2 -CONFIG_SOC_MCPWM_TIMERS_PER_GROUP=3 -CONFIG_SOC_MCPWM_OPERATORS_PER_GROUP=3 -CONFIG_SOC_MCPWM_COMPARATORS_PER_OPERATOR=2 -CONFIG_SOC_MCPWM_GENERATORS_PER_OPERATOR=2 -CONFIG_SOC_MCPWM_TRIGGERS_PER_OPERATOR=2 -CONFIG_SOC_MCPWM_GPIO_FAULTS_PER_GROUP=3 -CONFIG_SOC_MCPWM_CAPTURE_TIMERS_PER_GROUP=y -CONFIG_SOC_MCPWM_CAPTURE_CHANNELS_PER_TIMER=3 -CONFIG_SOC_MCPWM_GPIO_SYNCHROS_PER_GROUP=3 -CONFIG_SOC_MCPWM_SWSYNC_CAN_PROPAGATE=y -CONFIG_SOC_MMU_LINEAR_ADDRESS_REGION_NUM=1 -CONFIG_SOC_MMU_PERIPH_NUM=1 -CONFIG_SOC_PCNT_GROUPS=1 -CONFIG_SOC_PCNT_UNITS_PER_GROUP=4 -CONFIG_SOC_PCNT_CHANNELS_PER_UNIT=2 -CONFIG_SOC_PCNT_THRES_POINT_PER_UNIT=2 -CONFIG_SOC_RMT_GROUPS=1 -CONFIG_SOC_RMT_TX_CANDIDATES_PER_GROUP=4 -CONFIG_SOC_RMT_RX_CANDIDATES_PER_GROUP=4 -CONFIG_SOC_RMT_CHANNELS_PER_GROUP=8 -CONFIG_SOC_RMT_MEM_WORDS_PER_CHANNEL=48 -CONFIG_SOC_RMT_SUPPORT_RX_PINGPONG=y -CONFIG_SOC_RMT_SUPPORT_RX_DEMODULATION=y -CONFIG_SOC_RMT_SUPPORT_TX_ASYNC_STOP=y -CONFIG_SOC_RMT_SUPPORT_TX_LOOP_COUNT=y -CONFIG_SOC_RMT_SUPPORT_TX_LOOP_AUTO_STOP=y -CONFIG_SOC_RMT_SUPPORT_TX_SYNCHRO=y -CONFIG_SOC_RMT_SUPPORT_TX_CARRIER_DATA_ONLY=y -CONFIG_SOC_RMT_SUPPORT_XTAL=y -CONFIG_SOC_RMT_SUPPORT_RC_FAST=y -CONFIG_SOC_RMT_SUPPORT_APB=y -CONFIG_SOC_RMT_SUPPORT_DMA=y -CONFIG_SOC_LCD_I80_SUPPORTED=y -CONFIG_SOC_LCD_RGB_SUPPORTED=y -CONFIG_SOC_LCD_I80_BUSES=1 -CONFIG_SOC_LCD_RGB_PANELS=1 -CONFIG_SOC_LCD_I80_BUS_WIDTH=16 -CONFIG_SOC_LCD_RGB_DATA_WIDTH=16 -CONFIG_SOC_LCD_SUPPORT_RGB_YUV_CONV=y -CONFIG_SOC_RTC_CNTL_CPU_PD_DMA_BUS_WIDTH=128 -CONFIG_SOC_RTC_CNTL_CPU_PD_REG_FILE_NUM=549 -CONFIG_SOC_RTC_CNTL_TAGMEM_PD_DMA_BUS_WIDTH=128 -CONFIG_SOC_RTCIO_PIN_COUNT=22 -CONFIG_SOC_RTCIO_INPUT_OUTPUT_SUPPORTED=y -CONFIG_SOC_RTCIO_HOLD_SUPPORTED=y -CONFIG_SOC_RTCIO_WAKE_SUPPORTED=y -CONFIG_SOC_SDM_GROUPS=y -CONFIG_SOC_SDM_CHANNELS_PER_GROUP=8 -CONFIG_SOC_SDM_CLK_SUPPORT_APB=y -CONFIG_SOC_SPI_PERIPH_NUM=3 -CONFIG_SOC_SPI_MAX_CS_NUM=6 -CONFIG_SOC_SPI_MAXIMUM_BUFFER_SIZE=64 -CONFIG_SOC_SPI_SUPPORT_DDRCLK=y -CONFIG_SOC_SPI_SLAVE_SUPPORT_SEG_TRANS=y -CONFIG_SOC_SPI_SUPPORT_CD_SIG=y -CONFIG_SOC_SPI_SUPPORT_CONTINUOUS_TRANS=y -CONFIG_SOC_SPI_SUPPORT_SLAVE_HD_VER2=y -CONFIG_SOC_SPI_SUPPORT_CLK_APB=y -CONFIG_SOC_SPI_SUPPORT_CLK_XTAL=y -CONFIG_SOC_SPI_PERIPH_SUPPORT_CONTROL_DUMMY_OUT=y -CONFIG_SOC_MEMSPI_IS_INDEPENDENT=y -CONFIG_SOC_SPI_MAX_PRE_DIVIDER=16 -CONFIG_SOC_SPI_SUPPORT_OCT=y -CONFIG_SOC_MEMSPI_SRC_FREQ_120M=y -CONFIG_SOC_MEMSPI_SRC_FREQ_80M_SUPPORTED=y -CONFIG_SOC_MEMSPI_SRC_FREQ_40M_SUPPORTED=y -CONFIG_SOC_MEMSPI_SRC_FREQ_20M_SUPPORTED=y -CONFIG_SOC_SPIRAM_SUPPORTED=y -CONFIG_SOC_SPIRAM_XIP_SUPPORTED=y -CONFIG_SOC_SYSTIMER_COUNTER_NUM=2 -CONFIG_SOC_SYSTIMER_ALARM_NUM=3 -CONFIG_SOC_SYSTIMER_BIT_WIDTH_LO=32 -CONFIG_SOC_SYSTIMER_BIT_WIDTH_HI=20 -CONFIG_SOC_SYSTIMER_FIXED_DIVIDER=y -CONFIG_SOC_SYSTIMER_INT_LEVEL=y -CONFIG_SOC_SYSTIMER_ALARM_MISS_COMPENSATE=y -CONFIG_SOC_TIMER_GROUPS=2 -CONFIG_SOC_TIMER_GROUP_TIMERS_PER_GROUP=2 -CONFIG_SOC_TIMER_GROUP_COUNTER_BIT_WIDTH=54 -CONFIG_SOC_TIMER_GROUP_SUPPORT_XTAL=y -CONFIG_SOC_TIMER_GROUP_SUPPORT_APB=y -CONFIG_SOC_TIMER_GROUP_TOTAL_TIMERS=4 -CONFIG_SOC_TOUCH_VERSION_2=y -CONFIG_SOC_TOUCH_SENSOR_NUM=15 -CONFIG_SOC_TOUCH_PROXIMITY_CHANNEL_NUM=3 -CONFIG_SOC_TOUCH_PROXIMITY_MEAS_DONE_SUPPORTED=y -CONFIG_SOC_TOUCH_PAD_THRESHOLD_MAX=0x1FFFFF -CONFIG_SOC_TOUCH_PAD_MEASURE_WAIT_MAX=0xFF -CONFIG_SOC_TWAI_CONTROLLER_NUM=1 -CONFIG_SOC_TWAI_CLK_SUPPORT_APB=y -CONFIG_SOC_TWAI_BRP_MIN=2 -CONFIG_SOC_TWAI_BRP_MAX=16384 -CONFIG_SOC_TWAI_SUPPORTS_RX_STATUS=y -CONFIG_SOC_UART_NUM=3 -CONFIG_SOC_UART_FIFO_LEN=128 -CONFIG_SOC_UART_BITRATE_MAX=5000000 -CONFIG_SOC_UART_SUPPORT_FSM_TX_WAIT_SEND=y -CONFIG_SOC_UART_SUPPORT_WAKEUP_INT=y -CONFIG_SOC_UART_SUPPORT_APB_CLK=y -CONFIG_SOC_UART_SUPPORT_RTC_CLK=y -CONFIG_SOC_UART_SUPPORT_XTAL_CLK=y -CONFIG_SOC_UART_REQUIRE_CORE_RESET=y -CONFIG_SOC_USB_PERIPH_NUM=y -CONFIG_SOC_SHA_DMA_MAX_BUFFER_SIZE=3968 -CONFIG_SOC_SHA_SUPPORT_DMA=y -CONFIG_SOC_SHA_SUPPORT_RESUME=y -CONFIG_SOC_SHA_GDMA=y -CONFIG_SOC_SHA_SUPPORT_SHA1=y -CONFIG_SOC_SHA_SUPPORT_SHA224=y -CONFIG_SOC_SHA_SUPPORT_SHA256=y -CONFIG_SOC_SHA_SUPPORT_SHA384=y -CONFIG_SOC_SHA_SUPPORT_SHA512=y -CONFIG_SOC_SHA_SUPPORT_SHA512_224=y -CONFIG_SOC_SHA_SUPPORT_SHA512_256=y -CONFIG_SOC_SHA_SUPPORT_SHA512_T=y -CONFIG_SOC_RSA_MAX_BIT_LEN=4096 -CONFIG_SOC_AES_SUPPORT_DMA=y -CONFIG_SOC_AES_GDMA=y -CONFIG_SOC_AES_SUPPORT_AES_128=y -CONFIG_SOC_AES_SUPPORT_AES_256=y -CONFIG_SOC_PM_SUPPORT_EXT0_WAKEUP=y -CONFIG_SOC_PM_SUPPORT_EXT1_WAKEUP=y -CONFIG_SOC_PM_SUPPORT_EXT_WAKEUP=y -CONFIG_SOC_PM_SUPPORT_WIFI_WAKEUP=y -CONFIG_SOC_PM_SUPPORT_BT_WAKEUP=y -CONFIG_SOC_PM_SUPPORT_TOUCH_SENSOR_WAKEUP=y -CONFIG_SOC_PM_SUPPORT_CPU_PD=y -CONFIG_SOC_PM_SUPPORT_TAGMEM_PD=y -CONFIG_SOC_PM_SUPPORT_RTC_PERIPH_PD=y -CONFIG_SOC_PM_SUPPORT_RC_FAST_PD=y -CONFIG_SOC_PM_SUPPORT_VDDSDIO_PD=y -CONFIG_SOC_PM_SUPPORT_MAC_BB_PD=y -CONFIG_SOC_PM_SUPPORT_MODEM_PD=y -CONFIG_SOC_CONFIGURABLE_VDDSDIO_SUPPORTED=y -CONFIG_SOC_PM_SUPPORT_DEEPSLEEP_CHECK_STUB_ONLY=y -CONFIG_SOC_PM_CPU_RETENTION_BY_RTCCNTL=y -CONFIG_SOC_PM_MODEM_RETENTION_BY_BACKUPDMA=y -CONFIG_SOC_CLK_RC_FAST_D256_SUPPORTED=y -CONFIG_SOC_RTC_SLOW_CLK_SUPPORT_RC_FAST_D256=y -CONFIG_SOC_CLK_RC_FAST_SUPPORT_CALIBRATION=y -CONFIG_SOC_CLK_XTAL32K_SUPPORTED=y -CONFIG_SOC_EFUSE_DIS_DOWNLOAD_ICACHE=y -CONFIG_SOC_EFUSE_DIS_DOWNLOAD_DCACHE=y -CONFIG_SOC_EFUSE_HARD_DIS_JTAG=y -CONFIG_SOC_EFUSE_DIS_USB_JTAG=y -CONFIG_SOC_EFUSE_SOFT_DIS_JTAG=y -CONFIG_SOC_EFUSE_DIS_DIRECT_BOOT=y -CONFIG_SOC_EFUSE_DIS_ICACHE=y -CONFIG_SOC_EFUSE_BLOCK9_KEY_PURPOSE_QUIRK=y -CONFIG_SOC_SECURE_BOOT_V2_RSA=y -CONFIG_SOC_EFUSE_SECURE_BOOT_KEY_DIGESTS=3 -CONFIG_SOC_EFUSE_REVOKE_BOOT_KEY_DIGESTS=y -CONFIG_SOC_SUPPORT_SECURE_BOOT_REVOKE_KEY=y -CONFIG_SOC_FLASH_ENCRYPTED_XTS_AES_BLOCK_MAX=64 -CONFIG_SOC_FLASH_ENCRYPTION_XTS_AES=y -CONFIG_SOC_FLASH_ENCRYPTION_XTS_AES_OPTIONS=y -CONFIG_SOC_FLASH_ENCRYPTION_XTS_AES_128=y -CONFIG_SOC_FLASH_ENCRYPTION_XTS_AES_256=y -CONFIG_SOC_MEMPROT_CPU_PREFETCH_PAD_SIZE=16 -CONFIG_SOC_MEMPROT_MEM_ALIGN_SIZE=256 -CONFIG_SOC_PHY_DIG_REGS_MEM_SIZE=21 -CONFIG_SOC_MAC_BB_PD_MEM_SIZE=192 -CONFIG_SOC_WIFI_LIGHT_SLEEP_CLK_WIDTH=12 -CONFIG_SOC_SPI_MEM_SUPPORT_AUTO_WAIT_IDLE=y -CONFIG_SOC_SPI_MEM_SUPPORT_AUTO_SUSPEND=y -CONFIG_SOC_SPI_MEM_SUPPORT_AUTO_RESUME=y -CONFIG_SOC_SPI_MEM_SUPPORT_SW_SUSPEND=y -CONFIG_SOC_SPI_MEM_SUPPORT_OPI_MODE=y -CONFIG_SOC_SPI_MEM_SUPPORT_TIME_TUNING=y -CONFIG_SOC_SPI_MEM_SUPPORT_CONFIG_GPIO_BY_EFUSE=y -CONFIG_SOC_SPI_MEM_SUPPORT_WRAP=y -CONFIG_SOC_COEX_HW_PTI=y -CONFIG_SOC_EXTERNAL_COEX_LEADER_TX_LINE=y -CONFIG_SOC_SDMMC_USE_GPIO_MATRIX=y -CONFIG_SOC_SDMMC_NUM_SLOTS=2 -CONFIG_SOC_SDMMC_SUPPORT_XTAL_CLOCK=y -CONFIG_SOC_TEMPERATURE_SENSOR_SUPPORT_FAST_RC=y -CONFIG_SOC_WIFI_HW_TSF=y -CONFIG_SOC_WIFI_FTM_SUPPORT=y -CONFIG_SOC_WIFI_GCMP_SUPPORT=y -CONFIG_SOC_WIFI_WAPI_SUPPORT=y -CONFIG_SOC_WIFI_CSI_SUPPORT=y -CONFIG_SOC_WIFI_MESH_SUPPORT=y -CONFIG_SOC_WIFI_SUPPORT_VARIABLE_BEACON_WINDOW=y -CONFIG_SOC_BLE_SUPPORTED=y -CONFIG_SOC_BLE_MESH_SUPPORTED=y -CONFIG_SOC_BLE_50_SUPPORTED=y -CONFIG_SOC_BLE_DEVICE_PRIVACY_SUPPORTED=y -CONFIG_SOC_BLUFI_SUPPORTED=y -CONFIG_SOC_ULP_HAS_ADC=y -CONFIG_IDF_CMAKE=y -CONFIG_IDF_TARGET_ARCH_XTENSA=y -CONFIG_IDF_TARGET_ARCH="xtensa" -CONFIG_IDF_TARGET="esp32s3" -CONFIG_IDF_TARGET_ESP32S3=y -CONFIG_IDF_FIRMWARE_CHIP_ID=0x0009 - -# -# Build type -# -CONFIG_APP_BUILD_TYPE_APP_2NDBOOT=y -# CONFIG_APP_BUILD_TYPE_RAM is not set -CONFIG_APP_BUILD_GENERATE_BINARIES=y -CONFIG_APP_BUILD_BOOTLOADER=y -CONFIG_APP_BUILD_USE_FLASH_SECTIONS=y -# CONFIG_APP_REPRODUCIBLE_BUILD is not set -# CONFIG_APP_NO_BLOBS is not set -# end of Build type - -# -# Bootloader config -# -CONFIG_BOOTLOADER_OFFSET_IN_FLASH=0x0 -CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_SIZE=y -# CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_DEBUG is not set -# CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_PERF is not set -# CONFIG_BOOTLOADER_COMPILER_OPTIMIZATION_NONE is not set -# CONFIG_BOOTLOADER_LOG_LEVEL_NONE is not set -# CONFIG_BOOTLOADER_LOG_LEVEL_ERROR is not set -# CONFIG_BOOTLOADER_LOG_LEVEL_WARN is not set -CONFIG_BOOTLOADER_LOG_LEVEL_INFO=y -# CONFIG_BOOTLOADER_LOG_LEVEL_DEBUG is not set -# CONFIG_BOOTLOADER_LOG_LEVEL_VERBOSE is not set -CONFIG_BOOTLOADER_LOG_LEVEL=3 -CONFIG_BOOTLOADER_VDDSDIO_BOOST_1_9V=y -# CONFIG_BOOTLOADER_FACTORY_RESET is not set -# CONFIG_BOOTLOADER_APP_TEST is not set -CONFIG_BOOTLOADER_REGION_PROTECTION_ENABLE=y -CONFIG_BOOTLOADER_WDT_ENABLE=y -# CONFIG_BOOTLOADER_WDT_DISABLE_IN_USER_CODE is not set -CONFIG_BOOTLOADER_WDT_TIME_MS=9000 -# CONFIG_BOOTLOADER_APP_ROLLBACK_ENABLE is not set -# CONFIG_BOOTLOADER_SKIP_VALIDATE_IN_DEEP_SLEEP is not set -# CONFIG_BOOTLOADER_SKIP_VALIDATE_ON_POWER_ON is not set -# CONFIG_BOOTLOADER_SKIP_VALIDATE_ALWAYS is not set -CONFIG_BOOTLOADER_RESERVE_RTC_SIZE=0 -# CONFIG_BOOTLOADER_CUSTOM_RESERVE_RTC is not set -CONFIG_BOOTLOADER_FLASH_XMC_SUPPORT=y -# end of Bootloader config - -# -# Security features -# -CONFIG_SECURE_BOOT_V2_RSA_SUPPORTED=y -CONFIG_SECURE_BOOT_V2_PREFERRED=y -# CONFIG_SECURE_SIGNED_APPS_NO_SECURE_BOOT is not set -# CONFIG_SECURE_BOOT is not set -# CONFIG_SECURE_FLASH_ENC_ENABLED is not set -CONFIG_SECURE_ROM_DL_MODE_ENABLED=y -# end of Security features - -# -# Application manager -# -CONFIG_APP_COMPILE_TIME_DATE=y -# CONFIG_APP_EXCLUDE_PROJECT_VER_VAR is not set -# CONFIG_APP_EXCLUDE_PROJECT_NAME_VAR is not set -# CONFIG_APP_PROJECT_VER_FROM_CONFIG is not set -CONFIG_APP_RETRIEVE_LEN_ELF_SHA=16 -# end of Application manager - -CONFIG_ESP_ROM_HAS_CRC_LE=y -CONFIG_ESP_ROM_HAS_CRC_BE=y -CONFIG_ESP_ROM_HAS_MZ_CRC32=y -CONFIG_ESP_ROM_HAS_JPEG_DECODE=y -CONFIG_ESP_ROM_UART_CLK_IS_XTAL=y -CONFIG_ESP_ROM_HAS_RETARGETABLE_LOCKING=y -CONFIG_ESP_ROM_USB_SERIAL_DEVICE_NUM=4 -CONFIG_ESP_ROM_HAS_ERASE_0_REGION_BUG=y -CONFIG_ESP_ROM_GET_CLK_FREQ=y -CONFIG_ESP_ROM_HAS_HAL_WDT=y -CONFIG_ESP_ROM_NEEDS_SWSETUP_WORKAROUND=y -CONFIG_ESP_ROM_HAS_LAYOUT_TABLE=y -CONFIG_ESP_ROM_HAS_SPI_FLASH=y -CONFIG_ESP_ROM_HAS_ETS_PRINTF_BUG=y -CONFIG_ESP_ROM_HAS_NEWLIB_NANO_FORMAT=y -CONFIG_ESP_ROM_NEEDS_SET_CACHE_MMU_SIZE=y -CONFIG_ESP_ROM_RAM_APP_NEEDS_MMU_INIT=y -CONFIG_ESP_ROM_HAS_FLASH_COUNT_PAGES_BUG=y -CONFIG_ESP_ROM_HAS_CACHE_SUSPEND_WAITI_BUG=y -CONFIG_ESP_ROM_HAS_CACHE_WRITEBACK_BUG=y - -# -# Boot ROM Behavior -# -CONFIG_BOOT_ROM_LOG_ALWAYS_ON=y -# CONFIG_BOOT_ROM_LOG_ALWAYS_OFF is not set -# CONFIG_BOOT_ROM_LOG_ON_GPIO_HIGH is not set -# CONFIG_BOOT_ROM_LOG_ON_GPIO_LOW is not set -# end of Boot ROM Behavior - -# -# Serial flasher config -# -# CONFIG_ESPTOOLPY_NO_STUB is not set -# CONFIG_ESPTOOLPY_OCT_FLASH is not set -CONFIG_ESPTOOLPY_FLASH_MODE_AUTO_DETECT=y -# CONFIG_ESPTOOLPY_FLASHMODE_QIO is not set -# CONFIG_ESPTOOLPY_FLASHMODE_QOUT is not set -CONFIG_ESPTOOLPY_FLASHMODE_DIO=y -# CONFIG_ESPTOOLPY_FLASHMODE_DOUT is not set -CONFIG_ESPTOOLPY_FLASH_SAMPLE_MODE_STR=y -CONFIG_ESPTOOLPY_FLASHMODE="dio" -# CONFIG_ESPTOOLPY_FLASHFREQ_120M is not set -CONFIG_ESPTOOLPY_FLASHFREQ_80M=y -# CONFIG_ESPTOOLPY_FLASHFREQ_40M is not set -# CONFIG_ESPTOOLPY_FLASHFREQ_20M is not set -CONFIG_ESPTOOLPY_FLASHFREQ_80M_DEFAULT=y -CONFIG_ESPTOOLPY_FLASHFREQ="80m" -# CONFIG_ESPTOOLPY_FLASHSIZE_1MB is not set -CONFIG_ESPTOOLPY_FLASHSIZE_2MB=y -# CONFIG_ESPTOOLPY_FLASHSIZE_4MB is not set -# CONFIG_ESPTOOLPY_FLASHSIZE_8MB is not set -# CONFIG_ESPTOOLPY_FLASHSIZE_16MB is not set -# CONFIG_ESPTOOLPY_FLASHSIZE_32MB is not set -# CONFIG_ESPTOOLPY_FLASHSIZE_64MB is not set -# CONFIG_ESPTOOLPY_FLASHSIZE_128MB is not set -CONFIG_ESPTOOLPY_FLASHSIZE="2MB" -# CONFIG_ESPTOOLPY_HEADER_FLASHSIZE_UPDATE is not set -CONFIG_ESPTOOLPY_BEFORE_RESET=y -# CONFIG_ESPTOOLPY_BEFORE_NORESET is not set -CONFIG_ESPTOOLPY_BEFORE="default_reset" -CONFIG_ESPTOOLPY_AFTER_RESET=y -# CONFIG_ESPTOOLPY_AFTER_NORESET is not set -CONFIG_ESPTOOLPY_AFTER="hard_reset" -CONFIG_ESPTOOLPY_MONITOR_BAUD=115200 -# end of Serial flasher config - -# -# Partition Table -# -CONFIG_PARTITION_TABLE_SINGLE_APP=y -# CONFIG_PARTITION_TABLE_SINGLE_APP_LARGE is not set -# CONFIG_PARTITION_TABLE_TWO_OTA is not set -# CONFIG_PARTITION_TABLE_CUSTOM is not set -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions.csv" -CONFIG_PARTITION_TABLE_FILENAME="partitions_singleapp.csv" -CONFIG_PARTITION_TABLE_OFFSET=0x8000 -CONFIG_PARTITION_TABLE_MD5=y -# end of Partition Table - -# -# Compiler options -# -CONFIG_COMPILER_OPTIMIZATION_DEFAULT=y -# CONFIG_COMPILER_OPTIMIZATION_SIZE is not set -# CONFIG_COMPILER_OPTIMIZATION_PERF is not set -# CONFIG_COMPILER_OPTIMIZATION_NONE is not set -CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_ENABLE=y -# CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT is not set -# CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE is not set -CONFIG_COMPILER_FLOAT_LIB_FROM_GCCLIB=y -CONFIG_COMPILER_OPTIMIZATION_ASSERTION_LEVEL=2 -# CONFIG_COMPILER_OPTIMIZATION_CHECKS_SILENT is not set -CONFIG_COMPILER_HIDE_PATHS_MACROS=y -# CONFIG_COMPILER_CXX_EXCEPTIONS is not set -# CONFIG_COMPILER_CXX_RTTI is not set -CONFIG_COMPILER_STACK_CHECK_MODE_NONE=y -# CONFIG_COMPILER_STACK_CHECK_MODE_NORM is not set -# CONFIG_COMPILER_STACK_CHECK_MODE_STRONG is not set -# CONFIG_COMPILER_STACK_CHECK_MODE_ALL is not set -# CONFIG_COMPILER_WARN_WRITE_STRINGS is not set -# CONFIG_COMPILER_DISABLE_GCC12_WARNINGS is not set -# CONFIG_COMPILER_DUMP_RTL_FILES is not set -# end of Compiler options - -# -# Component config -# - -# -# Application Level Tracing -# -# CONFIG_APPTRACE_DEST_JTAG is not set -CONFIG_APPTRACE_DEST_NONE=y -# CONFIG_APPTRACE_DEST_UART1 is not set -# CONFIG_APPTRACE_DEST_UART2 is not set -# CONFIG_APPTRACE_DEST_USB_CDC is not set -CONFIG_APPTRACE_DEST_UART_NONE=y -CONFIG_APPTRACE_UART_TASK_PRIO=1 -CONFIG_APPTRACE_LOCK_ENABLE=y -# end of Application Level Tracing - -# -# Bluetooth -# -# CONFIG_BT_ENABLED is not set -# end of Bluetooth - -# -# Driver Configurations -# - -# -# Legacy ADC Configuration -# -# CONFIG_ADC_SUPPRESS_DEPRECATE_WARN is not set - -# -# Legacy ADC Calibration Configuration -# -# CONFIG_ADC_CALI_SUPPRESS_DEPRECATE_WARN is not set -# end of Legacy ADC Calibration Configuration -# end of Legacy ADC Configuration - -# -# SPI Configuration -# -# CONFIG_SPI_MASTER_IN_IRAM is not set -CONFIG_SPI_MASTER_ISR_IN_IRAM=y -# CONFIG_SPI_SLAVE_IN_IRAM is not set -CONFIG_SPI_SLAVE_ISR_IN_IRAM=y -# end of SPI Configuration - -# -# TWAI Configuration -# -# CONFIG_TWAI_ISR_IN_IRAM is not set -CONFIG_TWAI_ERRATA_FIX_LISTEN_ONLY_DOM=y -# end of TWAI Configuration - -# -# Temperature sensor Configuration -# -# CONFIG_TEMP_SENSOR_SUPPRESS_DEPRECATE_WARN is not set -# CONFIG_TEMP_SENSOR_ENABLE_DEBUG_LOG is not set -# end of Temperature sensor Configuration - -# -# UART Configuration -# -# CONFIG_UART_ISR_IN_IRAM is not set -# end of UART Configuration - -# -# GPIO Configuration -# -# CONFIG_GPIO_CTRL_FUNC_IN_IRAM is not set -# end of GPIO Configuration - -# -# Sigma Delta Modulator Configuration -# -# CONFIG_SDM_CTRL_FUNC_IN_IRAM is not set -# CONFIG_SDM_SUPPRESS_DEPRECATE_WARN is not set -# CONFIG_SDM_ENABLE_DEBUG_LOG is not set -# end of Sigma Delta Modulator Configuration - -# -# GPTimer Configuration -# -# CONFIG_GPTIMER_CTRL_FUNC_IN_IRAM is not set -# CONFIG_GPTIMER_ISR_IRAM_SAFE is not set -# CONFIG_GPTIMER_SUPPRESS_DEPRECATE_WARN is not set -# CONFIG_GPTIMER_ENABLE_DEBUG_LOG is not set -# end of GPTimer Configuration - -# -# PCNT Configuration -# -# CONFIG_PCNT_CTRL_FUNC_IN_IRAM is not set -# CONFIG_PCNT_ISR_IRAM_SAFE is not set -# CONFIG_PCNT_SUPPRESS_DEPRECATE_WARN is not set -# CONFIG_PCNT_ENABLE_DEBUG_LOG is not set -# end of PCNT Configuration - -# -# RMT Configuration -# -# CONFIG_RMT_ISR_IRAM_SAFE is not set -# CONFIG_RMT_SUPPRESS_DEPRECATE_WARN is not set -# CONFIG_RMT_ENABLE_DEBUG_LOG is not set -# end of RMT Configuration - -# -# MCPWM Configuration -# -# CONFIG_MCPWM_ISR_IRAM_SAFE is not set -# CONFIG_MCPWM_CTRL_FUNC_IN_IRAM is not set -# CONFIG_MCPWM_SUPPRESS_DEPRECATE_WARN is not set -# CONFIG_MCPWM_ENABLE_DEBUG_LOG is not set -# end of MCPWM Configuration - -# -# I2S Configuration -# -# CONFIG_I2S_ISR_IRAM_SAFE is not set -# CONFIG_I2S_SUPPRESS_DEPRECATE_WARN is not set -# CONFIG_I2S_ENABLE_DEBUG_LOG is not set -# end of I2S Configuration - -# -# USB Serial/JTAG Configuration -# -# end of USB Serial/JTAG Configuration -# end of Driver Configurations - -# -# eFuse Bit Manager -# -# CONFIG_EFUSE_CUSTOM_TABLE is not set -# CONFIG_EFUSE_VIRTUAL is not set -CONFIG_EFUSE_MAX_BLK_LEN=256 -# end of eFuse Bit Manager - -# -# ESP-TLS -# -CONFIG_ESP_TLS_USING_MBEDTLS=y -CONFIG_ESP_TLS_USE_DS_PERIPHERAL=y -# CONFIG_ESP_TLS_CLIENT_SESSION_TICKETS is not set -# CONFIG_ESP_TLS_SERVER is not set -# CONFIG_ESP_TLS_PSK_VERIFICATION is not set -# CONFIG_ESP_TLS_INSECURE is not set -# end of ESP-TLS - -# -# ADC and ADC Calibration -# -# CONFIG_ADC_ONESHOT_CTRL_FUNC_IN_IRAM is not set -# CONFIG_ADC_CONTINUOUS_ISR_IRAM_SAFE is not set -# CONFIG_ADC_CONTINUOUS_FORCE_USE_ADC2_ON_C3_S3 is not set -# end of ADC and ADC Calibration - -# -# Wireless Coexistence -# -# CONFIG_ESP_COEX_EXTERNAL_COEXIST_ENABLE is not set -# end of Wireless Coexistence - -# -# Common ESP-related -# -CONFIG_ESP_ERR_TO_NAME_LOOKUP=y -# end of Common ESP-related - -# -# Ethernet -# -CONFIG_ETH_ENABLED=y -CONFIG_ETH_USE_SPI_ETHERNET=y -# CONFIG_ETH_SPI_ETHERNET_DM9051 is not set -# CONFIG_ETH_SPI_ETHERNET_W5500 is not set -# CONFIG_ETH_SPI_ETHERNET_KSZ8851SNL is not set -# CONFIG_ETH_USE_OPENETH is not set -# CONFIG_ETH_TRANSMIT_MUTEX is not set -# end of Ethernet - -# -# Event Loop Library -# -# CONFIG_ESP_EVENT_LOOP_PROFILING is not set -CONFIG_ESP_EVENT_POST_FROM_ISR=y -CONFIG_ESP_EVENT_POST_FROM_IRAM_ISR=y -# end of Event Loop Library - -# -# GDB Stub -# -# end of GDB Stub - -# -# ESP HTTP client -# -CONFIG_ESP_HTTP_CLIENT_ENABLE_HTTPS=y -# CONFIG_ESP_HTTP_CLIENT_ENABLE_BASIC_AUTH is not set -# CONFIG_ESP_HTTP_CLIENT_ENABLE_DIGEST_AUTH is not set -# end of ESP HTTP client - -# -# HTTP Server -# -CONFIG_HTTPD_MAX_REQ_HDR_LEN=512 -CONFIG_HTTPD_MAX_URI_LEN=512 -CONFIG_HTTPD_ERR_RESP_NO_DELAY=y -CONFIG_HTTPD_PURGE_BUF_LEN=32 -# CONFIG_HTTPD_LOG_PURGE_DATA is not set -# CONFIG_HTTPD_WS_SUPPORT is not set -# CONFIG_HTTPD_QUEUE_WORK_BLOCKING is not set -# end of HTTP Server - -# -# ESP HTTPS OTA -# -# CONFIG_ESP_HTTPS_OTA_DECRYPT_CB is not set -# CONFIG_ESP_HTTPS_OTA_ALLOW_HTTP is not set -# end of ESP HTTPS OTA - -# -# ESP HTTPS server -# -# CONFIG_ESP_HTTPS_SERVER_ENABLE is not set -# end of ESP HTTPS server - -# -# Hardware Settings -# - -# -# Chip revision -# -CONFIG_ESP32S3_REV_MIN_0=y -# CONFIG_ESP32S3_REV_MIN_1 is not set -# CONFIG_ESP32S3_REV_MIN_2 is not set -CONFIG_ESP32S3_REV_MIN_FULL=0 -CONFIG_ESP_REV_MIN_FULL=0 - -# -# Maximum Supported ESP32-S3 Revision (Rev v0.99) -# -CONFIG_ESP32S3_REV_MAX_FULL=99 -CONFIG_ESP_REV_MAX_FULL=99 -# end of Chip revision - -# -# MAC Config -# -CONFIG_ESP_MAC_ADDR_UNIVERSE_WIFI_STA=y -CONFIG_ESP_MAC_ADDR_UNIVERSE_WIFI_AP=y -CONFIG_ESP_MAC_ADDR_UNIVERSE_BT=y -CONFIG_ESP_MAC_ADDR_UNIVERSE_ETH=y -CONFIG_ESP_MAC_UNIVERSAL_MAC_ADDRESSES_FOUR=y -# CONFIG_ESP32S3_UNIVERSAL_MAC_ADDRESSES_TWO is not set -CONFIG_ESP32S3_UNIVERSAL_MAC_ADDRESSES_FOUR=y -CONFIG_ESP32S3_UNIVERSAL_MAC_ADDRESSES=4 -# end of MAC Config - -# -# Sleep Config -# -# CONFIG_ESP_SLEEP_POWER_DOWN_FLASH is not set -CONFIG_ESP_SLEEP_FLASH_LEAKAGE_WORKAROUND=y -CONFIG_ESP_SLEEP_MSPI_NEED_ALL_IO_PU=y -CONFIG_ESP_SLEEP_RTC_BUS_ISO_WORKAROUND=y -CONFIG_ESP_SLEEP_GPIO_RESET_WORKAROUND=y -CONFIG_ESP_SLEEP_DEEP_SLEEP_WAKEUP_DELAY=2000 -# end of Sleep Config - -# -# RTC Clock Config -# -CONFIG_RTC_CLK_SRC_INT_RC=y -# CONFIG_RTC_CLK_SRC_EXT_CRYS is not set -# CONFIG_RTC_CLK_SRC_EXT_OSC is not set -# CONFIG_RTC_CLK_SRC_INT_8MD256 is not set -CONFIG_RTC_CLK_CAL_CYCLES=1024 -# end of RTC Clock Config - -# -# Peripheral Control -# -CONFIG_PERIPH_CTRL_FUNC_IN_IRAM=y -# end of Peripheral Control - -# -# GDMA Configuration -# -# CONFIG_GDMA_CTRL_FUNC_IN_IRAM is not set -# CONFIG_GDMA_ISR_IRAM_SAFE is not set -# end of GDMA Configuration - -# -# Main XTAL Config -# -CONFIG_XTAL_FREQ_40=y -CONFIG_XTAL_FREQ=40 -# end of Main XTAL Config -# end of Hardware Settings - -# -# LCD and Touch Panel -# - -# -# LCD Touch Drivers are maintained in the IDF Component Registry -# - -# -# LCD Peripheral Configuration -# -CONFIG_LCD_PANEL_IO_FORMAT_BUF_SIZE=32 -# CONFIG_LCD_ENABLE_DEBUG_LOG is not set -# CONFIG_LCD_RGB_ISR_IRAM_SAFE is not set -# CONFIG_LCD_RGB_RESTART_IN_VSYNC is not set -# end of LCD Peripheral Configuration -# end of LCD and Touch Panel - -# -# ESP NETIF Adapter -# -CONFIG_ESP_NETIF_IP_LOST_TIMER_INTERVAL=120 -CONFIG_ESP_NETIF_TCPIP_LWIP=y -# CONFIG_ESP_NETIF_LOOPBACK is not set -CONFIG_ESP_NETIF_USES_TCPIP_WITH_BSD_API=y -# CONFIG_ESP_NETIF_RECEIVE_REPORT_ERRORS is not set -# CONFIG_ESP_NETIF_L2_TAP is not set -# CONFIG_ESP_NETIF_BRIDGE_EN is not set -# end of ESP NETIF Adapter - -# -# Partition API Configuration -# -# end of Partition API Configuration - -# -# PHY -# -CONFIG_ESP_PHY_CALIBRATION_AND_DATA_STORAGE=y -# CONFIG_ESP_PHY_INIT_DATA_IN_PARTITION is not set -CONFIG_ESP_PHY_MAX_WIFI_TX_POWER=20 -CONFIG_ESP_PHY_MAX_TX_POWER=20 -# CONFIG_ESP_PHY_REDUCE_TX_POWER is not set -CONFIG_ESP_PHY_ENABLE_USB=y -# CONFIG_ESP_PHY_ENABLE_CERT_TEST is not set -CONFIG_ESP_PHY_RF_CAL_PARTIAL=y -# CONFIG_ESP_PHY_RF_CAL_NONE is not set -# CONFIG_ESP_PHY_RF_CAL_FULL is not set -CONFIG_ESP_PHY_CALIBRATION_MODE=0 -# end of PHY - -# -# Power Management -# -# CONFIG_PM_ENABLE is not set -CONFIG_PM_POWER_DOWN_CPU_IN_LIGHT_SLEEP=y -CONFIG_PM_POWER_DOWN_TAGMEM_IN_LIGHT_SLEEP=y -# end of Power Management - -# -# ESP PSRAM -# -# CONFIG_SPIRAM is not set -# end of ESP PSRAM - -# -# ESP Ringbuf -# -# CONFIG_RINGBUF_PLACE_FUNCTIONS_INTO_FLASH is not set -# end of ESP Ringbuf - -# -# ESP System Settings -# -# CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_80 is not set -CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_160=y -# CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ_240 is not set -CONFIG_ESP_DEFAULT_CPU_FREQ_MHZ=160 - -# -# Cache config -# -CONFIG_ESP32S3_INSTRUCTION_CACHE_16KB=y -# CONFIG_ESP32S3_INSTRUCTION_CACHE_32KB is not set -CONFIG_ESP32S3_INSTRUCTION_CACHE_SIZE=0x4000 -# CONFIG_ESP32S3_INSTRUCTION_CACHE_4WAYS is not set -CONFIG_ESP32S3_INSTRUCTION_CACHE_8WAYS=y -CONFIG_ESP32S3_ICACHE_ASSOCIATED_WAYS=8 -# CONFIG_ESP32S3_INSTRUCTION_CACHE_LINE_16B is not set -CONFIG_ESP32S3_INSTRUCTION_CACHE_LINE_32B=y -CONFIG_ESP32S3_INSTRUCTION_CACHE_LINE_SIZE=32 -# CONFIG_ESP32S3_DATA_CACHE_16KB is not set -CONFIG_ESP32S3_DATA_CACHE_32KB=y -# CONFIG_ESP32S3_DATA_CACHE_64KB is not set -CONFIG_ESP32S3_DATA_CACHE_SIZE=0x8000 -# CONFIG_ESP32S3_DATA_CACHE_4WAYS is not set -CONFIG_ESP32S3_DATA_CACHE_8WAYS=y -CONFIG_ESP32S3_DCACHE_ASSOCIATED_WAYS=8 -# CONFIG_ESP32S3_DATA_CACHE_LINE_16B is not set -CONFIG_ESP32S3_DATA_CACHE_LINE_32B=y -# CONFIG_ESP32S3_DATA_CACHE_LINE_64B is not set -CONFIG_ESP32S3_DATA_CACHE_LINE_SIZE=32 -# end of Cache config - -# -# Memory -# -# CONFIG_ESP32S3_RTCDATA_IN_FAST_MEM is not set -# CONFIG_ESP32S3_USE_FIXED_STATIC_RAM_SIZE is not set -# end of Memory - -# -# Trace memory -# -# CONFIG_ESP32S3_TRAX is not set -CONFIG_ESP32S3_TRACEMEM_RESERVE_DRAM=0x0 -# end of Trace memory - -# CONFIG_ESP_SYSTEM_PANIC_PRINT_HALT is not set -CONFIG_ESP_SYSTEM_PANIC_PRINT_REBOOT=y -# CONFIG_ESP_SYSTEM_PANIC_SILENT_REBOOT is not set -# CONFIG_ESP_SYSTEM_PANIC_GDBSTUB is not set -# CONFIG_ESP_SYSTEM_GDBSTUB_RUNTIME is not set -CONFIG_ESP_SYSTEM_PANIC_REBOOT_DELAY_SECONDS=0 -CONFIG_ESP_SYSTEM_RTC_FAST_MEM_AS_HEAP_DEPCHECK=y -CONFIG_ESP_SYSTEM_ALLOW_RTC_FAST_MEM_AS_HEAP=y - -# -# Memory protection -# -CONFIG_ESP_SYSTEM_MEMPROT_FEATURE=y -CONFIG_ESP_SYSTEM_MEMPROT_FEATURE_LOCK=y -# end of Memory protection - -CONFIG_ESP_SYSTEM_EVENT_QUEUE_SIZE=32 -CONFIG_ESP_SYSTEM_EVENT_TASK_STACK_SIZE=2304 -CONFIG_ESP_MAIN_TASK_STACK_SIZE=3584 -CONFIG_ESP_MAIN_TASK_AFFINITY_CPU0=y -# CONFIG_ESP_MAIN_TASK_AFFINITY_CPU1 is not set -# CONFIG_ESP_MAIN_TASK_AFFINITY_NO_AFFINITY is not set -CONFIG_ESP_MAIN_TASK_AFFINITY=0x0 -CONFIG_ESP_MINIMAL_SHARED_STACK_SIZE=2048 -CONFIG_ESP_CONSOLE_UART_DEFAULT=y -# CONFIG_ESP_CONSOLE_USB_CDC is not set -# CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG is not set -# CONFIG_ESP_CONSOLE_UART_CUSTOM is not set -# CONFIG_ESP_CONSOLE_NONE is not set -# CONFIG_ESP_CONSOLE_SECONDARY_NONE is not set -CONFIG_ESP_CONSOLE_SECONDARY_USB_SERIAL_JTAG=y -CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG_ENABLED=y -CONFIG_ESP_CONSOLE_UART=y -CONFIG_ESP_CONSOLE_MULTIPLE_UART=y -CONFIG_ESP_CONSOLE_UART_NUM=0 -CONFIG_ESP_CONSOLE_UART_BAUDRATE=115200 -CONFIG_ESP_INT_WDT=y -CONFIG_ESP_INT_WDT_TIMEOUT_MS=300 -CONFIG_ESP_INT_WDT_CHECK_CPU1=y -CONFIG_ESP_TASK_WDT_EN=y -CONFIG_ESP_TASK_WDT_INIT=y -# CONFIG_ESP_TASK_WDT_PANIC is not set -CONFIG_ESP_TASK_WDT_TIMEOUT_S=5 -CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU0=y -CONFIG_ESP_TASK_WDT_CHECK_IDLE_TASK_CPU1=y -# CONFIG_ESP_PANIC_HANDLER_IRAM is not set -# CONFIG_ESP_DEBUG_STUBS_ENABLE is not set -CONFIG_ESP_DEBUG_OCDAWARE=y -CONFIG_ESP_SYSTEM_CHECK_INT_LEVEL_4=y - -# -# Brownout Detector -# -CONFIG_ESP_BROWNOUT_DET=y -CONFIG_ESP_BROWNOUT_DET_LVL_SEL_7=y -# CONFIG_ESP_BROWNOUT_DET_LVL_SEL_6 is not set -# CONFIG_ESP_BROWNOUT_DET_LVL_SEL_5 is not set -# CONFIG_ESP_BROWNOUT_DET_LVL_SEL_4 is not set -# CONFIG_ESP_BROWNOUT_DET_LVL_SEL_3 is not set -# CONFIG_ESP_BROWNOUT_DET_LVL_SEL_2 is not set -# CONFIG_ESP_BROWNOUT_DET_LVL_SEL_1 is not set -CONFIG_ESP_BROWNOUT_DET_LVL=7 -# end of Brownout Detector - -CONFIG_ESP_SYSTEM_BROWNOUT_INTR=y -# end of ESP System Settings - -# -# IPC (Inter-Processor Call) -# -CONFIG_ESP_IPC_TASK_STACK_SIZE=1280 -CONFIG_ESP_IPC_USES_CALLERS_PRIORITY=y -CONFIG_ESP_IPC_ISR_ENABLE=y -# end of IPC (Inter-Processor Call) - -# -# High resolution timer (esp_timer) -# -# CONFIG_ESP_TIMER_PROFILING is not set -CONFIG_ESP_TIME_FUNCS_USE_RTC_TIMER=y -CONFIG_ESP_TIME_FUNCS_USE_ESP_TIMER=y -CONFIG_ESP_TIMER_TASK_STACK_SIZE=3584 -CONFIG_ESP_TIMER_INTERRUPT_LEVEL=1 -# CONFIG_ESP_TIMER_SHOW_EXPERIMENTAL is not set -CONFIG_ESP_TIMER_TASK_AFFINITY=0x0 -CONFIG_ESP_TIMER_TASK_AFFINITY_CPU0=y -CONFIG_ESP_TIMER_ISR_AFFINITY=0x1 -CONFIG_ESP_TIMER_ISR_AFFINITY_CPU0=y -# CONFIG_ESP_TIMER_SUPPORTS_ISR_DISPATCH_METHOD is not set -CONFIG_ESP_TIMER_IMPL_SYSTIMER=y -# end of High resolution timer (esp_timer) - -# -# Wi-Fi -# -CONFIG_ESP_WIFI_ENABLED=y -CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=10 -CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=32 -# CONFIG_ESP_WIFI_STATIC_TX_BUFFER is not set -CONFIG_ESP_WIFI_DYNAMIC_TX_BUFFER=y -CONFIG_ESP_WIFI_TX_BUFFER_TYPE=1 -CONFIG_ESP_WIFI_DYNAMIC_TX_BUFFER_NUM=32 -# CONFIG_ESP_WIFI_CSI_ENABLED is not set -CONFIG_ESP_WIFI_AMPDU_TX_ENABLED=y -CONFIG_ESP_WIFI_TX_BA_WIN=6 -CONFIG_ESP_WIFI_AMPDU_RX_ENABLED=y -CONFIG_ESP_WIFI_RX_BA_WIN=6 -CONFIG_ESP_WIFI_NVS_ENABLED=y -CONFIG_ESP_WIFI_TASK_PINNED_TO_CORE_0=y -# CONFIG_ESP_WIFI_TASK_PINNED_TO_CORE_1 is not set -CONFIG_ESP_WIFI_SOFTAP_BEACON_MAX_LEN=752 -CONFIG_ESP_WIFI_MGMT_SBUF_NUM=32 -CONFIG_ESP_WIFI_IRAM_OPT=y -CONFIG_ESP_WIFI_RX_IRAM_OPT=y -CONFIG_ESP_WIFI_ENABLE_WPA3_SAE=y -CONFIG_ESP_WIFI_ENABLE_SAE_PK=y -CONFIG_ESP_WIFI_SOFTAP_SAE_SUPPORT=y -CONFIG_ESP_WIFI_ENABLE_WPA3_OWE_STA=y -# CONFIG_ESP_WIFI_SLP_IRAM_OPT is not set -# CONFIG_ESP_WIFI_FTM_ENABLE is not set -CONFIG_ESP_WIFI_STA_DISCONNECTED_PM_ENABLE=y -# CONFIG_ESP_WIFI_GCMP_SUPPORT is not set -# CONFIG_ESP_WIFI_GMAC_SUPPORT is not set -CONFIG_ESP_WIFI_SOFTAP_SUPPORT=y -# CONFIG_ESP_WIFI_SLP_BEACON_LOST_OPT is not set -CONFIG_ESP_WIFI_ESPNOW_MAX_ENCRYPT_NUM=7 -CONFIG_ESP_WIFI_MBEDTLS_CRYPTO=y -CONFIG_ESP_WIFI_MBEDTLS_TLS_CLIENT=y -# CONFIG_ESP_WIFI_WAPI_PSK is not set -# CONFIG_ESP_WIFI_SUITE_B_192 is not set -# CONFIG_ESP_WIFI_11KV_SUPPORT is not set -# CONFIG_ESP_WIFI_MBO_SUPPORT is not set -# CONFIG_ESP_WIFI_DPP_SUPPORT is not set -# CONFIG_ESP_WIFI_11R_SUPPORT is not set -# CONFIG_ESP_WIFI_WPS_SOFTAP_REGISTRAR is not set - -# -# WPS Configuration Options -# -# CONFIG_ESP_WIFI_WPS_STRICT is not set -# CONFIG_ESP_WIFI_WPS_PASSPHRASE is not set -# end of WPS Configuration Options - -# CONFIG_ESP_WIFI_DEBUG_PRINT is not set -# CONFIG_ESP_WIFI_TESTING_OPTIONS is not set -# end of Wi-Fi - -# -# Core dump -# -# CONFIG_ESP_COREDUMP_ENABLE_TO_FLASH is not set -# CONFIG_ESP_COREDUMP_ENABLE_TO_UART is not set -CONFIG_ESP_COREDUMP_ENABLE_TO_NONE=y -# end of Core dump - -# -# FAT Filesystem support -# -CONFIG_FATFS_VOLUME_COUNT=2 -CONFIG_FATFS_LFN_NONE=y -# CONFIG_FATFS_LFN_HEAP is not set -# CONFIG_FATFS_LFN_STACK is not set -# CONFIG_FATFS_SECTOR_512 is not set -CONFIG_FATFS_SECTOR_4096=y -# CONFIG_FATFS_CODEPAGE_DYNAMIC is not set -CONFIG_FATFS_CODEPAGE_437=y -# CONFIG_FATFS_CODEPAGE_720 is not set -# CONFIG_FATFS_CODEPAGE_737 is not set -# CONFIG_FATFS_CODEPAGE_771 is not set -# CONFIG_FATFS_CODEPAGE_775 is not set -# CONFIG_FATFS_CODEPAGE_850 is not set -# CONFIG_FATFS_CODEPAGE_852 is not set -# CONFIG_FATFS_CODEPAGE_855 is not set -# CONFIG_FATFS_CODEPAGE_857 is not set -# CONFIG_FATFS_CODEPAGE_860 is not set -# CONFIG_FATFS_CODEPAGE_861 is not set -# CONFIG_FATFS_CODEPAGE_862 is not set -# CONFIG_FATFS_CODEPAGE_863 is not set -# CONFIG_FATFS_CODEPAGE_864 is not set -# CONFIG_FATFS_CODEPAGE_865 is not set -# CONFIG_FATFS_CODEPAGE_866 is not set -# CONFIG_FATFS_CODEPAGE_869 is not set -# CONFIG_FATFS_CODEPAGE_932 is not set -# CONFIG_FATFS_CODEPAGE_936 is not set -# CONFIG_FATFS_CODEPAGE_949 is not set -# CONFIG_FATFS_CODEPAGE_950 is not set -CONFIG_FATFS_CODEPAGE=437 -CONFIG_FATFS_FS_LOCK=0 -CONFIG_FATFS_TIMEOUT_MS=10000 -CONFIG_FATFS_PER_FILE_CACHE=y -# CONFIG_FATFS_USE_FASTSEEK is not set -CONFIG_FATFS_VFS_FSTAT_BLKSIZE=0 -# end of FAT Filesystem support - -# -# FreeRTOS -# - -# -# Kernel -# -# CONFIG_FREERTOS_SMP is not set -# CONFIG_FREERTOS_UNICORE is not set -CONFIG_FREERTOS_HZ=100 -# CONFIG_FREERTOS_CHECK_STACKOVERFLOW_NONE is not set -# CONFIG_FREERTOS_CHECK_STACKOVERFLOW_PTRVAL is not set -CONFIG_FREERTOS_CHECK_STACKOVERFLOW_CANARY=y -CONFIG_FREERTOS_THREAD_LOCAL_STORAGE_POINTERS=1 -CONFIG_FREERTOS_IDLE_TASK_STACKSIZE=1536 -# CONFIG_FREERTOS_USE_IDLE_HOOK is not set -# CONFIG_FREERTOS_USE_TICK_HOOK is not set -CONFIG_FREERTOS_MAX_TASK_NAME_LEN=16 -# CONFIG_FREERTOS_ENABLE_BACKWARD_COMPATIBILITY is not set -CONFIG_FREERTOS_TIMER_TASK_PRIORITY=1 -CONFIG_FREERTOS_TIMER_TASK_STACK_DEPTH=2048 -CONFIG_FREERTOS_TIMER_QUEUE_LENGTH=10 -CONFIG_FREERTOS_QUEUE_REGISTRY_SIZE=0 -CONFIG_FREERTOS_TASK_NOTIFICATION_ARRAY_ENTRIES=1 -# CONFIG_FREERTOS_USE_TRACE_FACILITY is not set -# CONFIG_FREERTOS_GENERATE_RUN_TIME_STATS is not set -# end of Kernel - -# -# Port -# -CONFIG_FREERTOS_TASK_FUNCTION_WRAPPER=y -# CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK is not set -CONFIG_FREERTOS_TLSP_DELETION_CALLBACKS=y -# CONFIG_FREERTOS_ENABLE_STATIC_TASK_CLEAN_UP is not set -CONFIG_FREERTOS_CHECK_MUTEX_GIVEN_BY_OWNER=y -CONFIG_FREERTOS_ISR_STACKSIZE=1536 -CONFIG_FREERTOS_INTERRUPT_BACKTRACE=y -CONFIG_FREERTOS_TICK_SUPPORT_SYSTIMER=y -CONFIG_FREERTOS_CORETIMER_SYSTIMER_LVL1=y -# CONFIG_FREERTOS_CORETIMER_SYSTIMER_LVL3 is not set -CONFIG_FREERTOS_SYSTICK_USES_SYSTIMER=y -# CONFIG_FREERTOS_PLACE_FUNCTIONS_INTO_FLASH is not set -# CONFIG_FREERTOS_PLACE_SNAPSHOT_FUNS_INTO_FLASH is not set -# CONFIG_FREERTOS_CHECK_PORT_CRITICAL_COMPLIANCE is not set -CONFIG_FREERTOS_ENABLE_TASK_SNAPSHOT=y -# end of Port - -CONFIG_FREERTOS_NO_AFFINITY=0x7FFFFFFF -CONFIG_FREERTOS_SUPPORT_STATIC_ALLOCATION=y -CONFIG_FREERTOS_DEBUG_OCDAWARE=y -# end of FreeRTOS - -# -# Hardware Abstraction Layer (HAL) and Low Level (LL) -# -CONFIG_HAL_ASSERTION_EQUALS_SYSTEM=y -# CONFIG_HAL_ASSERTION_DISABLE is not set -# CONFIG_HAL_ASSERTION_SILENT is not set -# CONFIG_HAL_ASSERTION_ENABLE is not set -CONFIG_HAL_DEFAULT_ASSERTION_LEVEL=2 -CONFIG_HAL_WDT_USE_ROM_IMPL=y -CONFIG_HAL_SPI_MASTER_FUNC_IN_IRAM=y -CONFIG_HAL_SPI_SLAVE_FUNC_IN_IRAM=y -# end of Hardware Abstraction Layer (HAL) and Low Level (LL) - -# -# Heap memory debugging -# -CONFIG_HEAP_POISONING_DISABLED=y -# CONFIG_HEAP_POISONING_LIGHT is not set -# CONFIG_HEAP_POISONING_COMPREHENSIVE is not set -CONFIG_HEAP_TRACING_OFF=y -# CONFIG_HEAP_TRACING_STANDALONE is not set -# CONFIG_HEAP_TRACING_TOHOST is not set -# CONFIG_HEAP_USE_HOOKS is not set -# CONFIG_HEAP_ABORT_WHEN_ALLOCATION_FAILS is not set -# CONFIG_HEAP_PLACE_FUNCTION_INTO_FLASH is not set -# end of Heap memory debugging - -CONFIG_IEEE802154_CCA_THRESHOLD=-60 -CONFIG_IEEE802154_PENDING_TABLE_SIZE=20 - -# -# Log output -# -# CONFIG_LOG_DEFAULT_LEVEL_NONE is not set -# CONFIG_LOG_DEFAULT_LEVEL_ERROR is not set -# CONFIG_LOG_DEFAULT_LEVEL_WARN is not set -CONFIG_LOG_DEFAULT_LEVEL_INFO=y -# CONFIG_LOG_DEFAULT_LEVEL_DEBUG is not set -# CONFIG_LOG_DEFAULT_LEVEL_VERBOSE is not set -CONFIG_LOG_DEFAULT_LEVEL=3 -CONFIG_LOG_MAXIMUM_EQUALS_DEFAULT=y -# CONFIG_LOG_MAXIMUM_LEVEL_DEBUG is not set -# CONFIG_LOG_MAXIMUM_LEVEL_VERBOSE is not set -CONFIG_LOG_MAXIMUM_LEVEL=3 -CONFIG_LOG_COLORS=y -CONFIG_LOG_TIMESTAMP_SOURCE_RTOS=y -# CONFIG_LOG_TIMESTAMP_SOURCE_SYSTEM is not set -# end of Log output - -# -# LWIP -# -CONFIG_LWIP_LOCAL_HOSTNAME="espressif" -# CONFIG_LWIP_NETIF_API is not set -# CONFIG_LWIP_TCPIP_CORE_LOCKING is not set -# CONFIG_LWIP_CHECK_THREAD_SAFETY is not set -CONFIG_LWIP_DNS_SUPPORT_MDNS_QUERIES=y -# CONFIG_LWIP_L2_TO_L3_COPY is not set -# CONFIG_LWIP_IRAM_OPTIMIZATION is not set -CONFIG_LWIP_TIMERS_ONDEMAND=y -CONFIG_LWIP_MAX_SOCKETS=10 -# CONFIG_LWIP_USE_ONLY_LWIP_SELECT is not set -# CONFIG_LWIP_SO_LINGER is not set -CONFIG_LWIP_SO_REUSE=y -CONFIG_LWIP_SO_REUSE_RXTOALL=y -# CONFIG_LWIP_SO_RCVBUF is not set -# CONFIG_LWIP_NETBUF_RECVINFO is not set -CONFIG_LWIP_IP4_FRAG=y -CONFIG_LWIP_IP6_FRAG=y -# CONFIG_LWIP_IP4_REASSEMBLY is not set -# CONFIG_LWIP_IP6_REASSEMBLY is not set -CONFIG_LWIP_IP_REASS_MAX_PBUFS=10 -# CONFIG_LWIP_IP_FORWARD is not set -# CONFIG_LWIP_STATS is not set -CONFIG_LWIP_ESP_GRATUITOUS_ARP=y -CONFIG_LWIP_GARP_TMR_INTERVAL=60 -CONFIG_LWIP_ESP_MLDV6_REPORT=y -CONFIG_LWIP_MLDV6_TMR_INTERVAL=40 -CONFIG_LWIP_TCPIP_RECVMBOX_SIZE=32 -CONFIG_LWIP_DHCP_DOES_ARP_CHECK=y -# CONFIG_LWIP_DHCP_DISABLE_CLIENT_ID is not set -CONFIG_LWIP_DHCP_DISABLE_VENDOR_CLASS_ID=y -# CONFIG_LWIP_DHCP_RESTORE_LAST_IP is not set -CONFIG_LWIP_DHCP_OPTIONS_LEN=68 -CONFIG_LWIP_NUM_NETIF_CLIENT_DATA=0 -CONFIG_LWIP_DHCP_COARSE_TIMER_SECS=1 - -# -# DHCP server -# -CONFIG_LWIP_DHCPS=y -CONFIG_LWIP_DHCPS_LEASE_UNIT=60 -CONFIG_LWIP_DHCPS_MAX_STATION_NUM=8 -# end of DHCP server - -# CONFIG_LWIP_AUTOIP is not set -CONFIG_LWIP_IPV4=y -CONFIG_LWIP_IPV6=y -# CONFIG_LWIP_IPV6_AUTOCONFIG is not set -CONFIG_LWIP_IPV6_NUM_ADDRESSES=3 -# CONFIG_LWIP_IPV6_FORWARD is not set -# CONFIG_LWIP_NETIF_STATUS_CALLBACK is not set -CONFIG_LWIP_NETIF_LOOPBACK=y -CONFIG_LWIP_LOOPBACK_MAX_PBUFS=8 - -# -# TCP -# -CONFIG_LWIP_MAX_ACTIVE_TCP=16 -CONFIG_LWIP_MAX_LISTENING_TCP=16 -CONFIG_LWIP_TCP_HIGH_SPEED_RETRANSMISSION=y -CONFIG_LWIP_TCP_MAXRTX=12 -CONFIG_LWIP_TCP_SYNMAXRTX=12 -CONFIG_LWIP_TCP_MSS=1440 -CONFIG_LWIP_TCP_TMR_INTERVAL=250 -CONFIG_LWIP_TCP_MSL=60000 -CONFIG_LWIP_TCP_FIN_WAIT_TIMEOUT=20000 -CONFIG_LWIP_TCP_SND_BUF_DEFAULT=5744 -CONFIG_LWIP_TCP_WND_DEFAULT=5744 -CONFIG_LWIP_TCP_RECVMBOX_SIZE=6 -CONFIG_LWIP_TCP_QUEUE_OOSEQ=y -# CONFIG_LWIP_TCP_SACK_OUT is not set -CONFIG_LWIP_TCP_OVERSIZE_MSS=y -# CONFIG_LWIP_TCP_OVERSIZE_QUARTER_MSS is not set -# CONFIG_LWIP_TCP_OVERSIZE_DISABLE is not set -CONFIG_LWIP_TCP_RTO_TIME=1500 -# end of TCP - -# -# UDP -# -CONFIG_LWIP_MAX_UDP_PCBS=16 -CONFIG_LWIP_UDP_RECVMBOX_SIZE=6 -# end of UDP - -# -# Checksums -# -# CONFIG_LWIP_CHECKSUM_CHECK_IP is not set -# CONFIG_LWIP_CHECKSUM_CHECK_UDP is not set -CONFIG_LWIP_CHECKSUM_CHECK_ICMP=y -# end of Checksums - -CONFIG_LWIP_TCPIP_TASK_STACK_SIZE=3072 -CONFIG_LWIP_TCPIP_TASK_AFFINITY_NO_AFFINITY=y -# CONFIG_LWIP_TCPIP_TASK_AFFINITY_CPU0 is not set -# CONFIG_LWIP_TCPIP_TASK_AFFINITY_CPU1 is not set -CONFIG_LWIP_TCPIP_TASK_AFFINITY=0x7FFFFFFF -# CONFIG_LWIP_PPP_SUPPORT is not set -CONFIG_LWIP_IPV6_MEMP_NUM_ND6_QUEUE=3 -CONFIG_LWIP_IPV6_ND6_NUM_NEIGHBORS=5 -# CONFIG_LWIP_SLIP_SUPPORT is not set - -# -# ICMP -# -CONFIG_LWIP_ICMP=y -# CONFIG_LWIP_MULTICAST_PING is not set -# CONFIG_LWIP_BROADCAST_PING is not set -# end of ICMP - -# -# LWIP RAW API -# -CONFIG_LWIP_MAX_RAW_PCBS=16 -# end of LWIP RAW API - -# -# SNTP -# -CONFIG_LWIP_SNTP_MAX_SERVERS=1 -# CONFIG_LWIP_DHCP_GET_NTP_SRV is not set -CONFIG_LWIP_SNTP_UPDATE_DELAY=3600000 -# end of SNTP - -CONFIG_LWIP_BRIDGEIF_MAX_PORTS=7 -CONFIG_LWIP_ESP_LWIP_ASSERT=y - -# -# Hooks -# -# CONFIG_LWIP_HOOK_TCP_ISN_NONE is not set -CONFIG_LWIP_HOOK_TCP_ISN_DEFAULT=y -# CONFIG_LWIP_HOOK_TCP_ISN_CUSTOM is not set -CONFIG_LWIP_HOOK_IP6_ROUTE_NONE=y -# CONFIG_LWIP_HOOK_IP6_ROUTE_DEFAULT is not set -# CONFIG_LWIP_HOOK_IP6_ROUTE_CUSTOM is not set -CONFIG_LWIP_HOOK_ND6_GET_GW_NONE=y -# CONFIG_LWIP_HOOK_ND6_GET_GW_DEFAULT is not set -# CONFIG_LWIP_HOOK_ND6_GET_GW_CUSTOM is not set -CONFIG_LWIP_HOOK_IP6_SELECT_SRC_ADDR_NONE=y -# CONFIG_LWIP_HOOK_IP6_SELECT_SRC_ADDR_DEFAULT is not set -# CONFIG_LWIP_HOOK_IP6_SELECT_SRC_ADDR_CUSTOM is not set -CONFIG_LWIP_HOOK_NETCONN_EXT_RESOLVE_NONE=y -# CONFIG_LWIP_HOOK_NETCONN_EXT_RESOLVE_DEFAULT is not set -# CONFIG_LWIP_HOOK_NETCONN_EXT_RESOLVE_CUSTOM is not set -CONFIG_LWIP_HOOK_IP6_INPUT_NONE=y -# CONFIG_LWIP_HOOK_IP6_INPUT_DEFAULT is not set -# CONFIG_LWIP_HOOK_IP6_INPUT_CUSTOM is not set -# end of Hooks - -# CONFIG_LWIP_DEBUG is not set -# end of LWIP - -# -# mbedTLS -# -CONFIG_MBEDTLS_INTERNAL_MEM_ALLOC=y -# CONFIG_MBEDTLS_DEFAULT_MEM_ALLOC is not set -# CONFIG_MBEDTLS_CUSTOM_MEM_ALLOC is not set -CONFIG_MBEDTLS_ASYMMETRIC_CONTENT_LEN=y -CONFIG_MBEDTLS_SSL_IN_CONTENT_LEN=16384 -CONFIG_MBEDTLS_SSL_OUT_CONTENT_LEN=4096 -# CONFIG_MBEDTLS_DYNAMIC_BUFFER is not set -# CONFIG_MBEDTLS_DEBUG is not set - -# -# mbedTLS v3.x related -# -# CONFIG_MBEDTLS_SSL_PROTO_TLS1_3 is not set -# CONFIG_MBEDTLS_SSL_VARIABLE_BUFFER_LENGTH is not set -# CONFIG_MBEDTLS_X509_TRUSTED_CERT_CALLBACK is not set -# CONFIG_MBEDTLS_SSL_CONTEXT_SERIALIZATION is not set -CONFIG_MBEDTLS_SSL_KEEP_PEER_CERTIFICATE=y -CONFIG_MBEDTLS_PKCS7_C=y -# end of mbedTLS v3.x related - -# -# Certificate Bundle -# -CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=y -CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL=y -# CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN is not set -# CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_NONE is not set -# CONFIG_MBEDTLS_CUSTOM_CERTIFICATE_BUNDLE is not set -CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_MAX_CERTS=200 -# end of Certificate Bundle - -# CONFIG_MBEDTLS_ECP_RESTARTABLE is not set -# CONFIG_MBEDTLS_CMAC_C is not set -CONFIG_MBEDTLS_HARDWARE_AES=y -CONFIG_MBEDTLS_AES_USE_INTERRUPT=y -CONFIG_MBEDTLS_HARDWARE_MPI=y -CONFIG_MBEDTLS_MPI_USE_INTERRUPT=y -CONFIG_MBEDTLS_HARDWARE_SHA=y -CONFIG_MBEDTLS_ROM_MD5=y -# CONFIG_MBEDTLS_ATCA_HW_ECDSA_SIGN is not set -# CONFIG_MBEDTLS_ATCA_HW_ECDSA_VERIFY is not set -CONFIG_MBEDTLS_HAVE_TIME=y -# CONFIG_MBEDTLS_PLATFORM_TIME_ALT is not set -# CONFIG_MBEDTLS_HAVE_TIME_DATE is not set -CONFIG_MBEDTLS_ECDSA_DETERMINISTIC=y -CONFIG_MBEDTLS_SHA512_C=y -CONFIG_MBEDTLS_TLS_SERVER_AND_CLIENT=y -# CONFIG_MBEDTLS_TLS_SERVER_ONLY is not set -# CONFIG_MBEDTLS_TLS_CLIENT_ONLY is not set -# CONFIG_MBEDTLS_TLS_DISABLED is not set -CONFIG_MBEDTLS_TLS_SERVER=y -CONFIG_MBEDTLS_TLS_CLIENT=y -CONFIG_MBEDTLS_TLS_ENABLED=y - -# -# TLS Key Exchange Methods -# -# CONFIG_MBEDTLS_PSK_MODES is not set -CONFIG_MBEDTLS_KEY_EXCHANGE_RSA=y -CONFIG_MBEDTLS_KEY_EXCHANGE_ELLIPTIC_CURVE=y -CONFIG_MBEDTLS_KEY_EXCHANGE_ECDHE_RSA=y -CONFIG_MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA=y -CONFIG_MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA=y -CONFIG_MBEDTLS_KEY_EXCHANGE_ECDH_RSA=y -# end of TLS Key Exchange Methods - -CONFIG_MBEDTLS_SSL_RENEGOTIATION=y -CONFIG_MBEDTLS_SSL_PROTO_TLS1_2=y -# CONFIG_MBEDTLS_SSL_PROTO_GMTSSL1_1 is not set -# CONFIG_MBEDTLS_SSL_PROTO_DTLS is not set -CONFIG_MBEDTLS_SSL_ALPN=y -CONFIG_MBEDTLS_CLIENT_SSL_SESSION_TICKETS=y -CONFIG_MBEDTLS_SERVER_SSL_SESSION_TICKETS=y - -# -# Symmetric Ciphers -# -CONFIG_MBEDTLS_AES_C=y -# CONFIG_MBEDTLS_CAMELLIA_C is not set -# CONFIG_MBEDTLS_DES_C is not set -# CONFIG_MBEDTLS_BLOWFISH_C is not set -# CONFIG_MBEDTLS_XTEA_C is not set -CONFIG_MBEDTLS_CCM_C=y -CONFIG_MBEDTLS_GCM_C=y -# CONFIG_MBEDTLS_NIST_KW_C is not set -# end of Symmetric Ciphers - -# CONFIG_MBEDTLS_RIPEMD160_C is not set - -# -# Certificates -# -CONFIG_MBEDTLS_PEM_PARSE_C=y -CONFIG_MBEDTLS_PEM_WRITE_C=y -CONFIG_MBEDTLS_X509_CRL_PARSE_C=y -CONFIG_MBEDTLS_X509_CSR_PARSE_C=y -# end of Certificates - -CONFIG_MBEDTLS_ECP_C=y -# CONFIG_MBEDTLS_DHM_C is not set -CONFIG_MBEDTLS_ECDH_C=y -CONFIG_MBEDTLS_ECDSA_C=y -# CONFIG_MBEDTLS_ECJPAKE_C is not set -CONFIG_MBEDTLS_ECP_DP_SECP192R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_SECP224R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_SECP256R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_SECP384R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_SECP521R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_SECP192K1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_SECP224K1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_SECP256K1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_BP256R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_BP384R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_BP512R1_ENABLED=y -CONFIG_MBEDTLS_ECP_DP_CURVE25519_ENABLED=y -CONFIG_MBEDTLS_ECP_NIST_OPTIM=y -# CONFIG_MBEDTLS_POLY1305_C is not set -# CONFIG_MBEDTLS_CHACHA20_C is not set -# CONFIG_MBEDTLS_HKDF_C is not set -# CONFIG_MBEDTLS_THREADING_C is not set -# CONFIG_MBEDTLS_LARGE_KEY_SOFTWARE_MPI is not set -# CONFIG_MBEDTLS_SECURITY_RISKS is not set -# end of mbedTLS - -# -# ESP-MQTT Configurations -# -CONFIG_MQTT_PROTOCOL_311=y -# CONFIG_MQTT_PROTOCOL_5 is not set -CONFIG_MQTT_TRANSPORT_SSL=y -CONFIG_MQTT_TRANSPORT_WEBSOCKET=y -CONFIG_MQTT_TRANSPORT_WEBSOCKET_SECURE=y -# CONFIG_MQTT_MSG_ID_INCREMENTAL is not set -# CONFIG_MQTT_SKIP_PUBLISH_IF_DISCONNECTED is not set -# CONFIG_MQTT_REPORT_DELETED_MESSAGES is not set -# CONFIG_MQTT_USE_CUSTOM_CONFIG is not set -# CONFIG_MQTT_TASK_CORE_SELECTION_ENABLED is not set -# CONFIG_MQTT_CUSTOM_OUTBOX is not set -# end of ESP-MQTT Configurations - -# -# Newlib -# -CONFIG_NEWLIB_STDOUT_LINE_ENDING_CRLF=y -# CONFIG_NEWLIB_STDOUT_LINE_ENDING_LF is not set -# CONFIG_NEWLIB_STDOUT_LINE_ENDING_CR is not set -# CONFIG_NEWLIB_STDIN_LINE_ENDING_CRLF is not set -# CONFIG_NEWLIB_STDIN_LINE_ENDING_LF is not set -CONFIG_NEWLIB_STDIN_LINE_ENDING_CR=y -# CONFIG_NEWLIB_NANO_FORMAT is not set -CONFIG_NEWLIB_TIME_SYSCALL_USE_RTC_HRT=y -# CONFIG_NEWLIB_TIME_SYSCALL_USE_RTC is not set -# CONFIG_NEWLIB_TIME_SYSCALL_USE_HRT is not set -# CONFIG_NEWLIB_TIME_SYSCALL_USE_NONE is not set -# end of Newlib - -# -# NVS -# -# CONFIG_NVS_ASSERT_ERROR_CHECK is not set -# end of NVS - -# -# OpenThread -# -# CONFIG_OPENTHREAD_ENABLED is not set - -# -# Thread Operational Dataset -# -CONFIG_OPENTHREAD_NETWORK_NAME="OpenThread-ESP" -CONFIG_OPENTHREAD_NETWORK_CHANNEL=15 -CONFIG_OPENTHREAD_NETWORK_PANID=0x1234 -CONFIG_OPENTHREAD_NETWORK_EXTPANID="dead00beef00cafe" -CONFIG_OPENTHREAD_NETWORK_MASTERKEY="00112233445566778899aabbccddeeff" -CONFIG_OPENTHREAD_NETWORK_PSKC="104810e2315100afd6bc9215a6bfac53" -# end of Thread Operational Dataset -# end of OpenThread - -# -# Protocomm -# -CONFIG_ESP_PROTOCOMM_SUPPORT_SECURITY_VERSION_0=y -CONFIG_ESP_PROTOCOMM_SUPPORT_SECURITY_VERSION_1=y -CONFIG_ESP_PROTOCOMM_SUPPORT_SECURITY_VERSION_2=y -# end of Protocomm - -# -# PThreads -# -CONFIG_PTHREAD_TASK_PRIO_DEFAULT=5 -CONFIG_PTHREAD_TASK_STACK_SIZE_DEFAULT=3072 -CONFIG_PTHREAD_STACK_MIN=768 -CONFIG_PTHREAD_DEFAULT_CORE_NO_AFFINITY=y -# CONFIG_PTHREAD_DEFAULT_CORE_0 is not set -# CONFIG_PTHREAD_DEFAULT_CORE_1 is not set -CONFIG_PTHREAD_TASK_CORE_DEFAULT=-1 -CONFIG_PTHREAD_TASK_NAME_DEFAULT="pthread" -# end of PThreads - -# -# MMU Config -# -CONFIG_MMU_PAGE_SIZE_64KB=y -CONFIG_MMU_PAGE_MODE="64KB" -CONFIG_MMU_PAGE_SIZE=0x10000 -# end of MMU Config - -# -# SPI Flash driver -# -# CONFIG_SPI_FLASH_VERIFY_WRITE is not set -# CONFIG_SPI_FLASH_ENABLE_COUNTERS is not set -CONFIG_SPI_FLASH_ROM_DRIVER_PATCH=y -# CONFIG_SPI_FLASH_ROM_IMPL is not set -CONFIG_SPI_FLASH_DANGEROUS_WRITE_ABORTS=y -# CONFIG_SPI_FLASH_DANGEROUS_WRITE_FAILS is not set -# CONFIG_SPI_FLASH_DANGEROUS_WRITE_ALLOWED is not set -# CONFIG_SPI_FLASH_BYPASS_BLOCK_ERASE is not set -CONFIG_SPI_FLASH_YIELD_DURING_ERASE=y -CONFIG_SPI_FLASH_ERASE_YIELD_DURATION_MS=20 -CONFIG_SPI_FLASH_ERASE_YIELD_TICKS=1 -CONFIG_SPI_FLASH_WRITE_CHUNK_SIZE=8192 -# CONFIG_SPI_FLASH_SIZE_OVERRIDE is not set -# CONFIG_SPI_FLASH_CHECK_ERASE_TIMEOUT_DISABLED is not set -# CONFIG_SPI_FLASH_OVERRIDE_CHIP_DRIVER_LIST is not set - -# -# SPI Flash behavior when brownout -# -CONFIG_SPI_FLASH_BROWNOUT_RESET_XMC=y -CONFIG_SPI_FLASH_BROWNOUT_RESET=y -# end of SPI Flash behavior when brownout - -# -# Auto-detect flash chips -# -CONFIG_SPI_FLASH_VENDOR_XMC_SUPPORTED=y -CONFIG_SPI_FLASH_VENDOR_GD_SUPPORTED=y -CONFIG_SPI_FLASH_VENDOR_ISSI_SUPPORTED=y -CONFIG_SPI_FLASH_VENDOR_MXIC_SUPPORTED=y -CONFIG_SPI_FLASH_VENDOR_WINBOND_SUPPORTED=y -CONFIG_SPI_FLASH_VENDOR_BOYA_SUPPORTED=y -CONFIG_SPI_FLASH_VENDOR_TH_SUPPORTED=y -CONFIG_SPI_FLASH_SUPPORT_ISSI_CHIP=y -CONFIG_SPI_FLASH_SUPPORT_MXIC_CHIP=y -CONFIG_SPI_FLASH_SUPPORT_GD_CHIP=y -CONFIG_SPI_FLASH_SUPPORT_WINBOND_CHIP=y -CONFIG_SPI_FLASH_SUPPORT_BOYA_CHIP=y -CONFIG_SPI_FLASH_SUPPORT_TH_CHIP=y -CONFIG_SPI_FLASH_SUPPORT_MXIC_OPI_CHIP=y -# end of Auto-detect flash chips - -CONFIG_SPI_FLASH_ENABLE_ENCRYPTED_READ_WRITE=y -# end of SPI Flash driver - -# -# SPIFFS Configuration -# -CONFIG_SPIFFS_MAX_PARTITIONS=3 - -# -# SPIFFS Cache Configuration -# -CONFIG_SPIFFS_CACHE=y -CONFIG_SPIFFS_CACHE_WR=y -# CONFIG_SPIFFS_CACHE_STATS is not set -# end of SPIFFS Cache Configuration - -CONFIG_SPIFFS_PAGE_CHECK=y -CONFIG_SPIFFS_GC_MAX_RUNS=10 -# CONFIG_SPIFFS_GC_STATS is not set -CONFIG_SPIFFS_PAGE_SIZE=256 -CONFIG_SPIFFS_OBJ_NAME_LEN=32 -# CONFIG_SPIFFS_FOLLOW_SYMLINKS is not set -CONFIG_SPIFFS_USE_MAGIC=y -CONFIG_SPIFFS_USE_MAGIC_LENGTH=y -CONFIG_SPIFFS_META_LENGTH=4 -CONFIG_SPIFFS_USE_MTIME=y - -# -# Debug Configuration -# -# CONFIG_SPIFFS_DBG is not set -# CONFIG_SPIFFS_API_DBG is not set -# CONFIG_SPIFFS_GC_DBG is not set -# CONFIG_SPIFFS_CACHE_DBG is not set -# CONFIG_SPIFFS_CHECK_DBG is not set -# CONFIG_SPIFFS_TEST_VISUALISATION is not set -# end of Debug Configuration -# end of SPIFFS Configuration - -# -# TCP Transport -# - -# -# Websocket -# -CONFIG_WS_TRANSPORT=y -CONFIG_WS_BUFFER_SIZE=1024 -# CONFIG_WS_DYNAMIC_BUFFER is not set -# end of Websocket -# end of TCP Transport - -# -# Ultra Low Power (ULP) Co-processor -# -# CONFIG_ULP_COPROC_ENABLED is not set -# end of Ultra Low Power (ULP) Co-processor - -# -# Unity unit testing library -# -CONFIG_UNITY_ENABLE_FLOAT=y -CONFIG_UNITY_ENABLE_DOUBLE=y -# CONFIG_UNITY_ENABLE_64BIT is not set -# CONFIG_UNITY_ENABLE_COLOR is not set -CONFIG_UNITY_ENABLE_IDF_TEST_RUNNER=y -# CONFIG_UNITY_ENABLE_FIXTURE is not set -# CONFIG_UNITY_ENABLE_BACKTRACE_ON_FAIL is not set -# end of Unity unit testing library - -# -# USB-OTG -# -CONFIG_USB_OTG_SUPPORTED=y -CONFIG_USB_HOST_CONTROL_TRANSFER_MAX_SIZE=256 -CONFIG_USB_HOST_HW_BUFFER_BIAS_BALANCED=y -# CONFIG_USB_HOST_HW_BUFFER_BIAS_IN is not set -# CONFIG_USB_HOST_HW_BUFFER_BIAS_PERIODIC_OUT is not set - -# -# Root Hub configuration -# -CONFIG_USB_HOST_DEBOUNCE_DELAY_MS=250 -CONFIG_USB_HOST_RESET_HOLD_MS=30 -CONFIG_USB_HOST_RESET_RECOVERY_MS=30 -CONFIG_USB_HOST_SET_ADDR_RECOVERY_MS=10 -# end of Root Hub configuration -# end of USB-OTG - -# -# Virtual file system -# -CONFIG_VFS_SUPPORT_IO=y -CONFIG_VFS_SUPPORT_DIR=y -CONFIG_VFS_SUPPORT_SELECT=y -CONFIG_VFS_SUPPRESS_SELECT_DEBUG_OUTPUT=y -CONFIG_VFS_SUPPORT_TERMIOS=y -CONFIG_VFS_MAX_COUNT=8 - -# -# Host File System I/O (Semihosting) -# -CONFIG_VFS_SEMIHOSTFS_MAX_MOUNT_POINTS=1 -# end of Host File System I/O (Semihosting) -# end of Virtual file system - -# -# Wear Levelling -# -# CONFIG_WL_SECTOR_SIZE_512 is not set -CONFIG_WL_SECTOR_SIZE_4096=y -CONFIG_WL_SECTOR_SIZE=4096 -# end of Wear Levelling - -# -# Wi-Fi Provisioning Manager -# -CONFIG_WIFI_PROV_SCAN_MAX_ENTRIES=16 -CONFIG_WIFI_PROV_AUTOSTOP_TIMEOUT=30 -# CONFIG_WIFI_PROV_BLE_FORCE_ENCRYPTION is not set -CONFIG_WIFI_PROV_STA_ALL_CHANNEL_SCAN=y -# CONFIG_WIFI_PROV_STA_FAST_SCAN is not set -# end of Wi-Fi Provisioning Manager -# end of Component config - -# CONFIG_IDF_EXPERIMENTAL_FEATURES is not set - -# Deprecated options for backward compatibility -# CONFIG_APP_BUILD_TYPE_ELF_RAM is not set -# CONFIG_NO_BLOBS is not set -# CONFIG_LOG_BOOTLOADER_LEVEL_NONE is not set -# CONFIG_LOG_BOOTLOADER_LEVEL_ERROR is not set -# CONFIG_LOG_BOOTLOADER_LEVEL_WARN is not set -CONFIG_LOG_BOOTLOADER_LEVEL_INFO=y -# CONFIG_LOG_BOOTLOADER_LEVEL_DEBUG is not set -# CONFIG_LOG_BOOTLOADER_LEVEL_VERBOSE is not set -CONFIG_LOG_BOOTLOADER_LEVEL=3 -# CONFIG_APP_ROLLBACK_ENABLE is not set -# CONFIG_FLASH_ENCRYPTION_ENABLED is not set -# CONFIG_FLASHMODE_QIO is not set -# CONFIG_FLASHMODE_QOUT is not set -CONFIG_FLASHMODE_DIO=y -# CONFIG_FLASHMODE_DOUT is not set -CONFIG_MONITOR_BAUD=115200 -CONFIG_OPTIMIZATION_LEVEL_DEBUG=y -CONFIG_COMPILER_OPTIMIZATION_LEVEL_DEBUG=y -# CONFIG_OPTIMIZATION_LEVEL_RELEASE is not set -# CONFIG_COMPILER_OPTIMIZATION_LEVEL_RELEASE is not set -CONFIG_OPTIMIZATION_ASSERTIONS_ENABLED=y -# CONFIG_OPTIMIZATION_ASSERTIONS_SILENT is not set -# CONFIG_OPTIMIZATION_ASSERTIONS_DISABLED is not set -CONFIG_OPTIMIZATION_ASSERTION_LEVEL=2 -# CONFIG_CXX_EXCEPTIONS is not set -CONFIG_STACK_CHECK_NONE=y -# CONFIG_STACK_CHECK_NORM is not set -# CONFIG_STACK_CHECK_STRONG is not set -# CONFIG_STACK_CHECK_ALL is not set -# CONFIG_WARN_WRITE_STRINGS is not set -# CONFIG_ESP32_APPTRACE_DEST_TRAX is not set -CONFIG_ESP32_APPTRACE_DEST_NONE=y -CONFIG_ESP32_APPTRACE_LOCK_ENABLE=y -# CONFIG_MCPWM_ISR_IN_IRAM is not set -# CONFIG_EXTERNAL_COEX_ENABLE is not set -# CONFIG_ESP_WIFI_EXTERNAL_COEXIST_ENABLE is not set -# CONFIG_EVENT_LOOP_PROFILING is not set -CONFIG_POST_EVENTS_FROM_ISR=y -CONFIG_POST_EVENTS_FROM_IRAM_ISR=y -# CONFIG_OTA_ALLOW_HTTP is not set -# CONFIG_ESP_SYSTEM_PD_FLASH is not set -CONFIG_ESP32S3_DEEP_SLEEP_WAKEUP_DELAY=2000 -CONFIG_ESP32S3_RTC_CLK_SRC_INT_RC=y -# CONFIG_ESP32S3_RTC_CLK_SRC_EXT_CRYS is not set -# CONFIG_ESP32S3_RTC_CLK_SRC_EXT_OSC is not set -# CONFIG_ESP32S3_RTC_CLK_SRC_INT_8MD256 is not set -CONFIG_ESP32S3_RTC_CLK_CAL_CYCLES=1024 -CONFIG_ESP32_PHY_CALIBRATION_AND_DATA_STORAGE=y -# CONFIG_ESP32_PHY_INIT_DATA_IN_PARTITION is not set -CONFIG_ESP32_PHY_MAX_WIFI_TX_POWER=20 -CONFIG_ESP32_PHY_MAX_TX_POWER=20 -# CONFIG_REDUCE_PHY_TX_POWER is not set -# CONFIG_ESP32_REDUCE_PHY_TX_POWER is not set -CONFIG_ESP_SYSTEM_PM_POWER_DOWN_CPU=y -# CONFIG_ESP32S3_SPIRAM_SUPPORT is not set -# CONFIG_ESP32S3_DEFAULT_CPU_FREQ_80 is not set -CONFIG_ESP32S3_DEFAULT_CPU_FREQ_160=y -# CONFIG_ESP32S3_DEFAULT_CPU_FREQ_240 is not set -CONFIG_ESP32S3_DEFAULT_CPU_FREQ_MHZ=160 -CONFIG_SYSTEM_EVENT_QUEUE_SIZE=32 -CONFIG_SYSTEM_EVENT_TASK_STACK_SIZE=2304 -CONFIG_MAIN_TASK_STACK_SIZE=3584 -CONFIG_CONSOLE_UART_DEFAULT=y -# CONFIG_CONSOLE_UART_CUSTOM is not set -# CONFIG_CONSOLE_UART_NONE is not set -# CONFIG_ESP_CONSOLE_UART_NONE is not set -CONFIG_CONSOLE_UART=y -CONFIG_CONSOLE_UART_NUM=0 -CONFIG_CONSOLE_UART_BAUDRATE=115200 -CONFIG_INT_WDT=y -CONFIG_INT_WDT_TIMEOUT_MS=300 -CONFIG_INT_WDT_CHECK_CPU1=y -CONFIG_TASK_WDT=y -CONFIG_ESP_TASK_WDT=y -# CONFIG_TASK_WDT_PANIC is not set -CONFIG_TASK_WDT_TIMEOUT_S=5 -CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU0=y -CONFIG_TASK_WDT_CHECK_IDLE_TASK_CPU1=y -# CONFIG_ESP32_DEBUG_STUBS_ENABLE is not set -CONFIG_ESP32S3_DEBUG_OCDAWARE=y -CONFIG_BROWNOUT_DET=y -CONFIG_ESP32S3_BROWNOUT_DET=y -CONFIG_ESP32S3_BROWNOUT_DET=y -CONFIG_BROWNOUT_DET_LVL_SEL_7=y -CONFIG_ESP32S3_BROWNOUT_DET_LVL_SEL_7=y -# CONFIG_BROWNOUT_DET_LVL_SEL_6 is not set -# CONFIG_ESP32S3_BROWNOUT_DET_LVL_SEL_6 is not set -# CONFIG_BROWNOUT_DET_LVL_SEL_5 is not set -# CONFIG_ESP32S3_BROWNOUT_DET_LVL_SEL_5 is not set -# CONFIG_BROWNOUT_DET_LVL_SEL_4 is not set -# CONFIG_ESP32S3_BROWNOUT_DET_LVL_SEL_4 is not set -# CONFIG_BROWNOUT_DET_LVL_SEL_3 is not set -# CONFIG_ESP32S3_BROWNOUT_DET_LVL_SEL_3 is not set -# CONFIG_BROWNOUT_DET_LVL_SEL_2 is not set -# CONFIG_ESP32S3_BROWNOUT_DET_LVL_SEL_2 is not set -# CONFIG_BROWNOUT_DET_LVL_SEL_1 is not set -# CONFIG_ESP32S3_BROWNOUT_DET_LVL_SEL_1 is not set -CONFIG_BROWNOUT_DET_LVL=7 -CONFIG_ESP32S3_BROWNOUT_DET_LVL=7 -CONFIG_IPC_TASK_STACK_SIZE=1280 -CONFIG_TIMER_TASK_STACK_SIZE=3584 -CONFIG_ESP32_WIFI_ENABLED=y -CONFIG_ESP32_WIFI_STATIC_RX_BUFFER_NUM=10 -CONFIG_ESP32_WIFI_DYNAMIC_RX_BUFFER_NUM=32 -# CONFIG_ESP32_WIFI_STATIC_TX_BUFFER is not set -CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER=y -CONFIG_ESP32_WIFI_TX_BUFFER_TYPE=1 -CONFIG_ESP32_WIFI_DYNAMIC_TX_BUFFER_NUM=32 -# CONFIG_ESP32_WIFI_CSI_ENABLED is not set -CONFIG_ESP32_WIFI_AMPDU_TX_ENABLED=y -CONFIG_ESP32_WIFI_TX_BA_WIN=6 -CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y -CONFIG_ESP32_WIFI_AMPDU_RX_ENABLED=y -CONFIG_ESP32_WIFI_RX_BA_WIN=6 -CONFIG_ESP32_WIFI_RX_BA_WIN=6 -CONFIG_ESP32_WIFI_NVS_ENABLED=y -CONFIG_ESP32_WIFI_TASK_PINNED_TO_CORE_0=y -# CONFIG_ESP32_WIFI_TASK_PINNED_TO_CORE_1 is not set -CONFIG_ESP32_WIFI_SOFTAP_BEACON_MAX_LEN=752 -CONFIG_ESP32_WIFI_MGMT_SBUF_NUM=32 -CONFIG_ESP32_WIFI_IRAM_OPT=y -CONFIG_ESP32_WIFI_RX_IRAM_OPT=y -CONFIG_ESP32_WIFI_ENABLE_WPA3_SAE=y -CONFIG_ESP32_WIFI_ENABLE_WPA3_OWE_STA=y -CONFIG_WPA_MBEDTLS_CRYPTO=y -CONFIG_WPA_MBEDTLS_TLS_CLIENT=y -# CONFIG_WPA_WAPI_PSK is not set -# CONFIG_WPA_SUITE_B_192 is not set -# CONFIG_WPA_11KV_SUPPORT is not set -# CONFIG_WPA_MBO_SUPPORT is not set -# CONFIG_WPA_DPP_SUPPORT is not set -# CONFIG_WPA_11R_SUPPORT is not set -# CONFIG_WPA_WPS_SOFTAP_REGISTRAR is not set -# CONFIG_WPA_WPS_STRICT is not set -# CONFIG_WPA_DEBUG_PRINT is not set -# CONFIG_WPA_TESTING_OPTIONS is not set -# CONFIG_ESP32_ENABLE_COREDUMP_TO_FLASH is not set -# CONFIG_ESP32_ENABLE_COREDUMP_TO_UART is not set -CONFIG_ESP32_ENABLE_COREDUMP_TO_NONE=y -CONFIG_TIMER_TASK_PRIORITY=1 -CONFIG_TIMER_TASK_STACK_DEPTH=2048 -CONFIG_TIMER_QUEUE_LENGTH=10 -# CONFIG_ENABLE_STATIC_TASK_CLEAN_UP_HOOK is not set -# CONFIG_HAL_ASSERTION_SILIENT is not set -# CONFIG_L2_TO_L3_COPY is not set -CONFIG_ESP_GRATUITOUS_ARP=y -CONFIG_GARP_TMR_INTERVAL=60 -CONFIG_TCPIP_RECVMBOX_SIZE=32 -CONFIG_TCP_MAXRTX=12 -CONFIG_TCP_SYNMAXRTX=12 -CONFIG_TCP_MSS=1440 -CONFIG_TCP_MSL=60000 -CONFIG_TCP_SND_BUF_DEFAULT=5744 -CONFIG_TCP_WND_DEFAULT=5744 -CONFIG_TCP_RECVMBOX_SIZE=6 -CONFIG_TCP_QUEUE_OOSEQ=y -CONFIG_TCP_OVERSIZE_MSS=y -# CONFIG_TCP_OVERSIZE_QUARTER_MSS is not set -# CONFIG_TCP_OVERSIZE_DISABLE is not set -CONFIG_UDP_RECVMBOX_SIZE=6 -CONFIG_TCPIP_TASK_STACK_SIZE=3072 -CONFIG_TCPIP_TASK_AFFINITY_NO_AFFINITY=y -# CONFIG_TCPIP_TASK_AFFINITY_CPU0 is not set -# CONFIG_TCPIP_TASK_AFFINITY_CPU1 is not set -CONFIG_TCPIP_TASK_AFFINITY=0x7FFFFFFF -# CONFIG_PPP_SUPPORT is not set -CONFIG_ESP32S3_TIME_SYSCALL_USE_RTC_SYSTIMER=y -CONFIG_ESP32S3_TIME_SYSCALL_USE_RTC_FRC1=y -# CONFIG_ESP32S3_TIME_SYSCALL_USE_RTC is not set -# CONFIG_ESP32S3_TIME_SYSCALL_USE_SYSTIMER is not set -# CONFIG_ESP32S3_TIME_SYSCALL_USE_FRC1 is not set -# CONFIG_ESP32S3_TIME_SYSCALL_USE_NONE is not set -CONFIG_ESP32_PTHREAD_TASK_PRIO_DEFAULT=5 -CONFIG_ESP32_PTHREAD_TASK_STACK_SIZE_DEFAULT=3072 -CONFIG_ESP32_PTHREAD_STACK_MIN=768 -CONFIG_ESP32_DEFAULT_PTHREAD_CORE_NO_AFFINITY=y -# CONFIG_ESP32_DEFAULT_PTHREAD_CORE_0 is not set -# CONFIG_ESP32_DEFAULT_PTHREAD_CORE_1 is not set -CONFIG_ESP32_PTHREAD_TASK_CORE_DEFAULT=-1 -CONFIG_ESP32_PTHREAD_TASK_NAME_DEFAULT="pthread" -CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ABORTS=y -# CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_FAILS is not set -# CONFIG_SPI_FLASH_WRITING_DANGEROUS_REGIONS_ALLOWED is not set -CONFIG_SUPPRESS_SELECT_DEBUG_OUTPUT=y -CONFIG_SUPPORT_TERMIOS=y -CONFIG_SEMIHOSTFS_MAX_MOUNT_POINTS=1 -# End of deprecated options diff --git a/ports/espressif/components/led_strip/idf_component.yml b/ports/espressif/components/led_strip/idf_component.yml index 1fd9b83ee..35f50bdaf 100644 --- a/ports/espressif/components/led_strip/idf_component.yml +++ b/ports/espressif/components/led_strip/idf_component.yml @@ -1,5 +1,11 @@ -version: "2.5.2" +dependencies: + idf: '>=5.2' description: Driver for Addressable LED Strip (WS2812, etc) +documentation: https://espressif.github.io/idf-extra-components/latest/led_strip/index.html +issues: https://github.com/espressif/idf-extra-components/issues +repository: git://github.com/espressif/idf-extra-components.git +repository_info: + commit_sha: f3a3b53e27141fadd10c154bac8a25eecb0b0bc6 + path: led_strip url: https://github.com/espressif/idf-extra-components/tree/master/led_strip -dependencies: - idf: ">=4.4" +version: 3.1.0~1 diff --git a/ports/espressif/components/led_strip/include/led_strip.h b/ports/espressif/components/led_strip/include/led_strip.h index 38711744a..676b5e63d 100644 --- a/ports/espressif/components/led_strip/include/led_strip.h +++ b/ports/espressif/components/led_strip/include/led_strip.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -8,11 +8,7 @@ #include #include "esp_err.h" #include "led_strip_rmt.h" -#include "esp_idf_version.h" - -#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 1, 0) #include "led_strip_spi.h" -#endif #ifdef __cplusplus extern "C" { @@ -60,8 +56,8 @@ esp_err_t led_strip_set_pixel_rgbw(led_strip_handle_t strip, uint32_t index, uin * @param strip: LED strip * @param index: index of pixel to set * @param hue: hue part of color (0 - 360) - * @param saturation: saturation part of color (0 - 255) - * @param value: value part of color (0 - 255) + * @param saturation: saturation part of color (0 - 255, rescaled from 0 - 1. e.g. saturation = 0.5, rescaled to 127) + * @param value: value part of color (0 - 255, rescaled from 0 - 1. e.g. value = 0.5, rescaled to 127) * * @return * - ESP_OK: Set HSV color for a specific pixel successfully @@ -70,6 +66,22 @@ esp_err_t led_strip_set_pixel_rgbw(led_strip_handle_t strip, uint32_t index, uin */ esp_err_t led_strip_set_pixel_hsv(led_strip_handle_t strip, uint32_t index, uint16_t hue, uint8_t saturation, uint8_t value); +/** + * @brief Set HSV for a specific pixel in 16-bit resolution + * + * @param strip: LED strip + * @param index: index of pixel to set + * @param hue: hue part of color (0 - 360) + * @param saturation: saturation part of color (0 - 65535, rescaled from 0 - 1. e.g. saturation = 0.5, rescaled to 32767) + * @param value: value part of color (0 - 65535, rescaled from 0 - 1. e.g. value = 0.5, rescaled to 32767) + * + * @return + * - ESP_OK: Set HSV color for a specific pixel successfully + * - ESP_ERR_INVALID_ARG: Set HSV color for a specific pixel failed because of an invalid argument + * - ESP_FAIL: Set HSV color for a specific pixel failed because other error occurred + */ +esp_err_t led_strip_set_pixel_hsv_16(led_strip_handle_t strip, uint32_t index, uint16_t hue, uint16_t saturation, uint16_t value); + /** * @brief Refresh memory colors to LEDs * @@ -79,11 +91,47 @@ esp_err_t led_strip_set_pixel_hsv(led_strip_handle_t strip, uint32_t index, uint * - ESP_OK: Refresh successfully * - ESP_FAIL: Refresh failed because some other error occurred * - * @note: - * After updating the LED colors in the memory, a following invocation of this API is needed to flush colors to strip. + * @note After updating the LED colors in the memory, a following invocation of this API is needed to flush colors to strip. */ esp_err_t led_strip_refresh(led_strip_handle_t strip); +/** + * @brief Refresh memory colors to LEDs asynchronously + * + * @param strip: LED strip + * + * @return + * - ESP_OK: Refresh successfully + * - ESP_ERR_INVALID_ARG: Refresh failed because of an invalid argument + * - ESP_ERR_INVALID_STATE: Refresh failed because another transaction is in progress + * - ESP_ERR_NOT_SUPPORTED: Async refresh is not supported by this backend + * - ESP_FAIL: Refresh failed because some other error occurred + * + * @note This function is non-blocking, but the driver does not have a double buffer yet. The pixel + * buffer is shared with the in-flight transaction, so you must still call + * `led_strip_refresh_async_done` before modifying the LED colors again. In other words, the + * only gain for now is that the waiting can be deferred and overlapped with CPU work that + * does not touch the strip. + * + */ +esp_err_t led_strip_refresh_async(led_strip_handle_t strip); + +/** + * @brief Wait for the async refresh to complete + * + * @param strip: LED strip + * + * @return + * - ESP_OK: Wait for the async refresh to complete successfully + * - ESP_ERR_INVALID_ARG: Wait failed because of an invalid argument + * - ESP_ERR_INVALID_STATE: Wait failed because there is no in-flight async refresh + * - ESP_ERR_NOT_SUPPORTED: Async refresh is not supported by this backend + * + * @note Pair this with `led_strip_refresh_async`. Until the double buffer is added, the pixel + * buffer must not be modified before this call returns. + */ +esp_err_t led_strip_refresh_async_done(led_strip_handle_t strip); + /** * @brief Clear LED strip (turn off all LEDs) * @@ -95,6 +143,24 @@ esp_err_t led_strip_refresh(led_strip_handle_t strip); */ esp_err_t led_strip_clear(led_strip_handle_t strip); +/** + * @brief Switch GPIO of LED strip + * + * @param strip: LED strip + * @param new_gpio_num: new GPIO number + * @param invert_output: invert output + * + * @note Only the RMT backend supports this operation + * + * @return + * - ESP_OK: Switch GPIO successfully + * - ESP_ERR_INVALID_ARG: Switch GPIO failed because of an invalid argument + * - ESP_ERR_INVALID_STATE: Switch GPIO failed because a transaction is in progress + * - ESP_ERR_NOT_SUPPORTED: Switch GPIO is not supported by this backend + * - ESP_FAIL: Switch GPIO failed because some other error occurred + */ +esp_err_t led_strip_switch_gpio(led_strip_handle_t strip, gpio_num_t new_gpio_num, bool invert_output); + /** * @brief Free LED strip resources * diff --git a/ports/espressif/components/led_strip/include/led_strip_rmt.h b/ports/espressif/components/led_strip/include/led_strip_rmt.h index b575aeaba..3b58276b1 100644 --- a/ports/espressif/components/led_strip/include/led_strip_rmt.h +++ b/ports/espressif/components/led_strip/include/led_strip_rmt.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -9,10 +9,7 @@ #include "esp_err.h" #include "led_strip_types.h" #include "esp_idf_version.h" - -#if ESP_IDF_VERSION >= ESP_IDF_VERSION_VAL(5, 0, 0) #include "driver/rmt_types.h" -#endif #ifdef __cplusplus extern "C" { @@ -22,14 +19,11 @@ extern "C" { * @brief LED Strip RMT specific configuration */ typedef struct { -#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 0, 0) - uint8_t rmt_channel; /*!< Specify the channel number, the legacy RMT driver doesn't support channel allocator */ -#else // new driver supports specify the clock source and clock resolution rmt_clock_source_t clk_src; /*!< RMT clock source */ uint32_t resolution_hz; /*!< RMT tick resolution, if set to zero, a default resolution (10MHz) will be applied */ -#endif size_t mem_block_symbols; /*!< How many RMT symbols can one RMT channel hold at one time. Set to 0 will fallback to use the default size. */ - struct { + /*!< Extra RMT specific driver flags */ + struct led_strip_rmt_extra_config { uint32_t with_dma: 1; /*!< Use DMA to transmit data */ } flags; /*!< Extra driver flags */ } led_strip_rmt_config_t; diff --git a/ports/espressif/components/led_strip/include/led_strip_spi.h b/ports/espressif/components/led_strip/include/led_strip_spi.h index eb3524936..8a19aacfd 100644 --- a/ports/espressif/components/led_strip/include/led_strip_spi.h +++ b/ports/espressif/components/led_strip/include/led_strip_spi.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2024 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -20,13 +20,14 @@ extern "C" { typedef struct { spi_clock_source_t clk_src; /*!< SPI clock source */ spi_host_device_t spi_bus; /*!< SPI bus ID. Which buses are available depends on the specific chip */ - struct { + struct led_strip_spi_extra_config { uint32_t with_dma: 1; /*!< Use DMA to transmit data */ } flags; /*!< Extra driver flags */ } led_strip_spi_config_t; /** * @brief Create LED strip based on SPI MOSI channel + * * @note Although only the MOSI line is used for generating the signal, the whole SPI bus can't be used for other purposes. * * @param led_config LED strip configuration diff --git a/ports/espressif/components/led_strip/include/led_strip_types.h b/ports/espressif/components/led_strip/include/led_strip_types.h index 691f0bc39..febcec33c 100644 --- a/ports/espressif/components/led_strip/include/led_strip_types.h +++ b/ports/espressif/components/led_strip/include/led_strip_types.h @@ -1,24 +1,21 @@ /* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ #pragma once #include +#include "hal/gpio_types.h" #ifdef __cplusplus extern "C" { #endif /** - * @brief LED strip pixel format + * @brief Type of LED strip handle */ -typedef enum { - LED_PIXEL_FORMAT_GRB, /*!< Pixel format: GRB */ - LED_PIXEL_FORMAT_GRBW, /*!< Pixel format: GRBW */ - LED_PIXEL_FORMAT_INVALID /*!< Invalid pixel format */ -} led_pixel_format_t; +typedef struct led_strip_t *led_strip_handle_t; /** * @brief LED strip model @@ -27,26 +24,68 @@ typedef enum { typedef enum { LED_MODEL_WS2812, /*!< LED strip model: WS2812 */ LED_MODEL_SK6812, /*!< LED strip model: SK6812 */ + LED_MODEL_WS2811, /*!< LED strip model: WS2811 */ + LED_MODEL_WS2816, /*!< LED strip model: WS2816 */ + LED_MODEL_CUSTOM, /*!< Custom LED strip model. Only used for RMT backend. The timings can be specified by the `led_strip_timings_t` */ LED_MODEL_INVALID /*!< Invalid LED strip model */ } led_model_t; /** - * @brief LED strip handle + * @brief LED strip timings. + * @note The bit timings are in nanoseconds and the reset timing is in microseconds. */ -typedef struct led_strip_t *led_strip_handle_t; +typedef struct { + uint32_t t0h_ns; /*!< High time for 0 bit, nanoseconds */ + uint32_t t1h_ns; /*!< High time for 1 bit, nanoseconds */ + uint32_t t0l_ns; /*!< Low time for 0 bit, nanoseconds */ + uint32_t t1l_ns; /*!< Low time for 1 bit, nanoseconds */ + uint32_t reset_us; /*!< Reset time, microseconds */ +} led_strip_timings_t; /** - * @brief LED Strip Configuration + * @brief LED color component format + * @note The format is used to specify the order of color components in each pixel, also the number of color components. */ -typedef struct { - int strip_gpio_num; /*!< GPIO number that used by LED strip */ - uint32_t max_leds; /*!< Maximum LEDs in a single strip */ - led_pixel_format_t led_pixel_format; /*!< LED pixel format */ - led_model_t led_model; /*!< LED model */ +typedef union { + struct format_layout { + uint32_t r_pos: 2; /*!< Position of the red channel in the color order: 0~3 */ + uint32_t g_pos: 2; /*!< Position of the green channel in the color order: 0~3 */ + uint32_t b_pos: 2; /*!< Position of the blue channel in the color order: 0~3 */ + uint32_t w_pos: 2; /*!< Position of the white channel in the color order: 0~3 */ + uint32_t reserved: 19; /*!< Reserved */ + uint32_t bytes_per_color: 2; /*!< Bytes per color component: 1 or 2. If set to 0, it will fallback to 1 */ + uint32_t num_components: 3; /*!< Number of color components per pixel: 3 or 4. If set to 0, it will fallback to 3 */ + } format; /*!< Format layout */ + uint32_t format_id; /*!< Format ID */ +} led_color_component_format_t; - struct { +/// Helper macros to set the color component format +#define LED_STRIP_COLOR_COMPONENT_FMT_GRB (led_color_component_format_t){.format = {.r_pos = 1, .g_pos = 0, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 1, .num_components = 3}} +#define LED_STRIP_COLOR_COMPONENT_FMT_GRB_16 (led_color_component_format_t){.format = {.r_pos = 1, .g_pos = 0, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 2, .num_components = 3}} +#define LED_STRIP_COLOR_COMPONENT_FMT_GRBW (led_color_component_format_t){.format = {.r_pos = 1, .g_pos = 0, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 1, .num_components = 4}} +#define LED_STRIP_COLOR_COMPONENT_FMT_GRBW_16 (led_color_component_format_t){.format = {.r_pos = 1, .g_pos = 0, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 2, .num_components = 4}} +#define LED_STRIP_COLOR_COMPONENT_FMT_RGB (led_color_component_format_t){.format = {.r_pos = 0, .g_pos = 1, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 1, .num_components = 3}} +#define LED_STRIP_COLOR_COMPONENT_FMT_RGB_16 (led_color_component_format_t){.format = {.r_pos = 0, .g_pos = 1, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 2, .num_components = 3}} +#define LED_STRIP_COLOR_COMPONENT_FMT_RGBW (led_color_component_format_t){.format = {.r_pos = 0, .g_pos = 1, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 1, .num_components = 4}} +#define LED_STRIP_COLOR_COMPONENT_FMT_RGBW_16 (led_color_component_format_t){.format = {.r_pos = 0, .g_pos = 1, .b_pos = 2, .w_pos = 3, .reserved = 0, .bytes_per_color = 2, .num_components = 4}} + +/** + * @brief LED Strip common configurations + * The common configurations are not specific to any backend peripheral. + */ +typedef struct { + int strip_gpio_num; /*!< GPIO number that used by LED strip */ + uint32_t max_leds; /*!< Maximum number of LEDs that can be controlled in a single strip */ + led_model_t led_model; /*!< Specifies the LED strip model (e.g., WS2812, SK6812) */ + led_color_component_format_t color_component_format; /*!< Specifies the order of color components in each pixel. + Use helper macros like `LED_STRIP_COLOR_COMPONENT_FMT_GRB` to set the format */ + led_strip_timings_t timings; /*!< When `led_model` is `LED_MODEL_CUSTOM`, this field is required and must + be initialized with appropriate timing values. For other LED models, it + may be left uninitialized, as model-specific defaults are used instead. */ + /*!< LED strip extra driver flags */ + struct led_strip_extra_flags { uint32_t invert_out: 1; /*!< Invert output signal */ - } flags; /*!< Extra driver flags */ + } flags; /*!< Extra driver flags */ } led_strip_config_t; #ifdef __cplusplus diff --git a/ports/espressif/components/led_strip/interface/led_strip_interface.h b/ports/espressif/components/led_strip/interface/led_strip_interface.h index 3de4c2715..ac089c452 100644 --- a/ports/espressif/components/led_strip/interface/led_strip_interface.h +++ b/ports/espressif/components/led_strip/interface/led_strip_interface.h @@ -1,12 +1,14 @@ /* - * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2026 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ #pragma once +#include #include #include "esp_err.h" +#include "hal/gpio_types.h" #ifdef __cplusplus extern "C" { @@ -55,22 +57,45 @@ struct led_strip_t { * @brief Refresh memory colors to LEDs * * @param strip: LED strip - * @param timeout_ms: timeout value for refreshing task * * @return * - ESP_OK: Refresh successfully * - ESP_FAIL: Refresh failed because some other error occurred * - * @note: - * After updating the LED colors in the memory, a following invocation of this API is needed to flush colors to strip. + * @note After updating the LED colors in the memory, a following invocation of this API is needed to flush colors to strip. */ esp_err_t (*refresh)(led_strip_t *strip); + /** + * @brief Refresh memory colors to LEDs asynchronously + * + * @param strip: LED strip + * + * @return + * - ESP_OK: Refresh successfully + * - ESP_FAIL: Refresh failed because some other error occurred + * + * @note This function is non-blocking, but the driver does not have a double buffer yet, so the pixel buffer is + * shared with the in-flight transaction. You still have to call `led_strip_refresh_async_done()` before + * modifying the LED colors again. + */ + esp_err_t (*refresh_async)(led_strip_t *strip); + + /** + * @brief Wait for the async refresh to complete + * + * @param strip: LED strip + * + * @return + * - ESP_OK: Wait for the async refresh to complete successfully + * - ESP_ERR_INVALID_STATE: Wait failed because there is no in-flight async refresh + */ + esp_err_t (*refresh_async_done)(led_strip_t *strip); + /** * @brief Clear LED strip (turn off all LEDs) * * @param strip: LED strip - * @param timeout_ms: timeout value for clearing task * * @return * - ESP_OK: Clear LEDs successfully @@ -78,6 +103,21 @@ struct led_strip_t { */ esp_err_t (*clear)(led_strip_t *strip); + /** + * @brief Switch GPIO of LED strip + * + * @param strip: LED strip + * @param new_gpio_num: new GPIO number + * @param invert_output: invert output + * + * @note Only support RMT backend now + * + * @return + * - ESP_OK: Switch GPIO successfully + * - ESP_FAIL: Switch GPIO failed because some other error occurred + */ + esp_err_t (*switch_gpio)(led_strip_t *strip, gpio_num_t new_gpio_num, bool invert_output); + /** * @brief Free LED strip resources * diff --git a/ports/espressif/components/led_strip/src/led_strip_api.c b/ports/espressif/components/led_strip/src/led_strip_api.c index 6eb86b8f1..9be8f5daf 100644 --- a/ports/espressif/components/led_strip/src/led_strip_api.c +++ b/ports/espressif/components/led_strip/src/led_strip_api.c @@ -13,6 +13,7 @@ static const char *TAG = "led_strip"; esp_err_t led_strip_set_pixel(led_strip_handle_t strip, uint32_t index, uint32_t red, uint32_t green, uint32_t blue) { ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->set_pixel, ESP_ERR_NOT_SUPPORTED, TAG, "set_pixel is not supported by this backend"); return strip->set_pixel(strip, index, red, green, blue); } @@ -25,7 +26,60 @@ esp_err_t led_strip_set_pixel_hsv(led_strip_handle_t strip, uint32_t index, uint uint32_t blue = 0; uint32_t rgb_max = value; - uint32_t rgb_min = rgb_max * (255 - saturation) / 255.0f; + uint32_t rgb_min = rgb_max * (255 - saturation) / 255; + + uint32_t i = hue / 60; + uint32_t diff = hue % 60; + + // RGB adjustment amount by hue + uint32_t rgb_adj = (rgb_max - rgb_min) * diff / 60; + + switch (i) { + case 0: + red = rgb_max; + green = rgb_min + rgb_adj; + blue = rgb_min; + break; + case 1: + red = rgb_max - rgb_adj; + green = rgb_max; + blue = rgb_min; + break; + case 2: + red = rgb_min; + green = rgb_max; + blue = rgb_min + rgb_adj; + break; + case 3: + red = rgb_min; + green = rgb_max - rgb_adj; + blue = rgb_max; + break; + case 4: + red = rgb_min + rgb_adj; + green = rgb_min; + blue = rgb_max; + break; + default: + red = rgb_max; + green = rgb_min; + blue = rgb_max - rgb_adj; + break; + } + + return strip->set_pixel(strip, index, red, green, blue); +} + +esp_err_t led_strip_set_pixel_hsv_16(led_strip_handle_t strip, uint32_t index, uint16_t hue, uint16_t saturation, uint16_t value) +{ + ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + + uint32_t red = 0; + uint32_t green = 0; + uint32_t blue = 0; + + uint32_t rgb_max = value; + uint32_t rgb_min = rgb_max * (65535 - saturation) / 65535; uint32_t i = hue / 60; uint32_t diff = hue % 60; @@ -72,23 +126,48 @@ esp_err_t led_strip_set_pixel_hsv(led_strip_handle_t strip, uint32_t index, uint esp_err_t led_strip_set_pixel_rgbw(led_strip_handle_t strip, uint32_t index, uint32_t red, uint32_t green, uint32_t blue, uint32_t white) { ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->set_pixel_rgbw, ESP_ERR_NOT_SUPPORTED, TAG, "set_pixel_rgbw is not supported by this backend"); return strip->set_pixel_rgbw(strip, index, red, green, blue, white); } esp_err_t led_strip_refresh(led_strip_handle_t strip) { ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->refresh, ESP_ERR_NOT_SUPPORTED, TAG, "refresh is not supported by this backend"); return strip->refresh(strip); } +esp_err_t led_strip_refresh_async(led_strip_handle_t strip) +{ + ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->refresh_async, ESP_ERR_NOT_SUPPORTED, TAG, "async refresh is not supported by this backend"); + return strip->refresh_async(strip); +} + +esp_err_t led_strip_refresh_async_done(led_strip_handle_t strip) +{ + ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->refresh_async_done, ESP_ERR_NOT_SUPPORTED, TAG, "async refresh is not supported by this backend"); + return strip->refresh_async_done(strip); +} + esp_err_t led_strip_clear(led_strip_handle_t strip) { ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->clear, ESP_ERR_NOT_SUPPORTED, TAG, "clear is not supported by this backend"); return strip->clear(strip); } esp_err_t led_strip_del(led_strip_handle_t strip) { ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->del, ESP_ERR_NOT_SUPPORTED, TAG, "del is not supported by this backend"); return strip->del(strip); } + +esp_err_t led_strip_switch_gpio(led_strip_handle_t strip, gpio_num_t new_gpio_num, bool invert_output) +{ + ESP_RETURN_ON_FALSE(strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); + ESP_RETURN_ON_FALSE(strip->switch_gpio, ESP_ERR_NOT_SUPPORTED, TAG, "switch_gpio is not supported by this backend"); + return strip->switch_gpio(strip, new_gpio_num, invert_output); +} diff --git a/ports/espressif/components/led_strip/src/led_strip_common.h b/ports/espressif/components/led_strip/src/led_strip_common.h new file mode 100644 index 000000000..bd64ac19f --- /dev/null +++ b/ports/espressif/components/led_strip/src/led_strip_common.h @@ -0,0 +1,43 @@ +/* + * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD + * + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief State of a strip transaction, used to serialize the access to the shared pixel buffer + * + * @note This state machine only prevents two tasks from starting a transaction at the same time. + * It does not provide ordering between tasks, nor does it make the driver as a whole + * thread-safe. See the "Thread safety" note in the public API documentation. + */ +typedef enum { + LED_STRIP_TRANS_IDLE, /*!< No transaction is running, the pixel buffer can be modified freely */ + LED_STRIP_TRANS_INFLIGHT, /*!< A transaction has been started but is still being transferred by the peripheral + (i.e. it is "in the air"). The pixel buffer must not be modified, otherwise the + peripheral may send a mix of old and new data. Use the wait API to join it */ + LED_STRIP_TRANS_LOCKED, /*!< The local critical section is taken by the current task, e.g. while it is writing + the pixel buffer or blocking to wait for a transaction to finish. The peripheral + itself may or may not be transmitting at this point. This state is only ever + held for the duration of a single driver call */ +} led_strip_trans_state_t; + +typedef atomic_int led_strip_trans_state_atomic_t; + +static inline bool led_strip_trans_state_try_set(led_strip_trans_state_atomic_t *state, led_strip_trans_state_t expected, led_strip_trans_state_t desired) +{ + int expected_state = expected; + return atomic_compare_exchange_strong(state, &expected_state, desired); +} + +#ifdef __cplusplus +} +#endif diff --git a/ports/espressif/components/led_strip/src/led_strip_rmt_dev.c b/ports/espressif/components/led_strip/src/led_strip_rmt_dev.c index 1cbf0e45a..662a9e533 100644 --- a/ports/espressif/components/led_strip/src/led_strip_rmt_dev.c +++ b/ports/espressif/components/led_strip/src/led_strip_rmt_dev.c @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -12,6 +12,7 @@ #include "led_strip.h" #include "led_strip_interface.h" #include "led_strip_rmt_encoder.h" +#include "led_strip_common.h" #define LED_STRIP_RMT_DEFAULT_RESOLUTION 10000000 // 10MHz resolution #define LED_STRIP_RMT_DEFAULT_TRANS_QUEUE_SIZE 4 @@ -30,67 +31,213 @@ typedef struct { rmt_encoder_handle_t strip_encoder; uint32_t strip_len; uint8_t bytes_per_pixel; + led_color_component_format_t component_fmt; + led_strip_trans_state_atomic_t trans_state; uint8_t pixel_buf[]; } led_strip_rmt_obj; +static esp_err_t led_strip_rmt_trans(led_strip_rmt_obj *rmt_strip) +{ + esp_err_t ret = ESP_OK; + rmt_transmit_config_t tx_conf = { + .loop_count = 0, + }; + + ESP_RETURN_ON_ERROR(rmt_enable(rmt_strip->rmt_chan), TAG, "enable RMT channel failed"); + ESP_GOTO_ON_ERROR(rmt_transmit(rmt_strip->rmt_chan, rmt_strip->strip_encoder, rmt_strip->pixel_buf, + rmt_strip->strip_len * rmt_strip->bytes_per_pixel, &tx_conf), err, TAG, "transmit pixels by RMT failed"); + return ret; +err: + rmt_disable(rmt_strip->rmt_chan); + return ret; +} + +static esp_err_t led_strip_rmt_wait_trans_done(led_strip_rmt_obj *rmt_strip) +{ + ESP_RETURN_ON_ERROR(rmt_tx_wait_all_done(rmt_strip->rmt_chan, -1), TAG, "wait for RMT done failed"); + ESP_RETURN_ON_ERROR(rmt_disable(rmt_strip->rmt_chan), TAG, "disable RMT channel failed"); + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ESP_OK; +} + static esp_err_t led_strip_rmt_set_pixel(led_strip_t *strip, uint32_t index, uint32_t red, uint32_t green, uint32_t blue) { led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); ESP_RETURN_ON_FALSE(index < rmt_strip->strip_len, ESP_ERR_INVALID_ARG, TAG, "index out of maximum number of LEDs"); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "RMT transaction is busy"); + + struct format_layout format = rmt_strip->component_fmt.format; uint32_t start = index * rmt_strip->bytes_per_pixel; - // In thr order of GRB, as LED strip like WS2812 sends out pixels in this order - rmt_strip->pixel_buf[start + 0] = green & 0xFF; - rmt_strip->pixel_buf[start + 1] = red & 0xFF; - rmt_strip->pixel_buf[start + 2] = blue & 0xFF; - if (rmt_strip->bytes_per_pixel > 3) { - rmt_strip->pixel_buf[start + 3] = 0; + uint8_t *pixel_buf = rmt_strip->pixel_buf; + uint8_t pos_bytes = format.bytes_per_color; + + for (uint8_t i = 0; i < format.bytes_per_color; i++) { + uint8_t color_shift = 8 * (format.bytes_per_color - 1 - i); + pixel_buf[start + format.r_pos * pos_bytes + i] = (red >> color_shift) & 0xFF; + pixel_buf[start + format.g_pos * pos_bytes + i] = (green >> color_shift) & 0xFF; + pixel_buf[start + format.b_pos * pos_bytes + i] = (blue >> color_shift) & 0xFF; + if (format.num_components > 3) { + pixel_buf[start + format.w_pos * pos_bytes + i] = 0; + } } + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); return ESP_OK; } static esp_err_t led_strip_rmt_set_pixel_rgbw(led_strip_t *strip, uint32_t index, uint32_t red, uint32_t green, uint32_t blue, uint32_t white) { led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); + struct format_layout format = rmt_strip->component_fmt.format; ESP_RETURN_ON_FALSE(index < rmt_strip->strip_len, ESP_ERR_INVALID_ARG, TAG, "index out of maximum number of LEDs"); - ESP_RETURN_ON_FALSE(rmt_strip->bytes_per_pixel == 4, ESP_ERR_INVALID_ARG, TAG, "wrong LED pixel format, expected 4 bytes per pixel"); - uint8_t *buf_start = rmt_strip->pixel_buf + index * 4; - // SK6812 component order is GRBW - *buf_start = green & 0xFF; - *++buf_start = red & 0xFF; - *++buf_start = blue & 0xFF; - *++buf_start = white & 0xFF; + ESP_RETURN_ON_FALSE(format.num_components == 4, ESP_ERR_INVALID_ARG, TAG, "led doesn't have 4 components"); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "RMT transaction is busy"); + + uint32_t start = index * rmt_strip->bytes_per_pixel; + uint8_t *pixel_buf = rmt_strip->pixel_buf; + uint8_t pos_bytes = format.bytes_per_color; + + for (uint8_t i = 0; i < format.bytes_per_color; i++) { + uint8_t color_shift = 8 * (format.bytes_per_color - 1 - i); + pixel_buf[start + format.r_pos * pos_bytes + i] = (red >> color_shift) & 0xFF; + pixel_buf[start + format.g_pos * pos_bytes + i] = (green >> color_shift) & 0xFF; + pixel_buf[start + format.b_pos * pos_bytes + i] = (blue >> color_shift) & 0xFF; + pixel_buf[start + format.w_pos * pos_bytes + i] = (white >> color_shift) & 0xFF; + } + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); return ESP_OK; } -static esp_err_t led_strip_rmt_refresh(led_strip_t *strip) +static esp_err_t led_strip_rmt_refresh_async(led_strip_t *strip) { + esp_err_t ret = ESP_OK; led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); - rmt_transmit_config_t tx_conf = { - .loop_count = 0, - }; + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_INFLIGHT), + ESP_ERR_INVALID_STATE, TAG, "RMT transaction is busy"); + ESP_GOTO_ON_ERROR(led_strip_rmt_trans(rmt_strip), err, TAG, "start RMT transaction failed"); + return ret; +err: + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; +} - ESP_RETURN_ON_ERROR(rmt_enable(rmt_strip->rmt_chan), TAG, "enable RMT channel failed"); - ESP_RETURN_ON_ERROR(rmt_transmit(rmt_strip->rmt_chan, rmt_strip->strip_encoder, rmt_strip->pixel_buf, - rmt_strip->strip_len * rmt_strip->bytes_per_pixel, &tx_conf), TAG, "transmit pixels by RMT failed"); - ESP_RETURN_ON_ERROR(rmt_tx_wait_all_done(rmt_strip->rmt_chan, -1), TAG, "flush RMT channel failed"); - ESP_RETURN_ON_ERROR(rmt_disable(rmt_strip->rmt_chan), TAG, "disable RMT channel failed"); - return ESP_OK; +static esp_err_t led_strip_rmt_refresh_async_done(led_strip_t *strip) +{ + esp_err_t ret = ESP_OK; + led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_INFLIGHT, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "no async refresh in progress"); + ESP_GOTO_ON_ERROR(led_strip_rmt_wait_trans_done(rmt_strip), err, TAG, "wait for done failed"); + return ret; +err: + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_INFLIGHT); + return ret; +} + +static esp_err_t led_strip_rmt_refresh(led_strip_t *strip) +{ + esp_err_t ret = ESP_OK; + led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "RMT transaction is busy"); + ESP_GOTO_ON_ERROR(led_strip_rmt_trans(rmt_strip), err, TAG, "start RMT transaction failed"); + ESP_GOTO_ON_ERROR(led_strip_rmt_wait_trans_done(rmt_strip), err, TAG, "wait for done failed"); + return ret; +err: + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; } static esp_err_t led_strip_rmt_clear(led_strip_t *strip) { + esp_err_t ret = ESP_OK; led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "RMT transaction is busy"); // Write zero to turn off all leds memset(rmt_strip->pixel_buf, 0, rmt_strip->strip_len * rmt_strip->bytes_per_pixel); - return led_strip_rmt_refresh(strip); + ESP_GOTO_ON_ERROR(led_strip_rmt_trans(rmt_strip), err, TAG, "start RMT transaction failed"); + ESP_GOTO_ON_ERROR(led_strip_rmt_wait_trans_done(rmt_strip), err, TAG, "wait for done failed"); + return ret; +err: + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; } static esp_err_t led_strip_rmt_del(led_strip_t *strip) { + esp_err_t ret = ESP_OK; led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); - ESP_RETURN_ON_ERROR(rmt_del_channel(rmt_strip->rmt_chan), TAG, "delete RMT channel failed"); - ESP_RETURN_ON_ERROR(rmt_del_encoder(rmt_strip->strip_encoder), TAG, "delete strip encoder failed"); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "RMT transaction is busy"); + ESP_GOTO_ON_ERROR(rmt_del_channel(rmt_strip->rmt_chan), err, TAG, "delete RMT channel failed"); + ESP_GOTO_ON_ERROR(rmt_del_encoder(rmt_strip->strip_encoder), err, TAG, "delete strip encoder failed"); free(rmt_strip); + return ret; +err: + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; +} + +static esp_err_t led_strip_rmt_switch_gpio(led_strip_t *strip, gpio_num_t new_gpio_num, bool invert_output) +{ + esp_err_t ret = ESP_OK; + led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "RMT transaction is busy"); + ret = rmt_tx_switch_gpio(rmt_strip->rmt_chan, new_gpio_num, invert_output); + atomic_store(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; +} + +static esp_err_t led_strip_get_timings(led_model_t led_model, const led_strip_timings_t *custom_timings, led_strip_timings_t *timings) +{ + if (led_model == LED_MODEL_CUSTOM) { + *timings = *custom_timings; + return ESP_OK; + } + + switch (led_model) { + case LED_MODEL_WS2812: + *timings = (led_strip_timings_t) { + .t0h_ns = 300, + .t1h_ns = 900, + .t0l_ns = 900, + .t1l_ns = 600, + .reset_us = 280, + }; + break; + case LED_MODEL_SK6812: + *timings = (led_strip_timings_t) { + .t0h_ns = 300, + .t1h_ns = 700, + .t0l_ns = 900, + .t1l_ns = 600, + .reset_us = 280, + }; + break; + case LED_MODEL_WS2811: + *timings = (led_strip_timings_t) { + .t0h_ns = 500, + .t1h_ns = 1200, + .t0l_ns = 2000, + .t1l_ns = 1300, + .reset_us = 50, + }; + break; + case LED_MODEL_WS2816: + *timings = (led_strip_timings_t) { + .t0h_ns = 300, + .t1h_ns = 750, + .t0l_ns = 950, + .t1l_ns = 500, + .reset_us = 280, + }; + break; + default: + return ESP_ERR_INVALID_ARG; + } return ESP_OK; } @@ -99,17 +246,39 @@ esp_err_t led_strip_new_rmt_device(const led_strip_config_t *led_config, const l led_strip_rmt_obj *rmt_strip = NULL; esp_err_t ret = ESP_OK; ESP_GOTO_ON_FALSE(led_config && rmt_config && ret_strip, ESP_ERR_INVALID_ARG, err, TAG, "invalid argument"); - ESP_GOTO_ON_FALSE(led_config->led_pixel_format < LED_PIXEL_FORMAT_INVALID, ESP_ERR_INVALID_ARG, err, TAG, "invalid led_pixel_format"); - uint8_t bytes_per_pixel = 3; - if (led_config->led_pixel_format == LED_PIXEL_FORMAT_GRBW) { - bytes_per_pixel = 4; - } else if (led_config->led_pixel_format == LED_PIXEL_FORMAT_GRB) { - bytes_per_pixel = 3; + led_strip_timings_t timings; + ESP_GOTO_ON_ERROR(led_strip_get_timings(led_config->led_model, &led_config->timings, &timings), err, TAG, "invalid LED model"); + led_color_component_format_t component_fmt = led_config->color_component_format; + // If R/G/B order is not specified, set default GRB order as fallback + if (component_fmt.format_id == 0) { + component_fmt = LED_STRIP_COLOR_COMPONENT_FMT_GRB; + } + if (led_config->led_model == LED_MODEL_WS2816) { + component_fmt.format.bytes_per_color = 2; + } + if (component_fmt.format.bytes_per_color == 0) { + component_fmt.format.bytes_per_color = 1; + } + // check the validation of the color component format + uint8_t mask = 0; + if (component_fmt.format.num_components == 3) { + mask = BIT(component_fmt.format.r_pos) | BIT(component_fmt.format.g_pos) | BIT(component_fmt.format.b_pos); + // Check for invalid values + ESP_RETURN_ON_FALSE(mask == 0x07, ESP_ERR_INVALID_ARG, TAG, "invalid order argument"); + } else if (component_fmt.format.num_components == 4) { + mask = BIT(component_fmt.format.r_pos) | BIT(component_fmt.format.g_pos) | BIT(component_fmt.format.b_pos) | BIT(component_fmt.format.w_pos); + // Check for invalid values + ESP_RETURN_ON_FALSE(mask == 0x0F, ESP_ERR_INVALID_ARG, TAG, "invalid order argument"); } else { - assert(false); + ESP_RETURN_ON_FALSE(false, ESP_ERR_INVALID_ARG, TAG, "invalid number of color components: %d", component_fmt.format.num_components); + } + uint8_t bytes_per_pixel = component_fmt.format.num_components; + if (component_fmt.format.bytes_per_color > 1) { + bytes_per_pixel *= component_fmt.format.bytes_per_color; } rmt_strip = calloc(1, sizeof(led_strip_rmt_obj) + led_config->max_leds * bytes_per_pixel); ESP_GOTO_ON_FALSE(rmt_strip, ESP_ERR_NO_MEM, err, TAG, "no mem for rmt strip"); + atomic_init(&rmt_strip->trans_state, LED_STRIP_TRANS_IDLE); uint32_t resolution = rmt_config->resolution_hz ? rmt_config->resolution_hz : LED_STRIP_RMT_DEFAULT_RESOLUTION; // for backward compatibility, if the user does not set the clk_src, use the default value @@ -135,21 +304,24 @@ esp_err_t led_strip_new_rmt_device(const led_strip_config_t *led_config, const l led_strip_encoder_config_t strip_encoder_conf = { .resolution = resolution, - .led_model = led_config->led_model + .timings = timings, }; ESP_GOTO_ON_ERROR(rmt_new_led_strip_encoder(&strip_encoder_conf, &rmt_strip->strip_encoder), err, TAG, "create LED strip encoder failed"); - + rmt_strip->component_fmt = component_fmt; rmt_strip->bytes_per_pixel = bytes_per_pixel; rmt_strip->strip_len = led_config->max_leds; rmt_strip->base.set_pixel = led_strip_rmt_set_pixel; rmt_strip->base.set_pixel_rgbw = led_strip_rmt_set_pixel_rgbw; rmt_strip->base.refresh = led_strip_rmt_refresh; + rmt_strip->base.refresh_async = led_strip_rmt_refresh_async; + rmt_strip->base.refresh_async_done = led_strip_rmt_refresh_async_done; + rmt_strip->base.switch_gpio = led_strip_rmt_switch_gpio; rmt_strip->base.clear = led_strip_rmt_clear; rmt_strip->base.del = led_strip_rmt_del; *ret_strip = &rmt_strip->base; - return ESP_OK; + return ret; err: if (rmt_strip) { if (rmt_strip->rmt_chan) { diff --git a/ports/espressif/components/led_strip/src/led_strip_rmt_dev_idf4.c b/ports/espressif/components/led_strip/src/led_strip_rmt_dev_idf4.c deleted file mode 100644 index a1067cd7c..000000000 --- a/ports/espressif/components/led_strip/src/led_strip_rmt_dev_idf4.c +++ /dev/null @@ -1,194 +0,0 @@ -/* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD - * - * SPDX-License-Identifier: Apache-2.0 - */ -#include -#include -#include -#include "esp_log.h" -#include "esp_check.h" -#include "driver/rmt.h" -#include "led_strip.h" -#include "led_strip_interface.h" - -static const char *TAG = "led_strip_rmt"; - -#define WS2812_T0H_NS (300) -#define WS2812_T0L_NS (900) -#define WS2812_T1H_NS (900) -#define WS2812_T1L_NS (300) - -#define SK6812_T0H_NS (300) -#define SK6812_T0L_NS (900) -#define SK6812_T1H_NS (600) -#define SK6812_T1L_NS (600) - -#define LED_STRIP_RESET_MS (10) - -// the memory size of each RMT channel, in words (4 bytes) -#if CONFIG_IDF_TARGET_ESP32 || CONFIG_IDF_TARGET_ESP32S2 -#define LED_STRIP_RMT_DEFAULT_MEM_BLOCK_SYMBOLS 64 -#else -#define LED_STRIP_RMT_DEFAULT_MEM_BLOCK_SYMBOLS 48 -#endif - -static uint32_t led_t0h_ticks = 0; -static uint32_t led_t1h_ticks = 0; -static uint32_t led_t0l_ticks = 0; -static uint32_t led_t1l_ticks = 0; - -typedef struct { - led_strip_t base; - rmt_channel_t rmt_channel; - uint32_t strip_len; - uint8_t bytes_per_pixel; - uint8_t buffer[0]; -} led_strip_rmt_obj; - -static void IRAM_ATTR ws2812_rmt_adapter(const void *src, rmt_item32_t *dest, size_t src_size, - size_t wanted_num, size_t *translated_size, size_t *item_num) -{ - if (src == NULL || dest == NULL) { - *translated_size = 0; - *item_num = 0; - return; - } - const rmt_item32_t bit0 = {{{ led_t0h_ticks, 1, led_t0l_ticks, 0 }}}; //Logical 0 - const rmt_item32_t bit1 = {{{ led_t1h_ticks, 1, led_t1l_ticks, 0 }}}; //Logical 1 - size_t size = 0; - size_t num = 0; - uint8_t *psrc = (uint8_t *)src; - rmt_item32_t *pdest = dest; - while (size < src_size && num < wanted_num) { - for (int i = 0; i < 8; i++) { - // MSB first - if (*psrc & (1 << (7 - i))) { - pdest->val = bit1.val; - } else { - pdest->val = bit0.val; - } - num++; - pdest++; - } - size++; - psrc++; - } - *translated_size = size; - *item_num = num; -} - -static esp_err_t led_strip_rmt_set_pixel(led_strip_t *strip, uint32_t index, uint32_t red, uint32_t green, uint32_t blue) -{ - led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); - ESP_RETURN_ON_FALSE(index < rmt_strip->strip_len, ESP_ERR_INVALID_ARG, TAG, "index out of the maximum number of leds"); - uint32_t start = index * rmt_strip->bytes_per_pixel; - // In thr order of GRB - rmt_strip->buffer[start + 0] = green & 0xFF; - rmt_strip->buffer[start + 1] = red & 0xFF; - rmt_strip->buffer[start + 2] = blue & 0xFF; - if (rmt_strip->bytes_per_pixel > 3) { - rmt_strip->buffer[start + 3] = 0; - } - return ESP_OK; -} - -static esp_err_t led_strip_rmt_refresh(led_strip_t *strip) -{ - led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); - ESP_RETURN_ON_ERROR(rmt_write_sample(rmt_strip->rmt_channel, rmt_strip->buffer, rmt_strip->strip_len * rmt_strip->bytes_per_pixel, true), TAG, - "transmit RMT samples failed"); - vTaskDelay(pdMS_TO_TICKS(LED_STRIP_RESET_MS)); - return ESP_OK; -} - -static esp_err_t led_strip_rmt_clear(led_strip_t *strip) -{ - led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); - // Write zero to turn off all LEDs - memset(rmt_strip->buffer, 0, rmt_strip->strip_len * rmt_strip->bytes_per_pixel); - return led_strip_rmt_refresh(strip); -} - -static esp_err_t led_strip_rmt_del(led_strip_t *strip) -{ - led_strip_rmt_obj *rmt_strip = __containerof(strip, led_strip_rmt_obj, base); - ESP_RETURN_ON_ERROR(rmt_driver_uninstall(rmt_strip->rmt_channel), TAG, "uninstall RMT driver failed"); - free(rmt_strip); - return ESP_OK; -} - -esp_err_t led_strip_new_rmt_device(const led_strip_config_t *led_config, const led_strip_rmt_config_t *dev_config, led_strip_handle_t *ret_strip) -{ - led_strip_rmt_obj *rmt_strip = NULL; - esp_err_t ret = ESP_OK; - ESP_RETURN_ON_FALSE(led_config && dev_config && ret_strip, ESP_ERR_INVALID_ARG, TAG, "invalid argument"); - ESP_RETURN_ON_FALSE(led_config->led_pixel_format < LED_PIXEL_FORMAT_INVALID, ESP_ERR_INVALID_ARG, TAG, "invalid led_pixel_format"); - ESP_RETURN_ON_FALSE(dev_config->flags.with_dma == 0, ESP_ERR_NOT_SUPPORTED, TAG, "DMA is not supported"); - - uint8_t bytes_per_pixel = 3; - if (led_config->led_pixel_format == LED_PIXEL_FORMAT_GRBW) { - bytes_per_pixel = 4; - } else if (led_config->led_pixel_format == LED_PIXEL_FORMAT_GRB) { - bytes_per_pixel = 3; - } else { - assert(false); - } - - // allocate memory for led_strip object - rmt_strip = calloc(1, sizeof(led_strip_rmt_obj) + led_config->max_leds * bytes_per_pixel); - ESP_RETURN_ON_FALSE(rmt_strip, ESP_ERR_NO_MEM, TAG, "request memory for les_strip failed"); - - // install RMT channel driver - rmt_config_t config = RMT_DEFAULT_CONFIG_TX(led_config->strip_gpio_num, dev_config->rmt_channel); - // set the minimal clock division because the LED strip needs a high clock resolution - config.clk_div = 2; - - uint8_t mem_block_num = 2; - // override the default value if the user specify the mem block size - if (dev_config->mem_block_symbols) { - mem_block_num = (dev_config->mem_block_symbols + LED_STRIP_RMT_DEFAULT_MEM_BLOCK_SYMBOLS / 2) / LED_STRIP_RMT_DEFAULT_MEM_BLOCK_SYMBOLS; - } - config.mem_block_num = mem_block_num; - - ESP_GOTO_ON_ERROR(rmt_config(&config), err, TAG, "RMT config failed"); - ESP_GOTO_ON_ERROR(rmt_driver_install(config.channel, 0, 0), err, TAG, "RMT install failed"); - - uint32_t counter_clk_hz = 0; - rmt_get_counter_clock((rmt_channel_t)dev_config->rmt_channel, &counter_clk_hz); - // ns -> ticks - float ratio = (float)counter_clk_hz / 1e9; - if (led_config->led_model == LED_MODEL_WS2812) { - led_t0h_ticks = (uint32_t)(ratio * WS2812_T0H_NS); - led_t0l_ticks = (uint32_t)(ratio * WS2812_T0L_NS); - led_t1h_ticks = (uint32_t)(ratio * WS2812_T1H_NS); - led_t1l_ticks = (uint32_t)(ratio * WS2812_T1L_NS); - } else if (led_config->led_model == LED_MODEL_SK6812) { - led_t0h_ticks = (uint32_t)(ratio * SK6812_T0H_NS); - led_t0l_ticks = (uint32_t)(ratio * SK6812_T0L_NS); - led_t1h_ticks = (uint32_t)(ratio * SK6812_T1H_NS); - led_t1l_ticks = (uint32_t)(ratio * SK6812_T1L_NS); - } else { - assert(false); - } - - // adapter to translates the LES strip date frame into RMT symbols - rmt_translator_init((rmt_channel_t)dev_config->rmt_channel, ws2812_rmt_adapter); - - rmt_strip->bytes_per_pixel = bytes_per_pixel; - rmt_strip->rmt_channel = (rmt_channel_t)dev_config->rmt_channel; - rmt_strip->strip_len = led_config->max_leds; - rmt_strip->base.set_pixel = led_strip_rmt_set_pixel; - rmt_strip->base.refresh = led_strip_rmt_refresh; - rmt_strip->base.clear = led_strip_rmt_clear; - rmt_strip->base.del = led_strip_rmt_del; - - *ret_strip = &rmt_strip->base; - return ESP_OK; - -err: - if (rmt_strip) { - free(rmt_strip); - } - return ret; -} diff --git a/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.c b/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.c index d352ac07f..8c9ef6bd5 100644 --- a/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.c +++ b/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.c @@ -1,14 +1,40 @@ /* - * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ +#include "sdkconfig.h" +#include "esp_idf_version.h" #include "esp_check.h" +#include "esp_attr.h" #include "led_strip_rmt_encoder.h" static const char *TAG = "led_rmt_encoder"; +/** RMT symbol duration is a 15-bit field. */ +#define LED_STRIP_RMT_DURATION_MAX 32767 + +static uint32_t led_strip_time_to_ticks(uint32_t time, uint32_t resolution_hz, uint64_t divisor) +{ + // Round up to avoid shortening timings that have a datasheet minimum. + uint64_t duration = (uint64_t)time * resolution_hz; + return (uint32_t)(duration / divisor + (duration % divisor != 0)); +} + +static bool led_strip_rmt_duration_in_range(uint32_t ticks) +{ + return ticks >= 1 && ticks <= LED_STRIP_RMT_DURATION_MAX; +} + +#if ESP_IDF_VERSION < ESP_IDF_VERSION_VAL(5, 5, 0) +#if CONFIG_RMT_ISR_IRAM_SAFE +#define RMT_ENCODER_FUNC_ATTR IRAM_ATTR +#else +#define RMT_ENCODER_FUNC_ATTR +#endif // CONFIG_RMT_ISR_IRAM_SAFE +#endif // ESP_IDF_VERSION + typedef struct { rmt_encoder_t base; rmt_encoder_t *bytes_encoder; @@ -17,6 +43,7 @@ typedef struct { rmt_symbol_word_t reset_code; } rmt_led_strip_encoder_t; +RMT_ENCODER_FUNC_ATTR static size_t rmt_encode_led_strip(rmt_encoder_t *encoder, rmt_channel_handle_t channel, const void *primary_data, size_t data_size, rmt_encode_state_t *ret_state) { rmt_led_strip_encoder_t *led_encoder = __containerof(encoder, rmt_led_strip_encoder_t, base); @@ -62,6 +89,7 @@ static esp_err_t rmt_del_led_strip_encoder(rmt_encoder_t *encoder) return ESP_OK; } +RMT_ENCODER_FUNC_ATTR static esp_err_t rmt_led_strip_encoder_reset(rmt_encoder_t *encoder) { rmt_led_strip_encoder_t *led_encoder = __containerof(encoder, rmt_led_strip_encoder_t, base); @@ -76,54 +104,41 @@ esp_err_t rmt_new_led_strip_encoder(const led_strip_encoder_config_t *config, rm esp_err_t ret = ESP_OK; rmt_led_strip_encoder_t *led_encoder = NULL; ESP_GOTO_ON_FALSE(config && ret_encoder, ESP_ERR_INVALID_ARG, err, TAG, "invalid argument"); - ESP_GOTO_ON_FALSE(config->led_model < LED_MODEL_INVALID, ESP_ERR_INVALID_ARG, err, TAG, "invalid led model"); led_encoder = calloc(1, sizeof(rmt_led_strip_encoder_t)); ESP_GOTO_ON_FALSE(led_encoder, ESP_ERR_NO_MEM, err, TAG, "no mem for led strip encoder"); led_encoder->base.encode = rmt_encode_led_strip; led_encoder->base.del = rmt_del_led_strip_encoder; led_encoder->base.reset = rmt_led_strip_encoder_reset; - rmt_bytes_encoder_config_t bytes_encoder_config; - if (config->led_model == LED_MODEL_SK6812) { - bytes_encoder_config = (rmt_bytes_encoder_config_t) { - .bit0 = { - .level0 = 1, - .duration0 = 0.3 * config->resolution / 1000000, // T0H=0.3us - .level1 = 0, - .duration1 = 0.9 * config->resolution / 1000000, // T0L=0.9us - }, - .bit1 = { - .level0 = 1, - .duration0 = 0.6 * config->resolution / 1000000, // T1H=0.6us - .level1 = 0, - .duration1 = 0.6 * config->resolution / 1000000, // T1L=0.6us - }, - .flags.msb_first = 1 // SK6812 transfer bit order: G7...G0R7...R0B7...B0(W7...W0) - }; - } else if (config->led_model == LED_MODEL_WS2812) { - // different led strip might have its own timing requirements, following parameter is for WS2812 - bytes_encoder_config = (rmt_bytes_encoder_config_t) { - .bit0 = { - .level0 = 1, - .duration0 = 0.3 * config->resolution / 1000000, // T0H=0.3us - .level1 = 0, - .duration1 = 0.9 * config->resolution / 1000000, // T0L=0.9us - }, - .bit1 = { - .level0 = 1, - .duration0 = 0.9 * config->resolution / 1000000, // T1H=0.9us - .level1 = 0, - .duration1 = 0.3 * config->resolution / 1000000, // T1L=0.3us - }, - .flags.msb_first = 1 // WS2812 transfer bit order: G7...G0R7...R0B7...B0 - }; - } else { - assert(false); - } + // Each duration is stored in a 15-bit RMT field, so reject values that become 0 or overflow. + uint32_t t0h_ticks = led_strip_time_to_ticks(config->timings.t0h_ns, config->resolution, 1000000000); + uint32_t t0l_ticks = led_strip_time_to_ticks(config->timings.t0l_ns, config->resolution, 1000000000); + uint32_t t1h_ticks = led_strip_time_to_ticks(config->timings.t1h_ns, config->resolution, 1000000000); + uint32_t t1l_ticks = led_strip_time_to_ticks(config->timings.t1l_ns, config->resolution, 1000000000); + uint32_t reset_half_ticks = led_strip_time_to_ticks(config->timings.reset_us, config->resolution, 1000000 * 2); + ESP_GOTO_ON_FALSE(led_strip_rmt_duration_in_range(t0h_ticks) && led_strip_rmt_duration_in_range(t0l_ticks) && + led_strip_rmt_duration_in_range(t1h_ticks) && led_strip_rmt_duration_in_range(t1l_ticks) && + led_strip_rmt_duration_in_range(reset_half_ticks), + ESP_ERR_INVALID_ARG, err, TAG, "timing is outside RMT duration range (1-%d ticks)", LED_STRIP_RMT_DURATION_MAX); + rmt_bytes_encoder_config_t bytes_encoder_config = { + .bit0 = { + .level0 = 1, + .duration0 = t0h_ticks, + .level1 = 0, + .duration1 = t0l_ticks, + }, + .bit1 = { + .level0 = 1, + .duration0 = t1h_ticks, + .level1 = 0, + .duration1 = t1l_ticks, + }, + .flags.msb_first = 1 + }; + uint32_t reset_ticks = reset_half_ticks; ESP_GOTO_ON_ERROR(rmt_new_bytes_encoder(&bytes_encoder_config, &led_encoder->bytes_encoder), err, TAG, "create bytes encoder failed"); rmt_copy_encoder_config_t copy_encoder_config = {}; ESP_GOTO_ON_ERROR(rmt_new_copy_encoder(©_encoder_config, &led_encoder->copy_encoder), err, TAG, "create copy encoder failed"); - uint32_t reset_ticks = config->resolution / 1000000 * 50 / 2; // reset code duration defaults to 50us led_encoder->reset_code = (rmt_symbol_word_t) { .level0 = 0, .duration0 = reset_ticks, diff --git a/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.h b/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.h index ba71e60ab..0b05b435b 100644 --- a/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.h +++ b/ports/espressif/components/led_strip/src/led_strip_rmt_encoder.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: 2022 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ @@ -17,8 +17,8 @@ extern "C" { * @brief Type of led strip encoder configuration */ typedef struct { - uint32_t resolution; /*!< Encoder resolution, in Hz */ - led_model_t led_model; /*!< LED model */ + uint32_t resolution; /*!< Encoder resolution, in Hz */ + led_strip_timings_t timings; /*!< Encoder timings */ } led_strip_encoder_config_t; /** diff --git a/ports/espressif/components/led_strip/src/led_strip_spi_dev.c b/ports/espressif/components/led_strip/src/led_strip_spi_dev.c index 70458d1c6..d0dc6c659 100644 --- a/ports/espressif/components/led_strip/src/led_strip_spi_dev.c +++ b/ports/espressif/components/led_strip/src/led_strip_spi_dev.c @@ -1,25 +1,29 @@ /* - * SPDX-FileCopyrightText: 2022-2023 Espressif Systems (Shanghai) CO LTD + * SPDX-FileCopyrightText: 2022-2025 Espressif Systems (Shanghai) CO LTD * * SPDX-License-Identifier: Apache-2.0 */ #include #include #include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" #include "esp_log.h" #include "esp_check.h" -#include "esp_heap_caps.h" #include "esp_rom_gpio.h" +#include "esp_heap_caps.h" #include "soc/spi_periph.h" #include "led_strip.h" #include "led_strip_interface.h" -#include "hal/spi_hal.h" +#include "led_strip_common.h" #define LED_STRIP_SPI_DEFAULT_RESOLUTION (2.5 * 1000 * 1000) // 2.5MHz resolution +#define LED_STRIP_SPI_WS2816_RESOLUTION (2.7 * 1000 * 1000) // 2.7MHz resolution #define LED_STRIP_SPI_DEFAULT_TRANS_QUEUE_SIZE 4 #define SPI_BYTES_PER_COLOR_BYTE 3 #define SPI_BITS_PER_COLOR_BYTE (SPI_BYTES_PER_COLOR_BYTE * 8) +#define SPI_TRANS_TIMEOUT portMAX_DELAY static const char *TAG = "led_strip_spi"; @@ -29,9 +33,29 @@ typedef struct { spi_device_handle_t spi_device; uint32_t strip_len; uint8_t bytes_per_pixel; + led_color_component_format_t component_fmt; + spi_transaction_t trans; + led_strip_trans_state_atomic_t trans_state; uint8_t pixel_buf[]; } led_strip_spi_obj; +static esp_err_t led_strip_spi_trans(led_strip_spi_obj *spi_strip) +{ + memset(&spi_strip->trans, 0, sizeof(spi_strip->trans)); + spi_strip->trans.length = spi_strip->strip_len * spi_strip->bytes_per_pixel * SPI_BITS_PER_COLOR_BYTE; + spi_strip->trans.tx_buffer = spi_strip->pixel_buf; + spi_strip->trans.rx_buffer = NULL; + return spi_device_queue_trans(spi_strip->spi_device, &spi_strip->trans, SPI_TRANS_TIMEOUT); +} + +static esp_err_t led_strip_spi_wait_trans_done(led_strip_spi_obj *spi_strip) +{ + spi_transaction_t *tx_conf = NULL; + ESP_RETURN_ON_ERROR(spi_device_get_trans_result(spi_strip->spi_device, &tx_conf, SPI_TRANS_TIMEOUT), TAG, "wait for SPI done failed"); + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ESP_OK; +} + // please make sure to zero-initialize the buf before calling this function static void __led_strip_spi_bit(uint8_t data, uint8_t *buf) { @@ -52,52 +76,100 @@ static esp_err_t led_strip_spi_set_pixel(led_strip_t *strip, uint32_t index, uin { led_strip_spi_obj *spi_strip = __containerof(strip, led_strip_spi_obj, base); ESP_RETURN_ON_FALSE(index < spi_strip->strip_len, ESP_ERR_INVALID_ARG, TAG, "index out of maximum number of LEDs"); - // LED_PIXEL_FORMAT_GRB takes 72bits(9bytes) + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "SPI transaction is busy"); + // 3 pixels take 72bits(9bytes) uint32_t start = index * spi_strip->bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE; - memset(spi_strip->pixel_buf + start, 0, spi_strip->bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE); - __led_strip_spi_bit(green, &spi_strip->pixel_buf[start]); - __led_strip_spi_bit(red, &spi_strip->pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE]); - __led_strip_spi_bit(blue, &spi_strip->pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * 2]); - if (spi_strip->bytes_per_pixel > 3) { - __led_strip_spi_bit(0, &spi_strip->pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * 3]); + uint8_t *pixel_buf = spi_strip->pixel_buf; + struct format_layout format = spi_strip->component_fmt.format; + memset(pixel_buf + start, 0, spi_strip->bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE); + + uint8_t pos_bytes = format.bytes_per_color; + for (uint8_t i = 0; i < format.bytes_per_color; i++) { + uint8_t color_shift = 8 * (format.bytes_per_color - 1 - i); + __led_strip_spi_bit((red >> color_shift) & 0xFF, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.r_pos * pos_bytes + i)]); + __led_strip_spi_bit((green >> color_shift) & 0xFF, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.g_pos * pos_bytes + i)]); + __led_strip_spi_bit((blue >> color_shift) & 0xFF, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.b_pos * pos_bytes + i)]); + if (format.num_components > 3) { + __led_strip_spi_bit(0, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.w_pos * pos_bytes + i)]); + } } + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); return ESP_OK; } static esp_err_t led_strip_spi_set_pixel_rgbw(led_strip_t *strip, uint32_t index, uint32_t red, uint32_t green, uint32_t blue, uint32_t white) { led_strip_spi_obj *spi_strip = __containerof(strip, led_strip_spi_obj, base); + struct format_layout format = spi_strip->component_fmt.format; ESP_RETURN_ON_FALSE(index < spi_strip->strip_len, ESP_ERR_INVALID_ARG, TAG, "index out of maximum number of LEDs"); - ESP_RETURN_ON_FALSE(spi_strip->bytes_per_pixel == 4, ESP_ERR_INVALID_ARG, TAG, "wrong LED pixel format, expected 4 bytes per pixel"); + ESP_RETURN_ON_FALSE(format.num_components == 4, ESP_ERR_INVALID_ARG, TAG, "led doesn't have 4 components"); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "SPI transaction is busy"); + // LED_PIXEL_FORMAT_GRBW takes 96bits(12bytes) uint32_t start = index * spi_strip->bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE; - // SK6812 component order is GRBW - memset(spi_strip->pixel_buf + start, 0, spi_strip->bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE); - __led_strip_spi_bit(green, &spi_strip->pixel_buf[start]); - __led_strip_spi_bit(red, &spi_strip->pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE]); - __led_strip_spi_bit(blue, &spi_strip->pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * 2]); - __led_strip_spi_bit(white, &spi_strip->pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * 3]); + uint8_t *pixel_buf = spi_strip->pixel_buf; + memset(pixel_buf + start, 0, spi_strip->bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE); + uint8_t pos_bytes = format.bytes_per_color; + for (uint8_t i = 0; i < format.bytes_per_color; i++) { + uint8_t color_shift = 8 * (format.bytes_per_color - 1 - i); + __led_strip_spi_bit((red >> color_shift) & 0xFF, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.r_pos * pos_bytes + i)]); + __led_strip_spi_bit((green >> color_shift) & 0xFF, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.g_pos * pos_bytes + i)]); + __led_strip_spi_bit((blue >> color_shift) & 0xFF, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.b_pos * pos_bytes + i)]); + __led_strip_spi_bit((white >> color_shift) & 0xFF, &pixel_buf[start + SPI_BYTES_PER_COLOR_BYTE * (format.w_pos * pos_bytes + i)]); + } + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); return ESP_OK; } -static esp_err_t led_strip_spi_refresh(led_strip_t *strip) +static esp_err_t led_strip_spi_refresh_async(led_strip_t *strip) { + esp_err_t ret = ESP_OK; led_strip_spi_obj *spi_strip = __containerof(strip, led_strip_spi_obj, base); - spi_transaction_t tx_conf; - memset(&tx_conf, 0, sizeof(tx_conf)); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_INFLIGHT), + ESP_ERR_INVALID_STATE, TAG, "SPI transaction is busy"); + ESP_GOTO_ON_ERROR(led_strip_spi_trans(spi_strip), err, TAG, "transmit pixels by SPI failed"); + return ret; +err: + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; +} - tx_conf.length = spi_strip->strip_len * spi_strip->bytes_per_pixel * SPI_BITS_PER_COLOR_BYTE; - tx_conf.tx_buffer = spi_strip->pixel_buf; - tx_conf.rx_buffer = NULL; - ESP_RETURN_ON_ERROR(spi_device_transmit(spi_strip->spi_device, &tx_conf), TAG, "transmit pixels by SPI failed"); +static esp_err_t led_strip_spi_refresh_async_done(led_strip_t *strip) +{ + esp_err_t ret = ESP_OK; + led_strip_spi_obj *spi_strip = __containerof(strip, led_strip_spi_obj, base); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&spi_strip->trans_state, LED_STRIP_TRANS_INFLIGHT, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "no async refresh in progress"); + ESP_GOTO_ON_ERROR(led_strip_spi_wait_trans_done(spi_strip), err, TAG, "wait for done failed"); + return ret; +err: + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_INFLIGHT); + return ret; +} - return ESP_OK; +static esp_err_t led_strip_spi_refresh(led_strip_t *strip) +{ + esp_err_t ret = ESP_OK; + led_strip_spi_obj *spi_strip = __containerof(strip, led_strip_spi_obj, base); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "SPI transaction is busy"); + ESP_GOTO_ON_ERROR(led_strip_spi_trans(spi_strip), err, TAG, "transmit pixels by SPI failed"); + ESP_GOTO_ON_ERROR(led_strip_spi_wait_trans_done(spi_strip), err, TAG, "wait for done failed"); + return ret; +err: + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; } static esp_err_t led_strip_spi_clear(led_strip_t *strip) { + esp_err_t ret = ESP_OK; led_strip_spi_obj *spi_strip = __containerof(strip, led_strip_spi_obj, base); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "SPI transaction is busy"); //Write zero to turn off all leds memset(spi_strip->pixel_buf, 0, spi_strip->strip_len * spi_strip->bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE); uint8_t *buf = spi_strip->pixel_buf; @@ -106,18 +178,29 @@ static esp_err_t led_strip_spi_clear(led_strip_t *strip) buf += SPI_BYTES_PER_COLOR_BYTE; } - return led_strip_spi_refresh(strip); + ESP_GOTO_ON_ERROR(led_strip_spi_trans(spi_strip), err, TAG, "transmit pixels by SPI failed"); + ESP_GOTO_ON_ERROR(led_strip_spi_wait_trans_done(spi_strip), err, TAG, "wait for done failed"); + return ret; +err: + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; } static esp_err_t led_strip_spi_del(led_strip_t *strip) { + esp_err_t ret = ESP_OK; led_strip_spi_obj *spi_strip = __containerof(strip, led_strip_spi_obj, base); - ESP_RETURN_ON_ERROR(spi_bus_remove_device(spi_strip->spi_device), TAG, "delete spi device failed"); - ESP_RETURN_ON_ERROR(spi_bus_free(spi_strip->spi_host), TAG, "free spi bus failed"); + ESP_RETURN_ON_FALSE(led_strip_trans_state_try_set(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE, LED_STRIP_TRANS_LOCKED), + ESP_ERR_INVALID_STATE, TAG, "SPI transaction is busy"); + ESP_GOTO_ON_ERROR(spi_bus_remove_device(spi_strip->spi_device), err, TAG, "delete spi device failed"); + ESP_GOTO_ON_ERROR(spi_bus_free(spi_strip->spi_host), err, TAG, "free spi bus failed"); free(spi_strip); - return ESP_OK; + return ret; +err: + atomic_store(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); + return ret; } esp_err_t led_strip_new_spi_device(const led_strip_config_t *led_config, const led_strip_spi_config_t *spi_config, led_strip_handle_t *ret_strip) @@ -125,14 +208,34 @@ esp_err_t led_strip_new_spi_device(const led_strip_config_t *led_config, const l led_strip_spi_obj *spi_strip = NULL; esp_err_t ret = ESP_OK; ESP_GOTO_ON_FALSE(led_config && spi_config && ret_strip, ESP_ERR_INVALID_ARG, err, TAG, "invalid argument"); - ESP_GOTO_ON_FALSE(led_config->led_pixel_format < LED_PIXEL_FORMAT_INVALID, ESP_ERR_INVALID_ARG, err, TAG, "invalid led_pixel_format"); - uint8_t bytes_per_pixel = 3; - if (led_config->led_pixel_format == LED_PIXEL_FORMAT_GRBW) { - bytes_per_pixel = 4; - } else if (led_config->led_pixel_format == LED_PIXEL_FORMAT_GRB) { - bytes_per_pixel = 3; + led_color_component_format_t component_fmt = led_config->color_component_format; + uint32_t clock_speed_hz = LED_STRIP_SPI_DEFAULT_RESOLUTION; + // If R/G/B order is not specified, set default GRB order as fallback + if (component_fmt.format_id == 0) { + component_fmt = LED_STRIP_COLOR_COMPONENT_FMT_GRB; + } + if (led_config->led_model == LED_MODEL_WS2816) { + component_fmt.format.bytes_per_color = 2; + clock_speed_hz = LED_STRIP_SPI_WS2816_RESOLUTION; + } + if (component_fmt.format.bytes_per_color == 0) { + component_fmt.format.bytes_per_color = 1; + } + uint8_t mask = 0; + if (component_fmt.format.num_components == 3) { + mask = BIT(component_fmt.format.r_pos) | BIT(component_fmt.format.g_pos) | BIT(component_fmt.format.b_pos); + // Check for invalid values + ESP_RETURN_ON_FALSE(mask == 0x07, ESP_ERR_INVALID_ARG, TAG, "invalid order argument"); + } else if (component_fmt.format.num_components == 4) { + mask = BIT(component_fmt.format.r_pos) | BIT(component_fmt.format.g_pos) | BIT(component_fmt.format.b_pos) | BIT(component_fmt.format.w_pos); + // Check for invalid values + ESP_RETURN_ON_FALSE(mask == 0x0F, ESP_ERR_INVALID_ARG, TAG, "invalid order argument"); } else { - assert(false); + ESP_RETURN_ON_FALSE(false, ESP_ERR_INVALID_ARG, TAG, "invalid number of color components: %d", component_fmt.format.num_components); + } + uint8_t bytes_per_pixel = component_fmt.format.num_components; + if (component_fmt.format.bytes_per_color > 1) { + bytes_per_pixel *= component_fmt.format.bytes_per_color; } uint32_t mem_caps = MALLOC_CAP_DEFAULT; if (spi_config->flags.with_dma) { @@ -143,6 +246,7 @@ esp_err_t led_strip_new_spi_device(const led_strip_config_t *led_config, const l ESP_GOTO_ON_FALSE(spi_strip, ESP_ERR_NO_MEM, err, TAG, "no mem for spi strip"); + atomic_init(&spi_strip->trans_state, LED_STRIP_TRANS_IDLE); spi_strip->spi_host = spi_config->spi_bus; // for backward compatibility, if the user does not set the clk_src, use the default value spi_clock_source_t clk_src = SPI_CLK_SRC_DEFAULT; @@ -159,18 +263,24 @@ esp_err_t led_strip_new_spi_device(const led_strip_config_t *led_config, const l .quadhd_io_num = -1, .max_transfer_sz = led_config->max_leds * bytes_per_pixel * SPI_BYTES_PER_COLOR_BYTE, }; +#ifdef SPICOMMON_BUSFLAG_DATA_OUT_INV + if (led_config->flags.invert_out == true) { + spi_bus_cfg.flags |= SPICOMMON_BUSFLAG_DATA_OUT_INV; + } +#endif ESP_GOTO_ON_ERROR(spi_bus_initialize(spi_strip->spi_host, &spi_bus_cfg, spi_config->flags.with_dma ? SPI_DMA_CH_AUTO : SPI_DMA_DISABLED), err, TAG, "create SPI bus failed"); +#ifndef SPICOMMON_BUSFLAG_DATA_OUT_INV if (led_config->flags.invert_out == true) { esp_rom_gpio_connect_out_signal(led_config->strip_gpio_num, spi_periph_signal[spi_strip->spi_host].spid_out, true, false); } - +#endif spi_device_interface_config_t spi_dev_cfg = { .clock_source = clk_src, .command_bits = 0, .address_bits = 0, .dummy_bits = 0, - .clock_speed_hz = LED_STRIP_SPI_DEFAULT_RESOLUTION, + .clock_speed_hz = clock_speed_hz, .mode = 0, //set -1 when CS is not used .spics_io_num = -1, @@ -178,24 +288,32 @@ esp_err_t led_strip_new_spi_device(const led_strip_config_t *led_config, const l }; ESP_GOTO_ON_ERROR(spi_bus_add_device(spi_strip->spi_host, &spi_dev_cfg, &spi_strip->spi_device), err, TAG, "Failed to add spi device"); - + //ensure the reset time is enough + esp_rom_delay_us(10); int clock_resolution_khz = 0; spi_device_get_actual_freq(spi_strip->spi_device, &clock_resolution_khz); // TODO: ideally we should decide the SPI_BYTES_PER_COLOR_BYTE by the real clock resolution - // But now, let's fixed the resolution, the downside is, we don't support a clock source whose frequency is not multiple of LED_STRIP_SPI_DEFAULT_RESOLUTION - ESP_GOTO_ON_FALSE(clock_resolution_khz == LED_STRIP_SPI_DEFAULT_RESOLUTION / 1000, ESP_ERR_NOT_SUPPORTED, err, + // But now, let's fixed the resolution + ESP_GOTO_ON_FALSE((clock_resolution_khz < clock_speed_hz / 1000 + 300) && (clock_resolution_khz > clock_speed_hz / 1000 - 300), ESP_ERR_NOT_SUPPORTED, err, TAG, "unsupported clock resolution:%dKHz", clock_resolution_khz); + if (led_config->led_model != LED_MODEL_WS2812 && led_config->led_model != LED_MODEL_WS2816) { + ESP_LOGW(TAG, "Only support WS2812 and WS2816. The timing requirements for other models may not be met"); + } + + spi_strip->component_fmt = component_fmt; spi_strip->bytes_per_pixel = bytes_per_pixel; spi_strip->strip_len = led_config->max_leds; spi_strip->base.set_pixel = led_strip_spi_set_pixel; spi_strip->base.set_pixel_rgbw = led_strip_spi_set_pixel_rgbw; spi_strip->base.refresh = led_strip_spi_refresh; + spi_strip->base.refresh_async = led_strip_spi_refresh_async; + spi_strip->base.refresh_async_done = led_strip_spi_refresh_async_done; spi_strip->base.clear = led_strip_spi_clear; spi_strip->base.del = led_strip_spi_del; *ret_strip = &spi_strip->base; - return ESP_OK; + return ret; err: if (spi_strip) { if (spi_strip->spi_device) { diff --git a/ports/espressif/components/tinyusb_src/tusb_config.h b/ports/espressif/components/tinyusb_src/tusb_config.h index c933a0cb4..2acb7601f 100644 --- a/ports/espressif/components/tinyusb_src/tusb_config.h +++ b/ports/espressif/components/tinyusb_src/tusb_config.h @@ -58,6 +58,11 @@ #define BOARD_TUD_RHPORT 0 #endif +#if CONFIG_IDF_TARGET_ESP32P4 && BOARD_TUD_RHPORT == 0 +// rhport 0 is the P4's OTG FS controller +#define CFG_TUD_MAX_SPEED OPT_MODE_FULL_SPEED +#endif + // can be defined by compiler in DEBUG build #ifndef CFG_TUSB_DEBUG #define CFG_TUSB_DEBUG 0 diff --git a/ports/espressif/family.cmake b/ports/espressif/family.cmake index 0e691c173..c263aa2aa 100644 --- a/ports/espressif/family.cmake +++ b/ports/espressif/family.cmake @@ -1,8 +1,11 @@ include(${CMAKE_CURRENT_LIST_DIR}/boards/${BOARD}/board.cmake) if(IDF_TARGET STREQUAL "esp32p4") - # TinyUF2 and the apps run on OTG HS - add_compile_definitions(BOARD_TUD_RHPORT=1) + # TinyUF2 and the apps run on OTG HS unless the board routes OTG FS + if(NOT DEFINED BOARD_TUD_RHPORT) + set(BOARD_TUD_RHPORT 1) + endif() + add_compile_definitions(BOARD_TUD_RHPORT=${BOARD_TUD_RHPORT}) # Chip revisions <3.0 and >=3.0 need separate images: standalone apps take the board sdkconfig, which selects one set(SDKCONFIG_DEFAULTS ${CMAKE_CURRENT_SOURCE_DIR}/sdkconfig.defaults ${BOARD_PATH}/sdkconfig) endif() diff --git a/ports/espressif/sdkconfig.defaults b/ports/espressif/sdkconfig.defaults index d3db6404a..ad4b226c7 100644 --- a/ports/espressif/sdkconfig.defaults +++ b/ports/espressif/sdkconfig.defaults @@ -1,5 +1,4 @@ CONFIG_IDF_CMAKE=y -CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y CONFIG_FREERTOS_SUPPORT_STATIC_ALLOCATION=y # Partition Table diff --git a/ports/family_support.cmake b/ports/family_support.cmake index 41e546b4a..3fdb1f511 100644 --- a/ports/family_support.cmake +++ b/ports/family_support.cmake @@ -369,9 +369,19 @@ function(family_flash_stlink TARGET) set(STM32_PROGRAMMER_CLI STM32_Programmer_CLI) endif () + if (ARGC GREATER 1) + set(BIN_FILE $/${TARGET}.${ARGV1}) + else () + set(BIN_FILE $) + endif () + + if (DEFINED STLINK_SERIAL) + set(STLINK_SN sn=${STLINK_SERIAL}) + endif () + add_custom_target(${TARGET}-stlink DEPENDS ${TARGET} - COMMAND ${STM32_PROGRAMMER_CLI} --connect port=swd --write $ --go + COMMAND ${STM32_PROGRAMMER_CLI} --connect port=swd ${STLINK_SN} --write ${BIN_FILE} --go ) endfunction() diff --git a/ports/samd51/CMakeLists.txt b/ports/samd51/CMakeLists.txt new file mode 100644 index 000000000..1458471fd --- /dev/null +++ b/ports/samd51/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.20) + +include(${CMAKE_CURRENT_LIST_DIR}/../family_support.cmake) + +project(tinyuf2 C ASM) +set(CMAKE_EXECUTABLE_SUFFIX .elf) + +#------------------------------------ +# TinyUF2 +#------------------------------------ +add_executable(tinyuf2 + board_flash.c + boards.c + ${TOP}/lib/tinyusb/src/portable/microchip/samd/dcd_samd.c + ) +target_link_options(tinyuf2 PUBLIC + "LINKER:--script=${CMAKE_CURRENT_LIST_DIR}/linker/samd51_boot.ld" + ) + +family_configure_tinyuf2(tinyuf2 OPT_MCU_SAMD51) +family_flash_openocd(tinyuf2) + +#------------------------------------ +# Application +#------------------------------------ +family_add_app(update-tinyuf2 ${TOP}/apps/self_update SELF_UPDATE) +family_add_app(blinky ${TOP}/apps/blinky) diff --git a/ports/samd51/README.md b/ports/samd51/README.md new file mode 100644 index 000000000..b9b400a3c --- /dev/null +++ b/ports/samd51/README.md @@ -0,0 +1,50 @@ +# TinyUF2 for SAMD51 + +Port for SAMD51 with the 1 MB / 256 KB parts (SAMD51N20A). Boards: + +| Board | Notes | +| --- | --- | +| `intech_grid` | Intech Studio Grid modules, volume `GRID26` | +| `intech_wrid` | Same hardware and USB IDs as `intech_grid`, volume `WRID26`, to tell the two bootloaders apart | + +## Flash layout + +| Address | Content | +| --- | --- | +| `0x00000000` | TinyUF2, 16 KB, write protected with BOOTPROT | +| `0x00004000` | Application | + +The application must be linked at `0x4000`. To create a UF2 image from a `.bin` file: + +``` +uf2conv.py -c -b 0x4000 -f 0x55114460 firmware.bin +``` + +## Entering the bootloader + +- Hold the button (`PC11`, active low) while the board resets or is plugged in. +- The bootloader also stays when the application is missing or invalid. +- Double tap and reboot from application are not implemented. + +## Self-update + +`update-tinyuf2.uf2` is built next to the bootloader in `_bin//apps`. Copy it to the bootloader drive: it runs from the application area, +rewrites the bootloader and invalidates itself, then the new bootloader starts. The application area is overwritten, so +copy the application again afterwards. + +- The bootloader protects itself on every boot: if the BOOTPROT fuse in the user page does not match the 16 KB boot size, it is rewritten + and the board resets once. The self-update lifts the protection until the next reset (`SBPDIS`) to write the new bootloader. +- Rewriting the BOOTPROT fuse erases the user page before it is programmed again. A power cut in between leaves the user page blank, + and the bootloader then leaves it alone: restore it with an SWD probe. +- Updating between `intech_grid` and `intech_wrid` works the same way, the drive label shows which one is installed. +- A power cut while the bootloader area is erased leaves the board without a bootloader: recover it with an SWD probe. To program the + bootloader over SWD, first remove the protection with `atsame5 bootloader 0` in OpenOCD and reset. + +## Debugging + +- `BOD33` is set to 2.7 V at start. The default level is too low and flash writes can store zeros. +- The application usually takes over the SWD pins. Hold the button while powering up so the bootloader runs and SWD stays usable. +- The bootloader must fit in 16 KB. A Debug build or logging (`-DLOG=1 -DLOGGER=RTT`) does not: the Debug build type increases the boot + size to 64 KB, and the application then has to be linked at `0x10000`. +- Flash with openocd and a CMSIS-DAP probe: `cmake --build build --target tinyuf2-openocd`. It lifts the bootloader protection first, + the bootloader protects itself again at boot. diff --git a/ports/samd51/app.cmake b/ports/samd51/app.cmake new file mode 100644 index 000000000..5d9fcf4db --- /dev/null +++ b/ports/samd51/app.cmake @@ -0,0 +1,8 @@ +include_guard(GLOBAL) + +function(family_configure_app TARGET) + family_configure_common(${TARGET}) + target_link_options(${TARGET} PUBLIC + "LINKER:--script=${CMAKE_CURRENT_FUNCTION_LIST_DIR}/linker/samd51_app.ld" + ) +endfunction() diff --git a/ports/samd51/board_flash.c b/ports/samd51/board_flash.c new file mode 100644 index 000000000..7c7454eca --- /dev/null +++ b/ports/samd51/board_flash.c @@ -0,0 +1,323 @@ +#include "board_api.h" + +//--------------------------------------------------------------------+ +// MACRO TYPEDEF CONSTANT ENUM +//--------------------------------------------------------------------+ + +// SAMD51 erases 8 KB blocks and programs quad words (16 bytes). UF2 payloads are 256 bytes +#define FLASH_BLOCK_SIZE NVMCTRL_BLOCK_SIZE +#define FLASH_ROW_SIZE 256 +#define QUAD_WORD_SIZE 16 + +#define USER_PAGE_ADDR 0x00804000 +#define USER_PAGE_SIZE 512 +#define BOOTPROT_POS 26 +#define BOOTPROT_MSK (0xFu << BOOTPROT_POS) + +enum { + BLOCK_COUNT = BOARD_FLASH_SIZE / FLASH_BLOCK_SIZE, + ROWS_PER_BLOCK = FLASH_BLOCK_SIZE / FLASH_ROW_SIZE +}; +_Static_assert(ROWS_PER_BLOCK <= 32, "good_rows is a 32 bit mask"); + +// rows of each block known to hold their data, written or found identical since the flash was initialized. +// They are written back when the block has to be erased. +static uint32_t good_rows[BLOCK_COUNT]; +// holds the rows to keep while their block is erased, or a block image while updating the bootloader +static uint8_t block_cache[FLASH_BLOCK_SIZE]; + +//--------------------------------------------------------------------+ +// Internal Helper +//--------------------------------------------------------------------+ +static inline void wait_ready(void) { + while (NVMCTRL->STATUS.bit.READY == 0) {} +} + +static inline void exec_cmd(uint32_t cmd) { + wait_ready(); + NVMCTRL->CTRLB.reg = NVMCTRL_CTRLB_CMDEX_KEY | cmd; + wait_ready(); +} + +// The cache in Rev A is not reliable when reading and writing to the NVM +static void cache_disable(void) { + NVMCTRL->CTRLA.bit.CACHEDIS0 = true; + NVMCTRL->CTRLA.bit.CACHEDIS1 = true; +} + +// Flash starts at address 0: volatile keeps reads of the bootloader from being treated as null dereferences +static bool is_blank(uint32_t addr, uint32_t size) { + const volatile uint32_t *flash = (const volatile uint32_t *)addr; + for (uint32_t i = 0; i < size / sizeof(uint32_t); i++) { + if (flash[i] != 0xffffffff) { + return false; + } + } + return true; +} + +static bool is_equal(uint32_t addr, const uint8_t *data, uint32_t size) { + const volatile uint8_t *flash = (const volatile uint8_t *)addr; + for (uint32_t i = 0; i < size; i++) { + if (flash[i] != data[i]) { + return false; + } + } + return true; +} + +static bool is_valid_row(uint32_t addr, uint32_t len) { + return len == FLASH_ROW_SIZE && (addr % FLASH_ROW_SIZE) == 0 && addr >= BOARD_FLASH_APP_START && + addr <= BOARD_FLASH_ADDR_ZERO + BOARD_FLASH_SIZE - len; +} + +// Erase a block, skipped if it is already blank +static bool flash_erase(uint32_t block) { + TUF2_ASSERT(block < BLOCK_COUNT); + + const uint32_t addr = BOARD_FLASH_ADDR_ZERO + block * FLASH_BLOCK_SIZE; + if (is_blank(addr, FLASH_BLOCK_SIZE)) { + return true; + } + + TUF2_LOG1("Erase: %08lX size = %u KB ... ", addr, FLASH_BLOCK_SIZE / 1024); + cache_disable(); + NVMCTRL->ADDR.reg = addr; + exec_cmd(NVMCTRL_CTRLB_CMD_EB); + TUF2_ASSERT(is_blank(addr, FLASH_BLOCK_SIZE)); + TUF2_LOG1("OK\r\n"); + + return true; +} + +// Quad word writes allow 256 byte UF2 payloads: a 512 byte page write would need the +// neighbouring payload read back from flash +static bool flash_program(uint32_t dst, const uint8_t *src, uint32_t len) { + TUF2_LOG1("Write flash at address %08lX\r\n", dst); + + cache_disable(); + NVMCTRL->CTRLA.bit.WMODE = NVMCTRL_CTRLA_WMODE_MAN_Val; + exec_cmd(NVMCTRL_CTRLB_CMD_PBC); + + for (uint32_t i = 0; i < len; i += QUAD_WORD_SIZE) { + const uint32_t chunk = (len - i) < QUAD_WORD_SIZE ? (len - i) : QUAD_WORD_SIZE; + volatile uint32_t *flash = (volatile uint32_t *)(dst + i); + + uint32_t words[QUAD_WORD_SIZE / sizeof(uint32_t)]; + memset(words, 0xff, sizeof(words)); + memcpy(words, src + i, chunk); + + // writes to the page buffer must be 16 or 32 bits wide + wait_ready(); + for (uint32_t w = 0; w < QUAD_WORD_SIZE / sizeof(uint32_t); w++) { + flash[w] = words[w]; + } + + NVMCTRL->ADDR.reg = dst + i; + exec_cmd(NVMCTRL_CTRLB_CMD_WQW); + } + + TUF2_VERIFY(is_equal(dst, src, len)); + return true; +} + +// Erase a block, writing back the rows to keep +static bool flash_erase_block_keep_rows(uint32_t block, uint32_t keep) { + const uint32_t base = BOARD_FLASH_ADDR_ZERO + block * FLASH_BLOCK_SIZE; + + for (uint32_t row = 0; row < ROWS_PER_BLOCK; row++) { + if (keep & (1u << row)) { + memcpy(block_cache + row * FLASH_ROW_SIZE, (const void *)(base + row * FLASH_ROW_SIZE), FLASH_ROW_SIZE); + } + } + + TUF2_VERIFY(flash_erase(block)); + + for (uint32_t row = 0; row < ROWS_PER_BLOCK; row++) { + if (keep & (1u << row)) { + TUF2_VERIFY(flash_program(base + row * FLASH_ROW_SIZE, block_cache + row * FLASH_ROW_SIZE, FLASH_ROW_SIZE)); + } + } + + return true; +} + +// BOOTPROT protects the first 15 - BOOTPROT blocks of 8 KB, it is loaded from the user page at reset +static uint32_t bootprot_value(void) { + return 15 - BOARD_FLASH_APP_START / FLASH_BLOCK_SIZE; +} + +//--------------------------------------------------------------------+ +// Board API +//--------------------------------------------------------------------+ +void board_flash_init(void) { + memset(good_rows, 0, sizeof(good_rows)); +} + +uint32_t board_flash_size(void) { + return BOARD_FLASH_SIZE; +} + +void board_flash_read(uint32_t addr, void *buffer, uint32_t len) { + memcpy(buffer, (void *)addr, len); +} + +void board_flash_flush(void) { +} + +bool board_flash_write(uint32_t addr, const void *data, uint32_t len) { + // a UF2 file may target anything: only accept whole rows of the application area + TUF2_VERIFY(is_valid_row(addr, len)); + + const uint32_t block = addr / FLASH_BLOCK_SIZE; + const uint32_t row_bit = 1u << ((addr % FLASH_BLOCK_SIZE) / FLASH_ROW_SIZE); + + // a row already holding its data is left alone + if (is_equal(addr, data, len)) { + good_rows[block] |= row_bit; + return true; + } + + // flash can only be programmed where it is blank: otherwise the block is erased, keeping the good rows + if (!is_blank(addr, len)) { + TUF2_VERIFY(flash_erase_block_keep_rows(block, good_rows[block] & ~row_bit)); + } + + TUF2_VERIFY(flash_program(addr, data, len)); + good_rows[block] |= row_bit; + return true; +} + +void board_flash_erase_app(void) { + // erasing the first block of the app is enough to invalidate it + flash_erase(BOARD_FLASH_APP_START / FLASH_BLOCK_SIZE); +} + +bool board_flash_protect_bootloader(bool protect) { + if (!protect) { + // lift the protection until the next reset + exec_cmd(NVMCTRL_CTRLB_CMD_SBPDIS); + return true; + } + + volatile uint32_t *const user_page = (volatile uint32_t *)USER_PAGE_ADDR; + // an erased user page holds no valid fuses: leave it alone + TUF2_VERIFY(user_page[0] != 0xffffffff); + + const uint32_t bootprot = bootprot_value(); + const uint32_t current = (user_page[0] & BOOTPROT_MSK) >> BOOTPROT_POS; + TUF2_LOG1("Protection: current = %lu, request = %lu\r\n", current, bootprot); + + if (current == bootprot) { + return true; + } + + uint32_t fuses[USER_PAGE_SIZE / sizeof(uint32_t)]; + memcpy(fuses, (const void *)USER_PAGE_ADDR, sizeof(fuses)); + fuses[0] = (fuses[0] & ~BOOTPROT_MSK) | (bootprot << BOOTPROT_POS); + + cache_disable(); + NVMCTRL->CTRLA.bit.WMODE = NVMCTRL_CTRLA_WMODE_MAN_Val; + NVMCTRL->ADDR.reg = USER_PAGE_ADDR; + exec_cmd(NVMCTRL_CTRLB_CMD_EP); + exec_cmd(NVMCTRL_CTRLB_CMD_PBC); + for (uint32_t i = 0; i < sizeof(fuses) / sizeof(uint32_t); i += QUAD_WORD_SIZE / sizeof(uint32_t)) { + for (uint32_t w = 0; w < QUAD_WORD_SIZE / sizeof(uint32_t); w++) { + user_page[i + w] = fuses[i + w]; + } + NVMCTRL->ADDR.reg = (uint32_t)&user_page[i]; + exec_cmd(NVMCTRL_CTRLB_CMD_WQW); + } + + // the new value is only loaded at reset, and must not be retried if the write failed + TUF2_VERIFY(((user_page[0] & BOOTPROT_MSK) >> BOOTPROT_POS) == bootprot); + NVIC_SystemReset(); + return true; +} + +#ifdef TINYUF2_SELF_UPDATE + +static bool is_new_bootloader_valid(const uint8_t *bootloader_bin, uint32_t bootloader_len) { + // at least larger than vector table, and must leave the application untouched + if (bootloader_len < 1024 || bootloader_len > BOARD_FLASH_APP_START) { + return false; + } + + const uint32_t *vector = (const uint32_t *)(uintptr_t)bootloader_bin; + const uint32_t sp = vector[0]; + const uint32_t entry = vector[1]; + + // 1st word is stack pointer (must be in SRAM region) + if ((sp & 0xff000003) != 0x20000000) { + return false; + } + + // 2nd word is the reset handler, must be inside the bootloader and have the Thumb bit set + if (entry >= BOARD_FLASH_APP_START || (entry & 1u) == 0) { + return false; + } + + return true; +} + +// Make a block of the bootloader area hold its part of the new bootloader and nothing else +static bool bootloader_block_update(uint32_t block, const uint8_t *bootloader_bin, uint32_t bootloader_len) { + const uint32_t base = BOARD_FLASH_ADDR_ZERO + block * FLASH_BLOCK_SIZE; + const uint32_t offset = block * FLASH_BLOCK_SIZE; + uint32_t size = 0; + if (offset < bootloader_len) { + size = ((bootloader_len - offset) < FLASH_BLOCK_SIZE) ? (bootloader_len - offset) : FLASH_BLOCK_SIZE; + } + + memset(block_cache, 0xff, sizeof(block_cache)); + memcpy(block_cache, bootloader_bin + offset, size); + if (is_equal(base, block_cache, FLASH_BLOCK_SIZE)) { + return true; + } + + TUF2_VERIFY(flash_erase(block)); + return flash_program(base, block_cache, size); +} + +// Runs from RAM, it erases the block holding the code that called it. Rewriting programmed flash with zeros +// instead would leave the quad word with an inconsistent ECC. +static __attribute__((section(".ramfunc"), noinline, noreturn)) void flash_erase_block_and_reset(uint32_t addr) { + __disable_irq(); + NVMCTRL->ADDR.reg = addr; + NVMCTRL->CTRLB.reg = NVMCTRL_CTRLB_CMDEX_KEY | NVMCTRL_CTRLB_CMD_EB; + while (NVMCTRL->STATUS.bit.READY == 0) {} + + SCB->AIRCR = (0x5FAu << SCB_AIRCR_VECTKEY_Pos) | SCB_AIRCR_SYSRESETREQ_Msk; + __DSB(); + while (1) {} +} + +void board_self_update(const uint8_t *bootloader_bin, uint32_t bootloader_len) { + // On failure this application stays in place so the update can be tried again + if (!is_new_bootloader_valid(bootloader_bin, bootloader_len)) { + TUF2_LOG1("Invalid bootloader image\r\n"); + return; + } + + // Note: Don't protect bootloader when done, leave that to the new bootloader + // since it may or may not enable protection. + board_flash_protect_bootloader(false); + + // keep writing until flash contents matches the new bootloader data, the rest of the bootloader area blank + bool updated = false; + for (uint32_t retry = 0; retry < 3 && !updated; retry++) { + updated = true; + for (uint32_t block = 0; block < BOARD_FLASH_APP_START / FLASH_BLOCK_SIZE && updated; block++) { + updated = bootloader_block_update(block, bootloader_bin, bootloader_len); + } + } + + if (!updated) { + TUF2_LOG1("Bootloader update failed\r\n"); + return; + } + + // self-destruct: erase the first block of this application so the bootloader does not run it again + flash_erase_block_and_reset(BOARD_FLASH_APP_START); +} +#endif diff --git a/ports/samd51/boards.c b/ports/samd51/boards.c new file mode 100644 index 000000000..a161e9d4c --- /dev/null +++ b/ports/samd51/boards.c @@ -0,0 +1,303 @@ +#include "board_api.h" + +#ifndef BUILD_NO_TINYUSB + #include "tusb.h" +#endif + +//--------------------------------------------------------------------+ +// MACRO TYPEDEF CONSTANT ENUM +//--------------------------------------------------------------------+ + +#define SAMD51_UID_WORD0 ((volatile uint32_t *)0x008061FC) +#define SAMD51_UID_WORD1 ((volatile uint32_t *)0x00806010) +#define SAMD51_UID_WORD2 ((volatile uint32_t *)0x00806014) +#define SAMD51_UID_WORD3 ((volatile uint32_t *)0x00806018) + +// BOD33 level: 1.5V + LEVEL * 6mV = 2.7V +#define BOD33_LEVEL 200 + +#define USB_DM_PIN 24 +#define USB_DP_PIN 25 + +//--------------------------------------------------------------------+ +// Internal Helper +//--------------------------------------------------------------------+ + +static inline void pin_output(uint8_t port, uint8_t pin, bool high) { + if (high) { + PORT->Group[port].OUTSET.reg = 1u << pin; + } else { + PORT->Group[port].OUTCLR.reg = 1u << pin; + } + PORT->Group[port].DIRSET.reg = 1u << pin; +} + +static void dwt_enable(void) { + CoreDebug->DEMCR |= CoreDebug_DEMCR_TRCENA_Msk; + DWT->CTRL |= DWT_CTRL_CYCCNTENA_Msk; +} + +static void delay_ms(uint32_t ms) { + const uint32_t start = DWT->CYCCNT; + const uint32_t cycles = (SystemCoreClock / 1000) * ms; + while ((DWT->CYCCNT - start) < cycles) {} +} + +// The default 1.7V level is too low: flash writes intermittently store zeros +static void bod33_init(void) { + SUPC->BOD33.bit.ENABLE = 0; + while (!SUPC->STATUS.bit.B33SRDY) {} + SUPC->BOD33.bit.LEVEL = BOD33_LEVEL; + SUPC->BOD33.bit.ACTION = SUPC_BOD33_ACTION_NONE_Val; + SUPC->BOD33.bit.RUNSTDBY = 1; + SUPC->BOD33.bit.ENABLE = 1; + + while (!SUPC->STATUS.bit.BOD33RDY) {} + while (SUPC->STATUS.bit.BOD33DET) {} + + if (RSTC->RCAUSE.bit.POR || RSTC->RCAUSE.bit.BODVDD) { + do { + delay_ms(100); + } while (SUPC->STATUS.bit.BOD33DET); + } + + SUPC->BOD33.bit.ENABLE = 0; + while (!SUPC->STATUS.bit.B33SRDY) {} + SUPC->BOD33.bit.ACTION = SUPC_BOD33_ACTION_RESET_Val; + SUPC->BOD33.bit.ENABLE = 1; +} + +// 48MHz DFLL in USB clock recovery mode, locks to the host SOF so no crystal is needed +static void clock_init(void) { + NVMCTRL->CTRLA.bit.AUTOWS = 1; + + GCLK->CTRLA.bit.SWRST = 1; + while (GCLK->SYNCBUSY.bit.SWRST) {} + + // run from the ultra low power 32k oscillator while the DFLL is reconfigured + GCLK->GENCTRL[0].reg = GCLK_GENCTRL_SRC(GCLK_GENCTRL_SRC_OSCULP32K) | GCLK_GENCTRL_OE | GCLK_GENCTRL_GENEN; + while (GCLK->SYNCBUSY.bit.GENCTRL0) {} + + OSCCTRL->DFLLCTRLA.reg = 0; + + OSCCTRL->DFLLMUL.reg = OSCCTRL_DFLLMUL_CSTEP(0x1) | OSCCTRL_DFLLMUL_FSTEP(0x1) | OSCCTRL_DFLLMUL_MUL(0xBB80); + while (OSCCTRL->DFLLSYNC.bit.DFLLMUL) {} + + OSCCTRL->DFLLCTRLB.reg = 0; + while (OSCCTRL->DFLLSYNC.bit.DFLLCTRLB) {} + + OSCCTRL->DFLLCTRLA.bit.ENABLE = true; + while (OSCCTRL->DFLLSYNC.bit.ENABLE) {} + + OSCCTRL->DFLLVAL.reg = OSCCTRL->DFLLVAL.reg; + while (OSCCTRL->DFLLSYNC.bit.DFLLVAL) {} + + OSCCTRL->DFLLCTRLB.reg = OSCCTRL_DFLLCTRLB_WAITLOCK | OSCCTRL_DFLLCTRLB_CCDIS | OSCCTRL_DFLLCTRLB_USBCRM; + while (!OSCCTRL->STATUS.bit.DFLLRDY) {} + + GCLK->GENCTRL[0].reg = + GCLK_GENCTRL_SRC(GCLK_GENCTRL_SRC_DFLL) | GCLK_GENCTRL_IDC | GCLK_GENCTRL_OE | GCLK_GENCTRL_GENEN; + while (GCLK->SYNCBUSY.bit.GENCTRL0) {} + + MCLK->CPUDIV.reg = MCLK_CPUDIV_DIV_DIV1; +} + +//--------------------------------------------------------------------+ +// Board API +//--------------------------------------------------------------------+ + +// Only touches what the application can inherit: the clocks stay at their reset +// state until the bootloader decides to stay (board_dfu_init) +void board_init(void) { + WDT->CTRLA.reg = 0; + while (WDT->SYNCBUSY.reg) {} + + dwt_enable(); + bod33_init(); + + pin_output(NEOPIXEL_POWER_PORT, NEOPIXEL_POWER_PIN, NEOPIXEL_POWER_STATE); + pin_output(NEOPIXEL_PORT, NEOPIXEL_PIN, false); + + PORT->Group[BUTTON_PORT].PINCFG[BUTTON_PIN].reg = PORT_PINCFG_INEN | PORT_PINCFG_PULLEN; + PORT->Group[BUTTON_PORT].DIRCLR.reg = 1u << BUTTON_PIN; + PORT->Group[BUTTON_PORT].OUTSET.reg = 1u << BUTTON_PIN; +} + +void board_dfu_init(void) { + clock_init(); + + PORT->Group[0].PINCFG[USB_DM_PIN].bit.PMUXEN = 1; + PORT->Group[0].PINCFG[USB_DP_PIN].bit.PMUXEN = 1; + PORT->Group[0].PMUX[USB_DM_PIN / 2].reg = PORT_PMUX_PMUXE(MUX_PA24H_USB_DM) | PORT_PMUX_PMUXO(MUX_PA25H_USB_DP); + + MCLK->AHBMASK.reg |= MCLK_AHBMASK_USB; + MCLK->APBBMASK.reg |= MCLK_APBBMASK_USB; + GCLK->PCHCTRL[USB_GCLK_ID].reg = GCLK_PCHCTRL_GEN_GCLK0 | GCLK_PCHCTRL_CHEN; +} + +void board_reset(void) { + NVIC_SystemReset(); +} + +void board_dfu_complete(void) { + NVIC_SystemReset(); +} + +// Holding the button at reset forces the bootloader. The line rises slowly after power-up +bool board_app_valid2(void) { + delay_ms(20); + const bool pressed = ((PORT->Group[BUTTON_PORT].IN.reg >> BUTTON_PIN) & 1u) == BUTTON_STATE_ACTIVE; + return !pressed; +} + +bool board_app_valid(void) { + const volatile uint32_t *app_vector = (const volatile uint32_t *)BOARD_FLASH_APP_START; + const uint32_t sp = app_vector[0]; + const uint32_t app_entry = app_vector[1]; + + TUF2_LOG1_HEX(sp); + TUF2_LOG1_HEX(app_entry); + + // 1st word is stack pointer (must be in SRAM region) + if ((sp & 0xff000003) != 0x20000000) { + return false; + } + + // 2nd word is app entry point (reset) + if (app_entry < BOARD_FLASH_APP_START || app_entry > BOARD_FLASH_ADDR_ZERO + BOARD_FLASH_SIZE) { + return false; + } + + return true; +} + +void board_teardown(void) { + PORT->Group[BUTTON_PORT].PINCFG[BUTTON_PIN].reg = 0; + PORT->Group[BUTTON_PORT].OUTCLR.reg = 1u << BUTTON_PIN; + + SysTick->CTRL = 0; + SysTick->LOAD = 0; + SysTick->VAL = 0; +} + +void board_app_jump(void) { + const volatile uint32_t *app_vector = (const volatile uint32_t *)BOARD_FLASH_APP_START; + const uint32_t sp = app_vector[0]; + const uint32_t app_entry = app_vector[1]; + + for (uint8_t i = 0; i < sizeof(NVIC->ICER) / sizeof(NVIC->ICER[0]); i++) { + NVIC->ICER[i] = 0xFFFFFFFFu; + } + + SCB->VTOR = BOARD_FLASH_APP_START; + + __set_MSP(sp); + __set_PSP(sp); + + asm("bx %0" ::"r"(app_entry)); +} + +uint8_t board_usb_get_serial(uint8_t serial_id[16]) { + uint32_t *serial_id32 = (uint32_t *)(uintptr_t)serial_id; + + serial_id32[0] = *SAMD51_UID_WORD0; + serial_id32[1] = *SAMD51_UID_WORD1; + serial_id32[2] = *SAMD51_UID_WORD2; + serial_id32[3] = *SAMD51_UID_WORD3; + + return 16; +} + +//--------------------------------------------------------------------+ +// LED pattern +//--------------------------------------------------------------------+ + +void board_led_write(uint32_t state) { + (void)state; +} + +static inline uint8_t apply_percentage(uint8_t brightness) { + return (uint8_t)((brightness * NEOPIXEL_BRIGHTNESS) >> 8); +} + +// The strip reads a pulse as a 1 once it is high for about 0.5us. The counts include the fixed cost of the PORT +// writes and cycle counter reads, they are tuned for a 48MHz core and give ~0.3us/0.6us high and ~1.2us per bit. +#define NEOPIXEL_T0H_CYCLES 4 +#define NEOPIXEL_T1H_CYCLES 25 +#define NEOPIXEL_BIT_CYCLES 36 +#define NEOPIXEL_RESET_NS 80000 + +static uint32_t ns_to_cycles(uint32_t ns) { + return (SystemCoreClock / 1000000) * ns / 1000; +} + +// Runs from RAM, so the pulse widths do not depend on flash wait states or the NVM cache +static __attribute__((section(".ramfunc"), noinline)) void neopixel_send(const uint8_t *grb, uint32_t count) { + for (uint32_t i = 0; i < count; i++) { + for (uint32_t c = 0; c < 3; c++) { + for (uint8_t mask = 0x80; mask != 0; mask >>= 1) { + const uint32_t high = (grb[c] & mask) ? NEOPIXEL_T1H_CYCLES : NEOPIXEL_T0H_CYCLES; + const uint32_t start = DWT->CYCCNT; + + PORT->Group[NEOPIXEL_PORT].OUTSET.reg = 1u << NEOPIXEL_PIN; + while ((DWT->CYCCNT - start) < high) {} + + PORT->Group[NEOPIXEL_PORT].OUTCLR.reg = 1u << NEOPIXEL_PIN; + while ((DWT->CYCCNT - start) < NEOPIXEL_BIT_CYCLES) {} + } + } + } +} + +void board_rgb_write(const uint8_t rgb[]) { + static uint32_t last_end = 0; + + // neopixel color order is GRB + const uint8_t grb[3] = {apply_percentage(rgb[1]), apply_percentage(rgb[0]), apply_percentage(rgb[2])}; + + // the previous frame is latched once the line has been low for the reset time + while ((DWT->CYCCNT - last_end) < ns_to_cycles(NEOPIXEL_RESET_NS)) {} + + const uint32_t primask = __get_PRIMASK(); + __disable_irq(); + neopixel_send(grb, NEOPIXEL_NUMBER); + __set_PRIMASK(primask); + + last_end = DWT->CYCCNT; +} + +//--------------------------------------------------------------------+ +// Timer +//--------------------------------------------------------------------+ +void board_timer_start(uint32_t ms) { + SysTick_Config((SystemCoreClock / 1000) * ms); +} + +void board_timer_stop(void) { + SysTick->CTRL &= ~SysTick_CTRL_ENABLE_Msk; +} + +void SysTick_Handler(void) { + board_timer_handler(); +} + +int board_uart_write(const void *buf, int len) { + (void)buf; + (void)len; + return 0; +} + +#ifndef BUILD_NO_TINYUSB +void USB_0_Handler(void) { + tud_int_handler(0); +} +void USB_1_Handler(void) { + tud_int_handler(0); +} +void USB_2_Handler(void) { + tud_int_handler(0); +} +void USB_3_Handler(void) { + tud_int_handler(0); +} +#endif diff --git a/ports/samd51/boards.h b/ports/samd51/boards.h new file mode 100644 index 000000000..fba7cb5f2 --- /dev/null +++ b/ports/samd51/boards.h @@ -0,0 +1,34 @@ +#ifndef BOARDS_H_ +#define BOARDS_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include "sam.h" + +#include "board.h" + +#define BOARD_FLASH_ADDR_ZERO 0x00000000 + +// Flash Start Address of Application +#ifndef BOARD_FLASH_APP_START +extern uint32_t __flash_boot_size[]; + #define BOARD_FLASH_APP_START (BOARD_FLASH_ADDR_ZERO + (uint32_t)__flash_boot_size) +#endif + +// Enable write protection +#ifndef TINYUF2_PROTECT_BOOTLOADER + #define TINYUF2_PROTECT_BOOTLOADER 1 +#endif + +// Brightness percentage from 1 to 255 +#ifndef NEOPIXEL_BRIGHTNESS + #define NEOPIXEL_BRIGHTNESS 0x10 +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* BOARDS_H_ */ diff --git a/ports/samd51/boards/intech_grid/board.cmake b/ports/samd51/boards/intech_grid/board.cmake new file mode 100644 index 000000000..99405f23c --- /dev/null +++ b/ports/samd51/boards/intech_grid/board.cmake @@ -0,0 +1,9 @@ +set(LD_FLASH_BOOT_SIZE 16K) +set(LD_FLASH_SIZE 1M) +set(LD_RAM_SIZE 256K) + +function(update_board TARGET) + target_compile_definitions(${TARGET} PUBLIC + __SAMD51N20A__ + ) +endfunction() diff --git a/ports/samd51/boards/intech_grid/board.h b/ports/samd51/boards/intech_grid/board.h new file mode 100644 index 000000000..147ea4d94 --- /dev/null +++ b/ports/samd51/boards/intech_grid/board.h @@ -0,0 +1,49 @@ +#ifndef BOARD_H_ +#define BOARD_H_ + +//--------------------------------------------------------------------+ +// Button, pulled up and active low. Holding it at reset enters the bootloader +//--------------------------------------------------------------------+ +#define BUTTON_PORT 2 +#define BUTTON_PIN 11 +#define BUTTON_STATE_ACTIVE 0 + +//--------------------------------------------------------------------+ +// Neopixel +//--------------------------------------------------------------------+ + +// Number of neopixels +#define NEOPIXEL_NUMBER 16 +// Brightness percentage from 1 to 255 +#define NEOPIXEL_BRIGHTNESS 0x08 + +#define NEOPIXEL_PORT 1 +#define NEOPIXEL_PIN 30 + +// Neopixel supply enable, STATE is the level that switches it on +#define NEOPIXEL_POWER_PORT 2 +#define NEOPIXEL_POWER_PIN 14 +#define NEOPIXEL_POWER_STATE 1 + +//--------------------------------------------------------------------+ +// Flash +//--------------------------------------------------------------------+ + +// Flash size of the board +#define BOARD_FLASH_SIZE 0x100000 + +//--------------------------------------------------------------------+ +// USB UF2 +//--------------------------------------------------------------------+ + +#define USB_VID 0x03EB +#define USB_PID 0x2402 +#define USB_MANUFACTURER "Intech Studio" +#define USB_PRODUCT "Bootloader 2026-10" + +#define UF2_PRODUCT_NAME USB_PRODUCT +#define UF2_BOARD_ID "SAMD51N20A-GRID-v0" +#define UF2_VOLUME_LABEL "GRID26" +#define UF2_INDEX_URL "https://github.com/intechstudio/grid-fw/" + +#endif diff --git a/ports/samd51/boards/intech_wrid/board.cmake b/ports/samd51/boards/intech_wrid/board.cmake new file mode 100644 index 000000000..99405f23c --- /dev/null +++ b/ports/samd51/boards/intech_wrid/board.cmake @@ -0,0 +1,9 @@ +set(LD_FLASH_BOOT_SIZE 16K) +set(LD_FLASH_SIZE 1M) +set(LD_RAM_SIZE 256K) + +function(update_board TARGET) + target_compile_definitions(${TARGET} PUBLIC + __SAMD51N20A__ + ) +endfunction() diff --git a/ports/samd51/boards/intech_wrid/board.h b/ports/samd51/boards/intech_wrid/board.h new file mode 100644 index 000000000..56618a805 --- /dev/null +++ b/ports/samd51/boards/intech_wrid/board.h @@ -0,0 +1,49 @@ +#ifndef BOARD_H_ +#define BOARD_H_ + +//--------------------------------------------------------------------+ +// Button, pulled up and active low. Holding it at reset enters the bootloader +//--------------------------------------------------------------------+ +#define BUTTON_PORT 2 +#define BUTTON_PIN 11 +#define BUTTON_STATE_ACTIVE 0 + +//--------------------------------------------------------------------+ +// Neopixel +//--------------------------------------------------------------------+ + +// Number of neopixels +#define NEOPIXEL_NUMBER 16 +// Brightness percentage from 1 to 255 +#define NEOPIXEL_BRIGHTNESS 0x08 + +#define NEOPIXEL_PORT 1 +#define NEOPIXEL_PIN 30 + +// Neopixel supply enable, STATE is the level that switches it on +#define NEOPIXEL_POWER_PORT 2 +#define NEOPIXEL_POWER_PIN 14 +#define NEOPIXEL_POWER_STATE 1 + +//--------------------------------------------------------------------+ +// Flash +//--------------------------------------------------------------------+ + +// Flash size of the board +#define BOARD_FLASH_SIZE 0x100000 + +//--------------------------------------------------------------------+ +// USB UF2 +//--------------------------------------------------------------------+ + +#define USB_VID 0x03EB +#define USB_PID 0x2402 +#define USB_MANUFACTURER "Intech Studio" +#define USB_PRODUCT "Bootloader 2026-10" + +#define UF2_PRODUCT_NAME USB_PRODUCT +#define UF2_BOARD_ID "SAMD51N20A-WRID-v0" +#define UF2_VOLUME_LABEL "WRID26" +#define UF2_INDEX_URL "https://github.com/intechstudio/grid-fw/" + +#endif diff --git a/ports/samd51/family.cmake b/ports/samd51/family.cmake new file mode 100644 index 000000000..4a30d5add --- /dev/null +++ b/ports/samd51/family.cmake @@ -0,0 +1,62 @@ +include_guard(GLOBAL) + +#------------------------------------ +# Config +#------------------------------------ + +set(UF2_FAMILY_ID 0x55114460) +set(SAM_SDK ${TOP}/lib/mcu/microchip/samd51) +set(CMSIS_5 ${TOP}/lib/CMSIS_5) + +include(${CMAKE_CURRENT_LIST_DIR}/boards/${BOARD}/board.cmake) + +# the bootloader area is write protected: lift BOOTPROT before programming, the bootloader protects itself again at boot +set(OPENOCD_OPTION "-f interface/cmsis-dap.cfg -c \"transport select swd\" -c \"set CHIPNAME at91samd51n20a\" -f target/atsame5x.cfg -c \"adapter speed 4000\" -c init -c \"reset halt\" -c \"atsame5 bootloader 0\" -c \"reset halt\"") + +# increase flash size if debug build +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(LD_FLASH_BOOT_SIZE 64K) +endif () + +# enable LTO +set(CMAKE_INTERPROCEDURAL_OPTIMIZATION TRUE) + +set(CMAKE_SYSTEM_PROCESSOR cortex-m4 CACHE INTERNAL "System Processor") +set(CMAKE_TOOLCHAIN_FILE ${TOP}/cmake/toolchain/arm_${TOOLCHAIN}.cmake) + +#------------------------------------ +# BOARD_TARGET +#------------------------------------ +function(family_add_board_target BOARD_TARGET) + if (TARGET ${BOARD_TARGET}) + return() + endif () + + add_library(${BOARD_TARGET} STATIC + ${SAM_SDK}/gcc/system_samd51.c + ${SAM_SDK}/gcc/gcc/startup_samd51.c + ) + target_include_directories(${BOARD_TARGET} PUBLIC + # port & board + ${CMAKE_CURRENT_FUNCTION_LIST_DIR} + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/boards/${BOARD} + # sdk + ${CMSIS_5}/CMSIS/Core/Include + ${SAM_SDK}/include + ) + + update_board(${BOARD_TARGET}) + + target_compile_definitions(${BOARD_TARGET} PUBLIC + BOARD_UF2_FAMILY_ID=${UF2_FAMILY_ID} + ) + set_source_files_properties(${SAM_SDK}/gcc/system_samd51.c ${SAM_SDK}/gcc/gcc/startup_samd51.c PROPERTIES + COMPILE_OPTIONS -w) + target_link_options(${BOARD_TARGET} PUBLIC + -nostartfiles + --specs=nosys.specs --specs=nano.specs + -Wl,--defsym=__flash_boot_size=${LD_FLASH_BOOT_SIZE} + -Wl,--defsym=__flash_size=${LD_FLASH_SIZE} + -Wl,--defsym=__ram_size=${LD_RAM_SIZE} + ) +endfunction() diff --git a/ports/samd51/linker/samd51_app.ld b/ports/samd51/linker/samd51_app.ld new file mode 100644 index 000000000..a3b107058 --- /dev/null +++ b/ports/samd51/linker/samd51_app.ld @@ -0,0 +1,102 @@ +/* Application linker script for SAMD51x20 (1 MB flash, 256 KB HSRAM), runs behind the bootloader */ + +__flash_boot_size = DEFINED(__flash_boot_size) ? __flash_boot_size : 16K; +__flash_size = DEFINED(__flash_size) ? __flash_size : 1M; +__ram_size = DEFINED(__ram_size) ? __ram_size : 256K; + +OUTPUT_FORMAT("elf32-littlearm", "elf32-littlearm", "elf32-littlearm") +OUTPUT_ARCH(arm) +ENTRY(Reset_Handler) + +/* Highest address of the stack */ +_estack = ORIGIN(RAM) + LENGTH(RAM); + +_Min_Stack_Size = 0x800; + +MEMORY +{ + FLASH (rx) : ORIGIN = __flash_boot_size, LENGTH = __flash_size - __flash_boot_size + RAM (rwx) : ORIGIN = 0x20000000, LENGTH = __ram_size +} + +SECTIONS +{ + .text : + { + . = ALIGN(4); + _sfixed = .; + KEEP(*(.vectors .vectors.*)) + *(.text .text.* .gnu.linkonce.t.*) + *(.glue_7t) *(.glue_7) + *(.rodata .rodata* .gnu.linkonce.r.*) + *(.ARM.extab* .gnu.linkonce.armextab.*) + + . = ALIGN(4); + KEEP(*(.init)) + . = ALIGN(4); + __preinit_array_start = .; + KEEP(*(.preinit_array)) + __preinit_array_end = .; + + . = ALIGN(4); + __init_array_start = .; + KEEP(*(SORT(.init_array.*))) + KEEP(*(.init_array)) + __init_array_end = .; + + . = ALIGN(4); + KEEP(*(.fini)) + + . = ALIGN(4); + __fini_array_start = .; + KEEP(*(.fini_array)) + KEEP(*(SORT(.fini_array.*))) + __fini_array_end = .; + + . = ALIGN(4); + _efixed = .; + } > FLASH + + .ARM.exidx : + { + __exidx_start = .; + *(.ARM.exidx* .gnu.linkonce.armexidx.*) + __exidx_end = .; + } > FLASH + + . = ALIGN(4); + _etext = .; + + .relocate : + { + . = ALIGN(4); + _srelocate = .; + *(.ramfunc .ramfunc.*) + *(.data .data.*) + . = ALIGN(4); + _erelocate = .; + } > RAM AT> FLASH + + .bss (NOLOAD) : + { + . = ALIGN(4); + _sbss = .; + _szero = .; + *(.bss .bss.*) + *(COMMON) + . = ALIGN(4); + _ebss = .; + _ezero = .; + } > RAM + + /* Fails the link if the stack has less than _Min_Stack_Size left */ + ._stack_check (NOLOAD) : + { + . = ALIGN(8); + PROVIDE(end = .); + PROVIDE(_end = .); + . = . + _Min_Stack_Size; + } > RAM + + .ARM.attributes 0 : { *(.ARM.attributes) } +} diff --git a/ports/samd51/linker/samd51_boot.ld b/ports/samd51/linker/samd51_boot.ld new file mode 100644 index 000000000..cf5ef2a7d --- /dev/null +++ b/ports/samd51/linker/samd51_boot.ld @@ -0,0 +1,101 @@ +/* Bootloader linker script for SAMD51x20 (1 MB flash, 256 KB HSRAM) */ + +__flash_boot_size = DEFINED(__flash_boot_size) ? __flash_boot_size : 16K; +__ram_size = DEFINED(__ram_size) ? __ram_size : 256K; + +OUTPUT_FORMAT("elf32-littlearm", "elf32-littlearm", "elf32-littlearm") +OUTPUT_ARCH(arm) +ENTRY(Reset_Handler) + +/* Highest address of the stack */ +_estack = ORIGIN(RAM) + LENGTH(RAM); + +_Min_Stack_Size = 0x800; + +MEMORY +{ + FLASH (rx) : ORIGIN = 0x00000000, LENGTH = __flash_boot_size + RAM (rwx) : ORIGIN = 0x20000000, LENGTH = __ram_size +} + +SECTIONS +{ + .text : + { + . = ALIGN(4); + _sfixed = .; + KEEP(*(.vectors .vectors.*)) + *(.text .text.* .gnu.linkonce.t.*) + *(.glue_7t) *(.glue_7) + *(.rodata .rodata* .gnu.linkonce.r.*) + *(.ARM.extab* .gnu.linkonce.armextab.*) + + . = ALIGN(4); + KEEP(*(.init)) + . = ALIGN(4); + __preinit_array_start = .; + KEEP(*(.preinit_array)) + __preinit_array_end = .; + + . = ALIGN(4); + __init_array_start = .; + KEEP(*(SORT(.init_array.*))) + KEEP(*(.init_array)) + __init_array_end = .; + + . = ALIGN(4); + KEEP(*(.fini)) + + . = ALIGN(4); + __fini_array_start = .; + KEEP(*(.fini_array)) + KEEP(*(SORT(.fini_array.*))) + __fini_array_end = .; + + . = ALIGN(4); + _efixed = .; + } > FLASH + + .ARM.exidx : + { + __exidx_start = .; + *(.ARM.exidx* .gnu.linkonce.armexidx.*) + __exidx_end = .; + } > FLASH + + . = ALIGN(4); + _etext = .; + + .relocate : + { + . = ALIGN(4); + _srelocate = .; + *(.ramfunc .ramfunc.*) + *(.data .data.*) + . = ALIGN(4); + _erelocate = .; + } > RAM AT> FLASH + + .bss (NOLOAD) : + { + . = ALIGN(4); + _sbss = .; + _szero = .; + *(.bss .bss.*) + *(COMMON) + . = ALIGN(4); + _ebss = .; + _ezero = .; + } > RAM + + /* Fails the link if the stack has less than _Min_Stack_Size left */ + ._stack_check (NOLOAD) : + { + . = ALIGN(8); + PROVIDE(end = .); + PROVIDE(_end = .); + . = . + _Min_Stack_Size; + } > RAM + + .ARM.attributes 0 : { *(.ARM.attributes) } +} diff --git a/ports/samd51/tusb_config.h b/ports/samd51/tusb_config.h new file mode 100644 index 000000000..09e62d471 --- /dev/null +++ b/ports/samd51/tusb_config.h @@ -0,0 +1,100 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2019 Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#ifndef TUSB_CONFIG_H_ +#define TUSB_CONFIG_H_ + +#ifdef __cplusplus + extern "C" { +#endif + +//-------------------------------------------------------------------- +// COMMON CONFIGURATION +//-------------------------------------------------------------------- + +#ifndef CFG_TUSB_MCU +#error CFG_TUSB_MCU must be defined in board.mk +#endif + +#define CFG_TUSB_OS OPT_OS_NONE + +// Enable Device stack +#define CFG_TUD_ENABLED 1 + +#ifndef BOARD_TUD_RHPORT +#define BOARD_TUD_RHPORT 0 +#endif + +// can be defined by compiler in DEBUG build +#ifndef CFG_TUSB_DEBUG +#define CFG_TUSB_DEBUG 0 +#endif + +/* USB DMA on some MCUs can only access a specific SRAM region with restriction on alignment. + * Tinyusb use follows macros to declare transferring memory so that they can be put + * into those specific section. + * e.g + * - CFG_TUSB_MEM SECTION : __attribute__ (( section(".usb_ram") )) + * - CFG_TUSB_MEM_ALIGN : __attribute__ ((aligned(4))) + */ +#ifndef CFG_TUSB_MEM_SECTION +#define CFG_TUSB_MEM_SECTION +#endif + +#ifndef CFG_TUSB_MEM_ALIGN +#define CFG_TUSB_MEM_ALIGN __attribute__ ((aligned(4))) +#endif + +//-------------------------------------------------------------------- +// DEVICE CONFIGURATION +//-------------------------------------------------------------------- + +#ifndef CFG_TUD_ENDPOINT0_SIZE +#define CFG_TUD_ENDPOINT0_SIZE 64 +#endif + +//------------- CLASS -------------// +#define CFG_TUD_CDC 0 +#define CFG_TUD_MSC 1 +#define CFG_TUD_HID 0 +#define CFG_TUD_MIDI 0 +#define CFG_TUD_VENDOR 0 + +// MSC Buffer size of Device Mass storage +#define CFG_TUD_MSC_BUFSIZE 4096 + +// HID buffer size Should be sufficient to hold ID (if any) + Data +#define CFG_TUD_HID_BUFSIZE 64 + +// Vendor FIFO size of TX and RX +// If not configured vendor endpoints will not be buffered +#define CFG_TUD_VENDOR_RX_BUFSIZE 64 +#define CFG_TUD_VENDOR_TX_BUFSIZE 64 + +#ifdef __cplusplus + } +#endif + +#endif diff --git a/ports/stm32h5/CMakeLists.txt b/ports/stm32h5/CMakeLists.txt index 73e4ee326..3a99b5f45 100644 --- a/ports/stm32h5/CMakeLists.txt +++ b/ports/stm32h5/CMakeLists.txt @@ -20,7 +20,7 @@ target_link_options(tinyuf2 PUBLIC family_configure_tinyuf2(tinyuf2 OPT_MCU_STM32H5) family_flash_jlink(tinyuf2) -family_flash_stlink(tinyuf2) +family_flash_stlink(tinyuf2 hex) #------------------------------------ # Application diff --git a/supported_boards.md b/supported_boards.md index ab121fdc0..6798f9aae 100644 --- a/supported_boards.md +++ b/supported_boards.md @@ -15,6 +15,7 @@ | adafruit_funhouse_esp32s2 | Adafruit FunHouse | 0x239A:0x00F9 | https://www.adafruit.com/ | | adafruit_magtag_29gray | Adafruit MagTag 2.9 Grayscale | 0x239A:0x00E5 | https://www.adafruit.com/product/4800 | | adafruit_matrixportal_s3 | Adafruit MatrixPortal S3 | 0x239A:0x0125 | https://www.adafruit.com/ | +| adafruit_metro_esp32p4 | Adafruit Metro ESP32-P4 | 0x239A:0x017B | https://www.adafruit.com | | adafruit_metro_esp32s2 | Adafruit Metro ESP32-S2 | 0x239A:0x00DF | https://www.adafruit.com/product/4775 | | adafruit_metro_esp32s3 | Adafruit Metro ESP32-S3 | 0x239A:0x0145 | https://www.adafruit.com/product/5500 | | adafruit_mylittlehacker_esp32s2 | Adafruit My Little Hacker ESP32-S2 | 0x239A:0x013F | https://www.adafruit.com/products/5742 | @@ -164,6 +165,13 @@ | teensy40 | PJRC Teensy 4.0 | 0x239A:0x0085 | https://www.pjrc.com/store/teensy40.html | | teensy41 | PJRC Teensy 4.1 | 0x239A:0x00AD | https://www.pjrc.com/store/teensy41.html | +## samd51 + +| Board | Name | VID PID | URL | +| --- | --- | --- | --- | +| intech_grid | Intech Studio Bootloader 2026-10 | 0x03EB:0x2402 | https://github.com/intechstudio/grid-fw/ | +| intech_wrid | Intech Studio Bootloader 2026-10 | 0x03EB:0x2402 | https://github.com/intechstudio/grid-fw/ | + ## stm32f3 | Board | Name | VID PID | URL | diff --git a/test/hil/ci.json b/test/hil/ci.json new file mode 100644 index 000000000..7d146c09c --- /dev/null +++ b/test/hil/ci.json @@ -0,0 +1,10 @@ +{ + "boards": [ + { + "name": "imxrt1064_evk", + "uid": "BAE96FB95AFA6DBB8F00005002001200", + "flasher": {"name": "jlink", "uid": "000725299165", "args": "-device MIMXRT1064xxx6A"}, + "tinyuf2": {"lock": "mimxrt1064_evk", "reentry": "retained-register"} + } + ] +} diff --git a/test/hil/local.json b/test/hil/local.json index 63ac9e4a4..ba7e643f1 100644 --- a/test/hil/local.json +++ b/test/hil/local.json @@ -22,7 +22,8 @@ "name": "espressif_p4_function_ev", "uid": "30EDA0E0C902", "flasher": {"name": "esptool", "uid": "1ec47e6369c3ee119baac50f9e1b1c54", "args": "--chip esp32p4"}, - "tinyuf2": {"reentry": "dfu-detach"} + "tinyuf2": {"reentry": "dfu-detach", + "routes": [{"flasher": {"name": "openocd_esp", "uid": "30:ED:A0:E0:C9:02", "args": "-f board/esp32p4-builtin.cfg"}}]} }, { "name": "stm32f303disco", @@ -35,6 +36,12 @@ "uid": "3A002A001051363332383332", "flasher": {"name": "jlink", "uid": "000770880814", "args": "-device stm32f411ve"}, "tinyuf2": {"lock": "stm32f411disco", "reentry": "retained-register"} + }, + { + "name": "stm32h563_nucleo", + "uid": "620041000751323237363130", + "flasher": {"name": "stlink", "uid": "002300273232510239353236", "args": "-device stm32h563zi"}, + "tinyuf2": {"lock": "stm32h563nucleo", "reentry": "retained-register"} } ] } diff --git a/tools/get_deps.py b/tools/get_deps.py index edd084110..87f1aedc5 100644 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -14,6 +14,9 @@ 'lib/mcu/analog/msdk' : ['https://github.com/analogdevicesinc/msdk.git', 'b20b398d3e5e2007594e54a74ba3d2a2e50ddd75', 'maxim'], + 'lib/mcu/microchip': ['https://github.com/hathach/microchip_driver.git', + '9e8b37e307d8404033bb881623a113931e1edf27', + 'samd51'], 'lib/mcu/nxp/mcux-sdk': ['https://github.com/nxp-mcuxpresso/mcux-sdk.git', '9990f264f98430f6d885041ab0f24224d68f4958', 'kinetis_k kinetis_k32l2 kinetis_kl lpc51 lpc54 lpc55 mcx mimxrt10xx'], @@ -62,6 +65,10 @@ TOP = Path(__file__).parent.parent.resolve() +def find_symlink_component(d): + return next((p for p in [*reversed(Path(d).parents), Path(d)] if (TOP / p).is_symlink()), None) + + def run_cmd(cmd): r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if r.returncode != 0: @@ -78,6 +85,12 @@ def get_a_dep(d): commit = deps_all[d][1] families = deps_all[d][2] + symlink = find_symlink_component(d) + if symlink is not None: + print(f'Error: refusing to update {d}: {symlink} is a symlink') + print(f'Fetch {commit} in the checkout that owns the dependency, then link it here') + return 1 + print(f'cloning {d} with {url}') p = Path(TOP / d)