From 54d94163a6d7169a554be4d58280221b113da0ca Mon Sep 17 00:00:00 2001 From: Danil Zagoskin Date: Wed, 27 May 2026 00:03:23 +0300 Subject: [PATCH 01/47] add Waveshare ESP32-S3-Tiny-N8R8 board --- .../waveshare_esp32_s3_tiny_n8r8/board.cmake | 2 + .../waveshare_esp32_s3_tiny_n8r8/board.h | 74 +++++++++++++++++++ .../waveshare_esp32_s3_tiny_n8r8/sdkconfig | 7 ++ 3 files changed, 83 insertions(+) create mode 100644 ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.cmake create mode 100644 ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.h create mode 100644 ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/sdkconfig diff --git a/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.cmake b/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.cmake new file mode 100644 index 000000000..9bac46d64 --- /dev/null +++ b/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.cmake @@ -0,0 +1,2 @@ +# Apply board specific content here +set(IDF_TARGET "esp32s3") diff --git a/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.h b/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.h new file mode 100644 index 000000000..fbda30e8f --- /dev/null +++ b/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/board.h @@ -0,0 +1,74 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2020 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef WAVESHARE_ESP32_S3_TINY_N8R8_H_ +#define WAVESHARE_ESP32_S3_TINY_N8R8_H_ + +//--------------------------------------------------------------------+ +// Button +//--------------------------------------------------------------------+ + +// Enter UF2 mode if GPIO is pressed while 2nd stage bootloader indicator +// is on e.g RGB = Purple. If it is GPIO0, user should not hold this while +// reset since that will instead run the 1st stage ROM bootloader +#define PIN_BUTTON_UF2 0 + +// GPIO that implement 1-bit memory with RC components which hold the +// pin value long enough for double reset detection. +// #define PIN_DOUBLE_RESET_RC + +//--------------------------------------------------------------------+ +// LED +//--------------------------------------------------------------------+ + +// GPIO connected to Neopixel data +#define NEOPIXEL_PIN 38 + +// Brightness percentage from 1 to 255 +#define NEOPIXEL_BRIGHTNESS 0x10 + +// Number of neopixels +#define NEOPIXEL_NUMBER 1 + +// Invert Neopixel red and green +#define NEOPIXEL_INVERT_RG 1 + +// LED for indicator and writing flash +// If not defined neopixel will be use for flash writing instead + +//--------------------------------------------------------------------+ +// USB UF2 +//--------------------------------------------------------------------+ + +#define USB_VID 0x303a +#define USB_PID 0x837a +#define USB_MANUFACTURER "Waveshare Electronics" +#define USB_PRODUCT "ESP32-S3-Tiny-N8R8" + +#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT +#define UF2_BOARD_ID "ESP32-S3-Tiny-N8R8" +#define UF2_VOLUME_LABEL "WS3TINYBOOT" +#define UF2_INDEX_URL "https://docs.waveshare.com/ESP32-S3-Tiny?variant=ESP32-S3-Tiny-N8R8" + +#endif diff --git a/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/sdkconfig b/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/sdkconfig new file mode 100644 index 000000000..329b9a9f2 --- /dev/null +++ b/ports/espressif/boards/waveshare_esp32_s3_tiny_n8r8/sdkconfig @@ -0,0 +1,7 @@ +# Board Specific Config + +# Partition Table +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-8MB.csv" + +# Serial flasher config +CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y From b0439df61ecc8ce3ea7699275457a38386caa61c Mon Sep 17 00:00:00 2001 From: Linar Yusupov Date: Tue, 9 Jun 2026 10:13:03 +0300 Subject: [PATCH 02/47] Add LilyGO T-Beam 1W --- .../boards/lilygo_ttgo_tbeam_1w/board.cmake | 5 ++ .../boards/lilygo_ttgo_tbeam_1w/board.cpp | 77 +++++++++++++++++++ .../boards/lilygo_ttgo_tbeam_1w/board.h | 62 +++++++++++++++ .../boards/lilygo_ttgo_tbeam_1w/sdkconfig | 28 +++++++ supported_boards.md | 1 + 5 files changed, 173 insertions(+) create mode 100644 ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cmake create mode 100644 ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cpp create mode 100644 ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.h create mode 100644 ports/espressif/boards/lilygo_ttgo_tbeam_1w/sdkconfig diff --git a/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cmake b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cmake new file mode 100644 index 000000000..71b78be75 --- /dev/null +++ b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cmake @@ -0,0 +1,5 @@ +# Apply board specific content here +set(IDF_TARGET "esp32s3") + +set(BOARD_SOURCES "${BOARD}/board.cpp") +#set(BOARD_COMPONENTS "ssd1306") diff --git a/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cpp b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cpp new file mode 100644 index 000000000..50c3617f6 --- /dev/null +++ b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.cpp @@ -0,0 +1,77 @@ + +#include +#include "board_api.h" + + +#if CONFIG_SSD1306_128x64 +#include "esp_log.h" +#include "sdkconfig.h" +#include "driver/gpio.h" +#include "driver/i2c.h" + + +#include "ssd1306.h" +#include "font8x8_basic.h" + +#define tag "SSD1306" +#endif + +#define I2C_MASTER_NUM I2C_NUM_1 +#define I2C_MASTER_FREQ_HZ 100000 /*!< I2C master clock frequency */ +#define I2C_MASTER_SDA_IO (gpio_num_t) 8 +#define I2C_MASTER_SCL_IO (gpio_num_t) 9 + + +#define I2C_MASTER_TX_BUF_DISABLE 0 /*!< I2C master doesn't need buffer */ +#define I2C_MASTER_RX_BUF_DISABLE 0 /*!< I2C master doesn't need buffer */ +#define I2C_MASTER_TIMEOUT_MS 1000 + +#define WRITE_BIT I2C_MASTER_WRITE /*!< I2C master write */ +#define READ_BIT I2C_MASTER_READ /*!< I2C master read */ +#define ACK_CHECK_EN 0x1 /*!< I2C master will check ack from slave*/ +#define ACK_CHECK_DIS 0x0 /*!< I2C master will not check ack from slave */ +#define ACK_VAL (i2c_ack_type_t)0x0 /*!< I2C ack value */ +#define NACK_VAL (i2c_ack_type_t)0x1 /*!< I2C nack value */ + +extern "C" bool board_init_extension() +{ + SSD1306_t dev; + + i2c_config_t i2c_conf ; + memset(&i2c_conf, 0, sizeof(i2c_conf)); + i2c_conf.mode = I2C_MODE_MASTER; + i2c_conf.sda_io_num = I2C_MASTER_SDA_IO; + i2c_conf.scl_io_num = I2C_MASTER_SCL_IO; + i2c_conf.sda_pullup_en = GPIO_PULLUP_ENABLE; + i2c_conf.scl_pullup_en = GPIO_PULLUP_ENABLE; + i2c_conf.master.clk_speed = I2C_MASTER_FREQ_HZ; + i2c_param_config(I2C_MASTER_NUM, &i2c_conf); + i2c_driver_install(I2C_MASTER_NUM, i2c_conf.mode, + I2C_MASTER_RX_BUF_DISABLE, I2C_MASTER_TX_BUF_DISABLE, 0); + + // use X axis offset for SH1106 OLED + dev._offset = CONFIG_OFFSETX; + +#if CONFIG_I2C_INTERFACE + i2c_master_init(&dev, I2C_MASTER_SDA_IO, I2C_MASTER_SCL_IO, CONFIG_RESET_GPIO, 0x3C); +#endif // CONFIG_I2C_INTERFACE + +#if CONFIG_FLIP + dev._flip = true; + ESP_LOGW(tag, "Flip upside down"); +#endif + + ESP_LOGI(tag, "Panel is 128x64"); + ssd1306_init(&dev, 128, 64); + + ssd1306_clear_screen(&dev, false); + ssd1306_contrast(&dev, 0xff); + + ssd1306_display_text(&dev, 1, " T-Beam 1W Boot ", 16, true); + ssd1306_display_text(&dev, 4, "Put UF2 firmware", 16, false); + ssd1306_display_text(&dev, 6, "on " UF2_VOLUME_LABEL " Vol", 16, false); + + //ssd1306_clear_line(&dev, 5, false); + + return true; +} diff --git a/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.h b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.h new file mode 100644 index 000000000..87e4b18c4 --- /dev/null +++ b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/board.h @@ -0,0 +1,62 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2020 Ha Thach (tinyusb.org) for Adafruit Industries + * Copyright (c) 2026 Linar Yusupov + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#pragma once + +#define BOARD_INIT_CUSTOM 1 + +//--------------------------------------------------------------------+ +// Button +//--------------------------------------------------------------------+ + +// Enter UF2 mode if GPIO is pressed while 2nd stage bootloader indicator +// is on e.g RGB = Purple. If it is GPIO0, user should not hold this while +// reset since that will instead run the 1st stage ROM bootloader +#define PIN_BUTTON_UF2 0 + +// Initial delay in milliseconds to detect user interaction to enter UF2 +#define UF2_DETECTION_DELAY_MS 1000 + +//--------------------------------------------------------------------+ +// LED +//--------------------------------------------------------------------+ + +// LED for indicator and writing flash +#define LED_PIN 18 +#define LED_STATE_ON 1 + +//--------------------------------------------------------------------+ +// USB UF2 +//--------------------------------------------------------------------+ + +#define USB_VID 0x303A // Espressif VID +#define USB_PID 0x8368 // Espressif assigned PID +#define USB_MANUFACTURER "LilyGO" +#define USB_PRODUCT "T-Beam 1W" + +#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT +#define UF2_BOARD_ID "ESP32S3-TBeam_1W-v1.1" +#define UF2_VOLUME_LABEL "TBEAMBOOT" +#define UF2_INDEX_URL "https://lilygo.cc/en-us/products/t-beam-1w" diff --git a/ports/espressif/boards/lilygo_ttgo_tbeam_1w/sdkconfig b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/sdkconfig new file mode 100644 index 000000000..a8fc5b095 --- /dev/null +++ b/ports/espressif/boards/lilygo_ttgo_tbeam_1w/sdkconfig @@ -0,0 +1,28 @@ +# Board Specific Config + +# Partition Table +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-16MB-ota4MB.csv" + +# Serial flasher config +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y + +# +# SSD1306 Configuration +# +#CONFIG_GPIO_RANGE_MAX=48 +#CONFIG_I2C_INTERFACE=y +# CONFIG_SPI_INTERFACE is not set +# CONFIG_SSD1306_128x32 is not set +#CONFIG_SSD1306_128x64=y + +# SSD1306 +#CONFIG_OFFSETX=0 + +# SH1106 +#CONFIG_OFFSETX=2 + +# CONFIG_FLIP is not set +#CONFIG_SCL_GPIO=9 +#CONFIG_SDA_GPIO=8 +#CONFIG_RESET_GPIO=-1 +# end of SSD1306 Configuration diff --git a/supported_boards.md b/supported_boards.md index 6fa608335..a9e9f51c9 100644 --- a/supported_boards.md +++ b/supported_boards.md @@ -61,6 +61,7 @@ | lilygo_ttgo_t8_s2_st7789 | LILYGO TTGO_T8_S2_Display | 0x303A:0x8008 | http://www.lilygo.cn/prod_view.aspx?TypeId=50033&Id=1321&FId=t3:50033:3 | | lilygo_ttgo_t8_s2_wroom | LILYGO TTGO_T8_S2_WROOM | 0x303A:0x80EB | http://www.lilygo.cn/prod_view.aspx?TypeId=50063&Id=1320&FId=t3:50063:3 | | lilygo_ttgo_t_twr_plus | LilyGO T-TWR Plus | 0x303A:0x8191 | https://www.lilygo.cc/products/t-twr-plus | +| lilygo_ttgo_tbeam_1w | LilyGO T-Beam 1W | 0x303A:0x8368 | https://lilygo.cc/en-us/products/t-beam-1w | | lilygo_ttgo_tbeam_s3 | LilyGO T-Beam Supreme | 0x303A:0x8134 | https://www.lilygo.cc/products/softrf-t-beamsupreme | | lilygo_twatch_s3 | LILYGO T-Watch-S3 | 0x303A:0x821D | https://github.com/Xinyuan-LilyGO/TTGO_TWatch_Library/tree/t-watch-s3 | | lolin_s2_mini | Lolin S2 Mini | 0x303A:0x80C4 | https://circuitpython.org/board/lolin_s2_mini/ | From 62b3fb97b93971f1e960eddc4dadd90f395cf248 Mon Sep 17 00:00:00 2001 From: Paula Scharf Date: Tue, 16 Jun 2026 16:10:25 +0200 Subject: [PATCH 03/47] update senseBox MCU Eye name and flashsize --- ports/espressif/boards/sensebox_eye_esp32s3/sdkconfig | 7 ------- .../board.cmake | 0 .../{sensebox_eye_esp32s3 => sensebox_mcu_eye}/board.h | 8 ++++---- ports/espressif/boards/sensebox_mcu_eye/sdkconfig | 7 +++++++ supported_boards.md | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 ports/espressif/boards/sensebox_eye_esp32s3/sdkconfig rename ports/espressif/boards/{sensebox_eye_esp32s3 => sensebox_mcu_eye}/board.cmake (100%) rename ports/espressif/boards/{sensebox_eye_esp32s3 => sensebox_mcu_eye}/board.h (93%) create mode 100644 ports/espressif/boards/sensebox_mcu_eye/sdkconfig diff --git a/ports/espressif/boards/sensebox_eye_esp32s3/sdkconfig b/ports/espressif/boards/sensebox_eye_esp32s3/sdkconfig deleted file mode 100644 index 329b9a9f2..000000000 --- a/ports/espressif/boards/sensebox_eye_esp32s3/sdkconfig +++ /dev/null @@ -1,7 +0,0 @@ -# Board Specific Config - -# Partition Table -CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-8MB.csv" - -# Serial flasher config -CONFIG_ESPTOOLPY_FLASHSIZE_8MB=y diff --git a/ports/espressif/boards/sensebox_eye_esp32s3/board.cmake b/ports/espressif/boards/sensebox_mcu_eye/board.cmake similarity index 100% rename from ports/espressif/boards/sensebox_eye_esp32s3/board.cmake rename to ports/espressif/boards/sensebox_mcu_eye/board.cmake diff --git a/ports/espressif/boards/sensebox_eye_esp32s3/board.h b/ports/espressif/boards/sensebox_mcu_eye/board.h similarity index 93% rename from ports/espressif/boards/sensebox_eye_esp32s3/board.h rename to ports/espressif/boards/sensebox_mcu_eye/board.h index b64ef6f67..d4e6ae10e 100644 --- a/ports/espressif/boards/sensebox_eye_esp32s3/board.h +++ b/ports/espressif/boards/sensebox_mcu_eye/board.h @@ -22,8 +22,8 @@ * THE SOFTWARE. */ -#ifndef SENSEBOX_EYE_ESP32S3_H_ -#define SENSEBOX_EYE_ESP32S3_H_ +#ifndef SENSEBOX_MCU_EYE_ESP32S3_H_ +#define SENSEBOX_MCU_EYE_ESP32S3_H_ //--------------------------------------------------------------------+ // Button @@ -56,10 +56,10 @@ #define USB_VID 0x303A #define USB_PID 0x82D3 #define USB_MANUFACTURER "senseBox" -#define USB_PRODUCT "eye ESP32S3" +#define USB_PRODUCT "MCU-Eye ESP32S3" #define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT -#define UF2_BOARD_ID "ESP32S3-senseBox-eye" +#define UF2_BOARD_ID "ESP32S3-senseBox-MCU-Eye" #define UF2_VOLUME_LABEL "senseBox" #define UF2_INDEX_URL "https://sensebox.de" diff --git a/ports/espressif/boards/sensebox_mcu_eye/sdkconfig b/ports/espressif/boards/sensebox_mcu_eye/sdkconfig new file mode 100644 index 000000000..e4fa04e07 --- /dev/null +++ b/ports/espressif/boards/sensebox_mcu_eye/sdkconfig @@ -0,0 +1,7 @@ +# Board Specific Config + +# Partition Table +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-16MB.csv" + +# Serial flasher config +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y diff --git a/supported_boards.md b/supported_boards.md index a9e9f51c9..0ddd57ab0 100644 --- a/supported_boards.md +++ b/supported_boards.md @@ -82,7 +82,7 @@ | muselab_nanoesp32-s2_wrover | Muse Lab nanoESP32-S2 WROVER | 0x303A:0x80B3 | https://github.com/wuxx/nanoESP32-S2 | | olimex_esp32s2_devkit_lipo_vB1 | Olimex ESP32S2 DevKit Lipo | 0x15BA:0x28DC | https://www.olimex.com/Products/IoT/ESP32-S2/ESP32-S2-DevKit-Lipo/open-source-hardware | | seeed_xiao_esp32s3 | Seeed Studio XIAO ESP32-S3 | 0x2886:0x8056 | https://www.seeedstudio.com/XIAO-ESP32S3-p-5627.html | -| sensebox_eye_esp32s3 | senseBox eye ESP32S3 | 0x303A:0x82D3 | https://sensebox.de | +| sensebox_mcu_eye | senseBox MCU-Eye ESP32S3 | 0x303A:0x82D3 | https://sensebox.de | | sensebox_mcu_esp32s2 | senseBox MCU-S2 ESP32S2 | 0x303A:0x81BA | https://sensebox.de | | smartbeedesigns_bee_motion_s3 | Smart Bee Designs Bee Motion S3 | 0x303A:0x8115 | N/A | | smartbeedesigns_bee_s3 | Smart Bee Designs Bee S3 | 0x303A:0x8112 | N/A | From 14bec0795e1f69d22a94affc8f2e2f5b94789fc2 Mon Sep 17 00:00:00 2001 From: Paula Scharf Date: Tue, 16 Jun 2026 16:31:46 +0200 Subject: [PATCH 04/47] no hyphen --- ports/espressif/boards/sensebox_mcu_eye/board.h | 2 +- supported_boards.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ports/espressif/boards/sensebox_mcu_eye/board.h b/ports/espressif/boards/sensebox_mcu_eye/board.h index d4e6ae10e..95aab4fbf 100644 --- a/ports/espressif/boards/sensebox_mcu_eye/board.h +++ b/ports/espressif/boards/sensebox_mcu_eye/board.h @@ -56,7 +56,7 @@ #define USB_VID 0x303A #define USB_PID 0x82D3 #define USB_MANUFACTURER "senseBox" -#define USB_PRODUCT "MCU-Eye ESP32S3" +#define USB_PRODUCT "MCU Eye ESP32S3" #define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT #define UF2_BOARD_ID "ESP32S3-senseBox-MCU-Eye" diff --git a/supported_boards.md b/supported_boards.md index 0ddd57ab0..6e311c5af 100644 --- a/supported_boards.md +++ b/supported_boards.md @@ -82,7 +82,7 @@ | muselab_nanoesp32-s2_wrover | Muse Lab nanoESP32-S2 WROVER | 0x303A:0x80B3 | https://github.com/wuxx/nanoESP32-S2 | | olimex_esp32s2_devkit_lipo_vB1 | Olimex ESP32S2 DevKit Lipo | 0x15BA:0x28DC | https://www.olimex.com/Products/IoT/ESP32-S2/ESP32-S2-DevKit-Lipo/open-source-hardware | | seeed_xiao_esp32s3 | Seeed Studio XIAO ESP32-S3 | 0x2886:0x8056 | https://www.seeedstudio.com/XIAO-ESP32S3-p-5627.html | -| sensebox_mcu_eye | senseBox MCU-Eye ESP32S3 | 0x303A:0x82D3 | https://sensebox.de | +| sensebox_mcu_eye | senseBox MCU Eye ESP32S3 | 0x303A:0x82D3 | https://sensebox.de | | sensebox_mcu_esp32s2 | senseBox MCU-S2 ESP32S2 | 0x303A:0x81BA | https://sensebox.de | | smartbeedesigns_bee_motion_s3 | Smart Bee Designs Bee Motion S3 | 0x303A:0x8115 | N/A | | smartbeedesigns_bee_s3 | Smart Bee Designs Bee S3 | 0x303A:0x8112 | N/A | From 222cd03dcc7d166804df0de46e18ed31d926c44f Mon Sep 17 00:00:00 2001 From: jdavis-codes <79899854+jdavis-codes@users.noreply.github.com> Date: Fri, 31 Jul 2026 01:50:27 -0400 Subject: [PATCH 05/47] Merge pull request #479 from jdavis-codes/add-waveshare-esp32-s3-eth-8di-8ro Add Waveshare ESP32-S3-ETH-8DI-8RO board support --- .../board.cmake | 2 + .../waveshare_esp32_s3_eth_8di_8ro/board.h | 71 +++++++++++++++++++ .../waveshare_esp32_s3_eth_8di_8ro/sdkconfig | 9 +++ 3 files changed, 82 insertions(+) create mode 100644 ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.cmake create mode 100644 ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.h create mode 100644 ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/sdkconfig diff --git a/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.cmake b/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.cmake new file mode 100644 index 000000000..9bac46d64 --- /dev/null +++ b/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.cmake @@ -0,0 +1,2 @@ +# Apply board specific content here +set(IDF_TARGET "esp32s3") diff --git a/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.h b/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.h new file mode 100644 index 000000000..8a18390a7 --- /dev/null +++ b/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/board.h @@ -0,0 +1,71 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2020 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef WAVESHARE_ESP32_S3_ETH_8DI_8RO_H_ +#define WAVESHARE_ESP32_S3_ETH_8DI_8RO_H_ + +//--------------------------------------------------------------------+ +// Button +//--------------------------------------------------------------------+ + +// Enter UF2 mode if GPIO is pressed while 2nd stage bootloader indicator +// is on e.g RGB = Purple. If it is GPIO0, user should not hold this while +// reset since that will instead run the 1st stage ROM bootloader +#define PIN_BUTTON_UF2 0 + +// GPIO that implement 1-bit memory with RC components which hold the +// pin value long enough for double reset detection. +// #define PIN_DOUBLE_RESET_RC + +//--------------------------------------------------------------------+ +// LED +//--------------------------------------------------------------------+ + +// GPIO connected to Neopixel data +#define NEOPIXEL_PIN 38 + +// Brightness percentage from 1 to 255 +#define NEOPIXEL_BRIGHTNESS 0x10 + +// Number of neopixels +#define NEOPIXEL_NUMBER 1 + +// Invert Neopixel red and green +#define NEOPIXEL_INVERT_RG 1 + +//--------------------------------------------------------------------+ +// USB UF2 +//--------------------------------------------------------------------+ + +#define USB_VID 0x303a +#define USB_PID 0x83A3 +#define USB_MANUFACTURER "Waveshare Electronics" +#define USB_PRODUCT "ESP32-S3-ETH-8DI-8RO" + +#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT +#define UF2_BOARD_ID "ESP32S3-ETH8DI8RO-v1.0" +#define UF2_VOLUME_LABEL "WSRELAYBOOT" +#define UF2_INDEX_URL "https://www.waveshare.com/esp32-s3-eth-8di-8ro.htm" + +#endif diff --git a/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/sdkconfig b/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/sdkconfig new file mode 100644 index 000000000..8996cfa8c --- /dev/null +++ b/ports/espressif/boards/waveshare_esp32_s3_eth_8di_8ro/sdkconfig @@ -0,0 +1,9 @@ +# Board Specific Config + +# Partition Table +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-16MB.csv" + +# Serial flasher config +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_FLASHMODE_QIO=n +CONFIG_FLASHMODE_DIO=y From 3484abeec4af9c696cb77c43ff38d2e5a3bb088e Mon Sep 17 00:00:00 2001 From: Mikey Sklar Date: Wed, 12 Aug 2026 10:31:22 -0700 Subject: [PATCH 06/47] espressif: add ESP32-S3-DevKitC-1-N32R16 board The existing espressif_esp32s3_devkitc_1 board is configured for 8MB flash, so on a 32MB board it leaves most of the flash unused. Adds a 32MB partition table and a board variant that uses it. Co-Authored-By: Claude Opus 5 (1M context) --- .../board.cmake | 2 + .../board.h | 74 +++++++++++++++++++ .../sdkconfig | 7 ++ ports/espressif/partitions-32MB.csv | 10 +++ 4 files changed, 93 insertions(+) create mode 100644 ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.cmake create mode 100644 ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.h create mode 100644 ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/sdkconfig create mode 100644 ports/espressif/partitions-32MB.csv diff --git a/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.cmake b/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.cmake new file mode 100644 index 000000000..c1426c950 --- /dev/null +++ b/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.cmake @@ -0,0 +1,2 @@ +# Apply board specific content here +set(IDF_TARGET "esp32s3") diff --git a/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.h b/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.h new file mode 100644 index 000000000..70a37be0e --- /dev/null +++ b/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/board.h @@ -0,0 +1,74 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2020 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef ESPRESSIF_S3_DEVKITC_N32R16_H_ +#define ESPRESSIF_S3_DEVKITC_N32R16_H_ + +//--------------------------------------------------------------------+ +// Button +//--------------------------------------------------------------------+ + +// Enter UF2 mode if GPIO is pressed while 2nd stage bootloader indicator +// is on e.g RGB = Purple. If it is GPIO0, user should not hold this while +// reset since that will instead run the 1st stage ROM bootloader +#define PIN_BUTTON_UF2 0 + +// GPIO that implement 1-bit memory with RC components which hold the +// pin value long enough for double reset detection. +// #define PIN_DOUBLE_RESET_RC + +//--------------------------------------------------------------------+ +// LED +//--------------------------------------------------------------------+ + +// GPIO connected to Neopixel data +#define NEOPIXEL_PIN 38 + +// Brightness percentage from 1 to 255 +#define NEOPIXEL_BRIGHTNESS 0x10 + +// Number of neopixels +#define NEOPIXEL_NUMBER 1 + + +// LED for indicator +// If not defined neopixel will be use for flash writing instead +// #define LED_PIN 42 +// #define LED_STATE_ON 1 + +//--------------------------------------------------------------------+ +// USB UF2 +//--------------------------------------------------------------------+ + +#define USB_VID 0x239A +#define USB_PID 0x00A5 // TODO temporarily shared with S2 saola wrover +#define USB_MANUFACTURER "Espressif" +#define USB_PRODUCT "ESP32S3 DevKitC 1 N32R16" + +#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT +#define UF2_BOARD_ID "ESP32S3-DevKitC-1-N32R16-v1.1" +#define UF2_VOLUME_LABEL "N32R16BOOT" +#define UF2_INDEX_URL "https://docs.espressif.com/projects/esp-dev-kits/en/latest/esp32s3/esp32-s3-devkitc-1/user_guide_v1.1.html" + +#endif diff --git a/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/sdkconfig b/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/sdkconfig new file mode 100644 index 000000000..751bef7c5 --- /dev/null +++ b/ports/espressif/boards/espressif_esp32s3_devkitc_1_n32r16/sdkconfig @@ -0,0 +1,7 @@ +# Board Specific Config + +# Partition Table +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-32MB.csv" + +# Serial flasher config +CONFIG_ESPTOOLPY_FLASHSIZE_32MB=y diff --git a/ports/espressif/partitions-32MB.csv b/ports/espressif/partitions-32MB.csv new file mode 100644 index 000000000..99f5c4d8c --- /dev/null +++ b/ports/espressif/partitions-32MB.csv @@ -0,0 +1,10 @@ +# ESP-IDF Partition Table +# Name, Type, SubType, Offset, Size, Flags +# bootloader.bin,, 0x1000, 32K +# partition table,, 0x8000, 4K +nvs, data, nvs, 0x9000, 20K, +otadata, data, ota, 0xe000, 8K, +ota_0, app, ota_0, 0x10000, 2048K, +ota_1, app, ota_1, 0x210000, 2048K, +uf2, app, factory,0x410000, 256K, +ffat, data, fat, 0x450000, 28352K, From 43c50cdd78815817fb342fa1ef7e54eea0398344 Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:17:12 -0400 Subject: [PATCH 07/47] Fix Metro M7 factory SD test path Co-authored-by: Limor Fried --- .../apps/factory_test_metro_sd/main.c | 78 ++++++++++++++----- .../middleware-sdmmc/sdspi/fsl_sdspi.c | 19 ++++- ports/mimxrt10xx/linker/app.ld | 3 +- 3 files changed, 75 insertions(+), 25 deletions(-) diff --git a/ports/mimxrt10xx/apps/factory_test_metro_sd/main.c b/ports/mimxrt10xx/apps/factory_test_metro_sd/main.c index 4b4617b5d..61a51cf9d 100644 --- a/ports/mimxrt10xx/apps/factory_test_metro_sd/main.c +++ b/ports/mimxrt10xx/apps/factory_test_metro_sd/main.c @@ -61,6 +61,11 @@ static uint8_t all_pins[] = { }; static bool test = false; +static bool run_sd_test = true; +static const uint8_t TEST_GPIO_POWER = 0xAE; +static const uint8_t TEST_GPIO_POWER_SD = 0xAF; +static const float DC_INPUT_MIN_VOLTS = 8.0f; +static const float DC_INPUT_MAX_VOLTS = 13.0f; bool testpins(uint8_t a, uint8_t b, uint8_t *allpins, uint8_t num_allpins); void test_print_adc(void); @@ -85,6 +90,8 @@ int main(void) { while (1) { loop(); + // Failure messages may have no newline; drain libc before the USB FIFO. + fflush(stdout); tud_task(); tud_cdc_write_flush(); @@ -98,7 +105,8 @@ void loop(void) { uint32_t count; count = tud_cdc_read(serial_buf, sizeof(serial_buf)); - if (count && serial_buf[0] == 0xAF) { + if (count && (serial_buf[0] == TEST_GPIO_POWER || serial_buf[0] == TEST_GPIO_POWER_SD)) { + run_sd_test = serial_buf[0] == TEST_GPIO_POWER_SD; test = true; } } @@ -117,6 +125,7 @@ void loop(void) { return; } + test = false; // Run one pass per start command, including failed passes. delay(100); Serial_printf("\n\r\n\rHello Metro M7 iMX RT1011 Test! %lu\n\r", millis()); @@ -149,15 +158,19 @@ void loop(void) { Serial_printf("5V power supply reading wrong?"); return; } - // Test 5V - int nine_mV = (float)analogRead(AD0) * 11.0 * 3.3 * 1000 / 4095.0; - Serial_printf("9V out = %d\n\r", (int)nine_mV); - if (abs(nine_mV - 9000) > 1000) { - Serial_printf("9V power supply reading wrong?"); + // Accept the fixture's 9V or 12V DC supply, with measurement tolerance. + float dc_input_volts = (float)analogRead(AD0) * 11.0f * 3.3f / 4095.0f; + Serial_printf("DC input = %d mV\n\r", (int)(dc_input_volts * 1000.0f)); + if (dc_input_volts < DC_INPUT_MIN_VOLTS || dc_input_volts > DC_INPUT_MAX_VOLTS) { + Serial_printf("DC input power supply reading wrong?"); return; } - if (! test_sd() ) return; + if (run_sd_test) { + if (!test_sd()) return; + } else { + Serial_printf("SD test skipped\r\n"); + } Serial_printf("*** TEST OK! ***\n\r"); @@ -285,6 +298,8 @@ void test_print_adc(void) { //--------------------------------------------------------------------+ #define LPSPI1_CLOCK_FREQ 105600000UL #define LPSPI_MAX_FREQ 25000000UL +static const uint32_t SD_INIT_TIMEOUT_MS = 10000; +static uint32_t sd_init_started; lpspi_master_config_t spi_config = { .baudRate = 400000UL, @@ -309,18 +324,15 @@ void sdhost_init(void){ /*!< SPI de-initialization */ void sdhost_deinit(void){ - + digitalWrite(SD_CS, HIGH); } -static bool force_cs = false; /*!< SPI CS active polarity */ void sdhost_csActivePolarity(sdspi_cs_active_polarity_t polarity){ if (polarity == kSDSPI_CsActivePolarityHigh) { digitalWrite(SD_CS, HIGH); - force_cs = true; }else { digitalWrite(SD_CS, LOW); - force_cs = false; } } @@ -336,6 +348,8 @@ status_t sdhost_setFrequency(uint32_t frequency){ spi_config.betweenTransferDelayInNanoSec = ns_delay; LPSPI_MasterInit(LPSPI1, &spi_config, LPSPI1_CLOCK_FREQ); + // SD requires MOSI high while clocking initialization and response bytes. + LPSPI_SetDummyData(LPSPI1, 0xFF); return kStatus_Success; } @@ -351,18 +365,14 @@ status_t sdhost_exchange(uint8_t *out, uint8_t *in, uint32_t size) { status_t status; - if (!force_cs) { - digitalWrite(SD_CS, LOW); - } - + // Keep CS asserted between the command and its separate response transfers. do { + tud_task(); + tud_cdc_write_flush(); + if (millis() - sd_init_started >= SD_INIT_TIMEOUT_MS) return kStatus_Timeout; status = LPSPI_MasterTransferBlocking(LPSPI1, &xfer); } while ( status == kStatus_LPSPI_Busy ); - if (!force_cs) { - digitalWrite(SD_CS, HIGH); - } - if ( status != kStatus_Success ) { printf("SPI Xfer: %ld\r\n", status); return status; @@ -384,6 +394,25 @@ sdspi_card_t sdcard = { .host = &sdhost, }; +// Temporary diagnostic: print only the first three CMD0 and CMD8 replies. +static uint8_t sd_command_trace_count[2]; + +void sdspi_trace_command(uint8_t command, status_t status, + const uint8_t *response, uint8_t response_length) { + uint8_t trace_index = command == 0 ? 0 : 1; + uint8_t count = sd_command_trace_count[trace_index]; + if (count >= 3) return; + sd_command_trace_count[trace_index]++; + printf("SD CMD%u attempt %u: status %ld R1=%02X", command, count + 1, + (long)status, response[0]); + if (response_length == 5) { + printf(" R7=%02X %02X %02X %02X", response[1], response[2], response[3], response[4]); + } + printf("\r\n"); + fflush(stdout); + delay(1); +} + bool test_sd(void) { // init SD CS & Detect pinMode(SD_CS, OUTPUT); @@ -407,16 +436,25 @@ bool test_sd(void) { } printf("SD card is detected\r\n"); + fflush(stdout); + delay(10); + memset(sd_command_trace_count, 0, sizeof(sd_command_trace_count)); + sdcard.host = &sdhost; // SDSPI_Deinit clears the card, including this pointer. + sd_init_started = millis(); int status = SDSPI_Init(&sdcard); + if (millis() - sd_init_started >= SD_INIT_TIMEOUT_MS) { + printf("SD initialization timeout after 10000 ms\r\n"); + } printf("SDSPI_Init: %d\r\n", status); if (status == kStatus_Success) { uint32_t card_size_mb = sdcard.blockCount / 2 / 1024; printf("Card size: %lu MB\r\n", card_size_mb); } tud_cdc_write_flush(); + SDSPI_Deinit(&sdcard); // Release CS after either success or initialization failure. - return true; + return status == kStatus_Success; } //--------------------------------------------------------------------+ diff --git a/ports/mimxrt10xx/apps/factory_test_metro_sd/middleware-sdmmc/sdspi/fsl_sdspi.c b/ports/mimxrt10xx/apps/factory_test_metro_sd/middleware-sdmmc/sdspi/fsl_sdspi.c index 572d66934..0f4e62659 100644 --- a/ports/mimxrt10xx/apps/factory_test_metro_sd/middleware-sdmmc/sdspi/fsl_sdspi.c +++ b/ports/mimxrt10xx/apps/factory_test_metro_sd/middleware-sdmmc/sdspi/fsl_sdspi.c @@ -10,6 +10,10 @@ #include #include "fsl_sdspi.h" +/* Temporary diagnostic callback; the app owns printing and USB service. */ +extern void sdspi_trace_command(uint8_t command, status_t status, + const uint8_t *response, uint8_t response_length); + //#define SDSPI_DEBUG #ifdef SDSPI_DEBUG @@ -328,7 +332,7 @@ static status_t SDSPI_SendCommand(sdspi_host_t *host, uint32_t command, uint32_t uint32_t responseType = SDSPI_COMMAND_FORMAT_GET_RESPONSE_TYPE(command); uint8_t index = (uint8_t)SDSPI_COMMAND_FORMAT_GET_INDEX(command); - if ((kStatus_Success != SDSPI_WaitReady(host)) && (index != (uint8_t)kSDMMC_GoIdleState)) + if ((index != (uint8_t)kSDMMC_GoIdleState) && (kStatus_Success != SDSPI_WaitReady(host))) { return kStatus_SDSPI_WaitReadyFailed; } @@ -400,7 +404,8 @@ static status_t SDSPI_SendCommand(sdspi_host_t *host, uint32_t command, uint32_t else if ((responseType == (uint32_t)kSDSPI_ResponseTypeR3) || (responseType == (uint32_t)kSDSPI_ResponseTypeR7)) { /* Left 4 bytes in response type R3 and R7(total 5 bytes in SPI mode) */ - if (kStatus_Success != host->exchange(&timingByte, &(response[1U]), 4U)) + /* Let the host send four dummy 0xFF bytes instead of reading past timingByte. */ + if (kStatus_Success != host->exchange(NULL, &(response[1U]), 4U)) { return kStatus_SDSPI_ExchangeFailed; } @@ -457,7 +462,10 @@ static status_t SDSPI_GoIdle(sdspi_card_t *card) and the card will be IDLE state. */ do { - if ((kStatus_Success == SDSPI_SendCommand(card->host, kSDSPI_CmdGoIdle, 0U, &response)) && + response = 0xFFU; + status_t commandStatus = SDSPI_SendCommand(card->host, kSDSPI_CmdGoIdle, 0U, &response); + sdspi_trace_command(0U, commandStatus, &response, 1U); + if ((kStatus_Success == commandStatus) && (response == (uint8_t)kSDSPI_R1InIdleStateFlag)) { return kStatus_Success; @@ -479,7 +487,10 @@ static status_t SDSPI_SendInterfaceCondition(sdspi_card_t *card, uint32_t *flags { /* CMD8 is used to check if the card accept the current supply voltage and check if the card support CMD8 */ - if (kStatus_Success == SDSPI_SendCommand(card->host, kSDSPI_CmdSendInterfaceCondition, 0x1AAU, response)) + (void)memset(response, 0xFF, sizeof(response)); + status_t commandStatus = SDSPI_SendCommand(card->host, kSDSPI_CmdSendInterfaceCondition, 0x1AAU, response); + sdspi_trace_command(8U, commandStatus, response, sizeof(response)); + if (kStatus_Success == commandStatus) { /* not support CMD8, clear hcs flag */ if ((response[0U] & (uint8_t)kSDSPI_R1IllegalCommandFlag) != 0U) diff --git a/ports/mimxrt10xx/linker/app.ld b/ports/mimxrt10xx/linker/app.ld index c93ea3eed..c6dd5a2f3 100644 --- a/ports/mimxrt10xx/linker/app.ld +++ b/ports/mimxrt10xx/linker/app.ld @@ -9,7 +9,8 @@ _interrupts_origin = _flash_base + 0xC000; PROVIDE(_interrupts_length = 0x0400); _text_origin = _interrupts_origin + _interrupts_length; -PROVIDE(_text_length = 64K); +/* The application runs from flash; override the bootloader RAM text limit. */ +_text_length = 64K; /* Specify the memory areas */ MEMORY From fc5799cd691cc238e7a6321e8a2222f74251a18e Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:21:23 -0400 Subject: [PATCH 08/47] Keep factory UF2 in application flash Co-authored-by: Limor Fried --- ports/mimxrt10xx/linker/app.ld | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/ports/mimxrt10xx/linker/app.ld b/ports/mimxrt10xx/linker/app.ld index c6dd5a2f3..ebdf87be4 100644 --- a/ports/mimxrt10xx/linker/app.ld +++ b/ports/mimxrt10xx/linker/app.ld @@ -12,6 +12,18 @@ _text_origin = _interrupts_origin + _interrupts_length; /* The application runs from flash; override the bootloader RAM text limit. */ _text_length = 64K; +/* Applications do not carry the ROM boot headers used by TinyUF2. */ +SECTIONS +{ + /DISCARD/ : + { + *(.boot_hdr.conf) + *(.boot_hdr.ivt) + *(.boot_hdr.boot_data) + *(.boot_hdr.dcd_data) + } +} + /* Specify the memory areas */ MEMORY { From fa335870a26f97f5e9b217fd2ebf3a407baef4ce Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:33:14 -0400 Subject: [PATCH 09/47] Release factory analog pins before ADC conversion Current board startup muxes A0 to TRACE_CLK. Restore GPIO input mode for every analog read so peripheral outputs cannot drive the voltage divider. Co-authored-by: Limor Fried --- .../apps/factory_test_metro_sd/arduino.c | 388 +++++++++--------- 1 file changed, 196 insertions(+), 192 deletions(-) diff --git a/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c b/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c index 3974696fb..ea45a6cf7 100644 --- a/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c +++ b/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c @@ -1,196 +1,200 @@ -#include "arduino.h" - -pinmap pinmapping[] = { - {.num=0 , .port=GPIO1, .pin=9 , .mux={IOMUXC_GPIO_09_GPIOMUX_IO09}} , - {.num=1 , .port=GPIO1, .pin=10, .mux={IOMUXC_GPIO_10_GPIOMUX_IO10}} , - {.num=2 , .port=GPIO1, .pin=13, .mux={IOMUXC_GPIO_13_GPIOMUX_IO13}} , - {.num=3 , .port=GPIO1, .pin=12, .mux={IOMUXC_GPIO_12_GPIOMUX_IO12}} , - - {.num=4 , .port=GPIO2, .pin=0 , .mux={IOMUXC_GPIO_SD_00_GPIO2_IO00}} , - {.num=5 , .port=GPIO2, .pin=1 , .mux={IOMUXC_GPIO_SD_01_GPIO2_IO01}} , - {.num=6 , .port=GPIO2, .pin=2 , .mux={IOMUXC_GPIO_SD_02_GPIO2_IO02}} , - {.num=7 , .port=GPIO1, .pin=11, .mux={IOMUXC_GPIO_11_GPIOMUX_IO11}} , - - {.num=8 , .port=GPIO1, .pin=8 , .mux={IOMUXC_GPIO_08_GPIOMUX_IO08}} , - {.num=9 , .port=GPIO1, .pin=7 , .mux={IOMUXC_GPIO_07_GPIOMUX_IO07}} , - {.num=10, .port=GPIO1, .pin=6 , .mux={IOMUXC_GPIO_06_GPIOMUX_IO06}} , - {.num=11, .port=GPIO1, .pin=5 , .mux={IOMUXC_GPIO_05_GPIOMUX_IO05}} , - {.num=12, .port=GPIO1, .pin=4 , .mux={IOMUXC_GPIO_04_GPIOMUX_IO04}} , - {.num=13, .port=GPIO1, .pin=3 , .mux={IOMUXC_GPIO_03_GPIOMUX_IO03}} , - - {.num=14, .port=GPIO1, .pin=16, .mux={IOMUXC_GPIO_AD_02_GPIOMUX_IO16}, .adc=ADC1, .adc_channel=2 }, // AD0 - {.num=15, .port=GPIO1, .pin=15, .mux={IOMUXC_GPIO_AD_01_GPIOMUX_IO15}, .adc=ADC1, .adc_channel=1 }, // AD1 - {.num=16, .port=GPIO1, .pin=14, .mux={IOMUXC_GPIO_AD_00_GPIOMUX_IO14}, .adc=ADC1, .adc_channel=0 }, // AD2 - {.num=17, .port=GPIO1, .pin=19, .mux={IOMUXC_GPIO_AD_05_GPIOMUX_IO19}, .adc=ADC1, .adc_channel=5 }, // AD3 - {.num=18, .port=GPIO1, .pin=24, .mux={IOMUXC_GPIO_AD_10_GPIOMUX_IO24}, .adc=ADC1, .adc_channel=10 }, // AD4 - {.num=19, .port=GPIO1, .pin=22, .mux={IOMUXC_GPIO_AD_08_GPIOMUX_IO22}, .adc=ADC1, .adc_channel=8 }, // AD5 - - {.num=PIN_SDA , .port=GPIO1, .pin=1 , .mux={IOMUXC_GPIO_01_GPIOMUX_IO01}} , - {.num=PIN_SCL , .port=GPIO1, .pin=2 , .mux={IOMUXC_GPIO_02_GPIOMUX_IO02}} , - - {.num=PIN_MISO, .port=GPIO1, .pin=17, .mux={IOMUXC_GPIO_AD_03_GPIOMUX_IO17}, .adc=ADC1}, // AD0 - {.num=PIN_MOSI, .port=GPIO1, .pin=18, .mux={IOMUXC_GPIO_AD_04_GPIOMUX_IO18}, .adc=ADC1}, // AD0 - {.num=PIN_SCK , .port=GPIO1, .pin=20, .mux={IOMUXC_GPIO_AD_06_GPIOMUX_IO20}, .adc=ADC1}, // AD0 - - {.num=PIN_SD_CS , .port=GPIO1, .pin=28, .mux={IOMUXC_GPIO_AD_14_GPIOMUX_IO28}}, - {.num=PIN_SD_DETECT, .port=GPIO1, .pin=25, .mux={IOMUXC_GPIO_AD_11_GPIOMUX_IO25}}, -}; - -static volatile uint32_t _millis = 0; - -void board_timer_handler(void) { - _millis++; -} - -uint32_t millis(void) { -#ifndef BUILD_APPLICATION - extern uint32_t board_millis(void); - return board_millis(); -#else - return _millis; -#endif -} - -void delay(uint32_t ms) { - uint32_t timestamp = millis(); - while ((timestamp+ms) > millis()) { - // Serial_printf("delay %d\n\r", millis()); - tud_task(); - tud_cdc_write_flush(); - } -} - - -static pinmap *getMapping(uint32_t pinnum) { - pinmap *pindeets = NULL; - - for (uint32_t i=0; i port %x, pin %d\n\r", pindeets->num, pindeets->port, pindeets->pin); - if (pinnum == pindeets->num) { - //Serial_printf("found!\n\r"); - return pindeets; - } - } - return NULL; -} - - -void digitalWrite(uint32_t pinnum, bool value) { - pinmap *pindeets = getMapping(pinnum); - if (! pindeets) return; - - GPIO_PinWrite(pindeets->port, pindeets->pin, value); -} - -bool digitalRead(uint32_t pinnum) { - pinmap *pindeets = getMapping(pinnum); - if (! pindeets) return 0; - - - return GPIO_PinRead(pindeets->port, pindeets->pin); -} - -void pinMode(uint32_t pinnum, uint8_t state) { - pinmap *pindeets = getMapping(pinnum); - if (! pindeets) return; - - IOMUXC_SetPinMux(pindeets->mux.muxReg, pindeets->mux.muxMode, - pindeets->mux.inputReg, pindeets->mux.idaisy, - pindeets->mux.configReg, 0U); - IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, - pindeets->mux.inputReg, pindeets->mux.idaisy, - pindeets->mux.configReg, 0x10B0U); - - if (state == OUTPUT) { - gpio_pin_config_t pin_config = { kGPIO_DigitalOutput, 0, kGPIO_NoIntmode }; - GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); - } else { - gpio_pin_config_t pin_config = { kGPIO_DigitalInput, 0, kGPIO_NoIntmode }; - GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); - - if (state == INPUT_PULLUP) { - IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, - pindeets->mux.inputReg, pindeets->mux.idaisy, - pindeets->mux.configReg, 0xB0B0U); - } - } -} - -void setColor(uint32_t color) { - uint8_t rgb[3] = {(color >> 16) & 0xFF, (color >> 8) & 0xFF, color & 0xFF}; - board_rgb_write(rgb); -} - -uint32_t makeColor(uint8_t r, uint8_t g, uint8_t b) { - uint32_t c = 0; - c |= r; - c <<= 8; - c |= g; - c <<= 8; - c |= b; - return c; -} - -uint32_t neoWheel(uint8_t WheelPos) { - WheelPos = 255 - WheelPos; - if (WheelPos < 85) { - return makeColor(255 - WheelPos * 3, 0, WheelPos * 3); - } - if (WheelPos < 170) { - WheelPos -= 85; - return makeColor(0, WheelPos * 3, 255 - WheelPos * 3); - } - WheelPos -= 170; - return makeColor(WheelPos * 3, 255 - WheelPos * 3, 0); -} - - -int analogRead(uint32_t pinnum) -{ - enum { ADC_CHANNEL_GROUP = 0 }; - static bool adc_hw_inited = false; - - pinmap *pindeets = getMapping(pinnum); +#include "arduino.h" + +pinmap pinmapping[] = { + {.num=0 , .port=GPIO1, .pin=9 , .mux={IOMUXC_GPIO_09_GPIOMUX_IO09}} , + {.num=1 , .port=GPIO1, .pin=10, .mux={IOMUXC_GPIO_10_GPIOMUX_IO10}} , + {.num=2 , .port=GPIO1, .pin=13, .mux={IOMUXC_GPIO_13_GPIOMUX_IO13}} , + {.num=3 , .port=GPIO1, .pin=12, .mux={IOMUXC_GPIO_12_GPIOMUX_IO12}} , + + {.num=4 , .port=GPIO2, .pin=0 , .mux={IOMUXC_GPIO_SD_00_GPIO2_IO00}} , + {.num=5 , .port=GPIO2, .pin=1 , .mux={IOMUXC_GPIO_SD_01_GPIO2_IO01}} , + {.num=6 , .port=GPIO2, .pin=2 , .mux={IOMUXC_GPIO_SD_02_GPIO2_IO02}} , + {.num=7 , .port=GPIO1, .pin=11, .mux={IOMUXC_GPIO_11_GPIOMUX_IO11}} , + + {.num=8 , .port=GPIO1, .pin=8 , .mux={IOMUXC_GPIO_08_GPIOMUX_IO08}} , + {.num=9 , .port=GPIO1, .pin=7 , .mux={IOMUXC_GPIO_07_GPIOMUX_IO07}} , + {.num=10, .port=GPIO1, .pin=6 , .mux={IOMUXC_GPIO_06_GPIOMUX_IO06}} , + {.num=11, .port=GPIO1, .pin=5 , .mux={IOMUXC_GPIO_05_GPIOMUX_IO05}} , + {.num=12, .port=GPIO1, .pin=4 , .mux={IOMUXC_GPIO_04_GPIOMUX_IO04}} , + {.num=13, .port=GPIO1, .pin=3 , .mux={IOMUXC_GPIO_03_GPIOMUX_IO03}} , + + {.num=14, .port=GPIO1, .pin=16, .mux={IOMUXC_GPIO_AD_02_GPIOMUX_IO16}, .adc=ADC1, .adc_channel=2 }, // AD0 + {.num=15, .port=GPIO1, .pin=15, .mux={IOMUXC_GPIO_AD_01_GPIOMUX_IO15}, .adc=ADC1, .adc_channel=1 }, // AD1 + {.num=16, .port=GPIO1, .pin=14, .mux={IOMUXC_GPIO_AD_00_GPIOMUX_IO14}, .adc=ADC1, .adc_channel=0 }, // AD2 + {.num=17, .port=GPIO1, .pin=19, .mux={IOMUXC_GPIO_AD_05_GPIOMUX_IO19}, .adc=ADC1, .adc_channel=5 }, // AD3 + {.num=18, .port=GPIO1, .pin=24, .mux={IOMUXC_GPIO_AD_10_GPIOMUX_IO24}, .adc=ADC1, .adc_channel=10 }, // AD4 + {.num=19, .port=GPIO1, .pin=22, .mux={IOMUXC_GPIO_AD_08_GPIOMUX_IO22}, .adc=ADC1, .adc_channel=8 }, // AD5 + + {.num=PIN_SDA , .port=GPIO1, .pin=1 , .mux={IOMUXC_GPIO_01_GPIOMUX_IO01}} , + {.num=PIN_SCL , .port=GPIO1, .pin=2 , .mux={IOMUXC_GPIO_02_GPIOMUX_IO02}} , + + {.num=PIN_MISO, .port=GPIO1, .pin=17, .mux={IOMUXC_GPIO_AD_03_GPIOMUX_IO17}, .adc=ADC1}, // AD0 + {.num=PIN_MOSI, .port=GPIO1, .pin=18, .mux={IOMUXC_GPIO_AD_04_GPIOMUX_IO18}, .adc=ADC1}, // AD0 + {.num=PIN_SCK , .port=GPIO1, .pin=20, .mux={IOMUXC_GPIO_AD_06_GPIOMUX_IO20}, .adc=ADC1}, // AD0 + + {.num=PIN_SD_CS , .port=GPIO1, .pin=28, .mux={IOMUXC_GPIO_AD_14_GPIOMUX_IO28}}, + {.num=PIN_SD_DETECT, .port=GPIO1, .pin=25, .mux={IOMUXC_GPIO_AD_11_GPIOMUX_IO25}}, +}; + +static volatile uint32_t _millis = 0; + +void board_timer_handler(void) { + _millis++; +} + +uint32_t millis(void) { +#ifndef BUILD_APPLICATION + extern uint32_t board_millis(void); + return board_millis(); +#else + return _millis; +#endif +} + +void delay(uint32_t ms) { + uint32_t timestamp = millis(); + while ((timestamp+ms) > millis()) { + // Serial_printf("delay %d\n\r", millis()); + tud_task(); + tud_cdc_write_flush(); + } +} + + +static pinmap *getMapping(uint32_t pinnum) { + pinmap *pindeets = NULL; + + for (uint32_t i=0; i port %x, pin %d\n\r", pindeets->num, pindeets->port, pindeets->pin); + if (pinnum == pindeets->num) { + //Serial_printf("found!\n\r"); + return pindeets; + } + } + return NULL; +} + + +void digitalWrite(uint32_t pinnum, bool value) { + pinmap *pindeets = getMapping(pinnum); + if (! pindeets) return; + + GPIO_PinWrite(pindeets->port, pindeets->pin, value); +} + +bool digitalRead(uint32_t pinnum) { + pinmap *pindeets = getMapping(pinnum); + if (! pindeets) return 0; + + + return GPIO_PinRead(pindeets->port, pindeets->pin); +} + +void pinMode(uint32_t pinnum, uint8_t state) { + pinmap *pindeets = getMapping(pinnum); + if (! pindeets) return; + + IOMUXC_SetPinMux(pindeets->mux.muxReg, pindeets->mux.muxMode, + pindeets->mux.inputReg, pindeets->mux.idaisy, + pindeets->mux.configReg, 0U); + IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, + pindeets->mux.inputReg, pindeets->mux.idaisy, + pindeets->mux.configReg, 0x10B0U); + + if (state == OUTPUT) { + gpio_pin_config_t pin_config = { kGPIO_DigitalOutput, 0, kGPIO_NoIntmode }; + GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); + } else { + gpio_pin_config_t pin_config = { kGPIO_DigitalInput, 0, kGPIO_NoIntmode }; + GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); + + if (state == INPUT_PULLUP) { + IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, + pindeets->mux.inputReg, pindeets->mux.idaisy, + pindeets->mux.configReg, 0xB0B0U); + } + } +} + +void setColor(uint32_t color) { + uint8_t rgb[3] = {(color >> 16) & 0xFF, (color >> 8) & 0xFF, color & 0xFF}; + board_rgb_write(rgb); +} + +uint32_t makeColor(uint8_t r, uint8_t g, uint8_t b) { + uint32_t c = 0; + c |= r; + c <<= 8; + c |= g; + c <<= 8; + c |= b; + return c; +} + +uint32_t neoWheel(uint8_t WheelPos) { + WheelPos = 255 - WheelPos; + if (WheelPos < 85) { + return makeColor(255 - WheelPos * 3, 0, WheelPos * 3); + } + if (WheelPos < 170) { + WheelPos -= 85; + return makeColor(0, WheelPos * 3, 255 - WheelPos * 3); + } + WheelPos -= 170; + return makeColor(WheelPos * 3, 255 - WheelPos * 3, 0); +} + + +int analogRead(uint32_t pinnum) +{ + enum { ADC_CHANNEL_GROUP = 0 }; + static bool adc_hw_inited = false; + + pinmap *pindeets = getMapping(pinnum); if (! pindeets) return -1; if (! pindeets->adc) return -1; - if (!pindeets->adc_inited) - { - if (!adc_hw_inited) - { - adc_config_t config = {0}; - - ADC_GetDefaultConfig(&config); - config.enableLongSample = true; - config.samplePeriodMode = kADC_SamplePeriod8or24Clocks; - - ADC_Init(pindeets->adc, &config); - ADC_SetHardwareAverageConfig(pindeets->adc, kADC_HardwareAverageCount32); - ADC_DoAutoCalibration(pindeets->adc); - - adc_hw_inited = true; - } - - pindeets->adc_inited = true; - } + // Board startup may mux an analog pin to a peripheral output (A0 is TRACE_CLK). + // Release it as an input before sampling the external voltage. + pinMode(pinnum, INPUT); - /* When in software trigger mode, each conversion would be launched once calling the "ADC_ChannelConfigure()" - function, which works like writing a conversion command and executing it. For another channel's conversion, - just to change the "channelNumber" field in channel's configuration structure, and call the - "ADC_ChannelConfigure() again. - */ - adc_channel_config_t config = { - .channelNumber = pindeets->adc_channel - }; - ADC_SetChannelConfig(pindeets->adc, ADC_CHANNEL_GROUP, &config); - - while (0U == ADC_GetChannelStatusFlags(pindeets->adc, ADC_CHANNEL_GROUP)) - { - } - - // value is 12-bit resolution - uint16_t value = ADC_GetChannelConversionValue(pindeets->adc, ADC_CHANNEL_GROUP); - - return value; -} + if (!pindeets->adc_inited) + { + if (!adc_hw_inited) + { + adc_config_t config = {0}; + + ADC_GetDefaultConfig(&config); + config.enableLongSample = true; + config.samplePeriodMode = kADC_SamplePeriod8or24Clocks; + + ADC_Init(pindeets->adc, &config); + ADC_SetHardwareAverageConfig(pindeets->adc, kADC_HardwareAverageCount32); + ADC_DoAutoCalibration(pindeets->adc); + + adc_hw_inited = true; + } + + pindeets->adc_inited = true; + } + + /* When in software trigger mode, each conversion would be launched once calling the "ADC_ChannelConfigure()" + function, which works like writing a conversion command and executing it. For another channel's conversion, + just to change the "channelNumber" field in channel's configuration structure, and call the + "ADC_ChannelConfigure() again. + */ + adc_channel_config_t config = { + .channelNumber = pindeets->adc_channel + }; + ADC_SetChannelConfig(pindeets->adc, ADC_CHANNEL_GROUP, &config); + + while (0U == ADC_GetChannelStatusFlags(pindeets->adc, ADC_CHANNEL_GROUP)) + { + } + + // value is 12-bit resolution + uint16_t value = ADC_GetChannelConversionValue(pindeets->adc, ADC_CHANNEL_GROUP); + + return value; +} From 04f5ac4aa05cd541bd6a182510da44cf58e58c53 Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:33:41 -0400 Subject: [PATCH 10/47] Preserve mixed line endings in factory Arduino helpers Co-authored-by: Limor Fried --- .../apps/factory_test_metro_sd/arduino.c | 384 +++++++++--------- 1 file changed, 192 insertions(+), 192 deletions(-) diff --git a/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c b/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c index ea45a6cf7..8b3c3e7f9 100644 --- a/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c +++ b/ports/mimxrt10xx/apps/factory_test_metro_sd/arduino.c @@ -1,157 +1,157 @@ -#include "arduino.h" - -pinmap pinmapping[] = { - {.num=0 , .port=GPIO1, .pin=9 , .mux={IOMUXC_GPIO_09_GPIOMUX_IO09}} , - {.num=1 , .port=GPIO1, .pin=10, .mux={IOMUXC_GPIO_10_GPIOMUX_IO10}} , - {.num=2 , .port=GPIO1, .pin=13, .mux={IOMUXC_GPIO_13_GPIOMUX_IO13}} , - {.num=3 , .port=GPIO1, .pin=12, .mux={IOMUXC_GPIO_12_GPIOMUX_IO12}} , - - {.num=4 , .port=GPIO2, .pin=0 , .mux={IOMUXC_GPIO_SD_00_GPIO2_IO00}} , - {.num=5 , .port=GPIO2, .pin=1 , .mux={IOMUXC_GPIO_SD_01_GPIO2_IO01}} , - {.num=6 , .port=GPIO2, .pin=2 , .mux={IOMUXC_GPIO_SD_02_GPIO2_IO02}} , - {.num=7 , .port=GPIO1, .pin=11, .mux={IOMUXC_GPIO_11_GPIOMUX_IO11}} , - - {.num=8 , .port=GPIO1, .pin=8 , .mux={IOMUXC_GPIO_08_GPIOMUX_IO08}} , - {.num=9 , .port=GPIO1, .pin=7 , .mux={IOMUXC_GPIO_07_GPIOMUX_IO07}} , - {.num=10, .port=GPIO1, .pin=6 , .mux={IOMUXC_GPIO_06_GPIOMUX_IO06}} , - {.num=11, .port=GPIO1, .pin=5 , .mux={IOMUXC_GPIO_05_GPIOMUX_IO05}} , - {.num=12, .port=GPIO1, .pin=4 , .mux={IOMUXC_GPIO_04_GPIOMUX_IO04}} , - {.num=13, .port=GPIO1, .pin=3 , .mux={IOMUXC_GPIO_03_GPIOMUX_IO03}} , - - {.num=14, .port=GPIO1, .pin=16, .mux={IOMUXC_GPIO_AD_02_GPIOMUX_IO16}, .adc=ADC1, .adc_channel=2 }, // AD0 - {.num=15, .port=GPIO1, .pin=15, .mux={IOMUXC_GPIO_AD_01_GPIOMUX_IO15}, .adc=ADC1, .adc_channel=1 }, // AD1 - {.num=16, .port=GPIO1, .pin=14, .mux={IOMUXC_GPIO_AD_00_GPIOMUX_IO14}, .adc=ADC1, .adc_channel=0 }, // AD2 - {.num=17, .port=GPIO1, .pin=19, .mux={IOMUXC_GPIO_AD_05_GPIOMUX_IO19}, .adc=ADC1, .adc_channel=5 }, // AD3 - {.num=18, .port=GPIO1, .pin=24, .mux={IOMUXC_GPIO_AD_10_GPIOMUX_IO24}, .adc=ADC1, .adc_channel=10 }, // AD4 - {.num=19, .port=GPIO1, .pin=22, .mux={IOMUXC_GPIO_AD_08_GPIOMUX_IO22}, .adc=ADC1, .adc_channel=8 }, // AD5 - - {.num=PIN_SDA , .port=GPIO1, .pin=1 , .mux={IOMUXC_GPIO_01_GPIOMUX_IO01}} , - {.num=PIN_SCL , .port=GPIO1, .pin=2 , .mux={IOMUXC_GPIO_02_GPIOMUX_IO02}} , - - {.num=PIN_MISO, .port=GPIO1, .pin=17, .mux={IOMUXC_GPIO_AD_03_GPIOMUX_IO17}, .adc=ADC1}, // AD0 - {.num=PIN_MOSI, .port=GPIO1, .pin=18, .mux={IOMUXC_GPIO_AD_04_GPIOMUX_IO18}, .adc=ADC1}, // AD0 - {.num=PIN_SCK , .port=GPIO1, .pin=20, .mux={IOMUXC_GPIO_AD_06_GPIOMUX_IO20}, .adc=ADC1}, // AD0 - - {.num=PIN_SD_CS , .port=GPIO1, .pin=28, .mux={IOMUXC_GPIO_AD_14_GPIOMUX_IO28}}, - {.num=PIN_SD_DETECT, .port=GPIO1, .pin=25, .mux={IOMUXC_GPIO_AD_11_GPIOMUX_IO25}}, -}; - -static volatile uint32_t _millis = 0; - -void board_timer_handler(void) { - _millis++; -} - -uint32_t millis(void) { -#ifndef BUILD_APPLICATION - extern uint32_t board_millis(void); - return board_millis(); -#else - return _millis; -#endif -} - -void delay(uint32_t ms) { - uint32_t timestamp = millis(); - while ((timestamp+ms) > millis()) { - // Serial_printf("delay %d\n\r", millis()); - tud_task(); - tud_cdc_write_flush(); - } -} - - -static pinmap *getMapping(uint32_t pinnum) { - pinmap *pindeets = NULL; - - for (uint32_t i=0; i port %x, pin %d\n\r", pindeets->num, pindeets->port, pindeets->pin); - if (pinnum == pindeets->num) { - //Serial_printf("found!\n\r"); - return pindeets; - } - } - return NULL; -} - - -void digitalWrite(uint32_t pinnum, bool value) { - pinmap *pindeets = getMapping(pinnum); - if (! pindeets) return; - - GPIO_PinWrite(pindeets->port, pindeets->pin, value); -} - -bool digitalRead(uint32_t pinnum) { - pinmap *pindeets = getMapping(pinnum); - if (! pindeets) return 0; - - - return GPIO_PinRead(pindeets->port, pindeets->pin); -} - -void pinMode(uint32_t pinnum, uint8_t state) { - pinmap *pindeets = getMapping(pinnum); - if (! pindeets) return; - - IOMUXC_SetPinMux(pindeets->mux.muxReg, pindeets->mux.muxMode, - pindeets->mux.inputReg, pindeets->mux.idaisy, - pindeets->mux.configReg, 0U); - IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, - pindeets->mux.inputReg, pindeets->mux.idaisy, - pindeets->mux.configReg, 0x10B0U); - - if (state == OUTPUT) { - gpio_pin_config_t pin_config = { kGPIO_DigitalOutput, 0, kGPIO_NoIntmode }; - GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); - } else { - gpio_pin_config_t pin_config = { kGPIO_DigitalInput, 0, kGPIO_NoIntmode }; - GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); - - if (state == INPUT_PULLUP) { - IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, - pindeets->mux.inputReg, pindeets->mux.idaisy, - pindeets->mux.configReg, 0xB0B0U); - } - } -} - -void setColor(uint32_t color) { - uint8_t rgb[3] = {(color >> 16) & 0xFF, (color >> 8) & 0xFF, color & 0xFF}; - board_rgb_write(rgb); -} - -uint32_t makeColor(uint8_t r, uint8_t g, uint8_t b) { - uint32_t c = 0; - c |= r; - c <<= 8; - c |= g; - c <<= 8; - c |= b; - return c; -} - -uint32_t neoWheel(uint8_t WheelPos) { - WheelPos = 255 - WheelPos; - if (WheelPos < 85) { - return makeColor(255 - WheelPos * 3, 0, WheelPos * 3); - } - if (WheelPos < 170) { - WheelPos -= 85; - return makeColor(0, WheelPos * 3, 255 - WheelPos * 3); - } - WheelPos -= 170; - return makeColor(WheelPos * 3, 255 - WheelPos * 3, 0); -} - - -int analogRead(uint32_t pinnum) -{ - enum { ADC_CHANNEL_GROUP = 0 }; - static bool adc_hw_inited = false; - - pinmap *pindeets = getMapping(pinnum); +#include "arduino.h" + +pinmap pinmapping[] = { + {.num=0 , .port=GPIO1, .pin=9 , .mux={IOMUXC_GPIO_09_GPIOMUX_IO09}} , + {.num=1 , .port=GPIO1, .pin=10, .mux={IOMUXC_GPIO_10_GPIOMUX_IO10}} , + {.num=2 , .port=GPIO1, .pin=13, .mux={IOMUXC_GPIO_13_GPIOMUX_IO13}} , + {.num=3 , .port=GPIO1, .pin=12, .mux={IOMUXC_GPIO_12_GPIOMUX_IO12}} , + + {.num=4 , .port=GPIO2, .pin=0 , .mux={IOMUXC_GPIO_SD_00_GPIO2_IO00}} , + {.num=5 , .port=GPIO2, .pin=1 , .mux={IOMUXC_GPIO_SD_01_GPIO2_IO01}} , + {.num=6 , .port=GPIO2, .pin=2 , .mux={IOMUXC_GPIO_SD_02_GPIO2_IO02}} , + {.num=7 , .port=GPIO1, .pin=11, .mux={IOMUXC_GPIO_11_GPIOMUX_IO11}} , + + {.num=8 , .port=GPIO1, .pin=8 , .mux={IOMUXC_GPIO_08_GPIOMUX_IO08}} , + {.num=9 , .port=GPIO1, .pin=7 , .mux={IOMUXC_GPIO_07_GPIOMUX_IO07}} , + {.num=10, .port=GPIO1, .pin=6 , .mux={IOMUXC_GPIO_06_GPIOMUX_IO06}} , + {.num=11, .port=GPIO1, .pin=5 , .mux={IOMUXC_GPIO_05_GPIOMUX_IO05}} , + {.num=12, .port=GPIO1, .pin=4 , .mux={IOMUXC_GPIO_04_GPIOMUX_IO04}} , + {.num=13, .port=GPIO1, .pin=3 , .mux={IOMUXC_GPIO_03_GPIOMUX_IO03}} , + + {.num=14, .port=GPIO1, .pin=16, .mux={IOMUXC_GPIO_AD_02_GPIOMUX_IO16}, .adc=ADC1, .adc_channel=2 }, // AD0 + {.num=15, .port=GPIO1, .pin=15, .mux={IOMUXC_GPIO_AD_01_GPIOMUX_IO15}, .adc=ADC1, .adc_channel=1 }, // AD1 + {.num=16, .port=GPIO1, .pin=14, .mux={IOMUXC_GPIO_AD_00_GPIOMUX_IO14}, .adc=ADC1, .adc_channel=0 }, // AD2 + {.num=17, .port=GPIO1, .pin=19, .mux={IOMUXC_GPIO_AD_05_GPIOMUX_IO19}, .adc=ADC1, .adc_channel=5 }, // AD3 + {.num=18, .port=GPIO1, .pin=24, .mux={IOMUXC_GPIO_AD_10_GPIOMUX_IO24}, .adc=ADC1, .adc_channel=10 }, // AD4 + {.num=19, .port=GPIO1, .pin=22, .mux={IOMUXC_GPIO_AD_08_GPIOMUX_IO22}, .adc=ADC1, .adc_channel=8 }, // AD5 + + {.num=PIN_SDA , .port=GPIO1, .pin=1 , .mux={IOMUXC_GPIO_01_GPIOMUX_IO01}} , + {.num=PIN_SCL , .port=GPIO1, .pin=2 , .mux={IOMUXC_GPIO_02_GPIOMUX_IO02}} , + + {.num=PIN_MISO, .port=GPIO1, .pin=17, .mux={IOMUXC_GPIO_AD_03_GPIOMUX_IO17}, .adc=ADC1}, // AD0 + {.num=PIN_MOSI, .port=GPIO1, .pin=18, .mux={IOMUXC_GPIO_AD_04_GPIOMUX_IO18}, .adc=ADC1}, // AD0 + {.num=PIN_SCK , .port=GPIO1, .pin=20, .mux={IOMUXC_GPIO_AD_06_GPIOMUX_IO20}, .adc=ADC1}, // AD0 + + {.num=PIN_SD_CS , .port=GPIO1, .pin=28, .mux={IOMUXC_GPIO_AD_14_GPIOMUX_IO28}}, + {.num=PIN_SD_DETECT, .port=GPIO1, .pin=25, .mux={IOMUXC_GPIO_AD_11_GPIOMUX_IO25}}, +}; + +static volatile uint32_t _millis = 0; + +void board_timer_handler(void) { + _millis++; +} + +uint32_t millis(void) { +#ifndef BUILD_APPLICATION + extern uint32_t board_millis(void); + return board_millis(); +#else + return _millis; +#endif +} + +void delay(uint32_t ms) { + uint32_t timestamp = millis(); + while ((timestamp+ms) > millis()) { + // Serial_printf("delay %d\n\r", millis()); + tud_task(); + tud_cdc_write_flush(); + } +} + + +static pinmap *getMapping(uint32_t pinnum) { + pinmap *pindeets = NULL; + + for (uint32_t i=0; i port %x, pin %d\n\r", pindeets->num, pindeets->port, pindeets->pin); + if (pinnum == pindeets->num) { + //Serial_printf("found!\n\r"); + return pindeets; + } + } + return NULL; +} + + +void digitalWrite(uint32_t pinnum, bool value) { + pinmap *pindeets = getMapping(pinnum); + if (! pindeets) return; + + GPIO_PinWrite(pindeets->port, pindeets->pin, value); +} + +bool digitalRead(uint32_t pinnum) { + pinmap *pindeets = getMapping(pinnum); + if (! pindeets) return 0; + + + return GPIO_PinRead(pindeets->port, pindeets->pin); +} + +void pinMode(uint32_t pinnum, uint8_t state) { + pinmap *pindeets = getMapping(pinnum); + if (! pindeets) return; + + IOMUXC_SetPinMux(pindeets->mux.muxReg, pindeets->mux.muxMode, + pindeets->mux.inputReg, pindeets->mux.idaisy, + pindeets->mux.configReg, 0U); + IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, + pindeets->mux.inputReg, pindeets->mux.idaisy, + pindeets->mux.configReg, 0x10B0U); + + if (state == OUTPUT) { + gpio_pin_config_t pin_config = { kGPIO_DigitalOutput, 0, kGPIO_NoIntmode }; + GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); + } else { + gpio_pin_config_t pin_config = { kGPIO_DigitalInput, 0, kGPIO_NoIntmode }; + GPIO_PinInit(pindeets->port, pindeets->pin, &pin_config); + + if (state == INPUT_PULLUP) { + IOMUXC_SetPinConfig(pindeets->mux.muxReg, pindeets->mux.muxMode, + pindeets->mux.inputReg, pindeets->mux.idaisy, + pindeets->mux.configReg, 0xB0B0U); + } + } +} + +void setColor(uint32_t color) { + uint8_t rgb[3] = {(color >> 16) & 0xFF, (color >> 8) & 0xFF, color & 0xFF}; + board_rgb_write(rgb); +} + +uint32_t makeColor(uint8_t r, uint8_t g, uint8_t b) { + uint32_t c = 0; + c |= r; + c <<= 8; + c |= g; + c <<= 8; + c |= b; + return c; +} + +uint32_t neoWheel(uint8_t WheelPos) { + WheelPos = 255 - WheelPos; + if (WheelPos < 85) { + return makeColor(255 - WheelPos * 3, 0, WheelPos * 3); + } + if (WheelPos < 170) { + WheelPos -= 85; + return makeColor(0, WheelPos * 3, 255 - WheelPos * 3); + } + WheelPos -= 170; + return makeColor(WheelPos * 3, 255 - WheelPos * 3, 0); +} + + +int analogRead(uint32_t pinnum) +{ + enum { ADC_CHANNEL_GROUP = 0 }; + static bool adc_hw_inited = false; + + pinmap *pindeets = getMapping(pinnum); if (! pindeets) return -1; if (! pindeets->adc) return -1; @@ -160,41 +160,41 @@ int analogRead(uint32_t pinnum) pinMode(pinnum, INPUT); if (!pindeets->adc_inited) - { - if (!adc_hw_inited) - { - adc_config_t config = {0}; - - ADC_GetDefaultConfig(&config); - config.enableLongSample = true; - config.samplePeriodMode = kADC_SamplePeriod8or24Clocks; - - ADC_Init(pindeets->adc, &config); - ADC_SetHardwareAverageConfig(pindeets->adc, kADC_HardwareAverageCount32); - ADC_DoAutoCalibration(pindeets->adc); - - adc_hw_inited = true; - } - - pindeets->adc_inited = true; - } - - /* When in software trigger mode, each conversion would be launched once calling the "ADC_ChannelConfigure()" - function, which works like writing a conversion command and executing it. For another channel's conversion, - just to change the "channelNumber" field in channel's configuration structure, and call the - "ADC_ChannelConfigure() again. - */ - adc_channel_config_t config = { - .channelNumber = pindeets->adc_channel - }; - ADC_SetChannelConfig(pindeets->adc, ADC_CHANNEL_GROUP, &config); - - while (0U == ADC_GetChannelStatusFlags(pindeets->adc, ADC_CHANNEL_GROUP)) - { - } - - // value is 12-bit resolution - uint16_t value = ADC_GetChannelConversionValue(pindeets->adc, ADC_CHANNEL_GROUP); - - return value; -} + { + if (!adc_hw_inited) + { + adc_config_t config = {0}; + + ADC_GetDefaultConfig(&config); + config.enableLongSample = true; + config.samplePeriodMode = kADC_SamplePeriod8or24Clocks; + + ADC_Init(pindeets->adc, &config); + ADC_SetHardwareAverageConfig(pindeets->adc, kADC_HardwareAverageCount32); + ADC_DoAutoCalibration(pindeets->adc); + + adc_hw_inited = true; + } + + pindeets->adc_inited = true; + } + + /* When in software trigger mode, each conversion would be launched once calling the "ADC_ChannelConfigure()" + function, which works like writing a conversion command and executing it. For another channel's conversion, + just to change the "channelNumber" field in channel's configuration structure, and call the + "ADC_ChannelConfigure() again. + */ + adc_channel_config_t config = { + .channelNumber = pindeets->adc_channel + }; + ADC_SetChannelConfig(pindeets->adc, ADC_CHANNEL_GROUP, &config); + + while (0U == ADC_GetChannelStatusFlags(pindeets->adc, ADC_CHANNEL_GROUP)) + { + } + + // value is 12-bit resolution + uint16_t value = ADC_GetChannelConversionValue(pindeets->adc, ADC_CHANNEL_GROUP); + + return value; +} From 577980a0d55eac916775c62d8970672cdc5a8ae4 Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:09:57 -0400 Subject: [PATCH 11/47] Make Metro M7 AirLift factory apps usable with USB-host testers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run one GPIO/power test per command, release analog inputs before conversion, accept measured 8–13 V DC input, and flush failure messages. Hold the ESP32 in reset only during the fixture pass and release tested outputs before restart. Enable CDC DTR/RTS programming resets, preserve USB TX backpressure, service both bridge directions between bounded writes, and respect UART read lengths. Reuse the flash-only application linker correction from PR #484. Co-authored-by: Limor Fried --- ports/mimxrt10xx/apps/esp32programmer/main.c | 11 +++--- ports/mimxrt10xx/apps/factory_test/arduino.c | 4 +++ ports/mimxrt10xx/apps/factory_test/main.c | 35 +++++++++++++++++--- ports/mimxrt10xx/boards.c | 2 +- ports/mimxrt10xx/linker/app.ld | 15 ++++++++- 5 files changed, 56 insertions(+), 11 deletions(-) diff --git a/ports/mimxrt10xx/apps/esp32programmer/main.c b/ports/mimxrt10xx/apps/esp32programmer/main.c index 867028e82..8ef9a9c2a 100644 --- a/ports/mimxrt10xx/apps/esp32programmer/main.c +++ b/ports/mimxrt10xx/apps/esp32programmer/main.c @@ -40,7 +40,7 @@ // Enable this for more reliable connection require esptool.py default reset option "--before default_reset" // i.e "--before no_reset" should not be include in the esptool.py command -#define ESP32_DTR_RTS_BOOT_RESET_SUPPORT 0 +#define ESP32_DTR_RTS_BOOT_RESET_SUPPORT 1 // optional API, not included in board_api.h int board_uart_read(uint8_t* buf, int len); @@ -132,7 +132,9 @@ int main(void) { uint32_t count; // UART -> USB - count = (uint32_t)board_uart_read(serial_buf, sizeof(serial_buf)); + // Leave UART bytes queued until the USB transmit FIFO has room for them. + count = tu_min32(tud_cdc_write_available(), sizeof(serial_buf)); + count = (uint32_t)board_uart_read(serial_buf, count); if (count) { board_led_write(0xff); @@ -143,10 +145,11 @@ int main(void) { } // USB -> UART - while (tud_cdc_available()) { + if (tud_cdc_available()) { board_led_write(0xff); - count = tud_cdc_read(serial_buf, sizeof(serial_buf)); + // Service USB and UART receive between full-speed USB packet-sized writes. + count = tud_cdc_read(serial_buf, 64); board_uart_write(serial_buf, count); board_led_write(0); diff --git a/ports/mimxrt10xx/apps/factory_test/arduino.c b/ports/mimxrt10xx/apps/factory_test/arduino.c index 53579445e..bb8d44cec 100644 --- a/ports/mimxrt10xx/apps/factory_test/arduino.c +++ b/ports/mimxrt10xx/apps/factory_test/arduino.c @@ -147,6 +147,10 @@ int analogRead(uint32_t pinnum) if (! pindeets) return -1; if (! pindeets->adc) return -1; + // Board startup may mux an analog pin to a peripheral output (A0 is TRACE_CLK). + // Release it as an input before sampling the external voltage. + pinMode(pinnum, INPUT); + if (!pindeets->adc_inited) { if (!adc_hw_inited) diff --git a/ports/mimxrt10xx/apps/factory_test/main.c b/ports/mimxrt10xx/apps/factory_test/main.c index cea667087..5e74b558a 100644 --- a/ports/mimxrt10xx/apps/factory_test/main.c +++ b/ports/mimxrt10xx/apps/factory_test/main.c @@ -43,9 +43,12 @@ uint8_t all_pins[] = {0, 1, 2, 3, 4, 5, 6, PIN_SDA, PIN_SCL, PIN_MOSI, PIN_MISO, PIN_SCK, AD5}; bool test = false; +static const float DC_INPUT_MIN_VOLTS = 8.0f; +static const float DC_INPUT_MAX_VOLTS = 13.0f; bool testpins(uint8_t a, uint8_t b, uint8_t *allpins, uint8_t num_allpins); void test_print_adc(void); +static void test_board(void); /* This is an application to test Metro M7 */ @@ -75,6 +78,8 @@ int main(void) { while (1) { loop(); + // Failure messages may have no newline; drain libc before the USB FIFO. + fflush(stdout); tud_task(); tud_cdc_write_flush(); } @@ -106,7 +111,27 @@ void loop(void) { return; } + test = false; // Run one pass per start command, including failed passes. + + // Keep the ESP32 from driving the UART and SPI loopback pins during testing. + gpio_pin_config_t reset_config = {kGPIO_DigitalOutput, 0, kGPIO_NoIntmode}; + IOMUXC_SetPinMux(ESP32_RESET_PINMUX, 0); + IOMUXC_SetPinConfig(ESP32_RESET_PINMUX, 0x10B0U); + GPIO_PinInit(ESP32_RESET_PORT, ESP32_RESET_PIN, &reset_config); delay(100); + test_board(); + + // Release every tested output before allowing the ESP32 to run, even on failure. + for (size_t i = 0; i < sizeof(all_pins); i++) { + pinMode(all_pins[i], INPUT); + } + for (uint8_t pin = AD0; pin <= AD5; pin++) { + pinMode(pin, INPUT); + } + GPIO_PinWrite(ESP32_RESET_PORT, ESP32_RESET_PIN, 1); +} + +static void test_board(void) { Serial_printf("\n\r\n\rHello Metro M7 iMX RT1011 Test! %lu\n\r", millis()); if (!testpins(0, 2, all_pins, sizeof(all_pins))) @@ -140,11 +165,11 @@ void loop(void) { Serial_printf("5V power supply reading wrong?"); return; } - // Test 5V - int nine_mV = (float)analogRead(AD0) * 11.0 * 3.3 * 1000 / 4095.0; - Serial_printf("9V out = %d\n\r", (int)nine_mV); - if (abs(nine_mV - 9000) > 1000) { - Serial_printf("9V power supply reading wrong?"); + // Accept the fixture's 9V or 12V DC supply, with measurement tolerance. + float dc_input_volts = (float)analogRead(AD0) * 11.0f * 3.3f / 4095.0f; + Serial_printf("DC input = %d mV\n\r", (int)(dc_input_volts * 1000.0f)); + if (dc_input_volts < DC_INPUT_MIN_VOLTS || dc_input_volts > DC_INPUT_MAX_VOLTS) { + Serial_printf("DC input power supply reading wrong?"); return; } Serial_printf("*** TEST OK! ***\n\r"); diff --git a/ports/mimxrt10xx/boards.c b/ports/mimxrt10xx/boards.c index 246543c4e..a6ff170b7 100644 --- a/ports/mimxrt10xx/boards.c +++ b/ports/mimxrt10xx/boards.c @@ -407,7 +407,7 @@ int board_uart_read(uint8_t *buf, int len) { break; } - for (int i = 0; i < rx_count; i++) { + for (int i = 0; i < rx_count && count < len; i++) { buf[count] = LPUART_ReadByte(UART_DEV); count++; } diff --git a/ports/mimxrt10xx/linker/app.ld b/ports/mimxrt10xx/linker/app.ld index c93ea3eed..ebdf87be4 100644 --- a/ports/mimxrt10xx/linker/app.ld +++ b/ports/mimxrt10xx/linker/app.ld @@ -9,7 +9,20 @@ _interrupts_origin = _flash_base + 0xC000; PROVIDE(_interrupts_length = 0x0400); _text_origin = _interrupts_origin + _interrupts_length; -PROVIDE(_text_length = 64K); +/* The application runs from flash; override the bootloader RAM text limit. */ +_text_length = 64K; + +/* Applications do not carry the ROM boot headers used by TinyUF2. */ +SECTIONS +{ + /DISCARD/ : + { + *(.boot_hdr.conf) + *(.boot_hdr.ivt) + *(.boot_hdr.boot_data) + *(.boot_hdr.dcd_data) + } +} /* Specify the memory areas */ MEMORY From 42ce79bb9e3c6d10e37263590b33060587cc9777 Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:58:11 -0400 Subject: [PATCH 12/47] Avoid AirLift fixture pin conflicts during programming and tests Release the UART loopback partner pins from trace output before starting the ESP32 bridge. Skip the SPI loopback because the Brains fixture leaves its SPI pogos covered; retain SPI pins in short detection. Co-authored-by: Limor Fried --- ports/mimxrt10xx/apps/esp32programmer/main.c | 10 ++++++++++ ports/mimxrt10xx/apps/factory_test/main.c | 3 +-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/ports/mimxrt10xx/apps/esp32programmer/main.c b/ports/mimxrt10xx/apps/esp32programmer/main.c index 8ef9a9c2a..ee407b473 100644 --- a/ports/mimxrt10xx/apps/esp32programmer/main.c +++ b/ports/mimxrt10xx/apps/esp32programmer/main.c @@ -105,6 +105,16 @@ void esp32_manual_enter_dfu(void) { int main(void) { board_init(); + // The fixture connects UART D0/D1 to D2/D3 for the GPIO loopback test. + // Release their startup trace outputs so they cannot drive the ESP32 UART. + gpio_pin_config_t input_config = { kGPIO_DigitalInput, 0, kGPIO_NoIntmode }; + GPIO_PinInit(GPIO1, 13, &input_config); + IOMUXC_SetPinConfig(IOMUXC_GPIO_13_GPIOMUX_IO13, 0x00B0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_13_GPIOMUX_IO13, 0); + GPIO_PinInit(GPIO1, 12, &input_config); + IOMUXC_SetPinConfig(IOMUXC_GPIO_12_GPIOMUX_IO12, 0x00B0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_12_GPIOMUX_IO12, 0); + gpio_pin_config_t pin_config = { kGPIO_DigitalOutput, 1, kGPIO_NoIntmode }; // ESP GPIO0 diff --git a/ports/mimxrt10xx/apps/factory_test/main.c b/ports/mimxrt10xx/apps/factory_test/main.c index 5e74b558a..b43ada7db 100644 --- a/ports/mimxrt10xx/apps/factory_test/main.c +++ b/ports/mimxrt10xx/apps/factory_test/main.c @@ -150,8 +150,7 @@ static void test_board(void) { return; if (!testpins(12, PIN_SCL, all_pins, sizeof(all_pins))) return; - if (!testpins(PIN_MOSI, PIN_MISO, all_pins, sizeof(all_pins))) - return; + // The SPI loopback is not connected in the Brains fixture. if (!testpins(AD2, AD4, all_pins, sizeof(all_pins))) return; if (!testpins(AD3, AD5, all_pins, sizeof(all_pins))) From 165bff8c85b685248bb4df7e0c2f63c8390c5707 Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:13:50 -0400 Subject: [PATCH 13/47] Require an AirLift Wi-Fi scan for factory pass and add an idle demo Run a bounded NINA SPI scan after the GPIO and power checks, accepting the factory pass only when at least one valid nonempty SSID is returned. Escape SSID output and keep USB responsive while waiting. Add a friendly serial greeting, LED/NeoPixel animation, and recurring idle scans. The Metro M7 factory app builds with Make. Update both Make and CMake source lists; CMake build and hardware validation are pending. Co-authored-by: Limor Fried --- .../apps/factory_test/CMakeLists.txt | 2 + ports/mimxrt10xx/apps/factory_test/Makefile | 2 + ports/mimxrt10xx/apps/factory_test/main.c | 96 +++++-- .../mimxrt10xx/apps/factory_test/wifi_scan.c | 242 ++++++++++++++++++ .../mimxrt10xx/apps/factory_test/wifi_scan.h | 11 + 5 files changed, 326 insertions(+), 27 deletions(-) create mode 100644 ports/mimxrt10xx/apps/factory_test/wifi_scan.c create mode 100644 ports/mimxrt10xx/apps/factory_test/wifi_scan.h diff --git a/ports/mimxrt10xx/apps/factory_test/CMakeLists.txt b/ports/mimxrt10xx/apps/factory_test/CMakeLists.txt index cd77ecfbc..c9413d4df 100644 --- a/ports/mimxrt10xx/apps/factory_test/CMakeLists.txt +++ b/ports/mimxrt10xx/apps/factory_test/CMakeLists.txt @@ -12,6 +12,8 @@ include(${CMAKE_CURRENT_LIST_DIR}/../../app.cmake) add_executable(factory_test arduino.c main.c + wifi_scan.c + ${TOP}/lib/mcu/nxp/mcux-sdk/drivers/common/fsl_common_arm.c usb_descriptors.c ${CMAKE_CURRENT_LIST_DIR}/../../boards.c ${TOP}/lib/tinyusb/src/portable/chipidea/ci_hs/dcd_ci_hs.c diff --git a/ports/mimxrt10xx/apps/factory_test/Makefile b/ports/mimxrt10xx/apps/factory_test/Makefile index 02e674a98..10476f718 100644 --- a/ports/mimxrt10xx/apps/factory_test/Makefile +++ b/ports/mimxrt10xx/apps/factory_test/Makefile @@ -4,8 +4,10 @@ SRC_C += \ $(PORT_DIR)/boards.c \ $(CURRENT_PATH)/main.c \ $(CURRENT_PATH)/arduino.c \ + $(CURRENT_PATH)/wifi_scan.c \ $(CURRENT_PATH)/usb_descriptors.c \ $(SDK_DIR)/drivers/adc_12b1msps_sar/fsl_adc.c \ + $(SDK_DIR)/drivers/common/fsl_common_arm.c \ INC += \ $(TOP)/$(CURRENT_PATH) \ diff --git a/ports/mimxrt10xx/apps/factory_test/main.c b/ports/mimxrt10xx/apps/factory_test/main.c index b43ada7db..313a79402 100644 --- a/ports/mimxrt10xx/apps/factory_test/main.c +++ b/ports/mimxrt10xx/apps/factory_test/main.c @@ -35,6 +35,7 @@ #include "arduino.h" #include "board_api.h" #include "tusb.h" +#include "wifi_scan.h" void loop(void); @@ -48,7 +49,9 @@ static const float DC_INPUT_MAX_VOLTS = 13.0f; bool testpins(uint8_t a, uint8_t b, uint8_t *allpins, uint8_t num_allpins); void test_print_adc(void); -static void test_board(void); +static bool test_board(void); +static uint32_t next_scan = 5000; +static bool serial_connected = false; /* This is an application to test Metro M7 */ @@ -85,8 +88,29 @@ int main(void) { } } -uint8_t loopcount = 0; +void demo_service(void) { + static uint32_t last_animation = 0; + static uint8_t color = 0; + uint32_t now = millis(); + if (now - last_animation >= 20) { + last_animation = now; + setColor(neoWheel(color++)); + digitalWrite(13, (now % 1000) < 500); + } + fflush(stdout); + tud_task(); + tud_cdc_write_flush(); +} + void loop(void) { + bool connected = tud_cdc_connected(); + if (connected && !serial_connected) { + Serial_printf("\r\nHello from your Metro M7 AirLift!\r\n"); + Serial_printf("Watch the red LED blink and the NeoPixel swirl.\r\n"); + Serial_printf("Nearby Wi-Fi networks are listed every 30 seconds.\r\n"); + next_scan = millis() + 5000; + } + serial_connected = connected; if (tud_cdc_available()) { uint8_t serial_buf[256]; uint32_t count; @@ -98,16 +122,13 @@ void loop(void) { } if (!test) { - setColor(neoWheel(loopcount++)); - - if ((loopcount % 32) == 0) { - digitalWrite(13, HIGH); + demo_service(); + if ((int32_t)(millis() - next_scan) >= 0) { + Serial_printf("\r\nScanning nearby Wi-Fi networks...\r\n"); + int networks = wifi_scan(); + if (networks == 0) Serial_printf("No named networks found. Trying again in 30 seconds.\r\n"); + next_scan = millis() + 30000; } - if ((loopcount % 32) == 16) { - digitalWrite(13, LOW); - } - - delay(10); return; } @@ -119,7 +140,7 @@ void loop(void) { IOMUXC_SetPinConfig(ESP32_RESET_PINMUX, 0x10B0U); GPIO_PinInit(ESP32_RESET_PORT, ESP32_RESET_PIN, &reset_config); delay(100); - test_board(); + bool gpio_passed = test_board(); // Release every tested output before allowing the ESP32 to run, even on failure. for (size_t i = 0; i < sizeof(all_pins); i++) { @@ -129,32 +150,44 @@ void loop(void) { pinMode(pin, INPUT); } GPIO_PinWrite(ESP32_RESET_PORT, ESP32_RESET_PIN, 1); + pinMode(13, OUTPUT); + if (gpio_passed) { + Serial_printf("WiFi scan required for factory pass...\r\n"); + int networks = wifi_scan(); + if (networks > 0) { + Serial_printf("WIFI SCAN OK\r\n"); + Serial_printf("*** TEST OK! ***\r\n"); + } else { + Serial_printf("WiFi FAILED: no valid SSIDs detected.\r\n"); + } + } + next_scan = millis() + 30000; } -static void test_board(void) { +static bool test_board(void) { Serial_printf("\n\r\n\rHello Metro M7 iMX RT1011 Test! %lu\n\r", millis()); if (!testpins(0, 2, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(1, 3, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(4, 6, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(5, 7, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(8, 10, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(9, 11, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(13, PIN_SDA, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(12, PIN_SCL, all_pins, sizeof(all_pins))) - return; + return false; // The SPI loopback is not connected in the Brains fixture. if (!testpins(AD2, AD4, all_pins, sizeof(all_pins))) - return; + return false; if (!testpins(AD3, AD5, all_pins, sizeof(all_pins))) - return; + return false; // test_print_adc(); // Test 5V @@ -162,16 +195,16 @@ static void test_board(void) { Serial_printf("5V out = %d\n\r", (int)five_mV); if (abs(five_mV - 5000) > 500) { Serial_printf("5V power supply reading wrong?"); - return; + return false; } // Accept the fixture's 9V or 12V DC supply, with measurement tolerance. float dc_input_volts = (float)analogRead(AD0) * 11.0f * 3.3f / 4095.0f; Serial_printf("DC input = %d mV\n\r", (int)(dc_input_volts * 1000.0f)); if (dc_input_volts < DC_INPUT_MIN_VOLTS || dc_input_volts > DC_INPUT_MAX_VOLTS) { Serial_printf("DC input power supply reading wrong?"); - return; + return false; } - Serial_printf("*** TEST OK! ***\n\r"); + return true; } bool testpins(uint8_t a, uint8_t b, uint8_t *allpins, uint8_t num_allpins) { @@ -298,5 +331,14 @@ void test_print_adc(void) { // retarget printf to usb cdc __attribute__((used)) int _write(int fhdl, const void *buf, size_t count) { (void)fhdl; - return tud_cdc_write(buf, count); + if (!tud_cdc_connected()) return (int)count; + const uint8_t *bytes = buf; + size_t written = 0; + uint32_t started = millis(); + while (written < count && tud_cdc_connected() && millis() - started < 1000) { + written += tud_cdc_write(bytes + written, count - written); + tud_cdc_write_flush(); + tud_task(); + } + return (int)written; } diff --git a/ports/mimxrt10xx/apps/factory_test/wifi_scan.c b/ports/mimxrt10xx/apps/factory_test/wifi_scan.c new file mode 100644 index 000000000..bf652c2f5 --- /dev/null +++ b/ports/mimxrt10xx/apps/factory_test/wifi_scan.c @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: Copyright (c) 2019 ladyada for Adafruit Industries +// SPDX-License-Identifier: MIT +// NINA framing and handshake follow Adafruit_CircuitPython_ESP32SPI. +// Pin assignments follow CircuitPython's metro_m7_1011 board definition. +#include "arduino.h" +#include "wifi_scan.h" + +#define ESP_CS_PIN 28 +#define ESP_BUSY_PIN 25 +#define SPI_MISO_PIN 17 +#define SPI_MOSI_PIN 18 +#define SPI_SCK_PIN 20 +#define SCAN_BUDGET_MS 20000U +#define MAX_SSIDS 64 +#define MAX_SSID_LENGTH 32 + +static uint32_t scan_start; +static bool timed_out; +static unsigned service_bytes; +static uint8_t names[MAX_SSIDS][MAX_SSID_LENGTH]; +static uint8_t lengths[MAX_SSIDS]; + +static bool within_budget(void) { + demo_service(); + if (millis() - scan_start >= SCAN_BUDGET_MS) { + timed_out = true; + } + return !timed_out; +} + +static bool pause_ms(uint32_t ms) { + uint32_t start = millis(); + while (millis() - start < ms) { + if (!within_budget()) { + return false; + } + } + return within_budget(); +} + +static void deselect(void) { + GPIO_PinWrite(GPIO1, ESP_CS_PIN, 1); +} + +static bool select_esp(void) { + while (GPIO_PinRead(GPIO1, ESP_BUSY_PIN)) { + if (!within_budget()) { + return false; + } + } + GPIO_PinWrite(GPIO1, ESP_CS_PIN, 0); + uint32_t start = millis(); + while (!GPIO_PinRead(GPIO1, ESP_BUSY_PIN)) { + if (!within_budget() || millis() - start >= 1000) { + deselect(); + return false; + } + } + return within_budget(); +} + +// Mode 0, MSB first. Delay each half cycle; actual clock is below 500 kHz. +static uint8_t exchange(uint8_t value) { + if ((service_bytes++ & 15U) == 0 && !within_budget()) { + return 0xff; + } + uint8_t reply = 0; + for (unsigned bit = 0; bit < 8; bit++) { + GPIO_PinWrite(GPIO1, SPI_MOSI_PIN, (value & 0x80) != 0); + value <<= 1; + SDK_DelayAtLeastUs(1, SystemCoreClock); + GPIO_PinWrite(GPIO1, SPI_SCK_PIN, 1); + SDK_DelayAtLeastUs(1, SystemCoreClock); + reply = (uint8_t)((reply << 1) | GPIO_PinRead(GPIO1, SPI_MISO_PIN)); + GPIO_PinWrite(GPIO1, SPI_SCK_PIN, 0); + } + return reply; +} + +static bool send_command(uint8_t command) { + if (!select_esp()) { + return false; + } + exchange(0xe0); + exchange(command); + exchange(0); + exchange(0xee); + deselect(); + return within_budget(); +} + +static bool read_response(uint8_t command, uint8_t *count) { + if (!select_esp()) { + return false; + } + bool started = false; + for (unsigned attempt = 0; attempt < 10; attempt++) { + uint8_t byte = exchange(0xff); + if (byte == 0xe0) { + started = true; + break; + } + if (byte == 0xef || !pause_ms(10)) { + break; + } + } + if (!started || exchange(0xff) != (command | 0x80)) { + deselect(); + return false; + } + *count = exchange(0xff); + if (*count > MAX_SSIDS || (command != 0x27 && *count != 1)) { + deselect(); + return false; + } + for (unsigned i = 0; i < *count; i++) { + lengths[i] = exchange(0xff); + if (lengths[i] > MAX_SSID_LENGTH) { + deselect(); + return false; + } + for (unsigned j = 0; j < lengths[i]; j++) { + names[i][j] = exchange(0xff); + } + } + bool valid = exchange(0xff) == 0xee && !timed_out; + deselect(); + return valid && within_budget(); +} + +static bool command_response(uint8_t command, uint8_t *count) { + return send_command(command) && read_response(command, count); +} + +// No control characters from radio data may affect the fixture's result parser. +static void print_escaped(const uint8_t *data, unsigned length) { + static const char hex[] = "0123456789ABCDEF"; + char escaped[MAX_SSID_LENGTH * 4 + 1]; + unsigned out = 0; + for (unsigned i = 0; i < length; i++) { + uint8_t byte = data[i]; + if (byte >= 0x20 && byte <= 0x7e && byte != '\\') { + escaped[out++] = (char)byte; + } else { + escaped[out++] = '\\'; + escaped[out++] = 'x'; + escaped[out++] = hex[byte >> 4]; + escaped[out++] = hex[byte & 15]; + } + } + escaped[out] = 0; + printf("%s\n", escaped); + fflush(stdout); + demo_service(); +} + +static bool has_name(unsigned index) { + for (unsigned i = 0; i < lengths[index]; i++) { + uint8_t byte = names[index][i]; + if (byte >= 0x20 && byte != 0x7f) { + return true; + } + } + return false; +} + +static bool init_esp(void) { + gpio_pin_config_t output_high = {kGPIO_DigitalOutput, 1, kGPIO_NoIntmode}; + gpio_pin_config_t output_low = {kGPIO_DigitalOutput, 0, kGPIO_NoIntmode}; + gpio_pin_config_t input = {kGPIO_DigitalInput, 0, kGPIO_NoIntmode}; + GPIO_PinInit(GPIO1, ESP_CS_PIN, &output_high); + IOMUXC_SetPinConfig(IOMUXC_GPIO_AD_14_GPIOMUX_IO28, 0x00B0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_AD_14_GPIOMUX_IO28, 0); + GPIO_PinInit(GPIO1, ESP_BUSY_PIN, &input); + IOMUXC_SetPinConfig(IOMUXC_GPIO_AD_11_GPIOMUX_IO25, 0x00B0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_AD_11_GPIOMUX_IO25, 0); + GPIO_PinInit(GPIO1, SPI_MISO_PIN, &input); + IOMUXC_SetPinConfig(IOMUXC_GPIO_AD_03_GPIOMUX_IO17, 0x00B0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_AD_03_GPIOMUX_IO17, 0); + GPIO_PinInit(GPIO1, SPI_MOSI_PIN, &output_low); + IOMUXC_SetPinConfig(IOMUXC_GPIO_AD_04_GPIOMUX_IO18, 0x00B0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_AD_04_GPIOMUX_IO18, 0); + GPIO_PinInit(GPIO1, SPI_SCK_PIN, &output_low); + IOMUXC_SetPinConfig(IOMUXC_GPIO_AD_06_GPIOMUX_IO20, 0x00B0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_AD_06_GPIOMUX_IO20, 0); + GPIO_PinInit(ESP32_GPIO0_PORT, ESP32_GPIO0_PIN, &output_high); + IOMUXC_SetPinConfig(ESP32_GPIO0_PINMUX, 0x00B0U); + IOMUXC_SetPinMux(ESP32_GPIO0_PINMUX, 0); + GPIO_PinInit(ESP32_RESET_PORT, ESP32_RESET_PIN, &output_low); + IOMUXC_SetPinConfig(ESP32_RESET_PINMUX, 0x00B0U); + IOMUXC_SetPinMux(ESP32_RESET_PINMUX, 0); + bool ready = pause_ms(10); + GPIO_PinWrite(ESP32_RESET_PORT, ESP32_RESET_PIN, 1); + ready = ready && pause_ms(750); + GPIO_PinInit(ESP32_GPIO0_PORT, ESP32_GPIO0_PIN, &input); + return ready; +} + +int wifi_scan(void) { + scan_start = millis(); + timed_out = false; + service_bytes = 0; + uint8_t count; + if (!init_esp() || !command_response(0x37, &count) || !lengths[0]) { + printf("WiFi FAILED: firmware response or timeout\n"); + goto fail; + } + printf("WiFi firmware: "); + print_escaped(names[0], lengths[0]); + if (!command_response(0x36, &count) || lengths[0] != 1 || names[0][0] != 1) { + printf("WiFi FAILED: start scan response or timeout\n"); + goto fail; + } + // Leave time for the radio scan; all waits share the same absolute deadline. + while (millis() - scan_start < SCAN_BUDGET_MS - 2000U) { + if (!pause_ms(2000) || !command_response(0x27, &count)) { + printf("WiFi FAILED: scan response or timeout\n"); + goto fail; + } + int valid = 0; + for (unsigned i = 0; i < count; i++) { + if (has_name(i)) { + printf("SSID: "); + print_escaped(names[i], lengths[i]); + valid++; + } + } + if (valid) { + return valid; + } + } + printf("WiFi FAILED: no visible SSIDs\n"); + fflush(stdout); + demo_service(); + return 0; + +fail: + deselect(); + fflush(stdout); + demo_service(); + return -1; +} diff --git a/ports/mimxrt10xx/apps/factory_test/wifi_scan.h b/ports/mimxrt10xx/apps/factory_test/wifi_scan.h new file mode 100644 index 000000000..67bc58cf1 --- /dev/null +++ b/ports/mimxrt10xx/apps/factory_test/wifi_scan.h @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: MIT +#ifndef WIFI_SCAN_H_ +#define WIFI_SCAN_H_ + +// Return valid SSID count, zero for none, or -1 for a protocol/timeout error. +int wifi_scan(void); + +// Provided by main.c; services USB and the idle indicators without starting a scan. +void demo_service(void); + +#endif From f5de380accd25132a67812dcf24108da0a2e085d Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Sun, 11 Oct 2026 15:36:53 +0700 Subject: [PATCH 14/47] Add agent instructions and a build contract (#488) Out-of-tree builds (make BUILD_ROOT, cmake/IDF ARTIFACT_PATH), the check_build.py build contract with tests, and CLAUDE.md for agentrc agents. --- .agents | 1 + .claude/skills/build/SKILL.md | 45 ++ .claude/skills/build/scripts/check_build.py | 695 ++++++++++++++++++ .claude/test/test_build.py | 497 +++++++++++++ .github/workflows/pre_commit.yml | 33 + .gitignore | 6 +- .pre-commit-config.yaml | 9 + AGENTS.md | 1 + CLAUDE.md | 35 + ports/espressif/CMakeLists.txt | 11 +- ports/espressif/apps/blinky/CMakeLists.txt | 1 + .../apps/update_tinyuf2/CMakeLists.txt | 1 + .../apps/update_tinyuf2/main/CMakeLists.txt | 4 +- ports/family_support.cmake | 6 +- ports/lpc55/Makefile | 5 +- ports/lpc55/apps/self_update/Makefile | 2 +- ports/make.mk | 13 +- ports/rules.mk | 2 +- ports/stm32f3/Makefile | 5 +- ports/stm32f3/apps/self_update/Makefile | 2 +- ports/stm32f4/Makefile | 5 +- ports/stm32f4/apps/self_update/Makefile | 2 +- ports/stm32h5/Makefile | 5 +- ports/stm32h5/apps/self_update/Makefile | 2 +- 24 files changed, 1363 insertions(+), 25 deletions(-) create mode 120000 .agents create mode 100644 .claude/skills/build/SKILL.md create mode 100755 .claude/skills/build/scripts/check_build.py create mode 100644 .claude/test/test_build.py create mode 100644 .github/workflows/pre_commit.yml create mode 120000 AGENTS.md create mode 100644 CLAUDE.md diff --git a/.agents b/.agents new file mode 120000 index 000000000..c8161850a --- /dev/null +++ b/.agents @@ -0,0 +1 @@ +.claude \ No newline at end of file diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md new file mode 100644 index 000000000..37edb86ba --- /dev/null +++ b/.claude/skills/build/SKILL.md @@ -0,0 +1,45 @@ +--- +name: build +description: The TinyUF2 build contract. Use when a change must be verified by building, when an agent needs the boards a diff affects, or when a workflow asks for "the project's build". Resolves scope to boards, builds each the way CI does in a private dir, runs the ghostfat self-test when its inputs change, reports pass/fail with the first error. +--- + +# Build contract + +`scripts/check_build.py` is the entry point for verifying a change; every agent and workflow that does so calls it rather than composing make, cmake or idf.py lines. A debug build for a probe session is not verification and uses the port's own build directly (below). + +```bash +B=.claude/skills/build/scripts/check_build.py +python3 $B --scope # boards a change affects +python3 $B --base master # same, from the branch diff +python3 $B --base master --worktree # same, plus uncommitted and untracked changes +python3 $B --select-only (--scope ... | --base REF [--worktree] | --diff-file FILE) # the selection alone +python3 $B --board feather_stm32f405_express [--board ...] +python3 $B --board lpcxpresso55s69 -D LOG=2 # a define, passed to the board's backend +python3 $B ... --shared # keep cmake-build/cmake-build-, the dir a flash or HIL run uses +python3 $B ... --idf local # exported ESP-IDF instead of CI's docker image +``` + +The last stdout line is JSON. An agent acts on `pass`; `firstError`, and `error` on exit 2, naming the cause or the remedy; `boards[]` (`status` `ok`, `failed` or `error`, `log`, `artifacts`, `notImplemented`); `built`, the artifacts of `--shared` roots, the only ones kept after a pass; `nothingToBuild` and `uncovered`, each path with its reason; and `idf.matchesCi`. `paths` gives every changed path's `rule` and `reason`, the answer to why a path selected what it did. Exit 0 pass, 1 a build, self-test or test failed, 2 usage, resolution or a missing prerequisite, 3 an uncovered path; 2 outranks 1, and 1 outranks 3. `-v` streams the build to stderr; each build's log is `/check_build.log`. + +`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". + +## Reporting + +An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the JSON this way: `buildOk` is `pass`. A pass with no board, self-test or test run is "nothing to build": quote each `nothingToBuild` reason in `notes`, so nobody mistakes it for a build that ran. Exit 3 is `buildOk` false with each `uncovered` reason quoted verbatim: that path is firmware no CI build compiles, and a human adds coverage or accepts the gap. Exit 1 or 2 quotes `firstError` (and `error`). A pass on `--idf local` whose `idf.matchesCi` is false is supplemental evidence only: say so in `notes`; acceptance needs CI's IDF. + +## Judgment + +- **Selection is the script's**, first match per path, and `paths[].reason` says why. A mixed change takes the union, and an uncovered path fails it whatever else built. The policy behind the rules: + - A change to a port, board, app or dependency builds the boards that compile it: a board's own files that board; other port files the port's `REPRESENTATIVES` (rig boards first, both ESP32-S2 and -S3, both i.MX RT10xx and RT1170); an app or dependency the representatives of the ports whose CI backend compiles it. + - Shared code (`src/`, `lib/`, the shared build files) and anything unclassified builds every port's representatives, plus the ghostfat self-test when a ghostfat input changed. + - A change to how boards build (this script, `build.yml`, `build_util.yml`, `ci_set_matrix.py`, `.github/actions/`) runs the contract's own tests and one smoke build per backend; no firmware build verifies workflow syntax or orchestration. A change to the contract's tests runs the tests only. + - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads. + - Firmware no CI build compiles is uncovered: `ports/template_port/`, `ports/ch32v20x/` (`--board ch32v203_r0_1v0` builds it on request, with a note), a build file the port's CI backend never reads, a stub app. + - Docs and agent, editor and repo metadata build nothing. +- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`; Espressif `idf.py build`. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds. A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. +- **ESP-IDF** builds in CI's image, `espressif/idf:`, through docker by default; a missing image is exit 2 naming the `docker pull`. `--idf local` uses the exported IDF and records its version and `matchesCi`: evidence from a different IDF is supplemental, not acceptance. +- **ghostfat** builds each `ports/test_ghostfat/boards/*` config natively and runs it against its known-good image in the private dir, one config at a time: `huge` needs about 3 GiB of disk while it runs. A passing config's dir is removed; a failing one keeps its log. +- **Private dirs for parallel agents.** Each build starts from an empty `cmake-build/cmake-build-agent--` (with `-` when a board builds with another backend than its port's) and is removed when it passes (`removed`; its `artifacts` say what it produced, nothing is left to flash); a failed one is kept for its log. Nothing is written into the source tree: make takes `BUILD_ROOT=`, cmake and idf `-DARTIFACT_PATH=/_bin`. `--shared` rebuilds `cmake-build/cmake-build-` from clean and keeps it for flashing; it needs exclusive ownership of that board. +- **Dependencies are never fetched here.** `lib/tinyusb` and `lib/uf2` are submodules (`git submodule update --init lib/tinyusb lib/uf2`); a port's `tools/get_deps.py` rows must exist. In a worktree, symlink them from the primary checkout (CLAUDE.md), never run `get_deps.py` through a symlink: it runs `git reset --hard` in each dependency. +- **Debug builds** for a probe session use the port's backend directly: make `make -C ports/ BOARD= DEBUG=1 LOG=2 LOGGER=rtt all` (`LOGGER=swo` too); cmake `-DCMAKE_BUILD_TYPE=Debug -DLOG=2 -DLOGGER=RTT` (only `RTT` adds the RTT sources). Flash targets: make `flash-jlink`, `flash-pyocd`, `flash-stlink`, `flash-uf2`; cmake `tinyuf2-jlink`, `-stlink`, `-openocd`; Espressif `idf.py -C ports/espressif -DBOARD= flash`. +- **Printing a command is not verification.** A verdict is the JSON of a run that happened; an agent that cannot run the script reports the build as not run. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py new file mode 100755 index 000000000..f1020e55a --- /dev/null +++ b/.claude/skills/build/scripts/check_build.py @@ -0,0 +1,695 @@ +#!/usr/bin/env python3 +"""Build TinyUF2 for the boards a change affects, or for named boards: the project's build contract. + + check_build.py (--scope PATH... | --base REF [--worktree] | --board B...) [-D VAR=VAL]... [--shared] [--idf docker|local] [-v] + check_build.py --select-only (--scope PATH... | --base REF [--worktree] | --diff-file FILE) + +Each board builds the way .github/workflows/build.yml builds its port (make, cmake or ESP-IDF) +into its own build root, cmake-build/cmake-build-agent-- (--shared: +cmake-build/cmake-build-), and must produce its expected artifacts. A ghostfat input +also builds and runs the native ghostfat self-test of every ports/test_ghostfat config. + +stdout ends with one JSON line (SKILL.md). Exit 0 pass, 1 a build or test failed, 2 usage, +resolution or a missing prerequisite ("error" names the remedy), 3 uncovered paths. +""" + +import argparse +import functools +import gzip +import json +import os +import re +import runpy +import shlex +import shutil +import subprocess +import sys +import zlib +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +BUILD_YML = ROOT / '.github' / 'workflows' / 'build.yml' +BUILD_DIR = ROOT / 'cmake-build' + +# Backend of every port CI builds, as build.yml's jobs build it; test_build.py holds the two equal. +BACKENDS = { + 'espressif': 'idf', + 'kinetis_k32l2': 'make', + 'lpc55': 'make', + 'maxim': 'cmake', + 'mimxrt10xx': 'make', + 'stm32f3': 'make', + 'stm32f4': 'cmake', + 'stm32h5': 'cmake', + 'stm32h7': 'make', + 'stm32l4': 'make', +} +# Boards that stand in for their port: rig boards first; espressif covers both IDF targets, +# mimxrt10xx both the RT10xx and RT1170 boot layouts. +REPRESENTATIVES = { + 'espressif': ['espressif_esp32s3_devkitm_1', 'espressif_esp32s2_devkitc_1'], + 'kinetis_k32l2': ['frdm_k32l2b'], + 'lpc55': ['lpcxpresso55s69'], + 'maxim': ['max32666fthr'], + 'mimxrt10xx': ['imxrt1064_evk', 'imxrt1170_evk'], + 'stm32f3': ['stm32f303disco'], + 'stm32f4': ['feather_stm32f405_express'], + 'stm32h5': ['stm32h563_nucleo'], + 'stm32h7': ['daisy_seed'], + 'stm32l4': ['swan_r5'], +} +# Ports no CI job builds: a change there is uncovered; --board may still build one explicitly. +UNBUILT = { + 'ch32v20x': ('make', 'its riscv job is disabled in build.yml'), + 'template_port': (None, 'a template no build compiles'), +} +# compiler a port's make/cmake build needs, when not arm-none-eabi-gcc +COMPILER = {'ch32v20x': 'riscv-none-elf-gcc'} +# make ports whose self-update target only echoes "not implemented yet" +NO_SELF_UPDATE = {'ch32v20x', 'kinetis_k32l2', 'stm32h7', 'stm32l4'} +GHOSTFAT = 'test_ghostfat' +GHOSTFAT_INPUTS = ('src/ghostfat.c', 'src/favicon/', 'src/board_api.h', 'src/compile_date.h', 'src/uf2.h', + 'lib/uf2', 'ports/make.mk', 'ports/rules.mk', 'ports/test_ghostfat/', + '.github/workflows/build_ghostfat.yml') +CORE = ('src/', 'lib/', 'ports/make.mk', 'ports/rules.mk', 'ports/family_support.cmake', 'CMakeLists.txt', + 'cmake/', 'tools/get_deps.py') +CONTRACT_TESTS = '.claude/test/test_build' +BUILD_CONTROL = ('.claude/skills/build/scripts/', '.github/workflows/build.yml', + '.github/workflows/build_util.yml', '.github/workflows/ci_set_matrix.py', '.github/actions/') +NOTHING = ('.claude/', '.github/', '.idea/', 'tools/codespell/', 'tools/gen_boards.py', 'tools/touch1200.py', + 'ports/build_all.py', 'CLAUDE.md', 'AGENTS.md', '.agents', 'LICENSE', '.gitignore', '.gitmodules', + '.pre-commit-config.yaml', '.codespellrc', '.clang-format') +CODEQL = '.github/workflows/codeql-buildscript.sh' +# one board per backend for a change to how boards build; lpc55 also runs make's self-update and apps +SMOKE = {'make': 'lpcxpresso55s69', 'cmake': 'feather_stm32f405_express', 'idf': 'espressif_esp32s3_devkitm_1'} + + +class Fail(Exception): + pass + + +def git(*args): + r = subprocess.run(['git', *args], capture_output=True, text=True, cwd=ROOT) + if r.returncode != 0: + raise Fail(f'git {" ".join(args)} failed: {r.stderr.strip()}') + return r.stdout + + +def ports_dir(port): + return ROOT / 'ports' / port + + +def boards_of(port): + d = ports_dir(port) / 'boards' + return sorted(p.name for p in d.iterdir() if p.is_dir()) if d.is_dir() else [] + + +def port_of(board): + hits = sorted(p.parent.parent.name for p in (ROOT / 'ports').glob(f'*/boards/{board}') if p.is_dir()) + hits = [h for h in hits if h != GHOSTFAT] + if len(hits) != 1: + raise Fail(f'board {board!r} matches {hits or "no"} port') + return hits[0] + + +def ci_idf_version(): + m = re.search(r"toolchain:\s*'esp-idf'\s*\n\s*toolchain_version:\s*'([^']+)'", BUILD_YML.read_text()) + if not m: + raise Fail(f'no esp-idf toolchain_version in {BUILD_YML.relative_to(ROOT)}') + return m.group(1) + + +def app_boards(app_dir): + """Boards an app's Makefile builds for: None for every board, [] for a stub that builds + nothing (no make.mk or app.mk include, only an echo).""" + mk = app_dir / 'Makefile' + if not mk.is_file(): + return None + text = mk.read_text() + if not re.search(r'^\s*include\s+\S*(make|app)\.mk', text, re.M): + return [] + return re.findall(r'ifeq\s*\(\$\(BOARD\),\s*(\w+)\)', text) or None + + +@functools.cache +def codeql_build(): + """(port, board) codeql-buildscript.sh builds with make `all` on every pull request.""" + text = (ROOT / CODEQL).read_text() + ports = set(re.findall(r'(?:cd|-C)\s+ports/(\w+)', text)) + boards = set(re.findall(r'^\s*make\b[^\n]*\bBOARD=(\w+)[^\n]*\ball\b', text, re.M)) + if len(ports) != 1 or len(boards) != 1: + raise Fail(f'cannot read one port and one `make ... all` board from {CODEQL}: update codeql_build()') + return ports.pop(), boards.pop() + + +def build_file_kind(name): + """'make' for a Makefile or *.mk, 'cmake' for a CMakeLists.txt or *.cmake, else None.""" + if name == 'Makefile' or name.endswith('.mk'): + return 'make' + if name == 'CMakeLists.txt' or name.endswith('.cmake'): + return 'cmake' + return None + + +def read_by_backend(path, backend): + """Whether CI's backend for the port reads this build file: a make port never reads CMake + files, a cmake or idf port never reads Makefiles. Sources and headers are read by both.""" + kind = build_file_kind(path.rsplit('/', 1)[-1]) + return kind is None or (kind == 'make') == (backend == 'make') + + +def code_lines(f): + return '\n'.join(l for l in f.read_text(errors='replace').splitlines() if not l.lstrip().startswith('#')) + + +def espressif_apps(): + """Apps ports/espressif/CMakeLists.txt builds as ExternalProjects.""" + return re.findall(r'SOURCE_DIR\s+\$\{CMAKE_CURRENT_LIST_DIR\}/apps/(\w+)', code_lines(ports_dir('espressif') / 'CMakeLists.txt')) + + +def port_app_selection(port, path): + """Boards that compile a change under ports//apps/: the app's own enabled boards, or for + a file shared by the port's apps (app.mk) those of every app that includes it.""" + parts = path.split('/') + apps_dir = ports_dir(port) / 'apps' + if BACKENDS[port] == 'idf': + if len(parts) >= 4 and parts[3] not in espressif_apps(): + return None, f'ports/espressif/CMakeLists.txt builds no ExternalProject for apps/{parts[3]}' + return rep_selection([port], f'builds {"/".join(parts[2:4])}'), f'{port} {"/".join(parts[2:4])}' + if len(parts) >= 5 and (apps_dir / parts[3]).is_dir(): + apps = [apps_dir / parts[3]] + else: + apps = [m.parent for m in apps_dir.glob('*/Makefile') if re.search(rf'^\s*include\s+\S*\b{re.escape(parts[-1])}\b', + m.read_text(), re.M)] + boards = {} + for app in apps: + enabled = app_boards(app) + if enabled is None: + boards.update(rep_selection([port], f'builds apps/{app.name}')) + for b in enabled or []: + boards[(b, BACKENDS[port])] = f'enables apps/{app.name}' + if not boards: + return None, f'no {port} board builds an app that reads {path}' + return boards, f'{port} apps: {", ".join(sorted(a.name for a in apps))}' + + +def ports_using_app(app): + """CI-built ports whose build, in the backend CI uses, compiles sources of the top-level apps/.""" + hits = [] + for port, backend in sorted(BACKENDS.items()): + kind = 'make' if backend == 'make' else 'cmake' + ref = re.compile(rf'(?= 2 and parts[0] == 'ports' and parts[1] in UNBUILT: + return out('unbuilt-port', 'gap', f'ports/{parts[1]}: {UNBUILT[parts[1]][1]}') + if len(parts) >= 3 and parts[0] == 'ports' and parts[1] in BACKENDS: + port = parts[1] + if not read_by_backend(path, BACKENDS[port]): + if port == codeql_port and parts[2] != 'apps' and (parts[2] != 'boards' or parts[3:4] == [codeql_board]): + return out('codeql', 'select', f'CodeQL builds {codeql_board} with make on every pull request', + {(codeql_board, 'make'): f'reads {path}'}) + return out('other-backend', 'gap', f'CI builds {port} with {BACKENDS[port]}, which never reads {parts[-1]}') + if parts[2] == 'boards' and len(parts) >= 4 and (ports_dir(port) / 'boards' / parts[3]).is_dir(): + return out('board', 'select', f'board {parts[3]} changed', + {(parts[3], BACKENDS[port]): f'{path} is its own'}, ghostfat) + if parts[2] == 'apps': + boards, reason = port_app_selection(port, path) + return out('port-app', 'select' if boards else 'gap', reason, boards) + return out('port', 'select', f'{port} port file', rep_selection([port], f'{port} representative'), ghostfat) + if len(parts) >= 2 and parts[0] == 'ports' and len(parts) >= 3 and (ROOT / 'ports' / parts[1]).is_dir(): + raise Fail(f'ports/{parts[1]} has no backend in check_build.py: add it to BACKENDS or UNBUILT') + if parts[0] == 'apps' and len(parts) >= 2: + users = ports_using_app(parts[1]) + if not users: + return out('app', 'gap', f'no CI-built port builds apps/{parts[1]}') + return out('app', 'select', f'apps/{parts[1]} is built by {", ".join(users)}', + rep_selection(users, f'builds apps/{parts[1]}')) + if path.startswith(('lib/mcu/', 'lib/sct_neopixel')): + families = deps_families(path) + if families is not None: + if not families: + return out('dep', 'gap', f'{path}: no CI-built port uses this dependency') + return out('dep', 'select', f'dependency of {", ".join(families)}', rep_selection(families, f'uses {path}')) + if path.startswith(NOTHING): + return out('nothing', 'none', 'no firmware build reads this path') + if path.startswith(CORE): + return out('core', 'core', 'shared by every port', ghostfat=ghostfat) + return out('unclassified', 'core', 'unclassified path: every port', ghostfat=ghostfat) + + +def repo_path(p): + """A path as git spells it, relative to the checkout: ./src/x.c and /abs/.../src/x.c are src/x.c.""" + # lexical, not resolve(): a worktree's dependency symlinks point into the primary checkout + p = os.path.relpath(os.path.abspath(p), ROOT) if os.path.isabs(p) else os.path.normpath(p) + if p.startswith('..'): + raise Fail(f'{p} is outside the checkout') + return p + + +def changed_paths(base=None, worktree=False, scope=None, diff_file=None): + if diff_file: + try: + lines = Path(diff_file).read_text().splitlines() + except OSError as e: + raise Fail(f'cannot read --diff-file: {e}') + return [repo_path(l.strip()) for l in lines if l.strip()], {'mode': 'diff-file'} + if scope: + paths = [] + for p in scope: + p = repo_path(p) + if (ROOT / p).is_dir() and not (ROOT / p / '.git').exists(): + files = [f for f in git('ls-files', '-z', '--cached', '--others', '--exclude-standard', '--', p).split('\0') if f] + if not files: + raise Fail(f'{p} is a directory with no files git reports') + paths += files + elif (ROOT / p).exists() or subprocess.run(['git', 'cat-file', '-e', f'HEAD:{p}'], cwd=ROOT, + capture_output=True).returncode == 0: + paths.append(p) + else: + raise Fail(f'{p} does not exist and is not tracked at HEAD') + return paths, {'mode': 'paths'} + merge_base = git('merge-base', base, 'HEAD').strip() + paths = git('diff', '--name-only', '--no-renames', merge_base, 'HEAD').split() + info = {'mode': 'worktree' if worktree else 'base', 'base': base, 'merge_base': merge_base, + 'head': git('rev-parse', 'HEAD').strip()} + if worktree: + paths += git('diff', '--name-only', '--no-renames', 'HEAD').split() + paths += git('ls-files', '--others', '--exclude-standard').split() + return sorted(set(paths)), info + + +def identity(): + """What a result was built from: HEAD, whether the tree differs from it, the submodule commits.""" + subs = {} + for line in git('submodule', 'status').splitlines(): + sha, path = line[1:].split()[:2] + subs[path] = sha + return {'head': git('rev-parse', 'HEAD').strip(), 'dirty': bool(git('status', '--porcelain').strip()), + 'submodules': subs} + + +def select(paths): + records = [classify(p) for p in paths] + full = any(r['effect'] == 'core' for r in records) + boards = rep_selection(sorted(REPRESENTATIVES), 'core: shared by every port') if full else {} + for r in records: + for k, why in r['boards'].items(): + boards.setdefault(k, why) + return { + 'full': full, + 'boards': boards, + 'ghostfat': any(r['ghostfat'] for r in records), + 'tests': any(r['tests'] for r in records), + 'records': records, + } + + +def path_view(records, effect=None, keys=('path', 'rule', 'effect', 'reason')): + return [{k: r[k] for k in keys} for r in records if effect in (None, r['effect'])] + + +def manifest(sel, info): + ports = {} + for (b, backend) in sel['boards']: + ports.setdefault(port_of(b), []).append(b) + return { + 'version': 1, + 'input': info, + 'build': { + 'full': sel['full'], + 'needed': bool(sel['boards'] or sel['ghostfat'] or sel['tests']), + 'ports': {p: sorted(set(v)) for p, v in sorted(ports.items())}, + 'builds': [{'board': b, 'backend': backend} for (b, backend) in sel['boards']], + 'ghostfat': sel['ghostfat'], + 'tests': sel['tests'], + 'paths': path_view(sel['records']), + }, + } + + +# ------------------------------------------------------------------ building + +def first_error(log): + lines = log.splitlines() + # a compiler or linker diagnostic names the cause; ninja's FAILED line only the object + for pattern in (r':\d+(:\d+)?: (fatal )?error:|undefined reference|region .* overflowed|multiple definition', + r'\berror\b|Error \d|FAILED|No rule to make|not found'): + for line in lines: + if re.search(pattern, line, re.I) and 'warning' not in line: + return line.strip()[:400] + tail = [l for l in lines if l.strip()] + return tail[-1].strip()[:400] if tail else '' + + +def run(cmd, log, verbose, cwd=ROOT): + with open(log, 'a') as f: + f.write('$ ' + ' '.join(map(str, cmd)) + '\n') + f.flush() + p = subprocess.Popen([str(c) for c in cmd], cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True, errors='replace') + for line in p.stdout: + f.write(line) + if verbose: + sys.stderr.write(line) + return p.wait() + + +def check_deps(port): + """Remedy for a missing submodule or family dependency, None when all are present.""" + missing = [s for s in ('lib/tinyusb', 'lib/uf2') if not (ROOT / s).is_dir() or not any((ROOT / s).iterdir())] + if missing: + return f'empty submodule {", ".join(missing)}: git submodule update --init {" ".join(missing)}' + for dep, families in deps_rows().items(): + if port in families: + d = ROOT / dep + if not d.is_dir() or not any(d.iterdir()): + return (f'{dep} missing: in a worktree symlink it from the primary checkout (CLAUDE.md), ' + f'else python3 tools/get_deps.py {port}') + return None + + +def check_tools(port, backend, idf_mode): + cc = COMPILER.get(port, 'arm-none-eabi-gcc') + need = {'make': ['make', cc], 'cmake': ['cmake', 'ninja', cc], + 'idf': ['docker'] if idf_mode == 'docker' else ['idf.py']}[backend] + missing = [t for t in need if not shutil.which(t)] + if missing: + return f'{", ".join(missing)} not on PATH' + (' (export ESP-IDF first)' if 'idf.py' in missing else '') + return None + + +def expected_artifacts(port, board, backend, root): + bin_dir = root / '_bin' + if backend == 'make': + exp = [bin_dir / f'tinyuf2-{board}.bin', bin_dir / f'tinyuf2-{board}.hex'] + if port not in NO_SELF_UPDATE: + exp.append(bin_dir / 'apps' / f'update-tinyuf2-{board}.uf2') + return exp + if backend == 'cmake': + exp = [bin_dir / 'tinyuf2.bin', bin_dir / 'tinyuf2.hex'] + cml = (ports_dir(port) / 'CMakeLists.txt').read_text() + for app in re.findall(r'add_subdirectory\(\$\{TOP\}/apps/(\w+)', cml): + exp.append(bin_dir / 'apps' / f'{"update-tinyuf2" if app == "self_update" else app}.uf2') + return exp + return [bin_dir / f for f in ('tinyuf2.bin', 'bootloader.bin', 'partition-table.bin', 'combined.bin', + 'apps/update-tinyuf2.uf2', 'apps/blinky.uf2')] + + +def build_make(port, board, root, defines, log, verbose, rec): + base = ['make', '-C', ports_dir(port), f'BOARD={board}', f'BUILD_ROOT={root}', *defines] + # copy-artifact runs after self-update, never beside it under -j + if run(base + [f'-j{os.cpu_count()}', 'all', 'self-update'], log, verbose) or run(base + ['copy-artifact'], log, verbose): + return False, [] + apps = [] + for app in sorted((ports_dir(port) / 'apps').glob('*/Makefile')): + if app.parent.name == 'self_update': + continue + enabled = app_boards(app.parent) + if run(['make', '-C', app.parent, f'BOARD={board}', f'BUILD_ROOT={root}', f'-j{os.cpu_count()}', *defines, + 'all'], log, verbose): + return False, apps + if enabled is None or board in enabled: + apps.append(root / 'ports' / port / 'apps' / app.parent.name) + elif enabled == []: + rec.setdefault('notImplemented', []).append(f'apps/{app.parent.name}') + return True, apps + + +def build_cmake(port, board, root, defines, log, verbose): + cfg = ['cmake', '-B', root, '-G', 'Ninja', '-DCMAKE_BUILD_TYPE=MinSizeRel', f'-DBOARD={board}', + f'-DARTIFACT_PATH={root / "_bin"}', *[f'-D{d}' for d in defines], ports_dir(port)] + return not (run(cfg, log, verbose) or run(['cmake', '--build', root], log, verbose)), [] + + +def build_idf(board, root, defines, log, verbose, idf_mode, image): + idf = ['idf.py', '-C', 'ports/espressif', '-B', str(root), f'-DBOARD={board}', + f'-DARTIFACT_PATH={root / "_bin"}', *[f'-D{d}' for d in defines], 'build'] + if idf_mode == 'local': + return not run(idf, log, verbose), [] + common = Path(git('rev-parse', '--path-format=absolute', '--git-common-dir').strip()) + mounts = {ROOT, common, root.parent} + cmd = ['docker', 'run', '--rm', '--user', f'{os.getuid()}:{os.getgid()}', '-e', 'HOME=/tmp', '-w', ROOT] + for m in sorted(mounts): + cmd += ['-v', f'{m}:{m}'] + inner = "git config --global --add safe.directory '*' && " + shlex.join(idf) + return not run(cmd + [image, 'bash', '-c', inner], log, verbose), [] + + +def backend_of(port): + """The backend CI builds a port with, or an unbuilt port's on-request backend (None for none).""" + if port not in BACKENDS and port not in UNBUILT: + raise Fail(f'ports/{port} has no backend in check_build.py: add it to BACKENDS or UNBUILT') + return BACKENDS.get(port) or UNBUILT[port][0] + + +def build_board(board, backend, args, idf): + port = port_of(board) + # a board built with another backend than its port's (codeql's make) gets its own root + name = board if backend == backend_of(port) else f'{board}-{backend}' + root = BUILD_DIR / (f'cmake-build-{name}' if args.shared else f'cmake-build-agent-{os.getpid()}-{name}') + rec = {'board': board, 'port': port, 'backend': backend, 'buildDir': str(root.relative_to(ROOT)), + 'status': 'ok', 'firstError': '', 'artifacts': []} + if port in UNBUILT: + rec['note'] = f'ports/{port}: {UNBUILT[port][1]}' + if port in NO_SELF_UPDATE: + rec['notImplemented'] = ['self-update'] + problem = check_tools(port, backend, args.idf) or check_deps(port) + if backend == 'idf' and args.idf == 'docker' and not problem: + if subprocess.run(['docker', 'image', 'inspect', idf['image']], capture_output=True).returncode: + problem = f'docker image {idf["image"]} missing: docker pull {idf["image"]}' + if problem: + rec.update(status='error', firstError=problem) + return rec + # always from clean: make cannot see a changed define or flag, so a reused root could pass stale firmware + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True) + log = root / 'check_build.log' + try: + if backend == 'make': + ok, apps = build_make(port, board, root, args.define, log, args.verbose, rec) + elif backend == 'cmake': + ok, apps = build_cmake(port, board, root, args.define, log, args.verbose) + else: + ok, apps = build_idf(board, root, args.define, log, args.verbose, args.idf, idf['image']) + except OSError as e: + rec.update(status='error', firstError=f'{type(e).__name__}: {e}', log=str(log.relative_to(ROOT))) + return rec + if not ok: + rec.update(status='failed', firstError=first_error(log.read_text(errors='replace')), log=str(log.relative_to(ROOT))) + return rec + missing = [p for p in expected_artifacts(port, board, backend, root) if not p.is_file()] + missing += [a for a in apps if not list(a.glob('*.elf'))] + if missing: + rec.update(status='failed', log=str(log.relative_to(ROOT)), firstError='missing expected artifact(s): ' + + ', '.join(str(p.relative_to(root)) for p in missing)) + return rec + rec['artifacts'] = sorted(str(p.relative_to(root)) for p in (root / '_bin').rglob('*') if p.is_file()) + rec['artifacts'] += sorted(str(e.relative_to(root)) for a in apps for e in a.glob('*.elf')) + if not args.shared: + shutil.rmtree(root) + rec['removed'] = True + return rec + + +def keep_only_log(root, log): + for p in root.iterdir(): + if p != log: + shutil.rmtree(p) if p.is_dir() else p.unlink() + + +def run_ghostfat(verbose): + """Build and run the native self-test of every ports/test_ghostfat config, one at a time: + the `huge` image is about 1.5 GiB, twice over with its known-good copy.""" + missing = [t for t in ('make', 'gcc') if not shutil.which(t)] + if missing: + return [{'config': cfg, 'status': 'error', 'firstError': f'{", ".join(missing)} not on PATH'} + for cfg in boards_of(GHOSTFAT)] + results = [] + for cfg in boards_of(GHOSTFAT): + root = BUILD_DIR / f'cmake-build-agent-{os.getpid()}-ghostfat-{cfg}' + shutil.rmtree(root, ignore_errors=True) + root.mkdir(parents=True) + log = root / 'check_build.log' + rec = {'config': cfg, 'status': 'ok', 'firstError': ''} + try: + if run(['make', '-C', ports_dir(GHOSTFAT), f'BOARD={cfg}', f'BUILD_ROOT={root}', f'-j{os.cpu_count()}', 'all'], + log, verbose): + rec.update(status='failed', firstError=first_error(log.read_text(errors='replace'))) + else: + work = root / 'ports' / GHOSTFAT + with gzip.open(ports_dir(GHOSTFAT) / 'boards' / cfg / 'knowngood.img.gz.gz') as outer, \ + gzip.open(outer) as inner, open(work / 'knowngood.img', 'wb') as dst: + shutil.copyfileobj(inner, dst, 1 << 20) + # the self-test writes ghostfat.img and reads knowngood.img in its working directory + if run([work / f'tinyuf2-{cfg}.elf'], log, verbose, cwd=work): + out = log.read_text(errors='replace') + rec.update(status='failed', firstError=(re.findall(r'^FAIL.*$', out, re.M) or [first_error(out)])[-1]) + except (OSError, EOFError, zlib.error) as e: + rec.update(status='error', firstError=f'{type(e).__name__}: {e}') + finally: + if rec['status'] == 'ok': + shutil.rmtree(root, ignore_errors=True) + else: + keep_only_log(root, log) + rec['log'] = str(log.relative_to(ROOT)) + results.append(rec) + return results + + +def run_tests(): + """The contract's own tests, for a change to how boards build.""" + r = subprocess.run([sys.executable, '-m', 'unittest', 'discover', '-s', '.claude/test', '-p', 'test_build*.py'], + capture_output=True, text=True, cwd=ROOT) + last = [l for l in r.stderr.splitlines() if l.strip()] + return {'status': 'ok' if r.returncode == 0 else 'failed', 'summary': last[-1] if last else '', + 'firstError': '' if r.returncode == 0 else first_error(r.stderr)} + + +class Parser(argparse.ArgumentParser): + def error(self, message): + print(json.dumps({'pass': False, 'error': f'usage: {message}'})) + sys.exit(2) + + +def main(argv=None): + ap = Parser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument('--scope', nargs='+', metavar='PATH', help='changed paths or directories') + src.add_argument('--base', metavar='REF', help='the branch diff against REF') + src.add_argument('--board', action='append', metavar='B', help='build named boards (repeatable)') + src.add_argument('--diff-file', metavar='FILE', help='changed paths, one per line (--select-only)') + ap.add_argument('--worktree', action='store_true', help='with --base: add uncommitted and untracked files') + ap.add_argument('--select-only', action='store_true', help='print the selection manifest and stop') + ap.add_argument('-D', '--define', action='append', default=[], metavar='VAR=VAL', + help="passed to the board's backend: make VAR=VAL, cmake/idf -DVAR=VAL") + ap.add_argument('--shared', action='store_true', help='build into cmake-build/cmake-build- and keep it') + ap.add_argument('--idf', choices=('docker', 'local'), default='docker', + help="ESP-IDF: CI's pinned docker image (default) or the exported local IDF") + ap.add_argument('-v', '--verbose', action='store_true', help='stream build output to stderr') + args = ap.parse_args(argv) + if args.worktree and not args.base: + ap.error('--worktree needs --base') + if args.diff_file and not args.select_only: + ap.error('--diff-file is for --select-only') + if args.select_only and args.board: + ap.error('--select-only takes --scope, --base or --diff-file') + + result = {'pass': False} + try: + if args.board: + boards = {} + for b in args.board: + port = port_of(b) + backend = backend_of(port) + if not backend: + raise Fail(f'ports/{port}: {UNBUILT[port][1]}') + boards[(b, backend)] = 'named' + sel = {'full': False, 'boards': boards, 'ghostfat': False, 'tests': False, 'records': []} + info = {'mode': 'boards'} + else: + paths, info = changed_paths(args.base, args.worktree, args.scope, args.diff_file) + sel = select(paths) + if args.select_only: + print(json.dumps(manifest(sel, info))) + return 0 + info.update(identity()) + result['input'] = info + idf = {} + if any(backend == 'idf' for _, backend in sel['boards']): + ci = ci_idf_version() + idf = {'mode': args.idf, 'ciVersion': ci, 'image': f'espressif/idf:{ci}'} + if args.idf == 'local' and shutil.which('idf.py'): + v = subprocess.run(['idf.py', '--version'], capture_output=True, text=True) + idf['version'] = (v.stdout or v.stderr).strip() + idf['matchesCi'] = ci in idf['version'] + records = [build_board(b, backend, args, idf) for (b, backend) in sel['boards']] + for r in records: + r['selectedBy'] = sel['boards'][(r['board'], r['backend'])] + ghostfat = run_ghostfat(args.verbose) if sel['ghostfat'] else [] + tests = run_tests() if sel['tests'] else None + nothing = path_view(sel['records'], 'none', ('path', 'reason')) + uncovered = path_view(sel['records'], 'gap', ('path', 'reason')) + checks = records + ghostfat + ([tests] if tests else []) + statuses = [c['status'] for c in checks] + code = 2 if 'error' in statuses else 1 if 'failed' in statuses else 3 if uncovered else 0 + result.update({ + 'pass': code == 0, + 'boards': records, + 'ghostfat': ghostfat, + 'built': [f"{r['buildDir']}/{a}" for r in records if not r.get('removed') for a in r['artifacts']], + 'resolution': 'full' if sel['full'] else 'boards' if args.board else 'selected', + 'paths': path_view(sel['records']), + 'nothingToBuild': nothing, + 'uncovered': uncovered, + }) + if tests: + result['tests'] = tests + if idf: + result['idf'] = idf + first = next((c['firstError'] for c in checks if c['status'] != 'ok'), '') + if first: + result['firstError'] = first + print(json.dumps(result)) + return code + except (Fail, OSError) as e: + result['error'] = str(e) + print(json.dumps(result)) + return 2 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py new file mode 100644 index 000000000..5b6bff705 --- /dev/null +++ b/.claude/test/test_build.py @@ -0,0 +1,497 @@ +"""The build contract: check_build.py's tables agree with CI, each selection rule picks the boards +that compile a change, and the JSON and exit code follow SKILL.md. Builds are stubbed here; a real +build is evidence only from a run of the script itself.""" +import contextlib +import importlib.util +import io +import json +import os +import re +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / '.claude' / 'skills' / 'build' / 'scripts' / 'check_build.py' +LPC55_ARTIFACTS = ['_bin/tinyuf2-lpcxpresso55s69.bin', '_bin/tinyuf2-lpcxpresso55s69.hex', + '_bin/apps/update-tinyuf2-lpcxpresso55s69.uf2'] + + +def build_tmpdir(): + """A scratch build root inside the checkout's ignored cmake-build/, where the script reports from.""" + (ROOT / 'cmake-build').mkdir(exist_ok=True) + return tempfile.TemporaryDirectory(dir=ROOT / 'cmake-build', prefix='test-') + + +spec = importlib.util.spec_from_file_location('check_build', SCRIPT) +cb = importlib.util.module_from_spec(spec) +spec.loader.exec_module(cb) + + +def ci_jobs(): + """{port: backend} of build.yml's enabled jobs, and the ports of its disabled ones.""" + text = (ROOT / '.github' / 'workflows' / 'build.yml').read_text() + jobs = re.split(r'^ (?=[\w-]+:\s*$)', text.split('\njobs:\n', 1)[1], flags=re.M) + enabled, disabled = {}, set() + for job in jobs: + code = '\n'.join(l for l in job.splitlines() if not l.lstrip().startswith('#')) + toolchain = re.search(r"toolchain:\s*'([^']+)'", code) + if not toolchain or 'build_util.yml' not in code: + continue + ports = re.findall(r"^\s*-\s*'(\w+)'", code, re.M) or re.findall(r"^\s*port:\s*'(\w+)'", code, re.M) + backend = 'idf' if toolchain.group(1) == 'esp-idf' else re.search(r"build-system:\s*'(\w+)'", code).group(1) + for port in ports: + if re.search(r'^\s*if:\s*false', code, re.M): + disabled.add(port) + else: + enabled[port] = backend + return enabled, disabled + + +def classify(path): + r = cb.classify(path) + return r['rule'], r['effect'], sorted(r['boards']), r['ghostfat'], r['tests'] + + +class TablesMatchTheTree(unittest.TestCase): + def test_backends_are_build_yml_jobs(self): + enabled, disabled = ci_jobs() + self.assertEqual(cb.BACKENDS, enabled) + self.assertLessEqual(disabled, set(cb.UNBUILT)) + + def test_every_port_dir_has_a_backend_or_a_reason(self): + ports = {p.name for p in (ROOT / 'ports').iterdir() if p.is_dir()} - {cb.GHOSTFAT} + self.assertEqual(ports, set(cb.BACKENDS) | set(cb.UNBUILT)) + + def test_representatives_exist(self): + self.assertEqual(set(cb.REPRESENTATIVES), set(cb.BACKENDS)) + for port, boards in cb.REPRESENTATIVES.items(): + for b in boards: + self.assertTrue((ROOT / 'ports' / port / 'boards' / b).is_dir(), f'{port}/{b}') + for backend, board in cb.SMOKE.items(): + self.assertEqual(cb.BACKENDS[cb.port_of(board)], backend) + + def test_espressif_representatives_cover_both_idf_targets(self): + targets = {re.search(r'esp32s[23]', (ROOT / 'ports/espressif/boards' / b / 'board.cmake').read_text()).group(0) + for b in cb.REPRESENTATIVES['espressif']} + self.assertEqual(targets, {'esp32s2', 'esp32s3'}) + + def test_no_self_update_ports_echo_a_stub(self): + for port in cb.NO_SELF_UPDATE: + mk = (ROOT / 'ports' / port / 'Makefile').read_text() + recipe = re.search(r'^self-update:.*\n((?:\t.*\n?)*)', mk.replace('\r', ''), re.M) + self.assertIn('not implemented', recipe.group(1), port) + + def test_ci_idf_version_is_read_from_build_yml(self): + self.assertRegex(cb.ci_idf_version(), r'^v\d+\.\d+') + + +class Selection(unittest.TestCase): + def test_board_file_selects_that_board(self): + self.assertEqual(classify('ports/stm32f4/boards/stm32f411ce_blackpill/board.h'), + ('board', 'select', [('stm32f411ce_blackpill', 'cmake')], False, False)) + + def test_build_file_the_backend_never_reads_is_uncovered(self): + for path in ('ports/stm32f4/boards/stm32f411ce_blackpill/board.mk', 'ports/stm32f4/apps/self_update/Makefile', + 'ports/mimxrt10xx/CMakeLists.txt'): + self.assertEqual(classify(path)[:2], ('other-backend', 'gap'), path) + + def test_make_files_codeql_builds_select_its_make_build(self): + for path in ('ports/stm32f4/Makefile', 'ports/stm32f4/port.mk', + 'ports/stm32f4/boards/feather_stm32f405_express/board.mk', '.github/workflows/codeql-buildscript.sh'): + self.assertEqual(classify(path)[2], [('feather_stm32f405_express', 'make')], path) + + def test_port_file_selects_its_representatives(self): + rule, effect, boards, _, _ = classify('ports/mimxrt10xx/board_flash.c') + self.assertEqual((rule, effect), ('port', 'select')) + self.assertEqual(boards, [('imxrt1064_evk', 'make'), ('imxrt1170_evk', 'make')]) + + def test_app_selects_the_boards_that_build_it(self): + self.assertEqual(classify('ports/mimxrt10xx/apps/factory_test/main.c')[2], [('metro_m7_1011', 'make')]) + self.assertEqual(classify('ports/mimxrt10xx/apps/app.mk')[2], + [('metro_m7_1011', 'make'), ('metro_m7_1011_sd', 'make')]) + + def test_app_nothing_builds_is_uncovered(self): + self.assertEqual(classify('ports/lpc55/apps/erase_firmware/Makefile')[:2], ('port-app', 'gap')) + self.assertEqual(classify('ports/espressif/apps/erase_firmware/main/CMakeLists.txt')[:2], ('port-app', 'gap')) + self.assertEqual(classify('ports/espressif/apps/blinky/main/CMakeLists.txt')[:2], ('port-app', 'select')) + + def test_top_level_app_selects_ports_whose_backend_compiles_it(self): + ports = {cb.port_of(b) for b, _ in classify('apps/blinky/src/blinky.c')[2]} + self.assertEqual(ports, {'espressif', 'maxim', 'stm32f4', 'stm32h5'}) + + def test_dependency_selects_the_ports_get_deps_names(self): + self.assertEqual(classify('lib/mcu/st/cmsis_device_f4/x.h')[2], [('feather_stm32f405_express', 'cmake')]) + self.assertEqual(classify('lib/sct_neopixel/x.c')[2], [('lpcxpresso55s69', 'make')]) + + def test_unbuilt_ports_are_uncovered(self): + for path in ('ports/template_port/boards.c', 'ports/ch32v20x/boards.c'): + self.assertEqual(classify(path)[:2], ('unbuilt-port', 'gap'), path) + + def test_ghostfat_inputs(self): + self.assertEqual(classify('ports/test_ghostfat/main.c'), ('ghostfat', 'select', [], True, False)) + self.assertEqual(classify('src/ghostfat.c')[:2] + classify('src/ghostfat.c')[3:4], ('core', 'core', True)) + self.assertFalse(classify('src/main.c')[3]) + + def test_build_control_runs_tests_and_one_smoke_build_per_backend(self): + rule, effect, boards, _, tests = classify('.claude/skills/build/scripts/check_build.py') + self.assertEqual((rule, effect, tests), ('build-control', 'select', True)) + self.assertEqual(sorted(b for _, b in boards), ['cmake', 'idf', 'make']) + + def test_unknown_port_fails_explicitly(self): + with self.assertRaises(cb.Fail): + cb.backend_of('no_such_port') + + def test_contract_test_edit_runs_the_tests_only(self): + self.assertEqual(classify('.claude/test/test_build.py'), ('contract-tests', 'select', [], False, True)) + + def test_unreadable_codeql_script_fails_explicitly(self): + cb.codeql_build.cache_clear() + self.addCleanup(cb.codeql_build.cache_clear) + with mock.patch.object(Path, 'read_text', lambda self, *a, **k: 'cmake -B build\n'): + with self.assertRaises(cb.Fail): + cb.codeql_build() + + def test_nothing_to_build(self): + for path in ('README.md', '.claude/skills/build/SKILL.md', 'tools/gen_boards.py', '.github/workflows/codeql.yml'): + self.assertEqual(classify(path)[1], 'none', path) + + def test_core_and_unclassified_select_every_representative(self): + for path in ('src/main.c', 'lib/tinyusb', 'ports/family_support.cmake', 'some/new/file.c'): + self.assertEqual(classify(path)[1], 'core', path) + sel = cb.select(['src/main.c', 'README.md']) + self.assertTrue(sel['full']) + self.assertEqual({b for b, _ in sel['boards']}, {b for v in cb.REPRESENTATIVES.values() for b in v}) + + def test_paths_are_normalised(self): + self.assertEqual(cb.repo_path('./ports/test_ghostfat/main.c'), 'ports/test_ghostfat/main.c') + self.assertEqual(cb.repo_path(str(ROOT / 'src' / 'main.c')), 'src/main.c') + with self.assertRaises(cb.Fail): + cb.repo_path('/') + + def test_scope_refuses_a_path_neither_present_nor_tracked(self): + with self.assertRaises(cb.Fail): + cb.changed_paths(scope=['no/such/file.c']) + + +class ChangedPaths(unittest.TestCase): + """--base and --worktree in a scratch repository. Every GIT_* variable is dropped first: + under pre-commit they point at the real checkout, and git would act on it.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.repo = Path(tmp.name) + env = mock.patch.dict(os.environ, {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}, clear=True) + env.start() + self.addCleanup(env.stop) + root = mock.patch.object(cb, 'ROOT', self.repo) + root.start() + self.addCleanup(root.stop) + self.git('init', '-q', '-b', 'master') + for name in ('a.c', 'b.c', 'c.c', 'e.c'): + (self.repo / name).write_text(name * 20) + self.git('add', '.') + self.git('commit', '-q', '-m', 'base') + self.git('checkout', '-q', '-b', 'topic') + (self.repo / 'a.c').write_text('changed') + self.git('rm', '-q', 'b.c') + self.git('mv', 'c.c', 'd.c') + self.git('commit', '-q', '-am', 'topic') + (self.repo / 'e.c').write_text('unstaged') + (self.repo / 'g.c').write_text('staged') + self.git('add', 'g.c') + (self.repo / 'f.c').write_text('untracked') + + def git(self, *args): + subprocess.run(['git', '-c', 'user.name=t', '-c', 'user.email=t@t', '-c', 'commit.gpgsign=false', *args], + cwd=self.repo, check=True, capture_output=True) + + def test_base_reports_committed_changes_deletions_and_both_sides_of_a_rename(self): + paths, info = cb.changed_paths(base='master') + self.assertEqual(paths, ['a.c', 'b.c', 'c.c', 'd.c']) + self.assertEqual(info['mode'], 'base') + # --select-only returns before identity(), so its input names the commit it selected from + head = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=self.repo, capture_output=True, text=True).stdout.strip() + self.assertEqual(info['head'], head) + + def test_worktree_adds_staged_unstaged_and_untracked(self): + paths, info = cb.changed_paths(base='master', worktree=True) + self.assertEqual(paths, ['a.c', 'b.c', 'c.c', 'd.c', 'e.c', 'f.c', 'g.c']) + self.assertEqual(info['mode'], 'worktree') + + def test_scope_accepts_a_tracked_path_deleted_from_the_tree(self): + (self.repo / 'a.c').unlink() + self.assertEqual(cb.changed_paths(scope=['a.c'])[0], ['a.c']) + with self.assertRaises(cb.Fail): # deleted by a commit: --base reports it, --scope cannot see it + cb.changed_paths(scope=['b.c']) + + +def run_main(argv, boards=None, ghostfat=None, tests=None): + """main() with builds stubbed: `boards` maps board -> status.""" + boards = boards or {} + + def build_board(board, backend, args, idf): + status = boards.get(board, 'ok') + return {'board': board, 'port': cb.port_of(board), 'backend': backend, 'buildDir': f'cmake-build/x-{board}', + 'status': status, 'firstError': '' if status == 'ok' else f'{board} {status}', 'artifacts': ['a.bin']} + + out = io.StringIO() + with mock.patch.object(cb, 'build_board', build_board), \ + mock.patch.object(cb, 'run_ghostfat', lambda v: ghostfat or []), \ + mock.patch.object(cb, 'run_tests', lambda: tests or {'status': 'ok', 'summary': 'OK', 'firstError': ''}), \ + mock.patch.object(cb, 'identity', lambda: {'head': 'h', 'dirty': False, 'submodules': {}}), \ + contextlib.redirect_stdout(out): + code = cb.main(argv) + return code, json.loads(out.getvalue().splitlines()[-1]) + + +class Verdict(unittest.TestCase): + def test_pass(self): + code, r = run_main(['--board', 'lpcxpresso55s69']) + self.assertEqual((code, r['pass']), (0, True)) + self.assertEqual(r['built'], ['cmake-build/x-lpcxpresso55s69/a.bin']) + + def test_docs_only_builds_nothing_and_passes_with_its_reason(self): + with mock.patch.object(cb, 'build_board', side_effect=AssertionError('built')): + code, r = run_main(['--scope', 'README.md']) + self.assertEqual((code, r['boards'], r['nothingToBuild'][0]['path']), (0, [], 'README.md')) + + def test_failed_board_is_exit_1(self): + code, r = run_main(['--board', 'lpcxpresso55s69', '--board', 'swan_r5'], boards={'swan_r5': 'failed'}) + self.assertEqual((code, r['pass'], r['firstError']), (1, False, 'swan_r5 failed')) + + def test_error_outranks_failure_and_failure_outranks_uncovered(self): + code, _ = run_main(['--board', 'lpcxpresso55s69', '--board', 'swan_r5'], + boards={'swan_r5': 'failed', 'lpcxpresso55s69': 'error'}) + self.assertEqual(code, 2) + code, _ = run_main(['--scope', 'ports/stm32f4/board_flash.c', 'ports/template_port/boards.c'], + boards={'feather_stm32f405_express': 'failed'}) + self.assertEqual(code, 1) + + def test_uncovered_path_fails_a_green_build(self): + code, r = run_main(['--scope', 'ports/stm32f4/board_flash.c', 'ports/template_port/boards.c']) + self.assertEqual((code, r['pass']), (3, False)) + self.assertEqual(r['uncovered'][0]['path'], 'ports/template_port/boards.c') + + def test_failed_ghostfat_or_tests_fail_the_run(self): + code, _ = run_main(['--scope', 'ports/test_ghostfat/main.c'], + ghostfat=[{'config': '4k', 'status': 'failed', 'firstError': 'FAIL: (-10)'}]) + self.assertEqual(code, 1) + code, r = run_main(['--scope', '.claude/test/test_build.py'], + tests={'status': 'failed', 'summary': 'FAILED (failures=1)', 'firstError': 'x'}) + self.assertEqual((code, r['tests']['status']), (1, 'failed')) + + def test_select_only_never_builds(self): + out = io.StringIO() + with mock.patch.object(cb, 'build_board', side_effect=AssertionError('built')), contextlib.redirect_stdout(out): + self.assertEqual(cb.main(['--select-only', '--scope', 'src/main.c']), 0) + m = json.loads(out.getvalue()) + self.assertEqual((m['version'], m['build']['full'], m['build']['needed']), (1, True, True)) + + def test_usage_error_is_json_exit_2(self): + out = io.StringIO() + with contextlib.redirect_stdout(out), self.assertRaises(SystemExit) as e: + cb.main(['--scope', 'README.md', '--worktree']) + self.assertEqual(e.exception.code, 2) + self.assertIn('--worktree needs --base', json.loads(out.getvalue())['error']) + + def test_passing_private_roots_leave_nothing_in_built(self): + def build_board(board, backend, args, idf): + return {'board': board, 'port': 'lpc55', 'backend': backend, 'buildDir': 'cmake-build/x', 'status': 'ok', + 'firstError': '', 'artifacts': ['a.bin'], 'removed': True} + with mock.patch.object(cb, 'build_board', build_board), \ + mock.patch.object(cb, 'identity', lambda: {}), contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(cb.main(['--board', 'lpcxpresso55s69']), 0) + self.assertEqual(json.loads(out.getvalue().splitlines()[-1])['built'], []) + + def test_unreadable_get_deps_is_json_exit_2(self): + cb.deps_rows.cache_clear() + self.addCleanup(cb.deps_rows.cache_clear) + with mock.patch.object(cb.runpy, 'run_path', side_effect=SyntaxError('invalid syntax')), \ + mock.patch.object(cb, 'changed_paths', lambda *a: (['lib/mcu/nxp/mcux-sdk/x.h'], {'mode': 'paths'})): + code, r = run_main(['--scope', 'lib/mcu/nxp/mcux-sdk/x.h']) + self.assertEqual((code, r['pass']), (2, False)) + self.assertIn('tools/get_deps.py', r['error']) + + def test_resolution_error_is_json_exit_2(self): + code, r = run_main(['--board', 'no_such_board']) + self.assertEqual((code, r['pass']), (2, False)) + self.assertIn('no_such_board', r['error']) + + +class BuildBoard(unittest.TestCase): + def setUp(self): + tmp = build_tmpdir() + self.addCleanup(tmp.cleanup) + self.build_dir = Path(tmp.name) + patches = [mock.patch.object(cb, 'BUILD_DIR', self.build_dir), mock.patch.object(cb, 'check_deps', lambda p: None), + mock.patch.object(cb, 'check_tools', lambda *a: None)] + for p in patches: + p.start() + self.addCleanup(p.stop) + self.args = mock.Mock(shared=False, define=['LOG=2'], verbose=False, idf='docker') + + def fake_make(self, write): + calls = [] + + def run(cmd, log, verbose, cwd=cb.ROOT): + cmd = [str(c) for c in cmd] + calls.append(cmd) + root = Path(next(c for c in cmd if c.startswith('BUILD_ROOT='))[len('BUILD_ROOT='):]) + for rel in write(cmd): + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text('x') + return 0 + return run, calls + + def test_make_builds_like_ci_and_removes_a_passing_private_root(self): + run, calls = self.fake_make(lambda cmd: LPC55_ARTIFACTS if 'copy-artifact' in cmd else []) + with mock.patch.object(cb, 'run', run): + rec = cb.build_board('lpcxpresso55s69', 'make', self.args, {}) + self.assertEqual(rec['status'], 'ok', rec) + self.assertTrue(rec['removed']) + self.assertIn('_bin/apps/update-tinyuf2-lpcxpresso55s69.uf2', rec['artifacts']) + self.assertEqual(rec['notImplemented'], ['apps/erase_firmware']) + main, copy = calls[0], calls[1] + self.assertIn('self-update', main) + self.assertNotIn('copy-artifact', main) # never beside self-update under -j + self.assertIn('copy-artifact', copy) + self.assertIn('LOG=2', main) + self.assertFalse(list(self.build_dir.iterdir())) + + def test_missing_expected_artifact_fails_and_keeps_the_root(self): + run, _ = self.fake_make(lambda cmd: LPC55_ARTIFACTS[:2] if 'copy-artifact' in cmd else []) + with mock.patch.object(cb, 'run', run): + rec = cb.build_board('lpcxpresso55s69', 'make', self.args, {}) + self.assertEqual(rec['status'], 'failed') + self.assertIn('update-tinyuf2-lpcxpresso55s69.uf2', rec['firstError']) + self.assertTrue((self.build_dir / f'cmake-build-agent-{cb.os.getpid()}-lpcxpresso55s69').is_dir()) + + def test_shared_root_is_rebuilt_from_clean_and_kept(self): + self.args.shared = True + root = self.build_dir / 'cmake-build-lpcxpresso55s69' + root.mkdir() + (root / 'stale.o').write_text('old configuration') + run, _ = self.fake_make(lambda cmd: LPC55_ARTIFACTS if 'copy-artifact' in cmd else []) + with mock.patch.object(cb, 'run', run): + rec = cb.build_board('lpcxpresso55s69', 'make', self.args, {}) + self.assertEqual(rec['status'], 'ok', rec) + self.assertNotIn('removed', rec) + self.assertFalse((root / 'stale.o').exists()) + self.assertTrue((root / '_bin' / 'tinyuf2-lpcxpresso55s69.bin').is_file()) + + def test_cmake_missing_app_image_fails(self): + def run(cmd, log, verbose, cwd=cb.ROOT): + log.write_text('ok') + if cmd[:2] == ['cmake', '--build']: + for f in ('tinyuf2.bin', 'tinyuf2.hex'): + (Path(cmd[2]) / '_bin' / f).parent.mkdir(parents=True, exist_ok=True) + (Path(cmd[2]) / '_bin' / f).write_text('x') + return 0 + with mock.patch.object(cb, 'run', run): + rec = cb.build_board('feather_stm32f405_express', 'cmake', self.args, {}) + self.assertEqual(rec['status'], 'failed') + self.assertIn('update-tinyuf2.uf2', rec['firstError']) + + def test_non_default_backend_gets_its_own_root(self): + with mock.patch.object(cb, 'run', lambda cmd, log, verbose, **k: log.write_text('make: *** Error 2') and 1): + rec = cb.build_board('feather_stm32f405_express', 'make', self.args, {}) + self.assertTrue(rec['buildDir'].endswith('feather_stm32f405_express-make')) + + def test_missing_prerequisite_is_an_error_not_a_failure(self): + with mock.patch.object(cb, 'check_deps', lambda p: 'lib/mcu/nxp/mcux-sdk missing'): + rec = cb.build_board('lpcxpresso55s69', 'make', self.args, {}) + self.assertEqual((rec['status'], rec['firstError']), ('error', 'lib/mcu/nxp/mcux-sdk missing')) + + +class Ghostfat(unittest.TestCase): + @staticmethod + def fake_ghostfat_make(cmd, log, verbose, cwd=None): + log.write_text('built') + root = Path(next(str(c) for c in cmd if str(c).startswith('BUILD_ROOT='))[len('BUILD_ROOT='):]) + work = root / 'ports' / cb.GHOSTFAT + (work / 'obj').mkdir(parents=True) + for f in ('obj/ghostfat.o', 'tinyuf2-4k.elf', 'knowngood.img'): + (work / f).write_text('x') + return 0 + + def test_a_damaged_known_good_image_is_an_error_and_keeps_only_the_log(self): + with build_tmpdir() as tmp, mock.patch.object(cb, 'BUILD_DIR', Path(tmp)), \ + mock.patch.object(cb, 'boards_of', lambda port: ['4k']), \ + mock.patch.object(cb, 'run', self.fake_ghostfat_make), \ + mock.patch.object(cb.gzip, 'open', side_effect=EOFError('truncated')): + [rec] = cb.run_ghostfat(False) + root = Path(tmp) / f'cmake-build-agent-{cb.os.getpid()}-ghostfat-4k' + self.assertEqual(rec['status'], 'error') + self.assertIn('truncated', rec['firstError']) + self.assertEqual([p.name for p in root.iterdir()], ['check_build.log']) + + +class BackendCommands(unittest.TestCase): + """Each backend builds into the root it is given and nowhere in the source tree.""" + + def setUp(self): + self.root = cb.BUILD_DIR / 'cmake-build-agent-0-x' + self.calls = [] + + def capture(self, results=()): + results = list(results) + + def run(cmd, log, verbose, cwd=cb.ROOT): + self.calls.append([str(c) for c in cmd]) + return results.pop(0) if results else 0 + return mock.patch.object(cb, 'run', run) + + def test_cmake_configures_into_the_root_with_artifacts_there(self): + with self.capture(): + ok, _ = cb.build_cmake('stm32f4', 'feather_stm32f405_express', self.root, ['LOG=2'], Path('log'), False) + self.assertTrue(ok) + cfg, build = self.calls + self.assertEqual(cfg[cfg.index('-B') + 1], str(self.root)) + self.assertIn(f'-DARTIFACT_PATH={self.root / "_bin"}', cfg) + self.assertIn('-DLOG=2', cfg) + self.assertEqual(build, ['cmake', '--build', str(self.root)]) + + def test_cmake_configure_failure_skips_the_build(self): + with self.capture([1]): + ok, _ = cb.build_cmake('stm32f4', 'feather_stm32f405_express', self.root, [], Path('log'), False) + self.assertFalse(ok) + self.assertEqual(len(self.calls), 1) + + def test_idf_runs_ci_image_with_the_git_common_dir_mounted(self): + with self.capture([1]), mock.patch.object(cb, 'git', lambda *a: '/srv/repo/.git\n'): + ok, _ = cb.build_idf('espressif_esp32s3_devkitm_1', self.root, ['LOG=2'], Path('log'), False, 'docker', + 'espressif/idf:v5.3.2') + self.assertFalse(ok) + [cmd] = self.calls + self.assertEqual(cmd[:2], ['docker', 'run']) + self.assertIn('espressif/idf:v5.3.2', cmd) + mounts = {cmd[i + 1] for i, c in enumerate(cmd) if c == '-v'} + self.assertLessEqual({f'{cb.ROOT}:{cb.ROOT}', '/srv/repo/.git:/srv/repo/.git'}, mounts) + inner = cmd[-1] + self.assertIn(f'-B {self.root}', inner) + self.assertIn(f'-DARTIFACT_PATH={self.root / "_bin"}', inner) + self.assertIn('-DLOG=2', inner) + + def test_expected_artifacts_per_backend(self): + names = lambda port, board, backend: {str(p.relative_to(self.root)) + for p in cb.expected_artifacts(port, board, backend, self.root)} + self.assertEqual(names('stm32f4', 'feather_stm32f405_express', 'cmake'), + {'_bin/tinyuf2.bin', '_bin/tinyuf2.hex', '_bin/apps/update-tinyuf2.uf2', '_bin/apps/blinky.uf2', + '_bin/apps/erase_firmware.uf2'}) + self.assertLessEqual({'_bin/combined.bin', '_bin/apps/update-tinyuf2.uf2'}, + names('espressif', 'espressif_esp32s3_devkitm_1', 'idf')) + self.assertNotIn('_bin/apps/update-tinyuf2-swan_r5.uf2', names('stm32l4', 'swan_r5', 'make')) + + +class FirstError(unittest.TestCase): + def test_compiler_diagnostic_beats_ninjas_failed_line(self): + log = 'FAILED: CMakeFiles/tinyuf2.dir/src/main.c.obj\n/x/src/main.c:12:3: error: unknown type name\n' + self.assertEqual(cb.first_error(log), '/x/src/main.c:12:3: error: unknown type name') + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/pre_commit.yml b/.github/workflows/pre_commit.yml new file mode 100644 index 000000000..ba79a2a37 --- /dev/null +++ b/.github/workflows/pre_commit.yml @@ -0,0 +1,33 @@ +name: pre-commit + +# build.yml runs pre-commit only for firmware paths; this covers the agent files and workflows +on: + push: + paths: + - '.claude/**' + - 'CLAUDE.md' + - '.github/workflows/**' + - '.pre-commit-config.yaml' + - 'tools/get_deps.py' + pull_request: + branches: [ master ] + paths: + - '.claude/**' + - 'CLAUDE.md' + - '.github/workflows/**' + - '.pre-commit-config.yaml' + - 'tools/get_deps.py' + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + pre-commit: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Run pre-commit + uses: pre-commit/action@v3.0.1 diff --git a/.gitignore b/.gitignore index 9278631bd..8217d1dd7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,8 @@ _bin/ *.cmake~ sdkconfig~ *.yml~ -ports/espressif/apps/self_update/main/bootloader_bin.c +cmake-build/ +lib/mcu/ +lib/sct_neopixel +__pycache__/ +*.pyc diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 19233e51c..a65fe33f1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,3 +29,12 @@ repos: (?x)^( lib/ ) + +- repo: local + hooks: + - id: claude-tests + name: agent skill script tests + entry: python3 -m unittest discover -s .claude/test -p 'test_*.py' + language: system + pass_filenames: false + files: ^(\.claude/|\.github/workflows/|ports/|tools/get_deps\.py$) diff --git a/AGENTS.md b/AGENTS.md new file mode 120000 index 000000000..681311eb9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +CLAUDE.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..d483ff773 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,35 @@ +# TinyUF2 Agent Instructions + +## Working Rules + +- Worktrees — `git submodule update --init lib/tinyusb lib/uf2`, then symlink the `deps_all` paths from `tools/get_deps.py` (`lib/mcu/...`, `lib/sct_neopixel`) to the primary checkout when it holds the pinned revisions. Never run `get_deps.py` through a symlink: it `git reset --hard`s each dependency in place. +- Code references — ports and boards: `ports//` (`boards//`, `board_flash.c`, `boards.c`), `supported_boards.md`; bootloader: `src/` (`main.c`, `ghostfat.c`, `msc.c`, `board_api.h`, the contract every port implements); apps: `apps/`, `ports//apps/`; build: `ports/make.mk`, `ports/rules.mk`, `ports/family_support.cmake`, `.github/workflows/build.yml` (which backend builds each port); deps: `tools/get_deps.py`. + +## Code Rules + +- Formatting follows `.clang-format`: 2-space indent, no tabs. +- Port-specific code stays in `ports//` behind the `board_api.h` interface. +- Some files use CRLF line endings; keep each file's own line endings when editing. + +## Skills + +- Agents, skills and workflows beyond this repo's `.claude/` come from [agentrc](https://github.com/hathach/agentrc); install it before agent work on TinyUF2. +- Skill script tests live in `.claude/test/test_*.py`; pre-commit runs them. +- Hardware diagnosis and repair use agentrc's `hw-debugger`; independent hardware claims and committed-fix validation use `hw-validator`; routine HIL runs use `hil-operator` — all orchestrated by `chief`. +- Bootloader audit: the `code-audit` workflow with `dirs` (e.g. `src`, `ports/`) and `dimensions`: `UF2 and MSC input: block validation, bounds and arithmetic, completion and error paths`; `flash and app boundaries: erase/write range and alignment, write protection, update integrity and recovery`; `boot decisions and app handoff: app validity, double-tap and reset reasons, jump, vector table, peripheral teardown`; `ISR and shared state, and register use vs reference manual and errata: cross-check the reference manual AND errata sheets via the read-doc skill; a lookup that cannot run is pending and its conclusion unverified (never a web/filesystem substitute); a missing erratum workaround is a finding`; `build and configuration: the backend CI uses per port, board and config selection, code only some configurations compile`; `style: repo conventions and line endings`. + +## Build and Validate + +- Build contract: `.claude/skills/build/SKILL.md`. It resolves a change to boards and builds each the way CI does, outside the source tree. +- No HIL contract yet: hardware validation of a change is reported pending, never inferred from a build. +- ESP-IDF: the build contract uses CI's docker image; for interactive work, `. "$IDF_PATH/export.sh"` and `idf.py -C ports/espressif -DBOARD= flash monitor`. +- Before submitting, and again after a rebase: `pre-commit run --all-files`. +- For code changes: `python3 .claude/skills/build/scripts/check_build.py --base master --worktree`. + +## PRs and Follow-ups + +- Before opening or updating a PR, follow Build and Validate. +- After opening a PR, ask the human to authorize one headless `chief` invocation to drive reviews and CI to green under agentrc chief's Authorization exception. Name the PR URL, head repository and branch, and actions: push commits to that branch, post replies to the PR's review feedback, and resolve review threads through `pr-babysit` with `autoPush: true`. On an affirmative answer, launch it in the PR worktree through agentrc's `headless-chief` skill (`chief_run.py`), never a bare `claude -p`, and relay its exit as that skill directs. The task names the PR, head repository and branch, expected HEAD and scope, and includes the question and answer verbatim; a relaunch for the same PR may carry that exchange copied verbatim from the earlier launch task. Leave the checkout to chief until it exits. TinyUF2 `pr-babysit` args: + `{"pr": , "reviewers": ["copilot", "code-scanning"], "protected": "^lib/(tinyusb|uf2)$"}` + (until CodeRabbit or Greptile is installed on the repository, only Copilot and code scanning review it, both harvested without gating; `protected` keeps the submodule pins out of automated fixes). +- The follow-up label is `followup`. diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index 5a8fc015b..06a88ca87 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -10,13 +10,16 @@ set(SDKCONFIG_DEFAULTS ${CMAKE_CURRENT_LIST_DIR}/sdkconfig.defaults ${CMAKE_CURR set(SDKCONFIG ${CMAKE_BINARY_DIR}/sdkconfig) include($ENV{IDF_PATH}/tools/cmake/project.cmake) +idf_build_set_property(DEPENDENCIES_LOCK ${CMAKE_BINARY_DIR}/dependencies.lock) # keep the source tree clean idf_build_set_property(EXTRA_CMAKE_ARGS -DBOARD=${BOARD}) # Pass BOARD to bootloader execute_process(COMMAND git describe --dirty --always --tags + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} OUTPUT_VARIABLE GIT_VERSION) string(STRIP ${GIT_VERSION} GIT_VERSION) execute_process(COMMAND bash "-c" "git submodule status ${TOP}/lib/tinyusb | cut -d\" \" -f3,4 | paste -s -d\" \" -" + WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR} OUTPUT_VARIABLE GIT_SUBMODULE_VERSIONS) string(REPLACE ../ "" GIT_SUBMODULE_VERSIONS ${GIT_SUBMODULE_VERSIONS}) @@ -83,7 +86,7 @@ add_custom_target(combined-flash # Post build: generate bootloader_bin.c for update-tinyuf2 and combined.bin add_custom_command(TARGET app POST_BUILD - COMMAND ${Python_EXECUTABLE} ${UF2CONV_PY} --carray -o ${CMAKE_CURRENT_LIST_DIR}/apps/update_tinyuf2/main/bootloader_bin.c ${CMAKE_BINARY_DIR}/tinyuf2.bin + COMMAND ${Python_EXECUTABLE} ${UF2CONV_PY} --carray -o ${CMAKE_BINARY_DIR}/apps/update_tinyuf2/bootloader_bin.c ${CMAKE_BINARY_DIR}/tinyuf2.bin ) externalproject_add(update-tinyuf2 @@ -91,10 +94,10 @@ externalproject_add(update-tinyuf2 BINARY_DIR ${CMAKE_BINARY_DIR}/apps/update_tinyuf2 # Modifying the list separator for the arguments, as such, we won't need to manually # replace the new separator by the default ';' in the subproject - CMAKE_ARGS -DBOARD=${BOARD} + CMAKE_ARGS -DBOARD=${BOARD} -DARTIFACT_PATH=${ARTIFACT_PATH} INSTALL_COMMAND "" BUILD_ALWAYS 1 - DEPENDS tinyuf2.elf + DEPENDS app # app post-build generates bootloader_bin.c ) externalproject_add(blinky @@ -102,7 +105,7 @@ externalproject_add(blinky BINARY_DIR ${CMAKE_BINARY_DIR}/apps/blinky # Modifying the list separator for the arguments, as such, we won't need to manually # replace the new separator by the default ';' in the subproject - CMAKE_ARGS -DBOARD=${BOARD} + CMAKE_ARGS -DBOARD=${BOARD} -DARTIFACT_PATH=${ARTIFACT_PATH} INSTALL_COMMAND "" BUILD_ALWAYS 1 ) diff --git a/ports/espressif/apps/blinky/CMakeLists.txt b/ports/espressif/apps/blinky/CMakeLists.txt index 967dd8df7..8e5502749 100644 --- a/ports/espressif/apps/blinky/CMakeLists.txt +++ b/ports/espressif/apps/blinky/CMakeLists.txt @@ -9,6 +9,7 @@ set(EXTRA_COMPONENT_DIRS "../../boards" "../../components") set(SDKCONFIG ${CMAKE_BINARY_DIR}/sdkconfig) include($ENV{IDF_PATH}/tools/cmake/project.cmake) +idf_build_set_property(DEPENDENCIES_LOCK ${CMAKE_BINARY_DIR}/dependencies.lock) # keep the source tree clean add_compile_definitions( BUILD_APPLICATION diff --git a/ports/espressif/apps/update_tinyuf2/CMakeLists.txt b/ports/espressif/apps/update_tinyuf2/CMakeLists.txt index 625ba6072..7a1bc07a3 100644 --- a/ports/espressif/apps/update_tinyuf2/CMakeLists.txt +++ b/ports/espressif/apps/update_tinyuf2/CMakeLists.txt @@ -9,6 +9,7 @@ set(EXTRA_COMPONENT_DIRS "../../boards" "../../components") set(SDKCONFIG ${CMAKE_BINARY_DIR}/sdkconfig) include($ENV{IDF_PATH}/tools/cmake/project.cmake) +idf_build_set_property(DEPENDENCIES_LOCK ${CMAKE_BINARY_DIR}/dependencies.lock) # keep the source tree clean add_compile_definitions( TINYUF2_SELF_UPDATE diff --git a/ports/espressif/apps/update_tinyuf2/main/CMakeLists.txt b/ports/espressif/apps/update_tinyuf2/main/CMakeLists.txt index 018e68468..8a5a88fe6 100644 --- a/ports/espressif/apps/update_tinyuf2/main/CMakeLists.txt +++ b/ports/espressif/apps/update_tinyuf2/main/CMakeLists.txt @@ -1,4 +1,4 @@ -# Note: bootloader_bin.c is generated by tinyuf2 app target -idf_component_register(SRCS ${TOP}/apps/self_update/self_update.c bootloader_bin.c +# Note: bootloader_bin.c is generated into this project's binary dir by tinyuf2 app target +idf_component_register(SRCS ${TOP}/apps/self_update/self_update.c ${CMAKE_BINARY_DIR}/bootloader_bin.c INCLUDE_DIRS ${TOP}/src REQUIRES boards) diff --git a/ports/family_support.cmake b/ports/family_support.cmake index 939879c07..a905ba9df 100644 --- a/ports/family_support.cmake +++ b/ports/family_support.cmake @@ -76,7 +76,9 @@ endif () set(FAMILY_PATH ${TOP}/ports/${FAMILY} CACHE INTERNAL "FAMILY_PATH") set(BOARD_PATH ${TOP}/ports/${FAMILY}/boards/${BOARD} CACHE INTERNAL "BOARD_PATH") -set(ARTIFACT_PATH ${FAMILY_PATH}/_bin/${BOARD}) +if (NOT DEFINED ARTIFACT_PATH) + set(ARTIFACT_PATH ${FAMILY_PATH}/_bin/${BOARD}) +endif () execute_process(COMMAND mkdir -p ${ARTIFACT_PATH}) execute_process(COMMAND mkdir -p ${ARTIFACT_PATH}/apps) @@ -207,7 +209,7 @@ function(family_configure_tinyuf2 TARGET OPT_MCU) family_add_tinyusb(${TARGET} ${OPT_MCU}) # Add uf2 version - execute_process(COMMAND ${GIT_EXECUTABLE} describe --dirty --always --tags OUTPUT_VARIABLE GIT_VERSION) + execute_process(COMMAND ${GIT_EXECUTABLE} describe --dirty --always --tags WORKING_DIRECTORY ${TOP} OUTPUT_VARIABLE GIT_VERSION) string(STRIP ${GIT_VERSION} GIT_VERSION) cmake_print_variables(GIT_VERSION) target_compile_definitions(${TARGET} PUBLIC diff --git a/ports/lpc55/Makefile b/ports/lpc55/Makefile index 63bdcf79d..85cb6f485 100644 --- a/ports/lpc55/Makefile +++ b/ports/lpc55/Makefile @@ -18,9 +18,10 @@ include ../rules.mk #-------------- Self-update -------------- SELF_DIR = apps/self_update -$(SELF_DIR)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin +$(SELF_BUILD)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin + @$(MKDIR) -p $(@D) $(PYTHON3) $(TOP)/lib/uf2/utils/uf2conv.py --carray $^ -o $@ -self-update: $(SELF_DIR)/bootloader_bin.c +self-update: $(SELF_BUILD)/bootloader_bin.c make -C $(SELF_DIR) BOARD=$(BOARD) self-update @rm $^ diff --git a/ports/lpc55/apps/self_update/Makefile b/ports/lpc55/apps/self_update/Makefile index 763ba3448..9fc5e4fb4 100644 --- a/ports/lpc55/apps/self_update/Makefile +++ b/ports/lpc55/apps/self_update/Makefile @@ -15,7 +15,7 @@ include ../../port.mk SRC_C += \ $(PORT_DIR)/boards.c \ apps/self_update/self_update.c \ - $(CURRENT_PATH)/bootloader_bin.c + $(BUILD)/bootloader_bin.c include ../../../rules.mk diff --git a/ports/make.mk b/ports/make.mk index a73a685e4..db4ec4d23 100644 --- a/ports/make.mk +++ b/ports/make.mk @@ -48,9 +48,16 @@ ifeq ($(wildcard $(TOP)/$(BOARD_DIR)/),) $(error Invalid BOARD specified) endif -# Build directory -BUILD = _build/$(BOARD) -BIN = $(TOP)/$(PORT_DIR)/_bin/$(BOARD) +# Build directory, BUILD_ROOT (an absolute path, one per board) moves the output of this make and its sub-makes out of the source tree +ifdef BUILD_ROOT + BUILD = $(BUILD_ROOT)/$(CURRENT_PATH) + BIN = $(BUILD_ROOT)/_bin + SELF_BUILD = $(BUILD_ROOT)/$(PORT_DIR)/apps/self_update +else + BUILD = _build/$(BOARD) + BIN = $(TOP)/$(PORT_DIR)/_bin/$(BOARD) + SELF_BUILD = $(TOP)/$(PORT_DIR)/apps/self_update/_build/$(BOARD) +endif # can be set manually by custom build such as flash_nuke OUTNAME ?= tinyuf2-$(BOARD) diff --git a/ports/rules.mk b/ports/rules.mk index a0712e2b4..557e60e23 100644 --- a/ports/rules.mk +++ b/ports/rules.mk @@ -70,7 +70,7 @@ get-deps: $(PYTHON3) $(TOP)/tools/get_deps.py --board $(BOARD) #-------------- Artifacts -------------- -SELF_UF2 ?= apps/self_update/$(BUILD)/update-$(OUTNAME).uf2 +SELF_UF2 ?= $(SELF_BUILD)/update-$(OUTNAME).uf2 $(BIN): @$(MKDIR) -p $@ diff --git a/ports/stm32f3/Makefile b/ports/stm32f3/Makefile index b90cda823..59904a29c 100644 --- a/ports/stm32f3/Makefile +++ b/ports/stm32f3/Makefile @@ -12,10 +12,11 @@ include ../rules.mk # directory containing Makefile for building update app SELF_DIR = apps/self_update -$(SELF_DIR)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin +$(SELF_BUILD)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin + @$(MKDIR) -p $(@D) $(PYTHON3) $(TOP)/lib/uf2/utils/uf2conv.py --carray $^ -o $@ # remove bootloader_bin.c at the end to force re-generate each time -self-update: $(SELF_DIR)/bootloader_bin.c +self-update: $(SELF_BUILD)/bootloader_bin.c make -C $(SELF_DIR) BOARD=$(BOARD) LOG=$(LOG) LOGGER=$(LOGGER) self-update @rm -f $^ diff --git a/ports/stm32f3/apps/self_update/Makefile b/ports/stm32f3/apps/self_update/Makefile index 5fcc7a235..27c8b1cf9 100644 --- a/ports/stm32f3/apps/self_update/Makefile +++ b/ports/stm32f3/apps/self_update/Makefile @@ -14,7 +14,7 @@ include ../../port.mk SRC_C += \ apps/self_update/self_update.c \ - $(CURRENT_PATH)/bootloader_bin.c + $(BUILD)/bootloader_bin.c include ../../../rules.mk diff --git a/ports/stm32f4/Makefile b/ports/stm32f4/Makefile index 5c3736c7a..049e48c25 100644 --- a/ports/stm32f4/Makefile +++ b/ports/stm32f4/Makefile @@ -12,10 +12,11 @@ include ../rules.mk # directory containing Makefile for building update app SELF_DIR = apps/self_update -$(SELF_DIR)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin +$(SELF_BUILD)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin + @$(MKDIR) -p $(@D) $(PYTHON3) $(TOP)/lib/uf2/utils/uf2conv.py --carray $^ -o $@ # remove bootloader_bin.c at the end to force re-generate each time -self-update: $(SELF_DIR)/bootloader_bin.c +self-update: $(SELF_BUILD)/bootloader_bin.c make -C $(SELF_DIR) BOARD=$(BOARD) LOG=$(LOG) LOGGER=$(LOGGER) self-update @rm -f $^ diff --git a/ports/stm32f4/apps/self_update/Makefile b/ports/stm32f4/apps/self_update/Makefile index 1b76357dd..f17e6da28 100644 --- a/ports/stm32f4/apps/self_update/Makefile +++ b/ports/stm32f4/apps/self_update/Makefile @@ -14,7 +14,7 @@ include ../../port.mk SRC_C += \ apps/self_update/self_update.c \ - $(CURRENT_PATH)/bootloader_bin.c + $(BUILD)/bootloader_bin.c include ../../../rules.mk diff --git a/ports/stm32h5/Makefile b/ports/stm32h5/Makefile index 5c3736c7a..049e48c25 100644 --- a/ports/stm32h5/Makefile +++ b/ports/stm32h5/Makefile @@ -12,10 +12,11 @@ include ../rules.mk # directory containing Makefile for building update app SELF_DIR = apps/self_update -$(SELF_DIR)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin +$(SELF_BUILD)/bootloader_bin.c: $(BUILD)/$(OUTNAME).bin + @$(MKDIR) -p $(@D) $(PYTHON3) $(TOP)/lib/uf2/utils/uf2conv.py --carray $^ -o $@ # remove bootloader_bin.c at the end to force re-generate each time -self-update: $(SELF_DIR)/bootloader_bin.c +self-update: $(SELF_BUILD)/bootloader_bin.c make -C $(SELF_DIR) BOARD=$(BOARD) LOG=$(LOG) LOGGER=$(LOGGER) self-update @rm -f $^ diff --git a/ports/stm32h5/apps/self_update/Makefile b/ports/stm32h5/apps/self_update/Makefile index 74b9c1001..341219743 100644 --- a/ports/stm32h5/apps/self_update/Makefile +++ b/ports/stm32h5/apps/self_update/Makefile @@ -14,7 +14,7 @@ include ../../port.mk SRC_C += \ apps/self_update/self_update.c \ - $(CURRENT_PATH)/bootloader_bin.c + $(BUILD)/bootloader_bin.c include ../../../rules.mk From 9718b8d6befe62caf8912e14e55236594602135f Mon Sep 17 00:00:00 2001 From: hathach Date: Mon, 12 Oct 2026 10:19:39 +0700 Subject: [PATCH 15/47] Add CodeRabbit config and README badges --- .coderabbit.yaml | 32 ++++++++++++++++++++++++++++++++ README.md | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 .coderabbit.yaml diff --git a/.coderabbit.yaml b/.coderabbit.yaml new file mode 100644 index 000000000..19626f177 --- /dev/null +++ b/.coderabbit.yaml @@ -0,0 +1,32 @@ +# yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json +reviews: + profile: quiet + high_level_summary: false + in_progress_fortune: false + auto_review: + auto_pause_after_reviewed_commits: 0 + + path_filters: + - "!lib/**" + + path_instructions: + - path: "src/**" + instructions: | + Cross-platform bootloader core shared by every port. Check that changes + stay port-agnostic: hardware access goes through the board_*() API in + src/board_api.h. In ghostfat.c, check the UF2 block checks (magic, + flags, family ID) before board_flash_write(). + - path: "ports/**" + instructions: | + Port and board code implementing src/board_api.h. Flag anything that + can brick a device: erasing or writing the bootloader region outside + self-update, a wrong application start address, or jumping to the + application without validating it and restoring clocks, interrupts + and peripherals left enabled by the bootloader. Linker scripts and + board flash or RAM sizes must match the target MCU. Some files are + vendor SDK or third-party drivers modified for tinyuf2; review them + like local code, but do not ask to match upstream style. + - path: "ports/*/boards/**" + instructions: | + Board definitions. Check that pins, USB VID/PID and board names are + consistent with the other files of the same board. diff --git a/README.md b/README.md index fc5079bb7..e61931a61 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TinyUF2 Bootloader -[![Build Status](https://github.com/adafruit/tinyuf2/workflows/Build/badge.svg)](https://github.com/adafruit/tinyuf2/actions)[![License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT) +[![License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](https://opensource.org/licenses/MIT)[![Build Status](https://github.com/adafruit/tinyuf2/workflows/Build/badge.svg)](https://github.com/adafruit/tinyuf2/actions)![CodeRabbit Pull Request Reviews](https://img.shields.io/coderabbit/prs/github/adafruit/tinyuf2?utm_source=oss&utm_medium=github&utm_campaign=adafruit%2Ftinyuf2&labelColor=171717&color=FF570A&link=https%3A%2F%2Fcoderabbit.ai&label=CodeRabbit+Reviews)[![Greptile: The War on Bugs](https://www.greptile.com/badge.svg)](https://www.greptile.com/?utm_source=oss_badge&utm_medium=readme&utm_campaign=greptile_for_open_source) This repo is cross-platform UF2 Bootloader projects for MCUs based on [TinyUSB](https://github.com/hathach/tinyusb) From 9d4b38227a032221e4ade92c5ca24bbe4c2293d8 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Mon, 12 Oct 2026 22:30:49 +0700 Subject: [PATCH 16/47] Add a local HIL contract and fix blank CURRENT.UF2 on mimxrt10xx (#507) The hil skill's battery runner, roster and contract for mimxrt1060_evk, HIL selection in check_build.py, and BOARD_FLASH_ADDR_ZERO for mimxrt10xx so CURRENT.UF2 reads the flash image. --- .claude/skills/build/SKILL.md | 10 +- .claude/skills/build/scripts/check_build.py | 183 ++- .claude/skills/hil/SKILL.md | 188 +++ .claude/skills/hil/scripts/hil_test.py | 1355 +++++++++++++++++++ .claude/test/test_build.py | 223 ++- .claude/test/test_hil.py | 1258 +++++++++++++++++ .github/workflows/build.yml | 6 + .github/workflows/pre_commit.yml | 8 + .pre-commit-config.yaml | 2 +- CLAUDE.md | 8 +- ports/mimxrt10xx/boards.h | 2 + test/hil/local.json | 10 + 12 files changed, 3217 insertions(+), 36 deletions(-) create mode 100644 .claude/skills/hil/SKILL.md create mode 100755 .claude/skills/hil/scripts/hil_test.py create mode 100644 .claude/test/test_hil.py create mode 100644 test/hil/local.json diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index 37edb86ba..f6f027251 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -12,16 +12,17 @@ B=.claude/skills/build/scripts/check_build.py python3 $B --scope # boards a change affects python3 $B --base master # same, from the branch diff python3 $B --base master --worktree # same, plus uncommitted and untracked changes -python3 $B --select-only (--scope ... | --base REF [--worktree] | --diff-file FILE) # the selection alone +python3 $B --select-only (--scope ... | --base REF [--worktree] | --diff-file FILE) [--hil-config ROSTER] # the selection alone python3 $B --board feather_stm32f405_express [--board ...] python3 $B --board lpcxpresso55s69 -D LOG=2 # a define, passed to the board's backend python3 $B ... --shared # keep cmake-build/cmake-build-, the dir a flash or HIL run uses +python3 $B --board imxrt1060_evk --backend cmake --shared # the HIL's receipt: cmake whatever CI uses python3 $B ... --idf local # exported ESP-IDF instead of CI's docker image ``` -The last stdout line is JSON. An agent acts on `pass`; `firstError`, and `error` on exit 2, naming the cause or the remedy; `boards[]` (`status` `ok`, `failed` or `error`, `log`, `artifacts`, `notImplemented`); `built`, the artifacts of `--shared` roots, the only ones kept after a pass; `nothingToBuild` and `uncovered`, each path with its reason; and `idf.matchesCi`. `paths` gives every changed path's `rule` and `reason`, the answer to why a path selected what it did. Exit 0 pass, 1 a build, self-test or test failed, 2 usage, resolution or a missing prerequisite, 3 an uncovered path; 2 outranks 1, and 1 outranks 3. `-v` streams the build to stderr; each build's log is `/check_build.log`. +The last stdout line is JSON. An agent acts on `pass`; `firstError`, and `error` on exit 2, naming the cause or the remedy; `boards[]` (`status` `ok`, `failed` or `error`, `log`, `artifacts`, `notImplemented`); `built`, the artifacts of `--shared` roots, the only ones kept after a pass, with each one's hash in `boards[].sha256`; `nothingToBuild` and `uncovered`, each path with its reason; and `idf.matchesCi`. `input` and `defines` record what a build used (HEAD, `dirty`, submodule and `get_deps` dependency commits, `dirtyInputs` for any of those with local changes, `inputsDirtiedByBuild` for any that changed or moved while building, `.` for the checkout itself): the HIL contract binds firmware to this record. `paths` gives every changed path's `rule` and `reason`, the answer to why a path selected what it did. Exit 0 pass, 1 a build, self-test or test failed, 2 usage, resolution or a missing prerequisite, 3 an uncovered path; 2 outranks 1, and 1 outranks 3. `-v` streams the build to stderr; each build's log is `/check_build.log`. -`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". +`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`), which the battery never runs. ## Reporting @@ -32,11 +33,12 @@ An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the J - **Selection is the script's**, first match per path, and `paths[].reason` says why. A mixed change takes the union, and an uncovered path fails it whatever else built. The policy behind the rules: - A change to a port, board, app or dependency builds the boards that compile it: a board's own files that board; other port files the port's `REPRESENTATIVES` (rig boards first, both ESP32-S2 and -S3, both i.MX RT10xx and RT1170); an app or dependency the representatives of the ports whose CI backend compiles it. - Shared code (`src/`, `lib/`, the shared build files) and anything unclassified builds every port's representatives, plus the ghostfat self-test when a ghostfat input changed. - - A change to how boards build (this script, `build.yml`, `build_util.yml`, `ci_set_matrix.py`, `.github/actions/`) runs the contract's own tests and one smoke build per backend; no firmware build verifies workflow syntax or orchestration. A change to the contract's tests runs the tests only. + - A change to how boards build (this script, `build.yml`, `build_util.yml`, `ci_set_matrix.py`, `.github/actions/`) runs the contract's own tests and one smoke build per backend, and a change to this script the HIL battery on every roster board, since the HIL runner imports it; no firmware build verifies workflow syntax or orchestration. A change to the contract's tests runs the tests only. - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads. - Firmware no CI build compiles is uncovered: `ports/template_port/`, `ports/ch32v20x/` (`--board ch32v203_r0_1v0` builds it on request, with a note), a build file the port's CI backend never reads, a stub app. - Docs and agent, editor and repo metadata build nothing. - **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`; Espressif `idf.py build`. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds. A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. +- **`--backend cmake`** is for the HIL's receipt only, with `--board` and `--shared`: CI still builds each port with its `build.yml` backend, so it is no CI evidence. A port without `CMakeLists.txt`, or Espressif, is refused. `-D` takes the cmake spellings: `LOGGER=RTT`, not `rtt`; `DEBUG` is ignored, the build stays MinSizeRel; `LOG` sets `CFG_TUSB_DEBUG` only, not `TUF2_LOG`; raw flags go through `CFLAGS_CLI`. - **ESP-IDF** builds in CI's image, `espressif/idf:`, through docker by default; a missing image is exit 2 naming the `docker pull`. `--idf local` uses the exported IDF and records its version and `matchesCi`: evidence from a different IDF is supplemental, not acceptance. - **ghostfat** builds each `ports/test_ghostfat/boards/*` config natively and runs it against its known-good image in the private dir, one config at a time: `huge` needs about 3 GiB of disk while it runs. A passing config's dir is removed; a failing one keeps its log. - **Private dirs for parallel agents.** Each build starts from an empty `cmake-build/cmake-build-agent--` (with `-` when a board builds with another backend than its port's) and is removed when it passes (`removed`; its `artifacts` say what it produced, nothing is left to flash); a failed one is kept for its log. Nothing is written into the source tree: make takes `BUILD_ROOT=`, cmake and idf `-DARTIFACT_PATH=/_bin`. `--shared` rebuilds `cmake-build/cmake-build-` from clean and keeps it for flashing; it needs exclusive ownership of that board. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index f1020e55a..3719507dd 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -2,7 +2,7 @@ """Build TinyUF2 for the boards a change affects, or for named boards: the project's build contract. check_build.py (--scope PATH... | --base REF [--worktree] | --board B...) [-D VAR=VAL]... [--shared] [--idf docker|local] [-v] - check_build.py --select-only (--scope PATH... | --base REF [--worktree] | --diff-file FILE) + check_build.py --select-only (--scope PATH... | --base REF [--worktree] | --diff-file FILE) [--hil-config ROSTER] Each board builds the way .github/workflows/build.yml builds its port (make, cmake or ESP-IDF) into its own build root, cmake-build/cmake-build-agent-- (--shared: @@ -16,6 +16,7 @@ import argparse import functools import gzip +import hashlib import json import os import re @@ -73,8 +74,12 @@ '.github/workflows/build_ghostfat.yml') CORE = ('src/', 'lib/', 'ports/make.mk', 'ports/rules.mk', 'ports/family_support.cmake', 'CMakeLists.txt', 'cmake/', 'tools/get_deps.py') -CONTRACT_TESTS = '.claude/test/test_build' -BUILD_CONTROL = ('.claude/skills/build/scripts/', '.github/workflows/build.yml', +CONTRACT_TESTS = ('.claude/test/test_build', '.claude/test/test_hil') +# the HIL contract's runner and rosters: a change there re-runs the battery on every roster board +HIL_CONTROL = ('.claude/skills/hil/scripts/', 'test/hil/') +# hil_test.py imports these: the identity, artifacts and build dirs the battery trusts +HIL_IMPORTS = '.claude/skills/build/scripts/' +BUILD_CONTROL = (HIL_IMPORTS, '.github/workflows/build.yml', '.github/workflows/build_util.yml', '.github/workflows/ci_set_matrix.py', '.github/actions/') NOTHING = ('.claude/', '.github/', '.idea/', 'tools/codespell/', 'tools/gen_boards.py', 'tools/touch1200.py', 'ports/build_all.py', 'CLAUDE.md', 'AGENTS.md', '.agents', 'LICENSE', '.gitignore', '.gitmodules', @@ -208,16 +213,25 @@ def ports_using_app(app): @functools.cache -def deps_rows(): - """{dependency path: [families]} from tools/get_deps.py's deps_all.""" +def get_deps_table(): + """{dependency path: (pinned commit, [families])} from tools/get_deps.py's deps_all.""" try: # run_path, not an import: no __pycache__ written into tools/ rows = runpy.run_path(str(ROOT / 'tools' / 'get_deps.py'), run_name='get_deps')['deps_all'] - return {dep: row[2].split() for dep, row in rows.items()} + return {dep: (row[1], row[2].split()) for dep, row in rows.items()} except Exception as e: raise Fail(f'cannot read deps_all from tools/get_deps.py: {type(e).__name__}: {e}') +def deps_rows(): + """{dependency path: [families]} from tools/get_deps.py's deps_all.""" + return {dep: families for dep, (_, families) in get_deps_table().items()} + + +def deps_pins(): + return {dep: pin for dep, (pin, _) in get_deps_table().items()} + + def deps_families(path): """Ports whose dependency rows in tools/get_deps.py cover the dependency directory `path`.""" for dep, families in deps_rows().items(): @@ -234,18 +248,24 @@ def classify(path): """One path's rule: effect none (nothing to build), gap (uncovered), select or core, the boards it selects as {(board, backend): reason}, and whether it adds the ghostfat self-test or the contract's own tests.""" - def out(rule, effect, reason, boards=None, ghostfat=False, tests=False): + def out(rule, effect, reason, boards=None, ghostfat=False, tests=False, hil=False): return {'path': path, 'rule': rule, 'effect': effect, 'reason': reason, 'boards': boards or {}, - 'ghostfat': ghostfat, 'tests': tests} + 'ghostfat': ghostfat, 'tests': tests, 'hil': hil} ghostfat = path.startswith(GHOSTFAT_INPUTS) parts = path.split('/') if path.startswith(CONTRACT_TESTS): return out('contract-tests', 'select', "the contract's own tests; a test edit changes no board build", tests=True) + if path.startswith(HIL_CONTROL): + return out('hil-control', 'select', 'the HIL runner or a roster changed: its tests and the battery on every ' + 'roster board', tests=True, hil=True) if path.startswith(BUILD_CONTROL): boards = {(b, be): f'{be} smoke build for {path}' for be, b in SMOKE.items()} + runner = path.startswith(HIL_IMPORTS) return out('build-control', 'select', 'how boards build changed: the contract tests and one smoke build ' - 'per backend; no firmware build verifies workflow syntax or orchestration', boards, tests=True) + 'per backend; no firmware build verifies workflow syntax or orchestration' + + ('; the HIL runner imports it: the battery on every roster board' if runner else ''), + boards, tests=True, hil=runner) if path.endswith('.md'): return out('docs', 'none', 'documentation') codeql_port, codeql_board = codeql_build() @@ -332,14 +352,56 @@ def changed_paths(base=None, worktree=False, scope=None, diff_file=None): return sorted(set(paths)), info -def identity(): - """What a result was built from: HEAD, whether the tree differs from it, the submodule commits.""" +def own_checkout(path): + """Whether `path` is a git checkout of its own: git -C on a plain or empty directory (an + uninitialised submodule, an unpacked dependency) silently answers for the parent repository.""" + r = subprocess.run(['git', '-C', str(ROOT / path), 'rev-parse', '--show-toplevel'], capture_output=True, text=True) + return r.returncode == 0 and os.path.realpath(r.stdout.strip()) == os.path.realpath(ROOT / path) + + +def git_in(path, *args): + r = subprocess.run(['git', '-C', str(ROOT / path), *args], capture_output=True, text=True) + if r.returncode != 0: + raise Fail(f'git -C {path} {" ".join(args)} failed: {r.stderr.strip()}') + return r.stdout + + +def dirty_inputs(paths): + return [p for p in paths if git_in(p, 'status', '--porcelain', '--untracked-files=all').strip()] + + +def identity(ports=()): + """What a result was built from: HEAD, whether the tree differs from it, the submodule commits, the + commits of the get_deps dependencies `ports` use, and which of those inputs cannot be trusted: + local changes, not a checkout of its own, or a dependency off its get_deps pin.""" subs = {} for line in git('submodule', 'status').splitlines(): sha, path = line[1:].split()[:2] subs[path] = sha - return {'head': git('rev-parse', 'HEAD').strip(), 'dirty': bool(git('status', '--porcelain').strip()), - 'submodules': subs} + used = [dep for dep, families in sorted(deps_rows().items()) if any(p in families for p in ports)] + # worktree deps are symlinks into the primary checkout, invisible to this checkout's status + missing = [p for p in [*subs, *used] if not own_checkout(p)] + deps = {dep: git_in(dep, 'rev-parse', 'HEAD').strip() for dep in used if dep not in missing} + pins = deps_pins() + info = {'head': git('rev-parse', 'HEAD').strip(), 'dirty': bool(git('status', '--porcelain').strip()), + 'submodules': subs, 'deps': deps, 'missingInputs': missing, + 'depsOffPin': [d for d, sha in deps.items() if sha != pins[d]]} + info['dirtyInputs'] = dirty_inputs(own_inputs(info)) + return info + + +def own_inputs(info): + """The submodules and dependencies `identity` could check, i.e. those that are checkouts of their own.""" + return [p for p in [*info['submodules'], *info['deps']] if p not in info['missingInputs']] + + +def moved_inputs(before, after): + """Inputs whose identity differs between two `identity` records: '.' for the checkout's HEAD or dirty + state, else each submodule or dependency whose commit or checkout changed.""" + moved = {'.'} if (before['head'], before['dirty']) != (after['head'], after['dirty']) else set() + commits = [{**i['submodules'], **i['deps']} for i in (before, after)] + moved |= {p for p in commits[0].keys() | commits[1].keys() if commits[0].get(p) != commits[1].get(p)} + return moved | (set(before['missingInputs']) ^ set(after['missingInputs'])) def select(paths): @@ -362,11 +424,69 @@ def path_view(records, effect=None, keys=('path', 'rule', 'effect', 'reason')): return [{k: r[k] for k in keys} for r in records if effect in (None, r['effect'])] -def manifest(sel, info): +def hil_selection(sel, config): + """Roster boards the selection puts under the HIL battery. A board's own change reaches that board, + an app change none, any other change every roster board of the ports it selects, a core or HIL-control + change the whole roster; what no roster board on this host can run is uncovered, whatever else was + selected.""" + try: + roster = sorted({b['name'] for b in json.loads((ROOT / config).read_text())['boards']}) # TinyUF2 names + except (OSError, ValueError, KeyError, TypeError, AttributeError) as e: + raise Fail(f'cannot read boards from {config}: {e}') + by_port = {} + for name in roster: + by_port.setdefault(port_of(name), []).append(name) + def kind(parts): # the HIL builds with cmake whatever CI uses: its build files decide, not CI's + if parts[0] == 'ports' and (len(parts) == 2 or parts[1] in BACKENDS): + return build_file_kind(parts[-1]) + return None + records = [(r, parts, kind(parts)) for r in sel['records'] for parts in [r['path'].split('/')]] + full = any(r['effect'] == 'core' and k != 'make' for r, _, k in records) + boards, uncovered = set(), {} + if full: + boards.update(roster) + for p in sorted(set(BACKENDS) - set(by_port)): + uncovered[p] = f'core change: no {config} board for {p}' + for r, parts, k in records: + if r['hil']: + boards.update(roster) + elif k == 'make': + uncovered[r['path']] = f'the HIL builds with cmake, which never reads {r["path"]}' + # the battery builds no port app, and a core file reaches every roster board above + elif k == 'cmake' and len(parts) >= 3 and r['effect'] != 'core' and parts[2] != 'apps': + if parts[2] == 'boards' and len(parts) >= 5: + own = [parts[3]] if parts[3] in roster else [] + else: + own = by_port.get(parts[1], []) + if own: + boards.update(own) + else: + uncovered[r['path']] = f'no {config} board builds {r["path"]}' + elif r['effect'] == 'gap': # firmware no build covers is no HIL board's either + uncovered[r['path']] = r['reason'] + elif r['rule'] in ('app', 'port-app'): + uncovered[r['path']] = f'the battery runs the bootloader and its generated fixture, never {r["path"]}' + elif r['rule'] == 'board': + (b, _), = r['boards'] + if b in roster: + boards.add(b) + else: + uncovered[b] = f'{r["path"]}: {b} is not in {config}' + else: + for p in sorted({port_of(b) for b, _ in r['boards']}): + if p in by_port: + boards.update(by_port[p]) + else: + uncovered[p] = f'{r["path"]}: no {config} board for {p}' + return {'config': config, 'full': full or any(r['hil'] for r in sel['records']), 'boards': sorted(boards), + 'uncovered': [{'target': t, 'reason': why} for t, why in sorted(uncovered.items())]} + + +def manifest(sel, info, hil_config=None): ports = {} for (b, backend) in sel['boards']: ports.setdefault(port_of(b), []).append(b) - return { + out = { 'version': 1, 'input': info, 'build': { @@ -379,6 +499,9 @@ def manifest(sel, info): 'paths': path_view(sel['records']), }, } + if hil_config: + out['hil'] = hil_selection(sel, hil_config) + return out # ------------------------------------------------------------------ building @@ -444,6 +567,7 @@ def expected_artifacts(port, board, backend, root): cml = (ports_dir(port) / 'CMakeLists.txt').read_text() for app in re.findall(r'add_subdirectory\(\$\{TOP\}/apps/(\w+)', cml): exp.append(bin_dir / 'apps' / f'{"update-tinyuf2" if app == "self_update" else app}.uf2') + exp += [bin_dir / a for a in re.findall(r'\$\{ARTIFACT_PATH\}/(apps/[\w.-]+\.uf2)', cml)] return exp return [bin_dir / f for f in ('tinyuf2.bin', 'bootloader.bin', 'partition-table.bin', 'combined.bin', 'apps/update-tinyuf2.uf2', 'apps/blinky.uf2')] @@ -539,7 +663,9 @@ def build_board(board, backend, args, idf): return rec rec['artifacts'] = sorted(str(p.relative_to(root)) for p in (root / '_bin').rglob('*') if p.is_file()) rec['artifacts'] += sorted(str(e.relative_to(root)) for a in apps for e in a.glob('*.elf')) - if not args.shared: + if args.shared: + rec['sha256'] = {a: hashlib.sha256((root / a).read_bytes()).hexdigest() for a in rec['artifacts']} + else: shutil.rmtree(root) rec['removed'] = True return rec @@ -591,8 +717,8 @@ def run_ghostfat(verbose): def run_tests(): - """The contract's own tests, for a change to how boards build.""" - r = subprocess.run([sys.executable, '-m', 'unittest', 'discover', '-s', '.claude/test', '-p', 'test_build*.py'], + """The .claude script tests, for a change to how boards build or to the HIL contract.""" + r = subprocess.run([sys.executable, '-m', 'unittest', 'discover', '-s', '.claude/test', '-p', 'test_*.py'], capture_output=True, text=True, cwd=ROOT) last = [l for l in r.stderr.splitlines() if l.strip()] return {'status': 'ok' if r.returncode == 0 else 'failed', 'summary': last[-1] if last else '', @@ -614,9 +740,12 @@ def main(argv=None): src.add_argument('--diff-file', metavar='FILE', help='changed paths, one per line (--select-only)') ap.add_argument('--worktree', action='store_true', help='with --base: add uncommitted and untracked files') ap.add_argument('--select-only', action='store_true', help='print the selection manifest and stop') + ap.add_argument('--hil-config', metavar='ROSTER', help="with --select-only: the host's HIL roster, adds `hil`") ap.add_argument('-D', '--define', action='append', default=[], metavar='VAR=VAL', help="passed to the board's backend: make VAR=VAL, cmake/idf -DVAR=VAL") ap.add_argument('--shared', action='store_true', help='build into cmake-build/cmake-build- and keep it') + ap.add_argument('--backend', choices=('cmake',), + help="with --board and --shared: build with cmake whatever CI uses (the HIL's receipt)") ap.add_argument('--idf', choices=('docker', 'local'), default='docker', help="ESP-IDF: CI's pinned docker image (default) or the exported local IDF") ap.add_argument('-v', '--verbose', action='store_true', help='stream build output to stderr') @@ -625,6 +754,10 @@ def main(argv=None): ap.error('--worktree needs --base') if args.diff_file and not args.select_only: ap.error('--diff-file is for --select-only') + if args.hil_config and not args.select_only: + ap.error('--hil-config is for --select-only') + if args.backend and not (args.board and args.shared): + ap.error('--backend needs --board and --shared') if args.select_only and args.board: ap.error('--select-only takes --scope, --base or --diff-file') @@ -634,9 +767,11 @@ def main(argv=None): boards = {} for b in args.board: port = port_of(b) - backend = backend_of(port) + backend = args.backend or backend_of(port) if not backend: raise Fail(f'ports/{port}: {UNBUILT[port][1]}') + if backend == 'cmake' and (backend_of(port) == 'idf' or not (ports_dir(port) / 'CMakeLists.txt').is_file()): + raise Fail(f'ports/{port} has no cmake build') boards[(b, backend)] = 'named' sel = {'full': False, 'boards': boards, 'ghostfat': False, 'tests': False, 'records': []} info = {'mode': 'boards'} @@ -644,9 +779,10 @@ def main(argv=None): paths, info = changed_paths(args.base, args.worktree, args.scope, args.diff_file) sel = select(paths) if args.select_only: - print(json.dumps(manifest(sel, info))) + print(json.dumps(manifest(sel, info, args.hil_config))) return 0 - info.update(identity()) + ports = {port_of(b) for b, _ in sel['boards']} + info.update(identity(ports)) result['input'] = info idf = {} if any(backend == 'idf' for _, backend in sel['boards']): @@ -659,6 +795,10 @@ def main(argv=None): records = [build_board(b, backend, args, idf) for (b, backend) in sel['boards']] for r in records: r['selectedBy'] = sel['boards'][(r['board'], r['backend'])] + # an input that moved while building, even if restored before the HIL checks it, leaves firmware + # the record cannot vouch for; hil_test.py refuses any entry here + now = identity(ports) + info['inputsDirtiedByBuild'] = sorted((set(now['dirtyInputs']) - set(info['dirtyInputs'])) | moved_inputs(info, now)) ghostfat = run_ghostfat(args.verbose) if sel['ghostfat'] else [] tests = run_tests() if sel['tests'] else None nothing = path_view(sel['records'], 'none', ('path', 'reason')) @@ -672,6 +812,7 @@ def main(argv=None): 'ghostfat': ghostfat, 'built': [f"{r['buildDir']}/{a}" for r in records if not r.get('removed') for a in r['artifacts']], 'resolution': 'full' if sel['full'] else 'boards' if args.board else 'selected', + 'defines': args.define, 'paths': path_view(sel['records']), 'nothingToBuild': nothing, 'uncovered': uncovered, diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md new file mode 100644 index 000000000..da64bbbbf --- /dev/null +++ b/.claude/skills/hil/SKILL.md @@ -0,0 +1,188 @@ +--- +name: hil +description: The TinyUF2 HIL contract. Use when a change needs hardware evidence on a rig board, when a debug or validation session must take, flash, clean up or release a TinyUF2 rig board, or when a workflow asks for "the project's HIL contract". Runs the bootloader battery on the roster's boards, owns the board lock, and reports pass/fail per board with target-side evidence. +--- + +# HIL contract + +`scripts/hil_test.py` holds the mechanics: roster and source checks, firmware binding, locking, the probe and USB steps, cleanup and the JSON report. Agents call it instead of composing J-Link, mtools or lock commands. This page is the judgment. + +## Roster + +- The roster is `test/hil/local.json`, the boards on htpc's bench. On a machine without a board, its run fails at `identity`, where J-Link finds no probe with that serial, before anything is written. ci.lan has no TinyUF2 rig yet (a follow-up). +- `TINYUSB_DIR` must name the tinyusb checkout whose `test/hil/helper/hil_lock.py` owns the board locks; every command refuses without it. Set it in the login profile, which headless sessions (chief, `hil-operator`) inherit. + +A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. The runner uses SWD at 4000 kHz; pass the same to those scripts, `--interface swd --speed 4000`. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: +- the board lock name (`lock`) when tinyusb names the board differently, as `mimxrt1060_evk` for `imxrt1060_evk`; both runners take that one lock, and the helper's `status`, holder records and the report rows' `lock` show it. It defaults to the board name; +- the commissioned re-entry path (`reentry`). + +The battery runs a board only after commissioning sets `reentry` and confirms `uid`; `install` and `invalidate` work without `reentry`, for commissioning. Only commissioned roster boards can be offered for a run. Adding a board is commissioning, the lead's (below), never a review-time or dispatch-time action: the runner refuses a build record from a dirty tree, an uncommitted roster entry included. + +## Locks + +Locks are tinyusb's: one kernel flock per board in `/tmp/tinyusb-hil-locks/`, taken through `$TINYUSB_DIR/test/hil/helper/hil_lock.py`. A board's lock is ``: its roster `tinyuf2.lock`, else the board name. It is tinyusb's name for the board, so TinyUF2 and tinyusb runs exclude each other; the helper's `hold` and `release` take it and `status` (no board argument) shows it, while `hil_test.py -b` and the probe scripts take the board name. +- **The battery locks itself,** as `hil_test.py`, which tinyusb's `release` refuses to kill. Never pre-hold a board you are about to run the battery on. +- **The lock fails closed.** An unusable lock dir refuses the run (exit 2); it never runs unlocked. `HIL_NO_BOARD_LOCK=1` bypasses the lock only when the task scope names forcing that board. The report then says `lockBypassed: true`. +- **Manual sessions** (hw-debugger, hw-validator): + 1. Take your own hold: `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py hold --reason ""`. + 2. Program a bootloader BIN, and invalidate the board when done: + + ```bash + python3 .claude/skills/hil/scripts/hil_test.py install -b --firmware --sha256 --under-hold "" test/hil/local.json + python3 .claude/skills/hil/scripts/hil_test.py invalidate -b --under-hold "" test/hil/local.json + ``` + + `install` writes one image from the board's flash base (the BIN, 0xFF up to the app start, then a zero page), reads it back, and refuses a BIN that is outside the install range or lacks this board's `UF2_BOARD_ID`; it needs the automount exclusion rule (Commissioning). `invalidate` is the cleanup (step 7 below). Both refuse unless the hold is provably yours: the record's reason, a live pid, a FLOCK on the lock file in that pid's own fds, and a failing probe of the flock. + 3. Release with `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py release `, only after an `invalidate` that exited 0 with no `survivors`. Otherwise the board is not cleaned up: keep the hold and report it as a blocker. + + Manual exits: 0 the operation succeeded (for `invalidate`, the zero page at the flash floor verified from a new session); 1 it failed and the board's state is unknown; 2 refused before any hardware access; 4 a child survived SIGKILL, so keep the hold until it is gone. + + On a refused hold, report the holder from `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` (no board argument; it prints every holder); never kill it. A `hil_test.py` holder whose record has `role: cleanup-holder` never frees on its own: report it as a blocker that needs the recovery dispatch (Exit 4), without waiting. Any other `hil_test.py` holder is a battery run; wait for it within the lock-wait budget. +- **Cleanup state** for a hw-validator or hw-debugger dispatch: no firmware is restored. `invalidate --under-hold` leaves the board unbootable, and its `cleanup.evidence` (the zero page at the flash floor verified from a new session after the reset) is the observation that Cleanup records. Whoever uses the board next flashes it first; tinyusb's HIL does. +- **Lock-wait budget:** htpc has no CI runner, so it is the dev-bench row of chief's budget, 2 min, unless the dispatch names another. + +## Run the battery + +```bash +mkdir -p cmake-build/hil +python3 .claude/skills/build/scripts/check_build.py --board --backend cmake --shared > cmake-build/hil/.build.json +python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json test/hil/local.json +``` + +`` is the roster name, the TinyUF2 board name. + +- **Admission.** Before taking the lock the run checks the automount exclusion rule, and every span it could write (BIN, zero page, the device's flash limits, board id), with the install and app spans taken from the candidate's source. +- **Build record.** It is this contract's build receipt: build it on the clean, pinned HEAD and pass it with `--build-record`. It must be a cmake build (`--backend cmake`), whatever backend CI builds the port with; board facts are read from the port's CMake files too. The run refuses a build record that is not clean, does not match the checkout now (HEAD, submodules, `get_deps` dependencies at their pins), or whose artifacts changed since. Commit first. A debug build goes through `install`, not the battery. +- **Background.** Run it in the background; one board takes about 2 minutes. Use one board per run. +- **Selection.** There are no examples or test names to select: a run is the whole battery on one board, and the board is the only selection. + +The battery's steps on one board, all under one lock: +1. **identity:** the probe is selected by serial. +2. **install:** one image is written from the flash base and read back: the candidate bootloader, 0xFF up to the app start, then a zero page. +3. **dfu:** the UF2 drive with this board's serial is found. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. +4. **copy:** a generated app is copied to the drive. +5. **app-ran:** the probe shows the app counting in thread mode with VTOR at the app start. +6. **reenter:** the bootloader is re-entered by the commissioned path, and the drive's `CURRENT.UF2` reads the app back. +7. **cleanup** (reported as `cleanup`, not a step): a zero page is written at the device's flash floor (`FLASH_LIMITS`, never the candidate's layout) and read back, the target reset, and the zero page verified again from a new session, so the board boots nothing (on MIMXRT1062 with BOOT_MODE 10b, Internal Boot, an error falls to the Serial Downloader: IMXRT1060RM rev 4 §9.3.5 p199). + +A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; a run triggered by the upload alone is unverified. "reenter" by `retained-register` is the forced-entry path, not a physical double tap. + +`--fail-at ` exists only for acceptance: it fails that step to exercise cleanup. + +## Reporting + +The last stdout line is JSON, also written to its `reportPath`; read that path, never an older report. Its fields: +- `pass`, `caveat`; +- per board: `ran`, `pass`, `locked`, `lockBypassed`, `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`), `jlinkRetried` (J-Link sessions run again after a crash before the target connect; Limits), and `detail`; +- `error` on a refusal or an operation failure. + +| Exit | Meaning | +|---|---| +| 0 | every board passed and was cleaned up | +| 1 | a board failed or did not run; it was cleaned up or never touched | +| 2 | refused before any hardware access (usage, roster, source, build record, lock helper, lock dir) | +| 4 | cleanup failed; the board is held when `cleanup.heldAfter` (not under `HIL_NO_BOARD_LOCK`). From `install` or `invalidate`: a child survived; keep your hold until it is gone | + +A delegated run (`hil-operator`) returns `{pass, results, caveat, wedged}` and, beside them, `reportPaths`: +- `pass`, `results` and `caveat` are copied verbatim from the JSON; +- `wedged` lists the boards the operator saw stop responding; +- `reportPaths` maps each board to its JSON's `reportPath`, which a recovery dispatch needs. +- With several boards, the operator runs one battery per board, one after another: `results` holds every board's row, `pass` is true only when every run's is, and `caveat` joins the runs' non-empty caveats. +- A launched run that ended without printing its JSON (killed, crashed) may have programmed the board and left it running a candidate. The operator writes `pass: false`, `caveat: "run ended without a report; board state unknown"` and one row per board, exactly `{board, ran: null, pass: false, locked: false, detail}`, with `detail` naming the exit status, the output tail and `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` at that moment. +- The operator runs the battery once and returns. It never waits on a lock and never retries: a `locked` row comes back as is, and the caller rules below are the caller's (chief's). +- A recovery dispatch names the board, the holder pid and the failed run's `reportPath`. It runs `invalidate --recover-holder` in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Exit 4). + +### Review mapping + +A review (`pr-review`) maps each board's row, except a commissioning run's (`commissioning: true`, from `--try-reentry`), which is never validation. A blocker takes precedence over every other mapping: +- `pass`: `ran`, `pass` and `cleanup.pass` all true; +- `fail`: `ran` and not `pass`, with a failing step other than `identity`, and that step's `detail`; +- `not-run`: `ran` false (`locked` or a refusal), or a failing `identity`, with its `detail`. `identity` runs before anything is written, so its failure is the rig's: no probe with that serial on this machine, or one unplugged or dead; +- a blocker, never a `fail` row: exit 4, `ran: null` (no report), and an abort, which is `ran` and not `pass` with no failing step (an interruption or an unexpected error, named in the board's `detail`). + +`testedHead` is the build record's `input.head`. `regression` is `unknown` unless the battery also ran on the merge base. + +### Caller rules +- **Retry.** A failed test may be retried once, and only when no board is `locked`, `wedged` or `heldAfter`, `caveat` is empty, and every board was cleaned up. + - Keep both reports. A fail-then-pass is a pass with `flaky: true`, quoting both. + - An abort (a failed run with no failing step), a cleanup failure, `heldAfter`, `wedged` or a missing report is never retried; it is a blocker until diagnosed. + - A board whose run ended without a report gets a manual `invalidate --under-hold` before its next run, and only once no such process is left on the host. Killing the runner frees its lock but not its children, which `invalidate` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: + + `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'` + + The last one also covers the runner's `fat-read` subcommand. While a process is left, or this cannot be established, the blocker stays. +- **Locked.** `locked` is an admission refusal, not a test failure. The caller admits again after the holder releases, within the lock-wait budget; the operator does not wait. Otherwise report the board not covered, naming the holder. +- **Exit 4.** + - The board stays held by a detached holder forked from the run: reason `hil_test.py`, `role: cleanup-holder`, with its pid in `cleanup.holder`. It ignores SIGTERM and SIGINT, so a timeout kill cannot free the board; only recovery releases it. + - If no holder could be forked, `cleanup.holder.inProcess` is set and the runner itself is the holder: it prints the report, closes its stdio and does not exit until recovered, so no completion notice arrives. When a run's notice is overdue, read the task's output: a final JSON line with `cleanup.holder.inProcess` true is this case. Do not wait for its exit. + - `cleanup.survivors` lists children that outlived SIGKILL (a J-Link, mtools or FAT read stuck on the device); recovery refuses while any of them is still alive. + - tinyusb sees that holder as a battery run and waits, which is safe but means the board is out of service until recovered. + - Do not release it by hand, and do not treat it as safe because tinyusb reflashes. The holder may stand for a stuck J-Link or mtools process. + - Recovery is a separate `hil-operator` dispatch whose scope names forcing that board. After process, probe and host readiness are established, it runs: + + `python3 .claude/skills/hil/scripts/hil_test.py invalidate -b --recover-holder --report test/hil/local.json` + + The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, invalidates and verifies while the lock stays held, and only then releases the holder. A second recovery of the same board while one runs is refused (exit 2). + - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide). + +## Selection + +`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config test/hil/local.json` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered: the battery runs the bootloader and its generated fixture, never an app. The runner, the build scripts it imports and the roster (`.claude/skills/hil/scripts/`, `.claude/skills/build/scripts/`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. + +## Commissioning a board + +Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under a manual hold, except for the re-entry run, which locks itself. Record each result in the board's commissioning note (agentrc `target-debug` project notes), and `uid`, `lock` and `reentry` in the roster. In this order: +- **Cabling:** cable the USB port TinyUF2 enumerates on (`BOARD_TUD_RHPORT`). It can differ from the one tinyusb examples use: P10, not P9, on lpcxpresso55s69; USB OTG1 on imxrt1060_evk. A DFU device "not seen" with no new device in `dmesg` is a cabling problem first. Keep the probe on a hub apart from the target's (Limits). +- **Silicon revision,** before any other result: read it from the chip marking or registers, and check that revision's errata through read-doc. RT1060's DIGPROG does not tell A from B, so the marking decides. +- **Automount exclusion:** `/etc/udev/rules.d/99-tinyuf2-hil.rules`, one line per board, with the VID and PID from the board's source, in exactly this form (the runner accepts only this form, with exact `==` matches): + + ``` + SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ATTRS{serial}=="", ENV{UDISKS_IGNORE}="1" + ``` + + Write it with the roster's `uid`, before the serial can be confirmed: `install` and `run` refuse without the rule, and a fresh board runs whatever tinyusb last flashed, not TinyUF2. A serial a running tinyusb example reports (`lsusb -v`) is an early cross-check, not a confirmation. Until the serial is confirmed, also inhibit automount for every board of this VID:PID with a temporary `/etc/udev/rules.d/98-tinyuf2-hil-commissioning.rules`: + + ``` + SUBSYSTEM=="block", ATTRS{idVendor}=="", ATTRS{idProduct}=="", ENV{UDISKS_IGNORE}="1" + ``` + + Then `sudo udevadm control --reload`. +- **Bootloader:** `install` a known-good TinyUF2 BIN under the hold and observe DFU. +- **USB serial:** confirm the enumerated serial equals `uid` (sysfs or `lsusb -v`), and record the USB port paths. If it differs, fix `uid` and the rule together, reload, and `install` again; its reset re-enumerates the board and udev applies the reloaded rules, so no replug should be needed (by mechanism, not yet observed on a uid fix). +- **Cleanup:** `invalidate` under the hold (its `cleanup.evidence` is the zero page verified from a new session), then confirm the ROM fallback (on MIMXRT1062, the serial downloader's USB device on the board's port), and that `install` succeeds from that state and DFU comes up. Record the reset notes J-Link prints with no boot image (`BENIGN`). +- **Exclusion proof:** delete the temporary rule and reload before the re-entry run below; its `dfu` step passes only when `udevadm info` on the enumerated block device shows `UDISKS_IGNORE=1` (`check_exclusive`). A host rule that ignores a whole VID, such as tinyusb's `88-tinyusb.rules` for 239a, sets the same property, so the proof cannot tell it from the serial rule; the runner still requires the serial line. +- **Loader footprint:** for each `loadbin` shape the runner issues, place sentinels just past the written range and elsewhere in its erase block. Confirm them from a fresh session before the operation under test, and again from another fresh session after it: a verify in the session that wrote them cannot show what a later operation destroys. Make the written data differ from what is on the flash, since a `loadbin` whose data already matches is skipped ("Contents already match") and erases nothing. Record the result in Limits. +- **Fixture prerequisites,** before anything executes the battery's fixture: + - the app start meets VTOR's alignment for the fixture's vector table (`fixture()`); + - its stack top, the `m_data` origin the runner reads (`ram_origin`) + 0x1000, is in RAM enabled when the bootloader hands off: the ROM default per read-doc, which TinyUF2 does not reconfigure (DTCM on imxrt1060_evk); + - under a hold, a word just below that stack top takes a written value and reads it back, and then its saved value (0x20000FFC on imxrt1060_evk). +- **Re-entry:** commit the roster entry without `reentry`, build the record, release the manual hold, and run the battery in commissioning mode; it locks itself and ends with the cleanup, so it also proves a run from and back to an invalidated board: + + ```bash + python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json --try-reentry retained-register test/hil/local.json + ``` + + `--try-reentry` is the lead's only, and is refused once `reentry` is set. The row's `reentryDecision` names the method when the run passed or failed only at `reenter`'s `CURRENT.UF2` readback, every observation of the tried method is true (`controlBootedApp`, `dfuCameUp` and `magicConsumed` for `retained-register`), and the board was cleaned up and not held. Set `reentry` to it and commit. A null decision leaves `reentry` unset: file a `followup` issue with the observations for diagnosis, not as a diagnosed bug. `app-invalidate` is set only after that path is validated on its own. When the drive copy itself fails, a fixture installed by hand with J-Link `loadbin` can isolate re-entry from the copy for diagnosis; it never sets `reentry`. +- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. + +## Limits + +- Covered: imxrt1060_evk on htpc. Not covered: ports without a cmake build, lpc55 among them (no battery, `install` or `invalidate` for lpcxpresso55s69; its 0A silicon also hangs TinyUF2 on a UF2 copy, #500), ESP32 boards (#502), self-update, an interrupted update, malformed UF2 input and the physical double tap. +- Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. +- The HIL's cmake image logs on the board UART: cmake builds leave `TUF2_LOG` at its default 1 (`src/board_api.h`), while make builds, the make ports' CI artifacts and the make-built releases pass 0. Its blocking writes at start, before the app jump and on DFU entry are timing the make image does not have. +- Cleanup leaves the board unbootable, so the next user, tinyusb's HIL included, must flash before it runs anything. Measured on imxrt1060_evk on 2026-10-12: + - the board then enumerates as the ROM serial downloader (1fc9:0135), and `install` from that state brings up DFU; + - J-Link prints its reset notes (`BENIGN`) with and without a boot image; + - with no boot image, a new J-Link session reads the flash wrong until a `loadbin` has run (zeros, and at 0x60010000 the bytes written at 0x60000200), so a plain `verifybin` fails. The cause is unknown. `invalidate` therefore re-verifies with a `loadbin` of the zero page that J-Link must report as "Skipped. Contents already match", then a `verifybin`. +- Never use a port's CMake flash or erase targets on a rig board: `tinyuf2-erase-external-jlink` erases [FLASH_BASE, FLASH_BASE + 64 KiB), the bootloader and the zero page with it. +- J-Link's QSPI loader on imxrt1060_evk is undocumented: SEGGER UM08001 has nothing on it. Measured at sampled sentinel points only: + - the install image (the cmake BIN, 0xFF up to 0x6000C000, then the zero page, ending at 0x6000C200) left 0x6000C200 and 0x6000FF00 reading 0xFF in the same session, before any reset. Earlier, a 0.35.0 release BIN `loadbin` (30304 B) erased every point measured past it in its block (0x60007800, 0x6000A000, 0x6000C000, 0x6000C200). J-Link reports 65536 bytes per operation, so [image end, 0x60010000) is treated as lost; + - a sentinel at 0x60010000, in the next block, survived both; + - the cleanup's zero-page `loadbin` kept sentinels at 0x60000200, 0x6000A000, 0x6000FF00 and 0x60010000, and 16-byte sentinel `loadbin`s kept their neighbours in the same block, so what makes J-Link erase the rest of a block is not established. + - That a candidate BIN of another length behaves the same is an inference, not a measurement. + - That `loadbin` erases only the 64 KiB blocks its range touches, and never the chip, is an unverified premise these points support. + Under that premise, losses stay inside [0x60000000, 0x60010000), in regions the battery rewrites or never reads, while the install span and the zero page lie in that block. +- `JLinkExe` sometimes crashes in `libjlinkarm` at the probe connect on htpc (SIGSEGV; once a glibc double-free abort); the cause is unknown. A session that crashed this way, before the target connect, runs once more after 2 s (`jlinkRetried`); a second crash fails the run. Seen in 2 of 94 runner sessions on 2026-10-12. +- A device hung mid-write makes the host controller retry its enumeration, which stalled a J-Link connect on the same hub for ~80 s (lpcxpresso55s69); that the hub is the cause is a hypothesis. +- If no cleanup holder can be forked (exit 4 with `cleanup.holder.inProcess`), the runner never exits until recovered. +- No reproducer for an instrumented build: the battery refuses a dirty build, and `install` only programs. A claim that needs one is reported pending. diff --git a/.claude/skills/hil/scripts/hil_test.py b/.claude/skills/hil/scripts/hil_test.py new file mode 100755 index 000000000..193bf10d4 --- /dev/null +++ b/.claude/skills/hil/scripts/hil_test.py @@ -0,0 +1,1355 @@ +#!/usr/bin/env python3 +"""TinyUF2 HIL battery and the probe operations debug sessions share with it. + + hil_test.py run -b BOARD --build-record FILE CONFIG [--fail-at STEP] [--try-reentry METHOD] + hil_test.py install -b BOARD --firmware BIN --sha256 HASH --under-hold REASON CONFIG + hil_test.py invalidate -b BOARD (--under-hold REASON | --recover-holder PID --report FILE) CONFIG + +The last stdout line is JSON. Exit 0 every board passed and was cleaned up, 1 a board failed or did not +run (cleaned up or never touched), 2 refused before any hardware access, 4 cleanup failed (the board is +held when `cleanup.heldAfter`; not under HIL_NO_BOARD_LOCK) or a manual operation left `survivors`. +SKILL.md beside this script is the contract; this file is its mechanics. +""" +import argparse +import contextlib +import fcntl +import hashlib +import importlib.util +import json +import os +import re +import select +import shutil +import signal +import struct +import subprocess +import sys +import time +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +sys.path.insert(0, str(ROOT / '.claude' / 'skills' / 'build' / 'scripts')) +sys.dont_write_bytecode = True # nothing the runner imports may write into a checkout it binds +import check_build as cb # noqa: E402 + +# [floor, ceiling) the runner ever programs, per J-Link device, independent of roster and source: +# LPC55S69 PFR starts at 0x9DE00 and a bad CMPA write can lock the part (UM11126 Table 173, Table 1064); +# 0x9D800-0x9DDFF is reserved and undocumented. MIMXRT1062: FlexSPI XIP at 0x60000000 (IMXRT1060RM +# Table 3-1) holding the EVK's 8 MiB IS25WP064A (EVK hardware user's guide 2.7). +FLASH_LIMITS = {'LPC55S69_M33_0': (0, 0x9D800), 'MIMXRT1062xxx6A': (0x60000000, 0x60800000)} +STEPS = ('identity', 'install', 'dfu', 'copy', 'app-ran', 'reenter') +REENTRY = ('retained-register', 'app-invalidate') +PROTECTED = {'hil_test.py', 'pool_check'} # tinyusb hil_lock.PROTECTED_REASONS, checked at load +HOLDER_ROLE = 'cleanup-holder' +SCB_VTOR, SCB_AIRCR, SCB_CPUID, DHCSR = 0xE000ED08, 0xE000ED0C, 0xE000ED00, 0xE000EDF0 +AIRCR_SYSRESETREQ = 0x05FA0004 +UF2_MAGIC = (0x0A324655, 0x9E5D5157, 0x0AB16F30) +UF2_FLAG_FAMILY = 0x2000 +PROBE_TIMEOUT, USB_TIMEOUT, QUIET_TIME, KILL_WAIT, UDEV_TIMEOUT = 120, 20, 3, 5, 5 +JLINK_RETRY_DELAY = 2 +JLINK_IF, JLINK_SPEED = 'SWD', '4000' +ZERO_PAGE = 512 # bytes zeroed at the app start so its reset vector reads 0 +UDEV_RULES = ('/etc/udev/rules.d', '/run/udev/rules.d') +# J-Link notes that are not failures, per device, each with the line that shows the operation completed +# (bench: every MIMXRT1062 reset with an invalid app vector, then the reset halts as usual) +BENIGN = {'MIMXRT1062xxx6A': r'(?m)^Could not find valid reset vector\.\nTarget halted on flash header read\.$'} + + +class Refused(Exception): + """Exit 2: nothing on the board was touched.""" + + +class StepFail(Exception): + def __init__(self, msg, evidence=None): + super().__init__(msg) + self.evidence = evidence or {} + + +class Interrupted(Exception): + pass + + +# ------------------------------------------------------------------ roster and source facts + +def load_board(config, name): + try: + cfg = json.loads(Path(config).read_text()) + boards = [b for b in cfg['boards'] if b.get('name') == name] + except (OSError, ValueError, KeyError, TypeError, AttributeError) as e: + raise Refused(f'cannot read {config}: {e}') + if len(boards) != 1: + raise Refused(f'{config} has {len(boards)} entries named {name!r}') + b = boards[0] + tinyusb = os.environ.get('TINYUSB_DIR') + if not (tinyusb and os.path.isabs(tinyusb)): + raise Refused('set TINYUSB_DIR to the absolute path of the tinyusb checkout whose test/hil/helper/hil_lock.py ' + 'owns the board locks') + fl, t = b.get('flasher') or {}, b.get('tinyuf2') or {} + m = re.fullmatch(r'-device (\S+)', fl.get('args') or '') + if fl.get('name') != 'jlink' or not isinstance(fl.get('uid'), str) or not m: + raise Refused(f'{name}: flasher must be jlink with a string uid and args exactly "-device NAME"') + if not isinstance(b.get('uid'), str) or not b['uid']: + raise Refused(f'{name}: uid (the bootloader USB serial) is required') + lock = t.get('lock', name) # tinyusb's name for the board when it differs: both runners take one lock + if not (isinstance(lock, str) and re.fullmatch(r'[\w.-]+', lock)): + raise Refused(f'{name}: tinyuf2.lock must be the board lock name') + try: + port = cb.port_of(name) + except cb.Fail as e: + raise Refused(f'{name}: the roster names a board by its TinyUF2 name: {e}') + if m.group(1) not in FLASH_LIMITS: + raise Refused(f'{name}: no programming limits known for J-Link device {m.group(1)}') + return {'name': name, 'lock': lock, 'uid': b['uid'], 'probe': fl['uid'], 'device': m.group(1), + 'if': JLINK_IF, 'speed': JLINK_SPEED, 'port': port, 'page': ZERO_PAGE, + 'helper': os.path.join(tinyusb, 'test', 'hil', 'helper', 'hil_lock.py'), + 'reentry': t.get('reentry')} + + +def define(path, name): + m = re.search(rf'^\s*#\s*define\s+{name}\s+(.+?)\s*(//.*|/\*.*)?$', (ROOT / path).read_text(), re.M) + if not m: + raise Refused(f'{path}: no #define {name}') + return m.group(1) + + +LITERAL = r'(0x[0-9a-fA-F]+|\d+)[uUlL]*' + + +def num(expr, what): + m = re.fullmatch(rf'\(?\s*{LITERAL}\s*\)?', expr) + if not m: + raise Refused(f'{what}: {expr!r} is not a literal') + return int(m.group(1), 0) + + +def product(expr, what): + """A literal or a product of literals, e.g. (8*1024*1024); nothing else.""" + inner = expr.strip() + if inner.startswith('(') and inner.endswith(')'): + inner = inner[1:-1] + terms = inner.split('*') + if not all(re.fullmatch(rf'\s*{LITERAL}\s*', t) for t in terms): + raise Refused(f'{what}: {expr!r} is not a literal or a product of literals') + v = 1 + for t in terms: + v *= num(t, what) + return v + + +def flash_addr(expr, flash_base, what): + """A literal, or FLASH_BASE + literal where the port defines FLASH_BASE; nothing else.""" + m = re.fullmatch(rf'\(?\s*FLASH_BASE\s*\+\s*{LITERAL}\s*\)?', expr) + if not m: + return num(expr, what) + if flash_base is None: + raise Refused(f'{what}: {expr!r} uses FLASH_BASE, which this port does not define') + return flash_base + int(m.group(1), 0) + + +def cmake_var(path, name): + """The value of the one `set(NAME value)` in a CMake file, ${TOP}/ read as the checkout; nothing else.""" + hits = re.findall(rf'\bset\s*\(\s*{name}\b([^)]*)\)', cb.code_lines(ROOT / path), re.I) + if len(hits) != 1: + raise Refused(f'{path}: expected one set({name} ...), found {len(hits)}') + v = hits[0].strip().removeprefix('${TOP}/') + if not v or re.search(r'[\s$"]', v): + raise Refused(f'{path}: set({name} {hits[0].strip()}) is not one plain value') + return v + + +def port_sources(port, bdir): + """(CMSIS device header, the SDK's flash linker script, FLASH_BASE) of a port the HIL supports, read from + the CMake files the HIL's build uses.""" + if not (ROOT / f'ports/{port}/CMakeLists.txt').is_file(): + raise Refused(f'ports/{port} has no cmake build, which is what the HIL builds') + if port != 'mimxrt10xx': + raise Refused(f'ports/{port}: the HIL runner has no source facts for this port') + sdk = cmake_var(f'ports/{port}/family.cmake', 'SDK_DIR') + mcu = cmake_var(f'{bdir}/board.cmake', 'MCU_VARIANT') + ld = f'ports/{port}/linker/{mcu}_ram.ld' # where family.cmake takes FLASH_BASE from + m = re.search(r'^\s*_flash_base\s*=\s*(0x[0-9a-fA-F]+)\s*;', (ROOT / ld).read_text(), re.M) + if not m: + raise Refused(f'{ld}: no _flash_base') + return f'{sdk}/devices/{mcu}/{mcu}.h', f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld', int(m.group(1), 16) + + +def source_facts(board): + """What the candidate's source says about this board.""" + port = board['port'] + bdir = f'ports/{port}/boards/{board["name"]}' + if not (ROOT / bdir).is_dir(): + raise Refused(f'{bdir} does not exist') + header, ld, flash_base = port_sources(port, bdir) + if re.search(r'^\s*#\s*define\s+BOARD_FLASH_APP_START\b', (ROOT / f'{bdir}/board.h').read_text(), re.M): + raise Refused(f'{bdir}/board.h overrides BOARD_FLASH_APP_START; the runner reads the port default only') + facts = { + 'flashBase': flash_base, + 'appStart': flash_addr(define(f'ports/{port}/boards.h', 'BOARD_FLASH_APP_START'), flash_base, + 'BOARD_FLASH_APP_START'), + 'flashSize': product(define(f'{bdir}/board.h', 'BOARD_FLASH_SIZE'), 'BOARD_FLASH_SIZE'), + 'vid': num(define(f'{bdir}/board.h', 'USB_VID'), 'USB_VID'), + 'pid': num(define(f'{bdir}/board.h', 'USB_PID'), 'USB_PID'), + 'boardId': define(f'{bdir}/board.h', 'UF2_BOARD_ID').strip('"'), + 'family': int(cmake_var(f'ports/{port}/family.cmake', 'UF2_FAMILY_ID'), 16), + 'vectors': num(define(header, 'NUMBER_OF_INT_VECTORS'), 'NUMBER_OF_INT_VECTORS'), + 'ram': ram_origin(ld), + 'dblTapDelayMs': num(define('src/board_api.h', 'TINYUF2_DBL_TAP_DELAY'), 'TINYUF2_DBL_TAP_DELAY'), + } + reg = define(f'ports/{port}/boards.h', 'TINYUF2_DBL_TAP_REG') + m = re.fullmatch(r'(\w+)->(\w+)\[(\d+)\]', reg) + if not m: + raise Refused(f'TINYUF2_DBL_TAP_REG {reg!r} is not PERIPH->ARRAY[N]') + base, offset, step = register_layout(header, m.group(1), m.group(2)) + facts['dblTapReg'] = base + offset + step * int(m.group(3)) + facts['dblTapMagic'] = 0xf01669ef # board_api.h DBL_TAP_MAGIC at TINYUF2_DBL_TAP_REG_SIZE 32 + if 'TINYUF2_DBL_TAP_REG_SIZE' in (ROOT / f'ports/{port}/boards.h').read_text(): + raise Refused(f'ports/{port} overrides TINYUF2_DBL_TAP_REG_SIZE: the magic would need its shift') + return facts + + +def ram_origin(ld): + m = re.search(r'm_data\s*\(RW\)\s*:\s*ORIGIN\s*=\s*(0x[0-9a-fA-F]+)', (ROOT / ld).read_text()) + if not m: + raise Refused(f'{ld}: no m_data ORIGIN') + return int(m.group(1), 16) + + +def register_layout(header, periph, member): + """(base, array offset, array step) of PERIPH->MEMBER[] from the CMSIS device header; on a TrustZone + part the base is the non-secure alias, the one a build without -mcmse (tinyuf2's) uses.""" + text = (ROOT / header).read_text() + end = re.search(r'\}\s*' + periph + r'_Type;', text) + start = text.rfind('typedef struct {', 0, end.start()) if end else -1 + if start < 0: + raise Refused(f'{header}: no {periph}_Type') + f = re.search(rf'\b{member}\[\d+\];.*?array offset:\s*(0x[0-9a-fA-F]+), array step:\s*(0x[0-9a-fA-F]+)', + text[start:end.start()]) + if not f: + raise Refused(f'{header}: no {periph}->{member} array offset') + bases = re.findall(rf'#define {periph}_BASE\s+\((0x[0-9a-fA-F]+)u\)', text) + if len(bases) not in (1, 2): + raise Refused(f'{header}: expected one {periph}_BASE, or a secure and a non-secure one, found {bases}') + return int(bases[-1], 16), int(f.group(1), 16), int(f.group(2), 16) + + +def apply_layout(board, facts): + """The install and app spans, from the candidate's source: the bootloader from the flash base up to the app + start, the app from there to the end of the board's flash, all inside the device's programming limits.""" + base, start, end = facts['flashBase'], facts['appStart'], facts['flashBase'] + facts['flashSize'] + floor, ceiling = FLASH_LIMITS[board['device']] + if not floor <= base < start < end <= ceiling: + raise Refused(f'{board["name"]}: flash [{base:#x}, {end:#x}) with app start {start:#x} leaves the ' + f'{board["device"]} flash [{floor:#x}, {ceiling:#x}) or is out of order') + if start % ZERO_PAGE: + raise Refused(f'{board["name"]}: app start {start:#x} is not {ZERO_PAGE}-byte aligned') + board.update(install=(base, start), app=(start, end)) + + +# ------------------------------------------------------------------ images + +def parse_hex(path): + """{address: bytes} of an Intel HEX file, contiguous runs merged; checksums verified.""" + data, base = {}, 0 + for n, line in enumerate(Path(path).read_text().splitlines(), 1): + if not line.strip(): + continue + try: + rec = bytes.fromhex(line.strip()[1:]) + except ValueError: + raise Refused(f'{path}:{n}: not Intel HEX') + if line[0] != ':' or len(rec) < 5 or len(rec) != rec[0] + 5 or sum(rec) & 0xFF: + raise Refused(f'{path}:{n}: bad Intel HEX record') + addr, kind, payload = int.from_bytes(rec[1:3], 'big'), rec[3], rec[4:-1] + if kind == 0: + for i, b in enumerate(payload): + data[base + addr + i] = b + elif kind == 1: + break + elif kind == 2: + base = int.from_bytes(payload, 'big') << 4 + elif kind == 4: + base = int.from_bytes(payload, 'big') << 16 + elif kind not in (3, 5): + raise Refused(f'{path}:{n}: unknown record type {kind}') + return data + + +def check_ranges(data, allowed, limits, what): + if not data: + raise Refused(f'{what}: no data') + lo, hi = min(data), max(data) + 1 + if lo < limits[0] or hi > limits[1]: + raise Refused(f'{what}: data [{lo:#x}, {hi:#x}) leaves the flash limits [{limits[0]:#x}, {limits[1]:#x})') + for a in (lo, hi - 1): + if not any(s <= a < e for s, e in allowed): + raise Refused(f'{what}: address {a:#x} is outside {[(hex(s), hex(e)) for s, e in allowed]}') + bad = [a for a in data if not any(s <= a < e for s, e in allowed)] + if bad: + raise Refused(f'{what}: {len(bad)} bytes outside the allowed ranges, first at {min(bad):#x}') + return lo, hi + + +def hex_matches_bin(data, bin_path, base): + """Whether a bootloader BIN is its HEX laid out from `base`: the HEX fixes the base the BIN lacks. The + BIN is what gets programmed, so gaps between sections are written and read back as the BIN holds them.""" + img = Path(bin_path).read_bytes() + return min(data) == base and max(data) + 1 == base + len(img) and all(img[a - base] == b for a, b in data.items()) + + +def embedded_version(bin_path): + hits = re.findall(rb'TinyUF2 Bootloader ([^\r\n\x00]+)\r\n', Path(bin_path).read_bytes()) + if len(hits) != 1: + raise Refused(f'{bin_path}: expected one "TinyUF2 Bootloader " string, found {len(hits)}') + return hits[0].decode() + + +def fixture(facts, nonce): + """A Cortex-M image the probe can prove ran: every vector but SP and reset traps in a fault loop, + the reset handler counts in r0 forever, and a nonce pattern makes an older run's image fail.""" + start = facts['appStart'] + table = 128 + while table < facts['vectors'] * 4: + table *= 2 + handler, fault = start + table, start + table + 4 + vectors = [facts['ram'] + 0x1000, handler | 1] + [fault | 1] * (table // 4 - 2) + code = struct.pack('{re.escape(command)}\s*$', out)[1:] + + +def regs(out): + def one(pat): + m = re.search(pat, out) + if not m: + raise StepFail(f'no {pat!r} in J-Link regs output') + return int(m.group(1), 16) + return {'pc': one(r'\bPC = ([0-9A-F]{8})'), 'r0': one(r'\bR0 = ([0-9A-F]{8})'), + 'msp': one(r'\bMSP= ([0-9A-F]{8})'), 'ipsr': one(r'\bIPSR = ([0-9A-F]{3})')} + + +# ------------------------------------------------------------------ USB mass storage + +def usb_device(vid, pid, serial): + hits = [] + for d in Path('/sys/bus/usb/devices').iterdir(): + try: + if (int((d / 'idVendor').read_text(), 16), int((d / 'idProduct').read_text(), 16)) == (vid, pid) and \ + (d / 'serial').read_text().strip() == serial: + hits.append(d.name) + except OSError: + continue + if len(hits) > 1: + raise StepFail(f'{len(hits)} USB devices with {vid:04x}:{pid:04x} serial {serial}: {hits}') + return hits[0] if hits else None + + +def block_device(usb_path): + for blk in Path('/sys/block').iterdir(): + if f'/{usb_path}:' in os.path.realpath(blk): + return f'/dev/{blk.name}' + return None + + +def wait_for(fn, timeout, what): + end = time.monotonic() + timeout + while time.monotonic() < end: + v = fn() + if v: + return v + time.sleep(0.2) + raise StepFail(f'{what} not seen within {timeout}s') + + +def exclusion_configured(facts, serial): + """A udev rule in exactly SKILL.md's form sets UDISKS_IGNORE=1 on this bootloader's VID, PID and + serial, so desktop automount cannot race the raw I/O: any other predicate could keep it from applying + when the device appears. Checked before any hardware access; the property on the enumerated device + (check_exclusive) is the proof.""" + want = {('ATTRS{idVendor}', '=='): f'{facts["vid"]:04x}', ('ATTRS{idProduct}', '=='): f'{facts["pid"]:04x}', + ('ATTRS{serial}', '=='): serial, ('ENV{UDISKS_IGNORE}', '='): '1'} + token = r'([A-Za-z_]+(?:\{[^}]*\})?)\s*(==|!=|:=|\+=|-=|=)\s*"([^"]*)"' + for d in UDEV_RULES: + for f in sorted(Path(d).glob('*.rules')) if Path(d).is_dir() else (): + for line in f.read_text(errors='replace').splitlines(): + found = re.findall(token, line) + rest = re.sub(token, '', line).replace(',', '').strip() + rule = {(k, op): v for k, op, v in found} + if rest or len(rule) != len(found) or rule.pop(('SUBSYSTEM', '=='), 'block') != 'block': + continue + if rule == want: + return str(f) + raise Refused(f'no udev rule in SKILL.md\'s form sets ENV{{UDISKS_IGNORE}}="1" on idVendor ' + f'{want[("ATTRS{idVendor}", "==")]}, idProduct {want[("ATTRS{idProduct}", "==")]} and serial {serial}') + + +def udisks_ignored(dev): + props = subprocess.run(['udevadm', 'info', '--query=property', f'--name={dev}'], capture_output=True, text=True) + return 'UDISKS_IGNORE=1' in props.stdout.split() + + +def check_exclusive(dev): + # the block node shows in sysfs before udev has run its rules on it (bench: RT1060 re-entry), so the + # property gets a bounded wait; the mount check still catches a rule that did not apply + start = time.monotonic() + try: + wait_for(lambda: udisks_ignored(dev), UDEV_TIMEOUT, f'UDISKS_IGNORE=1 on {dev}') + except StepFail: + raise StepFail(f'{dev} has no UDISKS_IGNORE=1: desktop automount could race the raw I/O (SKILL.md, Commissioning)') + mounted = [l.split()[0] for l in Path('/proc/mounts').read_text().splitlines()] + if any(os.path.realpath(m) == dev for m in mounted): + raise StepFail(f'{dev} is mounted; refusing raw I/O') + return round(time.monotonic() - start, 2) + + +class Fat: + """Read-only FAT12/16 reader for the ghostfat volume. Its geometry and chains come from the firmware + under test, so every one is bounded; it runs in its own process (fat_read) so a stuck USB read cannot + hold the process that owns cleanup.""" + def __init__(self, dev): + self.dev = dev + bs = self.read(0, 512) + self.bps, self.spc, self.res, self.nfat = struct.unpack_from('> 4 if c & 1 else v & 0xFFF + + def file(self, name, length=None): + ents = self.entries() + if name not in ents: + raise ValueError(f'{name} not on the volume (found {sorted(ents)})') + c, size = ents[name] + want, out, csize, seen = min(size, length or size), b'', self.spc * self.bps, set() + while len(out) < want: + if not 2 <= c < (0xFFF8 if self.fat16 else 0xFF8) or c in seen: + raise ValueError(f'{name}: cluster chain broken at {c:#x} after {len(out)} of {want} bytes') + seen.add(c) + out += self.read(self.data + (c - 2) * csize, min(csize, want - len(out))) + c = self.next_cluster(c) + return out, size + + +def fat_read(dev, name, length, log): + """{'entries': {...}, 'data': bytes} of `name` on `dev`, read by a bounded child process.""" + code, out = spawn([sys.executable, '-B', __file__, 'fat-read', '--dev', dev, '--file', name, + '--length', str(length)], log, USB_TIMEOUT) + try: + r = json.loads(out.strip().splitlines()[-1]) + except (ValueError, IndexError): + raise StepFail(f'fat-read of {name} gave no result (exit {code})', {'tail': out.strip().splitlines()[-3:]}) + if 'error' in r: + raise StepFail(f'{name} on {dev}: {r["error"]}') + return {'entries': r['entries'], 'data': bytes.fromhex(r['data']), 'size': r['size']} + + +def fat_read_main(args): + try: + fat = Fat(args.dev) + data, size = fat.file(args.file, args.length) if args.length else (b'', fat.entries()[args.file][1]) + print(json.dumps({'entries': fat.entries(), 'data': data.hex(), 'size': size})) + except (OSError, ValueError, KeyError, struct.error) as e: + print(json.dumps({'error': f'{type(e).__name__}: {e}'})) + return 0 + + +# ------------------------------------------------------------------ locks + +def load_helper(path): + spec = importlib.util.spec_from_file_location('tinyusb_hil_lock', path) + if not spec or not Path(path).is_file(): + raise Refused(f'lock helper {path} not found') + mod = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(mod) + except Exception as e: + raise Refused(f'lock helper {path} failed to load: {e}') + if set(getattr(mod, 'PROTECTED_REASONS', ())) != PROTECTED or getattr(mod, 'CI_REASON', None) != 'hil_test.py': + raise Refused(f'lock helper {path}: protected reasons are not {sorted(PROTECTED)}; recheck this contract') + return mod + + +def holds_flock(pid, lock_file): + """Whether process `pid` has an fd on `lock_file` whose open file description holds a FLOCK. The pid + in /proc/locks and in fdinfo's lock line names whoever took the lock, which after a fork-and-exec + handoff is the dead runner, so ownership is read through the holder's own fds.""" + st = os.stat(lock_file) + try: + fds = os.listdir(f'/proc/{pid}/fd') + except OSError: + return False + for fd in fds: + try: + s = os.stat(f'/proc/{pid}/fd/{fd}') + info = Path(f'/proc/{pid}/fdinfo/{fd}').read_text() + except OSError: + continue + if (s.st_dev, s.st_ino) == (st.st_dev, st.st_ino) and \ + re.search(rf'^lock:.*\bFLOCK\b.*:{st.st_ino}\b', info, re.M): + return True + return False + + +def flock_free(lock_file): + """No FLOCK on the lock file's inode in /proc/locks. A holder's fd goes before the kernel's deferred + release of its lock, so recovery waits on this rather than on the holder's fds, without probing.""" + st = os.stat(lock_file) + dev = f'{os.major(st.st_dev):02x}:{os.minor(st.st_dev):02x}:{st.st_ino}' + return not any(' FLOCK ' in l and l.split()[5] == dev for l in Path('/proc/locks').read_text().splitlines()) + + +def prove_hold(helper, board, pid, reason=None, role=None): + """Refuse unless `pid` really holds the board's flock under the expected record.""" + path = helper.lock_path(board) + rec = helper.read_record(board) + if not isinstance(rec, dict) or rec.get('pid') != pid: + raise Refused(f'{board}: lock record {rec} does not name pid {pid}') + if reason is not None and rec.get('reason') != reason: + raise Refused(f'{board}: held for {rec.get("reason")!r}, not {reason!r}') + if role is not None and rec.get('role') != role: + raise Refused(f'{board}: holder role {rec.get("role")!r}, not {role!r}') + try: + os.kill(pid, 0) + except OSError: + raise Refused(f'{board}: holder pid {pid} is not alive') + if not holds_flock(pid, path): + raise Refused(f'{board}: pid {pid} has no flock on {path}') + try: + fh = helper.flock_nb(board) + except OSError: + return rec + fh.close() + raise Refused(f'{board}: the flock on {path} is free') + + +def recovery_lock(helper, board): + """An exclusive flock on `.recover`, kept until the caller drops it: one recovery per board, so + none releases the holder while another is still between its probe sessions. tinyusb's helper lists `*.lock` + only, so it never reports this file as a holder.""" + path = helper.lock_path(board) + '.recover' + try: + fh = os.fdopen(os.open(path, os.O_RDONLY | os.O_CREAT, 0o666)) + except OSError as e: + raise Refused(f'cannot open {path}: {e}') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as e: + fh.close() + raise Refused(f'{board}: another recovery holds {path}: {e}') + return fh + + +def manual_holder(helper, board, reason): + if reason in PROTECTED: + raise Refused(f'{reason!r} is a protected holder; --under-hold takes the reason of your own manual hold') + rec = helper.read_record(board) + pid = rec.get('pid') if isinstance(rec, dict) else None + if not isinstance(pid, int): + raise Refused(f'{board}: no lock record; hold it first with {helper.__file__} hold {board} --reason ...') + prove_hold(helper, board, pid, reason=reason) + + +def probe_users(probe): + """Processes that have this probe open by serial (a JLinkExe or GDB server from an earlier session).""" + users = [] + for d in Path('/proc').iterdir(): + try: + cmd = (d / 'cmdline').read_bytes().split(b'\0') + except OSError: + continue + if d.name.isdigit() and int(d.name) != os.getpid() and any(b'JLink' in c for c in cmd[:1]) and \ + any(probe.encode() in c for c in cmd): + users.append(int(d.name)) + return users + + +def holder_record(fh, board, detail, survivors): + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': 'hil_test.py', 'role': HOLDER_ROLE, 'board': board, + 'detail': detail, 'survivors': survivors, 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + + +HELD_IN_PROCESS = [] + + +def keep_held(fh): + """Hold until recovery's SIGUSR1, taken by sigwait with SIGUSR1 blocked since hand_off: a handler and + pause() lose a signal that lands between the interpreter's check and the syscall. SIGTERM and SIGINT + are ignored: an operator's timeout kill must not free a board that was never cleaned up.""" + for sig in (signal.SIGTERM, signal.SIGINT): + signal.signal(sig, signal.SIG_IGN) + signal.pthread_sigmask(signal.SIG_SETMASK, {signal.SIGUSR1}) + for std in (0, 1, 2): # closed, not reopened on /dev/null: holding must not need a free descriptor + with contextlib.suppress(OSError): + os.close(std) + signal.sigwait({signal.SIGUSR1}) + fh.truncate(0) + os._exit(0) + + +def hand_off(fh, board, detail, survivors=()): + """Keep the board held after a failed cleanup, by a forked holder that shares the open file + description, so the flock is never dropped, and keeps none of the caller's stdio, so the caller's + wait ends with this process. When no holder starts, this process keeps the lock itself and main() + stays alive holding it after reporting.""" + survivors = list(survivors) + signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGUSR1}) # pending, not lost, until keep_held waits + try: + r, w = os.pipe() + pid = os.fork() + except OSError: + pid = None + if pid == 0: + try: + os.close(r) + os.setsid() + holder_record(fh, board, detail, survivors) + os.write(w, b'ok') + os.close(w) + keep_held(fh) + finally: + os._exit(1) + if pid: + os.close(w) + ready, _, _ = select.select([r], [], [], 10) + ack = os.read(r, 16) if ready else b'' + os.close(r) + if ack == b'ok': + fh.close() # never LOCK_UN or truncate: the description and its record are the holder's now + return {'pid': pid, 'reason': 'hil_test.py', 'ready': True} + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(ChildProcessError): + os.waitpid(pid, 0) + holder_record(fh, board, detail, survivors) + HELD_IN_PROCESS.append(fh) + return {'pid': os.getpid(), 'reason': 'hil_test.py', 'ready': True, 'inProcess': True} + + +# ------------------------------------------------------------------ the battery + +def verify_record(record_path, board, facts): + try: + rec = json.loads(Path(record_path).read_text()) + except (OSError, ValueError) as e: + raise Refused(f'cannot read build record {record_path}: {e}') + rows = [r for r in rec.get('boards', []) if r.get('board') == board['name']] + if not rec.get('pass') or len(rows) != 1 or rows[0].get('status') != 'ok' or rows[0].get('removed'): + raise Refused(f'{record_path}: not a passing --shared build of {board["name"]}') + row, info = rows[0], rec.get('input', {}) + if row.get('backend') != 'cmake': + raise Refused(f'{record_path}: a {row.get("backend")} build; the HIL takes cmake builds only ' + f'(check_build.py --board {board["name"]} --backend cmake --shared)') + try: + now = cb.identity({board['port']}) + except (cb.Fail, OSError) as e: + raise Refused(f'cannot establish the checkout identity: {e}') + for key in ('head', 'submodules', 'deps'): + if info.get(key) != now[key]: + raise Refused(f'{record_path}: {key} {info.get(key)} differs from the checkout now ({now[key]})') + for state in (info, now): + bad = state.get('dirty') or state.get('dirtyInputs') or state.get('missingInputs') or state.get('depsOffPin') \ + or state.get('inputsDirtiedByBuild') + if bad: + raise Refused(f'build inputs are not clean: {bad} (commit, or rebuild from a clean checkout)') + root = cb.ROOT / row['buildDir'] + files, digests = {}, {} + for p in cb.expected_artifacts(board['port'], board['name'], 'cmake', root): + kind = p.suffix[1:] + if kind not in ('hex', 'bin'): + continue + digests[kind] = sha256(p) + if digests[kind] is None: + raise Refused(f'{p}: missing or unreadable; rebuild with --shared') + if row.get('sha256', {}).get(str(p.relative_to(root))) != digests[kind]: + raise Refused(f'{p}: hash differs from the build record') + files[kind] = p + if files.keys() != {'hex', 'bin'}: + raise Refused(f'{record_path}: the build of {board["name"]} names no {" or ".join(sorted({"hex", "bin"} - files.keys()))}') + return files, {'path': str(files['bin']), 'sha256': digests['bin'], 'buildRecord': str(record_path), + 'defines': rec.get('defines', []), 'version': embedded_version(files['bin'])} + + +def sha256(p): + try: + return hashlib.sha256(Path(p).read_bytes()).hexdigest() + except OSError: + return None + + +def check_bootloader_bin(bin_path, board, facts, what): + """Every span a bootloader install writes: the BIN from the install start inside `install` and the zero + page at the start of `app` inside it, both within the device's flash limits; and the BIN is this + board's (its UF2_BOARD_ID).""" + install, app = board['install'], board['app'] + size = Path(bin_path).stat().st_size + limits = FLASH_LIMITS[board['device']] + check_ranges({install[0]: 0, install[0] + size - 1: 0}, [install], limits, what) + check_ranges({app[0]: 0, app[0] + board['page'] - 1: 0}, [app], limits, f'{what} zero page') + if facts['boardId'].encode() not in Path(bin_path).read_bytes(): + raise Refused(f'{what}: {bin_path} does not carry board id {facts["boardId"]!r}; not this board\'s bootloader') + + +class Battery: + def __init__(self, board, facts, workdir, fail_at=None, commissioning=False): + self.b, self.f, self.dir, self.fail_at, self.commissioning = board, facts, workdir, fail_at, commissioning + self.log = workdir / 'hil.log' + self.probe = Probe(board, workdir, self.log) + self.steps, self.usb_path, self.touched, self.at = [], None, False, 'setup' + self.zero = workdir / 'zero.bin' + self.zero.write_bytes(b'\0' * board['page']) + + def step(self, name, fn): + self.at = name # an abort names the step it cut short without recording it as a failing step + if self.fail_at == name: + self.steps.append({'step': name, 'pass': False, 'evidence': {}, 'detail': 'injected by --fail-at'}) + raise StepFail(f'--fail-at {name}: injected criterion failure (acceptance only)') + try: + ev = fn() or {} + except StepFail as e: + self.steps.append({'step': name, 'pass': False, 'evidence': e.evidence, 'detail': str(e)}) + raise + self.steps.append({'step': name, 'pass': True, 'evidence': ev, 'detail': ''}) + + def identity(self): + v = self.probe.mem32([SCB_CPUID, DHCSR]) + return {'cpuid': f'{v[SCB_CPUID]:#010x}', 'dhcsr': f'{v[DHCSR]:#010x}'} + + def program(self, bin_path, base, app_start): + """One image from `base`: the bootloader BIN, 0xFF up to app_start, then a zero page, written by a single + loadbin and read back, so no second write lands in the bootloader's erase block; the zeroed reset vector + makes the app invalid by construction, so the next boot stays in DFU. No range erase is ever issued. + Callers have range-checked both spans (check_bootloader_bin).""" + data = Path(bin_path).read_bytes() + if base + len(data) > app_start: + raise StepFail(f'{bin_path} [{base:#x}, {base + len(data):#x}) overlaps the app start {app_start:#x}') + image = self.dir / 'program.bin' + image.write_bytes(data + b'\xff' * (app_start - base - len(data)) + bytes(self.b['page'])) + self.touched = True + self.installed = (bin_path, base) + self.probe.run(['h', f'loadbin {image},{base:#x}', f'verifybin {image},{base:#x}', 'r', 'g']) + + def dfu(self, version): + f = self.f + path = wait_for(lambda: usb_device(f['vid'], f['pid'], self.b['uid']), USB_TIMEOUT, + f'{f["vid"]:04x}:{f["pid"]:04x} serial {self.b["uid"]}') + if self.usb_path and path != self.usb_path: + raise StepFail(f'came back on USB port {path}, not {self.usb_path}') + self.usb_path = path + dev = wait_for(lambda: block_device(path), USB_TIMEOUT, f'the block device of USB {path}') + udev_wait = check_exclusive(dev) + info = fat_read(dev, 'INFO_UF2.TXT', 512, self.log) + lines = info['data'].decode('ascii', 'replace').splitlines() + want = f'TinyUF2 Bootloader {version}' + if not lines or lines[0] != want or f'Board-ID: {f["boardId"]}' not in lines or \ + f'Flash Size: 0x{f["flashSize"]:08X} bytes' not in lines: + raise StepFail('INFO_UF2.TXT does not match this build', {'info': lines, 'want': want}) + size = (info['entries'].get('CURRENT.UF2') or [None, None])[1] + if size != f['flashSize'] // 256 * 512: + raise StepFail(f'CURRENT.UF2 is {size} bytes, not {f["flashSize"] // 256 * 512}') + return {'usb': path, 'dev': dev, 'udevWaitS': udev_wait, 'info': lines[0], 'currentUf2': size, + 'claim': 'app rejected (reset vector 0); DFU observed'} + + def current_uf2(self, length): + """CURRENT.UF2's first blocks, each checked for its place: block i maps to app_start + 256*i.""" + dev = block_device(self.usb_path) + data = fat_read(dev, 'CURRENT.UF2', length, self.log)['data'] + try: + blocks = parse_uf2(data) + except ValueError as e: + raise StepFail(f'CURRENT.UF2 is not UF2: {e}', {'head': data[:32].hex()}) + for i, bl in enumerate(blocks): + if (bl['no'], bl['addr'], bl['size']) != (i, self.f['appStart'] + 256 * i, 256): + raise StepFail(f'CURRENT.UF2 block {i} is {bl["no"]} at {bl["addr"]:#x}, not at {self.f["appStart"] + 256 * i:#x}') + return b''.join(bl['data'] for bl in blocks) + + def run(self, files, version): + f = self.f + nonce = os.urandom(16) + self.fix = fixture(f, nonce) + app_bin, app_uf2 = make_uf2(self.fix['image'], f['appStart'], f['family'], self.dir) + blocks = parse_uf2(app_uf2.read_bytes()) + payload = {} + for bl in blocks: + if bl['family'] != f['family'] or not bl['flags'] & UF2_FLAG_FAMILY: + raise Refused('generated UF2 has the wrong family') + payload.update({bl['addr'] + i: x for i, x in enumerate(bl['data'])}) + check_ranges(payload, [self.b['app']], FLASH_LIMITS[self.b['device']], 'fixture UF2') + self.step('identity', self.identity) + self.step('install', lambda: self.program(files['bin'], self.b['install'][0], f['appStart']) or + {'zeroPage': f'{f["appStart"]:#x}', 'verified': True}) + # cleared once the bootloader has run board_init (the register's clock), so a magic a failed reenter left + # cannot send the copied app back to DFU + self.step('dfu', lambda: dict(self.dfu(version), dblTapReg=self.probe.write32(f['dblTapReg'], 0))) + + def copy(): + dev = block_device(self.usb_path) + code, out = spawn(['mcopy', '-o', '-i', dev, app_uf2, '::APP.UF2'], self.log, 60) + if code: + raise StepFail(f'mcopy failed: {out.strip()[-200:]}') + spawn(['sync'], self.log, 60) + wait_for(lambda: usb_device(f['vid'], f['pid'], self.b['uid']) is None, USB_TIMEOUT, 'the DFU device leaving') + return {'blocks': len(blocks), 'nonce': nonce.hex()} + self.step('copy', copy) + + def app_ran(): + # the bootloader writes DBL_TAP_MAGIC as it boots: a probe connect that resets inside that window + # would send the next boot to DFU, so the first connect waits it out + window = 2 * f['dblTapDelayMs'] / 1000 + 0.5 + time.sleep(window) + ev = dict(self.observe_app(app_bin), waitedS=window) + time.sleep(QUIET_TIME) + if usb_device(f['vid'], f['pid'], self.b['uid']): + raise StepFail('the DFU device came back while the app should run', ev) + return ev + self.step('app-ran', app_ran) + + def reenter(): + ev = {'path': self.b['reentry']} + try: + return entered(ev) + except StepFail as e: # what was observed before the failure stays in the step's evidence + e.evidence = {**ev, **e.evidence} + raise + + def entered(ev): + method = ev['path'] + if method == 'retained-register': + if self.commissioning: + ev.update(self.control(app_bin)) + self.probe.write32(f['dblTapReg'], f['dblTapMagic']) + # SYSRESETREQ may drop the debug connection mid-write: its exit is not judged, enumeration is; + # no probe session opens again until DFU is observed + tail = self.probe.run([f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) + else: + bin_path, base = self.installed + tail = self.probe.run(['h', f'loadbin {self.zero},{f["appStart"]:#x}', f'verifybin {self.zero},{f["appStart"]:#x}', + f'verifybin {bin_path},{base:#x}', 'r', 'g']) + ev['bootloaderVerified'] = True + ev.update(self.dfu(version), dfuCameUp=True, resetTail=tail.strip().splitlines()[-3:]) + ev['claim'] = 'the write/reset sequence re-entered DFU (register retention inferred)' \ + if method == 'retained-register' else 'invalidate application and enter DFU' + if method == 'retained-register': + left = self.probe.mem32([f['dblTapReg']])[f['dblTapReg']] + ev.update(dblTapRegAfter=hex(left), magicConsumed=not left) + if left: + raise StepFail('the double-tap register was not consumed', ev) + expect = bytearray(self.fix['image']) + if method == 'app-invalidate': + expect[:self.b['page']] = bytes(self.b['page']) + n = -(-len(expect) // 256) + try: + got = self.current_uf2(n * 512) + if got[:len(expect)] != bytes(expect): + raise StepFail('CURRENT.UF2 does not read back the fixture from the app start') + except StepFail as e: + e.evidence = {**ev, **e.evidence, 'failedAt': 'readback', 'appIntact': 'unshown'} + raise + ev['appIntact'] = True + return ev + self.step('reenter', reenter) + + def observe_app(self, app_bin): + """The fixture running from the bootloader handoff: two halts in one session, the counter moving, + PC at its handler in thread mode, VTOR and MSP from its vector table, and its image intact. The + in-script wait covers a connect that resets the core and so restarts the double-tap window.""" + f = self.f + out = self.probe.run(['h', 'g', f'sleep {2 * f["dblTapDelayMs"] + 200}', 'h', 'regs', f'mem32 {SCB_VTOR:08X} 1', + 'g', 'sleep 10', 'h', 'regs', f'verifybin {app_bin},{f["appStart"]:#x}', 'g']) + chunks = after(out, 'regs') + if len(chunks) != 2: + raise StepFail(f'expected two regs dumps, got {len(chunks)}', {'tail': out.strip().splitlines()[-8:]}) + first, second = regs(chunks[0]), regs(chunks[1]) + m = re.search(rf'^{SCB_VTOR:08X} = ([0-9A-F]{{8}})', out, re.M) + if not m: + raise StepFail('no VTOR read in the transcript') + vtor = int(m.group(1), 16) + ev = {'pc': hex(first['pc']), 'ipsr': first['ipsr'], 'vtor': hex(vtor), 'msp': hex(first['msp']), + 'r0': [hex(first['r0']), hex(second['r0'])], 'verifybin': True, + 'scope': 'executes through bootloader handoff after the observed reset; upload-triggered run unverified'} + ok = (first['pc'] in (self.fix['handler'], self.fix['handler'] + 2) and first['pc'] != self.fix['fault'] and + first['ipsr'] == 0 and vtor == f['appStart'] and first['msp'] == self.fix['sp'] and + first['r0'] != second['r0']) + if not ok: + raise StepFail('the fixture is not running from the bootloader handoff', ev) + return ev + + def control(self, app_bin): + """Commissioning: the same SYSRESETREQ session without the magic must return to the app. The bootloader + arms the magic itself for its double-tap window (src/main.c), so a session that reset twice would + re-enter DFU with no magic written; USB is watched past that window before any probe reconnects.""" + f = self.f + self.probe.run([f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) + watch = 2 * f['dblTapDelayMs'] / 1000 + USB_TIMEOUT + end = time.monotonic() + watch + while time.monotonic() < end: + if usb_device(f['vid'], f['pid'], self.b['uid']): + raise StepFail('the reset without the magic entered DFU', {'controlBootedApp': False}) + time.sleep(0.2) + return {'control': self.observe_app(app_bin), 'controlWatchedS': watch, 'controlBootedApp': True} + + +def invalidate(board, facts, workdir, battery=None): + """The board left unbootable: a zero page at the device's flash floor (FLASH_LIMITS, never the candidate's + layout) written and read back, the target reset, then the zero page verified again from a new session.""" + base = FLASH_LIMITS[board['device']][0] + bat = battery or Battery(board, facts, workdir) + load, verify = f'loadbin {bat.zero},{base:#x}', f'verifybin {bat.zero},{base:#x}' + bat.probe.run(['h', load, verify, 'r', 'g']) + # with no boot image, a new session reads flash wrong until a loadbin has run (bench-observed on MIMXRT1062, + # cause unknown: verifybin alone fails), so the check is a loadbin J-Link skips as already matching + out = bat.probe.run(['h', load, verify, 'g']) + if 'Skipped. Contents already match' not in ''.join(after(out, load)[:1]).split('J-Link>')[0]: + raise StepFail('the zero page did not persist past the reset; the new session wrote it again') + return {'pass': True, 'state': 'bootloader invalidated', 'evidence': {'zeroPage': f'{base:#x}', 'verified': True}} + + +TERM_SIGNALS = {signal.SIGTERM, signal.SIGINT} + + +@contextlib.contextmanager +def signals(how): + old = signal.pthread_sigmask(how, TERM_SIGNALS) + try: + yield + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, old) + + +def on_signal(signum, _): + # blocked again before raising: no later signal can break into the unwinding or the cleanup + signal.pthread_sigmask(signal.SIG_BLOCK, TERM_SIGNALS) + if REGISTERING['on']: + REGISTERING['signum'] = REGISTERING['signum'] or signum + return + raise Interrupted(f'interrupted by {signal.Signals(signum).name}') + + +def check_jlink(): + if shutil.which('JLinkExe') is None: + raise Refused('JLinkExe must be on PATH') + + +def reentry_decision(row): + """The re-entry method a commissioning run establishes (SKILL.md, Commissioning), or None: every + observation of the tried method held, the board was cleaned up and released, and the run passed or failed + only at reenter's CURRENT.UF2 readback.""" + cleanup = row.get('cleanup') or {} + if not row.get('commissioning') or not cleanup.get('pass') or cleanup.get('heldAfter'): + return None + steps = row.get('steps', []) + failed = [x for x in steps if not x['pass']] + reenter = [x for x in steps if x['step'] == 'reenter'] + if len(reenter) != 1 or failed not in ([], reenter) or (not failed and not row.get('pass')) or \ + (failed and reenter[0]['evidence'].get('failedAt') != 'readback'): + return None + need = ('controlBootedApp', 'dfuCameUp', 'magicConsumed') if row['tried'] == 'retained-register' else \ + ('bootloaderVerified', 'dfuCameUp') + return row['tried'] if all(reenter[0]['evidence'].get(k) is True for k in need) else None + + +def run_board(args, board, facts, helper, workdir): + row = {'board': board['name'], 'lock': board['lock'], 'ran': False, 'pass': False, 'locked': False, + 'lockBypassed': False, 'steps': [], 'cleanup': None, 'detail': ''} + if args.try_reentry: + if board['reentry'] is not None: + raise Refused(f'{board["name"]}: reentry is already commissioned; --try-reentry is for commissioning only') + board = dict(board, reentry=args.try_reentry) + row.update(commissioning=True, tried=args.try_reentry) + elif board['reentry'] not in REENTRY: + raise Refused(f'{board["name"]}: tinyuf2.reentry must be one of {REENTRY} (set from commissioning)') + files, row['firmware'] = verify_record(args.build_record, board, facts) + data = parse_hex(files['hex']) + check_ranges(data, [board['install']], FLASH_LIMITS[board['device']], files['hex']) + if not hex_matches_bin(data, files['bin'], board['install'][0]): + raise Refused(f'{files["hex"]} and {files["bin"]} do not hold the same bytes') + check_bootloader_bin(files['bin'], board, facts, 'candidate') + exclusion_configured(facts, board['uid']) + if shutil.which('mcopy') is None: + raise Refused('mcopy must be on PATH') + check_jlink() + bypass = os.environ.get('HIL_NO_BOARD_LOCK') == '1' + # blocked from here until the process exits or becomes the holder (keep_held): a signal acts only + # inside the battery, and the report and any hold are settled first + signal.pthread_sigmask(signal.SIG_BLOCK, TERM_SIGNALS) + try: + fh = helper.acquire_board_lock(board['lock']) + except RuntimeError as e: + row.update(locked=True, detail=str(e)) + return deferred(row), 1 + if fh is None and not bypass: + raise Refused(f'{board["name"]}: the lock dir is unusable (tinyusb helper fails open); refusing to run unlocked') + row['lockBypassed'] = fh is None + bat = Battery(board, facts, workdir, args.fail_at, row.get('commissioning', False)) + code = 0 + try: + row['ran'] = True + with signals(signal.SIG_UNBLOCK): + bat.run(files, row['firmware']['version']) + row['pass'] = True + except BaseException as e: # anything after admission still cleans up below + row['detail'] = f'{bat.at}: {e}' if isinstance(e, StepFail) else f'{bat.at}: {type(e).__name__}: {e}' + code = 1 + row['steps'], row['jlinkRetried'] = bat.steps, bat.probe.retried # the cleanup's retries land here too + code = clean_up(row, code, bat, fh, board, facts, helper, workdir) + if row.get('commissioning'): + row['reentryDecision'] = reentry_decision(row) + return deferred(row), code + + +def deferred(row): + pending = sorted(signal.Signals(s).name for s in signal.sigpending() & TERM_SIGNALS) + if pending: + row['detail'] = '; '.join(filter(None, (row['detail'], f'{", ".join(pending)} deferred until the board was settled'))) + return row + + +def clean_up(row, code, bat, fh, board, facts, helper, workdir): + """Invalidate a touched board, then release it. A child that survived SIGKILL, or any unsuccessful + cleanup, leaves the board held (exit 4), touched or not.""" + stuck = [] + try: + stuck = stuck_children() + if stuck: + raise StepFail(f'children {[x["pid"] for x in stuck]} survived SIGKILL; no flash may overlap them') + if bat.touched: + if bat.fail_at == 'cleanup': + raise StepFail('--fail-at cleanup: injected cleanup failure (acceptance only)') + row['cleanup'] = dict(invalidate(board, facts, workdir, bat), heldAfter=False) + except BaseException as e: + stuck += [x for x in stuck_children() if x not in stuck] + row['cleanup'] = {'pass': False, 'state': 'unknown' if bat.touched else 'untouched', + 'evidence': getattr(e, 'evidence', {}), 'detail': f'{type(e).__name__}: {e}', + 'survivors': stuck, 'heldAfter': fh is not None} + row['pass'] = False + if fh is not None: + row['cleanup']['holder'] = hand_off(fh, board['lock'], row['cleanup']['detail'], stuck) + fh = None + return 4 + if fh is not None: + helper.clear_record(fh) + fh.close() + return code + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest='cmd', required=True) + r = sub.add_parser('run', help='the battery on one board (self-locking)') + r.add_argument('-b', '--board', required=True) + r.add_argument('--build-record', required=True, help='check_build.py --board B --backend cmake --shared JSON, under cmake-build/') + r.add_argument('--fail-at', choices=(*STEPS, 'cleanup'), help='acceptance only: fail this step, or the cleanup, ' + 'to exercise cleanup and its holder') + r.add_argument('--try-reentry', choices=REENTRY, help="commissioning only, while the roster's reentry is unset") + r.add_argument('config') + i = sub.add_parser('install', help='program a bootloader BIN under your own manual hold') + i.add_argument('-b', '--board', required=True) + i.add_argument('--firmware', required=True, + help='a bootloader BIN, programmed at the install start with a zero page at the app start') + i.add_argument('--sha256', required=True) + i.add_argument('--under-hold', required=True, metavar='REASON') + i.add_argument('config') + s = sub.add_parser('invalidate', help='leave the board unbootable, under your hold or to recover a holder') + s.add_argument('-b', '--board', required=True) + g = s.add_mutually_exclusive_group(required=True) + g.add_argument('--under-hold', metavar='REASON') + g.add_argument('--recover-holder', type=int, metavar='PID') + s.add_argument('--report', help="the failed run's hil_report.json (with --recover-holder)") + s.add_argument('config') + fr = sub.add_parser('fat-read') + fr.add_argument('--dev', required=True) + fr.add_argument('--file', required=True) + fr.add_argument('--length', type=int, required=True) + args = ap.parse_args(argv) + if args.cmd == 'fat-read': + return fat_read_main(args) + for sig in TERM_SIGNALS: + signal.signal(sig, on_signal) + run_dir = cb.BUILD_DIR / 'hil' / f'run-{time.strftime("%Y%m%d-%H%M%S")}-{os.getpid()}' + result = {'pass': False, 'caveat': '', 'results': [], 'reportPath': str(run_dir / 'hil_report.json')} + code, recovery = 2, None + try: + run_dir.mkdir(parents=True) + board = load_board(args.config, args.board) + facts = None # invalidate, recovery included, writes only at the device's flash floor: no candidate source + if args.cmd != 'invalidate': + facts = source_facts(board) + apply_layout(board, facts) + helper = load_helper(board['helper']) + if args.cmd == 'run': + row, code = run_board(args, board, facts, helper, run_dir) + result['results'].append(row) + result['pass'] = code == 0 and row['pass'] + elif args.cmd == 'install': + manual_holder(helper, board['lock'], args.under_hold) + if sha256(args.firmware) != args.sha256: + raise Refused(f'{args.firmware} does not match --sha256') + check_bootloader_bin(args.firmware, board, facts, args.firmware) + exclusion_configured(facts, board['uid']) + check_jlink() + try: + Battery(board, facts, run_dir).program(args.firmware, board['install'][0], facts['appStart']) + finally: + result['survivors'] = stuck_children() + result['results'].append({'board': board['name'], 'installed': {'path': args.firmware, 'sha256': args.sha256}}) + result['pass'], code = True, 0 + else: + check_jlink() + rec = None + if args.recover_holder is not None: + if not args.report: + raise Refused("--recover-holder needs --report, the failed run's report naming that holder") + try: + rows = json.loads(Path(args.report).read_text())['results'] + except (OSError, ValueError, KeyError, TypeError) as e: + raise Refused(f'cannot read {args.report}: {e}') + held = [x['cleanup'] for x in rows if ((x.get('cleanup') or {}).get('holder') or {}).get('pid') + == args.recover_holder and x.get('board') == board['name']] + if len(held) != 1: + raise Refused(f'{args.report} names no cleanup holder {args.recover_holder} of {board["name"]}') + recovery = recovery_lock(helper, board['lock']) + rec = prove_hold(helper, board['lock'], args.recover_holder, reason='hil_test.py', role=HOLDER_ROLE) + cmdline = Path(f'/proc/{args.recover_holder}/cmdline').read_bytes() + if b'hil_test.py' not in cmdline: + raise Refused(f'pid {args.recover_holder} is not a hil_test.py cleanup holder') + alive = still_alive(held[0].get('survivors', []) + rec.get('survivors', [])) + if alive: + raise Refused(f'the failed run\'s children {alive} are still alive; recovery may not overlap them') + else: + manual_holder(helper, board['lock'], args.under_hold) + users = probe_users(board['probe']) + if users: + raise Refused(f'probe {board["probe"]} is still in use by pids {users}; recovery may not overlap them') + try: + cleanup = invalidate(board, facts, run_dir) + finally: + result['survivors'] = stuck_children() + if result['survivors']: + raise StepFail('invalidate left children that survived SIGKILL') + if args.recover_holder is not None: + os.kill(args.recover_holder, signal.SIGUSR1) + wait_for(lambda: flock_free(helper.lock_path(board['lock'])), 10, + f'cleanup holder {args.recover_holder} releasing {board["name"]}') + cleanup['recovered'] = {'holder': args.recover_holder, 'record': rec} + result['results'].append({'board': board['name'], 'cleanup': cleanup}) + result['pass'], code = True, 0 + except Refused as e: + result['error'] = str(e) + if args.cmd == 'run' and not result['results']: + result['results'].append({'board': args.board, 'ran': False, 'pass': False, 'locked': False, + 'detail': str(e)}) + code = 2 + except (StepFail, Interrupted) as e: + result['error'] = str(e) + code = 1 + if recovery: + recovery.close() + if result.get('survivors'): # a manual operation's survivor: keep your hold until it is gone + result['pass'], code = False, 4 + try: + Path(result['reportPath']).write_text(json.dumps(result, indent=1)) + except OSError: + pass + print(json.dumps(result), flush=True) + if HELD_IN_PROCESS: + keep_held(HELD_IN_PROCESS[0]) + return code + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index 5b6bff705..b0431f2cf 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -139,6 +139,8 @@ def test_build_control_runs_tests_and_one_smoke_build_per_backend(self): rule, effect, boards, _, tests = classify('.claude/skills/build/scripts/check_build.py') self.assertEqual((rule, effect, tests), ('build-control', 'select', True)) self.assertEqual(sorted(b for _, b in boards), ['cmake', 'idf', 'make']) + self.assertTrue(cb.classify('.claude/skills/build/scripts/check_build.py')['hil']) # hil_test.py imports it + self.assertFalse(cb.classify('.github/workflows/build.yml')['hil']) def test_unknown_port_fails_explicitly(self): with self.assertRaises(cb.Fail): @@ -158,6 +160,13 @@ def test_nothing_to_build(self): for path in ('README.md', '.claude/skills/build/SKILL.md', 'tools/gen_boards.py', '.github/workflows/codeql.yml'): self.assertEqual(classify(path)[1], 'none', path) + def test_hil_runner_and_roster_select_tests_and_every_roster_board(self): + for path in ('.claude/skills/hil/scripts/hil_test.py', 'test/hil/local.json'): + r = cb.classify(path) + self.assertEqual((r['rule'], r['tests'], r['hil'], r['boards']), ('hil-control', True, True, {}), path) + self.assertEqual(cb.classify('.claude/skills/hil/SKILL.md')['effect'], 'none') # wording costs no hardware + self.assertEqual(classify('.claude/test/test_hil.py'), ('contract-tests', 'select', [], False, True)) + def test_core_and_unclassified_select_every_representative(self): for path in ('src/main.c', 'lib/tinyusb', 'ports/family_support.cmake', 'some/new/file.c'): self.assertEqual(classify(path)[1], 'core', path) @@ -229,9 +238,13 @@ def test_scope_accepts_a_tracked_path_deleted_from_the_tree(self): cb.changed_paths(scope=['b.c']) -def run_main(argv, boards=None, ghostfat=None, tests=None): - """main() with builds stubbed: `boards` maps board -> status.""" +CLEAN = {'head': 'h', 'dirty': False, 'submodules': {}, 'deps': {}, 'dirtyInputs': [], 'missingInputs': [], 'depsOffPin': []} + + +def run_main(argv, boards=None, ghostfat=None, tests=None, identities=None): + """main() with builds stubbed: `boards` maps board -> status; `identities`, the successive identity() results.""" boards = boards or {} + identities = iter(identities or []) def build_board(board, backend, args, idf): status = boards.get(board, 'ok') @@ -242,12 +255,166 @@ def build_board(board, backend, args, idf): with mock.patch.object(cb, 'build_board', build_board), \ mock.patch.object(cb, 'run_ghostfat', lambda v: ghostfat or []), \ mock.patch.object(cb, 'run_tests', lambda: tests or {'status': 'ok', 'summary': 'OK', 'firstError': ''}), \ - mock.patch.object(cb, 'identity', lambda: {'head': 'h', 'dirty': False, 'submodules': {}}), \ + mock.patch.object(cb, 'identity', lambda ports=(): dict(next(identities, CLEAN))), \ contextlib.redirect_stdout(out): code = cb.main(argv) return code, json.loads(out.getvalue().splitlines()[-1]) +class HilSelection(unittest.TestCase): + def setUp(self): + tmp = build_tmpdir() + self.addCleanup(tmp.cleanup) + self.roster = Path(tmp.name) / 'local.json' + self.roster.write_text(json.dumps({'boards': [{'name': 'lpcxpresso55s69'}]})) + self.config = str(self.roster.relative_to(ROOT)) + + def hil(self, *paths): + return cb.hil_selection(cb.select(list(paths)), self.config) + + def test_a_port_change_selects_its_roster_board(self): + self.assertEqual(self.hil('ports/lpc55/boards.c')['boards'], ['lpcxpresso55s69']) + + def test_docs_select_no_board(self): + h = self.hil('README.md') + self.assertEqual((h['boards'], h['uncovered']), ([], [])) + + def test_a_runner_change_selects_every_roster_board(self): + h = self.hil('.claude/skills/hil/scripts/hil_test.py') + self.assertEqual((h['full'], h['boards'], h['uncovered']), (True, ['lpcxpresso55s69'], [])) + + def test_a_change_to_the_build_script_the_runner_imports_selects_every_roster_board(self): + h = self.hil('.claude/skills/build/scripts/check_build.py') + self.assertEqual((h['full'], h['boards'], h['uncovered']), (True, ['lpcxpresso55s69'], [])) + + def test_a_core_change_selects_every_roster_board_and_names_ports_without_one(self): + h = self.hil('src/main.c') + self.assertEqual((h['full'], h['boards']), (True, ['lpcxpresso55s69'])) + self.assertIn('stm32f4', [u['target'] for u in h['uncovered']]) + self.assertNotIn('lpc55', [u['target'] for u in h['uncovered']]) + + def test_a_port_without_a_roster_board_is_uncovered_even_beside_a_runner_change(self): + h = self.hil('ports/stm32f4/board_flash.c', 'ports/lpc55/boards.c', '.claude/skills/hil/scripts/hil_test.py') + self.assertEqual(h['boards'], ['lpcxpresso55s69']) + self.assertEqual([u['target'] for u in h['uncovered']], ['stm32f4']) + + def test_a_build_gap_stays_uncovered_beside_a_runner_change(self): + h = self.hil('ports/ch32v20x/board_flash.c', '.claude/skills/hil/scripts/hil_test.py') + self.assertEqual(h['boards'], ['lpcxpresso55s69']) + self.assertEqual([u['target'] for u in h['uncovered']], ['ports/ch32v20x/board_flash.c']) + + def test_a_port_change_reaches_roster_boards_that_are_not_its_representative(self): + self.roster.write_text(json.dumps({'boards': [{'name': 'stm32f401_blackpill'}]})) + self.assertEqual(self.hil('ports/stm32f4/board_flash.c')['boards'], ['stm32f401_blackpill']) + + def test_another_boards_own_change_is_uncovered(self): + self.roster.write_text(json.dumps({'boards': [{'name': 'feather_stm32f405_express'}]})) + h = self.hil('ports/stm32f4/boards/stm32f401_blackpill/board.h') + self.assertEqual((h['boards'], [u['target'] for u in h['uncovered']]), ([], ['stm32f401_blackpill'])) + + def test_a_roster_entry_is_named_by_its_tinyuf2_board(self): + self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk', 'tinyuf2': {'lock': 'mimxrt1060_evk'}}]})) + self.assertEqual(self.hil('ports/mimxrt10xx/boards/imxrt1060_evk/board.h')['boards'], ['imxrt1060_evk']) + self.assertEqual(self.hil('ports/mimxrt10xx/board_flash.c')['boards'], ['imxrt1060_evk']) + self.assertNotIn('mimxrt10xx', [u['target'] for u in self.hil('src/main.c')['uncovered']]) + + def test_the_hil_follows_its_cmake_build_not_cis_make(self): + self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk'}]})) + h = self.hil('ports/mimxrt10xx/family.cmake', 'ports/mimxrt10xx/boards/imxrt1060_evk/board.cmake') + self.assertEqual((h['boards'], h['uncovered']), (['imxrt1060_evk'], [])) + h = self.hil('ports/mimxrt10xx/port.mk', 'ports/mimxrt10xx/boards/imxrt1064_evk/board.cmake') + self.assertEqual(h['boards'], []) + self.assertEqual({u['target']: u['reason'] for u in h['uncovered']}, + {'ports/mimxrt10xx/port.mk': 'the HIL builds with cmake, which never reads ports/mimxrt10xx/port.mk', + 'ports/mimxrt10xx/boards/imxrt1064_evk/board.cmake': + f'no {self.config} board builds ports/mimxrt10xx/boards/imxrt1064_evk/board.cmake'}) + + def test_shared_make_files_and_port_apps_are_outside_the_hil_build(self): + self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk'}]})) + h = self.hil('ports/make.mk', 'ports/mimxrt10xx/apps/factory_test/CMakeLists.txt') + self.assertEqual((h['full'], h['boards']), (False, [])) + self.assertEqual([u['target'] for u in h['uncovered']], + ['ports/make.mk', 'ports/mimxrt10xx/apps/factory_test/CMakeLists.txt']) + self.assertEqual(self.hil('ports/family_support.cmake')['boards'], ['imxrt1060_evk']) + + def test_an_app_source_is_uncovered_since_the_battery_runs_no_app(self): + self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk'}]})) + app, shared = 'ports/mimxrt10xx/apps/factory_test/main.c', 'apps/blinky/src/blinky.c' + self.assertEqual(sorted(cb.select([app])['boards']), [('metro_m7_1011', 'make')]) # its build is unchanged + h = self.hil(app, shared) + self.assertEqual((h['full'], h['boards']), (False, [])) + self.assertEqual([u['target'] for u in h['uncovered']], [shared, app]) + h = self.hil(app, 'ports/mimxrt10xx/board_flash.c') # a bootloader change beside it still runs the battery + self.assertEqual((h['boards'], [u['target'] for u in h['uncovered']]), (['imxrt1060_evk'], [app])) + + def test_unreadable_roster_fails(self): + self.roster.write_text('{}') + with self.assertRaises(cb.Fail): + self.hil('ports/lpc55/boards.c') + + def test_select_only_adds_hil_only_when_asked(self): + for argv, has in ((['--select-only', '--scope', 'README.md'], False), + (['--select-only', '--scope', 'README.md', '--hil-config', self.config], True)): + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(cb.main(argv), 0) + self.assertEqual('hil' in json.loads(out.getvalue()), has) + + +class Identity(unittest.TestCase): + """A build record names each input it used and whether it has local changes; a dependency is a + separate git checkout (a symlink in a worktree), so the parent's status cannot see its changes.""" + + def setUp(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.repo = Path(tmp.name) + env = mock.patch.dict(os.environ, {k: v for k, v in os.environ.items() if not k.startswith('GIT_')}, clear=True) + env.start() + self.addCleanup(env.stop) + self.pin = None + for patch in (mock.patch.object(cb, 'ROOT', self.repo), + mock.patch.object(cb, 'get_deps_table', lambda: {'lib/mcu/dep': (self.pin, ['lpc55']), + 'lib/mcu/other': ('x', ['maxim'])})): + patch.start() + self.addCleanup(patch.stop) + self.dep = self.repo / 'lib' / 'mcu' / 'dep' + for repo in (self.repo, self.dep): + repo.mkdir(parents=True, exist_ok=True) + self.git(repo, 'init', '-q', '-b', 'master') + (repo / 'x.c').write_text('x') + self.git(repo, 'add', 'x.c') + self.git(repo, 'commit', '-q', '-m', 'init') + self.pin = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=self.dep, capture_output=True, text=True).stdout.strip() + + def git(self, repo, *args): + subprocess.run(['git', '-c', 'user.name=t', '-c', 'user.email=t@t', '-c', 'commit.gpgsign=false', *args], + cwd=repo, check=True, capture_output=True) + + def test_records_the_used_dependency_commit(self): + ident = cb.identity({'lpc55'}) + self.assertEqual(ident['deps'], {'lib/mcu/dep': self.pin}) + self.assertEqual((ident['dirtyInputs'], ident['missingInputs'], ident['depsOffPin']), ([], [], [])) + + def test_a_dependency_off_its_pin_is_flagged(self): + self.pin = '0' * 40 + self.assertEqual(cb.identity({'lpc55'})['depsOffPin'], ['lib/mcu/dep']) + + def test_a_dependency_that_is_not_its_own_checkout_is_missing_not_the_parent(self): + import shutil + shutil.rmtree(self.dep / '.git') # a plain directory: git -C would answer for the parent + ident = cb.identity({'lpc55'}) + self.assertEqual((ident['missingInputs'], ident['deps']), (['lib/mcu/dep'], {})) + + def test_a_modified_or_untracked_dependency_file_at_the_same_commit_is_dirty(self): + sha = cb.identity({'lpc55'})['deps']['lib/mcu/dep'] + (self.dep / 'x.c').write_text('changed') + ident = cb.identity({'lpc55'}) + self.assertEqual((ident['deps']['lib/mcu/dep'], ident['dirtyInputs']), (sha, ['lib/mcu/dep'])) + self.git(self.dep, 'checkout', '-q', '--', 'x.c') + (self.dep / 'new.h').write_text('untracked') + self.assertEqual(cb.identity({'lpc55'})['dirtyInputs'], ['lib/mcu/dep']) + + class Verdict(unittest.TestCase): def test_pass(self): code, r = run_main(['--board', 'lpcxpresso55s69']) @@ -298,24 +465,65 @@ def test_usage_error_is_json_exit_2(self): self.assertEqual(e.exception.code, 2) self.assertIn('--worktree needs --base', json.loads(out.getvalue())['error']) + def test_backend_override_needs_board_and_shared_and_a_cmake_build(self): + for argv, error in ((['--board', 'imxrt1060_evk', '--backend', 'cmake'], '--backend needs --board and --shared'), + (['--scope', 'src', '--shared', '--backend', 'cmake'], '--backend needs --board and --shared'), + (['--board', 'lpcxpresso55s69', '--shared', '--backend', 'cmake'], 'ports/lpc55 has no cmake build'), + (['--board', 'espressif_esp32s3_devkitm_1', '--shared', '--backend', 'cmake'], + 'ports/espressif has no cmake build')): + with self.subTest(argv), contextlib.redirect_stdout(io.StringIO()) as out, \ + mock.patch.object(cb, 'build_board', side_effect=AssertionError('built')): + try: + code = cb.main(argv) + except SystemExit as e: + code = e.code + self.assertEqual(code, 2) + self.assertIn(error, json.loads(out.getvalue().splitlines()[-1])['error']) + + def test_backend_override_builds_with_cmake(self): + seen = [] + def build_board(board, backend, args, idf): + seen.append(backend) + return {'board': board, 'port': 'mimxrt10xx', 'backend': backend, 'buildDir': 'cmake-build/x', 'status': 'ok', + 'firstError': '', 'artifacts': [], 'sha256': {}} + with mock.patch.object(cb, 'build_board', build_board), \ + mock.patch.object(cb, 'identity', lambda ports=(): {'head': 'h', 'dirty': False, 'submodules': {}, 'deps': {}, 'dirtyInputs': [], 'missingInputs': [], 'depsOffPin': []}), contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(cb.main(['--board', 'imxrt1060_evk', '--shared', '--backend', 'cmake']), 0) + self.assertEqual(seen, ['cmake']) + def test_passing_private_roots_leave_nothing_in_built(self): def build_board(board, backend, args, idf): return {'board': board, 'port': 'lpc55', 'backend': backend, 'buildDir': 'cmake-build/x', 'status': 'ok', 'firstError': '', 'artifacts': ['a.bin'], 'removed': True} with mock.patch.object(cb, 'build_board', build_board), \ - mock.patch.object(cb, 'identity', lambda: {}), contextlib.redirect_stdout(io.StringIO()) as out: + mock.patch.object(cb, 'identity', lambda ports=(): {'head': 'h', 'dirty': False, 'submodules': {}, 'deps': {}, 'dirtyInputs': [], 'missingInputs': [], 'depsOffPin': []}), contextlib.redirect_stdout(io.StringIO()) as out: self.assertEqual(cb.main(['--board', 'lpcxpresso55s69']), 0) self.assertEqual(json.loads(out.getvalue().splitlines()[-1])['built'], []) def test_unreadable_get_deps_is_json_exit_2(self): - cb.deps_rows.cache_clear() - self.addCleanup(cb.deps_rows.cache_clear) + cb.get_deps_table.cache_clear() + self.addCleanup(cb.get_deps_table.cache_clear) with mock.patch.object(cb.runpy, 'run_path', side_effect=SyntaxError('invalid syntax')), \ mock.patch.object(cb, 'changed_paths', lambda *a: (['lib/mcu/nxp/mcux-sdk/x.h'], {'mode': 'paths'})): code, r = run_main(['--scope', 'lib/mcu/nxp/mcux-sdk/x.h']) self.assertEqual((code, r['pass']), (2, False)) self.assertIn('tools/get_deps.py', r['error']) + def test_an_input_the_build_dirtied_is_recorded(self): + _, r = run_main(['--board', 'lpcxpresso55s69'], identities=[CLEAN, dict(CLEAN, dirtyInputs=['lib/uf2'])]) + self.assertEqual(r['input']['inputsDirtiedByBuild'], ['lib/uf2']) + + def test_an_input_that_moved_during_the_build_is_recorded(self): + sdk = 'lib/mcu/nxp/mcux-sdk' + before = dict(CLEAN, submodules={'lib/uf2': 'a'}, deps={sdk: 'a'}) + for what, changed, moved in (('nothing', {}, []), ('head', {'head': 'h2'}, ['.']), ('tree', {'dirty': True}, ['.']), + ('submodule commit', {'submodules': {'lib/uf2': 'b'}}, ['lib/uf2']), + ('dependency commit', {'deps': {sdk: 'b'}}, [sdk]), + ('dependency unlinked', {'deps': {}, 'missingInputs': [sdk]}, [sdk])): + with self.subTest(what): + _, r = run_main(['--board', 'lpcxpresso55s69'], identities=[before, dict(before, **changed)]) + self.assertEqual(r['input']['inputsDirtiedByBuild'], moved) + def test_resolution_error_is_json_exit_2(self): code, r = run_main(['--board', 'no_such_board']) self.assertEqual((code, r['pass']), (2, False)) @@ -381,6 +589,7 @@ def test_shared_root_is_rebuilt_from_clean_and_kept(self): self.assertEqual(rec['status'], 'ok', rec) self.assertNotIn('removed', rec) self.assertFalse((root / 'stale.o').exists()) + self.assertEqual(set(rec['sha256']), set(rec['artifacts'])) self.assertTrue((root / '_bin' / 'tinyuf2-lpcxpresso55s69.bin').is_file()) def test_cmake_missing_app_image_fails(self): @@ -485,6 +694,8 @@ def test_expected_artifacts_per_backend(self): self.assertLessEqual({'_bin/combined.bin', '_bin/apps/update-tinyuf2.uf2'}, names('espressif', 'espressif_esp32s3_devkitm_1', 'idf')) self.assertNotIn('_bin/apps/update-tinyuf2-swan_r5.uf2', names('stm32l4', 'swan_r5', 'make')) + # mimxrt10xx copies its self-update image to ARTIFACT_PATH instead of building apps/self_update + self.assertIn('_bin/apps/update-tinyuf2.uf2', names('mimxrt10xx', 'imxrt1060_evk', 'cmake')) class FirstError(unittest.TestCase): diff --git a/.claude/test/test_hil.py b/.claude/test/test_hil.py new file mode 100644 index 000000000..b47fd59ee --- /dev/null +++ b/.claude/test/test_hil.py @@ -0,0 +1,1258 @@ +#!/usr/bin/env python3 +"""Tests of the HIL contract's runner. Stubs prove the plumbing only: every probe and USB path here is +unverified until a real run on the rig (SKILL.md, commissioning).""" +import contextlib +import importlib.util +import io +import json +import os +import signal +import struct +import subprocess +import sys +import tempfile +import textwrap +import time +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / '.claude' / 'skills' / 'hil' / 'scripts' / 'hil_test.py' +sys.dont_write_bytecode = True +spec = importlib.util.spec_from_file_location('hil_test', SCRIPT) +ht = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ht) +BOARD = 'lpcxpresso55s69' +RT = {'port': 'mimxrt10xx', 'name': 'imxrt1060_evk', 'lock': 'mimxrt1060_evk'} + +# the API hil_test.py uses from tinyusb's test/hil/helper/hil_lock.py, with the same flock and record +# semantics; set TINYUSB_HIL_LOCK to the real helper to run the lock tests against it instead +FAKE_HELPER = ''' +import fcntl, json, os, time +BOARD_LOCK_DIR = {dir!r} +CI_REASON = 'hil_test.py' +PROTECTED_REASONS = {{CI_REASON, 'pool_check'}} +def lock_path(board): return os.path.join(BOARD_LOCK_DIR, f'{{board}}.lock') +def flock_nb(board): + fh = os.fdopen(os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666), 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + fh.close(); raise + return fh +def write_record(fh, reason): + fh.truncate(0); fh.seek(0) + json.dump({{'pid': os.getpid(), 'reason': reason, 'since': ''}}, fh); fh.flush(); return True +def clear_record(fh): fh.truncate(0) +def read_record(board): + try: + with open(lock_path(board)) as f: return json.load(f) + except (OSError, ValueError): return None +def acquire_board_lock(board, reason=CI_REASON): + if os.environ.get('HIL_NO_BOARD_LOCK') == '1': return None + try: + os.makedirs(BOARD_LOCK_DIR, exist_ok=True) + fh = os.fdopen(os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666), 'r+') + except OSError: + return None + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + info = fh.read(500).strip(); fh.close(); raise RuntimeError(f'board locked: {{info}}') + write_record(fh, reason); return fh +''' + +HOLDER = ''' +import fcntl, json, os, sys, time +fh = os.fdopen(os.open(sys.argv[1], os.O_RDWR | os.O_CREAT, 0o666), 'r+') +if sys.argv[3] == 'lock': + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) +fh.truncate(0); fh.seek(0); json.dump({'pid': os.getpid(), 'reason': sys.argv[2]}, fh); fh.flush() +print('ready', flush=True) +time.sleep(60) +''' + + +def roster(tmp, **over): + t = {'reentry': 'retained-register'} + t.update(over.pop('tinyuf2', {})) + b = {'name': BOARD, 'uid': 'UID', 'flasher': {'name': 'jlink', 'uid': '1', 'args': '-device LPC55S69_M33_0'}, + 'tinyuf2': t} + b.update(over) + cfg = tmp / 'local.json' + cfg.write_text(json.dumps({'boards': [b]})) + return cfg + + +HELPER = Path('test/hil/helper/hil_lock.py') # under TINYUSB_DIR + + +# lpc55 has no cmake build, so the runner reads no facts for it; most tests keep its layout as synthetic facts +LPC_FACTS = {'flashBase': 0, 'appStart': 0x10000, 'flashSize': 0x98000, 'vid': 0x1fc9, 'pid': 0x0094, + 'boardId': 'LPC55S69-Xpresso-revA', 'family': 0x2abc77ec, 'vectors': 76, 'ram': 0x20000000, + 'dblTapDelayMs': 500, 'dblTapReg': 0x4002C05C, 'dblTapMagic': 0xF01669EF} +real_source_facts = ht.source_facts + + +def source_facts(board): + return dict(LPC_FACTS) if board['port'] == 'lpc55' else real_source_facts(board) + + +class Tmp(unittest.TestCase): + def setUp(self): + t = tempfile.TemporaryDirectory() + self.addCleanup(t.cleanup) + self.tmp = Path(t.name) + for p in (mock.patch.object(ht, 'source_facts', source_facts), + mock.patch.dict(os.environ, {'TINYUSB_DIR': str(self.tmp)})): + p.start() + self.addCleanup(p.stop) + + def load(self, cfg, name=BOARD): + """A roster entry with its layout applied, as main() admits it.""" + b = ht.load_board(cfg, name) + ht.apply_layout(b, ht.source_facts(b)) + return b + + +class Roster(Tmp): + def test_a_valid_entry_loads_with_its_layout_from_the_source(self): + b = self.load(roster(self.tmp)) + self.assertEqual((b['install'], b['app'], b['page'], b['device'], b['if'], b['speed']), + ((0, 0x10000), (0x10000, 0x98000), 512, 'LPC55S69_M33_0', 'SWD', '4000')) + self.assertEqual(b['helper'], str(self.tmp / HELPER)) + + def test_refusals(self): + cases = { + 'args': {'flasher': {'name': 'jlink', 'uid': '1', 'args': '-device LPC55S69_M33_0 -if SWD'}}, + 'int uid': {'flasher': {'name': 'jlink', 'uid': 1, 'args': '-device LPC55S69_M33_0'}}, + 'unknown device': {'flasher': {'name': 'jlink', 'uid': '1', 'args': '-device NRF52840_XXAA'}}, + 'bad lock name': {'tinyuf2': {'lock': '../x'}}, + 'not a TinyUF2 board': {'name': 'no_such_board'}, + } + for what, over in cases.items(): + with self.subTest(what), self.assertRaises(ht.Refused): + ht.load_board(roster(self.tmp, **over), over.get('name', BOARD)) + + def test_the_lock_helper_comes_from_an_absolute_tinyusb_dir(self): + for value in ('', 'tinyusb'): + with self.subTest(value), mock.patch.dict(os.environ, {'TINYUSB_DIR': value}), \ + self.assertRaisesRegex(ht.Refused, 'TINYUSB_DIR'): + ht.load_board(roster(self.tmp), BOARD) + with mock.patch.dict(os.environ), self.assertRaisesRegex(ht.Refused, 'TINYUSB_DIR'): + del os.environ['TINYUSB_DIR'] + ht.load_board(roster(self.tmp), BOARD) + + def test_the_lock_defaults_to_the_board_name_and_the_port_comes_from_it(self): + b = ht.load_board(roster(self.tmp), BOARD) + self.assertEqual((b['lock'], b['port']), (BOARD, 'lpc55')) + b = ht.load_board(roster(self.tmp, tinyuf2={'lock': 'other'}), BOARD) + self.assertEqual((b['name'], b['lock']), (BOARD, 'other')) + + def test_layout_refusals(self): + b = ht.load_board(roster(self.tmp), BOARD) + cases = {'app start not page aligned': {'appStart': 0x10100}, + 'app start at the flash base': {'appStart': 0}, + 'app start past the flash end': {'appStart': 0x98000}, + 'flash past the ceiling': {'flashSize': 0x9E000}, + 'flash below the floor': {'flashBase': 0x100000000 - 0x98000 - 1}} + for what, over in cases.items(): + with self.subTest(what), self.assertRaises(ht.Refused): + ht.apply_layout(dict(b), dict(LPC_FACTS, **over)) + + def test_the_committed_roster_is_well_formed_and_commissioned(self): + b = self.load(ROOT / 'test' / 'hil' / 'local.json', 'imxrt1060_evk') + self.assertEqual((b['lock'], b['reentry']), ('mimxrt1060_evk', 'retained-register')) + self.assertEqual(b['install'], (0x60000000, 0x6000C000)) + + +def bootloader_bin(path, size=0x5824, board_id=b'LPC55S69-Xpresso-revA', version=b'0.35.0 - tinyusb (x)'): + body = b'TinyUF2 Bootloader ' + version + b'\r\nModel: x\r\nBoard-ID: ' + board_id + b'\r\n' + path.write_bytes(body + b'\xff' * (size - len(body))) + return path + + +class Admission(Tmp): + """Every span a bootloader install writes is checked before the lock is taken.""" + + def setUp(self): + super().setUp() + self.board = self.load(roster(self.tmp)) + self.facts = ht.source_facts(self.board) + + def test_a_good_candidate_passes(self): + ht.check_bootloader_bin(bootloader_bin(self.tmp / 'c.bin'), self.board, self.facts, 'candidate') + + def test_candidate_refusals(self): + cases = {'too big for the install range': bootloader_bin(self.tmp / 'a.bin', size=0x10001), + 'another board': bootloader_bin(self.tmp / 'b.bin', board_id=b'OTHER')} + for what, path in cases.items(): + with self.subTest(what), self.assertRaises(ht.Refused): + ht.check_bootloader_bin(path, self.board, self.facts, 'c') + +def rt_roster(tmp): + return roster(tmp, name='imxrt1060_evk', flasher={'name': 'jlink', 'uid': '2', 'args': '-device MIMXRT1062xxx6A'}, + tinyuf2={'lock': 'mimxrt1060_evk'}) + + +class Rt1060(Tmp): + """The RT1060 entry: board imxrt1060_evk, lock mimxrt1060_evk (tinyusb's name), flash from 0x60000000.""" + + def setUp(self): + super().setUp() + self.board = self.load(rt_roster(self.tmp), 'imxrt1060_evk') + self.facts = ht.source_facts(self.board) + + def test_the_layout_comes_from_the_source_and_spans_check_against_the_flash_base(self): + self.assertEqual((self.board['install'], self.board['app']), ((0x60000000, 0x6000C000), (0x6000C000, 0x60800000))) + bl = bootloader_bin(self.tmp / 'c.bin', board_id=b'MIMXRT1060-EVK-revA') + ht.check_bootloader_bin(bl, self.board, self.facts, 'c') + with self.assertRaises(ht.Refused): # bigger than the install span at its base + ht.check_bootloader_bin(bootloader_bin(self.tmp / 'big.bin', size=0xC001, board_id=b'MIMXRT1060-EVK-revA'), + self.board, self.facts, 'c') + + def test_the_build_record_is_read_by_board_name_and_must_be_cmake(self): + build = self.tmp / 'b' + (build / '_bin').mkdir(parents=True) + rows = {} + for kind in ('hex', 'bin'): + f = build / '_bin' / f'tinyuf2.{kind}' + f.write_bytes(b'TinyUF2 Bootloader 1.0\r\n') + rows[f'_bin/{f.name}'] = ht.sha256(f) + now = {'head': 'h', 'submodules': {}, 'deps': {}, 'dirty': False} + rec = self.tmp / 'rec.json' + row = {'board': 'imxrt1060_evk', 'status': 'ok', 'backend': 'cmake', 'buildDir': str(build), 'sha256': rows} + rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [row]})) + with mock.patch.object(ht.cb, 'identity', lambda ports: now): # an absolute buildDir ignores cb.ROOT + files, fw = ht.verify_record(rec, self.board, self.facts) + self.assertEqual((files['bin'].name, fw['version']), ('tinyuf2.bin', '1.0')) + rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [dict(row, backend='make')]})) + with self.assertRaisesRegex(ht.Refused, 'cmake builds only .*--backend cmake --shared'): + ht.verify_record(rec, self.board, self.facts) + rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [row]})) + (build / '_bin' / 'tinyuf2.bin').unlink() + with self.assertRaisesRegex(ht.Refused, 'tinyuf2.bin: missing'): + ht.verify_record(rec, self.board, self.facts) + with mock.patch.object(ht.cb, 'identity', side_effect=ht.cb.Fail('git rev-parse HEAD failed')), \ + self.assertRaisesRegex(ht.Refused, 'cannot establish the checkout identity: git rev-parse'): + ht.verify_record(rec, self.board, self.facts) + + +class SourceFacts(unittest.TestCase): + def test_a_port_without_a_cmake_build_has_no_facts(self): + with self.assertRaisesRegex(ht.Refused, 'ports/lpc55 has no cmake build'): + ht.source_facts({'port': 'lpc55', 'name': BOARD}) + + def test_rt1060_facts_come_from_the_source(self): + f = ht.source_facts(RT) + self.assertEqual((f['flashBase'], f['appStart'], f['flashSize'], f['vid'], f['pid'], f['boardId']), + (0x60000000, 0x6000C000, 0x800000, 0x239a, 0x0083, 'MIMXRT1060-EVK-revA')) + self.assertEqual((f['dblTapReg'], f['ram'], f['vectors']), (0x400D410C, 0x20000000, 174)) # SNVS LPGPR[3] + self.assertEqual(f['family'], 0x4fb2d5bd) + + def test_a_cmake_value_is_one_plain_set(self): + t = tempfile.TemporaryDirectory() + self.addCleanup(t.cleanup) + (Path(t.name) / 'x.cmake').write_text(' # set(SDK_DIR old)\nset(SDK_DIR ${TOP}/lib/sdk)\nset(ID 0x1)\nset(ID 0x2)\nset(V ${OTHER}/x)\n' + 'set(Q "a")\nset(F 0x1)\nSET (F 0x2)\nset(E)\n') + with mock.patch.object(ht, 'ROOT', Path(t.name)): + self.assertEqual(ht.cmake_var('x.cmake', 'SDK_DIR'), 'lib/sdk') + for name in ('ID', 'V', 'Q', 'F', 'E', 'NONE'): + with self.subTest(name), self.assertRaises(ht.Refused): + ht.cmake_var('x.cmake', name) + + def test_only_literals_and_the_supported_expressions_are_read(self): + self.assertEqual(ht.product('(8*1024*1024)', 'x'), 0x800000) + self.assertEqual(ht.product('(0x98000U)', 'x'), 0x98000) + self.assertEqual(ht.flash_addr('0x10000', None, 'x'), 0x10000) + self.assertEqual(ht.flash_addr('(FLASH_BASE + 0xC000)', 0x60000000, 'x'), 0x6000C000) + for expr, base in (('(FLASH_BASE + 0xC000)', None), ('(OTHER + 0xC000)', 0), ('FLASH_BASE - 4', 0), ('(1 << 20)', 0)): + with self.subTest(expr), self.assertRaises(ht.Refused): + ht.flash_addr(expr, base, 'x') if 'BASE' in expr or 'OTHER' in expr else ht.product(expr, 'x') + +def ihex(records): + lines = [] + for kind, addr, data in records: + rec = bytes([len(data)]) + addr.to_bytes(2, 'big') + bytes([kind]) + data + lines.append(':' + (rec + bytes([(-sum(rec)) & 0xFF])).hex().upper()) + return '\n'.join(lines + [':00000001FF']) + '\n' + + +class Images(Tmp): + def test_hex_extended_address_and_range_checks(self): + p = self.tmp / 'a.hex' + p.write_text(ihex([(4, 0, b'\x00\x01'), (0, 0x0000, b'\xaa\xbb')])) # 0x10000 + data = ht.parse_hex(p) + self.assertEqual(data, {0x10000: 0xaa, 0x10001: 0xbb}) + lim = (0, 0x9D800) + self.assertEqual(ht.check_ranges(data, [(0x10000, 0x98000)], lim, 'x'), (0x10000, 0x10002)) + with self.assertRaises(ht.Refused): + ht.check_ranges(data, [(0, 0x10000)], lim, 'x') + with self.assertRaises(ht.Refused): + ht.check_ranges({0x9D7FF: 1, 0x9D800: 1}, [(0, 0x9E000)], lim, 'x') # one byte over the ceiling + self.assertEqual(ht.check_ranges({0x9D7FF: 1}, [(0, 0x9E000)], lim, 'x'), (0x9D7FF, 0x9D800)) + with self.assertRaises(ht.Refused): # one byte under the floor, though inside the allowed span + ht.check_ranges({0x5FFFFFFF: 1, 0x60000000: 1}, [(0x5FFF0000, 0x60010000)], (0x60000000, 0x60800000), 'x') + + def test_a_bad_checksum_is_refused(self): + p = self.tmp / 'a.hex' + p.write_text(':0100000000FE\n') + with self.assertRaises(ht.Refused): + ht.parse_hex(p) + + def test_bin_must_be_the_hex_laid_out_from_its_base(self): + p, b = self.tmp / 'a.hex', self.tmp / 'a.bin' + p.write_text(ihex([(0, 0, b'\x01\x02'), (0, 4, b'\x05')])) + b.write_bytes(b'\x01\x02\xff\xff\x05') + self.assertTrue(ht.hex_matches_bin(ht.parse_hex(p), b, 0)) + self.assertFalse(ht.hex_matches_bin(ht.parse_hex(p), b, 0x60000000)) + b.write_bytes(b'\x01\x02\xff\xff\x06') + self.assertFalse(ht.hex_matches_bin(ht.parse_hex(p), b, 0)) + p.write_text(ihex([(4, 0, b'\x60\x00'), (0, 0, b'\x01\x02'), (0, 4, b'\x06')])) # at 0x60000000 + self.assertTrue(ht.hex_matches_bin(ht.parse_hex(p), b, 0x60000000)) + + def test_embedded_version(self): + b = self.tmp / 'a.bin' + b.write_bytes(b'\0junk TinyUF2 Bootloader 0.35.0-1-gabc - tinyusb (x)\r\nModel') + self.assertEqual(ht.embedded_version(b), '0.35.0-1-gabc - tinyusb (x)') + + +class Exclusive(unittest.TestCase): + def test_the_udev_property_gets_a_bounded_wait(self): + seen = iter([False, False, True]) + with mock.patch.object(ht, 'udisks_ignored', lambda d: next(seen)), mock.patch.object(ht.time, 'sleep', lambda s: None), \ + mock.patch.object(ht, 'Path', lambda p: mock.Mock(read_text=lambda: '')): + ht.check_exclusive('/dev/sdz') + clock = [0.0] + with mock.patch.object(ht, 'udisks_ignored', lambda d: False), \ + mock.patch.object(ht.time, 'sleep', lambda s: clock.__setitem__(0, clock[0] + s)), \ + mock.patch.object(ht.time, 'monotonic', lambda: clock[0]), self.assertRaisesRegex(ht.StepFail, 'no UDISKS_IGNORE'): + ht.check_exclusive('/dev/sdz') + + +class Exclusion(Tmp): + def test_a_udev_rule_must_ignore_this_bootloader_by_serial(self): + facts = {'vid': 0x1fc9, 'pid': 0x0094} + rule = self.tmp / '99-tinyuf2-hil.rules' + with mock.patch.object(ht, 'UDEV_RULES', (str(self.tmp),)): + for text, ok in (('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="UID", ' + 'ENV{UDISKS_IGNORE}="1"', True), + ('# ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="UID", ' + 'ENV{UDISKS_IGNORE}="1"', False), + ('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="OTHER", ' + 'ENV{UDISKS_IGNORE}="1"', False), + ('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}!="UID", ' + 'ENV{UDISKS_IGNORE}="1"', False), + ('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="OTHERUID", ' + 'ENV{UDISKS_IGNORE}="1"', False), + ('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="UID", ' + 'ENV{UDISKS_IGNORE}="0"', False), + ('SUBSYSTEM=="block", ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ' + 'ATTRS{serial}=="UID", ENV{UDISKS_IGNORE}="1"', True), + ('ACTION=="remove", ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ' # Codex's case + 'ATTRS{serial}=="UID", ENV{UDISKS_IGNORE}="1"', False), + ('SUBSYSTEM=="usb", ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ' + 'ATTRS{serial}=="UID", ENV{UDISKS_IGNORE}="1"', False), + ('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="UID", ' + 'ATTRS{serial}=="X", ENV{UDISKS_IGNORE}="1"', False), + ('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="UID", ' + 'ENV{UDISKS_IGNORE}="1", GOTO="end"', False), + ('ATTRS{idVendor}=="1fc9", ATTRS{idProduct}=="0094", ATTRS{serial}=="UID", ' + 'ENV{UDISKS_IGNORE}="1" junk', False)): + rule.write_text(text + '\n') + with self.subTest(text): + if ok: + self.assertEqual(ht.exclusion_configured(facts, 'UID'), str(rule)) + else: + self.assertRaises(ht.Refused, ht.exclusion_configured, facts, 'UID') + + +class Fixture(Tmp): + def setUp(self): + super().setUp() + self.facts = dict(LPC_FACTS) + self.fx = ht.fixture(self.facts, b'n' * 16) + + def test_vector_table_covers_every_vector_and_no_code_sits_inside_it(self): + fx, img = self.fx, self.fx['image'] + self.assertEqual(fx['table'], 0x200) # 76 vectors * 4 = 0x130, rounded up to a power of two + self.assertGreaterEqual(fx['handler'] - self.facts['appStart'], 76 * 4) + vec = struct.unpack_from(f'<{fx["table"] // 4}I', img) + self.assertEqual((vec[0], vec[1]), (fx['sp'], fx['handler'] | 1)) + self.assertTrue(all(v == fx['fault'] | 1 for v in vec[2:])) + self.assertEqual(struct.unpack_from(' 4 -> end + img[data + 512:data + 1024] = b'a' * 512 + img[data + 1024:data + 1536] = b'b' * 512 + return img + + +class Fat(Tmp): + def vol(self, img): + dev = self.tmp / 'vol' + dev.write_bytes(bytes(img)) + return str(dev) + + def test_reads_entries_and_a_file_prefix(self): + fat = ht.Fat(self.vol(fat_image())) + self.assertEqual(fat.entries(), {'INFO_UF2.TXT': (2, 5), 'CURRENT.UF2': (3, 1024)}) + self.assertEqual(fat.file('INFO_UF2.TXT'), (b'hello', 5)) + self.assertEqual(fat.file('CURRENT.UF2', 600)[0], b'a' * 512 + b'b' * 88) + + def test_malformed_geometry_is_refused(self): + for what, (off, fmt, v) in {'no sectors per cluster': (13, ' 3: a loop, read until the file's 4 KiB + struct.pack_into(' free + with self.assertRaisesRegex(ValueError, 'chain broken'): + ht.Fat(self.vol(img)).file('CURRENT.UF2') + with self.assertRaisesRegex(ValueError, 'short read'): + ht.Fat(self.vol(fat_image()[:3 * 512])).entries() # the root dir is cut off + + def test_fat_read_runs_in_a_bounded_child_and_reports_errors_as_step_failures(self): + log = self.tmp / 'log' + r = ht.fat_read(self.vol(fat_image()), 'CURRENT.UF2', 600, log) + self.assertEqual((r['data'], r['size'], r['entries']['INFO_UF2.TXT']), (b'a' * 512 + b'b' * 88, 1024, [2, 5])) + with self.assertRaisesRegex(ht.StepFail, 'not on the volume'): + ht.fat_read(self.vol(fat_image()), 'NOPE.TXT', 10, log) + self.assertEqual(ht.CHILDREN, []) + + +class Spawn(Tmp): + def setUp(self): + super().setUp() + handler = signal.signal(signal.SIGTERM, ht.on_signal) + mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, ht.TERM_SIGNALS) + self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) + self.addCleanup(signal.signal, signal.SIGTERM, handler) + + def test_a_signal_right_after_the_child_starts_still_leaves_it_tracked_for_cleanup(self): + real, launched = subprocess.Popen, [] + self.addCleanup(lambda: [(ht.reap(p), p.communicate()) for p in launched]) + + def popen_then_signalled(*a, **k): + launched.append(real(*a, **k)) + os.kill(os.getpid(), signal.SIGTERM) + return launched[-1] + with mock.patch.object(ht.subprocess, 'Popen', popen_then_signalled), \ + self.assertRaisesRegex(ht.Interrupted, 'SIGTERM'): + ht.spawn([sys.executable, '-c', 'import time; time.sleep(30)'], self.tmp / 'log', 5) + self.assertEqual(ht.CHILDREN, launched) + self.assertEqual(ht.stuck_children(), []) # reaped by SIGKILL, not left running + self.assertIsNotNone(launched[0].poll()) + self.assertEqual(ht.CHILDREN, []) + + def test_registering_leaves_the_childs_signal_mask_alone(self): + code, out = ht.spawn(['grep', 'SigBlk', '/proc/self/status'], self.tmp / 'log', 5) + self.assertEqual(code, 0, out) + self.assertFalse(int(out.split()[1], 16) & (1 << (signal.SIGTERM - 1)), out) + + +class Locks(Tmp): + def setUp(self): + super().setUp() + mask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) # hand_off blocks SIGUSR1 in this process + self.addCleanup(signal.pthread_sigmask, signal.SIG_SETMASK, mask) + real = os.environ.get('TINYUSB_HIL_LOCK') + lockdir = self.tmp / 'locks' + lockdir.mkdir() + if real: + self.helper = ht.load_helper(real) + patch = mock.patch.object(self.helper, 'BOARD_LOCK_DIR', str(lockdir)) + patch.start() + self.addCleanup(patch.stop) + else: + (self.tmp / 'hil_lock.py').write_text(FAKE_HELPER.format(dir=str(lockdir))) + self.helper = ht.load_helper(self.tmp / 'hil_lock.py') + self.lock = self.helper.lock_path(BOARD) + + def holder(self, reason, lock=True): + p = subprocess.Popen([sys.executable, '-c', HOLDER, self.lock, reason, 'lock' if lock else 'nolock'], + stdout=subprocess.PIPE, text=True) + self.addCleanup(lambda: (p.kill(), p.wait(), p.stdout.close())) + self.assertEqual(p.stdout.readline().strip(), 'ready') + return p + + def test_a_real_manual_hold_is_proven(self): + p = self.holder('debug session 7') + ht.manual_holder(self.helper, BOARD, 'debug session 7') + self.assertTrue(ht.holds_flock(p.pid, self.lock)) + + def test_a_live_record_without_the_flock_is_refused(self): + self.holder('debug session 7', lock=False) + with self.assertRaisesRegex(ht.Refused, 'no flock'): + ht.manual_holder(self.helper, BOARD, 'debug session 7') + + def test_another_reason_or_a_protected_one_is_refused(self): + self.holder('someone else') + with self.assertRaises(ht.Refused): + ht.manual_holder(self.helper, BOARD, 'debug session 7') + with self.assertRaises(ht.Refused): + ht.manual_holder(self.helper, BOARD, 'hil_test.py') + + def test_cleanup_handoff_keeps_the_board_held_across_exec_and_recovery_can_end_it(self): + fh = self.helper.acquire_board_lock(BOARD) + holder = ht.hand_off(fh, BOARD, 'cleanup failed', [{'pid': 99, 'start': 1, 'cmd': ['mcopy']}]) + self.addCleanup(lambda: _kill(holder['pid'])) + self.assertTrue(holder['ready']) + self.assertNotIn('inProcess', holder) + with self.assertRaises(OSError): + self.helper.flock_nb(BOARD) + rec = ht.prove_hold(self.helper, BOARD, holder['pid'], reason='hil_test.py', role=ht.HOLDER_ROLE) + self.assertEqual((rec['detail'], rec['survivors'][0]['pid']), ('cleanup failed', 99)) + self.assertFalse(ht.flock_free(self.lock)) + for sig in (signal.SIGTERM, signal.SIGINT): # an operator's timeout kill does not free the board + os.kill(holder['pid'], sig) + time.sleep(0.3) + self.assertFalse(ht.flock_free(self.lock)) + os.kill(holder['pid'], signal.SIGUSR1) + ht.wait_for(lambda: ht.flock_free(self.lock), 5, 'release') + self.helper.flock_nb(BOARD).close() + self.assertIsNone(self.helper.read_record(BOARD)) + + + def test_a_holder_that_never_starts_leaves_the_lock_held_in_process(self): + parent = os.getpid() + real = ht.holder_record + + def dies_in_the_child(*a): + if os.getpid() != parent: + raise OSError('child died before its ack') + return real(*a) + for what, patch in (('fork fails', mock.patch.object(ht.os, 'fork', side_effect=OSError(11, 'EAGAIN'))), + ('pipe fails', mock.patch.object(ht.os, 'pipe', side_effect=OSError(24, 'EMFILE'))), + ('no ack', mock.patch.object(ht, 'holder_record', dies_in_the_child))): + with self.subTest(what): + fh = self.helper.acquire_board_lock(BOARD) + with patch: + holder = ht.hand_off(fh, BOARD, 'cleanup failed') + self.assertEqual((holder['pid'], holder.get('inProcess')), (parent, True)) + with self.assertRaises(OSError): + self.helper.flock_nb(BOARD) + rec = ht.prove_hold(self.helper, BOARD, parent, reason='hil_test.py', role=ht.HOLDER_ROLE) + self.assertEqual(rec['detail'], 'cleanup failed') + ht.HELD_IN_PROCESS.pop().close() + + def test_with_no_descriptor_left_the_runner_itself_holds_and_releases_only_on_sigusr1(self): + script = textwrap.dedent(f''' + import importlib.util, json, os, resource, signal, sys + spec = importlib.util.spec_from_file_location('ht', {str(SCRIPT)!r}); ht = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ht) + helper = ht.load_helper({str(self.helper.__file__)!r}) + helper.BOARD_LOCK_DIR = {str(Path(self.lock).parent)!r} + fh = helper.acquire_board_lock({BOARD!r}) + signal.pthread_sigmask(signal.SIG_BLOCK, ht.TERM_SIGNALS) + resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) + hog = [] + try: + while True: + hog.append(os.open(os.devnull, os.O_RDONLY)) + except OSError: + pass + holder = ht.hand_off(fh, {BOARD!r}, 'cleanup failed') + print(json.dumps(holder), flush=True) + ht.keep_held(ht.HELD_IN_PROCESS[0]) + ''') + p = subprocess.Popen([sys.executable, '-c', script], stdout=subprocess.PIPE, text=True) + self.addCleanup(lambda: (p.kill(), p.wait(), p.stdout.close())) + holder = json.loads(p.stdout.readline()) + self.assertEqual((holder['pid'], holder['inProcess']), (p.pid, True)) + self.assertEqual(p.stdout.read(), '') # stdio closed: the caller's read ends while the hold stays + for sig in (signal.SIGTERM, signal.SIGINT): + p.send_signal(sig) + time.sleep(0.3) + self.assertIsNone(p.poll()) + ht.prove_hold(self.helper, BOARD, p.pid, reason='hil_test.py', role=ht.HOLDER_ROLE) + p.send_signal(signal.SIGUSR1) + self.assertEqual(p.wait(5), 0) + self.helper.flock_nb(BOARD).close() + + def test_a_holder_handed_off_during_cleanup_still_ends_on_sigusr1(self): + fh = self.helper.acquire_board_lock(BOARD) + with ht.signals(signal.SIG_BLOCK): # cleanup blocks SIGTERM and SIGINT; the holder must not care + holder = ht.hand_off(fh, BOARD, 'cleanup failed') + self.addCleanup(lambda: _kill(holder['pid'])) + os.kill(holder['pid'], signal.SIGUSR1) + ht.wait_for(lambda: ht.flock_free(self.lock), 5, 'release') + + def test_one_recovery_per_board_and_tinyusb_never_lists_it_as_a_holder(self): + first = ht.recovery_lock(self.helper, BOARD) + with self.assertRaisesRegex(ht.Refused, 'another recovery holds'): + ht.recovery_lock(self.helper, BOARD) + self.helper.flock_nb(BOARD).close() # the board lock itself stays free + self.assertEqual(sorted(os.listdir(Path(self.lock).parent)), [f'{BOARD}.lock', f'{BOARD}.lock.recover']) + first.close() + ht.recovery_lock(self.helper, BOARD).close() + + def test_survivors_are_identified_against_pid_reuse(self): + me = {'pid': os.getpid(), 'start': ht.proc_start(os.getpid())} + self.assertEqual(ht.still_alive([me, dict(me, start=me['start'] + 1), {'pid': 2 ** 22 + 1, 'start': 5}]), [me]) + + def test_the_holder_keeps_none_of_the_callers_stdio(self): + fh = self.helper.acquire_board_lock(BOARD) + holder = ht.hand_off(fh, BOARD, 'x') + self.addCleanup(lambda: _kill(holder['pid'])) + ht.wait_for(lambda: not any(Path(f'/proc/{holder["pid"]}/fd/{std}').exists() for std in (0, 1, 2)), 5, + 'the holder closing its stdio') + + +def _kill(pid): + try: + os.kill(pid, signal.SIGKILL) + os.waitpid(pid, 0) + except (ProcessLookupError, ChildProcessError): + pass + + +class RunBoard(Tmp): + """The state machine around the battery: who cleans up, and what is left held.""" + + def setUp(self): + super().setUp() + mask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) + handlers = {sig: signal.getsignal(sig) for sig in ht.TERM_SIGNALS} + + def unblock(): # run_board leaves the signals blocked until exit; drop what is pending, then put the mask back + for sig in ht.TERM_SIGNALS: + signal.signal(sig, signal.SIG_IGN) + signal.pthread_sigmask(signal.SIG_SETMASK, mask) + for sig, h in handlers.items(): + signal.signal(sig, h) + self.addCleanup(unblock) + (self.tmp / 'hil_lock.py').write_text(FAKE_HELPER.format(dir=str(self.tmp / 'locks'))) + self.helper = ht.load_helper(self.tmp / 'hil_lock.py') + self.board = self.load(roster(self.tmp)) + self.facts = ht.source_facts(self.board) + self.args = mock.Mock(build_record='rec.json', fail_at=None, try_reentry=None) + for p in (mock.patch.object(ht, 'verify_record', lambda *a: ({'hex': 'x.hex', 'bin': 'x.bin'}, {'version': 'v'})), + mock.patch.object(ht, 'parse_hex', lambda p: {0: 1}), + mock.patch.object(ht, 'hex_matches_bin', lambda d, b, base: True), + mock.patch.object(ht, 'check_jlink', lambda: None), + mock.patch.object(ht, 'check_bootloader_bin', lambda *a: None), + mock.patch.object(ht, 'exclusion_configured', lambda *a: None), + mock.patch.object(ht.shutil, 'which', lambda t: '/usr/bin/' + t)): + p.start() + self.addCleanup(p.stop) + + def run_board(self, battery_run, cleanup_error=None): + cleaned = [] + + def fake_invalidate(*a, **k): + cleaned.append(True) + if cleanup_error: + raise cleanup_error + return {'pass': True, 'state': 'bootloader invalidated', 'evidence': {}} + with mock.patch.object(ht.Battery, 'run', battery_run), mock.patch.object(ht, 'invalidate', fake_invalidate): + row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) + return row, code, cleaned + + def test_pass_cleans_up_and_releases(self): + def ok(bat, files, version): + bat.touched = True + row, code, cleaned = self.run_board(ok) + self.assertEqual((code, row['pass'], cleaned, row['cleanup']['heldAfter']), (0, True, [True], False)) + self.helper.flock_nb(BOARD).close() + + def test_a_failed_step_after_touching_cleans_up_and_is_exit_1(self): + def fail(bat, files, version): + bat.touched = True + bat.step('app-ran', lambda: (_ for _ in ()).throw(ht.StepFail('pc not in the loop'))) + row, code, cleaned = self.run_board(fail) + self.assertEqual((code, row['pass'], cleaned), (1, False, [True])) + self.assertIn('app-ran', row['detail']) + + def test_an_untouched_board_is_not_cleaned_up(self): + def fail(bat, files, version): + raise ht.StepFail('probe not found') + row, code, cleaned = self.run_board(fail) + self.assertEqual((code, cleaned, row['cleanup']), (1, [], None)) + + def test_any_failed_cleanup_hands_the_lock_to_a_holder_and_is_exit_4(self): + def fail(bat, files, version): + bat.touched = True + raise ht.StepFail('x') + for err in (ht.StepFail('verify failed'), ht.Refused('range'), ValueError('parse'), ht.Interrupted('TERM')): + with self.subTest(type(err).__name__), \ + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): + row, code, _ = self.run_board(fail, cleanup_error=err) + self.assertEqual((code, row['pass'], row['cleanup']['heldAfter'], row['cleanup']['holder']['pid']), + (4, False, True, 1)) + + def test_fail_at_cleanup_skips_the_invalidate_and_hands_the_board_to_a_holder(self): + def ok(bat, files, version): + bat.touched = True + self.args.fail_at = 'cleanup' + with mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): + row, code, cleaned = self.run_board(ok) + self.assertEqual((code, cleaned, row['cleanup']['heldAfter']), (4, [], True)) + self.assertIn('--fail-at cleanup', row['cleanup']['detail']) + + def test_try_reentry_is_for_an_uncommissioned_board_only_and_reports_its_decision(self): + self.args.try_reentry = 'retained-register' + with self.assertRaisesRegex(ht.Refused, 'already commissioned'): + self.run_board(lambda *a: self.fail('ran')) + self.board['reentry'] = None + + def ok(bat, files, version): + self.assertEqual((bat.b['reentry'], bat.commissioning), ('retained-register', True)) + bat.touched = True + bat.steps.append({'step': 'reenter', 'pass': True, 'evidence': { + 'controlBootedApp': True, 'dfuCameUp': True, 'magicConsumed': True}}) + row, code, _ = self.run_board(ok) + self.assertEqual((code, row['commissioning'], row['reentryDecision']), (0, True, 'retained-register')) + + def test_an_uncommissioned_board_is_refused_before_the_lock(self): + self.board['reentry'] = None + with self.assertRaisesRegex(ht.Refused, 'commissioning'): + self.run_board(lambda *a: self.fail('ran')) + self.assertFalse(Path(self.helper.lock_path(BOARD)).exists()) + + def test_a_signal_during_cleanup_waits_for_it(self): + def ok(bat, files, version): + bat.touched = True + + def invalidate_then_signalled(*a, **k): + os.kill(os.getpid(), signal.SIGTERM) # blocked until cleanup has ended + return {'pass': True, 'state': 's', 'evidence': {}} + + signal.signal(signal.SIGTERM, ht.on_signal) + with mock.patch.object(ht.Battery, 'run', ok), mock.patch.object(ht, 'invalidate', invalidate_then_signalled): + row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) + self.assertEqual((code, row['pass'], row['cleanup']['heldAfter']), (0, True, False)) + self.assertIn('SIGTERM deferred', row['detail']) + self.helper.flock_nb(BOARD).close() + + def test_a_signal_pending_at_a_locked_board_still_gives_its_row(self): + held = self.helper.acquire_board_lock(BOARD, reason='someone') + self.addCleanup(held.close) + signal.signal(signal.SIGTERM, ht.on_signal) + real = self.helper.acquire_board_lock + + def signalled_while_locking(board): + os.kill(os.getpid(), signal.SIGTERM) + return real(board) + with mock.patch.object(self.helper, 'acquire_board_lock', signalled_while_locking): + row, code, _ = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['locked'], row['ran']), (1, True, False)) + self.assertIn('SIGTERM deferred', row['detail']) + + def test_a_stuck_child_is_a_cleanup_failure_with_no_invalidate(self): + def fail(bat, files, version): + bat.touched = True + raise ht.StepFail('x') + stuck = [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}] + with mock.patch.object(ht, 'stuck_children', lambda: list(stuck)), \ + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'survivors': s})[1]): + row, code, cleaned = self.run_board(fail) + self.assertEqual((code, cleaned, row['cleanup']['survivors']), (4, [], stuck)) + self.assertIn('99', row['cleanup']['detail']) + + def test_a_survivor_on_an_untouched_board_still_keeps_it_held(self): + def fail(bat, files, version): + raise ht.StepFail('identity: probe hung') + with mock.patch.object(ht, 'stuck_children', lambda: [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}]), \ + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1})[1]): + row, code, cleaned = self.run_board(fail) + self.assertEqual((code, cleaned, row['cleanup']['state'], row['cleanup']['heldAfter']), + (4, [], 'untouched', True)) + + def test_repeated_signals_interrupt_the_battery_but_never_the_cleanup(self): + signal.signal(signal.SIGTERM, ht.on_signal) + + def interrupted(bat, files, version): + bat.touched = True + os.kill(os.getpid(), signal.SIGTERM) + self.fail('the signal did not interrupt the battery') + + def invalidate_under_fire(*a, **k): + for _ in range(3): + os.kill(os.getpid(), signal.SIGTERM) + return {'pass': True, 'state': 's', 'evidence': {}} + with mock.patch.object(ht.Battery, 'run', interrupted), mock.patch.object(ht, 'invalidate', invalidate_under_fire): + row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) + self.assertEqual((code, row['cleanup']['pass'], row['cleanup']['heldAfter']), (1, True, False)) + self.assertIn('interrupted by SIGTERM', row['detail']) + self.assertIn('SIGTERM deferred', row['detail']) + self.helper.flock_nb(BOARD).close() + + def test_an_abort_names_the_step_it_cut_short_and_records_no_failing_step(self): + def cut_short(bat, files, version): + bat.touched = True + bat.step('dfu', lambda: {}) + bat.step('copy', lambda: (_ for _ in ()).throw(ht.Interrupted('interrupted by SIGTERM'))) + row, code, cleaned = self.run_board(cut_short) + self.assertEqual((code, cleaned, [s['pass'] for s in row['steps']]), (1, [True], [True])) + self.assertTrue(row['detail'].startswith('copy: Interrupted: interrupted by SIGTERM'), row['detail']) + + def test_a_locked_board_is_not_touched(self): + held = self.helper.acquire_board_lock(BOARD, reason='someone') + self.addCleanup(held.close) + row, code, cleaned = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['locked'], row['ran'], cleaned), (1, True, False, [])) + + def test_the_board_is_locked_and_reported_under_its_lock_name(self): + self.board = self.load(roster(self.tmp, tinyuf2={'lock': 'tinyusb_name'})) + held = self.helper.acquire_board_lock('tinyusb_name', reason='someone') + self.addCleanup(held.close) + row, code, _ = self.run_board(lambda *a: self.fail('ran')) + self.assertEqual((code, row['board'], row['lock'], row['locked']), (1, BOARD, 'tinyusb_name', True)) + + def test_an_unusable_lock_dir_is_refused_not_run_unlocked(self): + with mock.patch.object(self.helper, 'acquire_board_lock', lambda b: None), \ + mock.patch.dict(os.environ, {'HIL_NO_BOARD_LOCK': ''}): + with self.assertRaises(ht.Refused): + self.run_board(lambda *a: self.fail('ran')) + + def test_fail_at_injects_a_criterion_failure(self): + bat = ht.Battery(self.board, self.facts, self.tmp, 'dfu') + bat.step('identity', lambda: {}) + with self.assertRaisesRegex(ht.StepFail, 'injected'): + bat.step('dfu', lambda: {}) + + +class FakeRig: + """A probe and a UF2 drive that behave as the battery expects; each test breaks one thing.""" + + def __init__(self, bat, reentry): + self.bat, self.reentry, self.scripts, self.present = bat, reentry, [], True + self.r0, self.mem, self.double_reset, self.flash = 5, {}, False, set() + + def probe(self, commands, judge=True): + self.scripts.append(list(commands)) + f, out = self.bat.f, '' + for c in commands: + if c == 'regs': + self.r0 += 7 + fx = self.bat.fix + out += (f'J-Link>regs\nPC = {fx["handler"]:08X}, CycleCnt = 0\nR0 = {self.r0:08X}, R1 = 00000000\n' + f'SP(R13)= {fx["sp"]:08X}, MSP= {fx["sp"]:08X}, PSP= 00000000\nIPSR = 000 (NoException)\n') + elif c.startswith('mem32 '): + a = int(c.split()[1], 16) + v = f['appStart'] if a == ht.SCB_VTOR else self.mem.get(a, 0) + out += f'J-Link>{c}\n{a:08X} = {v:08X} \n' + elif c.startswith(f'w4 {ht.SCB_AIRCR:08X}'): + # DFU only with the magic retained, or when a second reset lands in the bootloader's own window + self.present = self.mem.pop(f['dblTapReg'], None) == f['dblTapMagic'] or self.double_reset + elif c == 'r' and self.reentry == 'app-invalidate': + self.present = True + elif c.startswith('w4 '): + _, a, v = c.split() + self.mem[int(a, 16)] = int(v, 16) + elif c.startswith('loadbin '): + note = 'Skipped. Contents already match' if c in self.flash else '1 range affected (65536 bytes)' + self.flash.add(c) + out += f'J-Link>{c}\nJ-Link: Flash download: Bank 0 @ 0x60000000: {note}\nO.K.\n' + return out + + def spawn(self, cmd, log, timeout, cwd=None): + if cmd[0] == 'mcopy': + self.present = False + return 0, '' + + def fat_read(self, dev, name, length, log): + f = self.bat.f + if name == 'INFO_UF2.TXT': + data = (f'TinyUF2 Bootloader v1\r\nModel: x\r\nBoard-ID: {f["boardId"]}\r\nDate: d\r\n' + f'Flash Size: 0x{f["flashSize"]:08X} bytes').encode() + else: + data = ht.make_uf2(bytes(self.current), f['appStart'], f['family'], self.bat.dir / 'cur')[1].read_bytes()[:length] + return {'entries': {'CURRENT.UF2': [9, f['flashSize'] // 256 * 512]}, 'data': data, 'size': len(data)} + + +class BatteryRun(Tmp): + """Battery.run end to end over a fake probe transcript and a fake drive: the plumbing, not the rig.""" + + def battery(self, reentry='retained-register', corrupt=None, commissioning=False): + board = self.load(roster(self.tmp, tinyuf2={'reentry': reentry})) + facts = ht.source_facts(board) + bat = ht.Battery(board, facts, self.tmp, commissioning=commissioning) + bootloader_bin(self.tmp / 'c.bin') + rig = FakeRig(bat, reentry) + (self.tmp / 'cur').mkdir() + bat.probe.run = rig.probe + clock = [0.0] # time passes only when the battery sleeps + + def fat_read(*a): + if a[1] == 'CURRENT.UF2': + rig.current = bytearray(bat.fix['image']) + if reentry == 'app-invalidate': + rig.current[:board['page']] = bytes(board['page']) + if corrupt: + corrupt(rig.current) + return rig.fat_read(*a) + for p in (mock.patch.object(ht, 'usb_device', lambda *a: '1-2' if rig.present else None), + mock.patch.object(ht, 'block_device', lambda p: '/dev/sdz'), + mock.patch.object(ht, 'check_exclusive', lambda d: None), + mock.patch.object(ht, 'fat_read', fat_read), + mock.patch.object(ht, 'spawn', rig.spawn), + mock.patch.object(ht.time, 'sleep', lambda s: clock.__setitem__(0, clock[0] + s)), + mock.patch.object(ht.time, 'monotonic', lambda: clock[0] + 1e-3)): + p.start() + self.addCleanup(p.stop) + return bat, rig + + def test_every_step_passes_and_resumes_only_after_a_halt(self): + for reentry in ht.REENTRY: + with self.subTest(reentry): + self.tmp = Path(tempfile.mkdtemp(dir=self.tmp)) + bat, rig = self.battery(reentry) + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + self.assertEqual([x['step'] for x in bat.steps if x['pass']], list(ht.STEPS)) + image = self.tmp / 'program.bin' + self.assertEqual(rig.scripts[1], ['h', f'loadbin {image},0x0', f'verifybin {image},0x0', 'r', 'g']) + self.assertEqual(rig.scripts[2][:2], ['h', f'w4 {bat.f["dblTapReg"]:08X} 00000000']) # cleared in dfu + self.assertIn(f'sleep {2 * bat.f["dblTapDelayMs"] + 200}', rig.scripts[3]) + for script in rig.scripts: + if 'g' in script: + self.assertEqual(script[0], 'h', script) + if reentry == 'retained-register': + self.assertIn([f'w4 {ht.SCB_AIRCR:08X} {ht.AIRCR_SYSRESETREQ:08X}'], rig.scripts) + + def test_the_whole_fixture_is_read_back_including_its_last_partial_block(self): + bat, _ = self.battery(corrupt=lambda img: img.__setitem__(len(img) - 1, img[-1] ^ 1)) + with self.assertRaisesRegex(ht.StepFail, 'read back the fixture'): + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + self.assertEqual(bat.steps[-1]['step'], 'reenter') + + def test_a_readback_failure_keeps_the_reentry_observations(self): + for what, corrupt, fat in (('mismatch', lambda img: img.__setitem__(0, img[0] ^ 1), None), + ('not UF2', None, lambda: bytes(1024)), # the empty CURRENT.UF2 suspected on RT1060 + ('truncated', None, lambda: ht.make_uf2(b'x' * 256, 0x10000, 1, self.tmp)[1].read_bytes() + b'x')): + with self.subTest(what): + self.tmp = Path(tempfile.mkdtemp(dir=self.tmp)) + bat, rig = self.battery(corrupt=corrupt) + if fat: + read = ht.fat_read + ht.fat_read = lambda *a: dict(read(*a), data=fat()) if a[1] == 'CURRENT.UF2' else read(*a) + self.addCleanup(setattr, ht, 'fat_read', read) + with self.assertRaises(ht.StepFail): + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + last = bat.steps[-1] + self.assertEqual((last['step'], last['pass'], last['evidence']['failedAt']), ('reenter', False, 'readback')) + self.assertTrue(last['evidence']['dfuCameUp'] and last['evidence']['magicConsumed']) + self.assertEqual((last['evidence']['appIntact'], last['evidence']['claim']), + ('unshown', 'the write/reset sequence re-entered DFU (register retention inferred)')) + + def test_commissioning_runs_the_control_first_and_no_probe_session_before_dfu(self): + bat, rig = self.battery(commissioning=True) + dfu = bat.dfu + bat.dfu = lambda v: rig.scripts.append('dfu observed') or dfu(v) + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + ev = bat.steps[-1]['evidence'] + self.assertTrue(ev['controlBootedApp'] and ev['dfuCameUp'] and ev['magicConsumed']) + aircr = [f'w4 {ht.SCB_AIRCR:08X} {ht.AIRCR_SYSRESETREQ:08X}'] + reenter = rig.scripts[rig.scripts.index(aircr):] + self.assertEqual(reenter[0], aircr) # the control: no magic written before it + self.assertIn('regs', reenter[1]) # the app observed again + self.assertEqual(reenter[2][1], f'w4 {bat.f["dblTapReg"]:08X} {bat.f["dblTapMagic"]:08X}') + self.assertEqual(reenter[3:5], [aircr, 'dfu observed']) # nothing opens the probe in between + + def test_a_failure_after_the_control_keeps_what_was_observed(self): + bat, rig = self.battery(commissioning=True) + dfu, calls = bat.dfu, [] + + def flaky(v): # the install's DFU passes; the re-entry's finds no exclusion property + calls.append(v) + if len(calls) > 1: + raise ht.StepFail('/dev/sdz has no UDISKS_IGNORE=1') + return dfu(v) + bat.dfu = flaky + with self.assertRaises(ht.StepFail): + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + ev = bat.steps[-1]['evidence'] + self.assertEqual((bat.steps[-1]['step'], ev['controlBootedApp'], 'dfuCameUp' in ev), ('reenter', True, False)) + + def test_a_control_that_lands_in_dfu_fails_without_a_decision(self): + bat, rig = self.battery(commissioning=True) + rig.double_reset = True + with self.assertRaisesRegex(ht.StepFail, 'without the magic'): + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + self.assertIs(bat.steps[-1]['evidence']['controlBootedApp'], False) + + def test_one_image_carries_the_bootloader_ff_padding_and_a_zero_page(self): + bat, rig = self.battery() + fw = bootloader_bin(self.tmp / 'fw.bin', size=0x5000) + bat.program(fw, 0, 0x10000) + image = (self.tmp / 'program.bin').read_bytes() + self.assertEqual(image, fw.read_bytes() + b'\xff' * 0xb000 + bytes(512)) + self.assertEqual(rig.scripts, [['h', f'loadbin {self.tmp / "program.bin"},0x0', + f'verifybin {self.tmp / "program.bin"},0x0', 'r', 'g']]) + self.assertEqual(bat.installed, (fw, 0)) # app-invalidate re-verifies the BIN, not the image + + def test_a_bin_that_fills_the_install_span_exactly_has_no_padding(self): + bat, _ = self.battery() + fw = bootloader_bin(self.tmp / 'fw.bin', size=0x10000) + bat.program(fw, 0, 0x10000) + self.assertEqual((self.tmp / 'program.bin').read_bytes(), fw.read_bytes() + bytes(512)) + + def test_a_bin_past_the_app_start_is_refused_before_any_probe_session(self): + bat, rig = self.battery() + fw = bootloader_bin(self.tmp / 'fw.bin', size=0x10001) + with self.assertRaisesRegex(ht.StepFail, 'overlaps the app start'): + bat.program(fw, 0, 0x10000) + self.assertEqual((rig.scripts, bat.touched), ([], False)) + + def test_cleanup_zeroes_the_device_flash_floor_and_reverifies_it_from_a_new_session(self): + bat, rig = self.battery() + b = dict(bat.b, device='MIMXRT1062xxx6A') # the floor is the device's, never the candidate's layout + out = ht.invalidate(b, bat.f, self.tmp, bat) + zero = f'{bat.zero},0x60000000' + self.assertEqual(rig.scripts, [['h', f'loadbin {zero}', f'verifybin {zero}', 'r', 'g'], + ['h', f'loadbin {zero}', f'verifybin {zero}', 'g']]) + self.assertEqual((out['state'], out['evidence']), ('bootloader invalidated', {'zeroPage': '0x60000000', 'verified': True})) + + def test_cleanup_fails_when_the_new_session_has_to_write_the_zero_page_again(self): + bat, rig = self.battery() + b = dict(bat.b, device='MIMXRT1062xxx6A') + bat.probe.run = lambda commands, judge=True: rig.flash.clear() or rig.probe(commands, judge) + with self.assertRaisesRegex(ht.StepFail, 'did not persist'): + ht.invalidate(b, bat.f, self.tmp, bat) + + def test_app_invalidate_reverifies_the_bootloader_before_its_reset(self): + bat, rig = self.battery('app-invalidate') + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + script = [x for x in rig.scripts if 'r' in x][-1] + self.assertLess(script.index(f'verifybin {self.tmp / "c.bin"},0x0'), script.index('r')) + self.assertTrue(bat.steps[-1]['evidence']['bootloaderVerified']) + + def test_a_wrong_version_on_the_drive_fails_dfu(self): + bat, _ = self.battery() + with self.assertRaisesRegex(ht.StepFail, 'INFO_UF2'): + bat.run({'bin': self.tmp / 'c.bin'}, 'v2') + + def test_a_left_magic_is_cleared_right_after_dfu_and_read_back(self): + bat, rig = self.battery() + reg = bat.f['dblTapReg'] + rig.mem[reg] = bat.f['dblTapMagic'] + dfu = bat.dfu + bat.dfu = lambda v: rig.scripts.append('dfu observed') or dfu(v) + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + ev = [x for x in bat.steps if x['step'] == 'dfu'][0]['evidence'] + self.assertEqual(ev['dblTapReg'], '0x0') + self.assertEqual(rig.scripts[2:4], ['dfu observed', ['h', f'w4 {reg:08X} 00000000', f'mem32 {reg:08X} 1', 'g']]) + + def test_a_double_tap_clear_that_does_not_read_back_fails_dfu(self): + bat, rig = self.battery() + real = rig.probe + rig.probe = lambda commands, judge=True: real([c for c in commands if not c.startswith('w4 ')], judge) + bat.probe.run = rig.probe + rig.mem[bat.f['dblTapReg']] = bat.f['dblTapMagic'] + with self.assertRaisesRegex(ht.StepFail, 'reads back'): + bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + self.assertEqual([(x['step'], x['pass']) for x in bat.steps][-1], ('dfu', False)) + + def test_a_write_the_bus_dropped_fails_its_read_back(self): + bat, rig = self.battery() + rig.probe = lambda commands, judge=True: f'J-Link>mem32 4002C05C 1\n4002C05C = 00000000 \n' + bat.probe.run = rig.probe + with self.assertRaisesRegex(ht.StepFail, 'reads back 00000000'): + bat.probe.write32(0x4002C05C, 0xF01669EF) + + def test_the_rt1062_missing_reset_vector_note_passes_only_with_its_completed_reset(self): + note = 'J-Link>r\nResetTarget() start\nCould not find valid reset vector.\nTarget halted on flash header read.\nJ-Link>g\n' + rt = ht.Probe({'probe': '1', 'device': 'MIMXRT1062xxx6A', 'if': 'SWD', 'speed': '4000'}, self.tmp, None) + other = ht.Probe({'probe': '1', 'device': 'LPC55S69_M33_0', 'if': 'SWD', 'speed': '4000'}, self.tmp, None) + with mock.patch.object(ht, 'spawn', lambda *a, **k: (0, note)): + rt.run(['r', 'g']) + for probe, out in ((rt, note + 'Could not read memory.\n'), # another failure beside it + (rt, note.replace('Target halted on flash header read.\n', '')), # reset not seen completing + (other, note)): # another device + with self.subTest(out[-40:]), mock.patch.object(ht, 'spawn', lambda *a, o=out, **k: (0, o)), \ + self.assertRaisesRegex(ht.StepFail, 'JLinkExe failed'): + probe.run(['r', 'g']) + + def test_only_the_probe_connect_crash_is_retried_once_and_any_crash_fails(self): + probe = ht.Probe({'probe': '000726129165', 'device': 'd', 'if': 'SWD', 'speed': '4000'}, self.tmp, self.tmp / 'hil.log') + banner = 'Connecting to J-Link ...O.K.\nFirmware: J-Link LPCXpresso V2\nS/N: 726129165\n' + segv, abort = (-11, banner), (-6, banner + 'double free or corruption (top)\n') + ok = (0, 'J-Link>exit\n') + cases = (([segv, ok], True, 1), ([abort, ok], True, 1), + ([segv, segv], False, 1), # retried once only + ([(-9, ''), ok], False, 0), # no transcript: state unknown + ([(-11, banner + 'VTref=3.300V\n'), ok], False, 0), # past the probe connect + ([(-11, banner.replace('726129165', '1')), ok], False, 0), # another probe's serial + ([(-11, banner + 'Connecting to target via SWD\n'), ok], False, 0)) + for outcomes, passes, retries in cases: + for judge in (True, False): + calls = iter(outcomes) + probe.retried.clear() + with self.subTest(outcomes=outcomes, judge=judge), mock.patch.object(ht, 'JLINK_RETRY_DELAY', 0), \ + mock.patch.object(ht, 'spawn', lambda *a, **k: next(calls)): + if passes: + probe.run(['h'], judge=judge) + else: + with self.assertRaisesRegex(ht.StepFail, r'exit -\d+'): + probe.run(['h'], judge=judge) + self.assertEqual(len(probe.retried), retries) + + def test_a_failed_reset_fails_the_session(self): + out = 'J-Link>r\nResetTarget failed (-1).\nFailed to halt CPU.\nJ-Link>g\n\n****** Error: CPU is not halted\n' + probe = ht.Probe({'probe': '1', 'device': 'd', 'if': 'SWD', 'speed': '4000'}, self.tmp, None) + with mock.patch.object(ht, 'spawn', lambda *a, **k: (0, out)), self.assertRaisesRegex(ht.StepFail, 'JLinkExe failed'): + probe.run(['r', 'g']) + + +class AppRan(Tmp): + """The probe transcript the app-ran step reads; its exact shape is bench-checked at commissioning.""" + + def transcript(self, pc, r0s, ipsr=0, vtor=0x10000, msp=0x20001000): + dump = lambda r0: textwrap.dedent(f'''\ + PC = {pc:08X}, CycleCnt = 00000000 + R0 = {r0:08X}, R1 = 00000000, R2 = 00000000, R3 = 00000000 + SP(R13)= {msp:08X}, MSP= {msp:08X}, PSP= 00000000, R14(LR) = FFFFFFFF + XPSR = 01000000: APSR = nzcvq, EPSR = 01000000, IPSR = {ipsr:03X} (NoException) + ''') + return (f'J-Link>regs\n{dump(r0s[0])}J-Link>mem32 E000ED08 1\nE000ED08 = {vtor:08X} \n' + f'J-Link>regs\n{dump(r0s[1])}') + + def test_regs_parse_two_dumps(self): + out = self.transcript(0x10200, [5, 9]) + first, second = (ht.regs(c) for c in ht.after(out, 'regs')) + self.assertEqual((first['pc'], first['r0'], second['r0'], first['ipsr'], first['msp']), + (0x10200, 5, 9, 0, 0x20001000)) + + +class Decision(unittest.TestCase): + def row(self, tried='retained-register', passed=True, failed_at=None, cleaned=True, held=False, **obs): + ev = {'controlBootedApp': True, 'dfuCameUp': True, 'magicConsumed': True, 'bootloaderVerified': True} + ev.update(obs) + if failed_at: + ev['failedAt'] = failed_at + steps = [{'step': s, 'pass': True, 'evidence': {}} for s in ht.STEPS[:-1]] + steps.append({'step': 'reenter', 'pass': passed, 'evidence': ev}) + return {'commissioning': True, 'tried': tried, 'pass': passed, 'steps': steps, + 'cleanup': {'pass': cleaned, 'heldAfter': held}} + + def test_the_commissioning_decision(self): + for method in ht.REENTRY: + with self.subTest(method): + self.assertEqual(ht.reentry_decision(self.row(method)), method) + self.assertEqual(ht.reentry_decision(self.row(method, passed=False, failed_at='readback')), method) + self.assertIsNone(ht.reentry_decision(self.row(method, passed=False))) # failed entry + self.assertIsNone(ht.reentry_decision(self.row(method, dfuCameUp=False))) + self.assertIsNone(ht.reentry_decision(self.row(method, cleaned=False))) # failed cleanup + self.assertIsNone(ht.reentry_decision(self.row(method, held=True))) + self.assertIsNone(ht.reentry_decision(self.row(controlBootedApp=False))) + self.assertEqual(ht.reentry_decision(self.row('app-invalidate', controlBootedApp=False)), 'app-invalidate') + early = self.row(passed=False, failed_at='readback') + early['steps'][3]['pass'] = False + self.assertIsNone(ht.reentry_decision(early)) # another step failed too + self.assertIsNone(ht.reentry_decision(dict(self.row(), commissioning=False))) + self.assertIsNone(ht.reentry_decision(dict(self.row(), **{'pass': False}))) # aborted after every step passed + + +class Main(Tmp): + def test_manual_install_needs_no_reentry_and_programs_the_bin_with_a_zero_page(self): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + fw = bootloader_bin(self.tmp / 'fw.bin') + programmed = [] + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp), \ + mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ + mock.patch.object(ht, 'exclusion_configured', lambda *a: None), \ + mock.patch.object(ht, 'check_jlink', lambda: None), \ + mock.patch.object(ht.Battery, 'program', lambda bat, b, base, a: programmed.append((b, base, a))): + code = ht.main(['install', '-b', BOARD, '--firmware', str(fw), '--sha256', ht.sha256(fw), + '--under-hold', 'debug 7', str(cfg)]) + self.assertEqual((code, programmed), (0, [(str(fw), 0, 0x10000)])) + + def test_invalidate_needs_no_candidate_source(self): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + invalidated = [] + with contextlib.redirect_stdout(io.StringIO()), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp), \ + mock.patch.object(ht, 'source_facts', side_effect=AssertionError('read the candidate source')), \ + mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ + mock.patch.object(ht, 'check_jlink', lambda: None), mock.patch.object(ht, 'probe_users', lambda p: []), \ + mock.patch.object(ht, 'invalidate', lambda b, f, d: invalidated.append(f) or {'pass': True}): + self.assertEqual(ht.main(['invalidate', '-b', BOARD, '--under-hold', 'x', str(cfg)]), 0) + self.assertEqual(invalidated, [None]) + + def test_a_recovery_while_another_runs_is_refused_before_it_checks_the_holder(self): + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + report = self.tmp / 'failed.json' + report.write_text(json.dumps({'results': [{'board': BOARD, 'cleanup': {'holder': {'pid': 4242}}}]})) + busy = mock.Mock(side_effect=ht.Refused(f'{BOARD}: another recovery holds x.lock.recover')) + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp), \ + mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'check_jlink', lambda: None), \ + mock.patch.object(ht, 'recovery_lock', busy), \ + mock.patch.object(ht, 'prove_hold', side_effect=AssertionError('checked the holder')), \ + mock.patch.object(ht, 'invalidate', side_effect=AssertionError('probed')): + code = ht.main(['invalidate', '-b', BOARD, '--recover-holder', '4242', '--report', str(report), str(cfg)]) + self.assertEqual((code, json.loads(out.getvalue().splitlines()[-1])['error']), + (2, f'{BOARD}: another recovery holds x.lock.recover')) + self.assertEqual(busy.call_count, 1) + + def test_no_jlinkexe_programs_nothing_on_any_path(self): + fw = bootloader_bin(self.tmp / 'fw.bin') + cfg = roster(self.tmp, tinyuf2={'reentry': None}) + probed = [] + for argv in (['install', '-b', BOARD, '--firmware', str(fw), '--sha256', ht.sha256(fw), '--under-hold', 'x'], + ['invalidate', '-b', BOARD, '--under-hold', 'x']): + with self.subTest(argv[0]), contextlib.redirect_stdout(io.StringIO()), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / argv[0]), mock.patch.object(ht, 'load_helper', lambda p: None), \ + mock.patch.object(ht, 'manual_holder', lambda *a: None), mock.patch.object(ht, 'exclusion_configured', lambda *a: None), \ + mock.patch.object(ht.shutil, 'which', lambda t: None if t == 'JLinkExe' else '/usr/bin/' + t), \ + mock.patch.object(ht.Probe, 'run', lambda *a, **k: probed.append(a)): + self.assertEqual(ht.main(argv + [str(cfg)]), 2) + self.assertEqual(probed, []) + + + def test_a_refusal_is_a_ran_false_row_and_exit_2(self): + cfg = roster(self.tmp, flasher={'name': 'jlink', 'uid': '1', 'args': '-device NOPE'}) + out = io.StringIO() + with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp): + code = ht.main(['run', '-b', BOARD, '--build-record', 'r.json', str(cfg)]) + r = json.loads(out.getvalue().splitlines()[-1]) + self.assertEqual((code, r['pass'], r['results'][0]['ran']), (2, False, False)) + self.assertTrue(Path(r['reportPath']).is_file()) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index caabfdfe4..8564de080 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -40,6 +40,12 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # the claude-tests hook reads lib/uf2's uf2conv.py and the RT1062 SDK source + - name: Fetch test inputs + run: | + git submodule update --init lib/uf2 + python3 tools/get_deps.py mimxrt10xx + - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/pre_commit.yml b/.github/workflows/pre_commit.yml index ba79a2a37..33153bb9f 100644 --- a/.github/workflows/pre_commit.yml +++ b/.github/workflows/pre_commit.yml @@ -9,6 +9,7 @@ on: - '.github/workflows/**' - '.pre-commit-config.yaml' - 'tools/get_deps.py' + - 'test/hil/**' pull_request: branches: [ master ] paths: @@ -17,6 +18,7 @@ on: - '.github/workflows/**' - '.pre-commit-config.yaml' - 'tools/get_deps.py' + - 'test/hil/**' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -29,5 +31,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + # the claude-tests hook reads lib/uf2's uf2conv.py and the RT1062 SDK source + - name: Fetch test inputs + run: | + git submodule update --init lib/uf2 + python3 tools/get_deps.py mimxrt10xx + - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a65fe33f1..052046b56 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,4 +37,4 @@ repos: entry: python3 -m unittest discover -s .claude/test -p 'test_*.py' language: system pass_filenames: false - files: ^(\.claude/|\.github/workflows/|ports/|tools/get_deps\.py$) + files: ^(\.claude/|\.github/workflows/|ports/|test/hil/|tools/get_deps\.py$) diff --git a/CLAUDE.md b/CLAUDE.md index d483ff773..ac7a94ef7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -2,7 +2,7 @@ ## Working Rules -- Worktrees — `git submodule update --init lib/tinyusb lib/uf2`, then symlink the `deps_all` paths from `tools/get_deps.py` (`lib/mcu/...`, `lib/sct_neopixel`) to the primary checkout when it holds the pinned revisions. Never run `get_deps.py` through a symlink: it `git reset --hard`s each dependency in place. +- Worktrees — `git submodule update --init lib/tinyusb lib/uf2`, then symlink the `deps_all` paths from `tools/get_deps.py` (`lib/mcu/...`, `lib/sct_neopixel`) to the primary checkout when it holds the pinned revisions. Never run `get_deps.py` through a symlink: it `git reset --hard`s each dependency in place. Where the primary's copy has local changes, use a detached `git worktree` of that dependency at its pin instead. - Code references — ports and boards: `ports//` (`boards//`, `board_flash.c`, `boards.c`), `supported_boards.md`; bootloader: `src/` (`main.c`, `ghostfat.c`, `msc.c`, `board_api.h`, the contract every port implements); apps: `apps/`, `ports//apps/`; build: `ports/make.mk`, `ports/rules.mk`, `ports/family_support.cmake`, `.github/workflows/build.yml` (which backend builds each port); deps: `tools/get_deps.py`. ## Code Rules @@ -21,7 +21,7 @@ ## Build and Validate - Build contract: `.claude/skills/build/SKILL.md`. It resolves a change to boards and builds each the way CI does, outside the source tree. -- No HIL contract yet: hardware validation of a change is reported pending, never inferred from a build. +- HIL contract: `.claude/skills/hil/SKILL.md`. Hardware validation comes from its report, never from a build; what no roster board on the host covers is reported pending. - ESP-IDF: the build contract uses CI's docker image; for interactive work, `. "$IDF_PATH/export.sh"` and `idf.py -C ports/espressif -DBOARD= flash monitor`. - Before submitting, and again after a rebase: `pre-commit run --all-files`. - For code changes: `python3 .claude/skills/build/scripts/check_build.py --base master --worktree`. @@ -30,6 +30,6 @@ - Before opening or updating a PR, follow Build and Validate. - After opening a PR, ask the human to authorize one headless `chief` invocation to drive reviews and CI to green under agentrc chief's Authorization exception. Name the PR URL, head repository and branch, and actions: push commits to that branch, post replies to the PR's review feedback, and resolve review threads through `pr-babysit` with `autoPush: true`. On an affirmative answer, launch it in the PR worktree through agentrc's `headless-chief` skill (`chief_run.py`), never a bare `claude -p`, and relay its exit as that skill directs. The task names the PR, head repository and branch, expected HEAD and scope, and includes the question and answer verbatim; a relaunch for the same PR may carry that exchange copied verbatim from the earlier launch task. Leave the checkout to chief until it exits. TinyUF2 `pr-babysit` args: - `{"pr": , "reviewers": ["copilot", "code-scanning"], "protected": "^lib/(tinyusb|uf2)$"}` - (until CodeRabbit or Greptile is installed on the repository, only Copilot and code scanning review it, both harvested without gating; `protected` keeps the submodule pins out of automated fixes). + `{"pr": , "protected": "^(lib/(tinyusb|uf2)|test/hil/[^/]+\\.json)$"}`; reviewers are agentrc's default. + (`protected` keeps the submodule pins and the HIL rosters out of automated fixes). - The follow-up label is `followup`. diff --git a/ports/mimxrt10xx/boards.h b/ports/mimxrt10xx/boards.h index f51367fb3..f4e277019 100644 --- a/ports/mimxrt10xx/boards.h +++ b/ports/mimxrt10xx/boards.h @@ -46,6 +46,8 @@ extern uint32_t _interrupts_length[]; extern uint32_t _board_boot_length[]; +#define BOARD_FLASH_ADDR_ZERO FLASH_BASE + // Flash Start Address of Application #ifndef BOARD_FLASH_APP_START #define BOARD_FLASH_APP_START (FLASH_BASE + 0xC000) diff --git a/test/hil/local.json b/test/hil/local.json new file mode 100644 index 000000000..9ccbefa63 --- /dev/null +++ b/test/hil/local.json @@ -0,0 +1,10 @@ +{ + "boards": [ + { + "name": "imxrt1060_evk", + "uid": "4629F865D2291713E000005002004200", + "flasher": {"name": "jlink", "uid": "000726129165", "args": "-device MIMXRT1062xxx6A"}, + "tinyuf2": {"lock": "mimxrt1060_evk", "reentry": "retained-register"} + } + ] +} From fb69a9209b4aa9d5de09c57b63e59a8fb7c9b973 Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:17:28 -0400 Subject: [PATCH 17/47] Support verified i.MX RT provisioning through full-speed USB hosts Select descriptors by negotiated speed and process one-shot erase requests before app validity. Verify flash erase and TinyUF2 installation, including current RT1176 image layout. Co-authored-by: Limor Fried --- ports/mimxrt10xx/board_flash.c | 56 +++- src/main.c | 35 ++- src/usb_descriptors.c | 489 ++++++++++++++++++--------------- 3 files changed, 352 insertions(+), 228 deletions(-) diff --git a/ports/mimxrt10xx/board_flash.c b/ports/mimxrt10xx/board_flash.c index 62842fa7a..7cac3d398 100644 --- a/ports/mimxrt10xx/board_flash.c +++ b/ports/mimxrt10xx/board_flash.c @@ -240,13 +240,65 @@ bool board_flash_write(uint32_t addr, const void *src, uint32_t len) { void board_flash_erase_app(void) { TUF2_LOG1("Erase whole chip\r\n"); + // Erase requests are handled before board_flash_init(). Initialize the ROM + // configuration here as well, without writing the image before the erase. +#ifdef USE_BLHOST + ROM_API_Init(); + flash_cfg = flash_nor_config_copy; +#else + flash_cfg = flash_nor_config; +#endif + // Perform chip erase first - ROM_FLEXSPI_NorFlash_Init(FLEXSPI_INSTANCE, &flash_cfg); - ROM_FLEXSPI_NorFlash_EraseAll(FLEXSPI_INSTANCE, &flash_cfg); + status_t status = ROM_FLEXSPI_NorFlash_Init(FLEXSPI_INSTANCE, &flash_cfg); + if (status != kStatus_Success) { + TUF2_LOG1("Flash init failed: status = %ld!\r\n", status); + while (1) {} + } + + status = ROM_FLEXSPI_NorFlash_EraseAll(FLEXSPI_INSTANCE, &flash_cfg); + if (status != kStatus_Success) { + TUF2_LOG1("Chip erase failed: status = %ld!\r\n", status); + while (1) {} + } + + // Read every flash word after discarding cached contents. + SCB_InvalidateDCache_by_Addr((uint32_t*) FLASH_BASE, BOARD_FLASH_SIZE); + volatile uint32_t const* flash = (volatile uint32_t const*) FLASH_BASE; + for (uint32_t i = 0; i < BOARD_FLASH_SIZE / sizeof(uint32_t); ++i) { + if (flash[i] != 0xffffffff) { + TUF2_LOG1("Chip erase verification failed at 0x%08lX!\r\n", FLASH_BASE + i * sizeof(uint32_t)); + while (1) {} + } + } + _flash_page_addr = FLASH_CACHE_INVALID_ADDR; + TUF2_LOG1("Whole-chip erase verified.\r\n"); // Re-write bootloader to flash after chip erase (only when running from RAM) TUF2_LOG1("Erase app firmware: "); write_tinyuf2_to_flash(); + + SCB_InvalidateDCache_by_Addr((uint32_t*) FLASH_BASE, BOARD_BOOT_LENGTH); +#ifdef USE_BLHOST + // RT1176 constructs its flash headers separately from the loaded image. + if (memcmp((void const*) FLASH_FCFB_ADDR, &flash_nor_config_copy, sizeof(flexspi_nor_config_t)) != 0 || + memcmp((void const*) FLASH_IVT_ADDR, &image_vector_table, sizeof(ivt)) != 0 || + memcmp((void const*) (FLASH_IVT_ADDR + sizeof(ivt)), &g_boot_data_copy, sizeof(BOOT_DATA_T)) != 0) { + TUF2_LOG1("TinyUF2 header verification failed!\r\n"); + while (1) {} + } + uint32_t const image_address = FLASH_IVT_ADDR + (uint32_t) _ivt_length; + void const* image_data = _interrupts_origin; +#else + uint32_t const image_address = FLASH_FCFB_ADDR; + void const* image_data = &flash_nor_config; +#endif + uint32_t const image_length = FLASH_BASE + BOARD_BOOT_LENGTH - image_address; + if (memcmp((void const*) image_address, image_data, image_length) != 0) { + TUF2_LOG1("TinyUF2 installation verification failed!\r\n"); + while (1) {} + } + TUF2_LOG1("TinyUF2 installation verified.\r\n"); } bool board_flash_protect_bootloader(bool protect) { diff --git a/src/main.c b/src/main.c index bf295f988..dff634f06 100644 --- a/src/main.c +++ b/src/main.c @@ -53,6 +53,12 @@ uint8_t RGB_OFF[] = {0x00, 0x00, 0x00}; static volatile uint32_t _timer_count = 0; +#if defined(MIMXRT1011_SERIES) +// SDP may overwrite this initializer in the SRAM image before jumping to it. +volatile uint32_t tester_erase_request = 0x13579BDF; +extern uint8_t __DATA_ROM[], __DATA_RAM[]; +#endif + //--------------------------------------------------------------------+ // //--------------------------------------------------------------------+ @@ -63,6 +69,15 @@ int main(void) { if (board_init2) { board_init2(); } +#if defined(MIMXRT1011_SERIES) + if (tester_erase_request == DBL_TAP_MAGIC_ERASE_APP) { + // Clear both copies so installing this image cannot repeat the erase. + uintptr_t const request_offset = (uintptr_t) &tester_erase_request - (uintptr_t) __DATA_RAM; + *((volatile uint32_t*) ((uintptr_t) __DATA_ROM + request_offset)) = 0; + tester_erase_request = 0; + TINYUF2_DBL_TAP_REG = DBL_TAP_MAGIC_ERASE_APP; + } +#endif TUF2_LOG1("TinyUF2\r\n"); #if TINYUF2_PROTECT_BOOTLOADER @@ -106,6 +121,18 @@ int main(void) { // return true if start DFU mode, else App mode static bool check_dfu_mode(void) { +#if TINYUF2_DBL_TAP_DFU + // Consume an explicit erase request even when the application is invalid. + if (TINYUF2_DBL_TAP_REG == DBL_TAP_MAGIC_ERASE_APP) { + TUF2_LOG1("Erase app\r\n"); + TINYUF2_DBL_TAP_REG = 0; + indicator_set(STATE_WRITING_STARTED); + board_flash_erase_app(); + indicator_set(STATE_WRITING_FINISHED); + return true; + } +#endif + // Check if app is valid if (!board_app_valid()) { TUF2_LOG1("App invalid\r\n"); @@ -132,14 +159,6 @@ static bool check_dfu_mode(void) { TINYUF2_DBL_TAP_REG = 0; return true; - case DBL_TAP_MAGIC_ERASE_APP: - TUF2_LOG1("Erase app\r\n"); - TINYUF2_DBL_TAP_REG = 0; - indicator_set(STATE_WRITING_STARTED); - board_flash_erase_app(); - indicator_set(STATE_WRITING_FINISHED); - return true; - default: break; } diff --git a/src/usb_descriptors.c b/src/usb_descriptors.c index 27face9a7..b5b21347f 100644 --- a/src/usb_descriptors.c +++ b/src/usb_descriptors.c @@ -1,218 +1,271 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2019 Ha Thach (tinyusb.org) - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ - -#include "board_api.h" -#include "tusb.h" - -// Interface number -enum { -#if CFG_TUD_CDC - ITF_NUM_CDC, - ITF_NUM_CDC_DATA, -#endif - ITF_NUM_MSC, - ITF_NUM_TOTAL -}; - -// String Descriptor Index -enum { - STRID_LANGID = 0, - STRID_MANUFACTURER, - STRID_PRODUCT, - STRID_SERIAL, -#if CFG_TUD_CDC - STRID_CDC, - STRID_CDC_DATA, -#endif - STRID_MSC, -}; - -//--------------------------------------------------------------------+ -// Device Descriptors -//--------------------------------------------------------------------+ -tusb_desc_device_t TINYUF2_CONST desc_device = { - .bLength = sizeof(tusb_desc_device_t), - .bDescriptorType = TUSB_DESC_DEVICE, - .bcdUSB = 0x0200, -#if CFG_TUD_CDC - // Use Interface Association Descriptor (IAD) for CDC - .bDeviceClass = TUSB_CLASS_MISC, - .bDeviceSubClass = MISC_SUBCLASS_COMMON, - .bDeviceProtocol = MISC_PROTOCOL_IAD, -#else - .bDeviceClass = 0x00, - .bDeviceSubClass = 0x00, - .bDeviceProtocol = 0x00, -#endif - .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, - .idVendor = USB_VID, - .idProduct = USB_PID, - .bcdDevice = 0x0101, - .iManufacturer = STRID_MANUFACTURER, - .iProduct = STRID_PRODUCT, - .iSerialNumber = STRID_SERIAL, - .bNumConfigurations = 0x01 -}; - -// Invoked when received GET DEVICE DESCRIPTOR -// Application return pointer to descriptor -uint8_t const* tud_descriptor_device_cb(void) { - return (uint8_t const*) &desc_device; -} - -//--------------------------------------------------------------------+ -// Configuration Descriptor -//--------------------------------------------------------------------+ - -#define CONFIG_TOTAL_LEN (TUD_CONFIG_DESC_LEN + TUD_MSC_DESC_LEN + \ - CFG_TUD_CDC*TUD_CDC_DESC_LEN + CFG_TUD_VENDOR*TUD_VENDOR_DESC_LEN) - -// MSC is mandatory -#if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) - // MCUs that don't support a same endpoint number with different direction - #define EPNUM_MSC_OUT 0x01 - #define EPNUM_MSC_IN 0x82 -#else - // Use endpoint 1 for most devices - #define EPNUM_MSC_OUT 0x01 - #define EPNUM_MSC_IN 0x81 -#endif - -// Board/Port can force CDC endpoint numbering -#if defined(BOARD_EPNUM_CDC_OUT) && defined(BOARD_EPNUM_CDC_IN) && defined(BOARD_EPNUM_CDC_NOTIF) - #define EPNUM_CDC_NOTIF BOARD_EPNUM_CDC_NOTIF - #define EPNUM_CDC_OUT BOARD_EPNUM_CDC_OUT - #define EPNUM_CDC_IN BOARD_EPNUM_CDC_IN -#else - #if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) - // MCUs that don't support a same endpoint number with different direction - #define EPNUM_CDC_NOTIF 0x83 - #define EPNUM_CDC_OUT 0x04 - #define EPNUM_CDC_IN 0x85 - #else - #define EPNUM_CDC_NOTIF 0x82 - #define EPNUM_CDC_OUT 0x03 - #define EPNUM_CDC_IN 0x83 - #endif -#endif - -uint8_t TINYUF2_CONST desc_configuration[] = { - // Config number, interface count, string index, total length, attribute, power in mA - TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, TUSB_DESC_CONFIG_ATT_REMOTE_WAKEUP, 100), -#if CFG_TUD_CDC - // Interface number, string index, EP notification address and size, EP data address (out, in) and size. - TUD_CDC_DESCRIPTOR(ITF_NUM_CDC, STRID_CDC, EPNUM_CDC_NOTIF, 8, EPNUM_CDC_OUT, EPNUM_CDC_IN, TUD_OPT_HIGH_SPEED ? 512 : 64), -#endif - // Interface number, string index, EP Out & EP In address, EP size - TUD_MSC_DESCRIPTOR(ITF_NUM_MSC, STRID_MSC, EPNUM_MSC_OUT, EPNUM_MSC_IN, TUD_OPT_HIGH_SPEED ? 512 : 64), -}; - - -// Invoked when received GET CONFIGURATION DESCRIPTOR -// Application return pointer to descriptor -// Descriptor contents must exist long enough for transfer to complete -uint8_t const* tud_descriptor_configuration_cb(uint8_t index) { - (void) index; // for multiple configurations - - // TODO when device is highspeed, host is fullspeed. - return desc_configuration; -} - -//--------------------------------------------------------------------+ -// String Descriptors -//--------------------------------------------------------------------+ - -// Serial is 64-bit DeviceID -> 16 chars len -static char desc_str_serial[1 + 16] = { 0 }; - -// array of pointer to string descriptors -char const* string_desc_arr[] = { - (const char[]) { 0x09, 0x04 }, // 0: is supported language is English (0x0409) - USB_MANUFACTURER, // 1: Manufacturer - USB_PRODUCT, // 2: Product - desc_str_serial, // 3: Serials, use default MAC address -#if CFG_TUD_CDC - "TinyUF2 CDC", // 4: CDC Interface - NULL, -#endif - "UF2", // 4: MSC Interface -}; - -static uint16_t _desc_str[48 + 1]; - -// Invoked when received GET STRING DESCRIPTOR request -// Application return pointer to descriptor, whose contents must exist long enough for transfer to complete -uint16_t const* tud_descriptor_string_cb(uint8_t index, uint16_t langid) { - (void) langid; - uint8_t chr_count; - - switch (index) { - case STRID_LANGID: - memcpy(&_desc_str[1], string_desc_arr[0], 2); - chr_count = 1; - break; - - // TODO light alternation such as +1 to prevent conflict with application - case STRID_SERIAL: { - uint8_t serial_id[16] TU_ATTR_ALIGNED(4); - uint8_t serial_len = board_usb_get_serial(serial_id); - chr_count = 2 * serial_len; - - for (uint8_t i = 0; i < serial_len; i++) { - for (uint8_t j = 0; j < 2; j++) { - const char nibble_to_hex[16] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' - }; - uint8_t nibble = (serial_id[i] >> (j * 4)) & 0xf; - _desc_str[1 + i * 2 + (1 - j)] = nibble_to_hex[nibble]; // UTF-16-LE - } - } - break; - } - - default: { - // Convert ASCII string into UTF-16 - if (!(index < sizeof(string_desc_arr) / sizeof(string_desc_arr[0]))) return NULL; - - uint16_t const max_count = (sizeof(_desc_str) / sizeof(_desc_str[0])) - 1; - - const char* str = string_desc_arr[index]; - chr_count = strlen(str); - - // Cap at max char - if (chr_count > max_count) chr_count = max_count; - - for (uint8_t i = 0; i < chr_count; i++) { - _desc_str[1 + i] = str[i]; - } - break; - } - } - - // first byte is length (including header), second byte is string type - _desc_str[0] = (TUSB_DESC_STRING << 8) | (2 * chr_count + 2); - - return _desc_str; -} +/* + * The MIT License (MIT) + * + * Copyright (c) 2019 Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "board_api.h" +#include "tusb.h" + +// Interface number +enum { +#if CFG_TUD_CDC + ITF_NUM_CDC, + ITF_NUM_CDC_DATA, +#endif + ITF_NUM_MSC, + ITF_NUM_TOTAL +}; + +// String Descriptor Index +enum { + STRID_LANGID = 0, + STRID_MANUFACTURER, + STRID_PRODUCT, + STRID_SERIAL, +#if CFG_TUD_CDC + STRID_CDC, + STRID_CDC_DATA, +#endif + STRID_MSC, +}; + +//--------------------------------------------------------------------+ +// Device Descriptors +//--------------------------------------------------------------------+ +tusb_desc_device_t TINYUF2_CONST desc_device = { + .bLength = sizeof(tusb_desc_device_t), + .bDescriptorType = TUSB_DESC_DEVICE, + .bcdUSB = 0x0200, +#if CFG_TUD_CDC + // Use Interface Association Descriptor (IAD) for CDC + .bDeviceClass = TUSB_CLASS_MISC, + .bDeviceSubClass = MISC_SUBCLASS_COMMON, + .bDeviceProtocol = MISC_PROTOCOL_IAD, +#else + .bDeviceClass = 0x00, + .bDeviceSubClass = 0x00, + .bDeviceProtocol = 0x00, +#endif + .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, + .idVendor = USB_VID, + .idProduct = USB_PID, + .bcdDevice = 0x0101, + .iManufacturer = STRID_MANUFACTURER, + .iProduct = STRID_PRODUCT, + .iSerialNumber = STRID_SERIAL, + .bNumConfigurations = 0x01 +}; + +// Invoked when received GET DEVICE DESCRIPTOR +// Application return pointer to descriptor +uint8_t const* tud_descriptor_device_cb(void) { + return (uint8_t const*) &desc_device; +} + +//--------------------------------------------------------------------+ +// Configuration Descriptor +//--------------------------------------------------------------------+ + +#define CONFIG_TOTAL_LEN (TUD_CONFIG_DESC_LEN + TUD_MSC_DESC_LEN + \ + CFG_TUD_CDC*TUD_CDC_DESC_LEN + CFG_TUD_VENDOR*TUD_VENDOR_DESC_LEN) + +// MSC is mandatory +#if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) + // MCUs that don't support a same endpoint number with different direction + #define EPNUM_MSC_OUT 0x01 + #define EPNUM_MSC_IN 0x82 +#else + // Use endpoint 1 for most devices + #define EPNUM_MSC_OUT 0x01 + #define EPNUM_MSC_IN 0x81 +#endif + +// Board/Port can force CDC endpoint numbering +#if defined(BOARD_EPNUM_CDC_OUT) && defined(BOARD_EPNUM_CDC_IN) && defined(BOARD_EPNUM_CDC_NOTIF) + #define EPNUM_CDC_NOTIF BOARD_EPNUM_CDC_NOTIF + #define EPNUM_CDC_OUT BOARD_EPNUM_CDC_OUT + #define EPNUM_CDC_IN BOARD_EPNUM_CDC_IN +#else + #if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) + // MCUs that don't support a same endpoint number with different direction + #define EPNUM_CDC_NOTIF 0x83 + #define EPNUM_CDC_OUT 0x04 + #define EPNUM_CDC_IN 0x85 + #else + #define EPNUM_CDC_NOTIF 0x82 + #define EPNUM_CDC_OUT 0x03 + #define EPNUM_CDC_IN 0x83 + #endif +#endif + +uint8_t TINYUF2_CONST desc_fs_configuration[] = { + // Config number, interface count, string index, total length, attribute, power in mA + TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, TUSB_DESC_CONFIG_ATT_REMOTE_WAKEUP, 100), +#if CFG_TUD_CDC + // Interface number, string index, EP notification address and size, EP data address (out, in) and size. + TUD_CDC_DESCRIPTOR(ITF_NUM_CDC, STRID_CDC, EPNUM_CDC_NOTIF, 8, EPNUM_CDC_OUT, EPNUM_CDC_IN, 64), +#endif + // Interface number, string index, EP Out & EP In address, EP size + TUD_MSC_DESCRIPTOR(ITF_NUM_MSC, STRID_MSC, EPNUM_MSC_OUT, EPNUM_MSC_IN, 64), +}; + +#if TUD_OPT_HIGH_SPEED +uint8_t TINYUF2_CONST desc_hs_configuration[] = { + // Config number, interface count, string index, total length, attribute, power in mA + TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, TUSB_DESC_CONFIG_ATT_REMOTE_WAKEUP, 100), +#if CFG_TUD_CDC + // Interface number, string index, EP notification address and size, EP data address (out, in) and size. + TUD_CDC_DESCRIPTOR(ITF_NUM_CDC, STRID_CDC, EPNUM_CDC_NOTIF, 8, EPNUM_CDC_OUT, EPNUM_CDC_IN, 512), +#endif + // Interface number, string index, EP Out & EP In address, EP size + TUD_MSC_DESCRIPTOR(ITF_NUM_MSC, STRID_MSC, EPNUM_MSC_OUT, EPNUM_MSC_IN, 512), +}; + +// High-speed capable devices also describe their operation at the other speed. +static uint8_t desc_other_speed_configuration[sizeof(desc_fs_configuration)]; + +tusb_desc_device_qualifier_t TINYUF2_CONST desc_device_qualifier = { + .bLength = sizeof(tusb_desc_device_qualifier_t), + .bDescriptorType = TUSB_DESC_DEVICE_QUALIFIER, + .bcdUSB = 0x0200, +#if CFG_TUD_CDC + .bDeviceClass = TUSB_CLASS_MISC, + .bDeviceSubClass = MISC_SUBCLASS_COMMON, + .bDeviceProtocol = MISC_PROTOCOL_IAD, +#else + .bDeviceClass = 0x00, + .bDeviceSubClass = 0x00, + .bDeviceProtocol = 0x00, +#endif + .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, + .bNumConfigurations = 0x01, + .bReserved = 0x00 +}; + +uint8_t const* tud_descriptor_device_qualifier_cb(void) { + return (uint8_t const*) &desc_device_qualifier; +} + +uint8_t const* tud_descriptor_other_speed_configuration_cb(uint8_t index) { + (void) index; // for multiple configurations + + memcpy(desc_other_speed_configuration, + (tud_speed_get() == TUSB_SPEED_HIGH) ? desc_fs_configuration : desc_hs_configuration, + sizeof(desc_other_speed_configuration)); + desc_other_speed_configuration[1] = TUSB_DESC_OTHER_SPEED_CONFIG; + + return desc_other_speed_configuration; +} +#endif + + +// Invoked when received GET CONFIGURATION DESCRIPTOR +// Application return pointer to descriptor +// Descriptor contents must exist long enough for transfer to complete +uint8_t const* tud_descriptor_configuration_cb(uint8_t index) { + (void) index; // for multiple configurations + +#if TUD_OPT_HIGH_SPEED + // The host may negotiate full speed even when the device supports high speed. + return (tud_speed_get() == TUSB_SPEED_HIGH) ? desc_hs_configuration : desc_fs_configuration; +#else + return desc_fs_configuration; +#endif +} + +//--------------------------------------------------------------------+ +// String Descriptors +//--------------------------------------------------------------------+ + +// Serial is 64-bit DeviceID -> 16 chars len +static char desc_str_serial[1 + 16] = { 0 }; + +// array of pointer to string descriptors +char const* string_desc_arr[] = { + (const char[]) { 0x09, 0x04 }, // 0: is supported language is English (0x0409) + USB_MANUFACTURER, // 1: Manufacturer + USB_PRODUCT, // 2: Product + desc_str_serial, // 3: Serials, use default MAC address +#if CFG_TUD_CDC + "TinyUF2 CDC", // 4: CDC Interface + NULL, +#endif + "UF2", // 4: MSC Interface +}; + +static uint16_t _desc_str[48 + 1]; + +// Invoked when received GET STRING DESCRIPTOR request +// Application return pointer to descriptor, whose contents must exist long enough for transfer to complete +uint16_t const* tud_descriptor_string_cb(uint8_t index, uint16_t langid) { + (void) langid; + uint8_t chr_count; + + switch (index) { + case STRID_LANGID: + memcpy(&_desc_str[1], string_desc_arr[0], 2); + chr_count = 1; + break; + + // TODO light alternation such as +1 to prevent conflict with application + case STRID_SERIAL: { + uint8_t serial_id[16] TU_ATTR_ALIGNED(4); + uint8_t serial_len = board_usb_get_serial(serial_id); + chr_count = 2 * serial_len; + + for (uint8_t i = 0; i < serial_len; i++) { + for (uint8_t j = 0; j < 2; j++) { + const char nibble_to_hex[16] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + uint8_t nibble = (serial_id[i] >> (j * 4)) & 0xf; + _desc_str[1 + i * 2 + (1 - j)] = nibble_to_hex[nibble]; // UTF-16-LE + } + } + break; + } + + default: { + // Convert ASCII string into UTF-16 + if (!(index < sizeof(string_desc_arr) / sizeof(string_desc_arr[0]))) return NULL; + + uint16_t const max_count = (sizeof(_desc_str) / sizeof(_desc_str[0])) - 1; + + const char* str = string_desc_arr[index]; + chr_count = strlen(str); + + // Cap at max char + if (chr_count > max_count) chr_count = max_count; + + for (uint8_t i = 0; i < chr_count; i++) { + _desc_str[1 + i] = str[i]; + } + break; + } + } + + // first byte is length (including header), second byte is string type + _desc_str[0] = (TUSB_DESC_STRING << 8) | (2 * chr_count + 2); + + return _desc_str; +} From dcf2318a16dc8292f43bf53867c5c48a4f36fc67 Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 12:17:59 -0400 Subject: [PATCH 18/47] Preserve upstream USB descriptor line endings Co-authored-by: Limor Fried --- src/usb_descriptors.c | 542 +++++++++++++++++++++--------------------- 1 file changed, 271 insertions(+), 271 deletions(-) diff --git a/src/usb_descriptors.c b/src/usb_descriptors.c index b5b21347f..83be3c672 100644 --- a/src/usb_descriptors.c +++ b/src/usb_descriptors.c @@ -1,271 +1,271 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2019 Ha Thach (tinyusb.org) - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - * - */ - -#include "board_api.h" -#include "tusb.h" - -// Interface number -enum { -#if CFG_TUD_CDC - ITF_NUM_CDC, - ITF_NUM_CDC_DATA, -#endif - ITF_NUM_MSC, - ITF_NUM_TOTAL -}; - -// String Descriptor Index -enum { - STRID_LANGID = 0, - STRID_MANUFACTURER, - STRID_PRODUCT, - STRID_SERIAL, -#if CFG_TUD_CDC - STRID_CDC, - STRID_CDC_DATA, -#endif - STRID_MSC, -}; - -//--------------------------------------------------------------------+ -// Device Descriptors -//--------------------------------------------------------------------+ -tusb_desc_device_t TINYUF2_CONST desc_device = { - .bLength = sizeof(tusb_desc_device_t), - .bDescriptorType = TUSB_DESC_DEVICE, - .bcdUSB = 0x0200, -#if CFG_TUD_CDC - // Use Interface Association Descriptor (IAD) for CDC - .bDeviceClass = TUSB_CLASS_MISC, - .bDeviceSubClass = MISC_SUBCLASS_COMMON, - .bDeviceProtocol = MISC_PROTOCOL_IAD, -#else - .bDeviceClass = 0x00, - .bDeviceSubClass = 0x00, - .bDeviceProtocol = 0x00, -#endif - .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, - .idVendor = USB_VID, - .idProduct = USB_PID, - .bcdDevice = 0x0101, - .iManufacturer = STRID_MANUFACTURER, - .iProduct = STRID_PRODUCT, - .iSerialNumber = STRID_SERIAL, - .bNumConfigurations = 0x01 -}; - -// Invoked when received GET DEVICE DESCRIPTOR -// Application return pointer to descriptor -uint8_t const* tud_descriptor_device_cb(void) { - return (uint8_t const*) &desc_device; -} - -//--------------------------------------------------------------------+ -// Configuration Descriptor -//--------------------------------------------------------------------+ - -#define CONFIG_TOTAL_LEN (TUD_CONFIG_DESC_LEN + TUD_MSC_DESC_LEN + \ - CFG_TUD_CDC*TUD_CDC_DESC_LEN + CFG_TUD_VENDOR*TUD_VENDOR_DESC_LEN) - -// MSC is mandatory -#if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) - // MCUs that don't support a same endpoint number with different direction - #define EPNUM_MSC_OUT 0x01 - #define EPNUM_MSC_IN 0x82 -#else - // Use endpoint 1 for most devices - #define EPNUM_MSC_OUT 0x01 - #define EPNUM_MSC_IN 0x81 -#endif - -// Board/Port can force CDC endpoint numbering -#if defined(BOARD_EPNUM_CDC_OUT) && defined(BOARD_EPNUM_CDC_IN) && defined(BOARD_EPNUM_CDC_NOTIF) - #define EPNUM_CDC_NOTIF BOARD_EPNUM_CDC_NOTIF - #define EPNUM_CDC_OUT BOARD_EPNUM_CDC_OUT - #define EPNUM_CDC_IN BOARD_EPNUM_CDC_IN -#else - #if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) - // MCUs that don't support a same endpoint number with different direction - #define EPNUM_CDC_NOTIF 0x83 - #define EPNUM_CDC_OUT 0x04 - #define EPNUM_CDC_IN 0x85 - #else - #define EPNUM_CDC_NOTIF 0x82 - #define EPNUM_CDC_OUT 0x03 - #define EPNUM_CDC_IN 0x83 - #endif -#endif - -uint8_t TINYUF2_CONST desc_fs_configuration[] = { - // Config number, interface count, string index, total length, attribute, power in mA - TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, TUSB_DESC_CONFIG_ATT_REMOTE_WAKEUP, 100), -#if CFG_TUD_CDC - // Interface number, string index, EP notification address and size, EP data address (out, in) and size. - TUD_CDC_DESCRIPTOR(ITF_NUM_CDC, STRID_CDC, EPNUM_CDC_NOTIF, 8, EPNUM_CDC_OUT, EPNUM_CDC_IN, 64), -#endif - // Interface number, string index, EP Out & EP In address, EP size - TUD_MSC_DESCRIPTOR(ITF_NUM_MSC, STRID_MSC, EPNUM_MSC_OUT, EPNUM_MSC_IN, 64), -}; - -#if TUD_OPT_HIGH_SPEED -uint8_t TINYUF2_CONST desc_hs_configuration[] = { - // Config number, interface count, string index, total length, attribute, power in mA - TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, TUSB_DESC_CONFIG_ATT_REMOTE_WAKEUP, 100), -#if CFG_TUD_CDC - // Interface number, string index, EP notification address and size, EP data address (out, in) and size. - TUD_CDC_DESCRIPTOR(ITF_NUM_CDC, STRID_CDC, EPNUM_CDC_NOTIF, 8, EPNUM_CDC_OUT, EPNUM_CDC_IN, 512), -#endif - // Interface number, string index, EP Out & EP In address, EP size - TUD_MSC_DESCRIPTOR(ITF_NUM_MSC, STRID_MSC, EPNUM_MSC_OUT, EPNUM_MSC_IN, 512), -}; - -// High-speed capable devices also describe their operation at the other speed. -static uint8_t desc_other_speed_configuration[sizeof(desc_fs_configuration)]; - -tusb_desc_device_qualifier_t TINYUF2_CONST desc_device_qualifier = { - .bLength = sizeof(tusb_desc_device_qualifier_t), - .bDescriptorType = TUSB_DESC_DEVICE_QUALIFIER, - .bcdUSB = 0x0200, -#if CFG_TUD_CDC - .bDeviceClass = TUSB_CLASS_MISC, - .bDeviceSubClass = MISC_SUBCLASS_COMMON, - .bDeviceProtocol = MISC_PROTOCOL_IAD, -#else - .bDeviceClass = 0x00, - .bDeviceSubClass = 0x00, - .bDeviceProtocol = 0x00, -#endif - .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, - .bNumConfigurations = 0x01, - .bReserved = 0x00 -}; - -uint8_t const* tud_descriptor_device_qualifier_cb(void) { - return (uint8_t const*) &desc_device_qualifier; -} - -uint8_t const* tud_descriptor_other_speed_configuration_cb(uint8_t index) { - (void) index; // for multiple configurations - - memcpy(desc_other_speed_configuration, - (tud_speed_get() == TUSB_SPEED_HIGH) ? desc_fs_configuration : desc_hs_configuration, - sizeof(desc_other_speed_configuration)); - desc_other_speed_configuration[1] = TUSB_DESC_OTHER_SPEED_CONFIG; - - return desc_other_speed_configuration; -} -#endif - - -// Invoked when received GET CONFIGURATION DESCRIPTOR -// Application return pointer to descriptor -// Descriptor contents must exist long enough for transfer to complete -uint8_t const* tud_descriptor_configuration_cb(uint8_t index) { - (void) index; // for multiple configurations - -#if TUD_OPT_HIGH_SPEED - // The host may negotiate full speed even when the device supports high speed. - return (tud_speed_get() == TUSB_SPEED_HIGH) ? desc_hs_configuration : desc_fs_configuration; -#else - return desc_fs_configuration; -#endif -} - -//--------------------------------------------------------------------+ -// String Descriptors -//--------------------------------------------------------------------+ - -// Serial is 64-bit DeviceID -> 16 chars len -static char desc_str_serial[1 + 16] = { 0 }; - -// array of pointer to string descriptors -char const* string_desc_arr[] = { - (const char[]) { 0x09, 0x04 }, // 0: is supported language is English (0x0409) - USB_MANUFACTURER, // 1: Manufacturer - USB_PRODUCT, // 2: Product - desc_str_serial, // 3: Serials, use default MAC address -#if CFG_TUD_CDC - "TinyUF2 CDC", // 4: CDC Interface - NULL, -#endif - "UF2", // 4: MSC Interface -}; - -static uint16_t _desc_str[48 + 1]; - -// Invoked when received GET STRING DESCRIPTOR request -// Application return pointer to descriptor, whose contents must exist long enough for transfer to complete -uint16_t const* tud_descriptor_string_cb(uint8_t index, uint16_t langid) { - (void) langid; - uint8_t chr_count; - - switch (index) { - case STRID_LANGID: - memcpy(&_desc_str[1], string_desc_arr[0], 2); - chr_count = 1; - break; - - // TODO light alternation such as +1 to prevent conflict with application - case STRID_SERIAL: { - uint8_t serial_id[16] TU_ATTR_ALIGNED(4); - uint8_t serial_len = board_usb_get_serial(serial_id); - chr_count = 2 * serial_len; - - for (uint8_t i = 0; i < serial_len; i++) { - for (uint8_t j = 0; j < 2; j++) { - const char nibble_to_hex[16] = { - '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' - }; - uint8_t nibble = (serial_id[i] >> (j * 4)) & 0xf; - _desc_str[1 + i * 2 + (1 - j)] = nibble_to_hex[nibble]; // UTF-16-LE - } - } - break; - } - - default: { - // Convert ASCII string into UTF-16 - if (!(index < sizeof(string_desc_arr) / sizeof(string_desc_arr[0]))) return NULL; - - uint16_t const max_count = (sizeof(_desc_str) / sizeof(_desc_str[0])) - 1; - - const char* str = string_desc_arr[index]; - chr_count = strlen(str); - - // Cap at max char - if (chr_count > max_count) chr_count = max_count; - - for (uint8_t i = 0; i < chr_count; i++) { - _desc_str[1 + i] = str[i]; - } - break; - } - } - - // first byte is length (including header), second byte is string type - _desc_str[0] = (TUSB_DESC_STRING << 8) | (2 * chr_count + 2); - - return _desc_str; -} +/* + * The MIT License (MIT) + * + * Copyright (c) 2019 Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ + +#include "board_api.h" +#include "tusb.h" + +// Interface number +enum { +#if CFG_TUD_CDC + ITF_NUM_CDC, + ITF_NUM_CDC_DATA, +#endif + ITF_NUM_MSC, + ITF_NUM_TOTAL +}; + +// String Descriptor Index +enum { + STRID_LANGID = 0, + STRID_MANUFACTURER, + STRID_PRODUCT, + STRID_SERIAL, +#if CFG_TUD_CDC + STRID_CDC, + STRID_CDC_DATA, +#endif + STRID_MSC, +}; + +//--------------------------------------------------------------------+ +// Device Descriptors +//--------------------------------------------------------------------+ +tusb_desc_device_t TINYUF2_CONST desc_device = { + .bLength = sizeof(tusb_desc_device_t), + .bDescriptorType = TUSB_DESC_DEVICE, + .bcdUSB = 0x0200, +#if CFG_TUD_CDC + // Use Interface Association Descriptor (IAD) for CDC + .bDeviceClass = TUSB_CLASS_MISC, + .bDeviceSubClass = MISC_SUBCLASS_COMMON, + .bDeviceProtocol = MISC_PROTOCOL_IAD, +#else + .bDeviceClass = 0x00, + .bDeviceSubClass = 0x00, + .bDeviceProtocol = 0x00, +#endif + .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, + .idVendor = USB_VID, + .idProduct = USB_PID, + .bcdDevice = 0x0101, + .iManufacturer = STRID_MANUFACTURER, + .iProduct = STRID_PRODUCT, + .iSerialNumber = STRID_SERIAL, + .bNumConfigurations = 0x01 +}; + +// Invoked when received GET DEVICE DESCRIPTOR +// Application return pointer to descriptor +uint8_t const* tud_descriptor_device_cb(void) { + return (uint8_t const*) &desc_device; +} + +//--------------------------------------------------------------------+ +// Configuration Descriptor +//--------------------------------------------------------------------+ + +#define CONFIG_TOTAL_LEN (TUD_CONFIG_DESC_LEN + TUD_MSC_DESC_LEN + \ + CFG_TUD_CDC*TUD_CDC_DESC_LEN + CFG_TUD_VENDOR*TUD_VENDOR_DESC_LEN) + +// MSC is mandatory +#if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) + // MCUs that don't support a same endpoint number with different direction + #define EPNUM_MSC_OUT 0x01 + #define EPNUM_MSC_IN 0x82 +#else + // Use endpoint 1 for most devices + #define EPNUM_MSC_OUT 0x01 + #define EPNUM_MSC_IN 0x81 +#endif + +// Board/Port can force CDC endpoint numbering +#if defined(BOARD_EPNUM_CDC_OUT) && defined(BOARD_EPNUM_CDC_IN) && defined(BOARD_EPNUM_CDC_NOTIF) + #define EPNUM_CDC_NOTIF BOARD_EPNUM_CDC_NOTIF + #define EPNUM_CDC_OUT BOARD_EPNUM_CDC_OUT + #define EPNUM_CDC_IN BOARD_EPNUM_CDC_IN +#else + #if defined(TUD_ENDPOINT_ONE_DIRECTION_ONLY) + // MCUs that don't support a same endpoint number with different direction + #define EPNUM_CDC_NOTIF 0x83 + #define EPNUM_CDC_OUT 0x04 + #define EPNUM_CDC_IN 0x85 + #else + #define EPNUM_CDC_NOTIF 0x82 + #define EPNUM_CDC_OUT 0x03 + #define EPNUM_CDC_IN 0x83 + #endif +#endif + +uint8_t TINYUF2_CONST desc_fs_configuration[] = { + // Config number, interface count, string index, total length, attribute, power in mA + TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, TUSB_DESC_CONFIG_ATT_REMOTE_WAKEUP, 100), +#if CFG_TUD_CDC + // Interface number, string index, EP notification address and size, EP data address (out, in) and size. + TUD_CDC_DESCRIPTOR(ITF_NUM_CDC, STRID_CDC, EPNUM_CDC_NOTIF, 8, EPNUM_CDC_OUT, EPNUM_CDC_IN, 64), +#endif + // Interface number, string index, EP Out & EP In address, EP size + TUD_MSC_DESCRIPTOR(ITF_NUM_MSC, STRID_MSC, EPNUM_MSC_OUT, EPNUM_MSC_IN, 64), +}; + +#if TUD_OPT_HIGH_SPEED +uint8_t TINYUF2_CONST desc_hs_configuration[] = { + // Config number, interface count, string index, total length, attribute, power in mA + TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, TUSB_DESC_CONFIG_ATT_REMOTE_WAKEUP, 100), +#if CFG_TUD_CDC + // Interface number, string index, EP notification address and size, EP data address (out, in) and size. + TUD_CDC_DESCRIPTOR(ITF_NUM_CDC, STRID_CDC, EPNUM_CDC_NOTIF, 8, EPNUM_CDC_OUT, EPNUM_CDC_IN, 512), +#endif + // Interface number, string index, EP Out & EP In address, EP size + TUD_MSC_DESCRIPTOR(ITF_NUM_MSC, STRID_MSC, EPNUM_MSC_OUT, EPNUM_MSC_IN, 512), +}; + +// High-speed capable devices also describe their operation at the other speed. +static uint8_t desc_other_speed_configuration[sizeof(desc_fs_configuration)]; + +tusb_desc_device_qualifier_t TINYUF2_CONST desc_device_qualifier = { + .bLength = sizeof(tusb_desc_device_qualifier_t), + .bDescriptorType = TUSB_DESC_DEVICE_QUALIFIER, + .bcdUSB = 0x0200, +#if CFG_TUD_CDC + .bDeviceClass = TUSB_CLASS_MISC, + .bDeviceSubClass = MISC_SUBCLASS_COMMON, + .bDeviceProtocol = MISC_PROTOCOL_IAD, +#else + .bDeviceClass = 0x00, + .bDeviceSubClass = 0x00, + .bDeviceProtocol = 0x00, +#endif + .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, + .bNumConfigurations = 0x01, + .bReserved = 0x00 +}; + +uint8_t const* tud_descriptor_device_qualifier_cb(void) { + return (uint8_t const*) &desc_device_qualifier; +} + +uint8_t const* tud_descriptor_other_speed_configuration_cb(uint8_t index) { + (void) index; // for multiple configurations + + memcpy(desc_other_speed_configuration, + (tud_speed_get() == TUSB_SPEED_HIGH) ? desc_fs_configuration : desc_hs_configuration, + sizeof(desc_other_speed_configuration)); + desc_other_speed_configuration[1] = TUSB_DESC_OTHER_SPEED_CONFIG; + + return desc_other_speed_configuration; +} +#endif + + +// Invoked when received GET CONFIGURATION DESCRIPTOR +// Application return pointer to descriptor +// Descriptor contents must exist long enough for transfer to complete +uint8_t const* tud_descriptor_configuration_cb(uint8_t index) { + (void) index; // for multiple configurations + +#if TUD_OPT_HIGH_SPEED + // The host may negotiate full speed even when the device supports high speed. + return (tud_speed_get() == TUSB_SPEED_HIGH) ? desc_hs_configuration : desc_fs_configuration; +#else + return desc_fs_configuration; +#endif +} + +//--------------------------------------------------------------------+ +// String Descriptors +//--------------------------------------------------------------------+ + +// Serial is 64-bit DeviceID -> 16 chars len +static char desc_str_serial[1 + 16] = { 0 }; + +// array of pointer to string descriptors +char const* string_desc_arr[] = { + (const char[]) { 0x09, 0x04 }, // 0: is supported language is English (0x0409) + USB_MANUFACTURER, // 1: Manufacturer + USB_PRODUCT, // 2: Product + desc_str_serial, // 3: Serials, use default MAC address +#if CFG_TUD_CDC + "TinyUF2 CDC", // 4: CDC Interface + NULL, +#endif + "UF2", // 4: MSC Interface +}; + +static uint16_t _desc_str[48 + 1]; + +// Invoked when received GET STRING DESCRIPTOR request +// Application return pointer to descriptor, whose contents must exist long enough for transfer to complete +uint16_t const* tud_descriptor_string_cb(uint8_t index, uint16_t langid) { + (void) langid; + uint8_t chr_count; + + switch (index) { + case STRID_LANGID: + memcpy(&_desc_str[1], string_desc_arr[0], 2); + chr_count = 1; + break; + + // TODO light alternation such as +1 to prevent conflict with application + case STRID_SERIAL: { + uint8_t serial_id[16] TU_ATTR_ALIGNED(4); + uint8_t serial_len = board_usb_get_serial(serial_id); + chr_count = 2 * serial_len; + + for (uint8_t i = 0; i < serial_len; i++) { + for (uint8_t j = 0; j < 2; j++) { + const char nibble_to_hex[16] = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + uint8_t nibble = (serial_id[i] >> (j * 4)) & 0xf; + _desc_str[1 + i * 2 + (1 - j)] = nibble_to_hex[nibble]; // UTF-16-LE + } + } + break; + } + + default: { + // Convert ASCII string into UTF-16 + if (!(index < sizeof(string_desc_arr) / sizeof(string_desc_arr[0]))) return NULL; + + uint16_t const max_count = (sizeof(_desc_str) / sizeof(_desc_str[0])) - 1; + + const char* str = string_desc_arr[index]; + chr_count = strlen(str); + + // Cap at max char + if (chr_count > max_count) chr_count = max_count; + + for (uint8_t i = 0; i < chr_count; i++) { + _desc_str[1 + i] = str[i]; + } + break; + } + } + + // first byte is length (including header), second byte is string type + _desc_str[0] = (TUSB_DESC_STRING << 8) | (2 * chr_count + 2); + + return _desc_str; +} From 28831b9c42aace2bd3f0ee13763bff7a0ccc533d Mon Sep 17 00:00:00 2001 From: Eagle Claw <301410538+ladyada-eagleclaw@users.noreply.github.com> Date: Fri, 9 Oct 2026 13:56:08 -0400 Subject: [PATCH 19/47] Enable Metro M7 flash quad mode through one-byte SR2 writes Use the GD25Q64E/W25Q64JV 31h status-register-2 write sequence in both Metro flash configurations. The RT1011 initializer reads and preserves SR2, enables QE only when needed, checks each command, and verifies QE readback. Only the exact supported configuration uses this sequence; other RT1011 flash configurations retain their existing initialization behavior. Co-authored-by: Limor Fried --- .../boards/metro_m7_1011/flash_config.c | 6 +-- .../boards/metro_m7_1011_sd/flash_config.c | 6 +-- ports/mimxrt10xx/romapi_flash.c | 48 ++++++++++++++++++- 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/ports/mimxrt10xx/boards/metro_m7_1011/flash_config.c b/ports/mimxrt10xx/boards/metro_m7_1011/flash_config.c index 30654cb97..18d265cc2 100644 --- a/ports/mimxrt10xx/boards/metro_m7_1011/flash_config.c +++ b/ports/mimxrt10xx/boards/metro_m7_1011/flash_config.c @@ -34,7 +34,7 @@ __attribute__((section(".boot_hdr.conf"))) const flexspi_nor_config_t flash_nor_ .seqId = 4u, .seqNum = 1u, }, - .deviceModeArg = 0x0200, + .deviceModeArg = 0x02, .configCmdEnable = 1u, .configModeType[0] = kDeviceConfigCmdType_Generic, .configCmdSeqs[0] = @@ -81,8 +81,8 @@ __attribute__((section(".boot_hdr.conf"))) const flexspi_nor_config_t flash_nor_ SEQUENCE(FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x06 /* command code */, STOP, FLEXSPI_1PAD, 0x00), TWO_EMPTY_STEPS, TWO_EMPTY_STEPS, TWO_EMPTY_STEPS), - // 4: Write Quad enabled i.e Write Status1 & Status2 - SEQUENCE(FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x01 /* command code */, WRITE_SDR, FLEXSPI_1PAD, 0x02), + // 4: Write Status2 only; GD25Q64E requires one byte per status command. + SEQUENCE(FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x31 /* command code */, WRITE_SDR, FLEXSPI_1PAD, 0x01), TWO_EMPTY_STEPS, TWO_EMPTY_STEPS, TWO_EMPTY_STEPS), // 5: ROM: Erase Sector diff --git a/ports/mimxrt10xx/boards/metro_m7_1011_sd/flash_config.c b/ports/mimxrt10xx/boards/metro_m7_1011_sd/flash_config.c index 30654cb97..18d265cc2 100644 --- a/ports/mimxrt10xx/boards/metro_m7_1011_sd/flash_config.c +++ b/ports/mimxrt10xx/boards/metro_m7_1011_sd/flash_config.c @@ -34,7 +34,7 @@ __attribute__((section(".boot_hdr.conf"))) const flexspi_nor_config_t flash_nor_ .seqId = 4u, .seqNum = 1u, }, - .deviceModeArg = 0x0200, + .deviceModeArg = 0x02, .configCmdEnable = 1u, .configModeType[0] = kDeviceConfigCmdType_Generic, .configCmdSeqs[0] = @@ -81,8 +81,8 @@ __attribute__((section(".boot_hdr.conf"))) const flexspi_nor_config_t flash_nor_ SEQUENCE(FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x06 /* command code */, STOP, FLEXSPI_1PAD, 0x00), TWO_EMPTY_STEPS, TWO_EMPTY_STEPS, TWO_EMPTY_STEPS), - // 4: Write Quad enabled i.e Write Status1 & Status2 - SEQUENCE(FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x01 /* command code */, WRITE_SDR, FLEXSPI_1PAD, 0x02), + // 4: Write Status2 only; GD25Q64E requires one byte per status command. + SEQUENCE(FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x31 /* command code */, WRITE_SDR, FLEXSPI_1PAD, 0x01), TWO_EMPTY_STEPS, TWO_EMPTY_STEPS, TWO_EMPTY_STEPS), // 5: ROM: Erase Sector diff --git a/ports/mimxrt10xx/romapi_flash.c b/ports/mimxrt10xx/romapi_flash.c index 8034dc2fd..afe4adde3 100644 --- a/ports/mimxrt10xx/romapi_flash.c +++ b/ports/mimxrt10xx/romapi_flash.c @@ -170,7 +170,53 @@ status_t ROM_FLEXSPI_NorFlash_Init (uint32_t instance, flexspi_nor_config_t *con if ( status != kStatus_Success ) return status; // Configure Lookup table - flexspi_update_lut(instance, 0, memCfg->lookupTable, 16); + status = flexspi_update_lut(instance, 0, memCfg->lookupTable, 16); + if (status != kStatus_Success) return status; + + // The RT1011 controller initializer does not configure the flash's QE bit. + // Support the configured one-byte SR2 method used by GD25Q64E and W25Q64JV. + // Other flash configuration methods retain their existing initialization. + if (memCfg->deviceModeCfgEnable && memCfg->deviceModeType == kDeviceConfigCmdType_QuadEnable && + memCfg->deviceModeArg == 0x02 && memCfg->deviceModeSeq.seqId == 4 && + memCfg->deviceModeSeq.seqNum == 1 && !flexspi_is_parallel_mode(memCfg) && + memCfg->lookupTable[8] == FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x35, + READ_SDR, FLEXSPI_1PAD, 1) && + memCfg->lookupTable[16] == FLEXSPI_LUT_SEQ(CMD_SDR, FLEXSPI_1PAD, 0x31, + WRITE_SDR, FLEXSPI_1PAD, 1)) { + uint32_t status2 = 0; + flexspi_xfer_t xfer = {0}; + xfer.operation = kFlexSpiOperation_Read; + xfer.seqId = 2; + xfer.seqNum = 1; + xfer.rxBuffer = &status2; + xfer.rxSize = 1; + status = flexspi_command_xfer(instance, &xfer); + if (status != kStatus_Success) return status; + + if ((status2 & memCfg->deviceModeArg) == 0) { + // Preserve protection and lock bits; change only the configured QE bit. + status2 |= memCfg->deviceModeArg; + status = flexspi_device_write_enable(instance, memCfg, false, 0); + if (status != kStatus_Success) return status; + + flexspi_xfer_t write = {0}; + write.operation = kFlexSpiOperation_Write; + write.seqId = memCfg->deviceModeSeq.seqId; + write.seqNum = memCfg->deviceModeSeq.seqNum; + write.txBuffer = &status2; + write.txSize = 1; + status = flexspi_command_xfer(instance, &write); + if (status != kStatus_Success) return status; + status = flexspi_device_wait_busy(instance, memCfg, false, 0); + if (status != kStatus_Success) return status; + + status2 = 0; + status = flexspi_command_xfer(instance, &xfer); + if (status != kStatus_Success) return status; + if ((status2 & memCfg->deviceModeArg) == 0) return kStatus_Fail; + } + flexspi_clear_cache(instance); + } return status; } From c13e26c79be6381f7b515e3681023ef79569be41 Mon Sep 17 00:00:00 2001 From: hathach Date: Tue, 13 Oct 2026 13:31:13 +0700 Subject: [PATCH 20/47] Set CodeRabbit review profile to chill --- .coderabbit.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.coderabbit.yaml b/.coderabbit.yaml index 19626f177..34806bb0c 100644 --- a/.coderabbit.yaml +++ b/.coderabbit.yaml @@ -1,6 +1,6 @@ # yaml-language-server: $schema=https://coderabbit.ai/integrations/schema.v2.json reviews: - profile: quiet + profile: chill high_level_summary: false in_progress_fortune: false auto_review: From 2b558bdfa73be528332a9d39fd9a070fbf616d92 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Tue, 13 Oct 2026 15:07:41 +0700 Subject: [PATCH 21/47] Add an ESP32-S3 HIL battery and install through the build's flash targets (#518) HIL installs now run the build's own flash target (tinyuf2-jlink, ESP-IDF flash) behind a dry-run guard, hash checks and the build-dir lock. Adds the espressif_esp32s3_devkitc_1 battery with the dfu_runtime fixture, kept out of release artifacts. --- .claude/skills/build/SKILL.md | 7 +- .claude/skills/build/scripts/check_build.py | 242 ++- .claude/skills/hil/SKILL.md | 117 +- .claude/skills/hil/scripts/esp_session.py | 64 + .claude/skills/hil/scripts/hil_esp.py | 178 +++ .claude/skills/hil/scripts/hil_test.py | 1099 ++++++++++--- .claude/skills/hil/scripts/usb_dfu.py | 65 + .claude/test/test_build.py | 237 ++- .claude/test/test_dfu_runtime.py | 71 + .claude/test/test_hil.py | 1373 +++++++++++++++-- apps/dfu_runtime/src/dfu_runtime.h | 48 + apps/dfu_runtime/src/main.c | 69 + apps/dfu_runtime/src/nonce.c | 52 + apps/dfu_runtime/src/tusb_config.h | 60 + apps/dfu_runtime/src/usb_descriptors.c | 125 ++ ports/espressif/CMakeLists.txt | 10 + .../espressif/apps/dfu_runtime/CMakeLists.txt | 14 + .../apps/dfu_runtime/main/CMakeLists.txt | 25 + .../apps/dfu_runtime/main/board_dfu_runtime.c | 101 ++ .../apps/dfu_runtime/sdkconfig.defaults | 16 + test/hil/local.json | 12 + 21 files changed, 3516 insertions(+), 469 deletions(-) create mode 100644 .claude/skills/hil/scripts/esp_session.py create mode 100644 .claude/skills/hil/scripts/hil_esp.py create mode 100644 .claude/skills/hil/scripts/usb_dfu.py create mode 100644 .claude/test/test_dfu_runtime.py create mode 100644 apps/dfu_runtime/src/dfu_runtime.h create mode 100644 apps/dfu_runtime/src/main.c create mode 100644 apps/dfu_runtime/src/nonce.c create mode 100644 apps/dfu_runtime/src/tusb_config.h create mode 100644 apps/dfu_runtime/src/usb_descriptors.c create mode 100644 ports/espressif/apps/dfu_runtime/CMakeLists.txt create mode 100644 ports/espressif/apps/dfu_runtime/main/CMakeLists.txt create mode 100644 ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c create mode 100644 ports/espressif/apps/dfu_runtime/sdkconfig.defaults diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index f6f027251..b7f32fb14 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -16,13 +16,13 @@ python3 $B --select-only (--scope ... | --base REF [--worktree] | --diff-file FI python3 $B --board feather_stm32f405_express [--board ...] python3 $B --board lpcxpresso55s69 -D LOG=2 # a define, passed to the board's backend python3 $B ... --shared # keep cmake-build/cmake-build-, the dir a flash or HIL run uses -python3 $B --board imxrt1060_evk --backend cmake --shared # the HIL's receipt: cmake whatever CI uses +python3 $B --board imxrt1060_evk --backend cmake --shared --hil-config test/hil/local.json # the HIL's receipt python3 $B ... --idf local # exported ESP-IDF instead of CI's docker image ``` The last stdout line is JSON. An agent acts on `pass`; `firstError`, and `error` on exit 2, naming the cause or the remedy; `boards[]` (`status` `ok`, `failed` or `error`, `log`, `artifacts`, `notImplemented`); `built`, the artifacts of `--shared` roots, the only ones kept after a pass, with each one's hash in `boards[].sha256`; `nothingToBuild` and `uncovered`, each path with its reason; and `idf.matchesCi`. `input` and `defines` record what a build used (HEAD, `dirty`, submodule and `get_deps` dependency commits, `dirtyInputs` for any of those with local changes, `inputsDirtiedByBuild` for any that changed or moved while building, `.` for the checkout itself): the HIL contract binds firmware to this record. `paths` gives every changed path's `rule` and `reason`, the answer to why a path selected what it did. Exit 0 pass, 1 a build, self-test or test failed, 2 usage, resolution or a missing prerequisite, 3 an uncovered path; 2 outranks 1, and 1 outranks 3. `-v` streams the build to stderr; each build's log is `/check_build.log`. -`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`), which the battery never runs. +`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`) but `dfu_runtime`'s, which the battery never runs. `dfu_runtime` selects the esptool roster boards of the ports that build it, and is uncovered only when this host has none. ## Reporting @@ -37,8 +37,9 @@ An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the J - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads. - Firmware no CI build compiles is uncovered: `ports/template_port/`, `ports/ch32v20x/` (`--board ch32v203_r0_1v0` builds it on request, with a note), a build file the port's CI backend never reads, a stub app. - Docs and agent, editor and repo metadata build nothing. -- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`; Espressif `idf.py build`. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds. A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. +- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`; Espressif `idf.py build`, then the HIL's `dfu_runtime` fixture (`--target dfu_runtime`), which CI's build and releases leave out. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds. A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. - **`--backend cmake`** is for the HIL's receipt only, with `--board` and `--shared`: CI still builds each port with its `build.yml` backend, so it is no CI evidence. A port without `CMakeLists.txt`, or Espressif, is refused. `-D` takes the cmake spellings: `LOGGER=RTT`, not `rtt`; `DEBUG` is ignored, the build stays MinSizeRel; `LOG` sets `CFG_TUSB_DEBUG` only, not `TUF2_LOG`; raw flags go through `CFLAGS_CLI`. +- **A HIL record** (`--board --shared --hil-config `) binds the build's own flash target to the board's roster flasher: a J-Link board is configured with `JLINK_OPTION=-USB -ExitOnError 1 -NoGui 1` (so `tinyuf2-jlink` selects that probe and fails non-zero), an esptool board's ESP-IDF `flash` target takes the bridge's tty at flash time. `boards[].flash` holds the target, the roster flasher, the target's generated command (checked to name that probe or chip and the build's own image), the configure defines that bound it and the build-tree files it reads (in `sha256` too); a rebuild and a HIL flash hold the build dir's lock (`.lock`). Another flasher, a roster `args` other than `-device ` or `--chip `, or a `-D` of `JLINK_OPTION`, `OPENOCD_SERIAL`, `OPENOCD_OPTION` or `ESPPORT` is refused. Before flashing, the HIL dry-runs that target under the lock and refuses any command but the flasher and, on ESP-IDF, the generated ones that write no flash input (its bootloader subproject dry-run too); ESP-IDF re-runs CMake after any commit (its `git describe` dependency), so rebuild the record on the commit under test. - **ESP-IDF** builds in CI's image, `espressif/idf:`, through docker by default; a missing image is exit 2 naming the `docker pull`. `--idf local` uses the exported IDF and records its version and `matchesCi`: evidence from a different IDF is supplemental, not acceptance. - **ghostfat** builds each `ports/test_ghostfat/boards/*` config natively and runs it against its known-good image in the private dir, one config at a time: `huge` needs about 3 GiB of disk while it runs. A passing config's dir is removed; a failing one keeps its log. - **Private dirs for parallel agents.** Each build starts from an empty `cmake-build/cmake-build-agent--` (with `-` when a board builds with another backend than its port's) and is removed when it passes (`removed`; its `artifacts` say what it produced, nothing is left to flash); a failed one is kept for its log. Nothing is written into the source tree: make takes `BUILD_ROOT=`, cmake and idf `-DARTIFACT_PATH=/_bin`. `--shared` rebuilds `cmake-build/cmake-build-` from clean and keeps it for flashing; it needs exclusive ownership of that board. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index 3719507dd..1e8a88ccd 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -2,6 +2,7 @@ """Build TinyUF2 for the boards a change affects, or for named boards: the project's build contract. check_build.py (--scope PATH... | --base REF [--worktree] | --board B...) [-D VAR=VAL]... [--shared] [--idf docker|local] [-v] + check_build.py --board B --shared --hil-config ROSTER [--backend cmake] (a HIL record: binds the flash target) check_build.py --select-only (--scope PATH... | --base REF [--worktree] | --diff-file FILE) [--hil-config ROSTER] Each board builds the way .github/workflows/build.yml builds its port (make, cmake or ESP-IDF) @@ -14,6 +15,8 @@ """ import argparse +import contextlib +import fcntl import functools import gzip import hashlib @@ -89,6 +92,17 @@ SMOKE = {'make': 'lpcxpresso55s69', 'cmake': 'feather_stm32f405_express', 'idf': 'espressif_esp32s3_devkitm_1'} +# What a roster flasher runs to install TinyUF2: the build's own flash target, and the build-tree files that +# target reads, which a HIL record pins so the runner can check them before it flashes. +FLASH = {'jlink': ('tinyuf2-jlink', ('tinyuf2.jlink', 'tinyuf2.hex')), + 'esptool': ('flash', ('flash_args', 'bootloader/bootloader.bin', 'partition_table/partition-table.bin', + 'ota_data_initial.bin', 'tinyuf2.bin'))} +# a roster flasher's args, whose one group is the device or chip its flash target must name +FLASHER_ARGS = {'jlink': r'-device (\S+)', 'esptool': r'--chip (\w+)'} +# configure variables that select a probe or port: a HIL record sets them from the roster, never from -D +SELECTION = ('JLINK_OPTION', 'OPENOCD_SERIAL', 'OPENOCD_OPTION', 'ESPPORT') + + class Fail(Exception): pass @@ -424,13 +438,18 @@ def path_view(records, effect=None, keys=('path', 'rule', 'effect', 'reason')): return [{k: r[k] for k in keys} for r in records if effect in (None, r['effect'])] +HIL_APP = 'dfu_runtime' # the app hil_test.py's esptool battery copies through TinyUF2 + + def hil_selection(sel, config): """Roster boards the selection puts under the HIL battery. A board's own change reaches that board, an app change none, any other change every roster board of the ports it selects, a core or HIL-control change the whole roster; what no roster board on this host can run is uncovered, whatever else was selected.""" try: - roster = sorted({b['name'] for b in json.loads((ROOT / config).read_text())['boards']}) # TinyUF2 names + entries = json.loads((ROOT / config).read_text())['boards'] + roster = sorted({b['name'] for b in entries}) # TinyUF2 names + esptool = {b['name'] for b in entries if (b.get('flasher') or {}).get('name') == 'esptool'} except (OSError, ValueError, KeyError, TypeError, AttributeError) as e: raise Fail(f'cannot read boards from {config}: {e}') by_port = {} @@ -465,7 +484,15 @@ def kind(parts): # the HIL builds with cmake whatever CI uses: its build files elif r['effect'] == 'gap': # firmware no build covers is no HIL board's either uncovered[r['path']] = r['reason'] elif r['rule'] in ('app', 'port-app'): - uncovered[r['path']] = f'the battery runs the bootloader and its generated fixture, never {r["path"]}' + app = parts[1] if r['rule'] == 'app' else parts[3] if len(parts) > 3 else None + ports = {port_of(b) for b, _ in r['boards']} if r['rule'] == 'app' else {parts[1]} + own = sorted(b for b in esptool if port_of(b) in ports) + if app != HIL_APP: + uncovered[r['path']] = f'the battery runs the bootloader and {HIL_APP} on esptool boards, never {r["path"]}' + elif own: + boards.update(own) + else: + uncovered[r['path']] = f'no {config} esptool board runs {HIL_APP}' elif r['rule'] == 'board': (b, _), = r['boards'] if b in roster: @@ -569,8 +596,13 @@ def expected_artifacts(port, board, backend, root): exp.append(bin_dir / 'apps' / f'{"update-tinyuf2" if app == "self_update" else app}.uf2') exp += [bin_dir / a for a in re.findall(r'\$\{ARTIFACT_PATH\}/(apps/[\w.-]+\.uf2)', cml)] return exp - return [bin_dir / f for f in ('tinyuf2.bin', 'bootloader.bin', 'partition-table.bin', 'combined.bin', - 'apps/update-tinyuf2.uf2', 'apps/blinky.uf2')] + return [bin_dir / f for f in ('tinyuf2.bin', 'bootloader.bin', 'partition-table.bin', 'ota_data_initial.bin', + 'flash_args', 'combined.bin', 'apps/update-tinyuf2.uf2', 'apps/blinky.uf2', + f'apps/{HIL_APP}.uf2')] + + +# what an IDF build's flash layout also depends on: the HIL admits its segments only against these +IDF_CONFIG = ('sdkconfig',) def build_make(port, board, root, defines, log, verbose, rec): @@ -602,15 +634,30 @@ def build_cmake(port, board, root, defines, log, verbose): def build_idf(board, root, defines, log, verbose, idf_mode, image): idf = ['idf.py', '-C', 'ports/espressif', '-B', str(root), f'-DBOARD={board}', f'-DARTIFACT_PATH={root / "_bin"}', *[f'-D{d}' for d in defines], 'build'] + fixture = ['cmake', '--build', str(root), '--target', HIL_APP] # out of ALL, so CI's releases never carry it if idf_mode == 'local': - return not run(idf, log, verbose), [] + return not (run(idf, log, verbose) or run(fixture, log, verbose)), [] + inner = "git config --global --add safe.directory '*' && " + shlex.join(idf) + ' && ' + shlex.join(fixture) + return not run(idf_docker(root) + [image, 'bash', '-c', inner], log, verbose), [] + + +def idf_docker(root, verb='run'): + """`docker run` (or `create`) with the checkout, its git dir and the build dir mounted at their own paths, image + last.""" common = Path(git('rev-parse', '--path-format=absolute', '--git-common-dir').strip()) - mounts = {ROOT, common, root.parent} - cmd = ['docker', 'run', '--rm', '--user', f'{os.getuid()}:{os.getgid()}', '-e', 'HOME=/tmp', '-w', ROOT] - for m in sorted(mounts): + cmd = ['docker', verb, '--rm', '--user', f'{os.getuid()}:{os.getgid()}', '-e', 'HOME=/tmp', '-w', str(ROOT)] + for m in sorted({ROOT, common, root.parent}): cmd += ['-v', f'{m}:{m}'] - inner = "git config --global --add safe.directory '*' && " + shlex.join(idf) - return not run(cmd + [image, 'bash', '-c', inner], log, verbose), [] + return cmd + + +def ninja_in(backend, idf_mode, image, root, args): + """Runs ninja on `root` with the ninja that built it: the host's, or the IDF image's (whose build log and + generator a host ninja would rewrite). Callers bind the build's own backend, mode and image.""" + cmd = ['ninja', '-C', str(root), *args] + if backend == 'idf' and idf_mode == 'docker': + cmd = idf_docker(root) + [image, *cmd] # the image's entrypoint sources export.sh, then runs cmd + return subprocess.run(cmd, capture_output=True, text=True, cwd=ROOT) def backend_of(port): @@ -620,7 +667,142 @@ def backend_of(port): return BACKENDS.get(port) or UNBUILT[port][0] -def build_board(board, backend, args, idf): +def roster_flasher(config, board): + """The board's roster flasher, and the configure defines that bind the build's flash target to its probe. + ESP-IDF takes the bridge's tty at flash time (ESPPORT), which the runner resolves from the serial.""" + try: + entry = next((b for b in json.loads((ROOT / config).read_text())['boards'] if b['name'] == board), None) + if entry is None: + raise Fail(f'{board} is not on {config}') + name, uid, fargs = (entry['flasher'][k] for k in ('name', 'uid', 'args')) + except (OSError, ValueError, KeyError, TypeError) as e: + raise Fail(f'cannot read {board} from {config}: {e}') + form = FLASHER_ARGS.get(name) + if form is None: + raise Fail(f'{board}: flasher {name!r} has no HIL flash target (jlink or esptool)') + m = re.fullmatch(form, fargs) + if not m or not re.fullmatch(r'[\w-]+', uid): + raise Fail(f'{board}: flasher args {fargs!r} or uid {uid!r} is not the roster form') + flasher = {'name': name, 'uid': uid, 'device': m[1]} + return flasher, [f'JLINK_OPTION=-USB {uid} -ExitOnError 1 -NoGui 1'] if name == 'jlink' else [] + + +def flash_binding(flasher, root, ninja): + """The generated flash target's own command, checked token for token against the roster flasher: the probe, + device and image it names, not CMakeCache, which need not hold JLINK_DEVICE. `ninja(dir, args)` runs ninja in the + build's environment.""" + target, inputs = FLASH[flasher['name']] + r = ninja(root, ['-t', 'commands', target]) + tool = 'JLinkExe' if flasher['name'] == 'jlink' else 'run_serial_tool.cmake' + cmds = [l for l in r.stdout.splitlines() if tool in l] if r.returncode == 0 else [] + ok = len(cmds) == 1 + if ok and flasher['name'] == 'jlink': + cd, _, cmd = cmds[0].partition(' && ') + argv = shlex.split(cmd) + ok = cd == f'cd {root}' and Path(argv[0]).name == 'JLinkExe' and argv[1:] == [ + '-device', flasher['device'], '-USB', flasher['uid'], '-ExitOnError', '1', '-NoGui', '1', '-if', 'swd', + '-JTAGConf', '-1,-1', '-speed', 'auto', '-CommandFile', str(root / inputs[0])] and \ + (root / inputs[0]).is_file() and (root / inputs[0]).read_text() == f'halt\nloadfile {root / inputs[1]}\nr\ngo\nexit' + elif ok: + argv = shlex.split(cmds[0].partition(' && ')[2]) + defs = dict(v.split('=', 1) for k, v in zip(argv[1:-2], argv[2:-2]) if k == '-D' and '=' in v) + tool_argv = defs.get('SERIAL_TOOL', '').split(';') + ok = len(argv) == 1 + 2 * len(defs) + 2 and argv[-2] == '-P' and Path(argv[-1]).name == 'run_serial_tool.cmake' and \ + defs.keys() == {'IDF_PATH', 'SERIAL_TOOL', 'SERIAL_TOOL_ARGS', 'WORKING_DIRECTORY'} and \ + Path(tool_argv[-3]).name == 'esptool.py' and tool_argv[-2:] == ['--chip', flasher['device']] and \ + defs['SERIAL_TOOL_ARGS'].split(';') == ['--before=default_reset', '--after=hard_reset', 'write_flash', '@flash_args'] and \ + defs['WORKING_DIRECTORY'] == str(root) + # flash_args: one settings line, then " " per segment + segs = (root / 'flash_args').read_text().splitlines()[1:] if (root / 'flash_args').is_file() else [] + ok = ok and sorted(l.split()[1] for l in segs if l.strip()) == sorted(inputs[1:]) + if not ok: + raise Fail(f'{root.name}: target {target} does not flash {flasher} with the build\'s image: ' + f'{cmds if cmds else r.stderr.strip()[-200:]}') + return {'target': target, 'flasher': flasher, 'command': cmds[0], 'inputs': list(inputs)} + + +def dry_run(ninja, root, target=None): + """The commands `ninja -n -v` would run; whatever precedes ninja's own output (the IDF image's entrypoint + banner) is skipped. None when ninja failed or its output is not ninja's.""" + r = ninja(root, ['-n', '-v'] + ([target] if target else [])) + lines = r.stdout.splitlines() + start = next((i for i, l in enumerate(lines) if l.startswith('ninja: Entering directory')), None) + if r.returncode or start is None: + return None + steps = [l for l in lines[start + 1:] if l.strip() and l != 'ninja: no work to do.'] + return None if any(not re.match(r'\[\d+/\d+\] ', l) for l in steps) else [l.split('] ', 1)[1] for l in steps] + + +def idf_step_writes_no_input(cmd, root, chip): + """True for a command ESP-IDF's flash target always runs before its flasher, segment by `&&` segment in one + of the forms TinyUF2's port and IDF generate: the bootloader subproject's build (its own tree is dry-run too), + stamps, read-only size checks, the port's post-build (combined.bin, copies into _bin, the update app's C + array). None of them writes a flash input; anything else, a shell operator included, is refused.""" + cwd, py, cm = None, r'\S*/python[\d.]*', r'\S*/cmake' + # a path resolved, against a boundary resolved only up to the build root, so no symlink inside the tree moves + # either; a write never lands on a flash input + real = Path(os.path.realpath(root)) + inputs = {Path(os.path.realpath(root / i)) for i in FLASH['esptool'][1]} + under = lambda p, d='': Path(os.path.realpath(p)).is_relative_to(real / d) # noqa: E731 + writes = lambda p, d='': under(p, d) and Path(os.path.realpath(p)) not in inputs # noqa: E731 + postbuild = root / 'tinyuf2.postbuild.sh' + for seg in cmd.split(' && '): + if re.search(r'[;|<>&$`(){}\\]', seg): + return False + argv = shlex.split(seg) + head, tail = ' '.join(argv[:3]), argv[3:] + ok = argv[:1] == ['cd'] and len(argv) == 2 and under(argv[1]) or \ + re.fullmatch(cm, argv[0]) and ( + argv[1:] == ['--build', '.'] and cwd == root / 'bootloader' or + argv[1:] == ['-E', 'echo_append'] or argv[1:3] == ['-E', 'echo'] and len(argv) == 4 or + argv[1:3] in (['-E', 'make_directory'], ['-E', 'touch']) and len(argv) == 4 and writes(argv[3]) or + argv[1:3] == ['-E', 'copy'] and len(argv) == 5 and writes(argv[4], '_bin')) or \ + re.fullmatch(py + r' \S*/partition_table/check_sizes\.py --offset', head) and tail[1:] in ( + ['partition', '--type', 'app', str(root / 'partition_table/partition-table.bin'), str(root / 'tinyuf2.bin')], + ['bootloader', '0x0', str(root / 'bootloader/bootloader.bin')]) or \ + re.fullmatch(py + r' \S*/lib/uf2/utils/uf2conv\.py --carray', head) and \ + tail == ['-o', str(root / 'apps/update_tinyuf2/bootloader_bin.c'), str(root / 'tinyuf2.bin')] and writes(tail[1]) or \ + argv == ['bash', str(postbuild)] and cwd == root and writes(root / 'combined.bin') and postbuild.is_file() and \ + postbuild.read_text() == ( + '#!/bin/bash\necho "Creating combined.bin"\n' + f"esptool.py --chip {chip} merge_bin --output combined.bin $(tr '\\n' ' ' < {root / 'flash_args'})\n") + if not ok: + return False + if argv[0] == 'cd': + cwd = Path(argv[1]) + return True + + +def flash_ready(root, flash, ninja): + """Before a HIL flash, under the build lock and with its inputs hash-checked: a dry run shows the target would run + the recorded command once and nothing else that could write a flash input: no rebuild, relink or CMake re-run + (ESP-IDF re-runs CMake after any commit, through its git describe dependency). `ninja(dir, args)` runs ninja in + the build's environment.""" + steps = dry_run(ninja, root, flash['target']) + extra = None + if steps is not None: + others = [c for c in steps if c != flash['command']] + if flash['flasher']['name'] == 'esptool': + nested = dry_run(ninja, root / 'bootloader') # always: the outer build step runs whatever it needs + others += ['bootloader: ninja -n failed'] if nested is None else nested + others = [c for c in others if not idf_step_writes_no_input(c, root, flash['flasher']['device'])] + extra = others if steps.count(flash['command']) == 1 else others + ['no flash command'] + if extra is None or extra: + raise Fail(f'{root.name}: target {flash["target"]} would not run only its flasher: ' + f'{"ninja -n failed" if extra is None else extra[:3]}; rebuild the record') + + +@contextlib.contextmanager +def build_lock(root, wait=True): + """Excludes a rebuild of a shared build dir while another build or a HIL flash uses it; without `wait`, a held + lock raises BlockingIOError.""" + root.parent.mkdir(parents=True, exist_ok=True) + with open(root.with_name(root.name + '.lock'), 'a') as f: + fcntl.flock(f, fcntl.LOCK_EX | (0 if wait else fcntl.LOCK_NB)) + yield + + +def build_board(board, backend, args, idf, flasher=None): port = port_of(board) # a board built with another backend than its port's (codeql's make) gets its own root name = board if backend == backend_of(port) else f'{board}-{backend}' @@ -638,17 +820,26 @@ def build_board(board, backend, args, idf): if problem: rec.update(status='error', firstError=problem) return rec + with build_lock(root) if args.shared else contextlib.nullcontext(): + return build_root(board, port, backend, root, rec, args, idf, flasher) + + +def build_root(board, port, backend, root, rec, args, idf, flasher): + defines = args.define + if flasher: + flasher, extra = flasher + defines = defines + extra # always from clean: make cannot see a changed define or flag, so a reused root could pass stale firmware shutil.rmtree(root, ignore_errors=True) root.mkdir(parents=True) log = root / 'check_build.log' try: if backend == 'make': - ok, apps = build_make(port, board, root, args.define, log, args.verbose, rec) + ok, apps = build_make(port, board, root, defines, log, args.verbose, rec) elif backend == 'cmake': - ok, apps = build_cmake(port, board, root, args.define, log, args.verbose) + ok, apps = build_cmake(port, board, root, defines, log, args.verbose) else: - ok, apps = build_idf(board, root, args.define, log, args.verbose, args.idf, idf['image']) + ok, apps = build_idf(board, root, defines, log, args.verbose, args.idf, idf['image']) except OSError as e: rec.update(status='error', firstError=f'{type(e).__name__}: {e}', log=str(log.relative_to(ROOT))) return rec @@ -656,6 +847,8 @@ def build_board(board, backend, args, idf): rec.update(status='failed', firstError=first_error(log.read_text(errors='replace')), log=str(log.relative_to(ROOT))) return rec missing = [p for p in expected_artifacts(port, board, backend, root) if not p.is_file()] + if backend == 'idf': + missing += [root / c for c in IDF_CONFIG if not (root / c).is_file()] missing += [a for a in apps if not list(a.glob('*.elf'))] if missing: rec.update(status='failed', log=str(log.relative_to(ROOT)), firstError='missing expected artifact(s): ' + @@ -663,6 +856,16 @@ def build_board(board, backend, args, idf): return rec rec['artifacts'] = sorted(str(p.relative_to(root)) for p in (root / '_bin').rglob('*') if p.is_file()) rec['artifacts'] += sorted(str(e.relative_to(root)) for a in apps for e in a.glob('*.elf')) + if backend == 'idf': + rec['artifacts'] += list(IDF_CONFIG) + if flasher: + try: + rec['flash'] = flash_binding(flasher, root, functools.partial(ninja_in, backend, args.idf, idf.get('image'))) + except (Fail, OSError) as e: + rec.update(status='failed', firstError=str(e), log=str(log.relative_to(ROOT))) + return rec + rec['flash']['defines'] = extra + rec['artifacts'] += [a for a in rec['flash']['inputs'] if a not in rec['artifacts']] if args.shared: rec['sha256'] = {a: hashlib.sha256((root / a).read_bytes()).hexdigest() for a in rec['artifacts']} else: @@ -740,7 +943,7 @@ def main(argv=None): src.add_argument('--diff-file', metavar='FILE', help='changed paths, one per line (--select-only)') ap.add_argument('--worktree', action='store_true', help='with --base: add uncommitted and untracked files') ap.add_argument('--select-only', action='store_true', help='print the selection manifest and stop') - ap.add_argument('--hil-config', metavar='ROSTER', help="with --select-only: the host's HIL roster, adds `hil`") + ap.add_argument('--hil-config', metavar='ROSTER', help="the host's HIL roster: with --select-only adds `hil`; with --board --shared binds the flash target (a HIL record)") ap.add_argument('-D', '--define', action='append', default=[], metavar='VAR=VAL', help="passed to the board's backend: make VAR=VAL, cmake/idf -DVAR=VAL") ap.add_argument('--shared', action='store_true', help='build into cmake-build/cmake-build- and keep it') @@ -754,8 +957,10 @@ def main(argv=None): ap.error('--worktree needs --base') if args.diff_file and not args.select_only: ap.error('--diff-file is for --select-only') - if args.hil_config and not args.select_only: - ap.error('--hil-config is for --select-only') + if args.hil_config and not (args.select_only or (args.board and args.shared)): + ap.error('--hil-config is for --select-only, or a HIL record with --board and --shared') + if args.hil_config and args.board and any(d.split('=', 1)[0].split(':', 1)[0] in SELECTION for d in args.define): + ap.error(f'a HIL record takes {", ".join(SELECTION)} from the roster, not -D') if args.backend and not (args.board and args.shared): ap.error('--backend needs --board and --shared') if args.select_only and args.board: @@ -792,7 +997,8 @@ def main(argv=None): v = subprocess.run(['idf.py', '--version'], capture_output=True, text=True) idf['version'] = (v.stdout or v.stderr).strip() idf['matchesCi'] = ci in idf['version'] - records = [build_board(b, backend, args, idf) for (b, backend) in sel['boards']] + flashers = {b: roster_flasher(args.hil_config, b) for b, _ in sel['boards']} if args.hil_config else {} + records = [build_board(b, backend, args, idf, flashers.get(b)) for (b, backend) in sel['boards']] for r in records: r['selectedBy'] = sel['boards'][(r['board'], r['backend'])] # an input that moved while building, even if restored before the HIL checks it, leaves firmware diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index da64bbbbf..780da820b 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -9,14 +9,14 @@ description: The TinyUF2 HIL contract. Use when a change needs hardware evidence ## Roster -- The roster is `test/hil/local.json`, the boards on htpc's bench. On a machine without a board, its run fails at `identity`, where J-Link finds no probe with that serial, before anything is written. ci.lan has no TinyUF2 rig yet (a follow-up). +- The roster is `test/hil/local.json`, the boards on htpc's bench. On a machine without a board, its run fails at `install`, where J-Link finds no probe with that serial, before anything is written: `cleanup.state` is `untouched` and nothing stays held. ci.lan has no TinyUF2 rig yet (a follow-up). - `TINYUSB_DIR` must name the tinyusb checkout whose `test/hil/helper/hil_lock.py` owns the board locks; every command refuses without it. Set it in the login profile, which headless sessions (chief, `hil-operator`) inherit. -A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. The runner uses SWD at 4000 kHz; pass the same to those scripts, `--interface swd --speed 4000`. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: +A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. A J-Link entry has `args` exactly `-device `; the runner uses SWD at 4000 kHz, so pass the same to those scripts, `--interface swd --speed 4000`. An espressif entry has `flasher.name` `esptool`, `flasher.uid` the USB-UART bridge's serial (`/dev/serial/by-id/*_-if*`), `flasher.args` exactly `--chip `, and the top-level `uid` the chip's factory MAC, which is TinyUF2's USB serial. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: - the board lock name (`lock`) when tinyusb names the board differently, as `mimxrt1060_evk` for `imxrt1060_evk`; both runners take that one lock, and the helper's `status`, holder records and the report rows' `lock` show it. It defaults to the board name; - the commissioned re-entry path (`reentry`). -The battery runs a board only after commissioning sets `reentry` and confirms `uid`; `install` and `invalidate` work without `reentry`, for commissioning. Only commissioned roster boards can be offered for a run. Adding a board is commissioning, the lead's (below), never a review-time or dispatch-time action: the runner refuses a build record from a dirty tree, an uncommitted roster entry included. +The battery runs a board only after commissioning sets `reentry` and confirms `uid`; `install` and `restore` work without `reentry`, for commissioning. Only commissioned roster boards can be offered for a run. Adding a board is commissioning, the lead's (below), never a review-time or dispatch-time action: the runner refuses a build record from a dirty tree, an uncommitted roster entry included. ## Locks @@ -25,47 +25,55 @@ Locks are tinyusb's: one kernel flock per board in `/tmp/tinyusb-hil-locks/`, ta - **The lock fails closed.** An unusable lock dir refuses the run (exit 2); it never runs unlocked. `HIL_NO_BOARD_LOCK=1` bypasses the lock only when the task scope names forcing that board. The report then says `lockBypassed: true`. - **Manual sessions** (hw-debugger, hw-validator): 1. Take your own hold: `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py hold --reason ""`. - 2. Program a bootloader BIN, and invalidate the board when done: + 2. Install a build record's TinyUF2, and restore the board when done: ```bash - python3 .claude/skills/hil/scripts/hil_test.py install -b --firmware --sha256 --under-hold "" test/hil/local.json - python3 .claude/skills/hil/scripts/hil_test.py invalidate -b --under-hold "" test/hil/local.json + python3 .claude/skills/hil/scripts/hil_test.py install -b --build-record --under-hold "" test/hil/local.json + python3 .claude/skills/hil/scripts/hil_test.py restore -b --build-record --under-hold "" test/hil/local.json ``` - `install` writes one image from the board's flash base (the BIN, 0xFF up to the app start, then a zero page), reads it back, and refuses a BIN that is outside the install range or lacks this board's `UF2_BOARD_ID`; it needs the automount exclusion rule (Commissioning). `invalidate` is the cleanup (step 7 below). Both refuse unless the hold is provably yours: the record's reason, a live pid, a FLOCK on the lock file in that pid's own fds, and a failing probe of the flock. - 3. Release with `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py release `, only after an `invalidate` that exited 0 with no `survivors`. Otherwise the board is not cleaned up: keep the hold and report it as a blocker. + `install` is the battery's install step from the record (Run the battery: a J-Link board runs the record's `tinyuf2-jlink` target, an esptool board takes the four IDF segments), then waits for DFU; it takes the record the battery takes, admitted the same way, so it needs the automount exclusion rule (Commissioning) and a clean build of HEAD. `--try-reentry `, while the roster's `reentry` is unset, gives DFU's fallback (Run the battery, dfu) for commissioning; `restore` takes it too. `restore` is the cleanup (below), from a clean build record of HEAD. Both refuse unless the hold is provably yours: the record's reason, a live pid, a FLOCK on the lock file in that pid's own fds, and a failing probe of the flock. + 3. Release with `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py release `, only after a `restore` that exited 0 with no `survivors`. Otherwise the board is not cleaned up: keep the hold and report it as a blocker. - Manual exits: 0 the operation succeeded (for `invalidate`, the zero page at the flash floor verified from a new session); 1 it failed and the board's state is unknown; 2 refused before any hardware access; 4 a child survived SIGKILL, so keep the hold until it is gone. + Manual exits: 0 the operation succeeded (for `install`, DFU came up; for `restore`, DFU left after the blinky copy and stayed away); 1 it failed before writing; 2 refused before any hardware access; 4 a child survived SIGKILL, or an `install` or `restore` failed after it began writing: keep the hold. On a refused hold, report the holder from `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` (no board argument; it prints every holder); never kill it. A `hil_test.py` holder whose record has `role: cleanup-holder` never frees on its own: report it as a blocker that needs the recovery dispatch (Exit 4), without waiting. Any other `hil_test.py` holder is a battery run; wait for it within the lock-wait budget. -- **Cleanup state** for a hw-validator or hw-debugger dispatch: no firmware is restored. `invalidate --under-hold` leaves the board unbootable, and its `cleanup.evidence` (the zero page at the flash floor verified from a new session after the reset) is the observation that Cleanup records. Whoever uses the board next flashes it first; tinyusb's HIL does. +- **Cleanup state** for a hw-validator or hw-debugger dispatch: `restore --under-hold` leaves the board on the record's TinyUF2 with `apps/blinky` copied through it, and its `cleanup.evidence` (DFU left after the copy and stayed away) is the observation that Cleanup records. Blinky running is not checked. Whoever uses the board next flashes it first; tinyusb's HIL does. - **Lock-wait budget:** htpc has no CI runner, so it is the dev-bench row of chief's budget, 2 min, unless the dispatch names another. ## Run the battery ```bash mkdir -p cmake-build/hil -python3 .claude/skills/build/scripts/check_build.py --board --backend cmake --shared > cmake-build/hil/.build.json +python3 .claude/skills/build/scripts/check_build.py --board --backend cmake --shared --hil-config test/hil/local.json > cmake-build/hil/.build.json python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json test/hil/local.json ``` -`` is the roster name, the TinyUF2 board name. +`` is the roster name, the TinyUF2 board name. An espressif board builds without `--backend cmake`: ESP-IDF in CI's docker image, as CI does. -- **Admission.** Before taking the lock the run checks the automount exclusion rule, and every span it could write (BIN, zero page, the device's flash limits, board id), with the install and app spans taken from the candidate's source. -- **Build record.** It is this contract's build receipt: build it on the clean, pinned HEAD and pass it with `--build-record`. It must be a cmake build (`--backend cmake`), whatever backend CI builds the port with; board facts are read from the port's CMake files too. The run refuses a build record that is not clean, does not match the checkout now (HEAD, submodules, `get_deps` dependencies at their pins), or whose artifacts changed since. Commit first. A debug build goes through `install`, not the battery. +- **Admission.** Before taking the lock the run checks the automount exclusion rule, and every span it could write (the HEX the flash target loads, inside the install span and the device's flash limits and laid out as the BIN; the BIN's board id), with the install and app spans taken from the candidate's source. +- **Build record.** It is this contract's build receipt: build it on the clean, pinned HEAD with `--hil-config` and pass it with `--build-record`. `--hil-config` binds the build's own flash target to the roster's probe or bridge (build contract); the run refuses a record whose binding names another flasher, and checks the files that target reads against the record. The build dir's lock covers that target only: the files the runner reads itself (the BIN it verifies, the segments it reads back, `apps/blinky.uf2` and `apps/dfu_runtime.uf2`) are copies in the run dir, taken at admission from bytes that hash as recorded, so a rebuild after the target cannot change them. For a J-Link board it must be a cmake build (`--backend cmake`), whatever backend CI builds the port with; board facts are read from the port's CMake files too. For an espressif board it must be an IDF build from CI's `espressif/idf` image; the run binds its generated `sdkconfig`, `flash_args`, the four segments (bootloader, partition table, otadata, `tinyuf2.bin` in the factory partition) and `apps/dfu_runtime.uf2`, and checks every segment against the generated partition table and the candidate's partition CSV. The run refuses a build record that is not clean, does not match the checkout now (HEAD, submodules, `get_deps` dependencies at their pins), or whose artifacts changed since. Commit first; ESP-IDF's records also go stale at the next commit (build contract). - **Background.** Run it in the background; one board takes about 2 minutes. Use one board per run. - **Selection.** There are no examples or test names to select: a run is the whole battery on one board, and the board is the only selection. -The battery's steps on one board, all under one lock: -1. **identity:** the probe is selected by serial. -2. **install:** one image is written from the flash base and read back: the candidate bootloader, 0xFF up to the app start, then a zero page. -3. **dfu:** the UF2 drive with this board's serial is found. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. -4. **copy:** a generated app is copied to the drive. -5. **app-ran:** the probe shows the app counting in thread mode with VTOR at the app start. -6. **reenter:** the bootloader is re-entered by the commissioned path, and the drive's `CURRENT.UF2` reads the app back. -7. **cleanup** (reported as `cleanup`, not a step): a zero page is written at the device's flash floor (`FLASH_LIMITS`, never the candidate's layout) and read back, the target reset, and the zero page verified again from a new session, so the board boots nothing (on MIMXRT1062 with BOOT_MODE 10b, Internal Boot, an error falls to the Serial Downloader: IMXRT1060RM rev 4 §9.3.5 p199). - -A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; a run triggered by the upload alone is unverified. "reenter" by `retained-register` is the forced-entry path, not a physical double tap. +The battery's steps on one board, all under one lock. The roster's probe serial selects the board (each probe is wired to one target), so no step identifies the chip: +1. **install:** the record's `tinyuf2-jlink` target runs (`cmake --build --target tinyuf2-jlink`) under its build dir's lock, once the files it reads hash as recorded and its dry run shows nothing but the flasher; JLinkExe can exit 0 with every command failed, so the transcript must show the record's HEX downloaded (programmed, or skipped as already matching), `O.K.` and the script completed, with no failure line. The files are hashed again after it, and a session of the runner's verifies the BIN at its flash address (`verifybin`), then resets. The board is touched from the target's start; only a transcript that ends at "Connecting to J-Link ...FAILED", before any target connect, leaves it untouched. +2. **dfu:** the UF2 drive with this board's serial is found. The install has left the app start erased on the bench (Limits), so DFU comes up; if it never enumerates, as when a valid app boots, the commissioned `retained-register` re-entry brings DFU up (its claim says so); any other DFU failure fails the step. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. +3. **copy:** a generated app is copied to the drive. +4. **app-ran:** the probe shows the app counting in thread mode with VTOR at the app start. +5. **reenter:** the bootloader is re-entered by the commissioned path, and the drive's `CURRENT.UF2` reads the app back. +6. **cleanup** (reported as `cleanup`, not a step, state `blinky copied`, or `untouched` with nothing to do when no session reached the board): the record's TinyUF2 is installed again as in install, DFU is found, the record's `apps/blinky.uf2` (admitted with the record: one family, every block in the app region) is copied to the drive, and the DFU device must leave and stay away for a quiet period. Any failure keeps the board held (Exit 4). + +A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; on a J-Link board a run triggered by the upload alone is unverified (an esptool board's app-ran observes the copied app itself over USB). "reenter" by `retained-register` is the forced-entry path, not a physical double tap. + +On an esptool board, each esptool connection is a child of the runner (`esp_session.py`), and so is each USB control request (`usb_dfu.py`, pyusb): +1. **install:** a first connection takes the MD5s of every partition the battery never writes (nvs, ota_1, ffat on the 8 MB layout) as the run's snapshot, kept in the row's `snapshot` however the run ends; that session is the first to touch the board, which only a missing bridge (no tty with its serial), a container left by an earlier flash, or a failed or interrupted `docker create` of the flash container leaves untouched. Then ESP-IDF's `flash` target runs in CI's image (`cmake --build --target flash`), with the same lock, hash and dry-run checks as a J-Link install: the bridge's tty is the only device passed in, as `ESPPORT`, and the container is named `tinyuf2-hil-` so a survivor is found and ended by name. Its transcript must show every segment written at its offset and hash-verified, then the hard reset. A runner session then reads each segment back and takes those MD5s again, then resets once; the runner compares the read-back bytes with the build's segment files and the MD5s with the snapshot, and any difference fails the step. esptool rewrites the bootloader header from `flash_args`, which the build already holds (dio, 80 MHz, 8 MB): on the bench (2026-10-13, esptool 4.8.1) all four segments read back identical to the files. The roster's bridge selects the board (it is wired to that board only), so the chip is not identified, and its security eFuses and flash size are commissioning's to check. The evidence lists each write's span and the 4 KiB-aligned span covering it. Blank otadata boots the factory partition, TinyUF2. +2. **dfu:** as above; `INFO_UF2.TXT` reports the app partition (ota_0) as its flash, and no register is written. +3. **copy:** `apps/dfu_runtime` from the record, plus a block carrying a fresh nonce in ota_0's last 256 bytes, where the app reads it, renumbered under one `numBlocks`; the image itself is never patched. The record is refused when the image reaches that block or TinyUF2's 64 KiB flash cache window over it (`FLASH_CACHE_SIZE`) leaves ota_0. The evidence lists the image and nonce spans and the cache windows covering them. +4. **app-ran:** the app enumerates with its own VID:PID and the board's serial on the DFU's USB path, its DFU runtime interface string read from the device is `HIL `, and TinyUF2 stays away for a quiet period. +5. **reenter** (`dfu-detach`): DFU_DETACH to the app is acknowledged, the app leaves, TinyUF2 comes back on the same path, and `CURRENT.UF2` reads back the image and the nonce block. +6. **recopy:** a copy with a fresh nonce boots the app again. +7. **cleanup:** as on a J-Link board, with the install as in step 1: the preserved regions' MD5s after the writes, and again after the blinky copy, must equal the run's snapshot (a recovery takes it from the failed report's `snapshot`; a manual `restore` takes its own first). A mismatch fails the cleanup and keeps the hold. `--fail-at ` exists only for acceptance: it fails that step to exercise cleanup. @@ -73,7 +81,7 @@ A board's evidence is scoped as the report says. "app-ran" is execution after th The last stdout line is JSON, also written to its `reportPath`; read that path, never an older report. Its fields: - `pass`, `caveat`; -- per board: `ran`, `pass`, `locked`, `lockBypassed`, `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`), `jlinkRetried` (J-Link sessions run again after a crash before the target connect; Limits), and `detail`; +- per board: `ran`, `pass`, `locked`, `lockBypassed`, `firmware` (path, sha256, version, build record), `steps[]` (`step`, `pass`, `evidence`, `detail`), `cleanup` (`pass`, `state`, `evidence`, `heldAfter`, `holder`, `survivors`), `jlinkRetried` (J-Link boards only: sessions run again after a crash before the target connect; Limits), and `detail`; - `error` on a refusal or an operation failure. | Exit | Meaning | @@ -81,7 +89,7 @@ The last stdout line is JSON, also written to its `reportPath`; read that path, | 0 | every board passed and was cleaned up | | 1 | a board failed or did not run; it was cleaned up or never touched | | 2 | refused before any hardware access (usage, roster, source, build record, lock helper, lock dir) | -| 4 | cleanup failed; the board is held when `cleanup.heldAfter` (not under `HIL_NO_BOARD_LOCK`). From `install` or `invalidate`: a child survived; keep your hold until it is gone | +| 4 | cleanup failed; the board is held when `cleanup.heldAfter` (not under `HIL_NO_BOARD_LOCK`). From `install` or `restore`: a child survived, or the operation failed after it began writing; keep your hold | A delegated run (`hil-operator`) returns `{pass, results, caveat, wedged}` and, beside them, `reportPaths`: - `pass`, `results` and `caveat` are copied verbatim from the JSON; @@ -90,14 +98,14 @@ A delegated run (`hil-operator`) returns `{pass, results, caveat, wedged}` and, - With several boards, the operator runs one battery per board, one after another: `results` holds every board's row, `pass` is true only when every run's is, and `caveat` joins the runs' non-empty caveats. - A launched run that ended without printing its JSON (killed, crashed) may have programmed the board and left it running a candidate. The operator writes `pass: false`, `caveat: "run ended without a report; board state unknown"` and one row per board, exactly `{board, ran: null, pass: false, locked: false, detail}`, with `detail` naming the exit status, the output tail and `python3 $TINYUSB_DIR/test/hil/helper/hil_lock.py status` at that moment. - The operator runs the battery once and returns. It never waits on a lock and never retries: a `locked` row comes back as is, and the caller rules below are the caller's (chief's). -- A recovery dispatch names the board, the holder pid and the failed run's `reportPath`. It runs `invalidate --recover-holder` in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Exit 4). +- A recovery dispatch names the board, the holder pid and the failed run's `reportPath`. It runs `restore --recover-holder` in the foreground and returns `{pass, results, caveat: "", wedged, exit, error, survivors}`: that JSON's `results`, `error` (`null` when absent) and `survivors` (`[]` when absent), and the exit status. Any exit but 0 is a blocker for the caller. A probe that cannot connect is `needs-user` (the board's recovery note, Exit 4). ### Review mapping A review (`pr-review`) maps each board's row, except a commissioning run's (`commissioning: true`, from `--try-reentry`), which is never validation. A blocker takes precedence over every other mapping: - `pass`: `ran`, `pass` and `cleanup.pass` all true; -- `fail`: `ran` and not `pass`, with a failing step other than `identity`, and that step's `detail`; -- `not-run`: `ran` false (`locked` or a refusal), or a failing `identity`, with its `detail`. `identity` runs before anything is written, so its failure is the rig's: no probe with that serial on this machine, or one unplugged or dead; +- `fail`: `ran` and not `pass`, with a failing step, `cleanup.state` other than `untouched`, and that step's `detail`; +- `not-run`: `ran` false (`locked` or a refusal), or a failing step with `cleanup.state` `untouched`, with its `detail`. Nothing was written, so the failure is the rig's: no probe with that serial on this machine, one unplugged or dead, or no esptool bridge with that serial; - a blocker, never a `fail` row: exit 4, `ran: null` (no report), and an abort, which is `ran` and not `pass` with no failing step (an interruption or an unexpected error, named in the board's `detail`). `testedHead` is the build record's `input.head`. `regression` is `unknown` unless the battery also ran on the merge base. @@ -106,28 +114,28 @@ A review (`pr-review`) maps each board's row, except a commissioning run's (`com - **Retry.** A failed test may be retried once, and only when no board is `locked`, `wedged` or `heldAfter`, `caveat` is empty, and every board was cleaned up. - Keep both reports. A fail-then-pass is a pass with `flaky: true`, quoting both. - An abort (a failed run with no failing step), a cleanup failure, `heldAfter`, `wedged` or a missing report is never retried; it is a blocker until diagnosed. - - A board whose run ended without a report gets a manual `invalidate --under-hold` before its next run, and only once no such process is left on the host. Killing the runner frees its lock but not its children, which `invalidate` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: + - A board whose run ended without a report gets a manual `restore --under-hold` before its next run, and only once no such process is left on the host. Killing the runner frees its lock but not its children, which `restore` cannot see, and nothing ties a surviving child to the run that died, so the check is host-wide: any other battery, holder or J-Link session also keeps the blocker. Run this as a command of its own, since a shell line carrying the plain name would match itself; none is left when it prints nothing: - `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'` + `pgrep -ax JLinkExe; pgrep -ax mcopy; pgrep -af '[h]il_test\.py'; pgrep -af '[e]sp_session\.py'; pgrep -af '[u]sb_dfu\.py'` - The last one also covers the runner's `fat-read` subcommand. While a process is left, or this cannot be established, the blocker stays. + The `hil_test.py` check also covers the runner's `fat-read` subcommand. While a process is left, or this cannot be established, the blocker stays. - **Locked.** `locked` is an admission refusal, not a test failure. The caller admits again after the holder releases, within the lock-wait budget; the operator does not wait. Otherwise report the board not covered, naming the holder. - **Exit 4.** - The board stays held by a detached holder forked from the run: reason `hil_test.py`, `role: cleanup-holder`, with its pid in `cleanup.holder`. It ignores SIGTERM and SIGINT, so a timeout kill cannot free the board; only recovery releases it. - If no holder could be forked, `cleanup.holder.inProcess` is set and the runner itself is the holder: it prints the report, closes its stdio and does not exit until recovered, so no completion notice arrives. When a run's notice is overdue, read the task's output: a final JSON line with `cleanup.holder.inProcess` true is this case. Do not wait for its exit. - - `cleanup.survivors` lists children that outlived SIGKILL (a J-Link, mtools or FAT read stuck on the device); recovery refuses while any of them is still alive. + - `cleanup.survivors` lists children that outlived SIGKILL (a J-Link, mtools or FAT read stuck on the device), and a flash container that outlived `docker rm -f` (`{"container": "tinyuf2-hil-"}`); recovery refuses while any of them is still alive. A runner killed outright leaves its container to run out; the next install refuses while `docker ps -a` lists it. Each child runs in a session of its own and the kill reaches the whole session, so a JLinkExe that ninja runs in a process group of its own under the flash target is killed and reported too. - tinyusb sees that holder as a battery run and waits, which is safe but means the board is out of service until recovered. - Do not release it by hand, and do not treat it as safe because tinyusb reflashes. The holder may stand for a stuck J-Link or mtools process. - Recovery is a separate `hil-operator` dispatch whose scope names forcing that board. After process, probe and host readiness are established, it runs: - `python3 .claude/skills/hil/scripts/hil_test.py invalidate -b --recover-holder --report test/hil/local.json` + `python3 .claude/skills/hil/scripts/hil_test.py restore -b --build-record --recover-holder --report test/hil/local.json` - The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, invalidates and verifies while the lock stays held, and only then releases the holder. A second recovery of the same board while one runs is refused (exit 2). - - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide). + `` is the failed report's `firmware.buildRecord`, or a record rebuilt on the clean HEAD when it no longer verifies. The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, restores the board while the lock stays held, and only then releases the holder. On an esptool board the restore checks the preserved regions against the snapshot in the failed report, and recovery is refused for a touched run without one or one whose regions differ from the checkout's. A candidate whose drive cannot take the blinky copy fails the restore; the board stays held until a record that can restore it is given. A second recovery of the same board while one runs is refused (exit 2). + - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide); on espressif_esp32s3_devkitc_1, ROM download mode: hold Boot (SW1, GPIO0 low), press and release RST (SW2, CHIP_PU), then release Boot (DevKitC-1 schematic V1.0; the TRM's strap table also needs GPIO46 low, its default pull-down). ## Selection -`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config test/hil/local.json` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered: the battery runs the bootloader and its generated fixture, never an app. The runner, the build scripts it imports and the roster (`.claude/skills/hil/scripts/`, `.claude/skills/build/scripts/`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. +`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config test/hil/local.json` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered, except `dfu_runtime`, which selects the esptool boards of the ports that build it: the battery runs the bootloader with its generated fixture on J-Link boards and with `dfu_runtime` on esptool boards. The runner, the build scripts it imports and the roster (`.claude/skills/hil/scripts/`, `.claude/skills/build/scripts/`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. ## Commissioning a board @@ -147,42 +155,37 @@ Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under ``` Then `sudo udevadm control --reload`. -- **Bootloader:** `install` a known-good TinyUF2 BIN under the hold and observe DFU. +- **Bootloader:** `install` a clean HIL record of HEAD under the hold and observe DFU (add `--try-reentry retained-register` if a valid app could keep DFU away). - **USB serial:** confirm the enumerated serial equals `uid` (sysfs or `lsusb -v`), and record the USB port paths. If it differs, fix `uid` and the rule together, reload, and `install` again; its reset re-enumerates the board and udev applies the reloaded rules, so no replug should be needed (by mechanism, not yet observed on a uid fix). -- **Cleanup:** `invalidate` under the hold (its `cleanup.evidence` is the zero page verified from a new session), then confirm the ROM fallback (on MIMXRT1062, the serial downloader's USB device on the board's port), and that `install` succeeds from that state and DFU comes up. Record the reset notes J-Link prints with no boot image (`BENIGN`). +- **Cleanup:** `restore` under the hold and observe DFU leave after the blinky copy. Record the reset notes J-Link prints on install (`BENIGN`). - **Exclusion proof:** delete the temporary rule and reload before the re-entry run below; its `dfu` step passes only when `udevadm info` on the enumerated block device shows `UDISKS_IGNORE=1` (`check_exclusive`). A host rule that ignores a whole VID, such as tinyusb's `88-tinyusb.rules` for 239a, sets the same property, so the proof cannot tell it from the serial rule; the runner still requires the serial line. -- **Loader footprint:** for each `loadbin` shape the runner issues, place sentinels just past the written range and elsewhere in its erase block. Confirm them from a fresh session before the operation under test, and again from another fresh session after it: a verify in the session that wrote them cannot show what a later operation destroys. Make the written data differ from what is on the flash, since a `loadbin` whose data already matches is skipped ("Contents already match") and erases nothing. Record the result in Limits. +- **Loader footprint:** for the flash target's `loadfile`, place sentinels just past the written range and elsewhere in its erase block. Confirm them from a fresh session before the operation under test, and again from another fresh session after it: a verify in the session that wrote them cannot show what a later operation destroys. Make the written data differ from what is on the flash, since a download whose data already matches is skipped ("Contents already match") and erases nothing. Record the result in Limits. - **Fixture prerequisites,** before anything executes the battery's fixture: - the app start meets VTOR's alignment for the fixture's vector table (`fixture()`); - its stack top, the `m_data` origin the runner reads (`ram_origin`) + 0x1000, is in RAM enabled when the bootloader hands off: the ROM default per read-doc, which TinyUF2 does not reconfigure (DTCM on imxrt1060_evk); - under a hold, a word just below that stack top takes a written value and reads it back, and then its saved value (0x20000FFC on imxrt1060_evk). -- **Re-entry:** commit the roster entry without `reentry`, build the record, release the manual hold, and run the battery in commissioning mode; it locks itself and ends with the cleanup, so it also proves a run from and back to an invalidated board: +- **Re-entry:** commit the roster entry without `reentry`, build the record, release the manual hold, and run the battery in commissioning mode; it locks itself and ends with the cleanup, so it also proves the restore: ```bash python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cmake-build/hil/.build.json --try-reentry retained-register test/hil/local.json ``` - `--try-reentry` is the lead's only, and is refused once `reentry` is set. The row's `reentryDecision` names the method when the run passed or failed only at `reenter`'s `CURRENT.UF2` readback, every observation of the tried method is true (`controlBootedApp`, `dfuCameUp` and `magicConsumed` for `retained-register`), and the board was cleaned up and not held. Set `reentry` to it and commit. A null decision leaves `reentry` unset: file a `followup` issue with the observations for diagnosis, not as a diagnosed bug. `app-invalidate` is set only after that path is validated on its own. When the drive copy itself fails, a fixture installed by hand with J-Link `loadbin` can isolate re-entry from the copy for diagnosis; it never sets `reentry`. -- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. + `--try-reentry` is the lead's only, and is refused once `reentry` is set. The row's `reentryDecision` names the method when the run passed or failed only at `reenter`'s `CURRENT.UF2` readback, every observation of the tried method is true (`controlBootedApp`, `dfuCameUp` and `magicConsumed` for `retained-register`), and the board was cleaned up and not held. Set `reentry` to it and commit. A null decision leaves `reentry` unset: file a `followup` issue with the observations for diagnosis, not as a diagnosed bug. When the drive copy itself fails, a fixture installed by hand with J-Link `loadbin` can isolate re-entry from the copy for diagnosis; it never sets `reentry`. +- **On an esptool board,** in place of the J-Link items: cable the native USB connector as well as the bridge's; check once by hand that the chip has no secure boot, secure download or flash encryption and that its flash holds the candidate's layout (`esptool get-security-info`, `esptool flash-id`), since the runner never checks them; record the bridge serial and the chip's MAC (`uid`, TinyUF2's USB serial); `install` from the record and observe DFU, then `restore` and observe DFU leave. Re-entry is `--try-reentry dfu-detach`, whose decision needs `detachAcked`, `appLeft` and `dfuCameUp`; before setting it, repeat DFU_DETACH ten times under the hold and check the host log shows no USB reset on the DETACH transitions (USB DFU 1.1 bitWillDetach). +- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid; on MIMXRT1011DAE5A the same note on every reset of a TinyUF2 image. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. ## Limits -- Covered: imxrt1060_evk on htpc. Not covered: ports without a cmake build, lpc55 among them (no battery, `install` or `invalidate` for lpcxpresso55s69; its 0A silicon also hangs TinyUF2 on a UF2 copy, #500), ESP32 boards (#502), self-update, an interrupted update, malformed UF2 input and the physical double tap. +- Covered: imxrt1060_evk, metro_m7_1011 and espressif_esp32s3_devkitc_1 on htpc. Not covered: ports without a cmake build, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69; its 0A silicon also hangs TinyUF2 on a UF2 copy, #500), espressif chips other than the ESP32-S3, self-update, an interrupted update, malformed UF2 input and the physical double tap. +- esptool boards: the runner needs esptool 5.2.0 and pyusb with libusb-1.0 for its own Python. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The bridge's UART carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. At the end of a UF2 copy TinyUF2 restarts while the host's usb-storage is still talking to it: the host log shows a reset of that device and read errors on its block device. Seen on the S3; other ports are unchecked. - Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. - The HIL's cmake image logs on the board UART: cmake builds leave `TUF2_LOG` at its default 1 (`src/board_api.h`), while make builds, the make ports' CI artifacts and the make-built releases pass 0. Its blocking writes at start, before the app jump and on DFU entry are timing the make image does not have. -- Cleanup leaves the board unbootable, so the next user, tinyusb's HIL included, must flash before it runs anything. Measured on imxrt1060_evk on 2026-10-12: - - the board then enumerates as the ROM serial downloader (1fc9:0135), and `install` from that state brings up DFU; - - J-Link prints its reset notes (`BENIGN`) with and without a boot image; - - with no boot image, a new J-Link session reads the flash wrong until a `loadbin` has run (zeros, and at 0x60010000 the bytes written at 0x60000200), so a plain `verifybin` fails. The cause is unknown. `invalidate` therefore re-verifies with a `loadbin` of the zero page that J-Link must report as "Skipped. Contents already match", then a `verifybin`. -- Never use a port's CMake flash or erase targets on a rig board: `tinyuf2-erase-external-jlink` erases [FLASH_BASE, FLASH_BASE + 64 KiB), the bootloader and the zero page with it. -- J-Link's QSPI loader on imxrt1060_evk is undocumented: SEGGER UM08001 has nothing on it. Measured at sampled sentinel points only: - - the install image (the cmake BIN, 0xFF up to 0x6000C000, then the zero page, ending at 0x6000C200) left 0x6000C200 and 0x6000FF00 reading 0xFF in the same session, before any reset. Earlier, a 0.35.0 release BIN `loadbin` (30304 B) erased every point measured past it in its block (0x60007800, 0x6000A000, 0x6000C000, 0x6000C200). J-Link reports 65536 bytes per operation, so [image end, 0x60010000) is treated as lost; - - a sentinel at 0x60010000, in the next block, survived both; - - the cleanup's zero-page `loadbin` kept sentinels at 0x60000200, 0x6000A000, 0x6000FF00 and 0x60010000, and 16-byte sentinel `loadbin`s kept their neighbours in the same block, so what makes J-Link erase the rest of a block is not established. - - That a candidate BIN of another length behaves the same is an inference, not a measurement. - - That `loadbin` erases only the 64 KiB blocks its range touches, and never the chip, is an unverified premise these points support. - Under that premise, losses stay inside [0x60000000, 0x60010000), in regions the battery rewrites or never reads, while the install span and the zero page lie in that block. +- `install` from a blank board, or one whose ROM found no boot image, brings up DFU (measured on imxrt1060_evk 2026-10-12 and metro_m7_1011 2026-10-13). With no boot image a new J-Link session reads the flash wrong (imxrt1060_evk: zeros; metro_m7_1011: bytes shifted) until a `loadbin` has run; a `loadbin` compare reads it right. On metro_m7_1011, J-Link then prints "Invalid flash header detected." at the install's first reset. +- Of a port's CMake flash targets the runner runs only `tinyuf2-jlink` and ESP-IDF's `flash`, bound by the record to the roster's probe or chip (the bridge's tty is passed at flash time). Never use the erase targets on a rig board (`tinyuf2-erase-external-jlink` erases [FLASH_BASE, FLASH_BASE + 64 KiB), the bootloader with it), nor ESP-IDF's `combined-flash`, which has no port selection and writes nvs and both OTA slots blank. +- On metro_m7_1011 (2026-10-13, J-Link V9.84), `tinyuf2-jlink`'s `loadfile` of the cmake HEX (0x60000400 on) reported "1 range affected (65536 bytes)" and, from a fresh session after it, 0x60000000, 0x6000BF00, 0x6000C000-0x6000C00F, 0x6000C200 and 0x6000FF00 read 0xFF while 0x60010000 kept its data. Its compare covered the whole block: over blinky it programmed, and right after a program it skipped ("Contents already match"). DFU came up within 2 s in each of these three runs. That every install by the target leaves [0x6000C000, 0x60010000), the app's first 16 KiB, erased is an inference from these samples and this HEX, not a measurement of every case; the re-entry fallback (Run the battery, dfu) covers an install that leaves a valid app. On imxrt1060_evk (2026-10-13), the same target over blinky also reported "1 range affected (65536 bytes)"; from a fresh session after it, blinky's vectors at 0x6000C000-0x6000C00F and 0x6000C200 read 0xFF, while the FCB at 0x60000000 and 0x60010000 kept their data. +- A `JLinkExe` crash inside the target is not retried (the crash retry below covers the runner's own sessions only): it fails the install, touched. +- J-Link's QSPI loader is undocumented (SEGGER UM08001 has nothing on it). Historically, at sampled sentinel points on both RT boards, the runner's former `loadbin` install (no longer issued) lost the rest of its 64 KiB block past the image while a sentinel at 0x60010000, in the next block, survived; a sample, not a guarantee for every image. - `JLinkExe` sometimes crashes in `libjlinkarm` at the probe connect on htpc (SIGSEGV; once a glibc double-free abort); the cause is unknown. A session that crashed this way, before the target connect, runs once more after 2 s (`jlinkRetried`); a second crash fails the run. Seen in 2 of 94 runner sessions on 2026-10-12. - A device hung mid-write makes the host controller retry its enumeration, which stalled a J-Link connect on the same hub for ~80 s (lpcxpresso55s69); that the hub is the cause is a hypothesis. - If no cleanup holder can be forked (exit 4 with `cleanup.holder.inProcess`), the runner never exits until recovered. -- No reproducer for an instrumented build: the battery refuses a dirty build, and `install` only programs. A claim that needs one is reported pending. +- No reproducer for an instrumented build: the battery and `install` take only a clean build record. A claim that needs one is reported pending. diff --git a/.claude/skills/hil/scripts/esp_session.py b/.claude/skills/hil/scripts/esp_session.py new file mode 100644 index 000000000..a4e1ff019 --- /dev/null +++ b/.claude/skills/hil/scripts/esp_session.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""One esptool connection for hil_test.py, run as its own child so the runner can time it out, kill it and +see it survive: connect into download mode, run the ops in order, then reset or not. + + esp_session.py --port DEV --chip CHIP --baud N --after hard-reset|no-reset OPS.json + +OPS.json is a list of {"op": "read", "addr", "size", "file"} and {"op": "md5", "addr", "size"}: no write, which +is the build's flash target's. +The last stdout line is JSON: {"ok": true, "results": [...]} or {"ok": false, "error", "results"}. +""" +import argparse +import json +import sys + +sys.dont_write_bytecode = True + + +def run(esp, op): + kind = op['op'] + if kind == 'read': + data = esp.read_flash(op['addr'], op['size']) + with open(op['file'], 'wb') as f: + f.write(data) + return {'size': len(data)} + if kind == 'md5': + return {'md5': esp.flash_md5sum(op['addr'], op['size'])} + raise ValueError(f'unknown op {kind!r}') + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--port', required=True) + ap.add_argument('--chip', required=True) + ap.add_argument('--baud', type=int, required=True) + ap.add_argument('--after', choices=('hard-reset', 'no-reset'), required=True) + ap.add_argument('ops') + args = ap.parse_args() + results, esp = [], None + try: + import esptool + from esptool import cmds + ops = json.load(open(args.ops)) + esp = cmds.detect_chip(args.port, connect_mode='default-reset') + if esp.CHIP_NAME.lower().replace('-', '') != args.chip: + raise RuntimeError(f'the port reaches a {esp.CHIP_NAME}, not an {args.chip}') + esp = cmds.run_stub(esp) + esp.change_baud(args.baud) + cmds.attach_flash(esp) + for op in ops: + results.append(dict(run(esp, op), op=op['op'])) + if args.after == 'hard-reset': + cmds.reset_chip(esp, 'hard-reset') + out = {'ok': True, 'esptool': esptool.__version__, 'results': results} + except Exception as e: # every failure is reported, never raised past the JSON line + out = {'ok': False, 'error': f'{type(e).__name__}: {e}', 'results': results} + finally: + if esp is not None: + esp._port.close() + print(json.dumps(out), flush=True) + return 0 if out['ok'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/skills/hil/scripts/hil_esp.py b/.claude/skills/hil/scripts/hil_esp.py new file mode 100644 index 000000000..a9f1d3231 --- /dev/null +++ b/.claude/skills/hil/scripts/hil_esp.py @@ -0,0 +1,178 @@ +"""Espressif facts and build-record admission for hil_test.py: what an IDF build of a TinyUF2 board +writes where, read from the candidate's source and checked against the generated build files.""" +import csv +import hashlib +import io +import re +import struct + +from pathlib import Path + +from check_build import FLASH, HIL_APP, IDF_CONFIG + +# 2nd-stage bootloader offset per chip, independent of the build: the ROM loads it from here (IDF v5.3.2 +# bootloader/Kconfig.projbuild BOOTLOADER_OFFSET_IN_FLASH, esptool 5.2.0 targets/esp32s3.py; no checked TRM states it) +BOOTLOADER_OFFSET = {'esp32s3': 0x0} +PARTITION_TABLE = (0x8000, 0x1000) # ports/espressif/sdkconfig.defaults CONFIG_PARTITION_TABLE_OFFSET; one sector +APP, DATA = 0x00, 0x01 +SUBTYPE = {(APP, 'factory'): 0x00, (APP, 'ota_0'): 0x10, (APP, 'ota_1'): 0x11, (DATA, 'ota'): 0x00, + (DATA, 'nvs'): 0x02, (DATA, 'fat'): 0x81} +TYPE = {'app': APP, 'data': DATA} +# the IDF build's own segments, the flash target's inputs after flash_args: build-dir path -> the segment's name +# (its roster region) +SEGMENT_PATHS = {p: Path(p).name for p in FLASH['esptool'][1][1:]} +SEGMENTS = tuple(SEGMENT_PATHS.values()) +NATIVE = {v: k for k, v in SEGMENT_PATHS.items()} +APP_UF2 = f'_bin/apps/{HIL_APP}.uf2' # the battery's app, copied through TinyUF2 +BLINKY_UF2 = '_bin/apps/blinky.uf2' # what cleanup leaves running +RECORD_FILES = (APP_UF2, BLINKY_UF2, *IDF_CONFIG) # besides the flash target's inputs, which admit_flash checks +IMAGE_MAGIC = 0xE9 + + +class Refused(Exception): + pass + + +def size_of(text): + m = re.fullmatch(r'(\d+)([KM]?)', text.strip()) + if not m: + raise Refused(f'{text!r} is not a partition size') + return int(m.group(1)) * {'': 1, 'K': 1024, 'M': 1024 * 1024}[m.group(2)] + + +def parse_csv(text, what): + """[(label, type, subtype, offset, size)] of an IDF partition CSV with explicit offsets only.""" + rows = [] + for row in csv.reader(io.StringIO(text)): + row = [c.strip() for c in row] + if not row or not row[0] or row[0].startswith('#'): + continue + if len(row) < 5 or row[1] not in TYPE or (TYPE[row[1]], row[2]) not in SUBTYPE or not row[3]: + raise Refused(f'{what}: unsupported row {row}') + rows.append((row[0], TYPE[row[1]], SUBTYPE[TYPE[row[1]], row[2]], int(row[3], 0), size_of(row[4]))) + return rows + + +def parse_table(blob, what): + """[(label, type, subtype, offset, size)] of a generated partition-table.bin, its MD5 entry checked.""" + rows, md5 = [], None + for i in range(0, len(blob), 32): + e = blob[i:i + 32] + if e[:2] == b'\xaa\x50': + t, st, off, size = struct.unpack_from('MB=y and a partitions-*.csv') + return {'flashSize': sizes[0] * 1024 * 1024, 'partitionCsv': name, + 'partitions': parse_csv((root / 'ports/espressif' / name).read_text(), name)} + + +def layout(f, chip): + """Where the runner may write, from the partitions: each install segment's region, the app (ota_0, which + only TinyUF2 writes, through copy), and every other partition, preserved.""" + parts = {p[0]: p for p in f['partitions']} + by_kind = {(p[1], p[2]): p for p in f['partitions']} + need = {'otadata': (DATA, 0x00), 'app': (APP, 0x10), 'factory': (APP, 0x00)} + missing = [k for k, v in need.items() if v not in by_kind] + if missing: + raise Refused(f'{f["partitionCsv"]}: no {", ".join(missing)} partition') + span = lambda p: (p[3], p[3] + p[4]) + install = {'bootloader.bin': (BOOTLOADER_OFFSET[chip], PARTITION_TABLE[0]), + 'partition-table.bin': (PARTITION_TABLE[0], sum(PARTITION_TABLE)), + 'ota_data_initial.bin': span(by_kind[need['otadata']]), 'tinyuf2.bin': span(by_kind[need['factory']])} + spans = sorted([(*install['bootloader.bin'], 'bootloader'), (*install['partition-table.bin'], 'partition table')] + + [(*span(p), p[0]) for p in f['partitions']]) + for (s0, e0, a), (s1, e1, b) in zip(spans, spans[1:]): + if e0 > s1: + raise Refused(f'{f["partitionCsv"]}: {a} [{s0:#x},{e0:#x}) overlaps {b} at {s1:#x}') + if spans[-1][1] > f['flashSize']: + raise Refused(f'{f["partitionCsv"]}: {spans[-1][2]} ends at {spans[-1][1]:#x}, past the {f["flashSize"]:#x} flash') + owned = {by_kind[v][0] for v in need.values()} + return {'install': install, 'app': span(by_kind[need['app']]), + 'preserved': {n: span(p) for n, p in parts.items() if n not in owned}} + + +def read_flash_args(text, what): + """(settings, {offset: basename}) of an IDF flash_args file.""" + lines = [l.split() for l in text.splitlines() if l.strip()] + if not lines: + raise Refused(f'{what}: empty') + opts = lines[0] + if len(opts) % 2 or not all(o.startswith('--') for o in opts[::2]): + raise Refused(f'{what}: first line is not --option value pairs: {opts}') + settings = {o[2:]: v for o, v in zip(opts[::2], opts[1::2])} + segs = {} + for l in lines[1:]: + if len(l) != 2 or not re.fullmatch(r'0x[0-9a-fA-F]+', l[0]): + raise Refused(f'{what}: {" ".join(l)!r} is not "OFFSET FILE"') + off = int(l[0], 16) + if off in segs: + raise Refused(f'{what}: two segments at {off:#x}') + if l[1] not in SEGMENT_PATHS: + raise Refused(f'{what}: {l[1]!r} is not one of the IDF build\'s segments {sorted(SEGMENT_PATHS)}') + segs[off] = SEGMENT_PATHS[l[1]] + return settings, segs + + +def admit(root, board, f, lay): + """The segments an install writes, [(offset, path)], from an IDF build dir already hash-checked against + its record: flash_args, the generated sdkconfig and partition table must agree with the candidate's + source, and every segment must sit inside its own region. Paths are the build tree's own files, which + ESP-IDF's flash target writes, not the _bin copies.""" + gen = sdkconfig((root / 'sdkconfig').read_text()) + want = {'CONFIG_IDF_TARGET': f'"{board["device"]}"', 'CONFIG_PARTITION_TABLE_OFFSET': hex(PARTITION_TABLE[0]), + 'CONFIG_PARTITION_TABLE_CUSTOM_FILENAME': f'"{f["partitionCsv"]}"', + 'CONFIG_ESPTOOLPY_FLASHSIZE': f'"{f["flashSize"] // (1024 * 1024)}MB"'} + bad = {k: gen.get(k) for k, v in want.items() if gen.get(k) != v} + if bad: + raise Refused(f'generated sdkconfig differs from the candidate source: {bad}') + settings, segs = read_flash_args((root / 'flash_args').read_text(), 'flash_args') + for k, v in (('flash_mode', gen.get('CONFIG_ESPTOOLPY_FLASHMODE')), ('flash_freq', gen.get('CONFIG_ESPTOOLPY_FLASHFREQ')), + ('flash_size', gen.get('CONFIG_ESPTOOLPY_FLASHSIZE'))): + if f'"{settings.get(k)}"' != v: + raise Refused(f'flash_args --{k} {settings.get(k)} differs from the generated sdkconfig {v}') + if sorted(segs.values()) != sorted(SEGMENTS): + raise Refused(f'flash_args names {sorted(segs.values())}, not exactly {sorted(SEGMENTS)}') + table = parse_table((root / NATIVE['partition-table.bin']).read_bytes(), 'partition-table.bin') + if table != f['partitions']: + raise Refused(f'partition-table.bin {table} differs from {f["partitionCsv"]} {f["partitions"]}') + out = [] + for off, name in sorted(segs.items()): + path = root / NATIVE[name] + lo, hi = lay['install'][name] + size = path.stat().st_size + if off != lo or off + size > hi: + raise Refused(f'{name} [{off:#x},{off + size:#x}) is not at the start of its region [{lo:#x},{hi:#x})') + with open(path, 'rb') as fh: + magic = fh.read(1) + if name in ('bootloader.bin', 'tinyuf2.bin') and magic != bytes([IMAGE_MAGIC]): + raise Refused(f'{name} is not an ESP image (no 0xE9 magic)') + out.append((off, path)) + return out diff --git a/.claude/skills/hil/scripts/hil_test.py b/.claude/skills/hil/scripts/hil_test.py index 193bf10d4..05fb9e261 100755 --- a/.claude/skills/hil/scripts/hil_test.py +++ b/.claude/skills/hil/scripts/hil_test.py @@ -2,17 +2,20 @@ """TinyUF2 HIL battery and the probe operations debug sessions share with it. hil_test.py run -b BOARD --build-record FILE CONFIG [--fail-at STEP] [--try-reentry METHOD] - hil_test.py install -b BOARD --firmware BIN --sha256 HASH --under-hold REASON CONFIG - hil_test.py invalidate -b BOARD (--under-hold REASON | --recover-holder PID --report FILE) CONFIG + hil_test.py install -b BOARD --build-record FILE --under-hold REASON [--try-reentry METHOD] CONFIG + hil_test.py restore -b BOARD --build-record FILE (--under-hold REASON [--try-reentry METHOD] | --recover-holder PID --report FILE) CONFIG The last stdout line is JSON. Exit 0 every board passed and was cleaned up, 1 a board failed or did not run (cleaned up or never touched), 2 refused before any hardware access, 4 cleanup failed (the board is -held when `cleanup.heldAfter`; not under HIL_NO_BOARD_LOCK) or a manual operation left `survivors`. +held when `cleanup.heldAfter`; not under HIL_NO_BOARD_LOCK), a manual operation left `survivors`, or a +`restore` failed after it began writing. SKILL.md beside this script is the contract; this file is its mechanics. """ import argparse import contextlib import fcntl +import functools +import glob import hashlib import importlib.util import json @@ -29,30 +32,44 @@ ROOT = Path(__file__).resolve().parents[4] sys.path.insert(0, str(ROOT / '.claude' / 'skills' / 'build' / 'scripts')) +sys.path.insert(0, str(Path(__file__).resolve().parent)) sys.dont_write_bytecode = True # nothing the runner imports may write into a checkout it binds import check_build as cb # noqa: E402 +import hil_esp as esp # noqa: E402 # [floor, ceiling) the runner ever programs, per J-Link device, independent of roster and source: # LPC55S69 PFR starts at 0x9DE00 and a bad CMPA write can lock the part (UM11126 Table 173, Table 1064); # 0x9D800-0x9DDFF is reserved and undocumented. MIMXRT1062: FlexSPI XIP at 0x60000000 (IMXRT1060RM -# Table 3-1) holding the EVK's 8 MiB IS25WP064A (EVK hardware user's guide 2.7). -FLASH_LIMITS = {'LPC55S69_M33_0': (0, 0x9D800), 'MIMXRT1062xxx6A': (0x60000000, 0x60800000)} -STEPS = ('identity', 'install', 'dfu', 'copy', 'app-ran', 'reenter') -REENTRY = ('retained-register', 'app-invalidate') +# Table 3-1) holding the EVK's 8 MiB IS25WP064A (EVK hardware user's guide 2.7). MIMXRT1011: FlexSPI XIP at +# 0x60000000 (IMXRT1010RM Table 3-1) holding the Metro M7 1011's W25Q64JV (Metro M7 RT1011 SD revA schematic), 8 MiB. +FLASH_LIMITS = {'LPC55S69_M33_0': (0, 0x9D800), 'MIMXRT1062xxx6A': (0x60000000, 0x60800000), + 'MIMXRT1011DAE5A': (0x60000000, 0x60800000)} +STEPS = {'jlink': ('install', 'dfu', 'copy', 'app-ran', 'reenter')} +STEPS['esptool'] = STEPS['jlink'] + ('recopy',) +REENTRY = {'jlink': ('retained-register',), 'esptool': ('dfu-detach',)} +REENTRY_METHODS = sorted({m for v in REENTRY.values() for m in v}) PROTECTED = {'hil_test.py', 'pool_check'} # tinyusb hil_lock.PROTECTED_REASONS, checked at load HOLDER_ROLE = 'cleanup-holder' -SCB_VTOR, SCB_AIRCR, SCB_CPUID, DHCSR = 0xE000ED08, 0xE000ED0C, 0xE000ED00, 0xE000EDF0 +SCB_VTOR, SCB_AIRCR = 0xE000ED08, 0xE000ED0C AIRCR_SYSRESETREQ = 0x05FA0004 UF2_MAGIC = (0x0A324655, 0x9E5D5157, 0x0AB16F30) UF2_FLAG_FAMILY = 0x2000 PROBE_TIMEOUT, USB_TIMEOUT, QUIET_TIME, KILL_WAIT, UDEV_TIMEOUT = 120, 20, 3, 5, 5 JLINK_RETRY_DELAY = 2 +ESPTOOL_VERSION = '5.2.0' # the CLI and API the esptool session helper is written and bench-checked against +ESP_BAUD, ESP_TIMEOUT = 921600, 300 +ESP_SESSION = Path(__file__).resolve().parent / 'esp_session.py' +USB_DFU = Path(__file__).resolve().parent / 'usb_dfu.py' +NONCE_MAGIC, NONCE_BLOCK = b'TUF2NONC', 256 # apps/dfu_runtime/src/dfu_runtime.h JLINK_IF, JLINK_SPEED = 'SWD', '4000' -ZERO_PAGE = 512 # bytes zeroed at the app start so its reset vector reads 0 +CMAKE_RECORD_FILES = ('_bin/tinyuf2.bin', '_bin/apps/blinky.uf2') # besides the flash target's inputs +JLINK_ERRORS = r'Verify failed|Could not|Cannot connect|ERROR|ResetTarget failed|Failed to halt|CPU is not halted' UDEV_RULES = ('/etc/udev/rules.d', '/run/udev/rules.d') # J-Link notes that are not failures, per device, each with the line that shows the operation completed -# (bench: every MIMXRT1062 reset with an invalid app vector, then the reset halts as usual) -BENIGN = {'MIMXRT1062xxx6A': r'(?m)^Could not find valid reset vector\.\nTarget halted on flash header read\.$'} +# (bench: every MIMXRT1062 reset with an invalid app vector, then the reset halts as usual; every MIMXRT1011 reset +# of a TinyUF2 image, which then boots) +_RESET_VECTOR_NOTE = r'(?m)^Could not find valid reset vector\.\nTarget halted on flash header read\.$' +BENIGN = {'MIMXRT1062xxx6A': _RESET_VECTOR_NOTE, 'MIMXRT1011DAE5A': _RESET_VECTOR_NOTE} class Refused(Exception): @@ -65,6 +82,10 @@ def __init__(self, msg, evidence=None): self.evidence = evidence or {} +class NotEnumerated(StepFail): + """The DFU device never appeared: the only failure after an install that re-entry may follow.""" + + class Interrupted(Exception): pass @@ -85,9 +106,10 @@ def load_board(config, name): raise Refused('set TINYUSB_DIR to the absolute path of the tinyusb checkout whose test/hil/helper/hil_lock.py ' 'owns the board locks') fl, t = b.get('flasher') or {}, b.get('tinyuf2') or {} - m = re.fullmatch(r'-device (\S+)', fl.get('args') or '') - if fl.get('name') != 'jlink' or not isinstance(fl.get('uid'), str) or not m: - raise Refused(f'{name}: flasher must be jlink with a string uid and args exactly "-device NAME"') + m = re.fullmatch(cb.FLASHER_ARGS.get(fl.get('name'), r'(?!)'), fl.get('args') or '') + if not isinstance(fl.get('uid'), str) or not m: + raise Refused(f'{name}: flasher must be jlink with args exactly "-device NAME", or esptool with args exactly ' + f'"--chip CHIP", with a string uid') if not isinstance(b.get('uid'), str) or not b['uid']: raise Refused(f'{name}: uid (the bootloader USB serial) is required') lock = t.get('lock', name) # tinyusb's name for the board when it differs: both runners take one lock @@ -97,10 +119,13 @@ def load_board(config, name): port = cb.port_of(name) except cb.Fail as e: raise Refused(f'{name}: the roster names a board by its TinyUF2 name: {e}') - if m.group(1) not in FLASH_LIMITS: + if fl['name'] == 'esptool': + if m.group(1) not in esp.BOOTLOADER_OFFSET or port != 'espressif': + raise Refused(f'{name}: esptool is supported for espressif chips {sorted(esp.BOOTLOADER_OFFSET)} only') + elif m.group(1) not in FLASH_LIMITS: raise Refused(f'{name}: no programming limits known for J-Link device {m.group(1)}') - return {'name': name, 'lock': lock, 'uid': b['uid'], 'probe': fl['uid'], 'device': m.group(1), - 'if': JLINK_IF, 'speed': JLINK_SPEED, 'port': port, 'page': ZERO_PAGE, + return {'name': name, 'lock': lock, 'uid': b['uid'], 'flasher': fl['name'], 'probe': fl['uid'], 'device': m.group(1), + 'if': JLINK_IF, 'speed': JLINK_SPEED, 'port': port, 'helper': os.path.join(tinyusb, 'test', 'hil', 'helper', 'hil_lock.py'), 'reentry': t.get('reentry')} @@ -158,19 +183,25 @@ def cmake_var(path, name): def port_sources(port, bdir): - """(CMSIS device header, the SDK's flash linker script, FLASH_BASE) of a port the HIL supports, read from - the CMake files the HIL's build uses.""" + """(CMSIS device header, the SDK's flash linker script, FLASH_BASE, where the BIN starts in flash) of a port + the HIL supports, read from the CMake files the HIL's build uses.""" if not (ROOT / f'ports/{port}/CMakeLists.txt').is_file(): raise Refused(f'ports/{port} has no cmake build, which is what the HIL builds') if port != 'mimxrt10xx': raise Refused(f'ports/{port}: the HIL runner has no source facts for this port') sdk = cmake_var(f'ports/{port}/family.cmake', 'SDK_DIR') mcu = cmake_var(f'{bdir}/board.cmake', 'MCU_VARIANT') - ld = f'ports/{port}/linker/{mcu}_ram.ld' # where family.cmake takes FLASH_BASE from - m = re.search(r'^\s*_flash_base\s*=\s*(0x[0-9a-fA-F]+)\s*;', (ROOT / ld).read_text(), re.M) - if not m: - raise Refused(f'{ld}: no _flash_base') - return f'{sdk}/devices/{mcu}/{mcu}.h', f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld', int(m.group(1), 16) + ld = f'ports/{port}/linker/{mcu}_ram.ld' # where family.cmake takes FLASH_BASE and FCFB_ORIGIN from + vals = {} + for name in ('_flash_base', '_fcfb_origin'): + m = re.search(rf'^\s*{name}\s*=\s*(0x[0-9a-fA-F]+)\s*;', (ROOT / ld).read_text(), re.M) + if not m: + raise Refused(f'{ld}: no {name}') + vals[name] = int(m.group(1), 16) + flash_base = vals['_flash_base'] + # family.cmake FLASH_FCFB_ADDR: the BIN starts at the FCB, 0x400 into flash on RT1010 (IMXRT1010RM Table 9-8) + bin_base = flash_base + (vals['_fcfb_origin'] & 0xFFF) + return f'{sdk}/devices/{mcu}/{mcu}.h', f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld', flash_base, bin_base def source_facts(board): @@ -179,11 +210,14 @@ def source_facts(board): bdir = f'ports/{port}/boards/{board["name"]}' if not (ROOT / bdir).is_dir(): raise Refused(f'{bdir} does not exist') - header, ld, flash_base = port_sources(port, bdir) + if board.get('flasher') == 'esptool': + return esp_source_facts(board, bdir) + header, ld, flash_base, bin_base = port_sources(port, bdir) if re.search(r'^\s*#\s*define\s+BOARD_FLASH_APP_START\b', (ROOT / f'{bdir}/board.h').read_text(), re.M): raise Refused(f'{bdir}/board.h overrides BOARD_FLASH_APP_START; the runner reads the port default only') facts = { 'flashBase': flash_base, + 'binBase': bin_base, 'appStart': flash_addr(define(f'ports/{port}/boards.h', 'BOARD_FLASH_APP_START'), flash_base, 'BOARD_FLASH_APP_START'), 'flashSize': product(define(f'{bdir}/board.h', 'BOARD_FLASH_SIZE'), 'BOARD_FLASH_SIZE'), @@ -207,6 +241,25 @@ def source_facts(board): return facts +def esp_source_facts(board, bdir): + try: + facts = esp.facts(ROOT, board) + except (esp.Refused, OSError, ValueError) as e: + raise Refused(str(e)) + delay = re.search(r'^\s*#\s*define\s+UF2_DETECTION_DELAY_MS\s+(\d+)', (ROOT / bdir / 'board.h').read_text(), re.M) + facts.update(vid=num(define(f'{bdir}/board.h', 'USB_VID'), 'USB_VID'), + pid=num(define(f'{bdir}/board.h', 'USB_PID'), 'USB_PID'), + boardId=define(f'{bdir}/board.h', 'UF2_BOARD_ID').strip('"'), + detectionDelayMs=int(delay.group(1)) if delay else + num(define('ports/espressif/bootloader_components/main/bootloader_start.c', 'UF2_DETECTION_DELAY_MS'), + 'UF2_DETECTION_DELAY_MS'), + appVid=num(define(f'apps/{cb.HIL_APP}/src/usb_descriptors.c', 'USB_VID'), 'USB_VID'), + appPid=num(define(f'apps/{cb.HIL_APP}/src/usb_descriptors.c', 'USB_PID'), 'USB_PID'), + family=int(cmake_var('ports/espressif/family.cmake', f'UF2_FAMILY_ID_{board["device"]}'), 16), + flashCache=product(define('ports/espressif/boards/board_flash.c', 'FLASH_CACHE_SIZE'), 'FLASH_CACHE_SIZE')) + return facts + + def ram_origin(ld): m = re.search(r'm_data\s*\(RW\)\s*:\s*ORIGIN\s*=\s*(0x[0-9a-fA-F]+)', (ROOT / ld).read_text()) if not m: @@ -235,13 +288,18 @@ def register_layout(header, periph, member): def apply_layout(board, facts): """The install and app spans, from the candidate's source: the bootloader from the flash base up to the app start, the app from there to the end of the board's flash, all inside the device's programming limits.""" + if board.get('flasher') == 'esptool': + try: + lay = esp.layout(facts, board['device']) + except esp.Refused as e: + raise Refused(str(e)) + board.update(regions=lay, app=lay['app']) + return base, start, end = facts['flashBase'], facts['appStart'], facts['flashBase'] + facts['flashSize'] floor, ceiling = FLASH_LIMITS[board['device']] if not floor <= base < start < end <= ceiling: raise Refused(f'{board["name"]}: flash [{base:#x}, {end:#x}) with app start {start:#x} leaves the ' f'{board["device"]} flash [{floor:#x}, {ceiling:#x}) or is out of order') - if start % ZERO_PAGE: - raise Refused(f'{board["name"]}: app start {start:#x} is not {ZERO_PAGE}-byte aligned') board.update(install=(base, start), app=(start, end)) @@ -347,6 +405,7 @@ def make_uf2(img, base, family, workdir): # ------------------------------------------------------------------ processes CHILDREN = [] +CONTAINERS = [] # docker containers a flash started, by name: outside every child's session REGISTERING = {'on': False, 'signum': None} @@ -385,36 +444,87 @@ def spawn(cmd, log, timeout, cwd=None): return p.returncode, out -def reap(p): - try: - os.killpg(p.pid, signal.SIGKILL) - except ProcessLookupError: - pass +def proc_stat(pid): + """proc(5) stat fields from the third (state) on, or None for a process that is gone or a zombie.""" try: + fields = Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split() + except (OSError, IndexError): + return None + return fields if fields and fields[0] != 'Z' else None + + +def session(sid): + """The live processes of session `sid` (stat field 6): a child started with start_new_session and everything it + started, a process ninja moved into a group of its own included.""" + return [int(d.name) for d in Path('/proc').iterdir() + if d.name.isdigit() and (st := proc_stat(d.name)) and st[3] == str(sid)] + + +def reap(p): + """SIGKILL to the child's whole session; the members still alive after KILL_WAIT (D state on a stuck USB + device), empty once the session is gone.""" + for pid in session(p.pid): + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + with contextlib.suppress(subprocess.TimeoutExpired): p.wait(KILL_WAIT) - except subprocess.TimeoutExpired: - return False - if p in CHILDREN: + end = time.monotonic() + KILL_WAIT + while (left := session(p.pid)) and time.monotonic() < end: + time.sleep(0.1) + if not left and p in CHILDREN: # tracked while any of its session lives, for cleanup to see CHILDREN.remove(p) - return True + return left def proc_start(pid): - """The start time of `pid` in clock ticks (proc(5) stat field 22), which tells a reused pid apart.""" + """The start time of a live `pid` in clock ticks (proc(5) stat field 22), which tells a reused pid apart.""" + st = proc_stat(pid) + return int(st[19]) if st and len(st) > 19 else None + + +def container_present(name): + r = subprocess.run(['docker', 'ps', '-aq', '-f', f'name=^{name}$'], capture_output=True, text=True, timeout=30) + if r.returncode: + raise StepFail(f'docker ps failed: {r.stderr.strip()[-200:]}') + return bool(r.stdout.strip()) + + +def container_gone(name): + """`docker rm -f` (a running, created or exited container alike), then whether docker no longer lists it; a + docker error is not an answer, so it counts as still there.""" try: - return int(Path(f'/proc/{pid}/stat').read_text().rsplit(')', 1)[1].split()[19]) - except (OSError, IndexError, ValueError): - return None + subprocess.run(['docker', 'rm', '-f', name], capture_output=True, timeout=30) + end = time.monotonic() + 2 * KILL_WAIT + while (left := container_present(name)) and time.monotonic() < end: + time.sleep(0.2) + return not left + except (StepFail, OSError, subprocess.SubprocessError): + return False def stuck_children(): - """Children that survived SIGKILL, identified so a later recovery can tell they are still alive.""" - return [{'pid': p.pid, 'start': proc_start(p.pid), 'cmd': [str(c) for c in p.args][:3]} - for p in list(CHILDREN) if not reap(p)] + """Processes of the children's sessions, and containers a flash created, that survived SIGKILL, identified so + a later recovery can tell they are still alive. The launchers go first, so no `docker` child can create or start + a container after its name is checked.""" + stuck = [] + for p in list(CHILDREN): + for pid in reap(p): + try: + cmd = Path(f'/proc/{pid}/cmdline').read_bytes().split(b'\0')[:3] + except OSError: + cmd = [] + stuck.append({'pid': pid, 'start': proc_start(pid), 'cmd': [c.decode(errors='replace') for c in cmd]}) + for name in list(CONTAINERS): + if container_gone(name): + CONTAINERS.remove(name) + else: + stuck.append({'container': name}) + return stuck def still_alive(survivors): - return [s for s in survivors if s.get('start') is not None and proc_start(s['pid']) == s['start']] + return [s for s in survivors if (not container_gone(s['container']) if 'container' in s else + s.get('start') is not None and proc_start(s['pid']) == s['start'])] # ------------------------------------------------------------------ probe @@ -429,6 +539,38 @@ def died_at_probe_connect(code, out, probe): 'Connecting to target' not in out +def stopped_before_target(code, out, probe): + """A session recognised as ending before any target connect, so it changed nothing: no probe with the serial + (bench: "Connecting to J-Link ...FAILED", then nothing more), or the crash at the probe connect. Any other + failure, an empty transcript included, may have reached the target.""" + no_probe = re.search(r'^Connecting to J-Link \.\.\.FAILED', out, re.M) and 'Connecting to target' not in out + return bool(no_probe) or died_at_probe_connect(code, out, probe) + + +def judge_jlink_target(code, out, board, hex_path): + """The tinyuf2-jlink transcript, through CMake and Ninja: JLinkExe can exit 0 with every command FAILED (no probe + with the serial), so success is read from the transcript: this HEX downloaded, programmed or already matching, + then O.K. and the script completed, with no failure line anywhere.""" + judged = re.sub(BENIGN[board['device']], '', out) if board['device'] in BENIGN else out + m = re.search(rf'^Downloading file \[{re.escape(str(hex_path))}\]\.\.\.$(?:\n.*)*?\nJ-Link: Flash download: Bank \d+ @ ' + r'0x[0-9A-F]+: (Skipped\. Contents already match|\d+ ranges? affected \(\d+ bytes\))$(?:\n.*)*?\nO\.K\.$', + judged, re.M) + if code or not m or 'Script processing completed.' not in judged or re.search(JLINK_ERRORS + '|FAILED', judged): + raise StepFail(f'tinyuf2-jlink failed (exit {code})', {'tail': out.strip().splitlines()[-5:], + 'beforeTarget': stopped_before_target(code, out, board['probe'])}) + return {'flashDownload': m[1]} + + +def judge_esp_target(code, out, segments): + """ESP-IDF's flash target, through CMake and Ninja in CI's image: esptool's write of every segment at its offset, + each hash-verified, then the hard reset (bench transcript, esptool v4.8.1 in espressif/idf:v5.3.2).""" + wrote = sorted(int(a, 16) for a in re.findall(r'^Wrote \d+ bytes \(\d+ compressed\) at 0x([0-9a-f]{8}) in ', out, re.M)) + if code or wrote != sorted(off for off, _ in segments) or out.count('Hash of data verified.') != len(segments) or \ + 'Hard resetting via RTS pin' not in out: + raise StepFail(f'the flash target failed (exit {code})', {'tail': out.strip().splitlines()[-5:]}) + return {'wrote': [hex(a) for a in wrote]} + + class Probe: def __init__(self, board, workdir, log): self.b, self.dir, self.log = board, workdir, log @@ -450,9 +592,9 @@ def run(self, commands, judge=True): code, out = spawn(cmd, self.log, PROBE_TIMEOUT) judged = re.sub(BENIGN[self.b['device']], '', out) if self.b['device'] in BENIGN else out # a crash fails even a reset session (judge=False): its commands may not have run - if code < 0 or judge and (code or re.search(r'Verify failed|Could not|Cannot connect|ERROR|ResetTarget failed|' - r'Failed to halt|CPU is not halted', judged)): - raise StepFail(f'JLinkExe failed (exit {code})', {'tail': out.strip().splitlines()[-5:]}) + if code < 0 or judge and (code or re.search(JLINK_ERRORS, judged)): + raise StepFail(f'JLinkExe failed (exit {code})', {'tail': out.strip().splitlines()[-5:], + 'beforeTarget': stopped_before_target(code, out, self.b['probe'])}) return out def mem32(self, addrs): @@ -473,6 +615,97 @@ def write32(self, addr, value): return hex(value) +def esp_port(uid): + """The tty of the USB-UART bridge with this serial: exactly one, never a guessed /dev/ttyUSBn.""" + hits = sorted(glob.glob(f'/dev/serial/by-id/*_{glob.escape(uid)}-if*')) + if len(hits) != 1: + raise StepFail(f'{len(hits)} serial ports for bridge {uid}: {hits}') + return os.path.realpath(hits[0]) + + +def child_json(cmd, log, timeout, what): + """The JSON last line of a helper child (esp_session.py, usb_dfu.py, fat-read); a missing line, a non-zero + exit or an `ok` other than true fails the step with the helper's `reason` (else `error`) and the output tail.""" + code, out = spawn(cmd, log, timeout) + tail = out.strip().splitlines()[-5:] + try: + res = json.loads(tail[-1]) + except (IndexError, ValueError): + res = None + if not isinstance(res, dict): + raise StepFail(f'{what} ended without its JSON (exit {code})', {'tail': tail}) + if code or res.get('ok') is not True: + raise StepFail(f'{what}: {res.get("reason") or res.get("error")}', {'tail': tail, 'error': res.get('error')}) + return res + + +class Esptool: + """esptool sessions on one board, each one connection in its own child (esp_session.py).""" + def __init__(self, board, workdir, log): + self.b, self.dir, self.log = board, workdir, log + + def run(self, ops, after): + spec = self.dir / f'esp-{time.monotonic_ns()}.json' + spec.write_text(json.dumps(ops)) + return child_json([sys.executable, '-B', ESP_SESSION, '--port', esp_port(self.b['probe']), '--chip', self.b['device'], + '--baud', str(ESP_BAUD), '--after', after, spec], self.log, ESP_TIMEOUT, 'esptool session')['results'] + + +def preserved_ops(regions): + return [{'op': 'md5', 'addr': lo, 'size': hi - lo} for lo, hi in regions.values()] + + +def preserved_md5(regions, results): + md5 = [r['md5'] for r in results if r['op'] == 'md5'] + return dict(zip(regions, md5)) + + +def check_preserved(before, after, by): + if before != after: + raise StepFail(f'a preserved region changed by the end of the {by}', {'before': before, 'after': after}) + + +def esp_verify(tool, board, segments, workdir, snapshot): + """After the flash target: every segment read back byte for byte in one connection of the runner's, then one + reset. The preserved regions' MD5s must equal `snapshot`, taken before the install.""" + reads = [(off, path, workdir / f'readback-{off:x}.bin') for off, path in segments] + keep = board['regions']['preserved'] + ops = [{'op': 'read', 'addr': off, 'size': path.stat().st_size, 'file': str(rb)} for off, path, rb in reads] + \ + preserved_ops(keep) + after = preserved_md5(keep, tool.run(ops, 'hard-reset')) + check_preserved(snapshot, after, 'install') + for off, path, rb in reads: + if rb.read_bytes() != path.read_bytes(): + raise StepFail(f'{path.name} does not read back as written', {'addr': hex(off)}) + return {'preserved': after, 'writes': [write_span(path.name, off, path.stat().st_size) for off, path in segments]} + + +def write_span(what, addr, length, block=0x1000): + """A write's payload span and the `block`-aligned span covering it.""" + lo, hi = addr // block * block, -(-(addr + length) // block) * block + return {'what': what, 'span': [hex(addr), hex(addr + length)], 'blocks': [hex(lo), hex(hi)]} + + +def report_snapshot(row, keep): + """The preserved-region MD5s a failed esptool run took before its install wrote, from its report row, over the + regions `keep` this checkout preserves. A board whose flash was written always has them (the snapshot precedes + the writes), so a touched row without them is refused, never recovered blind.""" + ev = row.get('snapshot') or {} + regions, md5 = ev.get('preservedRegions'), ev.get('preserved') + if regions is None and md5 is None and (row.get('cleanup') or {}).get('state') == 'untouched': + return None + if regions is None or md5 is None: + raise Refused('the failed run touched the board but its report has no preserved-region snapshot to check') + ok = isinstance(regions, dict) and isinstance(md5, dict) and regions and set(regions) == set(md5) and all( + isinstance(r, list) and len(r) == 2 and all(type(v) is int for v in r) and 0 <= r[0] < r[1] < 1 << 32 and + isinstance(md5[n], str) and re.fullmatch(r'[0-9a-f]{32}', md5[n]) for n, r in regions.items()) + if not ok: + raise Refused(f'the failed run\'s preserved-region snapshot is malformed: {regions} {md5}') + if {n: tuple(r) for n, r in regions.items()} != keep: + raise Refused(f'the failed run preserved {regions}; this checkout preserves {keep}') + return md5 + + def after(out, command): """The output chunks following each echo of `command` in a JLinkExe -CommandFile transcript; the transcript shape is bench-checked at commissioning.""" @@ -512,14 +745,24 @@ def block_device(usb_path): return None -def wait_for(fn, timeout, what): +def wait_for(fn, timeout, what, fail=None): end = time.monotonic() + timeout while time.monotonic() < end: v = fn() if v: return v time.sleep(0.2) - raise StepFail(f'{what} not seen within {timeout}s') + raise (fail or StepFail)(f'{what} not seen within {timeout}s') + + +def stays_away(fn, seconds, what, ev): + end = time.monotonic() + seconds + while True: + if fn(): + raise StepFail(what, ev) + if time.monotonic() >= end: # sampled once more after the last sleep: the window's end is checked too + return + time.sleep(min(0.2, max(0.0, end - time.monotonic()))) def exclusion_configured(facts, serial): @@ -566,9 +809,24 @@ def check_exclusive(dev): class Fat: """Read-only FAT12/16 reader for the ghostfat volume. Its geometry and chains come from the firmware under test, so every one is bounded; it runs in its own process (fat_read) so a stuck USB read cannot - hold the process that owns cleanup.""" + hold the process that owns cleanup. It keeps one descriptor for its life: the device's last close drops its + page cache (Linux block/bdev.c blkdev_put_whole -> kill_bdev), so a reader that reopened it per read would + cost a USB read per cluster (bench: the nonce block 4 MiB into CURRENT.UF2 took >20s).""" def __init__(self, dev): - self.dev = dev + self.dev, self.fd = dev, os.open(dev, os.O_RDONLY) + try: + self._geometry() + except BaseException: + os.close(self.fd) + raise + + def __enter__(self): + return self + + def __exit__(self, *exc): + os.close(self.fd) + + def _geometry(self): bs = self.read(0, 512) self.bps, self.spc, self.res, self.nfat = struct.unpack_from('> 4 if c & 1 else v & 0xFFF - def file(self, name, length=None): + def file(self, name, length=None, offset=0): + """`length` bytes of `name` from `offset`, a whole number of clusters in; the clusters before it are + walked in the FAT, not read.""" ents = self.entries() if name not in ents: raise ValueError(f'{name} not on the volume (found {sorted(ents)})') c, size = ents[name] - want, out, csize, seen = min(size, length or size), b'', self.spc * self.bps, set() + csize, seen = self.spc * self.bps, set() + if offset % csize or offset > size: + raise ValueError(f'{name}: offset {offset:#x} is not a cluster boundary inside its {size} bytes') + want, out, skip = min(size - offset, length or size), b'', offset // csize while len(out) < want: if not 2 <= c < (0xFFF8 if self.fat16 else 0xFF8) or c in seen: raise ValueError(f'{name}: cluster chain broken at {c:#x} after {len(out)} of {want} bytes') seen.add(c) - out += self.read(self.data + (c - 2) * csize, min(csize, want - len(out))) + if skip: + skip -= 1 + else: + out += self.read(self.data + (c - 2) * csize, min(csize, want - len(out))) c = self.next_cluster(c) return out, size -def fat_read(dev, name, length, log): +def fat_read(dev, name, length, log, offset=0): """{'entries': {...}, 'data': bytes} of `name` on `dev`, read by a bounded child process.""" - code, out = spawn([sys.executable, '-B', __file__, 'fat-read', '--dev', dev, '--file', name, - '--length', str(length)], log, USB_TIMEOUT) try: - r = json.loads(out.strip().splitlines()[-1]) - except (ValueError, IndexError): - raise StepFail(f'fat-read of {name} gave no result (exit {code})', {'tail': out.strip().splitlines()[-3:]}) - if 'error' in r: - raise StepFail(f'{name} on {dev}: {r["error"]}') + r = child_json([sys.executable, '-B', __file__, 'fat-read', '--dev', dev, '--file', name, '--length', str(length), + '--offset', str(offset)], log, USB_TIMEOUT, name) + except StepFail as e: + e.evidence['dev'] = dev + raise return {'entries': r['entries'], 'data': bytes.fromhex(r['data']), 'size': r['size']} def fat_read_main(args): try: - fat = Fat(args.dev) - data, size = fat.file(args.file, args.length) if args.length else (b'', fat.entries()[args.file][1]) - print(json.dumps({'entries': fat.entries(), 'data': data.hex(), 'size': size})) + with Fat(args.dev) as fat: + entries = fat.entries() + data, size = fat.file(args.file, args.length, args.offset) if args.length else (b'', entries[args.file][1]) + print(json.dumps({'ok': True, 'entries': entries, 'data': data.hex(), 'size': size})) except (OSError, ValueError, KeyError, struct.error) as e: - print(json.dumps({'error': f'{type(e).__name__}: {e}'})) + # reason: the same failure reads the same in every run; an OSError's text names the device node + reason = f'{type(e).__name__}: {e.strerror}' if isinstance(e, OSError) and e.strerror else f'{type(e).__name__}: {e}' + print(json.dumps({'ok': False, 'error': f'{type(e).__name__}: {e}', 'reason': reason})) return 0 @@ -742,9 +1005,12 @@ def manual_holder(helper, board, reason): prove_hold(helper, board, pid, reason=reason) -def probe_users(probe): - """Processes that have this probe open by serial (a JLinkExe or GDB server from an earlier session).""" - users = [] +def probe_users(board): + """Processes using this board's probe: for J-Link one opened by serial (a JLinkExe or GDB server from an + earlier session), for esptool any process holding the bridge's tty open.""" + if board['flasher'] == 'esptool': + return tty_users(esp_port(board['probe'])) + probe, users = board['probe'], [] for d in Path('/proc').iterdir(): try: cmd = (d / 'cmdline').read_bytes().split(b'\0') @@ -756,6 +1022,28 @@ def probe_users(probe): return users +def tty_users(dev): + users = [] + for d in Path('/proc').iterdir(): + if not d.name.isdigit() or int(d.name) == os.getpid(): + continue + try: + fds = list((d / 'fd').iterdir()) + except OSError: + continue + if any(readlink(fd) == dev for fd in fds): + users.append(int(d.name)) + return users + + +def readlink(fd): + """A /proc fd's target, or None once it has closed: one closing fd must not hide the process's others.""" + try: + return os.readlink(fd) + except OSError: + return None + + def holder_record(fh, board, detail, survivors): fh.truncate(0) fh.seek(0) @@ -823,7 +1111,7 @@ def hand_off(fh, board, detail, survivors=()): # ------------------------------------------------------------------ the battery -def verify_record(record_path, board, facts): +def verify_record(record_path, board, facts, workdir): try: rec = json.loads(Path(record_path).read_text()) except (OSError, ValueError) as e: @@ -832,9 +1120,19 @@ def verify_record(record_path, board, facts): if not rec.get('pass') or len(rows) != 1 or rows[0].get('status') != 'ok' or rows[0].get('removed'): raise Refused(f'{record_path}: not a passing --shared build of {board["name"]}') row, info = rows[0], rec.get('input', {}) - if row.get('backend') != 'cmake': - raise Refused(f'{record_path}: a {row.get("backend")} build; the HIL takes cmake builds only ' - f'(check_build.py --board {board["name"]} --backend cmake --shared)') + want = 'idf' if board.get('flasher') == 'esptool' else 'cmake' + if row.get('backend') != want: + raise Refused(f'{record_path}: a {row.get("backend")} build; the HIL takes {want} builds only for this board ' + f'(check_build.py --board {board["name"]}{" --backend cmake" if want == "cmake" else ""} --shared ' + f'--hil-config )') + if want == 'idf': + try: + image = f'espressif/idf:{cb.ci_idf_version()}' + except cb.Fail as e: + raise Refused(str(e)) + idf = rec.get('idf') or {} + if idf.get('mode') != 'docker' or idf.get('image') != image: + raise Refused(f'{record_path}: an IDF build from {idf.get("mode")} {idf.get("image")}; the HIL takes CI\'s {image} only') try: now = cb.identity({board['port']}) except (cb.Fail, OSError) as e: @@ -848,23 +1146,92 @@ def verify_record(record_path, board, facts): if bad: raise Refused(f'build inputs are not clean: {bad} (commit, or rebuild from a clean checkout)') root = cb.ROOT / row['buildDir'] - files, digests = {}, {} - for p in cb.expected_artifacts(board['port'], board['name'], 'cmake', root): - kind = p.suffix[1:] - if kind not in ('hex', 'bin'): - continue - digests[kind] = sha256(p) - if digests[kind] is None: - raise Refused(f'{p}: missing or unreadable; rebuild with --shared') - if row.get('sha256', {}).get(str(p.relative_to(root))) != digests[kind]: - raise Refused(f'{p}: hash differs from the build record') - files[kind] = p - if files.keys() != {'hex', 'bin'}: - raise Refused(f'{record_path}: the build of {board["name"]} names no {" or ".join(sorted({"hex", "bin"} - files.keys()))}') - return files, {'path': str(files['bin']), 'sha256': digests['bin'], 'buildRecord': str(record_path), + flash = admit_flash(record_path, rec, row, root, board) + # the runner reads its own copies of the BIN, segments and blinky: the build dir's lock covers the flash target + # only, and a rebuild may replace the dir after it + copy = workdir / 'record' + if want == 'idf': + bin_rel = 'tinyuf2.bin' + copied(root, (*esp.RECORD_FILES, *flash['sha256']), row, copy) + files = esp_record(record_path, copy, board, facts) + else: + bin_rel, blinky_rel = CMAKE_RECORD_FILES + copies = copied(root, CMAKE_RECORD_FILES, row, copy) + files = dict(check_blinky(copies[blinky_rel], board['app'], facts), bin=copies[bin_rel], hex=root / 'tinyuf2.hex') + files['flash'] = flash + return files, {'path': str(root / bin_rel), 'sha256': sha256(files['bin']), 'buildRecord': str(record_path), 'defines': rec.get('defines', []), 'version': embedded_version(files['bin'])} +def admit_flash(record_path, rec, row, root, board): + """The record's flash target, bound by check_build --hil-config to this roster flasher, with every file it reads + hash-checked; the install runs that target, under the build dir's lock.""" + fl = row.get('flash') or {} + target, inputs = cb.FLASH[board['flasher']] + want = {'name': board['flasher'], 'uid': board['probe'], 'device': board['device']} + if (fl.get('flasher'), fl.get('target'), fl.get('inputs')) != (want, target, list(inputs)): + raise Refused(f'{record_path}: its flash target is not bound to this roster flasher {want} (rebuild it with ' + f'check_build.py --board {board["name"]} ... --shared --hil-config )') + idf = rec.get('idf') or {} + return {'root': root, 'binding': fl, 'sha256': recorded(root, inputs, row), 'image': idf.get('image'), + 'ninja': functools.partial(cb.ninja_in, row['backend'], idf.get('mode'), idf.get('image'))} + + +def recorded(root, rels, row): + """Each file's sha256, which must be the build record's.""" + for rel in rels: + recorded_bytes(root, rel, row) + return {rel: row['sha256'][rel] for rel in rels} + + +def recorded_bytes(root, rel, row): + try: + data = (root / rel).read_bytes() + except OSError: + data = None + if data is None or row.get('sha256', {}).get(rel) != hashlib.sha256(data).hexdigest(): + raise Refused(f'{root / rel}: missing, unreadable or changed since the build record') + return data + + +def copied(root, rels, row, dest): + """Each file copied to the same relative path under `dest`, from bytes that hash as the build record's.""" + out = {} + for rel in rels: + data = recorded_bytes(root, rel, row) + out[rel] = dest / rel + out[rel].parent.mkdir(parents=True, exist_ok=True) + out[rel].write_bytes(data) + return out + + +def check_blinky(path, span, facts): + """The record's apps/blinky.uf2, which cleanup copies through TinyUF2: one family, numbered under one + numBlocks, every block inside `span` (the app region, in the UF2's own addresses).""" + try: + blocks = parse_uf2(Path(path).read_bytes()) + except (OSError, ValueError) as e: + raise Refused(f'{path}: {e}') + for i, bl in enumerate(blocks): + if (bl['no'], bl['total'], bl['family']) != (i, len(blocks), facts['family']) or not bl['flags'] & UF2_FLAG_FAMILY \ + or not span[0] <= bl['addr'] <= bl['addr'] + bl['size'] <= span[1]: + raise Refused(f'{path}: block {i} is not block {i} of {len(blocks)} for family {facts["family"]:#x} ' + f'inside [{span[0]:#x}, {span[1]:#x})') + if not blocks: + raise Refused(f'{path}: no UF2 blocks') + return {'blinky': Path(path), 'blinkyBlocks': len(blocks)} + + +def esp_record(record_path, root, board, facts): + """An IDF record's segments, from `root`, the run's copy of every file the record binds.""" + try: + segments = esp.admit(root, board, facts, board['regions']) + except (esp.Refused, OSError, ValueError) as e: + raise Refused(f'{record_path}: {e}') + return {'segments': segments, 'bin': root / 'tinyuf2.bin', **esp_app_chunks(root / esp.APP_UF2, facts, board), + **check_blinky(root / esp.BLINKY_UF2, (0, board['app'][1] - board['app'][0]), facts)} + + def sha256(p): try: return hashlib.sha256(Path(p).read_bytes()).hexdigest() @@ -873,14 +1240,10 @@ def sha256(p): def check_bootloader_bin(bin_path, board, facts, what): - """Every span a bootloader install writes: the BIN from the install start inside `install` and the zero - page at the start of `app` inside it, both within the device's flash limits; and the BIN is this + """The BIN from its flash address (`binBase`) inside `install`, within the device's flash limits, and this board's (its UF2_BOARD_ID).""" - install, app = board['install'], board['app'] - size = Path(bin_path).stat().st_size - limits = FLASH_LIMITS[board['device']] - check_ranges({install[0]: 0, install[0] + size - 1: 0}, [install], limits, what) - check_ranges({app[0]: 0, app[0] + board['page'] - 1: 0}, [app], limits, f'{what} zero page') + start, size = facts['binBase'], Path(bin_path).stat().st_size + check_ranges({start: 0, start + size - 1: 0}, [board['install']], FLASH_LIMITS[board['device']], what) if facts['boardId'].encode() not in Path(bin_path).read_bytes(): raise Refused(f'{what}: {bin_path} does not carry board id {facts["boardId"]!r}; not this board\'s bootloader') @@ -891,8 +1254,9 @@ def __init__(self, board, facts, workdir, fail_at=None, commissioning=False): self.log = workdir / 'hil.log' self.probe = Probe(board, workdir, self.log) self.steps, self.usb_path, self.touched, self.at = [], None, False, 'setup' - self.zero = workdir / 'zero.bin' - self.zero.write_bytes(b'\0' * board['page']) + # where CURRENT.UF2 starts and the flash size INFO_UF2.TXT reports + self.app_start, self.dfu_flash = facts.get('appStart'), facts['flashSize'] + self.dfu_claim = 'DFU observed after the install' def step(self, name, fn): self.at = name # an abort names the step it cut short without recording it as a failing step @@ -906,30 +1270,73 @@ def step(self, name, fn): raise self.steps.append({'step': name, 'pass': True, 'evidence': ev, 'detail': ''}) - def identity(self): - v = self.probe.mem32([SCB_CPUID, DHCSR]) - return {'cpuid': f'{v[SCB_CPUID]:#010x}', 'dhcsr': f'{v[DHCSR]:#010x}'} - - def program(self, bin_path, base, app_start): - """One image from `base`: the bootloader BIN, 0xFF up to app_start, then a zero page, written by a single - loadbin and read back, so no second write lands in the bootloader's erase block; the zeroed reset vector - makes the app invalid by construction, so the next boot stays in DFU. No range erase is ever issued. - Callers have range-checked both spans (check_bootloader_bin).""" - data = Path(bin_path).read_bytes() - if base + len(data) > app_start: - raise StepFail(f'{bin_path} [{base:#x}, {base + len(data):#x}) overlaps the app start {app_start:#x}') - image = self.dir / 'program.bin' - image.write_bytes(data + b'\xff' * (app_start - base - len(data)) + bytes(self.b['page'])) - self.touched = True - self.installed = (bin_path, base) - self.probe.run(['h', f'loadbin {image},{base:#x}', f'verifybin {image},{base:#x}', 'r', 'g']) + def flash_target(self, flash, cmd, timeout, judge): + """The record's flash target, under its build dir's lock: the inputs still hash as recorded and a dry run shows + the target runs nothing but its flasher (cb.flash_ready); after it, the inputs are hashed again. Touched from the + target's start; only a transcript recognised as ending before any target connect leaves the board as it was.""" + root, fl = flash['root'], flash['binding'] + with contextlib.ExitStack() as held: + try: + held.enter_context(cb.build_lock(root, wait=False)) + except BlockingIOError: + raise StepFail(f'{root.name} is in use by a build or another flash; nothing was flashed') + self.check_inputs(flash, 'before the flash') + try: + cb.flash_ready(root, fl, flash['ninja']) + except cb.Fail as e: + raise StepFail(str(e)) + was, self.touched = self.touched, True + code, out = spawn(cmd, self.log, timeout) + try: + ev = judge(code, out) + except StepFail as e: + if e.evidence.get('beforeTarget'): + self.touched = was + raise + self.check_inputs(flash, 'while flashing', ev) + return ev + + @staticmethod + def check_inputs(flash, when, ev=None): + changed = [rel for rel, digest in flash['sha256'].items() if sha256(flash['root'] / rel) != digest] + if changed: + raise StepFail(f'flash inputs changed {when}: {changed}', ev or {}) + + def dfu_after_install(self, version): + """DFU after an install: J-Link's download erased the bootloader's whole 64 KiB block on the bench, the app start + with it (SKILL.md Limits), so DFU comes up; should enumeration time out instead, as when a valid app boots, the + board's commissioned retained-register re-entry brings DFU up. Any other failure fails as it is.""" + try: + return self.dfu(version) + except NotEnumerated as e: + if self.b['reentry'] != 'retained-register': + raise StepFail(f'{e}, and {self.b["name"]} has no retained-register re-entry to try', e.evidence) + return self.retained_reentry(version, 'DFU did not enumerate after the install; the retained-register re-entry ' + 'brought it up') + + def retained_reentry(self, version, claim): + f = self.f + self.probe.write32(f['dblTapReg'], f['dblTapMagic']) + # SYSRESETREQ may drop the debug connection mid-write: its exit is not judged, enumeration is; + # no probe session opens again until DFU is observed + tail = self.probe.run([f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) + ev = dict(self.dfu(version), dfuCameUp=True, resetTail=tail.strip().splitlines()[-3:], claim=claim) + try: + left = self.probe.mem32([f['dblTapReg']])[f['dblTapReg']] + except StepFail as e: + e.evidence = {**ev, **e.evidence} + raise + ev.update(dblTapRegAfter=hex(left), magicConsumed=not left) + if left: + raise StepFail('the double-tap register was not consumed', ev) + return ev def dfu(self, version): f = self.f path = wait_for(lambda: usb_device(f['vid'], f['pid'], self.b['uid']), USB_TIMEOUT, - f'{f["vid"]:04x}:{f["pid"]:04x} serial {self.b["uid"]}') + f'{f["vid"]:04x}:{f["pid"]:04x} serial {self.b["uid"]}', NotEnumerated) if self.usb_path and path != self.usb_path: - raise StepFail(f'came back on USB port {path}, not {self.usb_path}') + raise StepFail('came back on another USB port', {'usb': path, 'expected': self.usb_path}) self.usb_path = path dev = wait_for(lambda: block_device(path), USB_TIMEOUT, f'the block device of USB {path}') udev_wait = check_exclusive(dev) @@ -937,27 +1344,37 @@ def dfu(self, version): lines = info['data'].decode('ascii', 'replace').splitlines() want = f'TinyUF2 Bootloader {version}' if not lines or lines[0] != want or f'Board-ID: {f["boardId"]}' not in lines or \ - f'Flash Size: 0x{f["flashSize"]:08X} bytes' not in lines: + f'Flash Size: 0x{self.dfu_flash:08X} bytes' not in lines: raise StepFail('INFO_UF2.TXT does not match this build', {'info': lines, 'want': want}) size = (info['entries'].get('CURRENT.UF2') or [None, None])[1] - if size != f['flashSize'] // 256 * 512: - raise StepFail(f'CURRENT.UF2 is {size} bytes, not {f["flashSize"] // 256 * 512}') + if size != self.dfu_flash // 256 * 512: + raise StepFail(f'CURRENT.UF2 is {size} bytes, not {self.dfu_flash // 256 * 512}') return {'usb': path, 'dev': dev, 'udevWaitS': udev_wait, 'info': lines[0], 'currentUf2': size, - 'claim': 'app rejected (reset vector 0); DFU observed'} + 'claim': self.dfu_claim} - def current_uf2(self, length): - """CURRENT.UF2's first blocks, each checked for its place: block i maps to app_start + 256*i.""" + def current_uf2(self, length, offset=0): + """CURRENT.UF2's blocks from byte `offset`, each checked for its place: block i maps to app_start + 256*i.""" dev = block_device(self.usb_path) - data = fat_read(dev, 'CURRENT.UF2', length, self.log)['data'] + data = fat_read(dev, 'CURRENT.UF2', length, self.log, offset)['data'] try: blocks = parse_uf2(data) except ValueError as e: raise StepFail(f'CURRENT.UF2 is not UF2: {e}', {'head': data[:32].hex()}) - for i, bl in enumerate(blocks): - if (bl['no'], bl['addr'], bl['size']) != (i, self.f['appStart'] + 256 * i, 256): - raise StepFail(f'CURRENT.UF2 block {i} is {bl["no"]} at {bl["addr"]:#x}, not at {self.f["appStart"] + 256 * i:#x}') + for i, bl in enumerate(blocks, offset // 512): + if (bl['no'], bl['addr'], bl['size']) != (i, self.app_start + 256 * i, 256): + raise StepFail(f'CURRENT.UF2 block {i} is {bl["no"]} at {bl["addr"]:#x}, not at {self.app_start + 256 * i:#x}') return b''.join(bl['data'] for bl in blocks) + def copy(self, uf2, blocks, nonce=None): + dev = block_device(self.usb_path) + code, out = spawn(['mcopy', '-o', '-i', dev, uf2, '::APP.UF2'], self.log, 60) + if code: + raise StepFail('mcopy failed', {'tail': out.strip().splitlines()[-5:]}) + spawn(['sync'], self.log, 60) + f = self.f + wait_for(lambda: usb_device(f['vid'], f['pid'], self.b['uid']) is None, USB_TIMEOUT, 'the DFU device leaving') + return {'blocks': blocks, **({'nonce': nonce.hex()} if nonce else {})} + def run(self, files, version): f = self.f nonce = os.urandom(16) @@ -970,22 +1387,12 @@ def run(self, files, version): raise Refused('generated UF2 has the wrong family') payload.update({bl['addr'] + i: x for i, x in enumerate(bl['data'])}) check_ranges(payload, [self.b['app']], FLASH_LIMITS[self.b['device']], 'fixture UF2') - self.step('identity', self.identity) - self.step('install', lambda: self.program(files['bin'], self.b['install'][0], f['appStart']) or - {'zeroPage': f'{f["appStart"]:#x}', 'verified': True}) + self.step('install', lambda: self.install(files)) # cleared once the bootloader has run board_init (the register's clock), so a magic a failed reenter left # cannot send the copied app back to DFU - self.step('dfu', lambda: dict(self.dfu(version), dblTapReg=self.probe.write32(f['dblTapReg'], 0))) - - def copy(): - dev = block_device(self.usb_path) - code, out = spawn(['mcopy', '-o', '-i', dev, app_uf2, '::APP.UF2'], self.log, 60) - if code: - raise StepFail(f'mcopy failed: {out.strip()[-200:]}') - spawn(['sync'], self.log, 60) - wait_for(lambda: usb_device(f['vid'], f['pid'], self.b['uid']) is None, USB_TIMEOUT, 'the DFU device leaving') - return {'blocks': len(blocks), 'nonce': nonce.hex()} - self.step('copy', copy) + self.step('dfu', lambda: dict(self.dfu_after_install(version), dblTapReg=self.probe.write32(f['dblTapReg'], 0))) + + self.step('copy', lambda: self.copy(app_uf2, len(blocks), nonce)) def app_ran(): # the bootloader writes DBL_TAP_MAGIC as it boots: a probe connect that resets inside that window @@ -993,9 +1400,8 @@ def app_ran(): window = 2 * f['dblTapDelayMs'] / 1000 + 0.5 time.sleep(window) ev = dict(self.observe_app(app_bin), waitedS=window) - time.sleep(QUIET_TIME) - if usb_device(f['vid'], f['pid'], self.b['uid']): - raise StepFail('the DFU device came back while the app should run', ev) + stays_away(lambda: usb_device(f['vid'], f['pid'], self.b['uid']), QUIET_TIME, + 'the DFU device came back while the app should run', ev) return ev self.step('app-ran', app_ran) @@ -1008,34 +1414,14 @@ def reenter(): raise def entered(ev): - method = ev['path'] - if method == 'retained-register': - if self.commissioning: - ev.update(self.control(app_bin)) - self.probe.write32(f['dblTapReg'], f['dblTapMagic']) - # SYSRESETREQ may drop the debug connection mid-write: its exit is not judged, enumeration is; - # no probe session opens again until DFU is observed - tail = self.probe.run([f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) - else: - bin_path, base = self.installed - tail = self.probe.run(['h', f'loadbin {self.zero},{f["appStart"]:#x}', f'verifybin {self.zero},{f["appStart"]:#x}', - f'verifybin {bin_path},{base:#x}', 'r', 'g']) - ev['bootloaderVerified'] = True - ev.update(self.dfu(version), dfuCameUp=True, resetTail=tail.strip().splitlines()[-3:]) - ev['claim'] = 'the write/reset sequence re-entered DFU (register retention inferred)' \ - if method == 'retained-register' else 'invalidate application and enter DFU' - if method == 'retained-register': - left = self.probe.mem32([f['dblTapReg']])[f['dblTapReg']] - ev.update(dblTapRegAfter=hex(left), magicConsumed=not left) - if left: - raise StepFail('the double-tap register was not consumed', ev) - expect = bytearray(self.fix['image']) - if method == 'app-invalidate': - expect[:self.b['page']] = bytes(self.b['page']) + if self.commissioning: + ev.update(self.control(app_bin)) + ev.update(self.retained_reentry(version, 'the write/reset sequence re-entered DFU (register retention inferred)')) + expect = self.fix['image'] n = -(-len(expect) // 256) try: got = self.current_uf2(n * 512) - if got[:len(expect)] != bytes(expect): + if got[:len(expect)] != expect: raise StepFail('CURRENT.UF2 does not read back the fixture from the app start') except StepFail as e: e.evidence = {**ev, **e.evidence, 'failedAt': 'readback', 'appIntact': 'unshown'} @@ -1044,6 +1430,30 @@ def entered(ev): return ev self.step('reenter', reenter) + def restore(self, files, version): + """Cleanup: the record's TinyUF2 installed again and its blinky copied through the drive; the board must + leave DFU and stay out of it.""" + self.install(files) + ev = {'dfu': self.dfu_after_install(version)['usb'], 'copy': self.copy(files['blinky'], files['blinkyBlocks'])} + ev.update(self.after_copy()) + stays_away(lambda: usb_device(self.f['vid'], self.f['pid'], self.b['uid']), QUIET_TIME, + 'the DFU device came back after the blinky copy', ev) + return {'pass': True, 'state': 'blinky copied', 'evidence': ev} + + def report_fields(self): + return {'jlinkRetried': self.probe.retried} # the cleanup's retries land here too: the same list + + def install(self, files): + """The record's tinyuf2-jlink target, then the BIN verified at its flash address in a session of the runner's.""" + fl = files['flash'] + ev = self.flash_target(fl, ['cmake', '--build', fl['root'], '--target', fl['binding']['target']], PROBE_TIMEOUT, + lambda code, out: judge_jlink_target(code, out, self.b, files['hex'])) + self.probe.run(['h', f'verifybin {files["bin"]},{self.f["binBase"]:#x}', 'r', 'g']) + return dict(ev, verified=True) + + def after_copy(self): + return {} + def observe_app(self, app_bin): """The fixture running from the bootloader handoff: two halts in one session, the counter moving, PC at its handler in thread mode, VTOR and MSP from its vector table, and its image intact. The @@ -1076,27 +1486,161 @@ def control(self, app_bin): f = self.f self.probe.run([f'w4 {SCB_AIRCR:08X} {AIRCR_SYSRESETREQ:08X}'], judge=False) watch = 2 * f['dblTapDelayMs'] / 1000 + USB_TIMEOUT - end = time.monotonic() + watch - while time.monotonic() < end: - if usb_device(f['vid'], f['pid'], self.b['uid']): - raise StepFail('the reset without the magic entered DFU', {'controlBootedApp': False}) - time.sleep(0.2) + stays_away(lambda: usb_device(f['vid'], f['pid'], self.b['uid']), watch, 'the reset without the magic entered DFU', + {'controlBootedApp': False}) return {'control': self.observe_app(app_bin), 'controlWatchedS': watch, 'controlBootedApp': True} -def invalidate(board, facts, workdir, battery=None): - """The board left unbootable: a zero page at the device's flash floor (FLASH_LIMITS, never the candidate's - layout) written and read back, the target reset, then the zero page verified again from a new session.""" - base = FLASH_LIMITS[board['device']][0] - bat = battery or Battery(board, facts, workdir) - load, verify = f'loadbin {bat.zero},{base:#x}', f'verifybin {bat.zero},{base:#x}' - bat.probe.run(['h', load, verify, 'r', 'g']) - # with no boot image, a new session reads flash wrong until a loadbin has run (bench-observed on MIMXRT1062, - # cause unknown: verifybin alone fails), so the check is a loadbin J-Link skips as already matching - out = bat.probe.run(['h', load, verify, 'g']) - if 'Skipped. Contents already match' not in ''.join(after(out, load)[:1]).split('J-Link>')[0]: - raise StepFail('the zero page did not persist past the reset; the new session wrote it again') - return {'pass': True, 'state': 'bootloader invalidated', 'evidence': {'zeroPage': f'{base:#x}', 'verified': True}} +def nonce_block(nonce): + """apps/dfu_runtime's nonce block (dfu_runtime.h): magic, LE length, the nonce, then each byte its offset.""" + head = NONCE_MAGIC + len(nonce).to_bytes(4, 'little') + nonce + return head + bytes(i & 0xff for i in range(len(head), NONCE_BLOCK)) + + +def pack_uf2(chunks, family): + """A UF2 of [(addr, 256 bytes)], numbered in order under one numBlocks.""" + return b''.join(struct.pack('<8I', UF2_MAGIC[0], UF2_MAGIC[1], UF2_FLAG_FAMILY, addr, 256, i, len(chunks), family) + + data + bytes(476 - len(data)) + struct.pack(' size: + raise Refused(f'{uf2_path}: the app [0, {256 * len(blocks):#x}) and the nonce block at {off:#x} (in a {cache:#x} ' + f'cache window) do not both fit in the {size:#x} app partition in order') + return {'appChunks': [bl['data'] for bl in blocks], 'nonceOffset': off} + + +class EspBattery(Battery): + """The battery on an esptool board. TinyUF2 copies apps/dfu_runtime with this run's nonce block past the + image; the app reports the nonce in its DFU runtime interface string and returns to DFU on DFU_DETACH.""" + def __init__(self, board, facts, workdir, fail_at=None, commissioning=False): + super().__init__(board, facts, workdir, fail_at, commissioning) + self.tool, self.preserved = Esptool(board, workdir, self.log), None + # TinyUF2 takes espressif UF2 addresses as ota_0 offsets and reports ota_0 as its flash + self.app_start, self.dfu_flash = 0, board['app'][1] - board['app'][0] + self.dfu_claim = 'DFU observed' + + def after_copy(self): + """The preserved regions checked again once blinky is copied, against the snapshot.""" + keep = self.b['regions']['preserved'] + check_preserved(self.preserved, preserved_md5(keep, self.tool.run(preserved_ops(keep), 'hard-reset')), 'restore') + return {'preserved': 'unchanged since the snapshot before the install'} + + def install(self, files): + """The preserved regions' snapshot first, in a session of its own, which the report keeps however the run ends: + a recovery's restore checks against it. A restore reuses the run's snapshot, or a recovery's. Then ESP-IDF's + flash target in CI's image, given only the bridge's tty, in a container named for the board so a survivor is + found and ended by name (cleanup, recovery); then the runner's own read-back.""" + tty = esp_port(self.b['probe']) # no bridge fails here, untouched; any session may leave the chip in its ROM loader + name = f'tinyuf2-hil-{self.b["name"]}' + if container_present(name): + raise StepFail(f'container {name} from an earlier flash still exists: check what it runs, then docker rm -f {name}') + fl = files['flash'] + # created, not run: the container exists, and its name is tracked, before anything can start it; cleanup + # removes it by name however the run ends + CONTAINERS.append(name) + code, out = spawn(cb.idf_docker(fl['root'], 'create') + [ + '--name', name, '--device', tty, '--group-add', str(os.stat(tty).st_gid), '-e', f'ESPPORT={tty}', fl['image'], + 'cmake', '--build', str(fl['root']), '--target', fl['binding']['target']], self.log, 60) + if code: + raise StepFail(f'docker create {name} failed (exit {code})', {'tail': out.strip().splitlines()[-3:]}) + self.touched = True + if self.preserved is None: + keep = self.b['regions']['preserved'] + self.preserved = preserved_md5(keep, self.tool.run(preserved_ops(keep), 'no-reset')) + ev = self.flash_target(fl, ['docker', 'start', '-a', name], ESP_TIMEOUT, + lambda code, out: judge_esp_target(code, out, files['segments'])) + if container_gone(name): + CONTAINERS.remove(name) + ev.update(esp_verify(self.tool, self.b, files['segments'], self.dir, self.preserved)) + return {'wrote': ev['wrote'], 'writes': ev['writes'], 'readBack': True} + + def report_fields(self): + keep = self.b['regions']['preserved'] + return {'snapshot': self.preserved and {'preserved': self.preserved, + 'preservedRegions': {n: list(r) for n, r in keep.items()}}} + + def usb_dfu(self, op): + f = self.f + return child_json([sys.executable, '-B', USB_DFU, '--vid', hex(f['appVid']), '--pid', hex(f['appPid']), + '--serial', self.b['uid'], '--path', self.usb_path, op], self.log, USB_TIMEOUT, f'usb_dfu.py {op}') + + def app(self): + return usb_device(self.f['appVid'], self.f['appPid'], self.b['uid']) + + def copy_app(self, step): + self.nonce, cache = os.urandom(16), self.f['flashCache'] + image = 256 * len(self.chunks) + uf2 = self.dir / f'{step}.uf2' + uf2.write_bytes(pack_uf2([(256 * i, c) for i, c in enumerate(self.chunks)] + + [(self.nonce_off, nonce_block(self.nonce))], self.f['family'])) + return dict(self.copy(uf2, len(self.chunks) + 1, self.nonce), + writes=[write_span('app image', 0, image, cache), write_span('nonce block', self.nonce_off, NONCE_BLOCK, cache)]) + + def app_ran(self): + f = self.f + path = wait_for(self.app, USB_TIMEOUT, f'the app {f["appVid"]:04x}:{f["appPid"]:04x} serial {self.b["uid"]}') + ev = {'usb': path} + if path != self.usb_path: + raise StepFail('the app came up on another USB port than DFU', dict(ev, expected=self.usb_path)) + ev['string'] = self.usb_dfu('string')['string'] + if ev['string'] != f'HIL {self.nonce.hex()}': + raise StepFail('the app does not report this copy\'s nonce', ev) + stays_away(lambda: usb_device(f['vid'], f['pid'], self.b['uid']), QUIET_TIME, + 'the DFU device came back while the app should run', ev) + if self.app() != path: + raise StepFail('the app left the bus', ev) + return ev + + def reenter(self, version): + ev = {'path': self.b['reentry']} + try: + self.usb_dfu('detach') + ev['detachAcked'] = True + wait_for(lambda: self.app() is None, USB_TIMEOUT, 'the app leaving the bus') + ev['appLeft'] = True + ev.update(self.dfu(version), dfuCameUp=True, claim='DFU_DETACH to the app re-entered DFU') + except StepFail as e: + e.evidence = {**ev, **e.evidence} + raise + try: + if self.current_uf2(512 * len(self.chunks)) != b''.join(self.chunks): + raise StepFail('CURRENT.UF2 does not read back the app image') + off = self.nonce_off + if self.current_uf2(512, off // 256 * 512) != nonce_block(self.nonce): + raise StepFail(f'CURRENT.UF2 does not read back the nonce block at {off:#x}') + except StepFail as e: + e.evidence = {**ev, **e.evidence, 'failedAt': 'readback', 'appIntact': 'unshown'} + raise + ev['appIntact'] = True + return ev + + def run(self, files, version): + self.chunks, self.nonce_off = files['appChunks'], files['nonceOffset'] + self.step('install', lambda: self.install(files)) + self.step('dfu', lambda: self.dfu(version)) + self.step('copy', lambda: self.copy_app('copy')) + self.step('app-ran', self.app_ran) + self.step('reenter', lambda: self.reenter(version)) + self.step('recopy', lambda: dict(self.copy_app('recopy'), **self.app_ran(), + claim='a fresh copy boots the app after re-entry')) + + +BATTERIES = {'jlink': Battery, 'esptool': EspBattery} TERM_SIGNALS = {signal.SIGTERM, signal.SIGINT} @@ -1125,6 +1669,29 @@ def check_jlink(): raise Refused('JLinkExe must be on PATH') +def check_esptool(): + import importlib.metadata + try: + v = importlib.metadata.version('esptool') + except importlib.metadata.PackageNotFoundError: + v = None + if v != ESPTOOL_VERSION: + raise Refused(f'esptool {v} is installed for {sys.executable}; the runner needs esptool {ESPTOOL_VERSION}') + + +def check_pyusb(): + try: + import usb.backend.libusb1 + except ImportError: + raise Refused(f'pyusb is not installed for {sys.executable}; usb_dfu.py needs it') + if usb.backend.libusb1.get_backend() is None: + raise Refused('pyusb finds no libusb-1.0; usb_dfu.py needs it') + + +def check_flasher(board): + check_esptool() if board['flasher'] == 'esptool' else check_jlink() + + def reentry_decision(row): """The re-entry method a commissioning run establishes (SKILL.md, Commissioning), or None: every observation of the tried method held, the board was cleaned up and released, and the run passed or failed @@ -1138,31 +1705,52 @@ def reentry_decision(row): if len(reenter) != 1 or failed not in ([], reenter) or (not failed and not row.get('pass')) or \ (failed and reenter[0]['evidence'].get('failedAt') != 'readback'): return None - need = ('controlBootedApp', 'dfuCameUp', 'magicConsumed') if row['tried'] == 'retained-register' else \ - ('bootloaderVerified', 'dfuCameUp') + need = {'retained-register': ('controlBootedApp', 'dfuCameUp', 'magicConsumed'), + 'dfu-detach': ('detachAcked', 'appLeft', 'dfuCameUp')}[row['tried']] return row['tried'] if all(reenter[0]['evidence'].get(k) is True for k in need) else None +def admit_record(record_path, board, facts, workdir): + """A build record the battery and restore may install, with what they need on this host.""" + files, fw = verify_record(record_path, board, facts, workdir) + if board['flasher'] == 'jlink': + data = parse_hex(files['hex']) + check_ranges(data, [board['install']], FLASH_LIMITS[board['device']], files['hex']) + if not hex_matches_bin(data, files['bin'], facts['binBase']): + raise Refused(f'{files["hex"]} and {files["bin"]} do not hold the same bytes') + check_bootloader_bin(files['bin'], board, facts, 'candidate') + exclusion_configured(facts, board['uid']) + if shutil.which('mcopy') is None: + raise Refused('mcopy must be on PATH') + check_flasher(board) + return files, fw + + +def trial_reentry(board, method): + """The board with `method` as its re-entry, for commissioning: refused once the roster's reentry is set.""" + if not method: + return board + if board['reentry'] is not None: + raise Refused(f'{board["name"]}: reentry is already commissioned; --try-reentry is for commissioning only') + if method not in REENTRY[board['flasher']]: + raise Refused(f'{board["name"]}: a {board["flasher"]} board re-enters by one of {REENTRY[board["flasher"]]}') + return dict(board, reentry=method) + + def run_board(args, board, facts, helper, workdir): row = {'board': board['name'], 'lock': board['lock'], 'ran': False, 'pass': False, 'locked': False, 'lockBypassed': False, 'steps': [], 'cleanup': None, 'detail': ''} + methods = REENTRY[board['flasher']] + if args.fail_at and args.fail_at not in (*STEPS[board['flasher']], 'cleanup'): + raise Refused(f'{board["name"]}: the {board["flasher"]} battery has no step {args.fail_at}') if args.try_reentry: - if board['reentry'] is not None: - raise Refused(f'{board["name"]}: reentry is already commissioned; --try-reentry is for commissioning only') - board = dict(board, reentry=args.try_reentry) + board = trial_reentry(board, args.try_reentry) row.update(commissioning=True, tried=args.try_reentry) - elif board['reentry'] not in REENTRY: - raise Refused(f'{board["name"]}: tinyuf2.reentry must be one of {REENTRY} (set from commissioning)') - files, row['firmware'] = verify_record(args.build_record, board, facts) - data = parse_hex(files['hex']) - check_ranges(data, [board['install']], FLASH_LIMITS[board['device']], files['hex']) - if not hex_matches_bin(data, files['bin'], board['install'][0]): - raise Refused(f'{files["hex"]} and {files["bin"]} do not hold the same bytes') - check_bootloader_bin(files['bin'], board, facts, 'candidate') - exclusion_configured(facts, board['uid']) - if shutil.which('mcopy') is None: - raise Refused('mcopy must be on PATH') - check_jlink() + elif board['reentry'] not in methods: + raise Refused(f'{board["name"]}: tinyuf2.reentry must be one of {methods} (set from commissioning)') + files, row['firmware'] = admit_record(args.build_record, board, facts, workdir) + if board['flasher'] == 'esptool': + check_pyusb() bypass = os.environ.get('HIL_NO_BOARD_LOCK') == '1' # blocked from here until the process exits or becomes the holder (keep_held): a signal acts only # inside the battery, and the report and any hold are settled first @@ -1175,7 +1763,7 @@ def run_board(args, board, facts, helper, workdir): if fh is None and not bypass: raise Refused(f'{board["name"]}: the lock dir is unusable (tinyusb helper fails open); refusing to run unlocked') row['lockBypassed'] = fh is None - bat = Battery(board, facts, workdir, args.fail_at, row.get('commissioning', False)) + bat = BATTERIES[board['flasher']](board, facts, workdir, args.fail_at, row.get('commissioning', False)) code = 0 try: row['ran'] = True @@ -1185,8 +1773,9 @@ def run_board(args, board, facts, helper, workdir): except BaseException as e: # anything after admission still cleans up below row['detail'] = f'{bat.at}: {e}' if isinstance(e, StepFail) else f'{bat.at}: {type(e).__name__}: {e}' code = 1 - row['steps'], row['jlinkRetried'] = bat.steps, bat.probe.retried # the cleanup's retries land here too - code = clean_up(row, code, bat, fh, board, facts, helper, workdir) + row['steps'] = bat.steps + code = clean_up(row, code, bat, fh, board, helper, files) + row.update(bat.report_fields()) # after the cleanup, whose install may take the snapshot a recovery needs if row.get('commissioning'): row['reentryDecision'] = reentry_decision(row) return deferred(row), code @@ -1199,18 +1788,20 @@ def deferred(row): return row -def clean_up(row, code, bat, fh, board, facts, helper, workdir): - """Invalidate a touched board, then release it. A child that survived SIGKILL, or any unsuccessful - cleanup, leaves the board held (exit 4), touched or not.""" +def clean_up(row, code, bat, fh, board, helper, files): + """Restore a touched board to the record's TinyUF2 with blinky copied, then release it. A child that survived + SIGKILL, or any unsuccessful cleanup, leaves the board held (exit 4), touched or not.""" stuck = [] try: stuck = stuck_children() if stuck: - raise StepFail(f'children {[x["pid"] for x in stuck]} survived SIGKILL; no flash may overlap them') + raise StepFail('children survived SIGKILL; no flash may overlap them') # their pids are in survivors if bat.touched: if bat.fail_at == 'cleanup': raise StepFail('--fail-at cleanup: injected cleanup failure (acceptance only)') - row['cleanup'] = dict(invalidate(board, facts, workdir, bat), heldAfter=False) + row['cleanup'] = dict(bat.restore(files, row['firmware']['version']), heldAfter=False) + else: + row['cleanup'] = {'pass': True, 'state': 'untouched', 'evidence': {}, 'heldAfter': False} except BaseException as e: stuck += [x for x in stuck_children() if x not in stuck] row['cleanup'] = {'pass': False, 'state': 'unknown' if bat.touched else 'untouched', @@ -1232,29 +1823,35 @@ def main(argv=None): sub = ap.add_subparsers(dest='cmd', required=True) r = sub.add_parser('run', help='the battery on one board (self-locking)') r.add_argument('-b', '--board', required=True) - r.add_argument('--build-record', required=True, help='check_build.py --board B --backend cmake --shared JSON, under cmake-build/') - r.add_argument('--fail-at', choices=(*STEPS, 'cleanup'), help='acceptance only: fail this step, or the cleanup, ' + r.add_argument('--build-record', required=True, help='check_build.py --board B [--backend cmake] --shared --hil-config ROSTER ' + 'JSON, under cmake-build/') + r.add_argument('--fail-at', choices=(*STEPS['esptool'], 'cleanup'), help='acceptance only: fail this step, or the cleanup, ' 'to exercise cleanup and its holder') - r.add_argument('--try-reentry', choices=REENTRY, help="commissioning only, while the roster's reentry is unset") + r.add_argument('--try-reentry', choices=REENTRY_METHODS, help="commissioning only, while the roster's reentry is unset") r.add_argument('config') - i = sub.add_parser('install', help='program a bootloader BIN under your own manual hold') + i = sub.add_parser('install', help='program a bootloader under your own manual hold') i.add_argument('-b', '--board', required=True) - i.add_argument('--firmware', required=True, - help='a bootloader BIN, programmed at the install start with a zero page at the app start') - i.add_argument('--sha256', required=True) + i.add_argument('--build-record', required=True, help="the HIL build record whose TinyUF2 is installed") i.add_argument('--under-hold', required=True, metavar='REASON') + i.add_argument('--try-reentry', choices=REENTRY_METHODS, + help="commissioning only, while the roster's reentry is unset: DFU's fallback after the install") i.add_argument('config') - s = sub.add_parser('invalidate', help='leave the board unbootable, under your hold or to recover a holder') + s = sub.add_parser('restore', help="install the record's TinyUF2 and copy its blinky, under your hold or to " + 'recover a holder') s.add_argument('-b', '--board', required=True) + s.add_argument('--build-record', required=True, help='the build record whose TinyUF2 and apps/blinky.uf2 are installed') g = s.add_mutually_exclusive_group(required=True) g.add_argument('--under-hold', metavar='REASON') g.add_argument('--recover-holder', type=int, metavar='PID') s.add_argument('--report', help="the failed run's hil_report.json (with --recover-holder)") + s.add_argument('--try-reentry', choices=REENTRY_METHODS, + help="commissioning only, under your hold, while the roster's reentry is unset") s.add_argument('config') fr = sub.add_parser('fat-read') fr.add_argument('--dev', required=True) fr.add_argument('--file', required=True) fr.add_argument('--length', type=int, required=True) + fr.add_argument('--offset', type=int, default=0) args = ap.parse_args(argv) if args.cmd == 'fat-read': return fat_read_main(args) @@ -1262,46 +1859,48 @@ def main(argv=None): signal.signal(sig, on_signal) run_dir = cb.BUILD_DIR / 'hil' / f'run-{time.strftime("%Y%m%d-%H%M%S")}-{os.getpid()}' result = {'pass': False, 'caveat': '', 'results': [], 'reportPath': str(run_dir / 'hil_report.json')} - code, recovery = 2, None + code, recovery, bat = 2, None, None try: run_dir.mkdir(parents=True) board = load_board(args.config, args.board) - facts = None # invalidate, recovery included, writes only at the device's flash floor: no candidate source - if args.cmd != 'invalidate': - facts = source_facts(board) - apply_layout(board, facts) + facts = source_facts(board) + apply_layout(board, facts) helper = load_helper(board['helper']) if args.cmd == 'run': row, code = run_board(args, board, facts, helper, run_dir) result['results'].append(row) result['pass'] = code == 0 and row['pass'] elif args.cmd == 'install': + board = trial_reentry(board, args.try_reentry) manual_holder(helper, board['lock'], args.under_hold) - if sha256(args.firmware) != args.sha256: - raise Refused(f'{args.firmware} does not match --sha256') - check_bootloader_bin(args.firmware, board, facts, args.firmware) - exclusion_configured(facts, board['uid']) - check_jlink() + files, fw = admit_record(args.build_record, board, facts, run_dir) + bat = BATTERIES[board['flasher']](board, facts, run_dir) try: - Battery(board, facts, run_dir).program(args.firmware, board['install'][0], facts['appStart']) + ev = bat.install(files) + ev['dfu'] = bat.dfu_after_install(fw['version']) finally: result['survivors'] = stuck_children() - result['results'].append({'board': board['name'], 'installed': {'path': args.firmware, 'sha256': args.sha256}}) + result['results'].append({'board': board['name'], 'installed': fw, 'evidence': ev}) result['pass'], code = True, 0 else: - check_jlink() - rec = None + files, fw = admit_record(args.build_record, board, facts, run_dir) + rec = snapshot = None if args.recover_holder is not None: + if args.try_reentry: + raise Refused('--try-reentry is for a restore under your own hold, not a recovery') if not args.report: raise Refused("--recover-holder needs --report, the failed run's report naming that holder") try: rows = json.loads(Path(args.report).read_text())['results'] except (OSError, ValueError, KeyError, TypeError) as e: raise Refused(f'cannot read {args.report}: {e}') - held = [x['cleanup'] for x in rows if ((x.get('cleanup') or {}).get('holder') or {}).get('pid') - == args.recover_holder and x.get('board') == board['name']] - if len(held) != 1: + failed = [x for x in rows if ((x.get('cleanup') or {}).get('holder') or {}).get('pid') + == args.recover_holder and x.get('board') == board['name']] + if len(failed) != 1: raise Refused(f'{args.report} names no cleanup holder {args.recover_holder} of {board["name"]}') + held = [failed[0]['cleanup']] + if board['flasher'] == 'esptool': + snapshot = report_snapshot(failed[0], board['regions']['preserved']) recovery = recovery_lock(helper, board['lock']) rec = prove_hold(helper, board['lock'], args.recover_holder, reason='hil_test.py', role=HOLDER_ROLE) cmdline = Path(f'/proc/{args.recover_holder}/cmdline').read_bytes() @@ -1311,22 +1910,26 @@ def main(argv=None): if alive: raise Refused(f'the failed run\'s children {alive} are still alive; recovery may not overlap them') else: + board = trial_reentry(board, args.try_reentry) manual_holder(helper, board['lock'], args.under_hold) - users = probe_users(board['probe']) + users = probe_users(board) if users: raise Refused(f'probe {board["probe"]} is still in use by pids {users}; recovery may not overlap them') + bat = BATTERIES[board['flasher']](board, facts, run_dir) try: - cleanup = invalidate(board, facts, run_dir) + if snapshot: + bat.preserved = snapshot + cleanup = bat.restore(files, fw['version']) finally: result['survivors'] = stuck_children() if result['survivors']: - raise StepFail('invalidate left children that survived SIGKILL') + raise StepFail('restore left children that survived SIGKILL') if args.recover_holder is not None: os.kill(args.recover_holder, signal.SIGUSR1) wait_for(lambda: flock_free(helper.lock_path(board['lock'])), 10, f'cleanup holder {args.recover_holder} releasing {board["name"]}') cleanup['recovered'] = {'holder': args.recover_holder, 'record': rec} - result['results'].append({'board': board['name'], 'cleanup': cleanup}) + result['results'].append({'board': board['name'], 'firmware': fw, 'cleanup': cleanup}) result['pass'], code = True, 0 except Refused as e: result['error'] = str(e) @@ -1336,7 +1939,7 @@ def main(argv=None): code = 2 except (StepFail, Interrupted) as e: result['error'] = str(e) - code = 1 + code = 4 if bat is not None and bat.touched else 1 # a failed restore keeps the hold if recovery: recovery.close() if result.get('survivors'): # a manual operation's survivor: keep your hold until it is gone diff --git a/.claude/skills/hil/scripts/usb_dfu.py b/.claude/skills/hil/scripts/usb_dfu.py new file mode 100644 index 000000000..4d7ee5b7b --- /dev/null +++ b/.claude/skills/hil/scripts/usb_dfu.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""USB control requests to a DFU runtime interface for hil_test.py, run as its own child so the runner can +time it out, kill it and see it survive. + + usb_dfu.py --vid V --pid P --serial S --path BUS-PORT[.PORT...] string|detach + +The device must be the only one with that VID, PID, serial and sysfs path, with exactly one DFU runtime +interface (class 0xFE, subclass 1, protocol 1). `string` reads that interface's string from the device +itself, not from a cache; `detach` sends DFU_DETACH (USB DFU 1.1 section 5.1) to it. +The last stdout line is JSON: {"ok": true, ...} or {"ok": false, "error"}. +""" +import argparse +import json +import sys + +sys.dont_write_bytecode = True + +DFU_CLASS = (0xFE, 0x01, 0x01) +DFU_DETACH = 0x00 +DETACH_TIMEOUT_MS = 1000 # wDetachTimeOut; a bitWillDetach device leaves the bus by itself +CONTROL_TIMEOUT_MS = 1000 + + +def find(vid, pid, serial, path): + import usb.core + hits = [] + for dev in usb.core.find(find_all=True, idVendor=vid, idProduct=pid): + where = f'{dev.bus}-{".".join(map(str, dev.port_numbers or ()))}' + if where == path and dev.serial_number == serial: + hits.append(dev) + if len(hits) != 1: + raise RuntimeError(f'{len(hits)} devices {vid:04x}:{pid:04x} serial {serial} at {path}') + dev = hits[0] + itfs = [i for i in dev.get_active_configuration() + if (i.bInterfaceClass, i.bInterfaceSubClass, i.bInterfaceProtocol) == DFU_CLASS] + if len(itfs) != 1: + raise RuntimeError(f'{len(itfs)} DFU runtime interfaces') + return dev, itfs[0] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('--vid', type=lambda s: int(s, 0), required=True) + ap.add_argument('--pid', type=lambda s: int(s, 0), required=True) + ap.add_argument('--serial', required=True) + ap.add_argument('--path', required=True) + ap.add_argument('op', choices=('string', 'detach')) + args = ap.parse_args() + try: + import usb.util + dev, itf = find(args.vid, args.pid, args.serial, args.path) + if args.op == 'string': + out = {'ok': True, 'interface': itf.bInterfaceNumber, + 'string': usb.util.get_string(dev, itf.iInterface) if itf.iInterface else None} + else: + dev.ctrl_transfer(0x21, DFU_DETACH, DETACH_TIMEOUT_MS, itf.bInterfaceNumber, None, CONTROL_TIMEOUT_MS) + out = {'ok': True, 'interface': itf.bInterfaceNumber} + except Exception as e: # every failure is reported, never raised past the JSON line + out = {'ok': False, 'error': f'{type(e).__name__}: {e}'} + print(json.dumps(out), flush=True) + return 0 if out['ok'] else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index b0431f2cf..2bf67e86b 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -246,7 +246,7 @@ def run_main(argv, boards=None, ghostfat=None, tests=None, identities=None): boards = boards or {} identities = iter(identities or []) - def build_board(board, backend, args, idf): + def build_board(board, backend, args, idf, flasher=None): status = boards.get(board, 'ok') return {'board': board, 'port': cb.port_of(board), 'backend': backend, 'buildDir': f'cmake-build/x-{board}', 'status': status, 'firstError': '' if status == 'ok' else f'{board} {status}', 'artifacts': ['a.bin']} @@ -318,6 +318,19 @@ def test_a_roster_entry_is_named_by_its_tinyuf2_board(self): self.assertEqual(self.hil('ports/mimxrt10xx/board_flash.c')['boards'], ['imxrt1060_evk']) self.assertNotIn('mimxrt10xx', [u['target'] for u in self.hil('src/main.c')['uncovered']]) + def test_the_battery_app_selects_the_esptool_roster_boards_and_other_apps_none(self): + self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk', 'flasher': {'name': 'jlink'}}, + {'name': 'espressif_esp32s3_devkitc_1', 'flasher': {'name': 'esptool'}}]})) + for path in ('apps/dfu_runtime/src/nonce.c', 'ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c'): + with self.subTest(path): + h = self.hil(path) + self.assertEqual((h['boards'], h['uncovered']), (['espressif_esp32s3_devkitc_1'], [])) + h = self.hil('apps/blinky/src/blinky.c', 'ports/espressif/apps/blinky/main/CMakeLists.txt') + self.assertEqual((h['boards'], len(h['uncovered'])), ([], 2)) + self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk', 'flasher': {'name': 'jlink'}}]})) + h = self.hil('apps/dfu_runtime/src/nonce.c') + self.assertEqual((h['boards'], [u['reason'] for u in h['uncovered']]), ([], [f'no {self.config} esptool board runs dfu_runtime'])) + def test_the_hil_follows_its_cmake_build_not_cis_make(self): self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk'}]})) h = self.hil('ports/mimxrt10xx/family.cmake', 'ports/mimxrt10xx/boards/imxrt1060_evk/board.cmake') @@ -482,7 +495,7 @@ def test_backend_override_needs_board_and_shared_and_a_cmake_build(self): def test_backend_override_builds_with_cmake(self): seen = [] - def build_board(board, backend, args, idf): + def build_board(board, backend, args, idf, flasher=None): seen.append(backend) return {'board': board, 'port': 'mimxrt10xx', 'backend': backend, 'buildDir': 'cmake-build/x', 'status': 'ok', 'firstError': '', 'artifacts': [], 'sha256': {}} @@ -492,7 +505,7 @@ def build_board(board, backend, args, idf): self.assertEqual(seen, ['cmake']) def test_passing_private_roots_leave_nothing_in_built(self): - def build_board(board, backend, args, idf): + def build_board(board, backend, args, idf, flasher=None): return {'board': board, 'port': 'lpc55', 'backend': backend, 'buildDir': 'cmake-build/x', 'status': 'ok', 'firstError': '', 'artifacts': ['a.bin'], 'removed': True} with mock.patch.object(cb, 'build_board', build_board), \ @@ -592,6 +605,28 @@ def test_shared_root_is_rebuilt_from_clean_and_kept(self): self.assertEqual(set(rec['sha256']), set(rec['artifacts'])) self.assertTrue((root / '_bin' / 'tinyuf2-lpcxpresso55s69.bin').is_file()) + def test_an_idf_build_hashes_its_flash_config_and_needs_it(self): + self.args.shared = True + root = self.build_dir / 'cmake-build-espressif_esp32s3_devkitc_1' + made = [str(p.relative_to(root)) for p in cb.expected_artifacts('espressif', 'x', 'idf', root)] + list(cb.IDF_CONFIG) + + def run(cmd, log, verbose, cwd=cb.ROOT): + for rel in made: + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text(rel) + return 0 + inspect = mock.Mock(return_value=mock.Mock(returncode=0)) + with mock.patch.object(cb, 'run', run), mock.patch.object(cb.subprocess, 'run', inspect), \ + mock.patch.object(cb, 'git', lambda *a: str(self.build_dir)): + rec = cb.build_board('espressif_esp32s3_devkitc_1', 'idf', self.args, {'image': 'espressif/idf:v5.3.2'}) + self.assertEqual(rec['status'], 'ok', rec) + self.assertLessEqual({'sdkconfig', '_bin/flash_args', '_bin/ota_data_initial.bin', '_bin/apps/dfu_runtime.uf2'}, + set(rec['sha256'])) + made.remove('sdkconfig') + rec = cb.build_board('espressif_esp32s3_devkitc_1', 'idf', self.args, {'image': 'espressif/idf:v5.3.2'}) + self.assertEqual(rec['status'], 'failed') + self.assertIn('sdkconfig', rec['firstError']) + def test_cmake_missing_app_image_fails(self): def run(cmd, log, verbose, cwd=cb.ROOT): log.write_text('ok') @@ -616,6 +651,202 @@ def test_missing_prerequisite_is_an_error_not_a_failure(self): self.assertEqual((rec['status'], rec['firstError']), ('error', 'lib/mcu/nxp/mcux-sdk missing')) +class HilRecord(unittest.TestCase): + """A HIL record binds the build's own flash target to the roster's probe or bridge.""" + + def setUp(self): + tmp = build_tmpdir() + self.addCleanup(tmp.cleanup) + self.tmp = Path(tmp.name) + self.roster = self.tmp / 'local.json' + self.roster.write_text(json.dumps({'boards': [ + {'name': 'metro_m7_1011', 'uid': 'U', 'flasher': {'name': 'jlink', 'uid': '000801012283', 'args': '-device MIMXRT1011DAE5A'}}, + {'name': 'espressif_esp32s3_devkitc_1', 'uid': 'M', 'flasher': {'name': 'esptool', 'uid': 'b1', 'args': '--chip esp32s3'}}, + {'name': 'lpcxpresso55s69', 'uid': 'L', 'flasher': {'name': 'openocd', 'uid': 'x', 'args': '-f a.cfg'}}]})) + self.config = str(self.roster.relative_to(ROOT)) + + def jlink_tree(self, root, script=None, options='-USB 000801012283 -ExitOnError 1 -NoGui 1', interface='swd'): + root.mkdir(parents=True, exist_ok=True) + (root / 'tinyuf2.jlink').write_text(script or f'halt\nloadfile {root / "tinyuf2.hex"}\nr\ngo\nexit') + line = (f'cd {root} && JLinkExe -device MIMXRT1011DAE5A {options} -if {interface} -JTAGConf -1,-1 -speed auto ' + f'-CommandFile {root / "tinyuf2.jlink"}') + return lambda d, args: subprocess.CompletedProcess(args, 0, f'arm-none-eabi-gcc ...\n{line}\n', '') + + def idf_tree(self, root, chip='esp32s3', extra='', args_file='flash_args'): + root.mkdir(parents=True, exist_ok=True) + (root / 'flash_args').write_text('--flash_mode dio --flash_freq 80m --flash_size 8MB\n0x0 bootloader/bootloader.bin\n' + '0x410000 tinyuf2.bin\n0x8000 partition_table/partition-table.bin\n' + f'0xe000 ota_data_initial.bin\n{extra}') + line = (f'cd /opt/esp/idf/components/esptool_py && cmake -D IDF_PATH=/opt/esp/idf ' + f'-D "SERIAL_TOOL=python;;/opt/esp/idf/components/esptool_py/esptool/esptool.py;--chip;{chip}" ' + f'-D "SERIAL_TOOL_ARGS=--before=default_reset;--after=hard_reset;write_flash;@{args_file}" ' + f'-D WORKING_DIRECTORY={root} -P /opt/esp/idf/components/esptool_py/run_serial_tool.cmake') + return lambda d, args: subprocess.CompletedProcess(args, 0, line + '\n', '') + + def test_the_roster_flasher_selects_the_probe_through_configure_defines(self): + self.assertEqual(cb.roster_flasher(self.config, 'metro_m7_1011'), + ({'name': 'jlink', 'uid': '000801012283', 'device': 'MIMXRT1011DAE5A'}, + ['JLINK_OPTION=-USB 000801012283 -ExitOnError 1 -NoGui 1'])) + self.assertEqual(cb.roster_flasher(self.config, 'espressif_esp32s3_devkitc_1'), + ({'name': 'esptool', 'uid': 'b1', 'device': 'esp32s3'}, [])) # the tty is chosen at flash time + for board, err in (('lpcxpresso55s69', 'no HIL flash target'), ('imxrt1060_evk', 'not on')): + with self.subTest(board), self.assertRaisesRegex(cb.Fail, err): + cb.roster_flasher(self.config, board) + + def test_the_generated_jlink_target_must_name_the_roster_probe_and_the_build_image(self): + flasher = {'name': 'jlink', 'uid': '000801012283', 'device': 'MIMXRT1011DAE5A'} + root = self.tmp / 'b' + out = cb.flash_binding(flasher, root, self.jlink_tree(root)) + self.assertEqual((out['target'], out['inputs']), ('tinyuf2-jlink', ['tinyuf2.jlink', 'tinyuf2.hex'])) + for what, tree in (('another probe', dict(options='-USB 1 -ExitOnError 1 -NoGui 1')), + ('no exit on error', dict(options='-USB 000801012283')), + ('two probes', dict(options='-USB 000801012283 -USB 2 -ExitOnError 1 -NoGui 1')), + ('jtag', dict(interface='jtag')), + ('another image', dict(script=f'halt\nloadfile {self.tmp / "other.hex"}\nr\ngo\nexit')), + ('a second image', dict(script=f'halt\nloadfile {root / "tinyuf2.hex"}\nloadfile /tmp/o.hex\nr\ngo\nexit'))): + with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'does not flash'): + cb.flash_binding(flasher, root, self.jlink_tree(root, **tree)) + + def test_the_idf_flash_target_must_write_flash_args_for_the_roster_chip(self): + flasher = {'name': 'esptool', 'uid': 'b1', 'device': 'esp32s3'} + root = self.tmp / 'e' + out = cb.flash_binding(flasher, root, self.idf_tree(root)) + self.assertEqual(out['inputs'][0], 'flash_args') + for what, tree in (('another chip', dict(chip='esp32s2')), + ('a chip prefix', dict(chip='esp32s3WRONG')), + ('another argument file', dict(args_file='flash_args.extra')), + ('another segment', dict(extra='0x10000 app.bin\n'))): + with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'does not flash'): + cb.flash_binding(flasher, root, self.idf_tree(root, **tree)) + + def ready(self, flasher, root, tree, steps, nested=(), code=0, banner=True): + """flash_ready on `root` whose target dry-runs `steps` and whose bootloader subproject dry-runs `nested`; + `banner` False drops ninja's banner from both, 'nested' from the subproject's only.""" + commands = tree(root) + command = cb.flash_binding(flasher, root, commands)['command'] + flash = {'target': cb.FLASH[flasher['name']][0], 'flasher': flasher, 'command': command} + + def ninja(d, args): + self.assertEqual(args[:2], ['-n', '-v']) + out = [command if x == 'FLASH' else x for x in (steps if d == root else nested)] + return subprocess.CompletedProcess(args, code, 'entrypoint banner\n' + ("ninja: Entering directory `x'\n" if banner is True or banner == 'nested' and d == root else '') + + ''.join(f'[{i}/9] {x}\n' for i, x in enumerate(out, 1)), '') + cb.flash_ready(root, flash, ninja) + + def test_a_flash_target_is_ready_only_when_its_dry_run_shows_just_the_flasher(self): + jlink = {'name': 'jlink', 'uid': '000801012283', 'device': 'MIMXRT1011DAE5A'} + root = self.tmp / 'b' + self.ready(jlink, root, self.jlink_tree, ['FLASH']) + rebuild = [f'/usr/bin/arm-none-eabi-gcc -c {ROOT}/src/main.c -o main.c.obj', ': && arm-none-eabi-gcc -o tinyuf2.elf', 'FLASH'] + for what, steps, kw in (('a rebuild', rebuild, {}), ('a CMake re-run', ['/usr/bin/cmake --regenerate-during-build -S.. -B.'], {}), + ('ninja fails', ['FLASH'], {'code': 1}), ('nothing', [], {}), + ('no ninja banner', ['FLASH'], {'banner': False}), + ('another command', ['JLinkExe -USB 1'], {}), ('the flasher twice', ['FLASH', 'FLASH'], {})): + with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): + self.ready(jlink, root, self.jlink_tree, steps, **kw) + + def test_an_idf_flash_target_may_also_run_steps_that_write_no_flash_input(self): + esp = {'name': 'esptool', 'uid': 'b1', 'device': 'esp32s3'} + root = self.tmp / 'e' + cmake, py = '/opt/esp/tools/cmake/3.30.2/bin/cmake', '/opt/esp/python_env/idf5.3_py3.10_env/bin/python' + sizes = f'{py} /opt/esp/idf/components/partition_table/check_sizes.py --offset 0x8000' + size = (f'cd {root}/esp-idf/esptool_py && {sizes} partition --type app {root}/partition_table/partition-table.bin ' + f'{root}/tinyuf2.bin') + boot_size = f'cd {root}/bootloader/esp-idf/esptool_py && {sizes} bootloader 0x0 {root}/bootloader/bootloader.bin' + copies = ' && '.join(f'{cmake} -E copy {root}/{f} {root}/_bin/{Path(f).name}' for f in cb.FLASH['esptool'][1]) + post = (f'cd {root} && bash {root}/tinyuf2.postbuild.sh && cd {root} && {cmake} -E echo "Copy binaries for artifact" && ' + f'{copies} && cd {root} && {py}3.10 {ROOT}/lib/uf2/utils/uf2conv.py --carray -o ' + f'{root}/apps/update_tinyuf2/bootloader_bin.c {root}/tinyuf2.bin') + stamps = (f'cd {root} && {cmake} -E make_directory {root}/CMakeFiles && {cmake} -E touch {root}/CMakeFiles/bootloader-complete ' + f'&& {cmake} -E touch {root}/bootloader-prefix/src/bootloader-stamp/bootloader-done') + fresh = [f'cd {root}/bootloader && {cmake} --build .', size, f'cd {root}/bootloader && {cmake} -E echo_append', post, stamps, + 'FLASH'] + script = root / 'tinyuf2.postbuild.sh' + good = ('#!/bin/bash\necho "Creating combined.bin"\n' + f"esptool.py --chip esp32s3 merge_bin --output combined.bin $(tr '\\n' ' ' < {root}/flash_args)\n") + self.idf_tree(root) + script.write_text(good) + self.ready(esp, root, self.idf_tree, fresh, [boot_size]) + cases = (('a CMake re-run', [f'cd {root} && {cmake} --regenerate-during-build -S/p -B{root}'], [boot_size], good), + ('an app rebuild', [f'{root}/xtensa-esp32s3-elf-g++ -o tinyuf2.elf'] + fresh, [boot_size], good), + ('a bootloader rebuild', fresh, ['xtensa-esp32s3-elf-gcc -c bootloader_start.c', boot_size], good), + ('a size check and a copy', [size + f' && cp /tmp/o.bin {root}/tinyuf2.bin', 'FLASH'], [], good), + ('a copy out of _bin', [f'{cmake} -E copy /tmp/o.bin {root}/_bin/../tinyuf2.bin', 'FLASH'], [], good), + ('a redirect', [f'{cmake} -E echo x > {root}/tinyuf2.bin', 'FLASH'], [], good), + ('a changed post-build script', fresh, [boot_size], good + f'cp /tmp/o.bin {root}/tinyuf2.bin\n'), + ('a step after the bootloader build', [f'cd {root}/bootloader && {cmake} --build . && {cmake} -E echo_append', 'FLASH'], + ['xtensa-esp32s3-elf-gcc -c bootloader_start.c'], good), + ('a copy through a symlink', [f'{cmake} -E copy /tmp/o.bin {root}/_bin/alias/tinyuf2.bin', 'FLASH'], [], good), + ('a touch of a flash input', [f'{cmake} -E touch {root}/flash_args', 'FLASH'], [], good)) + (root / '_bin').mkdir() + (root / '_bin/alias').symlink_to('..') + for what, steps, nested, text in cases: + with self.subTest(what), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): + script.write_text(text) + self.ready(esp, root, self.idf_tree, steps, nested) + script.write_text(good) + (root / '_bin/alias').unlink() + (root / '_bin').rmdir() + (root / '_bin').symlink_to('.') # a copy into _bin would land on the input itself + with self.subTest('a symlinked _bin'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): + self.ready(esp, root, self.idf_tree, [f'{cmake} -E copy /tmp/o.bin {root}/_bin/tinyuf2.bin', 'FLASH'], [boot_size]) + (root / 'apps').symlink_to('.') + (root / 'update_tinyuf2').symlink_to('.') + carray = 'cd ' + post.split(' && cd ')[-1] + self.ready(esp, root, self.idf_tree, [carray, 'FLASH'], [boot_size]) # before the alias + (root / 'bootloader_bin.c').symlink_to('tinyuf2.bin') + with self.subTest('a C array onto the image'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): + self.ready(esp, root, self.idf_tree, [carray, 'FLASH'], [boot_size]) + (root / 'combined.bin').symlink_to('tinyuf2.bin') + with self.subTest('a post-build onto the image'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): + self.ready(esp, root, self.idf_tree, [post.split(' && cd ')[0], 'FLASH'], [boot_size]) + (root / 'combined.bin').unlink() + self.ready(esp, root, self.idf_tree, [post.split(' && cd ')[0], 'FLASH'], [boot_size]) + (root / '_bin').unlink() + (root / '_bin').mkdir() + (root / '_bin/image.bin').write_text('x') + (root / 'ota_data_initial.bin').unlink(missing_ok=True) + (root / 'ota_data_initial.bin').symlink_to('_bin/image.bin') + with self.subTest('a copy onto a symlinked input'), self.assertRaisesRegex(cb.Fail, 'would not run only its flasher'): + self.ready(esp, root, self.idf_tree, [f'{cmake} -E copy /tmp/o.bin {root}/_bin/image.bin', 'FLASH'], [boot_size]) + with self.assertRaisesRegex(cb.Fail, 'bootloader: ninja -n failed'): + self.ready(esp, root, self.idf_tree, fresh, [boot_size], banner='nested') + + def test_a_hil_record_pins_the_target_inputs_and_holds_the_build_lock(self): + args = mock.Mock(shared=True, define=[], verbose=False, idf='docker') + built = [] + + def fake_cmake(port, board, root, defines, log, verbose): + built.append(defines) + lock = root.with_name(root.name + '.lock') + with open(lock) as f, self.assertRaises(BlockingIOError): # held while the build runs + cb.fcntl.flock(f, cb.fcntl.LOCK_EX | cb.fcntl.LOCK_NB) + for rel in ('_bin/tinyuf2.bin', '_bin/tinyuf2.hex', '_bin/apps/blinky.uf2', 'tinyuf2.hex'): + (root / rel).parent.mkdir(parents=True, exist_ok=True) + (root / rel).write_text(rel) + return True, [] + build_dir = self.tmp / 'cmake-build' # absent: the first shared build creates it before its lock + with mock.patch.object(cb, 'BUILD_DIR', build_dir), mock.patch.object(cb, 'check_deps', lambda p: None), \ + mock.patch.object(cb, 'check_tools', lambda *a: None), mock.patch.object(cb, 'build_cmake', fake_cmake), \ + mock.patch.object(cb, 'expected_artifacts', lambda *a: []), \ + mock.patch.object(cb, 'ninja_in', lambda backend, mode, image, root, args: self.jlink_tree(root)(root, args)): + rec = cb.build_board('metro_m7_1011', 'cmake', args, {}, cb.roster_flasher(self.config, 'metro_m7_1011')) + self.assertEqual(rec['status'], 'ok', rec) + self.assertEqual(built, [['JLINK_OPTION=-USB 000801012283 -ExitOnError 1 -NoGui 1']]) + self.assertIn('tinyuf2.hex', rec['sha256']) + self.assertIn('tinyuf2.jlink', rec['sha256']) + self.assertEqual(rec['flash']['defines'], ['JLINK_OPTION=-USB 000801012283 -ExitOnError 1 -NoGui 1']) + + def test_a_hil_record_takes_probe_selection_only_from_the_roster(self): + for argv in (['--board', 'metro_m7_1011', '--shared', '--hil-config', self.config, '-D', 'JLINK_OPTION=-USB 1'], + ['--board', 'metro_m7_1011', '--shared', '--hil-config', self.config, '-D', 'ESPPORT:STRING=/dev/x'], + ['--board', 'metro_m7_1011', '--hil-config', self.config]): + with self.subTest(argv), contextlib.redirect_stdout(io.StringIO()) as out, self.assertRaises(SystemExit) as e: + cb.main(argv) + self.assertEqual(e.exception.code, 2) + self.assertIn('usage', json.loads(out.getvalue())['error']) + + class Ghostfat(unittest.TestCase): @staticmethod def fake_ghostfat_make(cmd, log, verbose, cwd=None): diff --git a/.claude/test/test_dfu_runtime.py b/.claude/test/test_dfu_runtime.py new file mode 100644 index 000000000..69692909c --- /dev/null +++ b/.claude/test/test_dfu_runtime.py @@ -0,0 +1,71 @@ +"""apps/dfu_runtime's nonce parser, compiled for the host and called through ctypes.""" +import ctypes +import importlib.util +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +SRC = ROOT / 'apps' / 'dfu_runtime' / 'src' +spec = importlib.util.spec_from_file_location('hil_test', ROOT / '.claude' / 'skills' / 'hil' / 'scripts' / 'hil_test.py') +ht = importlib.util.module_from_spec(spec) +spec.loader.exec_module(ht) +STRING_SIZE = len('HIL ') + 1 + 2 * 16 + + +def block(nonce=b'\x01\x23\x45\x67\x89\xab\xcd\xef\x10\x32\x54\x76\x98\xba\xdc\xfe', length=None, magic=b'TUF2NONC'): + n = len(nonce) if length is None else length + head = magic + n.to_bytes(4, 'little') + nonce + return bytearray(head + bytes(i & 0xff for i in range(len(head), 256))) + + +class NonceBlock(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + lib = Path(cls.tmp.name) / 'nonce.so' + subprocess.run(['cc', '-shared', '-fPIC', '-Wall', '-Werror', '-I', SRC, '-o', lib, SRC / 'nonce.c'], check=True) + cls.fn = ctypes.CDLL(str(lib)).nonce_block_to_string + cls.fn.restype = ctypes.c_bool + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def parse(self, blk): + self.assertEqual(len(blk), 256) + out = ctypes.create_string_buffer(b'\xaa' * (STRING_SIZE + 8), STRING_SIZE + 8) # guard bytes past the string + ok = self.fn((ctypes.c_uint8 * 256).from_buffer_copy(bytes(blk)), out) + return ok, out.raw + + def test_valid_lengths(self): + for nonce in (b'\x5a', bytes(range(0xf0, 0x100))): + ok, raw = self.parse(block(nonce)) + self.assertTrue(ok) + text = 'HIL ' + nonce.hex() + self.assertEqual(raw[:len(text) + 1], text.encode() + b'\0') + self.assertEqual(raw[len(text) + 1:], b'\xaa' * (len(raw) - len(text) - 1)) + + def test_the_runner_block_parses(self): + nonce = os.urandom(16) + ok, raw = self.parse(ht.nonce_block(nonce)) + self.assertTrue(ok) + self.assertEqual(raw[:37], f'HIL {nonce.hex()}'.encode() + b'\0') + + def test_invalid_blocks_leave_the_string(self): + bad_pad = block() + bad_pad[200] ^= 1 + erased = bytearray(b'\xff' * 256) + cases = {'length 0': block(b'', 0), 'length 17': block(bytes(17)), 'large LE length': block(length=0x01000010), + 'bad magic': block(magic=b'TUF2NONX'), 'bad padding': bad_pad, 'erased': erased} + for what, blk in cases.items(): + with self.subTest(what): + ok, raw = self.parse(blk) + self.assertFalse(ok) + self.assertEqual(raw, b'\xaa' * len(raw)) + + +if __name__ == '__main__': + unittest.main() diff --git a/.claude/test/test_hil.py b/.claude/test/test_hil.py index b47fd59ee..4f89156cb 100644 --- a/.claude/test/test_hil.py +++ b/.claude/test/test_hil.py @@ -6,6 +6,7 @@ import io import json import os +import shutil import signal import struct import subprocess @@ -89,7 +90,7 @@ def roster(tmp, **over): # lpc55 has no cmake build, so the runner reads no facts for it; most tests keep its layout as synthetic facts -LPC_FACTS = {'flashBase': 0, 'appStart': 0x10000, 'flashSize': 0x98000, 'vid': 0x1fc9, 'pid': 0x0094, +LPC_FACTS = {'flashBase': 0, 'binBase': 0, 'appStart': 0x10000, 'flashSize': 0x98000, 'vid': 0x1fc9, 'pid': 0x0094, 'boardId': 'LPC55S69-Xpresso-revA', 'family': 0x2abc77ec, 'vectors': 76, 'ram': 0x20000000, 'dblTapDelayMs': 500, 'dblTapReg': 0x4002C05C, 'dblTapMagic': 0xF01669EF} real_source_facts = ht.source_facts @@ -105,6 +106,7 @@ def setUp(self): self.addCleanup(t.cleanup) self.tmp = Path(t.name) for p in (mock.patch.object(ht, 'source_facts', source_facts), + mock.patch.object(ht, 'CONTAINERS', []), # a test's tracked container never reaches real docker later mock.patch.dict(os.environ, {'TINYUSB_DIR': str(self.tmp)})): p.start() self.addCleanup(p.stop) @@ -119,8 +121,8 @@ def load(self, cfg, name=BOARD): class Roster(Tmp): def test_a_valid_entry_loads_with_its_layout_from_the_source(self): b = self.load(roster(self.tmp)) - self.assertEqual((b['install'], b['app'], b['page'], b['device'], b['if'], b['speed']), - ((0, 0x10000), (0x10000, 0x98000), 512, 'LPC55S69_M33_0', 'SWD', '4000')) + self.assertEqual((b['install'], b['app'], b['device'], b['if'], b['speed']), + ((0, 0x10000), (0x10000, 0x98000), 'LPC55S69_M33_0', 'SWD', '4000')) self.assertEqual(b['helper'], str(self.tmp / HELPER)) def test_refusals(self): @@ -152,8 +154,7 @@ def test_the_lock_defaults_to_the_board_name_and_the_port_comes_from_it(self): def test_layout_refusals(self): b = ht.load_board(roster(self.tmp), BOARD) - cases = {'app start not page aligned': {'appStart': 0x10100}, - 'app start at the flash base': {'appStart': 0}, + cases = {'app start at the flash base': {'appStart': 0}, 'app start past the flash end': {'appStart': 0x98000}, 'flash past the ceiling': {'flashSize': 0x9E000}, 'flash below the floor': {'flashBase': 0x100000000 - 0x98000 - 1}} @@ -220,23 +221,55 @@ def test_the_build_record_is_read_by_board_name_and_must_be_cmake(self): f = build / '_bin' / f'tinyuf2.{kind}' f.write_bytes(b'TinyUF2 Bootloader 1.0\r\n') rows[f'_bin/{f.name}'] = ht.sha256(f) + for rel, data in (('tinyuf2.hex', b'hex'), ('tinyuf2.jlink', b'halt')): + (build / rel).write_bytes(data) + rows[rel] = ht.sha256(build / rel) + (build / '_bin' / 'apps').mkdir() + blinky = build / '_bin' / 'apps' / 'blinky.uf2' + blinky.write_bytes(ht.pack_uf2([(0x6000C000, bytes(256)), (0x6000C100, bytes(256))], self.facts['family'])) + rows['_bin/apps/blinky.uf2'] = ht.sha256(blinky) now = {'head': 'h', 'submodules': {}, 'deps': {}, 'dirty': False} rec = self.tmp / 'rec.json' - row = {'board': 'imxrt1060_evk', 'status': 'ok', 'backend': 'cmake', 'buildDir': str(build), 'sha256': rows} + row = {'board': 'imxrt1060_evk', 'status': 'ok', 'backend': 'cmake', 'buildDir': str(build), 'sha256': rows, + 'flash': {'target': 'tinyuf2-jlink', 'flasher': {'name': 'jlink', 'uid': '2', 'device': 'MIMXRT1062xxx6A'}, + 'inputs': ['tinyuf2.jlink', 'tinyuf2.hex']}} rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [row]})) + work = self.tmp / 'run' with mock.patch.object(ht.cb, 'identity', lambda ports: now): # an absolute buildDir ignores cb.ROOT - files, fw = ht.verify_record(rec, self.board, self.facts) - self.assertEqual((files['bin'].name, fw['version']), ('tinyuf2.bin', '1.0')) + files, fw = ht.verify_record(rec, self.board, self.facts, work) + self.assertEqual((files['bin'], fw['path'], fw['version']), + (work / 'record/_bin/tinyuf2.bin', str(build / '_bin/tinyuf2.bin'), '1.0')) + self.assertEqual((files['hex'], files['flash']['root'], files['flash']['sha256']['tinyuf2.hex']), + (build / 'tinyuf2.hex', build, rows['tinyuf2.hex'])) # the HEX the target loads, not a copy + for what, flash in (('another probe', dict(row['flash'], flasher={'name': 'jlink', 'uid': '9', 'device': 'MIMXRT1062xxx6A'})), + ('no binding', None)): + with self.subTest(what), self.assertRaisesRegex(ht.Refused, 'not bound to this roster flasher'): + rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [dict(row, flash=flash)]})) + ht.verify_record(rec, self.board, self.facts, work) + rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [row]})) + self.assertEqual((files['blinky'], files['blinkyBlocks']), (work / 'record/_bin/apps/blinky.uf2', 2)) + good = blinky.read_bytes() + for what, data in (('outside the app', ht.pack_uf2([(0x60000000, bytes(256))], self.facts['family'])), + ('another family', ht.pack_uf2([(0x6000C000, bytes(256))], 0x1234))): + with self.subTest(what): + blinky.write_bytes(data) + rows['_bin/apps/blinky.uf2'] = ht.sha256(blinky) + rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [dict(row, sha256=rows)]})) + with self.assertRaisesRegex(ht.Refused, 'blinky.uf2: block 0'): + ht.verify_record(rec, self.board, self.facts, work) + blinky.write_bytes(good) + rows['_bin/apps/blinky.uf2'] = ht.sha256(blinky) rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [dict(row, backend='make')]})) with self.assertRaisesRegex(ht.Refused, 'cmake builds only .*--backend cmake --shared'): - ht.verify_record(rec, self.board, self.facts) + ht.verify_record(rec, self.board, self.facts, work) rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [row]})) (build / '_bin' / 'tinyuf2.bin').unlink() with self.assertRaisesRegex(ht.Refused, 'tinyuf2.bin: missing'): - ht.verify_record(rec, self.board, self.facts) + ht.verify_record(rec, self.board, self.facts, work) + self.assertEqual(files['bin'].read_bytes(), b'TinyUF2 Bootloader 1.0\r\n') # verifybin's copy outlives a rebuild with mock.patch.object(ht.cb, 'identity', side_effect=ht.cb.Fail('git rev-parse HEAD failed')), \ self.assertRaisesRegex(ht.Refused, 'cannot establish the checkout identity: git rev-parse'): - ht.verify_record(rec, self.board, self.facts) + ht.verify_record(rec, self.board, self.facts, work) class SourceFacts(unittest.TestCase): @@ -251,6 +284,13 @@ def test_rt1060_facts_come_from_the_source(self): self.assertEqual((f['dblTapReg'], f['ram'], f['vectors']), (0x400D410C, 0x20000000, 174)) # SNVS LPGPR[3] self.assertEqual(f['family'], 0x4fb2d5bd) + def test_rt1011_facts_put_the_bin_at_the_fcb(self): + f = ht.source_facts({'port': 'mimxrt10xx', 'name': 'metro_m7_1011'}) + self.assertEqual((f['flashBase'], f['binBase'], f['appStart'], f['flashSize'], f['pid'], f['boardId']), + (0x60000000, 0x60000400, 0x6000C000, 0x800000, 0x00e1, 'MIMXRT1011-Metro-revA')) + self.assertEqual((f['dblTapReg'], f['ram']), (0x400D410C, 0x20000000)) + self.assertEqual(ht.source_facts(RT)['binBase'], 0x60000000) + def test_a_cmake_value_is_one_plain_set(self): t = tempfile.TemporaryDirectory() self.addCleanup(t.cleanup) @@ -271,6 +311,778 @@ def test_only_literals_and_the_supported_expressions_are_read(self): with self.subTest(expr), self.assertRaises(ht.Refused): ht.flash_addr(expr, base, 'x') if 'BASE' in expr or 'OTHER' in expr else ht.product(expr, 'x') +S3 = 'espressif_esp32s3_devkitc_1' +S3_PARTS = [('nvs', 1, 0x02, 0x9000, 0x5000), ('otadata', 1, 0x00, 0xe000, 0x2000), ('ota_0', 0, 0x10, 0x10000, 0x200000), + ('ota_1', 0, 0x11, 0x210000, 0x200000), ('uf2', 0, 0x00, 0x410000, 0x40000), ('ffat', 1, 0x81, 0x450000, 0x3b0000)] + + +def s3_roster(tmp, **over): + b = {'name': S3, 'uid': '84F703C084E4', 'flasher': {'name': 'esptool', 'uid': 'cp2102n', 'args': '--chip esp32s3'}, + 'tinyuf2': {'reentry': 'dfu-detach'}} + b.update(over) + cfg = tmp / 'esp.json' + cfg.write_text(json.dumps({'boards': [b]})) + return cfg + + +def partition_table(rows): + body = b''.join(b'\xaa\x50' + struct.pack(' 3: a loop, read until the file's 4 KiB struct.pack_into(' free with self.assertRaisesRegex(ValueError, 'chain broken'): - ht.Fat(self.vol(img)).file('CURRENT.UF2') + with ht.Fat(self.vol(img)) as fat: + fat.file('CURRENT.UF2') with self.assertRaisesRegex(ValueError, 'short read'): - ht.Fat(self.vol(fat_image()[:3 * 512])).entries() # the root dir is cut off + with ht.Fat(self.vol(fat_image()[:3 * 512])) as fat: # the root dir is cut off + fat.entries() def test_fat_read_runs_in_a_bounded_child_and_reports_errors_as_step_failures(self): log = self.tmp / 'log' @@ -461,6 +1307,11 @@ def test_fat_read_runs_in_a_bounded_child_and_reports_errors_as_step_failures(se self.assertEqual((r['data'], r['size'], r['entries']['INFO_UF2.TXT']), (b'a' * 512 + b'b' * 88, 1024, [2, 5])) with self.assertRaisesRegex(ht.StepFail, 'not on the volume'): ht.fat_read(self.vol(fat_image()), 'NOPE.TXT', 10, log) + for missing in (self.tmp / 'sdy', self.tmp / 'sdz'): + with self.assertRaises(ht.StepFail) as cm: + ht.fat_read(str(missing), 'CURRENT.UF2', 10, log) + self.assertEqual(str(cm.exception), 'CURRENT.UF2: FileNotFoundError: No such file or directory') # no node name + self.assertIn(str(missing), cm.exception.evidence['error']) self.assertEqual(ht.CHILDREN, []) @@ -488,6 +1339,83 @@ def popen_then_signalled(*a, **k): self.assertIsNotNone(launched[0].poll()) self.assertEqual(ht.CHILDREN, []) + def test_a_kill_reaches_a_grandchild_in_a_process_group_of_its_own(self): + # as ninja runs JLinkExe: setpgid moves it out of the child's group, not out of its session + pidfile = self.tmp / 'grandchild' + code = ('import os, subprocess, sys, time\n' + 'g = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], process_group=0)\n' + f'open({str(pidfile)!r}, "w").write(str(g.pid)); time.sleep(60)') + with self.assertRaisesRegex(ht.StepFail, 'timed out'): + ht.spawn([sys.executable, '-c', code], self.tmp / 'log', 2) + stat = Path(f'/proc/{pidfile.read_text()}/stat') + self.assertFalse(stat.exists() and stat.read_text().rsplit(')', 1)[1].split()[0] != 'Z') # gone, or a zombie + self.assertEqual(ht.CHILDREN, []) + + def test_a_grandchild_that_outlives_the_kill_stays_tracked_until_it_is_gone(self): + pidfile = self.tmp / 'grandchild' + code = ('import subprocess, sys, time\n' + 'g = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"], process_group=0)\n' + f'open({str(pidfile)!r}, "w").write(str(g.pid)); time.sleep(60)') + kill = os.kill + spared = lambda pid, sig: None if pidfile.exists() and pid == int(pidfile.read_text()) else kill(pid, sig) # noqa: E731 + with mock.patch.object(ht.os, 'kill', spared), mock.patch.object(ht, 'KILL_WAIT', 0.5), \ + self.assertRaisesRegex(ht.StepFail, 'timed out'): + ht.spawn([sys.executable, '-c', code], self.tmp / 'log', 2) + grandchild = int(pidfile.read_text()) + + def end(): + with contextlib.suppress(ProcessLookupError): + kill(grandchild, signal.SIGKILL) + self.addCleanup(end) + with mock.patch.object(ht.os, 'kill', spared), mock.patch.object(ht, 'KILL_WAIT', 0.5): + stuck = ht.stuck_children() # as cleanup asks after the failed step + self.assertEqual([x['pid'] for x in stuck], [grandchild]) + self.assertEqual(ht.still_alive(stuck), stuck) + kill(grandchild, signal.SIGKILL) + self.assertEqual((ht.stuck_children(), ht.CHILDREN), ([], [])) + + def test_a_zombie_is_not_a_live_survivor(self): + p = subprocess.Popen([sys.executable, '-c', 'pass']) + self.addCleanup(p.wait) + start = ht.proc_start(p.pid) + while ht.proc_stat(p.pid): # exited, not yet waited for + time.sleep(0.01) + self.assertTrue(Path(f'/proc/{p.pid}').exists()) + self.assertEqual(ht.still_alive([{'pid': p.pid, 'start': start}]), []) + + def test_a_flash_container_is_removed_by_name_after_its_launcher_and_a_survivor_blocks_recovery(self): + order, present = [], {'tinyuf2-hil-b': True} + def run(cmd, **k): + order.append(cmd) + return subprocess.CompletedProcess(cmd, 0, '', '') + with mock.patch.object(ht, 'CONTAINERS', ['tinyuf2-hil-b']), mock.patch.object(ht.subprocess, 'run', run), \ + mock.patch.object(ht, 'container_present', lambda name: present[name]), mock.patch.object(ht, 'KILL_WAIT', 0.1): + stuck = ht.stuck_children() + self.assertEqual((stuck, order[0], ht.CONTAINERS), ([{'container': 'tinyuf2-hil-b'}], ['docker', 'rm', '-f', 'tinyuf2-hil-b'], + ['tinyuf2-hil-b'])) + self.assertEqual(ht.still_alive(stuck), stuck) + present['tinyuf2-hil-b'] = False + self.assertEqual((ht.still_alive(stuck), ht.stuck_children(), ht.CONTAINERS), ([], [], [])) + + def test_a_docker_error_in_cleanup_is_a_survivor_not_an_escape(self): + for what, run, present in (('docker ps fails', None, mock.Mock(side_effect=ht.StepFail('docker ps failed: x'))), + ('docker rm times out', mock.Mock(side_effect=subprocess.TimeoutExpired('docker', 30)), None), + ('no docker', mock.Mock(side_effect=FileNotFoundError('docker')), None)): + with self.subTest(what), mock.patch.object(ht, 'CONTAINERS', ['tinyuf2-hil-b']), \ + mock.patch.object(ht.subprocess, 'run', run or (lambda cmd, **k: subprocess.CompletedProcess(cmd, 0, '', ''))), \ + mock.patch.object(ht, 'container_present', present or (lambda name: True)), mock.patch.object(ht, 'KILL_WAIT', 0.1): + self.assertEqual(ht.stuck_children(), [{'container': 'tinyuf2-hil-b'}]) + self.assertEqual(ht.still_alive([{'container': 'tinyuf2-hil-b'}]), [{'container': 'tinyuf2-hil-b'}]) + + def test_cleanup_reaps_the_launchers_before_it_checks_a_container(self): + order = [] + p = mock.Mock(pid=1, args=['docker', 'create']) + with mock.patch.object(ht, 'CHILDREN', [p]), mock.patch.object(ht, 'CONTAINERS', ['tinyuf2-hil-b']), \ + mock.patch.object(ht, 'reap', lambda q: order.append('reap') or ht.CHILDREN.remove(q) or []), \ + mock.patch.object(ht, 'container_gone', lambda name: order.append('container') or True): + self.assertEqual(ht.stuck_children(), []) + self.assertEqual(order, ['reap', 'container']) + def test_registering_leaves_the_childs_signal_mask_alone(self): code, out = ht.spawn(['grep', 'SigBlk', '/proc/self/status'], self.tmp / 'log', 5) self.assertEqual(code, 0, out) @@ -683,12 +1611,12 @@ def unblock(): # run_board leaves the signals blocked until exit; drop what is def run_board(self, battery_run, cleanup_error=None): cleaned = [] - def fake_invalidate(*a, **k): + def fake_restore(*a, **k): cleaned.append(True) if cleanup_error: raise cleanup_error - return {'pass': True, 'state': 'bootloader invalidated', 'evidence': {}} - with mock.patch.object(ht.Battery, 'run', battery_run), mock.patch.object(ht, 'invalidate', fake_invalidate): + return {'pass': True, 'state': 'blinky copied', 'evidence': {}} + with mock.patch.object(ht.Battery, 'run', battery_run), mock.patch.object(ht.Battery, 'restore', fake_restore): row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) return row, code, cleaned @@ -711,7 +1639,8 @@ def test_an_untouched_board_is_not_cleaned_up(self): def fail(bat, files, version): raise ht.StepFail('probe not found') row, code, cleaned = self.run_board(fail) - self.assertEqual((code, cleaned, row['cleanup']), (1, [], None)) + self.assertEqual((code, cleaned, row['cleanup']), (1, [], {'pass': True, 'state': 'untouched', 'evidence': {}, + 'heldAfter': False})) def test_any_failed_cleanup_hands_the_lock_to_a_holder_and_is_exit_4(self): def fail(bat, files, version): @@ -724,7 +1653,26 @@ def fail(bat, files, version): self.assertEqual((code, row['pass'], row['cleanup']['heldAfter'], row['cleanup']['holder']['pid']), (4, False, True, 1)) - def test_fail_at_cleanup_skips_the_invalidate_and_hands_the_board_to_a_holder(self): + def test_a_snapshot_the_cleanups_install_took_reaches_the_report_when_the_cleanup_fails(self): + def no_snapshot(bat, files, version): # the run's snapshot session failed after touching the board + bat.touched = True + raise ht.StepFail('snapshot session failed') + + def snapshot_then_fail(bat, *a): + bat.snap = {'nvs': '0' * 32} + raise ht.StepFail('read-back failed') + with mock.patch.object(ht.Battery, 'report_fields', lambda bat: {'snapshot': getattr(bat, 'snap', None)}), \ + mock.patch.object(ht.Battery, 'run', no_snapshot), mock.patch.object(ht.Battery, 'restore', snapshot_then_fail), \ + mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'reason': 'hil_test.py'})[1]): + row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) + self.assertEqual((code, row['cleanup']['pass'], row['snapshot']), (4, False, {'nvs': '0' * 32})) + + def test_a_step_or_method_of_the_other_flasher_is_refused_before_the_lock(self): + for attr, val, err in (('fail_at', 'recopy', 'has no step recopy'), ('try_reentry', 'dfu-detach', 're-enters by one of')): + with self.subTest(val), mock.patch.object(self.args, attr, val), self.assertRaisesRegex(ht.Refused, err): + ht.run_board(self.args, dict(self.board, reentry=None), self.facts, self.helper, self.tmp) + + def test_fail_at_cleanup_skips_the_restore_and_hands_the_board_to_a_holder(self): def ok(bat, files, version): bat.touched = True self.args.fail_at = 'cleanup' @@ -757,12 +1705,12 @@ def test_a_signal_during_cleanup_waits_for_it(self): def ok(bat, files, version): bat.touched = True - def invalidate_then_signalled(*a, **k): + def restore_then_signalled(*a, **k): os.kill(os.getpid(), signal.SIGTERM) # blocked until cleanup has ended return {'pass': True, 'state': 's', 'evidence': {}} signal.signal(signal.SIGTERM, ht.on_signal) - with mock.patch.object(ht.Battery, 'run', ok), mock.patch.object(ht, 'invalidate', invalidate_then_signalled): + with mock.patch.object(ht.Battery, 'run', ok), mock.patch.object(ht.Battery, 'restore', restore_then_signalled): row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) self.assertEqual((code, row['pass'], row['cleanup']['heldAfter']), (0, True, False)) self.assertIn('SIGTERM deferred', row['detail']) @@ -782,7 +1730,7 @@ def signalled_while_locking(board): self.assertEqual((code, row['locked'], row['ran']), (1, True, False)) self.assertIn('SIGTERM deferred', row['detail']) - def test_a_stuck_child_is_a_cleanup_failure_with_no_invalidate(self): + def test_a_stuck_child_is_a_cleanup_failure_with_no_restore(self): def fail(bat, files, version): bat.touched = True raise ht.StepFail('x') @@ -791,11 +1739,11 @@ def fail(bat, files, version): mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1, 'survivors': s})[1]): row, code, cleaned = self.run_board(fail) self.assertEqual((code, cleaned, row['cleanup']['survivors']), (4, [], stuck)) - self.assertIn('99', row['cleanup']['detail']) + self.assertEqual(row['cleanup']['detail'], 'StepFail: children survived SIGKILL; no flash may overlap them') def test_a_survivor_on_an_untouched_board_still_keeps_it_held(self): def fail(bat, files, version): - raise ht.StepFail('identity: probe hung') + raise ht.StepFail('install: probe hung') with mock.patch.object(ht, 'stuck_children', lambda: [{'pid': 99, 'start': 7, 'cmd': ['JLinkExe']}]), \ mock.patch.object(ht, 'hand_off', lambda fh, b, d, s: (fh.close(), {'pid': 1})[1]): row, code, cleaned = self.run_board(fail) @@ -810,11 +1758,11 @@ def interrupted(bat, files, version): os.kill(os.getpid(), signal.SIGTERM) self.fail('the signal did not interrupt the battery') - def invalidate_under_fire(*a, **k): + def restore_under_fire(*a, **k): for _ in range(3): os.kill(os.getpid(), signal.SIGTERM) return {'pass': True, 'state': 's', 'evidence': {}} - with mock.patch.object(ht.Battery, 'run', interrupted), mock.patch.object(ht, 'invalidate', invalidate_under_fire): + with mock.patch.object(ht.Battery, 'run', interrupted), mock.patch.object(ht.Battery, 'restore', restore_under_fire): row, code = ht.run_board(self.args, self.board, self.facts, self.helper, self.tmp) self.assertEqual((code, row['cleanup']['pass'], row['cleanup']['heldAfter']), (1, True, False)) self.assertIn('interrupted by SIGTERM', row['detail']) @@ -851,17 +1799,33 @@ def test_an_unusable_lock_dir_is_refused_not_run_unlocked(self): def test_fail_at_injects_a_criterion_failure(self): bat = ht.Battery(self.board, self.facts, self.tmp, 'dfu') - bat.step('identity', lambda: {}) + bat.step('install', lambda: {}) with self.assertRaisesRegex(ht.StepFail, 'injected'): bat.step('dfu', lambda: {}) +def jlink_target(hex_path, download='1 range affected (65536 bytes)', tail='O.K.\n'): + """tinyuf2-jlink's transcript through CMake and Ninja, in the shape the bench recorded on metro_m7_1011.""" + return (f'[1/1] cd /b && JLinkExe -device D -USB 1 -ExitOnError 1 -NoGui 1 -if swd -JTAGConf -1,-1 -speed auto ' + f'-CommandFile /b/tinyuf2.jlink\nJ-Link>halt\nConnecting to J-Link ...O.K.\nS/N: 1\nConnecting to target via SWD\n' + f'Cortex-M7 identified.\nJ-Link>loadfile {hex_path}\nResetTarget() start\n' + f'ResetTarget() end - Took 117ms\nDownloading file [{hex_path}]...\n' + f'Comparing flash [000%]100%] Done.\nJ-Link: Flash download: Bank 0 @ 0x60000000: {download}\n{tail}' + f'J-Link>r\nJ-Link>go\nJ-Link>exit\n\nScript processing completed.\n') + + +JLINK_NO_PROBE = ('[1/1] cd /b && JLinkExe -USB 999999999 -ExitOnError 1\nJ-Link>halt\nJ-Link connection not established yet but ' + 'required for command.\nConnecting to J-Link ...FAILED: Cannot connect to J-Link.\nFAILED: CMakeFiles/tinyuf2-jlink\n' + 'ninja: build stopped: subcommand failed.\n') + + class FakeRig: """A probe and a UF2 drive that behave as the battery expects; each test breaks one thing.""" - def __init__(self, bat, reentry): - self.bat, self.reentry, self.scripts, self.present = bat, reentry, [], True - self.r0, self.mem, self.double_reset, self.flash = 5, {}, False, set() + def __init__(self, bat): + self.bat, self.scripts, self.present = bat, [], True + self.r0, self.mem, self.double_reset = 5, {}, False + self.hex, self.app_survives = None, False # the install erases the app start (bench), so DFU follows it def probe(self, commands, judge=True): self.scripts.append(list(commands)) @@ -879,23 +1843,21 @@ def probe(self, commands, judge=True): elif c.startswith(f'w4 {ht.SCB_AIRCR:08X}'): # DFU only with the magic retained, or when a second reset lands in the bootloader's own window self.present = self.mem.pop(f['dblTapReg'], None) == f['dblTapMagic'] or self.double_reset - elif c == 'r' and self.reentry == 'app-invalidate': - self.present = True elif c.startswith('w4 '): _, a, v = c.split() self.mem[int(a, 16)] = int(v, 16) - elif c.startswith('loadbin '): - note = 'Skipped. Contents already match' if c in self.flash else '1 range affected (65536 bytes)' - self.flash.add(c) - out += f'J-Link>{c}\nJ-Link: Flash download: Bank 0 @ 0x60000000: {note}\nO.K.\n' return out def spawn(self, cmd, log, timeout, cwd=None): if cmd[0] == 'mcopy': self.present = False + if cmd[0] == 'cmake': + self.scripts.append(['target ' + cmd[-1]]) + self.present = not self.app_survives + return 0, jlink_target(self.hex) return 0, '' - def fat_read(self, dev, name, length, log): + def fat_read(self, dev, name, length, log, offset=0): f = self.bat.f if name == 'INFO_UF2.TXT': data = (f'TinyUF2 Bootloader v1\r\nModel: x\r\nBoard-ID: {f["boardId"]}\r\nDate: d\r\n' @@ -913,7 +1875,14 @@ def battery(self, reentry='retained-register', corrupt=None, commissioning=False facts = ht.source_facts(board) bat = ht.Battery(board, facts, self.tmp, commissioning=commissioning) bootloader_bin(self.tmp / 'c.bin') - rig = FakeRig(bat, reentry) + rig = FakeRig(bat) + build = self.tmp / 'build' + build.mkdir(exist_ok=True) + rig.hex = build / 'tinyuf2.hex' + rig.hex.write_text(':00000001FF\n') + self.files = {'bin': self.tmp / 'c.bin', 'hex': rig.hex, + 'flash': {'root': build, 'binding': {'target': 'tinyuf2-jlink'}, 'sha256': {'tinyuf2.hex': ht.sha256(rig.hex)}, + 'ninja': None}} (self.tmp / 'cur').mkdir() bat.probe.run = rig.probe clock = [0.0] # time passes only when the battery sleeps @@ -921,12 +1890,11 @@ def battery(self, reentry='retained-register', corrupt=None, commissioning=False def fat_read(*a): if a[1] == 'CURRENT.UF2': rig.current = bytearray(bat.fix['image']) - if reentry == 'app-invalidate': - rig.current[:board['page']] = bytes(board['page']) if corrupt: corrupt(rig.current) return rig.fat_read(*a) - for p in (mock.patch.object(ht, 'usb_device', lambda *a: '1-2' if rig.present else None), + for p in (mock.patch.object(ht.cb, 'flash_ready', lambda *a: None), + mock.patch.object(ht, 'usb_device', lambda *a: '1-2' if rig.present else None), mock.patch.object(ht, 'block_device', lambda p: '/dev/sdz'), mock.patch.object(ht, 'check_exclusive', lambda d: None), mock.patch.object(ht, 'fat_read', fat_read), @@ -938,26 +1906,22 @@ def fat_read(*a): return bat, rig def test_every_step_passes_and_resumes_only_after_a_halt(self): - for reentry in ht.REENTRY: - with self.subTest(reentry): - self.tmp = Path(tempfile.mkdtemp(dir=self.tmp)) - bat, rig = self.battery(reentry) - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') - self.assertEqual([x['step'] for x in bat.steps if x['pass']], list(ht.STEPS)) - image = self.tmp / 'program.bin' - self.assertEqual(rig.scripts[1], ['h', f'loadbin {image},0x0', f'verifybin {image},0x0', 'r', 'g']) - self.assertEqual(rig.scripts[2][:2], ['h', f'w4 {bat.f["dblTapReg"]:08X} 00000000']) # cleared in dfu - self.assertIn(f'sleep {2 * bat.f["dblTapDelayMs"] + 200}', rig.scripts[3]) - for script in rig.scripts: - if 'g' in script: - self.assertEqual(script[0], 'h', script) - if reentry == 'retained-register': - self.assertIn([f'w4 {ht.SCB_AIRCR:08X} {ht.AIRCR_SYSRESETREQ:08X}'], rig.scripts) + bat, rig = self.battery() + bat.run(self.files, 'v1') + self.assertEqual([x['step'] for x in bat.steps if x['pass']], list(ht.STEPS['jlink'])) + self.assertEqual(rig.scripts[:2], [['target tinyuf2-jlink'], ['h', f'verifybin {self.tmp / "c.bin"},0x0', 'r', 'g']]) + self.assertEqual(rig.scripts[2][:2], ['h', f'w4 {bat.f["dblTapReg"]:08X} 00000000']) # cleared in dfu + self.assertIn(f'sleep {2 * bat.f["dblTapDelayMs"] + 200}', rig.scripts[3]) + self.assertEqual(bat.steps[0]['evidence'], {'flashDownload': '1 range affected (65536 bytes)', 'verified': True}) + for script in rig.scripts: + if 'g' in script: + self.assertEqual(script[0], 'h', script) + self.assertIn([f'w4 {ht.SCB_AIRCR:08X} {ht.AIRCR_SYSRESETREQ:08X}'], rig.scripts) def test_the_whole_fixture_is_read_back_including_its_last_partial_block(self): bat, _ = self.battery(corrupt=lambda img: img.__setitem__(len(img) - 1, img[-1] ^ 1)) with self.assertRaisesRegex(ht.StepFail, 'read back the fixture'): - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + bat.run(self.files, 'v1') self.assertEqual(bat.steps[-1]['step'], 'reenter') def test_a_readback_failure_keeps_the_reentry_observations(self): @@ -972,7 +1936,7 @@ def test_a_readback_failure_keeps_the_reentry_observations(self): ht.fat_read = lambda *a: dict(read(*a), data=fat()) if a[1] == 'CURRENT.UF2' else read(*a) self.addCleanup(setattr, ht, 'fat_read', read) with self.assertRaises(ht.StepFail): - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + bat.run(self.files, 'v1') last = bat.steps[-1] self.assertEqual((last['step'], last['pass'], last['evidence']['failedAt']), ('reenter', False, 'readback')) self.assertTrue(last['evidence']['dfuCameUp'] and last['evidence']['magicConsumed']) @@ -983,7 +1947,7 @@ def test_commissioning_runs_the_control_first_and_no_probe_session_before_dfu(se bat, rig = self.battery(commissioning=True) dfu = bat.dfu bat.dfu = lambda v: rig.scripts.append('dfu observed') or dfu(v) - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + bat.run(self.files, 'v1') ev = bat.steps[-1]['evidence'] self.assertTrue(ev['controlBootedApp'] and ev['dfuCameUp'] and ev['magicConsumed']) aircr = [f'w4 {ht.SCB_AIRCR:08X} {ht.AIRCR_SYSRESETREQ:08X}'] @@ -1004,7 +1968,7 @@ def flaky(v): # the install's DFU passes; the re-entry's finds no exclusion pr return dfu(v) bat.dfu = flaky with self.assertRaises(ht.StepFail): - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + bat.run(self.files, 'v1') ev = bat.steps[-1]['evidence'] self.assertEqual((bat.steps[-1]['step'], ev['controlBootedApp'], 'dfuCameUp' in ev), ('reenter', True, False)) @@ -1012,59 +1976,119 @@ def test_a_control_that_lands_in_dfu_fails_without_a_decision(self): bat, rig = self.battery(commissioning=True) rig.double_reset = True with self.assertRaisesRegex(ht.StepFail, 'without the magic'): - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + bat.run(self.files, 'v1') self.assertIs(bat.steps[-1]['evidence']['controlBootedApp'], False) - def test_one_image_carries_the_bootloader_ff_padding_and_a_zero_page(self): + def test_restore_reinstalls_the_record_and_leaves_dfu_after_blinky(self): bat, rig = self.battery() - fw = bootloader_bin(self.tmp / 'fw.bin', size=0x5000) - bat.program(fw, 0, 0x10000) - image = (self.tmp / 'program.bin').read_bytes() - self.assertEqual(image, fw.read_bytes() + b'\xff' * 0xb000 + bytes(512)) - self.assertEqual(rig.scripts, [['h', f'loadbin {self.tmp / "program.bin"},0x0', - f'verifybin {self.tmp / "program.bin"},0x0', 'r', 'g']]) - self.assertEqual(bat.installed, (fw, 0)) # app-invalidate re-verifies the BIN, not the image - - def test_a_bin_that_fills_the_install_span_exactly_has_no_padding(self): + blinky = self.tmp / 'blinky.uf2' + blinky.write_bytes(ht.pack_uf2([(bat.f['appStart'], bytes(256))], bat.f['family'])) + copies = [] + with mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: copies.append(cmd) or rig.spawn(cmd, *a, **k)): + out = bat.restore(dict(self.files, blinky=blinky, blinkyBlocks=1), 'v1') + self.assertEqual((out['state'], out['evidence']['copy']), ('blinky copied', {'blocks': 1})) + self.assertEqual(rig.scripts, [['target tinyuf2-jlink'], ['h', f'verifybin {self.tmp / "c.bin"},0x0', 'r', 'g']]) + self.assertEqual(copies[1][4], blinky) # after the target's cmake + rig.present, polls = True, [] + def blip(*a): # DFU shows once, mid-way through the quiet period, then is gone again + if rig.present: + return '1-2' + polls.append(1) + return '1-2' if len(polls) == 3 else None + with mock.patch.object(ht, 'usb_device', blip), self.assertRaisesRegex(ht.StepFail, 'came back after the blinky copy'): + bat.restore(dict(self.files, blinky=blinky, blinkyBlocks=1), 'v1') + + def test_the_target_transcript_is_judged_not_its_exit(self): bat, _ = self.battery() - fw = bootloader_bin(self.tmp / 'fw.bin', size=0x10000) - bat.program(fw, 0, 0x10000) - self.assertEqual((self.tmp / 'program.bin').read_bytes(), fw.read_bytes() + bytes(512)) - - def test_a_bin_past_the_app_start_is_refused_before_any_probe_session(self): - bat, rig = self.battery() - fw = bootloader_bin(self.tmp / 'fw.bin', size=0x10001) - with self.assertRaisesRegex(ht.StepFail, 'overlaps the app start'): - bat.program(fw, 0, 0x10000) - self.assertEqual((rig.scripts, bat.touched), ([], False)) + hexp = self.files['hex'] + self.assertEqual(ht.judge_jlink_target(0, jlink_target(hexp), bat.b, hexp), + {'flashDownload': '1 range affected (65536 bytes)'}) + skip = 'Skipped. Contents already match' # an already-matching image is a success + self.assertEqual(ht.judge_jlink_target(0, jlink_target(hexp, skip), bat.b, hexp)['flashDownload'], skip) + cases = (('every command FAILED, exit 0', 0, JLINK_NO_PROBE, True), ('ninja failed', 1, JLINK_NO_PROBE, True), + ('another image', 0, jlink_target(self.tmp / 'other.hex'), False), + ('no O.K.', 0, jlink_target(hexp, tail=''), False), + ('a failure line', 0, jlink_target(hexp, tail='O.K.\nCannot connect to target.\n'), False), + ('cut short', 0, jlink_target(hexp).replace('Script processing completed.\n', ''), False), + ('a nonzero exit', 2, jlink_target(hexp), False), ('nothing', 0, '', False)) + for what, code, out, before in cases: + with self.subTest(what), self.assertRaisesRegex(ht.StepFail, 'tinyuf2-jlink failed') as e: + ht.judge_jlink_target(code, out, bat.b, hexp) + self.assertEqual(e.exception.evidence['beforeTarget'], before, what) + + def test_the_rt_reset_vector_note_passes_in_the_target_transcript_on_rt_devices_only(self): + hexp = self.tmp / 'tinyuf2.hex' + out = jlink_target(hexp).replace('ResetTarget() start\n', 'ResetTarget() start\nCould not find valid reset vector.\n' + 'Target halted on flash header read.\n') + ht.judge_jlink_target(0, out, {'device': 'MIMXRT1011DAE5A', 'probe': '1'}, hexp) + with self.assertRaises(ht.StepFail): + ht.judge_jlink_target(0, out, {'device': 'LPC55S69_M33_0', 'probe': '1'}, hexp) - def test_cleanup_zeroes_the_device_flash_floor_and_reverifies_it_from_a_new_session(self): + def test_the_target_runs_under_the_build_lock_between_two_input_checks(self): bat, rig = self.battery() - b = dict(bat.b, device='MIMXRT1062xxx6A') # the floor is the device's, never the candidate's layout - out = ht.invalidate(b, bat.f, self.tmp, bat) - zero = f'{bat.zero},0x60000000' - self.assertEqual(rig.scripts, [['h', f'loadbin {zero}', f'verifybin {zero}', 'r', 'g'], - ['h', f'loadbin {zero}', f'verifybin {zero}', 'g']]) - self.assertEqual((out['state'], out['evidence']), ('bootloader invalidated', {'zeroPage': '0x60000000', 'verified': True})) - - def test_cleanup_fails_when_the_new_session_has_to_write_the_zero_page_again(self): - bat, rig = self.battery() - b = dict(bat.b, device='MIMXRT1062xxx6A') - bat.probe.run = lambda commands, judge=True: rig.flash.clear() or rig.probe(commands, judge) - with self.assertRaisesRegex(ht.StepFail, 'did not persist'): - ht.invalidate(b, bat.f, self.tmp, bat) - - def test_app_invalidate_reverifies_the_bootloader_before_its_reset(self): - bat, rig = self.battery('app-invalidate') - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') - script = [x for x in rig.scripts if 'r' in x][-1] - self.assertLess(script.index(f'verifybin {self.tmp / "c.bin"},0x0'), script.index('r')) - self.assertTrue(bat.steps[-1]['evidence']['bootloaderVerified']) + fl, hexp = self.files['flash'], self.files['hex'] + judge = lambda code, out: ht.judge_jlink_target(code, out, bat.b, hexp) # noqa: E731 + cmake = ['cmake', '--build', fl['root'], '--target', 'tinyuf2-jlink'] + with self.subTest('a build holds the dir'), ht.cb.build_lock(fl['root']), \ + self.assertRaisesRegex(ht.StepFail, 'in use by a build or another flash'): + bat.flash_target(fl, cmake, 5, judge) + good = hexp.read_text() + hexp.write_text('changed') + with self.subTest('an input changed'), self.assertRaisesRegex(ht.StepFail, 'changed before the flash'): + bat.flash_target(fl, cmake, 5, judge) + hexp.write_text(good) + with self.subTest('a dry run with more than the flasher'), \ + mock.patch.object(ht.cb, 'flash_ready', side_effect=ht.cb.Fail('would not run only its flasher')), \ + self.assertRaisesRegex(ht.StepFail, 'only its flasher'): + bat.flash_target(fl, cmake, 5, judge) + self.assertEqual((rig.scripts, bat.touched), ([], False)) # none of these ran the target + for what, result, touched in (('no probe', (1, JLINK_NO_PROBE), False), + ('the target reached', (1, jlink_target(hexp, tail='Cannot connect to target.\n')), True)): + bat.touched = False + with self.subTest(what), mock.patch.object(ht, 'spawn', lambda *a, r=result, **k: r), \ + self.assertRaisesRegex(ht.StepFail, 'tinyuf2-jlink failed'): + bat.flash_target(fl, cmake, 5, judge) + self.assertEqual(bat.touched, touched, what) + bat.touched = False + spawn = rig.spawn + with self.subTest('rebuilt while flashing'), \ + mock.patch.object(ht, 'spawn', lambda cmd, *a, **k: (hexp.write_text('rebuilt'), spawn(cmd, *a, **k))[1]), \ + self.assertRaisesRegex(ht.StepFail, 'changed while flashing') as e: + bat.flash_target(fl, cmake, 5, judge) + self.assertEqual((bat.touched, e.exception.evidence['flashDownload']), (True, '1 range affected (65536 bytes)')) + + def test_dfu_after_an_install_reenters_only_when_dfu_never_enumerates(self): + aircr = [f'w4 {ht.SCB_AIRCR:08X} {ht.AIRCR_SYSRESETREQ:08X}'] + for what, reentry, survives, version, err in ( + ('a valid app booted', 'retained-register', True, 'v1', None), + ('uncommissioned', None, True, 'v1', 'has no retained-register re-entry'), + ('another failure', 'retained-register', False, 'v2', 'INFO_UF2')): + with self.subTest(what): + self.tmp = Path(tempfile.mkdtemp(dir=self.tmp)) + bat, rig = self.battery(reentry=reentry) + rig.app_survives = survives + bat.install(self.files) + if err: + with self.assertRaisesRegex(ht.StepFail, err): + bat.dfu_after_install(version) + self.assertNotIn(aircr, rig.scripts) + else: + ev = bat.dfu_after_install(version) + self.assertTrue(ev['dfuCameUp'] and ev['magicConsumed']) + self.assertIn('did not enumerate after the install', ev['claim']) + self.assertIn(aircr, rig.scripts) + rig.app_survives = True # the last case's rig: a re-entry whose final register read fails keeps what it saw + bat.install(self.files) + with mock.patch.object(bat.probe, 'mem32', side_effect=ht.StepFail('mem32 read gave no value')), \ + self.assertRaisesRegex(ht.StepFail, 'no value') as e: + bat.dfu_after_install('v1') + self.assertEqual((e.exception.evidence['dfuCameUp'], e.exception.evidence['usb']), (True, '1-2')) def test_a_wrong_version_on_the_drive_fails_dfu(self): + bat, _ = self.battery() with self.assertRaisesRegex(ht.StepFail, 'INFO_UF2'): - bat.run({'bin': self.tmp / 'c.bin'}, 'v2') + bat.run(self.files, 'v2') def test_a_left_magic_is_cleared_right_after_dfu_and_read_back(self): bat, rig = self.battery() @@ -1072,7 +2096,7 @@ def test_a_left_magic_is_cleared_right_after_dfu_and_read_back(self): rig.mem[reg] = bat.f['dblTapMagic'] dfu = bat.dfu bat.dfu = lambda v: rig.scripts.append('dfu observed') or dfu(v) - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + bat.run(self.files, 'v1') ev = [x for x in bat.steps if x['step'] == 'dfu'][0]['evidence'] self.assertEqual(ev['dblTapReg'], '0x0') self.assertEqual(rig.scripts[2:4], ['dfu observed', ['h', f'w4 {reg:08X} 00000000', f'mem32 {reg:08X} 1', 'g']]) @@ -1084,12 +2108,12 @@ def test_a_double_tap_clear_that_does_not_read_back_fails_dfu(self): bat.probe.run = rig.probe rig.mem[bat.f['dblTapReg']] = bat.f['dblTapMagic'] with self.assertRaisesRegex(ht.StepFail, 'reads back'): - bat.run({'bin': self.tmp / 'c.bin'}, 'v1') + bat.run(self.files, 'v1') self.assertEqual([(x['step'], x['pass']) for x in bat.steps][-1], ('dfu', False)) def test_a_write_the_bus_dropped_fails_its_read_back(self): bat, rig = self.battery() - rig.probe = lambda commands, judge=True: f'J-Link>mem32 4002C05C 1\n4002C05C = 00000000 \n' + rig.probe = lambda commands, judge=True: 'J-Link>mem32 4002C05C 1\n4002C05C = 00000000 \n' bat.probe.run = rig.probe with self.assertRaisesRegex(ht.StepFail, 'reads back 00000000'): bat.probe.write32(0x4002C05C, 0xF01669EF) @@ -1107,6 +2131,15 @@ def test_the_rt1062_missing_reset_vector_note_passes_only_with_its_completed_res self.assertRaisesRegex(ht.StepFail, 'JLinkExe failed'): probe.run(['r', 'g']) + def test_the_rt1011_missing_reset_vector_note_is_benign(self): + note = 'J-Link>r\nResetTarget() start\nCould not find valid reset vector.\nTarget halted on flash header read.\nJ-Link>g\n' + metro = ht.Probe({'probe': '1', 'device': 'MIMXRT1011DAE5A', 'if': 'SWD', 'speed': '4000'}, self.tmp, None) + with mock.patch.object(ht, 'spawn', lambda *a, **k: (0, note)): + metro.run(['r', 'g']) + with mock.patch.object(ht, 'spawn', lambda *a, **k: (0, note + 'Could not read memory.\n')), \ + self.assertRaisesRegex(ht.StepFail, 'JLinkExe failed'): + metro.run(['r', 'g']) + def test_only_the_probe_connect_crash_is_retried_once_and_any_crash_fails(self): probe = ht.Probe({'probe': '000726129165', 'device': 'd', 'if': 'SWD', 'speed': '4000'}, self.tmp, self.tmp / 'hil.log') banner = 'Connecting to J-Link ...O.K.\nFirmware: J-Link LPCXpresso V2\nS/N: 726129165\n' @@ -1138,6 +2171,17 @@ def test_a_failed_reset_fails_the_session(self): probe.run(['r', 'g']) +class StaysAway(unittest.TestCase): + def test_the_window_end_is_sampled_after_the_last_sleep(self): + clock = [0.0] + sleep = lambda t: clock.__setitem__(0, clock[0] + t) # noqa: E731 + with mock.patch.object(ht.time, 'monotonic', lambda: clock[0]), mock.patch.object(ht.time, 'sleep', sleep): + with self.assertRaisesRegex(ht.StepFail, 'came back'): # back at 0.9 s, inside a 1 s window + ht.stays_away(lambda: clock[0] >= 0.9, 1.0, 'came back', {}) + clock[0] = 0.0 + ht.stays_away(lambda: clock[0] > 1.0, 1.0, 'came back', {}) # back only after the window + + class AppRan(Tmp): """The probe transcript the app-ran step reads; its exact shape is bench-checked at commissioning.""" @@ -1160,17 +2204,18 @@ def test_regs_parse_two_dumps(self): class Decision(unittest.TestCase): def row(self, tried='retained-register', passed=True, failed_at=None, cleaned=True, held=False, **obs): - ev = {'controlBootedApp': True, 'dfuCameUp': True, 'magicConsumed': True, 'bootloaderVerified': True} + ev = {'controlBootedApp': True, 'dfuCameUp': True, 'magicConsumed': True, + 'detachAcked': True, 'appLeft': True} ev.update(obs) if failed_at: ev['failedAt'] = failed_at - steps = [{'step': s, 'pass': True, 'evidence': {}} for s in ht.STEPS[:-1]] + steps = [{'step': s, 'pass': True, 'evidence': {}} for s in ht.STEPS['jlink'][:-1]] steps.append({'step': 'reenter', 'pass': passed, 'evidence': ev}) return {'commissioning': True, 'tried': tried, 'pass': passed, 'steps': steps, 'cleanup': {'pass': cleaned, 'heldAfter': held}} def test_the_commissioning_decision(self): - for method in ht.REENTRY: + for method in (*ht.REENTRY['jlink'], *ht.REENTRY['esptool']): with self.subTest(method): self.assertEqual(ht.reentry_decision(self.row(method)), method) self.assertEqual(ht.reentry_decision(self.row(method, passed=False, failed_at='readback')), method) @@ -1179,7 +2224,6 @@ def test_the_commissioning_decision(self): self.assertIsNone(ht.reentry_decision(self.row(method, cleaned=False))) # failed cleanup self.assertIsNone(ht.reentry_decision(self.row(method, held=True))) self.assertIsNone(ht.reentry_decision(self.row(controlBootedApp=False))) - self.assertEqual(ht.reentry_decision(self.row('app-invalidate', controlBootedApp=False)), 'app-invalidate') early = self.row(passed=False, failed_at='readback') early['steps'][3]['pass'] = False self.assertIsNone(ht.reentry_decision(early)) # another step failed too @@ -1188,30 +2232,45 @@ def test_the_commissioning_decision(self): class Main(Tmp): - def test_manual_install_needs_no_reentry_and_programs_the_bin_with_a_zero_page(self): + def test_manual_install_runs_the_record_and_brings_up_dfu_with_reentry_only_on_trial(self): + for what, reentry, extra, want, calls in ( + ('uncommissioned', None, [], 0, [('install', None), ('dfu', 'v')]), + ('a trial', None, ['--try-reentry', 'retained-register'], 0, [('install', 'retained-register'), ('dfu', 'v')]), + ('a trial once commissioned', 'retained-register', ['--try-reentry', 'retained-register'], 2, [])): + seen = [] + cfg = roster(self.tmp, tinyuf2={'reentry': reentry}) + with self.subTest(what), contextlib.redirect_stdout(io.StringIO()), \ + mock.patch.object(ht.cb, 'BUILD_DIR', Path(tempfile.mkdtemp(dir=self.tmp))), \ + mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({'bin': 'b'}, {'version': 'v'})), \ + mock.patch.object(ht.Battery, 'install', lambda bat, f: seen.append(('install', bat.b['reentry'])) or {}), \ + mock.patch.object(ht.Battery, 'dfu_after_install', lambda bat, v: seen.append(('dfu', v)) or {}): + code = ht.main(['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'debug 7', *extra, str(cfg)]) + self.assertEqual((code, seen), (want, calls)) + + def test_an_install_that_fails_after_touching_the_board_is_exit_4(self): cfg = roster(self.tmp, tinyuf2={'reentry': None}) - fw = bootloader_bin(self.tmp / 'fw.bin') - programmed = [] - out = io.StringIO() - with contextlib.redirect_stdout(out), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp), \ - mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ - mock.patch.object(ht, 'exclusion_configured', lambda *a: None), \ - mock.patch.object(ht, 'check_jlink', lambda: None), \ - mock.patch.object(ht.Battery, 'program', lambda bat, b, base, a: programmed.append((b, base, a))): - code = ht.main(['install', '-b', BOARD, '--firmware', str(fw), '--sha256', ht.sha256(fw), - '--under-hold', 'debug 7', str(cfg)]) - self.assertEqual((code, programmed), (0, [(str(fw), 0, 0x10000)])) - - def test_invalidate_needs_no_candidate_source(self): + for touched, want in ((True, 4), (False, 1)): + def fail(bat, files, touched=touched): + bat.touched = touched + raise ht.StepFail('tinyuf2-jlink failed (exit 1)') + with self.subTest(touched), contextlib.redirect_stdout(io.StringIO()), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / f'i{want}'), mock.patch.object(ht, 'load_helper', lambda p: None), \ + mock.patch.object(ht, 'manual_holder', lambda *a: None), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), mock.patch.object(ht.Battery, 'install', fail): + self.assertEqual(ht.main(['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x', str(cfg)]), want) + + def test_a_restore_that_fails_after_touching_the_board_is_exit_4(self): cfg = roster(self.tmp, tinyuf2={'reentry': None}) - invalidated = [] - with contextlib.redirect_stdout(io.StringIO()), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp), \ - mock.patch.object(ht, 'source_facts', side_effect=AssertionError('read the candidate source')), \ - mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'manual_holder', lambda *a: None), \ - mock.patch.object(ht, 'check_jlink', lambda: None), mock.patch.object(ht, 'probe_users', lambda p: []), \ - mock.patch.object(ht, 'invalidate', lambda b, f, d: invalidated.append(f) or {'pass': True}): - self.assertEqual(ht.main(['invalidate', '-b', BOARD, '--under-hold', 'x', str(cfg)]), 0) - self.assertEqual(invalidated, [None]) + for touched, want in ((True, 4), (False, 1)): + def fail(bat, files, version, touched=touched): + bat.touched = touched + raise ht.StepFail('the DFU device came back after the blinky copy') + with self.subTest(touched), contextlib.redirect_stdout(io.StringIO()), \ + mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / str(want)), mock.patch.object(ht, 'load_helper', lambda p: None), \ + mock.patch.object(ht, 'manual_holder', lambda *a: None), mock.patch.object(ht, 'probe_users', lambda b: []), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), mock.patch.object(ht.Battery, 'restore', fail): + self.assertEqual(ht.main(['restore', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x', str(cfg)]), want) def test_a_recovery_while_another_runs_is_refused_before_it_checks_the_holder(self): cfg = roster(self.tmp, tinyuf2={'reentry': None}) @@ -1223,19 +2282,53 @@ def test_a_recovery_while_another_runs_is_refused_before_it_checks_the_holder(se mock.patch.object(ht, 'load_helper', lambda p: None), mock.patch.object(ht, 'check_jlink', lambda: None), \ mock.patch.object(ht, 'recovery_lock', busy), \ mock.patch.object(ht, 'prove_hold', side_effect=AssertionError('checked the holder')), \ - mock.patch.object(ht, 'invalidate', side_effect=AssertionError('probed')): - code = ht.main(['invalidate', '-b', BOARD, '--recover-holder', '4242', '--report', str(report), str(cfg)]) + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ + mock.patch.object(ht.Battery, 'restore', side_effect=AssertionError('probed')): + code = ht.main(['restore', '-b', BOARD, '--build-record', 'r.json', '--recover-holder', '4242', '--report', str(report), + str(cfg)]) self.assertEqual((code, json.loads(out.getvalue().splitlines()[-1])['error']), (2, f'{BOARD}: another recovery holds x.lock.recover')) self.assertEqual(busy.call_count, 1) + def test_an_esptool_recovery_checks_the_snapshot_in_the_failed_report(self): + holder = self.tmp / 'hil_test.py' + holder.write_text('import time; time.sleep(30)') + p = subprocess.Popen([sys.executable, str(holder)]) + self.addCleanup(lambda: (p.kill(), p.wait())) + # recovery reads the holder's cmdline, which names hil_test.py only once the child has exec'd + end = time.monotonic() + 10 + while b'hil_test.py' not in Path(f'/proc/{p.pid}/cmdline').read_bytes() and time.monotonic() < end: + time.sleep(0.01) + cfg = s3_roster(self.tmp) + keep = self.load(cfg, S3)['regions']['preserved'] + md5 = {n: 'ab' * 16 for n in keep} + report = self.tmp / 'failed.json' + for what, regions, want in (('the same layout', keep, 0), ('another layout', {'nvs': (0x9000, 0xe000)}, 2)): + report.write_text(json.dumps({'results': [{'board': S3, 'cleanup': {'holder': {'pid': p.pid}, 'state': 'unknown'}, + 'snapshot': {'preserved': {n: md5[n] for n in regions}, + 'preservedRegions': {n: list(r) for n, r in regions.items()}}}]})) + seen = [] + with self.subTest(what), contextlib.redirect_stdout(io.StringIO()), mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / str(want)), \ + mock.patch.object(ht, 'load_helper', lambda p: mock.Mock()), \ + mock.patch.object(ht, 'admit_record', lambda *a: ({}, {'version': 'v'})), \ + mock.patch.object(ht, 'recovery_lock', lambda *a: mock.Mock()), mock.patch.object(ht, 'prove_hold', lambda *a, **k: {}), \ + mock.patch.object(ht, 'probe_users', lambda b: []), mock.patch.object(ht.os, 'kill', lambda *a: None), \ + mock.patch.object(ht, 'wait_for', lambda *a: True), \ + mock.patch.object(ht.EspBattery, 'restore', lambda bat, f, v: seen.append(bat.preserved) or {'pass': True}): + code = ht.main(['restore', '-b', S3, '--build-record', 'r.json', '--recover-holder', str(p.pid), '--report', + str(report), str(cfg)]) + self.assertEqual(code, want) + self.assertEqual(seen, [md5] if want == 0 else []) + def test_no_jlinkexe_programs_nothing_on_any_path(self): fw = bootloader_bin(self.tmp / 'fw.bin') cfg = roster(self.tmp, tinyuf2={'reentry': None}) probed = [] - for argv in (['install', '-b', BOARD, '--firmware', str(fw), '--sha256', ht.sha256(fw), '--under-hold', 'x'], - ['invalidate', '-b', BOARD, '--under-hold', 'x']): + for argv in (['install', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x'], + ['restore', '-b', BOARD, '--build-record', 'r.json', '--under-hold', 'x']): with self.subTest(argv[0]), contextlib.redirect_stdout(io.StringIO()), \ + mock.patch.object(ht, 'verify_record', lambda *a: ({'hex': 'x.hex', 'bin': fw}, {'version': 'v'})), \ + mock.patch.object(ht, 'parse_hex', lambda p: {0: 1}), mock.patch.object(ht, 'hex_matches_bin', lambda *a: True), \ mock.patch.object(ht.cb, 'BUILD_DIR', self.tmp / argv[0]), mock.patch.object(ht, 'load_helper', lambda p: None), \ mock.patch.object(ht, 'manual_holder', lambda *a: None), mock.patch.object(ht, 'exclusion_configured', lambda *a: None), \ mock.patch.object(ht.shutil, 'which', lambda t: None if t == 'JLinkExe' else '/usr/bin/' + t), \ diff --git a/apps/dfu_runtime/src/dfu_runtime.h b/apps/dfu_runtime/src/dfu_runtime.h new file mode 100644 index 000000000..38978f3ec --- /dev/null +++ b/apps/dfu_runtime/src/dfu_runtime.h @@ -0,0 +1,48 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef DFU_RUNTIME_H_ +#define DFU_RUNTIME_H_ + +#include +#include + +// Optional block a UF2 may carry past the image to label one copy of the app: magic, nonce length (LE, 1..16), +// nonce, then every remaining byte equal to its offset in the block. A valid block makes the DFU runtime +// interface string "HIL ". +#define NONCE_BLOCK_MAGIC "TUF2NONC" +#define NONCE_BLOCK_SIZE 256 +#define NONCE_MAX_LEN 16 +#define NONCE_STRING_SIZE (sizeof("HIL ") + 2 * NONCE_MAX_LEN) + +// Reboot into TinyUF2 DFU mode, no return +void board_reboot_to_bootloader(void); + +// Read the nonce block from where the port keeps it; false when it cannot be read +bool board_nonce_block_read(uint8_t block[NONCE_BLOCK_SIZE]); + +// "HIL " into str when the block is valid; str is left untouched otherwise +bool nonce_block_to_string(uint8_t const block[NONCE_BLOCK_SIZE], char str[NONCE_STRING_SIZE]); + +#endif diff --git a/apps/dfu_runtime/src/main.c b/apps/dfu_runtime/src/main.c new file mode 100644 index 000000000..786804878 --- /dev/null +++ b/apps/dfu_runtime/src/main.c @@ -0,0 +1,69 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "board_api.h" +#include "tusb.h" +#include "dfu_runtime.h" + +// Example app with a single USB DFU runtime interface: a DFU_DETACH request (e.g. dfu-util -e) reboots into TinyUF2. + +// lets the DETACH status stage complete before the device leaves the bus +#define DETACH_REBOOT_DELAY_MS 100 + +char dfu_rt_string[NONCE_STRING_SIZE] = "DFU runtime"; + +static volatile uint32_t _ms; +static volatile bool _detached; +static uint32_t _detach_ms; + +int main(void) { + uint8_t block[NONCE_BLOCK_SIZE]; + board_init(); + if (board_nonce_block_read(block)) { + nonce_block_to_string(block, dfu_rt_string); + } + board_timer_start(1); + + board_dfu_init(); + tud_init(BOARD_TUD_RHPORT); + + while (1) { + // bounded wait so an RTOS port still reaches the reboot check when the bus goes quiet + tud_task_ext(10, false); + + if (_detached && (uint32_t) (_ms - _detach_ms) >= DETACH_REBOOT_DELAY_MS) { + board_reboot_to_bootloader(); + } + } +} + +void board_timer_handler(void) { + _ms++; +} + +// TinyUSB calls this right after queuing the DETACH status stage, before the host has it +void tud_dfu_runtime_reboot_to_dfu_cb(void) { + _detach_ms = _ms; + _detached = true; +} diff --git a/apps/dfu_runtime/src/nonce.c b/apps/dfu_runtime/src/nonce.c new file mode 100644 index 000000000..43e2b7bca --- /dev/null +++ b/apps/dfu_runtime/src/nonce.c @@ -0,0 +1,52 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include +#include "dfu_runtime.h" + +bool nonce_block_to_string(uint8_t const block[NONCE_BLOCK_SIZE], char str[NONCE_STRING_SIZE]) { + if (memcmp(block, NONCE_BLOCK_MAGIC, 8) != 0) { + return false; + } + + uint32_t const len = block[8] | (block[9] << 8) | ((uint32_t) block[10] << 16) | ((uint32_t) block[11] << 24); + if (len == 0 || len > NONCE_MAX_LEN) { + return false; + } + for (uint32_t i = 12 + len; i < NONCE_BLOCK_SIZE; i++) { + if (block[i] != (uint8_t) i) { + return false; + } + } + + static const char hex[] = "0123456789abcdef"; + memcpy(str, "HIL ", 4); + str += 4; + for (uint32_t i = 0; i < len; i++) { + *str++ = hex[block[12 + i] >> 4]; + *str++ = hex[block[12 + i] & 0xf]; + } + *str = '\0'; + return true; +} diff --git a/apps/dfu_runtime/src/tusb_config.h b/apps/dfu_runtime/src/tusb_config.h new file mode 100644 index 000000000..0f57b4e08 --- /dev/null +++ b/apps/dfu_runtime/src/tusb_config.h @@ -0,0 +1,60 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef TUSB_CONFIG_H_ +#define TUSB_CONFIG_H_ + +#ifndef BOARD_TUD_RHPORT +#define BOARD_TUD_RHPORT 0 +#endif + +#ifndef CFG_TUSB_MCU +#error CFG_TUSB_MCU must be defined +#endif + +#ifndef CFG_TUSB_OS +#define CFG_TUSB_OS OPT_OS_NONE +#endif + +#ifndef CFG_TUSB_DEBUG +#define CFG_TUSB_DEBUG 0 +#endif + +#define CFG_TUD_ENABLED 1 + +#ifndef CFG_TUSB_MEM_SECTION +#define CFG_TUSB_MEM_SECTION +#endif + +#ifndef CFG_TUSB_MEM_ALIGN +#define CFG_TUSB_MEM_ALIGN __attribute__ ((aligned(4))) +#endif + +#ifndef CFG_TUD_ENDPOINT0_SIZE +#define CFG_TUD_ENDPOINT0_SIZE 64 +#endif + +#define CFG_TUD_DFU_RUNTIME 1 + +#endif diff --git a/apps/dfu_runtime/src/usb_descriptors.c b/apps/dfu_runtime/src/usb_descriptors.c new file mode 100644 index 000000000..68eab056e --- /dev/null +++ b/apps/dfu_runtime/src/usb_descriptors.c @@ -0,0 +1,125 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "board_api.h" +#include "tusb.h" + +// TinyUSB example IDs: 0x4000 is its product ID for a device with none of CDC, MSC, HID, MIDI or vendor +#define USB_VID 0xCafe +#define USB_PID 0x4000 + +extern char dfu_rt_string[]; + +tusb_desc_device_t const desc_device = { + .bLength = sizeof(tusb_desc_device_t), + .bDescriptorType = TUSB_DESC_DEVICE, + .bcdUSB = 0x0200, + .bDeviceClass = 0x00, + .bDeviceSubClass = 0x00, + .bDeviceProtocol = 0x00, + .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, + .idVendor = USB_VID, + .idProduct = USB_PID, + .bcdDevice = 0x0100, + .iManufacturer = 0x01, + .iProduct = 0x02, + .iSerialNumber = 0x03, + .bNumConfigurations = 0x01 +}; + +uint8_t const* tud_descriptor_device_cb(void) { + return (uint8_t const*) &desc_device; +} + +enum { + ITF_NUM_DFU_RT, + ITF_NUM_TOTAL +}; + +#define CONFIG_TOTAL_LEN (TUD_CONFIG_DESC_LEN + TUD_DFU_RT_DESC_LEN) + +// attributes 0x0d: bitWillDetach, bitManifestationTolerant, bitCanDnload; with bitWillDetach the device +// leaves the bus itself after DETACH and the host issues no USB reset +uint8_t const desc_configuration[] = { + TUD_CONFIG_DESCRIPTOR(1, ITF_NUM_TOTAL, 0, CONFIG_TOTAL_LEN, 0x00, 100), + TUD_DFU_RT_DESCRIPTOR(ITF_NUM_DFU_RT, 4, 0x0d, 1000, 4096), +}; + +uint8_t const* tud_descriptor_configuration_cb(uint8_t index) { + (void) index; + return desc_configuration; +} + +enum { + STRID_LANGID = 0, + STRID_MANUFACTURER, + STRID_PRODUCT, + STRID_SERIAL, + STRID_DFU_RT, +}; + +static uint16_t _desc_str[48 + 1]; + +uint16_t const* tud_descriptor_string_cb(uint8_t index, uint16_t langid) { + (void) langid; + char const* str = NULL; + uint8_t chr_count = 0; + + switch (index) { + case STRID_LANGID: + _desc_str[1] = 0x0409; + chr_count = 1; + break; + + case STRID_MANUFACTURER: str = "TinyUF2"; break; + case STRID_PRODUCT: str = "TinyUF2 DFU runtime example"; break; + case STRID_DFU_RT: str = dfu_rt_string; break; + + // same format as TinyUF2's serial, so the app and the bootloader report one identity + case STRID_SERIAL: { + uint8_t serial_id[16] TU_ATTR_ALIGNED(4); + uint8_t serial_len = board_usb_get_serial(serial_id); + static const char hex[] = "0123456789ABCDEF"; + chr_count = 2 * serial_len; + for (uint8_t i = 0; i < serial_len; i++) { + _desc_str[1 + 2 * i] = hex[serial_id[i] >> 4]; + _desc_str[2 + 2 * i] = hex[serial_id[i] & 0xf]; + } + break; + } + + default: + return NULL; + } + + if (str != NULL) { + chr_count = (uint8_t) TU_MIN(strlen(str), TU_ARRAY_SIZE(_desc_str) - 1); + for (uint8_t i = 0; i < chr_count; i++) { + _desc_str[1 + i] = str[i]; + } + } + + _desc_str[0] = (uint16_t) ((TUSB_DESC_STRING << 8) | (2 * chr_count + 2)); + return _desc_str; +} diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index 06a88ca87..fd6ef2912 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -110,6 +110,16 @@ externalproject_add(blinky BUILD_ALWAYS 1 ) +# the HIL battery's fixture: out of ALL, so no release carries it; build it with --target dfu_runtime +externalproject_add(dfu_runtime + SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/apps/dfu_runtime + BINARY_DIR ${CMAKE_BINARY_DIR}/apps/dfu_runtime + CMAKE_ARGS -DBOARD=${BOARD} -DARTIFACT_PATH=${ARTIFACT_PATH} + INSTALL_COMMAND "" + BUILD_ALWAYS 1 + EXCLUDE_FROM_ALL 1 + ) + # erase firmware need to use reset reason, not implemented yet #externalproject_add(erase_firmware # SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/apps/erase_firmware diff --git a/ports/espressif/apps/dfu_runtime/CMakeLists.txt b/ports/espressif/apps/dfu_runtime/CMakeLists.txt new file mode 100644 index 000000000..8a40b6e63 --- /dev/null +++ b/ports/espressif/apps/dfu_runtime/CMakeLists.txt @@ -0,0 +1,14 @@ +# The following five lines of boilerplate have to be in your project's +# CMakeLists in this exact order for cmake to work correctly +cmake_minimum_required(VERSION 3.17) + +include(${CMAKE_CURRENT_LIST_DIR}/../../../family_support.cmake) + +# No shared boards/tinyusb_src components: the app has its own TinyUSB configuration and no board_flash.c +set(SDKCONFIG ${CMAKE_BINARY_DIR}/sdkconfig) + +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +idf_build_set_property(DEPENDENCIES_LOCK ${CMAKE_BINARY_DIR}/dependencies.lock) # keep the source tree clean + +project(dfu_runtime) +family_gen_uf2(dfu_runtime ${UF2_FAMILY_ID}) diff --git a/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt b/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt new file mode 100644 index 000000000..aac46bf14 --- /dev/null +++ b/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt @@ -0,0 +1,25 @@ +idf_build_get_property(target IDF_TARGET) +string(TOUPPER ${target} target_upper) + +set(app_src ${TOP}/apps/dfu_runtime/src) +set(tusb_src ${TOP}/lib/tinyusb/src) + +idf_component_register(SRCS + board_dfu_runtime.c + ${app_src}/main.c + ${app_src}/nonce.c + ${app_src}/usb_descriptors.c + ${tusb_src}/tusb.c + ${tusb_src}/common/tusb_fifo.c + ${tusb_src}/device/usbd.c + ${tusb_src}/device/usbd_control.c + ${tusb_src}/class/dfu/dfu_rt_device.c + ${tusb_src}/portable/synopsys/dwc2/dcd_dwc2.c + ${tusb_src}/portable/synopsys/dwc2/dwc2_common.c + INCLUDE_DIRS ${app_src} ${tusb_src} ${TOP}/src ${FAMILY_PATH}/boards ${BOARD_PATH} + PRIV_REQUIRES esp_timer esp_partition app_update usb) + +target_compile_definitions(${COMPONENT_LIB} PRIVATE + CFG_TUSB_MCU=OPT_MCU_${target_upper} + CFG_TUSB_OS=OPT_OS_FREERTOS + CFG_TUSB_OS_INC_PATH=freertos/) diff --git a/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c new file mode 100644 index 000000000..6d4d9d3f2 --- /dev/null +++ b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c @@ -0,0 +1,101 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "esp_mac.h" +#include "esp_ota_ops.h" +#include "esp_partition.h" +#include "esp_system.h" +#include "esp_timer.h" +#include "esp_private/system_internal.h" +#include "esp_private/usb_phy.h" + +#include "board_api.h" +#include "dfu_runtime.h" + +// Espressif glue for the DFU runtime example, without the bootloader's boards component + +// ports/espressif/bootloader_components/main/bootloader_start.c APP_REQUEST_UF2_RESET_HINT +#define APP_REQUEST_UF2_RESET_HINT 0x11F2 + +extern int main(void); + +static esp_timer_handle_t timer_hdl; +static usb_phy_handle_t phy_hdl; + +void app_main(void) { + main(); +} + +static void timer_cb(void* arg) { + (void) arg; + board_timer_handler(); +} + +void board_init(void) { + esp_timer_create_args_t const args = { .callback = timer_cb }; + esp_timer_create(&args, &timer_hdl); +} + +void board_timer_start(uint32_t ms) { + esp_timer_stop(timer_hdl); + esp_timer_start_periodic(timer_hdl, ms * 1000); +} + +void board_timer_stop(void) { + esp_timer_stop(timer_hdl); +} + +void board_dfu_init(void) { + // same PHY setup as TinyUF2's boards.c board_dfu_init() + usb_phy_config_t const phy_conf = { + .controller = USB_PHY_CTRL_OTG, + .target = USB_PHY_TARGET_INT, + .otg_mode = USB_OTG_MODE_DEVICE, + // https://github.com/hathach/tinyusb/issues/2943#issuecomment-2601888322 + .otg_speed = USB_PHY_SPEED_UNDEFINED, + }; + usb_new_phy(&phy_conf, &phy_hdl); +} + +uint8_t board_usb_get_serial(uint8_t serial_id[16]) { + // factory MAC, as TinyUF2 reports it + esp_efuse_mac_get_default(serial_id); + return 6; +} + +void board_reboot_to_bootloader(void) { + // the call makes the link keep esp_reset_reason_set_hint() (ports/espressif/README.md) + (void) esp_reset_reason(); + esp_reset_reason_set_hint((esp_reset_reason_t) APP_REQUEST_UF2_RESET_HINT); + esp_restart(); +} + +// the HIL runner puts the nonce block in the running partition's last 256 bytes +bool board_nonce_block_read(uint8_t block[NONCE_BLOCK_SIZE]) { + esp_partition_t const* part = esp_ota_get_running_partition(); + if (part == NULL || part->size < NONCE_BLOCK_SIZE) { + return false; + } + return esp_partition_read(part, part->size - NONCE_BLOCK_SIZE, block, NONCE_BLOCK_SIZE) == ESP_OK; +} diff --git a/ports/espressif/apps/dfu_runtime/sdkconfig.defaults b/ports/espressif/apps/dfu_runtime/sdkconfig.defaults new file mode 100644 index 000000000..9d26177ac --- /dev/null +++ b/ports/espressif/apps/dfu_runtime/sdkconfig.defaults @@ -0,0 +1,16 @@ +CONFIG_IDF_CMAKE=y +CONFIG_FREERTOS_WATCHPOINT_END_OF_STACK=y +CONFIG_FREERTOS_SUPPORT_STATIC_ALLOCATION=y + +# Compiler options +CONFIG_COMPILER_OPTIMIZATION_SIZE=y +CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT=y + +# Newlib +CONFIG_NEWLIB_NANO_FORMAT=y + +# Virtual file system +CONFIG_VFS_SUPPORT_IO=n + +# Compatibility options +CONFIG_FLASHMODE_QIO=y diff --git a/test/hil/local.json b/test/hil/local.json index 9ccbefa63..d415ffa5a 100644 --- a/test/hil/local.json +++ b/test/hil/local.json @@ -5,6 +5,18 @@ "uid": "4629F865D2291713E000005002004200", "flasher": {"name": "jlink", "uid": "000726129165", "args": "-device MIMXRT1062xxx6A"}, "tinyuf2": {"lock": "mimxrt1060_evk", "reentry": "retained-register"} + }, + { + "name": "metro_m7_1011", + "uid": "7BE8715DD77116635100005002004200", + "flasher": {"name": "jlink", "uid": "000801012283", "args": "-device MIMXRT1011DAE5A"}, + "tinyuf2": {"reentry": "retained-register"} + }, + { + "name": "espressif_esp32s3_devkitc_1", + "uid": "7CDFA1E06FD0", + "flasher": {"name": "esptool", "uid": "8ecc58dfb1cdeb1185e7b806e93fd3f1", "args": "--chip esp32s3"}, + "tinyuf2": {"reentry": "dfu-detach"} } ] } From e7148ac669a3551fd7e04a1b18f8d5d0c6ed527f Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Tue, 13 Oct 2026 16:54:21 +0700 Subject: [PATCH 22/47] espressif: build with ESP-IDF v6.1 (#497) Move the espressif port, CI and the ESP32-S3 HIL to ESP-IDF v6.1 only. --- .claude/skills/build/scripts/check_build.py | 8 +++++--- .claude/skills/hil/SKILL.md | 2 +- .claude/skills/hil/scripts/hil_esp.py | 8 ++++---- .claude/skills/hil/scripts/hil_test.py | 12 +++++++----- .claude/test/test_build.py | 15 ++++++++------- .claude/test/test_hil.py | 19 ++++++++++--------- .github/workflows/build.yml | 2 +- apps/dfu_runtime/src/usb_descriptors.c | 8 ++++---- ports/espressif/README.md | 2 +- .../apps/dfu_runtime/main/CMakeLists.txt | 2 +- ports/espressif/boards/CMakeLists.txt | 2 +- ports/espressif/boards/board_flash.c | 1 - ports/espressif/boards/boards.c | 13 +++++++++---- .../bootloader_components/main/CMakeLists.txt | 17 ----------------- .../main/bootloader_start.c | 4 +++- ports/espressif/components/lcd/CMakeLists.txt | 3 ++- .../components/led_strip/CMakeLists.txt | 2 +- .../led_strip/src/led_strip_spi_dev.c | 1 + .../components/ssd1306/CMakeLists.txt | 3 ++- ports/espressif/components/ssd1306/ssd1306.c | 4 ++-- ports/espressif/components/ssd1306/ssd1306.h | 4 ++-- .../components/ssd1306/ssd1306_i2c.c | 1 + ports/espressif/sdkconfig.defaults | 3 --- src/ghostfat.c | 18 ++++++++++++++---- 24 files changed, 80 insertions(+), 74 deletions(-) diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index 1e8a88ccd..80fed36d9 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -708,9 +708,11 @@ def flash_binding(flasher, root, ninja): defs = dict(v.split('=', 1) for k, v in zip(argv[1:-2], argv[2:-2]) if k == '-D' and '=' in v) tool_argv = defs.get('SERIAL_TOOL', '').split(';') ok = len(argv) == 1 + 2 * len(defs) + 2 and argv[-2] == '-P' and Path(argv[-1]).name == 'run_serial_tool.cmake' and \ - defs.keys() == {'IDF_PATH', 'SERIAL_TOOL', 'SERIAL_TOOL_ARGS', 'WORKING_DIRECTORY'} and \ - Path(tool_argv[-3]).name == 'esptool.py' and tool_argv[-2:] == ['--chip', flasher['device']] and \ - defs['SERIAL_TOOL_ARGS'].split(';') == ['--before=default_reset', '--after=hard_reset', 'write_flash', '@flash_args'] and \ + defs.keys() == {'IDF_PATH', 'SERIAL_TOOL', 'SERIAL_TOOL_ARGS', 'SERIAL_TOOL_IS_WRITE_FLASH', + 'SERIAL_TOOL_FLASH_ARGS_FILE', 'WORKING_DIRECTORY'} and \ + tool_argv[1:] == ['-m', 'esptool', '--chip', flasher['device']] and \ + defs['SERIAL_TOOL_ARGS'].split(';') == ['--before=default-reset', '--after=hard-reset', 'write-flash', '@flash_args'] and \ + defs['SERIAL_TOOL_IS_WRITE_FLASH'] == '1' and defs['SERIAL_TOOL_FLASH_ARGS_FILE'] == 'flash_args' and \ defs['WORKING_DIRECTORY'] == str(root) # flash_args: one settings line, then " " per segment segs = (root / 'flash_args').read_text().splitlines()[1:] if (root / 'flash_args').is_file() else [] diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index 780da820b..e644dcfd0 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -67,7 +67,7 @@ The battery's steps on one board, all under one lock. The roster's probe serial A board's evidence is scoped as the report says. "app-ran" is execution after the bootloader handoff; on a J-Link board a run triggered by the upload alone is unverified (an esptool board's app-ran observes the copied app itself over USB). "reenter" by `retained-register` is the forced-entry path, not a physical double tap. On an esptool board, each esptool connection is a child of the runner (`esp_session.py`), and so is each USB control request (`usb_dfu.py`, pyusb): -1. **install:** a first connection takes the MD5s of every partition the battery never writes (nvs, ota_1, ffat on the 8 MB layout) as the run's snapshot, kept in the row's `snapshot` however the run ends; that session is the first to touch the board, which only a missing bridge (no tty with its serial), a container left by an earlier flash, or a failed or interrupted `docker create` of the flash container leaves untouched. Then ESP-IDF's `flash` target runs in CI's image (`cmake --build --target flash`), with the same lock, hash and dry-run checks as a J-Link install: the bridge's tty is the only device passed in, as `ESPPORT`, and the container is named `tinyuf2-hil-` so a survivor is found and ended by name. Its transcript must show every segment written at its offset and hash-verified, then the hard reset. A runner session then reads each segment back and takes those MD5s again, then resets once; the runner compares the read-back bytes with the build's segment files and the MD5s with the snapshot, and any difference fails the step. esptool rewrites the bootloader header from `flash_args`, which the build already holds (dio, 80 MHz, 8 MB): on the bench (2026-10-13, esptool 4.8.1) all four segments read back identical to the files. The roster's bridge selects the board (it is wired to that board only), so the chip is not identified, and its security eFuses and flash size are commissioning's to check. The evidence lists each write's span and the 4 KiB-aligned span covering it. Blank otadata boots the factory partition, TinyUF2. +1. **install:** a first connection takes the MD5s of every partition the battery never writes (nvs, ota_1, ffat on the 8 MB layout) as the run's snapshot, kept in the row's `snapshot` however the run ends; that session is the first to touch the board, which only a missing bridge (no tty with its serial), a container left by an earlier flash, or a failed or interrupted `docker create` of the flash container leaves untouched. Then ESP-IDF's `flash` target runs in CI's image (`cmake --build --target flash`), with the same lock, hash and dry-run checks as a J-Link install: the bridge's tty is the only device passed in, as `ESPPORT`, with `IDF_FLASH_FULL=1` so ESP-IDF v6 writes segments already in flash too, and the container is named `tinyuf2-hil-` so a survivor is found and ended by name. Its transcript must show every segment written at its offset and hash-verified, then the hard reset. A runner session then reads each segment back and takes those MD5s again, then resets once; the runner compares the read-back bytes with the build's segment files and the MD5s with the snapshot, and any difference fails the step. esptool rewrites the bootloader header from `flash_args`, which the build already holds (dio, 80 MHz, 8 MB): on the bench (2026-10-13, esptool 5.3.1 in espressif/idf:v6.1) all four segments read back identical to the files. The roster's bridge selects the board (it is wired to that board only), so the chip is not identified, and its security eFuses and flash size are commissioning's to check. The evidence lists each write's span and the 4 KiB-aligned span covering it. Blank otadata boots the factory partition, TinyUF2. 2. **dfu:** as above; `INFO_UF2.TXT` reports the app partition (ota_0) as its flash, and no register is written. 3. **copy:** `apps/dfu_runtime` from the record, plus a block carrying a fresh nonce in ota_0's last 256 bytes, where the app reads it, renumbered under one `numBlocks`; the image itself is never patched. The record is refused when the image reaches that block or TinyUF2's 64 KiB flash cache window over it (`FLASH_CACHE_SIZE`) leaves ota_0. The evidence lists the image and nonce spans and the cache windows covering them. 4. **app-ran:** the app enumerates with its own VID:PID and the board's serial on the DFU's USB path, its DFU runtime interface string read from the device is `HIL `, and TinyUF2 stays away for a quiet period. diff --git a/.claude/skills/hil/scripts/hil_esp.py b/.claude/skills/hil/scripts/hil_esp.py index a9f1d3231..9437fb15a 100644 --- a/.claude/skills/hil/scripts/hil_esp.py +++ b/.claude/skills/hil/scripts/hil_esp.py @@ -10,8 +10,8 @@ from check_build import FLASH, HIL_APP, IDF_CONFIG -# 2nd-stage bootloader offset per chip, independent of the build: the ROM loads it from here (IDF v5.3.2 -# bootloader/Kconfig.projbuild BOOTLOADER_OFFSET_IN_FLASH, esptool 5.2.0 targets/esp32s3.py; no checked TRM states it) +# 2nd-stage bootloader offset per chip, independent of the build: the ROM loads it from here (IDF v6.1 +# esp_rom/esp32s3/esp_rom_caps.h ESP_ROM_BOOTLOADER_OFFSET_FLASH, esptool 5.2.0 targets/esp32s3.py; no checked TRM states it) BOOTLOADER_OFFSET = {'esp32s3': 0x0} PARTITION_TABLE = (0x8000, 0x1000) # ports/espressif/sdkconfig.defaults CONFIG_PARTITION_TABLE_OFFSET; one sector APP, DATA = 0x00, 0x01 @@ -154,8 +154,8 @@ def admit(root, board, f, lay): if bad: raise Refused(f'generated sdkconfig differs from the candidate source: {bad}') settings, segs = read_flash_args((root / 'flash_args').read_text(), 'flash_args') - for k, v in (('flash_mode', gen.get('CONFIG_ESPTOOLPY_FLASHMODE')), ('flash_freq', gen.get('CONFIG_ESPTOOLPY_FLASHFREQ')), - ('flash_size', gen.get('CONFIG_ESPTOOLPY_FLASHSIZE'))): + for k, v in (('flash-mode', gen.get('CONFIG_ESPTOOLPY_FLASHMODE')), ('flash-freq', gen.get('CONFIG_ESPTOOLPY_FLASHFREQ')), + ('flash-size', gen.get('CONFIG_ESPTOOLPY_FLASHSIZE'))): if f'"{settings.get(k)}"' != v: raise Refused(f'flash_args --{k} {settings.get(k)} differs from the generated sdkconfig {v}') if sorted(segs.values()) != sorted(SEGMENTS): diff --git a/.claude/skills/hil/scripts/hil_test.py b/.claude/skills/hil/scripts/hil_test.py index 05fb9e261..6493e3aa1 100755 --- a/.claude/skills/hil/scripts/hil_test.py +++ b/.claude/skills/hil/scripts/hil_test.py @@ -253,8 +253,8 @@ def esp_source_facts(board, bdir): detectionDelayMs=int(delay.group(1)) if delay else num(define('ports/espressif/bootloader_components/main/bootloader_start.c', 'UF2_DETECTION_DELAY_MS'), 'UF2_DETECTION_DELAY_MS'), - appVid=num(define(f'apps/{cb.HIL_APP}/src/usb_descriptors.c', 'USB_VID'), 'USB_VID'), - appPid=num(define(f'apps/{cb.HIL_APP}/src/usb_descriptors.c', 'USB_PID'), 'USB_PID'), + appVid=num(define(f'apps/{cb.HIL_APP}/src/usb_descriptors.c', 'APP_USB_VID'), 'APP_USB_VID'), + appPid=num(define(f'apps/{cb.HIL_APP}/src/usb_descriptors.c', 'APP_USB_PID'), 'APP_USB_PID'), family=int(cmake_var('ports/espressif/family.cmake', f'UF2_FAMILY_ID_{board["device"]}'), 16), flashCache=product(define('ports/espressif/boards/board_flash.c', 'FLASH_CACHE_SIZE'), 'FLASH_CACHE_SIZE')) return facts @@ -563,7 +563,7 @@ def judge_jlink_target(code, out, board, hex_path): def judge_esp_target(code, out, segments): """ESP-IDF's flash target, through CMake and Ninja in CI's image: esptool's write of every segment at its offset, - each hash-verified, then the hard reset (bench transcript, esptool v4.8.1 in espressif/idf:v5.3.2).""" + each hash-verified, then the hard reset (bench transcript, esptool v5.3.1 in espressif/idf:v6.1).""" wrote = sorted(int(a, 16) for a in re.findall(r'^Wrote \d+ bytes \(\d+ compressed\) at 0x([0-9a-f]{8}) in ', out, re.M)) if code or wrote != sorted(off for off, _ in segments) or out.count('Hash of data verified.') != len(segments) or \ 'Hard resetting via RTS pin' not in out: @@ -1551,10 +1551,12 @@ def install(self, files): raise StepFail(f'container {name} from an earlier flash still exists: check what it runs, then docker rm -f {name}') fl = files['flash'] # created, not run: the container exists, and its name is tracked, before anything can start it; cleanup - # removes it by name however the run ends + # removes it by name however the run ends. IDF_FLASH_FULL: v6's fast reflash skips segments already in + # flash, and judge_esp_target needs every segment written and verified CONTAINERS.append(name) code, out = spawn(cb.idf_docker(fl['root'], 'create') + [ - '--name', name, '--device', tty, '--group-add', str(os.stat(tty).st_gid), '-e', f'ESPPORT={tty}', fl['image'], + '--name', name, '--device', tty, '--group-add', str(os.stat(tty).st_gid), '-e', f'ESPPORT={tty}', + '-e', 'IDF_FLASH_FULL=1', fl['image'], 'cmake', '--build', str(fl['root']), '--target', fl['binding']['target']], self.log, 60) if code: raise StepFail(f'docker create {name} failed (exit {code})', {'tail': out.strip().splitlines()[-3:]}) diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index 2bf67e86b..325865cd9 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -618,12 +618,12 @@ def run(cmd, log, verbose, cwd=cb.ROOT): inspect = mock.Mock(return_value=mock.Mock(returncode=0)) with mock.patch.object(cb, 'run', run), mock.patch.object(cb.subprocess, 'run', inspect), \ mock.patch.object(cb, 'git', lambda *a: str(self.build_dir)): - rec = cb.build_board('espressif_esp32s3_devkitc_1', 'idf', self.args, {'image': 'espressif/idf:v5.3.2'}) + rec = cb.build_board('espressif_esp32s3_devkitc_1', 'idf', self.args, {'image': 'espressif/idf:v6.1'}) self.assertEqual(rec['status'], 'ok', rec) self.assertLessEqual({'sdkconfig', '_bin/flash_args', '_bin/ota_data_initial.bin', '_bin/apps/dfu_runtime.uf2'}, set(rec['sha256'])) made.remove('sdkconfig') - rec = cb.build_board('espressif_esp32s3_devkitc_1', 'idf', self.args, {'image': 'espressif/idf:v5.3.2'}) + rec = cb.build_board('espressif_esp32s3_devkitc_1', 'idf', self.args, {'image': 'espressif/idf:v6.1'}) self.assertEqual(rec['status'], 'failed') self.assertIn('sdkconfig', rec['firstError']) @@ -674,12 +674,13 @@ def jlink_tree(self, root, script=None, options='-USB 000801012283 -ExitOnError def idf_tree(self, root, chip='esp32s3', extra='', args_file='flash_args'): root.mkdir(parents=True, exist_ok=True) - (root / 'flash_args').write_text('--flash_mode dio --flash_freq 80m --flash_size 8MB\n0x0 bootloader/bootloader.bin\n' + (root / 'flash_args').write_text('--flash-mode dio --flash-freq 80m --flash-size 8MB\n0x0 bootloader/bootloader.bin\n' '0x410000 tinyuf2.bin\n0x8000 partition_table/partition-table.bin\n' f'0xe000 ota_data_initial.bin\n{extra}') line = (f'cd /opt/esp/idf/components/esptool_py && cmake -D IDF_PATH=/opt/esp/idf ' - f'-D "SERIAL_TOOL=python;;/opt/esp/idf/components/esptool_py/esptool/esptool.py;--chip;{chip}" ' - f'-D "SERIAL_TOOL_ARGS=--before=default_reset;--after=hard_reset;write_flash;@{args_file}" ' + f'-D "SERIAL_TOOL=/opt/esp/python_env/idf6.1_py3.12_env/bin/python;-m;esptool;--chip;{chip}" ' + f'-D "SERIAL_TOOL_ARGS=--before=default-reset;--after=hard-reset;write-flash;@{args_file}" ' + f'-D SERIAL_TOOL_IS_WRITE_FLASH=1 -D SERIAL_TOOL_FLASH_ARGS_FILE={args_file} ' f'-D WORKING_DIRECTORY={root} -P /opt/esp/idf/components/esptool_py/run_serial_tool.cmake') return lambda d, args: subprocess.CompletedProcess(args, 0, line + '\n', '') @@ -904,11 +905,11 @@ def test_cmake_configure_failure_skips_the_build(self): def test_idf_runs_ci_image_with_the_git_common_dir_mounted(self): with self.capture([1]), mock.patch.object(cb, 'git', lambda *a: '/srv/repo/.git\n'): ok, _ = cb.build_idf('espressif_esp32s3_devkitm_1', self.root, ['LOG=2'], Path('log'), False, 'docker', - 'espressif/idf:v5.3.2') + 'espressif/idf:v6.1') self.assertFalse(ok) [cmd] = self.calls self.assertEqual(cmd[:2], ['docker', 'run']) - self.assertIn('espressif/idf:v5.3.2', cmd) + self.assertIn('espressif/idf:v6.1', cmd) mounts = {cmd[i + 1] for i, c in enumerate(cmd) if c == '-v'} self.assertLessEqual({f'{cb.ROOT}:{cb.ROOT}', '/srv/repo/.git:/srv/repo/.git'}, mounts) inner = cmd[-1] diff --git a/.claude/test/test_hil.py b/.claude/test/test_hil.py index 4f89156cb..19d1bc777 100644 --- a/.claude/test/test_hil.py +++ b/.claude/test/test_hil.py @@ -337,7 +337,7 @@ def idf_build(root, parts=S3_PARTS, flash_args=None, sdk=None): files = {'_bin/bootloader.bin': b'\xe9' + b'\0' * 0x5000, '_bin/tinyuf2.bin': b'\xe9' + b'TinyUF2 Bootloader 1.0\r\nBoard-ID: ESP32S3-DevKitC-v1.0\r\n' + b'\0' * 0x100, '_bin/partition-table.bin': partition_table(parts), '_bin/ota_data_initial.bin': b'\xff' * 0x2000, - '_bin/flash_args': flash_args or ('--flash_mode dio --flash_freq 80m --flash_size 8MB\n0x0 bootloader/bootloader.bin\n' + '_bin/flash_args': flash_args or ('--flash-mode dio --flash-freq 80m --flash-size 8MB\n0x0 bootloader/bootloader.bin\n' '0x410000 tinyuf2.bin\n0x8000 partition_table/partition-table.bin\n' '0xe000 ota_data_initial.bin\n'), 'sdkconfig': sdk or ('CONFIG_IDF_TARGET="esp32s3"\nCONFIG_ESPTOOLPY_FLASHMODE="dio"\nCONFIG_ESPTOOLPY_FLASHFREQ="80m"\n' @@ -369,12 +369,12 @@ def setUp(self): def record(self, row, idf=None): rec = self.tmp / 'rec.json' rec.write_text(json.dumps({'pass': True, 'input': self.now, 'boards': [row], - 'idf': idf or {'mode': 'docker', 'image': 'espressif/idf:v5.3.2'}})) + 'idf': idf or {'mode': 'docker', 'image': 'espressif/idf:v6.1'}})) return rec def verify(self, rec): with mock.patch.object(ht.cb, 'identity', lambda ports: self.now), \ - mock.patch.object(ht.cb, 'ci_idf_version', lambda: 'v5.3.2'): + mock.patch.object(ht.cb, 'ci_idf_version', lambda: 'v6.1'): return ht.verify_record(rec, self.board, self.facts, self.tmp / 'run') def test_the_facts_come_from_the_idf_port(self): @@ -424,10 +424,10 @@ def test_an_idf_record_gives_the_four_segments_at_their_regions(self): self.assertEqual(ht.sha256(p), row['sha256'][rel]) def test_idf_record_refusals(self): - fa = ('--flash_mode dio --flash_freq 80m --flash_size 8MB\n0x0 bootloader/bootloader.bin\n0x410000 tinyuf2.bin\n' + fa = ('--flash-mode dio --flash-freq 80m --flash-size 8MB\n0x0 bootloader/bootloader.bin\n0x410000 tinyuf2.bin\n' '0x8000 partition_table/partition-table.bin\n') cases = { - 'a local IDF': ({}, {'mode': 'local', 'image': 'espressif/idf:v5.3.2'}, 'CI\'s espressif/idf:v5.3.2 only'), + 'a local IDF': ({}, {'mode': 'local', 'image': 'espressif/idf:v6.1'}, 'CI\'s espressif/idf:v6.1 only'), 'another image': ({}, {'mode': 'docker', 'image': 'espressif/idf:v5.4'}, 'only'), 'a cmake build': ({'backend': 'cmake'}, None, 'idf builds only'), 'a segment moved': ({'flash_args': fa.replace('0x410000', '0x400000') + '0xe000 ota_data_initial.bin\n'}, None, @@ -546,7 +546,7 @@ def target(self, cmd): def files(self): root = self.tmp / 'b' return {'segments': self.segments, - 'flash': {'root': root, 'binding': {'target': 'flash'}, 'image': 'espressif/idf:v5.3.2', 'ninja': None, + 'flash': {'root': root, 'binding': {'target': 'flash'}, 'image': 'espressif/idf:v6.1', 'ninja': None, 'sha256': {rel: ht.sha256(root / rel) for rel in ht.cb.FLASH['esptool'][1]}}} @@ -575,6 +575,7 @@ def test_install_writes_the_segments_reads_them_back_and_keeps_the_preserved_reg self.assertEqual((docker[:2], start), (['docker', 'create'], ['docker', 'start', '-a', f'tinyuf2-hil-{S3}'])) self.assertEqual(docker[docker.index('--name') + 1], f'tinyuf2-hil-{S3}') self.assertEqual((docker[docker.index('--device') + 1], f'ESPPORT={tty}' in docker), (tty, True)) + self.assertIn('IDF_FLASH_FULL=1', docker) # every segment written, as judge_esp_target requires self.assertEqual(docker[-5:], ['cmake', '--build', str(self.tmp / 'b'), '--target', 'flash']) def test_a_session_that_fails_to_connect_writes_nothing(self): @@ -616,13 +617,13 @@ def test_an_admitted_record_installs(self): now = {'head': 'h', 'submodules': {}, 'deps': {}, 'dirty': False} rec = self.tmp / 'rec.json' rec.write_text(json.dumps({'pass': True, 'input': now, 'boards': [self.row], - 'idf': {'mode': 'docker', 'image': 'espressif/idf:v5.3.2'}})) - with mock.patch.object(ht.cb, 'identity', lambda ports: now), mock.patch.object(ht.cb, 'ci_idf_version', lambda: 'v5.3.2'): + 'idf': {'mode': 'docker', 'image': 'espressif/idf:v6.1'}})) + with mock.patch.object(ht.cb, 'identity', lambda ports: now), mock.patch.object(ht.cb, 'ci_idf_version', lambda: 'v6.1'): files, _ = ht.verify_record(rec, self.board, self.facts, self.work) self.bat = ht.EspBattery(self.board, self.facts, self.work) self.assertTrue(self.bat.install(files)['readBack']) self.assertEqual(self.targets[0][self.targets[0].index('--device') - 1], f'tinyuf2-hil-{S3}') - self.assertIn('espressif/idf:v5.3.2', self.targets[0]) + self.assertIn('espressif/idf:v6.1', self.targets[0]) def test_an_interrupted_create_leaves_the_name_for_cleanup_and_the_board_untouched(self): def interrupted(cmd): diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8564de080..14d5ba376 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -113,7 +113,7 @@ jobs: boards: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)['espressif'].board) }} build-system: 'make' toolchain: 'esp-idf' - toolchain_version: 'v5.3.2' + toolchain_version: 'v6.1' # --------------------------------------- # Build RISC-V diff --git a/apps/dfu_runtime/src/usb_descriptors.c b/apps/dfu_runtime/src/usb_descriptors.c index 68eab056e..430fd73f3 100644 --- a/apps/dfu_runtime/src/usb_descriptors.c +++ b/apps/dfu_runtime/src/usb_descriptors.c @@ -26,8 +26,8 @@ #include "tusb.h" // TinyUSB example IDs: 0x4000 is its product ID for a device with none of CDC, MSC, HID, MIDI or vendor -#define USB_VID 0xCafe -#define USB_PID 0x4000 +#define APP_USB_VID 0xCafe +#define APP_USB_PID 0x4000 extern char dfu_rt_string[]; @@ -39,8 +39,8 @@ tusb_desc_device_t const desc_device = { .bDeviceSubClass = 0x00, .bDeviceProtocol = 0x00, .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, - .idVendor = USB_VID, - .idProduct = USB_PID, + .idVendor = APP_USB_VID, + .idProduct = APP_USB_PID, .bcdDevice = 0x0100, .iManufacturer = 0x01, .iProduct = 0x02, diff --git a/ports/espressif/README.md b/ports/espressif/README.md index 2b250b4e6..d613ee892 100644 --- a/ports/espressif/README.md +++ b/ports/espressif/README.md @@ -2,7 +2,7 @@ The project is composed of customizing the 2nd stage bootloader from IDF and UF2 factory application as 3rd stage bootloader. -**Note**: IDF is actively developed and change very often, TinyUF2 is developed and tested with IDF v5.3.2. Should you have a problem please try to change your IDF version. +**Note**: IDF is actively developed and change very often, TinyUF2 is developed and tested with IDF v6.1. Should you have a problem please try to change your IDF version. ## Supported Boards diff --git a/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt b/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt index aac46bf14..dd0de6879 100644 --- a/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt +++ b/ports/espressif/apps/dfu_runtime/main/CMakeLists.txt @@ -17,7 +17,7 @@ idf_component_register(SRCS ${tusb_src}/portable/synopsys/dwc2/dcd_dwc2.c ${tusb_src}/portable/synopsys/dwc2/dwc2_common.c INCLUDE_DIRS ${app_src} ${tusb_src} ${TOP}/src ${FAMILY_PATH}/boards ${BOARD_PATH} - PRIV_REQUIRES esp_timer esp_partition app_update usb) + PRIV_REQUIRES esp_timer esp_partition app_update) target_compile_definitions(${COMPONENT_LIB} PRIVATE CFG_TUSB_MCU=OPT_MCU_${target_upper} diff --git a/ports/espressif/boards/CMakeLists.txt b/ports/espressif/boards/CMakeLists.txt index 5b80c35b9..02bb7d27f 100644 --- a/ports/espressif/boards/CMakeLists.txt +++ b/ports/espressif/boards/CMakeLists.txt @@ -1,4 +1,4 @@ idf_component_register(SRCS boards.c board_flash.c ${BOARD_SOURCES} INCLUDE_DIRS "." "${BOARD}" ${BOARD_INCLUDES} ${TOP}/src - PRIV_REQUIRES driver usb + PRIV_REQUIRES driver esp_driver_gpio esp_driver_spi esp_driver_ledc esp_hal_usb REQUIRES esp_timer app_update spi_flash led_strip lcd ssd1306 XPowersLib tinyusb_src) diff --git a/ports/espressif/boards/board_flash.c b/ports/espressif/boards/board_flash.c index 86cd0c7eb..4a9566ff0 100644 --- a/ports/espressif/boards/board_flash.c +++ b/ports/espressif/boards/board_flash.c @@ -28,7 +28,6 @@ #include "esp_partition.h" #include "esp_ota_ops.h" -#include "spi_flash_chip_driver.h" #include "board_api.h" #define FLASH_CACHE_SIZE (64*1024) diff --git a/ports/espressif/boards/boards.c b/ports/espressif/boards/boards.c index 4f596c33f..d2241301e 100644 --- a/ports/espressif/boards/boards.c +++ b/ports/espressif/boards/boards.c @@ -33,6 +33,7 @@ #include "soc/usb_periph.h" #include "esp_private/periph_ctrl.h" +#include "hal/usb_wrap_ll.h" #include "esp_partition.h" #include "esp_ota_ops.h" @@ -367,8 +368,10 @@ static void hw_cdc_reset_handler(void *arg) { static void usb_switch_to_cdc_jtag(void) { // Disable USB-OTG - periph_module_reset(PERIPH_USB_MODULE); - periph_module_disable(PERIPH_USB_MODULE); + PERIPH_RCC_ATOMIC() { + usb_wrap_ll_reset_register(); + usb_wrap_ll_enable_bus_clock(false); + } // Switch to hardware CDC+JTAG CLEAR_PERI_REG_MASK(RTC_CNTL_USB_CONF_REG, @@ -428,8 +431,10 @@ static void usb_switch_to_cdc_jtag(void) { static void IRAM_ATTR usb_persist_shutdown_handler(void) { // USB CDC Download: RESTART_BOOTLOADER #if CONFIG_IDF_TARGET_ESP32S2 - periph_module_reset(PERIPH_USB_MODULE); - periph_module_enable(PERIPH_USB_MODULE); + PERIPH_RCC_ATOMIC() { + usb_wrap_ll_reset_register(); + usb_wrap_ll_enable_bus_clock(true); + } #endif REG_WRITE(RTC_CNTL_OPTION1_REG, RTC_CNTL_FORCE_DOWNLOAD_BOOT); diff --git a/ports/espressif/bootloader_components/main/CMakeLists.txt b/ports/espressif/bootloader_components/main/CMakeLists.txt index 7cc560334..12cbce2de 100644 --- a/ports/espressif/bootloader_components/main/CMakeLists.txt +++ b/ports/espressif/bootloader_components/main/CMakeLists.txt @@ -1,20 +1,3 @@ idf_component_register(SRCS "bootloader_start.c" INCLUDE_DIRS "../../boards/${BOARD}" REQUIRES bootloader bootloader_support hal) - -idf_build_get_property(target IDF_TARGET) - -# TODO: [ESP32C5] IDF-9197 remove this when beta3 is removed -if(CONFIG_IDF_TARGET_ESP32C5_MP_VERSION) - set(target_folder "esp32c5/mp") -elseif(CONFIG_IDF_TARGET_ESP32C5_BETA3_VERSION) - set(target_folder "esp32c5/beta3") -else() - set(target_folder "${target}") -endif() - -# Use the linker script files from the actual bootloader -set(scripts "${IDF_PATH}/components/bootloader/subproject/main/ld/${target_folder}/bootloader.ld" - "${IDF_PATH}/components/bootloader/subproject/main/ld/${target_folder}/bootloader.rom.ld") - -target_linker_script(${COMPONENT_LIB} INTERFACE "${scripts}") diff --git a/ports/espressif/bootloader_components/main/bootloader_start.c b/ports/espressif/bootloader_components/main/bootloader_start.c index 22e12b157..402aabba7 100644 --- a/ports/espressif/bootloader_components/main/bootloader_start.c +++ b/ports/espressif/bootloader_components/main/bootloader_start.c @@ -4,6 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ #include +#include "sdkconfig.h" #include "esp_log.h" #include "esp_rom_sys.h" #include "bootloader_init.h" @@ -21,7 +22,6 @@ #include "board.h" #ifdef TCA9554_ADDR - #include "hal/i2c_types.h" // Using GPIO expander requires long reset delay (somehow) #define NEOPIXEL_RESET_DELAY ns2cycle(1000*1000) #endif @@ -270,11 +270,13 @@ static int selected_boot_partition(const bootloader_state_t *bs) { return boot_index; } +#if CONFIG_LIBC_NEWLIB // Return global reent struct if any newlib functions are linked to bootloader struct _reent *__getreent(void) { return _GLOBAL_REENT; } +#endif //--------------------------------------------------------------------+ // Board LED Indicator diff --git a/ports/espressif/components/lcd/CMakeLists.txt b/ports/espressif/components/lcd/CMakeLists.txt index 1c6d9643f..53b2927a3 100644 --- a/ports/espressif/components/lcd/CMakeLists.txt +++ b/ports/espressif/components/lcd/CMakeLists.txt @@ -1,3 +1,4 @@ idf_component_register(SRCS "lcd.c" INCLUDE_DIRS "include" - REQUIRES boards) + REQUIRES boards esp_driver_spi + PRIV_REQUIRES esp_driver_gpio) diff --git a/ports/espressif/components/led_strip/CMakeLists.txt b/ports/espressif/components/led_strip/CMakeLists.txt index 15de610cc..4738556a2 100644 --- a/ports/espressif/components/led_strip/CMakeLists.txt +++ b/ports/espressif/components/led_strip/CMakeLists.txt @@ -19,4 +19,4 @@ endif() idf_component_register(SRCS ${srcs} INCLUDE_DIRS "include" "interface" - REQUIRES "driver") + REQUIRES "esp_driver_rmt" "esp_driver_spi") diff --git a/ports/espressif/components/led_strip/src/led_strip_spi_dev.c b/ports/espressif/components/led_strip/src/led_strip_spi_dev.c index 12ea8fbf3..70458d1c6 100644 --- a/ports/espressif/components/led_strip/src/led_strip_spi_dev.c +++ b/ports/espressif/components/led_strip/src/led_strip_spi_dev.c @@ -8,6 +8,7 @@ #include #include "esp_log.h" #include "esp_check.h" +#include "esp_heap_caps.h" #include "esp_rom_gpio.h" #include "soc/spi_periph.h" #include "led_strip.h" diff --git a/ports/espressif/components/ssd1306/CMakeLists.txt b/ports/espressif/components/ssd1306/CMakeLists.txt index ac32f0ac7..98a730d77 100644 --- a/ports/espressif/components/ssd1306/CMakeLists.txt +++ b/ports/espressif/components/ssd1306/CMakeLists.txt @@ -1,5 +1,6 @@ set(component_srcs "ssd1306.c" "ssd1306_i2c.c" "ssd1306_spi.c") idf_component_register(SRCS "${component_srcs}" - PRIV_REQUIRES driver + REQUIRES esp_driver_spi + PRIV_REQUIRES driver esp_driver_gpio INCLUDE_DIRS ".") diff --git a/ports/espressif/components/ssd1306/ssd1306.c b/ports/espressif/components/ssd1306/ssd1306.c index 770da896d..e92b83d98 100644 --- a/ports/espressif/components/ssd1306/ssd1306.c +++ b/ports/espressif/components/ssd1306/ssd1306.c @@ -87,7 +87,7 @@ void ssd1306_display_image(SSD1306_t * dev, int page, int seg, uint8_t * images, memcpy(&dev->_page[page]._segs[seg], images, width); } -void ssd1306_display_text(SSD1306_t * dev, int page, char * text, int text_len, bool invert) +void ssd1306_display_text(SSD1306_t * dev, int page, const char * text, int text_len, bool invert) { if (page >= dev->_pages) return; int _text_len = text_len; @@ -113,7 +113,7 @@ void ssd1306_display_text(SSD1306_t * dev, int page, char * text, int text_len, // by Coert Vonk void -ssd1306_display_text_x3(SSD1306_t * dev, int page, char * text, int text_len, bool invert) +ssd1306_display_text_x3(SSD1306_t * dev, int page, const char * text, int text_len, bool invert) { if (page >= dev->_pages) return; int _text_len = text_len; diff --git a/ports/espressif/components/ssd1306/ssd1306.h b/ports/espressif/components/ssd1306/ssd1306.h index c512ca645..95e1f9461 100644 --- a/ports/espressif/components/ssd1306/ssd1306.h +++ b/ports/espressif/components/ssd1306/ssd1306.h @@ -114,8 +114,8 @@ void ssd1306_show_buffer(SSD1306_t * dev); void ssd1306_set_buffer(SSD1306_t * dev, uint8_t * buffer); void ssd1306_get_buffer(SSD1306_t * dev, uint8_t * buffer); void ssd1306_display_image(SSD1306_t * dev, int page, int seg, uint8_t * images, int width); -void ssd1306_display_text(SSD1306_t * dev, int page, char * text, int text_len, bool invert); -void ssd1306_display_text_x3(SSD1306_t * dev, int page, char * text, int text_len, bool invert); +void ssd1306_display_text(SSD1306_t * dev, int page, const char * text, int text_len, bool invert); +void ssd1306_display_text_x3(SSD1306_t * dev, int page, const char * text, int text_len, bool invert); void ssd1306_clear_screen(SSD1306_t * dev, bool invert); void ssd1306_clear_line(SSD1306_t * dev, int page, bool invert); void ssd1306_contrast(SSD1306_t * dev, int contrast); diff --git a/ports/espressif/components/ssd1306/ssd1306_i2c.c b/ports/espressif/components/ssd1306/ssd1306_i2c.c index db3153098..187915b10 100644 --- a/ports/espressif/components/ssd1306/ssd1306_i2c.c +++ b/ports/espressif/components/ssd1306/ssd1306_i2c.c @@ -4,6 +4,7 @@ #include "freertos/task.h" #include "driver/i2c.h" +#include "driver/gpio.h" #include "esp_log.h" #include "ssd1306.h" diff --git a/ports/espressif/sdkconfig.defaults b/ports/espressif/sdkconfig.defaults index 453a1a50f..c2a438a64 100644 --- a/ports/espressif/sdkconfig.defaults +++ b/ports/espressif/sdkconfig.defaults @@ -11,9 +11,6 @@ CONFIG_PARTITION_TABLE_MD5=y CONFIG_COMPILER_OPTIMIZATION_SIZE=y CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT=y -# Newlib -CONFIG_NEWLIB_NANO_FORMAT=y - # Virtual file system CONFIG_VFS_SUPPORT_IO=n diff --git a/src/ghostfat.c b/src/ghostfat.c index abdc45be4..b7345bb77 100644 --- a/src/ghostfat.c +++ b/src/ghostfat.c @@ -47,9 +47,19 @@ static uint32_t _flash_size; #define UF2_ARRAY_SIZE(_arr) ( sizeof(_arr) / sizeof(_arr[0]) ) #define UF2_DIV_CEIL(_v, _d) ( ((_v) / (_d)) + ((_v) % (_d) ? 1 : 0) ) +// FAT names and labels are fixed-width and not NUL terminated +#if defined(__has_attribute) + #if __has_attribute(nonstring) + #define ATTR_NONSTRING __attribute__((nonstring)) + #endif +#endif +#ifndef ATTR_NONSTRING + #define ATTR_NONSTRING +#endif + typedef struct { uint8_t JumpInstruction[3]; - uint8_t OEMInfo[8]; + uint8_t OEMInfo[8] ATTR_NONSTRING; uint16_t SectorSize; uint8_t SectorsPerCluster; uint16_t ReservedSectors; @@ -66,8 +76,8 @@ typedef struct { uint8_t Reserved; uint8_t ExtendedBootSig; uint32_t VolumeSerialNumber; - uint8_t VolumeLabel[11]; - uint8_t FilesystemIdentifier[8]; + uint8_t VolumeLabel[11] ATTR_NONSTRING; + uint8_t FilesystemIdentifier[8] ATTR_NONSTRING; } __attribute__((packed)) FAT_BootBlock; typedef struct { @@ -88,7 +98,7 @@ typedef struct { STATIC_ASSERT(sizeof(DirEntry) == 32); typedef struct FileContent { - char const name[11]; + char const name[11] ATTR_NONSTRING; void const * content; uint32_t size; // OK to use uint32_T b/c FAT32 limits filesize to (4GiB - 2) From bb54352aa0691302b7d00715bc151d7fbaa5927f Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Wed, 14 Oct 2026 01:24:46 +0700 Subject: [PATCH 23/47] Ship dfu_runtime as an example app and build it on mimxrt10xx (#519) --- .claude/skills/build/SKILL.md | 4 +-- .claude/skills/build/scripts/check_build.py | 18 ++++++----- .claude/skills/hil/SKILL.md | 2 +- .claude/test/test_build.py | 23 +++++++++---- apps/dfu_runtime/src/dfu_runtime.h | 3 -- apps/dfu_runtime/src/main.c | 2 +- apps/dfu_runtime/src/tusb_config.h | 5 +++ apps/dfu_runtime/src/usb_descriptors.c | 30 +++++++++++++++++ ports/espressif/CMakeLists.txt | 2 -- .../apps/dfu_runtime/main/board_dfu_runtime.c | 2 +- ports/mimxrt10xx/CMakeLists.txt | 1 + .../apps/dfu_runtime/CMakeLists.txt | 23 +++++++++++++ ports/mimxrt10xx/apps/dfu_runtime/Makefile | 14 ++++++++ .../apps/dfu_runtime/board_dfu_runtime.c | 32 +++++++++++++++++++ ports/mimxrt10xx/board_flash.c | 4 +-- ports/mimxrt10xx/boards.c | 6 ++++ ports/mimxrt10xx/boards.h | 3 ++ src/board_api.h | 5 ++- 18 files changed, 151 insertions(+), 28 deletions(-) create mode 100644 ports/mimxrt10xx/apps/dfu_runtime/CMakeLists.txt create mode 100644 ports/mimxrt10xx/apps/dfu_runtime/Makefile create mode 100644 ports/mimxrt10xx/apps/dfu_runtime/board_dfu_runtime.c diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index b7f32fb14..b7a2a11bd 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -22,7 +22,7 @@ python3 $B ... --idf local # exported ESP-IDF instead of CI The last stdout line is JSON. An agent acts on `pass`; `firstError`, and `error` on exit 2, naming the cause or the remedy; `boards[]` (`status` `ok`, `failed` or `error`, `log`, `artifacts`, `notImplemented`); `built`, the artifacts of `--shared` roots, the only ones kept after a pass, with each one's hash in `boards[].sha256`; `nothingToBuild` and `uncovered`, each path with its reason; and `idf.matchesCi`. `input` and `defines` record what a build used (HEAD, `dirty`, submodule and `get_deps` dependency commits, `dirtyInputs` for any of those with local changes, `inputsDirtiedByBuild` for any that changed or moved while building, `.` for the checkout itself): the HIL contract binds firmware to this record. `paths` gives every changed path's `rule` and `reason`, the answer to why a path selected what it did. Exit 0 pass, 1 a build, self-test or test failed, 2 usage, resolution or a missing prerequisite, 3 an uncovered path; 2 outranks 1, and 1 outranks 3. `-v` streams the build to stderr; each build's log is `/check_build.log`. -`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`) but `dfu_runtime`'s, which the battery never runs. `dfu_runtime` selects the esptool roster boards of the ports that build it, and is uncovered only when this host has none. +`--select-only` prints the selection (`build.{ports,builds,ghostfat,tests,paths}`) and stops before any toolchain check: what any agent or CI step reads for "what does this change affect". With `--hil-config ` it adds `hil.{boards,full,uncovered}`: the roster boards the change puts under the HIL battery, and as uncovered each selected port with no roster board on this host and each app source (`apps/`, `ports//apps/`) but `dfu_runtime`'s, which the battery never runs. `dfu_runtime` selects the esptool roster boards of the ports that build it, and is uncovered once for each port that builds it without one. ## Reporting @@ -37,7 +37,7 @@ An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the J - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads. - Firmware no CI build compiles is uncovered: `ports/template_port/`, `ports/ch32v20x/` (`--board ch32v203_r0_1v0` builds it on request, with a note), a build file the port's CI backend never reads, a stub app. - Docs and agent, editor and repo metadata build nothing. -- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`; Espressif `idf.py build`, then the HIL's `dfu_runtime` fixture (`--target dfu_runtime`), which CI's build and releases leave out. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds. A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. +- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`; Espressif `idf.py build`. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds. A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. - **`--backend cmake`** is for the HIL's receipt only, with `--board` and `--shared`: CI still builds each port with its `build.yml` backend, so it is no CI evidence. A port without `CMakeLists.txt`, or Espressif, is refused. `-D` takes the cmake spellings: `LOGGER=RTT`, not `rtt`; `DEBUG` is ignored, the build stays MinSizeRel; `LOG` sets `CFG_TUSB_DEBUG` only, not `TUF2_LOG`; raw flags go through `CFLAGS_CLI`. - **A HIL record** (`--board --shared --hil-config `) binds the build's own flash target to the board's roster flasher: a J-Link board is configured with `JLINK_OPTION=-USB -ExitOnError 1 -NoGui 1` (so `tinyuf2-jlink` selects that probe and fails non-zero), an esptool board's ESP-IDF `flash` target takes the bridge's tty at flash time. `boards[].flash` holds the target, the roster flasher, the target's generated command (checked to name that probe or chip and the build's own image), the configure defines that bound it and the build-tree files it reads (in `sha256` too); a rebuild and a HIL flash hold the build dir's lock (`.lock`). Another flasher, a roster `args` other than `-device ` or `--chip `, or a `-D` of `JLINK_OPTION`, `OPENOCD_SERIAL`, `OPENOCD_OPTION` or `ESPPORT` is refused. Before flashing, the HIL dry-runs that target under the lock and refuses any command but the flasher and, on ESP-IDF, the generated ones that write no flash input (its bootloader subproject dry-run too); ESP-IDF re-runs CMake after any commit (its `git describe` dependency), so rebuild the record on the commit under test. - **ESP-IDF** builds in CI's image, `espressif/idf:`, through docker by default; a missing image is exit 2 naming the `docker pull`. `--idf local` uses the exported IDF and records its version and `matchesCi`: evidence from a different IDF is supplemental, not acceptance. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index 80fed36d9..f603fe1cc 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -486,13 +486,16 @@ def kind(parts): # the HIL builds with cmake whatever CI uses: its build files elif r['rule'] in ('app', 'port-app'): app = parts[1] if r['rule'] == 'app' else parts[3] if len(parts) > 3 else None ports = {port_of(b) for b, _ in r['boards']} if r['rule'] == 'app' else {parts[1]} - own = sorted(b for b in esptool if port_of(b) in ports) if app != HIL_APP: uncovered[r['path']] = f'the battery runs the bootloader and {HIL_APP} on esptool boards, never {r["path"]}' - elif own: - boards.update(own) - else: - uncovered[r['path']] = f'no {config} esptool board runs {HIL_APP}' + continue + # a port that builds the app but has no esptool roster board: its battery never runs it + for p in sorted(ports): + own = [b for b in esptool if port_of(b) == p] + if own: + boards.update(own) + else: + uncovered[f'{r["path"]} on {p}'] = f'no {config} esptool board of {p} runs {HIL_APP}' elif r['rule'] == 'board': (b, _), = r['boards'] if b in roster: @@ -634,10 +637,9 @@ def build_cmake(port, board, root, defines, log, verbose): def build_idf(board, root, defines, log, verbose, idf_mode, image): idf = ['idf.py', '-C', 'ports/espressif', '-B', str(root), f'-DBOARD={board}', f'-DARTIFACT_PATH={root / "_bin"}', *[f'-D{d}' for d in defines], 'build'] - fixture = ['cmake', '--build', str(root), '--target', HIL_APP] # out of ALL, so CI's releases never carry it if idf_mode == 'local': - return not (run(idf, log, verbose) or run(fixture, log, verbose)), [] - inner = "git config --global --add safe.directory '*' && " + shlex.join(idf) + ' && ' + shlex.join(fixture) + return not run(idf, log, verbose), [] + inner = "git config --global --add safe.directory '*' && " + shlex.join(idf) return not run(idf_docker(root) + [image, 'bash', '-c', inner], log, verbose), [] diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index e644dcfd0..b3fa8a303 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -135,7 +135,7 @@ A review (`pr-review`) maps each board's row, except a commissioning run's (`com ## Selection -`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config test/hil/local.json` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered, except `dfu_runtime`, which selects the esptool boards of the ports that build it: the battery runs the bootloader with its generated fixture on J-Link boards and with `dfu_runtime` on esptool boards. The runner, the build scripts it imports and the roster (`.claude/skills/hil/scripts/`, `.claude/skills/build/scripts/`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. +`python3 .claude/skills/build/scripts/check_build.py --select-only (--diff-file | --base [--worktree]) --hil-config test/hil/local.json` adds `hil.{boards, full, uncovered}`. Run the battery when `full` is set or `boards` is non-empty. An empty diff file selects nothing; a caller without the changed-file list passes `--base`. A fresh worktree, a review worktree included, needs CLAUDE.md's worktree setup (submodules and dependency links) before `check_build.py` can build in it. `uncovered` names what no roster board on this host can run; report it, never treat it as covered. An app source (`apps/`, `ports//apps/`) selects no board and is uncovered, except `dfu_runtime`, which selects the esptool boards of the ports that build it, and leaves uncovered each port that builds it without one (mimxrt10xx today): the battery runs the bootloader with its generated fixture on J-Link boards and with `dfu_runtime` on esptool boards. The runner, the build scripts it imports and the roster (`.claude/skills/hil/scripts/`, `.claude/skills/build/scripts/`, `test/hil/*.json`) select every roster board; a wording change to this page selects none. ## Commissioning a board diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index 325865cd9..41970c8b4 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -110,8 +110,10 @@ def test_port_file_selects_its_representatives(self): def test_app_selects_the_boards_that_build_it(self): self.assertEqual(classify('ports/mimxrt10xx/apps/factory_test/main.c')[2], [('metro_m7_1011', 'make')]) + # dfu_runtime, built for every board, includes it too: the port's representatives join the factory tests' boards self.assertEqual(classify('ports/mimxrt10xx/apps/app.mk')[2], - [('metro_m7_1011', 'make'), ('metro_m7_1011_sd', 'make')]) + [('imxrt1064_evk', 'make'), ('imxrt1170_evk', 'make'), ('metro_m7_1011', 'make'), + ('metro_m7_1011_sd', 'make')]) def test_app_nothing_builds_is_uncovered(self): self.assertEqual(classify('ports/lpc55/apps/erase_firmware/Makefile')[:2], ('port-app', 'gap')) @@ -321,15 +323,24 @@ def test_a_roster_entry_is_named_by_its_tinyuf2_board(self): def test_the_battery_app_selects_the_esptool_roster_boards_and_other_apps_none(self): self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk', 'flasher': {'name': 'jlink'}}, {'name': 'espressif_esp32s3_devkitc_1', 'flasher': {'name': 'esptool'}}]})) - for path in ('apps/dfu_runtime/src/nonce.c', 'ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c'): - with self.subTest(path): - h = self.hil(path) - self.assertEqual((h['boards'], h['uncovered']), (['espressif_esp32s3_devkitc_1'], [])) + rt_gap = {'target': 'apps/dfu_runtime/src/nonce.c on mimxrt10xx', + 'reason': f'no {self.config} esptool board of mimxrt10xx runs dfu_runtime'} + h = self.hil('apps/dfu_runtime/src/nonce.c') # espressif and mimxrt10xx build it; only the S3 runs it + self.assertEqual((h['boards'], h['uncovered']), (['espressif_esp32s3_devkitc_1'], [rt_gap])) + h = self.hil('apps/dfu_runtime/src/nonce.c', 'src/main.c') # the whole roster's battery still never runs it on RT + self.assertEqual(h['boards'], ['espressif_esp32s3_devkitc_1', 'imxrt1060_evk']) + self.assertIn(rt_gap, h['uncovered']) + h = self.hil('ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c') + self.assertEqual((h['boards'], h['uncovered']), (['espressif_esp32s3_devkitc_1'], [])) + h = self.hil('ports/mimxrt10xx/apps/dfu_runtime/board_dfu_runtime.c') + self.assertEqual((h['boards'], [u['reason'] for u in h['uncovered']]), + ([], [f'no {self.config} esptool board of mimxrt10xx runs dfu_runtime'])) h = self.hil('apps/blinky/src/blinky.c', 'ports/espressif/apps/blinky/main/CMakeLists.txt') self.assertEqual((h['boards'], len(h['uncovered'])), ([], 2)) self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk', 'flasher': {'name': 'jlink'}}]})) h = self.hil('apps/dfu_runtime/src/nonce.c') - self.assertEqual((h['boards'], [u['reason'] for u in h['uncovered']]), ([], [f'no {self.config} esptool board runs dfu_runtime'])) + self.assertEqual((h['boards'], sorted(u['target'] for u in h['uncovered'])), + ([], ['apps/dfu_runtime/src/nonce.c on espressif', 'apps/dfu_runtime/src/nonce.c on mimxrt10xx'])) def test_the_hil_follows_its_cmake_build_not_cis_make(self): self.roster.write_text(json.dumps({'boards': [{'name': 'imxrt1060_evk'}]})) diff --git a/apps/dfu_runtime/src/dfu_runtime.h b/apps/dfu_runtime/src/dfu_runtime.h index 38978f3ec..cb93108b6 100644 --- a/apps/dfu_runtime/src/dfu_runtime.h +++ b/apps/dfu_runtime/src/dfu_runtime.h @@ -36,9 +36,6 @@ #define NONCE_MAX_LEN 16 #define NONCE_STRING_SIZE (sizeof("HIL ") + 2 * NONCE_MAX_LEN) -// Reboot into TinyUF2 DFU mode, no return -void board_reboot_to_bootloader(void); - // Read the nonce block from where the port keeps it; false when it cannot be read bool board_nonce_block_read(uint8_t block[NONCE_BLOCK_SIZE]); diff --git a/apps/dfu_runtime/src/main.c b/apps/dfu_runtime/src/main.c index 786804878..364fd4054 100644 --- a/apps/dfu_runtime/src/main.c +++ b/apps/dfu_runtime/src/main.c @@ -45,7 +45,7 @@ int main(void) { } board_timer_start(1); - board_dfu_init(); + board_usb_init(); tud_init(BOARD_TUD_RHPORT); while (1) { diff --git a/apps/dfu_runtime/src/tusb_config.h b/apps/dfu_runtime/src/tusb_config.h index 0f57b4e08..a516f9e58 100644 --- a/apps/dfu_runtime/src/tusb_config.h +++ b/apps/dfu_runtime/src/tusb_config.h @@ -48,8 +48,13 @@ #endif #ifndef CFG_TUSB_MEM_ALIGN +#if CFG_TUSB_MCU == OPT_MCU_MIMXRT1XXX +// Cortex-M7 D-cache is enabled by board_init(): USB buffers need 32-byte (cache line) alignment +#define CFG_TUSB_MEM_ALIGN __attribute__ ((aligned(32))) +#else #define CFG_TUSB_MEM_ALIGN __attribute__ ((aligned(4))) #endif +#endif #ifndef CFG_TUD_ENDPOINT0_SIZE #define CFG_TUD_ENDPOINT0_SIZE 64 diff --git a/apps/dfu_runtime/src/usb_descriptors.c b/apps/dfu_runtime/src/usb_descriptors.c index 430fd73f3..e9f5267f9 100644 --- a/apps/dfu_runtime/src/usb_descriptors.c +++ b/apps/dfu_runtime/src/usb_descriptors.c @@ -22,6 +22,7 @@ * THE SOFTWARE. */ + #include "board_api.h" #include "tusb.h" @@ -71,6 +72,35 @@ uint8_t const* tud_descriptor_configuration_cb(uint8_t index) { return desc_configuration; } +#if TUD_OPT_HIGH_SPEED +// a high-speed device must answer both requests (USB 2.0 9.2.6.6, 9.4.3); DFU runtime has no endpoints, so the +// other speed's configuration is this one +tusb_desc_device_qualifier_t const desc_device_qualifier = { + .bLength = sizeof(tusb_desc_device_qualifier_t), + .bDescriptorType = TUSB_DESC_DEVICE_QUALIFIER, + .bcdUSB = 0x0200, + .bDeviceClass = 0x00, + .bDeviceSubClass = 0x00, + .bDeviceProtocol = 0x00, + .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, + .bNumConfigurations = 0x01, + .bReserved = 0x00 +}; + +uint8_t const* tud_descriptor_device_qualifier_cb(void) { + return (uint8_t const*) &desc_device_qualifier; +} + +static uint8_t desc_other_speed_config[CONFIG_TOTAL_LEN]; + +uint8_t const* tud_descriptor_other_speed_configuration_cb(uint8_t index) { + (void) index; + memcpy(desc_other_speed_config, desc_configuration, CONFIG_TOTAL_LEN); + desc_other_speed_config[1] = TUSB_DESC_OTHER_SPEED_CONFIG; + return desc_other_speed_config; +} +#endif + enum { STRID_LANGID = 0, STRID_MANUFACTURER, diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index fd6ef2912..bf282fc3c 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -110,14 +110,12 @@ externalproject_add(blinky BUILD_ALWAYS 1 ) -# the HIL battery's fixture: out of ALL, so no release carries it; build it with --target dfu_runtime externalproject_add(dfu_runtime SOURCE_DIR ${CMAKE_CURRENT_LIST_DIR}/apps/dfu_runtime BINARY_DIR ${CMAKE_BINARY_DIR}/apps/dfu_runtime CMAKE_ARGS -DBOARD=${BOARD} -DARTIFACT_PATH=${ARTIFACT_PATH} INSTALL_COMMAND "" BUILD_ALWAYS 1 - EXCLUDE_FROM_ALL 1 ) # erase firmware need to use reset reason, not implemented yet diff --git a/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c index 6d4d9d3f2..78f926ea3 100644 --- a/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c +++ b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c @@ -66,7 +66,7 @@ void board_timer_stop(void) { esp_timer_stop(timer_hdl); } -void board_dfu_init(void) { +void board_usb_init(void) { // same PHY setup as TinyUF2's boards.c board_dfu_init() usb_phy_config_t const phy_conf = { .controller = USB_PHY_CTRL_OTG, diff --git a/ports/mimxrt10xx/CMakeLists.txt b/ports/mimxrt10xx/CMakeLists.txt index 62761374d..9374becbb 100644 --- a/ports/mimxrt10xx/CMakeLists.txt +++ b/ports/mimxrt10xx/CMakeLists.txt @@ -43,6 +43,7 @@ add_custom_command(TARGET tinyuf2 POST_BUILD #------------------------------------ add_subdirectory(${TOP}/apps/blinky ${CMAKE_BINARY_DIR}/apps/blinky) add_subdirectory(${TOP}/apps/erase_firmware ${CMAKE_BINARY_DIR}/apps/erase_firmware) +add_subdirectory(apps/dfu_runtime) if (BOARD STREQUAL metro_m7_1011) add_subdirectory(apps/esp32programmer) diff --git a/ports/mimxrt10xx/apps/dfu_runtime/CMakeLists.txt b/ports/mimxrt10xx/apps/dfu_runtime/CMakeLists.txt new file mode 100644 index 000000000..c17438528 --- /dev/null +++ b/ports/mimxrt10xx/apps/dfu_runtime/CMakeLists.txt @@ -0,0 +1,23 @@ +cmake_minimum_required(VERSION 3.17) + +include(${CMAKE_CURRENT_LIST_DIR}/../../app.cmake) + +set(app_src ${TOP}/apps/dfu_runtime/src) + +add_executable(dfu_runtime + board_dfu_runtime.c + ${app_src}/main.c + ${app_src}/nonce.c + ${app_src}/usb_descriptors.c + ${CMAKE_CURRENT_LIST_DIR}/../../boards.c + ${TOP}/lib/tinyusb/src/portable/chipidea/ci_hs/dcd_ci_hs.c + ) +target_include_directories(dfu_runtime PUBLIC + ${app_src} + ${TOP}/src + ) + +family_configure_app(dfu_runtime) +family_add_tinyusb(dfu_runtime OPT_MCU_MIMXRT1XXX) +family_gen_uf2(dfu_runtime ${UF2_FAMILY_ID}) +family_flash_uf2(dfu_runtime ${UF2_FAMILY_ID}) diff --git a/ports/mimxrt10xx/apps/dfu_runtime/Makefile b/ports/mimxrt10xx/apps/dfu_runtime/Makefile new file mode 100644 index 000000000..b5cfb822d --- /dev/null +++ b/ports/mimxrt10xx/apps/dfu_runtime/Makefile @@ -0,0 +1,14 @@ +OUTNAME = dfu_runtime-$(BOARD) + +SRC_C += \ + $(PORT_DIR)/boards.c \ + $(CURRENT_PATH)/board_dfu_runtime.c \ + apps/dfu_runtime/src/main.c \ + apps/dfu_runtime/src/nonce.c \ + apps/dfu_runtime/src/usb_descriptors.c \ + +INC += \ + $(TOP)/apps/dfu_runtime/src \ + $(TOP)/src \ + +include ../app.mk diff --git a/ports/mimxrt10xx/apps/dfu_runtime/board_dfu_runtime.c b/ports/mimxrt10xx/apps/dfu_runtime/board_dfu_runtime.c new file mode 100644 index 000000000..0a015a618 --- /dev/null +++ b/ports/mimxrt10xx/apps/dfu_runtime/board_dfu_runtime.c @@ -0,0 +1,32 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "board_api.h" +#include "dfu_runtime.h" + +// the nonce block sits in the last 256 bytes of the flash TinyUF2 serves, read through XIP +bool board_nonce_block_read(uint8_t block[NONCE_BLOCK_SIZE]) { + memcpy(block, (void const*) (FLASH_BASE + BOARD_FLASH_UF2_SIZE - NONCE_BLOCK_SIZE), NONCE_BLOCK_SIZE); + return true; +} diff --git a/ports/mimxrt10xx/board_flash.c b/ports/mimxrt10xx/board_flash.c index 62842fa7a..6400cfa81 100644 --- a/ports/mimxrt10xx/board_flash.c +++ b/ports/mimxrt10xx/board_flash.c @@ -160,9 +160,7 @@ void board_flash_init(void) { } uint32_t board_flash_size(void) { - // TODO currently limit at 8MB since the CURRENT.UF2 can occupies all 32MB virtual disk - const uint32_t max_size = 8 * 1024 * 1024; - return (BOARD_FLASH_SIZE < max_size) ? BOARD_FLASH_SIZE : max_size; + return BOARD_FLASH_UF2_SIZE; } void board_flash_read(uint32_t addr, void *buffer, uint32_t len) { diff --git a/ports/mimxrt10xx/boards.c b/ports/mimxrt10xx/boards.c index a6ff170b7..d7d5b18a7 100644 --- a/ports/mimxrt10xx/boards.c +++ b/ports/mimxrt10xx/boards.c @@ -196,6 +196,12 @@ void board_reset(void) { NVIC_SystemReset(); } +// the bootloader's double-tap check reads the magic from SNVS LPGPR, which a system reset keeps +void board_reboot_to_bootloader(void) { + TINYUF2_DBL_TAP_REG = DBL_TAP_MAGIC; + board_reset(); +} + void board_dfu_complete(void) { NVIC_SystemReset(); } diff --git a/ports/mimxrt10xx/boards.h b/ports/mimxrt10xx/boards.h index f4e277019..8029498f3 100644 --- a/ports/mimxrt10xx/boards.h +++ b/ports/mimxrt10xx/boards.h @@ -53,6 +53,9 @@ #define BOARD_FLASH_APP_START (FLASH_BASE + 0xC000) #endif +// Flash TinyUF2 serves, at most 8 MiB: CURRENT.UF2 could otherwise occupy the whole 32 MiB virtual disk +#define BOARD_FLASH_UF2_SIZE ((BOARD_FLASH_SIZE < 8*1024*1024) ? BOARD_FLASH_SIZE : 8*1024*1024) + // Double Reset tap to enter DFU #define TINYUF2_DBL_TAP_DFU 1 #define TINYUF2_DBL_TAP_REG SNVS->LPGPR[3] diff --git a/src/board_api.h b/src/board_api.h index 1563c871b..69202bc75 100644 --- a/src/board_api.h +++ b/src/board_api.h @@ -188,7 +188,7 @@ bool board_app_valid2(void) __attribute__ ((weak)); // Jump to Application, no return void board_app_jump(void); -// Init USB hardware (not used for now) +// Init USB hardware void board_usb_init(void); // Init DFU process, mostly for starting USB @@ -197,6 +197,9 @@ void board_dfu_init(void); // DFU is complete, should reset or jump to application mode and not return void board_dfu_complete(void); +// Reboot into TinyUF2 DFU mode, no return. For applications (apps/dfu_runtime); not called by the bootloader +void board_reboot_to_bootloader(void); + // Fill Serial Number and return its length (limit to 16 bytes) uint8_t board_usb_get_serial(uint8_t serial_id[16]); From 8e75969ef9ed15215522c080b0f9574af0a13d3c Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Wed, 14 Oct 2026 09:34:49 +0700 Subject: [PATCH 24/47] Add ESP32-P4 Function EV board (#532) TinyUF2 on OTG HS (J15), chip revision <3.0. Detach OTG HS before a software restart on P4. Add the board to the build representatives and HIL roster. --- .claude/skills/build/SKILL.md | 2 +- .claude/skills/build/scripts/check_build.py | 11 +-- .claude/skills/hil/SKILL.md | 6 +- .claude/skills/hil/scripts/hil_esp.py | 4 +- .claude/test/test_build.py | 17 +++-- .claude/test/test_hil.py | 71 ++++++++++++++++--- ports/espressif/README.md | 8 +++ .../apps/dfu_runtime/main/board_dfu_runtime.c | 13 +++- ports/espressif/boards/boards.c | 29 +++++++- .../espressif_p4_function_ev/board.cmake | 2 + .../boards/espressif_p4_function_ev/board.h | 58 +++++++++++++++ .../boards/espressif_p4_function_ev/sdkconfig | 10 +++ .../main/bootloader_start.c | 8 +-- .../components/ssd1306/Kconfig.projbuild | 4 ++ .../components/ssd1306/ssd1306_spi.c | 2 + .../components/tinyusb_src/CMakeLists.txt | 2 + .../components/tinyusb_src/tusb_config.h | 3 - ports/espressif/family.cmake | 7 ++ supported_boards.md | 1 + test/hil/local.json | 6 ++ 20 files changed, 229 insertions(+), 35 deletions(-) create mode 100644 ports/espressif/boards/espressif_p4_function_ev/board.cmake create mode 100644 ports/espressif/boards/espressif_p4_function_ev/board.h create mode 100644 ports/espressif/boards/espressif_p4_function_ev/sdkconfig diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index b7a2a11bd..6db5c4b30 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -31,7 +31,7 @@ An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the J ## Judgment - **Selection is the script's**, first match per path, and `paths[].reason` says why. A mixed change takes the union, and an uncovered path fails it whatever else built. The policy behind the rules: - - A change to a port, board, app or dependency builds the boards that compile it: a board's own files that board; other port files the port's `REPRESENTATIVES` (rig boards first, both ESP32-S2 and -S3, both i.MX RT10xx and RT1170); an app or dependency the representatives of the ports whose CI backend compiles it. + - A change to a port, board, app or dependency builds the boards that compile it: a board's own files that board; other port files the port's `REPRESENTATIVES` (rig boards first, every ESP32 target the port's boards use, both i.MX RT10xx and RT1170); an app or dependency the representatives of the ports whose CI backend compiles it. - Shared code (`src/`, `lib/`, the shared build files) and anything unclassified builds every port's representatives, plus the ghostfat self-test when a ghostfat input changed. - A change to how boards build (this script, `build.yml`, `build_util.yml`, `ci_set_matrix.py`, `.github/actions/`) runs the contract's own tests and one smoke build per backend, and a change to this script the HIL battery on every roster board, since the HIL runner imports it; no firmware build verifies workflow syntax or orchestration. A change to the contract's tests runs the tests only. - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index f603fe1cc..e73eca89b 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -48,10 +48,10 @@ 'stm32h7': 'make', 'stm32l4': 'make', } -# Boards that stand in for their port: rig boards first; espressif covers both IDF targets, +# Boards that stand in for their port: rig boards first; espressif covers every IDF target, # mimxrt10xx both the RT10xx and RT1170 boot layouts. REPRESENTATIVES = { - 'espressif': ['espressif_esp32s3_devkitm_1', 'espressif_esp32s2_devkitc_1'], + 'espressif': ['espressif_esp32s3_devkitm_1', 'espressif_esp32s2_devkitc_1', 'espressif_p4_function_ev'], 'kinetis_k32l2': ['frdm_k32l2b'], 'lpc55': ['lpcxpresso55s69'], 'maxim': ['max32666fthr'], @@ -761,9 +761,10 @@ def idf_step_writes_no_input(cmd, root, chip): argv[1:] == ['-E', 'echo_append'] or argv[1:3] == ['-E', 'echo'] and len(argv) == 4 or argv[1:3] in (['-E', 'make_directory'], ['-E', 'touch']) and len(argv) == 4 and writes(argv[3]) or argv[1:3] == ['-E', 'copy'] and len(argv) == 5 and writes(argv[4], '_bin')) or \ - re.fullmatch(py + r' \S*/partition_table/check_sizes\.py --offset', head) and tail[1:] in ( - ['partition', '--type', 'app', str(root / 'partition_table/partition-table.bin'), str(root / 'tinyuf2.bin')], - ['bootloader', '0x0', str(root / 'bootloader/bootloader.bin')]) or \ + re.fullmatch(py + r' \S*/partition_table/check_sizes\.py --offset', head) and ( + tail[1:] == ['partition', '--type', 'app', str(root / 'partition_table/partition-table.bin'), str(root / 'tinyuf2.bin')] or + len(tail) == 4 and tail[1] == 'bootloader' and re.fullmatch(r'0x[0-9a-f]+', tail[2]) and + tail[3] == str(root / 'bootloader/bootloader.bin')) or \ re.fullmatch(py + r' \S*/lib/uf2/utils/uf2conv\.py --carray', head) and \ tail == ['-o', str(root / 'apps/update_tinyuf2/bootloader_bin.c'), str(root / 'tinyuf2.bin')] and writes(tail[1]) or \ argv == ['bash', str(postbuild)] and cwd == root and writes(root / 'combined.bin') and postbuild.is_file() and \ diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index b3fa8a303..0ae912f1c 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -131,7 +131,7 @@ A review (`pr-review`) maps each board's row, except a commissioning run's (`com `python3 .claude/skills/hil/scripts/hil_test.py restore -b --build-record --recover-holder --report test/hil/local.json` `` is the failed report's `firmware.buildRecord`, or a record rebuilt on the clean HEAD when it no longer verifies. The command proves that pid holds the board as the failed run's cleanup holder and that none of the run's survivors and no J-Link process on the probe is alive, restores the board while the lock stays held, and only then releases the holder. On an esptool board the restore checks the preserved regions against the snapshot in the failed report, and recovery is refused for a touched run without one or one whose regions differ from the checkout's. A candidate whose drive cannot take the blinky copy fails the restore; the board stays held until a record that can restore it is given. A second recovery of the same board while one runs is refused (exit 2). - - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide); on espressif_esp32s3_devkitc_1, ROM download mode: hold Boot (SW1, GPIO0 low), press and release RST (SW2, CHIP_PU), then release Boot (DevKitC-1 schematic V1.0; the TRM's strap table also needs GPIO46 low, its default pull-down). + - If the probe cannot connect, recovery is the human's: on imxrt1060_evk, serial download mode (BOOT_MODE 01: SW7 OFF OFF OFF ON, inferred from the EVK guide); on espressif_esp32s3_devkitc_1, ROM download mode: hold Boot (SW1, GPIO0 low), press and release RST (SW2, CHIP_PU), then release Boot (DevKitC-1 schematic V1.0; the TRM's strap table also needs GPIO46 low, its default pull-down); on espressif_p4_function_ev, the same with BOOT (SW2, GPIO35 low) and RST (SW1, CHIP_PU) (Function-EV-Board schematic rev 1.4). ## Selection @@ -176,8 +176,8 @@ Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under ## Limits -- Covered: imxrt1060_evk, metro_m7_1011 and espressif_esp32s3_devkitc_1 on htpc. Not covered: ports without a cmake build, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69; its 0A silicon also hangs TinyUF2 on a UF2 copy, #500), espressif chips other than the ESP32-S3, self-update, an interrupted update, malformed UF2 input and the physical double tap. -- esptool boards: the runner needs esptool 5.2.0 and pyusb with libusb-1.0 for its own Python. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The bridge's UART carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. At the end of a UF2 copy TinyUF2 restarts while the host's usb-storage is still talking to it: the host log shows a reset of that device and read errors on its block device. Seen on the S3; other ports are unchecked. +- Covered: imxrt1060_evk, metro_m7_1011, espressif_esp32s3_devkitc_1 and espressif_p4_function_ev on htpc. Not covered: ports without a cmake build, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69; its 0A silicon also hangs TinyUF2 on a UF2 copy, #500), espressif chips other than the ESP32-S3 and ESP32-P4, self-update, an interrupted update, malformed UF2 input and the physical double tap. +- esptool boards: the runner needs esptool 5.2.0 and pyusb with libusb-1.0 for its own Python. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The bridge's UART carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. At the end of a UF2 copy TinyUF2 restarts while the host's usb-storage is still talking to it: the host log shows a reset of that device and read errors on its block device. Seen on the S3 and the P4; other ports are unchecked. On the P4, while an esptool session holds the chip, the host log shows a full-speed device on TinyUF2's USB path that never answers its descriptor request (htpc 2026-10-14); the battery passed with it. - Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. - The HIL's cmake image logs on the board UART: cmake builds leave `TUF2_LOG` at its default 1 (`src/board_api.h`), while make builds, the make ports' CI artifacts and the make-built releases pass 0. Its blocking writes at start, before the app jump and on DFU entry are timing the make image does not have. - `install` from a blank board, or one whose ROM found no boot image, brings up DFU (measured on imxrt1060_evk 2026-10-12 and metro_m7_1011 2026-10-13). With no boot image a new J-Link session reads the flash wrong (imxrt1060_evk: zeros; metro_m7_1011: bytes shifted) until a `loadbin` has run; a `loadbin` compare reads it right. On metro_m7_1011, J-Link then prints "Invalid flash header detected." at the install's first reset. diff --git a/.claude/skills/hil/scripts/hil_esp.py b/.claude/skills/hil/scripts/hil_esp.py index 9437fb15a..cb97bea2c 100644 --- a/.claude/skills/hil/scripts/hil_esp.py +++ b/.claude/skills/hil/scripts/hil_esp.py @@ -11,8 +11,8 @@ from check_build import FLASH, HIL_APP, IDF_CONFIG # 2nd-stage bootloader offset per chip, independent of the build: the ROM loads it from here (IDF v6.1 -# esp_rom/esp32s3/esp_rom_caps.h ESP_ROM_BOOTLOADER_OFFSET_FLASH, esptool 5.2.0 targets/esp32s3.py; no checked TRM states it) -BOOTLOADER_OFFSET = {'esp32s3': 0x0} +# esp_rom//esp_rom_caps.h ESP_ROM_BOOTLOADER_OFFSET_FLASH, esptool 5.2.0 targets/.py; no checked TRM states it) +BOOTLOADER_OFFSET = {'esp32s3': 0x0, 'esp32p4': 0x2000} PARTITION_TABLE = (0x8000, 0x1000) # ports/espressif/sdkconfig.defaults CONFIG_PARTITION_TABLE_OFFSET; one sector APP, DATA = 0x00, 0x01 SUBTYPE = {(APP, 'factory'): 0x00, (APP, 'ota_0'): 0x10, (APP, 'ota_1'): 0x11, (DATA, 'ota'): 0x00, diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index 41970c8b4..b1dfe1f03 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -73,10 +73,10 @@ def test_representatives_exist(self): for backend, board in cb.SMOKE.items(): self.assertEqual(cb.BACKENDS[cb.port_of(board)], backend) - def test_espressif_representatives_cover_both_idf_targets(self): - targets = {re.search(r'esp32s[23]', (ROOT / 'ports/espressif/boards' / b / 'board.cmake').read_text()).group(0) - for b in cb.REPRESENTATIVES['espressif']} - self.assertEqual(targets, {'esp32s2', 'esp32s3'}) + def test_espressif_representatives_cover_every_idf_target(self): + target = lambda b: re.search(r'IDF_TARGET "(\w+)"', (ROOT / 'ports/espressif/boards' / b / 'board.cmake').read_text())[1] + every = {target(b.name) for b in (ROOT / 'ports/espressif/boards').iterdir() if (b / 'board.cmake').is_file()} + self.assertEqual({target(b) for b in cb.REPRESENTATIVES['espressif']}, every) def test_no_self_update_ports_echo_a_stub(self): for port in cb.NO_SELF_UPDATE: @@ -824,6 +824,15 @@ def test_an_idf_flash_target_may_also_run_steps_that_write_no_flash_input(self): with self.assertRaisesRegex(cb.Fail, 'bootloader: ninja -n failed'): self.ready(esp, root, self.idf_tree, fresh, [boot_size], banner='nested') + def test_an_idf_bootloader_size_check_is_read_only_at_any_offset(self): + root = self.tmp / 'p4' + sizes = '/opt/esp/python_env/idf6.1_py3.12_env/bin/python /opt/esp/idf/components/partition_table/check_sizes.py --offset 0x8000' + boot_size = lambda arg: f'cd {root}/bootloader && {sizes} bootloader {arg} {root}/bootloader/bootloader.bin' + self.assertTrue(cb.idf_step_writes_no_input(boot_size('0x2000'), root, 'esp32p4')) + for arg in ('2000', '0x2000 x', '0x20g0'): + with self.subTest(arg): + self.assertFalse(cb.idf_step_writes_no_input(boot_size(arg), root, 'esp32p4')) + def test_a_hil_record_pins_the_target_inputs_and_holds_the_build_lock(self): args = mock.Mock(shared=True, define=[], verbose=False, idf='docker') built = [] diff --git a/.claude/test/test_hil.py b/.claude/test/test_hil.py index 19d1bc777..a08505454 100644 --- a/.claude/test/test_hil.py +++ b/.claude/test/test_hil.py @@ -331,11 +331,13 @@ def partition_table(rows): return body + b'\xeb\xeb' + b'\xff' * 14 + ht.hashlib.md5(body).digest() + b'\xff' * 32 -def idf_build(root, parts=S3_PARTS, flash_args=None, sdk=None): - """A synthetic IDF build of the S3 DevKitC as the port's CMakeLists lays it out, and its record row.""" +def idf_build(root, parts=S3_PARTS, flash_args=None, sdk=None, board=S3, device='esp32s3', + board_id='ESP32S3-DevKitC-v1.0', family=0xc47e5767): + """A synthetic IDF build of the S3 DevKitC (or another board) as the port's CMakeLists lays it out, and its + record row.""" (root / '_bin').mkdir(parents=True) files = {'_bin/bootloader.bin': b'\xe9' + b'\0' * 0x5000, - '_bin/tinyuf2.bin': b'\xe9' + b'TinyUF2 Bootloader 1.0\r\nBoard-ID: ESP32S3-DevKitC-v1.0\r\n' + b'\0' * 0x100, + '_bin/tinyuf2.bin': b'\xe9' + b'TinyUF2 Bootloader 1.0\r\nBoard-ID: ' + board_id.encode() + b'\r\n' + b'\0' * 0x100, '_bin/partition-table.bin': partition_table(parts), '_bin/ota_data_initial.bin': b'\xff' * 0x2000, '_bin/flash_args': flash_args or ('--flash-mode dio --flash-freq 80m --flash-size 8MB\n0x0 bootloader/bootloader.bin\n' '0x410000 tinyuf2.bin\n0x8000 partition_table/partition-table.bin\n' @@ -343,26 +345,27 @@ def idf_build(root, parts=S3_PARTS, flash_args=None, sdk=None): 'sdkconfig': sdk or ('CONFIG_IDF_TARGET="esp32s3"\nCONFIG_ESPTOOLPY_FLASHMODE="dio"\nCONFIG_ESPTOOLPY_FLASHFREQ="80m"\n' 'CONFIG_ESPTOOLPY_FLASHSIZE="8MB"\nCONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-8MB.csv"\n' 'CONFIG_PARTITION_TABLE_OFFSET=0x8000\n'), - '_bin/apps/dfu_runtime.uf2': ht.pack_uf2([(256 * i, bytes([i]) * 256) for i in range(3)], 0xc47e5767), - '_bin/apps/blinky.uf2': ht.pack_uf2([(256 * i, bytes([0xb0 + i]) * 256) for i in range(2)], 0xc47e5767)} + '_bin/apps/dfu_runtime.uf2': ht.pack_uf2([(256 * i, bytes([i]) * 256) for i in range(3)], family), + '_bin/apps/blinky.uf2': ht.pack_uf2([(256 * i, bytes([0xb0 + i]) * 256) for i in range(2)], family)} (root / '_bin' / 'apps').mkdir() for rel in ht.cb.FLASH['esptool'][1]: # what the flash target reads, as the build leaves it files[rel] = files['_bin/' + Path(rel).name] (root / rel).parent.mkdir(parents=True, exist_ok=True) for rel, data in files.items(): (root / rel).write_bytes(data if isinstance(data, bytes) else data.encode()) - return {'board': S3, 'status': 'ok', 'backend': 'idf', 'buildDir': str(root), + return {'board': board, 'status': 'ok', 'backend': 'idf', 'buildDir': str(root), 'sha256': {rel: ht.sha256(root / rel) for rel in files}, - 'flash': {'target': 'flash', 'flasher': {'name': 'esptool', 'uid': 'cp2102n', 'device': 'esp32s3'}, + 'flash': {'target': 'flash', 'flasher': {'name': 'esptool', 'uid': 'cp2102n', 'device': device}, 'inputs': list(ht.cb.FLASH['esptool'][1])}} -class Esp(Tmp): - """The S3 DevKitC entry: esptool by the CP2102N's serial, facts from the IDF port's own files.""" +class EspCase(Tmp): + """An esptool roster entry (`roster` overrides to s3_roster, board `name`) with its IDF record checks.""" + name, roster = S3, {} def setUp(self): super().setUp() - self.board = self.load(s3_roster(self.tmp), S3) + self.board = self.load(s3_roster(self.tmp, **self.roster), self.name) self.facts = ht.source_facts(self.board) self.now = {'head': 'h', 'submodules': {}, 'deps': {}, 'dirty': False} @@ -377,6 +380,10 @@ def verify(self, rec): mock.patch.object(ht.cb, 'ci_idf_version', lambda: 'v6.1'): return ht.verify_record(rec, self.board, self.facts, self.tmp / 'run') + +class Esp(EspCase): + """The S3 DevKitC entry: esptool by the CP2102N's serial, facts from the IDF port's own files.""" + def test_the_facts_come_from_the_idf_port(self): f = self.facts self.assertEqual((f['flashSize'], f['partitionCsv'], f['partitions']), (0x800000, 'partitions-8MB.csv', S3_PARTS)) @@ -474,6 +481,50 @@ def test_a_record_not_bound_to_the_roster_flasher_is_refused(self): self.verify(self.record(row)) +P4 = 'espressif_p4_function_ev' +P4_PARTS = S3_PARTS[:5] + [('ffat', 1, 0x81, 0x450000, 0xbb0000)] +P4_FLASH_ARGS = ('--flash-mode dio --flash-freq 80m --flash-size 16MB\n0x2000 bootloader/bootloader.bin\n' + '0x410000 tinyuf2.bin\n0x8000 partition_table/partition-table.bin\n0xe000 ota_data_initial.bin\n') +P4_SDK = ('CONFIG_IDF_TARGET="esp32p4"\nCONFIG_ESPTOOLPY_FLASHMODE="dio"\nCONFIG_ESPTOOLPY_FLASHFREQ="80m"\n' + 'CONFIG_ESPTOOLPY_FLASHSIZE="16MB"\nCONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-16MB.csv"\n' + 'CONFIG_PARTITION_TABLE_OFFSET=0x8000\n') + + +class EspP4(EspCase): + """The P4 Function EV entry: its 2nd-stage bootloader sits at 0x2000, below the partition table.""" + name = P4 + roster = {'name': P4, 'uid': '30EDA0E0C902', 'flasher': {'name': 'esptool', 'uid': 'cp2102n', 'args': '--chip esp32p4'}} + + def build(self, root, flash_args=P4_FLASH_ARGS): + return idf_build(root, P4_PARTS, flash_args, P4_SDK, board=P4, device='esp32p4', board_id='ESP32P4-FunctionEV-v1.4', + family=0x3d308e94) + + def test_the_facts_come_from_the_idf_port(self): + f = self.facts + self.assertEqual((f['flashSize'], f['partitionCsv'], f['partitions']), (0x1000000, 'partitions-16MB.csv', P4_PARTS)) + self.assertEqual((f['vid'], f['pid'], f['boardId'], f['family']), + (0x239a, 0x00a5, 'ESP32P4-FunctionEV-v1.4', 0x3d308e94)) + self.assertEqual(self.board['device'], 'esp32p4') + + def test_the_layout_names_what_an_install_writes_and_what_is_preserved(self): + lay = self.board['regions'] + self.assertEqual(lay['install'], {'bootloader.bin': (0x2000, 0x8000), 'partition-table.bin': (0x8000, 0x9000), + 'ota_data_initial.bin': (0xe000, 0x10000), 'tinyuf2.bin': (0x410000, 0x450000)}) + self.assertEqual(lay['preserved'], {'nvs': (0x9000, 0xe000), 'ota_1': (0x210000, 0x410000), + 'ffat': (0x450000, 0x1000000)}) + + def test_an_idf_record_gives_the_four_segments_at_their_regions(self): + files, _ = self.verify(self.record(self.build(self.tmp / 'b'))) + self.assertEqual([(off, p.name) for off, p in files['segments']], + [(0x2000, 'bootloader.bin'), (0x8000, 'partition-table.bin'), (0xe000, 'ota_data_initial.bin'), + (0x410000, 'tinyuf2.bin')]) + + def test_a_bootloader_at_the_s3_offset_is_refused(self): + row = self.build(self.tmp / 'b', P4_FLASH_ARGS.replace('0x2000 bootloader', '0x0 bootloader')) + with self.assertRaisesRegex(ht.Refused, 'bootloader.bin .* is not at the start of its region \\[0x2000,0x8000\\)'): + self.verify(self.record(row)) + + FAKE_ESP = textwrap.dedent(''' import hashlib, json, os, sys args, ops = sys.argv[1:-1], json.load(open(sys.argv[-1])) diff --git a/ports/espressif/README.md b/ports/espressif/README.md index d613ee892..86a299319 100644 --- a/ports/espressif/README.md +++ b/ports/espressif/README.md @@ -69,8 +69,16 @@ uf2conv.py firmware.bin -c -b 0x00 -f 0xbfdd4eee uf2conv.py firmware.bin -c -b 0x00 -f ESP32S3 uf2conv.py firmware.bin -c -b 0x00 -f 0xc47e5767 +uf2conv.py firmware.bin -c -b 0x00 -f 0x3d308e94 # ESP32P4 + ``` +## ESP32-P4 + +- TinyUF2 runs on the USB 2.0 OTG High-Speed controller (`BOARD_TUD_RHPORT` 1), e.g. the USB 2.0 Type-C port of the ESP32-P4-Function-EV-Board. It is tested at high speed only; enumeration through a full-speed-only host or hub is unverified. +- ESP32-P4 chip revisions below v3.0 and v3.x need separate images (`CONFIG_ESP32P4_SELECTS_REV_LESS_V3`). A board selects its revision in its `sdkconfig`, which `family.cmake` also passes to the standalone applications so their images boot on the same chip. `espressif_p4_function_ev` targets the ESP32-P4-Function-EV-Board (chip below v3.0, tested on chip v0.1 and v1.0), not the ESP32-P4X-Function-EV-Board (chip v3.1+). +- The 2nd stage bootloader sits at 0x2000, leaving 24 KiB before the partition table. + ## 2nd Stage Bootloader After 1st stage ROM bootloader runs, which mostly checks GPIO0 to determine whether it should go into ROM DFU, 2nd stage bootloader is loaded. It is responsible for determining and loading either UF2 or user application (OTA0, OTA1). This is the place where we added detection code for entering UF2 mode mentioned by above methods. diff --git a/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c index 78f926ea3..d90bf4f5c 100644 --- a/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c +++ b/ports/espressif/apps/dfu_runtime/main/board_dfu_runtime.c @@ -29,9 +29,11 @@ #include "esp_timer.h" #include "esp_private/system_internal.h" #include "esp_private/usb_phy.h" +#include "esp_rom_sys.h" #include "board_api.h" #include "dfu_runtime.h" +#include "tusb.h" // Espressif glue for the DFU runtime example, without the bootloader's boards component @@ -70,12 +72,16 @@ void board_usb_init(void) { // same PHY setup as TinyUF2's boards.c board_dfu_init() usb_phy_config_t const phy_conf = { .controller = USB_PHY_CTRL_OTG, +#if BOARD_TUD_RHPORT == 1 + .target = USB_PHY_TARGET_UTMI, // OTG HS +#else .target = USB_PHY_TARGET_INT, +#endif .otg_mode = USB_OTG_MODE_DEVICE, // https://github.com/hathach/tinyusb/issues/2943#issuecomment-2601888322 .otg_speed = USB_PHY_SPEED_UNDEFINED, }; - usb_new_phy(&phy_conf, &phy_hdl); + ESP_ERROR_CHECK(usb_new_phy(&phy_conf, &phy_hdl)); } uint8_t board_usb_get_serial(uint8_t serial_id[16]) { @@ -88,6 +94,11 @@ void board_reboot_to_bootloader(void) { // the call makes the link keep esp_reset_reason_set_hint() (ports/espressif/README.md) (void) esp_reset_reason(); esp_reset_reason_set_hint((esp_reset_reason_t) APP_REQUEST_UF2_RESET_HINT); +#if CONFIG_IDF_TARGET_ESP32P4 + // as TinyUF2's boards.c: a software restart leaves OTG HS attached + tud_disconnect(); + esp_rom_delay_us(20000); +#endif esp_restart(); } diff --git a/ports/espressif/boards/boards.c b/ports/espressif/boards/boards.c index d2241301e..7de9be1f4 100644 --- a/ports/espressif/boards/boards.c +++ b/ports/espressif/boards/boards.c @@ -29,7 +29,6 @@ #include "hal/gpio_ll.h" #include "esp_private/usb_phy.h" -#include "soc/usb_pins.h" #include "soc/usb_periph.h" #include "esp_private/periph_ctrl.h" @@ -45,6 +44,11 @@ #include "board_api.h" +#if CONFIG_IDF_TARGET_ESP32P4 +#include "soc/lp_system_reg.h" +#include "esp_rom_sys.h" +#endif + #ifndef BUILD_NO_TINYUSB #include "tusb.h" #endif @@ -260,19 +264,35 @@ void board_init(void) { esp_timer_create(&periodic_timer_args, &timer_hdl); } +#if CONFIG_IDF_TARGET_ESP32P4 && !defined(BUILD_NO_TINYUSB) +// A software restart leaves OTG HS attached: detach first, or the host keeps talking to a device that stopped answering +static void usb_disconnect_shutdown_handler(void) { + tud_disconnect(); + esp_rom_delay_us(20000); +} +#endif + static usb_phy_handle_t phy_hdl; void board_dfu_init(void) { // Configure USB PHY usb_phy_config_t phy_conf = { .controller = USB_PHY_CTRL_OTG, +#if BOARD_TUD_RHPORT == 1 + .target = USB_PHY_TARGET_UTMI, // OTG HS +#else .target = USB_PHY_TARGET_INT, +#endif .otg_mode = USB_OTG_MODE_DEVICE, // https://github.com/hathach/tinyusb/issues/2943#issuecomment-2601888322 // Set speed to undefined (auto-detect) to avoid timinng/racing issue with S3 with host such as macOS .otg_speed = USB_PHY_SPEED_UNDEFINED, }; - usb_new_phy(&phy_conf, &phy_hdl); + ESP_ERROR_CHECK(usb_new_phy(&phy_conf, &phy_hdl)); + +#if CONFIG_IDF_TARGET_ESP32P4 && !defined(BUILD_NO_TINYUSB) + esp_register_shutdown_handler(usb_disconnect_shutdown_handler); +#endif } void board_reset(void) { @@ -351,6 +371,7 @@ void board_timer_stop(void) { #if CONFIG_IDF_TARGET_ESP32S3 #include "hal/usb_serial_jtag_ll.h" +#include "soc/usb_pins.h" static void hw_cdc_reset_handler(void *arg) { portBASE_TYPE xTaskWoken = 0; @@ -437,7 +458,11 @@ static void IRAM_ATTR usb_persist_shutdown_handler(void) { } #endif +#if CONFIG_IDF_TARGET_ESP32P4 + REG_SET_BIT(LP_SYSTEM_REG_SYS_CTRL_REG, LP_SYSTEM_REG_FORCE_DOWNLOAD_BOOT); +#else REG_WRITE(RTC_CNTL_OPTION1_REG, RTC_CNTL_FORCE_DOWNLOAD_BOOT); +#endif } // Invoked when cdc when line state changed e.g connected/disconnected diff --git a/ports/espressif/boards/espressif_p4_function_ev/board.cmake b/ports/espressif/boards/espressif_p4_function_ev/board.cmake new file mode 100644 index 000000000..fe4db4fc1 --- /dev/null +++ b/ports/espressif/boards/espressif_p4_function_ev/board.cmake @@ -0,0 +1,2 @@ +# Apply board specific content here +set(IDF_TARGET "esp32p4") diff --git a/ports/espressif/boards/espressif_p4_function_ev/board.h b/ports/espressif/boards/espressif_p4_function_ev/board.h new file mode 100644 index 000000000..af4bd34df --- /dev/null +++ b/ports/espressif/boards/espressif_p4_function_ev/board.h @@ -0,0 +1,58 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2020 Ha Thach (tinyusb.org) for Adafruit Industries + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#ifndef ESPRESSIF_P4_FUNCTION_EV_H_ +#define ESPRESSIF_P4_FUNCTION_EV_H_ + +//--------------------------------------------------------------------+ +// Button +//--------------------------------------------------------------------+ + +// Enter UF2 mode if GPIO is pressed while 2nd stage bootloader is running. +// BOOT button is GPIO35, user should not hold this while reset since that +// will instead run the 1st stage ROM bootloader +#define PIN_BUTTON_UF2 35 + +//--------------------------------------------------------------------+ +// LED +//--------------------------------------------------------------------+ + +// No user LED on this board + +//--------------------------------------------------------------------+ +// USB UF2 +//--------------------------------------------------------------------+ + +// TinyUF2 runs on OTG HS (USB 2.0 Type-C port J15) +#define USB_VID 0x239A +#define USB_PID 0x00A5 // TODO temporarily shared with S2 saola wrover +#define USB_MANUFACTURER "Espressif" +#define USB_PRODUCT "ESP32P4 Function EV" + +#define UF2_PRODUCT_NAME USB_MANUFACTURER " " USB_PRODUCT +#define UF2_BOARD_ID "ESP32P4-FunctionEV-v1.4" +#define UF2_VOLUME_LABEL "P4FEVBOOT" +#define UF2_INDEX_URL "https://docs.espressif.com/projects/esp-dev-kits/en/latest/esp32p4/esp32-p4-function-ev-board/index.html" + +#endif diff --git a/ports/espressif/boards/espressif_p4_function_ev/sdkconfig b/ports/espressif/boards/espressif_p4_function_ev/sdkconfig new file mode 100644 index 000000000..571016c53 --- /dev/null +++ b/ports/espressif/boards/espressif_p4_function_ev/sdkconfig @@ -0,0 +1,10 @@ +# Board Specific Config + +# Partition Table +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="partitions-16MB.csv" + +# Serial flasher config +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y + +# ESP32-P4-Function-EV-Board has a chip below v3.0 (the ESP32-P4X-Function-EV-Board is v3.1+) +CONFIG_ESP32P4_SELECTS_REV_LESS_V3=y diff --git a/ports/espressif/bootloader_components/main/bootloader_start.c b/ports/espressif/bootloader_components/main/bootloader_start.c index 402aabba7..4f47616b7 100644 --- a/ports/espressif/bootloader_components/main/bootloader_start.c +++ b/ports/espressif/bootloader_components/main/bootloader_start.c @@ -205,7 +205,7 @@ static int selected_boot_partition(const bootloader_state_t *bs) { #ifdef PIN_DOUBLE_RESET_RC // Double reset detect if board implements 1-bit memory with RC components esp_rom_gpio_pad_select_gpio(PIN_DOUBLE_RESET_RC); - PIN_INPUT_ENABLE(GPIO_PIN_MUX_REG[PIN_DOUBLE_RESET_RC]); + gpio_ll_input_enable(&GPIO, PIN_DOUBLE_RESET_RC); if ( gpio_ll_get_level(&GPIO, PIN_DOUBLE_RESET_RC) == 1 ) { // RC pin is already high, double reset detected ESP_LOGI(TAG, "Detect double reset using RC on GPIO %d to enter UF2 bootloader", PIN_DOUBLE_RESET_RC); @@ -219,7 +219,7 @@ static int selected_boot_partition(const bootloader_state_t *bs) { #endif if (boot_index != FACTORY_INDEX) { - #if SOC_USB_SERIAL_JTAG_SUPPORTED + #if CONFIG_IDF_TARGET_ESP32S3 // startup with USB JTAG, while delaying here, USB JTAG will be enumerated which can cause confusion when // switching to OTG in application. Switch to OTG PHY here to avoid this. uint32_t const rtc_cntl_usb_conf = READ_PERI_REG(RTC_CNTL_USB_CONF_REG); @@ -232,7 +232,7 @@ static int selected_boot_partition(const bootloader_state_t *bs) { } esp_rom_gpio_pad_select_gpio(PIN_BUTTON_UF2); - PIN_INPUT_ENABLE(GPIO_PIN_MUX_REG[PIN_BUTTON_UF2]); + gpio_ll_input_enable(&GPIO, PIN_BUTTON_UF2); esp_rom_gpio_pad_pullup_only(PIN_BUTTON_UF2); // run the GPIO detection at least once even if UF2_DETECTION_DELAY_MS is set to zero @@ -249,7 +249,7 @@ static int selected_boot_partition(const bootloader_state_t *bs) { board_led_off(); } - #if SOC_USB_SERIAL_JTAG_SUPPORTED + #if CONFIG_IDF_TARGET_ESP32S3 WRITE_PERI_REG(RTC_CNTL_USB_CONF_REG, rtc_cntl_usb_conf); #endif } diff --git a/ports/espressif/components/ssd1306/Kconfig.projbuild b/ports/espressif/components/ssd1306/Kconfig.projbuild index 7b5a4bfd3..8fb3e6aba 100644 --- a/ports/espressif/components/ssd1306/Kconfig.projbuild +++ b/ports/espressif/components/ssd1306/Kconfig.projbuild @@ -6,6 +6,7 @@ menu "SSD1306 Configuration" default 46 if IDF_TARGET_ESP32S2 default 19 if IDF_TARGET_ESP32C3 default 48 if IDF_TARGET_ESP32S3 + default 54 if IDF_TARGET_ESP32P4 choice INTERFACE prompt "Interface" @@ -57,6 +58,7 @@ menu "SSD1306 Configuration" default 22 if IDF_TARGET_ESP32 default 12 if IDF_TARGET_ESP32S2 default 18 if IDF_TARGET_ESP32S3 + default 32 if IDF_TARGET_ESP32P4 default 9 if IDF_TARGET_ESP32C3 help GPIO number (IOxx) to I2C SCL. @@ -70,6 +72,7 @@ menu "SSD1306 Configuration" default 21 if IDF_TARGET_ESP32 default 11 if IDF_TARGET_ESP32S2 default 17 if IDF_TARGET_ESP32S3 + default 33 if IDF_TARGET_ESP32P4 default 10 if IDF_TARGET_ESP32C3 help GPIO number (IOxx) to I2C SDA. @@ -82,6 +85,7 @@ menu "SSD1306 Configuration" default 33 if IDF_TARGET_ESP32 default 38 if IDF_TARGET_ESP32S2 default -1 if IDF_TARGET_ESP32S3 + default -1 if IDF_TARGET_ESP32P4 default 3 if IDF_TARGET_ESP32C3 help GPIO number (IOxx) to RESET. diff --git a/ports/espressif/components/ssd1306/ssd1306_spi.c b/ports/espressif/components/ssd1306/ssd1306_spi.c index 431066c04..3e6fa20e6 100644 --- a/ports/espressif/components/ssd1306/ssd1306_spi.c +++ b/ports/espressif/components/ssd1306/ssd1306_spi.c @@ -17,6 +17,8 @@ #define LCD_HOST SPI2_HOST #elif CONFIG_IDF_TARGET_ESP32S3 #define LCD_HOST SPI2_HOST +#elif CONFIG_IDF_TARGET_ESP32P4 +#define LCD_HOST SPI2_HOST #elif CONFIG_IDF_TARGET_ESP32C3 #define LCD_HOST SPI2_HOST #endif diff --git a/ports/espressif/components/tinyusb_src/CMakeLists.txt b/ports/espressif/components/tinyusb_src/CMakeLists.txt index faab43a62..0bab280be 100644 --- a/ports/espressif/components/tinyusb_src/CMakeLists.txt +++ b/ports/espressif/components/tinyusb_src/CMakeLists.txt @@ -7,6 +7,8 @@ set(tusb_src "${CMAKE_CURRENT_SOURCE_DIR}/../../../../lib/tinyusb/src") if(target STREQUAL "esp32s3") set(tusb_mcu "OPT_MCU_ESP32S3") +elseif(target STREQUAL "esp32p4") + set(tusb_mcu "OPT_MCU_ESP32P4") elseif(target STREQUAL "esp32s2") set(tusb_mcu "OPT_MCU_ESP32S2") else() diff --git a/ports/espressif/components/tinyusb_src/tusb_config.h b/ports/espressif/components/tinyusb_src/tusb_config.h index 229d80454..5eb97151c 100644 --- a/ports/espressif/components/tinyusb_src/tusb_config.h +++ b/ports/espressif/components/tinyusb_src/tusb_config.h @@ -102,9 +102,6 @@ #define CFG_TUD_CDC_RX_BUFSIZE 512 #define CFG_TUD_CDC_TX_BUFSIZE 512 -// CDC Endpoint transfer buffer size, more is faster -#define CFG_TUD_CDC_EP_BUFSIZE 64 - //------------- MSC -------------// // MSC Buffer size of Device Mass storage #define CFG_TUD_MSC_BUFSIZE 4096 diff --git a/ports/espressif/family.cmake b/ports/espressif/family.cmake index 571fc4445..4751c5722 100644 --- a/ports/espressif/family.cmake +++ b/ports/espressif/family.cmake @@ -1,5 +1,12 @@ include(${CMAKE_CURRENT_LIST_DIR}/boards/${BOARD}/board.cmake) +if(IDF_TARGET STREQUAL "esp32p4") + # TinyUF2 and the apps run on OTG HS + add_compile_definitions(BOARD_TUD_RHPORT=1) + # Chip revisions <3.0 and >=3.0 need separate images: standalone apps take the board sdkconfig, which selects one + set(SDKCONFIG_DEFAULTS ${CMAKE_CURRENT_SOURCE_DIR}/sdkconfig.defaults ${BOARD_PATH}/sdkconfig) +endif() + set(UF2_FAMILY_ID_esp32s2 0xbfdd4eee) set(UF2_FAMILY_ID_esp32s3 0xc47e5767) set(UF2_FAMILY_ID_esp32p4 0x3d308e94) diff --git a/supported_boards.md b/supported_boards.md index 6e311c5af..88d738bf4 100644 --- a/supported_boards.md +++ b/supported_boards.md @@ -43,6 +43,7 @@ | espressif_esp32s3_eye | Espressif ESP32S3 EYE | 0x303A:0x700E | https://github.com/espressif/esp-who/blob/master/docs/en/get-started/ESP32-S3-EYE_Getting_Started_Guide.md | | espressif_hmi_1 | Espressif HMI 1 | 0x303A:0x7000 | https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/hw-reference/esp32s2/user-guide-esp32-s2-kaluga-1-kit.html | | espressif_kaluga_1 | Espressif Kaluga 1 | 0x239A:0x00C7 | https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/hw-reference/esp32s2/user-guide-esp32-s2-kaluga-1-kit.html | +| espressif_p4_function_ev | Espressif ESP32P4 Function EV | 0x239A:0x00A5 | https://docs.espressif.com/projects/esp-dev-kits/en/latest/esp32p4/esp32-p4-function-ev-board/index.html | | espressif_saola_1_wroom | Espressif Saola 1M WROOM | 0x239A:0x00A7 | https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/hw-reference/esp32s2/user-guide-saola-1-v1.2.html | | espressif_saola_1_wrover | Espressif Saola 1R WROVER | 0x239A:0x00A5 | https://docs.espressif.com/projects/esp-idf/en/latest/esp32s2/hw-reference/esp32s2/user-guide-saola-1-v1.2.html | | firebeetle2_esp32s3 | DFRobot FireBeetle 2 ESP32-S3 | 0x3343:0x83CF | https://www.dfrobot.com/product-2676.html | diff --git a/test/hil/local.json b/test/hil/local.json index d415ffa5a..71c05a01b 100644 --- a/test/hil/local.json +++ b/test/hil/local.json @@ -17,6 +17,12 @@ "uid": "7CDFA1E06FD0", "flasher": {"name": "esptool", "uid": "8ecc58dfb1cdeb1185e7b806e93fd3f1", "args": "--chip esp32s3"}, "tinyuf2": {"reentry": "dfu-detach"} + }, + { + "name": "espressif_p4_function_ev", + "uid": "30EDA0E0C902", + "flasher": {"name": "esptool", "uid": "1ec47e6369c3ee119baac50f9e1b1c54", "args": "--chip esp32p4"}, + "tinyuf2": {"reentry": "dfu-detach"} } ] } From ded0c41fc63602596a27f4f106f0f7cf561742b7 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Wed, 14 Oct 2026 12:18:22 +0700 Subject: [PATCH 25/47] build: report CMake errors and skip ninja progress lines in firstError (#522) --- .claude/skills/build/scripts/check_build.py | 16 +++++++--- .claude/test/test_build.py | 34 +++++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index e73eca89b..17cf27f0e 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -99,6 +99,7 @@ 'ota_data_initial.bin', 'tinyuf2.bin'))} # a roster flasher's args, whose one group is the device or chip its flash target must name FLASHER_ARGS = {'jlink': r'-device (\S+)', 'esptool': r'--chip (\w+)'} +NINJA_STEP = re.compile(r'\[\d+/\d+\] ') # ninja's progress prefix # configure variables that select a probe or port: a HIL record sets them from the roster, never from -D SELECTION = ('JLINK_OPTION', 'OPENOCD_SERIAL', 'OPENOCD_OPTION', 'ESPPORT') @@ -538,11 +539,18 @@ def manifest(sel, info, hil_config=None): def first_error(log): lines = log.splitlines() - # a compiler or linker diagnostic names the cause; ninja's FAILED line only the object - for pattern in (r':\d+(:\d+)?: (fatal )?error:|undefined reference|region .* overflowed|multiple definition', + # a compiler, linker or CMake diagnostic names the cause; ninja's FAILED line only the object + for pattern in (r':\d+(:\d+)?: (fatal )?error:|undefined reference|region .* overflowed|multiple definition|' + r'^CMake Error\b', r'\berror\b|Error \d|FAILED|No rule to make|not found'): - for line in lines: + for i, line in enumerate(lines): + if NINJA_STEP.match(line): # e.g. "Building ... error.c.obj" + continue if re.search(pattern, line, re.I) and 'warning' not in line: + if re.match(r'CMake Error\b.*:$', line): # its message is on the next indented line + nxt = next((l for l in lines[i + 1:] if l.strip()), '') + if nxt[:1].isspace(): + line = f'{line} {nxt.strip()}' return line.strip()[:400] tail = [l for l in lines if l.strip()] return tail[-1].strip()[:400] if tail else '' @@ -734,7 +742,7 @@ def dry_run(ninja, root, target=None): if r.returncode or start is None: return None steps = [l for l in lines[start + 1:] if l.strip() and l != 'ninja: no work to do.'] - return None if any(not re.match(r'\[\d+/\d+\] ', l) for l in steps) else [l.split('] ', 1)[1] for l in steps] + return None if any(not NINJA_STEP.match(l) for l in steps) else [l.split('] ', 1)[1] for l in steps] def idf_step_writes_no_input(cmd, root, chip): diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index b1dfe1f03..964fe3f36 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -955,6 +955,40 @@ def test_compiler_diagnostic_beats_ninjas_failed_line(self): log = 'FAILED: CMakeFiles/tinyuf2.dir/src/main.c.obj\n/x/src/main.c:12:3: error: unknown type name\n' self.assertEqual(cb.first_error(log), '/x/src/main.c:12:3: error: unknown type name') + def test_cmake_error_is_reported_with_its_message(self): + log = ('[736/1124] Building C object esp-idf/mbedtls/mbedtls/library/CMakeFiles/mbedx509.dir/error.c.obj\n' + 'FAILED: bootloader-prefix/src/bootloader-stamp/bootloader-configure\n' + 'CMake Error at /opt/esp/idf/tools/cmake/build.cmake:380 (message):\n' + '\n' + " Failed to resolve component 'usb' required by component 'main'\n" + 'Call Stack (most recent call first):\n') + self.assertEqual(cb.first_error(log), 'CMake Error at /opt/esp/idf/tools/cmake/build.cmake:380 (message): ' + "Failed to resolve component 'usb' required by component 'main'") + + def test_cmake_error_with_its_message_on_the_line(self): + log = 'CMake Error: The source directory "/x" does not exist.\n next\n' + self.assertEqual(cb.first_error(log), 'CMake Error: The source directory "/x" does not exist.') + + def test_cmake_error_in_form_takes_its_message(self): + log = 'CMake Error in main/CMakeLists.txt:\n Target "x" links to itself.\n' + self.assertEqual(cb.first_error(log), 'CMake Error in main/CMakeLists.txt: Target "x" links to itself.') + + def test_cmake_error_header_takes_no_indented_line_after_a_progress_line(self): + log = 'CMake Error at CMakeLists.txt:3 (message):\n[2/3] Running another command\n unrelated output\n' + self.assertEqual(cb.first_error(log), 'CMake Error at CMakeLists.txt:3 (message):') + + def test_cmake_error_header_takes_no_unindented_line(self): + log = 'CMake Error at CMakeLists.txt:3 (message):\nninja: build stopped: subcommand failed.\n' + self.assertEqual(cb.first_error(log), 'CMake Error at CMakeLists.txt:3 (message):') + + def test_cmake_error_with_continuation_is_capped(self): + log = 'CMake Error at CMakeLists.txt:3 (message):\n ' + 'x' * 500 + '\n' + self.assertEqual(len(cb.first_error(log)), 400) + + def test_fallback_skips_progress_lines(self): + log = '[5/9] Building C object x/error.c.obj\nninja: error: loading build.ninja\n' + self.assertEqual(cb.first_error(log), 'ninja: error: loading build.ninja') + if __name__ == '__main__': unittest.main() From c04cd725bb455bf033e6517e7b411ed28d1c79d7 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Wed, 14 Oct 2026 12:20:43 +0700 Subject: [PATCH 26/47] Fix D+ pin mask in S3 Touch1200 USB-Serial-JTAG switch (#537) Restore the USB-Serial-JTAG D+ pull-up that gpio_config() disables through the pull override; it survives esp_restart's CPU reset and kept the ROM from attaching. --- ports/espressif/boards/boards.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ports/espressif/boards/boards.c b/ports/espressif/boards/boards.c index 7de9be1f4..dadff5a7a 100644 --- a/ports/espressif/boards/boards.c +++ b/ports/espressif/boards/boards.c @@ -406,7 +406,7 @@ static void usb_switch_to_cdc_jtag(void) { // Force the host to re-enumerate (BUS_RESET) gpio_config_t dp_dm_conf = { - .pin_bit_mask = (1ULL << USBPHY_DM_NUM) | (1ULL < USBPHY_DP_NUM), + .pin_bit_mask = (1ULL << USBPHY_DM_NUM) | (1ULL << USBPHY_DP_NUM), .mode = GPIO_MODE_OUTPUT_OD, .pull_up_en = GPIO_PULLUP_DISABLE, .pull_down_en = GPIO_PULLDOWN_DISABLE, @@ -418,6 +418,10 @@ static void usb_switch_to_cdc_jtag(void) { // Initialize CDC+JTAG ISR to listen for BUS_RESET usb_serial_jtag_ll_phy_enable_external(false); + // gpio_config() turned off the D+ pull-up via the pull override; restore it, a CPU reset won't + const usb_serial_jtag_pull_override_vals_t pull_conf = {.dp_pu = 1, .dm_pu = 0, .dp_pd = 0, .dm_pd = 0}; + usb_serial_jtag_ll_phy_enable_pull_override(&pull_conf); + usb_serial_jtag_ll_phy_disable_pull_override(); usb_serial_jtag_ll_disable_intr_mask(USB_SERIAL_JTAG_LL_INTR_MASK); usb_serial_jtag_ll_clr_intsts_mask(USB_SERIAL_JTAG_LL_INTR_MASK); usb_serial_jtag_ll_ena_intr_mask(USB_SERIAL_JTAG_INTR_BUS_RESET); From 3c5e10c7be12cd5647ae1039d70e3e1491901133 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Wed, 14 Oct 2026 14:38:22 +0700 Subject: [PATCH 27/47] espressif: keep tinyuf2.bin inside the uf2 partition under ESP-IDF 6.1 (#523) * espressif: drop CONFIG_VFS_SUPPORT_IO=n, which links lwIP under ESP-IDF 6.1 With VFS off, IDF 6.1 resolves libc's _write_r from lwIP's no_vfs_syscalls.c, pulling in the TCP/IP stack (41 KB): adafruit_feather_esp32s3_tft's tinyuf2.bin outgrew its 0x40000 uf2 partition (0x41630). With VFS on it is 0x38cf0. The console gets no USB-Serial-JTAG secondary: TinyUSB takes the shared USB PHY, so that output could only delay writes. * espressif: fail the build when tinyuf2.bin outgrows the uf2 partition IDF's app_check_size only warns while ota_0 still fits. The check runs as part of app, so idf.py flash refuses an image that would spill into ffat; the HIL flash dry-run admits its command. * espressif: report a missing factory partition at build time, not configure * espressif: state the UART0-only console trade-off * espressif: drop the USB-Serial-JTAG secondary console only on ESP32-S3 --- .claude/skills/build/scripts/check_build.py | 5 ++++- .claude/test/test_build.py | 5 +++-- ports/espressif/CMakeLists.txt | 15 +++++++++++++++ ports/espressif/sdkconfig.defaults | 3 --- ports/espressif/sdkconfig.defaults.esp32s3 | 2 ++ 5 files changed, 24 insertions(+), 6 deletions(-) create mode 100644 ports/espressif/sdkconfig.defaults.esp32s3 diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index 17cf27f0e..d36d594fe 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -770,7 +770,10 @@ def idf_step_writes_no_input(cmd, root, chip): argv[1:3] in (['-E', 'make_directory'], ['-E', 'touch']) and len(argv) == 4 and writes(argv[3]) or argv[1:3] == ['-E', 'copy'] and len(argv) == 5 and writes(argv[4], '_bin')) or \ re.fullmatch(py + r' \S*/partition_table/check_sizes\.py --offset', head) and ( - tail[1:] == ['partition', '--type', 'app', str(root / 'partition_table/partition-table.bin'), str(root / 'tinyuf2.bin')] or + tail[1:] in ( + ['partition', '--type', 'app', str(root / 'partition_table/partition-table.bin'), str(root / 'tinyuf2.bin')], + ['partition', '--type', 'app', '--subtype', 'factory', str(root / 'partition_table/partition-table.bin'), + str(root / 'tinyuf2.bin')]) or len(tail) == 4 and tail[1] == 'bootloader' and re.fullmatch(r'0x[0-9a-f]+', tail[2]) and tail[3] == str(root / 'bootloader/bootloader.bin')) or \ re.fullmatch(py + r' \S*/lib/uf2/utils/uf2conv\.py --carray', head) and \ diff --git a/.claude/test/test_build.py b/.claude/test/test_build.py index 964fe3f36..86928fca8 100644 --- a/.claude/test/test_build.py +++ b/.claude/test/test_build.py @@ -765,14 +765,15 @@ def test_an_idf_flash_target_may_also_run_steps_that_write_no_flash_input(self): size = (f'cd {root}/esp-idf/esptool_py && {sizes} partition --type app {root}/partition_table/partition-table.bin ' f'{root}/tinyuf2.bin') boot_size = f'cd {root}/bootloader/esp-idf/esptool_py && {sizes} bootloader 0x0 {root}/bootloader/bootloader.bin' + uf2_size = f'cd {root} && {sizes} partition --type app --subtype factory {root}/partition_table/partition-table.bin {root}/tinyuf2.bin' copies = ' && '.join(f'{cmake} -E copy {root}/{f} {root}/_bin/{Path(f).name}' for f in cb.FLASH['esptool'][1]) post = (f'cd {root} && bash {root}/tinyuf2.postbuild.sh && cd {root} && {cmake} -E echo "Copy binaries for artifact" && ' f'{copies} && cd {root} && {py}3.10 {ROOT}/lib/uf2/utils/uf2conv.py --carray -o ' f'{root}/apps/update_tinyuf2/bootloader_bin.c {root}/tinyuf2.bin') stamps = (f'cd {root} && {cmake} -E make_directory {root}/CMakeFiles && {cmake} -E touch {root}/CMakeFiles/bootloader-complete ' f'&& {cmake} -E touch {root}/bootloader-prefix/src/bootloader-stamp/bootloader-done') - fresh = [f'cd {root}/bootloader && {cmake} --build .', size, f'cd {root}/bootloader && {cmake} -E echo_append', post, stamps, - 'FLASH'] + fresh = [f'cd {root}/bootloader && {cmake} --build .', size, uf2_size, f'cd {root}/bootloader && {cmake} -E echo_append', post, + stamps, 'FLASH'] script = root / 'tinyuf2.postbuild.sh' good = ('#!/bin/bash\necho "Creating combined.bin"\n' f"esptool.py --chip esp32s3 merge_bin --output combined.bin $(tr '\\n' ' ' < {root}/flash_args)\n") diff --git a/ports/espressif/CMakeLists.txt b/ports/espressif/CMakeLists.txt index bf282fc3c..2bb08a976 100644 --- a/ports/espressif/CMakeLists.txt +++ b/ports/espressif/CMakeLists.txt @@ -80,6 +80,21 @@ add_custom_target(combined-flash VERBATIM ) +# tinyuf2.bin must fit the factory (uf2) partition: IDF's app_check_size only warns while ota_0 is large enough +partition_table_get_partition_info(UF2_PARTITION_OFFSET "--partition-type app --partition-subtype factory" "offset") +if (UF2_PARTITION_OFFSET) + partition_table_add_check_size_target(tinyuf2_check_size + DEPENDS gen_project_binary + BINARY_PATH ${CMAKE_BINARY_DIR}/tinyuf2.bin + PARTITION_TYPE app + PARTITION_SUBTYPE factory + ) +else () + # fail the build, not configure, so menuconfig can still fix the partition table + fail_at_build_time(tinyuf2_check_size "${PARTITION_CSV_PATH} has no factory app partition for tinyuf2") +endif () +add_dependencies(app tinyuf2_check_size) + #------------------------------ # External project (update_tinyuf2, blinky etc ...) #------------------------------ diff --git a/ports/espressif/sdkconfig.defaults b/ports/espressif/sdkconfig.defaults index c2a438a64..d454e83a6 100644 --- a/ports/espressif/sdkconfig.defaults +++ b/ports/espressif/sdkconfig.defaults @@ -11,8 +11,5 @@ CONFIG_PARTITION_TABLE_MD5=y CONFIG_COMPILER_OPTIMIZATION_SIZE=y CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_SILENT=y -# Virtual file system -CONFIG_VFS_SUPPORT_IO=n - # Compatibility options CONFIG_FLASHMODE_QIO=y diff --git a/ports/espressif/sdkconfig.defaults.esp32s3 b/ports/espressif/sdkconfig.defaults.esp32s3 new file mode 100644 index 000000000..6b468d826 --- /dev/null +++ b/ports/espressif/sdkconfig.defaults.esp32s3 @@ -0,0 +1,2 @@ +# Console on UART0 only: TinyUSB takes the USB PHY that USB-Serial-JTAG shares, so it would carry only output before board_dfu_init +CONFIG_ESP_CONSOLE_SECONDARY_NONE=y From abf80dd2d8fc53fafe2219ae38841af6d294af5e Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Wed, 14 Oct 2026 17:26:31 +0700 Subject: [PATCH 28/47] Build every port with CMake (#538) Apps build as external projects via family_add_app(). CI uses CMake for ports with a board that passed HIL with it (maxim, stm32f4, stm32h5, mimxrt10xx, stm32f3); the rest stay on make until it is removed. CMake ports release with CMake file names, and Build runs on pull requests and master pushes only. Add stm32f303disco to the HIL rig: the runner power-cycles its hub port while DFU is awaited, since the F303 has no internal D+ pull-up. --- .claude/skills/build/SKILL.md | 12 +- .claude/skills/build/scripts/check_build.py | 74 +++-- .claude/skills/hil/SKILL.md | 17 +- .claude/skills/hil/scripts/hil_test.py | 253 +++++++++++++++--- .claude/test/test_build.py | 59 ++-- .claude/test/test_hil.py | 211 ++++++++++++++- .github/workflows/build.yml | 17 +- .github/workflows/build_ghostfat.yml | 26 +- .github/workflows/build_util.yml | 19 +- .github/workflows/pre_commit.yml | 4 +- README.md | 9 + apps/blinky/CMakeLists.txt | 19 +- apps/erase_firmware/CMakeLists.txt | 18 +- apps/self_update/CMakeLists.txt | 30 ++- ports/ch32v20x/CMakeLists.txt | 8 +- ports/ch32v20x/family.cmake | 2 +- ports/family_support.cmake | 83 +++++- ports/kinetis_k32l2/CMakeLists.txt | 21 ++ .../boards/frdm_k32l2b/board.cmake | 9 + ports/kinetis_k32l2/boards/kuiic/board.cmake | 20 ++ ports/kinetis_k32l2/family.cmake | 65 +++++ ports/lpc55/CMakeLists.txt | 23 ++ .../boards/double_m33_express/board.cmake | 16 ++ .../lpc55/boards/lpcxpresso55s28/board.cmake | 10 + .../lpc55/boards/lpcxpresso55s69/board.cmake | 10 + ports/lpc55/family.cmake | 70 +++++ ports/maxim/CMakeLists.txt | 8 +- ports/mimxrt10xx/CMakeLists.txt | 19 +- ports/mimxrt10xx/app.cmake | 4 +- .../apps/dfu_runtime/CMakeLists.txt | 15 +- .../apps/esp32programmer/CMakeLists.txt | 15 +- .../apps/factory_test/CMakeLists.txt | 15 +- .../apps/factory_test_metro_sd/CMakeLists.txt | 15 +- ports/mimxrt10xx/family.cmake | 7 +- ports/stm32f3/CMakeLists.txt | 9 +- ports/stm32f3/app.cmake | 8 + ports/stm32f3/apps/self_update/CMakeLists.txt | 33 --- ports/stm32f3/boards.c | 5 + ports/stm32f4/CMakeLists.txt | 8 +- ports/stm32h5/CMakeLists.txt | 8 +- ports/stm32h7/CMakeLists.txt | 28 ++ ports/stm32h7/app.cmake | 18 ++ ports/stm32h7/boards/daisy_seed/board.cmake | 17 ++ .../boards/mini_stm32h750_v11/board.cmake | 22 ++ ports/stm32h7/family.cmake | 105 ++++++++ ports/stm32l4/CMakeLists.txt | 24 ++ ports/stm32l4/boards/swan_r5/board.cmake | 16 ++ ports/stm32l4/family.cmake | 71 +++++ ports/template_port/CMakeLists.txt | 23 ++ ports/template_port/boards/dummy/board.cmake | 2 + ports/template_port/family.cmake | 37 +++ ports/test_ghostfat/CMakeLists.txt | 14 +- ports/test_ghostfat/boards/32k/board.cmake | 6 + .../boards/32k_favicon/board.cmake | 7 + .../boards/4k_favicon/board.cmake | 6 + ports/test_ghostfat/boards/512b/board.cmake | 2 + .../boards/512b_favicon/board.cmake | 5 + ports/test_ghostfat/boards/huge/board.cmake | 8 + test/hil/local.json | 6 + 59 files changed, 1444 insertions(+), 247 deletions(-) create mode 100644 ports/kinetis_k32l2/CMakeLists.txt create mode 100644 ports/kinetis_k32l2/boards/frdm_k32l2b/board.cmake create mode 100644 ports/kinetis_k32l2/boards/kuiic/board.cmake create mode 100644 ports/kinetis_k32l2/family.cmake create mode 100644 ports/lpc55/CMakeLists.txt create mode 100644 ports/lpc55/boards/double_m33_express/board.cmake create mode 100644 ports/lpc55/boards/lpcxpresso55s28/board.cmake create mode 100644 ports/lpc55/boards/lpcxpresso55s69/board.cmake create mode 100644 ports/lpc55/family.cmake create mode 100644 ports/stm32f3/app.cmake delete mode 100644 ports/stm32f3/apps/self_update/CMakeLists.txt create mode 100644 ports/stm32h7/CMakeLists.txt create mode 100644 ports/stm32h7/app.cmake create mode 100644 ports/stm32h7/boards/daisy_seed/board.cmake create mode 100644 ports/stm32h7/boards/mini_stm32h750_v11/board.cmake create mode 100644 ports/stm32h7/family.cmake create mode 100644 ports/stm32l4/CMakeLists.txt create mode 100644 ports/stm32l4/boards/swan_r5/board.cmake create mode 100644 ports/stm32l4/family.cmake create mode 100644 ports/template_port/CMakeLists.txt create mode 100644 ports/template_port/family.cmake create mode 100644 ports/test_ghostfat/boards/32k/board.cmake create mode 100644 ports/test_ghostfat/boards/32k_favicon/board.cmake create mode 100644 ports/test_ghostfat/boards/4k_favicon/board.cmake create mode 100644 ports/test_ghostfat/boards/512b/board.cmake create mode 100644 ports/test_ghostfat/boards/512b_favicon/board.cmake create mode 100644 ports/test_ghostfat/boards/huge/board.cmake diff --git a/.claude/skills/build/SKILL.md b/.claude/skills/build/SKILL.md index 6db5c4b30..4bd974db2 100644 --- a/.claude/skills/build/SKILL.md +++ b/.claude/skills/build/SKILL.md @@ -34,15 +34,15 @@ An agent that reports `buildOk` and `notes` (agentrc's `code-writer`) maps the J - A change to a port, board, app or dependency builds the boards that compile it: a board's own files that board; other port files the port's `REPRESENTATIVES` (rig boards first, every ESP32 target the port's boards use, both i.MX RT10xx and RT1170); an app or dependency the representatives of the ports whose CI backend compiles it. - Shared code (`src/`, `lib/`, the shared build files) and anything unclassified builds every port's representatives, plus the ghostfat self-test when a ghostfat input changed. - A change to how boards build (this script, `build.yml`, `build_util.yml`, `ci_set_matrix.py`, `.github/actions/`) runs the contract's own tests and one smoke build per backend, and a change to this script the HIL battery on every roster board, since the HIL runner imports it; no firmware build verifies workflow syntax or orchestration. A change to the contract's tests runs the tests only. - - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads. - - Firmware no CI build compiles is uncovered: `ports/template_port/`, `ports/ch32v20x/` (`--board ch32v203_r0_1v0` builds it on request, with a note), a build file the port's CI backend never reads, a stub app. + - CodeQL's make build of feather_stm32f405_express counts as coverage for the make files it reads; `ports/make.mk` and `ports/rules.mk` select the make ports' representatives and that build. + - Firmware no CI build compiles is uncovered: `ports/template_port/`, `ports/ch32v20x/` (`--board ch32v203_r0_1v0` builds it on request with make, with a note), a build file no CI build of the port reads (a make port's CMake files, a cmake port's make files but CodeQL's, Espressif's `Makefile`), a stub app. - Docs and agent, editor and repo metadata build nothing. -- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`; Espressif `idf.py build`. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds. A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. -- **`--backend cmake`** is for the HIL's receipt only, with `--board` and `--shared`: CI still builds each port with its `build.yml` backend, so it is no CI evidence. A port without `CMakeLists.txt`, or Espressif, is refused. `-D` takes the cmake spellings: `LOGGER=RTT`, not `rtt`; `DEBUG` is ignored, the build stays MinSizeRel; `LOG` sets `CFG_TUSB_DEBUG` only, not `TUF2_LOG`; raw flags go through `CFLAGS_CLI`. +- **Builds are CI's.** Each port uses the backend `build.yml` gives it (`BACKENDS` in the script; `test_build.py` fails when the two disagree): make `all self-update`, then `copy-artifact`, then every `ports//apps/*` but `self_update`; cmake `-G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel`, which also builds each app the port's `CMakeLists.txt` adds with `family_add_app()` as its own project; Espressif `idf.py build`. A port moves from make to cmake in `build.yml` once a board of it passes the HIL battery with cmake. A board passes only with the artifacts its build must produce (`expected_artifacts()`): the bootloader images and the self-update and app images the port builds (cmake: its `family_add_app()` calls enabled for the board). A port without self-update, or a stub app, is listed in `notImplemented`, which is not a failure. +- **`--backend cmake`** is for the HIL's receipt only, with `--board` and `--shared`: on a make port it is no CI evidence, and it is how such a port's board gets the HIL run that moves the port to cmake. Espressif is refused. `-D` takes the cmake spellings: `LOGGER=RTT`, not `rtt`; `DEBUG` is ignored, the build stays MinSizeRel; `LOG` (default 0, as in make) sets `TUF2_LOG` and `CFG_TUSB_DEBUG`; raw flags go through `CFLAGS_CLI`. - **A HIL record** (`--board --shared --hil-config `) binds the build's own flash target to the board's roster flasher: a J-Link board is configured with `JLINK_OPTION=-USB -ExitOnError 1 -NoGui 1` (so `tinyuf2-jlink` selects that probe and fails non-zero), an esptool board's ESP-IDF `flash` target takes the bridge's tty at flash time. `boards[].flash` holds the target, the roster flasher, the target's generated command (checked to name that probe or chip and the build's own image), the configure defines that bound it and the build-tree files it reads (in `sha256` too); a rebuild and a HIL flash hold the build dir's lock (`.lock`). Another flasher, a roster `args` other than `-device ` or `--chip `, or a `-D` of `JLINK_OPTION`, `OPENOCD_SERIAL`, `OPENOCD_OPTION` or `ESPPORT` is refused. Before flashing, the HIL dry-runs that target under the lock and refuses any command but the flasher and, on ESP-IDF, the generated ones that write no flash input (its bootloader subproject dry-run too); ESP-IDF re-runs CMake after any commit (its `git describe` dependency), so rebuild the record on the commit under test. - **ESP-IDF** builds in CI's image, `espressif/idf:`, through docker by default; a missing image is exit 2 naming the `docker pull`. `--idf local` uses the exported IDF and records its version and `matchesCi`: evidence from a different IDF is supplemental, not acceptance. -- **ghostfat** builds each `ports/test_ghostfat/boards/*` config natively and runs it against its known-good image in the private dir, one config at a time: `huge` needs about 3 GiB of disk while it runs. A passing config's dir is removed; a failing one keeps its log. +- **ghostfat** builds each `ports/test_ghostfat/boards/*` config natively with cmake and runs its `selftest` target, which compares the generated image with the config's known-good one, in the private dir, one config at a time: `huge` needs about 3 GiB of disk while it runs. A passing config's dir is removed; a failing one keeps its log. - **Private dirs for parallel agents.** Each build starts from an empty `cmake-build/cmake-build-agent--` (with `-` when a board builds with another backend than its port's) and is removed when it passes (`removed`; its `artifacts` say what it produced, nothing is left to flash); a failed one is kept for its log. Nothing is written into the source tree: make takes `BUILD_ROOT=`, cmake and idf `-DARTIFACT_PATH=/_bin`. `--shared` rebuilds `cmake-build/cmake-build-` from clean and keeps it for flashing; it needs exclusive ownership of that board. - **Dependencies are never fetched here.** `lib/tinyusb` and `lib/uf2` are submodules (`git submodule update --init lib/tinyusb lib/uf2`); a port's `tools/get_deps.py` rows must exist. In a worktree, symlink them from the primary checkout (CLAUDE.md), never run `get_deps.py` through a symlink: it runs `git reset --hard` in each dependency. -- **Debug builds** for a probe session use the port's backend directly: make `make -C ports/ BOARD= DEBUG=1 LOG=2 LOGGER=rtt all` (`LOGGER=swo` too); cmake `-DCMAKE_BUILD_TYPE=Debug -DLOG=2 -DLOGGER=RTT` (only `RTT` adds the RTT sources). Flash targets: make `flash-jlink`, `flash-pyocd`, `flash-stlink`, `flash-uf2`; cmake `tinyuf2-jlink`, `-stlink`, `-openocd`; Espressif `idf.py -C ports/espressif -DBOARD= flash`. +- **Debug builds** for a probe session use the port's backend directly: make `make -C ports/ BOARD= DEBUG=1 LOG=2 LOGGER=rtt all` (`LOGGER=swo` too); cmake `-DCMAKE_BUILD_TYPE=Debug -DLOG=2 -DLOGGER=RTT` (only `RTT` adds the RTT sources). Flash targets: make `flash-jlink`, `flash-pyocd`, `flash-stlink`, `flash-uf2`; cmake `tinyuf2-jlink`, `-stlink`, `-openocd`, `-pyocd` (lpc55, kinetis_k32l2), `-dfu-util` (stm32l4), and `-uf2` for each app; Espressif `idf.py -C ports/espressif -DBOARD= flash`. - **Printing a command is not verification.** A verdict is the JSON of a run that happened; an agent that cannot run the script reports the build as not run. diff --git a/.claude/skills/build/scripts/check_build.py b/.claude/skills/build/scripts/check_build.py index d36d594fe..7c1d522f6 100755 --- a/.claude/skills/build/scripts/check_build.py +++ b/.claude/skills/build/scripts/check_build.py @@ -18,7 +18,6 @@ import contextlib import fcntl import functools -import gzip import hashlib import json import os @@ -28,7 +27,6 @@ import shutil import subprocess import sys -import zlib from pathlib import Path ROOT = Path(__file__).resolve().parents[4] @@ -41,8 +39,8 @@ 'kinetis_k32l2': 'make', 'lpc55': 'make', 'maxim': 'cmake', - 'mimxrt10xx': 'make', - 'stm32f3': 'make', + 'mimxrt10xx': 'cmake', + 'stm32f3': 'cmake', 'stm32f4': 'cmake', 'stm32h5': 'cmake', 'stm32h7': 'make', @@ -69,14 +67,15 @@ } # compiler a port's make/cmake build needs, when not arm-none-eabi-gcc COMPILER = {'ch32v20x': 'riscv-none-elf-gcc'} -# make ports whose self-update target only echoes "not implemented yet" +# ports whose make self-update target only echoes "not implemented yet" NO_SELF_UPDATE = {'ch32v20x', 'kinetis_k32l2', 'stm32h7', 'stm32l4'} GHOSTFAT = 'test_ghostfat' GHOSTFAT_INPUTS = ('src/ghostfat.c', 'src/favicon/', 'src/board_api.h', 'src/compile_date.h', 'src/uf2.h', - 'lib/uf2', 'ports/make.mk', 'ports/rules.mk', 'ports/test_ghostfat/', + 'lib/uf2', 'ports/family_support.cmake', 'ports/test_ghostfat/', '.github/workflows/build_ghostfat.yml') -CORE = ('src/', 'lib/', 'ports/make.mk', 'ports/rules.mk', 'ports/family_support.cmake', 'CMakeLists.txt', - 'cmake/', 'tools/get_deps.py') +CORE = ('src/', 'lib/', 'ports/family_support.cmake', 'CMakeLists.txt', 'cmake/', 'tools/get_deps.py') +# make files every Makefile port includes: read by the make ports' builds and CodeQL's +MAKE_SHARED = ('ports/make.mk', 'ports/rules.mk') CONTRACT_TESTS = ('.claude/test/test_build', '.claude/test/test_hil') # the HIL contract's runner and rosters: a change there re-runs the battery on every roster board HIL_CONTROL = ('.claude/skills/hil/scripts/', 'test/hil/') @@ -187,6 +186,22 @@ def espressif_apps(): return re.findall(r'SOURCE_DIR\s+\$\{CMAKE_CURRENT_LIST_DIR\}/apps/(\w+)', code_lines(ports_dir('espressif') / 'CMakeLists.txt')) +def cmake_app_calls(port): + """The family_add_app() calls of a cmake port's CMakeLists.txt as (name, source dir, boards): boards + None for every board, else those its BOARDS argument names.""" + calls = [] + for m in re.finditer(r'^\s*family_add_app\(\s*([\w-]+)\s+([^\s)]+)([^)]*)\)', + code_lines(ports_dir(port) / 'CMakeLists.txt'), re.M): + boards = re.search(r'\bBOARDS\s+([^)]*)', m.group(3)) + calls.append((m.group(1), m.group(2), boards.group(1).split() if boards else None)) + return calls + + +def cmake_apps(port, board): + """Apps a cmake port builds for the board, each named after the app's target and so its .uf2.""" + return [name for name, _, boards in cmake_app_calls(port) if boards is None or board in boards] + + def port_app_selection(port, path): """Boards that compile a change under ports//apps/: the app's own enabled boards, or for a file shared by the port's apps (app.mk) those of every app that includes it.""" @@ -196,6 +211,15 @@ def port_app_selection(port, path): if len(parts) >= 4 and parts[3] not in espressif_apps(): return None, f'ports/espressif/CMakeLists.txt builds no ExternalProject for apps/{parts[3]}' return rep_selection([port], f'builds {"/".join(parts[2:4])}'), f'{port} {"/".join(parts[2:4])}' + if BACKENDS[port] == 'cmake': + calls = [c for c in cmake_app_calls(port) if c[1].endswith(f'/apps/{parts[3]}')] if len(parts) >= 5 else [] + if not calls: + return None, f'ports/{port}/CMakeLists.txt adds no app that reads {path}' + boards = {} + for name, _, enabled in calls: + boards.update(rep_selection([port], f'builds {name}') if enabled is None else + {(b, 'cmake'): f'enables {name}' for b in enabled}) + return boards, f'{port} apps: {", ".join(sorted(c[0] for c in calls))}' if len(parts) >= 5 and (apps_dir / parts[3]).is_dir(): apps = [apps_dir / parts[3]] else: @@ -319,6 +343,10 @@ def out(rule, effect, reason, boards=None, ghostfat=False, tests=False, hil=Fals if not families: return out('dep', 'gap', f'{path}: no CI-built port uses this dependency') return out('dep', 'select', f'dependency of {", ".join(families)}', rep_selection(families, f'uses {path}')) + if path in MAKE_SHARED: + return out('make-shared', 'select', 'every make port and the CodeQL build read it', + {**rep_selection([p for p, be in BACKENDS.items() if be == 'make'], f'make reads {path}'), + (codeql_board, 'make'): f'codeql build reads {path}'}) if path.startswith(NOTHING): return out('nothing', 'none', 'no firmware build reads this path') if path.startswith(CORE): @@ -603,8 +631,7 @@ def expected_artifacts(port, board, backend, root): if backend == 'cmake': exp = [bin_dir / 'tinyuf2.bin', bin_dir / 'tinyuf2.hex'] cml = (ports_dir(port) / 'CMakeLists.txt').read_text() - for app in re.findall(r'add_subdirectory\(\$\{TOP\}/apps/(\w+)', cml): - exp.append(bin_dir / 'apps' / f'{"update-tinyuf2" if app == "self_update" else app}.uf2') + exp += [bin_dir / 'apps' / f'{app}.uf2' for app in cmake_apps(port, board)] exp += [bin_dir / a for a in re.findall(r'\$\{ARTIFACT_PATH\}/(apps/[\w.-]+\.uf2)', cml)] return exp return [bin_dir / f for f in ('tinyuf2.bin', 'bootloader.bin', 'partition-table.bin', 'ota_data_initial.bin', @@ -827,7 +854,8 @@ def build_board(board, backend, args, idf, flasher=None): 'status': 'ok', 'firstError': '', 'artifacts': []} if port in UNBUILT: rec['note'] = f'ports/{port}: {UNBUILT[port][1]}' - if port in NO_SELF_UPDATE: + if backend == 'make' and port in NO_SELF_UPDATE or backend == 'cmake' and \ + 'update-tinyuf2.uf2' not in {p.name for p in expected_artifacts(port, board, backend, root)}: rec['notImplemented'] = ['self-update'] problem = check_tools(port, backend, args.idf) or check_deps(port) if backend == 'idf' and args.idf == 'docker' and not problem: @@ -897,9 +925,9 @@ def keep_only_log(root, log): def run_ghostfat(verbose): - """Build and run the native self-test of every ports/test_ghostfat config, one at a time: - the `huge` image is about 1.5 GiB, twice over with its known-good copy.""" - missing = [t for t in ('make', 'gcc') if not shutil.which(t)] + """Build and run the native self-test (its CMake `selftest` target) of every ports/test_ghostfat config, + one at a time: the `huge` image is about 1.5 GiB, twice over with its known-good copy.""" + missing = [t for t in ('cmake', 'ninja', 'gcc', 'gzip') if not shutil.which(t)] if missing: return [{'config': cfg, 'status': 'error', 'firstError': f'{", ".join(missing)} not on PATH'} for cfg in boards_of(GHOSTFAT)] @@ -911,19 +939,11 @@ def run_ghostfat(verbose): log = root / 'check_build.log' rec = {'config': cfg, 'status': 'ok', 'firstError': ''} try: - if run(['make', '-C', ports_dir(GHOSTFAT), f'BOARD={cfg}', f'BUILD_ROOT={root}', f'-j{os.cpu_count()}', 'all'], - log, verbose): - rec.update(status='failed', firstError=first_error(log.read_text(errors='replace'))) - else: - work = root / 'ports' / GHOSTFAT - with gzip.open(ports_dir(GHOSTFAT) / 'boards' / cfg / 'knowngood.img.gz.gz') as outer, \ - gzip.open(outer) as inner, open(work / 'knowngood.img', 'wb') as dst: - shutil.copyfileobj(inner, dst, 1 << 20) - # the self-test writes ghostfat.img and reads knowngood.img in its working directory - if run([work / f'tinyuf2-{cfg}.elf'], log, verbose, cwd=work): - out = log.read_text(errors='replace') - rec.update(status='failed', firstError=(re.findall(r'^FAIL.*$', out, re.M) or [first_error(out)])[-1]) - except (OSError, EOFError, zlib.error) as e: + if run(['cmake', '-B', root, '-G', 'Ninja', f'-DBOARD={cfg}', ports_dir(GHOSTFAT)], log, verbose) or \ + run(['cmake', '--build', root, '--target', 'selftest'], log, verbose): + out = log.read_text(errors='replace') + rec.update(status='failed', firstError=(re.findall(r'^FAIL.*$', out, re.M) or [first_error(out)])[-1]) + except OSError as e: rec.update(status='error', firstError=f'{type(e).__name__}: {e}') finally: if rec['status'] == 'ok': diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index 0ae912f1c..ddf70611c 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -14,7 +14,8 @@ description: The TinyUF2 HIL contract. Use when a change needs hardware evidence A roster entry is named by its TinyUF2 board (`ports//boards/`, which also gives the port) and keeps tinyusb's shape (`name`, `uid`, `flasher {name, uid, args}`), so agentrc's `rtt.py`, `pc_sample.py` and `probe_state.py` resolve the probe with `--hil-config test/hil/local.json --board `. A J-Link entry has `args` exactly `-device `; the runner uses SWD at 4000 kHz, so pass the same to those scripts, `--interface swd --speed 4000`. An espressif entry has `flasher.name` `esptool`, `flasher.uid` the USB-UART bridge's serial (`/dev/serial/by-id/*_-if*`), `flasher.args` exactly `--chip `, and the top-level `uid` the chip's factory MAC, which is TinyUF2's USB serial. The roster holds the board's hardware only; layouts come from the candidate's source. The `tinyuf2` block holds: - the board lock name (`lock`) when tinyusb names the board differently, as `mimxrt1060_evk` for `imxrt1060_evk`; both runners take that one lock, and the helper's `status`, holder records and the report rows' `lock` show it. It defaults to the board name; -- the commissioned re-entry path (`reentry`). +- the commissioned re-entry path (`reentry`); +- for a board whose TinyUF2 signals no disconnect when it comes up after a reset, the per-port-power hub port its USB is on (`usbPower`: `{"hub": "", "port": N}`; the serial, not the bus location, so a replugged hub is still found). J-Link boards only. While `dfu` waits, the runner power-cycles that port once it has sat connected but unaddressed (sysfs state `default`) for 2 s of that wait, at most once per 10 s, and at no other time: it writes the port's sysfs `disable` 1, then 0 a second later, which the kernel runs under the hub's lock, after any enumeration in progress, disconnecting the child before VBUS goes off. Each write and read-back runs in a bounded child; a cycle that fails or is interrupted after the off write still writes 0, and says in `hil.log` whether that restored power. The report lists the cycles (`portCycles`). The battery runs a board only after commissioning sets `reentry` and confirms `uid`; `install` and `restore` work without `reentry`, for commissioning. Only commissioned roster boards can be offered for a run. Adding a board is commissioning, the lead's (below), never a review-time or dispatch-time action: the runner refuses a build record from a dirty tree, an uncommitted roster entry included. @@ -58,7 +59,7 @@ python3 .claude/skills/hil/scripts/hil_test.py run -b --build-record cma The battery's steps on one board, all under one lock. The roster's probe serial selects the board (each probe is wired to one target), so no step identifies the chip: 1. **install:** the record's `tinyuf2-jlink` target runs (`cmake --build --target tinyuf2-jlink`) under its build dir's lock, once the files it reads hash as recorded and its dry run shows nothing but the flasher; JLinkExe can exit 0 with every command failed, so the transcript must show the record's HEX downloaded (programmed, or skipped as already matching), `O.K.` and the script completed, with no failure line. The files are hashed again after it, and a session of the runner's verifies the BIN at its flash address (`verifybin`), then resets. The board is touched from the target's start; only a transcript that ends at "Connecting to J-Link ...FAILED", before any target connect, leaves it untouched. -2. **dfu:** the UF2 drive with this board's serial is found. The install has left the app start erased on the bench (Limits), so DFU comes up; if it never enumerates, as when a valid app boots, the commissioned `retained-register` re-entry brings DFU up (its claim says so); any other DFU failure fails the step. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. +2. **dfu:** the UF2 drive with this board's serial is found. On the RT boards the install has left the app start erased on the bench (Limits), so DFU comes up; if it never enumerates, as when a valid app boots (an install on stm32f303disco keeps the app, Limits), the commissioned `retained-register` re-entry brings DFU up (its claim says so); any other DFU failure fails the step. `INFO_UF2.TXT` must name this build's version and board, and `CURRENT.UF2` must have its size. The double-tap register is then cleared and read back, once the bootloader runs. 3. **copy:** a generated app is copied to the drive. 4. **app-ran:** the probe shows the app counting in thread mode with VTOR at the app start. 5. **reenter:** the bootloader is re-entered by the commissioned path, and the drive's `CURRENT.UF2` reads the app back. @@ -140,8 +141,9 @@ A review (`pr-review`) maps each board's row, except a commissioning run's (`com ## Commissioning a board Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under a manual hold, except for the re-entry run, which locks itself. Record each result in the board's commissioning note (agentrc `target-debug` project notes), and `uid`, `lock` and `reentry` in the roster. In this order: -- **Cabling:** cable the USB port TinyUF2 enumerates on (`BOARD_TUD_RHPORT`). It can differ from the one tinyusb examples use: P10, not P9, on lpcxpresso55s69; USB OTG1 on imxrt1060_evk. A DFU device "not seen" with no new device in `dmesg` is a cabling problem first. Keep the probe on a hub apart from the target's (Limits). +- **Cabling:** cable the USB port TinyUF2 enumerates on (`BOARD_TUD_RHPORT`). It can differ from the one tinyusb examples use: P10, not P9, on lpcxpresso55s69; USB OTG1 on imxrt1060_evk; USB USER (CN3) on stm32f303disco. A DFU device "not seen" with no new device in `dmesg` is a cabling problem first. Keep the probe on a hub apart from the target's (Limits). - **Silicon revision,** before any other result: read it from the chip marking or registers, and check that revision's errata through read-doc. RT1060's DIGPROG does not tell A from B, so the marking decides. +- **Port power** (a board with `usbPower`): the hub's port `disable` files writable by the runner's user, through `/etc/udev/rules.d/52-tinyuf2-hil-uhubctl.rules`, one line per hub: on `ACTION=="add|bind"` of its `DRIVER=="hub"` interface, matched by the hub's VID, PID and serial, `RUN` `chgrp plugdev` and `chmod g+w` on `/sys$devpath/*-port*/disable`. The runner refuses the board while that file is not writable. Check once that writing `disable` 1 switches the port off (`uhubctl -l -p ` reads `off`), then write 0. - **Automount exclusion:** `/etc/udev/rules.d/99-tinyuf2-hil.rules`, one line per board, with the VID and PID from the board's source, in exactly this form (the runner accepts only this form, with exact `==` matches): ``` @@ -172,17 +174,20 @@ Commissioning is the lead's, never `hil-operator`'s or a review's. It runs under `--try-reentry` is the lead's only, and is refused once `reentry` is set. The row's `reentryDecision` names the method when the run passed or failed only at `reenter`'s `CURRENT.UF2` readback, every observation of the tried method is true (`controlBootedApp`, `dfuCameUp` and `magicConsumed` for `retained-register`), and the board was cleaned up and not held. Set `reentry` to it and commit. A null decision leaves `reentry` unset: file a `followup` issue with the observations for diagnosis, not as a diagnosed bug. When the drive copy itself fails, a fixture installed by hand with J-Link `loadbin` can isolate re-entry from the copy for diagnosis; it never sets `reentry`. - **On an esptool board,** in place of the J-Link items: cable the native USB connector as well as the bridge's; check once by hand that the chip has no secure boot, secure download or flash encryption and that its flash holds the candidate's layout (`esptool get-security-info`, `esptool flash-id`), since the runner never checks them; record the bridge serial and the chip's MAC (`uid`, TinyUF2's USB serial); `install` from the record and observe DFU, then `restore` and observe DFU leave. Re-entry is `--try-reentry dfu-detach`, whose decision needs `detachAcked`, `appLeft` and `dfuCameUp`; before setting it, repeat DFU_DETACH ten times under the hold and check the host log shows no USB reset on the DETACH transitions (USB DFU 1.1 bitWillDetach). -- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid; on MIMXRT1011DAE5A the same note on every reset of a TinyUF2 image. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. +- **J-Link transcript shape:** that `regs` and `mem32` output parses (`after()`, `regs()`), and which reset notes are benign. A benign note is exempted by device and exact text in `BENIGN`: on MIMXRT1062xxx6A, "Could not find valid reset vector." followed by "Target halted on flash header read.", printed on every reset while the app is invalid; on MIMXRT1011DAE5A the same note on every reset of a TinyUF2 image; on stm32f303vc, "Active write protection detected.", "Device will be unsecured now.", "Failed to unlock device." and "Can not attach to CPU. Trying connect under reset.", the connect that removes TinyUF2's write protection (Limits). A reset session (`w4` to AIRCR) passes only once J-Link reports the write issued ("Writing VALUE -> ADDR") with no failure before it. In a hand-written session, `-ExitOnError 0` still exits at a failed verify, before any later command runs, and leaves the target halted: resume it from a new session (`g`, or `r` then `g`), or keep verifies in a session ahead of the one that resumes. ## Limits -- Covered: imxrt1060_evk, metro_m7_1011, espressif_esp32s3_devkitc_1 and espressif_p4_function_ev on htpc. Not covered: ports without a cmake build, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69; its 0A silicon also hangs TinyUF2 on a UF2 copy, #500), espressif chips other than the ESP32-S3 and ESP32-P4, self-update, an interrupted update, malformed UF2 input and the physical double tap. +- Covered: imxrt1060_evk, metro_m7_1011, stm32f303disco, espressif_esp32s3_devkitc_1 and espressif_p4_function_ev on htpc. Not covered: ports without source facts in the runner, lpc55 among them (no battery, `install` or `restore` for lpcxpresso55s69; its 0A silicon also hangs TinyUF2 on a UF2 copy, #500), espressif chips other than the ESP32-S3 and ESP32-P4, self-update, an interrupted update, malformed UF2 input and the physical double tap. - esptool boards: the runner needs esptool 5.2.0 and pyusb with libusb-1.0 for its own Python. Re-entry by DFU_DETACH is the only method built; the GPIO0 detection window is not. The bridge's UART carries every esptool session. A helper that survives SIGKILL is handled like a stuck J-Link, but no way to wedge one on the board was found, so that path is unit-tested only. At the end of a UF2 copy TinyUF2 restarts while the host's usb-storage is still talking to it: the host log shows a reset of that device and read errors on its block device. Seen on the S3 and the P4; other ports are unchecked. On the P4, while an esptool session holds the chip, the host log shows a full-speed device on TinyUF2's USB path that never answers its descriptor request (htpc 2026-10-14); the battery passed with it. - Every probe and USB path is unverified until commissioning has run on the board, and commissioning is manual. -- The HIL's cmake image logs on the board UART: cmake builds leave `TUF2_LOG` at its default 1 (`src/board_api.h`), while make builds, the make ports' CI artifacts and the make-built releases pass 0. Its blocking writes at start, before the app jump and on DFU entry are timing the make image does not have. +- cmake builds pass `TUF2_LOG=0` unless `LOG` is set, as make does, so the HIL image does not log on the board UART. - `install` from a blank board, or one whose ROM found no boot image, brings up DFU (measured on imxrt1060_evk 2026-10-12 and metro_m7_1011 2026-10-13). With no boot image a new J-Link session reads the flash wrong (imxrt1060_evk: zeros; metro_m7_1011: bytes shifted) until a `loadbin` has run; a `loadbin` compare reads it right. On metro_m7_1011, J-Link then prints "Invalid flash header detected." at the install's first reset. - Of a port's CMake flash targets the runner runs only `tinyuf2-jlink` and ESP-IDF's `flash`, bound by the record to the roster's probe or chip (the bridge's tty is passed at flash time). Never use the erase targets on a rig board (`tinyuf2-erase-external-jlink` erases [FLASH_BASE, FLASH_BASE + 64 KiB), the bootloader with it), nor ESP-IDF's `combined-flash`, which has no port selection and writes nvs and both OTA slots blank. - On metro_m7_1011 (2026-10-13, J-Link V9.84), `tinyuf2-jlink`'s `loadfile` of the cmake HEX (0x60000400 on) reported "1 range affected (65536 bytes)" and, from a fresh session after it, 0x60000000, 0x6000BF00, 0x6000C000-0x6000C00F, 0x6000C200 and 0x6000FF00 read 0xFF while 0x60010000 kept its data. Its compare covered the whole block: over blinky it programmed, and right after a program it skipped ("Contents already match"). DFU came up within 2 s in each of these three runs. That every install by the target leaves [0x6000C000, 0x60010000), the app's first 16 KiB, erased is an inference from these samples and this HEX, not a measurement of every case; the re-entry fallback (Run the battery, dfu) covers an install that leaves a valid app. On imxrt1060_evk (2026-10-13), the same target over blinky also reported "1 range affected (65536 bytes)"; from a fresh session after it, blinky's vectors at 0x6000C000-0x6000C00F and 0x6000C200 read 0xFF, while the FCB at 0x60000000 and 0x60010000 kept their data. +- On stm32f303disco (2026-10-14, J-Link V9.84 on the board's J-Link OB), `tinyuf2-jlink`'s `loadfile` compared [0x08000000, 0x08004000) and reprogrammed only the 2 KiB pages that differed, erasing the rest of such a page past the image (a sentinel at 0x08003F00 read 0xFF from a fresh session after it), while 0x08004000 (the app start) and 0x08004800 kept their data. So an install keeps a valid app. DFU still came up without the fallback in the battery of 2026-10-14: the runner's verify session halted the core in TinyUF2's double-tap wait (`src/main.c`, magic armed), so its reset entered DFU as a double tap, an inference from that trace. When that reset misses the window the app boots, and the retained-register fallback brings DFU up after `dfu`'s 20 s wait. +- On stm32f303vc, TinyUF2 write-protects its pages 0-7 at boot through the option bytes unless RCC_CSR's OBLRSTF is set, and it never clears that flag, so after any option-byte reload it skips protection until a power-on reset. A J-Link connect to a chip with that protection set prints the `BENIGN` unlock note and, read from a fresh `-device Cortex-M4` session after it, leaves the option bytes at RDP 0xAA with no WRP and flash intact (three samples, 2026-10-14). Its double-tap word is the last SRAM word, 0x20009FFC; that it survives SYSRESETREQ is undocumented (RM0316 Rev 10, DS9118, PM0214, ES0204) and inferred from the commissioning run (2026-10-14: the reset without the magic booted the app, the one with it entered DFU and consumed it). A battery therefore runs unprotected after its install; the protection itself is not covered. +- stm32f303disco does not signal a disconnect when TinyUF2 comes up after a reset (the F303 has no internal D+ pull-up, DS9118 Rev 14 Table 66 note 2; the MB1035 schematic is not in the library, and on the bench the host kept the old device), so the host does not re-enumerate it unless it was polling the device (TinyUF2 as an MSC disk) when the reset came. For the same reason the `stays_away` checks (the control trial's, the copy's) cannot see a DFU entry there; `observe_app` shows the fixture running when probed, not that no transient DFU entry came before. Its roster `usbPower` names its USB USER port on the Port Authority hub (PA40004, port 3), which the runner cycles while `dfu` waits (Roster). A bench `uhubctl` cycle, a raw hub request, that met an enumeration in progress on hub 1-1 of an AMD 500-series xHCI controller killed that controller (2026-10-14: "HC died", every device on its buses lost, recovered by unbinding and binding `xhci_hcd`; the Port Authority hub needed a replug); hence the kernel's serialized `disable` and the 2 s in state `default`. On this hub `disable` 1 reads `off` from `uhubctl` (2026-10-14). - A `JLinkExe` crash inside the target is not retried (the crash retry below covers the runner's own sessions only): it fails the install, touched. - J-Link's QSPI loader is undocumented (SEGGER UM08001 has nothing on it). Historically, at sampled sentinel points on both RT boards, the runner's former `loadbin` install (no longer issued) lost the rest of its 64 KiB block past the image while a sentinel at 0x60010000, in the next block, survived; a sample, not a guarantee for every image. - `JLinkExe` sometimes crashes in `libjlinkarm` at the probe connect on htpc (SIGSEGV; once a glibc double-free abort); the cause is unknown. A session that crashed this way, before the target connect, runs once more after 2 s (`jlinkRetried`); a second crash fails the run. Seen in 2 of 94 runner sessions on 2026-10-12. diff --git a/.claude/skills/hil/scripts/hil_test.py b/.claude/skills/hil/scripts/hil_test.py index 6493e3aa1..beac3da40 100755 --- a/.claude/skills/hil/scripts/hil_test.py +++ b/.claude/skills/hil/scripts/hil_test.py @@ -42,8 +42,9 @@ # 0x9D800-0x9DDFF is reserved and undocumented. MIMXRT1062: FlexSPI XIP at 0x60000000 (IMXRT1060RM # Table 3-1) holding the EVK's 8 MiB IS25WP064A (EVK hardware user's guide 2.7). MIMXRT1011: FlexSPI XIP at # 0x60000000 (IMXRT1010RM Table 3-1) holding the Metro M7 1011's W25Q64JV (Metro M7 RT1011 SD revA schematic), 8 MiB. +# STM32F303VC: 256 KiB main flash at 0x08000000 (RM0316 Rev 10 Table 2); option bytes at 0x1FFFF800 stay outside. FLASH_LIMITS = {'LPC55S69_M33_0': (0, 0x9D800), 'MIMXRT1062xxx6A': (0x60000000, 0x60800000), - 'MIMXRT1011DAE5A': (0x60000000, 0x60800000)} + 'MIMXRT1011DAE5A': (0x60000000, 0x60800000), 'stm32f303vc': (0x08000000, 0x08040000)} STEPS = {'jlink': ('install', 'dfu', 'copy', 'app-ran', 'reenter')} STEPS['esptool'] = STEPS['jlink'] + ('recopy',) REENTRY = {'jlink': ('retained-register',), 'esptool': ('dfu-detach',)} @@ -63,13 +64,17 @@ NONCE_MAGIC, NONCE_BLOCK = b'TUF2NONC', 256 # apps/dfu_runtime/src/dfu_runtime.h JLINK_IF, JLINK_SPEED = 'SWD', '4000' CMAKE_RECORD_FILES = ('_bin/tinyuf2.bin', '_bin/apps/blinky.uf2') # besides the flash target's inputs -JLINK_ERRORS = r'Verify failed|Could not|Cannot connect|ERROR|ResetTarget failed|Failed to halt|CPU is not halted' +JLINK_ERRORS = (r'Verify failed|Could not|Cannot connect|ERROR|ResetTarget failed|Failed to halt|CPU is not halted|' + r'Failed to unlock|Can not attach') UDEV_RULES = ('/etc/udev/rules.d', '/run/udev/rules.d') # J-Link notes that are not failures, per device, each with the line that shows the operation completed # (bench: every MIMXRT1062 reset with an invalid app vector, then the reset halts as usual; every MIMXRT1011 reset -# of a TinyUF2 image, which then boots) +# of a TinyUF2 image, which then boots; STM32F303VC: the connect that removes TinyUF2's write protection, SKILL.md +# Limits) _RESET_VECTOR_NOTE = r'(?m)^Could not find valid reset vector\.\nTarget halted on flash header read\.$' -BENIGN = {'MIMXRT1062xxx6A': _RESET_VECTOR_NOTE, 'MIMXRT1011DAE5A': _RESET_VECTOR_NOTE} +_F3_UNPROTECT_NOTE = (r'(?m)^Active write protection detected\.\nDevice will be unsecured now\.\nFailed to unlock device\.\n' + r'Can not attach to CPU\. Trying connect under reset\.$') +BENIGN = {'MIMXRT1062xxx6A': _RESET_VECTOR_NOTE, 'MIMXRT1011DAE5A': _RESET_VECTOR_NOTE, 'stm32f303vc': _F3_UNPROTECT_NOTE} class Refused(Exception): @@ -124,10 +129,21 @@ def load_board(config, name): raise Refused(f'{name}: esptool is supported for espressif chips {sorted(esp.BOOTLOADER_OFFSET)} only') elif m.group(1) not in FLASH_LIMITS: raise Refused(f'{name}: no programming limits known for J-Link device {m.group(1)}') + power = t.get('usbPower') + if power is not None: + if not (isinstance(power, dict) and set(power) == {'hub', 'port'} and isinstance(power['hub'], str) and + re.fullmatch(r'[\w-]+', power['hub']) and type(power['port']) is int and 1 <= power['port'] <= 15): + raise Refused(f'{name}: tinyuf2.usbPower must be {{"hub": "", "port": <1-15>}}') + if fl['name'] != 'jlink': + raise Refused(f'{name}: tinyuf2.usbPower is for J-Link boards only') return {'name': name, 'lock': lock, 'uid': b['uid'], 'flasher': fl['name'], 'probe': fl['uid'], 'device': m.group(1), 'if': JLINK_IF, 'speed': JLINK_SPEED, 'port': port, 'helper': os.path.join(tinyusb, 'test', 'hil', 'helper', 'hil_lock.py'), - 'reentry': t.get('reentry')} + 'reentry': t.get('reentry'), 'usbPower': power} + + +def has_define(path, name): + return bool(re.search(rf'^\s*#\s*define\s+{name}\b', (ROOT / path).read_text(), re.M)) def define(path, name): @@ -183,10 +199,16 @@ def cmake_var(path, name): def port_sources(port, bdir): - """(CMSIS device header, the SDK's flash linker script, FLASH_BASE, where the BIN starts in flash) of a port - the HIL supports, read from the CMake files the HIL's build uses.""" - if not (ROOT / f'ports/{port}/CMakeLists.txt').is_file(): - raise Refused(f'ports/{port} has no cmake build, which is what the HIL builds') + """(CMSIS device header, the bootloader's linker script, FLASH_BASE, where the BIN starts in flash, RAM origin) of + a port the HIL supports, read from the CMake files the HIL's build uses.""" + if port == 'stm32f3': + ld = cmake_script(f'ports/{port}/CMakeLists.txt') + mcus = set(re.findall(r'\bSTM32F3\d\dx[A-Z]\b', cb.code_lines(ROOT / bdir / 'board.cmake'))) + if len(mcus) != 1: + raise Refused(f'{bdir}/board.cmake: expected one STM32F3 device define, found {sorted(mcus)}') + header = f'{cmake_var(f"ports/{port}/family.cmake", "ST_CMSIS")}/Include/{mcus.pop().lower()}.h' + flash_base, ram = ld_memory(ld, 'FLASH')[0], ld_memory(ld, 'RAM')[0] + return header, ld, flash_base, flash_base, ram if port != 'mimxrt10xx': raise Refused(f'ports/{port}: the HIL runner has no source facts for this port') sdk = cmake_var(f'ports/{port}/family.cmake', 'SDK_DIR') @@ -201,7 +223,67 @@ def port_sources(port, bdir): flash_base = vals['_flash_base'] # family.cmake FLASH_FCFB_ADDR: the BIN starts at the FCB, 0x400 into flash on RT1010 (IMXRT1010RM Table 9-8) bin_base = flash_base + (vals['_fcfb_origin'] & 0xFFF) - return f'{sdk}/devices/{mcu}/{mcu}.h', f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld', flash_base, bin_base + return (f'{sdk}/devices/{mcu}/{mcu}.h', ld, flash_base, bin_base, + ram_origin(f'{sdk}/devices/{mcu}/gcc/{mcu}xxxxx_flexspi_nor.ld')) + + +def cmake_script(path): + """The one linker script a port's CMakeLists.txt passes with --script=${CMAKE_CURRENT_LIST_DIR}/...""" + hits = re.findall(r'--script=\$\{CMAKE_CURRENT_LIST_DIR\}/([\w./-]+\.ld)(?=["\s)])', cb.code_lines(ROOT / path)) + if len(hits) != 1: + raise Refused(f'{path}: expected one --script=${{CMAKE_CURRENT_LIST_DIR}}/