From 69dcec747d69974446feb2132011cb2c157d9192 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Wed, 22 Jul 2026 23:32:52 -0700 Subject: [PATCH 01/29] Move HMAC auth key off firmware image onto internal-flash key store The command-authentication key was compiled into the firmware image as AUTH_DEFAULT_KEY, making it trivially extractable from a shipped binary (issue #220). This moves the key to a dedicated littlefs partition on internal flash (mounted at /keys), alongside the anti-replay sequence number file which previously lived on the unreliable SD-card FAT filesystem. TcSecurityDeframer now boots keyless and supports provisioning (PROVISION_KEY, bypass-allowlisted, trust-on-first-use while the store is empty) and rotation with up to 2 active keys (ADD_KEY/REMOVE_KEY, both requiring an authenticated frame). SPI validation checks the active key store instead of a hard-coded SPI 0. The key store is shared across all deframer instances (UART/LoRa/Sband): an unknown SPI triggers a reload from disk, so a rotation issued over one link propagates to the others. Also decouples Validator.cpp from the FPP-generated AuthKeyStore type via a plain ActiveSpiSlots array, keeping it (and its host-side gtest coverage) free of F Prime dependencies as documented in test/unit-tests/README.md. --- .github/workflows/ci.yaml | 41 +-- AGENTS.md | 16 +- Framing/src/authenticate_plugin.py | 74 ++--- Makefile | 20 +- .../Components/ProvesRouter/Bypasser.cpp | 3 + .../TcSecurityDeframer/Authenticator.cpp | 38 ++- .../TcSecurityDeframer/Authenticator.hpp | 20 +- .../TcSecurityDeframer/TcSecurityDeframer.cpp | 289 ++++++++++++++++-- .../TcSecurityDeframer/TcSecurityDeframer.fpp | 67 +++- .../TcSecurityDeframer/TcSecurityDeframer.hpp | 54 +++- .../Components/TcSecurityDeframer/Types.hpp | 11 + .../TcSecurityDeframer/Validator.cpp | 17 +- .../TcSecurityDeframer/Validator.hpp | 3 +- .../Components/TcSecurityDeframer/docs/sdd.md | 64 +++- .../Top/ReferenceDeploymentPackets.fppi | 4 + .../test/int/provision_key_test.py | 62 ++++ .../test_TcSecurityDeframer_Authenticator.cpp | 50 +++ .../test_TcSecurityDeframer_Validator.cpp | 49 ++- README.md | 2 +- TODO.md | 119 ++++++++ .../proves_flight_control_board_v5.dtsi | 20 +- docs-site/components/TcSecurityDeframer.md | 64 +++- prj.conf | 3 + pytest.ini | 1 + scripts/generate_auth_default_key.h | 11 - scripts/generate_auth_key_header.py | 138 --------- tools/yamcs/proves_adapter.py | 8 +- 27 files changed, 911 insertions(+), 337 deletions(-) create mode 100644 PROVESFlightControllerReference/test/int/provision_key_test.py create mode 100644 TODO.md delete mode 100644 scripts/generate_auth_default_key.h delete mode 100755 scripts/generate_auth_key_header.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 335bd032..183b2d31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -85,12 +85,6 @@ jobs: run: | make generate - - name: Set Authentication Key - env: - AUTH_KEY: ${{ secrets.AUTH_KEY }} - run: | - echo "#define AUTH_DEFAULT_KEY \"$AUTH_KEY\"" > PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h - - name: Build MCUBoot run: | make build-mcuboot @@ -154,12 +148,6 @@ jobs: # here to keep the YAMCS instance config / adapter in sync. make make-ci-spacecraft-id - - name: Set Authentication Key - env: - AUTH_KEY: ${{ secrets.AUTH_KEY }} - run: | - echo "#define AUTH_DEFAULT_KEY \"$AUTH_KEY\"" > PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h - - name: Install Framer Plugin run: | make framer-plugin @@ -177,10 +165,18 @@ jobs: pkill -9 python || true - name: Start GDS + env: + PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} run: | nohup make gds-integration UART_DEVICE="$UART_DEVICE" > gds-bootstrap.log 2>&1 & echo $! > server.pid + - name: Provision Key + env: + PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} + run: | + make test-integration FILTER=provision_key + - name: Sync Sequence Number run: | make test-integration FILTER=sync_sequence_number @@ -205,6 +201,8 @@ jobs: echo "UART_DEVICE=$ZEPHYR_TTY" >> $GITHUB_ENV - name: Start GDS + env: + PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} run: | nohup make gds-integration UART_DEVICE="$UART_DEVICE" > gds.log 2>&1 & echo $! > server.pid @@ -413,12 +411,6 @@ jobs: run: | make submodules fprime-venv - - name: Set Authentication Key - env: - AUTH_KEY: ${{ secrets.AUTH_KEY }} - run: | - echo "#define AUTH_DEFAULT_KEY \"$AUTH_KEY\"" > PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h - - name: Install Framer Plugin run: | make framer-plugin @@ -432,6 +424,8 @@ jobs: echo "UART_DEVICE=$ZEPHYR_TTY" >> $GITHUB_ENV - name: Bootstrap Sequence Number over UART + env: + PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} run: | # DIVIDER_PRM_SET and lora.TRANSMIT both pass through the authentication # router and require the GDS-side sequence number to match FSW. On a @@ -446,6 +440,11 @@ jobs: BOOTSTRAP_GDS_PID=$! echo $BOOTSTRAP_GDS_PID > server-bootstrap.pid sleep 5 + # Provision the HMAC key on the UART deframer instance; the key store + # is shared board-wide (see TcSecurityDeframer key-store propagation), + # so the LoRa instance picks it up on its next frame without a + # separate provision over radio. + make test-integration FILTER=provision_key # Sync against the LoRa deframer instance: the radio traffic that # follows is validated by it, not by the UART instance this # bootstrap link talks through. @@ -469,6 +468,8 @@ jobs: echo "UART_DEVICE=$ZEPHYR_TTY" >> $GITHUB_ENV - name: Sync Sequence Number over UART + env: + PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} run: | nohup make gds-integration UART_DEVICE="$UART_DEVICE" > gds-sync.log 2>&1 & BOOTSTRAP_GDS_PID=$! @@ -522,6 +523,8 @@ jobs: pkill -9 python || true - name: Start GDS on LoRa Passthrough + env: + PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} run: | # Capture GDS's own stdout/stderr for the whole session (not just what # pytest captures on failure) so a crash mid-run has a full record of @@ -544,7 +547,7 @@ jobs: env: TLM_SAMPLE_LOG: tlm_sample_radio.csv run: | - make test-integration FILTER="not sync_sequence_number and not format_filesystem and not uart_only" PYTEST_ARGS=--with-radio + make test-integration FILTER="not sync_sequence_number and not format_filesystem and not provision_key and not uart_only" PYTEST_ARGS=--with-radio - name: Format Filesystem if: always() diff --git a/AGENTS.md b/AGENTS.md index a6e88670..dd02b3f8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -270,7 +270,6 @@ make generate # Generate F Prime build cache (force) make generate-if-needed # Generate only if build directory missing make build # Build firmware (runs generate-if-needed) make build-mcuboot # Build firmware with MCUBoot bootloader signing -make generate-auth-key # Generate AuthDefaultKey.h with a random HMAC key make fmt # Run linters and formatters (pre-commit) make data-budget # Analyze telemetry data budget (use VERBOSE=1 for details) make docs-sync # Sync component SDD files to docs-site/components/ @@ -581,13 +580,22 @@ After compiling, upload the sequence through GDS for execution on the board. ### Authentication & Security -Commands can be HMAC-authenticated using the `TcSecurityDeframer` component. The authentication key is stored in `PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h`. +Commands can be HMAC-authenticated using the `TcSecurityDeframer` component. The authentication key is **never compiled into the firmware image** (see issue #220); it lives in a dedicated `keystore_partition` on internal flash, mounted as littlefs at `/keys` (`/keys/authkeys.bin`), separate from the SD-card FAT filesystem used for other storage. A keyless board still boots, so it can be provisioned in the field or on the bench. + +The key store holds up to 2 slots (`{valid, spi, key}`), keyed by the CCSDS Security Parameter Index (SPI) already present in every frame. Three commands manage it: + +- `PROVISION_KEY(spi, key)` — bootstrap. Bypass-allowlisted so it works on a keyless satellite, but the handler only honors it while the store is empty. +- `ADD_KEY(spi, key)` — rotation. Requires an authenticated frame; fails once 2 keys already exist. +- `REMOVE_KEY(spi)` — rotation. Requires an authenticated frame; fails if it would drop the active key count below 1. + +The store is the single source of truth shared by all `TcSecurityDeframer` instances (UART/LoRa/Sband): on an unknown SPI, a deframer reloads the store from disk once and retries before rejecting the frame, so a rotation issued over any one link propagates to the others automatically. The ground plugin (`Framing/src/authenticate_plugin.py`) sources its key from the `--authentication-key` CLI arg or the `PROVES_AUTH_KEY` env var — it no longer reads a compiled-in default. ```bash -make generate-auth-key # Generate a new random HMAC key (only if file doesn't exist) -make copy-secrets SECRETS_DIR= # Copy production keys and auth key from a secure directory +make copy-secrets SECRETS_DIR= # Copy production keys from a secure directory ``` +To provision a keyless board, uplink `PROVISION_KEY(spi=0, key=)` over a trusted bench/CI link (see `PROVESFlightControllerReference/test/int/provision_key_test.py`), then give ground the same key via `PROVES_AUTH_KEY`. + Firmware is signed with MCUBoot for secure boot. The signing key is at `keys/proves.pem` (copied from the MCUBoot test key or a production key). Build signed firmware with: ```bash diff --git a/Framing/src/authenticate_plugin.py b/Framing/src/authenticate_plugin.py index 0b99fc9a..ef126a88 100644 --- a/Framing/src/authenticate_plugin.py +++ b/Framing/src/authenticate_plugin.py @@ -19,54 +19,31 @@ SEQUENCE_NUMBER_FILE = os.path.join(_SEQUENCE_NUMBER_DIR, _SEQUENCE_NUMBER_FILENAME) -def get_default_auth_key_from_header() -> str: +def get_auth_key_from_env() -> str: """ - Read the authentication key from AuthDefaultKey.h file. + Read the authentication key from the PROVES_AUTH_KEY environment variable. + + The key is never compiled into the flight image (see issue #220), so ground + tooling must be told the key out-of-band: via --authentication-key or this + environment variable. Returns: - Default authentication key (without 0x prefix) from AuthDefaultKey.h + Authentication key as a hex string (without 0x prefix) from PROVES_AUTH_KEY Raises: - FileNotFoundError: If AuthDefaultKey.h file is not found - ValueError: If AuthDefaultKey.h does not contain a valid key - IOError: If there is an error reading the file + ValueError: If PROVES_AUTH_KEY is not set """ - path = ( - "PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h" - ) - - if not os.path.exists(path): - raise FileNotFoundError( - f"AuthDefaultKey.h not found at {path}. " - "Authentication plugin requires AuthDefaultKey.h to be present. " - "Ensure the file exists or run 'make generate-auth-key' to create it." + key = os.environ.get("PROVES_AUTH_KEY") + if not key: + raise ValueError( + "No authentication key available: pass --authentication-key or set " + "the PROVES_AUTH_KEY environment variable. The key is provisioned " + "onto the satellite with the PROVISION_KEY command and is never " + "compiled into the flight image." ) - - try: - with open(path, "r") as f: - for line in f: - # Look for line like: #define AUTH_DEFAULT_KEY "4916d208d40612daad6edbc7333c4c13" - if "AUTH_DEFAULT_KEY" in line and '"' in line: - # Extract key from between quotes - start = line.find('"') + 1 - end = line.find('"', start) - if start > 0 and end > start: - key = line[start:end] - # Remove 0x prefix if present (shouldn't be, but handle it) - if key.startswith("0x") or key.startswith("0X"): - key = key[2:] - return key - except (IOError, OSError) as e: - raise IOError( - f"Error reading AuthDefaultKey.h from {path}: {e}. " - "Authentication plugin cannot proceed without a valid AuthDefaultKey.h file." - ) from e - - # If we get here, file exists but contains no valid key - raise ValueError( - f"No valid key found in {path}. " - 'AuthDefaultKey.h must contain a line with: #define AUTH_DEFAULT_KEY ""' - ) + if key.startswith("0x") or key.startswith("0X"): + key = key[2:] + return key # pragma: no cover @@ -87,7 +64,7 @@ def __init__( spi: Security Parameter Index (default: 0) window_size: Window size for authentication (default: 50) authentication_type: Type of authentication (default: "HMAC") - authentication_key: Authentication key as hex string without 0x prefix (default: reads from spi_dict.txt) + authentication_key: Authentication key as hex string without 0x prefix (default: reads from PROVES_AUTH_KEY env var) **kwargs: Additional keyword arguments (ignored for now) """ super().__init__() @@ -101,9 +78,9 @@ def __init__( self.spi = spi self.window_size = window_size self.authentication_type = authentication_type - # Use provided key or read from AuthDefaultKey.h + # Use provided key or read from the PROVES_AUTH_KEY environment variable if authentication_key is None: - authentication_key = get_default_auth_key_from_header() + authentication_key = get_auth_key_from_env() self.authentication_key = authentication_key def get_sequence_number_from_file(self, filename: str, addition: bool) -> int: @@ -177,11 +154,6 @@ def deframe(self, data: bytes, no_copy=False) -> tuple[bytes, bytes, bytes]: @classmethod def get_arguments(cls) -> dict: """Return CLI argument definitions for this plugin""" - # Get default key from AuthDefaultKey.h for help text - try: - default_key = get_default_auth_key_from_header() - except (FileNotFoundError, ValueError, IOError): - default_key = "" return { ("--spi",): { "type": int, @@ -200,8 +172,8 @@ def get_arguments(cls) -> dict: }, ("--authentication-key",): { "type": str, - "help": f"Authentication key as hex string without 0x prefix (default: {default_key} from AuthDefaultKey.h)", - "default": None, # Will be set to key from AuthDefaultKey.h in __init__ + "help": "Authentication key as hex string without 0x prefix (default: reads from PROVES_AUTH_KEY env var)", + "default": None, # Will be set from PROVES_AUTH_KEY in __init__ }, } diff --git a/Makefile b/Makefile index bb3e819f..c51f588a 100644 --- a/Makefile +++ b/Makefile @@ -132,7 +132,7 @@ docs-build: uv ## Build MkDocs documentation site @$(UVX) --from mkdocs-material mkdocs build .PHONY: generate -generate: submodules fprime-venv zephyr generate-auth-key keys/proves.pem ## Generate FPrime-Zephyr Proves Core Reference +generate: submodules fprime-venv zephyr keys/proves.pem ## Generate FPrime-Zephyr Proves Core Reference @$(UV_RUN) fprime-util generate --force .PHONY: generate-if-needed @@ -153,21 +153,6 @@ ZEPHYR_CONFIG ?= $(BUILD_DIR)/zephyr/.config check-console-disabled: uv ## Fail if the Zephyr UART console is enabled (it corrupts the F' downlink); run after 'make build' @$(UV_RUN) python3 scripts/check_console_disabled.py "$(ZEPHYR_CONFIG)" -##@ Authentication Keys - -AUTH_DEFAULT_KEY_HEADER ?= PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h -AUTH_KEY_TEMPLATE ?= scripts/generate_auth_default_key.h - -.PHONY: generate-auth-key -generate-auth-key: ## Generate AuthDefaultKey.h with a random HMAC key - @if [ -f "$(AUTH_DEFAULT_KEY_HEADER)" ]; then \ - echo "$(AUTH_DEFAULT_KEY_HEADER) already exists. Skipping generation."; \ - else \ - echo "Generating $(AUTH_DEFAULT_KEY_HEADER) with random key..."; \ - $(UV_RUN) python3 scripts/generate_auth_key_header.py --output $(AUTH_DEFAULT_KEY_HEADER) --template $(AUTH_KEY_TEMPLATE); \ - fi - @echo "Generated $(AUTH_DEFAULT_KEY_HEADER)" - keys/proves.pem: @mkdir -p keys @cp lib/zephyr-workspace/bootloader/mcuboot/root-rsa-2048.pem keys/proves.pem @@ -224,7 +209,7 @@ test-unit: ## Run unit tests cmake --build build-gtest ctest --test-dir build-gtest -FILTER ?= not sync_sequence_number and not format_filesystem +FILTER ?= not sync_sequence_number and not format_filesystem and not provision_key .PHONY: test-integration test-integration: uv ## Run integration tests (set TEST= or pass test targets) @@ -465,7 +450,6 @@ copy-secrets: @mkdir -p ./keys/ @cp $(SECRETS_DIR)/proves.pem ./keys/ @cp $(SECRETS_DIR)/proves.pub.pem ./keys/ - @cp $(SECRETS_DIR)/AuthDefaultKey.h ./PROVESFlightControllerReference/Components/TcSecurityDeframer/ @echo "Copied secret files 🤫" .PHONY: make-ci-spacecraft-id diff --git a/PROVESFlightControllerReference/Components/ProvesRouter/Bypasser.cpp b/PROVESFlightControllerReference/Components/ProvesRouter/Bypasser.cpp index df10645f..7fe1a4d3 100644 --- a/PROVESFlightControllerReference/Components/ProvesRouter/Bypasser.cpp +++ b/PROVESFlightControllerReference/Components/ProvesRouter/Bypasser.cpp @@ -44,6 +44,9 @@ static constexpr uint32_t kBypassOpCodes[] = { 0x2100B000, //!< ComCcsdsUart.tcSecurityDeframer.GET_SEQ_NUM 0x2200B000, //!< ComCcsdsLora.tcSecurityDeframer.GET_SEQ_NUM 0x2300B000, //!< ComCcsdsSband.tcSecurityDeframer.GET_SEQ_NUM + 0x2100B002, //!< ComCcsdsUart.tcSecurityDeframer.PROVISION_KEY + 0x2200B002, //!< ComCcsdsLora.tcSecurityDeframer.PROVISION_KEY + 0x2300B002, //!< ComCcsdsSband.tcSecurityDeframer.PROVISION_KEY 0x10065000, //!< ReferenceDeployment.amateurRadio.TELL_JOKE }; diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp index 5e760442..d88a6364 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp @@ -35,8 +35,9 @@ bool hexToNibble(char ch, uint8_t& nibble) { return false; } +} // namespace + // Parse a 32-character hex string (16 bytes) into a byte array. -// Returns true on success and fills `keyBytes` with the parsed bytes. bool parseHexKey(const char* key, uint8_t (&keyBytes)[Ccsds355_0_B_2::kTCSecurityTrailer]) { if (key == nullptr) { return false; @@ -58,22 +59,15 @@ bool parseHexKey(const char* key, uint8_t (&keyBytes)[Ccsds355_0_B_2::kTCSecurit return true; } -} // namespace - -// Import an HMAC key into PSA for message verification. -PacketAuthenticator::KeyImportResult importHmacKey(const char* key, uint32_t& keyId) { +// Import a raw 128-bit HMAC key into PSA for message verification. +PacketAuthenticator::KeyImportResult importHmacKeyBytes(const uint8_t (&keyBytes)[Ccsds355_0_B_2::kTCSecurityTrailer], + uint32_t& keyId) { // Initialize PSA crypto library const psa_status_t initStatus = psa_crypto_init(); if (initStatus != PSA_SUCCESS) { return {PacketAuthenticator::KeyImportStatus::InitError, initStatus}; } - // Parse the hex-encoded default key into raw bytes - uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; - if (!parseHexKey(key, keyBytes)) { - return {PacketAuthenticator::KeyImportStatus::ParseKeyError, PSA_ERROR_INVALID_ARGUMENT}; - } - // Set up the key attributes psa_key_attributes_t attributes = PSA_KEY_ATTRIBUTES_INIT; psa_set_key_type(&attributes, PSA_KEY_TYPE_HMAC); @@ -84,11 +78,10 @@ PacketAuthenticator::KeyImportResult importHmacKey(const char* key, uint32_t& ke psa_set_key_lifetime(&attributes, PSA_KEY_LIFETIME_VOLATILE); // Import the key into PSA key store - const psa_status_t status = psa_import_key(&attributes, keyBytes, sizeof(keyBytes), &keyId); + const psa_status_t status = psa_import_key(&attributes, keyBytes, Ccsds355_0_B_2::kTCSecurityTrailer, &keyId); // Clean up sensitive data regardless of import outcome psa_reset_key_attributes(&attributes); - mbedtls_platform_zeroize(keyBytes, sizeof keyBytes); if (status != PSA_SUCCESS) { return {PacketAuthenticator::KeyImportStatus::ImportKeyError, status}; @@ -97,10 +90,27 @@ PacketAuthenticator::KeyImportResult importHmacKey(const char* key, uint32_t& ke return {PacketAuthenticator::KeyImportStatus::Success, PSA_SUCCESS}; } +// Parse a hex-encoded key and import it into PSA for message verification. +PacketAuthenticator::KeyImportResult importHmacKey(const char* key, uint32_t& keyId) { + // Parse the hex-encoded key into raw bytes + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + if (!parseHexKey(key, keyBytes)) { + return {PacketAuthenticator::KeyImportStatus::ParseKeyError, PSA_ERROR_INVALID_ARGUMENT}; + } + + const PacketAuthenticator::KeyImportResult result = importHmacKeyBytes(keyBytes, keyId); + mbedtls_platform_zeroize(keyBytes, sizeof keyBytes); + return result; +} + +void destroyHmacKey(uint32_t keyId) { + (void)psa_destroy_key(keyId); +} + PacketAuthenticator::AuthenticationResult authenticatePacket(const uint8_t* dataBuffer, size_t dataSize, const Mac& hmac, - uint32_t& keyId) { + uint32_t keyId) { // Basic input validation: buffer present and at least trailer-sized if (!dataBuffer || dataSize < Ccsds355_0_B_2::kTCSecurityTrailer) { return {PacketAuthenticator::AuthenticationStatus::VerifyError, PSA_ERROR_INVALID_ARGUMENT}; diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp index 296fd038..ec984aa0 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp @@ -41,17 +41,33 @@ struct AuthenticationResult { } // namespace PacketAuthenticator -//! Import an HMAC key into PSA for message verification. +//! Parse a 32-character hex string (16 bytes) into a byte array. +//! Returns true on success and fills `keyBytes` with the parsed bytes. +bool parseHexKey(const char* key, //!< The hex-encoded key string to parse + uint8_t (&keyBytes)[Ccsds355_0_B_2::kTCSecurityTrailer] //!< The parsed key bytes +); + +//! Import a raw 128-bit HMAC key into PSA for message verification. +PacketAuthenticator::KeyImportResult importHmacKeyBytes( + const uint8_t (&keyBytes)[Ccsds355_0_B_2::kTCSecurityTrailer], //!< The raw key bytes to import + uint32_t& keyId //!< The key ID to use for the imported key +); + +//! Parse a hex-encoded key and import it into PSA for message verification. PacketAuthenticator::KeyImportResult importHmacKey(const char* key, //!< The hex-encoded authentication key to import uint32_t& keyId //!< The key ID to use for the imported key ); +//! Destroy a previously-imported PSA key. Used to release the old key on rotation. +void destroyHmacKey(uint32_t keyId //!< The PSA key ID to destroy +); + //! Check the validity of the packet HMAC PacketAuthenticator::AuthenticationResult authenticatePacket( const uint8_t* buffer, //!< The packet data buffer size_t size, //!< The size of the data buffer const Mac& hmac, //!< The HMAC extracted from the packet to validate against - uint32_t& keyId //!< The hex-encoded authentication key to use for validation + uint32_t keyId //!< The PSA key ID to use for validation ); } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp index 11a5a036..f6d7d1fc 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp @@ -13,9 +13,6 @@ #include "TcSecurityDeframer.hpp" #include "Types.hpp" -// Include generated header with default key (generated at build time) -#include "AuthDefaultKey.h" - namespace Components { // ---------------------------------------------------------------------- @@ -26,7 +23,10 @@ TcSecurityDeframer ::TcSecurityDeframer(const char* const compName) : TcSecurityDeframerComponentBase(compName), m_sequenceNumberFilePath(), m_sequenceNumber(0), - m_sequenceNumberWindow(0) {} + m_sequenceNumberWindow(0), + m_keyStoreFilePath(), + m_keyStore(), + m_keyIds{0} {} TcSecurityDeframer ::~TcSecurityDeframer() {} @@ -54,11 +54,23 @@ void TcSecurityDeframer ::dataIn_handler(FwIndexType portNum, Fw::Buffer& data, this->log_WARNING_HI_ParsingFailed_ThrottleClear(); { - Os::ScopeLock lock(this->m_sequenceNumberLock); + // Lock order: key store lock, then sequence number lock. Every other handler that takes + // both locks (none currently do) must follow the same order to avoid deadlock. + Os::ScopeLock keyLock(this->m_keyStoreLock); + Os::ScopeLock seqLock(this->m_sequenceNumberLock); // --- Validate SPI and anti-replay sequence number --- - const PacketValidator::Status validationStatus = - validatePacket(parseResult.securityHeader, this->m_sequenceNumber, this->m_sequenceNumberWindow); + PacketValidator::Status validationStatus = validatePacket(parseResult.securityHeader, this->m_sequenceNumber, + this->m_sequenceNumberWindow, this->activeSpiSlots()); + + if (validationStatus == PacketValidator::Status::SpiInvalid) { + // The key store is shared across all TcSecurityDeframer instances (UART/LoRa/Sband). + // A rotation issued over one link is picked up here so the others don't need their + // own commands re-run: reload from disk once and retry before giving up. + this->loadKeyStore(); + validationStatus = validatePacket(parseResult.securityHeader, this->m_sequenceNumber, + this->m_sequenceNumberWindow, this->activeSpiSlots()); + } if (validationStatus == PacketValidator::Status::SpiInvalid) { this->log_WARNING_HI_SpiInvalid(parseResult.securityHeader.spi); @@ -69,9 +81,15 @@ void TcSecurityDeframer ::dataIn_handler(FwIndexType portNum, Fw::Buffer& data, this->log_WARNING_HI_SpiInvalid_ThrottleClear(); this->log_WARNING_HI_SequenceNumberInvalid_ThrottleClear(); + uint32_t hmacKeyId = 0; + const bool keyFound = this->findKeyIdForSpi(parseResult.securityHeader.spi, hmacKeyId); + // --- Authenticate: HMAC over Security Header + Data Field --- const PacketAuthenticator::AuthenticationResult authResult = - authenticatePacket(data.getData(), data.getSize(), parseResult.securityTrailer.mac, this->m_hmacKeyId); + keyFound + ? authenticatePacket(data.getData(), data.getSize(), parseResult.securityTrailer.mac, hmacKeyId) + : PacketAuthenticator::AuthenticationResult{PacketAuthenticator::AuthenticationStatus::VerifyError, + 0}; if (authResult.status != PacketAuthenticator::AuthenticationStatus::Authenticated) { this->log_WARNING_HI_AuthenticationFailed(static_cast(authResult.status), @@ -150,35 +168,168 @@ void TcSecurityDeframer ::SET_SEQ_NUM_cmdHandler(FwOpcodeType opCode, U32 cmdSeq this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); } +void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, + U32 cmdSeq, + U16 spi, + const Fw::CmdStringArg& key) { + Os::ScopeLock lock(this->m_keyStoreLock); + + // PROVISION_KEY is trust-on-first-use bootstrap: only honored while the store is empty. + // Once any key exists, rotation must go through ADD_KEY/REMOVE_KEY (which require auth). + if (this->activeKeyCount() != 0) { + this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::NotEmpty); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + if (!parseHexKey(key.toChar(), keyBytes)) { + this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::ParseKeyError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + this->m_keyStore[0].set_valid(true); + this->m_keyStore[0].set_spi(spi); + this->m_keyStore[0].set_key(keyBytes); + + if (this->writeKeyStore() != Os::File::OP_OK) { + this->m_keyStore[0].set_valid(false); + this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::WriteError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + this->importKeyStore(); + this->tlmWrite_ActiveKeyCount(this->activeKeyCount()); + this->log_ACTIVITY_HI_KeyProvisioned(spi); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); +} + +void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U16 spi, const Fw::CmdStringArg& key) { + Os::ScopeLock lock(this->m_keyStoreLock); + + const U8 count = this->activeKeyCount(); + if (count >= AuthKeyStore::SIZE) { + this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::StoreFull); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + if (!parseHexKey(key.toChar(), keyBytes)) { + this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::ParseKeyError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + // Find the first empty slot for the new key + U32 emptySlot = AuthKeyStore::SIZE; + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + if (!this->m_keyStore[i].get_valid()) { + emptySlot = i; + break; + } + } + FW_ASSERT(emptySlot < AuthKeyStore::SIZE); + + this->m_keyStore[emptySlot].set_valid(true); + this->m_keyStore[emptySlot].set_spi(spi); + this->m_keyStore[emptySlot].set_key(keyBytes); + + if (this->writeKeyStore() != Os::File::OP_OK) { + this->m_keyStore[emptySlot].set_valid(false); + this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::WriteError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + this->importKeyStore(); + this->tlmWrite_ActiveKeyCount(this->activeKeyCount()); + this->log_ACTIVITY_HI_KeyAdded(spi); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); +} + +void TcSecurityDeframer ::REMOVE_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U16 spi) { + Os::ScopeLock lock(this->m_keyStoreLock); + + if (this->activeKeyCount() <= 1) { + this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::LastKey); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + U32 targetSlot = AuthKeyStore::SIZE; + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + if (this->m_keyStore[i].get_valid() && this->m_keyStore[i].get_spi() == spi) { + targetSlot = i; + break; + } + } + + if (targetSlot >= AuthKeyStore::SIZE) { + this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::SpiNotFound); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + this->m_keyStore[targetSlot].set_valid(false); + + if (this->writeKeyStore() != Os::File::OP_OK) { + this->m_keyStore[targetSlot].set_valid(true); + this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::WriteError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + + this->importKeyStore(); + this->tlmWrite_ActiveKeyCount(this->activeKeyCount()); + this->log_ACTIVITY_HI_KeyRemoved(spi); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); +} + // ---------------------------------------------------------------------- // Public helper methods // ---------------------------------------------------------------------- void TcSecurityDeframer ::configure() { - Os::ScopeLock lock(this->m_sequenceNumberLock); Fw::ParamValid is_valid; - // Get the sequence number window size from the parameter - this->m_sequenceNumberWindow = this->paramGet_SEQ_NUM_WINDOW(is_valid); - FW_ASSERT(is_valid == Fw::ParamValid::VALID || is_valid == Fw::ParamValid::DEFAULT); + { + Os::ScopeLock lock(this->m_sequenceNumberLock); - // Get the file path from the parameter - this->m_sequenceNumberFilePath = this->paramGet_SEQ_NUM_FILE_PATH(is_valid); - FW_ASSERT(is_valid == Fw::ParamValid::VALID || is_valid == Fw::ParamValid::DEFAULT); + // Get the sequence number window size from the parameter + this->m_sequenceNumberWindow = this->paramGet_SEQ_NUM_WINDOW(is_valid); + FW_ASSERT(is_valid == Fw::ParamValid::VALID || is_valid == Fw::ParamValid::DEFAULT); - // Get the sequence number from the file system. On a read failure (already evented - // by readSequenceNumber) fall back to 0 rather than refusing to boot; the operator - // can correct the counter with SET_SEQ_NUM. - U32 sequenceNumber = 0; - (void)this->readSequenceNumber(sequenceNumber); - this->m_sequenceNumber = sequenceNumber; + // Get the file path from the parameter + this->m_sequenceNumberFilePath = this->paramGet_SEQ_NUM_FILE_PATH(is_valid); + FW_ASSERT(is_valid == Fw::ParamValid::VALID || is_valid == Fw::ParamValid::DEFAULT); - // Telemeter the current sequence number - this->tlmWrite_CurrentSequenceNumber(this->m_sequenceNumber); + // Get the sequence number from the file system. On a read failure (already evented + // by readSequenceNumber) fall back to 0 rather than refusing to boot; the operator + // can correct the counter with SET_SEQ_NUM. + U32 sequenceNumber = 0; + (void)this->readSequenceNumber(sequenceNumber); + this->m_sequenceNumber = sequenceNumber; - // Import the HMAC key - PacketAuthenticator::KeyImportResult result = importHmacKey(AUTH_DEFAULT_KEY, this->m_hmacKeyId); - FW_ASSERT(result.status == PacketAuthenticator::KeyImportStatus::Success); + // Telemeter the current sequence number + this->tlmWrite_CurrentSequenceNumber(this->m_sequenceNumber); + } + + { + Os::ScopeLock lock(this->m_keyStoreLock); + + // Get the key store file path from the parameter + this->m_keyStoreFilePath = this->paramGet_KEY_STORE_FILE_PATH(is_valid); + FW_ASSERT(is_valid == Fw::ParamValid::VALID || is_valid == Fw::ParamValid::DEFAULT); + + // Load the key store from the file system and import any valid keys into PSA. On a + // missing/unreadable store (already evented by loadKeyStore), m_keyStore is left with + // no valid slots: a keyless board must still boot so it can be provisioned. + (void)this->loadKeyStore(); + this->tlmWrite_ActiveKeyCount(this->activeKeyCount()); + } } // ---------------------------------------------------------------------- @@ -217,4 +368,90 @@ Os::File::Status TcSecurityDeframer ::writeSequenceNumber(const U32 value) { return status; } +Os::File::Status TcSecurityDeframer ::loadKeyStore() { + AuthKeyStore store; + Os::File::Status status = Utilities::FileHelper::readFromFile(this->m_keyStoreFilePath.toChar(), store); + if (status == Os::File::OP_OK) { + this->m_keyStore = store; + this->log_WARNING_HI_KeyStoreReadFailed_ThrottleClear(); + } else if (status == Os::File::DOESNT_EXIST) { + // No store file yet: keyless state. A keyless board must still boot so it can be + // provisioned; leave m_keyStore at its default (no valid slots). + this->m_keyStore = AuthKeyStore(); + this->log_WARNING_HI_KeyStoreReadFailed_ThrottleClear(); + } else { + this->log_WARNING_HI_KeyStoreReadFailed(static_cast(status)); + } + + this->importKeyStore(); + return status; +} + +Os::File::Status TcSecurityDeframer ::writeKeyStore() { + Os::File::Status status = Utilities::FileHelper::writeToFile(this->m_keyStoreFilePath.toChar(), this->m_keyStore); + if (status != Os::File::OP_OK) { + this->log_WARNING_HI_KeyStoreWriteFailed(static_cast(status)); + } else { + this->log_WARNING_HI_KeyStoreWriteFailed_ThrottleClear(); + } + + return status; +} + +void TcSecurityDeframer ::importKeyStore() { + // Release any previously-imported keys before re-importing, so rotation (and reloads that + // pick up another link's rotation) never leaves a stale key importable in PSA. + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + if (this->m_keyIds[i] != 0) { + destroyHmacKey(this->m_keyIds[i]); + this->m_keyIds[i] = 0; + } + } + + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + if (!this->m_keyStore[i].get_valid()) { + continue; + } + + uint32_t keyId = 0; + const PacketAuthenticator::KeyImportResult result = importHmacKeyBytes(this->m_keyStore[i].get_key(), keyId); + if (result.status == PacketAuthenticator::KeyImportStatus::Success) { + this->m_keyIds[i] = keyId; + } + // On import failure the slot stays without a usable PSA key id; the store on disk is + // unaffected, so a subsequent reload/rotation can recover once the underlying PSA issue + // clears. + } +} + +bool TcSecurityDeframer ::findKeyIdForSpi(uint32_t spi, uint32_t& keyId) const { + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + if (this->m_keyStore[i].get_valid() && this->m_keyStore[i].get_spi() == spi && this->m_keyIds[i] != 0) { + keyId = this->m_keyIds[i]; + return true; + } + } + return false; +} + +U8 TcSecurityDeframer ::activeKeyCount() const { + U8 count = 0; + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + if (this->m_keyStore[i].get_valid()) { + count++; + } + } + return count; +} + +ActiveSpiSlots TcSecurityDeframer ::activeSpiSlots() const { + static_assert(AuthKeyStore::SIZE == kMaxActiveKeys, "ActiveSpiSlots must match AuthKeyStore::SIZE"); + ActiveSpiSlots slots{}; + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + slots[i].valid = this->m_keyStore[i].get_valid(); + slots[i].spi = this->m_keyStore[i].get_spi(); + } + return slots; +} + } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp index f4b0d7c7..828f9717 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp @@ -13,6 +13,29 @@ module Components { MacParseError, @< MAC could not be parsed from packet } + @ A single HMAC authentication key slot + struct AuthKeySlot { + valid: bool @< Whether this slot holds an active key + spi: U16 @< Security Parameter Index selecting this key (CCSDS 355.0-B-2 key selector) + key: [16] U8 @< Raw 128-bit HMAC key bytes + } + + @ On-flash store of active authentication keys. Holds at most 2 keys so rotation can add a new + @ key before removing the old one (see issue #220: keys must never be compiled into the image) + array AuthKeyStore = [2] AuthKeySlot + + @ FPP shadow-enum representing Components::PacketKeyStore::ProvisionStatus + enum KeyStoreProvisionStatus { + Ok, @< Command succeeded + NotEmpty, @< PROVISION_KEY was rejected because the store already has a key + StoreFull, @< ADD_KEY was rejected because the store already has 2 keys + LastKey, @< REMOVE_KEY was rejected because it would remove the last remaining key + SpiNotFound, @< REMOVE_KEY was rejected because no slot has the given SPI + ParseKeyError, @< The supplied key was not a valid 32-character hex string + WriteError, @< The updated store could not be written to the file system + ImportError, @< The updated key could not be imported into PSA + } + @ Component placed between the TcDeframer and SpacePacketDeframer components. It @ implements the TC ProcessSecurity flow of CCSDS 355.0-B-2: parse the Security @ Header and Trailer, validate the SPI and anti-replay sequence number, and verify @@ -29,11 +52,26 @@ module Components { @ Command to set the current sequence number sync command SET_SEQ_NUM(seq_num: U32) + @ Command to bootstrap a key onto a keyless satellite. Bypass-allowlisted so it can run + @ before any key exists; rejected once the store already holds a key (use ADD_KEY to rotate) + sync command PROVISION_KEY(spi: U16, key: string size 33) + + @ Command to add a second active key for rotation. Requires an authenticated frame. Fails + @ if the store already has 2 keys + sync command ADD_KEY(spi: U16, key: string size 33) + + @ Command to remove an active key by SPI. Requires an authenticated frame. Fails if it + @ would drop the key count below 1 + sync command REMOVE_KEY(spi: U16) + ### Telemetry ### @ Telemetry for the current sequence number, updated on each successfully authenticated packet telemetry CurrentSequenceNumber : U32 + @ Telemetry for the number of active keys in the key store, updated on configure() and every key store mutation + telemetry ActiveKeyCount : U8 + ### Events ### @ SequenceNumberGet returns the current sequence number from the file system in response to a command @@ -60,13 +98,40 @@ module Components { @ SpiInvalid indicates that a received packet had an invalid SPI value event SpiInvalid(packet_spi: U32) severity warning high id 4 format "SPI invalid: Received={}" throttle 2 + @ KeyStoreReadFailed indicates that there was an error reading the key store from the file system + event KeyStoreReadFailed(status: Os.FileStatus) severity warning high id 16 format "Failed to read key store, error: {}" throttle 2 + + @ KeyStoreWriteFailed indicates that there was an error writing the key store to the file system + event KeyStoreWriteFailed(status: Os.FileStatus) severity warning high id 17 format "Failed to write key store, error: {}" throttle 2 + + @ KeyProvisioned indicates PROVISION_KEY succeeded + event KeyProvisioned(spi: U16) severity activity high id 9 format "Key provisioned for SPI={}" + + @ KeyProvisionFailed indicates PROVISION_KEY was rejected or failed + event KeyProvisionFailed(status: KeyStoreProvisionStatus) severity warning high id 10 format "Key provisioning failed: {}" throttle 2 + + @ KeyAdded indicates ADD_KEY succeeded + event KeyAdded(spi: U16) severity activity high id 11 format "Key added for SPI={}" + + @ KeyAddFailed indicates ADD_KEY was rejected or failed + event KeyAddFailed(status: KeyStoreProvisionStatus) severity warning high id 12 format "Key add failed: {}" throttle 2 + + @ KeyRemoved indicates REMOVE_KEY succeeded + event KeyRemoved(spi: U16) severity activity high id 13 format "Key removed for SPI={}" + + @ KeyRemoveFailed indicates REMOVE_KEY was rejected or failed + event KeyRemoveFailed(status: KeyStoreProvisionStatus) severity warning high id 15 format "Key remove failed: {}" throttle 2 + ### Parameters ### @ Parameter for the sequence numbers window size, used to prevent replay attacks. The window allows no reuse of previous sequence numbers but allows for new sequence numbers to be accepted within the window size param SEQ_NUM_WINDOW : U32 default 50000 @ Parameter for the file path where the current sequence number is stored - param SEQ_NUM_FILE_PATH : string default "//sequence_number.txt" + param SEQ_NUM_FILE_PATH : string default "/keys/sequence_number.bin" + + @ Parameter for the file path where the HMAC authentication key store is stored + param KEY_STORE_FILE_PATH : string default "/keys/authkeys.bin" ### Ports ### diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp index 08040a52..5a149226 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp @@ -13,6 +13,7 @@ #include #include +#include "PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthKeyStoreArrayAc.hpp" #include "PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp" #include "PROVESFlightControllerReference/Components/TcSecurityDeframer/Parser.hpp" #include "PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframerComponentAc.hpp" @@ -74,6 +75,26 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { U32 seqNum //!< The sequence number to set ) override; + //! Handler implementation for command PROVISION_KEY + void PROVISION_KEY_cmdHandler(FwOpcodeType opCode, //!< The opcode + U32 cmdSeq, //!< The command sequence number + U16 spi, //!< The SPI to provision + const Fw::CmdStringArg& key //!< The hex-encoded key to provision + ) override; + + //! Handler implementation for command ADD_KEY + void ADD_KEY_cmdHandler(FwOpcodeType opCode, //!< The opcode + U32 cmdSeq, //!< The command sequence number + U16 spi, //!< The SPI to add + const Fw::CmdStringArg& key //!< The hex-encoded key to add + ) override; + + //! Handler implementation for command REMOVE_KEY + void REMOVE_KEY_cmdHandler(FwOpcodeType opCode, //!< The opcode + U32 cmdSeq, //!< The command sequence number + U16 spi //!< The SPI to remove + ) override; + public: // ---------------------------------------------------------------------- // Public helper methods @@ -81,7 +102,7 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { //! Initialize component //! - //! Loads the sequence number from persistent storage + //! Loads the sequence number and key store from persistent storage void configure(); private: @@ -97,6 +118,29 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { Os::File::Status writeSequenceNumber(const U32 value //!< The sequence number to write ); + //! Loads the key store from the file system into m_keyStore and (re)imports every valid slot + //! into PSA, destroying any previously-imported keys first. Must be called with m_keyStoreLock held. + //! On a missing/unreadable file, m_keyStore is left with no valid slots (keyless state). + Os::File::Status loadKeyStore(); + + //! Writes m_keyStore to the file system. Must be called with m_keyStoreLock held. + Os::File::Status writeKeyStore(); + + //! (Re)imports every valid slot in m_keyStore into PSA, destroying any previously-imported + //! keys first, and updates m_keyIds. Must be called with m_keyStoreLock held. + void importKeyStore(); + + //! Finds the PSA key id for the given SPI among currently-imported keys. + //! Must be called with m_keyStoreLock held. + bool findKeyIdForSpi(uint32_t spi, uint32_t& keyId) const; + + //! Returns the number of valid slots in m_keyStore. Must be called with m_keyStoreLock held. + U8 activeKeyCount() const; + + //! Projects m_keyStore's valid/spi fields into the plain-C++ ActiveSpiSlots type consumed by + //! the pure-C++ Validator. Must be called with m_keyStoreLock held. + ActiveSpiSlots activeSpiSlots() const; + private: // ---------------------------------------------------------------------- // Private member variables @@ -109,7 +153,13 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { U32 m_sequenceNumber; //!< The current sequence number U32 m_sequenceNumberWindow; //!< The allowed window for sequence number validation - uint32_t m_hmacKeyId; //!< The HMAC key ID used for authentication + // Key store state is coupled between in-memory runtime state (m_keyStore/m_keyIds) and on-disk + // persistent storage; protected by the same mutex to keep PSA-imported keys consistent with the + // on-disk store shared across all TcSecurityDeframer instances (UART/LoRa/Sband) + Os::Mutex m_keyStoreLock; //!< Mutex protecting key store state atomicity + Fw::String m_keyStoreFilePath; //!< File path where the key store is stored + AuthKeyStore m_keyStore; //!< The active key store, up to 2 slots + uint32_t m_keyIds[AuthKeyStore::SIZE]; //!< PSA key ids parallel to m_keyStore, valid iff the slot is valid }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp index 3d4b5937..b6084c4b 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp @@ -14,6 +14,17 @@ namespace Components { using Mac = std::array; //!< The MAC field 16 octets in length +constexpr size_t kMaxActiveKeys = 2; //!< Max simultaneously active auth keys (mirrors AuthKeyStore::SIZE) + +//! A single active-key SPI slot. Mirrors the FPP-generated AuthKeySlot's `valid`/`spi` fields +//! without depending on the FPP/F Prime type, so Validator.cpp can stay pure C++. +struct ActiveSpiSlot { + bool valid; //!< Whether this slot holds an active key + uint32_t spi; //!< The SPI associated with the active key +}; + +using ActiveSpiSlots = std::array; //!< The set of currently active SPIs + //! CCSDS 355.0-B-2 //! https://ccsds.org/Pubs/355x0b2.pdf namespace Ccsds355_0_B_2 { diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.cpp index 0e494f34..0fc771b1 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.cpp @@ -8,10 +8,14 @@ namespace Components { namespace { -//! Validate the SPI field of the packet -bool spiValid(uint32_t spi) { - // For now we only support SPI 0, which indicates no additional security processing beyond HMAC - return spi == 0; +//! Validate the SPI field of the packet against the active SPI slots: it must match a valid slot +bool spiValid(uint32_t spi, const ActiveSpiSlots& activeSpis) { + for (const ActiveSpiSlot& slot : activeSpis) { + if (slot.valid && slot.spi == spi) { + return true; + } + } + return false; } //! Validate packet sequence number must be greater than the last accepted sequence number and within the window @@ -35,8 +39,9 @@ bool sequenceNumberValid(uint32_t packetSequenceNumber, uint32_t sequenceNumber, PacketValidator::Status validatePacket(const Ccsds355_0_B_2::TCSecurityHeader& secHeader, uint32_t sequenceNumber, - uint32_t sequenceNumberWindow) { - if (!spiValid(secHeader.spi)) { + uint32_t sequenceNumberWindow, + const ActiveSpiSlots& activeSpis) { + if (!spiValid(secHeader.spi, activeSpis)) { return PacketValidator::Status::SpiInvalid; } diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.hpp index ee16cda7..9689d6ad 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Validator.hpp @@ -26,7 +26,8 @@ enum class Status { PacketValidator::Status validatePacket( const Ccsds355_0_B_2::TCSecurityHeader& secHeader, //!< The parsed security header uint32_t sequenceNumber, //!< The current sequence number - uint32_t sequenceNumberWindow //!< The acceptable sequence number window + uint32_t sequenceNumberWindow, //!< The acceptable sequence number window + const ActiveSpiSlots& activeSpis //!< The active SPI slots, used to validate the SPI ); } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md index cbdc9687..d61b0b9b 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md @@ -9,10 +9,20 @@ The component does not enforce policy. Frames that fail verification are still f The component is a thin stateful shell over pure-function namespaces: - `Ccsds355_0_B_2::parse` (Parser) — Security Header (SPI, sequence number) and Trailer (MAC) extraction -- `Components::validatePacket` (Validator) — SPI validation and anti-replay sequence-number window validation -- `Components::authenticatePacket` / `importHmacKey` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto +- `Components::validatePacket` (Validator) — SPI validation against the active key store and anti-replay sequence-number window validation +- `Components::authenticatePacket` / `importHmacKey` / `importHmacKeyBytes` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto -The only component state is the last accepted sequence number (mutex-guarded, persisted to file) and the imported HMAC key id. +`Validator` takes the active SPI set as a plain `ActiveSpiSlots` array (`Types.hpp`) rather than the FPP-generated key store type directly, so it — and its unit tests — stay pure C++ with no F Prime dependency; `TcSecurityDeframer::activeSpiSlots()` projects the real key store into that shape before calling `validatePacket`. + +Component state is the last accepted sequence number and the on-flash key store (each mutex-guarded and persisted to file), plus the PSA key ids imported from the store's valid slots. + +### Key Storage + +The HMAC authentication key is **never compiled into the firmware image** (issue #220). It is persisted at `KEY_STORE_FILE_PATH` (default `/keys/authkeys.bin`) on a dedicated littlefs `keystore_partition` on internal flash, holding up to 2 slots (`{valid, spi, key}`). `configure()` loads the store and imports every valid slot into PSA; a missing/empty store is not an error — a keyless board still boots so it can be provisioned. The sequence-number file lives on the same partition (`SEQ_NUM_FILE_PATH`, default `/keys/sequence_number.bin`). + +The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): on an unknown SPI, `dataIn_handler` reloads the store from disk once and retries validation before rejecting the frame, so a rotation issued over one link is picked up by the others without a separate command per link. + +Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-import the store into PSA and update `ActiveKeyCount` telemetry on success. Primary data path connections: @@ -33,11 +43,21 @@ class TcSecurityDeframer { -dataReturnIn_handler(portNum, data, context) -GET_SEQ_NUM_cmdHandler(opCode, cmdSeq) -SET_SEQ_NUM_cmdHandler(opCode, cmdSeq, seqNum) + -PROVISION_KEY_cmdHandler(opCode, cmdSeq, spi, key) + -ADD_KEY_cmdHandler(opCode, cmdSeq, spi, key) + -REMOVE_KEY_cmdHandler(opCode, cmdSeq, spi) -readSequenceNumber(value) -writeSequenceNumber(value) + -loadKeyStore() + -writeKeyStore() + -importKeyStore() + -findKeyIdForSpi(spi, keyId) + -activeKeyCount() + -activeSpiSlots() -m_sequenceNumber : U32 -m_sequenceNumberWindow : U32 - -m_hmacKeyId : uint32_t + -m_keyStore : AuthKeyStore + -m_keyIds : uint32_t[2] } class Ccsds355_0_B_2 { @@ -47,7 +67,7 @@ class Ccsds355_0_B_2 { class PacketValidator { <> - +validatePacket(secHeader, sequenceNumber, window) Status + +validatePacket(secHeader, sequenceNumber, window, activeSpis) Status } class PacketAuthenticator { @@ -101,19 +121,20 @@ The MAC is HMAC-SHA-256 truncated to 16 bytes, computed over the Security Header ## Behavior 1. Parse the Security Header and Trailer. If the frame is too short to contain them it cannot be stripped for downstream deframing: log ParsingFailed and return the buffer upstream (drop). -2. Validate the SPI (only SPI 0 is currently supported) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). -3. If validation passes, verify the MAC. +2. Validate the SPI (must match a valid slot in the active key store — see [Key Storage](#key-storage)) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). On an unknown SPI, the key store is reloaded from disk once and validation retried, so a rotation issued over another link is picked up here. +3. If validation passes, look up the PSA key id for the packet's SPI and verify the MAC with it. 4. Only when all checks pass: store and persist the received sequence number, telemeter it, and set `authenticated = true` in the frame context. Frames failing any check never advance the sequence number (issue #426). 5. Strip the Security Header and Trailer and forward on dataOut with the resulting `authenticated` flag. ProvesRouter rejects unauthenticated packets unless their opcode is on the bypass allowlist. -At startup, `configure()` loads the persisted sequence number and telemeters it so the first downlinked value is correct before any command is accepted (issue #427). +At startup, `configure()` loads the persisted sequence number and key store, telemetering both so the first downlinked values are correct before any command is accepted (issue #427). A missing or empty key store leaves the board keyless (bootable, awaiting `PROVISION_KEY`) rather than failing to boot. ## Parameters | Name | Type | Default | Description | |---|---|---|---| | SEQ_NUM_WINDOW | U32 | 50000 | Maximum allowed forward sequence-number distance before rejecting a packet as out-of-window. | -| SEQ_NUM_FILE_PATH | string | "//sequence_number.txt" | File path used to persist and restore the sequence number across restarts. | +| SEQ_NUM_FILE_PATH | string | "/keys/sequence_number.bin" | File path used to persist and restore the sequence number across restarts. | +| KEY_STORE_FILE_PATH | string | "/keys/authkeys.bin" | File path used to persist and restore the authentication key store across restarts. | ## Port Descriptions @@ -131,6 +152,7 @@ Standard AC ports are also present for command handling, events, telemetry, para | Name | Type | Description | |---|---|---| | CurrentSequenceNumber | U32 | Current accepted sequence number tracked by the component. Emitted at startup and on each accepted packet. | +| ActiveKeyCount | U8 | Number of valid slots (0-2) in the key store. Emitted at startup and after PROVISION_KEY/ADD_KEY/REMOVE_KEY. | Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which owns the accept/reject policy. @@ -146,6 +168,14 @@ Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which ow | AuthenticationFailed | Warning High (throttle 2) | auth_status: PacketAuthenticatorStatus, rc: I32 | Logged when MAC verification fails. Format: "Authentication failed: Status={}, PSA Return Code={}" | | ParsingFailed | Warning High (throttle 2) | parse_status: PacketParserStatus | Logged when frame parsing fails. Format: "Parsing failed: {}" | | SpiInvalid | Warning High (throttle 2) | packet_spi: U32 | Logged when SPI validation fails. Format: "SPI invalid: Received={}" | +| KeyStoreReadFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store read fails (not thrown for a missing file — that's the keyless state). Format: "Failed to read key store, error: {}" | +| KeyStoreWriteFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store write fails. Format: "Failed to write key store, error: {}" | +| KeyProvisioned | Activity High | spi: U16 | Logged by PROVISION_KEY on success. Format: "Key provisioned for SPI={}" | +| KeyProvisionFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when PROVISION_KEY fails (store not empty, bad hex key, or write failure). Format: "Key provisioning failed: {}" | +| KeyAdded | Activity High | spi: U16 | Logged by ADD_KEY on success. Format: "Key added for SPI={}" | +| KeyAddFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when ADD_KEY fails (store full, bad hex key, or write failure). Format: "Key add failed: {}" | +| KeyRemoved | Activity High | spi: U16 | Logged by REMOVE_KEY on success. Format: "Key removed for SPI={}" | +| KeyRemoveFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when REMOVE_KEY fails (last remaining key, SPI not found, or write failure). Format: "Key remove failed: {}" | ## Commands @@ -153,6 +183,9 @@ Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which ow |---|---|---|---| | GET_SEQ_NUM | Sync | None | Reads and reports the current sequence number (SequenceNumberGet event). | | SET_SEQ_NUM | Sync | seq_num: U32 | Sets and persists a new sequence number (SequenceNumberSet event). | +| PROVISION_KEY | Sync | spi: U16, key: string | Bootstrap: writes the first key slot and imports it into PSA. Only succeeds while the store is empty; bypass-allowlisted in ProvesRouter so it works on a keyless board (KeyProvisioned/KeyProvisionFailed events). | +| ADD_KEY | Sync | spi: U16, key: string | Rotation: adds a key to an empty slot. Requires an authenticated frame (not bypass-allowlisted); fails if 2 keys are already active (KeyAdded/KeyAddFailed events). | +| REMOVE_KEY | Sync | spi: U16 | Rotation: invalidates the slot matching `spi`. Requires an authenticated frame (not bypass-allowlisted); fails if it would drop the active key count below 1 (KeyRemoved/KeyRemoveFailed events). | ## Unit Tests @@ -161,8 +194,10 @@ TcSecurityDeframer helper functionality is covered by unit tests in PROVESFlight | Test File | Coverage | |---|---| | test_TcSecurityDeframer_Parser.cpp | Valid parse path plus parse failures for SPI, sequence number, and MAC size checks. | -| test_TcSecurityDeframer_Validator.cpp | SPI validation, out-of-window and replayed sequence numbers, window boundary, and wraparound handling. | -| test_TcSecurityDeframer_Authenticator.cpp | Key import failures, successful MAC verification, and failed verification with corrupted MAC or data. | +| test_TcSecurityDeframer_Validator.cpp | SPI validation against the active `ActiveSpiSlots` set (single slot, second slot, no valid slots), out-of-window and replayed sequence numbers, window boundary, and wraparound handling. | +| test_TcSecurityDeframer_Authenticator.cpp | `parseHexKey` (valid upper/lowercase, null, wrong length, non-hex characters), key import via hex and raw bytes, `destroyHmacKey`, successful MAC verification, and failed verification with corrupted MAC, corrupted data, or a destroyed key. | + +These cover only the pure-function layer (Parser/Validator/Authenticator; no F Prime or Zephyr dependency). The key store mutation rules enforced in the command handlers (`PROVISION_KEY`/`ADD_KEY`/`REMOVE_KEY` — provision-only-when-empty, add fails at 2, remove fails at 1) are F-Prime-component-dependent and are not covered here; see the commented-out `register_fprime_ut` block in `CMakeLists.txt` for a future on-target/component test pass. Run unit tests with: @@ -180,16 +215,16 @@ make framer-plugin Then run GDS with the framing plugin enabled as configured by the project tooling. -## Generating Keys +## Provisioning Keys -The default authentication key header (AuthDefaultKey.h) is generated at build time from project key material via `make generate-auth-key` or `make copy-secrets`. This generated file is machine-local and not committed. +The authentication key is no longer compiled into the image, so there is nothing to generate at build time. Instead, a keyless board is provisioned after flashing by uplinking `PROVISION_KEY(spi, key)` over a bypass-allowlisted link (the flight-side integration test `PROVESFlightControllerReference/test/int/provision_key_test.py` does this in CI). Ground must be given the same key, via the `--authentication-key` CLI arg or `PROVES_AUTH_KEY` env var to the framing plugin (`Framing/src/authenticate_plugin.py`). `make copy-secrets` still copies production key material from a secure directory for use in provisioning. ## Requirements | Name | Description | Validation | |---|---|---| | AUTH001 | The component shall parse incoming frames to extract the SPI, sequence number, and MAC fields. | Unit Test | -| AUTH003 | The component shall validate that the SPI value corresponds to a configured Security Association. | Unit Test | +| AUTH003 | The component shall validate that the SPI value corresponds to a configured Security Association (an active slot in the on-flash key store). | Unit Test | | AUTH004 | The component shall validate the received sequence number against the stored sequence number. | Unit Test | | AUTH004-A | The component shall not authenticate packets with sequence numbers that are outside the acceptable window and shall log an event. | Unit Test, Inspection | | AUTH004-B | The component shall set the stored sequence number to the sequence number transmitted in the packet only when a packet is fully validated and authenticated. | Inspection | @@ -207,3 +242,4 @@ Opcode-based bypass policy (formerly AUTH002) is owned by ProvesRouter; see its | --- | --- | | 2025-11-26 | Initial design. | | 2026-07-17 | Renamed to TcSecurityDeframer, refactor to discrete responsibilities: Authenticator, Parser, Validator. Pass-through interface between TcDeframer and SpacePacketDeframer; verification result carried in frame context; policy enforcement moved to ProvesRouter. | +| 2026-07-23 | Moved the authentication key off the compiled-in image onto a littlefs key store on internal flash, alongside the sequence-number file (issue #220). Added PROVISION_KEY/ADD_KEY/REMOVE_KEY commands supporting up to 2 active keys; SPI validation now checks the active key store instead of a hard-coded SPI 0. | diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi index 5ca221ec..f2ad7ef1 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi @@ -171,6 +171,10 @@ telemetry packets ReferenceDeploymentPackets { amateurRadio.count_names + #ComCcsdsSband.tcSecurityDeframer.ActiveKeyCount + ComCcsdsLora.tcSecurityDeframer.ActiveKeyCount + ComCcsdsUart.tcSecurityDeframer.ActiveKeyCount + } packet DetumblePerformance id 16 group 5 { diff --git a/PROVESFlightControllerReference/test/int/provision_key_test.py b/PROVESFlightControllerReference/test/int/provision_key_test.py new file mode 100644 index 00000000..f077a372 --- /dev/null +++ b/PROVESFlightControllerReference/test/int/provision_key_test.py @@ -0,0 +1,62 @@ +""" +provision_key_test.py: + +This module provisions the HMAC authentication key onto a keyless satellite +so ground and flight share a key. Must run before any auth-required command +is sent, on the bypass-allowlisted link (PROVISION_KEY is refused once the +on-flash key store already holds a key). Idempotent: if the board was +already provisioned by a previous run (the key store lives on internal +flash and survives reflashing), a NotEmpty rejection is treated as success +rather than a failure, since the store already holds the CI secret key. +""" + +import os + +import pytest +from fprime_gds.common.data_types.event_data import EventData +from fprime_gds.common.testing_fw.api import IntegrationTestAPI +from fprime_gds.common.testing_fw.predicates import satisfies_any + + +@pytest.mark.provision_key +def test_provision_key( + fprime_test_api: IntegrationTestAPI, start_gds, request: pytest.FixtureRequest +): + """Provision the HMAC key (spi=0) on a keyless board, tolerating a prior provision""" + link = request.config.getoption("--sync-deframer", default=None) + if link is None: + link = ( + "lora" + if request.config.getoption("--with-radio", default=False) + else "uart" + ) + deframer = { + "uart": "ComCcsdsUart.tcSecurityDeframer", + "lora": "ComCcsdsLora.tcSecurityDeframer", + }[link] + + key = os.environ.get("PROVES_AUTH_KEY") + if not key: + pytest.fail( + "PROVES_AUTH_KEY environment variable not set; cannot provision key" + ) + + fprime_test_api.clear_histories() + fprime_test_api.send_command(f"{deframer}.PROVISION_KEY", ["0", key]) + + evt: EventData = fprime_test_api.await_event( + satisfies_any( + [ + fprime_test_api.get_event_pred(f"{deframer}.KeyProvisioned"), + fprime_test_api.get_event_pred(f"{deframer}.KeyProvisionFailed"), + ] + ), + timeout=10, + ) + + if evt.template.get_full_name().endswith("KeyProvisionFailed"): + status = evt.args[0].val + assert status == "NotEmpty", ( + f"PROVISION_KEY failed with unexpected status {status!r}; " + "board should either be keyless or already hold the CI key" + ) diff --git a/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp index dbcb5e8c..a5eb8829 100644 --- a/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp +++ b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp @@ -45,6 +45,56 @@ TEST(PacketAuthenticatorTest, ImportNullKey) { EXPECT_EQ(res.psaStatus, PSA_ERROR_INVALID_ARGUMENT); } +TEST(ParseHexKeyTest, ValidLowercaseKey) { + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + EXPECT_TRUE(parseHexKey(kTestKeyHex, keyBytes)); + EXPECT_EQ(keyBytes[0], 0x14); + EXPECT_EQ(keyBytes[1], 0x40); + EXPECT_EQ(keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer - 1], 0xfa); +} + +TEST(ParseHexKeyTest, ValidUppercaseKey) { + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + EXPECT_TRUE(parseHexKey("14408C2711281F4D70452CE3730BB4FA", keyBytes)); +} + +TEST(ParseHexKeyTest, NullKeyRejected) { + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + EXPECT_FALSE(parseHexKey(nullptr, keyBytes)); +} + +TEST(ParseHexKeyTest, WrongLengthRejected) { + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + EXPECT_FALSE(parseHexKey("1234", keyBytes)); +} + +TEST(ParseHexKeyTest, NonHexCharacterRejected) { + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + EXPECT_FALSE(parseHexKey("zz408c2711281f4d70452ce3730bb4fa", keyBytes)); +} + +TEST(PacketAuthenticatorTest, ImportKeyBytesDirectly) { + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + ASSERT_TRUE(parseHexKey(kTestKeyHex, keyBytes)); + + uint32_t keyId = 0; + auto res = importHmacKeyBytes(keyBytes, keyId); + EXPECT_EQ(res.status, PacketAuthenticator::KeyImportStatus::Success); + EXPECT_EQ(res.psaStatus, PSA_SUCCESS); + EXPECT_NE(keyId, 0u); + + destroyHmacKey(keyId); +} + +TEST(PacketAuthenticatorTest, DestroyedKeyFailsToVerify) { + uint32_t keyId = importTestKey(); + destroyHmacKey(keyId); + + auto res = authenticatePacket(kTestPacket.data(), kTestPacket.size(), macOf(kTestPacket), keyId); + EXPECT_EQ(res.status, PacketAuthenticator::AuthenticationStatus::VerifyError); + EXPECT_NE(res.psaStatus, PSA_SUCCESS); +} + TEST(PacketAuthenticatorTest, NullBuffer) { uint32_t keyId = importTestKey(); Mac mac{}; diff --git a/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Validator.cpp b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Validator.cpp index 998276d3..54f4afbd 100644 --- a/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Validator.cpp +++ b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Validator.cpp @@ -5,27 +5,58 @@ using namespace Components; using Header = Ccsds355_0_B_2::TCSecurityHeader; +namespace { + +//! Build an ActiveSpiSlots with a single valid slot at the given SPI +ActiveSpiSlots singleActiveSpi(uint32_t spi) { + return ActiveSpiSlots{{{true, spi}, {false, 0u}}}; +} + +//! Build an ActiveSpiSlots with no valid slots (keyless state) +ActiveSpiSlots noActiveSpis() { + return ActiveSpiSlots{{{false, 0u}, {false, 0u}}}; +} + +//! Build an ActiveSpiSlots with two valid slots (rotation state) +ActiveSpiSlots twoActiveSpis(uint32_t spiA, uint32_t spiB) { + return ActiveSpiSlots{{{true, spiA}, {true, spiB}}}; +} + +} // namespace + TEST(PacketValidatorTest, ValidPacket) { Header h{0u, 11u}; // spi 0 valid, seq 11 within window of expected 10 - auto res = validatePacket(h, 10u, 5u); + auto res = validatePacket(h, 10u, 5u, singleActiveSpi(0u)); EXPECT_EQ(res, PacketValidator::Status::Valid); } TEST(PacketValidatorTest, SpiInvalid) { - Header h{1u, 11u}; // non-zero SPI invalid - auto res = validatePacket(h, 10u, 5u); + Header h{1u, 11u}; // spi 1 is not among the active slots + auto res = validatePacket(h, 10u, 5u, singleActiveSpi(0u)); + EXPECT_EQ(res, PacketValidator::Status::SpiInvalid); +} + +TEST(PacketValidatorTest, SpiInvalidWhenStoreEmpty) { + Header h{0u, 11u}; // keyless board: no slot is valid, so no SPI matches + auto res = validatePacket(h, 10u, 5u, noActiveSpis()); EXPECT_EQ(res, PacketValidator::Status::SpiInvalid); } +TEST(PacketValidatorTest, SpiValidInSecondSlot) { + Header h{7u, 11u}; // rotation state: spi 7 lives in the second active slot + auto res = validatePacket(h, 10u, 5u, twoActiveSpis(0u, 7u)); + EXPECT_EQ(res, PacketValidator::Status::Valid); +} + TEST(PacketValidatorTest, SequenceNumberInvalid) { Header h{0u, 20u}; // sequence 20, expected 10, window 5 -> out - auto res = validatePacket(h, 10u, 5u); + auto res = validatePacket(h, 10u, 5u, singleActiveSpi(0u)); EXPECT_EQ(res, PacketValidator::Status::SequenceNumberInvalid); } TEST(PacketValidatorTest, SequenceNumberReplayRejected) { Header h{0u, 9u}; // below last accepted -> replay - auto res = validatePacket(h, 10u, 5u); + auto res = validatePacket(h, 10u, 5u, singleActiveSpi(0u)); EXPECT_EQ(res, PacketValidator::Status::SequenceNumberInvalid); } @@ -36,7 +67,7 @@ TEST(PacketValidatorTest, SequenceNumberWrapWithinWindow) { // distance = seq - expected = 1 - 0xFFFFFFFE = 3 (mod 2^32) Header h{0u, seq}; - auto res = validatePacket(h, expected, 5u); + auto res = validatePacket(h, expected, 5u, singleActiveSpi(0u)); EXPECT_EQ(res, PacketValidator::Status::Valid); } @@ -47,18 +78,18 @@ TEST(PacketValidatorTest, SequenceNumberWrapOutOfWindow) { // distance = 10 - 0xFFFFFFF0 = 26 (mod 2^32) Header h{0u, seq}; - auto res = validatePacket(h, expected, 5u); + auto res = validatePacket(h, expected, 5u, singleActiveSpi(0u)); EXPECT_EQ(res, PacketValidator::Status::SequenceNumberInvalid); } TEST(PacketValidatorTest, SequenceNumberEqualToExpected) { Header h{0u, 10u}; // sequence equal to last accepted must be rejected (no reuse) - auto res = validatePacket(h, 10u, 5u); + auto res = validatePacket(h, 10u, 5u, singleActiveSpi(0u)); EXPECT_EQ(res, PacketValidator::Status::SequenceNumberInvalid); } TEST(PacketValidatorTest, SequenceNumberAtWindowBoundary) { Header h{0u, 15u}; // expected 10, window 5 -> distance = 5 = window -> valid - auto res = validatePacket(h, 10u, 5u); + auto res = validatePacket(h, 10u, 5u, singleActiveSpi(0u)); EXPECT_EQ(res, PacketValidator::Status::Valid); } diff --git a/README.md b/README.md index 1ad4c26e..cd946667 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ MCUBoot only boots images that are **signed with the same key** the bootloader i If you regenerate/replace the bootloader (or switch computers and flash a bootloader built elsewhere), make sure you also update `keys/proves.pem` to the matching signing key, or your built images will not boot. -You also want to make sure the authentication key the gds runs with is the same as the authentication key on the board. For that, you want to make sure the authentication key in PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h matches. +You also want to make sure the authentication key the gds runs with is the same as the authentication key provisioned on the board. The board's key lives in its on-flash key store (never in the image); ground reads its key from the `--authentication-key` CLI arg or the `PROVES_AUTH_KEY` env var. Make sure these match the key you provisioned with `PROVISION_KEY`/`ADD_KEY`. ## Running Integration Tests diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..1d690a51 --- /dev/null +++ b/TODO.md @@ -0,0 +1,119 @@ +# HMAC key → internal-flash storage — progress tracker + +Plan source: `~/.claude/plans/quirky-forging-possum.md` +Branch: `hmac-to-storage` + +Status legend: [ ] todo, [~] in progress, [x] done + +## 1. Flash partition + littlefs mount +- [x] Add `keystore_partition` to `proves_flight_control_board_v5.dtsi` `&flash0`, shrink `storage_partition` + (keystore_partition@0x400000 256KB; storage_partition@0x440000 0xBC0000) +- [x] Add `zephyr,fstab,littlefs` node mounted at `/keys` (lfs1, automount) +- [x] `prj.conf`: `CONFIG_FILE_SYSTEM_LITTLEFS=y` (correct Kconfig name, not FS_LITTLEFS) +- [x] Automount confirmed via Zephyr Kconfig.littlefs: FS_LITTLEFS_FSTAB_AUTOMOUNT defaults y when DT + node has `automount` + CONFIG_FLASH_MAP=y (already on for all 3 board defconfigs). No explicit + fs_mount() needed in Main.cpp. + +## 2. Key store type + persistence (TcSecurityDeframer) +- [x] Add FPP `AuthKeySlot` / `AuthKeyStore` (2 slots) type + `KeyStoreProvisionStatus` enum, in `TcSecurityDeframer.fpp` + (verified accessor names `getvalid/getspi/getkey/setvalid/setspi/setkey`, `AuthKeyStore::SIZE`, `operator[](U32)` + by running `fpp-to-cpp` on a scratch copy of the struct/array — see scratch notes below) +- [x] `Authenticator.cpp/.hpp`: added `importHmacKeyBytes` (raw bytes, used by key-store reimport) with + `importHmacKey` (hex) now a thin wrapper; added `destroyHmacKey`; exposed `parseHexKey`; + `authenticatePacket` now takes `keyId` by value (was `uint32_t&`, never mutated) +- [x] `TcSecurityDeframer.hpp/.cpp`: `KEY_STORE_FILE_PATH` param, `loadKeyStore()`/`writeKeyStore()`/`importKeyStore()`/ + `findKeyIdForSpi()`/`activeKeyCount()` helpers, second `m_keyStoreLock` mutex (separate from seq-num lock; + dataIn_handler takes keyStoreLock then seqLock, only ordering with both — no deadlock risk). + configure() loads store, no FW_ASSERT on missing/empty (keyless boot supported) +- [x] `dataIn_handler`: `validatePacket` now takes the key store; on `SpiInvalid` reloads store from disk once + and retries before giving up +- [x] `PROVISION_KEY` / `ADD_KEY` / `REMOVE_KEY` command handlers + KeyProvisioned/Failed, KeyAdded/Failed, + KeyRemoved/Failed, KeyStoreReadFailed/WriteFailed events, ActiveKeyCount telemetry +- [x] `Validator.cpp`: `spiValid` checks against active slots instead of `spi == 0` (takes `const AuthKeyStore&`) +- [x] `TcSecurityDeframer.fpp`: `SEQ_NUM_FILE_PATH` default -> `/keys/sequence_number.bin` +- [x] Removed `#include "AuthDefaultKey.h"` / `AUTH_DEFAULT_KEY` usage from `TcSecurityDeframer.cpp`; + deleted generated `AuthDefaultKey.h` from TcSecurityDeframer dir (it's gitignored, not tracked) + +## 3. Router bootstrap allowlist +- [x] `Bypasser.cpp`: added `0x2100B002`/`0x2200B002`/`0x2300B002` (UART/LoRa/Sband PROVISION_KEY) to + `kBypassOpCodes`. Derivation verified against the real (pre-existing, stale) dictionary at + `build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json`: + opcodes are `instance_base + local_index`, where local_index is 0-based over *user commands only* + (GET_SEQ_NUM=0, SET_SEQ_NUM=1) followed by PRM_SET/PRM_SAVE pairs per param in declaration order. + My new commands are declared right after SET_SEQ_NUM and before any params, so + PROVISION_KEY=2, ADD_KEY=3, REMOVE_KEY=4 (only PROVISION_KEY needs a bypass entry). + Caveat: the stale dict only had ComCcsdsUart/ComCcsdsLora instances (no Sband), so 0x2300B002 + is derived by pattern from the pre-existing (already in file) 0x2300B000 Sband entry, not directly + confirmed. **Re-verify opcodes with `make build` + fresh dict.json before flight/CI trust.** + +## 4. Ground plugin +- [x] `Framing/src/authenticate_plugin.py`: `get_default_auth_key_from_header` -> `get_auth_key_from_env`, + reads `PROVES_AUTH_KEY` env var, raises ValueError with clear message if neither CLI arg nor env set +- [x] `tools/yamcs/proves_adapter.py` also imported the removed function — updated to `get_auth_key_from_env` + (not called out in plan explicitly but same removal would have broken this importer) + +## 5. Build / Makefile / provisioning +- [x] `Makefile`: dropped `generate-auth-key` target + from `generate` deps, `AUTH_DEFAULT_KEY_HEADER`/ + `AUTH_KEY_TEMPLATE` vars, `AuthDefaultKey.h` copy line in `copy-secrets` +- [x] Deleted `scripts/generate_auth_key_header.py`, `scripts/generate_auth_default_key.h` +- [x] Added `PROVESFlightControllerReference/test/int/provision_key_test.py` (marker `provision_key`), + registered marker in `pytest.ini`, added `and not provision_key` to Makefile default `FILTER`. + Idempotent: PROVISION_KEY fails with `NotEmpty` if the board was already provisioned by a + prior CI run (key store lives on internal flash, survives reflashing) — the test treats that + as success rather than failure, since the store already holds the CI secret key. + +## 6. CI +- [x] Removed all 3 "Set Authentication Key" steps (build, integration-uart, integration-radio) that + wrote `AuthDefaultKey.h` in `.github/workflows/ci.yaml`. +- [x] No job-wide `PROVES_AUTH_KEY` export. Instead, each step that either starts a GDS process + (`make gds-integration`, whose `AuthenticateFramer` plugin reads the env var at construction + and raises if unset) or itself needs the key value (`provision_key_test.py`, which reads + `os.environ["PROVES_AUTH_KEY"]` directly to build the command arg) gets its own `env:` block + with `PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }}`. Steps that only talk to an already-running + GDS over the network (Sync Sequence Number, Format Filesystem, Run UART/Radio Integration + Tests) don't need it. 6 steps scoped this way total (3 in integration-uart: two "Start GDS" + + "Provision Key"; 3 in integration-radio: "Bootstrap Sequence Number over UART", "Sync Sequence + Number over UART", "Start GDS on LoRa Passthrough"). The build job needs no key at all now + (nothing to bake into the image). No explicit `--authentication-key` CLI flag needed since the + plugin reads the env var by default. +- [x] Added a "Provision Key" step (`make test-integration FILTER=provision_key`) right after the + first `Start GDS` in integration-uart, and inside the "Bootstrap Sequence Number over UART" + block in integration-radio (before the LoRa sync, since key-store propagation means + provisioning once over UART covers the LoRa instance too). + Added `and not provision_key` to the radio job's main test-run FILTER. + +## 7. Docs +- [x] Update `AGENTS.md` "Authentication & Security" section +- [x] Update `PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md` + +## Build verification +- `make generate build` (not `uv run ...` directly — Makefile targets resolve the right Python env) + ran a real build and caught real errors, now fixed: + - FPP-generated accessor names are `get_valid/get_spi/get_key/set_valid/set_spi/set_key` (with + underscores), not `getvalid/getspi/...` as guessed earlier. Fixed all call sites in + `TcSecurityDeframer.cpp` and `Validator.cpp` via sed. + - New `ActiveKeyCount` telemetry channel (per deframer instance) was never referenced in any + telemetry packet, which `fpp-to-dict` treats as a hard error ("neither used nor marked as + omitted"). Added `ComCcsdsLora.tcSecurityDeframer.ActiveKeyCount` / + `ComCcsdsUart.tcSecurityDeframer.ActiveKeyCount` to the `Security` packet in + `ReferenceDeploymentPackets.fppi` (Sband entry commented out, matching the existing + `CurrentSequenceNumber` pattern in that same packet — Sband instance not present in this build). + - Re-ran `make generate build` after these fixes: **clean full build**, FLASH 69.81%, RAM 62.32%, + `zephyr.uf2`/`bootable.uf2`/dictionary/XTCE all generated successfully. + +## Verification +- [x] Unit tests (`make test-unit`, pure C++ only, no F Prime/Zephyr): `parseHexKey` (direct + wrapped), + `importHmacKeyBytes`/`destroyHmacKey`, and Validator SPI/sequence-number rules. Note: + Validator.cpp had started depending on the FPP-generated `AuthKeyStore` (`Fw::Serializable`), + which broke this test target's no-F-Prime contract — decoupled it via a new plain + `ActiveSpiSlots`/`ActiveSpiSlot` type in `Types.hpp`; `TcSecurityDeframer::activeSpiSlots()` + projects `m_keyStore` into it before calling `validatePacket`. Store-mutation rules + (provision-only-when-empty, add fails at 2, remove fails at 1) live in the F-Prime-dependent + command handlers, not pure functions, so they're out of scope for this gtest target — the + commented-out `register_fprime_ut` block remains for a future on-target/component test pass. +- [x] `make build` with no `AuthDefaultKey.h` — clean build confirmed (see above) +- [x] `make check-console-disabled` — OK, Zephyr console disabled +- [ ] CI green + +## Notes / decisions while implementing +(append here as work progresses) diff --git a/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi b/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi index 5b9a1132..9c4a0edd 100644 --- a/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi +++ b/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi @@ -23,6 +23,17 @@ disk-access; mount-point = "/"; }; + lfs1: lfs1 { + compatible = "zephyr,fstab,littlefs"; + mount-point = "/keys"; + partition = <&keystore_partition>; + automount; + read-size = <16>; + prog-size = <16>; + cache-size = <64>; + lookahead-size = <32>; + block-cycles = <512>; + }; }; @@ -89,9 +100,14 @@ zephyr_udc0: &usbd { reg = <0x300000 0x100000>; }; - storage_partition: partition@400000 { + keystore_partition: partition@400000 { + label = "keystore_partition"; + reg = <0x400000 0x40000>; + }; + + storage_partition: partition@440000 { label = "storage_partition"; - reg = <0x400000 0xC00000>; + reg = <0x440000 0xBC0000>; }; }; }; diff --git a/docs-site/components/TcSecurityDeframer.md b/docs-site/components/TcSecurityDeframer.md index cbdc9687..d61b0b9b 100644 --- a/docs-site/components/TcSecurityDeframer.md +++ b/docs-site/components/TcSecurityDeframer.md @@ -9,10 +9,20 @@ The component does not enforce policy. Frames that fail verification are still f The component is a thin stateful shell over pure-function namespaces: - `Ccsds355_0_B_2::parse` (Parser) — Security Header (SPI, sequence number) and Trailer (MAC) extraction -- `Components::validatePacket` (Validator) — SPI validation and anti-replay sequence-number window validation -- `Components::authenticatePacket` / `importHmacKey` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto +- `Components::validatePacket` (Validator) — SPI validation against the active key store and anti-replay sequence-number window validation +- `Components::authenticatePacket` / `importHmacKey` / `importHmacKeyBytes` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto -The only component state is the last accepted sequence number (mutex-guarded, persisted to file) and the imported HMAC key id. +`Validator` takes the active SPI set as a plain `ActiveSpiSlots` array (`Types.hpp`) rather than the FPP-generated key store type directly, so it — and its unit tests — stay pure C++ with no F Prime dependency; `TcSecurityDeframer::activeSpiSlots()` projects the real key store into that shape before calling `validatePacket`. + +Component state is the last accepted sequence number and the on-flash key store (each mutex-guarded and persisted to file), plus the PSA key ids imported from the store's valid slots. + +### Key Storage + +The HMAC authentication key is **never compiled into the firmware image** (issue #220). It is persisted at `KEY_STORE_FILE_PATH` (default `/keys/authkeys.bin`) on a dedicated littlefs `keystore_partition` on internal flash, holding up to 2 slots (`{valid, spi, key}`). `configure()` loads the store and imports every valid slot into PSA; a missing/empty store is not an error — a keyless board still boots so it can be provisioned. The sequence-number file lives on the same partition (`SEQ_NUM_FILE_PATH`, default `/keys/sequence_number.bin`). + +The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): on an unknown SPI, `dataIn_handler` reloads the store from disk once and retries validation before rejecting the frame, so a rotation issued over one link is picked up by the others without a separate command per link. + +Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-import the store into PSA and update `ActiveKeyCount` telemetry on success. Primary data path connections: @@ -33,11 +43,21 @@ class TcSecurityDeframer { -dataReturnIn_handler(portNum, data, context) -GET_SEQ_NUM_cmdHandler(opCode, cmdSeq) -SET_SEQ_NUM_cmdHandler(opCode, cmdSeq, seqNum) + -PROVISION_KEY_cmdHandler(opCode, cmdSeq, spi, key) + -ADD_KEY_cmdHandler(opCode, cmdSeq, spi, key) + -REMOVE_KEY_cmdHandler(opCode, cmdSeq, spi) -readSequenceNumber(value) -writeSequenceNumber(value) + -loadKeyStore() + -writeKeyStore() + -importKeyStore() + -findKeyIdForSpi(spi, keyId) + -activeKeyCount() + -activeSpiSlots() -m_sequenceNumber : U32 -m_sequenceNumberWindow : U32 - -m_hmacKeyId : uint32_t + -m_keyStore : AuthKeyStore + -m_keyIds : uint32_t[2] } class Ccsds355_0_B_2 { @@ -47,7 +67,7 @@ class Ccsds355_0_B_2 { class PacketValidator { <> - +validatePacket(secHeader, sequenceNumber, window) Status + +validatePacket(secHeader, sequenceNumber, window, activeSpis) Status } class PacketAuthenticator { @@ -101,19 +121,20 @@ The MAC is HMAC-SHA-256 truncated to 16 bytes, computed over the Security Header ## Behavior 1. Parse the Security Header and Trailer. If the frame is too short to contain them it cannot be stripped for downstream deframing: log ParsingFailed and return the buffer upstream (drop). -2. Validate the SPI (only SPI 0 is currently supported) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). -3. If validation passes, verify the MAC. +2. Validate the SPI (must match a valid slot in the active key store — see [Key Storage](#key-storage)) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). On an unknown SPI, the key store is reloaded from disk once and validation retried, so a rotation issued over another link is picked up here. +3. If validation passes, look up the PSA key id for the packet's SPI and verify the MAC with it. 4. Only when all checks pass: store and persist the received sequence number, telemeter it, and set `authenticated = true` in the frame context. Frames failing any check never advance the sequence number (issue #426). 5. Strip the Security Header and Trailer and forward on dataOut with the resulting `authenticated` flag. ProvesRouter rejects unauthenticated packets unless their opcode is on the bypass allowlist. -At startup, `configure()` loads the persisted sequence number and telemeters it so the first downlinked value is correct before any command is accepted (issue #427). +At startup, `configure()` loads the persisted sequence number and key store, telemetering both so the first downlinked values are correct before any command is accepted (issue #427). A missing or empty key store leaves the board keyless (bootable, awaiting `PROVISION_KEY`) rather than failing to boot. ## Parameters | Name | Type | Default | Description | |---|---|---|---| | SEQ_NUM_WINDOW | U32 | 50000 | Maximum allowed forward sequence-number distance before rejecting a packet as out-of-window. | -| SEQ_NUM_FILE_PATH | string | "//sequence_number.txt" | File path used to persist and restore the sequence number across restarts. | +| SEQ_NUM_FILE_PATH | string | "/keys/sequence_number.bin" | File path used to persist and restore the sequence number across restarts. | +| KEY_STORE_FILE_PATH | string | "/keys/authkeys.bin" | File path used to persist and restore the authentication key store across restarts. | ## Port Descriptions @@ -131,6 +152,7 @@ Standard AC ports are also present for command handling, events, telemetry, para | Name | Type | Description | |---|---|---| | CurrentSequenceNumber | U32 | Current accepted sequence number tracked by the component. Emitted at startup and on each accepted packet. | +| ActiveKeyCount | U8 | Number of valid slots (0-2) in the key store. Emitted at startup and after PROVISION_KEY/ADD_KEY/REMOVE_KEY. | Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which owns the accept/reject policy. @@ -146,6 +168,14 @@ Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which ow | AuthenticationFailed | Warning High (throttle 2) | auth_status: PacketAuthenticatorStatus, rc: I32 | Logged when MAC verification fails. Format: "Authentication failed: Status={}, PSA Return Code={}" | | ParsingFailed | Warning High (throttle 2) | parse_status: PacketParserStatus | Logged when frame parsing fails. Format: "Parsing failed: {}" | | SpiInvalid | Warning High (throttle 2) | packet_spi: U32 | Logged when SPI validation fails. Format: "SPI invalid: Received={}" | +| KeyStoreReadFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store read fails (not thrown for a missing file — that's the keyless state). Format: "Failed to read key store, error: {}" | +| KeyStoreWriteFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store write fails. Format: "Failed to write key store, error: {}" | +| KeyProvisioned | Activity High | spi: U16 | Logged by PROVISION_KEY on success. Format: "Key provisioned for SPI={}" | +| KeyProvisionFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when PROVISION_KEY fails (store not empty, bad hex key, or write failure). Format: "Key provisioning failed: {}" | +| KeyAdded | Activity High | spi: U16 | Logged by ADD_KEY on success. Format: "Key added for SPI={}" | +| KeyAddFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when ADD_KEY fails (store full, bad hex key, or write failure). Format: "Key add failed: {}" | +| KeyRemoved | Activity High | spi: U16 | Logged by REMOVE_KEY on success. Format: "Key removed for SPI={}" | +| KeyRemoveFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when REMOVE_KEY fails (last remaining key, SPI not found, or write failure). Format: "Key remove failed: {}" | ## Commands @@ -153,6 +183,9 @@ Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which ow |---|---|---|---| | GET_SEQ_NUM | Sync | None | Reads and reports the current sequence number (SequenceNumberGet event). | | SET_SEQ_NUM | Sync | seq_num: U32 | Sets and persists a new sequence number (SequenceNumberSet event). | +| PROVISION_KEY | Sync | spi: U16, key: string | Bootstrap: writes the first key slot and imports it into PSA. Only succeeds while the store is empty; bypass-allowlisted in ProvesRouter so it works on a keyless board (KeyProvisioned/KeyProvisionFailed events). | +| ADD_KEY | Sync | spi: U16, key: string | Rotation: adds a key to an empty slot. Requires an authenticated frame (not bypass-allowlisted); fails if 2 keys are already active (KeyAdded/KeyAddFailed events). | +| REMOVE_KEY | Sync | spi: U16 | Rotation: invalidates the slot matching `spi`. Requires an authenticated frame (not bypass-allowlisted); fails if it would drop the active key count below 1 (KeyRemoved/KeyRemoveFailed events). | ## Unit Tests @@ -161,8 +194,10 @@ TcSecurityDeframer helper functionality is covered by unit tests in PROVESFlight | Test File | Coverage | |---|---| | test_TcSecurityDeframer_Parser.cpp | Valid parse path plus parse failures for SPI, sequence number, and MAC size checks. | -| test_TcSecurityDeframer_Validator.cpp | SPI validation, out-of-window and replayed sequence numbers, window boundary, and wraparound handling. | -| test_TcSecurityDeframer_Authenticator.cpp | Key import failures, successful MAC verification, and failed verification with corrupted MAC or data. | +| test_TcSecurityDeframer_Validator.cpp | SPI validation against the active `ActiveSpiSlots` set (single slot, second slot, no valid slots), out-of-window and replayed sequence numbers, window boundary, and wraparound handling. | +| test_TcSecurityDeframer_Authenticator.cpp | `parseHexKey` (valid upper/lowercase, null, wrong length, non-hex characters), key import via hex and raw bytes, `destroyHmacKey`, successful MAC verification, and failed verification with corrupted MAC, corrupted data, or a destroyed key. | + +These cover only the pure-function layer (Parser/Validator/Authenticator; no F Prime or Zephyr dependency). The key store mutation rules enforced in the command handlers (`PROVISION_KEY`/`ADD_KEY`/`REMOVE_KEY` — provision-only-when-empty, add fails at 2, remove fails at 1) are F-Prime-component-dependent and are not covered here; see the commented-out `register_fprime_ut` block in `CMakeLists.txt` for a future on-target/component test pass. Run unit tests with: @@ -180,16 +215,16 @@ make framer-plugin Then run GDS with the framing plugin enabled as configured by the project tooling. -## Generating Keys +## Provisioning Keys -The default authentication key header (AuthDefaultKey.h) is generated at build time from project key material via `make generate-auth-key` or `make copy-secrets`. This generated file is machine-local and not committed. +The authentication key is no longer compiled into the image, so there is nothing to generate at build time. Instead, a keyless board is provisioned after flashing by uplinking `PROVISION_KEY(spi, key)` over a bypass-allowlisted link (the flight-side integration test `PROVESFlightControllerReference/test/int/provision_key_test.py` does this in CI). Ground must be given the same key, via the `--authentication-key` CLI arg or `PROVES_AUTH_KEY` env var to the framing plugin (`Framing/src/authenticate_plugin.py`). `make copy-secrets` still copies production key material from a secure directory for use in provisioning. ## Requirements | Name | Description | Validation | |---|---|---| | AUTH001 | The component shall parse incoming frames to extract the SPI, sequence number, and MAC fields. | Unit Test | -| AUTH003 | The component shall validate that the SPI value corresponds to a configured Security Association. | Unit Test | +| AUTH003 | The component shall validate that the SPI value corresponds to a configured Security Association (an active slot in the on-flash key store). | Unit Test | | AUTH004 | The component shall validate the received sequence number against the stored sequence number. | Unit Test | | AUTH004-A | The component shall not authenticate packets with sequence numbers that are outside the acceptable window and shall log an event. | Unit Test, Inspection | | AUTH004-B | The component shall set the stored sequence number to the sequence number transmitted in the packet only when a packet is fully validated and authenticated. | Inspection | @@ -207,3 +242,4 @@ Opcode-based bypass policy (formerly AUTH002) is owned by ProvesRouter; see its | --- | --- | | 2025-11-26 | Initial design. | | 2026-07-17 | Renamed to TcSecurityDeframer, refactor to discrete responsibilities: Authenticator, Parser, Validator. Pass-through interface between TcDeframer and SpacePacketDeframer; verification result carried in frame context; policy enforcement moved to ProvesRouter. | +| 2026-07-23 | Moved the authentication key off the compiled-in image onto a littlefs key store on internal flash, alongside the sequence-number file (issue #220). Added PROVISION_KEY/ADD_KEY/REMOVE_KEY commands supporting up to 2 active keys; SPI validation now checks the active key store instead of a hard-coded SPI 0. | diff --git a/prj.conf b/prj.conf index 946a59e3..b6135c54 100644 --- a/prj.conf +++ b/prj.conf @@ -74,6 +74,9 @@ CONFIG_FS_FATFS_EXFAT=y CONFIG_FS_FATFS_MOUNT_MKFS=y CONFIG_FS_FATFS_FSTAB_AUTOMOUNT=y CONFIG_FILE_SYSTEM_MKFS=y +# littlefs on the internal-flash keystore_partition, mounted at /keys for the +# HMAC key store and anti-replay sequence number (see TcSecurityDeframer) +CONFIG_FILE_SYSTEM_LITTLEFS=y CONFIG_MCUBOOT_SIGNATURE_KEY_FILE="keys/proves.pem" CONFIG_HAPTICS=y diff --git a/pytest.ini b/pytest.ini index 07ca9309..3da70b0b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -2,6 +2,7 @@ markers = uart_only: marks tests that sever the RF link (resets, TRANSMIT toggle) and should only be run when connected via UART sync_sequence_number: marks the test that synchronizes the sequence number between GDS and flight software; should be run before any other tests to avoid sequence number mismatches + provision_key: marks the test that provisions the HMAC authentication key onto a keyless board; should be run before any auth-required command is sent format_filesystem: marks the test that formats the filesystem; should be run before any other tests to ensure a clean state requires_face: marks tests that require a face board (TMP112 / VEML6031 / DRV2605 sensors) to be plugged in; skip on a bare flight controller requires_antenna: marks tests that require the antenna board to be plugged in and the burnwire capacitor installed; skip on a bare flight controller diff --git a/scripts/generate_auth_default_key.h b/scripts/generate_auth_default_key.h deleted file mode 100644 index d344f741..00000000 --- a/scripts/generate_auth_default_key.h +++ /dev/null @@ -1,11 +0,0 @@ -// This file is auto-generated at build time. -// DO NOT EDIT MANUALLY - it will be overwritten during build. - -#ifndef AUTH_DEFAULT_KEY_H -#define AUTH_DEFAULT_KEY_H - -// Default authentication key generated at build time -// This ensures the code works even if external key files are not available on the satellite -#define AUTH_DEFAULT_KEY "@AUTH_DEFAULT_KEY@" - -#endif // AUTH_DEFAULT_KEY_H diff --git a/scripts/generate_auth_key_header.py b/scripts/generate_auth_key_header.py deleted file mode 100755 index 17636423..00000000 --- a/scripts/generate_auth_key_header.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python3 -""" -Generate AuthDefaultKey.h file with a random HMAC key. - -This script generates AuthDefaultKey.h directly without needing spi_dict.txt. -""" - -import argparse -import os -import secrets -import sys - - -def generate_random_key() -> str: - """Generate a random 32-character hex key (16 bytes).""" - return secrets.token_hex(16) - - -def generate_auth_key_header(key: str, output_path: str, template_path: str) -> None: - """ - Generate AuthDefaultKey.h file from template. - - Args: - key: The authentication key (hex string without 0x prefix) - output_path: Path to output header file - template_path: Path to template file - """ - # Ensure output directory exists - os.makedirs(os.path.dirname(output_path), exist_ok=True) - - # Read template - with open(template_path, "r") as f: - template = f.read() - - # Replace placeholder with actual key - content = template.replace("@AUTH_DEFAULT_KEY@", key) - - # Write output file - with open(output_path, "w") as f: - f.write(content) - - -def extract_key_from_header(header_path: str) -> str: - """ - Extract the key from an existing AuthDefaultKey.h file. - - Args: - header_path: Path to AuthDefaultKey.h file - - Returns: - The authentication key (hex string without 0x prefix) - """ - if not os.path.exists(header_path): - raise FileNotFoundError(f"AuthDefaultKey.h not found at {header_path}") - - with open(header_path, "r") as f: - for line in f: - if "AUTH_DEFAULT_KEY" in line and '"' in line: - # Extract key from line like: #define AUTH_DEFAULT_KEY "4916d208d40612daad6edbc7333c4c13" - start = line.find('"') + 1 - end = line.find('"', start) - if start > 0 and end > start: - return line[start:end] - - raise ValueError("No valid key found in AuthDefaultKey.h") - - -def main(): - """Main function to parse arguments and generate/extract AuthDefaultKey.h.""" - parser = argparse.ArgumentParser( - description="Generate AuthDefaultKey.h with a random HMAC key" - ) - parser.add_argument( - "--output", - type=str, - default="PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h", - help="Output path for AuthDefaultKey.h", - ) - parser.add_argument( - "--template", - type=str, - default="scripts/generate_auth_default_key.h", - help="Path to template file", - ) - parser.add_argument( - "--key", - type=str, - default=None, - help="Use this specific key instead of generating a random one", - ) - parser.add_argument( - "--print-key", - action="store_true", - help="Print the key to stdout (for use in Makefile or debugging)", - ) - parser.add_argument( - "--extract", - action="store_true", - help="Extract key from existing header file instead of generating", - ) - - args = parser.parse_args() - - if args.extract: - # Extract from existing file - try: - key = extract_key_from_header(args.output) - if args.print_key: - print(key) - else: - print(f"Extracted key from {args.output}") - except (FileNotFoundError, ValueError) as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - else: - # Generate new file - try: - if args.key: - key = args.key - # Remove 0x prefix if present - if key.startswith("0x") or key.startswith("0X"): - key = key[2:] - else: - key = generate_random_key() - - generate_auth_key_header(key, args.output, args.template) - - if args.print_key: - print(key) - else: - print(f"Generated {args.output}") - except (FileNotFoundError, ValueError) as e: - print(f"Error: {e}", file=sys.stderr) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/tools/yamcs/proves_adapter.py b/tools/yamcs/proves_adapter.py index 1fd1ba55..ffa6d6b4 100644 --- a/tools/yamcs/proves_adapter.py +++ b/tools/yamcs/proves_adapter.py @@ -44,7 +44,7 @@ def _crc16_ccitt(data: bytes) -> int: from authenticate_plugin import ( # noqa: E402 AuthenticateFramer, - get_default_auth_key_from_header, + get_auth_key_from_env, ) from fprime_gds.common.communication.ccsds.space_data_link import ( # noqa: E402 SpaceDataLinkFramerDeframer, @@ -404,7 +404,7 @@ def _parse_scid_list(val: str) -> list[int]: p.add_argument( "--auth-key", default=None, - help="HMAC key as hex string (no 0x prefix). Defaults to key from AuthDefaultKey.h.", + help="HMAC key as hex string (no 0x prefix). Defaults to the PROVES_AUTH_KEY env var.", ) # Frame size and CCSDS identifiers @@ -448,8 +448,8 @@ def main(): # Resolve auth key auth_key = args.auth_key if auth_key is None: - auth_key = get_default_auth_key_from_header() - print("[auth] Loaded key from AuthDefaultKey.h") + auth_key = get_auth_key_from_env() + print("[auth] Loaded key from PROVES_AUTH_KEY env var") auth_framer = AuthenticateFramer(authentication_key=auth_key) From 1ff5d8bb21fb56e3cc5043c7d8932ba4607154b6 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 00:17:33 -0700 Subject: [PATCH 02/29] debug(ci): capture fault registers after boot to diagnose integration-uart silence integration-uart/integration-radio are both failing with zero bytes ever received from the board after flashing this branch's firmware (PR #472). The USB CDC ACM link repeatedly drops and re-enumerates, consistent with a boot-time crash loop rather than a framing/auth logic bug. Add a temporary step that halts both RP2350 cores via OpenOCD ~20s after boot and dumps CFSR/HFSR/registers/backtrace so we can see the actual fault cause. --- .github/workflows/ci.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 183b2d31..53bd8e4a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,6 +127,30 @@ jobs: - name: Flash Firmware uses: ./.github/actions/flash-firmware + - name: DEBUG capture fault registers after boot + run: | + sleep 20 + ~/openocd/src/openocd -s ~/openocd/tcl -f ~/openocd/tcl/interface/cmsis-dap.cfg -f ~/openocd/tcl/target/rp2350.cfg -c "adapter speed 5000" \ + -c "init" \ + -c "targets" \ + -c "rp2350.cm0 halt" \ + -c "echo {--- cm0 regs ---}" -c "rp2350.cm0 reg" \ + -c "echo {--- cm0 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ + -c "rp2350.cm0 mdw 0xE000ED28 5" \ + -c "echo {--- cm0 backtrace ---}" -c "rp2350.cm0 arm semihosting enable" \ + -c "rp2350.cm0 bt" \ + -c "rp2350.cm1 halt" \ + -c "echo {--- cm1 regs ---}" -c "rp2350.cm1 reg" \ + -c "echo {--- cm1 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ + -c "rp2350.cm1 mdw 0xE000ED28 5" \ + -c "shutdown" || true + # Power-cycle to clear the halted-core state and get back to a clean boot + # for the real test steps that follow. + ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 0 + sleep 3 + ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 1 + sleep 5 + - name: Load .env file run: | while IFS= read -r line || [ -n "$line" ]; do From 78ca10f7700a2be4501cb83696f28f442a46fcd2 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 06:46:08 -0700 Subject: [PATCH 03/29] debug(ci): fix OpenOCD target-selection syntax in fault-register diagnostic The previous attempt used "rp2350.cm0 halt" as a single command, which isn't valid OpenOCD syntax (only arp_halt exists per-target instance); OpenOCD printed the target's command-usage listing and exited before reaching any of the reg/mdw/bt commands, so the run produced no diagnostic data. Select the target explicitly with "targets rp2350.cmN" first, then issue plain halt/reg/mdw/bt commands against it. --- .github/workflows/ci.yaml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53bd8e4a..1b591dac 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -133,16 +133,18 @@ jobs: ~/openocd/src/openocd -s ~/openocd/tcl -f ~/openocd/tcl/interface/cmsis-dap.cfg -f ~/openocd/tcl/target/rp2350.cfg -c "adapter speed 5000" \ -c "init" \ -c "targets" \ - -c "rp2350.cm0 halt" \ - -c "echo {--- cm0 regs ---}" -c "rp2350.cm0 reg" \ + -c "targets rp2350.cm0" \ + -c "halt" \ + -c "echo {--- cm0 regs ---}" -c "reg" \ -c "echo {--- cm0 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ - -c "rp2350.cm0 mdw 0xE000ED28 5" \ - -c "echo {--- cm0 backtrace ---}" -c "rp2350.cm0 arm semihosting enable" \ - -c "rp2350.cm0 bt" \ - -c "rp2350.cm1 halt" \ - -c "echo {--- cm1 regs ---}" -c "rp2350.cm1 reg" \ + -c "mdw 0xE000ED28 5" \ + -c "echo {--- cm0 backtrace ---}" -c "arm semihosting enable" \ + -c "bt" \ + -c "targets rp2350.cm1" \ + -c "halt" \ + -c "echo {--- cm1 regs ---}" -c "reg" \ -c "echo {--- cm1 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ - -c "rp2350.cm1 mdw 0xE000ED28 5" \ + -c "mdw 0xE000ED28 5" \ -c "shutdown" || true # Power-cycle to clear the halted-core state and get back to a clean boot # for the real test steps that follow. From 0949dfbd90fd5d3c1b8fbbf66604c421442512c4 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 07:06:48 -0700 Subject: [PATCH 04/29] debug(ci): query individual registers with explicit poll/settle in fault diagnostic The previous attempt's bulk "reg" dump came back with register names but no values, and OpenOCD logged "target was in unknown state when halt was requested" for both cores -- the halt raced the initial poll cycle right after init. Also "bt" isn't a valid OpenOCD console command (that's GDB), so it aborted the remaining -c chain before the cm1 section ever ran. Force an explicit poll before and after halt, add a settle delay, and query pc/lr/sp/xpsr/r0-r3 individually instead of the bulk dump. --- .github/workflows/ci.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1b591dac..2f855b62 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -134,15 +134,21 @@ jobs: -c "init" \ -c "targets" \ -c "targets rp2350.cm0" \ + -c "poll" \ -c "halt" \ - -c "echo {--- cm0 regs ---}" -c "reg" \ + -c "sleep 200" \ + -c "poll" \ + -c "echo {--- cm0 pc/lr/sp/xpsr/r0-r3 ---}" \ + -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" -c "reg r0" -c "reg r1" -c "reg r2" -c "reg r3" \ -c "echo {--- cm0 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ -c "mdw 0xE000ED28 5" \ - -c "echo {--- cm0 backtrace ---}" -c "arm semihosting enable" \ - -c "bt" \ -c "targets rp2350.cm1" \ + -c "poll" \ -c "halt" \ - -c "echo {--- cm1 regs ---}" -c "reg" \ + -c "sleep 200" \ + -c "poll" \ + -c "echo {--- cm1 pc/lr/sp/xpsr/r0-r3 ---}" \ + -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" -c "reg r0" -c "reg r1" -c "reg r2" -c "reg r3" \ -c "echo {--- cm1 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ -c "mdw 0xE000ED28 5" \ -c "shutdown" || true From 693eb5d8a5330ab782f69ab04a3e3dd6370c8c6d Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 07:30:07 -0700 Subject: [PATCH 05/29] debug(ci): remove temporary fault-register diagnostic step Diagnostics are done: the board is not crashing (CFSR/HFSR both read 0, core0 was live in Thread mode with no fault flags set). Restore integration-uart to its pre-diagnostic form. --- .github/workflows/ci.yaml | 32 -------------------------------- 1 file changed, 32 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2f855b62..183b2d31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,38 +127,6 @@ jobs: - name: Flash Firmware uses: ./.github/actions/flash-firmware - - name: DEBUG capture fault registers after boot - run: | - sleep 20 - ~/openocd/src/openocd -s ~/openocd/tcl -f ~/openocd/tcl/interface/cmsis-dap.cfg -f ~/openocd/tcl/target/rp2350.cfg -c "adapter speed 5000" \ - -c "init" \ - -c "targets" \ - -c "targets rp2350.cm0" \ - -c "poll" \ - -c "halt" \ - -c "sleep 200" \ - -c "poll" \ - -c "echo {--- cm0 pc/lr/sp/xpsr/r0-r3 ---}" \ - -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" -c "reg r0" -c "reg r1" -c "reg r2" -c "reg r3" \ - -c "echo {--- cm0 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ - -c "mdw 0xE000ED28 5" \ - -c "targets rp2350.cm1" \ - -c "poll" \ - -c "halt" \ - -c "sleep 200" \ - -c "poll" \ - -c "echo {--- cm1 pc/lr/sp/xpsr/r0-r3 ---}" \ - -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" -c "reg r0" -c "reg r1" -c "reg r2" -c "reg r3" \ - -c "echo {--- cm1 CFSR/HFSR/DFSR/MMFAR/BFAR/AFSR ---}" \ - -c "mdw 0xE000ED28 5" \ - -c "shutdown" || true - # Power-cycle to clear the halted-core state and get back to a clean boot - # for the real test steps that follow. - ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 0 - sleep 3 - ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 1 - sleep 5 - - name: Load .env file run: | while IFS= read -r line || [ -n "$line" ]; do From 5570974c7d457c4457d4009da8b6617e83c42b31 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 09:44:29 -0700 Subject: [PATCH 06/29] docs: record hardware CI investigation findings for integration-uart/radio failure Three rounds of live SWD diagnostics on the integration-uart bench confirm the board is not crashing (CFSR/HFSR read 0 on both cores; core0 was live in Thread mode executing fs_open when halted). The ground side sees real USB serial disconnects instead. Leading theory: moving the sequence-number file and key-store reload onto the internal QSPI flash that also serves the running code trades the old SD-card path's interrupt-safety for the RP2350's requirement to disable interrupts system-wide during any internal flash write, now hit on the hot path (every accepted frame, and every frame with an unrecognized SPI while unprovisioned). Not yet confirmed or fixed -- see PROBLEM.md for the full writeup and candidate next steps. --- PROBLEM.md | 151 +++++++++++++++++++++++++++++++++++++++++++++++++++++ TODO.md | 10 +++- 2 files changed, 160 insertions(+), 1 deletion(-) create mode 100644 PROBLEM.md diff --git a/PROBLEM.md b/PROBLEM.md new file mode 100644 index 00000000..c1645335 --- /dev/null +++ b/PROBLEM.md @@ -0,0 +1,151 @@ +# CI hardware failure on hmac-to-storage (PR #472): board never responds over UART/LoRa + +## Symptom + +`integration-uart` and `integration-radio` both fail at the very first test +(`provision_key_test.py::test_provision_key`), inside the `start_gds` fixture's +`CMD_NO_OP` retry loop. GDS never sees any reply: + +- `recv.bin` is 0 bytes for the whole test window (one run showed 48 bytes — see below). +- `sent.bin` is 0 bytes: the ground side's write to the serial device itself is failing. +- `comm.py.log` shows real OS-level serial errors, not just logical timeouts: + ``` + [WARNING] serial_adapter: Serial exception caught: device reports readiness to + read but returned no data (device disconnected or multiple access on port?). + Reconnecting. + [WARNING] uplink: Uplink failed to send 41 bytes of data after 3 retries + ``` +- This reproduced identically on two different physical benches (UART bench and + LoRa/radio bench), ruling out a single flaky cable/bench. +- `lint`, `unit-test`, `build`, `yamcs-build` are all green. Only the two + hardware-integration jobs fail. + +## Investigation + +Three rounds of live SWD diagnostics were run against the UART bench (via a +temporary CI step in `.github/workflows/ci.yaml`, since removed — see commits +`1ff5d8bb`, `78ca10f7`, `0949dfbd`, `693eb5d8` on this branch for the added/fixed/ +removed diagnostic). Each halted both RP2350 cores ~20s after boot via OpenOCD and +dumped registers + CFSR/HFSR/DFSR/MMFAR/BFAR (`0xE000ED28`). + +Attempt 1: broken OpenOCD command syntax (`rp2350.cm0 halt` isn't valid — only +`arp_halt` exists per-target-instance); OpenOCD printed a command-usage listing and +exited before reaching any read. No data. + +Attempt 2: fixed target selection (`targets rp2350.cm0` then plain `halt`/`reg`), but +hit two problems: OpenOCD warned `target was in unknown state when halt was +requested` (a race between `halt` and its own initial poll cycle) so the bulk `reg` +dump came back with register **names** but no **values**; and `bt` (not a valid +OpenOCD console command — that's GDB) aborted the remaining `-c` chain before core1 +was ever queried. Partial data: CFSR=0, HFSR=0, DFSR=1 for core0 (a real debug-halt, +no fault). + +Attempt 3: added explicit `poll` before/after `halt`, a settle delay, and queried +`pc`/`lr`/`sp`/`xpsr`/`r0`-`r3` individually instead of the bulk dump; dropped `bt`. +This got real data for both cores: + +``` +cm0: pc=0x101864b8 lr=0x1010fb69 sp=0x20034410 xpsr=0x61000000 (Thread mode, no fault) + CFSR=0 HFSR=0 DFSR=0 +cm1: pc=0x0000019e lr=0x00000203 sp=0xf0000000 xpsr=0x09000000 + CFSR=0 HFSR=0 DFSR=1 (plain debug halt) +``` + +Resolved against a local build of the same commit's `zephyr.elf` +(`build-artifacts/zephyr.elf`, built by a prior session — see TODO.md): + +``` +$ arm-none-eabi-addr2line -e build-artifacts/zephyr.elf -f -C 0x101864b8 0x1010fb69 +fs_open +lib/zephyr-workspace/zephyr/subsys/fs/fs.c:140 +idle +lib/zephyr-workspace/zephyr/kernel/idle.c:30 +``` + +**Core0 (the only core Zephyr actually runs on this board) was live, in Thread +mode, with zero fault flags set, executing `fs_open()` at the moment it was +halted.** Core1 is just idling in the RP2350 boot ROM waiting for a multicore +launch that never comes (expected/benign for a single-core app). + +This rules out a crash, a hard fault, a null pointer dereference reaching a fault +handler, and a boot-time panic. The board is alive and running normal application +code; it is *not* in a fault-loop. + +## Leading hypothesis (not yet confirmed) + +This PR (per the plan) moves two things onto a new internal-flash littlefs +partition (`keystore_partition`, mounted at `/keys`) that previously lived on the +SD-card FAT filesystem: + +1. The anti-replay sequence number (`SEQ_NUM_FILE_PATH` default now + `/keys/sequence_number.bin`, was on `/` = SD/FAT). +2. The new HMAC key store (`/keys/authkeys.bin`), which `TcSecurityDeframer` + reloads from disk once whenever an incoming frame's SPI doesn't match any + active slot — i.e., on **every single received frame** while the board is + unprovisioned (keyless), which is exactly the state under test here. + +Both files now live on the **same physical QSPI/XIP flash chip that serves the +running firmware code**. On RP2040/RP2350, writing or erasing that flash requires +suspending code execution from flash and disabling interrupts system-wide for the +duration of the operation (a well-documented Pico SDK / Zephyr internal-flash +driver constraint — this is *not* true of the old SD-card/SDMMC path, which uses a +separate SPI bus with no such restriction). + +`TcSecurityDeframer::dataIn_handler` (see +`PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp`, +around lines 56–119) holds both `m_keyStoreLock` and `m_sequenceNumberLock` for the +full validate+authenticate+persist sequence on every frame, and: +- writes the sequence-number file (`writeSequenceNumber`) on every *accepted* + frame, and +- calls `loadKeyStore()` (a fresh `fs_open`+read) on every frame with an + unrecognized SPI, before this PR happens to be *every* frame during the keyless + window this test exercises. + +The working theory is that this internal-flash I/O — now on a hot path that used +to be interrupt-safe SD-card I/O — is disabling interrupts long enough, and/or +often enough, to stall USB CDC-ACM servicing, producing exactly the "device +disconnected" / failed-write symptom seen on the ground side. The single mid-flight +`fs_open` sample is consistent with this but is not proof by itself (no flash- +operation timing instrumentation was captured); a controlled repro (e.g. +instrumenting write duration with a GPIO toggle or RTT, or scoping bus activity) +would be needed to confirm the magnitude and pin down which specific call +(`writeSequenceNumber` vs `loadKeyStore` vs the very first-ever littlefs auto-format +at boot) is responsible. + +## What this is NOT + +- Not a build/compile problem — `build`, `unit-test`, `lint`, `yamcs-build` are all + green. +- Not a crash/fault/panic — confirmed via live register + CFSR/HFSR read on real + hardware, core0 healthy and running. +- Not bench/cable flakiness alone — reproduced identically on two separate physical + benches (UART and LoRa). +- Not the CI runner being stuck — earlier apparent multi-hour queue delays during + this investigation turned out to be real backlog on the shared hardware runner, + unrelated to this bug; runs did eventually execute and complete. + +## Suggested next steps (not yet implemented — needs a decision) + +The sequence-number-and-key-store-onto-internal-flash design is a locked decision +from the original plan (`~/.claude/plans/quirky-forging-possum.md`), so the fix +should stay within that design rather than reverting to SD-card storage. Candidates +worth evaluating: + +1. Debounce/throttle `writeSequenceNumber` so it isn't a synchronous internal-flash + write on every single accepted frame — e.g. only persist every N frames or after + a time interval, accepting a small anti-replay window on power loss (arguably + already implicitly tolerated by the existing `SEQ_NUM_WINDOW` mechanism). +2. Stop calling `loadKeyStore()` on *every* unrecognized-SPI frame; e.g. rate-limit + reloads (the plan's stated purpose — cross-link key-rotation propagation — is an + infrequent event, not something that needs a fresh disk read on every single + uplinked frame while unprovisioned). +3. Confirm whether the very first-ever littlefs mount on a blank `keystore_partition` + performs a full-partition format synchronously at `SYS_INIT` (before `main()`), + and if so, whether that's the dominant one-time stall rather than (or in addition + to) the per-frame writes. +4. Instrument actual flash-operation duration on real hardware (GPIO toggle around + `flash_area_write`/`erase`, or Zephyr's flash driver trace hooks) to know the real + magnitude before choosing a fix, rather than guessing further. + +None of the above has been implemented yet — see `TODO.md` for the standing +checklist item. diff --git a/TODO.md b/TODO.md index 1d690a51..d003d036 100644 --- a/TODO.md +++ b/TODO.md @@ -113,7 +113,15 @@ Status legend: [ ] todo, [~] in progress, [x] done commented-out `register_fprime_ut` block remains for a future on-target/component test pass. - [x] `make build` with no `AuthDefaultKey.h` — clean build confirmed (see above) - [x] `make check-console-disabled` — OK, Zephyr console disabled -- [ ] CI green +- [ ] CI green — **blocked**, see `PROBLEM.md`: `integration-uart`/`integration-radio` + fail on real hardware. Board is confirmed alive (not crashed/faulted — verified + via live SWD register dump), but the ground side sees real USB serial + disconnects and never gets a response to the first command. Leading theory: + moving the sequence-number file + key-store reload onto the same internal QSPI + flash chip that serves the running code means every write now disables + interrupts system-wide (RP2350 XIP constraint) on what used to be + interrupt-safe SD-card I/O, stalling USB. Needs a decision + fix — not yet + implemented. ## Notes / decisions while implementing (append here as work progresses) From a9245d9ce658373c9294f2017d4b36e6d73e4e31 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 11:41:44 -0700 Subject: [PATCH 07/29] fix(boards): correct flash0 size to 16MB, unblocking keystore partition mount keystore_partition@0x400000 sat past the declared 4MB flash0 boundary, so is_valid_range() rejected every flash op with -EINVAL before littlefs could even attempt to mount /keys -- masquerading as the interrupt-stall symptom described in PROBLEM.md. storage_partition already extended to 16MB against this same 4MB declaration on main, unnoticed only because nothing mounted it, which is strong evidence the physical chip is 16MB and the declaration was simply wrong. Awaiting hardware CI to confirm the integration-uart/integration-radio jobs go green with a mountable /keys. --- .codespell-ignore-words.txt | 1 + INVESTIGATION.md | 155 ++++++++++++++++++ TODO.md | 39 ++++- .../proves_flight_control_board_v5.dtsi | 2 +- 4 files changed, 187 insertions(+), 10 deletions(-) create mode 100644 INVESTIGATION.md diff --git a/.codespell-ignore-words.txt b/.codespell-ignore-words.txt index 9c69ee4e..76847f0c 100644 --- a/.codespell-ignore-words.txt +++ b/.codespell-ignore-words.txt @@ -1,4 +1,5 @@ ALS +FRAM comIn bufferIn commandIn diff --git a/INVESTIGATION.md b/INVESTIGATION.md new file mode 100644 index 00000000..42e8f549 --- /dev/null +++ b/INVESTIGATION.md @@ -0,0 +1,155 @@ +# Investigation: hmac-to-storage CI hardware failure — root-cause re-analysis + +Follow-up to `PROBLEM.md`. This document records a static re-analysis of the +`integration-uart` / `integration-radio` failure on branch `hmac-to-storage` +(PR #472), tracing the device tree, the RP2350 flash driver, the littlefs +automount path, and the `TcSecurityDeframer` hot path. + +**Headline conclusion:** the "internal-flash writes disable interrupts and stall +USB" hypothesis in `PROBLEM.md` is very likely a phantom. The keystore littlefs +partition is placed **beyond the declared flash boundary**, so every keystore +flash operation is rejected with `-EINVAL` *before* any interrupt is disabled, +and `/keys` can never mount. The most probable real fix is a one-line flash-size +correction in the device tree. + +Confidence: the partition-out-of-bounds and read-vs-write facts below are +**code-confirmed**. Whether the mount failure is the *sole* cause of the USB +symptom is **not yet confirmed on hardware** — see "Next steps". + +--- + +## Finding 1 — Reads do NOT disable interrupts (contradicts PROBLEM.md hypothesis #2) + +`lib/zephyr-workspace/zephyr/drivers/flash/flash_rpi_pico.c`: + +- `flash_rpi_read` (line 46) is a bare `memcpy` from the XIP-mapped flash window. + **No `irq_lock`.** +- Only `flash_rpi_write` (line 82) and `flash_rpi_erase` (line 132) take + `irq_lock()` / `irq_unlock()`. + +Therefore `loadKeyStore()` — a pure `fs_open`+read invoked on every +unrecognized-SPI frame (`TcSecurityDeframer.cpp:70`) — **cannot** stall USB via +interrupt disable. The mid-flight `fs_open` PC sample that `PROBLEM.md` treats as +its strongest evidence (`cm0: pc=0x101864b8 → fs_open`) is a *harmless read*, not +a stall. `PROBLEM.md` item #2 (per-frame reload) is a non-issue for USB timing. + +## Finding 2 — `keystore_partition` is placed past the end of the declared flash + +`boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi:77`: + +``` +&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; } /* 4 MB */ +``` +Confirmed in the generated build: `CONFIG_FLASH_SIZE=4096`, +`CONFIG_FLASH_BASE_ADDRESS=0x10000000` +(`build-fprime-automatic-zephyr/zephyr/.config`). + +Partition map (`.dtsi` / generated `zephyr.dts`): + +| partition | offset | end | note | +|----------------------|------------|------------|------------------------------| +| boot (mcuboot) | 0x000000 | 0x100000 | | +| slot0 (current) | 0x100000 | 0x200000 | | +| slot1 (golden) | 0x200000 | 0x300000 | | +| slot2 (test) | 0x300000 | 0x400000 | fills the entire 4 MB | +| **keystore_partition** | **0x400000** | 0x440000 | **starts AT the 4 MB end** | +| storage_partition | 0x440000 | 0x1000000 | runs to 16 MB | + +`keystore_partition@0x400000` begins exactly at the declared flash boundary and +is entirely out of bounds. The flash driver gate: + +```c +#define FLASH_SIZE KB(CONFIG_FLASH_SIZE) /* = 4 MB = 0x400000 */ +static bool is_valid_range(off_t offset, uint32_t size) { + return (offset >= 0) && ((offset + size) <= FLASH_SIZE); +} +``` + +`flash_area_*` passes the partition's flash-relative offset (`0x400000`) to the +driver, so `is_valid_range(0x400000, size)` → `(0x400000 + size) <= 0x400000` → +**false → `-EINVAL`**. This check runs **before** the `irq_lock` in both +`flash_rpi_write` and `flash_rpi_erase`. + +Consequences: + +1. `lfs_mount` → format (`littlefs_fs.c:966–971`) → `erase` → `-EINVAL` → format + fails → **`/keys` never mounts.** The feature as shipped cannot persist keys + or the sequence number at all. +2. Since no keystore flash op ever executes, **no interrupt is ever disabled for + it** — the interrupt-stall mechanism in `PROBLEM.md` has nothing to act on. + +## Finding 3 — This is a latent flash-size mis-declaration, newly exposed + +On `main`, `storage_partition` already ran `0x400000 → 0x1000000` (12 MB at +offset 4 MB) against the same 4 MB `flash0` declaration (confirmed via +`git diff main..hmac-to-storage` on the `.dtsi`). It never mattered because +**nothing mounted `storage_partition`** — the SD card (FAT, `disk-access`) serves +`/`, and that is the only fstab mount on `main`. + +This branch is the first to actually *mount* a filesystem at an offset ≥ 4 MB +(littlefs at `/keys`), which is what exposed the pre-existing mismatch. The 16 MB +extent of `storage_partition` strongly implies the physical chip is a 16 MB +W25Q128-class part and the `DT_SIZE_M(4)` declaration is simply wrong. + +## What still needs hardware confirmation + +The mount failure alone would normally degrade gracefully (the deframer tolerates +a missing store / keyless boot), so it is not yet proven that it is the *sole* +cause of the observed USB "device disconnected" symptom. It is, however, a +definite defect that makes the feature non-functional and that removes the basis +for the interrupt-stall theory. Confirm on hardware before ruling other causes +in or out. + +--- + +## Storage layout for v5e (as built) + +- `zephyr,flash = &flash0` — internal QSPI/XIP flash, declared 4 MB (**suspected + under-declaration**; see Finding 3). +- fstab has two mounts: + - `ffs1` — **FAT on the SD card** (spi0 / sdmmc-disk, mount `/`, `disk-access`). + Separate SPI bus, no XIP interrupt-disable constraint. This is where the + sequence number lived before this branch. + - `lfs1` — **littlefs on `keystore_partition`** (mount `/keys`, `automount`) — + added by this branch; currently unmountable (Finding 2). +- USB CDC-ACM (`CONFIG_USB_DEVICE_STACK_NEXT`, v5e defconfig) is the GDS + transport; the "device disconnected" errors are on this link. + +--- + +## Fix applied + +No hardware bench was available this session to do the SWD-read / console-log +confirmation described below as the original "Step 0". Proceeded on the +strength of the existing evidence instead (Finding 3: `storage_partition` +already ran to 16 MB against this same 4 MB declaration on `main`, unnoticed +only because nothing mounted it). + +Change made: `&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; }` → +`DT_SIZE_M(16)` in `proves_flight_control_board_v5.dtsi` (shared by the v5c/v5d/v5e +board variants via `#include`). `make generate build` confirms `CONFIG_FLASH_SIZE` +now follows to 16384 (was 4096) with an otherwise identical, clean build (FLASH +69.82%, RAM 62.32% — unchanged from before the edit). + +**This is not yet confirmed on hardware.** If the physical chip is actually +4 MB, re-running `integration-uart`/`integration-radio` should surface a +different failure mode than before (a hardware-level flash access fault or +hang when littlefs actually reaches into out-of-range silicon, rather than the +old software-level `-EINVAL`-before-mount short-circuit) — that would be the +signal to fall back to the original plan: carve `keystore_partition` from +within 0–4 MB (e.g. shrink `slot2/test`) and fix `storage_partition` too, since +it's also out of bounds. + +**Robustness follow-ups (independent of the size fix, evaluate after Step 0):** + +1. Consider replacing littlefs for this data with raw `flash_area_*` or the NVS + backend — it is a fixed-size key store + a 4-byte counter; a filesystem is + overkill and littlefs rewrites a whole file (extra erases) on every accepted + frame. NVS/raw erase far less often. +2. Rate-limit `loadKeyStore()` so it is not a fresh `fs_open` on every + unrecognized-SPI frame (read-only and harmless to USB, but wasteful). +3. If per-frame sequence-number persistence proves to be a real flash-wear or + timing problem after Step 0, either throttle `writeSequenceNumber` (persist + every N frames / on a timer, bounded replay window) or move only the seq + counter to a byte-writable, no-erase medium (RV3028 RTC battery-backed + user RAM/EEPROM on i2c1 — verify ≥4 usable bytes — or FRAM/MRAM if present). diff --git a/TODO.md b/TODO.md index d003d036..afa4ebf3 100644 --- a/TODO.md +++ b/TODO.md @@ -113,15 +113,36 @@ Status legend: [ ] todo, [~] in progress, [x] done commented-out `register_fprime_ut` block remains for a future on-target/component test pass. - [x] `make build` with no `AuthDefaultKey.h` — clean build confirmed (see above) - [x] `make check-console-disabled` — OK, Zephyr console disabled -- [ ] CI green — **blocked**, see `PROBLEM.md`: `integration-uart`/`integration-radio` - fail on real hardware. Board is confirmed alive (not crashed/faulted — verified - via live SWD register dump), but the ground side sees real USB serial - disconnects and never gets a response to the first command. Leading theory: - moving the sequence-number file + key-store reload onto the same internal QSPI - flash chip that serves the running code means every write now disables - interrupts system-wide (RP2350 XIP constraint) on what used to be - interrupt-safe SD-card I/O, stalling USB. Needs a decision + fix — not yet - implemented. +- [ ] CI green — **pending re-run**. `integration-uart`/`integration-radio` failed on + real hardware. Board was confirmed alive (not crashed/faulted — verified via live + SWD register dump), but the ground side saw real USB serial disconnects and never + got a response to the first command. See `INVESTIGATION.md` for a static + re-analysis: the `PROBLEM.md` interrupt-stall theory is **likely wrong** (reads + don't `irq_lock` in `flash_rpi_pico.c`, and `keystore_partition@0x400000` sat + past the declared 4 MB `flash0` boundary → every keystore op returned `-EINVAL` + before any `irq_lock`, so `/keys` never mounted and no interrupt was ever + disabled for it). Real root cause is almost certainly the flash-size + mis-declaration. **Fix implemented below — awaiting CI hardware confirmation.** + +## Suggested fix (blocked CI) — see INVESTIGATION.md +- [x] **Primary fix**: changed `&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; }` → + `DT_SIZE_M(16)` in `proves_flight_control_board_v5.dtsi` (shared by v5c/v5d/v5e). + No hardware bench available this session to do the SWD/console confirmation from + the old Step 0, but the evidence was already strong without it: `storage_partition` + on `main` has run `0x400000 → 0x1000000` (16 MB extent) against this same 4 MB + declaration for a while, unnoticed only because nothing mounted it. `make generate + build` confirms `CONFIG_FLASH_SIZE` now follows to 16384 (was 4096), same + FLASH/RAM usage as before (69.82%/62.32%) — build is otherwise unaffected. + **Re-run both integration jobs on real hardware to confirm** — if the chip is + actually 4 MB this will surface as a hardware-level flash access fault/hang + distinct from the old `-EINVAL`-before-mount failure, at which point fall back to + carving `keystore_partition` from within 0–4 MB (e.g. shrink `slot2/test`) and + fixing `storage_partition` too. +- [ ] **Robustness follow-ups (evaluate only after CI confirms the fix):** (a) consider raw + `flash_area_*`/NVS instead of littlefs for this fixed-size store; (b) rate-limit + `loadKeyStore()` so it isn't a fresh `fs_open` per unrecognized-SPI frame; (c) if + per-frame `writeSequenceNumber` proves a real wear/timing problem, throttle it or + move only the seq counter to a no-erase medium (RV3028 RTC user RAM / FRAM). ## Notes / decisions while implementing (append here as work progresses) diff --git a/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi b/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi index 9c4a0edd..96813f80 100644 --- a/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi +++ b/boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi @@ -74,7 +74,7 @@ zephyr_udc0: &usbd { }; &flash0 { - reg = <0x10000000 DT_SIZE_M(4)>; + reg = <0x10000000 DT_SIZE_M(16)>; partitions { compatible = "fixed-partitions"; #address-cells = <0x1>; From 5eee6190c0424f8e6bca6a08eb931383d1b17b60 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 11:55:24 -0700 Subject: [PATCH 08/29] docs: record hardware CI result for flash-size fix, narrow root cause CI run 30034861047 confirms the physical chip is 16MB (OpenOCD: w25q128fv/jv, 16384 KiB), so the flash0 size fix was correct and necessary. But integration-uart/integration-radio still fail with the identical USB-disconnect symptom, so the size fix alone doesn't unblock CI. Revised theory: now that /keys can actually mount, littlefs's first-ever format reaches flash_rpi_write/flash_rpi_erase for the first time, both of which hold irq_lock() for the whole erase/program call -- PROBLEM.md's original interrupt-stall theory may be correct after all, it just had nothing to act on before this fix. --- INVESTIGATION.md | 57 +++++++++++++++++++++++++++++++++++++------ TODO.md | 63 ++++++++++++++++++++++++++++-------------------- 2 files changed, 86 insertions(+), 34 deletions(-) diff --git a/INVESTIGATION.md b/INVESTIGATION.md index 42e8f549..90fbe7a1 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -131,14 +131,55 @@ board variants via `#include`). `make generate build` confirms `CONFIG_FLASH_SIZ now follows to 16384 (was 4096) with an otherwise identical, clean build (FLASH 69.82%, RAM 62.32% — unchanged from before the edit). -**This is not yet confirmed on hardware.** If the physical chip is actually -4 MB, re-running `integration-uart`/`integration-radio` should surface a -different failure mode than before (a hardware-level flash access fault or -hang when littlefs actually reaches into out-of-range silicon, rather than the -old software-level `-EINVAL`-before-mount short-circuit) — that would be the -signal to fall back to the original plan: carve `keystore_partition` from -within 0–4 MB (e.g. shrink `slot2/test`) and fix `storage_partition` too, since -it's also out of bounds. +**Confirmed on hardware (CI run 30034861047, 2026-07-23):** `Flash Firmware` +step's OpenOCD output reports `RP2350 rev 3, QSPI Flash win w25q128fv/jv id = +0x1840ef size = 16384 KiB in 4096 sectors` — the chip really is 16 MB, so the +`.dtsi` fix is correct and `CONFIG_FLASH_SIZE`/`is_valid_range` now permit the +`keystore_partition` range. + +**But `integration-uart`/`integration-radio` still fail identically** — GDS's +`comm.py.log` shows the same repeated +`Serial exception caught: device reports readiness to read but returned no +data (device disconnected or multiple access on port?). Reconnecting.` +starting ~19s after GDS start, and `start_gds`'s `CMD_NO_OP` never gets a +response within the whole 30s test window (both `provision_key_test.py` and +`format_filesystem_test.py` fail the same way in `integration-uart`; the radio +job fails at the identical `Bootstrap Sequence Number over UART` / +`provision_key` step). So the flash-size mis-declaration was real and worth +fixing, but it was **not the sole cause** of the CI failure. + +## Revised theory: PROBLEM.md's interrupt-stall theory may now apply for real + +Before this fix, every keystore flash op was rejected by `-EINVAL` *before* +`flash_rpi_write`/`flash_rpi_erase` ever reached `irq_lock()` — that's why +Finding 1/2 above concluded the interrupt-stall theory had "nothing to act on." +Now that `/keys` can actually mount, `lfs_mount`'s first-ever format on this +partition (superblock write, at minimum) is the first code in this branch that +actually reaches `flash_rpi_erase`/`flash_rpi_write`, both of which wrap the +*entire* erase/program call in `irq_lock()`/`irq_unlock()` +(`flash_rpi_pico.c:132-136`, `:82-107`) — with **no yielding** for however long +`flash_range_erase`/`flash_range_program` (Pico SDK bootrom calls) take. If +that takes long enough, USB CDC-ACM polling stalls exactly like the +symptom shows. This would mean the size fix was a necessary but not +sufficient change — PROBLEM.md's mechanism is real, it just couldn't fire +until the partition became mountable. + +**Not yet confirmed**: whether it's actually the littlefs format path (one-time, +at first boot on a virgin partition) vs. per-frame `loadKeyStore()`/ +`writeSequenceNumber` reloads causing repeated stalls throughout the test. The +timing (~19s in, then continuing through the whole 30s window) is at least +consistent with either. Console/log output is disabled in these builds +(`make check-console-disabled` is a required check), so there's no boot log +visibility in this CI run to distinguish the two — a temporary +`CONFIG_LOG=y` diagnostic CI run (same pattern as the prior fault-register +diagnostic commits `1ff5d8bb`/`78ca10f7`/`0949dfbd`/`693eb5d8`) would show +whether the stall is at first-mount/format or ongoing. + +## Original next steps (superseded above, kept for the 4 MB fallback) + +Everything below was written before hardware confirmation; kept for +reference/fallback only — the chip is now confirmed 16 MB, so the fallback +branch does not apply: **Robustness follow-ups (independent of the size fix, evaluate after Step 0):** diff --git a/TODO.md b/TODO.md index afa4ebf3..491b19c5 100644 --- a/TODO.md +++ b/TODO.md @@ -113,36 +113,47 @@ Status legend: [ ] todo, [~] in progress, [x] done commented-out `register_fprime_ut` block remains for a future on-target/component test pass. - [x] `make build` with no `AuthDefaultKey.h` — clean build confirmed (see above) - [x] `make check-console-disabled` — OK, Zephyr console disabled -- [ ] CI green — **pending re-run**. `integration-uart`/`integration-radio` failed on - real hardware. Board was confirmed alive (not crashed/faulted — verified via live - SWD register dump), but the ground side saw real USB serial disconnects and never - got a response to the first command. See `INVESTIGATION.md` for a static - re-analysis: the `PROBLEM.md` interrupt-stall theory is **likely wrong** (reads - don't `irq_lock` in `flash_rpi_pico.c`, and `keystore_partition@0x400000` sat - past the declared 4 MB `flash0` boundary → every keystore op returned `-EINVAL` - before any `irq_lock`, so `/keys` never mounted and no interrupt was ever - disabled for it). Real root cause is almost certainly the flash-size - mis-declaration. **Fix implemented below — awaiting CI hardware confirmation.** +- [ ] CI green — **still blocked, but root cause narrowed**. Pushed the 16MB flash + fix (below) and re-ran CI (run 30034861047): `Flash Firmware` step's OpenOCD + output confirms the chip really is 16MB (`w25q128fv/jv ... size = 16384 KiB`), + so the fix itself is correct and necessary. But `integration-uart`/ + `integration-radio` **still fail with the identical symptom** — GDS + `comm.py.log` shows repeated `device disconnected` serial exceptions starting + ~19s after boot, `CMD_NO_OP` never gets a response. So the flash-size + mis-declaration was real but **not the sole cause**. Revised theory: now that + `/keys` can actually mount, littlefs's first-ever format on this partition is + the first code path in this branch that reaches `flash_rpi_write`/ + `flash_rpi_erase`, both of which hold `irq_lock()` for the entire + erase/program call with no yielding — i.e. the original `PROBLEM.md` + interrupt-stall theory may be correct after all, it just couldn't fire before + (every op was rejected by `-EINVAL` pre-`irq_lock`). See `INVESTIGATION.md` + "Revised theory" section. **Next: a temporary `CONFIG_LOG=y` diagnostic CI + run** (same pattern as the prior fault-register diagnostic commits) to see + whether the stall is the one-time format or ongoing per-frame reloads. ## Suggested fix (blocked CI) — see INVESTIGATION.md - [x] **Primary fix**: changed `&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; }` → `DT_SIZE_M(16)` in `proves_flight_control_board_v5.dtsi` (shared by v5c/v5d/v5e). - No hardware bench available this session to do the SWD/console confirmation from - the old Step 0, but the evidence was already strong without it: `storage_partition` - on `main` has run `0x400000 → 0x1000000` (16 MB extent) against this same 4 MB - declaration for a while, unnoticed only because nothing mounted it. `make generate - build` confirms `CONFIG_FLASH_SIZE` now follows to 16384 (was 4096), same - FLASH/RAM usage as before (69.82%/62.32%) — build is otherwise unaffected. - **Re-run both integration jobs on real hardware to confirm** — if the chip is - actually 4 MB this will surface as a hardware-level flash access fault/hang - distinct from the old `-EINVAL`-before-mount failure, at which point fall back to - carving `keystore_partition` from within 0–4 MB (e.g. shrink `slot2/test`) and - fixing `storage_partition` too. -- [ ] **Robustness follow-ups (evaluate only after CI confirms the fix):** (a) consider raw - `flash_area_*`/NVS instead of littlefs for this fixed-size store; (b) rate-limit - `loadKeyStore()` so it isn't a fresh `fs_open` per unrecognized-SPI frame; (c) if - per-frame `writeSequenceNumber` proves a real wear/timing problem, throttle it or - move only the seq counter to a no-erase medium (RV3028 RTC user RAM / FRAM). + Confirmed correct by CI hardware run 30034861047: OpenOCD reports the real chip + is `w25q128fv/jv ... size = 16384 KiB`. `CONFIG_FLASH_SIZE` now follows to 16384 + (was 4096), build otherwise unaffected (FLASH/RAM usage unchanged). **Necessary + but not sufficient** — see next item. +- [ ] **Diagnose the remaining stall**: same CI run still fails identically + (repeated GDS `device disconnected` serial exceptions, no response to + `CMD_NO_OP`). Revised theory: `/keys` mounting for the first time means + littlefs's format now actually reaches `flash_rpi_write`/`flash_rpi_erase`, + which hold `irq_lock()` for the whole erase/program call — `PROBLEM.md`'s + original interrupt-stall theory may be correct, it just had nothing to act on + before this fix. Next: temporary `CONFIG_LOG=y` diagnostic CI run (pattern + from commits `1ff5d8bb`/`78ca10f7`/`0949dfbd`/`693eb5d8`) to see whether the + stall is the one-time format or ongoing per-frame reloads. +- [ ] **Robustness follow-ups (evaluate once the stall is diagnosed):** (a) consider + raw `flash_area_*`/NVS instead of littlefs for this fixed-size store (avoids + the format-time erase burst and any long single-call erase/program under + `irq_lock`); (b) rate-limit `loadKeyStore()` so it isn't a fresh `fs_open` per + unrecognized-SPI frame; (c) if per-frame `writeSequenceNumber` proves a real + wear/timing problem, throttle it or move only the seq counter to a no-erase + medium (RV3028 RTC user RAM / FRAM). ## Notes / decisions while implementing (append here as work progresses) From 77708d002001c969097ec33995eedfb2064a2f6c Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 12:06:33 -0700 Subject: [PATCH 09/29] debug(ci): capture boot console log to diagnose littlefs mount stall Enable Zephyr console+logging (prj.conf) to see whether the remaining integration-uart/integration-radio USB-disconnect stall lines up with littlefs's first-ever format on the newly-mountable keystore partition, per the revised interrupt-stall theory in INVESTIGATION.md. Console shares cdc_acm_uart0 with the F' downlink, so this intentionally desyncs GDS for this one CI run -- capture raw boot text from a fresh power cycle before anything else opens the port, same tradeoff as the earlier fault-register diagnostic commits. check-console-disabled is temporarily disabled in the build job since it would otherwise fail on this intentional change. --- .github/workflows/ci.yaml | 39 +++++++++++++++++++++++++++++++++------ prj.conf | 12 ++++++++---- 2 files changed, 41 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 183b2d31..1c01d469 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,12 +93,13 @@ jobs: run: | make build - - name: Ensure console disabled - # The Zephyr console shares cdc_acm_uart0 with the F' downlink; if it is - # re-enabled, console text interleaves with the CCSDS TM frames and - # desyncs the GDS deframer. Fail the build so this cannot reach main. - run: | - make check-console-disabled + # TEMPORARY DIAGNOSTIC (revert before merge): console is intentionally + # enabled right now (see prj.conf) to capture boot-time littlefs text, + # so this gate would fail the build. Re-enable once the diagnostic is + # removed and prj.conf is reverted. + # - name: Ensure console disabled + # run: | + # make check-console-disabled - name: Upload build artifacts uses: actions/upload-artifact@v4 @@ -164,6 +165,32 @@ jobs: run: | pkill -9 python || true + # TEMPORARY DIAGNOSTIC (revert before merge): console is enabled in this + # build (see prj.conf), sharing cdc_acm_uart0 with the F' downlink, so + # capture raw boot text from a fresh power cycle before anything else + # opens the port. This intentionally desyncs the rest of this job's GDS + # steps below -- that's expected for this one-off diagnostic run. + - name: Capture Boot Console Log + run: | + timeout 40 cat "$UART_DEVICE" > console-boot.log 2>&1 & + CAT_PID=$! + sleep 1 + ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 0 + sleep 3 + ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 1 + wait "$CAT_PID" || true + echo "=== console-boot.log ===" + cat console-boot.log || true + + - name: Upload Boot Console Log + if: always() + uses: actions/upload-artifact@v4 + with: + name: console-boot-log + path: console-boot.log + if-no-files-found: ignore + retention-days: 7 + - name: Start GDS env: PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} diff --git a/prj.conf b/prj.conf index b6135c54..e1d3d1eb 100644 --- a/prj.conf +++ b/prj.conf @@ -21,9 +21,12 @@ CONFIG_UART_INTERRUPT_DRIVEN=y # console on means printk/Os::Console text interleaves with the binary TM frames # and desyncs the GDS deframer. Disable the UART console so the F' UART carries # frames only; F' still drives the device directly via comDriver. -CONFIG_CONSOLE=n -CONFIG_UART_CONSOLE=n -CONFIG_PRINTK=n +# TEMPORARY DIAGNOSTIC (revert before merge): console+log enabled to capture +# boot-time littlefs mount/format text on the shared UART; this intentionally +# desyncs the GDS deframer for this one CI run. +CONFIG_CONSOLE=y +CONFIG_UART_CONSOLE=y +CONFIG_PRINTK=y CONFIG_SERIAL=y CONFIG_GPIO=y CONFIG_LED=y @@ -58,7 +61,8 @@ CONFIG_COMMON_LIBC_MALLOC=y CONFIG_SENSOR=y # Enable detailed logging for I2C and sensor debugging -CONFIG_LOG=n +# TEMPORARY DIAGNOSTIC (revert before merge): see CONFIG_CONSOLE note above. +CONFIG_LOG=y CONFIG_LOG_DEFAULT_LEVEL=3 CONFIG_CBPRINTF_FP_SUPPORT=y From 3cdfa541926be339c6d5cb40160f096a61428377 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 12:24:24 -0700 Subject: [PATCH 10/29] debug(ci): revert console diagnostic, record boot-hang finding Diagnostic done (CI run 30036653676): boot log shows normal USB init through ~1.16s then total silence for the rest of the 40s capture -- no more log lines and no TM-frame noise either, consistent with a full hang very early in boot. Ruled out a stale PICO_FLASH_SIZE_BYTES hard_assert in the Pico SDK's flash_range_erase (macro isn't defined in this build). Restore prj.conf/ci.yaml to their pre-diagnostic state since console can't coexist with a working GDS link. --- .github/workflows/ci.yaml | 39 +++++------------------------- INVESTIGATION.md | 51 +++++++++++++++++++++++++++++++++++++++ TODO.md | 16 +++++++++--- prj.conf | 12 +++------ 4 files changed, 74 insertions(+), 44 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1c01d469..183b2d31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,13 +93,12 @@ jobs: run: | make build - # TEMPORARY DIAGNOSTIC (revert before merge): console is intentionally - # enabled right now (see prj.conf) to capture boot-time littlefs text, - # so this gate would fail the build. Re-enable once the diagnostic is - # removed and prj.conf is reverted. - # - name: Ensure console disabled - # run: | - # make check-console-disabled + - name: Ensure console disabled + # The Zephyr console shares cdc_acm_uart0 with the F' downlink; if it is + # re-enabled, console text interleaves with the CCSDS TM frames and + # desyncs the GDS deframer. Fail the build so this cannot reach main. + run: | + make check-console-disabled - name: Upload build artifacts uses: actions/upload-artifact@v4 @@ -165,32 +164,6 @@ jobs: run: | pkill -9 python || true - # TEMPORARY DIAGNOSTIC (revert before merge): console is enabled in this - # build (see prj.conf), sharing cdc_acm_uart0 with the F' downlink, so - # capture raw boot text from a fresh power cycle before anything else - # opens the port. This intentionally desyncs the rest of this job's GDS - # steps below -- that's expected for this one-off diagnostic run. - - name: Capture Boot Console Log - run: | - timeout 40 cat "$UART_DEVICE" > console-boot.log 2>&1 & - CAT_PID=$! - sleep 1 - ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 0 - sleep 3 - ~/korad_control/.venv/bin/python ~/korad_control/korad_control.py -d /dev/ttyPWR --output 1 - wait "$CAT_PID" || true - echo "=== console-boot.log ===" - cat console-boot.log || true - - - name: Upload Boot Console Log - if: always() - uses: actions/upload-artifact@v4 - with: - name: console-boot-log - path: console-boot.log - if-no-files-found: ignore - retention-days: 7 - - name: Start GDS env: PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} diff --git a/INVESTIGATION.md b/INVESTIGATION.md index 90fbe7a1..bbfa68f4 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -175,6 +175,57 @@ visibility in this CI run to distinguish the two — a temporary diagnostic commits `1ff5d8bb`/`78ca10f7`/`0949dfbd`/`693eb5d8`) would show whether the stall is at first-mount/format or ongoing. +## Console-log diagnostic (CI run 30036653676, 2026-07-23, reverted) + +Temporarily enabled `CONFIG_CONSOLE`/`CONFIG_UART_CONSOLE`/`CONFIG_PRINTK`/ +`CONFIG_LOG` and captured raw serial from a fresh power-cycle for 40s before +GDS ever opened the port (console shares `cdc_acm_uart0` with the F' downlink +per `scripts/check_console_disabled.py`, so this intentionally desynced GDS +for the run — same tradeoff as the prior fault-register diagnostics). + +Captured log (`console-boot.log`, full contents): + +``` +[00:00:00.034,000] LSM6DSO: Initialize device LSM6DSO +[00:00:00.035,000] LSM6DSO: chip id 0x6c +[00:00:00.785,000] sd: Maximum SD clock is under 25MHz, using clock of 24000000Hz +*** Booting Zephyr OS build v4.4.1 *** +[00:00:00.795,000] usbd_init: bNumInterfaces 2 wTotalLength 75 +[00:00:00.927,000] usbd_core: Actual device speed 1 +[00:00:01.040,000] usbd_core: Actual device speed 1 +[00:00:01.162,000] usbd_ch9: protocol error: (GET_DESCRIPTOR/DEVICE_QUALIFIER, "not supported" — routine/benign for a full-speed-only device, seen twice) +``` + +...then **nothing** for the remaining ~39s of the 40s capture window — no +further log lines, and critically no raw TM-frame bytes either (F' emits +periodic telemetry within the first second or two of a normal boot; even +console-corrupted binary noise from that would show up as bytes in the +capture). Ruled out one specific hypothesis: the Pico SDK's own +`flash_range_erase()` has a separate `hard_assert(flash_offs + count <= +PICO_FLASH_SIZE_BYTES)` guard (`lib/zephyr-workspace/modules/hal/rpi_pico/src/ +rp2_common/hardware_flash/flash.c`) independent of Zephyr's `CONFIG_FLASH_SIZE` +— but `PICO_FLASH_SIZE_BYTES` is not defined anywhere in this Zephyr build +(confirmed via grep across `lib/zephyr-workspace/zephyr/` and the generated +build's compile flags), so that `#ifdef`-guarded assert is compiled out +entirely and cannot be firing. + +**Reading**: total silence this early (before first telemetry) is consistent +with a full system hang very early in boot — plausibly right around where the +`/keys` fstab automount would run — but the console log alone doesn't prove +*where*. It could be `irq_lock()` held for a very long single erase/program +call (the "revised theory" above), or something else entirely blocking +further interrupt/scheduler activity. Diagnostic reverted (prj.conf + +ci.yaml back to pre-diagnostic state) since it can't safely coexist with a +working GDS link. + +**Next diagnostic (not yet run)**: repeat the original fault-register/SWD +approach (commits `1ff5d8bb` etc.) but at multiple time offsets *after* a +fresh flash — e.g. halt+dump PC at t=+2s, +10s, +20s — to see whether PC is +parked inside `flash_range_erase`/`flash_range_program`/the bootrom routines +they call for an extended period. That would confirm or rule out the +interrupt-stall theory directly without touching console/USB at all, avoiding +the corruption tradeoff entirely. + ## Original next steps (superseded above, kept for the 4 MB fallback) Everything below was written before hardware confirmation; kept for diff --git a/TODO.md b/TODO.md index 491b19c5..6a40f025 100644 --- a/TODO.md +++ b/TODO.md @@ -144,9 +144,19 @@ Status legend: [ ] todo, [~] in progress, [x] done littlefs's format now actually reaches `flash_rpi_write`/`flash_rpi_erase`, which hold `irq_lock()` for the whole erase/program call — `PROBLEM.md`'s original interrupt-stall theory may be correct, it just had nothing to act on - before this fix. Next: temporary `CONFIG_LOG=y` diagnostic CI run (pattern - from commits `1ff5d8bb`/`78ca10f7`/`0949dfbd`/`693eb5d8`) to see whether the - stall is the one-time format or ongoing per-frame reloads. + before this fix. + Ran (and reverted) a temporary `CONFIG_LOG=y`+console diagnostic (CI run + 30036653676): boot log shows normal USB init through ~1.16s then **total + silence** for the remaining ~39s — no more log lines, no TM-frame noise + either. Consistent with a full hang very early in boot, but doesn't pinpoint + where. Ruled out a stale `PICO_FLASH_SIZE_BYTES` hard_assert in the Pico + SDK's `flash_range_erase` — that macro isn't defined in this Zephyr build. + Diagnostic reverted (console can't coexist with a working GDS link — see + `scripts/check_console_disabled.py`). **Next: SWD PC-sampling at multiple + time offsets** after flash (same pattern as commits `1ff5d8bb`/`78ca10f7`/ + `0949dfbd`/`693eb5d8`, but timed sweeps instead of one dump) to see if PC is + parked inside `flash_range_erase`/`flash_range_program` — see + `INVESTIGATION.md` "Next diagnostic" for details. Not yet run. - [ ] **Robustness follow-ups (evaluate once the stall is diagnosed):** (a) consider raw `flash_area_*`/NVS instead of littlefs for this fixed-size store (avoids the format-time erase burst and any long single-call erase/program under diff --git a/prj.conf b/prj.conf index e1d3d1eb..b6135c54 100644 --- a/prj.conf +++ b/prj.conf @@ -21,12 +21,9 @@ CONFIG_UART_INTERRUPT_DRIVEN=y # console on means printk/Os::Console text interleaves with the binary TM frames # and desyncs the GDS deframer. Disable the UART console so the F' UART carries # frames only; F' still drives the device directly via comDriver. -# TEMPORARY DIAGNOSTIC (revert before merge): console+log enabled to capture -# boot-time littlefs mount/format text on the shared UART; this intentionally -# desyncs the GDS deframer for this one CI run. -CONFIG_CONSOLE=y -CONFIG_UART_CONSOLE=y -CONFIG_PRINTK=y +CONFIG_CONSOLE=n +CONFIG_UART_CONSOLE=n +CONFIG_PRINTK=n CONFIG_SERIAL=y CONFIG_GPIO=y CONFIG_LED=y @@ -61,8 +58,7 @@ CONFIG_COMMON_LIBC_MALLOC=y CONFIG_SENSOR=y # Enable detailed logging for I2C and sensor debugging -# TEMPORARY DIAGNOSTIC (revert before merge): see CONFIG_CONSOLE note above. -CONFIG_LOG=y +CONFIG_LOG=n CONFIG_LOG_DEFAULT_LEVEL=3 CONFIG_CBPRINTF_FP_SUPPORT=y From f5c3a124353b1cc74ec7950fe37a2de8dad96a44 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 13:53:26 -0700 Subject: [PATCH 11/29] debug(ci): sample core PC/LR at intervals after boot to locate the hang The console-log diagnostic (reverted, see INVESTIGATION.md) showed total silence from ~1.16s after boot through the rest of a 40s capture, no firmware/config changes needed here -- halt both cores via OpenOCD and read pc/lr/sp/xpsr at t=+2s/+10s/+20s after a fresh reset to see whether either core is parked inside flash_range_erase/flash_range_program (or a bootrom routine they call), consistent with the revised interrupt-stall theory. Halt/resume only, so the rest of the job's normal (still-failing) GDS flow is undisturbed. --- .github/workflows/ci.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 183b2d31..f6c0157f 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,6 +127,42 @@ jobs: - name: Flash Firmware uses: ./.github/actions/flash-firmware + # TEMPORARY DIAGNOSTIC (remove once root-caused): sample PC/LR on both + # cores at several offsets after a fresh reset to see whether either + # core is parked inside flash_range_erase/flash_range_program (or the + # bootrom routines they call) for an extended period, per the revised + # interrupt-stall theory in INVESTIGATION.md. Halt/resume only -- no + # firmware or config changes, so this doesn't disturb the rest of the + # job's normal (still-failing) GDS flow below. + - name: PC Sweep Diagnostic + run: | + ~/openocd/src/openocd -s ~/openocd/tcl -f ~/openocd/tcl/interface/cmsis-dap.cfg -f ~/openocd/tcl/target/rp2350.cfg \ + -c "adapter speed 5000" \ + -c "init" \ + -c "reset run" \ + -c "sleep 2000" \ + -c "targets rp2350.cm0" -c "poll" -c "halt" -c "poll" \ + -c "echo {--- t=2s cm0 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ + -c "resume" \ + -c "targets rp2350.cm1" -c "poll" -c "halt" -c "poll" \ + -c "echo {--- t=2s cm1 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ + -c "resume" \ + -c "sleep 8000" \ + -c "targets rp2350.cm0" -c "poll" -c "halt" -c "poll" \ + -c "echo {--- t=10s cm0 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ + -c "resume" \ + -c "targets rp2350.cm1" -c "poll" -c "halt" -c "poll" \ + -c "echo {--- t=10s cm1 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ + -c "resume" \ + -c "sleep 10000" \ + -c "targets rp2350.cm0" -c "poll" -c "halt" -c "poll" \ + -c "echo {--- t=20s cm0 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ + -c "resume" \ + -c "targets rp2350.cm1" -c "poll" -c "halt" -c "poll" \ + -c "echo {--- t=20s cm1 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ + -c "resume" \ + -c "shutdown" || true + - name: Load .env file run: | while IFS= read -r line || [ -n "$line" ]; do From 59b8178cd0d5797d3759483306348ae914e0f3ea Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 14:08:45 -0700 Subject: [PATCH 12/29] debug(ci): revert PC-sweep diagnostic, record boot-hang location Diagnostic done (CI run 30043983799): cm0's pc/lr/sp/xpsr are identical at t=+2s/+10s/+20s after reset -- zero progress for 20+ seconds. pc resolves to fs_open+2, lr resolves into idle() (the early SYS_INIT/ fstab-automount context), i.e. the hang is at the first real file operation this branch performs after /keys mounts (almost certainly loadKeyStore() opening a virgin key-store file for the first time). cm1 sampled a bootrom address unchanged throughout -- it was never launched into Zephyr code, this app runs single-core. See INVESTIGATION.md "PC-sweep diagnostic" for full reasoning and the two candidate mechanisms, both of which point at replacing the littlefs /keys mount with raw flash_area_*/NVS. --- .github/workflows/ci.yaml | 36 -------------------- INVESTIGATION.md | 72 +++++++++++++++++++++++++++++++++++++++ TODO.md | 29 +++++++++++++--- 3 files changed, 96 insertions(+), 41 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f6c0157f..183b2d31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,42 +127,6 @@ jobs: - name: Flash Firmware uses: ./.github/actions/flash-firmware - # TEMPORARY DIAGNOSTIC (remove once root-caused): sample PC/LR on both - # cores at several offsets after a fresh reset to see whether either - # core is parked inside flash_range_erase/flash_range_program (or the - # bootrom routines they call) for an extended period, per the revised - # interrupt-stall theory in INVESTIGATION.md. Halt/resume only -- no - # firmware or config changes, so this doesn't disturb the rest of the - # job's normal (still-failing) GDS flow below. - - name: PC Sweep Diagnostic - run: | - ~/openocd/src/openocd -s ~/openocd/tcl -f ~/openocd/tcl/interface/cmsis-dap.cfg -f ~/openocd/tcl/target/rp2350.cfg \ - -c "adapter speed 5000" \ - -c "init" \ - -c "reset run" \ - -c "sleep 2000" \ - -c "targets rp2350.cm0" -c "poll" -c "halt" -c "poll" \ - -c "echo {--- t=2s cm0 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ - -c "resume" \ - -c "targets rp2350.cm1" -c "poll" -c "halt" -c "poll" \ - -c "echo {--- t=2s cm1 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ - -c "resume" \ - -c "sleep 8000" \ - -c "targets rp2350.cm0" -c "poll" -c "halt" -c "poll" \ - -c "echo {--- t=10s cm0 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ - -c "resume" \ - -c "targets rp2350.cm1" -c "poll" -c "halt" -c "poll" \ - -c "echo {--- t=10s cm1 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ - -c "resume" \ - -c "sleep 10000" \ - -c "targets rp2350.cm0" -c "poll" -c "halt" -c "poll" \ - -c "echo {--- t=20s cm0 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ - -c "resume" \ - -c "targets rp2350.cm1" -c "poll" -c "halt" -c "poll" \ - -c "echo {--- t=20s cm1 ---}" -c "reg pc" -c "reg lr" -c "reg sp" -c "reg xpsr" \ - -c "resume" \ - -c "shutdown" || true - - name: Load .env file run: | while IFS= read -r line || [ -n "$line" ]; do diff --git a/INVESTIGATION.md b/INVESTIGATION.md index bbfa68f4..27845857 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -226,6 +226,78 @@ they call for an extended period. That would confirm or rule out the interrupt-stall theory directly without touching console/USB at all, avoiding the corruption tradeoff entirely. +## PC-sweep diagnostic (CI run 30043983799, 2026-07-23) — hang located + +Ran the sweep above (`PC Sweep Diagnostic` step, halt/resume only, no +firmware/config changes). Result for `cm0` (the core running Zephyr/F'): + +| offset | pc | lr | sp | xpsr | +|--------|------------|------------|------------|------------| +| t=2s | 0x101864b8 | 0x1010fb69 | 0x20034410 | 0x61000000 | +| t=10s | 0x101864b8 | 0x1010fb69 | 0x20034410 | 0x61000000 | +| t=20s | 0x101864b8 | 0x1010fb69 | 0x20034410 | 0x61000000 | + +**Identical PC/LR/SP/xPSR at all three offsets spanning 20 seconds** — cm0 +made zero forward progress the entire window. Resolved against this exact +commit's local build (`build-fprime-automatic-zephyr/zephyr/zephyr.elf`, +same `f5c3a124` the CI run built from): + +``` +101864b6 T fs_open +101864b6 T fs_open <- pc 0x101864b8 is fs_open+2 (its first real instruction) +1010fb48 T idle +1010fb90 t unpend_thread_no_timeout <- lr 0x1010fb69 is idle()+0x21 (its caller) +``` + +So cm0 is parked at the very entry of `fs_open()`, called from Zephyr's early +init sequence (which runs in the context that becomes the idle thread before +the scheduler starts other threads — this is the boot-time `SYS_INIT`/fstab +automount call chain, not literally the CPU-idle loop). This is consistent +with the **first real file operation this branch performs after a successful +`/keys` mount** — almost certainly `TcSecurityDeframer::configure()` calling +`loadKeyStore()`'s `fs_open()` on a virgin key-store file, i.e. exactly the +code path the flash-size fix newly unblocked. + +`cm1` (the second RP2350 core) sampled `pc=0x19e`/`msp=0xf0000000` unchanged +at all three offsets too — `0x19e` is a bootrom address (well below the +`0x10000000` XIP flash base), meaning **cm1 was never launched into Zephyr +code at all** and has been idling in the bootrom's core-1 launch stub since +reset. This app apparently runs single-core (cm1 unused/unlaunched). + +This is a real, reproducible, total hang (not a slow operation) at the exact +point the new keystore feature first touches the filesystem, and it fully +explains the console-log diagnostic's total silence after ~1.16s. It doesn't +by itself prove the *mechanism* (why `fs_open`/whatever it calls never +returns), but two mechanisms fit the facts and are worth checking first: + +1. **RP2350 dual-core flash lockout deadlock**: the vendored + `flash_range_erase`/`flash_range_program` in + `lib/zephyr-workspace/modules/hal/rpi_pico/src/rp2_common/hardware_flash/flash.c` + call `flash_exit_xip_func()`/ROM erase-program routines directly — no + `multicore_lockout`/`flash_safe_execute` handshake is visible in this + vendored copy, so a genuine multicore lockout wait is less likely here + than on stock Pico SDK, but worth double-checking the ROM functions + themselves don't internally expect core1's cooperation given cm1 was never + launched. +2. **A global fs/littlefs lock held by a different, already-wedged context**: + if some earlier code path (e.g. an interrupt-context or another thread) + is genuinely stuck inside a flash erase/program with `irq_lock()` held + indefinitely, `fs_open()`'s first action (typically taking a shared fs + mutex) would block forever waiting for a lock that will never be + released — cm0's halted PC/LR here would be showing the *this* thread's + blocked-on-mutex state, not literally an infinite loop inside `fs_open` + itself. Under this reading the real hang is still likely to be an + erase/program call somewhere in the mount/format path, just not the one + `fs_open` is calling right now. + +Both point at the same practical fix: avoid littlefs's first-time +format/create path on this partition. The pre-existing "Robustness +follow-ups" item — replacing the littlefs `/keys` mount with raw +`flash_area_*`/NVS for this fixed-size key store — sidesteps this class of +bug entirely regardless of which exact mechanism is at fault, since NVS +doesn't take a global fs lock and its record writes are simple, bounded +`flash_area_write`/`erase` calls with well-understood timing. + ## Original next steps (superseded above, kept for the 4 MB fallback) Everything below was written before hardware confirmation; kept for diff --git a/TODO.md b/TODO.md index 6a40f025..cf8c4954 100644 --- a/TODO.md +++ b/TODO.md @@ -152,11 +152,30 @@ Status legend: [ ] todo, [~] in progress, [x] done where. Ruled out a stale `PICO_FLASH_SIZE_BYTES` hard_assert in the Pico SDK's `flash_range_erase` — that macro isn't defined in this Zephyr build. Diagnostic reverted (console can't coexist with a working GDS link — see - `scripts/check_console_disabled.py`). **Next: SWD PC-sampling at multiple - time offsets** after flash (same pattern as commits `1ff5d8bb`/`78ca10f7`/ - `0949dfbd`/`693eb5d8`, but timed sweeps instead of one dump) to see if PC is - parked inside `flash_range_erase`/`flash_range_program` — see - `INVESTIGATION.md` "Next diagnostic" for details. Not yet run. + `scripts/check_console_disabled.py`). + **Ran the SWD PC-sweep (CI run 30043983799, reverted after): hang located.** + cm0's pc/lr/sp/xpsr are byte-for-byte identical at t=+2s/+10s/+20s after + reset — zero forward progress for 20+ seconds. Resolved against the + build's symbols: `pc=0x101864b8` is `fs_open+2` (its first real + instruction), `lr=0x1010fb69` is inside Zephyr's `idle()` (the context + that runs early `SYS_INIT`/fstab-automount code before the scheduler + starts other threads) — i.e. cm0 is frozen at the very first file + operation this branch performs after `/keys` mounts (almost certainly + `TcSecurityDeframer::configure()`'s `loadKeyStore()` opening a virgin + key-store file for the first time). cm1 sampled `pc=0x19e` (a bootrom + address) unchanged too — cm1 was never launched into Zephyr code at all, + this app runs single-core. Two candidate mechanisms (both point at the + same fix, see `INVESTIGATION.md` "PC-sweep diagnostic" for full + reasoning): (a) something in the mount/format path already holds + `irq_lock()` in a flash erase/program that never returns, and `fs_open`'s + first action (a shared fs mutex) blocks on it forever; (b) a dual-core + interaction given cm1's unusual unlaunched state, though the vendored + `flash_range_erase`/`flash_range_program` in this tree don't show an + obvious multicore-lockout wait. **Next: either replace the littlefs + `/keys` mount with raw `flash_area_*`/NVS (sidesteps this class of bug + regardless of exact mechanism — see Robustness follow-ups below), or dig + further into which specific call inside the mount/format/create path + never returns.** - [ ] **Robustness follow-ups (evaluate once the stall is diagnosed):** (a) consider raw `flash_area_*`/NVS instead of littlefs for this fixed-size store (avoids the format-time erase burst and any long single-call erase/program under From 683c2f8e3b8a8765272aa598a4d63d0720787532 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 15:50:07 -0700 Subject: [PATCH 13/29] debug(ci): read-only SWD fault/XIP/stack forensics at the /keys hang The PC sweep (reverted, see INVESTIGATION.md) located a permanent cm0 freeze at ~fs_open but could not say why. The one sampled PC (0x101864b8) is mid-way through fs_open's opening 4-byte stmdb (fs_open starts at 0x101864b6), which reads more like a stalled instruction fetch / fault than code cleanly blocked at a call site -- so before committing to the littlefs->NVS rework, capture what actually distinguishes the candidate mechanisms: - SCB fault regs (CFSR/HFSR/DFSR/DHCSR) -> faulted / locked-up core - QMI/XIP state (DIRECT_CSR.EN, M0_RFMT/RCMD, XIP_CTRL) -> XIP left wedged by the erase/program XIP-exit dance (a flash-layer bug that NVS would hit identically, making the rework useless) - SRAM stack dump -> hand-unwind below fs_open (mount/create vs. a k_mutex wait that the rework would actually fix) Read-only: halt/read/resume, no firmware or config change, so the rest of the (still-failing) GDS flow is undisturbed. Deliberately never reads a 0x10xx_xxxx flash address over SWD -- if XIP is wedged such a read can stall the adapter; the instruction at PC is already known from zephyr.elf. Added to integration-uart only (both jobs fail identically). Remove once root-caused. --- .github/workflows/ci.yaml | 14 ++++ scripts/diag/hang_forensics.tcl | 109 ++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+) create mode 100644 scripts/diag/hang_forensics.tcl diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 183b2d31..f64ad7a6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,6 +127,20 @@ jobs: - name: Flash Firmware uses: ./.github/actions/flash-firmware + # TEMPORARY DIAGNOSTIC (remove once root-caused): read-only SWD forensics + # to distinguish core-fault vs. wedged-XIP vs. fs-lock at the /keys boot + # hang, before deciding on the littlefs->NVS rework. See INVESTIGATION.md + # and scripts/diag/hang_forensics.tcl. Halt/read only; no firmware or + # config change, so the rest of the (still-failing) GDS flow below is + # undisturbed. + - name: Hang Forensics Diagnostic + run: | + ~/openocd/src/openocd -s ~/openocd/tcl \ + -f ~/openocd/tcl/interface/cmsis-dap.cfg \ + -f ~/openocd/tcl/target/rp2350.cfg \ + -c "adapter speed 5000" \ + -f "$GITHUB_WORKSPACE/scripts/diag/hang_forensics.tcl" || true + - name: Load .env file run: | while IFS= read -r line || [ -n "$line" ]; do diff --git a/scripts/diag/hang_forensics.tcl b/scripts/diag/hang_forensics.tcl new file mode 100644 index 00000000..0f7cea35 --- /dev/null +++ b/scripts/diag/hang_forensics.tcl @@ -0,0 +1,109 @@ +# hang_forensics.tcl -- read-only SWD forensics for the /keys boot hang +# (branch hmac-to-storage, PR #472). See INVESTIGATION.md. +# +# Purpose: decide *why* cm0 is frozen at ~fs_open after the flash-size fix, +# BEFORE committing to the littlefs->NVS rework. Distinguishes: +# (A) core faulted / locked up -> CFSR/HFSR/DFSR/DHCSR non-zero +# (B) QMI/XIP interface left wedged -> QMI_DIRECT_CSR.EN=1 / bad M0_RFMT +# by the flash erase/program dance (flash-layer bug; NVS would ALSO +# hang -> rework is wasted) +# (C) cleanly blocked on an fs/lfs lock -> no fault, XIP sane, stack shows a +# k_mutex/k_sem wait (rework helps) +# +# Usage (CI or MOSAIC/GDS bench): +# openocd -s ~/openocd/tcl \ +# -f interface/cmsis-dap.cfg -f target/rp2350.cfg \ +# -c "adapter speed 5000" \ +# -f scripts/diag/hang_forensics.tcl +# +# SAFETY -- DO NOT read any 0x10xx_xxxx (XIP flash) address over SWD here. If +# the QMI/XIP interface is wedged (hypothesis B), an SWD read of a flash +# address can stall the debug adapter and lose the whole capture. Every read +# below targets the PPB (0xE000_xxxx), the QMI/XIP peripherals (0x400C/D_xxxx), +# or SRAM (0x2003_xxxx) only. The instruction at PC is already known from the +# ELF (fs_open begins with a 4-byte `stmdb` at 0x101864b6), so we never read +# code back over the wire. +# +# Memory-AP reads (mdw) work whether or not the core actually halts, so even a +# locked-up core still yields its fault registers and peripheral state. Each +# section is wrapped in `catch` so one failing read never aborts the rest. + +proc rd {label addr} { + if {[catch {set line [capture "mdw $addr"]} err]} { + echo " $label ($addr): " + } else { + echo " $label ($addr): [string trim $line]" + } +} + +init + +echo "=== reset + free-run so boot reaches the hang ===" +# Console-log diagnostic put the hang at ~1.16 s; 8 s leaves a wide margin and +# the PC sweep already proved the freeze is permanent (0 progress over 20 s), +# so a single halt is sufficient -- no need to sweep offsets again. +reset run +sleep 8000 + +targets rp2350.cm0 +poll +# Tolerate a halt that never completes (lockup / bus stall): the memory reads +# below go through the debug MEM-AP and do not require the core to be halted. +catch {halt} +poll + +echo "" +echo "=== CORE REGISTERS (cm0) -- expect pc ~= fs_open (0x101864b6) ===" +catch { + reg pc + reg lr + reg sp + reg msp + reg psp + reg xpsr + reg r0 + reg r1 + reg r2 + reg r3 +} + +echo "" +echo "=== FAULT / DEBUG STATUS (Cortex-M SCB, architectural addrs) ===" +echo " non-zero CFSR/HFSR => a fault escalated; DHCSR bit19 (0x00080000)" +echo " S_LOCKUP => core is locked up, PC is stale (rules out the fs-lock" +echo " theory outright)." +rd "ICSR " 0xE000ED04 +rd "SHCSR " 0xE000ED24 +rd "CFSR " 0xE000ED28 +rd "HFSR " 0xE000ED2C +rd "DFSR " 0xE000ED30 +rd "MMFAR " 0xE000ED34 +rd "BFAR " 0xE000ED38 +rd "DHCSR " 0xE000EDF0 + +echo "" +echo "=== QMI / XIP PERIPHERAL STATE -- is XIP still restored after erase? ===" +echo " QMI_DIRECT_CSR bit0 (EN, 0x1) SET => still in direct/serial mode, XIP" +echo " NOT re-enabled -> next flash fetch stalls forever == the wedge. A" +echo " clobbered M0_RFMT/M0_RCMD means the XIP read cmd config was lost." +rd "QMI_DIRECT_CSR" 0x400D0000 +rd "QMI_M0_TIMING " 0x400D000C +rd "QMI_M0_RFMT " 0x400D0010 +rd "QMI_M0_RCMD " 0x400D0014 +rd "XIP_CTRL " 0x400C8000 +rd "XIP_STAT " 0x400C8008 + +echo "" +echo "=== STACK DUMP for hand-unwind (SRAM only) ===" +echo " Return addresses appear as 0x1018_xxxx / 0x1010_xxxx words on the" +echo " stack; resolve them against zephyr.elf to reconstruct the call chain" +echo " below fs_open (mount? create? a k_mutex_lock wait?). SP was a rock" +echo " stable 0x20034410 across the whole PC sweep, so dump a fixed window" +echo " from just below it (does not require the live reg read to succeed)." +catch { reg sp } +mdw 0x20034380 96 + +echo "" +echo "=== done ===" +catch { resume } +shutdown From 07a78c79a467f6186223c2f3de84daa8ed50544f Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 16:08:53 -0700 Subject: [PATCH 14/29] debug(ci): capture reg/stack/mask output in hang forensics (v2) Run 30051402310 confirmed the hang is NOT a fault or lockup (CFSR/HFSR/DFSR=0, DHCSR S_LOCKUP=0, S_SLEEP=0) and NOT a wedged flash/XIP interface (XIP enabled, QMI_M0_RCMD=0xeb intact, DIRECT_CSR.EN=0) -- ICSR.ISRPENDING=1 with interrupts unserviced points at a spin holding irq_lock/a spinlock, which starves the USB IRQ and drops the CDC link. But the core-register and stack-dump blocks came back empty: bare reg/mdw output does not reach the CI step stdout in batch mode, only echo/capture does. Wrap those reads via a `dump` helper, and add PRIMASK/BASEPRI/FAULTMASK/CONTROL plus r4-r7 so the next run yields the stack unwind (which call chain holds the lock) and the masking mechanism. --- scripts/diag/hang_forensics.tcl | 37 +++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/scripts/diag/hang_forensics.tcl b/scripts/diag/hang_forensics.tcl index 0f7cea35..8d4bbaaf 100644 --- a/scripts/diag/hang_forensics.tcl +++ b/scripts/diag/hang_forensics.tcl @@ -36,6 +36,18 @@ proc rd {label addr} { } } +# Bare `reg`/`mdw` output does NOT reach the CI step stdout in batch (-c/-f) +# mode -- only `echo` and `capture` do (see run 30051402310, where the core-reg +# and stack blocks came back empty). Wrap every such command so its output is +# echoed into the captured log. +proc dump {cmd} { + if {[catch {set out [capture $cmd]} err]} { + echo " \[$cmd]: " + } else { + echo " \[$cmd]: [string trim $out]" + } +} + init echo "=== reset + free-run so boot reaches the hang ===" @@ -54,17 +66,16 @@ poll echo "" echo "=== CORE REGISTERS (cm0) -- expect pc ~= fs_open (0x101864b6) ===" -catch { - reg pc - reg lr - reg sp - reg msp - reg psp - reg xpsr - reg r0 - reg r1 - reg r2 - reg r3 +foreach r {pc lr sp msp psp xpsr r0 r1 r2 r3 r4 r5 r6 r7} { + dump "reg $r" +} + +echo "" +echo "=== INTERRUPT MASKING -- what is holding IRQs off? ===" +echo " PRIMASK=1 => irq_lock via PRIMASK; BASEPRI!=0 => Zephyr BASEPRI mask." +echo " Confirms the ISRPENDING-but-unserviced spin seen in run 30051402310." +foreach r {primask basepri faultmask control} { + dump "reg $r" } echo "" @@ -100,8 +111,8 @@ echo " stack; resolve them against zephyr.elf to reconstruct the call chain" echo " below fs_open (mount? create? a k_mutex_lock wait?). SP was a rock" echo " stable 0x20034410 across the whole PC sweep, so dump a fixed window" echo " from just below it (does not require the live reg read to succeed)." -catch { reg sp } -mdw 0x20034380 96 +dump "reg sp" +dump "mdw 0x20034380 96" echo "" echo "=== done ===" From b336f5f741eba520689d105828c7edc6545ba650 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 22:24:56 -0700 Subject: [PATCH 15/29] debug(ci): GDB DWARF backtrace at the /keys hang (v3); record forensics The register/QMI/stack capture (runs 30051402310 + 30052303346) ruled out a CPU fault, a lockup, and a wedged flash/XIP interface (XIP enabled, QMI RCMD=0xeb intact) -- so the littlefs->NVS rework is NOT provably futile -- but showed the core spinning with interrupts masked via PRIMASK=1, an off-boundary PC inside fs_open, an incoherent LR (idle), and a shallow garbage stack. That is a software control-flow failure (wild jump or a panic-spin), not the clean fs mutex block the rework was pitched against. v3 attaches arm-zephyr-eabi-gdb (or gdb-multiarch/arm-none-eabi-gdb, or the Zephyr SDK gdb) to the OpenOCD gdb server for a real DWARF backtrace, disassembly around PC, and single-stepping, to decide wild-jump vs. __ASSERT/SPIN_VALIDATE panic vs. lock-spin -- which determines the fix. Read-only; skips gracefully if no gdb is on the integration runner. Uses the flight ELF from the build artifact (build-artifacts/zephyr/fprime-zephyr-deployment). INVESTIGATION.md: add the forensics section, caveat the earlier NVS-rework recommendation, and update the diagnostic-history table + branch-state note. --- .github/workflows/ci.yaml | 8 +++ INVESTIGATION.md | 113 ++++++++++++++++++++++++++++++++++++++ scripts/diag/hang_gdb.sh | 99 +++++++++++++++++++++++++++++++++ 3 files changed, 220 insertions(+) create mode 100755 scripts/diag/hang_gdb.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f64ad7a6..8d618e58 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -141,6 +141,14 @@ jobs: -c "adapter speed 5000" \ -f "$GITHUB_WORKSPACE/scripts/diag/hang_forensics.tcl" || true + # TEMPORARY DIAGNOSTIC (remove once root-caused): v3 -- attach GDB to the + # OpenOCD gdb server for a DWARF backtrace + single-step, to decide + # wild-jump vs. panic vs. lock-spin at the /keys hang. Read-only; skips + # gracefully if no arm/multiarch gdb is on the runner. See INVESTIGATION.md + # and scripts/diag/hang_gdb.sh. + - name: Hang Forensics GDB Backtrace + run: bash "$GITHUB_WORKSPACE/scripts/diag/hang_gdb.sh" || true + - name: Load .env file run: | while IFS= read -r line || [ -n "$line" ]; do diff --git a/INVESTIGATION.md b/INVESTIGATION.md index 27845857..80ce5dba 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -298,6 +298,119 @@ bug entirely regardless of which exact mechanism is at fault, since NVS doesn't take a global fs lock and its record writes are simple, bounded `flash_area_write`/`erase` calls with well-understood timing. +One thing that's **ruled out** as the mechanism: a stale +`PICO_FLASH_SIZE_BYTES` hard_assert in the Pico SDK's +`flash_range_erase`/`flash_range_program` (`hard_assert(flash_offs + count <= +PICO_FLASH_SIZE_BYTES)`) — that macro is not defined anywhere in this Zephyr +build (confirmed via grep across `lib/zephyr-workspace/zephyr/` and the +generated build's compile flags), so the `#ifdef`-guarded assert is compiled +out entirely and cannot be firing. + +### Recommended fix + +> **Superseded / caveated by the SWD register forensics below (runs +> 30051402310 + 30052303346).** Those runs **ruled out** a wedged flash/XIP +> interface — the scenario in which switching to `flash_area_*`/NVS would have +> been wasted effort — but they also showed the hang is a *software* +> control-flow failure (interrupts masked via `PRIMASK`, an off-boundary PC, +> starved IRQ), not a clean littlefs mutex block. So "avoid the littlefs +> format path" is no longer established as *the* fix; the exact software cause +> is still being pinned (GDB backtrace, v3). Read the forensics section before +> acting on the recommendation below. + +Regardless of which exact mechanism is at fault, both candidates above point +at the same remedy: **avoid littlefs's mount/format/first-file-create path on +this partition entirely.** Replace the `/keys` littlefs mount with raw +`flash_area_*` calls or the Zephyr NVS backend for this fixed-size key store + +sequence-number counter. This is a nontrivial rework of +`TcSecurityDeframer`'s `loadKeyStore()`/`writeKeyStore()`/`writeSequenceNumber()` +(currently built on Zephyr's `fs_open`/`fs_read`/`fs_write` POSIX-ish file +API) to instead use `flash_area_open`/`flash_area_read`/`flash_area_write`/ +`flash_area_erase` directly against `keystore_partition`, or to adopt the NVS +subsystem's key-value API instead. Either avoids the littlefs mount/format +path entirely. **Not yet implemented.** + +### Diagnostic history (all reverted except the flash-size fix; CI is currently back to pre-diagnostic state otherwise) + +| run | change | result | reverted commit | +|-----|--------|--------|------------------| +| 30026433728 | (baseline, pre-fix) | fail, same USB-disconnect symptom | — | +| 30034861047 | flash0 `DT_SIZE_M(4)` → `DT_SIZE_M(16)` (kept, not reverted) | fail, same symptom; confirmed chip is 16MB | n/a — this is the real fix | +| 30036653676 | `CONFIG_CONSOLE`/`CONFIG_LOG` enabled | fail (expected); boot log silent after ~1.16s | `3cdfa541` | +| 30043983799 | SWD PC-sweep at t=+2s/+10s/+20s | fail (expected); cm0 frozen at `fs_open+2` throughout | `59b8178c` | +| 30051402310 | SWD forensics: SCB fault regs + QMI/XIP state (`hang_forensics.tcl`) | fail (expected); no fault/lockup, XIP healthy, `ISRPENDING`=1 — see forensics section | pending | +| 30052303346 | SWD forensics v2: capture-wrapped reg/stack + PRIMASK/BASEPRI | fail (expected); `PRIMASK`=1, off-boundary PC, shallow garbage stack | pending | + +The `hmac-to-storage` branch currently carries the flash-size fix (kept) plus +the **read-only SWD forensics diagnostic** (`scripts/diag/hang_forensics.tcl` +and a `Hang Forensics Diagnostic` step in `integration-uart`) — this one is +still active (not reverted) because the GDB-backtrace follow-up (v3) builds on +it. It must be reverted before merge, same as the earlier diagnostics. The +prior console/PC-sweep diagnostics remain cleanly reverted. + +## SWD register/QMI/stack forensics (runs 30051402310 + 30052303346, 2026-07-24) + +Read-only SWD capture (`scripts/diag/hang_forensics.tcl`): reset, free-run 8 s +so boot reaches the hang, halt cm0 once, and read the core registers, the +Cortex-M fault status block, the QMI/XIP peripheral state, and an SRAM stack +window. Deliberately reads **no** `0x10xx_xxxx` (XIP flash) address over SWD — +if the flash interface were wedged, such a read could stall the adapter. Two +runs: v1 got the fault/QMI reads; v2 fixed a capture bug (bare `reg`/`mdw` +output does not reach CI stdout in batch mode — only `echo`/`capture` does) and +added the register/stack dump plus the interrupt-mask registers. + +**Two hardware hypotheses ruled out (confirmed on both runs):** + +- **Not a CPU fault or lockup.** `CFSR = HFSR = DFSR = 0`; `DHCSR = 0x00130003` + → `S_HALT=1` but `S_LOCKUP=0` and `S_SLEEP=0` (not locked up, not in WFI). +- **Not a wedged flash/XIP interface** — this is the scenario in which a + littlefs→`flash_area_*`/NVS rewrite would have been *wasted*, because NVS + hits the same `flash_range_erase`. Ruled out: `XIP_CTRL=0x00000083` (XIP + enabled), `QMI_M0_RCMD=0x000000eb` (quad-read `0xEB` command intact), + `QMI_DIRECT_CSR=0x00c10800` (EN=0, BUSY=0 — not stuck in a direct/serial + transaction). The flash controller is idle and healthy. + +**Also ruled out:** SMP/second-core bringup. `CONFIG_MP_MAX_NUM_CPUS=1` — this +is a single-core build, so the `idle.c` SMP-without-IPI spin path is not +compiled and cm1 sitting in the bootrom (per the PC sweep) is expected and +irrelevant. + +**What the state actually is (v2, run 30052303346):** + +| register | value | reading | +|----------|-------|---------| +| `PRIMASK` | `0x01` | IRQs masked by `cpsid i` — **not** Zephyr's normal `irq_lock` (which uses `BASEPRI`, here `0x00`) | +| `ICSR` | `0x00400000` | `ISRPENDING=1`: an IRQ is pending and **starved** — this is what drops the USB CDC link | +| `control` | `0x02` | privileged thread on PSP | +| `pc` | `0x101864b8` | **not an instruction boundary** — `fs_open` opens with a 4-byte `stmdb` at `0x101864b6`; `…b8` is the second halfword, *inside* it | +| `lr` / `r5`| `0x1010fb69` (`idle+0x21`) / `0x1010fb49` (`&idle`) | but `idle()` never calls `fs_open` — incoherent as a live frame | +| `r1` | `0x0` | if this were `fs_open`'s `file_name` arg it is NULL, which returns `-EINVAL` immediately (`fs.c:145`) — can't hang *inside* `fs_open` | +| stack > SP | 2 words (`k_is_pre_kernel` ×2) then uninitialized garbage | shallow, not a coherent call chain | + +The earlier "`fs_open+2` = its first real instruction" reading (PC-sweep +section) was **wrong**: `0x101864b8` is mid-`stmdb`, not an instruction +boundary. Taken together — IRQs masked via `PRIMASK`, an off-boundary PC, an +incoherent LR, a shallow garbage stack, and no CPU fault — this is **not** a +clean mutex/lock block. It reads as execution gone off the rails: a +wild/corrupted PC, or a software panic-spin (`arch_system_halt()` also does +`cpsid`+infinite-loop with no CPU fault set), reached right when the keystore +feature first touches the filesystem. + +**Net for the fix decision:** the failure is a *firmware control-flow* bug, not +a flash/XIP hardware wedge — so this is not the case where moving off littlefs +is provably futile. But it is also not the clean fs-lock the rework was pitched +against, so the rework is not yet established as *the* fix either. The exact +software cause (wild jump vs. a tripped `__ASSERT`/`SPIN_VALIDATE` panic — +`CONFIG_ASSERT=y`, `CONFIG_SPIN_VALIDATE=y` are both set — vs. a lock-spin) +needs one more probe. + +**Next diagnostic (v3, in progress):** attach `arm-zephyr-eabi-gdb` to the +OpenOCD gdb server (port 3333, already opened by the same step) for a real +DWARF backtrace + `info threads` (`_current` thread) + a few single-steps to +see whether the PC advances and whether a `z_fatal_error`/`arch_system_halt` +frame is present. That distinguishes panic vs. wild-jump vs. lock-spin +directly and decides the fix. + ## Original next steps (superseded above, kept for the 4 MB fallback) Everything below was written before hardware confirmation; kept for diff --git a/scripts/diag/hang_gdb.sh b/scripts/diag/hang_gdb.sh new file mode 100755 index 00000000..7ac6bbcd --- /dev/null +++ b/scripts/diag/hang_gdb.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash +# hang_gdb.sh -- v3 of the /keys boot-hang forensics (see INVESTIGATION.md). +# +# The register/QMI/stack capture (hang_forensics.tcl) ruled out a CPU fault, a +# lockup, and a wedged flash/XIP interface, and showed the core spinning with +# interrupts masked (PRIMASK=1), an off-boundary PC inside fs_open, an +# incoherent LR (idle), and a shallow garbage stack -- i.e. a software +# control-flow failure, not a clean fs mutex block. This step attaches GDB to +# the OpenOCD gdb server for a real DWARF backtrace + single-step, to decide +# between: (a) a wild/corrupted PC, (b) a Zephyr __ASSERT/SPIN_VALIDATE panic +# (a z_fatal_error/arch_system_halt frame would show), or (c) a genuine +# lock-spin. That distinction determines whether the littlefs->NVS rework is +# the right fix or a red herring. +# +# Read-only: halts and inspects, never writes flash or changes config. Safe to +# read code over SWD now that v2 confirmed XIP is healthy. Best-effort: if no +# suitable GDB is found on the runner it prints a notice and exits 0 (the CI +# step is `|| true` anyway) -- the register capture already stands on its own. +set -u + +OCD=~/openocd/src/openocd +OCD_ARGS=(-s ~/openocd/tcl + -f ~/openocd/tcl/interface/cmsis-dap.cfg + -f ~/openocd/tcl/target/rp2350.cfg + -c "adapter speed 5000") + +# --- locate the symbol ELF (flight-software deployment, load addrs 0x1018_xxxx) +ELF="" +for cand in \ + build-artifacts/zephyr/fprime-zephyr-deployment \ + "$GITHUB_WORKSPACE"/build-artifacts/zephyr/fprime-zephyr-deployment; do + [ -f "$cand" ] && { ELF="$cand"; break; } +done +if [ -z "$ELF" ]; then + echo "hang_gdb: symbol ELF not found (build-artifacts/zephyr/fprime-zephyr-deployment); skipping." + exit 0 +fi +echo "hang_gdb: using ELF $ELF" + +# --- locate an ARM / multiarch GDB +GDB="" +for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do + command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } +done +if [ -z "$GDB" ]; then + GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | head -1) +fi +if [ -z "$GDB" ]; then + echo "hang_gdb: no arm/multiarch gdb on this runner (tried arm-zephyr-eabi-gdb," + echo " gdb-multiarch, arm-none-eabi-gdb, ~/zephyr-sdk*). Skipping backtrace." + exit 0 +fi +echo "hang_gdb: using GDB $GDB" + +# --- start OpenOCD: reset, free-run to the hang, halt, then KEEP the gdb server +# alive (no `shutdown`) so GDB can attach to the halted core. +"$OCD" "${OCD_ARGS[@]}" \ + -c "init" \ + -c "reset run" \ + -c "sleep 8000" \ + -c "targets rp2350.cm0" \ + -c "halt" \ + -c "echo {hang_gdb: core halted, gdb server ready on :3333}" & +OCD_PID=$! +trap 'kill "$OCD_PID" 2>/dev/null' EXIT +# wait past the 8s free-run + halt before connecting +sleep 12 + +"$GDB" -q -nx -batch "$ELF" \ + -ex "set pagination off" \ + -ex "set confirm off" \ + -ex "target extended-remote localhost:3333" \ + -ex "echo \n==== core registers ====\n" \ + -ex "info registers" \ + -ex "echo \n==== masking / psr ====\n" \ + -ex "info registers primask basepri faultmask control xpsr" \ + -ex "echo \n==== DWARF backtrace (the decisive read) ====\n" \ + -ex "bt -full" \ + -ex "echo \n==== frame 0 detail ====\n" \ + -ex "info frame" \ + -ex "echo \n==== disassembly around PC ====\n" \ + -ex "x/16i \$pc-16" \ + -ex "echo \n==== single-step x8: does the PC advance, and to where? ====\n" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ + -ex "echo \n==== backtrace after stepping ====\n" \ + -ex "bt" \ + -ex "echo \n==== threads (openocd hwthread view) ====\n" \ + -ex "info threads" \ + -ex "detach" \ + || echo "hang_gdb: gdb session returned non-zero (see output above)" + +echo "hang_gdb: done" From a302cdec870206d5be996d0545eb8ee679f9e140 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Thu, 23 Jul 2026 23:23:10 -0700 Subject: [PATCH 16/29] docs: record hardware-confirmed root cause (UsageFault -> fatal-halt spin -> dead SysTick) Reproduced on a local board via SWD (raspberrypi openocd + zephyr-sdk gdb, matching local zephyr.elf). Definitive chain, read off the hung target: wild jump to 0x20010480 (inside FileHandling::fileManager data; lr=0x1019 garbage) -> UsageFault (z_arm_usage_fault/z_arm_fault) -> non-recoverable -> z_arm_fatal_error fatal-halt loop "msr BASEPRI_MAX; b ." (PC pinned across 16 halts + 30 single-steps, primask=1 basepri=0x10, MSP/handler) -> SysTick ISR starved -> cycle_count/curr_tick frozen at ~8.1s -> rate-group k_timer_status_sync never ticks (main parked in arch_swap) -> no telemetry -> GDS "device disconnected". Board confirmed silent 38s (>30s cadence). This supersedes the littlefs-format, flash/XIP-wedge, and interrupt-stall theories and the recommended littlefs->NVS rework -- none address the actual defect. Open: the source of the corrupted pointer (stack overflow / garbage F-prime handler pointer / overrun near fileManager). The flash-size fix stays. --- INVESTIGATION.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/INVESTIGATION.md b/INVESTIGATION.md index 80ce5dba..ca12b153 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -18,6 +18,56 @@ symptom is **not yet confirmed on hardware** — see "Next steps". --- +## ✅ ROOT CAUSE CONFIRMED ON HARDWARE (2026-07-24) — supersedes every hypothesis below + +Reproduced on a local board (SWD via the raspberrypi OpenOCD fork + Zephyr-SDK +GDB, symbols from the matching local `zephyr.elf`). The board is in the failed +state: **no F' telemetry for 38 s** (spanning the 30 s default cadence). The +fault chain, read directly off the hung target, is: + +1. A worker context (unwinds through `FileHandling::fileDownlink`, faulting + `pc = 0x20010480`) **branches through a corrupted pointer into a RAM/data + address** (`0x20010480` is *inside* the `FileHandling::fileManager` object, + not code; stacked `lr = 0x1019` is garbage). Executing a data address raises + a **UsageFault** (`z_arm_usage_fault` -> `z_arm_fault`, `fault.c:1090`). +2. Zephyr's fault handler judges it **non-recoverable** (`recoverable = false`) + and calls `z_arm_fatal_error`, which lands in the fatal-halt loop: + ``` + msr BASEPRI_MAX, r3 ; mask interrupts + isb + b . ; spin forever (addr2line mislabels this "get_fat") + ``` + Confirmed stable: PC pinned here across 16 halts + 30 single-steps + (`b .` self-branch), `primask=1`, `basepri=0x10`, MSP/handler mode. +3. The masked spin means the **SysTick ISR never runs** -> the system clock is + **frozen** (`cycle_count`/`curr_tick` byte-identical 4 s apart, stuck at + ~8.1 s uptime = when the fault fired). +4. With no ticks, the F' rate-group loop `startRateGroups()` -> + `timer.cycle()` -> **`k_timer_status_sync()` never returns** (main thread + parked in `arch_swap`, confirmed by walking `_kernel.threads`). Rate groups + never cycle -> no telemetry -> the GDS/CI `comm.py` sees + "device disconnected / no data". **This is the CI symptom.** + +**What this means for the prior analysis:** the flash-size fix +(`DT_SIZE_M(4)`->`16`) was real and correct, but everything downstream of it in +this document — the littlefs mount/format theory, the interrupt-stall-from- +flash-erase theory, the "wild PC at `fs_open`" reading (the CI PC-sweep almost +certainly caught this *same* `b .` fatal-halt loop, just mislabeled `fs_open` +in the CI build the way it's mislabeled `get_fat` locally), and the +**recommended littlefs->NVS rework** — are **not the bug** and would not fix it. +The SWD "masked-IRQ spin" reading (`PRIMASK=1`, `ISRPENDING=1`) was pointing at +this fatal-halt loop, not a flash lock. + +**Still open (the actual defect):** *why* control jumps to `0x20010480` — a +wild/corrupted function pointer or return address (candidates: a stack overflow +in a worker thread, an uninitialized/garbage F' port or handler pointer, or a +buffer overrun near the `fileManager` instance). That corruption — not the +filesystem — is what must be fixed. The fatal-halt-with-interrupts-masked +behavior is also worth revisiting (it converts one fault into a total, silent +system death), but it is the symptom amplifier, not the cause. + +--- + ## Finding 1 — Reads do NOT disable interrupts (contradicts PROBLEM.md hypothesis #2) `lib/zephyr-workspace/zephyr/drivers/flash/flash_rpi_pico.c`: From ec0bdb37afdecc878a32757d1298471ec2f77ef0 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Tue, 28 Jul 2026 01:16:42 +0200 Subject: [PATCH 17/29] fix: unblock CI boot failure and make the /keys key store real Two independent defects, both confirmed on hardware with single-variable A/B tests (full clean `make generate build` each side, measuring F' telemetry off the board CDC). 1. CommandDispatcher opcode-table overflow -- the CI blocker. CMD_DISPATCHER_DISPATCH_TABLE_SIZE was 350 against a deployment already at 348 commands. This branch adds PROVISION_KEY/ADD_KEY/REMOVE_KEY to TcSecurityDeframer, and there are two instances (ComCcsdsUart, ComCcsdsLora), so 3 x 2 = 6 new commands -> 354 > 350. CommandDispatcherImpl.cpp:35 FW_ASSERTs when the RedBlackTreeMap insert fails, so the 351st registration panics the board during boot (z_fatal_error(reason=4) via z_arm_svc -- a k_panic, not a CPU fault, which is why every fault-vector probe found nothing). The downlink never starts, GDS reports "device disconnected", integration-uart/radio fail. Raised to 512 for headroom. Measured: main 1328 bytes @t+1.0s; branch as shipped 0; branch + table 512 1392 bytes @t+1.01s; branch + table back to 350 again 0. 2. littlefs was never compiled in -- the feature was inert. west.yml's name-allowlist imported fatfs but not littlefs, so the module never reached zephyr_modules.txt, ZEPHYR_LITTLEFS_MODULE was undefined, and Kconfig silently dropped CONFIG_FILE_SYSTEM_LITTLEFS=y. No lfs_* symbols in the image, /keys never existed, every fs_open("/keys/...") failed -- while the build stayed clean. Added to the allowlist and pinned as an explicit project so it lands under lib/zephyr-workspace/ rather than the workspace topdir. Verified end to end on hardware: /keys mounts and formats on internal flash; PROVISION_KEY succeeds on a keyless board over the bypass-allowlisted link; the key survives a cold reboot (re-read by loadKeyStore()); the flash-stored key authenticates uplink (SET_SEQ_NUM took effect); the sequence number survives a cold reboot; erasing the partition re-formats cleanly. The PROVISION_KEY opcodes were re-verified against a fresh dictionary and match Bypasser.cpp. Also adds the SWD/GDB diagnostics used to find this (scripts/diag/) and rewrites INVESTIGATION.md around the confirmed cause -- its earlier conclusions (wild jump, stack overflow, fs-lock deadlock, flash/XIP wedge, littlefs->NVS rework) were all wrong and are marked as such. Follow-ups tracked in TODO.md, including removing these diagnostics before merge and promoting the useful ones to an ADR. --- INVESTIGATION.md | 400 +++++++++++++++--- .../config/CommandDispatcherImplCfg.hpp | 9 +- TODO.md | 156 ++++++- scripts/diag/downlink_trace.gdb | 87 ++++ scripts/diag/downlink_trace.sh | 111 +++++ scripts/diag/hang_fault_bp.gdb | 102 +++++ scripts/diag/hang_fault_bp.sh | 130 ++++++ scripts/diag/hang_thread_walk.gdb | 377 +++++++++++++++++ scripts/diag/hang_thread_walk.sh | 126 ++++++ west.yml | 16 +- 10 files changed, 1450 insertions(+), 64 deletions(-) create mode 100644 scripts/diag/downlink_trace.gdb create mode 100755 scripts/diag/downlink_trace.sh create mode 100644 scripts/diag/hang_fault_bp.gdb create mode 100755 scripts/diag/hang_fault_bp.sh create mode 100644 scripts/diag/hang_thread_walk.gdb create mode 100755 scripts/diag/hang_thread_walk.sh diff --git a/INVESTIGATION.md b/INVESTIGATION.md index ca12b153..ae73a47e 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -1,70 +1,348 @@ -# Investigation: hmac-to-storage CI hardware failure — root-cause re-analysis +# Investigation: hmac-to-storage CI hardware failure -Follow-up to `PROBLEM.md`. This document records a static re-analysis of the -`integration-uart` / `integration-radio` failure on branch `hmac-to-storage` -(PR #472), tracing the device tree, the RP2350 flash driver, the littlefs -automount path, and the `TcSecurityDeframer` hot path. +## ROOT CAUSE — CONFIRMED ON HARDWARE (2026-07-27), single-variable A/B -**Headline conclusion:** the "internal-flash writes disable interrupts and stall -USB" hypothesis in `PROBLEM.md` is very likely a phantom. The keystore littlefs -partition is placed **beyond the declared flash boundary**, so every keystore -flash operation is rejected with `-EINVAL` *before* any interrupt is disabled, -and `/keys` can never mount. The most probable real fix is a one-line flash-size -correction in the device tree. +**The branch overflows the CommandDispatcher opcode table.** -Confidence: the partition-out-of-bounds and read-vs-write facts below are -**code-confirmed**. Whether the mount failure is the *sole* cause of the USB -symptom is **not yet confirmed on hardware** — see "Next steps". +`PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp` +sized `CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350` against a deployment that +already had **348** commands. This branch adds `PROVISION_KEY`, `ADD_KEY` and +`REMOVE_KEY` to `TcSecurityDeframer`, and there are **two** deframer instances +(`ComCcsdsUart`, `ComCcsdsLora`) — 3 x 2 = **6 new commands, total 354 > 350**. + +`CommandDispatcherImpl::compCmdReg_handler` inserts every opcode into a +fixed-capacity `Fw::RedBlackTreeMap` and asserts on failure: + +```c +const Fw::Success status = this->m_entryTable.insert(opCode, portNum); +FW_ASSERT(status == Fw::Success::SUCCESS, ...); // CommandDispatcherImpl.cpp:35 +``` + +So the 351st registration **panics the board during boot command registration**. +The downlink never comes up, GDS sees `device disconnected` / no response to +`CMD_NO_OP`, and `integration-uart` / `integration-radio` fail. There is no CPU +fault — it is a `k_panic` (`z_fatal_error(reason=4)` via `z_arm_svc`), which is +why every fault-vector probe in this document came up empty. + +### Evidence (local bench, board `/dev/tty.usbmodem1101`, probe `/dev/tty.usbmodem102`) + +Bytes of F' telemetry read from the board CDC in a 25 s window after flashing, +each a full clean `make generate build`: + +| build | telemetry | +|---|---| +| `main` | **1328 bytes**, first at t+1.0 s | +| `hmac-to-storage` (as shipped) | **0 bytes** | +| `hmac-to-storage` + table 512 | **1392 bytes**, first at t+1.01 s | +| `hmac-to-storage` + table back to 350 | **0 bytes** | + +The last two differ **only** in that one constant, both fully regenerated — this +is the root cause, not a correlate. + +### The fix + +``` +CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350 -> 512 +``` + +Raised to 512 rather than 354 so the next few commands do not repeat this. Note +the same zero-headroom pattern elsewhere in `project/config`: +`MAX_PACKETIZER_CHANNELS = 202` vs 191 channels in use, and +`MAX_PACKETIZER_PACKETS = 22` vs exactly 22 packets. Any of these overflowing +fails the same way — a boot-time `FW_ASSERT`, not a graceful degradation. +**Adding commands, channels or packets to this deployment requires checking +these constants against the generated dictionary.** + +--- + +## Second, independent defect found and fixed: littlefs was never compiled in + +`prj.conf` sets `CONFIG_FILE_SYSTEM_LITTLEFS=y`, but the repo's `west.yml` +`name-allowlist` imports `fatfs` and **not `littlefs`**. Without the module in +`zephyr_modules.txt`, `ZEPHYR_LITTLEFS_MODULE` is undefined and Kconfig +**silently drops** the symbol (`# LittleFS module not available.`). The build +stayed clean and `/keys` simply never existed: no `lfs_*` symbols in the image, +every `fs_open("/keys/...")` in `TcSecurityDeframer` failing, the whole key-store +feature inert. + +Fixed by adding `littlefs` to the allowlist **and** pinning it as an explicit +project (like every other module here) so it lands under `lib/zephyr-workspace/` +instead of the workspace topdir. Verified: `CONFIG_FILE_SYSTEM_LITTLEFS=y`, +`CONFIG_FS_LITTLEFS_FSTAB_AUTOMOUNT=y`, 20 `lfs_*` symbols in the ELF. + +This was *not* the cause of the CI failure (the branch failed identically with +littlefs absent and with it present) but the feature cannot work without it. + +## Build-system trap that cost several hours + +**`make build` does not re-derive Kconfig from device-tree changes.** After +editing the board `.dtsi`, `make build` left a stale `CONFIG_FLASH_SIZE=4096` +while the DTS said 16 MB, putting `keystore_partition` out of bounds so +`get_block_size()` returned 0 and the `/keys` automount hit +`__ASSERT_NO_MSG(block_size != 0)` (`littlefs_fs.c:787`) — a boot panic that was +purely an artifact of the stale config. **Always `make generate build` when the +device tree changes.** Several intermediate bisect results in this session were +invalidated by this and had to be re-run. + +## Diagnostics built this session (remove before merge) + +- `scripts/diag/hang_thread_walk.{sh,gdb}` — reset, free-run, halt twice N s + apart; dumps clock/SysTick/interrupt state, walks `_kernel.threads` (naming + each F' task by symbolizing `entry.parameter1`, since `CONFIG_THREAD_NAME` is + off), dumps `timeout_list` and the whole downlink chain's state, and diffs the + two halts using each thread's monotonic `base.usage.total`. +- `scripts/diag/downlink_trace.{sh,gdb}` — breakpoints every hop of the + `comStub -> framer -> aggregator -> spacePacketFramer -> comQueue` status path. +- Pre-existing: `hang_forensics.tcl`, `hang_gdb.sh`, `hang_fault_bp.{sh,gdb}`. +- Stray capture files to delete: `board-serial.log`, + `hang-thread-walk-openocd.log`, `downlink-trace-openocd.log`. + +## Corrections to earlier conclusions in this document + +Everything below this line predates the hardware A/B and is **wrong in its +conclusions**, kept only for the audit trail: + +- There is **no hang, no wild jump, no stack overflow, no fs-lock deadlock and + no flash/XIP wedge.** With the failure present the system is fully healthy: + clock advancing, all three rate groups cycling, ~89% idle. Only the downlink + is dead. +- `PRIMASK=1` at `arch_cpu_idle+18` is the **normal** `cpsid i; wfi; cpsie i` + idle sequence, not a masked spin. +- `pc=0x101864b8` "inside `fs_open`" and `pc=0x20010480` "inside `fileManager`" + were artifacts — a stale FPB breakpoint, and callee-saved registers misread as + a PC. (A swapped-out Cortex-M thread's `callee_saved.psp` points straight at + the exception frame: stacked LR at +0x14, PC at +0x18. Reading +0x34/+0x38 + yields F' object addresses that look exactly like plausible wild pointers.) +- The `DT_SIZE_M(4) -> DT_SIZE_M(16)` flash-size fix is correct and necessary + (the chip really is a 16 MB w25q128), but it was never the CI blocker. +- The recommended littlefs -> NVS rework is **not** needed for this failure. --- -## ✅ ROOT CAUSE CONFIRMED ON HARDWARE (2026-07-24) — supersedes every hypothesis below - -Reproduced on a local board (SWD via the raspberrypi OpenOCD fork + Zephyr-SDK -GDB, symbols from the matching local `zephyr.elf`). The board is in the failed -state: **no F' telemetry for 38 s** (spanning the 30 s default cadence). The -fault chain, read directly off the hung target, is: - -1. A worker context (unwinds through `FileHandling::fileDownlink`, faulting - `pc = 0x20010480`) **branches through a corrupted pointer into a RAM/data - address** (`0x20010480` is *inside* the `FileHandling::fileManager` object, - not code; stacked `lr = 0x1019` is garbage). Executing a data address raises - a **UsageFault** (`z_arm_usage_fault` -> `z_arm_fault`, `fault.c:1090`). -2. Zephyr's fault handler judges it **non-recoverable** (`recoverable = false`) - and calls `z_arm_fatal_error`, which lands in the fatal-halt loop: - ``` - msr BASEPRI_MAX, r3 ; mask interrupts - isb - b . ; spin forever (addr2line mislabels this "get_fat") - ``` - Confirmed stable: PC pinned here across 16 halts + 30 single-steps - (`b .` self-branch), `primask=1`, `basepri=0x10`, MSP/handler mode. -3. The masked spin means the **SysTick ISR never runs** -> the system clock is - **frozen** (`cycle_count`/`curr_tick` byte-identical 4 s apart, stuck at - ~8.1 s uptime = when the fault fired). -4. With no ticks, the F' rate-group loop `startRateGroups()` -> - `timer.cycle()` -> **`k_timer_status_sync()` never returns** (main thread - parked in `arch_swap`, confirmed by walking `_kernel.threads`). Rate groups - never cycle -> no telemetry -> the GDS/CI `comm.py` sees - "device disconnected / no data". **This is the CI symptom.** - -**What this means for the prior analysis:** the flash-size fix -(`DT_SIZE_M(4)`->`16`) was real and correct, but everything downstream of it in -this document — the littlefs mount/format theory, the interrupt-stall-from- -flash-erase theory, the "wild PC at `fs_open`" reading (the CI PC-sweep almost -certainly caught this *same* `b .` fatal-halt loop, just mislabeled `fs_open` -in the CI build the way it's mislabeled `get_fat` locally), and the -**recommended littlefs->NVS rework** — are **not the bug** and would not fix it. -The SWD "masked-IRQ spin" reading (`PRIMASK=1`, `ISRPENDING=1`) was pointing at -this fatal-halt loop, not a flash lock. - -**Still open (the actual defect):** *why* control jumps to `0x20010480` — a -wild/corrupted function pointer or return address (candidates: a stack overflow -in a worker thread, an uninitialized/garbage F' port or handler pointer, or a -buffer overrun near the `fileManager` instance). That corruption — not the -filesystem — is what must be fixed. The fatal-halt-with-interrupts-masked -behavior is also worth revisiting (it converts one fault into a total, silent -system death), but it is the symptom amplifier, not the cause. +## Local-bench result (2026-07-27) — v5 thread-walk probe: THERE IS NO HANG + +Built and ran the thread-walk probe INVESTIGATION.md called for +(`scripts/diag/hang_thread_walk.{sh,gdb}`): reset, free-run 12 s into the failed +state, halt, dump the clock/SysTick/interrupt state, walk every thread in +`_kernel.threads`, dump `kernel/timeout.c`'s `timeout_list`, then free-run 4 s +more and repeat, comparing the two halts. Board CDC `/dev/tty.usbmodem1101`, +Debug Probe `/dev/tty.usbmodem102`, current `zephyr.elf` (carries the +`DT_SIZE_M(16)` fix). + +**The firmware is not hung, not faulted, and not blocked. It is running +normally.** Measured across the two halts 4 s apart: + +| observation | value | +|---|---| +| `curr_tick` | 119896 → 159936 (**+40040 ticks = 4.004 s** at 10 kHz) | +| `cycle_count` | +600,603,820 (150 MHz, exactly 4 s) | +| `SYST_CSR` | `0x7` (ENABLE+TICKINT+CLKSOURCE), `SYST_CVR` advancing | +| `PRIMASK`/`BASEPRI`/`CFSR`/`HFSR` | `0`/`0`/`0`/`0` | +| `timeout_list` | 1 entry: `z_timer_expiration_handler`, `dticks=10` (the 1 ms base-rate `k_timer`, re-armed every ms) | +| thread CPU time (`base.usage.total`, summed) | +597,388,374 cycles in 4 s | + +Per-thread CPU consumed in that 4 s window (probe names each thread by +symbolizing `entry.parameter1`, since `CONFIG_THREAD_NAME` is off and all 21 +F′ tasks share the `zephyrEntryWrapper` entry symbol): + +``` +rateGroup50Hz 19.2M cmdSeq 2.2M fileManager 0.59M CdhCore::cmdDisp 0.19M +bg_thread_main 19.9M safeModeSeq 1.6M ComCcsdsLora::comQueue 0.41M +rateGroup1Hz 7.2M payloadSeq 1.6M fileDownlink 0.26M CdhCore::events 0.09M +rateGroup10Hz 5.3M ComCcsdsUart::aggregator 1.4M prmDb 0.16M CdhCore::tlmSend 0.08M +idle 537.2M (89% idle) fileUplink 0.13M +``` + +All three rate groups cycle, `CdhCore::tlmSend`/`events`/`cmdDisp` run, and the +main thread is looping through `startRateGroups()` → `timer.cycle()` → +`k_timer_status_sync()` exactly as designed. One halt caught the main thread +mid-`Svc::ActiveRateGroup::CycleIn_handlerBase` → `Os::Queue::send` to +`rateGroup50Hz`; another caught the CPU inside `sys_clock_isr` → +`z_timer_expiration_handler`. + +**Every prior "hang" reading was a misread of a healthy idle CPU.** `PRIMASK=1` ++ `pc=arch_cpu_idle+18` is not a masked spin — it is the normal +`cpsid i; wfi; cpsie i` idle sequence, halted at the `cpsie i`. `pc=0x101864b8` +"inside `fs_open`" and `pc=0x20010480` "inside `fileManager`" were likewise +artifacts (a stale FPB breakpoint, and callee-saved register values misread as +a PC). There is no wild jump, no stack overflow, no fs-lock deadlock, no +flash/XIP wedge, and no fatal-halt loop. **Items 1–4 of the "Next steps" below +are chasing a defect that does not exist.** + +### What IS broken: the downlink, not the system + +Three threads consumed **exactly zero** cycles in the 4 s window: + +- `usbd_thread` — parked in `k_msgq_get`, never woken +- `udc_rpi_pico_thread_0` — parked in `k_event_wait`, never woken +- `ComCcsdsUart::comQueue` — parked on its queue condvar, never woken, **while + `ComCcsdsUart::aggregator` on the same path burned 1.4M cycles** + +and the host sees **0 bytes in 15 s** on the board's CDC. (Checked against +`/dev/cu.usbmodem1101`, not `/dev/tty.*` — on macOS a `tty.` open blocks on +carrier detect and would produce a false "no data". The silence is real either +way.) + +So the CI symptom — GDS `device disconnected`, `CMD_NO_OP` never answered — is a +**dead USB CDC / UART downlink path on a fully healthy flight system**, not a +boot hang. That is where the investigation goes next: telemetry is aggregated +but never dequeued to the com driver, and the USB device stack is dormant. + +### Notes on the probe itself + +- Register resync matters: run/halt is sequenced with `monitor`, so gdb keeps + serving registers cached from the reset halt unless forced to re-read. The + first version reported `pc=z_arm_reset` at a halt 12 s into the run. + `monitor gdb sync` + `stepi` is the usual recipe but resumes the target when + the halt lands mid-ISR; the probe detaches and reconnects instead. +- The first "did anything move?" metric summed each thread's saved `psp` + + resume PC. That is **unsound** — a healthy thread that blocks at the same + line every cycle reproduces byte-identical values, and it reported + "IDENTICAL: no thread made any progress" on a running system. The probe now + sums `base.usage.total` (`CONFIG_SCHED_THREAD_USAGE=y`), which is monotonic. +- A swapped-out Cortex-M thread's `callee_saved.psp` points *straight at* the + hardware exception frame (stacked LR at `+0x14`, PC at `+0x18`); the callee + registers live in the `k_thread`, not on the stack. Reading them at `+0x34`/ + `+0x38` yields F′ object addresses that look like plausible wild pointers — + which is very likely the origin of the `0x20010480` red herring above. + +## Local-bench result (2026-07-27) — v6 downlink trace: ComQueue never leaves WAITING + +Follow-on to the v5 thread-walk. `scripts/diag/downlink_trace.{sh,gdb}` traces +the status path that is supposed to release the downlink, live from reset. + +**`Svc::ComQueue` is constructed in `WAITING` (`ComQueue.cpp:35`) and only ever +reaches `READY` via `comStatusIn` carrying `SUCCESS` (`ComQueue.cpp:236-247`). +Until that happens it never dequeues, so nothing is framed and the link is +silent from boot.** On this board it never happens, on *both* Com paths: + +| read | ComCcsdsUart | ComCcsdsLora | +|---|---|---| +| `comQueue.m_state` | `WAITING` | `WAITING` | +| `framer.m_masterFrameCount` | **0** | **0** | +| `aggregator.m_allow_timeout` | true (FILL) | false (WAIT_STATUS) | +| `comStub.m_reinitialize` | 0 (ready seen) | n/a (no comStub) | + +`m_masterFrameCount = 0` means **no TM frame has ever been framed on either +link** — this is not "telemetry stopped", it is "downlink never started". + +The status path is `comStub.comStatusOut -> framer -> aggregator -> +spacePacketFramer -> comQueue.comStatusIn`. Breakpointing every hop: + +- `ComStub::drvConnected_handler` **fires**, from + `ZephyrUartDriver::configure` ← `setupTopology`, and emits its one status. +- `ComAggregator::preamble` **fires for both aggregators** — this is the F′ + active-component preamble and the only place the aggregator emits an + unprovoked `comStatusOut` (`ComAggregator.cpp:24-27`); its other one is in + `doFill`, which needs data that cannot flow until ComQueue is released. +- `ComAggregator::comStatusIn_handler` **fires** with `condition.e = SUCCESS`. +- `SpacePacketFramer::comStatusIn_handler` **fires for both instances** — the + last hop before ComQueue, and a pure pass-through + (`SpacePacketFramer.cpp:79-83`). +- `ComQueue::comStatusIn_handler` **never fires, on either instance.** + +The gap is not wiring. Both `comStatusOut` ports are connected at runtime — +`ComCcsdsUart::spacePacketFramer.m_comStatusOut_OutputPort[0].m_port == +&ComCcsdsUart::comQueue.m_comStatusIn_InputPort[0]` (and likewise for LoRa), +matching `ReferenceDeploymentTopologyAc.cpp:2396`. Nor is it a dead thread: +`ComCcsdsLora::comQueue.run_handler` is dispatched every second on that same +comQueue thread, so the thread is alive and draining its queue — it simply +never receives a `comStatusIn` message. + +Two things this rules out, both of which looked promising: + +- **`ComCcsdsUart.comQueue.run` being unconnected is NOT the bug.** It really is + unconnected (only `ComCcsdsLora.comQueue.run` is wired, `topology.fpp:268`), + but `ComQueue::run_handler` only publishes queue-depth telemetry + (`ComQueue.cpp:257`); the dequeue is driven by `comStatusIn`. It is also + unconnected on `main`. Cost: one missing telemetry channel. +- **The v5 per-thread zeros were partly a short-window artifact.** A single 4 s + sample showed `ComCcsdsUart::comQueue` at 0 cycles and `ComCcsdsLora:: + aggregator` at 0 while their opposite numbers ran — a mirror-image asymmetry + that does not survive contact with the trace above (both paths are equally + stuck). The probe now takes a configurable window (`WINDOW_A_MS`/ + `WINDOW_B_MS`, default 12 s / 20 s) so a thread that merely had nothing to do + is not read as wedged. + +**What this means for the branch.** `git diff main...HEAD` touches *no* code in +the com path — `topology.fpp`, `instances.fpp` and all of `lib/fprime` are +byte-identical to `main`. So if this is a regression it is being caused +indirectly, and the only boot-time behavioural change the branch makes is the +new `/keys` littlefs automount (`prj.conf` `CONFIG_FILE_SYSTEM_LITTLEFS=y` plus +the `lfs1` fstab node + `keystore_partition` in the board `.dtsi`). + +**Control experiment (in progress):** same tree, `automount` removed from the +`lfs1` node so `/keys` is not mounted at boot, everything else unchanged. If the +downlink returns, the boot-time littlefs mount is the cause and the key store +must move off an automounted filesystem; if it does not, the failure predates +the branch's config and the next control is a `main` build on this bench. + +## Next steps (post-root-cause) — pin *why* control jumps to `0x20010480` + +> **Caveat (see local-bench result above):** on the current build the hang does +> **not** raise a CPU fault, so steps 1–2's fault-entry breakpoints won't fire +> as written. Treat the "wild jump to `0x20010480`" framing as unconfirmed on +> today's build — the observed state is a no-fault idle/starved-scheduler hang. +> The stack-overflow-vs-wild-pointer question is still the crux, but the catch +> mechanism must change (watchpoint / thread walk, not a fault breakpoint). + +Ordered by diagnostic-value-per-effort. The whole task now is separating the two +live candidates: a **stack overflow** in a worker thread that smashed a +neighbouring return address / function pointer, vs. a **wild/uninitialized +pointer** (bad F′ port or handler) branched through directly. + +**1. Catch the fault at ENTRY, not after the halt (do this first — no rebuild).** +Every diagnostic so far halted the board *after* `z_arm_fatal_error` had already +run and masked IRQs, which is why the backtrace is garbage (`lr=0x1019`, shallow +stack). Set a **hardware** breakpoint on the UsageFault vector entry +`z_arm_usage_fault` (`0x10104fa0`), `monitor reset halt`, `continue`, and let it +hit at ~8 s with the exception frame fresh and `EXC_RETURN` still in `LR`. From +that frame, read the **true stacked PC** (the faulting instruction), the +**stacked LR** (the real caller/return), and compare the **pre-fault SP** to the +faulting thread's `stack_info.start/size` (valid — `CONFIG_THREAD_STACK_INFO=y`). +That single frame decides overflow (SP past its bound) vs. wild pointer (SP +sane, PC/LR corrupted), and the UFSR bits (`INVSTATE`/`UNDEFINSTR` @ `0xE000ED28`) +confirm "executed a data address". Runs on the *current* build over SWD — zero +turnaround. Script: **`scripts/diag/hang_fault_bp.sh`** + `hang_fault_bp.gdb`. +If the fault doesn't route through the UsageFault vector (e.g. a HardFault +escalation), rerun with `FAULT_SYM=z_arm_fault` to break on the common C handler +instead (it reads `EXC_RETURN`/`msp`/`psp` from `r2`/`r0`/`r1` per `fault.c:1025`). +This supersedes the earlier "walk `_kernel.threads` on the hung state" idea (same +data, but you'd have to dig the frame out of post-halt state by hand). + +**2. If (1) points at overflow, prove the site with a rebuild.** The build has +**both** stack-overflow traps OFF — `# CONFIG_HW_STACK_PROTECTION is not set`, +`# CONFIG_STACK_SENTINEL is not set` — so an overrun silently corrupts adjacent +RAM (exactly the "wild pointer *inside* the `fileManager` object" signature). A +diagnostic build with `CONFIG_HW_STACK_PROTECTION=y` (+ `CONFIG_STACK_SENTINEL=y`, +`CONFIG_THREAD_ANALYZER=y`) faults on the *overflowing write* and names the +thread + high-water mark — earlier and more precise than waiting for the eventual +wild jump. + +**3. Audit the branch's new on-stack buffers.** `TcSecurityDeframer.cpp:185` and +`:219` put `uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]` on the stack +(plus HMAC scratch) in the +289-line feature. Verify `kTCSecurityTrailer` sizing +against every write into `keyBytes`, and check the F′ thread stack the deframer +runs on. A bounded overrun in the hot path the flash-size fix newly unblocked +fits the timeline. + +**4. Make the fatal handler talk before it spins (kills the guessing loop).** +Override `k_sys_fatal_error_handler` (or set `CONFIG_EXTRA_EXCEPTION_INFO=y`) to +emit faulting-thread + real stacked PC/LR/CFSR over SWD/RTT *before* the masked +`b .`. + +**5. Treat the amplifier as its own defect (post-root-cause follow-up).** One +non-recoverable fault → IRQs masked → infinite spin → dead SysTick → silent +system death is a reliability hole independent of the cause. Track a hardware +watchdog so the board *resets* (CI sees a reboot, not a permanent "disconnect") +instead of going dark. + +**Housekeeping:** `scripts/diag/hang_fault_bp.{sh,gdb}` and +`scripts/diag/hang_thread_walk.{sh,gdb}` join the existing +`hang_forensics.tcl` / `hang_gdb.sh` diagnostics that **must be removed before +merge**, along with the `board-serial.log` / `hang-thread-walk-openocd.log` +capture files they drop in the repo root. (These v4/v5 scripts are local-bench +only — no `ci.yaml` change — so nothing new to revert in CI.) --- diff --git a/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp b/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp index b28b21a5..a4042255 100644 --- a/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp +++ b/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp @@ -11,7 +11,14 @@ // Define configuration values for dispatcher enum { - CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350, // !< The size of the table holding opcodes to dispatch + // Must be >= the deployment's total command count (see the dictionary's + // `commands` array). CommandDispatcher inserts every opcode into a + // fixed-capacity RedBlackTreeMap and FW_ASSERTs when the insert fails + // (CommandDispatcherImpl.cpp:35), so exceeding this bricks the boot rather + // than degrading. hmac-to-storage pushed the count from 348 to 354 by + // adding PROVISION_KEY/ADD_KEY/REMOVE_KEY to both TcSecurityDeframer + // instances; headroom raised so the next few commands do not repeat this. + CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 512, // !< The size of the table holding opcodes to dispatch CMD_DISPATCHER_SEQUENCER_TABLE_SIZE = 10, // !< The size of the table holding commands in progress }; diff --git a/TODO.md b/TODO.md index cf8c4954..a7b0d2d7 100644 --- a/TODO.md +++ b/TODO.md @@ -131,7 +131,144 @@ Status legend: [ ] todo, [~] in progress, [x] done run** (same pattern as the prior fault-register diagnostic commits) to see whether the stall is the one-time format or ongoing per-frame reloads. -## Suggested fix (blocked CI) — see INVESTIGATION.md +## CI blocker — ROOT-CAUSED AND FIXED (2026-07-27), see INVESTIGATION.md + +- [x] **Root cause: CommandDispatcher opcode-table overflow.** + `project/config/CommandDispatcherImplCfg.hpp` had + `CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350` against a deployment already at + **348** commands. This branch adds PROVISION_KEY/ADD_KEY/REMOVE_KEY to + `TcSecurityDeframer`, and there are **two** instances (ComCcsdsUart, + ComCcsdsLora) -> 6 new commands, **354 > 350**. + `CommandDispatcherImpl.cpp:35` `FW_ASSERT`s when the RedBlackTreeMap + insert fails, so the 351st registration **panics the board during boot** + (`z_fatal_error(reason=4)` via `z_arm_svc` -- a `k_panic`, not a CPU + fault, which is why every fault-vector probe found nothing). Downlink + never starts -> GDS `device disconnected` -> CI fails. + **Fix: raised to 512.** + Confirmed on the local bench with a single-variable A/B, both full clean + `make generate build`s, measuring F' telemetry bytes off the board CDC in + a 25s window: `main` 1328 bytes @t+1.0s; branch as-shipped **0**; branch + + table 512 **1392 bytes @t+1.01s**; branch + table back to 350 **0**. + Final verified build: **2200 bytes in 30s @t+1.01s**. + +- [x] **Second, independent defect: littlefs was never compiled in.** + `west.yml`'s `name-allowlist` imported `fatfs` but not `littlefs`, so the + module never reached `zephyr_modules.txt`, `ZEPHYR_LITTLEFS_MODULE` was + undefined, and Kconfig **silently dropped** `CONFIG_FILE_SYSTEM_LITTLEFS=y` + ("LittleFS module not available"). No `lfs_*` symbols in the image, `/keys` + never existed, every `fs_open("/keys/...")` failed -- the entire key-store + feature was inert while the build stayed clean. + **Fix: added `littlefs` to the allowlist and pinned it as an explicit + project** (like every other module) so it lands under `lib/zephyr-workspace/` + rather than the workspace topdir. Verified `CONFIG_FILE_SYSTEM_LITTLEFS=y`, + `CONFIG_FS_LITTLEFS_FSTAB_AUTOMOUNT=y`, 20 `lfs_*` symbols in the ELF. + Not the CI blocker, but the feature cannot work without it. + +- [x] **Feature verified end to end on hardware (2026-07-27).** Board on + `/dev/tty.usbmodem1101`, GDS on the USB CDC, state read back over SWD: + 1. **`/keys` mounts and formats on internal flash.** littlefs superblock + magic present at `0x10400008` with block_size 0x1000 and block_count + 0x40 -- exactly the 256 KB `keystore_partition` geometry. + 2. **PROVISION_KEY works on a keyless board** over the unauthenticated + (bypass-allowlisted) link: `m_keyStore.elements[0]` went to + `valid=1 spi=0` with the provisioned key bytes. + 3. **The key survives a cold reboot.** After `reset` (RAM re-zeroed by + `arch_bss_zero`), slot 0 is `valid=1` with the same bytes -- i.e. + `configure()` -> `loadKeyStore()` read it back off flash. + 4. **The flash-stored key authenticates uplink.** `SET_SEQ_NUM 12345` + requires authentication (not bypass-allowlisted) and took effect: + `m_sequenceNumber == 12345`. + 5. **The sequence number survives a cold reboot**: still 12345 after + reset, read back from `/keys/sequence_number.bin`, key still valid. + 6. **Erasing the partition re-formats cleanly.** After erasing + `0x10400000+0x40000` over SWD the board came back keyless + (`valid=0`, `seqnum=0`) with a fresh littlefs superblock. + 7. **Bypass path confirmed:** on a keyless board the router shows + `routed=3 bypassed=3 rejected=0` -- allowlisted opcodes are dispatched + without a key, which is what makes bootstrap possible. + **Section 3 opcodes re-verified against the fresh dictionary:** + `ComCcsdsUart/Lora.tcSecurityDeframer.PROVISION_KEY` are `0x2100B002` / + `0x2200B002`, matching `Bypasser.cpp` exactly. The TODO caveat there is + resolved. (The Sband entry `0x2300B002` is still unconfirmed -- that + instance is not built.) + +- [ ] **Could not get `provision_key_test.py` to pass through the pytest + fixture path on the bench.** The firmware side is proven (item 2 above -- + the same command sent directly provisions the board), but + `start_gds`'s `CdhCore.cmdDisp.CMD_NO_OP` kept timing out, and + `recover_from_safe_mode` is `autouse=True` and depends on `start_gds`, so + every test in the directory errors in setup. The board *is* dispatching + those commands (`bypassed=3 rejected=0`), so this looks like a GDS-side + downlink desync from my reset-heavy bench session -- GDS logged + `APID 2 received sequence count: 4 (expected: 1)` after each board reset, + and CI power-cycles before starting GDS, which would avoid it. **Not + confirmed either way -- re-check once CI runs.** + +- [ ] **No over-the-air recovery from a mis-provisioned key.** PROVISION_KEY is + refused once the store is non-empty (`NotEmpty`) and REMOVE_KEY refuses to + remove the last key, so a board provisioned with the wrong key cannot be + re-keyed from the ground -- it needs a physical SWD flash erase of + `keystore_partition` (which is how the bench board was recovered). Worth a + deliberate decision before flight. + +- [ ] **Watch the other zero-headroom config constants.** Same failure mode, + same file tree: `MAX_PACKETIZER_CHANNELS = 202` vs 191 channels in use, + `MAX_PACKETIZER_PACKETS = 22` vs exactly 22 packets. Adding commands, + channels or packets to this deployment requires checking these against the + generated dictionary -- they assert at boot rather than degrading. + +- [ ] **Write up the SWD/GDB diagnostic helpers as an ADR, and point the agent + instructions at it.** The probes built while root-causing this branch are + generally useful for any "board is silent / GDS sees nothing" failure on + this hardware, and re-deriving them cost most of a session. Capture in a + new ADR (there is no `docs/adr/` yet -- this would be the first): + - `scripts/diag/hang_thread_walk.{sh,gdb}` -- reset, free-run, halt twice + N s apart; clock/SysTick/interrupt state, a walk of `_kernel.threads` + naming each F' task, the `timeout_list`, the whole downlink chain's + state, and a per-thread CPU-time diff between the two halts. + - `scripts/diag/downlink_trace.{sh,gdb}` -- breakpoints on every hop of + the `comStub -> framer -> aggregator -> spacePacketFramer -> comQueue` + com-status path. + - The older `hang_forensics.tcl` / `hang_gdb.sh` / `hang_fault_bp.*`. + Non-obvious things the ADR should record, all of which cost real time: + - Use the **raspberrypi OpenOCD fork**, not a nix/brew build. + - Sequencing run/halt via `monitor` leaves gdb serving **stale registers**; + detach + reconnect to resync (`monitor gdb sync` + `stepi` can resume the + target when the halt lands mid-ISR). + - **Always resume the target** before detaching -- a halted board drops its + USB CDC, which makes GDS see nothing and silently no-ops any command. + - A swapped-out Cortex-M thread's `callee_saved.psp` points straight at the + exception frame (LR `+0x14`, PC `+0x18`); callee regs live in the + `k_thread`. Reading `+0x34`/`+0x38` yields F' object addresses that look + exactly like plausible wild pointers -- this produced a multi-day red + herring in `INVESTIGATION.md`. + - `PRIMASK=1` at `arch_cpu_idle+18` is the **normal** idle sequence, not a + masked spin. + - Prefer monotonic `base.usage.total` over saved psp/PC when asking "did + this thread make progress" -- a healthy thread re-blocking at the same + line reproduces byte-identical values. + - **`make build` does not re-derive Kconfig from device-tree changes**; use + `make generate build`. + Then add a diagnostics section to the repo's agent instructions pointing at + the ADR. **Note:** the repo has `AGENTS.md`, not `CLAUDE.md` -- decide + whether to add the section to `AGENTS.md`, or add a `CLAUDE.md` (symlink or + stub) so both agent toolchains pick it up. + +- [ ] **Remove the diagnostics before merge:** `scripts/diag/hang_thread_walk.*`, + `scripts/diag/downlink_trace.*`, `scripts/diag/hang_fault_bp.*`, + `scripts/diag/hang_forensics.tcl`, `scripts/diag/hang_gdb.sh`, the + `Hang Forensics Diagnostic` step in `ci.yaml`, and the stray capture logs. + +### Build-system trap (cost several hours this session) +`make build` does **not** re-derive Kconfig from device-tree changes -- it left +`CONFIG_FLASH_SIZE=4096` while the DTS said 16 MB, putting `keystore_partition` +out of bounds so the `/keys` automount panicked on +`__ASSERT_NO_MSG(block_size != 0)` (`littlefs_fs.c:787`). Purely an artifact of +the stale config, and it invalidated several intermediate bisect results. +**Always `make generate build` after touching the device tree.** + +## Superseded — original "Suggested fix" notes, kept for the audit trail + - [x] **Primary fix**: changed `&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; }` → `DT_SIZE_M(16)` in `proves_flight_control_board_v5.dtsi` (shared by v5c/v5d/v5e). Confirmed correct by CI hardware run 30034861047: OpenOCD reports the real chip @@ -176,6 +313,23 @@ Status legend: [ ] todo, [~] in progress, [x] done regardless of exact mechanism — see Robustness follow-ups below), or dig further into which specific call inside the mount/format/create path never returns.** + **2026-07-27 — the "stall" does not exist.** Built and ran the v5 + thread-walk probe (`scripts/diag/hang_thread_walk.{sh,gdb}`) on the local + bench. Across two halts 4s apart the board is *fully healthy*: clock + advancing (+40040 ticks = 4.004s), no fault (`CFSR=HFSR=0`), no masked + IRQs, the 1ms base-rate `k_timer` queued and firing, all three rate + groups cycling, main looping in `startRateGroups()`, and 89% idle. Every + earlier "hang" reading was a misread healthy idle CPU (`PRIMASK=1` + + `arch_cpu_idle` is the normal `cpsid i; wfi; cpsie i`), a stale FPB + breakpoint, or callee-saved registers misread as a PC — there is no wild + jump, stack overflow, fs-lock deadlock or fatal-halt spin. + **The real failure is the downlink:** `usbd_thread`, + `udc_rpi_pico_thread_0` and `ComCcsdsUart::comQueue` consume *zero* CPU + cycles while `ComCcsdsUart::aggregator` burns 1.4M, and the host reads 0 + bytes in 15s from the board CDC. Telemetry is produced and aggregated but + never dequeued to the com driver, and the USB device stack is dormant. + **Next: chase the UART/USB downlink path**, not the filesystem. See + `INVESTIGATION.md` "v5 thread-walk probe: THERE IS NO HANG". - [ ] **Robustness follow-ups (evaluate once the stall is diagnosed):** (a) consider raw `flash_area_*`/NVS instead of littlefs for this fixed-size store (avoids the format-time erase burst and any long single-call erase/program under diff --git a/scripts/diag/downlink_trace.gdb b/scripts/diag/downlink_trace.gdb new file mode 100644 index 00000000..2b725f69 --- /dev/null +++ b/scripts/diag/downlink_trace.gdb @@ -0,0 +1,87 @@ +# downlink_trace.gdb -- v6 of the /keys CI-failure forensics (INVESTIGATION.md). +# +# v5 (hang_thread_walk.*) proved the firmware is healthy and localised the +# failure to the downlink: at t=12s both Com subtopologies show +# ComQueue.m_state = WAITING (on BOTH the UART and LoRa paths) +# TmFramer.m_masterFrameCount = 0 (on BOTH -- no TM frame has EVER +# been framed, on either link) +# ComCcsdsUart::comStub.m_reinitialize = 0 (the driver's ready DID arrive and +# comStub DID emit its one status) +# +# ComQueue is constructed in WAITING (ComQueue.cpp:35) and only ever reaches +# READY via comStatusIn carrying SUCCESS (ComQueue.cpp:236-247). Until then it +# never dequeues, so nothing is framed and the link is silent from boot. The +# status has to travel +# comStub.comStatusOut -> framer -> aggregator -> spacePacketFramer +# -> comQueue.comStatusIn +# and it demonstrably reaches the aggregator (its m_allow_timeout is true, i.e. +# the FILL state) but not comQueue. This probe watches that whole path live +# from reset and reports exactly where the status dies. +# +# Read-only apart from breakpoints/watchpoints, all set on a target that is +# reset at the start of the run. Driven by downlink_trace.sh. + +monitor log_output downlink-trace-openocd.log + +echo \n================ arming the downlink status path ================\n +monitor reset halt + +# Every stage that must forward the status upward. Plain linespec form, NOT +# `*func`: the `*` forces expression parsing, which needs Svc::ComStub as a +# *type* in the current context and fails with "No type ComStub within class or +# namespace Svc" before the program has run. Letting gdb skip the prologue is +# fine here because the arguments are read with `info args` (DWARF locations) +# rather than out of raw registers. +# +# ComAggregator::preamble is the one that matters most: it is the F' active +# component preamble, run on the aggregator's own thread when tasks start, and +# it is the ONLY place the aggregator emits an unprovoked comStatusOut +# (ComAggregator.cpp:24-27). Its other comStatusOut is in doFill, which needs +# data -- and data cannot flow until ComQueue is released. So if preamble +# never runs, or its status never reaches ComQueue, the chain deadlocks from +# boot exactly as observed. +hbreak Svc::ComStub::drvConnected_handler +hbreak Svc::ComAggregator::preamble +hbreak Svc::ComAggregator::comStatusIn_handler +hbreak Svc::ComQueue::comStatusIn_handler +info breakpoints + +# Report a stop, then keep going. $stops bounds the run so a fast-repeating +# hit cannot spin forever. +set $stops = 0 +set $limit = 14 + +echo \n================ running to topology setup ================\n +continue +printf "\n---- reached %s; arming the ComQueue.m_state watchpoint ----\n", "drvConnected" +# The gate itself. A hardware watchpoint fires on every write, so "never +# fires again" is itself an answer, and each hit names the writer. Armed here +# rather than at reset because bss-zeroing and ComQueue's own constructor write +# it during early boot and drown the trace in noise. +watch ComCcsdsUart::comQueue.m_state + +echo \n================ tracing the status path ================\n +while $stops < $limit + continue + set $stops = $stops + 1 + printf "\n---- stop %d ----------------------------------------------\n", $stops + printf " pc = %#lx -> ", (unsigned long)$pc + info symbol $pc + printf " ComCcsdsUart::comQueue.m_state = %d (0=READY 1=WAITING)\n", \ + (int)ComCcsdsUart::comQueue.m_state + printf " ComCcsdsUart::aggregator FILL? %d comStub.reinit=%d\n", \ + (int)ComCcsdsUart::aggregator.m_allow_timeout, \ + (int)ComCcsdsUart::comStub.m_reinitialize + printf " TmFramer mfc=%d vfc=%d\n", \ + (int)ComCcsdsUart::framer.m_masterFrameCount, \ + (int)ComCcsdsUart::framer.m_virtualFrameCount + # For the comStatusIn breakpoints `condition` is the Fw::Success& being + # forwarded -- SUCCESS=1, FAILURE=0. A FAILURE arriving at ComQueue leaves it + # WAITING (ComQueue.cpp:246) and is just as fatal as no status at all. + printf " args at this stop:\n" + info args + bt 8 +end + +echo \n================ stop limit reached; detaching ================\n +detach diff --git a/scripts/diag/downlink_trace.sh b/scripts/diag/downlink_trace.sh new file mode 100755 index 00000000..c62eb301 --- /dev/null +++ b/scripts/diag/downlink_trace.sh @@ -0,0 +1,111 @@ +#!/usr/bin/env bash +# downlink_trace.sh -- v6 of the /keys CI-failure forensics (INVESTIGATION.md). +# +# v5 (hang_thread_walk.*) showed the firmware is healthy and the DOWNLINK is +# dead from boot: ComQueue never leaves its initial WAITING state, so nothing is +# ever framed (TmFramer master frame count stays 0 on both the UART and LoRa +# paths) and the CDC stays silent. This probe watches the status path that is +# supposed to release ComQueue -- +# comStub.comStatusOut -> framer -> aggregator -> spacePacketFramer +# -> comQueue.comStatusIn +# -- live from reset, with a hardware watchpoint on ComQueue.m_state plus +# breakpoints on each forwarding stage, and reports where the status dies. +# Read logic lives in downlink_trace.gdb. +# +# Local bench only (not CI). Drives the board over a Raspberry Pi Debug Probe +# (CMSIS-DAP SWD). The two USB CDC ttys involved: +# BOARD_TTY /dev/tty.usbmodem1101 - the target's own USB CDC (F'/GDS link) +# PROBE_TTY /dev/tty.usbmodem102 - the Debug Probe's UART bridge +# OpenOCD reaches SWD via the probe's CMSIS-DAP USB interface (not a tty). +# +# Read-only apart from the breakpoints/watchpoint it sets, all on a target that +# is reset at the start of the run. Never writes flash or config. +set -u + +BOARD_TTY=${BOARD_TTY:-/dev/tty.usbmodem1101} +PROBE_TTY=${PROBE_TTY:-/dev/tty.usbmodem102} + +# OpenOCD: the raspberrypi fork (do NOT substitute a nix/brew openocd -- the RP2350 +# support and the CMSIS-DAP build the bench relies on live in this fork). Local +# bench keeps it under ~/code/...; the CI runner uses ~/openocd. Override with +# OOCD_HOME. +OOCD_HOME=${OOCD_HOME:-} +if [ -z "$OOCD_HOME" ]; then + for cand in ~/code/github.com/raspberrypi/openocd ~/openocd; do + [ -x "$cand/src/openocd" ] && { OOCD_HOME="$cand"; break; } + done +fi +[ -x "$OOCD_HOME/src/openocd" ] || { echo "downlink_trace: openocd not found (set OOCD_HOME to the raspberrypi openocd checkout)"; exit 1; } +OCD="$OOCD_HOME/src/openocd" +OCD_ARGS=(-s "$OOCD_HOME/tcl" + -f "$OOCD_HOME/tcl/interface/cmsis-dap.cfg" + -f "$OOCD_HOME/tcl/target/rp2350.cfg" + -c "adapter speed 5000") + +HERE=$(cd "$(dirname "$0")" && pwd) +GDBCMDS="$HERE/downlink_trace.gdb" +[ -f "$GDBCMDS" ] || { echo "downlink_trace: missing $GDBCMDS"; exit 1; } + +# --- symbol ELF (local flight build; load addrs 0x1018_xxxx) --- +ELF="" +for cand in \ + build-fprime-automatic-zephyr/zephyr/zephyr.elf \ + build-artifacts/zephyr/fprime-zephyr-deployment; do + [ -f "$cand" ] && { ELF="$cand"; break; } +done +[ -z "$ELF" ] && { echo "downlink_trace: symbol ELF not found; run 'make generate build' first."; exit 1; } +echo "downlink_trace: ELF $ELF" +echo "downlink_trace: board $BOARD_TTY probe $PROBE_TTY" + +# --- ARM gdb --- +GDB="" +for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do + command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } +done +[ -z "$GDB" ] && GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | tail -1) +[ -z "$GDB" ] && { echo "downlink_trace: no arm/multiarch gdb found."; exit 1; } +echo "downlink_trace: gdb $GDB" + +# --- optional: capture the board CDC so telemetry-going-silent is timestamped --- +SERIAL_LOG=${SERIAL_LOG:-board-serial.log} +SNIFF_PID="" +if [ -e "$BOARD_TTY" ]; then + ( cat "$BOARD_TTY" > "$SERIAL_LOG" 2>/dev/null ) & + SNIFF_PID=$! + echo "downlink_trace: sniffing $BOARD_TTY -> $SERIAL_LOG" +else + echo "downlink_trace: note: $BOARD_TTY not present; skipping serial sniff" +fi + +# --- OpenOCD: init + keep the gdb server up; the .gdb file drives reset/run/halt --- +"$OCD" "${OCD_ARGS[@]}" -c "init" -c "echo {downlink_trace: gdb server on :3333}" & +OCD_PID=$! +cleanup() { [ -n "$SNIFF_PID" ] && kill "$SNIFF_PID" 2>/dev/null; kill "$OCD_PID" 2>/dev/null; } +trap cleanup EXIT +sleep 2 + +# All run/halt sequencing is done with `monitor` inside the command file, so gdb +# never thinks the target is running. That means gdb would happily serve stale +# cached data across a resume, so disable both memory caches here; the command +# file flushes the register cache after each halt. +GDB_ARGS=(-q -nx -batch "$ELF" + -ex "set pagination off" + -ex "set confirm off" + -ex "set print pretty on" + -ex "set backtrace past-main on" + -ex "set stack-cache off" + -ex "set code-cache off" + -ex "target extended-remote localhost:3333" + -x "$GDBCMDS") + +TIMEOUT_BIN=$(command -v timeout || command -v gtimeout || true) +if [ -n "$TIMEOUT_BIN" ]; then + "$TIMEOUT_BIN" 300 "$GDB" "${GDB_ARGS[@]}" \ + || echo "downlink_trace: gdb exited non-zero / timed out" +else + "$GDB" "${GDB_ARGS[@]}" \ + || echo "downlink_trace: gdb exited non-zero (see output above)" +fi + +echo "downlink_trace: done (board serial capture in $SERIAL_LOG," +echo " openocd log in downlink-trace-openocd.log)" diff --git a/scripts/diag/hang_fault_bp.gdb b/scripts/diag/hang_fault_bp.gdb new file mode 100644 index 00000000..4f820da5 --- /dev/null +++ b/scripts/diag/hang_fault_bp.gdb @@ -0,0 +1,102 @@ +# hang_fault_bp.gdb -- read the pre-fault (hardware-stacked) CPU state once a +# fault breakpoint has hit, before z_arm_fatal_error masks IRQs and spins in +# `b .`. Driven by hang_fault_bp.sh, which has already: connected to the gdb +# server, `monitor reset halt`, armed the HW breakpoint, and set $FAULT_ARGS. +# See INVESTIGATION.md "Next diagnostic (v4): catch the fault at entry". +# +# Two entry points, selected by hang_fault_bp.sh via FAULT_SYM -> $FAULT_ARGS: +# $FAULT_ARGS==0 break at z_arm_usage_fault (the UsageFault *vector* entry): +# the HW-stacked frame is fresh and LR still holds EXC_RETURN, +# and live $msp/$psp are the pre-fault stack pointers. +# $FAULT_ARGS==1 break at *z_arm_fault (the common C handler) instead, for +# faults that don't route through z_arm_usage_fault. Its args +# carry the state: z_arm_fault(msp=r0, psp=r1, exc_return=r2, +# callee=r3) -- confirmed in this build's fault.c:1025 -- so we +# take EXC_RETURN/msp/psp from r2/r0/r1 (LR is stale here). +# +# Build facts relied on (build-fprime-automatic-zephyr/zephyr/.config): +# CONFIG_FPU is not set -> plain 8-word (0x20) exception frame +# CONFIG_MP_MAX_NUM_CPUS=1 -> current thread is _kernel.cpus[0].current +# CONFIG_THREAD_STACK_INFO=y -> stack_info.start/size are valid +# CONFIG_THREAD_NAME is not set -> k_thread has no .name member (don't read it) + +echo \n==== running to the fault breakpoint (expected ~8s into boot) ====\n +continue + +echo \n==== FAULT CAUGHT -- pre-halt, exception frame intact ====\n +info registers lr primask basepri control xpsr + +# Recover EXC_RETURN and the pre-fault MSP/PSP for whichever entry we stopped at. +if $FAULT_ARGS + set $exc = (unsigned long)$r2 + set $msp_v = (unsigned long)$r0 + set $psp_v = (unsigned long)$r1 + printf "entry=z_arm_fault (args): exc_return=r2 msp=r0 psp=r1\n" +else + set $exc = (unsigned long)$lr + set $msp_v = (unsigned long)$msp + set $psp_v = (unsigned long)$psp + printf "entry=z_arm_usage_fault (vector): exc_return=lr msp/psp live\n" +end + +# EXC_RETURN bit2 selects the stack the CPU pushed the exception frame onto: +# 0 -> MSP (fault happened in handler mode) 1 -> PSP (fault in a thread) +set $usepsp = ($exc >> 2) & 1 +set $frame = $usepsp ? $psp_v : $msp_v +printf "exc_return=%#lx pre-fault stack=%s frame_sp=%#lx\n", \ + $exc, ($usepsp ? "PSP(thread)" : "MSP(handler)"), $frame + +echo \n==== stacked exception frame = the exact pre-fault CPU state ====\n +set $sr0 = *(unsigned long*)($frame+0x00) +set $sr1 = *(unsigned long*)($frame+0x04) +set $sr2 = *(unsigned long*)($frame+0x08) +set $sr3 = *(unsigned long*)($frame+0x0c) +set $sr12 = *(unsigned long*)($frame+0x10) +set $slr = *(unsigned long*)($frame+0x14) +set $spc = *(unsigned long*)($frame+0x18) +set $sxpsr = *(unsigned long*)($frame+0x1c) +printf " r0=%#lx r1=%#lx r2=%#lx r3=%#lx r12=%#lx\n", $sr0,$sr1,$sr2,$sr3,$sr12 +printf " stacked LR (caller/return) = %#lx\n", $slr +printf " stacked PC (faulting instr) = %#lx\n", $spc +printf " stacked xPSR = %#lx\n", $sxpsr +echo -- symbolize the pre-fault PC and LR --\n +printf " faulting PC -> " +info symbol $spc +printf " stacked LR -> " +info symbol $slr + +echo \n==== why: CFSR / UFSR (UsageFault status @ 0xE000ED28) ====\n +# UFSR = upper halfword of CFSR. Key bits for a wild jump: +# bit16 UNDEFINSTR : jumped into non-code / bad opcode +# bit17 INVSTATE : Thumb (EPSR.T) bit clear -> branched to an even/data addr +# bit18 INVPC : bad EXC_RETURN / integrity check +# INVSTATE or UNDEFINSTR here == executed a data/garbage address (matches the +# observed pc=0x20010480 inside the fileManager object). +x/1xw 0xE000ED28 + +echo \n==== stack-overflow test: is frame_sp below the faulting thread's stack? ====\n +set $thr = _kernel.cpus[0].current +set $sbase = (unsigned long)$thr->stack_info.start +set $ssize = (unsigned long)$thr->stack_info.size +printf " current k_thread @ %#lx\n", (unsigned long)$thr +printf " stack: base=%#lx size=%#lx top=%#lx\n", $sbase, $ssize, $sbase+$ssize +printf " frame_sp=%#lx -> %s\n", $frame, \ + ($frame < $sbase ? "*** SP BELOW STACK BASE == STACK OVERFLOW ***" : \ + ($frame > $sbase+$ssize ? "*** SP ABOVE STACK TOP (wrong stack?) ***" : \ + "within stack extent (points to wild pointer, not overflow)")) + +echo \n==== reconstructed backtrace of the FAULTING thread ====\n +# Rewind GDB's view to the pre-fault frame so `bt` unwinds the culprit rather +# than the fault handler. Frame is 0x20 bytes; xPSR bit9 set => +4 align pad. +# NOTE: this writes core registers on the (about-to-be-reset) target. +set $pad = (($sxpsr >> 9) & 1) ? 4 : 0 +set $sp = $frame + 0x20 + $pad +set $pc = $spc +set $lr = $slr +bt + +echo \n==== raw stack window above frame_sp (find real 0x10xx return addrs) ====\n +x/64xw $frame + +echo \n==== done (detaching; board left halted) ====\n +detach diff --git a/scripts/diag/hang_fault_bp.sh b/scripts/diag/hang_fault_bp.sh new file mode 100755 index 00000000..caf7598e --- /dev/null +++ b/scripts/diag/hang_fault_bp.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# hang_fault_bp.sh -- v4 of the /keys boot-hang forensics (see INVESTIGATION.md). +# +# All prior forensics (hang_forensics.tcl / hang_gdb.sh) halted the board AFTER +# it was already spinning in the masked `b .` fatal-halt loop -- i.e. after +# z_arm_fatal_error ran -- so the backtrace was incoherent (lr=0x1019, garbage +# stack). This step instead sets a HARDWARE breakpoint on the UsageFault vector +# entry (z_arm_usage_fault), resets, and lets it hit at ~8s, catching the fault +# with the exception frame fresh and EXC_RETURN still in LR. From that frame it +# reads the true faulting PC, the caller LR, and the pre-fault SP, and compares +# SP to the faulting thread's stack bounds -- deciding STACK OVERFLOW vs. a +# WILD/CORRUPTED POINTER directly. See hang_fault_bp.gdb for the read logic. +# +# Local bench only (not CI). Drives the board over a Raspberry Pi Debug Probe +# (CMSIS-DAP SWD). The two USB CDC ttys involved: +# BOARD_TTY /dev/tty.usbmodem3101 - the target's own USB CDC (F'/GDS link); +# telemetry goes silent here at the hang +# PROBE_TTY /dev/tty.usbmodem102 - the Debug Probe's UART bridge +# OpenOCD reaches SWD via the probe's CMSIS-DAP USB interface (not a tty); the +# ttys are used only to correlate/timestamp the hang. +# +# Read-mostly: it resets+halts and reads state; the only writes are the GDB +# breakpoint and a register rewind for the reconstructed backtrace, both on a +# target that is reset at the start of the run. Never writes flash or config. +set -u + +BOARD_TTY=${BOARD_TTY:-/dev/tty.usbmodem3101} +PROBE_TTY=${PROBE_TTY:-/dev/tty.usbmodem102} + +# Which fault entry to break on. Default z_arm_usage_fault (the UsageFault +# vector -- LR still holds EXC_RETURN there). Set FAULT_SYM=z_arm_fault to +# instead catch the common C handler, for faults that don't route through the +# UsageFault vector (e.g. a HardFault escalation); its args carry EXC_RETURN. +FAULT_SYM=${FAULT_SYM:-z_arm_usage_fault} +if [ "$FAULT_SYM" = "z_arm_fault" ]; then + FAULT_ARGS=1 # read exc_return/msp/psp from r2/r0/r1 (see hang_fault_bp.gdb) +else + FAULT_ARGS=0 # read exc_return from LR, msp/psp live + [ "$FAULT_SYM" != "z_arm_usage_fault" ] && \ + echo "hang_fault_bp: warning: FAULT_SYM=$FAULT_SYM is unrecognized; assuming LR holds EXC_RETURN (vector-entry mode)" +fi +# Break at the exact address (`*`) so GDB does not skip a prologue and clobber +# the argument registers before we read them. +BP_SPEC="*$FAULT_SYM" + +# OpenOCD: the raspberrypi fork (do NOT substitute a nix/brew openocd -- the RP2350 +# support and the CMSIS-DAP build the bench relies on live in this fork). Local +# bench keeps it under ~/code/...; the CI runner uses ~/openocd. Override with +# OOCD_HOME. +OOCD_HOME=${OOCD_HOME:-} +if [ -z "$OOCD_HOME" ]; then + for cand in ~/code/github.com/raspberrypi/openocd ~/openocd; do + [ -x "$cand/src/openocd" ] && { OOCD_HOME="$cand"; break; } + done +fi +[ -x "$OOCD_HOME/src/openocd" ] || { echo "hang_fault_bp: openocd not found (set OOCD_HOME to the raspberrypi openocd checkout)"; exit 1; } +OCD="$OOCD_HOME/src/openocd" +OCD_ARGS=(-s "$OOCD_HOME/tcl" + -f "$OOCD_HOME/tcl/interface/cmsis-dap.cfg" + -f "$OOCD_HOME/tcl/target/rp2350.cfg" + -c "adapter speed 5000") + +HERE=$(cd "$(dirname "$0")" && pwd) +GDBCMDS="$HERE/hang_fault_bp.gdb" +[ -f "$GDBCMDS" ] || { echo "hang_fault_bp: missing $GDBCMDS"; exit 1; } + +# --- symbol ELF (local flight build; load addrs 0x1018_xxxx) --- +ELF="" +for cand in \ + build-fprime-automatic-zephyr/zephyr/zephyr.elf \ + build-artifacts/zephyr/fprime-zephyr-deployment; do + [ -f "$cand" ] && { ELF="$cand"; break; } +done +[ -z "$ELF" ] && { echo "hang_fault_bp: symbol ELF not found; run 'make generate build' first."; exit 1; } +echo "hang_fault_bp: ELF $ELF" +echo "hang_fault_bp: board $BOARD_TTY probe $PROBE_TTY" +echo "hang_fault_bp: break $BP_SPEC (FAULT_ARGS=$FAULT_ARGS)" + +# --- ARM gdb --- +GDB="" +for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do + command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } +done +[ -z "$GDB" ] && GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | head -1) +[ -z "$GDB" ] && { echo "hang_fault_bp: no arm/multiarch gdb found."; exit 1; } +echo "hang_fault_bp: gdb $GDB" + +# --- optional: capture the board CDC so telemetry-going-silent is timestamped --- +SERIAL_LOG="board-serial.log" +SNIFF_PID="" +if [ -e "$BOARD_TTY" ]; then + ( cat "$BOARD_TTY" > "$SERIAL_LOG" 2>/dev/null ) & + SNIFF_PID=$! + echo "hang_fault_bp: sniffing $BOARD_TTY -> $SERIAL_LOG" +else + echo "hang_fault_bp: note: $BOARD_TTY not present; skipping serial sniff" +fi + +# --- OpenOCD: init + keep the gdb server up; GDB drives reset/breakpoint --- +"$OCD" "${OCD_ARGS[@]}" -c "init" -c "echo {hang_fault_bp: gdb server on :3333}" & +OCD_PID=$! +cleanup() { [ -n "$SNIFF_PID" ] && kill "$SNIFF_PID" 2>/dev/null; kill "$OCD_PID" 2>/dev/null; } +trap cleanup EXIT +sleep 2 + +# Connect, reset+halt, and arm the HW breakpoint here (before boot runs), then +# hand off to the command file which continues to the fault and reads the frame. +# Guard with a timeout so a fault that never fires (continue blocks forever) +# doesn't wedge the run. +GDB_ARGS=(-q -nx -batch "$ELF" + -ex "set pagination off" + -ex "set confirm off" + -ex "set print pretty on" + -ex "target extended-remote localhost:3333" + -ex "monitor reset halt" + -ex "hbreak $BP_SPEC" + -ex "set \$FAULT_ARGS = $FAULT_ARGS" + -x "$GDBCMDS") + +TIMEOUT_BIN=$(command -v timeout || command -v gtimeout || true) +if [ -n "$TIMEOUT_BIN" ]; then + "$TIMEOUT_BIN" 90 "$GDB" "${GDB_ARGS[@]}" \ + || echo "hang_fault_bp: gdb exited non-zero / timed out (fault may not have fired within 90s)" +else + echo "hang_fault_bp: note: no timeout(1); gdb 'continue' will block until the fault fires." + "$GDB" "${GDB_ARGS[@]}" \ + || echo "hang_fault_bp: gdb exited non-zero (see output above)" +fi + +echo "hang_fault_bp: done (board serial capture in $SERIAL_LOG)" diff --git a/scripts/diag/hang_thread_walk.gdb b/scripts/diag/hang_thread_walk.gdb new file mode 100644 index 00000000..850e11e9 --- /dev/null +++ b/scripts/diag/hang_thread_walk.gdb @@ -0,0 +1,377 @@ +# hang_thread_walk.gdb -- v5 of the /keys boot-hang forensics (INVESTIGATION.md). +# +# The v4 fault-entry probe (hang_fault_bp.*) established there is NO CPU fault: +# breakpoints on z_arm_usage_fault / z_arm_fault never hit, and a clean halt of +# the board shows cm0 in arch_cpu_idle with CFSR=HFSR=0. The open question was +# whether the scheduler had stopped making progress with every thread blocked +# forever. +# +# This probe answers the two questions that state raises: +# 1. Is the system clock still running? (compare cycle_count / curr_tick and +# the live SysTick registers across two halts several seconds apart) +# 2. Who is blocked, and on what? (walk _kernel.threads and, for each +# swapped-out thread, recover its resume PC/LR from the saved PSP frame) +# +# Driven by hang_thread_walk.sh, which has connected to the OpenOCD gdb server. +# All run/halt sequencing happens below via `monitor`, so gdb never believes the +# target is running -- hence the explicit register-cache flush after each halt +# and the stack/code caches disabled by the driver script. +# +# Build facts relied on (build-fprime-automatic-zephyr/zephyr/.config): +# CONFIG_THREAD_MONITOR=y -> _kernel.threads list + k_thread.entry exist +# CONFIG_THREAD_STACK_INFO=y -> stack_info.start/size are valid +# CONFIG_THREAD_NAME is not set -> no k_thread.name; we symbolize entry.pEntry +# CONFIG_USE_SWITCH is not set -> classic Cortex-M PendSV swap. Note the +# callee-saved regs live in the k_thread +# (struct _callee_saved = v1-v8 + psp), NOT +# on the stack, so callee_saved.psp points +# straight at the hardware exception frame: +# [r0,r1,r2,r3,r12,lr,pc,xpsr] +# CONFIG_FPU is not set -> that frame is exactly 0x20 bytes +# CONFIG_MP_MAX_NUM_CPUS=1 -> current thread is _kernel.cpus[0].current +# CONFIG_TICKLESS_KERNEL=y -> SysTick is reloaded per-timeout (last_load) +# CONFIG_CORTEX_M_SYSTICK_64BIT_CYCLE_COUNTER=y -> cycle_count is 64-bit + +# Cortex-M33 register block addresses used below (read as words): +# 0xE000E010 SYST_CSR 0xE000E014 SYST_RVR 0xE000E018 SYST_CVR +# 0xE000E100 NVIC_ISER0 0xE000E200 NVIC_ISPR0 +# 0xE000ED04 ICSR 0xE000ED24 SHCSR 0xE000ED28 CFSR 0xE000ED2C HFSR + +define hw_snapshot + printf "-- kernel time base --\n" + printf " cycle_count = %llu\n", (unsigned long long)cycle_count + printf " announced_cycles = %llu\n", (unsigned long long)announced_cycles + printf " curr_tick = %lld\n", (long long)curr_tick + printf " last_load = %#lx\n", (unsigned long)last_load + printf "-- SysTick (live peripheral) --\n" + printf " SYST_CSR = %#010lx (bit0 ENABLE, bit1 TICKINT, bit16 COUNTFLAG)\n", \ + *(unsigned long*)0xE000E010 + printf " SYST_RVR = %#010lx SYST_CVR = %#010lx\n", \ + *(unsigned long*)0xE000E014, *(unsigned long*)0xE000E018 + printf "-- interrupt state --\n" + printf " PRIMASK=%#lx BASEPRI=%#lx FAULTMASK=%#lx CONTROL=%#lx\n", \ + (unsigned long)$primask, (unsigned long)$basepri, \ + (unsigned long)$faultmask, (unsigned long)$control + printf " ICSR = %#010lx (bit22 ISRPENDING, bits[8:0] VECTACTIVE)\n", \ + *(unsigned long*)0xE000ED04 + printf " NVIC_ISER0=%#010lx NVIC_ISPR0=%#010lx SHCSR=%#010lx\n", \ + *(unsigned long*)0xE000E100, *(unsigned long*)0xE000E200, \ + *(unsigned long*)0xE000ED24 + printf " CFSR = %#010lx HFSR = %#010lx (both 0 => no CPU fault)\n", \ + *(unsigned long*)0xE000ED28, *(unsigned long*)0xE000ED2C + printf "-- live core --\n" + printf " pc=%#lx sp=%#lx msp=%#lx psp=%#lx\n", \ + (unsigned long)$pc, (unsigned long)$sp, \ + (unsigned long)$msp, (unsigned long)$psp + printf " pc -> " + info symbol $pc + printf " backtrace of the live context:\n" + bt 12 + printf " -- call chain (text-looking words above live sp) --\n" + stack_ras $sp 0x200 +end + +# kernel/timeout.c's static list -- the answer to "is anything still +# scheduled to wake, and when". Kept in its own command (invoked after +# thread_walk) so a problem here never costs us the thread data. +define timeout_walk + # An empty sys_dlist_t points at itself, so head == &timeout_list means + # nothing at all is waiting on the clock. + printf "-- timeout queue (kernel/timeout.c timeout_list @ %#lx) --\n", \ + (unsigned long)&timeout_list + printf " head = %#lx%s\n", (unsigned long)timeout_list.head, \ + (timeout_list.head == &timeout_list ? \ + " (EMPTY: nothing is waiting on time)" : " (timeouts pending)") + # dticks is a *delta* chain: entry N fires `sum(dticks[0..N])` ticks after the + # last announcement. A head whose remaining delta never shrinks between the + # two halts is the smoking gun for "the clock counts but nothing is announced + # to the timeout layer". + set $to_n = 0 + set $to_sum = (long long)0 + set $to_p = (struct _timeout *)timeout_list.head + while $to_p != (struct _timeout *)&timeout_list && $to_n < 12 + set $to_sum = $to_sum + (long long)$to_p->dticks + printf " [%d] _timeout @ %#lx dticks=%lld (fires in %lld ticks = %lld ms)\n", \ + $to_n, (unsigned long)$to_p, (long long)$to_p->dticks, $to_sum, \ + $to_sum * 1000 / 10000 + printf " fn = %#lx -> ", (unsigned long)$to_p->fn + info symbol $to_p->fn + set $to_p = (struct _timeout *)$to_p->node.next + set $to_n = $to_n + 1 + end + printf " (%d timeouts queued)\n", $to_n +end + +# The downlink chain, per Com subtopology instance. Every stage of it waits on +# a status handed back from the stage below: +# comQueue -> spacePacketFramer -> aggregator -> framer -> comStub -> driver +# comStub.comStatusOut -> framer -> aggregator -> spacePacketFramer -> comQueue +# so a status that never comes back parks the whole chain and the link goes +# silent while the rest of the system stays perfectly healthy. This dump says +# which stage is parked and whether frames are moving at all. +# ComQueue.m_state READY | WAITING (WAITING = sent, awaiting status) +# ComAggregator.m_allow_timeout false => in WAIT_STATUS, discarding timeouts +# TmFramer.m_masterFrameCount increments per frame emitted downstream -- +# compare across the two halts: not advancing +# means nothing is being framed at all +# ComStub.m_reinitialize true => still waiting for a drvConnected +# NOTE: ComQueue::run only publishes queue-depth telemetry and is NOT what +# drives the dequeue, so `run` being unconnected cannot cause silence. +# NOTE: only the UART subtopology instantiates a comStub; the LoRa one reaches +# its radio by another path, so `ComCcsdsLora::comStub` does not exist. +define com_state + printf "-- downlink chain state --\n" + printf " %-14s %-9s %-14s %-14s %s\n", \ + "instance", "ComQueue", "Aggregator", "TmFramer", "ComStub" + com_state_one ComCcsdsUart + printf " %-14s %-9s %-14s %-14s %s\n", "", "", "", "", "" + com_state_one ComCcsdsLora + printf " ComCcsdsUart::comStub: reinitialize=%d retry_count=%d\n", \ + (int)ComCcsdsUart::comStub.m_reinitialize, \ + (int)ComCcsdsUart::comStub.m_retry_count +end + +define com_state_one + printf " %-14s ", "$arg0" + # ComQueue::SendState: READY=0, WAITING=1 (ComQueue.hpp:103). Compared + # numerically because gdb loses the Svc:: enum context across the reconnect. + printf "%-9s ", ($arg0::comQueue.m_state == 0 ? "READY" : "*WAITING*") + printf "%-14s ", ($arg0::aggregator.m_allow_timeout ? \ + "FILL" : "*WAIT_STATUS*") + printf "mfc=%-3d vfc=%-3d ", (int)$arg0::framer.m_masterFrameCount, \ + (int)$arg0::framer.m_virtualFrameCount + set $cs_mfc = (unsigned long)$arg0::framer.m_masterFrameCount +end + +# Decode _thread_base.thread_state (include/zephyr/kernel_structs.h:52-72). +define state_bits + set $st = (unsigned long)$arg0 + printf "%#04lx [", $st + if $st == 0 + printf "READY/RUNNING" + end + if $st & 0x01 + printf "DUMMY " + end + if $st & 0x02 + printf "PENDING " + end + if $st & 0x04 + printf "SLEEPING " + end + if $st & 0x08 + printf "DEAD " + end + if $st & 0x10 + printf "SUSPENDED " + end + if $st & 0x20 + printf "ABORTING " + end + if $st & 0x40 + printf "SUSPENDING " + end + if $st & 0x80 + printf "QUEUED " + end + printf "]" +end + +# Symbolize every word in [$arg0, $arg0+$arg1) that looks like a Thumb return +# address into .text -- a hand-rolled unwind. Used instead of rewinding gdb's +# $pc/$sp into each blocked thread: that writes core registers and, when a frame +# is unrecoverable, wedges gdb ("attempt to assign to an unmodifiable value"). +# This is purely read-only and works no matter how mangled the frame is. +define stack_ras + set $ra_p = (unsigned long)$arg0 + set $ra_e = (unsigned long)$arg0 + (unsigned long)$arg1 + set $ra_n = 0 + while $ra_p < $ra_e && $ra_n < 20 + set $ra_w = *(unsigned long*)$ra_p + # Thumb code pointer inside this image's .text region. + if ($ra_w & 1) && $ra_w > (unsigned long)&__text_region_start && $ra_w < (unsigned long)&__text_region_end + printf " +%#04lx %#010lx ", $ra_p - (unsigned long)$arg0, $ra_w + info symbol $ra_w - 1 + set $ra_n = $ra_n + 1 + end + set $ra_p = $ra_p + 4 + end + if $ra_n == 0 + printf " (no text-looking return addresses in this window)\n" + end +end + +# Walk the CONFIG_THREAD_MONITOR list of every thread in the system. For each +# swapped-out thread recover where it will resume: callee_saved.psp points at +# the hardware exception frame PendSV entry pushed, so the stacked LR is at +# +0x14 and the stacked PC at +0x18 (callee regs v1-v8 are in the k_thread). +# +# $walk_sum accumulates each thread's CONFIG_SCHED_THREAD_USAGE cycle counter +# (base.usage.total), which is monotonic: comparing it between the two halts is +# a one-number answer to "did ANY thread get CPU time?". Do NOT use saved +# psp/resume-PC for this -- a healthy thread that blocks at the same line every +# cycle reproduces byte-identical values and would read as frozen. +define thread_walk + set $cur = _kernel.cpus[0].current + set $t = _kernel.threads + set $n = 0 + set $walk_sum = (unsigned long long)0 + printf "-- thread walk (current = %#lx) --\n", (unsigned long)$cur + while $t != 0 && $n < 32 + printf "\n [%d] k_thread @ %#lx%s\n", $n, (unsigned long)$t, \ + ($t == $cur ? " <== CURRENT" : "") + printf " entry = %#lx -> ", (unsigned long)$t->entry.pEntry + info symbol $t->entry.pEntry + # CONFIG_THREAD_NAME is off, so all 21 F' task threads share one entry + # symbol (Os::Zephyr::Task::zephyrEntryWrapper). The entry *argument* is + # the per-task pointer, which lands inside the owning F' component object + # -- symbolizing it is what actually names the thread. + printf " arg = %#lx -> ", (unsigned long)$t->entry.parameter1 + info symbol $t->entry.parameter1 + printf " state = " + state_bits $t->base.thread_state + printf " prio=%d preempt=%#x\n", (int)$t->base.prio, \ + (unsigned int)$t->base.preempt + printf " pended_on = %#lx%s\n", (unsigned long)$t->base.pended_on, \ + ($t->base.pended_on != 0 ? " (blocked on a wait queue)" : "") + printf " timeout = dticks=%lld node.next=%#lx%s\n", \ + (long long)$t->base.timeout.dticks, \ + (unsigned long)$t->base.timeout.node.next, \ + ($t->base.timeout.node.next != 0 ? " (queued in _kernel.timeouts)" : " (NO timeout armed)") + + printf " cpu cycles= %llu (base.usage.total)\n", \ + (unsigned long long)$t->base.usage.total + set $walk_sum = $walk_sum + (unsigned long long)$t->base.usage.total + set $sbase = (unsigned long)$t->stack_info.start + set $ssize = (unsigned long)$t->stack_info.size + set $tpsp = (unsigned long)$t->callee_saved.psp + printf " stack = base=%#lx size=%#lx top=%#lx\n", \ + $sbase, $ssize, $sbase + $ssize + printf " saved psp = %#lx", $tpsp + if $tpsp < $sbase + printf " *** BELOW STACK BASE == OVERFLOW ***\n" + else + if $tpsp > $sbase + $ssize + printf " *** ABOVE STACK TOP (wrong stack / not yet swapped) ***\n" + else + printf " used=%#lx of %#lx (%lu%% headroom left)\n", \ + ($sbase + $ssize - $tpsp), $ssize, \ + (unsigned long)(($tpsp - $sbase) * 100 / $ssize) + end + end + + # Resume PC/LR are only meaningful for a thread that is actually swapped + # out with a PendSV frame on its own stack. Skip the running thread (its + # live $pc/$sp were printed by hw_snapshot) and any bogus psp. + if $t != $cur && $tpsp >= $sbase && $tpsp + 0x20 <= $sbase + $ssize + set $rlr = *(unsigned long*)($tpsp + 0x14) + set $rpc = *(unsigned long*)($tpsp + 0x18) + printf " resume LR = %#lx -> ", $rlr + info symbol $rlr + printf " resume PC = %#lx -> ", $rpc + info symbol $rpc + echo -- call chain (text-looking words above the exception frame) --\n + # gdb splits user-command arguments on whitespace, so an expression like + # ($tpsp + 0x20) would arrive as three separate args -- precompute. + set $rs_a = $tpsp + 0x20 + set $rs_l = ($sbase + $ssize) - $rs_a + stack_ras $rs_a $rs_l + end + + set $t = $t->next_thread + set $n = $n + 1 + end + printf "\n-- %d threads walked; total CPU cycles across all threads = %llu --\n", \ + $n, (unsigned long long)$walk_sum +end + +# OpenOCD forwards its own log to the attached gdb, which interleaves +# "[rp2350.cm1] halted due to debug-request" mid-printf and shreds the report. +# Send it to a file instead (hang_thread_walk.sh prints the path). +monitor log_output hang-thread-walk-openocd.log + +echo \n================ RESET + FREE-RUN INTO THE HANG ================\n +monitor reset halt +monitor resume +echo hang_thread_walk: running to reach the failed state (window A)...\n +eval "monitor sleep %d", $WINDOW_A_MS +monitor halt +# The run/halt above went through `monitor`, so gdb still believes the target +# never moved and would serve register values cached from the reset halt (this +# bit the first version of this script: it reported pc=z_arm_reset at a halt +# 12s into the run). Reconnecting is the reliable resync: gdb re-queries the +# stop reason and gets the true state. (`monitor gdb sync` + `stepi` is the +# usual recipe but is not safe here -- when the halt lands mid-ISR that stepi +# can resume the target, after which every later read fails with "Cannot +# execute this command while the target is running".) Convenience variables +# survive the reconnect, so the A/B comparison below is unaffected. +detach +target extended-remote localhost:3333 +maintenance flush register-cache + +echo \n================ HALT A ================\n +hw_snapshot +set $A_cycles = (unsigned long long)cycle_count +set $A_tick = (long long)curr_tick +set $A_cvr = *(unsigned long*)0xE000E018 +set $A_current = (unsigned long)_kernel.cpus[0].current +thread_walk +timeout_walk +com_state +set $A_sum = $walk_sum + +eval "echo \\n================ FREE-RUN %d ms ================\\n", $WINDOW_B_MS +monitor resume +eval "monitor sleep %d", $WINDOW_B_MS +monitor halt +# The run/halt above went through `monitor`, so gdb still believes the target +# never moved and would serve register values cached from the reset halt (this +# bit the first version of this script: it reported pc=z_arm_reset at a halt +# 12s into the run). Reconnecting is the reliable resync: gdb re-queries the +# stop reason and gets the true state. (`monitor gdb sync` + `stepi` is the +# usual recipe but is not safe here -- when the halt lands mid-ISR that stepi +# can resume the target, after which every later read fails with "Cannot +# execute this command while the target is running".) Convenience variables +# survive the reconnect, so the A/B comparison below is unaffected. +detach +target extended-remote localhost:3333 +maintenance flush register-cache + +echo \n================ HALT B ================\n +hw_snapshot +set $B_cycles = (unsigned long long)cycle_count +set $B_tick = (long long)curr_tick +set $B_cvr = *(unsigned long*)0xE000E018 +set $B_current = (unsigned long)_kernel.cpus[0].current +thread_walk +timeout_walk +com_state +set $B_sum = $walk_sum + +echo \n================ VERDICT ================\n +printf "cycle_count A=%llu B=%llu delta=%lld\n", \ + $A_cycles, $B_cycles, (long long)($B_cycles - $A_cycles) +printf "curr_tick A=%lld B=%lld delta=%lld\n", \ + $A_tick, $B_tick, ($B_tick - $A_tick) +printf "SYST_CVR A=%#lx B=%#lx (free-running counter; equal is suspicious\n", \ + $A_cvr, $B_cvr +printf " but not conclusive -- it wraps every RVR ticks)\n" +if $B_cycles == $A_cycles && $B_tick == $A_tick + echo *** CLOCK IS FROZEN: no ticks announced across 4s -- SysTick ISR is not\n + echo *** running. Look at PRIMASK/BASEPRI/ICSR above and at whoever masked.\n +else + echo *** CLOCK IS ALIVE: ticks still advancing, so the hang is a scheduler /\n + echo *** blocked-thread problem, not a dead timer. The thread that should be\n + echo *** running is blocked -- see its pended_on + backtrace above.\n +end +printf "current thread A=%#lx B=%#lx -> %s\n", $A_current, $B_current, \ + ($A_current == $B_current ? "SAME (no context switch in 4s)" : "changed") +printf "thread CPU cyc A=%llu B=%llu delta=%llu -> %s\n", \ + (unsigned long long)$A_sum, (unsigned long long)$B_sum, \ + (unsigned long long)($B_sum - $A_sum), \ + ($A_sum == $B_sum ? \ + "*** NO thread got any CPU time in 4s: the system really is wedged" : \ + "threads are still being scheduled: the system is RUNNING") + +echo \n================ done (detaching; board left halted) ================\n +detach diff --git a/scripts/diag/hang_thread_walk.sh b/scripts/diag/hang_thread_walk.sh new file mode 100755 index 00000000..c3ba2489 --- /dev/null +++ b/scripts/diag/hang_thread_walk.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# hang_thread_walk.sh -- v5 of the /keys boot-hang forensics (see INVESTIGATION.md). +# +# v4 (hang_fault_bp.*) proved the hang is NOT a CPU fault: the fault-entry +# breakpoints never fire and a clean halt shows cm0 idle with CFSR=HFSR=0. So +# there is no faulting frame to catch -- the scheduler simply stops making +# progress. This probe takes the other approach INVESTIGATION.md calls for: +# reset, free-run into the hang, then halt TWICE a few seconds apart and +# * compare cycle_count / curr_tick / SysTick to see whether the clock is +# frozen or still ticking, and +# * walk _kernel.threads, recovering each swapped-out thread's resume PC/LR +# from its saved PendSV frame and unwinding it, to see exactly who is +# blocked and on what. +# Read logic lives in hang_thread_walk.gdb. +# +# Local bench only (not CI). Drives the board over a Raspberry Pi Debug Probe +# (CMSIS-DAP SWD). The two USB CDC ttys involved: +# BOARD_TTY /dev/tty.usbmodem1101 - the target's own USB CDC (F'/GDS link); +# telemetry goes silent here at the hang +# PROBE_TTY /dev/tty.usbmodem102 - the Debug Probe's UART bridge +# OpenOCD reaches SWD via the probe's CMSIS-DAP USB interface (not a tty); the +# ttys are used only to correlate/timestamp the hang. +# +# Read-mostly: it resets, runs, halts and reads state. The only target writes +# are core-register rewinds used to unwind each blocked thread's stack (restored +# right after, on a target that was reset at the start of the run and is left +# halted at the end). Never writes flash or config. +set -u + +# Free-run windows, in ms. A = reset -> first halt (long enough to reach the +# failed state), B = gap between the two halts (long enough to span the 30s +# telemetry cadence, so a thread that simply had nothing to do in a short window +# is not mistaken for a wedged one). +WINDOW_A_MS=${WINDOW_A_MS:-12000} +WINDOW_B_MS=${WINDOW_B_MS:-20000} + +BOARD_TTY=${BOARD_TTY:-/dev/tty.usbmodem1101} +PROBE_TTY=${PROBE_TTY:-/dev/tty.usbmodem102} + +# OpenOCD: the raspberrypi fork (do NOT substitute a nix/brew openocd -- the RP2350 +# support and the CMSIS-DAP build the bench relies on live in this fork). Local +# bench keeps it under ~/code/...; the CI runner uses ~/openocd. Override with +# OOCD_HOME. +OOCD_HOME=${OOCD_HOME:-} +if [ -z "$OOCD_HOME" ]; then + for cand in ~/code/github.com/raspberrypi/openocd ~/openocd; do + [ -x "$cand/src/openocd" ] && { OOCD_HOME="$cand"; break; } + done +fi +[ -x "$OOCD_HOME/src/openocd" ] || { echo "hang_thread_walk: openocd not found (set OOCD_HOME to the raspberrypi openocd checkout)"; exit 1; } +OCD="$OOCD_HOME/src/openocd" +OCD_ARGS=(-s "$OOCD_HOME/tcl" + -f "$OOCD_HOME/tcl/interface/cmsis-dap.cfg" + -f "$OOCD_HOME/tcl/target/rp2350.cfg" + -c "adapter speed 5000") + +HERE=$(cd "$(dirname "$0")" && pwd) +GDBCMDS="$HERE/hang_thread_walk.gdb" +[ -f "$GDBCMDS" ] || { echo "hang_thread_walk: missing $GDBCMDS"; exit 1; } + +# --- symbol ELF (local flight build; load addrs 0x1018_xxxx) --- +ELF="" +for cand in \ + build-fprime-automatic-zephyr/zephyr/zephyr.elf \ + build-artifacts/zephyr/fprime-zephyr-deployment; do + [ -f "$cand" ] && { ELF="$cand"; break; } +done +[ -z "$ELF" ] && { echo "hang_thread_walk: symbol ELF not found; run 'make generate build' first."; exit 1; } +echo "hang_thread_walk: ELF $ELF" +echo "hang_thread_walk: board $BOARD_TTY probe $PROBE_TTY" +echo "hang_thread_walk: windows A=${WINDOW_A_MS}ms B=${WINDOW_B_MS}ms" + +# --- ARM gdb --- +GDB="" +for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do + command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } +done +[ -z "$GDB" ] && GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | tail -1) +[ -z "$GDB" ] && { echo "hang_thread_walk: no arm/multiarch gdb found."; exit 1; } +echo "hang_thread_walk: gdb $GDB" + +# --- optional: capture the board CDC so telemetry-going-silent is timestamped --- +SERIAL_LOG=${SERIAL_LOG:-board-serial.log} +SNIFF_PID="" +if [ -e "$BOARD_TTY" ]; then + ( cat "$BOARD_TTY" > "$SERIAL_LOG" 2>/dev/null ) & + SNIFF_PID=$! + echo "hang_thread_walk: sniffing $BOARD_TTY -> $SERIAL_LOG" +else + echo "hang_thread_walk: note: $BOARD_TTY not present; skipping serial sniff" +fi + +# --- OpenOCD: init + keep the gdb server up; the .gdb file drives reset/run/halt --- +"$OCD" "${OCD_ARGS[@]}" -c "init" -c "echo {hang_thread_walk: gdb server on :3333}" & +OCD_PID=$! +cleanup() { [ -n "$SNIFF_PID" ] && kill "$SNIFF_PID" 2>/dev/null; kill "$OCD_PID" 2>/dev/null; } +trap cleanup EXIT +sleep 2 + +# All run/halt sequencing is done with `monitor` inside the command file, so gdb +# never thinks the target is running. That means gdb would happily serve stale +# cached data across a resume, so disable both memory caches here; the command +# file flushes the register cache after each halt. +GDB_ARGS=(-q -nx -batch "$ELF" + -ex "set pagination off" + -ex "set confirm off" + -ex "set print pretty on" + -ex "set backtrace past-main on" + -ex "set stack-cache off" + -ex "set code-cache off" + -ex "set \$WINDOW_A_MS = $WINDOW_A_MS" + -ex "set \$WINDOW_B_MS = $WINDOW_B_MS" + -ex "target extended-remote localhost:3333" + -x "$GDBCMDS") + +TIMEOUT_BIN=$(command -v timeout || command -v gtimeout || true) +if [ -n "$TIMEOUT_BIN" ]; then + "$TIMEOUT_BIN" 300 "$GDB" "${GDB_ARGS[@]}" \ + || echo "hang_thread_walk: gdb exited non-zero / timed out" +else + "$GDB" "${GDB_ARGS[@]}" \ + || echo "hang_thread_walk: gdb exited non-zero (see output above)" +fi + +echo "hang_thread_walk: done (board serial capture in $SERIAL_LOG," +echo " openocd log in hang-thread-walk-openocd.log)" diff --git a/west.yml b/west.yml index 73f68164..f5de61c3 100644 --- a/west.yml +++ b/west.yml @@ -29,7 +29,13 @@ manifest: - picolibc # C library - mbedtls # Crypto library - mcuboot # Bootloader support - - fatfs # FatFS (file system) support + - fatfs # FatFS (file system) support (SD card, mounted at /) + - littlefs # LittleFS - internal-flash key store, mounted at /keys. + # Without this the module is not in zephyr_modules.txt, + # so ZEPHYR_LITTLEFS_MODULE is undefined and Kconfig + # SILENTLY drops CONFIG_FILE_SYSTEM_LITTLEFS=y ("LittleFS + # module not available") -- the build stays clean and + # /keys simply never exists. - hal_st # Required for certain sensors - name: loramac-node @@ -87,6 +93,14 @@ manifest: revision: f4ead3bf4a6dab3a07d7b5f5315795c073db568d path: lib/zephyr-workspace/modules/fatfs + # Pinned explicitly, like every other module here, so it lands under + # lib/zephyr-workspace/ instead of the workspace topdir: the allowlist + # import above only makes the project visible, it keeps Zephyr's own + # `path: modules/fs/littlefs`, which resolves to the repo root. + - name: littlefs + revision: 8f5ca347843363882619d8f96c00d8dbd88a8e79 + path: lib/zephyr-workspace/modules/fs/littlefs + self: path: . west-commands: west-commands.yml From 466ce0cd23e6d3dbe9eacc083dd71efbd0327183 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Tue, 28 Jul 2026 01:22:11 +0200 Subject: [PATCH 18/29] docs: reset INVESTIGATION.md onto the two remaining open items The firmware blockers are fixed and hardware-verified (ec0bdb37), so the old investigation -- which chased a boot "hang" that turned out not to exist -- has served its purpose and is superseded. Its content stays in git history. New INVESTIGATION.md is scoped to the actual remaining goal: integration tests green on the local bench AND in CI. 1. provision_key_test.py errors in pytest *setup*, so the test body never runs. start_gds loops on CdhCore.cmdDisp.CMD_NO_OP and its two-item event sequence times out; recover_from_safe_mode is autouse=True and depends on start_gds, so every test in test/int/ errors with it. The firmware side is proven -- the identical PROVISION_KEY sent outside pytest provisions the board, and the router read routed=3 bypassed=3 rejected=0, so keyless commands are dispatched and none rejected. Leading hypothesis is a GDS downlink desync from resetting the board underneath a long-lived GDS (it logged "APID 2 received sequence count: 4 (expected: 1)"), which CI should not be exposed to since it power-cycles before starting GDS. Recorded as UNCONFIRMED: only 3 bypassed packets were counted against more attempts than that, so uplink loss is not ruled out. Ordered next steps included. 2. No over-the-air recovery from a mis-provisioned key. PROVISION_KEY is refused on a non-empty store, REMOVE_KEY refuses the last key, and ADD_KEY needs an authenticated link -- so a board keyed with the wrong value is unreachable from the ground; bench recovery needed an SWD erase of keystore_partition. Five options are laid out with their security/recoverability trade-offs; this is a decision for the project owners, not a quiet patch. Also adds a TODO to decouple the autouse fixture from start_gds regardless of the cause, records the bench procedure gotchas that cost time, and repoints the stale INVESTIGATION.md section references in TODO.md at git history. --- INVESTIGATION.md | 873 ++++++++--------------------------------------- TODO.md | 67 ++-- 2 files changed, 194 insertions(+), 746 deletions(-) diff --git a/INVESTIGATION.md b/INVESTIGATION.md index ae73a47e..fbe8daa4 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -1,760 +1,187 @@ -# Investigation: hmac-to-storage CI hardware failure +# Investigation: integration tests green on the local bench and in CI -## ROOT CAUSE — CONFIRMED ON HARDWARE (2026-07-27), single-variable A/B +**Goal: `provision_key_test.py` and the rest of the integration suite pass both +on the local bench and in CI, on a board that starts keyless.** -**The branch overflows the CommandDispatcher opcode table.** +The firmware-side blockers are fixed and verified on hardware (see commit +`ec0bdb37`; the previous edition of this file, which chased a boot "hang" that +turned out not to exist, is in git history and is superseded). What remains is +getting the *test path* to work, plus one design decision the provisioning flow +forces. -`PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp` -sized `CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350` against a deployment that -already had **348** commands. This branch adds `PROVISION_KEY`, `ADD_KEY` and -`REMOVE_KEY` to `TcSecurityDeframer`, and there are **two** deframer instances -(`ComCcsdsUart`, `ComCcsdsLora`) — 3 x 2 = **6 new commands, total 354 > 350**. +Two open items: -`CommandDispatcherImpl::compCmdReg_handler` inserts every opcode into a -fixed-capacity `Fw::RedBlackTreeMap` and asserts on failure: +1. `provision_key_test.py` errors in pytest setup on the bench — **cause not yet + established**. +2. A mis-provisioned key cannot be recovered from the ground — **needs a + deliberate decision**. -```c -const Fw::Success status = this->m_entryTable.insert(opCode, portNum); -FW_ASSERT(status == Fw::Success::SUCCESS, ...); // CommandDispatcherImpl.cpp:35 -``` - -So the 351st registration **panics the board during boot command registration**. -The downlink never comes up, GDS sees `device disconnected` / no response to -`CMD_NO_OP`, and `integration-uart` / `integration-radio` fail. There is no CPU -fault — it is a `k_panic` (`z_fatal_error(reason=4)` via `z_arm_svc`), which is -why every fault-vector probe in this document came up empty. - -### Evidence (local bench, board `/dev/tty.usbmodem1101`, probe `/dev/tty.usbmodem102`) +--- -Bytes of F' telemetry read from the board CDC in a 25 s window after flashing, -each a full clean `make generate build`: +## Issue 1 — `provision_key_test.py` never reaches its test body -| build | telemetry | -|---|---| -| `main` | **1328 bytes**, first at t+1.0 s | -| `hmac-to-storage` (as shipped) | **0 bytes** | -| `hmac-to-storage` + table 512 | **1392 bytes**, first at t+1.01 s | -| `hmac-to-storage` + table back to 350 | **0 bytes** | +### Status: unconfirmed. Firmware side proven; test/GDS side not. -The last two differ **only** in that one constant, both fully regenerated — this -is the root cause, not a correlate. +`make test-integration FILTER=provision_key` fails with +`ERROR ... test_provision_key` — an error in *setup*, not an assertion failure. +The test body never runs. -### The fix +### What is proven -``` -CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350 -> 512 -``` +- **The firmware provisioning path works.** Sending the identical command + outside pytest — + `fprime-cli command-send ComCcsdsUart.tcSecurityDeframer.PROVISION_KEY + --arguments 0 <32 hex chars>` — provisions the board: the key store went to + `valid=1 spi=0` with the expected bytes, survived a cold reboot, and + subsequently authenticated an uplink `SET_SEQ_NUM`. +- **The bypass allowlist works on a keyless board.** `CMD_NO_OP` + (`0x01000000`) and `PROVISION_KEY` (`0x2100B002` / `0x2200B002`) are all in + `kBypassOpCodes` (`Bypasser.cpp`), and the opcodes match a freshly generated + dictionary. +- **Commands were reaching the dispatcher.** After a bench session containing + several `CMD_NO_OP` attempts, `ComCcsdsUart::provesRouter` read + `routed=3 bypassed=3 rejected=0` over SWD — some uplink commands were accepted + and dispatched with no key present, and **nothing** was rejected. +- **The board was healthy and transmitting** during the failing runs: telemetry + flowing on the CDC, and GDS's `comm.py.log` showing deframed downlink. -Raised to 512 rather than 354 so the next few commands do not repeat this. Note -the same zero-headroom pattern elsewhere in `project/config`: -`MAX_PACKETIZER_CHANNELS = 202` vs 191 channels in use, and -`MAX_PACKETIZER_PACKETS = 22` vs exactly 22 packets. Any of these overflowing -fails the same way — a boot-time `FW_ASSERT`, not a graceful degradation. -**Adding commands, channels or packets to this deployment requires checking -these constants against the generated dictionary.** +### What fails ---- +`start_gds` (`test/int/conftest.py:113`, session-scoped) loops for 30 s doing +`send_and_assert_command("CdhCore.cmdDisp.CMD_NO_OP")`, which waits on a +two-item event sequence (`OpCodeDispatched` + `OpCodeCompleted`). That sequence +search times out on every attempt. -## Second, independent defect found and fixed: littlefs was never compiled in - -`prj.conf` sets `CONFIG_FILE_SYSTEM_LITTLEFS=y`, but the repo's `west.yml` -`name-allowlist` imports `fatfs` and **not `littlefs`**. Without the module in -`zephyr_modules.txt`, `ZEPHYR_LITTLEFS_MODULE` is undefined and Kconfig -**silently drops** the symbol (`# LittleFS module not available.`). The build -stayed clean and `/keys` simply never existed: no `lfs_*` symbols in the image, -every `fs_open("/keys/...")` in `TcSecurityDeframer` failing, the whole key-store -feature inert. - -Fixed by adding `littlefs` to the allowlist **and** pinning it as an explicit -project (like every other module here) so it lands under `lib/zephyr-workspace/` -instead of the workspace topdir. Verified: `CONFIG_FILE_SYSTEM_LITTLEFS=y`, -`CONFIG_FS_LITTLEFS_FSTAB_AUTOMOUNT=y`, 20 `lfs_*` symbols in the ELF. - -This was *not* the cause of the CI failure (the branch failed identically with -littlefs absent and with it present) but the feature cannot work without it. - -## Build-system trap that cost several hours - -**`make build` does not re-derive Kconfig from device-tree changes.** After -editing the board `.dtsi`, `make build` left a stale `CONFIG_FLASH_SIZE=4096` -while the DTS said 16 MB, putting `keystore_partition` out of bounds so -`get_block_size()` returned 0 and the `/keys` automount hit -`__ASSERT_NO_MSG(block_size != 0)` (`littlefs_fs.c:787`) — a boot panic that was -purely an artifact of the stale config. **Always `make generate build` when the -device tree changes.** Several intermediate bisect results in this session were -invalidated by this and had to be re-run. - -## Diagnostics built this session (remove before merge) - -- `scripts/diag/hang_thread_walk.{sh,gdb}` — reset, free-run, halt twice N s - apart; dumps clock/SysTick/interrupt state, walks `_kernel.threads` (naming - each F' task by symbolizing `entry.parameter1`, since `CONFIG_THREAD_NAME` is - off), dumps `timeout_list` and the whole downlink chain's state, and diffs the - two halts using each thread's monotonic `base.usage.total`. -- `scripts/diag/downlink_trace.{sh,gdb}` — breakpoints every hop of the - `comStub -> framer -> aggregator -> spacePacketFramer -> comQueue` status path. -- Pre-existing: `hang_forensics.tcl`, `hang_gdb.sh`, `hang_fault_bp.{sh,gdb}`. -- Stray capture files to delete: `board-serial.log`, - `hang-thread-walk-openocd.log`, `downlink-trace-openocd.log`. - -## Corrections to earlier conclusions in this document - -Everything below this line predates the hardware A/B and is **wrong in its -conclusions**, kept only for the audit trail: - -- There is **no hang, no wild jump, no stack overflow, no fs-lock deadlock and - no flash/XIP wedge.** With the failure present the system is fully healthy: - clock advancing, all three rate groups cycling, ~89% idle. Only the downlink - is dead. -- `PRIMASK=1` at `arch_cpu_idle+18` is the **normal** `cpsid i; wfi; cpsie i` - idle sequence, not a masked spin. -- `pc=0x101864b8` "inside `fs_open`" and `pc=0x20010480` "inside `fileManager`" - were artifacts — a stale FPB breakpoint, and callee-saved registers misread as - a PC. (A swapped-out Cortex-M thread's `callee_saved.psp` points straight at - the exception frame: stacked LR at +0x14, PC at +0x18. Reading +0x34/+0x38 - yields F' object addresses that look exactly like plausible wild pointers.) -- The `DT_SIZE_M(4) -> DT_SIZE_M(16)` flash-size fix is correct and necessary - (the chip really is a 16 MB w25q128), but it was never the CI blocker. -- The recommended littlefs -> NVS rework is **not** needed for this failure. +Because `recover_from_safe_mode` (`conftest.py:177`) is `autouse=True` **and** +depends on `start_gds`, that fixture runs for *every* test in +`PROVESFlightControllerReference/test/int/` — so when it fails, the whole +directory errors in setup, including `provision_key_test.py`. ---- +### Leading hypothesis (not yet tested) -## Local-bench result (2026-07-27) — v5 thread-walk probe: THERE IS NO HANG - -Built and ran the thread-walk probe INVESTIGATION.md called for -(`scripts/diag/hang_thread_walk.{sh,gdb}`): reset, free-run 12 s into the failed -state, halt, dump the clock/SysTick/interrupt state, walk every thread in -`_kernel.threads`, dump `kernel/timeout.c`'s `timeout_list`, then free-run 4 s -more and repeat, comparing the two halts. Board CDC `/dev/tty.usbmodem1101`, -Debug Probe `/dev/tty.usbmodem102`, current `zephyr.elf` (carries the -`DT_SIZE_M(16)` fix). - -**The firmware is not hung, not faulted, and not blocked. It is running -normally.** Measured across the two halts 4 s apart: - -| observation | value | -|---|---| -| `curr_tick` | 119896 → 159936 (**+40040 ticks = 4.004 s** at 10 kHz) | -| `cycle_count` | +600,603,820 (150 MHz, exactly 4 s) | -| `SYST_CSR` | `0x7` (ENABLE+TICKINT+CLKSOURCE), `SYST_CVR` advancing | -| `PRIMASK`/`BASEPRI`/`CFSR`/`HFSR` | `0`/`0`/`0`/`0` | -| `timeout_list` | 1 entry: `z_timer_expiration_handler`, `dticks=10` (the 1 ms base-rate `k_timer`, re-armed every ms) | -| thread CPU time (`base.usage.total`, summed) | +597,388,374 cycles in 4 s | - -Per-thread CPU consumed in that 4 s window (probe names each thread by -symbolizing `entry.parameter1`, since `CONFIG_THREAD_NAME` is off and all 21 -F′ tasks share the `zephyrEntryWrapper` entry symbol): +A GDS-side downlink desync caused by the bench workflow rather than a product +defect. After every board reset during the session GDS logged ``` -rateGroup50Hz 19.2M cmdSeq 2.2M fileManager 0.59M CdhCore::cmdDisp 0.19M -bg_thread_main 19.9M safeModeSeq 1.6M ComCcsdsLora::comQueue 0.41M -rateGroup1Hz 7.2M payloadSeq 1.6M fileDownlink 0.26M CdhCore::events 0.09M -rateGroup10Hz 5.3M ComCcsdsUart::aggregator 1.4M prmDb 0.16M CdhCore::tlmSend 0.08M -idle 537.2M (89% idle) fileUplink 0.13M +[WARNING] framing: APID 2 received sequence count: 4 (expected: 1) ``` -All three rate groups cycle, `CdhCore::tlmSend`/`events`/`cmdDisp` run, and the -main thread is looping through `startRateGroups()` → `timer.cycle()` → -`k_timer_status_sync()` exactly as designed. One halt caught the main thread -mid-`Svc::ActiveRateGroup::CycleIn_handlerBase` → `Os::Queue::send` to -`rateGroup50Hz`; another caught the CPU inside `sys_clock_isr` → -`z_timer_expiration_handler`. - -**Every prior "hang" reading was a misread of a healthy idle CPU.** `PRIMASK=1` -+ `pc=arch_cpu_idle+18` is not a masked spin — it is the normal -`cpsid i; wfi; cpsie i` idle sequence, halted at the `cpsie i`. `pc=0x101864b8` -"inside `fs_open`" and `pc=0x20010480` "inside `fileManager`" were likewise -artifacts (a stale FPB breakpoint, and callee-saved register values misread as -a PC). There is no wild jump, no stack overflow, no fs-lock deadlock, no -flash/XIP wedge, and no fatal-halt loop. **Items 1–4 of the "Next steps" below -are chasing a defect that does not exist.** - -### What IS broken: the downlink, not the system - -Three threads consumed **exactly zero** cycles in the 4 s window: - -- `usbd_thread` — parked in `k_msgq_get`, never woken -- `udc_rpi_pico_thread_0` — parked in `k_event_wait`, never woken -- `ComCcsdsUart::comQueue` — parked on its queue condvar, never woken, **while - `ComCcsdsUart::aggregator` on the same path burned 1.4M cycles** - -and the host sees **0 bytes in 15 s** on the board's CDC. (Checked against -`/dev/cu.usbmodem1101`, not `/dev/tty.*` — on macOS a `tty.` open blocks on -carrier detect and would produce a false "no data". The silence is real either -way.) - -So the CI symptom — GDS `device disconnected`, `CMD_NO_OP` never answered — is a -**dead USB CDC / UART downlink path on a fully healthy flight system**, not a -boot hang. That is where the investigation goes next: telemetry is aggregated -but never dequeued to the com driver, and the USB device stack is dormant. - -### Notes on the probe itself - -- Register resync matters: run/halt is sequenced with `monitor`, so gdb keeps - serving registers cached from the reset halt unless forced to re-read. The - first version reported `pc=z_arm_reset` at a halt 12 s into the run. - `monitor gdb sync` + `stepi` is the usual recipe but resumes the target when - the halt lands mid-ISR; the probe detaches and reconnects instead. -- The first "did anything move?" metric summed each thread's saved `psp` + - resume PC. That is **unsound** — a healthy thread that blocks at the same - line every cycle reproduces byte-identical values, and it reported - "IDENTICAL: no thread made any progress" on a running system. The probe now - sums `base.usage.total` (`CONFIG_SCHED_THREAD_USAGE=y`), which is monotonic. -- A swapped-out Cortex-M thread's `callee_saved.psp` points *straight at* the - hardware exception frame (stacked LR at `+0x14`, PC at `+0x18`); the callee - registers live in the `k_thread`, not on the stack. Reading them at `+0x34`/ - `+0x38` yields F′ object addresses that look like plausible wild pointers — - which is very likely the origin of the `0x20010480` red herring above. - -## Local-bench result (2026-07-27) — v6 downlink trace: ComQueue never leaves WAITING - -Follow-on to the v5 thread-walk. `scripts/diag/downlink_trace.{sh,gdb}` traces -the status path that is supposed to release the downlink, live from reset. - -**`Svc::ComQueue` is constructed in `WAITING` (`ComQueue.cpp:35`) and only ever -reaches `READY` via `comStatusIn` carrying `SUCCESS` (`ComQueue.cpp:236-247`). -Until that happens it never dequeues, so nothing is framed and the link is -silent from boot.** On this board it never happens, on *both* Com paths: - -| read | ComCcsdsUart | ComCcsdsLora | -|---|---|---| -| `comQueue.m_state` | `WAITING` | `WAITING` | -| `framer.m_masterFrameCount` | **0** | **0** | -| `aggregator.m_allow_timeout` | true (FILL) | false (WAIT_STATUS) | -| `comStub.m_reinitialize` | 0 (ready seen) | n/a (no comStub) | - -`m_masterFrameCount = 0` means **no TM frame has ever been framed on either -link** — this is not "telemetry stopped", it is "downlink never started". - -The status path is `comStub.comStatusOut -> framer -> aggregator -> -spacePacketFramer -> comQueue.comStatusIn`. Breakpointing every hop: - -- `ComStub::drvConnected_handler` **fires**, from - `ZephyrUartDriver::configure` ← `setupTopology`, and emits its one status. -- `ComAggregator::preamble` **fires for both aggregators** — this is the F′ - active-component preamble and the only place the aggregator emits an - unprovoked `comStatusOut` (`ComAggregator.cpp:24-27`); its other one is in - `doFill`, which needs data that cannot flow until ComQueue is released. -- `ComAggregator::comStatusIn_handler` **fires** with `condition.e = SUCCESS`. -- `SpacePacketFramer::comStatusIn_handler` **fires for both instances** — the - last hop before ComQueue, and a pure pass-through - (`SpacePacketFramer.cpp:79-83`). -- `ComQueue::comStatusIn_handler` **never fires, on either instance.** - -The gap is not wiring. Both `comStatusOut` ports are connected at runtime — -`ComCcsdsUart::spacePacketFramer.m_comStatusOut_OutputPort[0].m_port == -&ComCcsdsUart::comQueue.m_comStatusIn_InputPort[0]` (and likewise for LoRa), -matching `ReferenceDeploymentTopologyAc.cpp:2396`. Nor is it a dead thread: -`ComCcsdsLora::comQueue.run_handler` is dispatched every second on that same -comQueue thread, so the thread is alive and draining its queue — it simply -never receives a `comStatusIn` message. - -Two things this rules out, both of which looked promising: - -- **`ComCcsdsUart.comQueue.run` being unconnected is NOT the bug.** It really is - unconnected (only `ComCcsdsLora.comQueue.run` is wired, `topology.fpp:268`), - but `ComQueue::run_handler` only publishes queue-depth telemetry - (`ComQueue.cpp:257`); the dequeue is driven by `comStatusIn`. It is also - unconnected on `main`. Cost: one missing telemetry channel. -- **The v5 per-thread zeros were partly a short-window artifact.** A single 4 s - sample showed `ComCcsdsUart::comQueue` at 0 cycles and `ComCcsdsLora:: - aggregator` at 0 while their opposite numbers ran — a mirror-image asymmetry - that does not survive contact with the trace above (both paths are equally - stuck). The probe now takes a configurable window (`WINDOW_A_MS`/ - `WINDOW_B_MS`, default 12 s / 20 s) so a thread that merely had nothing to do - is not read as wedged. - -**What this means for the branch.** `git diff main...HEAD` touches *no* code in -the com path — `topology.fpp`, `instances.fpp` and all of `lib/fprime` are -byte-identical to `main`. So if this is a regression it is being caused -indirectly, and the only boot-time behavioural change the branch makes is the -new `/keys` littlefs automount (`prj.conf` `CONFIG_FILE_SYSTEM_LITTLEFS=y` plus -the `lfs1` fstab node + `keystore_partition` in the board `.dtsi`). - -**Control experiment (in progress):** same tree, `automount` removed from the -`lfs1` node so `/keys` is not mounted at boot, everything else unchanged. If the -downlink returns, the boot-time littlefs mount is the cause and the key store -must move off an automounted filesystem; if it does not, the failure predates -the branch's config and the next control is a `main` build on this bench. - -## Next steps (post-root-cause) — pin *why* control jumps to `0x20010480` - -> **Caveat (see local-bench result above):** on the current build the hang does -> **not** raise a CPU fault, so steps 1–2's fault-entry breakpoints won't fire -> as written. Treat the "wild jump to `0x20010480`" framing as unconfirmed on -> today's build — the observed state is a no-fault idle/starved-scheduler hang. -> The stack-overflow-vs-wild-pointer question is still the crux, but the catch -> mechanism must change (watchpoint / thread walk, not a fault breakpoint). - -Ordered by diagnostic-value-per-effort. The whole task now is separating the two -live candidates: a **stack overflow** in a worker thread that smashed a -neighbouring return address / function pointer, vs. a **wild/uninitialized -pointer** (bad F′ port or handler) branched through directly. - -**1. Catch the fault at ENTRY, not after the halt (do this first — no rebuild).** -Every diagnostic so far halted the board *after* `z_arm_fatal_error` had already -run and masked IRQs, which is why the backtrace is garbage (`lr=0x1019`, shallow -stack). Set a **hardware** breakpoint on the UsageFault vector entry -`z_arm_usage_fault` (`0x10104fa0`), `monitor reset halt`, `continue`, and let it -hit at ~8 s with the exception frame fresh and `EXC_RETURN` still in `LR`. From -that frame, read the **true stacked PC** (the faulting instruction), the -**stacked LR** (the real caller/return), and compare the **pre-fault SP** to the -faulting thread's `stack_info.start/size` (valid — `CONFIG_THREAD_STACK_INFO=y`). -That single frame decides overflow (SP past its bound) vs. wild pointer (SP -sane, PC/LR corrupted), and the UFSR bits (`INVSTATE`/`UNDEFINSTR` @ `0xE000ED28`) -confirm "executed a data address". Runs on the *current* build over SWD — zero -turnaround. Script: **`scripts/diag/hang_fault_bp.sh`** + `hang_fault_bp.gdb`. -If the fault doesn't route through the UsageFault vector (e.g. a HardFault -escalation), rerun with `FAULT_SYM=z_arm_fault` to break on the common C handler -instead (it reads `EXC_RETURN`/`msp`/`psp` from `r2`/`r0`/`r1` per `fault.c:1025`). -This supersedes the earlier "walk `_kernel.threads` on the hung state" idea (same -data, but you'd have to dig the frame out of post-halt state by hand). - -**2. If (1) points at overflow, prove the site with a rebuild.** The build has -**both** stack-overflow traps OFF — `# CONFIG_HW_STACK_PROTECTION is not set`, -`# CONFIG_STACK_SENTINEL is not set` — so an overrun silently corrupts adjacent -RAM (exactly the "wild pointer *inside* the `fileManager` object" signature). A -diagnostic build with `CONFIG_HW_STACK_PROTECTION=y` (+ `CONFIG_STACK_SENTINEL=y`, -`CONFIG_THREAD_ANALYZER=y`) faults on the *overflowing write* and names the -thread + high-water mark — earlier and more precise than waiting for the eventual -wild jump. - -**3. Audit the branch's new on-stack buffers.** `TcSecurityDeframer.cpp:185` and -`:219` put `uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]` on the stack -(plus HMAC scratch) in the +289-line feature. Verify `kTCSecurityTrailer` sizing -against every write into `keyBytes`, and check the F′ thread stack the deframer -runs on. A bounded overrun in the hot path the flash-size fix newly unblocked -fits the timeline. - -**4. Make the fatal handler talk before it spins (kills the guessing loop).** -Override `k_sys_fatal_error_handler` (or set `CONFIG_EXTRA_EXCEPTION_INFO=y`) to -emit faulting-thread + real stacked PC/LR/CFSR over SWD/RTT *before* the masked -`b .`. - -**5. Treat the amplifier as its own defect (post-root-cause follow-up).** One -non-recoverable fault → IRQs masked → infinite spin → dead SysTick → silent -system death is a reliability hole independent of the cause. Track a hardware -watchdog so the board *resets* (CI sees a reboot, not a permanent "disconnect") -instead of going dark. - -**Housekeeping:** `scripts/diag/hang_fault_bp.{sh,gdb}` and -`scripts/diag/hang_thread_walk.{sh,gdb}` join the existing -`hang_forensics.tcl` / `hang_gdb.sh` diagnostics that **must be removed before -merge**, along with the `board-serial.log` / `hang-thread-walk-openocd.log` -capture files they drop in the repo root. (These v4/v5 scripts are local-bench -only — no `ci.yaml` change — so nothing new to revert in CI.) +i.e. its deframer's expected space-packet sequence count was stale relative to a +board that had rebooted underneath a long-lived GDS process. If GDS is +discarding out-of-sequence frames, the command *is* dispatched on the board (the +router counters agree) but the responding events never reach the test API — an +exact match for the observed symptom. + +CI does not have this exposure: `.github/actions/flash-firmware` power-cycles the +board (korad) *before* flashing and again after, and GDS is started afterwards, +so GDS never outlives a board reset. + +**Counter-evidence that keeps this unconfirmed:** the router counted only 3 +bypassed packets, while the failing runs plus the manual send should have +produced more `CMD_NO_OP` attempts than that. So it is *not* established that +every attempt reached the board; some may have been lost on the uplink instead. +Do not treat the desync theory as settled. + +### How to settle it + +Cheapest first: + +1. **Clean-slate bench run, mimicking CI ordering.** Power-cycle (or + `reset run`) the board, *then* start GDS, *then* run + `make test-integration FILTER=provision_key` — with no SWD session attached + and no resets while GDS lives. If it passes, the desync theory holds and the + bench procedure (not the code) was at fault. +2. **Count uplink arrivals directly.** Before/after a single `CMD_NO_OP`, read + `ComCcsdsUart::provesRouter.m_routedPackets` / `m_bypassedPackets` / + `m_rejectedPackets` over SWD. A `+1` per attempt proves the uplink is intact + and moves the problem entirely to the downlink/GDS side; no change proves the + uplink is dropping frames. **Resume the target before detaching** — a halted + board drops its USB CDC, which silently no-ops everything (this bit us). +3. **If it is the downlink**, check whether GDS's space-packet sequence check + should reset on a detected discontinuity, and whether the deframer's APID + sequence state needs a resync path after a spacecraft reboot. Note there is + already a `sync-sequence-number` make target and a "Sync Sequence Number" + CI step for the *anti-replay* counter — a different counter, but the same + class of ground/flight desync. +4. **Let CI settle it.** CI has never been green on this branch, so the + dispatch-table fix may simply expose this step for the first time. Push and + read the result before doing more bench work. + +### Independent of the cause: the fixture coupling is worth fixing + +`recover_from_safe_mode` is `autouse=True` and pulls in `start_gds` for every +test in the directory, so one uncooperative `CMD_NO_OP` takes out the entire +suite in setup — including the very test whose job is to bootstrap the board +into a state where commanding works. Making that fixture opt-in (or having it +tolerate an unavailable link) would decouple "the board is keyless" from "no +test can run", and would make failures report as failures rather than errors. --- -## Finding 1 — Reads do NOT disable interrupts (contradicts PROBLEM.md hypothesis #2) +## Issue 2 — No over-the-air recovery from a mis-provisioned key -`lib/zephyr-workspace/zephyr/drivers/flash/flash_rpi_pico.c`: +### Status: confirmed behaviour, needs a decision. -- `flash_rpi_read` (line 46) is a bare `memcpy` from the XIP-mapped flash window. - **No `irq_lock`.** -- Only `flash_rpi_write` (line 82) and `flash_rpi_erase` (line 132) take - `irq_lock()` / `irq_unlock()`. +The key store cannot be re-keyed from the ground once it holds a wrong key: -Therefore `loadKeyStore()` — a pure `fs_open`+read invoked on every -unrecognized-SPI frame (`TcSecurityDeframer.cpp:70`) — **cannot** stall USB via -interrupt disable. The mid-flight `fs_open` PC sample that `PROBLEM.md` treats as -its strongest evidence (`cm0: pc=0x101864b8 → fs_open`) is a *harmless read*, not -a stall. `PROBLEM.md` item #2 (per-frame reload) is a non-issue for USB timing. +- `PROVISION_KEY` is refused when the store is non-empty (`KeyProvisionFailed` + with `NotEmpty`) — by design, so an attacker cannot overwrite the key. +- `REMOVE_KEY` refuses to remove the **last** key (`LastKey`) — by design, so + the board cannot be locked out. +- `ADD_KEY` requires an already-authenticated link, which a wrong key cannot + provide. -## Finding 2 — `keystore_partition` is placed past the end of the declared flash +Together these mean a board provisioned with the wrong value is unreachable: +every recovery command needs either an empty store or a valid key, and neither +is obtainable. Hit during this session's bench work; recovery required a +physical SWD erase of `keystore_partition`: -`boards/bronco_space/proves_flight_control_board_v5/proves_flight_control_board_v5.dtsi:77`: - -``` -&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; } /* 4 MB */ ``` -Confirmed in the generated build: `CONFIG_FLASH_SIZE=4096`, -`CONFIG_FLASH_BASE_ADDRESS=0x10000000` -(`build-fprime-automatic-zephyr/zephyr/.config`). - -Partition map (`.dtsi` / generated `zephyr.dts`): - -| partition | offset | end | note | -|----------------------|------------|------------|------------------------------| -| boot (mcuboot) | 0x000000 | 0x100000 | | -| slot0 (current) | 0x100000 | 0x200000 | | -| slot1 (golden) | 0x200000 | 0x300000 | | -| slot2 (test) | 0x300000 | 0x400000 | fills the entire 4 MB | -| **keystore_partition** | **0x400000** | 0x440000 | **starts AT the 4 MB end** | -| storage_partition | 0x440000 | 0x1000000 | runs to 16 MB | - -`keystore_partition@0x400000` begins exactly at the declared flash boundary and -is entirely out of bounds. The flash driver gate: - -```c -#define FLASH_SIZE KB(CONFIG_FLASH_SIZE) /* = 4 MB = 0x400000 */ -static bool is_valid_range(off_t offset, uint32_t size) { - return (offset >= 0) && ((offset + size) <= FLASH_SIZE); -} +openocd ... -c "init; halt; flash erase_address 0x10400000 0x40000; reset run; exit" ``` -`flash_area_*` passes the partition's flash-relative offset (`0x400000`) to the -driver, so `is_valid_range(0x400000, size)` → `(0x400000 + size) <= 0x400000` → -**false → `-EINVAL`**. This check runs **before** the `irq_lock` in both -`flash_rpi_write` and `flash_rpi_erase`. - -Consequences: +after which littlefs re-formatted the partition on the next boot and the board +came back keyless (`valid=0`, `seqnum=0`). -1. `lfs_mount` → format (`littlefs_fs.c:966–971`) → `erase` → `-EINVAL` → format - fails → **`/keys` never mounts.** The feature as shipped cannot persist keys - or the sequence number at all. -2. Since no keystore flash op ever executes, **no interrupt is ever disabled for - it** — the interrupt-stall mechanism in `PROBLEM.md` has nothing to act on. +This is fine on the bench and fatal in flight. -## Finding 3 — This is a latent flash-size mis-declaration, newly exposed +### Options to weigh -On `main`, `storage_partition` already ran `0x400000 → 0x1000000` (12 MB at -offset 4 MB) against the same 4 MB `flash0` declaration (confirmed via -`git diff main..hmac-to-storage` on the `.dtsi`). It never mattered because -**nothing mounted `storage_partition`** — the SD card (FAT, `disk-access`) serves -`/`, and that is the only fstab mount on `main`. +- **Accept it**, and make provisioning a controlled ground procedure with a + verification read-back before the board is buttoned up. Cheapest; leaves a + single-point failure with no in-orbit remedy. +- **Allow `ADD_KEY` from the bypass allowlist** so a second key can be added + unauthenticated, then the bad one removed. Restores recoverability but + substantially weakens the security posture — an attacker could inject a key. +- **Add an authenticated `CLEAR_KEY_STORE`/`FORMAT_KEYS`**, usable only over an + authenticated link. Does not help if the *only* key is wrong. +- **Two-slot bootstrap**: provision two keys at manufacture (`AuthKeyStore` has + 2 slots), so a rotation error still leaves one working key. Recoverable + without weakening bypass, at the cost of a provisioning-procedure change. +- **Time-boxed bypass window after boot**, e.g. accept `PROVISION_KEY` on a + non-empty store only within N seconds of reset. Recoverable via a power cycle; + needs care that it is not a standing attack window. -This branch is the first to actually *mount* a filesystem at an offset ≥ 4 MB -(littlefs at `/keys`), which is what exposed the pre-existing mismatch. The 16 MB -extent of `storage_partition` strongly implies the physical chip is a 16 MB -W25Q128-class part and the `DT_SIZE_M(4)` declaration is simply wrong. - -## What still needs hardware confirmation - -The mount failure alone would normally degrade gracefully (the deframer tolerates -a missing store / keyless boot), so it is not yet proven that it is the *sole* -cause of the observed USB "device disconnected" symptom. It is, however, a -definite defect that makes the feature non-functional and that removes the basis -for the interrupt-stall theory. Confirm on hardware before ruling other causes -in or out. +No option is obviously right — this is a security/recoverability trade the +project owners should make explicitly, not a bug to be quietly patched. --- -## Storage layout for v5e (as built) - -- `zephyr,flash = &flash0` — internal QSPI/XIP flash, declared 4 MB (**suspected - under-declaration**; see Finding 3). -- fstab has two mounts: - - `ffs1` — **FAT on the SD card** (spi0 / sdmmc-disk, mount `/`, `disk-access`). - Separate SPI bus, no XIP interrupt-disable constraint. This is where the - sequence number lived before this branch. - - `lfs1` — **littlefs on `keystore_partition`** (mount `/keys`, `automount`) — - added by this branch; currently unmountable (Finding 2). -- USB CDC-ACM (`CONFIG_USB_DEVICE_STACK_NEXT`, v5e defconfig) is the GDS - transport; the "device disconnected" errors are on this link. - ---- - -## Fix applied - -No hardware bench was available this session to do the SWD-read / console-log -confirmation described below as the original "Step 0". Proceeded on the -strength of the existing evidence instead (Finding 3: `storage_partition` -already ran to 16 MB against this same 4 MB declaration on `main`, unnoticed -only because nothing mounted it). - -Change made: `&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; }` → -`DT_SIZE_M(16)` in `proves_flight_control_board_v5.dtsi` (shared by the v5c/v5d/v5e -board variants via `#include`). `make generate build` confirms `CONFIG_FLASH_SIZE` -now follows to 16384 (was 4096) with an otherwise identical, clean build (FLASH -69.82%, RAM 62.32% — unchanged from before the edit). - -**Confirmed on hardware (CI run 30034861047, 2026-07-23):** `Flash Firmware` -step's OpenOCD output reports `RP2350 rev 3, QSPI Flash win w25q128fv/jv id = -0x1840ef size = 16384 KiB in 4096 sectors` — the chip really is 16 MB, so the -`.dtsi` fix is correct and `CONFIG_FLASH_SIZE`/`is_valid_range` now permit the -`keystore_partition` range. - -**But `integration-uart`/`integration-radio` still fail identically** — GDS's -`comm.py.log` shows the same repeated -`Serial exception caught: device reports readiness to read but returned no -data (device disconnected or multiple access on port?). Reconnecting.` -starting ~19s after GDS start, and `start_gds`'s `CMD_NO_OP` never gets a -response within the whole 30s test window (both `provision_key_test.py` and -`format_filesystem_test.py` fail the same way in `integration-uart`; the radio -job fails at the identical `Bootstrap Sequence Number over UART` / -`provision_key` step). So the flash-size mis-declaration was real and worth -fixing, but it was **not the sole cause** of the CI failure. - -## Revised theory: PROBLEM.md's interrupt-stall theory may now apply for real - -Before this fix, every keystore flash op was rejected by `-EINVAL` *before* -`flash_rpi_write`/`flash_rpi_erase` ever reached `irq_lock()` — that's why -Finding 1/2 above concluded the interrupt-stall theory had "nothing to act on." -Now that `/keys` can actually mount, `lfs_mount`'s first-ever format on this -partition (superblock write, at minimum) is the first code in this branch that -actually reaches `flash_rpi_erase`/`flash_rpi_write`, both of which wrap the -*entire* erase/program call in `irq_lock()`/`irq_unlock()` -(`flash_rpi_pico.c:132-136`, `:82-107`) — with **no yielding** for however long -`flash_range_erase`/`flash_range_program` (Pico SDK bootrom calls) take. If -that takes long enough, USB CDC-ACM polling stalls exactly like the -symptom shows. This would mean the size fix was a necessary but not -sufficient change — PROBLEM.md's mechanism is real, it just couldn't fire -until the partition became mountable. - -**Not yet confirmed**: whether it's actually the littlefs format path (one-time, -at first boot on a virgin partition) vs. per-frame `loadKeyStore()`/ -`writeSequenceNumber` reloads causing repeated stalls throughout the test. The -timing (~19s in, then continuing through the whole 30s window) is at least -consistent with either. Console/log output is disabled in these builds -(`make check-console-disabled` is a required check), so there's no boot log -visibility in this CI run to distinguish the two — a temporary -`CONFIG_LOG=y` diagnostic CI run (same pattern as the prior fault-register -diagnostic commits `1ff5d8bb`/`78ca10f7`/`0949dfbd`/`693eb5d8`) would show -whether the stall is at first-mount/format or ongoing. - -## Console-log diagnostic (CI run 30036653676, 2026-07-23, reverted) - -Temporarily enabled `CONFIG_CONSOLE`/`CONFIG_UART_CONSOLE`/`CONFIG_PRINTK`/ -`CONFIG_LOG` and captured raw serial from a fresh power-cycle for 40s before -GDS ever opened the port (console shares `cdc_acm_uart0` with the F' downlink -per `scripts/check_console_disabled.py`, so this intentionally desynced GDS -for the run — same tradeoff as the prior fault-register diagnostics). - -Captured log (`console-boot.log`, full contents): - -``` -[00:00:00.034,000] LSM6DSO: Initialize device LSM6DSO -[00:00:00.035,000] LSM6DSO: chip id 0x6c -[00:00:00.785,000] sd: Maximum SD clock is under 25MHz, using clock of 24000000Hz -*** Booting Zephyr OS build v4.4.1 *** -[00:00:00.795,000] usbd_init: bNumInterfaces 2 wTotalLength 75 -[00:00:00.927,000] usbd_core: Actual device speed 1 -[00:00:01.040,000] usbd_core: Actual device speed 1 -[00:00:01.162,000] usbd_ch9: protocol error: (GET_DESCRIPTOR/DEVICE_QUALIFIER, "not supported" — routine/benign for a full-speed-only device, seen twice) -``` - -...then **nothing** for the remaining ~39s of the 40s capture window — no -further log lines, and critically no raw TM-frame bytes either (F' emits -periodic telemetry within the first second or two of a normal boot; even -console-corrupted binary noise from that would show up as bytes in the -capture). Ruled out one specific hypothesis: the Pico SDK's own -`flash_range_erase()` has a separate `hard_assert(flash_offs + count <= -PICO_FLASH_SIZE_BYTES)` guard (`lib/zephyr-workspace/modules/hal/rpi_pico/src/ -rp2_common/hardware_flash/flash.c`) independent of Zephyr's `CONFIG_FLASH_SIZE` -— but `PICO_FLASH_SIZE_BYTES` is not defined anywhere in this Zephyr build -(confirmed via grep across `lib/zephyr-workspace/zephyr/` and the generated -build's compile flags), so that `#ifdef`-guarded assert is compiled out -entirely and cannot be firing. - -**Reading**: total silence this early (before first telemetry) is consistent -with a full system hang very early in boot — plausibly right around where the -`/keys` fstab automount would run — but the console log alone doesn't prove -*where*. It could be `irq_lock()` held for a very long single erase/program -call (the "revised theory" above), or something else entirely blocking -further interrupt/scheduler activity. Diagnostic reverted (prj.conf + -ci.yaml back to pre-diagnostic state) since it can't safely coexist with a -working GDS link. - -**Next diagnostic (not yet run)**: repeat the original fault-register/SWD -approach (commits `1ff5d8bb` etc.) but at multiple time offsets *after* a -fresh flash — e.g. halt+dump PC at t=+2s, +10s, +20s — to see whether PC is -parked inside `flash_range_erase`/`flash_range_program`/the bootrom routines -they call for an extended period. That would confirm or rule out the -interrupt-stall theory directly without touching console/USB at all, avoiding -the corruption tradeoff entirely. - -## PC-sweep diagnostic (CI run 30043983799, 2026-07-23) — hang located - -Ran the sweep above (`PC Sweep Diagnostic` step, halt/resume only, no -firmware/config changes). Result for `cm0` (the core running Zephyr/F'): - -| offset | pc | lr | sp | xpsr | -|--------|------------|------------|------------|------------| -| t=2s | 0x101864b8 | 0x1010fb69 | 0x20034410 | 0x61000000 | -| t=10s | 0x101864b8 | 0x1010fb69 | 0x20034410 | 0x61000000 | -| t=20s | 0x101864b8 | 0x1010fb69 | 0x20034410 | 0x61000000 | - -**Identical PC/LR/SP/xPSR at all three offsets spanning 20 seconds** — cm0 -made zero forward progress the entire window. Resolved against this exact -commit's local build (`build-fprime-automatic-zephyr/zephyr/zephyr.elf`, -same `f5c3a124` the CI run built from): - -``` -101864b6 T fs_open -101864b6 T fs_open <- pc 0x101864b8 is fs_open+2 (its first real instruction) -1010fb48 T idle -1010fb90 t unpend_thread_no_timeout <- lr 0x1010fb69 is idle()+0x21 (its caller) -``` - -So cm0 is parked at the very entry of `fs_open()`, called from Zephyr's early -init sequence (which runs in the context that becomes the idle thread before -the scheduler starts other threads — this is the boot-time `SYS_INIT`/fstab -automount call chain, not literally the CPU-idle loop). This is consistent -with the **first real file operation this branch performs after a successful -`/keys` mount** — almost certainly `TcSecurityDeframer::configure()` calling -`loadKeyStore()`'s `fs_open()` on a virgin key-store file, i.e. exactly the -code path the flash-size fix newly unblocked. - -`cm1` (the second RP2350 core) sampled `pc=0x19e`/`msp=0xf0000000` unchanged -at all three offsets too — `0x19e` is a bootrom address (well below the -`0x10000000` XIP flash base), meaning **cm1 was never launched into Zephyr -code at all** and has been idling in the bootrom's core-1 launch stub since -reset. This app apparently runs single-core (cm1 unused/unlaunched). - -This is a real, reproducible, total hang (not a slow operation) at the exact -point the new keystore feature first touches the filesystem, and it fully -explains the console-log diagnostic's total silence after ~1.16s. It doesn't -by itself prove the *mechanism* (why `fs_open`/whatever it calls never -returns), but two mechanisms fit the facts and are worth checking first: - -1. **RP2350 dual-core flash lockout deadlock**: the vendored - `flash_range_erase`/`flash_range_program` in - `lib/zephyr-workspace/modules/hal/rpi_pico/src/rp2_common/hardware_flash/flash.c` - call `flash_exit_xip_func()`/ROM erase-program routines directly — no - `multicore_lockout`/`flash_safe_execute` handshake is visible in this - vendored copy, so a genuine multicore lockout wait is less likely here - than on stock Pico SDK, but worth double-checking the ROM functions - themselves don't internally expect core1's cooperation given cm1 was never - launched. -2. **A global fs/littlefs lock held by a different, already-wedged context**: - if some earlier code path (e.g. an interrupt-context or another thread) - is genuinely stuck inside a flash erase/program with `irq_lock()` held - indefinitely, `fs_open()`'s first action (typically taking a shared fs - mutex) would block forever waiting for a lock that will never be - released — cm0's halted PC/LR here would be showing the *this* thread's - blocked-on-mutex state, not literally an infinite loop inside `fs_open` - itself. Under this reading the real hang is still likely to be an - erase/program call somewhere in the mount/format path, just not the one - `fs_open` is calling right now. - -Both point at the same practical fix: avoid littlefs's first-time -format/create path on this partition. The pre-existing "Robustness -follow-ups" item — replacing the littlefs `/keys` mount with raw -`flash_area_*`/NVS for this fixed-size key store — sidesteps this class of -bug entirely regardless of which exact mechanism is at fault, since NVS -doesn't take a global fs lock and its record writes are simple, bounded -`flash_area_write`/`erase` calls with well-understood timing. - -One thing that's **ruled out** as the mechanism: a stale -`PICO_FLASH_SIZE_BYTES` hard_assert in the Pico SDK's -`flash_range_erase`/`flash_range_program` (`hard_assert(flash_offs + count <= -PICO_FLASH_SIZE_BYTES)`) — that macro is not defined anywhere in this Zephyr -build (confirmed via grep across `lib/zephyr-workspace/zephyr/` and the -generated build's compile flags), so the `#ifdef`-guarded assert is compiled -out entirely and cannot be firing. - -### Recommended fix - -> **Superseded / caveated by the SWD register forensics below (runs -> 30051402310 + 30052303346).** Those runs **ruled out** a wedged flash/XIP -> interface — the scenario in which switching to `flash_area_*`/NVS would have -> been wasted effort — but they also showed the hang is a *software* -> control-flow failure (interrupts masked via `PRIMASK`, an off-boundary PC, -> starved IRQ), not a clean littlefs mutex block. So "avoid the littlefs -> format path" is no longer established as *the* fix; the exact software cause -> is still being pinned (GDB backtrace, v3). Read the forensics section before -> acting on the recommendation below. - -Regardless of which exact mechanism is at fault, both candidates above point -at the same remedy: **avoid littlefs's mount/format/first-file-create path on -this partition entirely.** Replace the `/keys` littlefs mount with raw -`flash_area_*` calls or the Zephyr NVS backend for this fixed-size key store + -sequence-number counter. This is a nontrivial rework of -`TcSecurityDeframer`'s `loadKeyStore()`/`writeKeyStore()`/`writeSequenceNumber()` -(currently built on Zephyr's `fs_open`/`fs_read`/`fs_write` POSIX-ish file -API) to instead use `flash_area_open`/`flash_area_read`/`flash_area_write`/ -`flash_area_erase` directly against `keystore_partition`, or to adopt the NVS -subsystem's key-value API instead. Either avoids the littlefs mount/format -path entirely. **Not yet implemented.** - -### Diagnostic history (all reverted except the flash-size fix; CI is currently back to pre-diagnostic state otherwise) - -| run | change | result | reverted commit | -|-----|--------|--------|------------------| -| 30026433728 | (baseline, pre-fix) | fail, same USB-disconnect symptom | — | -| 30034861047 | flash0 `DT_SIZE_M(4)` → `DT_SIZE_M(16)` (kept, not reverted) | fail, same symptom; confirmed chip is 16MB | n/a — this is the real fix | -| 30036653676 | `CONFIG_CONSOLE`/`CONFIG_LOG` enabled | fail (expected); boot log silent after ~1.16s | `3cdfa541` | -| 30043983799 | SWD PC-sweep at t=+2s/+10s/+20s | fail (expected); cm0 frozen at `fs_open+2` throughout | `59b8178c` | -| 30051402310 | SWD forensics: SCB fault regs + QMI/XIP state (`hang_forensics.tcl`) | fail (expected); no fault/lockup, XIP healthy, `ISRPENDING`=1 — see forensics section | pending | -| 30052303346 | SWD forensics v2: capture-wrapped reg/stack + PRIMASK/BASEPRI | fail (expected); `PRIMASK`=1, off-boundary PC, shallow garbage stack | pending | - -The `hmac-to-storage` branch currently carries the flash-size fix (kept) plus -the **read-only SWD forensics diagnostic** (`scripts/diag/hang_forensics.tcl` -and a `Hang Forensics Diagnostic` step in `integration-uart`) — this one is -still active (not reverted) because the GDB-backtrace follow-up (v3) builds on -it. It must be reverted before merge, same as the earlier diagnostics. The -prior console/PC-sweep diagnostics remain cleanly reverted. - -## SWD register/QMI/stack forensics (runs 30051402310 + 30052303346, 2026-07-24) - -Read-only SWD capture (`scripts/diag/hang_forensics.tcl`): reset, free-run 8 s -so boot reaches the hang, halt cm0 once, and read the core registers, the -Cortex-M fault status block, the QMI/XIP peripheral state, and an SRAM stack -window. Deliberately reads **no** `0x10xx_xxxx` (XIP flash) address over SWD — -if the flash interface were wedged, such a read could stall the adapter. Two -runs: v1 got the fault/QMI reads; v2 fixed a capture bug (bare `reg`/`mdw` -output does not reach CI stdout in batch mode — only `echo`/`capture` does) and -added the register/stack dump plus the interrupt-mask registers. - -**Two hardware hypotheses ruled out (confirmed on both runs):** - -- **Not a CPU fault or lockup.** `CFSR = HFSR = DFSR = 0`; `DHCSR = 0x00130003` - → `S_HALT=1` but `S_LOCKUP=0` and `S_SLEEP=0` (not locked up, not in WFI). -- **Not a wedged flash/XIP interface** — this is the scenario in which a - littlefs→`flash_area_*`/NVS rewrite would have been *wasted*, because NVS - hits the same `flash_range_erase`. Ruled out: `XIP_CTRL=0x00000083` (XIP - enabled), `QMI_M0_RCMD=0x000000eb` (quad-read `0xEB` command intact), - `QMI_DIRECT_CSR=0x00c10800` (EN=0, BUSY=0 — not stuck in a direct/serial - transaction). The flash controller is idle and healthy. - -**Also ruled out:** SMP/second-core bringup. `CONFIG_MP_MAX_NUM_CPUS=1` — this -is a single-core build, so the `idle.c` SMP-without-IPI spin path is not -compiled and cm1 sitting in the bootrom (per the PC sweep) is expected and -irrelevant. - -**What the state actually is (v2, run 30052303346):** - -| register | value | reading | -|----------|-------|---------| -| `PRIMASK` | `0x01` | IRQs masked by `cpsid i` — **not** Zephyr's normal `irq_lock` (which uses `BASEPRI`, here `0x00`) | -| `ICSR` | `0x00400000` | `ISRPENDING=1`: an IRQ is pending and **starved** — this is what drops the USB CDC link | -| `control` | `0x02` | privileged thread on PSP | -| `pc` | `0x101864b8` | **not an instruction boundary** — `fs_open` opens with a 4-byte `stmdb` at `0x101864b6`; `…b8` is the second halfword, *inside* it | -| `lr` / `r5`| `0x1010fb69` (`idle+0x21`) / `0x1010fb49` (`&idle`) | but `idle()` never calls `fs_open` — incoherent as a live frame | -| `r1` | `0x0` | if this were `fs_open`'s `file_name` arg it is NULL, which returns `-EINVAL` immediately (`fs.c:145`) — can't hang *inside* `fs_open` | -| stack > SP | 2 words (`k_is_pre_kernel` ×2) then uninitialized garbage | shallow, not a coherent call chain | - -The earlier "`fs_open+2` = its first real instruction" reading (PC-sweep -section) was **wrong**: `0x101864b8` is mid-`stmdb`, not an instruction -boundary. Taken together — IRQs masked via `PRIMASK`, an off-boundary PC, an -incoherent LR, a shallow garbage stack, and no CPU fault — this is **not** a -clean mutex/lock block. It reads as execution gone off the rails: a -wild/corrupted PC, or a software panic-spin (`arch_system_halt()` also does -`cpsid`+infinite-loop with no CPU fault set), reached right when the keystore -feature first touches the filesystem. - -**Net for the fix decision:** the failure is a *firmware control-flow* bug, not -a flash/XIP hardware wedge — so this is not the case where moving off littlefs -is provably futile. But it is also not the clean fs-lock the rework was pitched -against, so the rework is not yet established as *the* fix either. The exact -software cause (wild jump vs. a tripped `__ASSERT`/`SPIN_VALIDATE` panic — -`CONFIG_ASSERT=y`, `CONFIG_SPIN_VALIDATE=y` are both set — vs. a lock-spin) -needs one more probe. - -**Next diagnostic (v3, in progress):** attach `arm-zephyr-eabi-gdb` to the -OpenOCD gdb server (port 3333, already opened by the same step) for a real -DWARF backtrace + `info threads` (`_current` thread) + a few single-steps to -see whether the PC advances and whether a `z_fatal_error`/`arch_system_halt` -frame is present. That distinguishes panic vs. wild-jump vs. lock-spin -directly and decides the fix. - -## Original next steps (superseded above, kept for the 4 MB fallback) - -Everything below was written before hardware confirmation; kept for -reference/fallback only — the chip is now confirmed 16 MB, so the fallback -branch does not apply: - -**Robustness follow-ups (independent of the size fix, evaluate after Step 0):** - -1. Consider replacing littlefs for this data with raw `flash_area_*` or the NVS - backend — it is a fixed-size key store + a 4-byte counter; a filesystem is - overkill and littlefs rewrites a whole file (extra erases) on every accepted - frame. NVS/raw erase far less often. -2. Rate-limit `loadKeyStore()` so it is not a fresh `fs_open` on every - unrecognized-SPI frame (read-only and harmless to USB, but wasteful). -3. If per-frame sequence-number persistence proves to be a real flash-wear or - timing problem after Step 0, either throttle `writeSequenceNumber` (persist - every N frames / on a timer, bounded replay window) or move only the seq - counter to a byte-writable, no-erase medium (RV3028 RTC battery-backed - user RAM/EEPROM on i2c1 — verify ≥4 usable bytes — or FRAM/MRAM if present). +## Bench procedure notes (learned the hard way) + +- **Always resume the target before detaching from OpenOCD/GDB.** A halted board + drops its USB CDC; GDS then sees nothing and commands silently no-op. Several + hours of this session were spent chasing failures that were only a halted + board. +- **Do not reset the board while GDS is running** — it desyncs the downlink + deframer and is the leading suspect for Issue 1. Restart GDS after any reset. +- **`make build` does not re-derive Kconfig from device-tree changes.** Use + `make generate build`; a stale `CONFIG_FLASH_SIZE` invalidated several bisect + results in this session. +- Use `/dev/cu.*`, not `/dev/tty.*`, when reading the board CDC from macOS — a + `tty.` open blocks on carrier detect and looks like a dead link. +- The `--active` flag on the vendored `uv` can resolve to a stale system + `fprime_gds`; `fprime-venv/bin/fprime-cli` is the reliable path, and it needs + `--deployment build-artifacts/zephyr/fprime-zephyr-deployment`. + +Diagnostics live in `scripts/diag/` (see the ADR item in `TODO.md`). diff --git a/TODO.md b/TODO.md index a7b0d2d7..bd4df75b 100644 --- a/TODO.md +++ b/TODO.md @@ -126,12 +126,11 @@ Status legend: [ ] todo, [~] in progress, [x] done `flash_rpi_erase`, both of which hold `irq_lock()` for the entire erase/program call with no yielding — i.e. the original `PROBLEM.md` interrupt-stall theory may be correct after all, it just couldn't fire before - (every op was rejected by `-EINVAL` pre-`irq_lock`). See `INVESTIGATION.md` - "Revised theory" section. **Next: a temporary `CONFIG_LOG=y` diagnostic CI + (every op was rejected by `-EINVAL` pre-`irq_lock`). (That theory was later disproved; see commit `ec0bdb37`.) **Next: a temporary `CONFIG_LOG=y` diagnostic CI run** (same pattern as the prior fault-register diagnostic commits) to see whether the stall is the one-time format or ongoing per-frame reloads. -## CI blocker — ROOT-CAUSED AND FIXED (2026-07-27), see INVESTIGATION.md +## CI blocker — ROOT-CAUSED AND FIXED (2026-07-27), see commit ec0bdb37 - [x] **Root cause: CommandDispatcher opcode-table overflow.** `project/config/CommandDispatcherImplCfg.hpp` had @@ -192,24 +191,46 @@ Status legend: [ ] todo, [~] in progress, [x] done resolved. (The Sband entry `0x2300B002` is still unconfirmed -- that instance is not built.) -- [ ] **Could not get `provision_key_test.py` to pass through the pytest - fixture path on the bench.** The firmware side is proven (item 2 above -- - the same command sent directly provisions the board), but - `start_gds`'s `CdhCore.cmdDisp.CMD_NO_OP` kept timing out, and - `recover_from_safe_mode` is `autouse=True` and depends on `start_gds`, so - every test in the directory errors in setup. The board *is* dispatching - those commands (`bypassed=3 rejected=0`), so this looks like a GDS-side - downlink desync from my reset-heavy bench session -- GDS logged - `APID 2 received sequence count: 4 (expected: 1)` after each board reset, - and CI power-cycles before starting GDS, which would avoid it. **Not - confirmed either way -- re-check once CI runs.** +## Current goal: integration tests green on the local bench AND in CI -- [ ] **No over-the-air recovery from a mis-provisioned key.** PROVISION_KEY is - refused once the store is non-empty (`NotEmpty`) and REMOVE_KEY refuses to - remove the last key, so a board provisioned with the wrong key cannot be - re-keyed from the ground -- it needs a physical SWD flash erase of - `keystore_partition` (which is how the bench board was recovered). Worth a - deliberate decision before flight. +Tracked in `INVESTIGATION.md`. The firmware blockers are fixed and verified on +hardware; what is left is the test path plus one design decision. + +- [ ] **Make `provision_key_test.py` pass on the bench and in CI.** It currently + errors in *setup*, so the test body never runs: `start_gds` loops for 30s + on `CdhCore.cmdDisp.CMD_NO_OP` (`conftest.py:113`) and its two-item event + sequence always times out. `recover_from_safe_mode` (`conftest.py:177`) is + `autouse=True` and depends on `start_gds`, so **every** test in + `test/int/` errors with it. + Firmware side is proven -- the identical PROVISION_KEY sent outside pytest + provisions the board, and the router showed `routed=3 bypassed=3 + rejected=0`, i.e. keyless commands are dispatched and none rejected. + Leading (unconfirmed) hypothesis: GDS downlink desync from resetting the + board underneath a long-lived GDS -- it logged `APID 2 received sequence + count: 4 (expected: 1)` after each reset. CI power-cycles before starting + GDS so it should not be exposed. **Counter-evidence: only 3 bypassed + packets were counted against more attempts than that, so uplink loss is + not ruled out.** Ordered next steps in `INVESTIGATION.md` -- start with a + clean-slate bench run in CI order (power-cycle, then GDS, then test, no + SWD attached), and push to let CI settle it. + +- [ ] **Decouple the autouse fixture from `start_gds` regardless of the cause.** + One uncooperative `CMD_NO_OP` currently takes out the whole suite in + setup -- including the very test whose job is to bootstrap a keyless board + into a commandable state. Make `recover_from_safe_mode` opt-in, or have it + tolerate an unavailable link, so failures report as failures rather than + errors. + +- [ ] **Decide how a mis-provisioned key is recovered.** Confirmed behaviour, + needs an explicit call rather than a quiet patch: `PROVISION_KEY` is + refused on a non-empty store (`NotEmpty`), `REMOVE_KEY` refuses the last + key (`LastKey`), and `ADD_KEY` needs an already-authenticated link -- so a + board keyed with the wrong value is unreachable from the ground. Recovery + on the bench required an SWD erase of `keystore_partition`. Fine on the + bench, fatal in flight. Options weighed in `INVESTIGATION.md` (accept it + with a verified ground procedure; bypass-allowlist `ADD_KEY`; authenticated + `CLEAR_KEY_STORE`; two-slot bootstrap provisioning; time-boxed post-boot + bypass window) -- each trades security against recoverability. - [ ] **Watch the other zero-headroom config constants.** Same failure mode, same file tree: `MAX_PACKETIZER_CHANNELS = 202` vs 191 channels in use, @@ -241,7 +262,7 @@ Status legend: [ ] todo, [~] in progress, [x] done exception frame (LR `+0x14`, PC `+0x18`); callee regs live in the `k_thread`. Reading `+0x34`/`+0x38` yields F' object addresses that look exactly like plausible wild pointers -- this produced a multi-day red - herring in `INVESTIGATION.md`. + herring in the earlier investigation (git history, pre-`ec0bdb37`). - `PRIMASK=1` at `arch_cpu_idle+18` is the **normal** idle sequence, not a masked spin. - Prefer monotonic `base.usage.total` over saved psp/PC when asking "did @@ -302,7 +323,7 @@ the stale config, and it invalidated several intermediate bisect results. key-store file for the first time). cm1 sampled `pc=0x19e` (a bootrom address) unchanged too — cm1 was never launched into Zephyr code at all, this app runs single-core. Two candidate mechanisms (both point at the - same fix, see `INVESTIGATION.md` "PC-sweep diagnostic" for full + same fix, see the earlier investigation in git history for full reasoning): (a) something in the mount/format path already holds `irq_lock()` in a flash erase/program that never returns, and `fs_open`'s first action (a shared fs mutex) blocks on it forever; (b) a dual-core @@ -329,7 +350,7 @@ the stale config, and it invalidated several intermediate bisect results. bytes in 15s from the board CDC. Telemetry is produced and aggregated but never dequeued to the com driver, and the USB device stack is dormant. **Next: chase the UART/USB downlink path**, not the filesystem. See - `INVESTIGATION.md` "v5 thread-walk probe: THERE IS NO HANG". + the earlier investigation in git history (pre-`ec0bdb37`). - [ ] **Robustness follow-ups (evaluate once the stall is diagnosed):** (a) consider raw `flash_area_*`/NVS instead of littlefs for this fixed-size store (avoids the format-time erase burst and any long single-call erase/program under From cb1d743c2d39c708b7101db8c1cf522d16e66cfe Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Tue, 28 Jul 2026 02:11:21 +0200 Subject: [PATCH 19/29] fix(test): make the integration suite pass on bench and unblock CI provision_key_test never reached its body. It awaited "either KeyProvisioned or KeyProvisionFailed" via await_event(satisfies_any([get_event_pred(a), get_event_pred(b)])) but IntegrationTestAPI.get_event_pred only passes its argument through when it is already an event_predicate. A satisfies_any is a predicate and not an event_predicate, so it was used as the event-ID predicate and its inner EventData checks were evaluated against an int -- never true. The event did arrive; the search timed out anyway and the None result surfaced as an AttributeError. Match over ids instead, and assert the event is not None so a real no-response failure reports as itself. The accompanying "every test in test/int/ errors in setup" was a halted board dropping its USB CDC, not a GDS downlink desync. start_gds used a bare `assert gds_working`, so that presented as 40-odd unexplained setup errors; it now names the command, attempt count and last exception. The redundant start_gds dependency is dropped from recover_from_safe_mode (every test file already requests start_gds directly). Four unrelated bench failures also fixed or marked: - rtc_test's uplink helper fired CreateDirectory /seq and uplinked immediately, racing the mkdir -- FileOpenError landed 30ms before CreateDirectorySucceeded whenever /seq did not already exist. Wait for the directory to resolve either way first. Real pre-existing race. - Without a battery the power monitor reads 0.012V, so modeManager auto-enters SAFE_MODE every debounce period and its sequence switches the face load switch off; enough cycles wedge the face I2C bus for the session. New --no-battery option drops SafeModeEntryVoltage to 0 before each test (per-test, since PRM_SET does not survive the reboots reset_manager performs). - drv2605 asserts a 0.3W rise in INA219 system power, which reads 0.0W on both samples without a battery -> requires_battery. - safe_09 asserts the boot count increments after a watchdog-driven power cycle, which cannot happen with JP6 open -> requires_watchdog_jumper. Markers are inert in CI, which never passes --bare-flight-controler-board. Also drop the two Hang Forensics CI steps: the root cause is found, and they halted the target over SWD immediately before GDS started -- the exact failure mode above. Bench result: 30 passed, 0 failed. Both key-store paths verified on hardware -- a keyless board (keystore erased over SWD) provisions and then authenticates, and an already-provisioned board reports NotEmpty. --- .github/workflows/ci.yaml | 22 -- INVESTIGATION.md | 212 +++++++++--------- .../test/int/antenna_deployer_test.py | 6 +- .../test/int/common.py | 20 ++ .../test/int/conftest.py | 50 ++++- .../test/int/drv2605_test.py | 5 +- .../test/int/mode_manager_test.py | 4 + .../test/int/provision_key_test.py | 23 +- .../test/int/rtc_test.py | 20 +- .../test/int/tmp112_test.py | 5 +- .../test/int/veml6031_test.py | 5 +- TODO.md | 65 ++++-- pytest.ini | 1 + 13 files changed, 271 insertions(+), 167 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8d618e58..183b2d31 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -127,28 +127,6 @@ jobs: - name: Flash Firmware uses: ./.github/actions/flash-firmware - # TEMPORARY DIAGNOSTIC (remove once root-caused): read-only SWD forensics - # to distinguish core-fault vs. wedged-XIP vs. fs-lock at the /keys boot - # hang, before deciding on the littlefs->NVS rework. See INVESTIGATION.md - # and scripts/diag/hang_forensics.tcl. Halt/read only; no firmware or - # config change, so the rest of the (still-failing) GDS flow below is - # undisturbed. - - name: Hang Forensics Diagnostic - run: | - ~/openocd/src/openocd -s ~/openocd/tcl \ - -f ~/openocd/tcl/interface/cmsis-dap.cfg \ - -f ~/openocd/tcl/target/rp2350.cfg \ - -c "adapter speed 5000" \ - -f "$GITHUB_WORKSPACE/scripts/diag/hang_forensics.tcl" || true - - # TEMPORARY DIAGNOSTIC (remove once root-caused): v3 -- attach GDB to the - # OpenOCD gdb server for a DWARF backtrace + single-step, to decide - # wild-jump vs. panic vs. lock-spin at the /keys hang. Read-only; skips - # gracefully if no arm/multiarch gdb is on the runner. See INVESTIGATION.md - # and scripts/diag/hang_gdb.sh. - - name: Hang Forensics GDB Backtrace - run: bash "$GITHUB_WORKSPACE/scripts/diag/hang_gdb.sh" || true - - name: Load .env file run: | while IFS= read -r line || [ -n "$line" ]; do diff --git a/INVESTIGATION.md b/INVESTIGATION.md index fbe8daa4..182770b2 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -3,124 +3,115 @@ **Goal: `provision_key_test.py` and the rest of the integration suite pass both on the local bench and in CI, on a board that starts keyless.** -The firmware-side blockers are fixed and verified on hardware (see commit -`ec0bdb37`; the previous edition of this file, which chased a boot "hang" that -turned out not to exist, is in git history and is superseded). What remains is -getting the *test path* to work, plus one design decision the provisioning flow -forces. +Status as of 2026-07-28: **the bench is green — 30 passed, 0 failed**, on a +flight control board with a face attached, no battery board, no antenna board +and JP6 open. Both key-store paths are verified on hardware: a freshly erased +(keyless) board provisions and then authenticates, and an already-provisioned +board reports `NotEmpty` and is treated as success. -Two open items: - -1. `provision_key_test.py` errors in pytest setup on the bench — **cause not yet - established**. -2. A mis-provisioned key cannot be recovered from the ground — **needs a - deliberate decision**. +One item remains open, and it is a design decision rather than a defect: +recovery from a board provisioned with the wrong key (§2). --- -## Issue 1 — `provision_key_test.py` never reaches its test body +## Issue 1 — `provision_key_test.py` never reached its test body — RESOLVED + +The previous edition of this file suspected a GDS downlink desync caused by +resetting the board underneath a long-lived GDS. **That theory was wrong.** Two +real defects were behind it, plus one bench-procedure artifact. + +### Cause A (the actual test failure): a mis-built event predicate + +`provision_key_test.py` awaited "either KeyProvisioned or KeyProvisionFailed" +with + +```python +await_event(satisfies_any([get_event_pred(...), get_event_pred(...)])) +``` -### Status: unconfirmed. Firmware side proven; test/GDS side not. +`IntegrationTestAPI.get_event_pred` returns its argument unchanged only when it +is already an `event_predicate`. A `satisfies_any` is a predicate but *not* an +`event_predicate`, so it fell through to being used as the **event-ID** +predicate — the two inner `EventData` checks were evaluated against an integer +id and could never be true. The search timed out even though the event had +arrived, and `evt` came back `None`, which then blew up as +`AttributeError: 'NoneType' object has no attribute 'template'`. -`make test-integration FILTER=provision_key` fails with -`ERROR ... test_provision_key` — an error in *setup*, not an assertion failure. -The test body never runs. +Fixed by matching over ids instead: -### What is proven +```python +await_event(is_a_member_of([translate_event_name(...), translate_event_name(...)])) +``` -- **The firmware provisioning path works.** Sending the identical command - outside pytest — - `fprime-cli command-send ComCcsdsUart.tcSecurityDeframer.PROVISION_KEY - --arguments 0 <32 hex chars>` — provisions the board: the key store went to - `valid=1 spi=0` with the expected bytes, survived a cold reboot, and - subsequently authenticated an uplink `SET_SEQ_NUM`. -- **The bypass allowlist works on a keyless board.** `CMD_NO_OP` - (`0x01000000`) and `PROVISION_KEY` (`0x2100B002` / `0x2200B002`) are all in - `kBypassOpCodes` (`Bypasser.cpp`), and the opcodes match a freshly generated - dictionary. -- **Commands were reaching the dispatcher.** After a bench session containing - several `CMD_NO_OP` attempts, `ComCcsdsUart::provesRouter` read - `routed=3 bypassed=3 rejected=0` over SWD — some uplink commands were accepted - and dispatched with no key present, and **nothing** was rejected. -- **The board was healthy and transmitting** during the failing runs: telemetry - flowing on the CDC, and GDS's `comm.py.log` showing deframed downlink. +plus an explicit `assert evt is not None` so a genuine no-response failure +reports as itself rather than as an `AttributeError`. -### What fails +### Cause B (why it looked like a setup error): a halted board -`start_gds` (`test/int/conftest.py:113`, session-scoped) loops for 30 s doing -`send_and_assert_command("CdhCore.cmdDisp.CMD_NO_OP")`, which waits on a -two-item event sequence (`OpCodeDispatched` + `OpCodeCompleted`). That sequence -search times out on every attempt. +The earlier bench sessions had left the target halted under SWD. A halted board +drops its USB CDC, so GDS saw nothing and `start_gds`'s `CMD_NO_OP` loop failed +— erroring every test in the directory during setup. With the board simply left +running, `start_gds` passes first time. -Because `recover_from_safe_mode` (`conftest.py:177`) is `autouse=True` **and** -depends on `start_gds`, that fixture runs for *every* test in -`PROVESFlightControllerReference/test/int/` — so when it fails, the whole -directory errors in setup, including `provision_key_test.py`. +`start_gds` used a bare `assert gds_working`, which is why this presented as an +unexplained error on 40-odd tests. It now reports the command, the attempt +count and the last exception. -### Leading hypothesis (not yet tested) +### Not a cause: the space-packet sequence-count warning -A GDS-side downlink desync caused by the bench workflow rather than a product -defect. After every board reset during the session GDS logged +GDS does log ``` -[WARNING] framing: APID 2 received sequence count: 4 (expected: 1) +[WARNING] framing: APID 2 received sequence count: 35 (expected: 1) +``` + +on every startup against an already-running board, but it is a warning only — +GDS adopts the received count and carries on. `reset_manager`'s cold- and +warm-reset tests both pass with GDS running across the reboot. + +--- + +## Bench-only failures, and what they actually were + +Four tests failed on the bench for reasons unrelated to the key store. Each is +now either fixed or correctly marked. + +| Test | Cause | Resolution | +| --- | --- | --- | +| `tmp112`, `veml6031` | With nothing at the battery terminals the power monitor reads 0.012 V, so modeManager auto-enters SAFE_MODE (`reason=LOW_BATTERY`) every debounce period; its safe-mode sequence switches the face load switch **off**, and enough of those cycles wedges the face I2C bus for the rest of the session. | New `--no-battery` pytest option drops `SafeModeEntryVoltage` to 0 before each test, so auto-entry never fires. Verified: 0 `AutoSafeModeEntry` events in a full run. | +| `antenna_deployer::test_deployment_prevention_after_success` | After `format_filesystem`, `/antenna` no longer exists, so `SET_DEPLOYMENT_STATE` fails with `FileOperationError ... on file open_write`. The directory is recreated at boot. | Bench procedure: power-cycle after formatting, which is what CI already does (`Format Filesystem` → `Power-Cycle Satellite`). Also added an `exit_safe_mode` to its fixture, since deployment is inhibited in safe mode. | +| `rtc_test::test_04_sequence_cancellation_on_time_set` | `uplink_sequence_and_await_completion` fired `CreateDirectory /seq` and uplinked immediately, racing the mkdir: `FileOpenError: Could not open file /seq/no_op.bin` 30 ms *before* `CreateDirectorySucceeded`. Only visible when `/seq` did not already exist. | Real pre-existing race, fixed: wait for `CreateDirectorySucceeded` **or** `DirectoryCreateError` (already-exists) before uplinking. | +| `drv2605::test_01_magnetorquer_power_draw` | Asserts a ≥0.3 W rise in INA219 system power; that rail reads 0.0 W in both samples without a battery. No command can fix an unpowered rail. | Marked `requires_battery` in addition to `requires_face`. | +| `mode_manager::test_safe_09` | Asserts the boot count increments after a watchdog-driven hardware power cycle; with JP6 open the reboot never happens (177 → 177). The command-loss detection and SAFE_MODE entry it also asserts both pass. | Marked `requires_watchdog_jumper`. | + +These markers are inert in CI: CI never passes `--bare-flight-controler-board`, +which is the only thing that acts on them. + +### Reproducing the green bench run + +```sh +# 1. board running and NOT halted under SWD; no GDS yet +PROVES_AUTH_KEY=<32 hex chars> make gds-integration UART_DEVICE=/dev/cu.usbmodem1101 & + +# 2. bootstrap, in CI's order +make test-integration FILTER=provision_key +make test-integration FILTER=sync_sequence_number + +# 3. the suite +make test-integration \ + FILTER="not sync_sequence_number and not format_filesystem and not provision_key \ + and not requires_antenna and not requires_battery and not requires_watchdog_jumper" \ + PYTEST_ARGS=--no-battery ``` -i.e. its deframer's expected space-packet sequence count was stale relative to a -board that had rebooted underneath a long-lived GDS process. If GDS is -discarding out-of-sequence frames, the command *is* dispatched on the board (the -router counters agree) but the responding events never reach the test API — an -exact match for the observed symptom. - -CI does not have this exposure: `.github/actions/flash-firmware` power-cycles the -board (korad) *before* flashing and again after, and GDS is started afterwards, -so GDS never outlives a board reset. - -**Counter-evidence that keeps this unconfirmed:** the router counted only 3 -bypassed packets, while the failing runs plus the manual send should have -produced more `CMD_NO_OP` attempts than that. So it is *not* established that -every attempt reached the board; some may have been lost on the uplink instead. -Do not treat the desync theory as settled. - -### How to settle it - -Cheapest first: - -1. **Clean-slate bench run, mimicking CI ordering.** Power-cycle (or - `reset run`) the board, *then* start GDS, *then* run - `make test-integration FILTER=provision_key` — with no SWD session attached - and no resets while GDS lives. If it passes, the desync theory holds and the - bench procedure (not the code) was at fault. -2. **Count uplink arrivals directly.** Before/after a single `CMD_NO_OP`, read - `ComCcsdsUart::provesRouter.m_routedPackets` / `m_bypassedPackets` / - `m_rejectedPackets` over SWD. A `+1` per attempt proves the uplink is intact - and moves the problem entirely to the downlink/GDS side; no change proves the - uplink is dropping frames. **Resume the target before detaching** — a halted - board drops its USB CDC, which silently no-ops everything (this bit us). -3. **If it is the downlink**, check whether GDS's space-packet sequence check - should reset on a detected discontinuity, and whether the deframer's APID - sequence state needs a resync path after a spacecraft reboot. Note there is - already a `sync-sequence-number` make target and a "Sync Sequence Number" - CI step for the *anti-replay* counter — a different counter, but the same - class of ground/flight desync. -4. **Let CI settle it.** CI has never been green on this branch, so the - dispatch-table fix may simply expose this step for the first time. Push and - read the result before doing more bench work. - -### Independent of the cause: the fixture coupling is worth fixing - -`recover_from_safe_mode` is `autouse=True` and pulls in `start_gds` for every -test in the directory, so one uncooperative `CMD_NO_OP` takes out the entire -suite in setup — including the very test whose job is to bootstrap the board -into a state where commanding works. Making that fixture opt-in (or having it -tolerate an unavailable link) would decouple "the board is keyless" from "no -test can run", and would make failures report as failures rather than errors. +If you run `format_filesystem`, reset the board before the suite so `/antenna` +and friends are recreated. --- ## Issue 2 — No over-the-air recovery from a mis-provisioned key -### Status: confirmed behaviour, needs a decision. +### Status: confirmed behaviour, still needs a decision. The key store cannot be re-keyed from the ground once it holds a wrong key: @@ -133,15 +124,15 @@ The key store cannot be re-keyed from the ground once it holds a wrong key: Together these mean a board provisioned with the wrong value is unreachable: every recovery command needs either an empty store or a valid key, and neither -is obtainable. Hit during this session's bench work; recovery required a -physical SWD erase of `keystore_partition`: +is obtainable. Recovery on the bench requires a physical SWD erase: ``` openocd ... -c "init; halt; flash erase_address 0x10400000 0x40000; reset run; exit" ``` -after which littlefs re-formatted the partition on the next boot and the board -came back keyless (`valid=0`, `seqnum=0`). +after which littlefs re-formats the partition on the next boot and the board +comes back keyless (`valid=0`, `seqnum=0`). This was used repeatedly during +this session's verification and works reliably. This is fine on the bench and fatal in flight. @@ -170,18 +161,25 @@ project owners should make explicitly, not a bug to be quietly patched. ## Bench procedure notes (learned the hard way) - **Always resume the target before detaching from OpenOCD/GDB.** A halted board - drops its USB CDC; GDS then sees nothing and commands silently no-op. Several - hours of this session were spent chasing failures that were only a halted - board. -- **Do not reset the board while GDS is running** — it desyncs the downlink - deframer and is the leading suspect for Issue 1. Restart GDS after any reset. + drops its USB CDC; GDS then sees nothing and commands silently no-op. This was + the single biggest time sink across two sessions, and it is what made Issue 1 + look like a product defect. +- Resetting the board while GDS runs is **fine** — GDS logs a sequence-count + warning and resyncs. (The earlier claim to the contrary was wrong.) +- After `format_filesystem`, **reset the board** before running the suite: + `/antenna` and `/seq` are recreated at boot. - **`make build` does not re-derive Kconfig from device-tree changes.** Use `make generate build`; a stale `CONFIG_FLASH_SIZE` invalidated several bisect - results in this session. + results in an earlier session. - Use `/dev/cu.*`, not `/dev/tty.*`, when reading the board CDC from macOS — a `tty.` open blocks on carrier detect and looks like a dead link. +- OpenOCD lives at `~/code/github.com/raspberrypi/openocd` (the raspberrypi + fork — do not substitute a nix/brew build). - The `--active` flag on the vendored `uv` can resolve to a stale system `fprime_gds`; `fprime-venv/bin/fprime-cli` is the reliable path, and it needs `--deployment build-artifacts/zephyr/fprime-zephyr-deployment`. -Diagnostics live in `scripts/diag/` (see the ADR item in `TODO.md`). +Diagnostics live in `scripts/diag/` (see the ADR item in `TODO.md`). The +`Hang Forensics` CI steps that drove them have been removed — they halted the +target over SWD immediately before GDS started, which is the exact failure mode +above. diff --git a/PROVESFlightControllerReference/test/int/antenna_deployer_test.py b/PROVESFlightControllerReference/test/int/antenna_deployer_test.py index cf724b51..e3ee9469 100644 --- a/PROVESFlightControllerReference/test/int/antenna_deployer_test.py +++ b/PROVESFlightControllerReference/test/int/antenna_deployer_test.py @@ -5,7 +5,7 @@ """ import pytest -from common import proves_send_and_assert_command +from common import exit_safe_mode, proves_send_and_assert_command from fprime_gds.common.data_types.event_data import EventData from fprime_gds.common.testing_fw.api import IntegrationTestAPI @@ -30,6 +30,10 @@ def configure_antenna_deployer(fprime_test_api: IntegrationTestAPI, start_gds): ("MAX_DEPLOY_ATTEMPTS", 3), ] + # Deployment is inhibited while FSW is in SAFE_MODE, which a bench with no + # power at the battery terminals auto-enters on a LOW_BATTERY reading. + exit_safe_mode(fprime_test_api) + proves_send_and_assert_command( fprime_test_api, f"{antenna_deployer}.SET_DEPLOYMENT_STATE", [False] ) diff --git a/PROVESFlightControllerReference/test/int/common.py b/PROVESFlightControllerReference/test/int/common.py index 88ae3f59..429701a2 100644 --- a/PROVESFlightControllerReference/test/int/common.py +++ b/PROVESFlightControllerReference/test/int/common.py @@ -40,6 +40,26 @@ def set_default_retries(n: int) -> None: _DEFAULT_RETRIES = n +def exit_safe_mode(fprime_test_api: IntegrationTestAPI) -> None: + """Command FSW out of SAFE_MODE so that face hardware stays powered. + + On a bench with no power at the battery terminals the power monitor reads + ~0 V, so modeManager auto-enters SAFE_MODE with reason=LOW_BATTERY and the + safe-mode sequence switches the face load switches off. Any subsequent + sensor read on that face then fails with EXECUTION_ERROR. Sending + EXIT_SAFE_MODE first clears the condition long enough for a short test to + run (auto-entry re-arms only after SafeModeDebounceSeconds). + + Best-effort: EXIT_SAFE_MODE is a no-op when the board is already in NORMAL, + and a failure here should surface as the real test's failure, not as a + setup error. + """ + try: + fprime_test_api.send_command("ReferenceDeployment.modeManager.EXIT_SAFE_MODE") + except Exception: # noqa: BLE001 - advisory only; the test itself is the assertion + pass + + def set_radio_recover_fn(fn: Callable[[], None] | None) -> None: """Register a callable to re-establish the radio link. diff --git a/PROVESFlightControllerReference/test/int/conftest.py b/PROVESFlightControllerReference/test/int/conftest.py index 933ebcee..9dec6bac 100644 --- a/PROVESFlightControllerReference/test/int/conftest.py +++ b/PROVESFlightControllerReference/test/int/conftest.py @@ -89,6 +89,17 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help="Override retry count for proves_send_and_assert_command (default: 3 UART, 5 radio).", ) + parser.addoption( + "--no-battery", + action="store_true", + default=False, + help="Bench is running on USB power with nothing at the battery terminals. " + "The power monitor then reads ~0 V, so modeManager auto-enters SAFE_MODE " + "with reason=LOW_BATTERY every SafeModeDebounceSeconds and its safe-mode " + "sequence switches the face load switches off; enough of those cycles " + "wedges the face I2C bus for the rest of the session. This option drops " + "SafeModeEntryVoltage to 0 before each test so the auto-entry never fires.", + ) parser.addoption( "--bare-flight-controler-board", action="store_true", @@ -119,8 +130,11 @@ def start_gds( GDS is used to send commands and receive telemetry/events. """ gds_working = False + attempts = 0 + last_error: Exception | None = None timeout_time = time.time() + 30 while time.time() < timeout_time: + attempts += 1 try: if request.config.getoption("--with-radio"): _enable_radio(fprime_test_api_session) @@ -129,9 +143,16 @@ def start_gds( ) gds_working = True break - except Exception: + except Exception as exc: # noqa: BLE001 - retried until the 30s budget runs out + last_error = exc time.sleep(1) - assert gds_working + # This fixture gates every test in the directory, so a bare assert here reports + # as an unexplained setup error on all of them. Say what actually failed. + assert gds_working, ( + f"No response to {cmdDispatch}.CMD_NO_OP after {attempts} attempts over 30s " + "- the GDS<->board link is not working, so no test in this directory can " + f"run. Last error: {last_error!r}" + ) if request.config.getoption("--with-radio"): # Allow the boot-time event backlog to drain before any test commands @@ -160,6 +181,30 @@ def _enable_radio(fprime_test_api: IntegrationTestAPI) -> None: ) +@pytest.fixture(autouse=True) +def suppress_low_battery_safe_mode( + request: pytest.FixtureRequest, fprime_test_api: IntegrationTestAPI +): + """With --no-battery, keep FSW out of LOW_BATTERY safe mode for each test. + + Applied per test rather than once per session because SAFEMODEENTRYVOLTAGE is + set with PRM_SET (not PRM_SAVE), so it reverts to its 6.7 V default on every + reboot - and the reset_manager tests deliberately reboot mid-suite. + + Fire-and-forget: this is bench scaffolding, and a failure to send it should + show up as the real test failing rather than as a setup error. + """ + if not request.config.getoption("--no-battery", default=False): + return + try: + fprime_test_api.send_command( + "ReferenceDeployment.modeManager.SAFEMODEENTRYVOLTAGE_PRM_SET", [0.0] + ) + fprime_test_api.send_command("ReferenceDeployment.modeManager.EXIT_SAFE_MODE") + except Exception: # noqa: BLE001 - advisory only + pass + + @pytest.fixture(autouse=True) def start_radio(request: pytest.FixtureRequest, fprime_test_api: IntegrationTestAPI): """Fixture to start the radio before tests""" @@ -178,7 +223,6 @@ def start_radio(request: pytest.FixtureRequest, fprime_test_api: IntegrationTest def recover_from_safe_mode( request: pytest.FixtureRequest, fprime_test_api: IntegrationTestAPI, - start_gds, ): """Best-effort: after each test, if FSW slipped into SAFE_MODE (e.g. low voltage brownout from a burnwire-heavy test, or a partial file upload diff --git a/PROVESFlightControllerReference/test/int/drv2605_test.py b/PROVESFlightControllerReference/test/int/drv2605_test.py index 49963330..29011271 100644 --- a/PROVESFlightControllerReference/test/int/drv2605_test.py +++ b/PROVESFlightControllerReference/test/int/drv2605_test.py @@ -12,7 +12,10 @@ from fprime_gds.common.models.serialize.time_type import TimeType from fprime_gds.common.testing_fw.api import IntegrationTestAPI -pytestmark = [pytest.mark.requires_face] +# requires_battery as well as requires_face: the assertion compares INA219 system +# power before and during the magnetorquer pulse, and on an unpowered bench that +# rail reads 0.0 W in both samples. +pytestmark = [pytest.mark.requires_face, pytest.mark.requires_battery] drv2605Manager = "ReferenceDeployment.drv2605Face0Manager" ina219SysManager = "ReferenceDeployment.ina219SysManager" diff --git a/PROVESFlightControllerReference/test/int/mode_manager_test.py b/PROVESFlightControllerReference/test/int/mode_manager_test.py index cb00ae73..2abb1738 100644 --- a/PROVESFlightControllerReference/test/int/mode_manager_test.py +++ b/PROVESFlightControllerReference/test/int/mode_manager_test.py @@ -543,6 +543,10 @@ def test_safe_08_clean_reboot_no_safe_mode( @pytest.mark.slow @pytest.mark.uart_only(reason="Requires reboot and GDS reconnect") +# The reboot half of this test is a hardware power cycle driven by the watchdog, +# so with JP6 open the boot count never increments even though the command-loss +# detection and SAFE_MODE entry it also asserts both work. +@pytest.mark.requires_watchdog_jumper def test_safe_09_command_loss_triggers_safe_mode_and_reboot( fprime_test_api: IntegrationTestAPI, start_gds ): diff --git a/PROVESFlightControllerReference/test/int/provision_key_test.py b/PROVESFlightControllerReference/test/int/provision_key_test.py index f077a372..14bdf043 100644 --- a/PROVESFlightControllerReference/test/int/provision_key_test.py +++ b/PROVESFlightControllerReference/test/int/provision_key_test.py @@ -15,7 +15,7 @@ import pytest from fprime_gds.common.data_types.event_data import EventData from fprime_gds.common.testing_fw.api import IntegrationTestAPI -from fprime_gds.common.testing_fw.predicates import satisfies_any +from fprime_gds.common.testing_fw.predicates import is_a_member_of @pytest.mark.provision_key @@ -41,19 +41,28 @@ def test_provision_key( "PROVES_AUTH_KEY environment variable not set; cannot provision key" ) + # await_event coerces any non-event_predicate into an *id* predicate, so the + # "either outcome" match has to be expressed over event IDs. Passing a list of + # event_predicates here would silently never match (they'd be evaluated against + # an int). + outcome_ids = [ + fprime_test_api.translate_event_name(f"{deframer}.KeyProvisioned"), + fprime_test_api.translate_event_name(f"{deframer}.KeyProvisionFailed"), + ] + fprime_test_api.clear_histories() fprime_test_api.send_command(f"{deframer}.PROVISION_KEY", ["0", key]) evt: EventData = fprime_test_api.await_event( - satisfies_any( - [ - fprime_test_api.get_event_pred(f"{deframer}.KeyProvisioned"), - fprime_test_api.get_event_pred(f"{deframer}.KeyProvisionFailed"), - ] - ), + is_a_member_of(outcome_ids), timeout=10, ) + assert evt is not None, ( + f"No KeyProvisioned/KeyProvisionFailed event from {deframer} within 10s of " + "PROVISION_KEY; the command may not have reached the board" + ) + if evt.template.get_full_name().endswith("KeyProvisionFailed"): status = evt.args[0].val assert status == "NotEmpty", ( diff --git a/PROVESFlightControllerReference/test/int/rtc_test.py b/PROVESFlightControllerReference/test/int/rtc_test.py index 2c0ec765..bc78e604 100644 --- a/PROVESFlightControllerReference/test/int/rtc_test.py +++ b/PROVESFlightControllerReference/test/int/rtc_test.py @@ -18,7 +18,7 @@ from fprime_gds.common.models.serialize.numerical_types import U32Type from fprime_gds.common.models.serialize.time_type import TimeType from fprime_gds.common.testing_fw.api import IntegrationTestAPI -from fprime_gds.common.testing_fw.predicates import event_predicate +from fprime_gds.common.testing_fw.predicates import event_predicate, is_a_member_of from fprime_gds.common.tools.seqgen import SeqGenException, generateSequence rtcManager = "ReferenceDeployment.rtcManager" @@ -105,6 +105,24 @@ def uplink_sequence_and_await_completion( fprime_test_api.__log(msg, TestLogger.RED) raise fprime_test_api.send_command(f"{fileManager}.CreateDirectory", ["/seq"]) + # CreateDirectory is dispatched asynchronously. Uplinking straight away + # races it, and when /seq does not already exist the receiving end fails + # with FileOpenError before the mkdir lands. Wait for the command to + # resolve either way first - "already exists" comes back as + # DirectoryCreateError, which is just as good for our purposes. + fprime_test_api.await_event( + is_a_member_of( + [ + fprime_test_api.translate_event_name( + f"{fileManager}.CreateDirectorySucceeded" + ), + fprime_test_api.translate_event_name( + f"{fileManager}.DirectoryCreateError" + ), + ] + ), + timeout=timeout, + ) fprime_test_api.uplink_file(temp_bin_path, destination) fprime_test_api.await_event("FileReceived", timeout=timeout) diff --git a/PROVESFlightControllerReference/test/int/tmp112_test.py b/PROVESFlightControllerReference/test/int/tmp112_test.py index be73d011..bf6b6e51 100644 --- a/PROVESFlightControllerReference/test/int/tmp112_test.py +++ b/PROVESFlightControllerReference/test/int/tmp112_test.py @@ -9,7 +9,7 @@ from datetime import datetime import pytest -from common import FIB_BACKOFF, proves_send_and_assert_command +from common import FIB_BACKOFF, exit_safe_mode, proves_send_and_assert_command from fprime_gds.common.data_types.event_data import EventData from fprime_gds.common.models.serialize.numerical_types import F32Type from fprime_gds.common.models.serialize.time_type import TimeType @@ -23,6 +23,9 @@ @pytest.fixture(autouse=True) def setup_test(fprime_test_api: IntegrationTestAPI, start_gds): """Fixture to turn on face 0 before each test""" + # Must precede TURN_ON: entering safe mode switches the face load switch + # back off, so powering the face first would just be undone. + exit_safe_mode(fprime_test_api) proves_send_and_assert_command( fprime_test_api, "ReferenceDeployment.face0LoadSwitch.TURN_ON", diff --git a/PROVESFlightControllerReference/test/int/veml6031_test.py b/PROVESFlightControllerReference/test/int/veml6031_test.py index 7b729b56..fb3f862c 100644 --- a/PROVESFlightControllerReference/test/int/veml6031_test.py +++ b/PROVESFlightControllerReference/test/int/veml6031_test.py @@ -7,7 +7,7 @@ from datetime import datetime import pytest -from common import proves_send_and_assert_command +from common import exit_safe_mode, proves_send_and_assert_command from fprime_gds.common.data_types.event_data import EventData from fprime_gds.common.models.serialize.numerical_types import F32Type from fprime_gds.common.models.serialize.time_type import TimeType @@ -21,6 +21,9 @@ @pytest.fixture(autouse=True) def setup_test(fprime_test_api: IntegrationTestAPI, start_gds): """Fixture to turn on face 0 before each test""" + # Must precede TURN_ON: entering safe mode switches the face load switch + # back off, so powering the face first would just be undone. + exit_safe_mode(fprime_test_api) proves_send_and_assert_command( fprime_test_api, "ReferenceDeployment.face0LoadSwitch.TURN_ON", diff --git a/TODO.md b/TODO.md index bd4df75b..172b0968 100644 --- a/TODO.md +++ b/TODO.md @@ -196,30 +196,49 @@ Status legend: [ ] todo, [~] in progress, [x] done Tracked in `INVESTIGATION.md`. The firmware blockers are fixed and verified on hardware; what is left is the test path plus one design decision. -- [ ] **Make `provision_key_test.py` pass on the bench and in CI.** It currently - errors in *setup*, so the test body never runs: `start_gds` loops for 30s - on `CdhCore.cmdDisp.CMD_NO_OP` (`conftest.py:113`) and its two-item event - sequence always times out. `recover_from_safe_mode` (`conftest.py:177`) is - `autouse=True` and depends on `start_gds`, so **every** test in - `test/int/` errors with it. - Firmware side is proven -- the identical PROVISION_KEY sent outside pytest - provisions the board, and the router showed `routed=3 bypassed=3 - rejected=0`, i.e. keyless commands are dispatched and none rejected. - Leading (unconfirmed) hypothesis: GDS downlink desync from resetting the - board underneath a long-lived GDS -- it logged `APID 2 received sequence - count: 4 (expected: 1)` after each reset. CI power-cycles before starting - GDS so it should not be exposed. **Counter-evidence: only 3 bypassed - packets were counted against more attempts than that, so uplink loss is - not ruled out.** Ordered next steps in `INVESTIGATION.md` -- start with a - clean-slate bench run in CI order (power-cycle, then GDS, then test, no - SWD attached), and push to let CI settle it. +- [x] **`provision_key_test.py` passes on the bench (2026-07-28).** The GDS + desync hypothesis was **wrong**. Two real causes: + 1. **Mis-built predicate.** `await_event(satisfies_any([event_pred, ...]))` + -- `get_event_pred` only passes an argument through when it is already + an `event_predicate`, so a `satisfies_any` was used as the *event-ID* + predicate and its inner `EventData` checks were evaluated against an + int. Never matched; `evt` came back `None` and blew up as + `AttributeError`. Fixed with + `is_a_member_of([translate_event_name(...), ...])` + an explicit + `assert evt is not None`. + 2. **A halted board.** Earlier sessions left the target halted under SWD, + which drops the USB CDC, so `start_gds`'s `CMD_NO_OP` failed and + errored the whole directory in setup. + Verified on hardware both ways: keyless board (keystore erased over SWD) + -> `KeyProvisioned` -> `sync_sequence_number` passes, i.e. the + flash-stored key authenticates; and already-provisioned board -> + `NotEmpty` -> treated as success. -- [ ] **Decouple the autouse fixture from `start_gds` regardless of the cause.** - One uncooperative `CMD_NO_OP` currently takes out the whole suite in - setup -- including the very test whose job is to bootstrap a keyless board - into a commandable state. Make `recover_from_safe_mode` opt-in, or have it - tolerate an unavailable link, so failures report as failures rather than - errors. +- [x] **`start_gds` now explains itself.** The bare `assert gds_working` is + replaced by a message naming the command, the attempt count and the last + exception -- it gates every test in the directory, so its failure used to + surface as 40-odd unexplained setup errors. The redundant `start_gds` + dependency was also dropped from `recover_from_safe_mode`; note the + originally-planned "make it opt-in" fix would **not** have helped, because + every test file already requests `start_gds` directly. + +- [x] **Full bench suite green: 30 passed, 0 failed (2026-07-28).** Four + unrelated bench failures diagnosed and resolved (table in + `INVESTIGATION.md`): LOW_BATTERY auto-safe-mode cycling the face load + switch until the face I2C bus wedged (new `--no-battery` option drops + `SafeModeEntryVoltage` to 0 per test); a real pre-existing race in + `rtc_test`'s `uplink_sequence_and_await_completion`, which uplinked + without waiting for `CreateDirectory /seq` to land; `/antenna` missing + after a format until the next boot; and two tests that genuinely need + hardware this bench lacks (`drv2605` -> `requires_battery`, `safe_09` -> + `requires_watchdog_jumper`). Markers are inert in CI, which never passes + `--bare-flight-controler-board`. + +- [ ] **Get CI green.** Bench is green and the remote branch is 3 commits + behind, so CI has never run with the CommandDispatcher table fix. Removed + the two `Hang Forensics` diagnostic steps from `ci.yaml` -- they halted the + target over SWD immediately before GDS started, which is precisely the + failure mode that made Issue 1 look real. - [ ] **Decide how a mis-provisioned key is recovered.** Confirmed behaviour, needs an explicit call rather than a quiet patch: `PROVISION_KEY` is diff --git a/pytest.ini b/pytest.ini index 3da70b0b..97b1fc54 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,6 +8,7 @@ markers = requires_antenna: marks tests that require the antenna board to be plugged in and the burnwire capacitor installed; skip on a bare flight controller requires_battery: marks tests that require the battery board connected with power flowing from the battery terminals; skip on a bare flight controller requires_watchdog_jumper: marks tests that require the JP6 watchdog jumper to be bridged so the watchdog can reset the MCU; skip when JP6 is open + slow: marks tests that take tens of seconds because they wait on a reboot or a timeout expiring filterwarnings = ignore::DeprecationWarning:yamcs\..* ignore::DeprecationWarning:google\.protobuf\..* From 622d866178db757396d086c06d54fb797feb923f Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Tue, 28 Jul 2026 03:47:01 +0200 Subject: [PATCH 20/29] fix(ci): give the YAMCS stack the provisioned auth key The Start YAMCS Stack step runs `make yamcs`, which launches tools/yamcs/proves_adapter.py. That adapter builds the same authenticated framing the GDS plugin does, so it resolves the HMAC key at construction. While the key was compiled into the flight image the step needed no environment; now that it is provisioned onto the satellite and read from PROVES_AUTH_KEY, the adapter died at startup with ValueError: No authentication key available: pass --authentication-key or set the PROVES_AUTH_KEY environment variable. so YAMCS came up with no uplink path and test_noop_round_trip timed out after 18 CMD_NO_OP attempts. Scope the secret to the step like every other step that starts a framing process. Audited the rest of the workflow for the same gap: the only other steps matching `make yamcs` are Stop YAMCS Stack (`make yamcs-stop`) and Validate YAMCS server boots with current MDB (`make yamcs-build-check`), neither of which starts a framing process. Run 30316568128 otherwise validated the branch end to end on hardware from a keyless board: Provision Key, Sync Sequence Number and Run UART Integration Tests all passed in integration-uart, and integration-radio passed in full. --- .github/workflows/ci.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 183b2d31..ae474ca0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -282,6 +282,12 @@ jobs: - name: Start YAMCS Stack env: PYTHONUNBUFFERED: "1" + # proves_adapter.py builds the same authenticated framing the GDS + # plugin does, so it needs the key at construction. Now that the key + # is provisioned onto the satellite instead of compiled into the + # image, this step has to be given it explicitly like every other + # step that starts a framing process. + PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }} run: | # SPACECRAFT_ID=67 matches the override applied by `make # make-ci-spacecraft-id` in the build job so the adapter syncs on From 30ffaef043457e3462ecd55877f3d053cc25477b Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Tue, 28 Jul 2026 04:12:55 +0200 Subject: [PATCH 21/29] docs: record first green CI run on hmac-to-storage (30321256393) --- INVESTIGATION.md | 24 +++++++++++++++++++----- TODO.md | 24 +++++++++++++++++++----- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/INVESTIGATION.md b/INVESTIGATION.md index 182770b2..038cabae 100644 --- a/INVESTIGATION.md +++ b/INVESTIGATION.md @@ -3,11 +3,25 @@ **Goal: `provision_key_test.py` and the rest of the integration suite pass both on the local bench and in CI, on a board that starts keyless.** -Status as of 2026-07-28: **the bench is green — 30 passed, 0 failed**, on a -flight control board with a face attached, no battery board, no antenna board -and JP6 open. Both key-store paths are verified on hardware: a freshly erased -(keyless) board provisions and then authenticates, and an already-provisioned -board reports `NotEmpty` and is treated as success. +Status as of 2026-07-28: **both are green.** + +- **Bench: 30 passed, 0 failed**, on a flight control board with a face + attached, no battery board, no antenna board and JP6 open. +- **CI: run 30321256393 passes all six jobs** — `lint`, `unit-test`, `build`, + `yamcs-build`, `integration-uart`, `integration-radio`. First green CI on + this branch. + +Both key-store paths are verified on hardware: a freshly erased (keyless) board +provisions and then authenticates, and an already-provisioned board reports +`NotEmpty` and is treated as success. CI exercises the keyless path for real — +its board had never been provisioned, since no prior run on this branch got +past boot. + +The last CI-only defect was `Start YAMCS Stack` missing `PROVES_AUTH_KEY`: it +runs `make yamcs` → `tools/yamcs/proves_adapter.py`, which resolves the HMAC key +at construction. Free while the key was compiled into the image; with the key on +the satellite the adapter died at startup, YAMCS had no uplink, and +`test_noop_round_trip` timed out. Fixed in `622d8661`. One item remains open, and it is a design decision rather than a defect: recovery from a board provisioned with the wrong key (§2). diff --git a/TODO.md b/TODO.md index 172b0968..4efcce3f 100644 --- a/TODO.md +++ b/TODO.md @@ -234,11 +234,25 @@ hardware; what is left is the test path plus one design decision. `requires_watchdog_jumper`). Markers are inert in CI, which never passes `--bare-flight-controler-board`. -- [ ] **Get CI green.** Bench is green and the remote branch is 3 commits - behind, so CI has never run with the CommandDispatcher table fix. Removed - the two `Hang Forensics` diagnostic steps from `ci.yaml` -- they halted the - target over SWD immediately before GDS started, which is precisely the - failure mode that made Issue 1 look real. +- [x] **CI green (run 30321256393, 2026-07-28).** All six jobs pass: `lint`, + `unit-test`, `build`, `yamcs-build`, `integration-uart`, + `integration-radio`. First green CI on this branch. + Two things had to change beyond the bench work: + - Removed the two `Hang Forensics` diagnostic steps -- they halted the + target over SWD immediately before GDS started, which is precisely the + failure mode that made Issue 1 look real. + - **`Start YAMCS Stack` was missing `PROVES_AUTH_KEY`.** It runs + `make yamcs` -> `tools/yamcs/proves_adapter.py`, which resolves the HMAC + key at construction. That was free while the key was compiled into the + image; with the key provisioned onto the satellite the adapter died at + startup (`ValueError: No authentication key available`), YAMCS came up + with no uplink path, and `test_noop_round_trip` timed out after 18 + `CMD_NO_OP` attempts. This was the *only* failure in run 30316568128 -- + Provision Key, Sync Sequence Number, Run UART Integration Tests and the + entire `integration-radio` job all passed there, on a keyless board. + Audited the rest of the workflow: the only other `make yamcs` matches + are `yamcs-stop` and `yamcs-build-check`, neither of which starts a + framing process. - [ ] **Decide how a mis-provisioned key is recovered.** Confirmed behaviour, needs an explicit call rather than a quiet patch: `PROVISION_KEY` is From 7dcd9421b100e53c4673e0f74b0eda3cf4adcb3e Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Tue, 28 Jul 2026 15:15:48 +0200 Subject: [PATCH 22/29] chore: drop investigation scratch files and bench diagnostics The key-store work needed a long hardware investigation; its notes and one-off debug tooling were committed along the way. None of it belongs in the reviewable change set: - TODO.md / INVESTIGATION.md / PROBLEM.md: working notes for the boot-hang and CI investigations, now resolved. - scripts/diag/: throwaway OpenOCD/GDB probes (thread walk, downlink trace, fault breakpoints, forensics) used to find the UsageFault and the /keys mount stall. Also drops the FRAM codespell exception (only TODO.md used the word), the stale AuthDefaultKey.h include path in the unit-test CMakeLists, and the branch name from the CommandDispatcher table-size comment. Implementation, tests, and documentation are unchanged; unit tests pass. --- .codespell-ignore-words.txt | 1 - INVESTIGATION.md | 199 --------- PROBLEM.md | 151 ------- .../config/CommandDispatcherImplCfg.hpp | 6 +- .../test/unit-tests/CMakeLists.txt | 1 - TODO.md | 396 ------------------ scripts/diag/downlink_trace.gdb | 87 ---- scripts/diag/downlink_trace.sh | 111 ----- scripts/diag/hang_fault_bp.gdb | 102 ----- scripts/diag/hang_fault_bp.sh | 130 ------ scripts/diag/hang_forensics.tcl | 120 ------ scripts/diag/hang_gdb.sh | 99 ----- scripts/diag/hang_thread_walk.gdb | 377 ----------------- scripts/diag/hang_thread_walk.sh | 126 ------ 14 files changed, 3 insertions(+), 1903 deletions(-) delete mode 100644 INVESTIGATION.md delete mode 100644 PROBLEM.md delete mode 100644 TODO.md delete mode 100644 scripts/diag/downlink_trace.gdb delete mode 100755 scripts/diag/downlink_trace.sh delete mode 100644 scripts/diag/hang_fault_bp.gdb delete mode 100755 scripts/diag/hang_fault_bp.sh delete mode 100644 scripts/diag/hang_forensics.tcl delete mode 100755 scripts/diag/hang_gdb.sh delete mode 100644 scripts/diag/hang_thread_walk.gdb delete mode 100755 scripts/diag/hang_thread_walk.sh diff --git a/.codespell-ignore-words.txt b/.codespell-ignore-words.txt index 76847f0c..9c69ee4e 100644 --- a/.codespell-ignore-words.txt +++ b/.codespell-ignore-words.txt @@ -1,5 +1,4 @@ ALS -FRAM comIn bufferIn commandIn diff --git a/INVESTIGATION.md b/INVESTIGATION.md deleted file mode 100644 index 038cabae..00000000 --- a/INVESTIGATION.md +++ /dev/null @@ -1,199 +0,0 @@ -# Investigation: integration tests green on the local bench and in CI - -**Goal: `provision_key_test.py` and the rest of the integration suite pass both -on the local bench and in CI, on a board that starts keyless.** - -Status as of 2026-07-28: **both are green.** - -- **Bench: 30 passed, 0 failed**, on a flight control board with a face - attached, no battery board, no antenna board and JP6 open. -- **CI: run 30321256393 passes all six jobs** — `lint`, `unit-test`, `build`, - `yamcs-build`, `integration-uart`, `integration-radio`. First green CI on - this branch. - -Both key-store paths are verified on hardware: a freshly erased (keyless) board -provisions and then authenticates, and an already-provisioned board reports -`NotEmpty` and is treated as success. CI exercises the keyless path for real — -its board had never been provisioned, since no prior run on this branch got -past boot. - -The last CI-only defect was `Start YAMCS Stack` missing `PROVES_AUTH_KEY`: it -runs `make yamcs` → `tools/yamcs/proves_adapter.py`, which resolves the HMAC key -at construction. Free while the key was compiled into the image; with the key on -the satellite the adapter died at startup, YAMCS had no uplink, and -`test_noop_round_trip` timed out. Fixed in `622d8661`. - -One item remains open, and it is a design decision rather than a defect: -recovery from a board provisioned with the wrong key (§2). - ---- - -## Issue 1 — `provision_key_test.py` never reached its test body — RESOLVED - -The previous edition of this file suspected a GDS downlink desync caused by -resetting the board underneath a long-lived GDS. **That theory was wrong.** Two -real defects were behind it, plus one bench-procedure artifact. - -### Cause A (the actual test failure): a mis-built event predicate - -`provision_key_test.py` awaited "either KeyProvisioned or KeyProvisionFailed" -with - -```python -await_event(satisfies_any([get_event_pred(...), get_event_pred(...)])) -``` - -`IntegrationTestAPI.get_event_pred` returns its argument unchanged only when it -is already an `event_predicate`. A `satisfies_any` is a predicate but *not* an -`event_predicate`, so it fell through to being used as the **event-ID** -predicate — the two inner `EventData` checks were evaluated against an integer -id and could never be true. The search timed out even though the event had -arrived, and `evt` came back `None`, which then blew up as -`AttributeError: 'NoneType' object has no attribute 'template'`. - -Fixed by matching over ids instead: - -```python -await_event(is_a_member_of([translate_event_name(...), translate_event_name(...)])) -``` - -plus an explicit `assert evt is not None` so a genuine no-response failure -reports as itself rather than as an `AttributeError`. - -### Cause B (why it looked like a setup error): a halted board - -The earlier bench sessions had left the target halted under SWD. A halted board -drops its USB CDC, so GDS saw nothing and `start_gds`'s `CMD_NO_OP` loop failed -— erroring every test in the directory during setup. With the board simply left -running, `start_gds` passes first time. - -`start_gds` used a bare `assert gds_working`, which is why this presented as an -unexplained error on 40-odd tests. It now reports the command, the attempt -count and the last exception. - -### Not a cause: the space-packet sequence-count warning - -GDS does log - -``` -[WARNING] framing: APID 2 received sequence count: 35 (expected: 1) -``` - -on every startup against an already-running board, but it is a warning only — -GDS adopts the received count and carries on. `reset_manager`'s cold- and -warm-reset tests both pass with GDS running across the reboot. - ---- - -## Bench-only failures, and what they actually were - -Four tests failed on the bench for reasons unrelated to the key store. Each is -now either fixed or correctly marked. - -| Test | Cause | Resolution | -| --- | --- | --- | -| `tmp112`, `veml6031` | With nothing at the battery terminals the power monitor reads 0.012 V, so modeManager auto-enters SAFE_MODE (`reason=LOW_BATTERY`) every debounce period; its safe-mode sequence switches the face load switch **off**, and enough of those cycles wedges the face I2C bus for the rest of the session. | New `--no-battery` pytest option drops `SafeModeEntryVoltage` to 0 before each test, so auto-entry never fires. Verified: 0 `AutoSafeModeEntry` events in a full run. | -| `antenna_deployer::test_deployment_prevention_after_success` | After `format_filesystem`, `/antenna` no longer exists, so `SET_DEPLOYMENT_STATE` fails with `FileOperationError ... on file open_write`. The directory is recreated at boot. | Bench procedure: power-cycle after formatting, which is what CI already does (`Format Filesystem` → `Power-Cycle Satellite`). Also added an `exit_safe_mode` to its fixture, since deployment is inhibited in safe mode. | -| `rtc_test::test_04_sequence_cancellation_on_time_set` | `uplink_sequence_and_await_completion` fired `CreateDirectory /seq` and uplinked immediately, racing the mkdir: `FileOpenError: Could not open file /seq/no_op.bin` 30 ms *before* `CreateDirectorySucceeded`. Only visible when `/seq` did not already exist. | Real pre-existing race, fixed: wait for `CreateDirectorySucceeded` **or** `DirectoryCreateError` (already-exists) before uplinking. | -| `drv2605::test_01_magnetorquer_power_draw` | Asserts a ≥0.3 W rise in INA219 system power; that rail reads 0.0 W in both samples without a battery. No command can fix an unpowered rail. | Marked `requires_battery` in addition to `requires_face`. | -| `mode_manager::test_safe_09` | Asserts the boot count increments after a watchdog-driven hardware power cycle; with JP6 open the reboot never happens (177 → 177). The command-loss detection and SAFE_MODE entry it also asserts both pass. | Marked `requires_watchdog_jumper`. | - -These markers are inert in CI: CI never passes `--bare-flight-controler-board`, -which is the only thing that acts on them. - -### Reproducing the green bench run - -```sh -# 1. board running and NOT halted under SWD; no GDS yet -PROVES_AUTH_KEY=<32 hex chars> make gds-integration UART_DEVICE=/dev/cu.usbmodem1101 & - -# 2. bootstrap, in CI's order -make test-integration FILTER=provision_key -make test-integration FILTER=sync_sequence_number - -# 3. the suite -make test-integration \ - FILTER="not sync_sequence_number and not format_filesystem and not provision_key \ - and not requires_antenna and not requires_battery and not requires_watchdog_jumper" \ - PYTEST_ARGS=--no-battery -``` - -If you run `format_filesystem`, reset the board before the suite so `/antenna` -and friends are recreated. - ---- - -## Issue 2 — No over-the-air recovery from a mis-provisioned key - -### Status: confirmed behaviour, still needs a decision. - -The key store cannot be re-keyed from the ground once it holds a wrong key: - -- `PROVISION_KEY` is refused when the store is non-empty (`KeyProvisionFailed` - with `NotEmpty`) — by design, so an attacker cannot overwrite the key. -- `REMOVE_KEY` refuses to remove the **last** key (`LastKey`) — by design, so - the board cannot be locked out. -- `ADD_KEY` requires an already-authenticated link, which a wrong key cannot - provide. - -Together these mean a board provisioned with the wrong value is unreachable: -every recovery command needs either an empty store or a valid key, and neither -is obtainable. Recovery on the bench requires a physical SWD erase: - -``` -openocd ... -c "init; halt; flash erase_address 0x10400000 0x40000; reset run; exit" -``` - -after which littlefs re-formats the partition on the next boot and the board -comes back keyless (`valid=0`, `seqnum=0`). This was used repeatedly during -this session's verification and works reliably. - -This is fine on the bench and fatal in flight. - -### Options to weigh - -- **Accept it**, and make provisioning a controlled ground procedure with a - verification read-back before the board is buttoned up. Cheapest; leaves a - single-point failure with no in-orbit remedy. -- **Allow `ADD_KEY` from the bypass allowlist** so a second key can be added - unauthenticated, then the bad one removed. Restores recoverability but - substantially weakens the security posture — an attacker could inject a key. -- **Add an authenticated `CLEAR_KEY_STORE`/`FORMAT_KEYS`**, usable only over an - authenticated link. Does not help if the *only* key is wrong. -- **Two-slot bootstrap**: provision two keys at manufacture (`AuthKeyStore` has - 2 slots), so a rotation error still leaves one working key. Recoverable - without weakening bypass, at the cost of a provisioning-procedure change. -- **Time-boxed bypass window after boot**, e.g. accept `PROVISION_KEY` on a - non-empty store only within N seconds of reset. Recoverable via a power cycle; - needs care that it is not a standing attack window. - -No option is obviously right — this is a security/recoverability trade the -project owners should make explicitly, not a bug to be quietly patched. - ---- - -## Bench procedure notes (learned the hard way) - -- **Always resume the target before detaching from OpenOCD/GDB.** A halted board - drops its USB CDC; GDS then sees nothing and commands silently no-op. This was - the single biggest time sink across two sessions, and it is what made Issue 1 - look like a product defect. -- Resetting the board while GDS runs is **fine** — GDS logs a sequence-count - warning and resyncs. (The earlier claim to the contrary was wrong.) -- After `format_filesystem`, **reset the board** before running the suite: - `/antenna` and `/seq` are recreated at boot. -- **`make build` does not re-derive Kconfig from device-tree changes.** Use - `make generate build`; a stale `CONFIG_FLASH_SIZE` invalidated several bisect - results in an earlier session. -- Use `/dev/cu.*`, not `/dev/tty.*`, when reading the board CDC from macOS — a - `tty.` open blocks on carrier detect and looks like a dead link. -- OpenOCD lives at `~/code/github.com/raspberrypi/openocd` (the raspberrypi - fork — do not substitute a nix/brew build). -- The `--active` flag on the vendored `uv` can resolve to a stale system - `fprime_gds`; `fprime-venv/bin/fprime-cli` is the reliable path, and it needs - `--deployment build-artifacts/zephyr/fprime-zephyr-deployment`. - -Diagnostics live in `scripts/diag/` (see the ADR item in `TODO.md`). The -`Hang Forensics` CI steps that drove them have been removed — they halted the -target over SWD immediately before GDS started, which is the exact failure mode -above. diff --git a/PROBLEM.md b/PROBLEM.md deleted file mode 100644 index c1645335..00000000 --- a/PROBLEM.md +++ /dev/null @@ -1,151 +0,0 @@ -# CI hardware failure on hmac-to-storage (PR #472): board never responds over UART/LoRa - -## Symptom - -`integration-uart` and `integration-radio` both fail at the very first test -(`provision_key_test.py::test_provision_key`), inside the `start_gds` fixture's -`CMD_NO_OP` retry loop. GDS never sees any reply: - -- `recv.bin` is 0 bytes for the whole test window (one run showed 48 bytes — see below). -- `sent.bin` is 0 bytes: the ground side's write to the serial device itself is failing. -- `comm.py.log` shows real OS-level serial errors, not just logical timeouts: - ``` - [WARNING] serial_adapter: Serial exception caught: device reports readiness to - read but returned no data (device disconnected or multiple access on port?). - Reconnecting. - [WARNING] uplink: Uplink failed to send 41 bytes of data after 3 retries - ``` -- This reproduced identically on two different physical benches (UART bench and - LoRa/radio bench), ruling out a single flaky cable/bench. -- `lint`, `unit-test`, `build`, `yamcs-build` are all green. Only the two - hardware-integration jobs fail. - -## Investigation - -Three rounds of live SWD diagnostics were run against the UART bench (via a -temporary CI step in `.github/workflows/ci.yaml`, since removed — see commits -`1ff5d8bb`, `78ca10f7`, `0949dfbd`, `693eb5d8` on this branch for the added/fixed/ -removed diagnostic). Each halted both RP2350 cores ~20s after boot via OpenOCD and -dumped registers + CFSR/HFSR/DFSR/MMFAR/BFAR (`0xE000ED28`). - -Attempt 1: broken OpenOCD command syntax (`rp2350.cm0 halt` isn't valid — only -`arp_halt` exists per-target-instance); OpenOCD printed a command-usage listing and -exited before reaching any read. No data. - -Attempt 2: fixed target selection (`targets rp2350.cm0` then plain `halt`/`reg`), but -hit two problems: OpenOCD warned `target was in unknown state when halt was -requested` (a race between `halt` and its own initial poll cycle) so the bulk `reg` -dump came back with register **names** but no **values**; and `bt` (not a valid -OpenOCD console command — that's GDB) aborted the remaining `-c` chain before core1 -was ever queried. Partial data: CFSR=0, HFSR=0, DFSR=1 for core0 (a real debug-halt, -no fault). - -Attempt 3: added explicit `poll` before/after `halt`, a settle delay, and queried -`pc`/`lr`/`sp`/`xpsr`/`r0`-`r3` individually instead of the bulk dump; dropped `bt`. -This got real data for both cores: - -``` -cm0: pc=0x101864b8 lr=0x1010fb69 sp=0x20034410 xpsr=0x61000000 (Thread mode, no fault) - CFSR=0 HFSR=0 DFSR=0 -cm1: pc=0x0000019e lr=0x00000203 sp=0xf0000000 xpsr=0x09000000 - CFSR=0 HFSR=0 DFSR=1 (plain debug halt) -``` - -Resolved against a local build of the same commit's `zephyr.elf` -(`build-artifacts/zephyr.elf`, built by a prior session — see TODO.md): - -``` -$ arm-none-eabi-addr2line -e build-artifacts/zephyr.elf -f -C 0x101864b8 0x1010fb69 -fs_open -lib/zephyr-workspace/zephyr/subsys/fs/fs.c:140 -idle -lib/zephyr-workspace/zephyr/kernel/idle.c:30 -``` - -**Core0 (the only core Zephyr actually runs on this board) was live, in Thread -mode, with zero fault flags set, executing `fs_open()` at the moment it was -halted.** Core1 is just idling in the RP2350 boot ROM waiting for a multicore -launch that never comes (expected/benign for a single-core app). - -This rules out a crash, a hard fault, a null pointer dereference reaching a fault -handler, and a boot-time panic. The board is alive and running normal application -code; it is *not* in a fault-loop. - -## Leading hypothesis (not yet confirmed) - -This PR (per the plan) moves two things onto a new internal-flash littlefs -partition (`keystore_partition`, mounted at `/keys`) that previously lived on the -SD-card FAT filesystem: - -1. The anti-replay sequence number (`SEQ_NUM_FILE_PATH` default now - `/keys/sequence_number.bin`, was on `/` = SD/FAT). -2. The new HMAC key store (`/keys/authkeys.bin`), which `TcSecurityDeframer` - reloads from disk once whenever an incoming frame's SPI doesn't match any - active slot — i.e., on **every single received frame** while the board is - unprovisioned (keyless), which is exactly the state under test here. - -Both files now live on the **same physical QSPI/XIP flash chip that serves the -running firmware code**. On RP2040/RP2350, writing or erasing that flash requires -suspending code execution from flash and disabling interrupts system-wide for the -duration of the operation (a well-documented Pico SDK / Zephyr internal-flash -driver constraint — this is *not* true of the old SD-card/SDMMC path, which uses a -separate SPI bus with no such restriction). - -`TcSecurityDeframer::dataIn_handler` (see -`PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp`, -around lines 56–119) holds both `m_keyStoreLock` and `m_sequenceNumberLock` for the -full validate+authenticate+persist sequence on every frame, and: -- writes the sequence-number file (`writeSequenceNumber`) on every *accepted* - frame, and -- calls `loadKeyStore()` (a fresh `fs_open`+read) on every frame with an - unrecognized SPI, before this PR happens to be *every* frame during the keyless - window this test exercises. - -The working theory is that this internal-flash I/O — now on a hot path that used -to be interrupt-safe SD-card I/O — is disabling interrupts long enough, and/or -often enough, to stall USB CDC-ACM servicing, producing exactly the "device -disconnected" / failed-write symptom seen on the ground side. The single mid-flight -`fs_open` sample is consistent with this but is not proof by itself (no flash- -operation timing instrumentation was captured); a controlled repro (e.g. -instrumenting write duration with a GPIO toggle or RTT, or scoping bus activity) -would be needed to confirm the magnitude and pin down which specific call -(`writeSequenceNumber` vs `loadKeyStore` vs the very first-ever littlefs auto-format -at boot) is responsible. - -## What this is NOT - -- Not a build/compile problem — `build`, `unit-test`, `lint`, `yamcs-build` are all - green. -- Not a crash/fault/panic — confirmed via live register + CFSR/HFSR read on real - hardware, core0 healthy and running. -- Not bench/cable flakiness alone — reproduced identically on two separate physical - benches (UART and LoRa). -- Not the CI runner being stuck — earlier apparent multi-hour queue delays during - this investigation turned out to be real backlog on the shared hardware runner, - unrelated to this bug; runs did eventually execute and complete. - -## Suggested next steps (not yet implemented — needs a decision) - -The sequence-number-and-key-store-onto-internal-flash design is a locked decision -from the original plan (`~/.claude/plans/quirky-forging-possum.md`), so the fix -should stay within that design rather than reverting to SD-card storage. Candidates -worth evaluating: - -1. Debounce/throttle `writeSequenceNumber` so it isn't a synchronous internal-flash - write on every single accepted frame — e.g. only persist every N frames or after - a time interval, accepting a small anti-replay window on power loss (arguably - already implicitly tolerated by the existing `SEQ_NUM_WINDOW` mechanism). -2. Stop calling `loadKeyStore()` on *every* unrecognized-SPI frame; e.g. rate-limit - reloads (the plan's stated purpose — cross-link key-rotation propagation — is an - infrequent event, not something that needs a fresh disk read on every single - uplinked frame while unprovisioned). -3. Confirm whether the very first-ever littlefs mount on a blank `keystore_partition` - performs a full-partition format synchronously at `SYS_INIT` (before `main()`), - and if so, whether that's the dominant one-time stall rather than (or in addition - to) the per-frame writes. -4. Instrument actual flash-operation duration on real hardware (GPIO toggle around - `flash_area_write`/`erase`, or Zephyr's flash driver trace hooks) to know the real - magnitude before choosing a fix, rather than guessing further. - -None of the above has been implemented yet — see `TODO.md` for the standing -checklist item. diff --git a/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp b/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp index a4042255..5b2c69b6 100644 --- a/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp +++ b/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp @@ -15,9 +15,9 @@ enum { // `commands` array). CommandDispatcher inserts every opcode into a // fixed-capacity RedBlackTreeMap and FW_ASSERTs when the insert fails // (CommandDispatcherImpl.cpp:35), so exceeding this bricks the boot rather - // than degrading. hmac-to-storage pushed the count from 348 to 354 by - // adding PROVISION_KEY/ADD_KEY/REMOVE_KEY to both TcSecurityDeframer - // instances; headroom raised so the next few commands do not repeat this. + // than degrading. Adding PROVISION_KEY/ADD_KEY/REMOVE_KEY to the + // TcSecurityDeframer instances pushed the count from 348 to 354; headroom + // raised here so the next few commands do not have to repeat this. CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 512, // !< The size of the table holding opcodes to dispatch CMD_DISPATCHER_SEQUENCER_TABLE_SIZE = 10, // !< The size of the table holding commands in progress }; diff --git a/PROVESFlightControllerReference/test/unit-tests/CMakeLists.txt b/PROVESFlightControllerReference/test/unit-tests/CMakeLists.txt index 389ccaf6..18ec6d5d 100644 --- a/PROVESFlightControllerReference/test/unit-tests/CMakeLists.txt +++ b/PROVESFlightControllerReference/test/unit-tests/CMakeLists.txt @@ -53,7 +53,6 @@ add_library(security_deframer_authenticator STATIC ) target_include_directories(security_deframer_authenticator PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/../../.. - ${CMAKE_CURRENT_SOURCE_DIR} # for AuthDefaultKey.h ) # RtcManager RtcHelper diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 4efcce3f..00000000 --- a/TODO.md +++ /dev/null @@ -1,396 +0,0 @@ -# HMAC key → internal-flash storage — progress tracker - -Plan source: `~/.claude/plans/quirky-forging-possum.md` -Branch: `hmac-to-storage` - -Status legend: [ ] todo, [~] in progress, [x] done - -## 1. Flash partition + littlefs mount -- [x] Add `keystore_partition` to `proves_flight_control_board_v5.dtsi` `&flash0`, shrink `storage_partition` - (keystore_partition@0x400000 256KB; storage_partition@0x440000 0xBC0000) -- [x] Add `zephyr,fstab,littlefs` node mounted at `/keys` (lfs1, automount) -- [x] `prj.conf`: `CONFIG_FILE_SYSTEM_LITTLEFS=y` (correct Kconfig name, not FS_LITTLEFS) -- [x] Automount confirmed via Zephyr Kconfig.littlefs: FS_LITTLEFS_FSTAB_AUTOMOUNT defaults y when DT - node has `automount` + CONFIG_FLASH_MAP=y (already on for all 3 board defconfigs). No explicit - fs_mount() needed in Main.cpp. - -## 2. Key store type + persistence (TcSecurityDeframer) -- [x] Add FPP `AuthKeySlot` / `AuthKeyStore` (2 slots) type + `KeyStoreProvisionStatus` enum, in `TcSecurityDeframer.fpp` - (verified accessor names `getvalid/getspi/getkey/setvalid/setspi/setkey`, `AuthKeyStore::SIZE`, `operator[](U32)` - by running `fpp-to-cpp` on a scratch copy of the struct/array — see scratch notes below) -- [x] `Authenticator.cpp/.hpp`: added `importHmacKeyBytes` (raw bytes, used by key-store reimport) with - `importHmacKey` (hex) now a thin wrapper; added `destroyHmacKey`; exposed `parseHexKey`; - `authenticatePacket` now takes `keyId` by value (was `uint32_t&`, never mutated) -- [x] `TcSecurityDeframer.hpp/.cpp`: `KEY_STORE_FILE_PATH` param, `loadKeyStore()`/`writeKeyStore()`/`importKeyStore()`/ - `findKeyIdForSpi()`/`activeKeyCount()` helpers, second `m_keyStoreLock` mutex (separate from seq-num lock; - dataIn_handler takes keyStoreLock then seqLock, only ordering with both — no deadlock risk). - configure() loads store, no FW_ASSERT on missing/empty (keyless boot supported) -- [x] `dataIn_handler`: `validatePacket` now takes the key store; on `SpiInvalid` reloads store from disk once - and retries before giving up -- [x] `PROVISION_KEY` / `ADD_KEY` / `REMOVE_KEY` command handlers + KeyProvisioned/Failed, KeyAdded/Failed, - KeyRemoved/Failed, KeyStoreReadFailed/WriteFailed events, ActiveKeyCount telemetry -- [x] `Validator.cpp`: `spiValid` checks against active slots instead of `spi == 0` (takes `const AuthKeyStore&`) -- [x] `TcSecurityDeframer.fpp`: `SEQ_NUM_FILE_PATH` default -> `/keys/sequence_number.bin` -- [x] Removed `#include "AuthDefaultKey.h"` / `AUTH_DEFAULT_KEY` usage from `TcSecurityDeframer.cpp`; - deleted generated `AuthDefaultKey.h` from TcSecurityDeframer dir (it's gitignored, not tracked) - -## 3. Router bootstrap allowlist -- [x] `Bypasser.cpp`: added `0x2100B002`/`0x2200B002`/`0x2300B002` (UART/LoRa/Sband PROVISION_KEY) to - `kBypassOpCodes`. Derivation verified against the real (pre-existing, stale) dictionary at - `build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json`: - opcodes are `instance_base + local_index`, where local_index is 0-based over *user commands only* - (GET_SEQ_NUM=0, SET_SEQ_NUM=1) followed by PRM_SET/PRM_SAVE pairs per param in declaration order. - My new commands are declared right after SET_SEQ_NUM and before any params, so - PROVISION_KEY=2, ADD_KEY=3, REMOVE_KEY=4 (only PROVISION_KEY needs a bypass entry). - Caveat: the stale dict only had ComCcsdsUart/ComCcsdsLora instances (no Sband), so 0x2300B002 - is derived by pattern from the pre-existing (already in file) 0x2300B000 Sband entry, not directly - confirmed. **Re-verify opcodes with `make build` + fresh dict.json before flight/CI trust.** - -## 4. Ground plugin -- [x] `Framing/src/authenticate_plugin.py`: `get_default_auth_key_from_header` -> `get_auth_key_from_env`, - reads `PROVES_AUTH_KEY` env var, raises ValueError with clear message if neither CLI arg nor env set -- [x] `tools/yamcs/proves_adapter.py` also imported the removed function — updated to `get_auth_key_from_env` - (not called out in plan explicitly but same removal would have broken this importer) - -## 5. Build / Makefile / provisioning -- [x] `Makefile`: dropped `generate-auth-key` target + from `generate` deps, `AUTH_DEFAULT_KEY_HEADER`/ - `AUTH_KEY_TEMPLATE` vars, `AuthDefaultKey.h` copy line in `copy-secrets` -- [x] Deleted `scripts/generate_auth_key_header.py`, `scripts/generate_auth_default_key.h` -- [x] Added `PROVESFlightControllerReference/test/int/provision_key_test.py` (marker `provision_key`), - registered marker in `pytest.ini`, added `and not provision_key` to Makefile default `FILTER`. - Idempotent: PROVISION_KEY fails with `NotEmpty` if the board was already provisioned by a - prior CI run (key store lives on internal flash, survives reflashing) — the test treats that - as success rather than failure, since the store already holds the CI secret key. - -## 6. CI -- [x] Removed all 3 "Set Authentication Key" steps (build, integration-uart, integration-radio) that - wrote `AuthDefaultKey.h` in `.github/workflows/ci.yaml`. -- [x] No job-wide `PROVES_AUTH_KEY` export. Instead, each step that either starts a GDS process - (`make gds-integration`, whose `AuthenticateFramer` plugin reads the env var at construction - and raises if unset) or itself needs the key value (`provision_key_test.py`, which reads - `os.environ["PROVES_AUTH_KEY"]` directly to build the command arg) gets its own `env:` block - with `PROVES_AUTH_KEY: ${{ secrets.AUTH_KEY }}`. Steps that only talk to an already-running - GDS over the network (Sync Sequence Number, Format Filesystem, Run UART/Radio Integration - Tests) don't need it. 6 steps scoped this way total (3 in integration-uart: two "Start GDS" + - "Provision Key"; 3 in integration-radio: "Bootstrap Sequence Number over UART", "Sync Sequence - Number over UART", "Start GDS on LoRa Passthrough"). The build job needs no key at all now - (nothing to bake into the image). No explicit `--authentication-key` CLI flag needed since the - plugin reads the env var by default. -- [x] Added a "Provision Key" step (`make test-integration FILTER=provision_key`) right after the - first `Start GDS` in integration-uart, and inside the "Bootstrap Sequence Number over UART" - block in integration-radio (before the LoRa sync, since key-store propagation means - provisioning once over UART covers the LoRa instance too). - Added `and not provision_key` to the radio job's main test-run FILTER. - -## 7. Docs -- [x] Update `AGENTS.md` "Authentication & Security" section -- [x] Update `PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md` - -## Build verification -- `make generate build` (not `uv run ...` directly — Makefile targets resolve the right Python env) - ran a real build and caught real errors, now fixed: - - FPP-generated accessor names are `get_valid/get_spi/get_key/set_valid/set_spi/set_key` (with - underscores), not `getvalid/getspi/...` as guessed earlier. Fixed all call sites in - `TcSecurityDeframer.cpp` and `Validator.cpp` via sed. - - New `ActiveKeyCount` telemetry channel (per deframer instance) was never referenced in any - telemetry packet, which `fpp-to-dict` treats as a hard error ("neither used nor marked as - omitted"). Added `ComCcsdsLora.tcSecurityDeframer.ActiveKeyCount` / - `ComCcsdsUart.tcSecurityDeframer.ActiveKeyCount` to the `Security` packet in - `ReferenceDeploymentPackets.fppi` (Sband entry commented out, matching the existing - `CurrentSequenceNumber` pattern in that same packet — Sband instance not present in this build). - - Re-ran `make generate build` after these fixes: **clean full build**, FLASH 69.81%, RAM 62.32%, - `zephyr.uf2`/`bootable.uf2`/dictionary/XTCE all generated successfully. - -## Verification -- [x] Unit tests (`make test-unit`, pure C++ only, no F Prime/Zephyr): `parseHexKey` (direct + wrapped), - `importHmacKeyBytes`/`destroyHmacKey`, and Validator SPI/sequence-number rules. Note: - Validator.cpp had started depending on the FPP-generated `AuthKeyStore` (`Fw::Serializable`), - which broke this test target's no-F-Prime contract — decoupled it via a new plain - `ActiveSpiSlots`/`ActiveSpiSlot` type in `Types.hpp`; `TcSecurityDeframer::activeSpiSlots()` - projects `m_keyStore` into it before calling `validatePacket`. Store-mutation rules - (provision-only-when-empty, add fails at 2, remove fails at 1) live in the F-Prime-dependent - command handlers, not pure functions, so they're out of scope for this gtest target — the - commented-out `register_fprime_ut` block remains for a future on-target/component test pass. -- [x] `make build` with no `AuthDefaultKey.h` — clean build confirmed (see above) -- [x] `make check-console-disabled` — OK, Zephyr console disabled -- [ ] CI green — **still blocked, but root cause narrowed**. Pushed the 16MB flash - fix (below) and re-ran CI (run 30034861047): `Flash Firmware` step's OpenOCD - output confirms the chip really is 16MB (`w25q128fv/jv ... size = 16384 KiB`), - so the fix itself is correct and necessary. But `integration-uart`/ - `integration-radio` **still fail with the identical symptom** — GDS - `comm.py.log` shows repeated `device disconnected` serial exceptions starting - ~19s after boot, `CMD_NO_OP` never gets a response. So the flash-size - mis-declaration was real but **not the sole cause**. Revised theory: now that - `/keys` can actually mount, littlefs's first-ever format on this partition is - the first code path in this branch that reaches `flash_rpi_write`/ - `flash_rpi_erase`, both of which hold `irq_lock()` for the entire - erase/program call with no yielding — i.e. the original `PROBLEM.md` - interrupt-stall theory may be correct after all, it just couldn't fire before - (every op was rejected by `-EINVAL` pre-`irq_lock`). (That theory was later disproved; see commit `ec0bdb37`.) **Next: a temporary `CONFIG_LOG=y` diagnostic CI - run** (same pattern as the prior fault-register diagnostic commits) to see - whether the stall is the one-time format or ongoing per-frame reloads. - -## CI blocker — ROOT-CAUSED AND FIXED (2026-07-27), see commit ec0bdb37 - -- [x] **Root cause: CommandDispatcher opcode-table overflow.** - `project/config/CommandDispatcherImplCfg.hpp` had - `CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350` against a deployment already at - **348** commands. This branch adds PROVISION_KEY/ADD_KEY/REMOVE_KEY to - `TcSecurityDeframer`, and there are **two** instances (ComCcsdsUart, - ComCcsdsLora) -> 6 new commands, **354 > 350**. - `CommandDispatcherImpl.cpp:35` `FW_ASSERT`s when the RedBlackTreeMap - insert fails, so the 351st registration **panics the board during boot** - (`z_fatal_error(reason=4)` via `z_arm_svc` -- a `k_panic`, not a CPU - fault, which is why every fault-vector probe found nothing). Downlink - never starts -> GDS `device disconnected` -> CI fails. - **Fix: raised to 512.** - Confirmed on the local bench with a single-variable A/B, both full clean - `make generate build`s, measuring F' telemetry bytes off the board CDC in - a 25s window: `main` 1328 bytes @t+1.0s; branch as-shipped **0**; branch + - table 512 **1392 bytes @t+1.01s**; branch + table back to 350 **0**. - Final verified build: **2200 bytes in 30s @t+1.01s**. - -- [x] **Second, independent defect: littlefs was never compiled in.** - `west.yml`'s `name-allowlist` imported `fatfs` but not `littlefs`, so the - module never reached `zephyr_modules.txt`, `ZEPHYR_LITTLEFS_MODULE` was - undefined, and Kconfig **silently dropped** `CONFIG_FILE_SYSTEM_LITTLEFS=y` - ("LittleFS module not available"). No `lfs_*` symbols in the image, `/keys` - never existed, every `fs_open("/keys/...")` failed -- the entire key-store - feature was inert while the build stayed clean. - **Fix: added `littlefs` to the allowlist and pinned it as an explicit - project** (like every other module) so it lands under `lib/zephyr-workspace/` - rather than the workspace topdir. Verified `CONFIG_FILE_SYSTEM_LITTLEFS=y`, - `CONFIG_FS_LITTLEFS_FSTAB_AUTOMOUNT=y`, 20 `lfs_*` symbols in the ELF. - Not the CI blocker, but the feature cannot work without it. - -- [x] **Feature verified end to end on hardware (2026-07-27).** Board on - `/dev/tty.usbmodem1101`, GDS on the USB CDC, state read back over SWD: - 1. **`/keys` mounts and formats on internal flash.** littlefs superblock - magic present at `0x10400008` with block_size 0x1000 and block_count - 0x40 -- exactly the 256 KB `keystore_partition` geometry. - 2. **PROVISION_KEY works on a keyless board** over the unauthenticated - (bypass-allowlisted) link: `m_keyStore.elements[0]` went to - `valid=1 spi=0` with the provisioned key bytes. - 3. **The key survives a cold reboot.** After `reset` (RAM re-zeroed by - `arch_bss_zero`), slot 0 is `valid=1` with the same bytes -- i.e. - `configure()` -> `loadKeyStore()` read it back off flash. - 4. **The flash-stored key authenticates uplink.** `SET_SEQ_NUM 12345` - requires authentication (not bypass-allowlisted) and took effect: - `m_sequenceNumber == 12345`. - 5. **The sequence number survives a cold reboot**: still 12345 after - reset, read back from `/keys/sequence_number.bin`, key still valid. - 6. **Erasing the partition re-formats cleanly.** After erasing - `0x10400000+0x40000` over SWD the board came back keyless - (`valid=0`, `seqnum=0`) with a fresh littlefs superblock. - 7. **Bypass path confirmed:** on a keyless board the router shows - `routed=3 bypassed=3 rejected=0` -- allowlisted opcodes are dispatched - without a key, which is what makes bootstrap possible. - **Section 3 opcodes re-verified against the fresh dictionary:** - `ComCcsdsUart/Lora.tcSecurityDeframer.PROVISION_KEY` are `0x2100B002` / - `0x2200B002`, matching `Bypasser.cpp` exactly. The TODO caveat there is - resolved. (The Sband entry `0x2300B002` is still unconfirmed -- that - instance is not built.) - -## Current goal: integration tests green on the local bench AND in CI - -Tracked in `INVESTIGATION.md`. The firmware blockers are fixed and verified on -hardware; what is left is the test path plus one design decision. - -- [x] **`provision_key_test.py` passes on the bench (2026-07-28).** The GDS - desync hypothesis was **wrong**. Two real causes: - 1. **Mis-built predicate.** `await_event(satisfies_any([event_pred, ...]))` - -- `get_event_pred` only passes an argument through when it is already - an `event_predicate`, so a `satisfies_any` was used as the *event-ID* - predicate and its inner `EventData` checks were evaluated against an - int. Never matched; `evt` came back `None` and blew up as - `AttributeError`. Fixed with - `is_a_member_of([translate_event_name(...), ...])` + an explicit - `assert evt is not None`. - 2. **A halted board.** Earlier sessions left the target halted under SWD, - which drops the USB CDC, so `start_gds`'s `CMD_NO_OP` failed and - errored the whole directory in setup. - Verified on hardware both ways: keyless board (keystore erased over SWD) - -> `KeyProvisioned` -> `sync_sequence_number` passes, i.e. the - flash-stored key authenticates; and already-provisioned board -> - `NotEmpty` -> treated as success. - -- [x] **`start_gds` now explains itself.** The bare `assert gds_working` is - replaced by a message naming the command, the attempt count and the last - exception -- it gates every test in the directory, so its failure used to - surface as 40-odd unexplained setup errors. The redundant `start_gds` - dependency was also dropped from `recover_from_safe_mode`; note the - originally-planned "make it opt-in" fix would **not** have helped, because - every test file already requests `start_gds` directly. - -- [x] **Full bench suite green: 30 passed, 0 failed (2026-07-28).** Four - unrelated bench failures diagnosed and resolved (table in - `INVESTIGATION.md`): LOW_BATTERY auto-safe-mode cycling the face load - switch until the face I2C bus wedged (new `--no-battery` option drops - `SafeModeEntryVoltage` to 0 per test); a real pre-existing race in - `rtc_test`'s `uplink_sequence_and_await_completion`, which uplinked - without waiting for `CreateDirectory /seq` to land; `/antenna` missing - after a format until the next boot; and two tests that genuinely need - hardware this bench lacks (`drv2605` -> `requires_battery`, `safe_09` -> - `requires_watchdog_jumper`). Markers are inert in CI, which never passes - `--bare-flight-controler-board`. - -- [x] **CI green (run 30321256393, 2026-07-28).** All six jobs pass: `lint`, - `unit-test`, `build`, `yamcs-build`, `integration-uart`, - `integration-radio`. First green CI on this branch. - Two things had to change beyond the bench work: - - Removed the two `Hang Forensics` diagnostic steps -- they halted the - target over SWD immediately before GDS started, which is precisely the - failure mode that made Issue 1 look real. - - **`Start YAMCS Stack` was missing `PROVES_AUTH_KEY`.** It runs - `make yamcs` -> `tools/yamcs/proves_adapter.py`, which resolves the HMAC - key at construction. That was free while the key was compiled into the - image; with the key provisioned onto the satellite the adapter died at - startup (`ValueError: No authentication key available`), YAMCS came up - with no uplink path, and `test_noop_round_trip` timed out after 18 - `CMD_NO_OP` attempts. This was the *only* failure in run 30316568128 -- - Provision Key, Sync Sequence Number, Run UART Integration Tests and the - entire `integration-radio` job all passed there, on a keyless board. - Audited the rest of the workflow: the only other `make yamcs` matches - are `yamcs-stop` and `yamcs-build-check`, neither of which starts a - framing process. - -- [ ] **Decide how a mis-provisioned key is recovered.** Confirmed behaviour, - needs an explicit call rather than a quiet patch: `PROVISION_KEY` is - refused on a non-empty store (`NotEmpty`), `REMOVE_KEY` refuses the last - key (`LastKey`), and `ADD_KEY` needs an already-authenticated link -- so a - board keyed with the wrong value is unreachable from the ground. Recovery - on the bench required an SWD erase of `keystore_partition`. Fine on the - bench, fatal in flight. Options weighed in `INVESTIGATION.md` (accept it - with a verified ground procedure; bypass-allowlist `ADD_KEY`; authenticated - `CLEAR_KEY_STORE`; two-slot bootstrap provisioning; time-boxed post-boot - bypass window) -- each trades security against recoverability. - -- [ ] **Watch the other zero-headroom config constants.** Same failure mode, - same file tree: `MAX_PACKETIZER_CHANNELS = 202` vs 191 channels in use, - `MAX_PACKETIZER_PACKETS = 22` vs exactly 22 packets. Adding commands, - channels or packets to this deployment requires checking these against the - generated dictionary -- they assert at boot rather than degrading. - -- [ ] **Write up the SWD/GDB diagnostic helpers as an ADR, and point the agent - instructions at it.** The probes built while root-causing this branch are - generally useful for any "board is silent / GDS sees nothing" failure on - this hardware, and re-deriving them cost most of a session. Capture in a - new ADR (there is no `docs/adr/` yet -- this would be the first): - - `scripts/diag/hang_thread_walk.{sh,gdb}` -- reset, free-run, halt twice - N s apart; clock/SysTick/interrupt state, a walk of `_kernel.threads` - naming each F' task, the `timeout_list`, the whole downlink chain's - state, and a per-thread CPU-time diff between the two halts. - - `scripts/diag/downlink_trace.{sh,gdb}` -- breakpoints on every hop of - the `comStub -> framer -> aggregator -> spacePacketFramer -> comQueue` - com-status path. - - The older `hang_forensics.tcl` / `hang_gdb.sh` / `hang_fault_bp.*`. - Non-obvious things the ADR should record, all of which cost real time: - - Use the **raspberrypi OpenOCD fork**, not a nix/brew build. - - Sequencing run/halt via `monitor` leaves gdb serving **stale registers**; - detach + reconnect to resync (`monitor gdb sync` + `stepi` can resume the - target when the halt lands mid-ISR). - - **Always resume the target** before detaching -- a halted board drops its - USB CDC, which makes GDS see nothing and silently no-ops any command. - - A swapped-out Cortex-M thread's `callee_saved.psp` points straight at the - exception frame (LR `+0x14`, PC `+0x18`); callee regs live in the - `k_thread`. Reading `+0x34`/`+0x38` yields F' object addresses that look - exactly like plausible wild pointers -- this produced a multi-day red - herring in the earlier investigation (git history, pre-`ec0bdb37`). - - `PRIMASK=1` at `arch_cpu_idle+18` is the **normal** idle sequence, not a - masked spin. - - Prefer monotonic `base.usage.total` over saved psp/PC when asking "did - this thread make progress" -- a healthy thread re-blocking at the same - line reproduces byte-identical values. - - **`make build` does not re-derive Kconfig from device-tree changes**; use - `make generate build`. - Then add a diagnostics section to the repo's agent instructions pointing at - the ADR. **Note:** the repo has `AGENTS.md`, not `CLAUDE.md` -- decide - whether to add the section to `AGENTS.md`, or add a `CLAUDE.md` (symlink or - stub) so both agent toolchains pick it up. - -- [ ] **Remove the diagnostics before merge:** `scripts/diag/hang_thread_walk.*`, - `scripts/diag/downlink_trace.*`, `scripts/diag/hang_fault_bp.*`, - `scripts/diag/hang_forensics.tcl`, `scripts/diag/hang_gdb.sh`, the - `Hang Forensics Diagnostic` step in `ci.yaml`, and the stray capture logs. - -### Build-system trap (cost several hours this session) -`make build` does **not** re-derive Kconfig from device-tree changes -- it left -`CONFIG_FLASH_SIZE=4096` while the DTS said 16 MB, putting `keystore_partition` -out of bounds so the `/keys` automount panicked on -`__ASSERT_NO_MSG(block_size != 0)` (`littlefs_fs.c:787`). Purely an artifact of -the stale config, and it invalidated several intermediate bisect results. -**Always `make generate build` after touching the device tree.** - -## Superseded — original "Suggested fix" notes, kept for the audit trail - -- [x] **Primary fix**: changed `&flash0 { reg = <0x10000000 DT_SIZE_M(4)>; }` → - `DT_SIZE_M(16)` in `proves_flight_control_board_v5.dtsi` (shared by v5c/v5d/v5e). - Confirmed correct by CI hardware run 30034861047: OpenOCD reports the real chip - is `w25q128fv/jv ... size = 16384 KiB`. `CONFIG_FLASH_SIZE` now follows to 16384 - (was 4096), build otherwise unaffected (FLASH/RAM usage unchanged). **Necessary - but not sufficient** — see next item. -- [ ] **Diagnose the remaining stall**: same CI run still fails identically - (repeated GDS `device disconnected` serial exceptions, no response to - `CMD_NO_OP`). Revised theory: `/keys` mounting for the first time means - littlefs's format now actually reaches `flash_rpi_write`/`flash_rpi_erase`, - which hold `irq_lock()` for the whole erase/program call — `PROBLEM.md`'s - original interrupt-stall theory may be correct, it just had nothing to act on - before this fix. - Ran (and reverted) a temporary `CONFIG_LOG=y`+console diagnostic (CI run - 30036653676): boot log shows normal USB init through ~1.16s then **total - silence** for the remaining ~39s — no more log lines, no TM-frame noise - either. Consistent with a full hang very early in boot, but doesn't pinpoint - where. Ruled out a stale `PICO_FLASH_SIZE_BYTES` hard_assert in the Pico - SDK's `flash_range_erase` — that macro isn't defined in this Zephyr build. - Diagnostic reverted (console can't coexist with a working GDS link — see - `scripts/check_console_disabled.py`). - **Ran the SWD PC-sweep (CI run 30043983799, reverted after): hang located.** - cm0's pc/lr/sp/xpsr are byte-for-byte identical at t=+2s/+10s/+20s after - reset — zero forward progress for 20+ seconds. Resolved against the - build's symbols: `pc=0x101864b8` is `fs_open+2` (its first real - instruction), `lr=0x1010fb69` is inside Zephyr's `idle()` (the context - that runs early `SYS_INIT`/fstab-automount code before the scheduler - starts other threads) — i.e. cm0 is frozen at the very first file - operation this branch performs after `/keys` mounts (almost certainly - `TcSecurityDeframer::configure()`'s `loadKeyStore()` opening a virgin - key-store file for the first time). cm1 sampled `pc=0x19e` (a bootrom - address) unchanged too — cm1 was never launched into Zephyr code at all, - this app runs single-core. Two candidate mechanisms (both point at the - same fix, see the earlier investigation in git history for full - reasoning): (a) something in the mount/format path already holds - `irq_lock()` in a flash erase/program that never returns, and `fs_open`'s - first action (a shared fs mutex) blocks on it forever; (b) a dual-core - interaction given cm1's unusual unlaunched state, though the vendored - `flash_range_erase`/`flash_range_program` in this tree don't show an - obvious multicore-lockout wait. **Next: either replace the littlefs - `/keys` mount with raw `flash_area_*`/NVS (sidesteps this class of bug - regardless of exact mechanism — see Robustness follow-ups below), or dig - further into which specific call inside the mount/format/create path - never returns.** - **2026-07-27 — the "stall" does not exist.** Built and ran the v5 - thread-walk probe (`scripts/diag/hang_thread_walk.{sh,gdb}`) on the local - bench. Across two halts 4s apart the board is *fully healthy*: clock - advancing (+40040 ticks = 4.004s), no fault (`CFSR=HFSR=0`), no masked - IRQs, the 1ms base-rate `k_timer` queued and firing, all three rate - groups cycling, main looping in `startRateGroups()`, and 89% idle. Every - earlier "hang" reading was a misread healthy idle CPU (`PRIMASK=1` + - `arch_cpu_idle` is the normal `cpsid i; wfi; cpsie i`), a stale FPB - breakpoint, or callee-saved registers misread as a PC — there is no wild - jump, stack overflow, fs-lock deadlock or fatal-halt spin. - **The real failure is the downlink:** `usbd_thread`, - `udc_rpi_pico_thread_0` and `ComCcsdsUart::comQueue` consume *zero* CPU - cycles while `ComCcsdsUart::aggregator` burns 1.4M, and the host reads 0 - bytes in 15s from the board CDC. Telemetry is produced and aggregated but - never dequeued to the com driver, and the USB device stack is dormant. - **Next: chase the UART/USB downlink path**, not the filesystem. See - the earlier investigation in git history (pre-`ec0bdb37`). -- [ ] **Robustness follow-ups (evaluate once the stall is diagnosed):** (a) consider - raw `flash_area_*`/NVS instead of littlefs for this fixed-size store (avoids - the format-time erase burst and any long single-call erase/program under - `irq_lock`); (b) rate-limit `loadKeyStore()` so it isn't a fresh `fs_open` per - unrecognized-SPI frame; (c) if per-frame `writeSequenceNumber` proves a real - wear/timing problem, throttle it or move only the seq counter to a no-erase - medium (RV3028 RTC user RAM / FRAM). - -## Notes / decisions while implementing -(append here as work progresses) diff --git a/scripts/diag/downlink_trace.gdb b/scripts/diag/downlink_trace.gdb deleted file mode 100644 index 2b725f69..00000000 --- a/scripts/diag/downlink_trace.gdb +++ /dev/null @@ -1,87 +0,0 @@ -# downlink_trace.gdb -- v6 of the /keys CI-failure forensics (INVESTIGATION.md). -# -# v5 (hang_thread_walk.*) proved the firmware is healthy and localised the -# failure to the downlink: at t=12s both Com subtopologies show -# ComQueue.m_state = WAITING (on BOTH the UART and LoRa paths) -# TmFramer.m_masterFrameCount = 0 (on BOTH -- no TM frame has EVER -# been framed, on either link) -# ComCcsdsUart::comStub.m_reinitialize = 0 (the driver's ready DID arrive and -# comStub DID emit its one status) -# -# ComQueue is constructed in WAITING (ComQueue.cpp:35) and only ever reaches -# READY via comStatusIn carrying SUCCESS (ComQueue.cpp:236-247). Until then it -# never dequeues, so nothing is framed and the link is silent from boot. The -# status has to travel -# comStub.comStatusOut -> framer -> aggregator -> spacePacketFramer -# -> comQueue.comStatusIn -# and it demonstrably reaches the aggregator (its m_allow_timeout is true, i.e. -# the FILL state) but not comQueue. This probe watches that whole path live -# from reset and reports exactly where the status dies. -# -# Read-only apart from breakpoints/watchpoints, all set on a target that is -# reset at the start of the run. Driven by downlink_trace.sh. - -monitor log_output downlink-trace-openocd.log - -echo \n================ arming the downlink status path ================\n -monitor reset halt - -# Every stage that must forward the status upward. Plain linespec form, NOT -# `*func`: the `*` forces expression parsing, which needs Svc::ComStub as a -# *type* in the current context and fails with "No type ComStub within class or -# namespace Svc" before the program has run. Letting gdb skip the prologue is -# fine here because the arguments are read with `info args` (DWARF locations) -# rather than out of raw registers. -# -# ComAggregator::preamble is the one that matters most: it is the F' active -# component preamble, run on the aggregator's own thread when tasks start, and -# it is the ONLY place the aggregator emits an unprovoked comStatusOut -# (ComAggregator.cpp:24-27). Its other comStatusOut is in doFill, which needs -# data -- and data cannot flow until ComQueue is released. So if preamble -# never runs, or its status never reaches ComQueue, the chain deadlocks from -# boot exactly as observed. -hbreak Svc::ComStub::drvConnected_handler -hbreak Svc::ComAggregator::preamble -hbreak Svc::ComAggregator::comStatusIn_handler -hbreak Svc::ComQueue::comStatusIn_handler -info breakpoints - -# Report a stop, then keep going. $stops bounds the run so a fast-repeating -# hit cannot spin forever. -set $stops = 0 -set $limit = 14 - -echo \n================ running to topology setup ================\n -continue -printf "\n---- reached %s; arming the ComQueue.m_state watchpoint ----\n", "drvConnected" -# The gate itself. A hardware watchpoint fires on every write, so "never -# fires again" is itself an answer, and each hit names the writer. Armed here -# rather than at reset because bss-zeroing and ComQueue's own constructor write -# it during early boot and drown the trace in noise. -watch ComCcsdsUart::comQueue.m_state - -echo \n================ tracing the status path ================\n -while $stops < $limit - continue - set $stops = $stops + 1 - printf "\n---- stop %d ----------------------------------------------\n", $stops - printf " pc = %#lx -> ", (unsigned long)$pc - info symbol $pc - printf " ComCcsdsUart::comQueue.m_state = %d (0=READY 1=WAITING)\n", \ - (int)ComCcsdsUart::comQueue.m_state - printf " ComCcsdsUart::aggregator FILL? %d comStub.reinit=%d\n", \ - (int)ComCcsdsUart::aggregator.m_allow_timeout, \ - (int)ComCcsdsUart::comStub.m_reinitialize - printf " TmFramer mfc=%d vfc=%d\n", \ - (int)ComCcsdsUart::framer.m_masterFrameCount, \ - (int)ComCcsdsUart::framer.m_virtualFrameCount - # For the comStatusIn breakpoints `condition` is the Fw::Success& being - # forwarded -- SUCCESS=1, FAILURE=0. A FAILURE arriving at ComQueue leaves it - # WAITING (ComQueue.cpp:246) and is just as fatal as no status at all. - printf " args at this stop:\n" - info args - bt 8 -end - -echo \n================ stop limit reached; detaching ================\n -detach diff --git a/scripts/diag/downlink_trace.sh b/scripts/diag/downlink_trace.sh deleted file mode 100755 index c62eb301..00000000 --- a/scripts/diag/downlink_trace.sh +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env bash -# downlink_trace.sh -- v6 of the /keys CI-failure forensics (INVESTIGATION.md). -# -# v5 (hang_thread_walk.*) showed the firmware is healthy and the DOWNLINK is -# dead from boot: ComQueue never leaves its initial WAITING state, so nothing is -# ever framed (TmFramer master frame count stays 0 on both the UART and LoRa -# paths) and the CDC stays silent. This probe watches the status path that is -# supposed to release ComQueue -- -# comStub.comStatusOut -> framer -> aggregator -> spacePacketFramer -# -> comQueue.comStatusIn -# -- live from reset, with a hardware watchpoint on ComQueue.m_state plus -# breakpoints on each forwarding stage, and reports where the status dies. -# Read logic lives in downlink_trace.gdb. -# -# Local bench only (not CI). Drives the board over a Raspberry Pi Debug Probe -# (CMSIS-DAP SWD). The two USB CDC ttys involved: -# BOARD_TTY /dev/tty.usbmodem1101 - the target's own USB CDC (F'/GDS link) -# PROBE_TTY /dev/tty.usbmodem102 - the Debug Probe's UART bridge -# OpenOCD reaches SWD via the probe's CMSIS-DAP USB interface (not a tty). -# -# Read-only apart from the breakpoints/watchpoint it sets, all on a target that -# is reset at the start of the run. Never writes flash or config. -set -u - -BOARD_TTY=${BOARD_TTY:-/dev/tty.usbmodem1101} -PROBE_TTY=${PROBE_TTY:-/dev/tty.usbmodem102} - -# OpenOCD: the raspberrypi fork (do NOT substitute a nix/brew openocd -- the RP2350 -# support and the CMSIS-DAP build the bench relies on live in this fork). Local -# bench keeps it under ~/code/...; the CI runner uses ~/openocd. Override with -# OOCD_HOME. -OOCD_HOME=${OOCD_HOME:-} -if [ -z "$OOCD_HOME" ]; then - for cand in ~/code/github.com/raspberrypi/openocd ~/openocd; do - [ -x "$cand/src/openocd" ] && { OOCD_HOME="$cand"; break; } - done -fi -[ -x "$OOCD_HOME/src/openocd" ] || { echo "downlink_trace: openocd not found (set OOCD_HOME to the raspberrypi openocd checkout)"; exit 1; } -OCD="$OOCD_HOME/src/openocd" -OCD_ARGS=(-s "$OOCD_HOME/tcl" - -f "$OOCD_HOME/tcl/interface/cmsis-dap.cfg" - -f "$OOCD_HOME/tcl/target/rp2350.cfg" - -c "adapter speed 5000") - -HERE=$(cd "$(dirname "$0")" && pwd) -GDBCMDS="$HERE/downlink_trace.gdb" -[ -f "$GDBCMDS" ] || { echo "downlink_trace: missing $GDBCMDS"; exit 1; } - -# --- symbol ELF (local flight build; load addrs 0x1018_xxxx) --- -ELF="" -for cand in \ - build-fprime-automatic-zephyr/zephyr/zephyr.elf \ - build-artifacts/zephyr/fprime-zephyr-deployment; do - [ -f "$cand" ] && { ELF="$cand"; break; } -done -[ -z "$ELF" ] && { echo "downlink_trace: symbol ELF not found; run 'make generate build' first."; exit 1; } -echo "downlink_trace: ELF $ELF" -echo "downlink_trace: board $BOARD_TTY probe $PROBE_TTY" - -# --- ARM gdb --- -GDB="" -for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do - command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } -done -[ -z "$GDB" ] && GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | tail -1) -[ -z "$GDB" ] && { echo "downlink_trace: no arm/multiarch gdb found."; exit 1; } -echo "downlink_trace: gdb $GDB" - -# --- optional: capture the board CDC so telemetry-going-silent is timestamped --- -SERIAL_LOG=${SERIAL_LOG:-board-serial.log} -SNIFF_PID="" -if [ -e "$BOARD_TTY" ]; then - ( cat "$BOARD_TTY" > "$SERIAL_LOG" 2>/dev/null ) & - SNIFF_PID=$! - echo "downlink_trace: sniffing $BOARD_TTY -> $SERIAL_LOG" -else - echo "downlink_trace: note: $BOARD_TTY not present; skipping serial sniff" -fi - -# --- OpenOCD: init + keep the gdb server up; the .gdb file drives reset/run/halt --- -"$OCD" "${OCD_ARGS[@]}" -c "init" -c "echo {downlink_trace: gdb server on :3333}" & -OCD_PID=$! -cleanup() { [ -n "$SNIFF_PID" ] && kill "$SNIFF_PID" 2>/dev/null; kill "$OCD_PID" 2>/dev/null; } -trap cleanup EXIT -sleep 2 - -# All run/halt sequencing is done with `monitor` inside the command file, so gdb -# never thinks the target is running. That means gdb would happily serve stale -# cached data across a resume, so disable both memory caches here; the command -# file flushes the register cache after each halt. -GDB_ARGS=(-q -nx -batch "$ELF" - -ex "set pagination off" - -ex "set confirm off" - -ex "set print pretty on" - -ex "set backtrace past-main on" - -ex "set stack-cache off" - -ex "set code-cache off" - -ex "target extended-remote localhost:3333" - -x "$GDBCMDS") - -TIMEOUT_BIN=$(command -v timeout || command -v gtimeout || true) -if [ -n "$TIMEOUT_BIN" ]; then - "$TIMEOUT_BIN" 300 "$GDB" "${GDB_ARGS[@]}" \ - || echo "downlink_trace: gdb exited non-zero / timed out" -else - "$GDB" "${GDB_ARGS[@]}" \ - || echo "downlink_trace: gdb exited non-zero (see output above)" -fi - -echo "downlink_trace: done (board serial capture in $SERIAL_LOG," -echo " openocd log in downlink-trace-openocd.log)" diff --git a/scripts/diag/hang_fault_bp.gdb b/scripts/diag/hang_fault_bp.gdb deleted file mode 100644 index 4f820da5..00000000 --- a/scripts/diag/hang_fault_bp.gdb +++ /dev/null @@ -1,102 +0,0 @@ -# hang_fault_bp.gdb -- read the pre-fault (hardware-stacked) CPU state once a -# fault breakpoint has hit, before z_arm_fatal_error masks IRQs and spins in -# `b .`. Driven by hang_fault_bp.sh, which has already: connected to the gdb -# server, `monitor reset halt`, armed the HW breakpoint, and set $FAULT_ARGS. -# See INVESTIGATION.md "Next diagnostic (v4): catch the fault at entry". -# -# Two entry points, selected by hang_fault_bp.sh via FAULT_SYM -> $FAULT_ARGS: -# $FAULT_ARGS==0 break at z_arm_usage_fault (the UsageFault *vector* entry): -# the HW-stacked frame is fresh and LR still holds EXC_RETURN, -# and live $msp/$psp are the pre-fault stack pointers. -# $FAULT_ARGS==1 break at *z_arm_fault (the common C handler) instead, for -# faults that don't route through z_arm_usage_fault. Its args -# carry the state: z_arm_fault(msp=r0, psp=r1, exc_return=r2, -# callee=r3) -- confirmed in this build's fault.c:1025 -- so we -# take EXC_RETURN/msp/psp from r2/r0/r1 (LR is stale here). -# -# Build facts relied on (build-fprime-automatic-zephyr/zephyr/.config): -# CONFIG_FPU is not set -> plain 8-word (0x20) exception frame -# CONFIG_MP_MAX_NUM_CPUS=1 -> current thread is _kernel.cpus[0].current -# CONFIG_THREAD_STACK_INFO=y -> stack_info.start/size are valid -# CONFIG_THREAD_NAME is not set -> k_thread has no .name member (don't read it) - -echo \n==== running to the fault breakpoint (expected ~8s into boot) ====\n -continue - -echo \n==== FAULT CAUGHT -- pre-halt, exception frame intact ====\n -info registers lr primask basepri control xpsr - -# Recover EXC_RETURN and the pre-fault MSP/PSP for whichever entry we stopped at. -if $FAULT_ARGS - set $exc = (unsigned long)$r2 - set $msp_v = (unsigned long)$r0 - set $psp_v = (unsigned long)$r1 - printf "entry=z_arm_fault (args): exc_return=r2 msp=r0 psp=r1\n" -else - set $exc = (unsigned long)$lr - set $msp_v = (unsigned long)$msp - set $psp_v = (unsigned long)$psp - printf "entry=z_arm_usage_fault (vector): exc_return=lr msp/psp live\n" -end - -# EXC_RETURN bit2 selects the stack the CPU pushed the exception frame onto: -# 0 -> MSP (fault happened in handler mode) 1 -> PSP (fault in a thread) -set $usepsp = ($exc >> 2) & 1 -set $frame = $usepsp ? $psp_v : $msp_v -printf "exc_return=%#lx pre-fault stack=%s frame_sp=%#lx\n", \ - $exc, ($usepsp ? "PSP(thread)" : "MSP(handler)"), $frame - -echo \n==== stacked exception frame = the exact pre-fault CPU state ====\n -set $sr0 = *(unsigned long*)($frame+0x00) -set $sr1 = *(unsigned long*)($frame+0x04) -set $sr2 = *(unsigned long*)($frame+0x08) -set $sr3 = *(unsigned long*)($frame+0x0c) -set $sr12 = *(unsigned long*)($frame+0x10) -set $slr = *(unsigned long*)($frame+0x14) -set $spc = *(unsigned long*)($frame+0x18) -set $sxpsr = *(unsigned long*)($frame+0x1c) -printf " r0=%#lx r1=%#lx r2=%#lx r3=%#lx r12=%#lx\n", $sr0,$sr1,$sr2,$sr3,$sr12 -printf " stacked LR (caller/return) = %#lx\n", $slr -printf " stacked PC (faulting instr) = %#lx\n", $spc -printf " stacked xPSR = %#lx\n", $sxpsr -echo -- symbolize the pre-fault PC and LR --\n -printf " faulting PC -> " -info symbol $spc -printf " stacked LR -> " -info symbol $slr - -echo \n==== why: CFSR / UFSR (UsageFault status @ 0xE000ED28) ====\n -# UFSR = upper halfword of CFSR. Key bits for a wild jump: -# bit16 UNDEFINSTR : jumped into non-code / bad opcode -# bit17 INVSTATE : Thumb (EPSR.T) bit clear -> branched to an even/data addr -# bit18 INVPC : bad EXC_RETURN / integrity check -# INVSTATE or UNDEFINSTR here == executed a data/garbage address (matches the -# observed pc=0x20010480 inside the fileManager object). -x/1xw 0xE000ED28 - -echo \n==== stack-overflow test: is frame_sp below the faulting thread's stack? ====\n -set $thr = _kernel.cpus[0].current -set $sbase = (unsigned long)$thr->stack_info.start -set $ssize = (unsigned long)$thr->stack_info.size -printf " current k_thread @ %#lx\n", (unsigned long)$thr -printf " stack: base=%#lx size=%#lx top=%#lx\n", $sbase, $ssize, $sbase+$ssize -printf " frame_sp=%#lx -> %s\n", $frame, \ - ($frame < $sbase ? "*** SP BELOW STACK BASE == STACK OVERFLOW ***" : \ - ($frame > $sbase+$ssize ? "*** SP ABOVE STACK TOP (wrong stack?) ***" : \ - "within stack extent (points to wild pointer, not overflow)")) - -echo \n==== reconstructed backtrace of the FAULTING thread ====\n -# Rewind GDB's view to the pre-fault frame so `bt` unwinds the culprit rather -# than the fault handler. Frame is 0x20 bytes; xPSR bit9 set => +4 align pad. -# NOTE: this writes core registers on the (about-to-be-reset) target. -set $pad = (($sxpsr >> 9) & 1) ? 4 : 0 -set $sp = $frame + 0x20 + $pad -set $pc = $spc -set $lr = $slr -bt - -echo \n==== raw stack window above frame_sp (find real 0x10xx return addrs) ====\n -x/64xw $frame - -echo \n==== done (detaching; board left halted) ====\n -detach diff --git a/scripts/diag/hang_fault_bp.sh b/scripts/diag/hang_fault_bp.sh deleted file mode 100755 index caf7598e..00000000 --- a/scripts/diag/hang_fault_bp.sh +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env bash -# hang_fault_bp.sh -- v4 of the /keys boot-hang forensics (see INVESTIGATION.md). -# -# All prior forensics (hang_forensics.tcl / hang_gdb.sh) halted the board AFTER -# it was already spinning in the masked `b .` fatal-halt loop -- i.e. after -# z_arm_fatal_error ran -- so the backtrace was incoherent (lr=0x1019, garbage -# stack). This step instead sets a HARDWARE breakpoint on the UsageFault vector -# entry (z_arm_usage_fault), resets, and lets it hit at ~8s, catching the fault -# with the exception frame fresh and EXC_RETURN still in LR. From that frame it -# reads the true faulting PC, the caller LR, and the pre-fault SP, and compares -# SP to the faulting thread's stack bounds -- deciding STACK OVERFLOW vs. a -# WILD/CORRUPTED POINTER directly. See hang_fault_bp.gdb for the read logic. -# -# Local bench only (not CI). Drives the board over a Raspberry Pi Debug Probe -# (CMSIS-DAP SWD). The two USB CDC ttys involved: -# BOARD_TTY /dev/tty.usbmodem3101 - the target's own USB CDC (F'/GDS link); -# telemetry goes silent here at the hang -# PROBE_TTY /dev/tty.usbmodem102 - the Debug Probe's UART bridge -# OpenOCD reaches SWD via the probe's CMSIS-DAP USB interface (not a tty); the -# ttys are used only to correlate/timestamp the hang. -# -# Read-mostly: it resets+halts and reads state; the only writes are the GDB -# breakpoint and a register rewind for the reconstructed backtrace, both on a -# target that is reset at the start of the run. Never writes flash or config. -set -u - -BOARD_TTY=${BOARD_TTY:-/dev/tty.usbmodem3101} -PROBE_TTY=${PROBE_TTY:-/dev/tty.usbmodem102} - -# Which fault entry to break on. Default z_arm_usage_fault (the UsageFault -# vector -- LR still holds EXC_RETURN there). Set FAULT_SYM=z_arm_fault to -# instead catch the common C handler, for faults that don't route through the -# UsageFault vector (e.g. a HardFault escalation); its args carry EXC_RETURN. -FAULT_SYM=${FAULT_SYM:-z_arm_usage_fault} -if [ "$FAULT_SYM" = "z_arm_fault" ]; then - FAULT_ARGS=1 # read exc_return/msp/psp from r2/r0/r1 (see hang_fault_bp.gdb) -else - FAULT_ARGS=0 # read exc_return from LR, msp/psp live - [ "$FAULT_SYM" != "z_arm_usage_fault" ] && \ - echo "hang_fault_bp: warning: FAULT_SYM=$FAULT_SYM is unrecognized; assuming LR holds EXC_RETURN (vector-entry mode)" -fi -# Break at the exact address (`*`) so GDB does not skip a prologue and clobber -# the argument registers before we read them. -BP_SPEC="*$FAULT_SYM" - -# OpenOCD: the raspberrypi fork (do NOT substitute a nix/brew openocd -- the RP2350 -# support and the CMSIS-DAP build the bench relies on live in this fork). Local -# bench keeps it under ~/code/...; the CI runner uses ~/openocd. Override with -# OOCD_HOME. -OOCD_HOME=${OOCD_HOME:-} -if [ -z "$OOCD_HOME" ]; then - for cand in ~/code/github.com/raspberrypi/openocd ~/openocd; do - [ -x "$cand/src/openocd" ] && { OOCD_HOME="$cand"; break; } - done -fi -[ -x "$OOCD_HOME/src/openocd" ] || { echo "hang_fault_bp: openocd not found (set OOCD_HOME to the raspberrypi openocd checkout)"; exit 1; } -OCD="$OOCD_HOME/src/openocd" -OCD_ARGS=(-s "$OOCD_HOME/tcl" - -f "$OOCD_HOME/tcl/interface/cmsis-dap.cfg" - -f "$OOCD_HOME/tcl/target/rp2350.cfg" - -c "adapter speed 5000") - -HERE=$(cd "$(dirname "$0")" && pwd) -GDBCMDS="$HERE/hang_fault_bp.gdb" -[ -f "$GDBCMDS" ] || { echo "hang_fault_bp: missing $GDBCMDS"; exit 1; } - -# --- symbol ELF (local flight build; load addrs 0x1018_xxxx) --- -ELF="" -for cand in \ - build-fprime-automatic-zephyr/zephyr/zephyr.elf \ - build-artifacts/zephyr/fprime-zephyr-deployment; do - [ -f "$cand" ] && { ELF="$cand"; break; } -done -[ -z "$ELF" ] && { echo "hang_fault_bp: symbol ELF not found; run 'make generate build' first."; exit 1; } -echo "hang_fault_bp: ELF $ELF" -echo "hang_fault_bp: board $BOARD_TTY probe $PROBE_TTY" -echo "hang_fault_bp: break $BP_SPEC (FAULT_ARGS=$FAULT_ARGS)" - -# --- ARM gdb --- -GDB="" -for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do - command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } -done -[ -z "$GDB" ] && GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | head -1) -[ -z "$GDB" ] && { echo "hang_fault_bp: no arm/multiarch gdb found."; exit 1; } -echo "hang_fault_bp: gdb $GDB" - -# --- optional: capture the board CDC so telemetry-going-silent is timestamped --- -SERIAL_LOG="board-serial.log" -SNIFF_PID="" -if [ -e "$BOARD_TTY" ]; then - ( cat "$BOARD_TTY" > "$SERIAL_LOG" 2>/dev/null ) & - SNIFF_PID=$! - echo "hang_fault_bp: sniffing $BOARD_TTY -> $SERIAL_LOG" -else - echo "hang_fault_bp: note: $BOARD_TTY not present; skipping serial sniff" -fi - -# --- OpenOCD: init + keep the gdb server up; GDB drives reset/breakpoint --- -"$OCD" "${OCD_ARGS[@]}" -c "init" -c "echo {hang_fault_bp: gdb server on :3333}" & -OCD_PID=$! -cleanup() { [ -n "$SNIFF_PID" ] && kill "$SNIFF_PID" 2>/dev/null; kill "$OCD_PID" 2>/dev/null; } -trap cleanup EXIT -sleep 2 - -# Connect, reset+halt, and arm the HW breakpoint here (before boot runs), then -# hand off to the command file which continues to the fault and reads the frame. -# Guard with a timeout so a fault that never fires (continue blocks forever) -# doesn't wedge the run. -GDB_ARGS=(-q -nx -batch "$ELF" - -ex "set pagination off" - -ex "set confirm off" - -ex "set print pretty on" - -ex "target extended-remote localhost:3333" - -ex "monitor reset halt" - -ex "hbreak $BP_SPEC" - -ex "set \$FAULT_ARGS = $FAULT_ARGS" - -x "$GDBCMDS") - -TIMEOUT_BIN=$(command -v timeout || command -v gtimeout || true) -if [ -n "$TIMEOUT_BIN" ]; then - "$TIMEOUT_BIN" 90 "$GDB" "${GDB_ARGS[@]}" \ - || echo "hang_fault_bp: gdb exited non-zero / timed out (fault may not have fired within 90s)" -else - echo "hang_fault_bp: note: no timeout(1); gdb 'continue' will block until the fault fires." - "$GDB" "${GDB_ARGS[@]}" \ - || echo "hang_fault_bp: gdb exited non-zero (see output above)" -fi - -echo "hang_fault_bp: done (board serial capture in $SERIAL_LOG)" diff --git a/scripts/diag/hang_forensics.tcl b/scripts/diag/hang_forensics.tcl deleted file mode 100644 index 8d4bbaaf..00000000 --- a/scripts/diag/hang_forensics.tcl +++ /dev/null @@ -1,120 +0,0 @@ -# hang_forensics.tcl -- read-only SWD forensics for the /keys boot hang -# (branch hmac-to-storage, PR #472). See INVESTIGATION.md. -# -# Purpose: decide *why* cm0 is frozen at ~fs_open after the flash-size fix, -# BEFORE committing to the littlefs->NVS rework. Distinguishes: -# (A) core faulted / locked up -> CFSR/HFSR/DFSR/DHCSR non-zero -# (B) QMI/XIP interface left wedged -> QMI_DIRECT_CSR.EN=1 / bad M0_RFMT -# by the flash erase/program dance (flash-layer bug; NVS would ALSO -# hang -> rework is wasted) -# (C) cleanly blocked on an fs/lfs lock -> no fault, XIP sane, stack shows a -# k_mutex/k_sem wait (rework helps) -# -# Usage (CI or MOSAIC/GDS bench): -# openocd -s ~/openocd/tcl \ -# -f interface/cmsis-dap.cfg -f target/rp2350.cfg \ -# -c "adapter speed 5000" \ -# -f scripts/diag/hang_forensics.tcl -# -# SAFETY -- DO NOT read any 0x10xx_xxxx (XIP flash) address over SWD here. If -# the QMI/XIP interface is wedged (hypothesis B), an SWD read of a flash -# address can stall the debug adapter and lose the whole capture. Every read -# below targets the PPB (0xE000_xxxx), the QMI/XIP peripherals (0x400C/D_xxxx), -# or SRAM (0x2003_xxxx) only. The instruction at PC is already known from the -# ELF (fs_open begins with a 4-byte `stmdb` at 0x101864b6), so we never read -# code back over the wire. -# -# Memory-AP reads (mdw) work whether or not the core actually halts, so even a -# locked-up core still yields its fault registers and peripheral state. Each -# section is wrapped in `catch` so one failing read never aborts the rest. - -proc rd {label addr} { - if {[catch {set line [capture "mdw $addr"]} err]} { - echo " $label ($addr): " - } else { - echo " $label ($addr): [string trim $line]" - } -} - -# Bare `reg`/`mdw` output does NOT reach the CI step stdout in batch (-c/-f) -# mode -- only `echo` and `capture` do (see run 30051402310, where the core-reg -# and stack blocks came back empty). Wrap every such command so its output is -# echoed into the captured log. -proc dump {cmd} { - if {[catch {set out [capture $cmd]} err]} { - echo " \[$cmd]: " - } else { - echo " \[$cmd]: [string trim $out]" - } -} - -init - -echo "=== reset + free-run so boot reaches the hang ===" -# Console-log diagnostic put the hang at ~1.16 s; 8 s leaves a wide margin and -# the PC sweep already proved the freeze is permanent (0 progress over 20 s), -# so a single halt is sufficient -- no need to sweep offsets again. -reset run -sleep 8000 - -targets rp2350.cm0 -poll -# Tolerate a halt that never completes (lockup / bus stall): the memory reads -# below go through the debug MEM-AP and do not require the core to be halted. -catch {halt} -poll - -echo "" -echo "=== CORE REGISTERS (cm0) -- expect pc ~= fs_open (0x101864b6) ===" -foreach r {pc lr sp msp psp xpsr r0 r1 r2 r3 r4 r5 r6 r7} { - dump "reg $r" -} - -echo "" -echo "=== INTERRUPT MASKING -- what is holding IRQs off? ===" -echo " PRIMASK=1 => irq_lock via PRIMASK; BASEPRI!=0 => Zephyr BASEPRI mask." -echo " Confirms the ISRPENDING-but-unserviced spin seen in run 30051402310." -foreach r {primask basepri faultmask control} { - dump "reg $r" -} - -echo "" -echo "=== FAULT / DEBUG STATUS (Cortex-M SCB, architectural addrs) ===" -echo " non-zero CFSR/HFSR => a fault escalated; DHCSR bit19 (0x00080000)" -echo " S_LOCKUP => core is locked up, PC is stale (rules out the fs-lock" -echo " theory outright)." -rd "ICSR " 0xE000ED04 -rd "SHCSR " 0xE000ED24 -rd "CFSR " 0xE000ED28 -rd "HFSR " 0xE000ED2C -rd "DFSR " 0xE000ED30 -rd "MMFAR " 0xE000ED34 -rd "BFAR " 0xE000ED38 -rd "DHCSR " 0xE000EDF0 - -echo "" -echo "=== QMI / XIP PERIPHERAL STATE -- is XIP still restored after erase? ===" -echo " QMI_DIRECT_CSR bit0 (EN, 0x1) SET => still in direct/serial mode, XIP" -echo " NOT re-enabled -> next flash fetch stalls forever == the wedge. A" -echo " clobbered M0_RFMT/M0_RCMD means the XIP read cmd config was lost." -rd "QMI_DIRECT_CSR" 0x400D0000 -rd "QMI_M0_TIMING " 0x400D000C -rd "QMI_M0_RFMT " 0x400D0010 -rd "QMI_M0_RCMD " 0x400D0014 -rd "XIP_CTRL " 0x400C8000 -rd "XIP_STAT " 0x400C8008 - -echo "" -echo "=== STACK DUMP for hand-unwind (SRAM only) ===" -echo " Return addresses appear as 0x1018_xxxx / 0x1010_xxxx words on the" -echo " stack; resolve them against zephyr.elf to reconstruct the call chain" -echo " below fs_open (mount? create? a k_mutex_lock wait?). SP was a rock" -echo " stable 0x20034410 across the whole PC sweep, so dump a fixed window" -echo " from just below it (does not require the live reg read to succeed)." -dump "reg sp" -dump "mdw 0x20034380 96" - -echo "" -echo "=== done ===" -catch { resume } -shutdown diff --git a/scripts/diag/hang_gdb.sh b/scripts/diag/hang_gdb.sh deleted file mode 100755 index 7ac6bbcd..00000000 --- a/scripts/diag/hang_gdb.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# hang_gdb.sh -- v3 of the /keys boot-hang forensics (see INVESTIGATION.md). -# -# The register/QMI/stack capture (hang_forensics.tcl) ruled out a CPU fault, a -# lockup, and a wedged flash/XIP interface, and showed the core spinning with -# interrupts masked (PRIMASK=1), an off-boundary PC inside fs_open, an -# incoherent LR (idle), and a shallow garbage stack -- i.e. a software -# control-flow failure, not a clean fs mutex block. This step attaches GDB to -# the OpenOCD gdb server for a real DWARF backtrace + single-step, to decide -# between: (a) a wild/corrupted PC, (b) a Zephyr __ASSERT/SPIN_VALIDATE panic -# (a z_fatal_error/arch_system_halt frame would show), or (c) a genuine -# lock-spin. That distinction determines whether the littlefs->NVS rework is -# the right fix or a red herring. -# -# Read-only: halts and inspects, never writes flash or changes config. Safe to -# read code over SWD now that v2 confirmed XIP is healthy. Best-effort: if no -# suitable GDB is found on the runner it prints a notice and exits 0 (the CI -# step is `|| true` anyway) -- the register capture already stands on its own. -set -u - -OCD=~/openocd/src/openocd -OCD_ARGS=(-s ~/openocd/tcl - -f ~/openocd/tcl/interface/cmsis-dap.cfg - -f ~/openocd/tcl/target/rp2350.cfg - -c "adapter speed 5000") - -# --- locate the symbol ELF (flight-software deployment, load addrs 0x1018_xxxx) -ELF="" -for cand in \ - build-artifacts/zephyr/fprime-zephyr-deployment \ - "$GITHUB_WORKSPACE"/build-artifacts/zephyr/fprime-zephyr-deployment; do - [ -f "$cand" ] && { ELF="$cand"; break; } -done -if [ -z "$ELF" ]; then - echo "hang_gdb: symbol ELF not found (build-artifacts/zephyr/fprime-zephyr-deployment); skipping." - exit 0 -fi -echo "hang_gdb: using ELF $ELF" - -# --- locate an ARM / multiarch GDB -GDB="" -for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do - command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } -done -if [ -z "$GDB" ]; then - GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | head -1) -fi -if [ -z "$GDB" ]; then - echo "hang_gdb: no arm/multiarch gdb on this runner (tried arm-zephyr-eabi-gdb," - echo " gdb-multiarch, arm-none-eabi-gdb, ~/zephyr-sdk*). Skipping backtrace." - exit 0 -fi -echo "hang_gdb: using GDB $GDB" - -# --- start OpenOCD: reset, free-run to the hang, halt, then KEEP the gdb server -# alive (no `shutdown`) so GDB can attach to the halted core. -"$OCD" "${OCD_ARGS[@]}" \ - -c "init" \ - -c "reset run" \ - -c "sleep 8000" \ - -c "targets rp2350.cm0" \ - -c "halt" \ - -c "echo {hang_gdb: core halted, gdb server ready on :3333}" & -OCD_PID=$! -trap 'kill "$OCD_PID" 2>/dev/null' EXIT -# wait past the 8s free-run + halt before connecting -sleep 12 - -"$GDB" -q -nx -batch "$ELF" \ - -ex "set pagination off" \ - -ex "set confirm off" \ - -ex "target extended-remote localhost:3333" \ - -ex "echo \n==== core registers ====\n" \ - -ex "info registers" \ - -ex "echo \n==== masking / psr ====\n" \ - -ex "info registers primask basepri faultmask control xpsr" \ - -ex "echo \n==== DWARF backtrace (the decisive read) ====\n" \ - -ex "bt -full" \ - -ex "echo \n==== frame 0 detail ====\n" \ - -ex "info frame" \ - -ex "echo \n==== disassembly around PC ====\n" \ - -ex "x/16i \$pc-16" \ - -ex "echo \n==== single-step x8: does the PC advance, and to where? ====\n" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "stepi" -ex "printf \"pc=%#x\\n\", \$pc" \ - -ex "echo \n==== backtrace after stepping ====\n" \ - -ex "bt" \ - -ex "echo \n==== threads (openocd hwthread view) ====\n" \ - -ex "info threads" \ - -ex "detach" \ - || echo "hang_gdb: gdb session returned non-zero (see output above)" - -echo "hang_gdb: done" diff --git a/scripts/diag/hang_thread_walk.gdb b/scripts/diag/hang_thread_walk.gdb deleted file mode 100644 index 850e11e9..00000000 --- a/scripts/diag/hang_thread_walk.gdb +++ /dev/null @@ -1,377 +0,0 @@ -# hang_thread_walk.gdb -- v5 of the /keys boot-hang forensics (INVESTIGATION.md). -# -# The v4 fault-entry probe (hang_fault_bp.*) established there is NO CPU fault: -# breakpoints on z_arm_usage_fault / z_arm_fault never hit, and a clean halt of -# the board shows cm0 in arch_cpu_idle with CFSR=HFSR=0. The open question was -# whether the scheduler had stopped making progress with every thread blocked -# forever. -# -# This probe answers the two questions that state raises: -# 1. Is the system clock still running? (compare cycle_count / curr_tick and -# the live SysTick registers across two halts several seconds apart) -# 2. Who is blocked, and on what? (walk _kernel.threads and, for each -# swapped-out thread, recover its resume PC/LR from the saved PSP frame) -# -# Driven by hang_thread_walk.sh, which has connected to the OpenOCD gdb server. -# All run/halt sequencing happens below via `monitor`, so gdb never believes the -# target is running -- hence the explicit register-cache flush after each halt -# and the stack/code caches disabled by the driver script. -# -# Build facts relied on (build-fprime-automatic-zephyr/zephyr/.config): -# CONFIG_THREAD_MONITOR=y -> _kernel.threads list + k_thread.entry exist -# CONFIG_THREAD_STACK_INFO=y -> stack_info.start/size are valid -# CONFIG_THREAD_NAME is not set -> no k_thread.name; we symbolize entry.pEntry -# CONFIG_USE_SWITCH is not set -> classic Cortex-M PendSV swap. Note the -# callee-saved regs live in the k_thread -# (struct _callee_saved = v1-v8 + psp), NOT -# on the stack, so callee_saved.psp points -# straight at the hardware exception frame: -# [r0,r1,r2,r3,r12,lr,pc,xpsr] -# CONFIG_FPU is not set -> that frame is exactly 0x20 bytes -# CONFIG_MP_MAX_NUM_CPUS=1 -> current thread is _kernel.cpus[0].current -# CONFIG_TICKLESS_KERNEL=y -> SysTick is reloaded per-timeout (last_load) -# CONFIG_CORTEX_M_SYSTICK_64BIT_CYCLE_COUNTER=y -> cycle_count is 64-bit - -# Cortex-M33 register block addresses used below (read as words): -# 0xE000E010 SYST_CSR 0xE000E014 SYST_RVR 0xE000E018 SYST_CVR -# 0xE000E100 NVIC_ISER0 0xE000E200 NVIC_ISPR0 -# 0xE000ED04 ICSR 0xE000ED24 SHCSR 0xE000ED28 CFSR 0xE000ED2C HFSR - -define hw_snapshot - printf "-- kernel time base --\n" - printf " cycle_count = %llu\n", (unsigned long long)cycle_count - printf " announced_cycles = %llu\n", (unsigned long long)announced_cycles - printf " curr_tick = %lld\n", (long long)curr_tick - printf " last_load = %#lx\n", (unsigned long)last_load - printf "-- SysTick (live peripheral) --\n" - printf " SYST_CSR = %#010lx (bit0 ENABLE, bit1 TICKINT, bit16 COUNTFLAG)\n", \ - *(unsigned long*)0xE000E010 - printf " SYST_RVR = %#010lx SYST_CVR = %#010lx\n", \ - *(unsigned long*)0xE000E014, *(unsigned long*)0xE000E018 - printf "-- interrupt state --\n" - printf " PRIMASK=%#lx BASEPRI=%#lx FAULTMASK=%#lx CONTROL=%#lx\n", \ - (unsigned long)$primask, (unsigned long)$basepri, \ - (unsigned long)$faultmask, (unsigned long)$control - printf " ICSR = %#010lx (bit22 ISRPENDING, bits[8:0] VECTACTIVE)\n", \ - *(unsigned long*)0xE000ED04 - printf " NVIC_ISER0=%#010lx NVIC_ISPR0=%#010lx SHCSR=%#010lx\n", \ - *(unsigned long*)0xE000E100, *(unsigned long*)0xE000E200, \ - *(unsigned long*)0xE000ED24 - printf " CFSR = %#010lx HFSR = %#010lx (both 0 => no CPU fault)\n", \ - *(unsigned long*)0xE000ED28, *(unsigned long*)0xE000ED2C - printf "-- live core --\n" - printf " pc=%#lx sp=%#lx msp=%#lx psp=%#lx\n", \ - (unsigned long)$pc, (unsigned long)$sp, \ - (unsigned long)$msp, (unsigned long)$psp - printf " pc -> " - info symbol $pc - printf " backtrace of the live context:\n" - bt 12 - printf " -- call chain (text-looking words above live sp) --\n" - stack_ras $sp 0x200 -end - -# kernel/timeout.c's static list -- the answer to "is anything still -# scheduled to wake, and when". Kept in its own command (invoked after -# thread_walk) so a problem here never costs us the thread data. -define timeout_walk - # An empty sys_dlist_t points at itself, so head == &timeout_list means - # nothing at all is waiting on the clock. - printf "-- timeout queue (kernel/timeout.c timeout_list @ %#lx) --\n", \ - (unsigned long)&timeout_list - printf " head = %#lx%s\n", (unsigned long)timeout_list.head, \ - (timeout_list.head == &timeout_list ? \ - " (EMPTY: nothing is waiting on time)" : " (timeouts pending)") - # dticks is a *delta* chain: entry N fires `sum(dticks[0..N])` ticks after the - # last announcement. A head whose remaining delta never shrinks between the - # two halts is the smoking gun for "the clock counts but nothing is announced - # to the timeout layer". - set $to_n = 0 - set $to_sum = (long long)0 - set $to_p = (struct _timeout *)timeout_list.head - while $to_p != (struct _timeout *)&timeout_list && $to_n < 12 - set $to_sum = $to_sum + (long long)$to_p->dticks - printf " [%d] _timeout @ %#lx dticks=%lld (fires in %lld ticks = %lld ms)\n", \ - $to_n, (unsigned long)$to_p, (long long)$to_p->dticks, $to_sum, \ - $to_sum * 1000 / 10000 - printf " fn = %#lx -> ", (unsigned long)$to_p->fn - info symbol $to_p->fn - set $to_p = (struct _timeout *)$to_p->node.next - set $to_n = $to_n + 1 - end - printf " (%d timeouts queued)\n", $to_n -end - -# The downlink chain, per Com subtopology instance. Every stage of it waits on -# a status handed back from the stage below: -# comQueue -> spacePacketFramer -> aggregator -> framer -> comStub -> driver -# comStub.comStatusOut -> framer -> aggregator -> spacePacketFramer -> comQueue -# so a status that never comes back parks the whole chain and the link goes -# silent while the rest of the system stays perfectly healthy. This dump says -# which stage is parked and whether frames are moving at all. -# ComQueue.m_state READY | WAITING (WAITING = sent, awaiting status) -# ComAggregator.m_allow_timeout false => in WAIT_STATUS, discarding timeouts -# TmFramer.m_masterFrameCount increments per frame emitted downstream -- -# compare across the two halts: not advancing -# means nothing is being framed at all -# ComStub.m_reinitialize true => still waiting for a drvConnected -# NOTE: ComQueue::run only publishes queue-depth telemetry and is NOT what -# drives the dequeue, so `run` being unconnected cannot cause silence. -# NOTE: only the UART subtopology instantiates a comStub; the LoRa one reaches -# its radio by another path, so `ComCcsdsLora::comStub` does not exist. -define com_state - printf "-- downlink chain state --\n" - printf " %-14s %-9s %-14s %-14s %s\n", \ - "instance", "ComQueue", "Aggregator", "TmFramer", "ComStub" - com_state_one ComCcsdsUart - printf " %-14s %-9s %-14s %-14s %s\n", "", "", "", "", "" - com_state_one ComCcsdsLora - printf " ComCcsdsUart::comStub: reinitialize=%d retry_count=%d\n", \ - (int)ComCcsdsUart::comStub.m_reinitialize, \ - (int)ComCcsdsUart::comStub.m_retry_count -end - -define com_state_one - printf " %-14s ", "$arg0" - # ComQueue::SendState: READY=0, WAITING=1 (ComQueue.hpp:103). Compared - # numerically because gdb loses the Svc:: enum context across the reconnect. - printf "%-9s ", ($arg0::comQueue.m_state == 0 ? "READY" : "*WAITING*") - printf "%-14s ", ($arg0::aggregator.m_allow_timeout ? \ - "FILL" : "*WAIT_STATUS*") - printf "mfc=%-3d vfc=%-3d ", (int)$arg0::framer.m_masterFrameCount, \ - (int)$arg0::framer.m_virtualFrameCount - set $cs_mfc = (unsigned long)$arg0::framer.m_masterFrameCount -end - -# Decode _thread_base.thread_state (include/zephyr/kernel_structs.h:52-72). -define state_bits - set $st = (unsigned long)$arg0 - printf "%#04lx [", $st - if $st == 0 - printf "READY/RUNNING" - end - if $st & 0x01 - printf "DUMMY " - end - if $st & 0x02 - printf "PENDING " - end - if $st & 0x04 - printf "SLEEPING " - end - if $st & 0x08 - printf "DEAD " - end - if $st & 0x10 - printf "SUSPENDED " - end - if $st & 0x20 - printf "ABORTING " - end - if $st & 0x40 - printf "SUSPENDING " - end - if $st & 0x80 - printf "QUEUED " - end - printf "]" -end - -# Symbolize every word in [$arg0, $arg0+$arg1) that looks like a Thumb return -# address into .text -- a hand-rolled unwind. Used instead of rewinding gdb's -# $pc/$sp into each blocked thread: that writes core registers and, when a frame -# is unrecoverable, wedges gdb ("attempt to assign to an unmodifiable value"). -# This is purely read-only and works no matter how mangled the frame is. -define stack_ras - set $ra_p = (unsigned long)$arg0 - set $ra_e = (unsigned long)$arg0 + (unsigned long)$arg1 - set $ra_n = 0 - while $ra_p < $ra_e && $ra_n < 20 - set $ra_w = *(unsigned long*)$ra_p - # Thumb code pointer inside this image's .text region. - if ($ra_w & 1) && $ra_w > (unsigned long)&__text_region_start && $ra_w < (unsigned long)&__text_region_end - printf " +%#04lx %#010lx ", $ra_p - (unsigned long)$arg0, $ra_w - info symbol $ra_w - 1 - set $ra_n = $ra_n + 1 - end - set $ra_p = $ra_p + 4 - end - if $ra_n == 0 - printf " (no text-looking return addresses in this window)\n" - end -end - -# Walk the CONFIG_THREAD_MONITOR list of every thread in the system. For each -# swapped-out thread recover where it will resume: callee_saved.psp points at -# the hardware exception frame PendSV entry pushed, so the stacked LR is at -# +0x14 and the stacked PC at +0x18 (callee regs v1-v8 are in the k_thread). -# -# $walk_sum accumulates each thread's CONFIG_SCHED_THREAD_USAGE cycle counter -# (base.usage.total), which is monotonic: comparing it between the two halts is -# a one-number answer to "did ANY thread get CPU time?". Do NOT use saved -# psp/resume-PC for this -- a healthy thread that blocks at the same line every -# cycle reproduces byte-identical values and would read as frozen. -define thread_walk - set $cur = _kernel.cpus[0].current - set $t = _kernel.threads - set $n = 0 - set $walk_sum = (unsigned long long)0 - printf "-- thread walk (current = %#lx) --\n", (unsigned long)$cur - while $t != 0 && $n < 32 - printf "\n [%d] k_thread @ %#lx%s\n", $n, (unsigned long)$t, \ - ($t == $cur ? " <== CURRENT" : "") - printf " entry = %#lx -> ", (unsigned long)$t->entry.pEntry - info symbol $t->entry.pEntry - # CONFIG_THREAD_NAME is off, so all 21 F' task threads share one entry - # symbol (Os::Zephyr::Task::zephyrEntryWrapper). The entry *argument* is - # the per-task pointer, which lands inside the owning F' component object - # -- symbolizing it is what actually names the thread. - printf " arg = %#lx -> ", (unsigned long)$t->entry.parameter1 - info symbol $t->entry.parameter1 - printf " state = " - state_bits $t->base.thread_state - printf " prio=%d preempt=%#x\n", (int)$t->base.prio, \ - (unsigned int)$t->base.preempt - printf " pended_on = %#lx%s\n", (unsigned long)$t->base.pended_on, \ - ($t->base.pended_on != 0 ? " (blocked on a wait queue)" : "") - printf " timeout = dticks=%lld node.next=%#lx%s\n", \ - (long long)$t->base.timeout.dticks, \ - (unsigned long)$t->base.timeout.node.next, \ - ($t->base.timeout.node.next != 0 ? " (queued in _kernel.timeouts)" : " (NO timeout armed)") - - printf " cpu cycles= %llu (base.usage.total)\n", \ - (unsigned long long)$t->base.usage.total - set $walk_sum = $walk_sum + (unsigned long long)$t->base.usage.total - set $sbase = (unsigned long)$t->stack_info.start - set $ssize = (unsigned long)$t->stack_info.size - set $tpsp = (unsigned long)$t->callee_saved.psp - printf " stack = base=%#lx size=%#lx top=%#lx\n", \ - $sbase, $ssize, $sbase + $ssize - printf " saved psp = %#lx", $tpsp - if $tpsp < $sbase - printf " *** BELOW STACK BASE == OVERFLOW ***\n" - else - if $tpsp > $sbase + $ssize - printf " *** ABOVE STACK TOP (wrong stack / not yet swapped) ***\n" - else - printf " used=%#lx of %#lx (%lu%% headroom left)\n", \ - ($sbase + $ssize - $tpsp), $ssize, \ - (unsigned long)(($tpsp - $sbase) * 100 / $ssize) - end - end - - # Resume PC/LR are only meaningful for a thread that is actually swapped - # out with a PendSV frame on its own stack. Skip the running thread (its - # live $pc/$sp were printed by hw_snapshot) and any bogus psp. - if $t != $cur && $tpsp >= $sbase && $tpsp + 0x20 <= $sbase + $ssize - set $rlr = *(unsigned long*)($tpsp + 0x14) - set $rpc = *(unsigned long*)($tpsp + 0x18) - printf " resume LR = %#lx -> ", $rlr - info symbol $rlr - printf " resume PC = %#lx -> ", $rpc - info symbol $rpc - echo -- call chain (text-looking words above the exception frame) --\n - # gdb splits user-command arguments on whitespace, so an expression like - # ($tpsp + 0x20) would arrive as three separate args -- precompute. - set $rs_a = $tpsp + 0x20 - set $rs_l = ($sbase + $ssize) - $rs_a - stack_ras $rs_a $rs_l - end - - set $t = $t->next_thread - set $n = $n + 1 - end - printf "\n-- %d threads walked; total CPU cycles across all threads = %llu --\n", \ - $n, (unsigned long long)$walk_sum -end - -# OpenOCD forwards its own log to the attached gdb, which interleaves -# "[rp2350.cm1] halted due to debug-request" mid-printf and shreds the report. -# Send it to a file instead (hang_thread_walk.sh prints the path). -monitor log_output hang-thread-walk-openocd.log - -echo \n================ RESET + FREE-RUN INTO THE HANG ================\n -monitor reset halt -monitor resume -echo hang_thread_walk: running to reach the failed state (window A)...\n -eval "monitor sleep %d", $WINDOW_A_MS -monitor halt -# The run/halt above went through `monitor`, so gdb still believes the target -# never moved and would serve register values cached from the reset halt (this -# bit the first version of this script: it reported pc=z_arm_reset at a halt -# 12s into the run). Reconnecting is the reliable resync: gdb re-queries the -# stop reason and gets the true state. (`monitor gdb sync` + `stepi` is the -# usual recipe but is not safe here -- when the halt lands mid-ISR that stepi -# can resume the target, after which every later read fails with "Cannot -# execute this command while the target is running".) Convenience variables -# survive the reconnect, so the A/B comparison below is unaffected. -detach -target extended-remote localhost:3333 -maintenance flush register-cache - -echo \n================ HALT A ================\n -hw_snapshot -set $A_cycles = (unsigned long long)cycle_count -set $A_tick = (long long)curr_tick -set $A_cvr = *(unsigned long*)0xE000E018 -set $A_current = (unsigned long)_kernel.cpus[0].current -thread_walk -timeout_walk -com_state -set $A_sum = $walk_sum - -eval "echo \\n================ FREE-RUN %d ms ================\\n", $WINDOW_B_MS -monitor resume -eval "monitor sleep %d", $WINDOW_B_MS -monitor halt -# The run/halt above went through `monitor`, so gdb still believes the target -# never moved and would serve register values cached from the reset halt (this -# bit the first version of this script: it reported pc=z_arm_reset at a halt -# 12s into the run). Reconnecting is the reliable resync: gdb re-queries the -# stop reason and gets the true state. (`monitor gdb sync` + `stepi` is the -# usual recipe but is not safe here -- when the halt lands mid-ISR that stepi -# can resume the target, after which every later read fails with "Cannot -# execute this command while the target is running".) Convenience variables -# survive the reconnect, so the A/B comparison below is unaffected. -detach -target extended-remote localhost:3333 -maintenance flush register-cache - -echo \n================ HALT B ================\n -hw_snapshot -set $B_cycles = (unsigned long long)cycle_count -set $B_tick = (long long)curr_tick -set $B_cvr = *(unsigned long*)0xE000E018 -set $B_current = (unsigned long)_kernel.cpus[0].current -thread_walk -timeout_walk -com_state -set $B_sum = $walk_sum - -echo \n================ VERDICT ================\n -printf "cycle_count A=%llu B=%llu delta=%lld\n", \ - $A_cycles, $B_cycles, (long long)($B_cycles - $A_cycles) -printf "curr_tick A=%lld B=%lld delta=%lld\n", \ - $A_tick, $B_tick, ($B_tick - $A_tick) -printf "SYST_CVR A=%#lx B=%#lx (free-running counter; equal is suspicious\n", \ - $A_cvr, $B_cvr -printf " but not conclusive -- it wraps every RVR ticks)\n" -if $B_cycles == $A_cycles && $B_tick == $A_tick - echo *** CLOCK IS FROZEN: no ticks announced across 4s -- SysTick ISR is not\n - echo *** running. Look at PRIMASK/BASEPRI/ICSR above and at whoever masked.\n -else - echo *** CLOCK IS ALIVE: ticks still advancing, so the hang is a scheduler /\n - echo *** blocked-thread problem, not a dead timer. The thread that should be\n - echo *** running is blocked -- see its pended_on + backtrace above.\n -end -printf "current thread A=%#lx B=%#lx -> %s\n", $A_current, $B_current, \ - ($A_current == $B_current ? "SAME (no context switch in 4s)" : "changed") -printf "thread CPU cyc A=%llu B=%llu delta=%llu -> %s\n", \ - (unsigned long long)$A_sum, (unsigned long long)$B_sum, \ - (unsigned long long)($B_sum - $A_sum), \ - ($A_sum == $B_sum ? \ - "*** NO thread got any CPU time in 4s: the system really is wedged" : \ - "threads are still being scheduled: the system is RUNNING") - -echo \n================ done (detaching; board left halted) ================\n -detach diff --git a/scripts/diag/hang_thread_walk.sh b/scripts/diag/hang_thread_walk.sh deleted file mode 100755 index c3ba2489..00000000 --- a/scripts/diag/hang_thread_walk.sh +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env bash -# hang_thread_walk.sh -- v5 of the /keys boot-hang forensics (see INVESTIGATION.md). -# -# v4 (hang_fault_bp.*) proved the hang is NOT a CPU fault: the fault-entry -# breakpoints never fire and a clean halt shows cm0 idle with CFSR=HFSR=0. So -# there is no faulting frame to catch -- the scheduler simply stops making -# progress. This probe takes the other approach INVESTIGATION.md calls for: -# reset, free-run into the hang, then halt TWICE a few seconds apart and -# * compare cycle_count / curr_tick / SysTick to see whether the clock is -# frozen or still ticking, and -# * walk _kernel.threads, recovering each swapped-out thread's resume PC/LR -# from its saved PendSV frame and unwinding it, to see exactly who is -# blocked and on what. -# Read logic lives in hang_thread_walk.gdb. -# -# Local bench only (not CI). Drives the board over a Raspberry Pi Debug Probe -# (CMSIS-DAP SWD). The two USB CDC ttys involved: -# BOARD_TTY /dev/tty.usbmodem1101 - the target's own USB CDC (F'/GDS link); -# telemetry goes silent here at the hang -# PROBE_TTY /dev/tty.usbmodem102 - the Debug Probe's UART bridge -# OpenOCD reaches SWD via the probe's CMSIS-DAP USB interface (not a tty); the -# ttys are used only to correlate/timestamp the hang. -# -# Read-mostly: it resets, runs, halts and reads state. The only target writes -# are core-register rewinds used to unwind each blocked thread's stack (restored -# right after, on a target that was reset at the start of the run and is left -# halted at the end). Never writes flash or config. -set -u - -# Free-run windows, in ms. A = reset -> first halt (long enough to reach the -# failed state), B = gap between the two halts (long enough to span the 30s -# telemetry cadence, so a thread that simply had nothing to do in a short window -# is not mistaken for a wedged one). -WINDOW_A_MS=${WINDOW_A_MS:-12000} -WINDOW_B_MS=${WINDOW_B_MS:-20000} - -BOARD_TTY=${BOARD_TTY:-/dev/tty.usbmodem1101} -PROBE_TTY=${PROBE_TTY:-/dev/tty.usbmodem102} - -# OpenOCD: the raspberrypi fork (do NOT substitute a nix/brew openocd -- the RP2350 -# support and the CMSIS-DAP build the bench relies on live in this fork). Local -# bench keeps it under ~/code/...; the CI runner uses ~/openocd. Override with -# OOCD_HOME. -OOCD_HOME=${OOCD_HOME:-} -if [ -z "$OOCD_HOME" ]; then - for cand in ~/code/github.com/raspberrypi/openocd ~/openocd; do - [ -x "$cand/src/openocd" ] && { OOCD_HOME="$cand"; break; } - done -fi -[ -x "$OOCD_HOME/src/openocd" ] || { echo "hang_thread_walk: openocd not found (set OOCD_HOME to the raspberrypi openocd checkout)"; exit 1; } -OCD="$OOCD_HOME/src/openocd" -OCD_ARGS=(-s "$OOCD_HOME/tcl" - -f "$OOCD_HOME/tcl/interface/cmsis-dap.cfg" - -f "$OOCD_HOME/tcl/target/rp2350.cfg" - -c "adapter speed 5000") - -HERE=$(cd "$(dirname "$0")" && pwd) -GDBCMDS="$HERE/hang_thread_walk.gdb" -[ -f "$GDBCMDS" ] || { echo "hang_thread_walk: missing $GDBCMDS"; exit 1; } - -# --- symbol ELF (local flight build; load addrs 0x1018_xxxx) --- -ELF="" -for cand in \ - build-fprime-automatic-zephyr/zephyr/zephyr.elf \ - build-artifacts/zephyr/fprime-zephyr-deployment; do - [ -f "$cand" ] && { ELF="$cand"; break; } -done -[ -z "$ELF" ] && { echo "hang_thread_walk: symbol ELF not found; run 'make generate build' first."; exit 1; } -echo "hang_thread_walk: ELF $ELF" -echo "hang_thread_walk: board $BOARD_TTY probe $PROBE_TTY" -echo "hang_thread_walk: windows A=${WINDOW_A_MS}ms B=${WINDOW_B_MS}ms" - -# --- ARM gdb --- -GDB="" -for cand in arm-zephyr-eabi-gdb gdb-multiarch arm-none-eabi-gdb; do - command -v "$cand" >/dev/null 2>&1 && { GDB="$cand"; break; } -done -[ -z "$GDB" ] && GDB=$(ls ~/zephyr-sdk*/arm-zephyr-eabi/bin/arm-zephyr-eabi-gdb 2>/dev/null | tail -1) -[ -z "$GDB" ] && { echo "hang_thread_walk: no arm/multiarch gdb found."; exit 1; } -echo "hang_thread_walk: gdb $GDB" - -# --- optional: capture the board CDC so telemetry-going-silent is timestamped --- -SERIAL_LOG=${SERIAL_LOG:-board-serial.log} -SNIFF_PID="" -if [ -e "$BOARD_TTY" ]; then - ( cat "$BOARD_TTY" > "$SERIAL_LOG" 2>/dev/null ) & - SNIFF_PID=$! - echo "hang_thread_walk: sniffing $BOARD_TTY -> $SERIAL_LOG" -else - echo "hang_thread_walk: note: $BOARD_TTY not present; skipping serial sniff" -fi - -# --- OpenOCD: init + keep the gdb server up; the .gdb file drives reset/run/halt --- -"$OCD" "${OCD_ARGS[@]}" -c "init" -c "echo {hang_thread_walk: gdb server on :3333}" & -OCD_PID=$! -cleanup() { [ -n "$SNIFF_PID" ] && kill "$SNIFF_PID" 2>/dev/null; kill "$OCD_PID" 2>/dev/null; } -trap cleanup EXIT -sleep 2 - -# All run/halt sequencing is done with `monitor` inside the command file, so gdb -# never thinks the target is running. That means gdb would happily serve stale -# cached data across a resume, so disable both memory caches here; the command -# file flushes the register cache after each halt. -GDB_ARGS=(-q -nx -batch "$ELF" - -ex "set pagination off" - -ex "set confirm off" - -ex "set print pretty on" - -ex "set backtrace past-main on" - -ex "set stack-cache off" - -ex "set code-cache off" - -ex "set \$WINDOW_A_MS = $WINDOW_A_MS" - -ex "set \$WINDOW_B_MS = $WINDOW_B_MS" - -ex "target extended-remote localhost:3333" - -x "$GDBCMDS") - -TIMEOUT_BIN=$(command -v timeout || command -v gtimeout || true) -if [ -n "$TIMEOUT_BIN" ]; then - "$TIMEOUT_BIN" 300 "$GDB" "${GDB_ARGS[@]}" \ - || echo "hang_thread_walk: gdb exited non-zero / timed out" -else - "$GDB" "${GDB_ARGS[@]}" \ - || echo "hang_thread_walk: gdb exited non-zero (see output above)" -fi - -echo "hang_thread_walk: done (board serial capture in $SERIAL_LOG," -echo " openocd log in hang-thread-walk-openocd.log)" From 7c13d5d50f219e6f7000084633e693f13c027323 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Tue, 28 Jul 2026 17:52:23 +0200 Subject: [PATCH 23/29] fix(TcSecurityDeframer): harden the on-flash key store Addresses the review callouts on the key-store rotation path. The store file is shared by all three deframer instances (UART/LoRa/Sband), but the code treated it as per-instance state. - Share the key store mutex across instances. It was a per-instance member, so a PROVISION_KEY/ADD_KEY on one instance's command thread could rewrite the file while another instance's dataIn_handler read it on the com thread, at worst yielding a half-old/half-new record whose valid byte is set but whose key bytes are mixed. Replaced with a function-local static taken by every access site. - Make writeKeyStore() durable: write to a temp file, flush, rename over the target, mirroring StartupManager::persist_boot_count. Overwriting in place meant a reset mid-write (e.g. the watchdog power cycle used for command-loss recovery) could truncate the store, which reads back as BAD_SIZE and boots the board keyless with no authenticated way in. - Reload the store before the read-modify-write in PROVISION_KEY, ADD_KEY and REMOVE_KEY. An instance holding a copy that predates a rotation issued over another link would otherwise write it back, resurrecting a revoked key on flash and dropping the current one. This also makes the NotEmpty/StoreFull/LastKey/SpiNotFound rejections truthful against the store that is actually on flash. - Report PSA import failures instead of swallowing them. importKeyStore() now returns whether every valid slot imported; on failure the handlers emit ImportError and EXECUTION_ERROR rather than KeyAdded and OK, so the operator no longer believes a key is live that the board cannot use. The store stays on flash so a reboot or later reload can retry. - Reject ADD_KEY for an SPI that already has a slot, via a new DuplicateSpi status. Two slots with one SPI are unusable: findKeyIdForSpi only returns the first match, so the new key would authenticate nothing while REMOVE_KEY would clear only one of them. - Zeroize the parsed key bytes in PROVISION_KEY and ADD_KEY, matching what importHmacKey already does, covering the early-return paths. DuplicateSpi is appended to KeyStoreProvisionStatus, so existing serialized enum values are unchanged. --- .../TcSecurityDeframer/TcSecurityDeframer.cpp | 147 ++++++++++++++++-- .../TcSecurityDeframer/TcSecurityDeframer.fpp | 1 + .../TcSecurityDeframer/TcSecurityDeframer.hpp | 28 ++-- 3 files changed, 151 insertions(+), 25 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp index f6d7d1fc..5fb4b0f2 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp @@ -5,8 +5,11 @@ #include "PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp" +#include + #include #include +#include #include #include "Authenticator.hpp" @@ -15,6 +18,26 @@ namespace Components { +namespace { + +//! Mutex protecting the key store, shared by every TcSecurityDeframer instance. +//! +//! All three instances (UART/LoRa/Sband) back onto a single key store file, so this cannot be a +//! per-instance member: without a shared lock, a PROVISION_KEY/ADD_KEY on one instance's command +//! thread can rewrite the file while another instance's dataIn_handler is reading it on the com +//! thread, which at worst yields a half-old/half-new fixed-layout record whose valid byte is set +//! but whose key bytes are mixed. It also guards each instance's in-memory m_keyStore/m_keyIds, +//! which is per-instance state; one lock covering both is sufficient and keeps the lock order +//! simple, since contention here is limited to key rotation and SPI misses. +//! +//! Function-local static so initialization order relative to component construction is defined. +Os::Mutex& keyStoreLock() { + static Os::Mutex lock; + return lock; +} + +} // namespace + // ---------------------------------------------------------------------- // Component construction and destruction // ---------------------------------------------------------------------- @@ -56,7 +79,7 @@ void TcSecurityDeframer ::dataIn_handler(FwIndexType portNum, Fw::Buffer& data, { // Lock order: key store lock, then sequence number lock. Every other handler that takes // both locks (none currently do) must follow the same order to avoid deadlock. - Os::ScopeLock keyLock(this->m_keyStoreLock); + Os::ScopeLock keyLock(keyStoreLock()); Os::ScopeLock seqLock(this->m_sequenceNumberLock); // --- Validate SPI and anti-replay sequence number --- @@ -172,7 +195,12 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U16 spi, const Fw::CmdStringArg& key) { - Os::ScopeLock lock(this->m_keyStoreLock); + Os::ScopeLock lock(keyStoreLock()); + + // Refresh from disk before inspecting the store: this instance's in-memory copy may predate a + // rotation issued over another link, and the trust-on-first-use check below is only meaningful + // against the store that is actually on flash. + (void)this->loadKeyStore(); // PROVISION_KEY is trust-on-first-use bootstrap: only honored while the store is empty. // Once any key exists, rotation must go through ADD_KEY/REMOVE_KEY (which require auth). @@ -192,6 +220,8 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, this->m_keyStore[0].set_valid(true); this->m_keyStore[0].set_spi(spi); this->m_keyStore[0].set_key(keyBytes); + // The key now lives in m_keyStore; drop the plaintext copy from this stack frame. + mbedtls_platform_zeroize(keyBytes, sizeof keyBytes); if (this->writeKeyStore() != Os::File::OP_OK) { this->m_keyStore[0].set_valid(false); @@ -200,14 +230,28 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, return; } - this->importKeyStore(); + // The store is durable at this point. A PSA import failure is reported but not rolled back: + // the key stays on flash so a reboot or a later reload can retry the import, and discarding a + // key the operator may not be able to re-send would be the worse outcome. + const bool imported = this->importKeyStore(); this->tlmWrite_ActiveKeyCount(this->activeKeyCount()); + if (!imported) { + this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::ImportError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + this->log_ACTIVITY_HI_KeyProvisioned(spi); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); } void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U16 spi, const Fw::CmdStringArg& key) { - Os::ScopeLock lock(this->m_keyStoreLock); + Os::ScopeLock lock(keyStoreLock()); + + // Refresh from disk before the read-modify-write below. Without this, an instance whose + // in-memory store predates a rotation issued over another link would write its stale copy + // back, resurrecting a revoked key on flash and dropping the current one. + (void)this->loadKeyStore(); const U8 count = this->activeKeyCount(); if (count >= AuthKeyStore::SIZE) { @@ -216,6 +260,14 @@ void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U1 return; } + // Two slots with the same SPI are unusable: findKeyIdForSpi only ever returns the first match, + // so the new key would authenticate nothing while REMOVE_KEY would clear only one of the two. + if (this->hasSpi(spi)) { + this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::DuplicateSpi); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; if (!parseHexKey(key.toChar(), keyBytes)) { this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::ParseKeyError); @@ -236,6 +288,8 @@ void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U1 this->m_keyStore[emptySlot].set_valid(true); this->m_keyStore[emptySlot].set_spi(spi); this->m_keyStore[emptySlot].set_key(keyBytes); + // The key now lives in m_keyStore; drop the plaintext copy from this stack frame. + mbedtls_platform_zeroize(keyBytes, sizeof keyBytes); if (this->writeKeyStore() != Os::File::OP_OK) { this->m_keyStore[emptySlot].set_valid(false); @@ -244,14 +298,26 @@ void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U1 return; } - this->importKeyStore(); + // See PROVISION_KEY: a durable store with a failed import is reported, not rolled back. + const bool imported = this->importKeyStore(); this->tlmWrite_ActiveKeyCount(this->activeKeyCount()); + if (!imported) { + this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::ImportError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + this->log_ACTIVITY_HI_KeyAdded(spi); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); } void TcSecurityDeframer ::REMOVE_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U16 spi) { - Os::ScopeLock lock(this->m_keyStoreLock); + Os::ScopeLock lock(keyStoreLock()); + + // Refresh from disk before the read-modify-write below, for the same reason as ADD_KEY. This + // also keeps the LastKey and SpiNotFound rejections below truthful against the current store + // rather than a stale copy. + (void)this->loadKeyStore(); if (this->activeKeyCount() <= 1) { this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::LastKey); @@ -282,8 +348,16 @@ void TcSecurityDeframer ::REMOVE_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, return; } - this->importKeyStore(); + // See PROVISION_KEY: a durable store with a failed import is reported, not rolled back. On + // removal the failure can only concern the surviving slot(s), which are re-imported here. + const bool imported = this->importKeyStore(); this->tlmWrite_ActiveKeyCount(this->activeKeyCount()); + if (!imported) { + this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::ImportError); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } + this->log_ACTIVITY_HI_KeyRemoved(spi); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); } @@ -318,7 +392,7 @@ void TcSecurityDeframer ::configure() { } { - Os::ScopeLock lock(this->m_keyStoreLock); + Os::ScopeLock lock(keyStoreLock()); // Get the key store file path from the parameter this->m_keyStoreFilePath = this->paramGet_KEY_STORE_FILE_PATH(is_valid); @@ -383,12 +457,41 @@ Os::File::Status TcSecurityDeframer ::loadKeyStore() { this->log_WARNING_HI_KeyStoreReadFailed(static_cast(status)); } - this->importKeyStore(); + // Import failures are surfaced to the operator by the command handlers, which call + // importKeyStore() directly; a reload has no command context to report into. + (void)this->importKeyStore(); return status; } Os::File::Status TcSecurityDeframer ::writeKeyStore() { - Os::File::Status status = Utilities::FileHelper::writeToFile(this->m_keyStoreFilePath.toChar(), this->m_keyStore); + // Write to a temp file, flush, then rename over the target - the same pattern as + // StartupManager::persist_boot_count, and for a more serious failure mode. Overwriting the + // live store in place meant a reset mid-write (e.g. the watchdog power cycle used for + // command-loss recovery) could leave a truncated file, which reads back as BAD_SIZE and boots + // the board keyless with no authenticated way back in. The rename is not guaranteed power-cut + // atomic on the flight FS, but the new store is fully written and flushed before it replaces + // the old one, so the worst case shrinks to a missing file during the rename window. + Fw::String tempPath(this->m_keyStoreFilePath); + tempPath += ".tmp"; + + Os::File file; + Os::File::Status status = + file.open(tempPath.toChar(), Os::File::Mode::OPEN_CREATE, Os::File::OverwriteType::OVERWRITE); + if (status == Os::File::OP_OK) { + status = Utilities::FileHelper::writeToFile(file, this->m_keyStore); + if (status == Os::File::OP_OK) { + // close() returns void and cannot report a flush failure, so flush explicitly before + // the rename makes the new store authoritative. + status = file.flush(); + } + (void)file.close(); + } + + if (status == Os::File::OP_OK && + Os::FileSystem::rename(tempPath.toChar(), this->m_keyStoreFilePath.toChar()) != Os::FileSystem::OP_OK) { + status = Os::File::OTHER_ERROR; + } + if (status != Os::File::OP_OK) { this->log_WARNING_HI_KeyStoreWriteFailed(static_cast(status)); } else { @@ -398,7 +501,7 @@ Os::File::Status TcSecurityDeframer ::writeKeyStore() { return status; } -void TcSecurityDeframer ::importKeyStore() { +bool TcSecurityDeframer ::importKeyStore() { // Release any previously-imported keys before re-importing, so rotation (and reloads that // pick up another link's rotation) never leaves a stale key importable in PSA. for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { @@ -408,6 +511,7 @@ void TcSecurityDeframer ::importKeyStore() { } } + bool allImported = true; for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { if (!this->m_keyStore[i].get_valid()) { continue; @@ -417,11 +521,17 @@ void TcSecurityDeframer ::importKeyStore() { const PacketAuthenticator::KeyImportResult result = importHmacKeyBytes(this->m_keyStore[i].get_key(), keyId); if (result.status == PacketAuthenticator::KeyImportStatus::Success) { this->m_keyIds[i] = keyId; + } else { + // The slot stays without a usable PSA key id, so findKeyIdForSpi will miss it and every + // frame for that SPI fails authentication. The store on disk is unaffected, so a + // subsequent reload/rotation can recover once the underlying PSA issue clears - but the + // caller must not report success, or the operator would believe a key is live that the + // board cannot actually use. + allImported = false; } - // On import failure the slot stays without a usable PSA key id; the store on disk is - // unaffected, so a subsequent reload/rotation can recover once the underlying PSA issue - // clears. } + + return allImported; } bool TcSecurityDeframer ::findKeyIdForSpi(uint32_t spi, uint32_t& keyId) const { @@ -434,6 +544,15 @@ bool TcSecurityDeframer ::findKeyIdForSpi(uint32_t spi, uint32_t& keyId) const { return false; } +bool TcSecurityDeframer ::hasSpi(uint16_t spi) const { + for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { + if (this->m_keyStore[i].get_valid() && this->m_keyStore[i].get_spi() == spi) { + return true; + } + } + return false; +} + U8 TcSecurityDeframer ::activeKeyCount() const { U8 count = 0; for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp index 828f9717..20e0e109 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp @@ -34,6 +34,7 @@ module Components { ParseKeyError, @< The supplied key was not a valid 32-character hex string WriteError, @< The updated store could not be written to the file system ImportError, @< The updated key could not be imported into PSA + DuplicateSpi, @< ADD_KEY was rejected because a slot already holds the given SPI } @ Component placed between the TcDeframer and SpacePacketDeframer components. It diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp index 5a149226..eb61417d 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp @@ -119,26 +119,31 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { ); //! Loads the key store from the file system into m_keyStore and (re)imports every valid slot - //! into PSA, destroying any previously-imported keys first. Must be called with m_keyStoreLock held. - //! On a missing/unreadable file, m_keyStore is left with no valid slots (keyless state). + //! into PSA, destroying any previously-imported keys first. Must be called with the key store + //! lock held. On a missing/unreadable file, m_keyStore is left with no valid slots (keyless state). Os::File::Status loadKeyStore(); - //! Writes m_keyStore to the file system. Must be called with m_keyStoreLock held. + //! Writes m_keyStore to the file system. Must be called with the key store lock held. Os::File::Status writeKeyStore(); //! (Re)imports every valid slot in m_keyStore into PSA, destroying any previously-imported - //! keys first, and updates m_keyIds. Must be called with m_keyStoreLock held. - void importKeyStore(); + //! keys first, and updates m_keyIds. Returns false if any valid slot failed to import, leaving + //! that slot's m_keyIds entry at 0. Must be called with the key store lock held. + bool importKeyStore(); //! Finds the PSA key id for the given SPI among currently-imported keys. - //! Must be called with m_keyStoreLock held. + //! Must be called with the key store lock held. bool findKeyIdForSpi(uint32_t spi, uint32_t& keyId) const; - //! Returns the number of valid slots in m_keyStore. Must be called with m_keyStoreLock held. + //! Returns true if any valid slot already holds the given SPI. + //! Must be called with the key store lock held. + bool hasSpi(uint16_t spi) const; + + //! Returns the number of valid slots in m_keyStore. Must be called with the key store lock held. U8 activeKeyCount() const; //! Projects m_keyStore's valid/spi fields into the plain-C++ ActiveSpiSlots type consumed by - //! the pure-C++ Validator. Must be called with m_keyStoreLock held. + //! the pure-C++ Validator. Must be called with the key store lock held. ActiveSpiSlots activeSpiSlots() const; private: @@ -154,9 +159,10 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { U32 m_sequenceNumberWindow; //!< The allowed window for sequence number validation // Key store state is coupled between in-memory runtime state (m_keyStore/m_keyIds) and on-disk - // persistent storage; protected by the same mutex to keep PSA-imported keys consistent with the - // on-disk store shared across all TcSecurityDeframer instances (UART/LoRa/Sband) - Os::Mutex m_keyStoreLock; //!< Mutex protecting key store state atomicity + // persistent storage; both are protected by keyStoreLock() in the .cpp. That mutex is + // deliberately NOT a member: the store file is shared across all TcSecurityDeframer instances + // (UART/LoRa/Sband), so a per-instance lock would let one instance rewrite the file while + // another is reading it. See keyStoreLock() for the full rationale. Fw::String m_keyStoreFilePath; //!< File path where the key store is stored AuthKeyStore m_keyStore; //!< The active key store, up to 2 slots uint32_t m_keyIds[AuthKeyStore::SIZE]; //!< PSA key ids parallel to m_keyStore, valid iff the slot is valid From b214dd88c26f338fda10307903bdc21cd67f89cd Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Wed, 29 Jul 2026 01:33:11 +0200 Subject: [PATCH 24/29] fix: address PR #472 review on the key store, ground key, and int tests Flight side (TcSecurityDeframer): - Gate the unknown-SPI store reload on a process-wide generation counter. SPI validation runs before MAC verification, so an unauthenticated frame carrying a bogus SPI reached the reload path: a littlefs read plus a full PSA destroy/re-import of every slot, on the com thread, holding the lock shared by all three uplinks. A stream of such frames serialized UART, LoRa and Sband behind that work with no credential needed. Every writer is in this process and bumps the counter under the same lock, so the reload stays exact while costing nothing in the common case. - Refuse PROVISION_KEY/ADD_KEY/REMOVE_KEY on a store that could not be read back (new StoreUnreadable status). loadKeyStore() only resets m_keyStore on OP_OK and DOESNT_EXIST; on any other status a freshly booted instance kept the default all-invalid value, so activeKeyCount() returned 0 and the trust-on-first-use check in PROVISION_KEY passed. Since PROVISION_KEY is bypass-allowlisted, any induced read failure - a truncated record reading back as BAD_SIZE, a littlefs error, a mount not ready at configure() time - let an unauthenticated party install their own key while a valid one was still on flash. TOFU now requires proof the store is empty. The same guard keeps ADD_KEY/REMOVE_KEY from writing a stale guess over the real store. - Zeroize revoked key bytes in REMOVE_KEY before persisting. Clearing `valid` alone left the raw key in the fixed-layout record, recoverable from /keys/authkeys.bin; the slot is restored if the durable write fails. The write-failure rollbacks in PROVISION_KEY/ADD_KEY zeroize too, so an invalid slot never carries key bytes into a later successful write. - Zeroize keyBytes on the parse-failure returns. parseHexKey fills the buffer as it scans, so a key rejected part-way through left a prefix of real key bytes on the stack. - destroyHmacKey() returns the PSA status instead of discarding it, and importKeyStore() reports a failed release. A destroy that fails leaves the old key live in PSA while the slot was being treated as recycled - on REMOVE_KEY that told the operator a key was revoked when it could still authenticate frames. Ground: - authenticate_plugin.py validates the key at startup, from both --authentication-key and PROVES_AUTH_KEY: exactly 32 hex characters after normalizing an optional 0x prefix. Malformed values previously blew up in bytes.fromhex() mid-run or produced frames the board silently rejected. - README documents the ground-side key transition during rotation, since ground uses one key at a time while the board holds two. Integration tests: - provision_key_test retries PROVISION_KEY on the same Fibonacci backoff the rest of the suite uses. Every authenticated test depends on this one, so a single dropped uplink failed the whole provisioning gate. - exit_safe_mode() asserts the command completes instead of swallowing every failure. If it never lands, deployment stays inhibited and the faces stay unpowered, and the caller fails for a reason unrelated to what it tested. - rtc_test asserts the CreateDirectory outcome event arrived, instead of falling through into the uplink race the wait exists to close. --- Framing/src/authenticate_plugin.py | 61 ++++++++-- .../TcSecurityDeframer/Authenticator.cpp | 7 +- .../TcSecurityDeframer/Authenticator.hpp | 9 +- .../TcSecurityDeframer/TcSecurityDeframer.cpp | 110 ++++++++++++++++-- .../TcSecurityDeframer/TcSecurityDeframer.fpp | 1 + .../TcSecurityDeframer/TcSecurityDeframer.hpp | 4 + .../Components/TcSecurityDeframer/docs/sdd.md | 16 ++- .../test/int/common.py | 16 +-- .../test/int/provision_key_test.py | 29 +++-- .../test/int/rtc_test.py | 8 +- README.md | 8 ++ docs-site/components/TcSecurityDeframer.md | 16 ++- 12 files changed, 234 insertions(+), 51 deletions(-) diff --git a/Framing/src/authenticate_plugin.py b/Framing/src/authenticate_plugin.py index ef126a88..a09c2cbb 100644 --- a/Framing/src/authenticate_plugin.py +++ b/Framing/src/authenticate_plugin.py @@ -19,6 +19,44 @@ SEQUENCE_NUMBER_FILE = os.path.join(_SEQUENCE_NUMBER_DIR, _SEQUENCE_NUMBER_FILENAME) +# The flight side (Authenticator.cpp parseHexKey) accepts exactly a 128-bit key as 32 hex +# characters. Anything else either blows up later in bytes.fromhex() during framing or produces +# frames the board silently rejects, so both key sources are normalized and checked here instead. +AUTH_KEY_HEX_LENGTH = 32 + + +def normalize_auth_key(key: str, source: str) -> str: + """Strip any 0x prefix and validate the key is exactly 128 bits of hex. + + Args: + key: The key as supplied by the operator + source: Where it came from, for the error message + + Returns: + The key as 32 hex characters, without 0x prefix + + Raises: + ValueError: If the key is not exactly 32 hexadecimal characters + """ + normalized = key.strip() + if normalized[:2].lower() == "0x": + normalized = normalized[2:] + + if len(normalized) != AUTH_KEY_HEX_LENGTH: + raise ValueError( + f"Authentication key from {source} is {len(normalized)} hex characters; " + f"expected exactly {AUTH_KEY_HEX_LENGTH} (a 128-bit key)." + ) + try: + bytes.fromhex(normalized) + except ValueError as exc: + raise ValueError( + f"Authentication key from {source} is not valid hexadecimal: {exc}" + ) from exc + + return normalized + + def get_auth_key_from_env() -> str: """ Read the authentication key from the PROVES_AUTH_KEY environment variable. @@ -31,7 +69,7 @@ def get_auth_key_from_env() -> str: Authentication key as a hex string (without 0x prefix) from PROVES_AUTH_KEY Raises: - ValueError: If PROVES_AUTH_KEY is not set + ValueError: If PROVES_AUTH_KEY is unset or not a 128-bit hex key """ key = os.environ.get("PROVES_AUTH_KEY") if not key: @@ -41,9 +79,7 @@ def get_auth_key_from_env() -> str: "onto the satellite with the PROVISION_KEY command and is never " "compiled into the flight image." ) - if key.startswith("0x") or key.startswith("0X"): - key = key[2:] - return key + return normalize_auth_key(key, "the PROVES_AUTH_KEY environment variable") # pragma: no cover @@ -78,9 +114,15 @@ def __init__( self.spi = spi self.window_size = window_size self.authentication_type = authentication_type - # Use provided key or read from the PROVES_AUTH_KEY environment variable + # Use provided key or read from the PROVES_AUTH_KEY environment variable. Either way the + # key is validated up front, so a malformed key fails at startup with a clear message + # rather than mid-run inside frame(). if authentication_key is None: authentication_key = get_auth_key_from_env() + else: + authentication_key = normalize_auth_key( + authentication_key, "--authentication-key" + ) self.authentication_key = authentication_key def get_sequence_number_from_file(self, filename: str, addition: bool) -> int: @@ -130,12 +172,9 @@ def frame(self, data: bytes) -> bytes: # Security Trailer of 16 octets in length (TM Baseline) # the output MAC is 2*128 bits in total length. (32 bytes) - # Convert hex string to bytes (16 bytes) - # Keys are stored without 0x prefix, but handle it if present for backward compatibility - key_hex = self.authentication_key - if key_hex.startswith("0x") or key_hex.startswith("0X"): - key_hex = key_hex[2:] - key = bytes.fromhex(key_hex) + # Convert hex string to bytes (16 bytes). The key was normalized and validated in + # __init__, so this cannot fail here. + key = bytes.fromhex(self.authentication_key) hmac_object = hmac.new(key, data, hashlib.sha256) diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp index d88a6364..3b9b6a6a 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp @@ -11,6 +11,9 @@ #include namespace Components { + +static_assert(PacketAuthenticator::kPsaSuccess == PSA_SUCCESS, "kPsaSuccess must mirror PSA_SUCCESS"); + namespace { constexpr size_t kKeyHexLength = @@ -103,8 +106,8 @@ PacketAuthenticator::KeyImportResult importHmacKey(const char* key, uint32_t& ke return result; } -void destroyHmacKey(uint32_t keyId) { - (void)psa_destroy_key(keyId); +int32_t destroyHmacKey(uint32_t keyId) { + return psa_destroy_key(keyId); } PacketAuthenticator::AuthenticationResult authenticatePacket(const uint8_t* dataBuffer, diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp index ec984aa0..a264e2a9 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp @@ -21,6 +21,11 @@ enum class KeyImportStatus { ImportKeyError, //!< There was an error importing the authentication key }; +//! Mirror of PSA_SUCCESS, so callers can check a psaStatus without including +//! (this header is part of the pure-C++ layer covered by the gtest unit tests). Authenticator.cpp +//! static_asserts that it still matches. +constexpr int32_t kPsaSuccess = 0; + struct KeyImportResult { KeyImportStatus status; //!< The status of the key import attempt int32_t psaStatus; //!< The status code returned by the PSA crypto functions @@ -59,7 +64,9 @@ PacketAuthenticator::KeyImportResult importHmacKey(const char* key, //!< The he ); //! Destroy a previously-imported PSA key. Used to release the old key on rotation. -void destroyHmacKey(uint32_t keyId //!< The PSA key ID to destroy +//! Returns the PSA status: a failed destroy leaves the old key usable in PSA, which the caller +//! must not silently treat as a released slot. +int32_t destroyHmacKey(uint32_t keyId //!< The PSA key ID to destroy ); //! Check the validity of the packet HMAC diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp index 5fb4b0f2..6d993371 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp @@ -10,6 +10,8 @@ #include #include #include +#include +#include #include #include "Authenticator.hpp" @@ -36,6 +38,18 @@ Os::Mutex& keyStoreLock() { return lock; } +//! Counter bumped by every successful writeKeyStore(), read under keyStoreLock(). +//! +//! An instance whose in-memory copy is at the current generation knows no one has rewritten the +//! store since it last read, so an unknown SPI really is unknown and needs no flash read. Without +//! this, any unauthenticated frame carrying a bogus SPI - SPI validation runs before MAC +//! verification - would drive a littlefs read plus a full PSA destroy/re-import of every slot, on +//! the com thread, holding the lock shared by all three uplinks. +U32& keyStoreGeneration() { + static U32 generation = 0; + return generation; +} + } // namespace // ---------------------------------------------------------------------- @@ -49,7 +63,10 @@ TcSecurityDeframer ::TcSecurityDeframer(const char* const compName) m_sequenceNumberWindow(0), m_keyStoreFilePath(), m_keyStore(), - m_keyIds{0} {} + m_keyIds{0}, + // Sentinel: no generation ever takes this value, so the first unknown-SPI check reloads even + // if configure() has not run. + m_keyStoreGeneration(std::numeric_limits::max()) {} TcSecurityDeframer ::~TcSecurityDeframer() {} @@ -86,11 +103,18 @@ void TcSecurityDeframer ::dataIn_handler(FwIndexType portNum, Fw::Buffer& data, PacketValidator::Status validationStatus = validatePacket(parseResult.securityHeader, this->m_sequenceNumber, this->m_sequenceNumberWindow, this->activeSpiSlots()); - if (validationStatus == PacketValidator::Status::SpiInvalid) { + if (validationStatus == PacketValidator::Status::SpiInvalid && + this->m_keyStoreGeneration != keyStoreGeneration()) { // The key store is shared across all TcSecurityDeframer instances (UART/LoRa/Sband). // A rotation issued over one link is picked up here so the others don't need their // own commands re-run: reload from disk once and retry before giving up. - this->loadKeyStore(); + // + // Gated on the generation counter because this path is reachable by an unauthenticated + // frame: SPI validation runs before MAC verification, so without the gate a stream of + // bogus-SPI frames would serialize all three uplinks behind a flash read and a full PSA + // re-import per frame. Every writer of the store is in this process and bumps the + // counter under the same lock, so the reload is exact and free in the common case. + (void)this->loadKeyStore(); validationStatus = validatePacket(parseResult.securityHeader, this->m_sequenceNumber, this->m_sequenceNumberWindow, this->activeSpiSlots()); } @@ -200,7 +224,19 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, // Refresh from disk before inspecting the store: this instance's in-memory copy may predate a // rotation issued over another link, and the trust-on-first-use check below is only meaningful // against the store that is actually on flash. - (void)this->loadKeyStore(); + const Os::File::Status loadStatus = this->loadKeyStore(); + + // Trust-on-first-use must be gated on proof that the store is empty, not merely on the absence + // of a key we managed to read. PROVISION_KEY is bypass-allowlisted so it works on a keyless + // board, so if an unreadable store (a truncated record reading back as BAD_SIZE, a littlefs + // error, a mount not ready yet) counted as "keyless", anyone in radio range could induce a read + // failure and install their own key while a valid one still sits on flash. DOESNT_EXIST is the + // one failure that *is* proof of emptiness. + if (loadStatus != Os::File::OP_OK && loadStatus != Os::File::DOESNT_EXIST) { + this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::StoreUnreadable); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } // PROVISION_KEY is trust-on-first-use bootstrap: only honored while the store is empty. // Once any key exists, rotation must go through ADD_KEY/REMOVE_KEY (which require auth). @@ -212,6 +248,9 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; if (!parseHexKey(key.toChar(), keyBytes)) { + // parseHexKey fills the buffer as it scans, so a key rejected part-way through leaves a + // prefix of real key bytes behind on this stack frame. + mbedtls_platform_zeroize(keyBytes, sizeof keyBytes); this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::ParseKeyError); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -225,6 +264,9 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, if (this->writeKeyStore() != Os::File::OP_OK) { this->m_keyStore[0].set_valid(false); + // Don't leave the rejected key's bytes in an invalid slot: a later successful write would + // persist them to flash. + mbedtls_platform_zeroize(this->m_keyStore[0].get_key(), sizeof(AuthKeySlot::Type_of_key)); this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::WriteError); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -251,7 +293,16 @@ void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U1 // Refresh from disk before the read-modify-write below. Without this, an instance whose // in-memory store predates a rotation issued over another link would write its stale copy // back, resurrecting a revoked key on flash and dropping the current one. - (void)this->loadKeyStore(); + const Os::File::Status loadStatus = this->loadKeyStore(); + + // If the store could not be read back, its contents are unknown, and the read-modify-write + // below would persist a guess: writing the last in-memory copy over whatever is actually on + // flash. Refuse rather than risk dropping a live key. + if (loadStatus != Os::File::OP_OK && loadStatus != Os::File::DOESNT_EXIST) { + this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::StoreUnreadable); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } const U8 count = this->activeKeyCount(); if (count >= AuthKeyStore::SIZE) { @@ -270,6 +321,8 @@ void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U1 uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; if (!parseHexKey(key.toChar(), keyBytes)) { + // See PROVISION_KEY: a partially-parsed key leaves real key bytes on the stack. + mbedtls_platform_zeroize(keyBytes, sizeof keyBytes); this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::ParseKeyError); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -293,6 +346,8 @@ void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U1 if (this->writeKeyStore() != Os::File::OP_OK) { this->m_keyStore[emptySlot].set_valid(false); + // See PROVISION_KEY: an invalid slot must not carry key bytes into a later write. + mbedtls_platform_zeroize(this->m_keyStore[emptySlot].get_key(), sizeof(AuthKeySlot::Type_of_key)); this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::WriteError); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -317,7 +372,14 @@ void TcSecurityDeframer ::REMOVE_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, // Refresh from disk before the read-modify-write below, for the same reason as ADD_KEY. This // also keeps the LastKey and SpiNotFound rejections below truthful against the current store // rather than a stale copy. - (void)this->loadKeyStore(); + const Os::File::Status loadStatus = this->loadKeyStore(); + + // See ADD_KEY: an unreadable store makes the read-modify-write a guess. + if (loadStatus != Os::File::OP_OK && loadStatus != Os::File::DOESNT_EXIST) { + this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::StoreUnreadable); + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); + return; + } if (this->activeKeyCount() <= 1) { this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::LastKey); @@ -339,10 +401,24 @@ void TcSecurityDeframer ::REMOVE_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, return; } + // Clearing `valid` alone would leave the revoked key's raw bytes in the fixed-layout record on + // flash, recoverable by anyone who can read the partition. Wipe the slot's key material too, + // keeping a copy only long enough to restore the slot if the durable write fails. + uint8_t revokedKey[Ccsds355_0_B_2::kTCSecurityTrailer]; + static_assert(sizeof(revokedKey) == sizeof(AuthKeySlot::Type_of_key), "key slot size mismatch"); + std::memcpy(revokedKey, this->m_keyStore[targetSlot].get_key(), sizeof revokedKey); + this->m_keyStore[targetSlot].set_valid(false); + mbedtls_platform_zeroize(this->m_keyStore[targetSlot].get_key(), sizeof(AuthKeySlot::Type_of_key)); - if (this->writeKeyStore() != Os::File::OP_OK) { + const Os::File::Status writeStatus = this->writeKeyStore(); + if (writeStatus != Os::File::OP_OK) { + this->m_keyStore[targetSlot].set_key(revokedKey); this->m_keyStore[targetSlot].set_valid(true); + } + mbedtls_platform_zeroize(revokedKey, sizeof revokedKey); + + if (writeStatus != Os::File::OP_OK) { this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::WriteError); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -457,6 +533,11 @@ Os::File::Status TcSecurityDeframer ::loadKeyStore() { this->log_WARNING_HI_KeyStoreReadFailed(static_cast(status)); } + // m_keyStore now reflects the store as of this generation, whether or not the read succeeded: + // on a failure a retry would read the same bytes, so re-reading per frame buys nothing. The + // next writer bumps the generation and forces a fresh attempt. + this->m_keyStoreGeneration = keyStoreGeneration(); + // Import failures are surfaced to the operator by the command handlers, which call // importKeyStore() directly; a reload has no command context to report into. (void)this->importKeyStore(); @@ -496,22 +577,33 @@ Os::File::Status TcSecurityDeframer ::writeKeyStore() { this->log_WARNING_HI_KeyStoreWriteFailed(static_cast(status)); } else { this->log_WARNING_HI_KeyStoreWriteFailed_ThrottleClear(); + // Tell the other instances their copy is stale, and record that ours is not: m_keyStore is + // exactly what was just written. + keyStoreGeneration()++; + this->m_keyStoreGeneration = keyStoreGeneration(); } return status; } bool TcSecurityDeframer ::importKeyStore() { + bool allImported = true; + // Release any previously-imported keys before re-importing, so rotation (and reloads that // pick up another link's rotation) never leaves a stale key importable in PSA. for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { if (this->m_keyIds[i] != 0) { - destroyHmacKey(this->m_keyIds[i]); + // A failed destroy leaves the old key live in PSA, so the slot is not actually + // recycled. Nothing more can be done here, but the caller must not report success: + // on REMOVE_KEY in particular that would tell the operator a key is revoked when it + // can still authenticate frames. + if (destroyHmacKey(this->m_keyIds[i]) != PacketAuthenticator::kPsaSuccess) { + allImported = false; + } this->m_keyIds[i] = 0; } } - bool allImported = true; for (U32 i = 0; i < AuthKeyStore::SIZE; i++) { if (!this->m_keyStore[i].get_valid()) { continue; diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp index 20e0e109..5d4fe39b 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.fpp @@ -35,6 +35,7 @@ module Components { WriteError, @< The updated store could not be written to the file system ImportError, @< The updated key could not be imported into PSA DuplicateSpi, @< ADD_KEY was rejected because a slot already holds the given SPI + StoreUnreadable, @< The store could not be read back, so its contents are unknown } @ Component placed between the TcDeframer and SpacePacketDeframer components. It diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp index eb61417d..40227f3b 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp @@ -166,6 +166,10 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { Fw::String m_keyStoreFilePath; //!< File path where the key store is stored AuthKeyStore m_keyStore; //!< The active key store, up to 2 slots uint32_t m_keyIds[AuthKeyStore::SIZE]; //!< PSA key ids parallel to m_keyStore, valid iff the slot is valid + //! Value of keyStoreGeneration() when m_keyStore was last read from flash. Lets an unknown-SPI + //! frame tell "another instance rotated the store" from "this SPI is simply not ours" without + //! re-reading flash. Initialized to a sentinel no generation takes, so the first check reloads. + U32 m_keyStoreGeneration; }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md index d61b0b9b..9e32424b 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md @@ -20,9 +20,13 @@ Component state is the last accepted sequence number and the on-flash key store The HMAC authentication key is **never compiled into the firmware image** (issue #220). It is persisted at `KEY_STORE_FILE_PATH` (default `/keys/authkeys.bin`) on a dedicated littlefs `keystore_partition` on internal flash, holding up to 2 slots (`{valid, spi, key}`). `configure()` loads the store and imports every valid slot into PSA; a missing/empty store is not an error — a keyless board still boots so it can be provisioned. The sequence-number file lives on the same partition (`SEQ_NUM_FILE_PATH`, default `/keys/sequence_number.bin`). -The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): on an unknown SPI, `dataIn_handler` reloads the store from disk once and retries validation before rejecting the frame, so a rotation issued over one link is picked up by the others without a separate command per link. +The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): on an unknown SPI, `dataIn_handler` reloads the store from disk once and retries validation before rejecting the frame, so a rotation issued over one link is picked up by the others without a separate command per link. That reload is gated on a process-wide generation counter bumped by every successful store write: SPI validation runs before MAC verification, so the path is reachable by unauthenticated frames, and without the gate a stream of bogus-SPI frames would drive a flash read plus a full PSA re-import per frame on the com thread while holding the lock shared by all three uplinks. -Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-import the store into PSA and update `ActiveKeyCount` telemetry on success. +Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-read the store from flash first, then re-import it into PSA and update `ActiveKeyCount` telemetry on success. + +All three also refuse to act (`StoreUnreadable`) when that re-read fails with anything other than "file does not exist", because the store's contents are then unknown. For `ADD_KEY`/`REMOVE_KEY` that prevents a read-modify-write from persisting a stale guess over the real store. For `PROVISION_KEY` it is a security property: the command is bypass-allowlisted so it works on a keyless board, so trust-on-first-use must be gated on *proof* that the store is empty. Treating an unreadable store as keyless would let anyone in radio range induce a read failure and install their own key while a valid one still sits on flash. + +`REMOVE_KEY` zeroizes the revoked slot's key bytes before persisting, so a removed key is not recoverable from the on-flash record; the slot is restored if the durable write fails. Primary data path connections: @@ -121,7 +125,7 @@ The MAC is HMAC-SHA-256 truncated to 16 bytes, computed over the Security Header ## Behavior 1. Parse the Security Header and Trailer. If the frame is too short to contain them it cannot be stripped for downstream deframing: log ParsingFailed and return the buffer upstream (drop). -2. Validate the SPI (must match a valid slot in the active key store — see [Key Storage](#key-storage)) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). On an unknown SPI, the key store is reloaded from disk once and validation retried, so a rotation issued over another link is picked up here. +2. Validate the SPI (must match a valid slot in the active key store — see [Key Storage](#key-storage)) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). On an unknown SPI, the key store is reloaded from disk once and validation retried — but only if another instance has written the store since this one last read it — so a rotation issued over another link is picked up here without letting unauthenticated frames drive repeated flash reads. 3. If validation passes, look up the PSA key id for the packet's SPI and verify the MAC with it. 4. Only when all checks pass: store and persist the received sequence number, telemeter it, and set `authenticated = true` in the frame context. Frames failing any check never advance the sequence number (issue #426). 5. Strip the Security Header and Trailer and forward on dataOut with the resulting `authenticated` flag. ProvesRouter rejects unauthenticated packets unless their opcode is on the bypass allowlist. @@ -171,11 +175,11 @@ Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which ow | KeyStoreReadFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store read fails (not thrown for a missing file — that's the keyless state). Format: "Failed to read key store, error: {}" | | KeyStoreWriteFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store write fails. Format: "Failed to write key store, error: {}" | | KeyProvisioned | Activity High | spi: U16 | Logged by PROVISION_KEY on success. Format: "Key provisioned for SPI={}" | -| KeyProvisionFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when PROVISION_KEY fails (store not empty, bad hex key, or write failure). Format: "Key provisioning failed: {}" | +| KeyProvisionFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when PROVISION_KEY fails (store not empty, unreadable store, bad hex key, write failure, or PSA import failure). Format: "Key provisioning failed: {}" | | KeyAdded | Activity High | spi: U16 | Logged by ADD_KEY on success. Format: "Key added for SPI={}" | -| KeyAddFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when ADD_KEY fails (store full, bad hex key, or write failure). Format: "Key add failed: {}" | +| KeyAddFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when ADD_KEY fails (store full, duplicate SPI, unreadable store, bad hex key, write failure, or PSA import failure). Format: "Key add failed: {}" | | KeyRemoved | Activity High | spi: U16 | Logged by REMOVE_KEY on success. Format: "Key removed for SPI={}" | -| KeyRemoveFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when REMOVE_KEY fails (last remaining key, SPI not found, or write failure). Format: "Key remove failed: {}" | +| KeyRemoveFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when REMOVE_KEY fails (last remaining key, SPI not found, unreadable store, write failure, or PSA import failure). Format: "Key remove failed: {}" | ## Commands diff --git a/PROVESFlightControllerReference/test/int/common.py b/PROVESFlightControllerReference/test/int/common.py index 429701a2..627c4274 100644 --- a/PROVESFlightControllerReference/test/int/common.py +++ b/PROVESFlightControllerReference/test/int/common.py @@ -50,14 +50,16 @@ def exit_safe_mode(fprime_test_api: IntegrationTestAPI) -> None: EXIT_SAFE_MODE first clears the condition long enough for a short test to run (auto-entry re-arms only after SafeModeDebounceSeconds). - Best-effort: EXIT_SAFE_MODE is a no-op when the board is already in NORMAL, - and a failure here should surface as the real test's failure, not as a - setup error. + EXIT_SAFE_MODE is a no-op that still responds OK when the board is already + in NORMAL, so the command completing is a meaningful precondition rather + than something to swallow: if it never lands, deployment stays inhibited and + the face load switches stay off, and the caller's real assertion fails for a + reason that has nothing to do with what it was testing. Asserting here (with + the usual retries) points at the actual cause. """ - try: - fprime_test_api.send_command("ReferenceDeployment.modeManager.EXIT_SAFE_MODE") - except Exception: # noqa: BLE001 - advisory only; the test itself is the assertion - pass + proves_send_and_assert_command( + fprime_test_api, "ReferenceDeployment.modeManager.EXIT_SAFE_MODE" + ) def set_radio_recover_fn(fn: Callable[[], None] | None) -> None: diff --git a/PROVESFlightControllerReference/test/int/provision_key_test.py b/PROVESFlightControllerReference/test/int/provision_key_test.py index 14bdf043..4aab80bd 100644 --- a/PROVESFlightControllerReference/test/int/provision_key_test.py +++ b/PROVESFlightControllerReference/test/int/provision_key_test.py @@ -11,8 +11,11 @@ """ import os +import random +import time import pytest +from common import FIB_BACKOFF from fprime_gds.common.data_types.event_data import EventData from fprime_gds.common.testing_fw.api import IntegrationTestAPI from fprime_gds.common.testing_fw.predicates import is_a_member_of @@ -50,17 +53,27 @@ def test_provision_key( fprime_test_api.translate_event_name(f"{deframer}.KeyProvisionFailed"), ] - fprime_test_api.clear_histories() - fprime_test_api.send_command(f"{deframer}.PROVISION_KEY", ["0", key]) + # Every authenticated test downstream depends on this one, so a single dropped uplink must not + # fail the whole provisioning gate. proves_send_and_assert_command can't be used here (the + # outcome is an either/or event pair, and a command response alone doesn't tell us which), so + # retry by hand on the same Fibonacci backoff it uses to absorb LoRa's half-duplex collisions. + evt: EventData | None = None + for attempt in range(len(FIB_BACKOFF)): + fprime_test_api.clear_histories() + fprime_test_api.send_command(f"{deframer}.PROVISION_KEY", ["0", key]) - evt: EventData = fprime_test_api.await_event( - is_a_member_of(outcome_ids), - timeout=10, - ) + evt = fprime_test_api.await_event( + is_a_member_of(outcome_ids), + timeout=10, + ) + if evt is not None: + break + + time.sleep(FIB_BACKOFF[attempt] * random.uniform(0.5, 1.5)) assert evt is not None, ( - f"No KeyProvisioned/KeyProvisionFailed event from {deframer} within 10s of " - "PROVISION_KEY; the command may not have reached the board" + f"No KeyProvisioned/KeyProvisionFailed event from {deframer} after " + f"{len(FIB_BACKOFF)} PROVISION_KEY attempts; the command may not be reaching the board" ) if evt.template.get_full_name().endswith("KeyProvisionFailed"): diff --git a/PROVESFlightControllerReference/test/int/rtc_test.py b/PROVESFlightControllerReference/test/int/rtc_test.py index bc78e604..7e6ce944 100644 --- a/PROVESFlightControllerReference/test/int/rtc_test.py +++ b/PROVESFlightControllerReference/test/int/rtc_test.py @@ -110,7 +110,7 @@ def uplink_sequence_and_await_completion( # with FileOpenError before the mkdir lands. Wait for the command to # resolve either way first - "already exists" comes back as # DirectoryCreateError, which is just as good for our purposes. - fprime_test_api.await_event( + create_evt = fprime_test_api.await_event( is_a_member_of( [ fprime_test_api.translate_event_name( @@ -123,6 +123,12 @@ def uplink_sequence_and_await_completion( ), timeout=timeout, ) + # Falling through on a timeout would put us back in the race this wait exists to close, + # and report it as a confusing FileOpenError/FileReceived timeout further downstream. + assert create_evt is not None, ( + "No CreateDirectorySucceeded/DirectoryCreateError from " + f"{fileManager} within {timeout}s of CreateDirectory" + ) fprime_test_api.uplink_file(temp_bin_path, destination) fprime_test_api.await_event("FileReceived", timeout=timeout) diff --git a/README.md b/README.md index cd946667..3497b447 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,14 @@ If you regenerate/replace the bootloader (or switch computers and flash a bootlo You also want to make sure the authentication key the gds runs with is the same as the authentication key provisioned on the board. The board's key lives in its on-flash key store (never in the image); ground reads its key from the `--authentication-key` CLI arg or the `PROVES_AUTH_KEY` env var. Make sure these match the key you provisioned with `PROVISION_KEY`/`ADD_KEY`. +The board holds up to two active keys so a rotation never leaves you locked out. Ground uses exactly one key at a time (whichever `--authentication-key`/`PROVES_AUTH_KEY` it was started with), so rotate in this order: + +1. Keep running GDS with the **old** key and send `ADD_KEY(new_spi, new_key)` — the command itself has to authenticate under the old key. +2. Restart GDS with the **new** key (and `--spi new_spi`), and confirm commands are accepted. +3. Only then send `REMOVE_KEY(old_spi)`, authenticated under the new key. + +Doing step 3 before step 2 works too, but leaves nothing to fall back on if the new key turns out to be wrong. `REMOVE_KEY` refuses to remove the last remaining key. + ## Running Integration Tests First, start GDS with: diff --git a/docs-site/components/TcSecurityDeframer.md b/docs-site/components/TcSecurityDeframer.md index d61b0b9b..9e32424b 100644 --- a/docs-site/components/TcSecurityDeframer.md +++ b/docs-site/components/TcSecurityDeframer.md @@ -20,9 +20,13 @@ Component state is the last accepted sequence number and the on-flash key store The HMAC authentication key is **never compiled into the firmware image** (issue #220). It is persisted at `KEY_STORE_FILE_PATH` (default `/keys/authkeys.bin`) on a dedicated littlefs `keystore_partition` on internal flash, holding up to 2 slots (`{valid, spi, key}`). `configure()` loads the store and imports every valid slot into PSA; a missing/empty store is not an error — a keyless board still boots so it can be provisioned. The sequence-number file lives on the same partition (`SEQ_NUM_FILE_PATH`, default `/keys/sequence_number.bin`). -The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): on an unknown SPI, `dataIn_handler` reloads the store from disk once and retries validation before rejecting the frame, so a rotation issued over one link is picked up by the others without a separate command per link. +The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): on an unknown SPI, `dataIn_handler` reloads the store from disk once and retries validation before rejecting the frame, so a rotation issued over one link is picked up by the others without a separate command per link. That reload is gated on a process-wide generation counter bumped by every successful store write: SPI validation runs before MAC verification, so the path is reachable by unauthenticated frames, and without the gate a stream of bogus-SPI frames would drive a flash read plus a full PSA re-import per frame on the com thread while holding the lock shared by all three uplinks. -Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-import the store into PSA and update `ActiveKeyCount` telemetry on success. +Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-read the store from flash first, then re-import it into PSA and update `ActiveKeyCount` telemetry on success. + +All three also refuse to act (`StoreUnreadable`) when that re-read fails with anything other than "file does not exist", because the store's contents are then unknown. For `ADD_KEY`/`REMOVE_KEY` that prevents a read-modify-write from persisting a stale guess over the real store. For `PROVISION_KEY` it is a security property: the command is bypass-allowlisted so it works on a keyless board, so trust-on-first-use must be gated on *proof* that the store is empty. Treating an unreadable store as keyless would let anyone in radio range induce a read failure and install their own key while a valid one still sits on flash. + +`REMOVE_KEY` zeroizes the revoked slot's key bytes before persisting, so a removed key is not recoverable from the on-flash record; the slot is restored if the durable write fails. Primary data path connections: @@ -121,7 +125,7 @@ The MAC is HMAC-SHA-256 truncated to 16 bytes, computed over the Security Header ## Behavior 1. Parse the Security Header and Trailer. If the frame is too short to contain them it cannot be stripped for downstream deframing: log ParsingFailed and return the buffer upstream (drop). -2. Validate the SPI (must match a valid slot in the active key store — see [Key Storage](#key-storage)) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). On an unknown SPI, the key store is reloaded from disk once and validation retried, so a rotation issued over another link is picked up here. +2. Validate the SPI (must match a valid slot in the active key store — see [Key Storage](#key-storage)) and the anti-replay sequence number (must be strictly ahead of the last accepted value, within SEQ_NUM_WINDOW, with U32 wraparound handled). On an unknown SPI, the key store is reloaded from disk once and validation retried — but only if another instance has written the store since this one last read it — so a rotation issued over another link is picked up here without letting unauthenticated frames drive repeated flash reads. 3. If validation passes, look up the PSA key id for the packet's SPI and verify the MAC with it. 4. Only when all checks pass: store and persist the received sequence number, telemeter it, and set `authenticated = true` in the frame context. Frames failing any check never advance the sequence number (issue #426). 5. Strip the Security Header and Trailer and forward on dataOut with the resulting `authenticated` flag. ProvesRouter rejects unauthenticated packets unless their opcode is on the bypass allowlist. @@ -171,11 +175,11 @@ Routed/bypassed/rejected packet counts are telemetered by ProvesRouter, which ow | KeyStoreReadFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store read fails (not thrown for a missing file — that's the keyless state). Format: "Failed to read key store, error: {}" | | KeyStoreWriteFailed | Warning High (throttle 2) | status: Os.FileStatus | Logged when the key store write fails. Format: "Failed to write key store, error: {}" | | KeyProvisioned | Activity High | spi: U16 | Logged by PROVISION_KEY on success. Format: "Key provisioned for SPI={}" | -| KeyProvisionFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when PROVISION_KEY fails (store not empty, bad hex key, or write failure). Format: "Key provisioning failed: {}" | +| KeyProvisionFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when PROVISION_KEY fails (store not empty, unreadable store, bad hex key, write failure, or PSA import failure). Format: "Key provisioning failed: {}" | | KeyAdded | Activity High | spi: U16 | Logged by ADD_KEY on success. Format: "Key added for SPI={}" | -| KeyAddFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when ADD_KEY fails (store full, bad hex key, or write failure). Format: "Key add failed: {}" | +| KeyAddFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when ADD_KEY fails (store full, duplicate SPI, unreadable store, bad hex key, write failure, or PSA import failure). Format: "Key add failed: {}" | | KeyRemoved | Activity High | spi: U16 | Logged by REMOVE_KEY on success. Format: "Key removed for SPI={}" | -| KeyRemoveFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when REMOVE_KEY fails (last remaining key, SPI not found, or write failure). Format: "Key remove failed: {}" | +| KeyRemoveFailed | Warning High (throttle 2) | status: KeyStoreProvisionStatus | Logged when REMOVE_KEY fails (last remaining key, SPI not found, unreadable store, write failure, or PSA import failure). Format: "Key remove failed: {}" | ## Commands From 8bd372570eba162f1d7f39b7f8652a80151bc871 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Wed, 29 Jul 2026 01:56:59 +0200 Subject: [PATCH 25/29] refactor(TcSecurityDeframer): drop the unused importHmacKey() wrapper Since the key store moved to flash, importKeyStore() imports each slot's raw bytes with importHmacKeyBytes(); the hex-parsing wrapper had no callers left outside its own tests. KeyImportStatus::ParseKeyError goes with it, as importHmacKey was the only thing that produced it - the identically named KeyStoreProvisionStatus::ParseKeyError on the command handlers is unrelated and stays. The two tests covering the wrapper's parse path are dropped (parseHexKey has its own null/invalid-hex coverage) and importTestKey() now goes through parseHexKey + importHmacKeyBytes. --- .../TcSecurityDeframer/Authenticator.cpp | 14 -------------- .../TcSecurityDeframer/Authenticator.hpp | 6 ------ .../Components/TcSecurityDeframer/docs/sdd.md | 6 ++++-- .../test_TcSecurityDeframer_Authenticator.cpp | 19 ++++--------------- docs-site/components/TcSecurityDeframer.md | 6 ++++-- 5 files changed, 12 insertions(+), 39 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp index 3b9b6a6a..5bbcbdc4 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.cpp @@ -5,7 +5,6 @@ #include "Authenticator.hpp" -#include #include #include @@ -93,19 +92,6 @@ PacketAuthenticator::KeyImportResult importHmacKeyBytes(const uint8_t (&keyBytes return {PacketAuthenticator::KeyImportStatus::Success, PSA_SUCCESS}; } -// Parse a hex-encoded key and import it into PSA for message verification. -PacketAuthenticator::KeyImportResult importHmacKey(const char* key, uint32_t& keyId) { - // Parse the hex-encoded key into raw bytes - uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; - if (!parseHexKey(key, keyBytes)) { - return {PacketAuthenticator::KeyImportStatus::ParseKeyError, PSA_ERROR_INVALID_ARGUMENT}; - } - - const PacketAuthenticator::KeyImportResult result = importHmacKeyBytes(keyBytes, keyId); - mbedtls_platform_zeroize(keyBytes, sizeof keyBytes); - return result; -} - int32_t destroyHmacKey(uint32_t keyId) { return psa_destroy_key(keyId); } diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp index a264e2a9..2d1859e1 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Authenticator.hpp @@ -17,7 +17,6 @@ namespace PacketAuthenticator { enum class KeyImportStatus { Success, //!< Key was successfully imported InitError, //!< There was an error initializing the authentication process - ParseKeyError, //!< There was an error parsing the key from storage ImportKeyError, //!< There was an error importing the authentication key }; @@ -58,11 +57,6 @@ PacketAuthenticator::KeyImportResult importHmacKeyBytes( uint32_t& keyId //!< The key ID to use for the imported key ); -//! Parse a hex-encoded key and import it into PSA for message verification. -PacketAuthenticator::KeyImportResult importHmacKey(const char* key, //!< The hex-encoded authentication key to import - uint32_t& keyId //!< The key ID to use for the imported key -); - //! Destroy a previously-imported PSA key. Used to release the old key on rotation. //! Returns the PSA status: a failed destroy leaves the old key usable in PSA, which the caller //! must not silently treat as a released slot. diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md index 9e32424b..41cb4493 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md @@ -10,7 +10,7 @@ The component is a thin stateful shell over pure-function namespaces: - `Ccsds355_0_B_2::parse` (Parser) — Security Header (SPI, sequence number) and Trailer (MAC) extraction - `Components::validatePacket` (Validator) — SPI validation against the active key store and anti-replay sequence-number window validation -- `Components::authenticatePacket` / `importHmacKey` / `importHmacKeyBytes` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto +- `Components::authenticatePacket` / `parseHexKey` / `importHmacKeyBytes` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto `Validator` takes the active SPI set as a plain `ActiveSpiSlots` array (`Types.hpp`) rather than the FPP-generated key store type directly, so it — and its unit tests — stay pure C++ with no F Prime dependency; `TcSecurityDeframer::activeSpiSlots()` projects the real key store into that shape before calling `validatePacket`. @@ -76,7 +76,9 @@ class PacketValidator { class PacketAuthenticator { <> - +importHmacKey(key, keyId) KeyImportResult + +parseHexKey(key, keyBytes) bool + +importHmacKeyBytes(keyBytes, keyId) KeyImportResult + +destroyHmacKey(keyId) int32_t +authenticatePacket(buffer, size, mac, keyId) AuthenticationResult } diff --git a/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp index a5eb8829..598aff24 100644 --- a/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp +++ b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_Authenticator.cpp @@ -17,8 +17,11 @@ static const std::vector kTestPacket = {1, 2, 3, 4, 5, 6 //! Import the test key, asserting success, and return the PSA key id static uint32_t importTestKey() { + uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; + EXPECT_TRUE(parseHexKey(kTestKeyHex, keyBytes)); + uint32_t keyId = 0; - auto res = importHmacKey(kTestKeyHex, keyId); + auto res = importHmacKeyBytes(keyBytes, keyId); EXPECT_EQ(res.status, PacketAuthenticator::KeyImportStatus::Success); EXPECT_EQ(res.psaStatus, PSA_SUCCESS); return keyId; @@ -31,20 +34,6 @@ static Mac macOf(const std::vector& packet) { return mac; } -TEST(PacketAuthenticatorTest, ImportInvalidHexKey) { - uint32_t keyId = 0; - auto res = importHmacKey("invalidkey", keyId); - EXPECT_EQ(res.status, PacketAuthenticator::KeyImportStatus::ParseKeyError); - EXPECT_EQ(res.psaStatus, PSA_ERROR_INVALID_ARGUMENT); -} - -TEST(PacketAuthenticatorTest, ImportNullKey) { - uint32_t keyId = 0; - auto res = importHmacKey(nullptr, keyId); - EXPECT_EQ(res.status, PacketAuthenticator::KeyImportStatus::ParseKeyError); - EXPECT_EQ(res.psaStatus, PSA_ERROR_INVALID_ARGUMENT); -} - TEST(ParseHexKeyTest, ValidLowercaseKey) { uint8_t keyBytes[Ccsds355_0_B_2::kTCSecurityTrailer]; EXPECT_TRUE(parseHexKey(kTestKeyHex, keyBytes)); diff --git a/docs-site/components/TcSecurityDeframer.md b/docs-site/components/TcSecurityDeframer.md index 9e32424b..41cb4493 100644 --- a/docs-site/components/TcSecurityDeframer.md +++ b/docs-site/components/TcSecurityDeframer.md @@ -10,7 +10,7 @@ The component is a thin stateful shell over pure-function namespaces: - `Ccsds355_0_B_2::parse` (Parser) — Security Header (SPI, sequence number) and Trailer (MAC) extraction - `Components::validatePacket` (Validator) — SPI validation against the active key store and anti-replay sequence-number window validation -- `Components::authenticatePacket` / `importHmacKey` / `importHmacKeyBytes` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto +- `Components::authenticatePacket` / `parseHexKey` / `importHmacKeyBytes` (Authenticator) — HMAC-SHA-256 (truncated to 16 bytes) verification via PSA crypto `Validator` takes the active SPI set as a plain `ActiveSpiSlots` array (`Types.hpp`) rather than the FPP-generated key store type directly, so it — and its unit tests — stay pure C++ with no F Prime dependency; `TcSecurityDeframer::activeSpiSlots()` projects the real key store into that shape before calling `validatePacket`. @@ -76,7 +76,9 @@ class PacketValidator { class PacketAuthenticator { <> - +importHmacKey(key, keyId) KeyImportResult + +parseHexKey(key, keyBytes) bool + +importHmacKeyBytes(keyBytes, keyId) KeyImportResult + +destroyHmacKey(keyId) int32_t +authenticatePacket(buffer, size, mac, keyId) AuthenticationResult } From 5e81efd823844f2229389e16f99757de942ca9f4 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Wed, 29 Jul 2026 02:09:47 +0200 Subject: [PATCH 26/29] Comment cleanup --- prj.conf | 2 -- west.yml | 9 --------- 2 files changed, 11 deletions(-) diff --git a/prj.conf b/prj.conf index b6135c54..d5788b49 100644 --- a/prj.conf +++ b/prj.conf @@ -74,8 +74,6 @@ CONFIG_FS_FATFS_EXFAT=y CONFIG_FS_FATFS_MOUNT_MKFS=y CONFIG_FS_FATFS_FSTAB_AUTOMOUNT=y CONFIG_FILE_SYSTEM_MKFS=y -# littlefs on the internal-flash keystore_partition, mounted at /keys for the -# HMAC key store and anti-replay sequence number (see TcSecurityDeframer) CONFIG_FILE_SYSTEM_LITTLEFS=y CONFIG_MCUBOOT_SIGNATURE_KEY_FILE="keys/proves.pem" diff --git a/west.yml b/west.yml index f5de61c3..6e2d67ac 100644 --- a/west.yml +++ b/west.yml @@ -31,11 +31,6 @@ manifest: - mcuboot # Bootloader support - fatfs # FatFS (file system) support (SD card, mounted at /) - littlefs # LittleFS - internal-flash key store, mounted at /keys. - # Without this the module is not in zephyr_modules.txt, - # so ZEPHYR_LITTLEFS_MODULE is undefined and Kconfig - # SILENTLY drops CONFIG_FILE_SYSTEM_LITTLEFS=y ("LittleFS - # module not available") -- the build stays clean and - # /keys simply never exists. - hal_st # Required for certain sensors - name: loramac-node @@ -93,10 +88,6 @@ manifest: revision: f4ead3bf4a6dab3a07d7b5f5315795c073db568d path: lib/zephyr-workspace/modules/fatfs - # Pinned explicitly, like every other module here, so it lands under - # lib/zephyr-workspace/ instead of the workspace topdir: the allowlist - # import above only makes the project visible, it keeps Zephyr's own - # `path: modules/fs/littlefs`, which resolves to the repo root. - name: littlefs revision: 8f5ca347843363882619d8f96c00d8dbd88a8e79 path: lib/zephyr-workspace/modules/fs/littlefs From 42534c03b141147cf847e6086abf8896876eb1c4 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Fri, 31 Jul 2026 07:06:58 -0700 Subject: [PATCH 27/29] fix(TcSecurityDeframer): allow PROVISION_KEY on a genuinely blank key store (#485) A board whose key store file does not exist - a brand-new board, or one whose /keys partition was erased or reformatted - could not be provisioned. It came up keyless (correct) and then refused PROVISION_KEY with StoreUnreadable forever, which is total, unrecoverable command loss over every link. Root cause is a host/target divergence. The gate added in b214dd88c26 reads if (loadStatus != OP_OK && loadStatus != DOESNT_EXIST) -> StoreUnreadable but ZephyrFile::open (lib/fprime-zephyr Os/File.cpp) discards fs_open's errno and returns OTHER_ERROR for every failure, so Os::File::DOESNT_EXIST is unreachable on flight hardware. On the POSIX host the same missing file maps ENOENT -> DOESNT_EXIST and the gate passes, which is why host unit tests never saw it. CI hardware never saw it either: every bench board was seeded under 69dcec747d6 (which gated only on activeKeyCount(), reading the default-empty in-memory store), and the littlefs keystore partition survives reflashing, so the cold path is never exercised. The naive fix - map -ENOENT to DOESNT_EXIST in the Zephyr shim - would reintroduce the hole b214dd88c26 closed. Zephyr's fs_get_mnt_point returns -ENOENT when the mount point itself is absent, identically to a missing file, so ENOENT alone would let an attacker who can keep /keys from mounting provision their own key over a board that still holds a valid one (PROVISION_KEY is bypass-allowlisted, i.e. unauthenticated and reachable over RF). Instead, stop inferring anything from the read status and probe the filesystem: * fs_stat on the store file (via FileSystem::_getPathType, not getPathType(), which folds every error into NOT_EXIST) for a positive absence answer; * fs_statvfs on the mount point (via FileSystem::getFreeSpace) as independent proof that /keys is actually mounted. Provisioning is permitted only for a demonstrably-absent file on a demonstrably live mount, or a store that read back cleanly and holds no key. An unreadable store is never provisionable, so the security property is preserved. ADD_KEY and REMOVE_KEY use the same predicate, since their gates had the same defect. The decision itself lives in Components::KeyStore in Types.hpp as pure C++ over probe results rather than over an Os status, so the divergence has nowhere to hide and the host gtest suite can cover the whole domain. A plain component test could not have caught this bug, since on the host the buggy code passes. - Types.hpp: KeyStore::MountProbe / StoreProbe / storeStateIsKnown / storeIsProvisionable (pure, header-only, no Fw:: or Os:: dependency) - TcSecurityDeframer: probeKeyStore() + the three rewritten gates - test_TcSecurityDeframer_KeyStorePolicy.cpp: exhaustive mount x store x key-count matrix, plus a test that documents the host/target divergence - provision_key_test.py: spell out why NotEmpty (and only NotEmpty) is tolerated, and add an explicit skipped test marking the cold-provision coverage gap - sdd.md: document the probe and why the read status is not a usable signal Co-authored-by: Claude Opus 4.8 --- .../TcSecurityDeframer/TcSecurityDeframer.cpp | 87 ++++++++++- .../TcSecurityDeframer/TcSecurityDeframer.hpp | 9 ++ .../Components/TcSecurityDeframer/Types.hpp | 50 ++++++ .../Components/TcSecurityDeframer/docs/sdd.md | 5 +- .../test/int/provision_key_test.py | 38 +++++ ...test_TcSecurityDeframer_KeyStorePolicy.cpp | 142 ++++++++++++++++++ docs-site/components/TcSecurityDeframer.md | 5 +- 7 files changed, 327 insertions(+), 9 deletions(-) create mode 100644 PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_KeyStorePolicy.cpp diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp index 6d993371..a75f6d64 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.cpp @@ -50,6 +50,34 @@ U32& keyStoreGeneration() { return generation; } +//! Asks the filesystem holding `filePath` whether it is actually mounted. +//! +//! fs_statvfs resolves the mount point and queries the underlying FS, so a success is direct +//! evidence that /keys is up. This is the signal that makes "stat says the file is missing" safe to +//! act on: on Zephyr, fs_stat returns -ENOENT for an unmounted mount point just as it does for a +//! missing file (subsys/fs/fs.c: fs_get_mnt_point), so absence alone proves nothing. +KeyStore::MountProbe probeMount(const char* filePath) { + char directory[64]; + const char* const lastSlash = std::strrchr(filePath, '/'); + if (lastSlash == nullptr || lastSlash == filePath) { + directory[0] = '/'; + directory[1] = '\0'; + } else { + const size_t length = static_cast(lastSlash - filePath); + if (length >= sizeof directory) { + return KeyStore::MountProbe::Unknown; + } + std::memcpy(directory, filePath, length); + directory[length] = '\0'; + } + + FwSizeType totalBytes = 0; + FwSizeType freeBytes = 0; + return (Os::FileSystem::getFreeSpace(directory, totalBytes, freeBytes) == Os::FileSystem::OP_OK) + ? KeyStore::MountProbe::Live + : KeyStore::MountProbe::Unknown; +} + } // namespace // ---------------------------------------------------------------------- @@ -230,9 +258,17 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, // of a key we managed to read. PROVISION_KEY is bypass-allowlisted so it works on a keyless // board, so if an unreadable store (a truncated record reading back as BAD_SIZE, a littlefs // error, a mount not ready yet) counted as "keyless", anyone in radio range could induce a read - // failure and install their own key while a valid one still sits on flash. DOESNT_EXIST is the - // one failure that *is* proof of emptiness. - if (loadStatus != Os::File::OP_OK && loadStatus != Os::File::DOESNT_EXIST) { + // failure and install their own key while a valid one still sits on flash. + // + // The proof cannot come from loadStatus alone: on the Zephyr target an absent file reports + // OTHER_ERROR, not DOESNT_EXIST, so gating on the status made a factory-fresh (or /keys-erased) + // board refuse its own bootstrap forever - keyless and unprovisionable, i.e. total command + // loss. probeKeyStore() asks the filesystem instead. See Components::KeyStore in Types.hpp. + KeyStore::MountProbe mountProbe = KeyStore::MountProbe::Unknown; + KeyStore::StoreProbe storeProbe = KeyStore::StoreProbe::Unreadable; + this->probeKeyStore(loadStatus, mountProbe, storeProbe); + + if (!KeyStore::storeStateIsKnown(mountProbe, storeProbe)) { this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::StoreUnreadable); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -240,7 +276,7 @@ void TcSecurityDeframer ::PROVISION_KEY_cmdHandler(FwOpcodeType opCode, // PROVISION_KEY is trust-on-first-use bootstrap: only honored while the store is empty. // Once any key exists, rotation must go through ADD_KEY/REMOVE_KEY (which require auth). - if (this->activeKeyCount() != 0) { + if (!KeyStore::storeIsProvisionable(mountProbe, storeProbe, this->activeKeyCount())) { this->log_WARNING_HI_KeyProvisionFailed(KeyStoreProvisionStatus::NotEmpty); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -297,8 +333,13 @@ void TcSecurityDeframer ::ADD_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, U1 // If the store could not be read back, its contents are unknown, and the read-modify-write // below would persist a guess: writing the last in-memory copy over whatever is actually on - // flash. Refuse rather than risk dropping a live key. - if (loadStatus != Os::File::OP_OK && loadStatus != Os::File::DOESNT_EXIST) { + // flash. Refuse rather than risk dropping a live key. As in PROVISION_KEY, "could not be read" + // has to be established by probing the filesystem, not by reading loadStatus. + KeyStore::MountProbe mountProbe = KeyStore::MountProbe::Unknown; + KeyStore::StoreProbe storeProbe = KeyStore::StoreProbe::Unreadable; + this->probeKeyStore(loadStatus, mountProbe, storeProbe); + + if (!KeyStore::storeStateIsKnown(mountProbe, storeProbe)) { this->log_WARNING_HI_KeyAddFailed(KeyStoreProvisionStatus::StoreUnreadable); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -375,7 +416,11 @@ void TcSecurityDeframer ::REMOVE_KEY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, const Os::File::Status loadStatus = this->loadKeyStore(); // See ADD_KEY: an unreadable store makes the read-modify-write a guess. - if (loadStatus != Os::File::OP_OK && loadStatus != Os::File::DOESNT_EXIST) { + KeyStore::MountProbe mountProbe = KeyStore::MountProbe::Unknown; + KeyStore::StoreProbe storeProbe = KeyStore::StoreProbe::Unreadable; + this->probeKeyStore(loadStatus, mountProbe, storeProbe); + + if (!KeyStore::storeStateIsKnown(mountProbe, storeProbe)) { this->log_WARNING_HI_KeyRemoveFailed(KeyStoreProvisionStatus::StoreUnreadable); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::EXECUTION_ERROR); return; @@ -544,6 +589,34 @@ Os::File::Status TcSecurityDeframer ::loadKeyStore() { return status; } +void TcSecurityDeframer ::probeKeyStore(const Os::File::Status loadStatus, + KeyStore::MountProbe& mount, + KeyStore::StoreProbe& store) const { + if (loadStatus == Os::File::OP_OK) { + // A full, successful read is self-evidently proof the filesystem served the file. + mount = KeyStore::MountProbe::Live; + store = KeyStore::StoreProbe::Present; + return; + } + + // The read failed. Do not infer why from its status: ZephyrFile::open collapses every fs_open + // errno into OTHER_ERROR, so on flight hardware a missing file and a corrupt filesystem are + // indistinguishable here. Ask the filesystem directly instead. + // + // Os::FileSystem::getPathType() is deliberately not used: it folds every error into NOT_EXIST, + // which would turn an I/O error into a false "absent" and re-open the very hole this gate + // exists to close. Go through the interface to keep the real status. + Os::FileSystem::PathType pathType = Os::FileSystem::PathType::NOT_EXIST; + const Os::FileSystem::Status statStatus = + Os::FileSystem::getSingleton()._getPathType(this->m_keyStoreFilePath.toChar(), pathType); + + // DOESNT_EXIST from stat is a positive answer ("this path is not there"), but on Zephyr it is + // also what an unmounted /keys reports, so it only counts once probeMount() corroborates it. + store = + (statStatus == Os::FileSystem::DOESNT_EXIST) ? KeyStore::StoreProbe::Absent : KeyStore::StoreProbe::Unreadable; + mount = probeMount(this->m_keyStoreFilePath.toChar()); +} + Os::File::Status TcSecurityDeframer ::writeKeyStore() { // Write to a temp file, flush, then rename over the target - the same pattern as // StartupManager::persist_boot_count, and for a more serious failure mode. Overwriting the diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp index 40227f3b..0c09263d 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/TcSecurityDeframer.hpp @@ -123,6 +123,15 @@ class TcSecurityDeframer final : public TcSecurityDeframerComponentBase { //! lock held. On a missing/unreadable file, m_keyStore is left with no valid slots (keyless state). Os::File::Status loadKeyStore(); + //! Determines what is actually known about the on-flash key store after a loadKeyStore() + //! attempt, by interrogating the filesystem rather than trusting the read status (which cannot + //! distinguish "missing" from "broken" on the Zephyr target). Feeds the pure + //! Components::KeyStore policy predicates. Must be called with the key store lock held. + void probeKeyStore(const Os::File::Status loadStatus, //!< Status returned by loadKeyStore() + KeyStore::MountProbe& mount, //!< Set to whether /keys is demonstrably mounted + KeyStore::StoreProbe& store //!< Set to what is known about the store file + ) const; + //! Writes m_keyStore to the file system. Must be called with the key store lock held. Os::File::Status writeKeyStore(); diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp index b6084c4b..28c797da 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp @@ -25,6 +25,56 @@ struct ActiveSpiSlot { using ActiveSpiSlots = std::array; //!< The set of currently active SPIs +//! Key-store admission policy. +//! +//! The decision "may this board be provisioned / may its key store be rewritten" is expressed here +//! as pure C++ over probe results, deliberately away from any Os:: type, so it can be exercised +//! directly by the host gtest suite across the full status matrix. +//! +//! Why it cannot be written directly against Os::File::Status: on the Zephyr target a *genuinely +//! missing* file does not report Os::File::DOESNT_EXIST. ZephyrFile::open discards fs_open's errno +//! and returns OTHER_ERROR for every failure, so DOESNT_EXIST is unreachable on flight hardware +//! while it is the normal result on the POSIX host. Any gate written as +//! `status != OP_OK && status != DOESNT_EXIST -> refuse` therefore passes on the host and refuses +//! on the target, which is exactly how a brand-new board became unprovisionable. The read status +//! alone is not a usable signal; the caller must probe the filesystem for the answer. +namespace KeyStore { + +//! Whether the filesystem holding the key store is demonstrably mounted. +enum class MountProbe { + Live, //!< The mount point answered a statvfs, so the filesystem is up + Unknown, //!< The mount point could not be interrogated: not mounted, not ready, or erroring +}; + +//! What is demonstrably true about the key store file itself. +enum class StoreProbe { + Present, //!< The store was read back in full; its contents are known + Absent, //!< The store was positively confirmed not to exist (stat says so) + Unreadable, //!< The store may or may not exist, and its contents are unknown +}; + +//! True when the on-flash key store's contents are known with certainty. +//! +//! Note the asymmetry: `Present` needs no mount probe, because a successful full read is itself +//! proof the filesystem served the file. `Absent` does need one, because on Zephyr fs_stat reports +//! -ENOENT both for "file missing from a mounted filesystem" and for "no such mount point" +//! (subsys/fs/fs.c fs_get_mnt_point). Treating the latter as emptiness would let an attacker who +//! can keep /keys from mounting provision their own key over a board that still holds a valid one. +constexpr bool storeStateIsKnown(MountProbe mount, StoreProbe store) { + return (store == StoreProbe::Present) || (store == StoreProbe::Absent && mount == MountProbe::Live); +} + +//! True when trust-on-first-use provisioning (PROVISION_KEY) may proceed. +//! +//! PROVISION_KEY is bypass-allowlisted, i.e. reachable unauthenticated over RF, so it is only safe +//! while the store is *proven* empty. Proven means: we know what is on flash (storeStateIsKnown) +//! and it holds no active key. An unreadable store is never provisionable. +constexpr bool storeIsProvisionable(MountProbe mount, StoreProbe store, uint8_t activeKeyCount) { + return storeStateIsKnown(mount, store) && activeKeyCount == 0; +} + +} // namespace KeyStore + //! CCSDS 355.0-B-2 //! https://ccsds.org/Pubs/355x0b2.pdf namespace Ccsds355_0_B_2 { diff --git a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md index 41cb4493..5343ec95 100644 --- a/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TcSecurityDeframer/docs/sdd.md @@ -24,7 +24,9 @@ The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-read the store from flash first, then re-import it into PSA and update `ActiveKeyCount` telemetry on success. -All three also refuse to act (`StoreUnreadable`) when that re-read fails with anything other than "file does not exist", because the store's contents are then unknown. For `ADD_KEY`/`REMOVE_KEY` that prevents a read-modify-write from persisting a stale guess over the real store. For `PROVISION_KEY` it is a security property: the command is bypass-allowlisted so it works on a keyless board, so trust-on-first-use must be gated on *proof* that the store is empty. Treating an unreadable store as keyless would let anyone in radio range induce a read failure and install their own key while a valid one still sits on flash. +All three also refuse to act (`StoreUnreadable`) unless the store's contents are *known*, because otherwise they would be acting on a guess. + +"Known" cannot be decided from the read status. On the Zephyr target `ZephyrFile::open` discards `fs_open`'s errno and reports `OTHER_ERROR` for every failure, so `Os::File::DOESNT_EXIST` is unreachable on flight hardware even though it is the normal result for a missing file on the POSIX host. Gating on the status therefore made a factory-fresh (or `/keys`-erased) board refuse `PROVISION_KEY` forever: keyless *and* unprovisionable, i.e. total command loss. Instead `probeKeyStore()` interrogates the filesystem — `fs_stat` on the store file for a positive absence answer, plus `fs_statvfs` on the mount point to prove `/keys` is actually mounted — and feeds the pure predicates in `Components::KeyStore` (`Types.hpp`). Both signals are required: Zephyr returns `-ENOENT` for an unmounted mount point exactly as it does for a missing file, so absence alone proves nothing. For `ADD_KEY`/`REMOVE_KEY` that prevents a read-modify-write from persisting a stale guess over the real store. For `PROVISION_KEY` it is a security property: the command is bypass-allowlisted so it works on a keyless board, so trust-on-first-use must be gated on *proof* that the store is empty. Treating an unreadable store as keyless would let anyone in radio range induce a read failure and install their own key while a valid one still sits on flash. `REMOVE_KEY` zeroizes the revoked slot's key bytes before persisting, so a removed key is not recoverable from the on-flash record; the slot is restored if the durable write fails. @@ -201,6 +203,7 @@ TcSecurityDeframer helper functionality is covered by unit tests in PROVESFlight |---|---| | test_TcSecurityDeframer_Parser.cpp | Valid parse path plus parse failures for SPI, sequence number, and MAC size checks. | | test_TcSecurityDeframer_Validator.cpp | SPI validation against the active `ActiveSpiSlots` set (single slot, second slot, no valid slots), out-of-window and replayed sequence numbers, window boundary, and wraparound handling. | +| test_TcSecurityDeframer_KeyStorePolicy.cpp | The key store admission predicates (`Components::KeyStore::storeStateIsKnown` / `storeIsProvisionable`) over the exhaustive mount x store-probe x active-key-count matrix: a cold board on a live mount is provisionable; an unreadable store never is; an absent store on an unproven mount never is; and a regression test documenting why a gate written over `Os::File::Status` diverges between host and target. | | test_TcSecurityDeframer_Authenticator.cpp | `parseHexKey` (valid upper/lowercase, null, wrong length, non-hex characters), key import via hex and raw bytes, `destroyHmacKey`, successful MAC verification, and failed verification with corrupted MAC, corrupted data, or a destroyed key. | These cover only the pure-function layer (Parser/Validator/Authenticator; no F Prime or Zephyr dependency). The key store mutation rules enforced in the command handlers (`PROVISION_KEY`/`ADD_KEY`/`REMOVE_KEY` — provision-only-when-empty, add fails at 2, remove fails at 1) are F-Prime-component-dependent and are not covered here; see the commented-out `register_fprime_ut` block in `CMakeLists.txt` for a future on-target/component test pass. diff --git a/PROVESFlightControllerReference/test/int/provision_key_test.py b/PROVESFlightControllerReference/test/int/provision_key_test.py index 4aab80bd..0b42d87f 100644 --- a/PROVESFlightControllerReference/test/int/provision_key_test.py +++ b/PROVESFlightControllerReference/test/int/provision_key_test.py @@ -78,7 +78,45 @@ def test_provision_key( if evt.template.get_full_name().endswith("KeyProvisionFailed"): status = evt.args[0].val + # NotEmpty is the ONLY tolerated failure, and only because CI boards keep the key store on + # a littlefs partition that survives reflashing, so every run after the first re-provisions + # a board that is already provisioned. + # + # StoreUnreadable in particular must never be tolerated here. On a board that already holds + # a key it means the store went unreadable (a real fault); on a keyless board it is the + # signature of the cold-provisioning lockout - the board cannot be commanded at all, and + # letting this pass is precisely how that shipped. See test_cold_provision_gap below. assert status == "NotEmpty", ( f"PROVISION_KEY failed with unexpected status {status!r}; " "board should either be keyless or already hold the CI key" ) + + +@pytest.mark.provision_key +def test_cold_provision_gap(): + """Placeholder for the cold-provision path, which is not yet reachable from CI. + + The bug this file's assertions now guard against (PROVISION_KEY refused with StoreUnreadable on + a board whose key store file does not exist) can only be exercised against a *genuinely blank* + key store. Nothing in the test fleet can produce that state: + + * every bench/CI board was provisioned under an earlier firmware revision, and the littlefs + keystore_partition survives reflashing, so the store is never absent; + * the existing ``fsFormat.FORMAT`` command formats the FatFS root ``/``, NOT the littlefs + ``/keys`` partition, so it cannot clear the store either. + + Automating this therefore needs new firmware capability - either a command that erases the + keystore partition (which must itself be authenticated, since it is a remote-bricking primitive + if it is not) or a CI step that flashes a blank keystore partition image over SWD. Rather than + fake a cold board with a mock that would re-hide the host/target divergence, the invariant is + covered exhaustively at the unit level in + ``test/unit-tests/test_TcSecurityDeframer_KeyStorePolicy.cpp``, and this test is left as an + explicit, visible gap. + + TODO(#472): implement once /keys can be erased or pre-flashed blank from CI. + """ + pytest.skip( + "Cold-provision path needs a blank /keys partition; no mechanism exists yet to produce " + "one from CI (fsFormat.FORMAT targets FatFS /, not littlefs /keys). Covered at unit level " + "by test_TcSecurityDeframer_KeyStorePolicy.cpp." + ) diff --git a/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_KeyStorePolicy.cpp b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_KeyStorePolicy.cpp new file mode 100644 index 00000000..79c1c40c --- /dev/null +++ b/PROVESFlightControllerReference/test/unit-tests/test_TcSecurityDeframer_KeyStorePolicy.cpp @@ -0,0 +1,142 @@ +// ====================================================================== +// \title test_TcSecurityDeframer_KeyStorePolicy.cpp +// \brief Unit tests for the pure key-store admission policy (Components::KeyStore) +// +// These tests guard a bug class that a component-level host test structurally cannot catch: the +// original defect only manifested on the Zephyr target, because ZephyrFile::open collapses every +// fs_open errno into Os::File::OTHER_ERROR, while the POSIX host maps ENOENT to +// Os::File::DOESNT_EXIST. A gate written as "refuse unless OP_OK or DOESNT_EXIST" therefore passed +// every host test and bricked a factory-fresh board. +// +// The fix pushes the decision behind a pure predicate over *probe results* rather than over a read +// status, so the divergence has nowhere to hide: there is no "missing file" status input to get +// wrong, and the matrix below is exhaustive over the predicate's entire domain. +// ====================================================================== + +#include + +#include "PROVESFlightControllerReference/Components/TcSecurityDeframer/Types.hpp" + +using namespace Components; +using KeyStore::MountProbe; +using KeyStore::storeIsProvisionable; +using KeyStore::StoreProbe; +using KeyStore::storeStateIsKnown; + +namespace { + +constexpr MountProbe kMounts[] = {MountProbe::Live, MountProbe::Unknown}; +constexpr StoreProbe kStores[] = {StoreProbe::Present, StoreProbe::Absent, StoreProbe::Unreadable}; + +} // namespace + +// ---------------------------------------------------------------------- +// The bootstrap path: a keyless board must be able to accept its first key +// ---------------------------------------------------------------------- + +//! The C1 regression itself: a brand-new board (/keys freshly formatted, store file never written) +//! must be provisionable, or it is permanently keyless and unreachable by command. +TEST(KeyStorePolicy, ColdBoardOnLiveMountIsProvisionable) { + EXPECT_TRUE(storeIsProvisionable(MountProbe::Live, StoreProbe::Absent, 0)); +} + +//! A store file that exists and reads back cleanly but holds no valid slot is equally proven empty. +TEST(KeyStorePolicy, ReadableButEmptyStoreIsProvisionable) { + EXPECT_TRUE(storeIsProvisionable(MountProbe::Live, StoreProbe::Present, 0)); + // A successful read is its own proof the filesystem is up, so it does not need a mount probe. + EXPECT_TRUE(storeIsProvisionable(MountProbe::Unknown, StoreProbe::Present, 0)); +} + +// ---------------------------------------------------------------------- +// The security property: trust-on-first-use may never overwrite a live key +// ---------------------------------------------------------------------- + +//! PROVISION_KEY is bypass-allowlisted (unauthenticated, reachable over RF), so once any key is on +//! flash it must be refused; rotation has to go through the authenticated ADD_KEY/REMOVE_KEY. +TEST(KeyStorePolicy, ProvisionedBoardIsNotProvisionable) { + for (uint8_t count = 1; count <= kMaxActiveKeys; count++) { + EXPECT_FALSE(storeIsProvisionable(MountProbe::Live, StoreProbe::Present, count)) + << "count=" << static_cast(count); + } +} + +//! The core hardening property: if we cannot tell what is on flash, we must not write to it. An +//! attacker who can induce a read failure (glitching, a corrupt record, a wedged littlefs) must not +//! thereby be able to install their own key over a valid one. +TEST(KeyStorePolicy, UnreadableStoreIsNeverProvisionable) { + for (MountProbe mount : kMounts) { + for (uint8_t count = 0; count <= kMaxActiveKeys; count++) { + EXPECT_FALSE(storeIsProvisionable(mount, StoreProbe::Unreadable, count)); + EXPECT_FALSE(storeStateIsKnown(mount, StoreProbe::Unreadable)); + } + } +} + +//! The trap in the naive fix. On Zephyr, fs_open/fs_stat return -ENOENT both for "file missing from +//! a mounted filesystem" and for "that mount point does not exist" (subsys/fs/fs.c +//! fs_get_mnt_point). So mapping ENOENT straight to "absent, therefore empty, therefore +//! provisionable" would let anyone who can keep /keys from mounting take over a board that still +//! holds a valid key. Absence only counts when the mount independently answered. +TEST(KeyStorePolicy, AbsentFileWithoutLiveMountIsNeverProvisionable) { + EXPECT_FALSE(storeIsProvisionable(MountProbe::Unknown, StoreProbe::Absent, 0)); + EXPECT_FALSE(storeStateIsKnown(MountProbe::Unknown, StoreProbe::Absent)); +} + +// ---------------------------------------------------------------------- +// Exhaustive matrix +// ---------------------------------------------------------------------- + +//! Pin the entire domain, so any future widening of the predicate has to be a deliberate edit here. +TEST(KeyStorePolicy, ExhaustiveMatrix) { + for (MountProbe mount : kMounts) { + for (StoreProbe store : kStores) { + const bool expectKnown = + (store == StoreProbe::Present) || (store == StoreProbe::Absent && mount == MountProbe::Live); + EXPECT_EQ(expectKnown, storeStateIsKnown(mount, store)); + + for (uint8_t count = 0; count <= kMaxActiveKeys; count++) { + EXPECT_EQ(expectKnown && count == 0, storeIsProvisionable(mount, store, count)); + } + } + } +} + +// ---------------------------------------------------------------------- +// Host/target divergence documentation +// ---------------------------------------------------------------------- + +//! Mirrors of the Os::File::Status values a key-store read can produce, kept as plain ints so this +//! pure test needs no Os:: dependency. The point of this test is the *classification* step that +//! TcSecurityDeframer::probeKeyStore performs, and specifically that the classification is not +//! allowed to depend on the read status. +enum class ReadStatus { OpOk, DoesntExist, OtherError, BadSize }; + +//! What the old (buggy) gate concluded, purely from the read status. +bool legacyGateAllowed(ReadStatus status) { + return status == ReadStatus::OpOk || status == ReadStatus::DoesntExist; +} + +//! This is the whole bug in one assertion. On the POSIX host a missing file reads back as +//! DOESNT_EXIST and the legacy gate lets provisioning through; on the Zephyr target the identical +//! situation reads back as OTHER_ERROR and the legacy gate refuses. Same board state, opposite +//! answer, decided by which Os layer you happened to compile against. Any future gate that consumes +//! a raw read status reintroduces this. +TEST(KeyStorePolicy, LegacyStatusGateDivergesBetweenHostAndTarget) { + // The same physical condition - key store file genuinely absent: + const ReadStatus onHost = ReadStatus::DoesntExist; // POSIX open() sets ENOENT + const ReadStatus onTarget = ReadStatus::OtherError; // ZephyrFile::open discards fs_open's errno + + EXPECT_TRUE(legacyGateAllowed(onHost)); + EXPECT_FALSE(legacyGateAllowed(onTarget)); + EXPECT_NE(legacyGateAllowed(onHost), legacyGateAllowed(onTarget)) + << "A missing key store must not produce different decisions on host and target"; + + // The replacement predicate takes probe results, not a status, so both platforms feed it the + // same StoreProbe::Absent + MountProbe::Live and get the same answer. + EXPECT_TRUE(storeIsProvisionable(MountProbe::Live, StoreProbe::Absent, 0)); + + // And OTHER_ERROR that is *not* a missing file (a truncated record, a wedged FS) still lands on + // Unreadable, which is refused - the hardening property survives. + EXPECT_FALSE(storeIsProvisionable(MountProbe::Live, StoreProbe::Unreadable, 0)); + EXPECT_FALSE(storeIsProvisionable(MountProbe::Live, StoreProbe::Unreadable, 1)); +} diff --git a/docs-site/components/TcSecurityDeframer.md b/docs-site/components/TcSecurityDeframer.md index 41cb4493..5343ec95 100644 --- a/docs-site/components/TcSecurityDeframer.md +++ b/docs-site/components/TcSecurityDeframer.md @@ -24,7 +24,9 @@ The store is shared across all `TcSecurityDeframer` instances (UART/LoRa/Sband): Three commands manage the store (see [Commands](#commands)): `PROVISION_KEY` (bootstrap, only while the store is empty), `ADD_KEY` (rotation, fails at 2 active keys), and `REMOVE_KEY` (rotation, fails at 1 remaining key). All three re-read the store from flash first, then re-import it into PSA and update `ActiveKeyCount` telemetry on success. -All three also refuse to act (`StoreUnreadable`) when that re-read fails with anything other than "file does not exist", because the store's contents are then unknown. For `ADD_KEY`/`REMOVE_KEY` that prevents a read-modify-write from persisting a stale guess over the real store. For `PROVISION_KEY` it is a security property: the command is bypass-allowlisted so it works on a keyless board, so trust-on-first-use must be gated on *proof* that the store is empty. Treating an unreadable store as keyless would let anyone in radio range induce a read failure and install their own key while a valid one still sits on flash. +All three also refuse to act (`StoreUnreadable`) unless the store's contents are *known*, because otherwise they would be acting on a guess. + +"Known" cannot be decided from the read status. On the Zephyr target `ZephyrFile::open` discards `fs_open`'s errno and reports `OTHER_ERROR` for every failure, so `Os::File::DOESNT_EXIST` is unreachable on flight hardware even though it is the normal result for a missing file on the POSIX host. Gating on the status therefore made a factory-fresh (or `/keys`-erased) board refuse `PROVISION_KEY` forever: keyless *and* unprovisionable, i.e. total command loss. Instead `probeKeyStore()` interrogates the filesystem — `fs_stat` on the store file for a positive absence answer, plus `fs_statvfs` on the mount point to prove `/keys` is actually mounted — and feeds the pure predicates in `Components::KeyStore` (`Types.hpp`). Both signals are required: Zephyr returns `-ENOENT` for an unmounted mount point exactly as it does for a missing file, so absence alone proves nothing. For `ADD_KEY`/`REMOVE_KEY` that prevents a read-modify-write from persisting a stale guess over the real store. For `PROVISION_KEY` it is a security property: the command is bypass-allowlisted so it works on a keyless board, so trust-on-first-use must be gated on *proof* that the store is empty. Treating an unreadable store as keyless would let anyone in radio range induce a read failure and install their own key while a valid one still sits on flash. `REMOVE_KEY` zeroizes the revoked slot's key bytes before persisting, so a removed key is not recoverable from the on-flash record; the slot is restored if the durable write fails. @@ -201,6 +203,7 @@ TcSecurityDeframer helper functionality is covered by unit tests in PROVESFlight |---|---| | test_TcSecurityDeframer_Parser.cpp | Valid parse path plus parse failures for SPI, sequence number, and MAC size checks. | | test_TcSecurityDeframer_Validator.cpp | SPI validation against the active `ActiveSpiSlots` set (single slot, second slot, no valid slots), out-of-window and replayed sequence numbers, window boundary, and wraparound handling. | +| test_TcSecurityDeframer_KeyStorePolicy.cpp | The key store admission predicates (`Components::KeyStore::storeStateIsKnown` / `storeIsProvisionable`) over the exhaustive mount x store-probe x active-key-count matrix: a cold board on a live mount is provisionable; an unreadable store never is; an absent store on an unproven mount never is; and a regression test documenting why a gate written over `Os::File::Status` diverges between host and target. | | test_TcSecurityDeframer_Authenticator.cpp | `parseHexKey` (valid upper/lowercase, null, wrong length, non-hex characters), key import via hex and raw bytes, `destroyHmacKey`, successful MAC verification, and failed verification with corrupted MAC, corrupted data, or a destroyed key. | These cover only the pure-function layer (Parser/Validator/Authenticator; no F Prime or Zephyr dependency). The key store mutation rules enforced in the command handlers (`PROVISION_KEY`/`ADD_KEY`/`REMOVE_KEY` — provision-only-when-empty, add fails at 2, remove fails at 1) are F-Prime-component-dependent and are not covered here; see the commented-out `register_fprime_ut` block in `CMakeLists.txt` for a future on-target/component test pass. From 8d2a5aed2ae37e01ae5e3187ac7ba57873209a05 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Mon, 3 Aug 2026 14:05:56 +0200 Subject: [PATCH 28/29] feat(Makefile): let make gds pass --spi to fprime-gds Operators provisioning a non-default SPI had no way to point make gds at it and had to fall back to invoking fprime-gds directly. SPI= now forwards --spi the same way UART_DEVICE forwards --uart-device. Addresses PR #472 review comment from ineskhou. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c51f588a..d78000ab 100644 --- a/Makefile +++ b/Makefile @@ -408,13 +408,13 @@ sequence: fprime-venv ## Compile a sequence file (usage: make sequence SEQ=start @$(UV_RUN) fprime-seqgen sequences/$(SEQ).seq -d $(ARTIFACT_DIR)/zephyr/fprime-zephyr-deployment .PHONY: gds -gds: ## Run FPrime GDS +gds: ## Run FPrime GDS (set SPI= to authenticate with a non-default key slot) @echo "Running FPrime GDS..." @if [ -n "$(UART_DEVICE)" ]; then \ echo "Using UART_DEVICE=$(UART_DEVICE)"; \ - $(GDS_COMMAND) --uart-device $(UART_DEVICE); \ + $(GDS_COMMAND) --uart-device $(UART_DEVICE) $(if $(SPI),--spi $(SPI)); \ fi - $(GDS_COMMAND) + $(GDS_COMMAND) $(if $(SPI),--spi $(SPI)) .PHONY: delete-shadow-gds delete-shadow-gds: From a04d469cf36a6a06c0fa96f29e0c6d5c7c231c9f Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Mon, 3 Aug 2026 14:06:02 +0200 Subject: [PATCH 29/29] docs(README): walk through provisioning the first key The rotation section assumed a key was already provisioned and gave a beginner nowhere to start. Add a short walkthrough of PROVISION_KEY's trust-on-first-use bootstrap (allowed once, only while the on-flash store is empty) and point at the new make gds SPI=. Addresses PR #472 review comments from ineskhou on README.md and Framing/src/authenticate_plugin.py. --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 3497b447..d3338976 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,14 @@ If you regenerate/replace the bootloader (or switch computers and flash a bootlo You also want to make sure the authentication key the gds runs with is the same as the authentication key provisioned on the board. The board's key lives in its on-flash key store (never in the image); ground reads its key from the `--authentication-key` CLI arg or the `PROVES_AUTH_KEY` env var. Make sure these match the key you provisioned with `PROVISION_KEY`/`ADD_KEY`. +##### Provisioning your first key + +A freshly flashed board boots with an empty on-flash key store — no authentication key ever ships in the image. Because the store is empty, the board allows exactly one unauthenticated command: `PROVISION_KEY`. Once any key is provisioned, `PROVISION_KEY` is refused, so this only works the first time (or again after every key has been removed). + +1. Start GDS as normal (`make gds`). +2. From the GDS command view, send `PROVISION_KEY(spi=, key=<32 hex chars>)`, e.g. `PROVISION_KEY(0, 00112233445566778899aabbccddeeff)`. +3. Tell ground to use the same key for every command after that: set `PROVES_AUTH_KEY` to the same hex string (or pass `--authentication-key`). If you provisioned a non-zero SPI, also run gds with `make gds SPI=` (or pass `--spi ` directly) so ground's outgoing frames carry the matching SPI. + The board holds up to two active keys so a rotation never leaves you locked out. Ground uses exactly one key at a time (whichever `--authentication-key`/`PROVES_AUTH_KEY` it was started with), so rotate in this order: 1. Keep running GDS with the **old** key and send `ADD_KEY(new_spi, new_key)` — the command itself has to authenticate under the old key.