OTA tests + essential fixes for the key-store branch: devicetree-safe FlashWorker region, ComQueue arming fix, hardware-in-the-loop OTA CI

.github/workflows/ci.yaml

@@ -5,6 +5,10 @@ on:
   push:
     branches:
       - main
+  schedule:
+    # Nightly run (03:00 UTC) exercises the slow/over-the-air OTA variant on
+    # the radio rig; PR/push runs only do the fast UART OTA test.
+    - cron: "0 3 * * *"
 
 jobs:
   lint:
@@ -112,10 +116,95 @@ jobs:
             yamcs/yamcs-data/mdb/fprime.xtce.xml
           retention-days: 30
 
+  ota-test-image:
+    # Builds the dedicated OTA update image consumed by ota_update_test.py.
+    # Runs in PARALLEL with the `build` job (no `needs:`) so it does not
+    # gate the main firmware build; both feed the integration jobs.
+    runs-on: deathstar
+    outputs:
+      ota-build-id: ${{ steps.ota-build-id.outputs.id }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Download bin tools
+        if: steps.cache-bin.outputs.cache-hit != 'true'
+        run: |
+          make download-bin
+
+      - name: Setup submodules
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
+        run: |
+          make submodules
+
+      - name: Create python venv
+        run: |
+          make fprime-venv
+
+      - name: Setup Zephyr
+        if: steps.cache-zephyr-workspace.outputs.cache-hit != 'true'
+        run: |
+          make zephyr-workspace
+
+      - name: Setup Zephyr SDK
+        if: steps.cache-zephyr-sdk.outputs.cache-hit != 'true'
+        run: |
+          make zephyr-sdk
+
+      - name: Setup Zephyr Export
+        run: |
+          make zephyr-export
+
+      - name: Install Zephyr Python Dependencies
+        run: |
+          make zephyr-python-deps
+
+      - name: Change to CI Spacecraft ID
+        run: |
+          make make-ci-spacecraft-id
+
+      - name: Generate
+        run: |
+          make generate
+
+      - name: Set Authentication Key
+        env:
+          AUTH_KEY: ${{ secrets.AUTH_KEY }}
+        run: |
+          echo "#define AUTH_DEFAULT_KEY \"$AUTH_KEY\"" > PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h
+
+      - name: Build MCUBoot
+        run: |
+          make build-mcuboot
+
+      - name: Derive OTA build ID
+        id: ota-build-id
+        run: |
+          # Unique per run so the FSW can distinguish this image from the one
+          # currently flashed; consumed as pytest --ota-build-id in the
+          # integration jobs via this job's `ota-build-id` output.
+          OTA_BUILD_ID="${GITHUB_SHA::8}-${GITHUB_RUN_ID}"
+          echo "id=$OTA_BUILD_ID" >> "$GITHUB_OUTPUT"
+          echo "OTA_BUILD_ID=$OTA_BUILD_ID" >> "$GITHUB_ENV"
+          echo "Derived OTA_BUILD_ID=$OTA_BUILD_ID"
+
+      - name: Build OTA Test Image
+        run: |
+          make ota-test-image OTA_BUILD_ID="$OTA_BUILD_ID"
+
+      - name: Upload OTA test image
+        uses: actions/upload-artifact@v4
+        with:
+          name: ota-test-image
+          path: |
+            build-ota-test/ota-test-image.signed.bin
+          retention-days: 30
+
   integration-uart:
     runs-on:
       - integration
-    needs: build
+    needs:
+      - build
+      - ota-test-image
     steps:
       - uses: actions/checkout@v6
 
@@ -124,6 +213,12 @@ jobs:
           name: artifacts
           path: .
 
+      - name: Download OTA test image
+        uses: actions/download-artifact@v6
+        with:
+          name: ota-test-image
+          path: ota-image
+
       - name: Flash Firmware
         uses: ./.github/actions/flash-firmware
 
@@ -215,6 +310,15 @@ jobs:
         run: |
           make test-integration
 
+      - name: Run OTA Update Test (UART)
+        # Runs LAST in the GDS phase: ota_update_test.py reboots the board
+        # twice, so it must not precede the other UART tests. FILTER clears the
+        # default marker exclusions so the (fast, non-slow) UART OTA variant runs.
+        run: |
+          make test-integration TEST=ota_update_test.py \
+            FILTER="not slow" \
+            PYTEST_ARGS="--ota-image=$GITHUB_WORKSPACE/ota-image/ota-test-image.signed.bin --ota-build-id=${{ needs.ota-test-image.outputs.ota-build-id }}"
+
       - name: Format Filesystem
         if: always()
         run: |
@@ -394,6 +498,7 @@ jobs:
       - integration
     needs:
       - build
+      - ota-test-image
     steps:
       - uses: actions/checkout@v6
 
@@ -402,6 +507,15 @@ jobs:
           name: artifacts
           path: .
 
+      - name: Download OTA test image
+        # Only the nightly schedule runs the slow over-the-air OTA variant, so
+        # only fetch the image on that path.
+        if: github.event_name == 'schedule'
+        uses: actions/download-artifact@v6
+        with:
+          name: ota-test-image
+          path: ota-image
+
       - name: Flash Firmware
         uses: ./.github/actions/flash-firmware
 
@@ -555,6 +669,19 @@ jobs:
         run: |
           make test-integration FILTER="not sync_sequence_number and not format_filesystem and not provision_key and not uart_only" PYTEST_ARGS=--with-radio
 
+      - name: Run OTA Update Test (LoRa, nightly)
+        # Nightly-only: the LoRa OTA variant is marked slow and takes the full
+        # over-the-air transfer time. Runs after the other radio tests and
+        # reboots the board twice. Gated to the schedule event, mirroring how
+        # the slow variant is kept off the per-PR path.
+        if: github.event_name == 'schedule'
+        env:
+          TLM_SAMPLE_LOG: tlm_sample_ota.csv
+        run: |
+          make test-integration TEST=ota_update_test.py \
+            FILTER="slow" \
+            PYTEST_ARGS="--with-radio --ota-image=$GITHUB_WORKSPACE/ota-image/ota-test-image.signed.bin --ota-build-id=${{ needs.ota-test-image.outputs.ota-build-id }}"
+
       - name: Format Filesystem
         if: always()
         env:

Makefile

@@ -463,6 +463,46 @@ make-ci-spacecraft-id: ## Generate a unique spacecraft ID for CI builds (also re
 	rm yamcs/yamcs-data/etc/yamcs.fprime-project.yaml.bak
 	@! grep -q 'spacecraftId: 68' yamcs/yamcs-data/etc/yamcs.fprime-project.yaml || (echo "Failed to patch all spacecraftId entries in yamcs.fprime-project.yaml" && exit 1)
 
+##@ OTA Test Image
+
+# Build a uniquely-marked, MCUBoot-signed OTA test image. Reuses the exact same
+# FSW build + signing pipeline as `make build` (same board defconfig, same
+# prj.conf CONFIG_MCUBOOT_SIGNATURE_KEY_FILE="keys/proves.pem", same swap-using-
+# offset footer), changing nothing about signing. The only difference is that the
+# supplied OTA_BUILD_ID is injected into the project version string that
+# Svc::Version reports at startup, so the running image can be identified over the
+# link via the `CdhCore.version.ProjectVersion` event/telemetry.
+#
+# Injection mechanism: F' derives PROJECT_VERSION from `git describe --tags` run
+# in the project root (lib/fprime/cmake/target/version/generate_version_info.py).
+# We create an ephemeral lightweight tag at HEAD carrying the id so `git describe`
+# resolves to it (0 commits distance => exact match), then delete it afterwards.
+# No tracked file is modified. Keep OTA_BUILD_ID <= 30 chars: the ProjectVersion
+# event string is capped at 40 chars and git may append a "-dirty" suffix.
+OTA_BUILD_DIR ?= $(shell pwd)/build-ota-test
+OTA_IMAGE ?= $(OTA_BUILD_DIR)/ota-test-image.signed.bin
+OTA_TAG_PREFIX ?= ota-test
+OTA_SIGNED_BIN ?= $(shell pwd)/build-artifacts/zephyr.signed.bin
+
+.PHONY: ota-test-image
+ota-test-image: submodules zephyr fprime-venv generate-if-needed ## Build a uniquely-marked MCUBoot-signed OTA test image (OTA_BUILD_ID=<id>)
+	@if [ -z "$(OTA_BUILD_ID)" ]; then \
+		echo "Error: set OTA_BUILD_ID=<id>. Usage: make ota-test-image OTA_BUILD_ID=<id>"; \
+		echo "       <id> lands in the CdhCore.version.ProjectVersion startup event; keep it <=30 chars."; \
+		exit 1; \
+	fi
+	@echo "Building OTA test image (OTA_BUILD_ID=$(OTA_BUILD_ID))"
+	@TAG="$(OTA_TAG_PREFIX)-$(OTA_BUILD_ID)"; \
+	  git tag -f "$$TAG" >/dev/null 2>&1 || { echo "Error: failed to create version tag $$TAG"; exit 1; }; \
+	  trap 'git tag -d "'"$$TAG"'" >/dev/null 2>&1 || true' EXIT INT TERM; \
+	  echo "Injected project version: $$(git describe --tags --always --dirty --broken)"; \
+	  rm -f "$(BUILD_DIR)/versions/version.cpp" "$(BUILD_DIR)/versions/version.hpp" "$(BUILD_DIR)/versions/version.json"; \
+	  $(UV_RUN) fprime-util build || exit 1; \
+	  test -f "$(OTA_SIGNED_BIN)" || { echo "Error: signed image not found at $(OTA_SIGNED_BIN)"; exit 1; }; \
+	  mkdir -p "$(OTA_BUILD_DIR)"; \
+	  cp "$(OTA_SIGNED_BIN)" "$(OTA_IMAGE)"; \
+	  $(UV_RUN) python3 -c "import sys,zlib; d=open(sys.argv[1],'rb').read(); print('OTA image:  '+sys.argv[1]); print('Size:       %d bytes'%len(d)); print('CRC32 (fileManager.CalculateCrc): 0x%08x'%((zlib.crc32(d)&0xffffffff)^0xffffffff))" "$(OTA_IMAGE)"
+
 include makelib/build-tools.mk
 include makelib/ci.mk
 include makelib/zephyr.mk

PROVESFlightControllerReference/Components/FlashWorker/FlashWorker.hpp

@@ -8,12 +8,24 @@
 #define Update_FlashWorker_HPP
 #include "Os/File.hpp"
 #include "PROVESFlightControllerReference/Components/FlashWorker/FlashWorkerComponentAc.hpp"
+#include <zephyr/devicetree.h>
 #include <zephyr/dfu/flash_img.h>
+#include <zephyr/storage/flash_map.h>
 namespace Components {
 
 class FlashWorker final : public FlashWorkerComponentBase {
   public:
-    constexpr static U8 REGION_NUMBER = 2;  // 0: bootloader, 1: slot0, **2: slot1**
+    //! MCUboot secondary slot: where an uploaded image is staged before the bootloader swaps it in.
+    //!
+    //! Resolved from the devicetree label, never hardcoded. Zephyr hands out flash-area IDs in
+    //! devicetree dependency-ordinal order, so adding a partition anywhere in the DT renumbers every
+    //! area. A hardcoded ID silently starts pointing at a different partition -- and erasing the
+    //! wrong one here wipes the running firmware.
+    constexpr static U8 REGION_NUMBER = PARTITION_ID(slot1_partition);
+
+    //! Guard the failure above: the staging region must never be the slot we are executing from.
+    static_assert(PARTITION_OFFSET(slot1_partition) != DT_REG_ADDR(DT_CHOSEN(zephyr_code_partition)),
+                  "FlashWorker update region overlaps the running code partition");
     enum Step { IDLE, PREPARE, UPDATE };
     // ----------------------------------------------------------------------
     // Component construction and destruction

PROVESFlightControllerReference/test/int/common.py

@@ -75,6 +75,35 @@ def set_radio_recover_fn(fn: Callable[[], None] | None) -> None:
     _radio_recover_fn = fn
 
 
+def resync_sequence_number(
+    fprime_test_api: IntegrationTestAPI,
+    deframer: str = "ComCcsdsUart.tcSecurityDeframer",
+) -> None:
+    """Fast-forward the framer plugin's sequence file to the board's counter.
+
+    After a reboot the board resumes from its write-ahead persisted sequence
+    number (issue #461), which can be ahead of the ground counter -- every
+    authenticated command is then rejected until ground catches up. GET_SEQ_NUM
+    is bypass-listed so it works even while desynced. Ground being ahead is
+    normal and left untouched.
+    """
+    fprime_test_api.clear_histories()
+    fprime_test_api.send_command(f"{deframer}.GET_SEQ_NUM")
+    evt = fprime_test_api.await_event(f"{deframer}.SequenceNumberGet", timeout=5)
+    if evt is None:
+        return
+    board_seq = int(evt.args[0].val)
+    seq_file = "./Framing/src/sequence_number.bin"
+    try:
+        with open(seq_file, "r", encoding="utf-8") as f:
+            ground_seq = int(f.read().strip() or 0)
+    except (OSError, ValueError):
+        ground_seq = -1
+    if board_seq > ground_seq:
+        with open(seq_file, "w", encoding="utf-8") as f:
+            f.write(str(board_seq))
+
+
 def proves_send_and_assert_command(
     fprime_test_api: IntegrationTestAPI,
     command: str,
@@ -120,6 +149,15 @@ def proves_send_and_assert_command(
                 and (attempt + 1) % RADIO_RECOVER_THRESHOLD == 0
             ):
                 _radio_recover_fn()
+            # A mid-test reboot (safe-mode entry, reset, watchdog) leaves the
+            # board expecting a written-ahead sequence number (issue #461) and
+            # silently rejecting every authenticated command. GET_SEQ_NUM is
+            # bypass-listed, so resyncing here works even in that state and
+            # costs one round-trip per failed attempt.
+            try:
+                resync_sequence_number(fprime_test_api)
+            except Exception:  # noqa: BLE001 -- recovery must not mask the retry
+                pass
             # Fibonacci backoff with ±50% jitter before the next retry.
             # The LoRa radio link is half-duplex: the satellite cannot receive
             # an uplink command while it is transmitting events/telemetry