diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ae474ca0..93b8d002 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,6 +5,10 @@ on: push: branches: - main + schedule: + # Nightly run (03:00 UTC) exercises the slow/over-the-air OTA variant on + # the radio rig; PR/push runs only do the fast UART OTA test. + - cron: "0 3 * * *" jobs: lint: @@ -112,10 +116,95 @@ jobs: yamcs/yamcs-data/mdb/fprime.xtce.xml retention-days: 30 + ota-test-image: + # Builds the dedicated OTA update image consumed by ota_update_test.py. + # Runs in PARALLEL with the `build` job (no `needs:`) so it does not + # gate the main firmware build; both feed the integration jobs. + runs-on: deathstar + outputs: + ota-build-id: ${{ steps.ota-build-id.outputs.id }} + steps: + - uses: actions/checkout@v6 + + - name: Download bin tools + if: steps.cache-bin.outputs.cache-hit != 'true' + run: | + make download-bin + + - name: Setup submodules + if: steps.cache-submodules.outputs.cache-hit != 'true' + run: | + make submodules + + - name: Create python venv + run: | + make fprime-venv + + - name: Setup Zephyr + if: steps.cache-zephyr-workspace.outputs.cache-hit != 'true' + run: | + make zephyr-workspace + + - name: Setup Zephyr SDK + if: steps.cache-zephyr-sdk.outputs.cache-hit != 'true' + run: | + make zephyr-sdk + + - name: Setup Zephyr Export + run: | + make zephyr-export + + - name: Install Zephyr Python Dependencies + run: | + make zephyr-python-deps + + - name: Change to CI Spacecraft ID + run: | + make make-ci-spacecraft-id + + - name: Generate + run: | + make generate + + - name: Set Authentication Key + env: + AUTH_KEY: ${{ secrets.AUTH_KEY }} + run: | + echo "#define AUTH_DEFAULT_KEY \"$AUTH_KEY\"" > PROVESFlightControllerReference/Components/TcSecurityDeframer/AuthDefaultKey.h + + - name: Build MCUBoot + run: | + make build-mcuboot + + - name: Derive OTA build ID + id: ota-build-id + run: | + # Unique per run so the FSW can distinguish this image from the one + # currently flashed; consumed as pytest --ota-build-id in the + # integration jobs via this job's `ota-build-id` output. + OTA_BUILD_ID="${GITHUB_SHA::8}-${GITHUB_RUN_ID}" + echo "id=$OTA_BUILD_ID" >> "$GITHUB_OUTPUT" + echo "OTA_BUILD_ID=$OTA_BUILD_ID" >> "$GITHUB_ENV" + echo "Derived OTA_BUILD_ID=$OTA_BUILD_ID" + + - name: Build OTA Test Image + run: | + make ota-test-image OTA_BUILD_ID="$OTA_BUILD_ID" + + - name: Upload OTA test image + uses: actions/upload-artifact@v4 + with: + name: ota-test-image + path: | + build-ota-test/ota-test-image.signed.bin + retention-days: 30 + integration-uart: runs-on: - integration - needs: build + needs: + - build + - ota-test-image steps: - uses: actions/checkout@v6 @@ -124,6 +213,12 @@ jobs: name: artifacts path: . + - name: Download OTA test image + uses: actions/download-artifact@v6 + with: + name: ota-test-image + path: ota-image + - name: Flash Firmware uses: ./.github/actions/flash-firmware @@ -215,6 +310,15 @@ jobs: run: | make test-integration + - name: Run OTA Update Test (UART) + # Runs LAST in the GDS phase: ota_update_test.py reboots the board + # twice, so it must not precede the other UART tests. FILTER clears the + # default marker exclusions so the (fast, non-slow) UART OTA variant runs. + run: | + make test-integration TEST=ota_update_test.py \ + FILTER="not slow" \ + PYTEST_ARGS="--ota-image=$GITHUB_WORKSPACE/ota-image/ota-test-image.signed.bin --ota-build-id=${{ needs.ota-test-image.outputs.ota-build-id }}" + - name: Format Filesystem if: always() run: | @@ -394,6 +498,7 @@ jobs: - integration needs: - build + - ota-test-image steps: - uses: actions/checkout@v6 @@ -402,6 +507,15 @@ jobs: name: artifacts path: . + - name: Download OTA test image + # Only the nightly schedule runs the slow over-the-air OTA variant, so + # only fetch the image on that path. + if: github.event_name == 'schedule' + uses: actions/download-artifact@v6 + with: + name: ota-test-image + path: ota-image + - name: Flash Firmware uses: ./.github/actions/flash-firmware @@ -555,6 +669,19 @@ jobs: run: | make test-integration FILTER="not sync_sequence_number and not format_filesystem and not provision_key and not uart_only" PYTEST_ARGS=--with-radio + - name: Run OTA Update Test (LoRa, nightly) + # Nightly-only: the LoRa OTA variant is marked slow and takes the full + # over-the-air transfer time. Runs after the other radio tests and + # reboots the board twice. Gated to the schedule event, mirroring how + # the slow variant is kept off the per-PR path. + if: github.event_name == 'schedule' + env: + TLM_SAMPLE_LOG: tlm_sample_ota.csv + run: | + make test-integration TEST=ota_update_test.py \ + FILTER="slow" \ + PYTEST_ARGS="--with-radio --ota-image=$GITHUB_WORKSPACE/ota-image/ota-test-image.signed.bin --ota-build-id=${{ needs.ota-test-image.outputs.ota-build-id }}" + - name: Format Filesystem if: always() env: diff --git a/Makefile b/Makefile index d78000ab..085a5f28 100644 --- a/Makefile +++ b/Makefile @@ -463,6 +463,46 @@ make-ci-spacecraft-id: ## Generate a unique spacecraft ID for CI builds (also re rm yamcs/yamcs-data/etc/yamcs.fprime-project.yaml.bak @! grep -q 'spacecraftId: 68' yamcs/yamcs-data/etc/yamcs.fprime-project.yaml || (echo "Failed to patch all spacecraftId entries in yamcs.fprime-project.yaml" && exit 1) +##@ OTA Test Image + +# Build a uniquely-marked, MCUBoot-signed OTA test image. Reuses the exact same +# FSW build + signing pipeline as `make build` (same board defconfig, same +# prj.conf CONFIG_MCUBOOT_SIGNATURE_KEY_FILE="keys/proves.pem", same swap-using- +# offset footer), changing nothing about signing. The only difference is that the +# supplied OTA_BUILD_ID is injected into the project version string that +# Svc::Version reports at startup, so the running image can be identified over the +# link via the `CdhCore.version.ProjectVersion` event/telemetry. +# +# Injection mechanism: F' derives PROJECT_VERSION from `git describe --tags` run +# in the project root (lib/fprime/cmake/target/version/generate_version_info.py). +# We create an ephemeral lightweight tag at HEAD carrying the id so `git describe` +# resolves to it (0 commits distance => exact match), then delete it afterwards. +# No tracked file is modified. Keep OTA_BUILD_ID <= 30 chars: the ProjectVersion +# event string is capped at 40 chars and git may append a "-dirty" suffix. +OTA_BUILD_DIR ?= $(shell pwd)/build-ota-test +OTA_IMAGE ?= $(OTA_BUILD_DIR)/ota-test-image.signed.bin +OTA_TAG_PREFIX ?= ota-test +OTA_SIGNED_BIN ?= $(shell pwd)/build-artifacts/zephyr.signed.bin + +.PHONY: ota-test-image +ota-test-image: submodules zephyr fprime-venv generate-if-needed ## Build a uniquely-marked MCUBoot-signed OTA test image (OTA_BUILD_ID=) + @if [ -z "$(OTA_BUILD_ID)" ]; then \ + echo "Error: set OTA_BUILD_ID=. Usage: make ota-test-image OTA_BUILD_ID="; \ + echo " lands in the CdhCore.version.ProjectVersion startup event; keep it <=30 chars."; \ + exit 1; \ + fi + @echo "Building OTA test image (OTA_BUILD_ID=$(OTA_BUILD_ID))" + @TAG="$(OTA_TAG_PREFIX)-$(OTA_BUILD_ID)"; \ + git tag -f "$$TAG" >/dev/null 2>&1 || { echo "Error: failed to create version tag $$TAG"; exit 1; }; \ + trap 'git tag -d "'"$$TAG"'" >/dev/null 2>&1 || true' EXIT INT TERM; \ + echo "Injected project version: $$(git describe --tags --always --dirty --broken)"; \ + rm -f "$(BUILD_DIR)/versions/version.cpp" "$(BUILD_DIR)/versions/version.hpp" "$(BUILD_DIR)/versions/version.json"; \ + $(UV_RUN) fprime-util build || exit 1; \ + test -f "$(OTA_SIGNED_BIN)" || { echo "Error: signed image not found at $(OTA_SIGNED_BIN)"; exit 1; }; \ + mkdir -p "$(OTA_BUILD_DIR)"; \ + cp "$(OTA_SIGNED_BIN)" "$(OTA_IMAGE)"; \ + $(UV_RUN) python3 -c "import sys,zlib; d=open(sys.argv[1],'rb').read(); print('OTA image: '+sys.argv[1]); print('Size: %d bytes'%len(d)); print('CRC32 (fileManager.CalculateCrc): 0x%08x'%((zlib.crc32(d)&0xffffffff)^0xffffffff))" "$(OTA_IMAGE)" + include makelib/build-tools.mk include makelib/ci.mk include makelib/zephyr.mk diff --git a/PROVESFlightControllerReference/Components/FlashWorker/FlashWorker.hpp b/PROVESFlightControllerReference/Components/FlashWorker/FlashWorker.hpp index 657fdca2..58739a8d 100644 --- a/PROVESFlightControllerReference/Components/FlashWorker/FlashWorker.hpp +++ b/PROVESFlightControllerReference/Components/FlashWorker/FlashWorker.hpp @@ -8,12 +8,24 @@ #define Update_FlashWorker_HPP #include "Os/File.hpp" #include "PROVESFlightControllerReference/Components/FlashWorker/FlashWorkerComponentAc.hpp" +#include #include +#include namespace Components { class FlashWorker final : public FlashWorkerComponentBase { public: - constexpr static U8 REGION_NUMBER = 2; // 0: bootloader, 1: slot0, **2: slot1** + //! MCUboot secondary slot: where an uploaded image is staged before the bootloader swaps it in. + //! + //! Resolved from the devicetree label, never hardcoded. Zephyr hands out flash-area IDs in + //! devicetree dependency-ordinal order, so adding a partition anywhere in the DT renumbers every + //! area. A hardcoded ID silently starts pointing at a different partition -- and erasing the + //! wrong one here wipes the running firmware. + constexpr static U8 REGION_NUMBER = PARTITION_ID(slot1_partition); + + //! Guard the failure above: the staging region must never be the slot we are executing from. + static_assert(PARTITION_OFFSET(slot1_partition) != DT_REG_ADDR(DT_CHOSEN(zephyr_code_partition)), + "FlashWorker update region overlaps the running code partition"); enum Step { IDLE, PREPARE, UPDATE }; // ---------------------------------------------------------------------- // Component construction and destruction diff --git a/PROVESFlightControllerReference/test/int/common.py b/PROVESFlightControllerReference/test/int/common.py index 627c4274..b8cb313c 100644 --- a/PROVESFlightControllerReference/test/int/common.py +++ b/PROVESFlightControllerReference/test/int/common.py @@ -75,6 +75,35 @@ def set_radio_recover_fn(fn: Callable[[], None] | None) -> None: _radio_recover_fn = fn +def resync_sequence_number( + fprime_test_api: IntegrationTestAPI, + deframer: str = "ComCcsdsUart.tcSecurityDeframer", +) -> None: + """Fast-forward the framer plugin's sequence file to the board's counter. + + After a reboot the board resumes from its write-ahead persisted sequence + number (issue #461), which can be ahead of the ground counter -- every + authenticated command is then rejected until ground catches up. GET_SEQ_NUM + is bypass-listed so it works even while desynced. Ground being ahead is + normal and left untouched. + """ + fprime_test_api.clear_histories() + fprime_test_api.send_command(f"{deframer}.GET_SEQ_NUM") + evt = fprime_test_api.await_event(f"{deframer}.SequenceNumberGet", timeout=5) + if evt is None: + return + board_seq = int(evt.args[0].val) + seq_file = "./Framing/src/sequence_number.bin" + try: + with open(seq_file, "r", encoding="utf-8") as f: + ground_seq = int(f.read().strip() or 0) + except (OSError, ValueError): + ground_seq = -1 + if board_seq > ground_seq: + with open(seq_file, "w", encoding="utf-8") as f: + f.write(str(board_seq)) + + def proves_send_and_assert_command( fprime_test_api: IntegrationTestAPI, command: str, @@ -120,6 +149,15 @@ def proves_send_and_assert_command( and (attempt + 1) % RADIO_RECOVER_THRESHOLD == 0 ): _radio_recover_fn() + # A mid-test reboot (safe-mode entry, reset, watchdog) leaves the + # board expecting a written-ahead sequence number (issue #461) and + # silently rejecting every authenticated command. GET_SEQ_NUM is + # bypass-listed, so resyncing here works even in that state and + # costs one round-trip per failed attempt. + try: + resync_sequence_number(fprime_test_api) + except Exception: # noqa: BLE001 -- recovery must not mask the retry + pass # Fibonacci backoff with ±50% jitter before the next retry. # The LoRa radio link is half-duplex: the satellite cannot receive # an uplink command while it is transmitting events/telemetry diff --git a/PROVESFlightControllerReference/test/int/conftest.py b/PROVESFlightControllerReference/test/int/conftest.py index 9dec6bac..bdf2260e 100644 --- a/PROVESFlightControllerReference/test/int/conftest.py +++ b/PROVESFlightControllerReference/test/int/conftest.py @@ -10,7 +10,7 @@ import time import pytest -from common import cmdDispatch, set_radio_recover_fn +from common import cmdDispatch, resync_sequence_number, set_radio_recover_fn from fprime_gds.common.testing_fw.api import IntegrationTestAPI # After TRANSMIT is first enabled the satellite flushes the event backlog that @@ -107,6 +107,20 @@ def pytest_addoption(parser: pytest.Parser) -> None: help="Skip tests that require add-on hardware (face, antenna, battery, or " "the JP6 watchdog jumper) so the suite can run on a bare flight control board.", ) + parser.addoption( + "--ota-image", + default=None, + help="Path to the signed flight-software image (zephyr.signed.bin) uplinked " + "by the OTA image-swap tests (ota_update_test.py). When unset those tests skip.", + ) + parser.addoption( + "--ota-build-id", + default=None, + help="Unique marker string baked into the --ota-image build (e.g. its git " + "describe / project version). The OTA swap test asserts this string appears in " + "the CdhCore.version.ProjectVersion event after booting the new image, and is " + "absent again after the MCUBoot auto-revert. Required for ota_update_test.py.", + ) def pytest_configure(config: pytest.Config) -> None: @@ -219,6 +233,50 @@ def start_radio(request: pytest.FixtureRequest, fprime_test_api: IntegrationTest set_radio_recover_fn(lambda: _enable_radio(fprime_test_api)) +@pytest.fixture(autouse=True) +def resync_sequence_number_after_reboot( + request: pytest.FixtureRequest, + fprime_test_api: IntegrationTestAPI, + start_gds, +): + """Keep the ground authentication sequence number aligned across in-suite + reboots (issue #473 CI cascade). + + The TcSecurityDeframer persists a write-ahead high-water mark (issue #461), + so after a reboot the board can legitimately expect a sequence number ahead + of the ground counter, and every authenticated command is rejected until + ground catches up. Reboots happen mid-suite (safe-mode entry, reset tests, + watchdog tests), so before each test read the board's counter via + GET_SEQ_NUM (bypass-listed, works even while desynced) and fast-forward the + framer plugin's sequence file if the board is ahead. Ground being ahead is + normal and left alone (the acceptance window extends forward). + """ + # Don't recurse into the dedicated sync/format plumbing tests. + if request.node.get_closest_marker("sync_sequence_number") or ( + request.node.get_closest_marker("format_filesystem") + ): + yield + return + + link = request.config.getoption("--sync-deframer", default=None) + if link is None: + link = ( + "lora" + if request.config.getoption("--with-radio", default=False) + else "uart" + ) + deframer = { + "uart": "ComCcsdsUart.tcSecurityDeframer", + "lora": "ComCcsdsLora.tcSecurityDeframer", + }[link] + + try: + resync_sequence_number(fprime_test_api, deframer) + except Exception: # noqa: BLE001 -- recovery must never fail a test itself + pass + yield + + @pytest.fixture(autouse=True) def recover_from_safe_mode( request: pytest.FixtureRequest, @@ -318,3 +376,23 @@ def sample_loop(): stop.set() t.join(timeout=3) fprime_test_api_session.remove_telemetry_subhistory(subhist) + + +@pytest.fixture +def ota_config(request: pytest.FixtureRequest): + """Provide the OTA image path and build-id marker, skipping if either is unset. + + The OTA image-swap tests (ota_update_test.py) need a real signed image to + uplink and a unique marker string to prove which image is running. Both are + supplied on the command line via --ota-image / --ota-build-id; without them + there is nothing meaningful to test, so the tests skip rather than fail. + """ + image = request.config.getoption("--ota-image", default=None) + build_id = request.config.getoption("--ota-build-id", default=None) + if not image or not build_id: + pytest.skip( + "OTA tests require --ota-image= and --ota-build-id=" + ) + if not os.path.isfile(image): + pytest.skip(f"OTA image not found: {image}") + return image, build_id diff --git a/PROVESFlightControllerReference/test/int/ota_update_test.py b/PROVESFlightControllerReference/test/int/ota_update_test.py new file mode 100644 index 00000000..a1d49090 --- /dev/null +++ b/PROVESFlightControllerReference/test/int/ota_update_test.py @@ -0,0 +1,444 @@ +""" +ota_update_test.py: + +Integration tests for the over-the-air (OTA) flight-software image swap path +(Update.updater + Components.FlashWorker, backed by MCUBoot). + +These tests exercise the full "swap and revert" lifecycle end to end against real +hardware: + + 1. Uplink a signed image to a unique destination on the on-board filesystem. + 2. Verify the uplink with the on-board CalculateCrc oracle. + 3. PREPARE_UPDATE -> UPDATE_IMAGE_FROM -> CONFIGURE_NEXT_BOOT[TEST]. + 4. COLD_RESET and assert the newly-booted image reports the --ota-build-id. + 5. Deliberately skip CONFIRM_UPDATE, COLD_RESET again, and assert MCUBoot has + auto-reverted to the previous (confirmed) image. + +They are parametrized over the uplink transport: + * ``uart`` — fast, reliable link. The image is enqueued on the GDS uplinker + and we poll for IDLE. Runs on a UART bench (skipped with --with-radio). + * ``lora`` — lossy half-duplex radio. The image is re-uplinked to the same + destination until the on-board CRC matches (idempotent offset + writes). Marked ``slow`` and only runs with --with-radio. + +Because every variant issues COLD_RESET (which severs the RF link mid-run) each +is also tagged ``uart_only`` so the suite's collection logic keeps them off the +pure-radio regression path. + +Requires --ota-image= and --ota-build-id=; +without them the ``ota_config`` fixture skips the whole module. See issue for +context: OTA image-swap integration coverage. +""" + +import os +import shutil +import tempfile +import time +import zlib +from datetime import datetime + +import pytest +from common import proves_send_and_assert_command +from fprime_gds.common.files.helpers import FileStates +from fprime_gds.common.models.serialize.time_type import TimeType +from fprime_gds.common.testing_fw.api import IntegrationTestAPI + +# Every variant COLD_RESETs the board, severing the RF link, so this module is +# uart_only in the same sense as reset_manager_test / radio_test. +pytestmark = [pytest.mark.ota, pytest.mark.uart_only] + +UPDATER = "Update.updater" +WORKER = "Update.worker" +FILE_MANAGER = "FileHandling.fileManager" +RESET_MANAGER = "ReferenceDeployment.resetManager" +VERSION = "CdhCore.version" + +# Flash write of a full signed image is the slowest step in the sequence; give it +# a generous ceiling so a legitimately-long erase/write does not time out. +UPDATE_TIMEOUT_S = 180 +PREPARE_TIMEOUT_S = 60 +BOOT_TIMEOUT_S = 30 + +# UART uplink of a multi-hundred-KB image; the LoRa path uses its own longer budget. +UART_UPLINK_TIMEOUT_S = 900 +# LoRa is airtime-bound and lossy: a full image can take many minutes and may need +# re-uplinking a few times before the on-board CRC matches. +LORA_UPLINK_TIMEOUT_S = 3600 +LORA_UPLINK_ATTEMPTS = 4 + + +def _local_crc(path: str) -> int: + """CRC32 matching FileHandling.fileManager.CalculateCrc (zlib.crc32 ^ 0xFFFFFFFF).""" + crc = 0 + with open(path, "rb") as fh: + while chunk := fh.read(8192): + crc = zlib.crc32(chunk, crc) + return ~crc & 0xFFFFFFFF + + +def _onboard_crc(api: IntegrationTestAPI, dest: str, timeout: float = 30) -> int | None: + """Return the board-computed CRC32 of ``dest`` via CalculateCrc, or None on failure.""" + api.send_command(f"{FILE_MANAGER}.CalculateCrc", [dest]) + evt = api.await_event(f"{FILE_MANAGER}.CalculateCrcSucceeded", timeout=timeout) + if evt is None: + return None + # CalculateCrcSucceeded args: (file_name, crc) — the CRC is arg[1]. + return evt.args[1].val + + +def _uplink_uart( + api: IntegrationTestAPI, local: str, dest: str, timeout: float +) -> bool: + """Enqueue ``local`` for uplink to ``dest`` and poll the uplinker to IDLE. + + The GDS uplinker deletes its source file once the transfer completes (it + expects a staging copy), so enqueue a sacrificial temp copy of ``local``. + """ + with tempfile.NamedTemporaryFile(suffix=".bin", delete=False) as tmp: + staged = tmp.name + shutil.copyfile(local, staged) + uplinker = api.pipeline.files.uplinker + uplinker.enqueue(staged, dest) + deadline = time.time() + timeout + time.sleep(1) + while time.time() < deadline and uplinker.state != FileStates.IDLE: + time.sleep(1) + return uplinker.state == FileStates.IDLE + + +def _uplink_lora_until_crc( + api: IntegrationTestAPI, + local: str, + dest: str, + expected_crc: int, + timeout: float, + attempts: int, +) -> bool: + """Re-uplink ``local`` to the same ``dest`` until the on-board CRC matches. + + Offset writes are idempotent (no truncate), so re-sending the whole file over + a lossy radio link repairs any DATA frames dropped over the air. Returns True + once CalculateCrc reports a match, False if all attempts are exhausted. + """ + for attempt in range(attempts): + # Half-duplex: the flight's own TM transmissions blind its receiver, so + # uplink into a silent board (commanding works with TX disabled) and only + # re-enable the downlink to read the CRC back. + api.send_command("ReferenceDeployment.lora.TRANSMIT", ["DISABLED"]) + time.sleep(5) + _uplink_uart(api, local, dest, timeout) + api.send_command("ReferenceDeployment.downlinkDelay.DIVIDER_PRM_SET", [20]) + time.sleep(2) + api.send_command("ReferenceDeployment.lora.TRANSMIT", ["ENABLED"]) + time.sleep(10) + # A 700KB on-board CRC plus a divider-paced radio downlink far exceeds + # the UART-tuned default timeout; retry the command itself as well since + # a single command frame can be lost over the air. + actual = None + for _ in range(3): + actual = _onboard_crc(api, dest, timeout=240) + if actual is not None: + break + if actual == expected_crc: + return True + print( + f"[ota] lora uplink attempt {attempt + 1}/{attempts}: " + f"crc mismatch (expected 0x{expected_crc:08x}, got " + f"{'None' if actual is None else f'0x{actual:08x}'}) — re-uplinking" + ) + return False + + +def _uplink_and_verify_crc( + api: IntegrationTestAPI, transport: str, local: str, dest: str +) -> int: + """Uplink ``local`` to ``dest`` for the given transport and assert on-board CRC. + + Returns the verified CRC32 (== local CRC) for use with UPDATE_IMAGE_FROM. + """ + expected = _local_crc(local) + if transport == "lora": + ok = _uplink_lora_until_crc( + api, local, dest, expected, LORA_UPLINK_TIMEOUT_S, LORA_UPLINK_ATTEMPTS + ) + assert ok, f"LoRa uplink of {local} never matched CRC 0x{expected:08x}" + else: + idle = _uplink_uart(api, local, dest, UART_UPLINK_TIMEOUT_S) + assert idle, f"UART uplinker did not return to IDLE for {dest}" + actual = _onboard_crc(api, dest) + actual_str = "None" if actual is None else f"0x{actual:08x}" + assert actual == expected, ( + f"on-board CRC {actual_str} != expected 0x{expected:08x} for {dest}" + ) + return expected + + +def _resync_sequence_number( + api: IntegrationTestAPI, request: pytest.FixtureRequest +) -> None: + """Re-read the flight sequence number and rewrite the GDS framing file. + + Mirrors sync_sequence_number_test: after a reboot the flight-side counter has + advanced past whatever the GDS framing plugin last persisted, so the next few + uplink commands would be rejected as replays until we resync. + """ + link = request.config.getoption("--sync-deframer", default=None) + if link is None: + link = ( + "lora" + if request.config.getoption("--with-radio", default=False) + else "uart" + ) + deframer = { + "uart": "ComCcsdsUart.tcSecurityDeframer", + "lora": "ComCcsdsLora.tcSecurityDeframer", + }[link] + proves_send_and_assert_command(api, f"{deframer}.GET_SEQ_NUM") + evt = api.assert_event(f"{deframer}.SequenceNumberGet", timeout=5) + seq_num = evt.args[0].val + with open("./Framing/src/sequence_number.bin", "w", encoding="utf-8") as f: + f.write(str(seq_num)) + + +def _reenable_radio_after_boot(api: IntegrationTestAPI, transport: str) -> None: + """Blindly re-enable the flight LoRa downlink after a reboot (lora only). + + TRANSMIT resets to DISABLED on every flight reset, so post-reboot boot events + never downlink over the radio until we re-enable. The uplink direction works + with TX disabled, so these fire-and-forget sends get through; the divider must + be set while TX is still DISABLED (parameters latch on enable). + """ + if transport != "lora": + return + time.sleep(10) # let the board finish booting before commanding + for _ in range(3): + api.send_command("ReferenceDeployment.downlinkDelay.DIVIDER_PRM_SET", [20]) + time.sleep(2) + api.send_command("ReferenceDeployment.lora.TRANSMIT", ["ENABLED"]) + time.sleep(8) + + +def _configure_repeater_for_uart(api: IntegrationTestAPI, transport: str) -> None: + """Route file downlink to the UART channel only (uart transport). + + downlinkRepeater.CHANNEL_ENABLED is [uart, lora, sband] and RAM-only: it + resets on every reboot, and this test reboots the board twice. If the LoRa + channel is left enabled during a UART run, file downlink buffers queue on + the (slow or dead) radio path and the transfer throttles to the slowest + consumer -- a few-minute uplink becomes tens of minutes and reads as a rig + failure. Re-apply after every reboot, not just at test start. + + lora transport is left untouched: its channel setup is handled by + _reenable_radio_after_boot and the board's saved parameters. + """ + if transport != "uart": + return + # The GDS command layer takes array arguments as a JSON string, not a list. + api.send_command( + "ReferenceDeployment.downlinkRepeater.CHANNEL_ENABLED_PRM_SET", + ['["ENABLED", "DISABLED", "DISABLED"]'], + ) + time.sleep(1) + + +def _cold_reset(api: IntegrationTestAPI) -> TimeType: + """Issue COLD_RESET without expecting an OK response and return the send time. + + The board reboots before the command can complete, so the command dispatcher + reports EXECUTION_ERROR — exactly as reset_manager_test relies on. We assert + the restart by looking for the boot-time version events instead. + """ + start = TimeType().set_datetime( + datetime.now(), time_base=TimeType.TimeBase("TB_DONT_CARE") + ) + api.send_command(f"{RESET_MANAGER}.COLD_RESET") + return start + + +def _project_version_after_boot( + api: IntegrationTestAPI, + start: TimeType, + request: pytest.FixtureRequest, + timeout: float = BOOT_TIMEOUT_S, + transport: str = "uart", +) -> str: + """Wait for the boot to complete and return the reported project version string. + + The Version component emits FrameworkVersion/ProjectVersion at startup (see + reset_manager_test, which keys restart detection off FrameworkVersion). We wait + for FrameworkVersion to confirm the reboot, then read ProjectVersion — falling + back to an explicit VERSION[PROJECT] command if the boot-time event was missed. + The fallback must resync the auth sequence number first: commands sent before + resyncing after a reboot are silently rejected by the security deframer. + """ + if transport == "lora": + # Boot-time events are emitted while LoRa TX is still DISABLED (the + # default after every reset) and are lost over the air; go straight to + # the commanded fallback below. + evt = None + else: + api.assert_event(f"{VERSION}.FrameworkVersion", start=start, timeout=timeout) + evt = api.await_event(f"{VERSION}.ProjectVersion", timeout=5) + if evt is None: + _resync_sequence_number(api, request) + for _ in range(3): + api.send_command(f"{VERSION}.VERSION", ["PROJECT"]) + evt = api.await_event(f"{VERSION}.ProjectVersion", timeout=10) + if evt is not None: + break + _resync_sequence_number(api, request) + assert evt is not None, "no ProjectVersion event after reboot" + return str(evt.args[0].val) + + +@pytest.mark.parametrize( + "transport", + [ + pytest.param("uart", id="uart"), + pytest.param("lora", id="lora", marks=pytest.mark.slow), + ], +) +def test_ota_swap_and_revert( + fprime_test_api: IntegrationTestAPI, + start_gds, + request: pytest.FixtureRequest, + ota_config, + transport: str, +): + """OTA image swap in TEST mode boots the new image, then auto-reverts. + + Uplinks --ota-image, verifies it on board, stages it as the TEST next-boot, + reboots and asserts the new build-id is running, then (without CONFIRM_UPDATE) + reboots again and asserts MCUBoot has reverted to the previous image. + """ + with_radio = request.config.getoption("--with-radio", default=False) + if transport == "lora" and not with_radio: + pytest.skip("lora transport requires --with-radio") + if transport == "uart" and with_radio: + pytest.skip( + "uart transport is skipped on the radio path (use --with-radio for lora)" + ) + + image, build_id = ota_config + api = fprime_test_api + # Unique 8.3-friendly destination (GRC FATFS is 8.3 only); include the pid so + # reruns do not collide with a stale file. + dest = f"/ota{os.getpid() % 100000}.bin" + + try: + # (a)/(b) Uplink the signed image and verify the on-board CRC. + _configure_repeater_for_uart(api, transport) + crc = _uplink_and_verify_crc(api, transport, image, dest) + + # (c) Stage the update: prepare -> write image -> configure TEST next boot. + api.clear_histories() + proves_send_and_assert_command(api, f"{UPDATER}.PREPARE_UPDATE") + api.assert_event(f"{UPDATER}.PrepareUpdateSucceeded", timeout=PREPARE_TIMEOUT_S) + + api.clear_histories() + proves_send_and_assert_command( + api, f"{UPDATER}.UPDATE_IMAGE_FROM", args=[dest, str(crc)] + ) + api.assert_event(f"{UPDATER}.UpdateSucceeded", timeout=UPDATE_TIMEOUT_S) + + proves_send_and_assert_command( + api, f"{UPDATER}.CONFIGURE_NEXT_BOOT", args=["TEST"] + ) + api.assert_event(f"{UPDATER}.SetNextBoot", timeout=10) + + # (d) Reboot into the TEST image and assert the new build is running. + start = _cold_reset(api) + _reenable_radio_after_boot(api, transport) + version = _project_version_after_boot(api, start, request, transport=transport) + _resync_sequence_number(api, request) + # Repeater state is RAM-only and just got wiped by the reboot; commands + # are seq-gated, so re-apply only after the sequence resync above -- and + # resync AGAIN afterwards: an extra authenticated send between a resync + # and the next COLD_RESET desyncs the framing plugin's persisted counter + # (observed as SequenceNumberInvalid rejecting the reset, 2/2 runs). + _configure_repeater_for_uart(api, transport) + _resync_sequence_number(api, request) + assert build_id in version, ( + f"booted project version {version!r} does not contain build id " + f"{build_id!r} — TEST image did not take" + ) + + # (e) Do NOT CONFIRM_UPDATE. Reboot again; MCUBoot must auto-revert. + start = _cold_reset(api) + _reenable_radio_after_boot(api, transport) + reverted = _project_version_after_boot(api, start, request, transport=transport) + _resync_sequence_number(api, request) + _configure_repeater_for_uart(api, transport) + _resync_sequence_number(api, request) + assert build_id not in reverted, ( + f"project version {reverted!r} still contains build id {build_id!r} " + f"after second reboot — MCUBoot did not auto-revert" + ) + finally: + # (f) Teardown backstop: force a revert if we bailed mid-TEST, and delete + # the uploaded image. Best-effort — never mask the real failure. + try: + api.send_command(f"{RESET_MANAGER}.COLD_RESET") + api.await_event(f"{VERSION}.FrameworkVersion", timeout=BOOT_TIMEOUT_S) + _resync_sequence_number(api, request) + except Exception: + pass + try: + api.send_command(f"{FILE_MANAGER}.RemoveFile", [dest, "true"]) + except Exception: + pass + + +def test_ota_negative_paths( + fprime_test_api: IntegrationTestAPI, + start_gds, + ota_config, +): + """Failure-mode coverage that does NOT reboot the board. + + * UPDATE_IMAGE_FROM without a prior PREPARE_UPDATE -> FlashWorker.NoImagePrepared + (the no-prepare gate lives in the worker's updateImage handler, not in + CONFIGURE_NEXT_BOOT, which is an unconditional boot_request_upgrade). + * UPDATE_IMAGE_FROM with a wrong CRC -> FlashWorker.ImageFileCrcMismatch. + + Both worker faults surface as (warning) events; the async commands themselves + dispatch successfully, so we assert on the events rather than a command error. + The no-prepare case must run FIRST, before this test issues any PREPARE_UPDATE. + """ + image, _build_id = ota_config + _configure_repeater_for_uart(fprime_test_api, "uart") + api = fprime_test_api + dest = f"/otaneg{os.getpid() % 100000}.bin" + + try: + # Upload a good image so UPDATE_IMAGE_FROM has a real file to point at. + idle = _uplink_uart(api, image, dest, UART_UPLINK_TIMEOUT_S) + assert idle, f"UART uplinker did not return to IDLE for {dest}" + good_crc = _onboard_crc(api, dest) + assert good_crc is not None, "could not read on-board CRC of uploaded image" + wrong_crc = (good_crc ^ 0xFFFFFFFF) & 0xFFFFFFFF + + # UPDATE_IMAGE_FROM with no prior PREPARE_UPDATE -> NoImagePrepared. + api.clear_histories() + api.send_command(f"{UPDATER}.UPDATE_IMAGE_FROM", [dest, str(good_crc)]) + evt = api.await_event(f"{WORKER}.NoImagePrepared", timeout=30) + assert evt is not None, ( + "expected NoImagePrepared for UPDATE_IMAGE_FROM without PREPARE_UPDATE" + ) + + api.clear_histories() + proves_send_and_assert_command(api, f"{UPDATER}.PREPARE_UPDATE") + api.assert_event(f"{UPDATER}.PrepareUpdateSucceeded", timeout=PREPARE_TIMEOUT_S) + + api.clear_histories() + api.send_command(f"{UPDATER}.UPDATE_IMAGE_FROM", [dest, str(wrong_crc)]) + # FlashWorker rejects the image on CRC mismatch (warning event). + evt = api.await_event( + f"{WORKER}.ImageFileCrcMismatch", timeout=UPDATE_TIMEOUT_S + ) + assert evt is not None, "expected ImageFileCrcMismatch on wrong-CRC update" + finally: + try: + api.send_command(f"{FILE_MANAGER}.RemoveFile", [dest, "true"]) + except Exception: + pass diff --git a/lib/fprime b/lib/fprime index 8a62e455..03c64512 160000 --- a/lib/fprime +++ b/lib/fprime @@ -1 +1 @@ -Subproject commit 8a62e455a90b6d4f498c332d45d65a2a819988d8 +Subproject commit 03c6451237d4442248703947693c1985a1948e88 diff --git a/pytest.ini b/pytest.ini index 97b1fc54..adae094b 100644 --- a/pytest.ini +++ b/pytest.ini @@ -8,7 +8,8 @@ markers = requires_antenna: marks tests that require the antenna board to be plugged in and the burnwire capacitor installed; skip on a bare flight controller requires_battery: marks tests that require the battery board connected with power flowing from the battery terminals; skip on a bare flight controller requires_watchdog_jumper: marks tests that require the JP6 watchdog jumper to be bridged so the watchdog can reset the MCU; skip when JP6 is open - slow: marks tests that take tens of seconds because they wait on a reboot or a timeout expiring + ota: marks the over-the-air image-swap tests (ota_update_test.py); require --ota-image and --ota-build-id + slow: marks long-running tests -- tens of seconds waiting on a reboot/timeout, up to many minutes for the LoRa OTA image uplink which transfers a full signed image over the radio filterwarnings = ignore::DeprecationWarning:yamcs\..* ignore::DeprecationWarning:google\.protobuf\..*