From f220cbb3c44a55824492d836d7d5dcc1fe3d6427 Mon Sep 17 00:00:00 2001 From: Nate Gay Date: Mon, 20 Jul 2026 18:09:08 -0500 Subject: [PATCH 1/3] Make mode manager more reliable --- .../test/int/mode_manager_test.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/PROVESFlightControllerReference/test/int/mode_manager_test.py b/PROVESFlightControllerReference/test/int/mode_manager_test.py index cb00ae73..3bb1a72f 100644 --- a/PROVESFlightControllerReference/test/int/mode_manager_test.py +++ b/PROVESFlightControllerReference/test/int/mode_manager_test.py @@ -40,6 +40,7 @@ import pytest from common import proves_send_and_assert_command from fprime_gds.common.data_types.event_data import EventData +from fprime_gds.common.models.serialize.time_type import TimeType from fprime_gds.common.testing_fw.api import IntegrationTestAPI logger = logging.getLogger(__name__) @@ -543,6 +544,7 @@ def test_safe_08_clean_reboot_no_safe_mode( @pytest.mark.slow @pytest.mark.uart_only(reason="Requires reboot and GDS reconnect") +@pytest.mark.requires_watchdog_jumper def test_safe_09_command_loss_triggers_safe_mode_and_reboot( fprime_test_api: IntegrationTestAPI, start_gds ): @@ -594,8 +596,13 @@ def test_safe_09_command_loss_triggers_safe_mode_and_reboot( ) # stopWatchdog was called after safe mode entry — hardware reset expected in ~30 seconds - logger.info("Waiting for hardware reboot triggered by watchdog stop (~60s)...") - time.sleep(60.0) + reboot_start: TimeType = TimeType().set_datetime( + datetime.now(), time_base=TimeType.TimeBase("TB_DONT_CARE") + ) + logger.info("Waiting for hardware reboot triggered by watchdog stop...") + fprime_test_api.assert_event( + "CdhCore.version.FrameworkVersion", start=reboot_start, timeout=90 + ) # Verify reboot occurred final_boot_count = _get_boot_count(fprime_test_api) From b1b4235db5a268c9f2d50f41528c369c562a158e Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:13:15 -0700 Subject: [PATCH 2/3] =?UTF-8?q?test:=20retry=20boot-count=20read=20after?= =?UTF-8?q?=20reboot=20=E2=80=94=20first=201Hz=20tick=20does=20the=20lazy?= =?UTF-8?q?=20increment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FrameworkVersion reboot-detection assert fires during topology startup, before StartupManager's first 1Hz run tick performs the lazy boot-count increment (StartupManager.cpp get_boot_count(true) behind the m_boot_count==0 guard). Querying GET_BOOT_COUNT immediately after reboot detection therefore races the increment and can read the pre-reboot count — the failure seen in run 29894110572 (query at TB_PROC_TIME 4.8s, count still 2). Keep the +1 assertion (it distinguishes exactly-one reboot from a boot loop, which a startup-EVR assert cannot) but retry the read for up to 30s until the increment lands. Co-Authored-By: Claude Fable 5 --- .../test/int/mode_manager_test.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/test/int/mode_manager_test.py b/PROVESFlightControllerReference/test/int/mode_manager_test.py index 3bb1a72f..93298b03 100644 --- a/PROVESFlightControllerReference/test/int/mode_manager_test.py +++ b/PROVESFlightControllerReference/test/int/mode_manager_test.py @@ -604,8 +604,18 @@ def test_safe_09_command_loss_triggers_safe_mode_and_reboot( "CdhCore.version.FrameworkVersion", start=reboot_start, timeout=90 ) - # Verify reboot occurred + # Verify reboot occurred. StartupManager increments the boot count lazily + # on its first 1Hz run tick, and FrameworkVersion is emitted during topology + # startup before rate groups run — so immediately after reboot detection + # GET_BOOT_COUNT can still return the pre-reboot count. Retry briefly until + # the increment lands. Asserting +1 (not just "a startup EVR arrived") also + # catches a boot loop: a watchdog that isn't re-fed on the new boot would + # keep resetting and drive the count past initial + 1. + deadline = time.monotonic() + 30.0 final_boot_count = _get_boot_count(fprime_test_api) + while final_boot_count == initial_boot_count and time.monotonic() < deadline: + time.sleep(2.0) + final_boot_count = _get_boot_count(fprime_test_api) assert final_boot_count == initial_boot_count + 1, ( f"Boot count should increment by 1 after command loss reboot. " f"Before: {initial_boot_count}, After: {final_boot_count}" From 52ed1b86ae92376a762bfdb0204578c34cdc8e0b Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:13:58 -0700 Subject: [PATCH 3/3] test: wait for EnteringSafeMode instead of scraping event history Second flake in test_safe_09, seen on PR #470's CI (run 29983198334): CommandLossDetected asserted fine, but the immediate history scrape ran before EnteringSafeMode (emitted ~40ms later in the same firmware code path) reached the GDS. Use assert_event with a timeout so the check waits for the event instead of racing it. Co-Authored-By: Claude Fable 5 --- .../test/int/mode_manager_test.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/PROVESFlightControllerReference/test/int/mode_manager_test.py b/PROVESFlightControllerReference/test/int/mode_manager_test.py index 93298b03..7f42eb88 100644 --- a/PROVESFlightControllerReference/test/int/mode_manager_test.py +++ b/PROVESFlightControllerReference/test/int/mode_manager_test.py @@ -583,15 +583,13 @@ def test_safe_09_command_loss_triggers_safe_mode_and_reboot( # Wait for the 1Hz run_handler to detect command loss (at most 2 seconds) fprime_test_api.assert_event(f"{component}.CommandLossDetected", timeout=5) - # Verify EnteringSafeMode event mentions loss of contact - events = fprime_test_api.get_event_test_history() - entering_events = [ - e for e in events if "EnteringSafeMode" in str(e.get_template().get_name()) - ] - assert len(entering_events) > 0, ( - "EnteringSafeMode event should be emitted on command loss" + # Verify EnteringSafeMode event mentions loss of contact. The firmware emits it a few + # tens of ms after CommandLossDetected, so wait for it rather than scraping the history + # snapshot (which races the EVR's arrival at the GDS). + entering_event = fprime_test_api.assert_event( + f"{component}.EnteringSafeMode", timeout=5 ) - assert "contact" in entering_events[-1].get_display_text().lower(), ( + assert "contact" in entering_event.get_display_text().lower(), ( "EnteringSafeMode should mention loss of contact" )