feat(sband) 3/3: topology re-enable — HWIL-validated, gated on fault-injection + 24h soak

.pre-commit-config.yaml

@@ -1,3 +1,8 @@
+# patches/*.patch are byte-exact submodule patches (see patches/README.md):
+# trailing whitespace in diff context lines is significant and any rewrite
+# breaks `git apply`. No hook may touch them.
+exclude: ^patches/.*\.patch$
+
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v5.0.0

Makefile

@@ -21,6 +21,28 @@ submodules: ## Initialize and update git submodules
 			echo "❌ Error: Unable to apply patch. Run 'cd lib/fprime && git status' to check."; \
 			exit 1; \
 		fi
+	@echo "Applying ComQueue re-prime tolerance patch (see patches/README.md)..."
+	@cd lib/fprime && \
+		if git apply --check ../../patches/fprime-comqueue-reprime-tolerance.patch 2>/dev/null; then \
+			git apply ../../patches/fprime-comqueue-reprime-tolerance.patch && \
+			echo "✓ Applied ComQueue re-prime tolerance patch"; \
+		elif git apply --reverse --check ../../patches/fprime-comqueue-reprime-tolerance.patch 2>/dev/null; then \
+			echo "⚠ Patch already applied"; \
+		else \
+			echo "❌ Error: Unable to apply ComQueue patch. Run 'cd lib/fprime && git status' to check."; \
+			exit 1; \
+		fi
+	@echo "Applying CDC-ACM poll-mode TX drain patch (see patches/README.md)..."
+	@cd lib/zephyr-workspace/zephyr && \
+		if git apply --check ../../../patches/zephyr-cdc-acm-pollmode-enable-drain.patch 2>/dev/null; then \
+			git apply ../../../patches/zephyr-cdc-acm-pollmode-enable-drain.patch && \
+			echo "✓ Applied CDC-ACM poll-mode drain patch"; \
+		elif git apply --reverse --check ../../../patches/zephyr-cdc-acm-pollmode-enable-drain.patch 2>/dev/null; then \
+			echo "⚠ Patch already applied"; \
+		else \
+			echo "❌ Error: Unable to apply CDC-ACM patch. Run 'cd lib/zephyr-workspace/zephyr && git status' to check."; \
+			exit 1; \
+		fi
 
 export VIRTUAL_ENV ?= $(shell pwd)/fprime-venv
 .PHONY: fprime-venv

PROVESFlightControllerReference/ComCcsdsSband/ComCcsds.fpp

@@ -4,7 +4,7 @@ module ComCcsdsSband {
     # Active Components
     # ----------------------------------------------------------------------
     instance comQueue: Svc.ComQueue base id ComCcsdsConfig.BASE_ID_SBAND + 0x00000 \
-        queue size ComCcsdsConfig.QueueSizes.comQueue \
+        queue size ComCcsdsConfig.Sband.QueueSizes.comQueue \
         stack size ComCcsdsConfig.StackSizes.comQueue \
         priority ComCcsdsConfig.Priorities.comQueue \
     {
@@ -13,15 +13,15 @@ module ComCcsdsSband {
         Svc::ComQueue::QueueConfigurationTable configurationTableSband;
 
         // Events (highest-priority)
-        configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::EVENTS].depth = ComCcsdsConfig::QueueDepths::events;
+        configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::EVENTS].depth = ComCcsdsConfig::Sband::QueueDepths::events;
         configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::EVENTS].priority = ComCcsdsConfig::QueuePriorities::events;
 
         // Telemetry
-        configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::TELEMETRY].depth = ComCcsdsConfig::QueueDepths::tlm;
+        configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::TELEMETRY].depth = ComCcsdsConfig::Sband::QueueDepths::tlm;
         configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::TELEMETRY].priority = ComCcsdsConfig::QueuePriorities::tlm;
 
         // File Downlink Queue (buffer queue using NUM_CONSTANTS offset)
-        configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::NUM_CONSTANTS + ComCcsds::Ports_ComBufferQueue::FILE].depth = ComCcsdsConfig::QueueDepths::file;
+        configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::NUM_CONSTANTS + ComCcsds::Ports_ComBufferQueue::FILE].depth = ComCcsdsConfig::Sband::QueueDepths::file;
         configurationTableSband.entries[ComCcsds::Ports_ComPacketQueue::NUM_CONSTANTS + ComCcsds::Ports_ComBufferQueue::FILE].priority = ComCcsdsConfig::QueuePriorities::file;
 
         // Allocation identifier is 0 as the MallocAllocator discards it
@@ -64,9 +64,9 @@ module ComCcsdsSband {
         phase Fpp.ToCpp.Phases.configComponents """
         memset(&ConfigObjects::ComCcsdsSband_commsBufferManager::bins, 0, sizeof(ConfigObjects::ComCcsdsSband_commsBufferManager::bins));
         ConfigObjects::ComCcsdsSband_commsBufferManager::bins.bins[0].bufferSize = ComCcsdsConfig::BuffMgr::commsBuffSize;
-        ConfigObjects::ComCcsdsSband_commsBufferManager::bins.bins[0].numBuffers = ComCcsdsConfig::BuffMgr::commsBuffCount;
+        ConfigObjects::ComCcsdsSband_commsBufferManager::bins.bins[0].numBuffers = ComCcsdsConfig::Sband::BuffMgr::commsBuffCount;
         ConfigObjects::ComCcsdsSband_commsBufferManager::bins.bins[1].bufferSize = ComCcsdsConfig::BuffMgr::commsFileBuffSize;
-        ConfigObjects::ComCcsdsSband_commsBufferManager::bins.bins[1].numBuffers = ComCcsdsConfig::BuffMgr::commsFileBuffCount;
+        ConfigObjects::ComCcsdsSband_commsBufferManager::bins.bins[1].numBuffers = ComCcsdsConfig::Sband::BuffMgr::commsFileBuffCount;
         ComCcsdsSband::commsBufferManager.setup(
             ComCcsdsConfig::BuffMgr::commsBuffMgrId,
             0,
@@ -87,7 +87,7 @@ module ComCcsdsSband {
     instance spacePacketDeframer: Svc.Ccsds.SpacePacketDeframer base id ComCcsdsConfig.BASE_ID_SBAND + 0x05000
 
     instance aggregator: Svc.ComAggregator base id ComCcsdsConfig.BASE_ID_SBAND + 0x06000 \
-        queue size ComCcsdsConfig.QueueSizes.aggregator \
+        queue size ComCcsdsConfig.Sband.QueueSizes.aggregator \
         stack size ComCcsdsConfig.StackSizes.aggregator \
         priority ComCcsdsConfig.Priorities.aggregator
 

PROVESFlightControllerReference/ReferenceDeployment/Main.cpp

@@ -22,7 +22,7 @@ const struct device* ina219Sys = DEVICE_DT_GET(DT_NODELABEL(ina219_0));
 const struct device* ina219Sol = DEVICE_DT_GET(DT_NODELABEL(ina219_1));
 const struct device* serial = DEVICE_DT_GET(DT_NODELABEL(cdc_acm_uart0));
 const struct device* lora = DEVICE_DT_GET(DT_NODELABEL(lora0));
-// const struct device* spi0 = DEVICE_DT_GET(DT_NODELABEL(spi0));
+const struct device* spi0 = DEVICE_DT_GET(DT_NODELABEL(spi0));
 const struct device* peripheral_uart = DEVICE_DT_GET(DT_NODELABEL(uart0));
 const struct device* peripheral_uart1 = DEVICE_DT_GET(DT_NODELABEL(uart1));
 const struct device* lsm6dso = DEVICE_DT_GET(DT_NODELABEL(lsm6dso0));
@@ -71,7 +71,7 @@ int main(int argc, char* argv[]) {
 
     // Object for communicating state to the topology
     ReferenceDeployment::TopologyState inputs;
-    // inputs.spi0Device = spi0;
+    inputs.spi0Device = spi0;
 
     // Flight Control Board device bindings
     inputs.ina219SysDevice = ina219Sys;
@@ -129,6 +129,16 @@ int main(int argc, char* argv[]) {
 
     // Setup, cycle, and teardown topology
     ReferenceDeployment::setupTopology(inputs);
+
+    // Re-prime the downlink a few times with settling gaps: the one-shot READY
+    // that primes each ComQueue is dropped when the bring-up event storm
+    // overflows its dispatch queue (see primeDownlinkQueues). The sleeps let
+    // the storm drain between attempts so at least one READY is delivered.
+    for (int i = 0; i < 3; i++) {
+        k_sleep(K_MSEC(1000));
+        ReferenceDeployment::primeDownlinkQueues();
+    }
+
     ReferenceDeployment::startRateGroups();  // Program loop
     ReferenceDeployment::teardownTopology(inputs);
     return 0;

PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi

@@ -25,7 +25,7 @@ telemetry packets ReferenceDeploymentPackets {
     ReferenceDeployment.powerMonitor.TotalPowerGenerated
 
     # Communications Data
-    # ComCcsdsSband.authenticatesband.CurrentSequenceNumber
+    ComCcsdsSband.authenticatesband.CurrentSequenceNumber
     ComCcsdsLora.authenticatelora.CurrentSequenceNumber
     ComCcsdsUart.authenticate.CurrentSequenceNumber
 
@@ -49,8 +49,8 @@ telemetry packets ReferenceDeploymentPackets {
     lora.LastRssi
     lora.LastSnr
     lora.BytesSent
-#    sband.LastRssi
-#    sband.LastSnr
+    sband.LastRssi
+    sband.LastSnr
   }
 
   packet PowerMonitor id 11 group 2 {
@@ -119,8 +119,8 @@ telemetry packets ReferenceDeploymentPackets {
   packet Health id 2 group 5 {
     ComCcsdsLora.comQueue.comQueueDepth
     ComCcsdsLora.commsBufferManager.HiBuffs
-#    ComCcsdsSband.comQueue.comQueueDepth
-#    ComCcsdsSband.commsBufferManager.HiBuffs
+    ComCcsdsSband.comQueue.comQueueDepth
+    ComCcsdsSband.commsBufferManager.HiBuffs
     CdhCore.cmdDisp.CommandsDispatched
     CdhCore.cmdDisp.CommandsDropped
     ReferenceDeployment.rateGroup50Hz.RgMaxTime
@@ -134,12 +134,17 @@ telemetry packets ReferenceDeploymentPackets {
     ReferenceDeployment.stackMonitor.WorstThread
     ReferenceDeployment.stackMonitor.ThreadsBelowThreshold
     ReferenceDeployment.stackMonitor.SampleOverflow
+    # New in PR 3: SBand's fault-policy state (PR 2) -- radio fault state
+    # belongs in Health alongside the other component health/status channels.
+    sband.RadioFaulted
+    sband.ConsecutiveRadioFailures
+    sband.RadioResetCount
   }
 
   packet HealthWarnings id 3 group 5 {
     CdhCore.$health.PingLateWarnings
     ComCcsdsLora.commsBufferManager.NoBuffs
-    #ComCcsdsSband.commsBufferManager.NoBuffs
+    ComCcsdsSband.commsBufferManager.NoBuffs
     ComCcsdsUart.commsBufferManager.NoBuffs
     payloadBufferManager.NoBuffs
     ReferenceDeployment.rateGroup50Hz.RgCycleSlips
@@ -150,7 +155,7 @@ telemetry packets ReferenceDeploymentPackets {
 
   packet HealthAuxiliary id 4 group 5 {
     ComCcsdsLora.comQueue.buffQueueDepth
-#    ComCcsdsSband.comQueue.buffQueueDepth
+    ComCcsdsSband.comQueue.buffQueueDepth
     payloadBufferManager.HiBuffs
     CdhCore.tlmSend.SendLevel
   }
@@ -161,23 +166,23 @@ telemetry packets ReferenceDeploymentPackets {
   }
 
   packet Authenticate id 6 group 5 {
-    #ComCcsdsSband.authenticatesband.AuthenticatedPacketsCount
+    ComCcsdsSband.authenticatesband.AuthenticatedPacketsCount
     ComCcsdsLora.authenticatelora.AuthenticatedPacketsCount
     ComCcsdsUart.authenticate.AuthenticatedPacketsCount
 
-    #ComCcsdsSband.authenticatesband.RejectedPacketsCount
+    ComCcsdsSband.authenticatesband.RejectedPacketsCount
     ComCcsdsLora.authenticatelora.RejectedPacketsCount
     ComCcsdsUart.authenticate.RejectedPacketsCount
 
-    #ComCcsdsSband.authenticationRouter.ByPassedRouter
+    ComCcsdsSband.authenticationRouter.ByPassedRouter
     ComCcsdsLora.authenticationRouter.ByPassedRouter
     ComCcsdsUart.authenticationRouter.ByPassedRouter
 
-    #ComCcsdsSband.authenticationRouter.PassedRouter
+    ComCcsdsSband.authenticationRouter.PassedRouter
     ComCcsdsLora.authenticationRouter.PassedRouter
     ComCcsdsUart.authenticationRouter.PassedRouter
 
-    #ComCcsdsSband.authenticationRouter.FailedRouter
+    ComCcsdsSband.authenticationRouter.FailedRouter
     ComCcsdsLora.authenticationRouter.FailedRouter
     ComCcsdsUart.authenticationRouter.FailedRouter
 
@@ -306,14 +311,14 @@ telemetry packets ReferenceDeploymentPackets {
   ComCcsdsUart.comQueue.buffQueueDepth
 
   ComCcsdsLora.commsBufferManager.EmptyBuffs
-#  ComCcsdsSband.commsBufferManager.EmptyBuffs
+  ComCcsdsSband.commsBufferManager.EmptyBuffs
   ComCcsdsUart.commsBufferManager.EmptyBuffs
   payloadBufferManager.EmptyBuffs
 
   ComCcsdsLora.commsBufferManager.TotalBuffs
   ComCcsdsLora.commsBufferManager.CurrBuffs
-#  ComCcsdsSband.commsBufferManager.TotalBuffs
-#  ComCcsdsSband.commsBufferManager.CurrBuffs
+  ComCcsdsSband.commsBufferManager.TotalBuffs
+  ComCcsdsSband.commsBufferManager.CurrBuffs
 
   payloadBufferManager.TotalBuffs
   payloadBufferManager.CurrBuffs

PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentTopology.cpp

@@ -26,10 +26,18 @@ static const struct gpio_dt_spec face5LoadSwitchGpio = GPIO_DT_SPEC_GET(DT_NODEL
 static const struct gpio_dt_spec payloadPowerLoadSwitchGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(payload_pwr_enable), gpios);
 static const struct gpio_dt_spec payloadBatteryLoadSwitchGpio =
     GPIO_DT_SPEC_GET(DT_NODELABEL(payload_batt_enable), gpios);
-// static const struct gpio_dt_spec sbandNrstGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(sband_nrst), gpios);
-// static const struct gpio_dt_spec sbandRxEnGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(sband_rx_en), gpios);
-// static const struct gpio_dt_spec sbandTxEnGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(sband_tx_en), gpios);
-// static const struct gpio_dt_spec sbandTxEnIRQ = GPIO_DT_SPEC_GET(DT_NODELABEL(rf2_io1), gpios);
+static const struct gpio_dt_spec sbandNrstGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(sband_nrst), gpios);
+static const struct gpio_dt_spec sbandRxEnGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(sband_rx_en), gpios);
+static const struct gpio_dt_spec sbandTxEnGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(sband_tx_en), gpios);
+// rf2_io0/rf2_io1 are the two remaining lines of the generic 4-pin RF header
+// shared by every board variant (see boards/bronco_space's v5.dtsi); this
+// deployment repurposes them as the SX1280's IRQ/BUSY lines. rf2_io1 = IRQ
+// carries forward the mapping this draft already used; rf2_io0 = BUSY is a
+// PR 3 addition for SBand.getBusyLine (added PR 2) and has not been bench-
+// verified against a schematic -- confirm on real hardware in HWIL Slice 3.2
+// (see SBAND-HWIL-PROCEDURE.md) before relying on it operationally.
+static const struct gpio_dt_spec sbandIrqGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(rf2_io1), gpios);
+static const struct gpio_dt_spec sbandBusyGpio = GPIO_DT_SPEC_GET(DT_NODELABEL(rf2_io0), gpios);
 
 // Allows easy reference to objects in FPP/autocoder required namespaces
 using namespace ReferenceDeployment;
@@ -89,10 +97,11 @@ void configureTopology() {
     gpioface5LS.open(face5LoadSwitchGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
     gpioPayloadPowerLS.open(payloadPowerLoadSwitchGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
     gpioPayloadBatteryLS.open(payloadBatteryLoadSwitchGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
-    //    gpioSbandNrst.open(sbandNrstGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
-    //    gpioSbandRxEn.open(sbandRxEnGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
-    //    gpioSbandTxEn.open(sbandTxEnGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
-    //    gpioSbandIRQ.open(sbandTxEnIRQ, Zephyr::ZephyrGpioDriver::GpioConfiguration::IN);
+    gpioSbandNrst.open(sbandNrstGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
+    gpioSbandRxEn.open(sbandRxEnGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
+    gpioSbandTxEn.open(sbandTxEnGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::OUT);
+    gpioSbandIRQ.open(sbandIrqGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::IN);
+    gpioSbandBusy.open(sbandBusyGpio, Zephyr::ZephyrGpioDriver::GpioConfiguration::IN);
 }
 
 // Public functions for use in main program are namespaced with deployment name ReferenceDeployment
@@ -126,21 +135,29 @@ void setupTopology(const TopologyState& state) {
     lora.start(state.loraDevice, Zephyr::TransmitState::DISABLED);
     comDriver.configure(state.uartDevice, state.baudRate);
 
-    // static struct spi_cs_control cs_ctrl = {
-    //     .gpio = GPIO_DT_SPEC_GET_BY_IDX(DT_NODELABEL(spi0), cs_gpios, 1),
-    //     .delay = 0U, /* us to wait after asserting CS before transfer */
-    //     .cs_is_gpio = true,
-    // };
-
-    // struct spi_config cfg = {
-    //     .frequency = 100000,  // 100 KHz -- sx1280 has maximum 18.18 MHz -- there is a 12MHz oscillator on-board
-    //     .operation = SPI_WORD_SET(8),
-    //     .slave = 0,
-    //     .cs = cs_ctrl,
-    //     .word_delay = 0,
-    // };
-    //    spiDriver.configure(state.spi0Device, cfg);
-    //    sband.configureRadio();
+    // S-Band SPI: spi0 is shared with the onboard SD card (cs_gpios index 0);
+    // the SX1280 uses the second chip-select on the same bus (cs_gpios index 1,
+    // gpio0 7 -- see boards/bronco_space's v5.dtsi &spi0 node).
+    static struct spi_cs_control sbandCsCtrl = {
+        .gpio = GPIO_DT_SPEC_GET_BY_IDX(DT_NODELABEL(spi0), cs_gpios, 1),
+        .delay = 0U, /* us to wait after asserting CS before transfer */
+        .cs_is_gpio = true,
+    };
+
+    struct spi_config sbandSpiCfg = {
+        .frequency = 4000000,  // 4 MHz (SX1280 max 18.18 MHz; 100 kHz cost ~25 ms per 252 B buffer
+                               // upload -- see REPORT-sband-goodput.md; the same spi0 bus already
+                               // runs the SD card at 24 MHz)
+        .operation = SPI_WORD_SET(8),
+        .slave = 0,
+        .cs = sbandCsCtrl,
+        .word_delay = 0,
+    };
+    spiDriver.configure(state.spi0Device, sbandSpiCfg);
+    // Discarding the Status here matches SBand.cpp's own (void)configureRadio()
+    // call site; a failed initial configure is not fatal at boot -- SBandFaultPolicy
+    // (PR 2) surfaces RadioNotConfigured/RadioLibFailed events on first use instead.
+    (void)sband.configureRadio();
 
     // UART from the board to the payload
     peripheralUartDriver.configure(state.peripheralUart, state.peripheralBaudRate);
@@ -185,6 +202,22 @@ void setupTopology(const TopologyState& state) {
     fsFormat.configure(state.storagePartitionId);
 }
 
+void primeDownlinkQueues() {
+    // Re-prime the downlink queues. Each ComQueue is primed by a single READY
+    // status (ComAggregator::preamble) delivered over a lossy hop: comStatusIn
+    // is async, and the 20-deep dispatch queue overflows during the bring-up
+    // event storm (on boards with absent sensors, dozens of warning events),
+    // silently dropping priming statuses and deadlocking the downlink forever —
+    // late statuses die in the aggregator's assertNoStatus action, so there is
+    // no recovery path. A redundant READY is benign (see
+    // ComQueue::comStatusIn_handler), so main() calls this several times after
+    // bring-up settles to guarantee delivery.
+    Fw::Success primeReady = Fw::Success::SUCCESS;
+    ComCcsdsUart::comQueue.get_comStatusIn_InputPort(0)->invoke(primeReady);
+    ComCcsdsLora::comQueue.get_comStatusIn_InputPort(0)->invoke(primeReady);
+    ComCcsdsSband::comQueue.get_comStatusIn_InputPort(0)->invoke(primeReady);
+}
+
 void startRateGroups() {
     timer.configure(BASE_RATEGROUP_PERIOD_MS);
     timer.start();

PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentTopology.hpp

@@ -38,6 +38,16 @@ namespace ReferenceDeployment {
  */
 void setupTopology(const TopologyState& state);
 
+/**
+ * \brief re-send the downlink priming status to every ComQueue
+ *
+ * The one-shot READY that primes each ComQueue travels over a lossy async hop
+ * and is silently dropped when the dispatch queue overflows during the
+ * bring-up event storm, permanently deadlocking the downlink. Redundant READY
+ * statuses are benign, so main() calls this several times after bring-up.
+ */
+void primeDownlinkQueues();
+
 /**
  * \brief teardown the F´ topology
  *