From a7b34ce4f2eaea011e415d3dbb876a18f9f2b4b4 Mon Sep 17 00:00:00 2001 From: Michael Pham <61564344+Mikefly123@users.noreply.github.com> Date: Fri, 10 Jul 2026 22:23:59 -0700 Subject: [PATCH] Svc: drop periodic Sched/Ping ticks on queue-full instead of asserting Adds the drop queue-full annotation to two classes of periodic async input ports across Svc, where none currently exists: 1. The Svc.Sched async input on active components that receive a rate- group tick but did not have drop set: CmdSequencer.schedIn, CmdDispatcher.run, TlmChan.Run, TlmPacketizer.Run, FileDownlink.Run, BufferLogger.schedIn, DpManager.schedIn, DpWriter.schedIn. 2. PingIn/pingIn async ports across Svc, including ActiveRateGroup.PingIn and FpySequencer.pingIn (the latter previously priority 10 assert, with an open TODO questioning priority/behavior), plus two ports added upstream since this defect class was first scoped: BufferAccumulator.pingIn and FileDispatcher.pingIn. This is the same defect class fixed for ComAggregator (see companion PR): any active component fed periodic ticks by a rate group can, if its own dispatch thread stalls longer than queue_depth / tick_rate, accumulate enough queued ticks to trip the autocoded FW_ASSERT on queue-full -- turning a transient stall into an unrecoverable FATAL/reboot. Captured on hardware (PROVES CubeSat, RP2350/Zephyr) during HWIL soak testing, in two independently reproduced instances: - CmdSequencer::schedIn_handlerBase (10 Hz sched tick) hit the identical QUEUE_FULL assert as the ComAggregator case, while a sequencer's dispatch thread was stalled. - ActiveRateGroup::PingIn_handlerBase hit the same assert via Svc::Health's 1 Hz ping -- the health-check mechanism itself killed the board. Health's ping-timeout policy exists specifically to detect an unresponsive component and react gracefully; asserting on the ping enqueue short-circuits that design. Both classes of tick (rate-group sched, and ping) are periodic and idempotent by construction -- a dropped tick is simply retried on the next cycle -- so drop is behaviorally safe. Upstream already uses drop for exactly this reason on ComQueue.run and ActiveRateGroup.CycleIn; this extends the same reasoning to the remaining periodic producers that were missed. Related to issue #4195 ("Add a 'Drop But Warn' on Queue Full"), which is in the same design space but currently leans assert-by-default; this change is narrower in scope (periodic/idempotent producers only, per-port opt-in via the existing drop annotation, no new mechanism). This is an fpp-only annotation change consumed by the autocoder; no new C++ logic. A reviewer sweep of existing per-component UTs that assert on QUEUE_FULL behavior for these ports is recommended before merge. Generative AI (Claude, Anthropic) was used to help root-cause this defect class during HWIL debugging (two independently captured hardware instances of the same assert) and to enumerate affected ports; disclosed per AI_POLICY.md in the upstream pull request description. --- Svc/ActiveRateGroup/ActiveRateGroup.fpp | 2 +- Svc/BufferLogger/BufferLogger.fpp | 4 ++-- Svc/CmdDispatcher/CmdDispatcher.fpp | 4 ++-- Svc/CmdSequencer/CmdSequencer.fpp | 4 ++-- Svc/ComLogger/ComLogger.fpp | 2 +- Svc/DpCatalog/DpCatalog.fpp | 2 +- Svc/DpManager/DpManager.fpp | 2 +- Svc/DpWriter/DpWriter.fpp | 2 +- Svc/EventManager/EventManager.fpp | 2 +- Svc/FileDownlink/FileDownlink.fpp | 4 ++-- Svc/FileManager/FileManager.fpp | 2 +- Svc/FileUplink/FileUplink.fpp | 2 +- Svc/FpySequencer/FpySequencer.fpp | 2 +- Svc/PrmDb/PrmDb.fpp | 2 +- Svc/TlmChan/TlmChan.fpp | 4 ++-- Svc/TlmPacketizer/TlmPacketizer.fpp | 4 ++-- 16 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Svc/ActiveRateGroup/ActiveRateGroup.fpp b/Svc/ActiveRateGroup/ActiveRateGroup.fpp index 3ee1488e375..c60ad6d0d1e 100644 --- a/Svc/ActiveRateGroup/ActiveRateGroup.fpp +++ b/Svc/ActiveRateGroup/ActiveRateGroup.fpp @@ -15,7 +15,7 @@ module Svc { output port RateGroupMemberOut: [ActiveRateGroupOutputPorts] Sched @ Ping input port for health - async input port PingIn: Ping + async input port PingIn: Ping drop @ Ping output port for health output port PingOut: Ping diff --git a/Svc/BufferLogger/BufferLogger.fpp b/Svc/BufferLogger/BufferLogger.fpp index 6da2ffd6930..e045b185625 100644 --- a/Svc/BufferLogger/BufferLogger.fpp +++ b/Svc/BufferLogger/BufferLogger.fpp @@ -16,12 +16,12 @@ module Svc { async input port comIn: Fw.Com @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping - async input port schedIn: Svc.Sched + async input port schedIn: Svc.Sched drop # ---------------------------------------------------------------------- # Special ports diff --git a/Svc/CmdDispatcher/CmdDispatcher.fpp b/Svc/CmdDispatcher/CmdDispatcher.fpp index 66343b886e8..2f77328966e 100644 --- a/Svc/CmdDispatcher/CmdDispatcher.fpp +++ b/Svc/CmdDispatcher/CmdDispatcher.fpp @@ -24,10 +24,10 @@ module Svc { async input port seqCmdBuff: [CmdDispatcherSequencePorts] Fw.Com hook @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Run port used to emit telemetry - async input port run: Svc.Sched + async input port run: Svc.Sched drop @ Ping output port output port pingOut: Svc.Ping diff --git a/Svc/CmdSequencer/CmdSequencer.fpp b/Svc/CmdSequencer/CmdSequencer.fpp index f1c4012784f..ebc38165324 100644 --- a/Svc/CmdSequencer/CmdSequencer.fpp +++ b/Svc/CmdSequencer/CmdSequencer.fpp @@ -68,7 +68,7 @@ module Svc { async input port cmdResponseIn: Fw.CmdResponse @ Ping in port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping out port output port pingOut: Svc.Ping @@ -86,7 +86,7 @@ module Svc { output port comCmdOut: Fw.Com @ Schedule in port - async input port schedIn: Svc.Sched + async input port schedIn: Svc.Sched drop @ Notifies that a sequence has started running output port seqStartOut: Svc.CmdSeqIn diff --git a/Svc/ComLogger/ComLogger.fpp b/Svc/ComLogger/ComLogger.fpp index c4327b7ddb7..cf4bc223871 100644 --- a/Svc/ComLogger/ComLogger.fpp +++ b/Svc/ComLogger/ComLogger.fpp @@ -11,7 +11,7 @@ module Svc { async input port comIn: Fw.Com @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping diff --git a/Svc/DpCatalog/DpCatalog.fpp b/Svc/DpCatalog/DpCatalog.fpp index e1fd468dd2a..59ffc363f75 100644 --- a/Svc/DpCatalog/DpCatalog.fpp +++ b/Svc/DpCatalog/DpCatalog.fpp @@ -32,7 +32,7 @@ module Svc { # Component specific ports @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping diff --git a/Svc/DpManager/DpManager.fpp b/Svc/DpManager/DpManager.fpp index ebbf5a46fb4..74574f97f36 100644 --- a/Svc/DpManager/DpManager.fpp +++ b/Svc/DpManager/DpManager.fpp @@ -8,7 +8,7 @@ module Svc { # ---------------------------------------------------------------------- @ Schedule in port - async input port schedIn: Svc.Sched + async input port schedIn: Svc.Sched drop # ---------------------------------------------------------------------- # Ports for handling buffer requests diff --git a/Svc/DpWriter/DpWriter.fpp b/Svc/DpWriter/DpWriter.fpp index f24594f0925..152be91baf4 100644 --- a/Svc/DpWriter/DpWriter.fpp +++ b/Svc/DpWriter/DpWriter.fpp @@ -8,7 +8,7 @@ module Svc { # ---------------------------------------------------------------------- @ Schedule in port - async input port schedIn: Svc.Sched + async input port schedIn: Svc.Sched drop # ---------------------------------------------------------------------- # Ports for handling data products diff --git a/Svc/EventManager/EventManager.fpp b/Svc/EventManager/EventManager.fpp index b97a8d807e3..2ea88960ae9 100644 --- a/Svc/EventManager/EventManager.fpp +++ b/Svc/EventManager/EventManager.fpp @@ -53,7 +53,7 @@ module Svc { output port FatalAnnounce: Svc.FatalEvent @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping diff --git a/Svc/FileDownlink/FileDownlink.fpp b/Svc/FileDownlink/FileDownlink.fpp index eed9376aa41..5e710a3f41f 100644 --- a/Svc/FileDownlink/FileDownlink.fpp +++ b/Svc/FileDownlink/FileDownlink.fpp @@ -8,7 +8,7 @@ module Svc { # ---------------------------------------------------------------------- @ Run input port - async input port Run: Svc.Sched + async input port Run: Svc.Sched drop @ Mutexed Sendfile input port guarded input port SendFile: Svc.SendFileRequest @@ -23,7 +23,7 @@ module Svc { output port bufferSendOut: Fw.BufferSend @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping diff --git a/Svc/FileManager/FileManager.fpp b/Svc/FileManager/FileManager.fpp index bef777c7133..d85f42002f7 100644 --- a/Svc/FileManager/FileManager.fpp +++ b/Svc/FileManager/FileManager.fpp @@ -8,7 +8,7 @@ module Svc { # ---------------------------------------------------------------------- @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Scheduler input port for rate group operations sync input port schedIn: Sched diff --git a/Svc/FileUplink/FileUplink.fpp b/Svc/FileUplink/FileUplink.fpp index 90ef4e9a821..b2b67bd5c39 100644 --- a/Svc/FileUplink/FileUplink.fpp +++ b/Svc/FileUplink/FileUplink.fpp @@ -14,7 +14,7 @@ module Svc { output port bufferSendOut: Fw.BufferSend @ Ping in - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping out output port pingOut: Svc.Ping diff --git a/Svc/FpySequencer/FpySequencer.fpp b/Svc/FpySequencer/FpySequencer.fpp index 8af40d3d7ce..b9e80e1af03 100644 --- a/Svc/FpySequencer/FpySequencer.fpp +++ b/Svc/FpySequencer/FpySequencer.fpp @@ -38,7 +38,7 @@ module Svc { @ Ping in port # TODO should ping have highest prio? or lowest? - async input port pingIn: Svc.Ping priority 10 assert + async input port pingIn: Svc.Ping priority 10 drop @ port to trigger a wakeup or timeout check. increase frequency @ to increase temporal resolution of sequencer diff --git a/Svc/PrmDb/PrmDb.fpp b/Svc/PrmDb/PrmDb.fpp index bca4f172faa..b2a4b95b1e1 100644 --- a/Svc/PrmDb/PrmDb.fpp +++ b/Svc/PrmDb/PrmDb.fpp @@ -69,7 +69,7 @@ module Svc { async input port setPrm: Fw.PrmSet @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping diff --git a/Svc/TlmChan/TlmChan.fpp b/Svc/TlmChan/TlmChan.fpp index d15072ed08a..52b73b2d818 100644 --- a/Svc/TlmChan/TlmChan.fpp +++ b/Svc/TlmChan/TlmChan.fpp @@ -10,13 +10,13 @@ module Svc { guarded input port TlmGet: Fw.TlmGet @ Run port for starting packet send cycle - async input port Run: Svc.Sched + async input port Run: Svc.Sched drop @ Packet send port output port PktSend: Fw.Com @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping diff --git a/Svc/TlmPacketizer/TlmPacketizer.fpp b/Svc/TlmPacketizer/TlmPacketizer.fpp index 1655c3e2eb7..4125fdac9ca 100644 --- a/Svc/TlmPacketizer/TlmPacketizer.fpp +++ b/Svc/TlmPacketizer/TlmPacketizer.fpp @@ -28,13 +28,13 @@ module Svc { async input port controlIn: EnableSection @ Ping input port - async input port pingIn: Svc.Ping + async input port pingIn: Svc.Ping drop @ Ping output port output port pingOut: Svc.Ping @ Run port for starting packet send cycle - async input port Run: Svc.Sched + async input port Run: Svc.Sched drop @ Input configuration port async input port configureSectionGroupRate: ConfigureGroupRate