From 598d9018dc2fc8a4247633afce30c2abbf3bbb2c Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 21 Jul 2026 15:04:13 -0500 Subject: [PATCH 01/38] initial commit --- .../Components/CMakeLists.txt | 1 + .../Components/TlmArchive/CMakeLists.txt | 36 +++++ .../Components/TlmArchive/TlmArchive.cpp | 140 ++++++++++++++++++ .../Components/TlmArchive/TlmArchive.fpp | 40 +++++ .../Components/TlmArchive/TlmArchive.hpp | 58 ++++++++ .../Components/TlmArchive/docs/sdd.md | 21 +++ .../Top/ReferenceDeploymentPackets.fppi | 2 + .../ReferenceDeployment/Top/instances.fpp | 2 + .../ReferenceDeployment/Top/topology.fpp | 2 + 9 files changed, 302 insertions(+) create mode 100644 PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt create mode 100644 PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp create mode 100644 PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp create mode 100644 PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp create mode 100644 PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md diff --git a/PROVESFlightControllerReference/Components/CMakeLists.txt b/PROVESFlightControllerReference/Components/CMakeLists.txt index 743a451f..8bf54493 100644 --- a/PROVESFlightControllerReference/Components/CMakeLists.txt +++ b/PROVESFlightControllerReference/Components/CMakeLists.txt @@ -26,3 +26,4 @@ add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/ResetManager/") add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/StartupManager/") add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/ThermalManager/") add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/Watchdog") +add_fprime_subdirectory("${CMAKE_CURRENT_LIST_DIR}/TlmArchive/") diff --git a/PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt b/PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt new file mode 100644 index 00000000..52477a89 --- /dev/null +++ b/PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt @@ -0,0 +1,36 @@ +#### +# F Prime CMakeLists.txt: +# +# SOURCES: list of source files (to be compiled) +# AUTOCODER_INPUTS: list of files to be passed to the autocoders +# DEPENDS: list of libraries that this module depends on +# +# More information in the F´ CMake API documentation: +# https://fprime.jpl.nasa.gov/latest/docs/reference/api/cmake/API/ +# +#### + +# Module names are derived from the path from the nearest project/library/framework +# root when not specifically overridden by the developer, i.e. the module defined by +# `MyProj/Some/Path/CMakeLists.txt` will be named `MyProj_Some_Path`. + +register_fprime_library( + AUTOCODER_INPUTS + "${CMAKE_CURRENT_LIST_DIR}/TlmArchive.fpp" + SOURCES + "${CMAKE_CURRENT_LIST_DIR}/TlmArchive.cpp" +# DEPENDS +# MyPackage_MyOtherModule +) + +### Unit Tests ### +# register_fprime_ut( +# AUTOCODER_INPUTS +# "${CMAKE_CURRENT_LIST_DIR}/TlmArchive.fpp" +# SOURCES +# "${CMAKE_CURRENT_LIST_DIR}/test/ut/TlmArchiveTestMain.cpp" +# "${CMAKE_CURRENT_LIST_DIR}/test/ut/TlmArchiveTester.cpp" +# DEPENDS +# STest # For rules-based testing +# UT_AUTO_HELPERS +# ) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp new file mode 100644 index 00000000..a3a64f8e --- /dev/null +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -0,0 +1,140 @@ +// ====================================================================== +// \title TlmArchive.cpp +// \author aychar +// \brief cpp file for TlmArchive component implementation class +// ====================================================================== + +#include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp" + +#include +#include + +#include "Os/Directory.hpp" +#include "Os/FileSystem.hpp" + +namespace Components { + +TlmArchive ::TlmArchive(const char* const compName) : TlmArchiveComponentBase(compName) {} + +TlmArchive ::~TlmArchive() {} + +void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 context) { + (void)portNum; + (void)context; + + Os::ScopeLock lock(this->m_mutex); + if (!this->m_enabled) { + return; + } + + if (this->m_packetCount == MAX_STORED_PACKETS) { + bool status = this->writeRecord(); + if (!status) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry record")); + } + } + + if (this->m_packetCount < MAX_STORED_PACKETS) { + this->m_packetArr[this->m_packetCount] = data; + this->m_packetCount++; + } +} + +void TlmArchive::run_handler(FwIndexType portNum, U32 context) { + (void)portNum; + (void)context; + + Os::ScopeLock lock(this->m_mutex); + if (this->m_enabled && (this->m_packetCount > 0U) && !this->writeRecord()) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry record")); + } +} + +void TlmArchive::RECORDING_STATUS_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, Components::Status status) { + { + Os::ScopeLock lock(this->m_mutex); + this->m_enabled = (status == Components::Status::ENABLED); + this->tlmWrite_RecordingEnabled(this->m_enabled ? Fw::On::ON : Fw::On::OFF); + } + this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); +} + +bool TlmArchive::writeRecord() { + // TODO: Ensure capacity & prune old if at capacity + // TODO: Check boot status for if packet should be permanent + // TODO: Add record to db.txt file + + if (!this->openFile()) { + return false; + } + + FwSizeType fileDataSize = 0; + std::memset(this->m_fileData, 0, sizeof(this->m_fileData)); + + for (U32 i = 0; i < this->m_packetCount; i++) { + const FwSizeType pktSize = this->m_packetArr[i].getSize(); + std::memcpy(&this->m_fileData[fileDataSize], this->m_packetArr[i].getBuffAddr(), + static_cast(pktSize)); + fileDataSize += pktSize; + } + + if (!this->writeToFile(fileDataSize)) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry data to file")); + return false; + } + if (!this->closeFile()) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to close telemetry file")); + return false; + } + + this->m_packetCount = 0; + return true; +} + +bool TlmArchive::openFile() { + if (this->m_fileOpen || !this->ensureDirectories()) { + return false; + } + + char fileName[96U] = {}; + std::snprintf(fileName, sizeof(fileName), "//tlm/tlm_%08u.tlm", this->m_fileCount++); + Os::FileInterface::Status status = this->m_file.open(fileName, Os::File::OPEN_CREATE, Os::File::NO_OVERWRITE); + if (status != Os::File::OP_OK) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to open telemetry file")); + return false; + } + + this->m_fileOpen = true; + return true; +} + +bool TlmArchive::closeFile() { + if (!this->m_fileOpen) { + return false; + } + + this->m_file.close(); + this->m_fileOpen = false; + return true; +} + +bool TlmArchive::writeToFile(FwSizeType fileDataSize) { + if (!this->m_fileOpen) { + return false; + } + + const FwSizeType expectedSize = fileDataSize; + if ((this->m_file.write(this->m_fileData, fileDataSize) != Os::File::OP_OK) || (fileDataSize != expectedSize)) { + this->closeFile(); + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write to telemetry file")); + return false; + } + + return true; +} + +bool TlmArchive::ensureDirectories() { + return Os::FileSystem::createDirectory("//tlm", false) == Os::FileSystem::OP_OK; +} + +} // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp new file mode 100644 index 00000000..07ff5e5c --- /dev/null +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -0,0 +1,40 @@ +module Components { + @ Component for F Prime FSW framework. + enum Status { + ENABLED + DISABLED + } + + passive component TlmArchive { + sync command RECORDING_STATUS(status: Status) + + telemetry RecordingEnabled: Fw.On + + event Debug(message: string) severity activity low format "{}" + + sync input port run: Svc.Sched + sync input port comIn: Fw.Com + + ############################################################################### + # Standard AC Ports: Required for Channels, Events, Commands, and Parameters # + ############################################################################### + @ Port for requesting the current time + time get port timeCaller + + @ Enables command handling + import Fw.Command + + @ Enables event handling + import Fw.Event + + @ Enables telemetry channels handling + import Fw.Channel + + @ Port to return the value of a parameter + param get port prmGetOut + + @Port to set the value of a parameter + param set port prmSetOut + + } +} diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp new file mode 100644 index 00000000..9caec2f7 --- /dev/null +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -0,0 +1,58 @@ +// ====================================================================== +// \title TlmArchive.hpp +// \author aychar +// \brief hpp file for TlmArchive component implementation class +// ====================================================================== + +#ifndef Components_TlmArchive_HPP +#define Components_TlmArchive_HPP + +#include "Os/File.hpp" +#include "Os/Mutex.hpp" +#include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchiveComponentAc.hpp" + +namespace Components { + +class TlmArchive final : public TlmArchiveComponentBase { + public: + // ---------------------------------------------------------------------- + // Component construction and destruction + // ---------------------------------------------------------------------- + + //! Construct TlmArchive object + TlmArchive(const char* const compName //!< The component name + ); + + //! Destroy TlmArchive object + ~TlmArchive(); + + private: + static constexpr U32 MAX_STORED_PACKETS = 32; + static constexpr FwSizeType MAX_FILE_SIZE = 233 * MAX_STORED_PACKETS; + + void comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 context) override; + void run_handler(FwIndexType portNum, U32 context) override; + void RECORDING_STATUS_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, Components::Status status) override; + + bool writeRecord(); + bool openFile(); + bool closeFile(); + bool writeToFile(FwSizeType fileDataSize); + bool ensureDirectories(); + + Os::Mutex m_mutex; + Os::File m_file; + bool m_fileOpen = false; + U8 m_fileData[MAX_FILE_SIZE] = {}; + + bool m_enabled = true; + + U32 m_packetCount = 0; + U32 m_fileCount = 0; + + Fw::ComBuffer m_packetArr[MAX_STORED_PACKETS] = {}; +}; + +} // namespace Components + +#endif diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md new file mode 100644 index 00000000..04f98031 --- /dev/null +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -0,0 +1,21 @@ +# PROVESFlightControllerReference::TlmArchive + +Component for F Prime FSW framework. + +## Introduction + + + +## Requirements + +| Name | Description | Rationale | Validation | +|---|---|---|---| +| | | | | + +## Design + + + +## Configuration + + diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi index 74ceb781..2ce78477 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi @@ -308,4 +308,6 @@ telemetry packets ReferenceDeploymentPackets { ReferenceDeployment.watchdog.WatchdogTransitions + ReferenceDeployment.tlmArchive.RecordingEnabled + } diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/instances.fpp b/PROVESFlightControllerReference/ReferenceDeployment/Top/instances.fpp index c1a541eb..316eb706 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/instances.fpp +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/instances.fpp @@ -242,4 +242,6 @@ module ReferenceDeployment { instance picoTempManager: Drv.PicoTempManager base id 0x10079000 + instance tlmArchive: Components.TlmArchive base id 0x1007A000 + } diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp index 61e9a496..a04645f3 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp @@ -117,6 +117,7 @@ module ReferenceDeployment { instance dropDetector instance picoTempManager + instance tlmArchive # ---------------------------------------------------------------------- # Pattern graph specifiers @@ -150,6 +151,7 @@ module ReferenceDeployment { CdhCore.tlmSend.PktSend -> comSplitterTelemetry.comIn comSplitterTelemetry.comOut -> ComCcsdsLora.comQueue.comPacketQueueIn[ComCcsds.Ports_ComPacketQueue.TELEMETRY] comSplitterTelemetry.comOut -> ComCcsdsUart.comQueue.comPacketQueueIn[ComCcsds.Ports_ComPacketQueue.TELEMETRY] + comSplitterTelemetry.comOut -> tlmArchive.comIn #comSplitterTelemetry.comOut -> ComCcsdsSband.comQueue.comPacketQueueIn[ComCcsds.Ports_ComPacketQueue.TELEMETRY] # Router to Command Dispatcher From 5f38c963aa54fa4346fc097e4708030d24ced505 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 21 Jul 2026 15:17:10 -0500 Subject: [PATCH 02/38] logs --- .../Components/TlmArchive/TlmArchive.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index a3a64f8e..cdf64de7 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -27,11 +27,15 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con return; } + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Got a packet from comIn")); + if (this->m_packetCount == MAX_STORED_PACKETS) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Writing record.")); bool status = this->writeRecord(); if (!status) { this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry record")); } + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Done writing record.")); } if (this->m_packetCount < MAX_STORED_PACKETS) { From a3080a98a0e4a8b92814cb959ff17fd00057e348 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 21 Jul 2026 15:26:23 -0500 Subject: [PATCH 03/38] file downilnk test --- .github/workflows/ci.yaml | 9 ++ .../test/int/conftest.py | 6 +- .../test/int/tlm_archive_radio_test.py | 119 ++++++++++++++++++ fprime-gds.yml | 1 + pytest.ini | 1 + 5 files changed, 135 insertions(+), 1 deletion(-) create mode 100644 PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 90c9e1d1..9704055b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -498,6 +498,15 @@ jobs: if-no-files-found: ignore retention-days: 14 + - name: Upload downlinked telemetry archives + if: always() + uses: actions/upload-artifact@v4 + with: + name: radio-downlinked-tlm-archives + path: build-artifacts/radio-downlinked-tlm/*.tlm + if-no-files-found: warn + retention-days: 14 + - name: Kill GDS & Power Off Satellite if: always() run: | diff --git a/PROVESFlightControllerReference/test/int/conftest.py b/PROVESFlightControllerReference/test/int/conftest.py index 550a9562..f8d9adb3 100644 --- a/PROVESFlightControllerReference/test/int/conftest.py +++ b/PROVESFlightControllerReference/test/int/conftest.py @@ -59,7 +59,11 @@ def pytest_collection_modifyitems( ) ) - if not config.getoption("--with-radio", default=False): + with_radio = config.getoption("--with-radio", default=False) + if not with_radio: + for item in items: + if item.get_closest_marker("radio_only") is not None: + item.add_marker(pytest.mark.skip(reason="requires --with-radio")) return rtc_items = [i for i in items if "rtc_test" in i.nodeid] diff --git a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py new file mode 100644 index 00000000..ec3a0556 --- /dev/null +++ b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py @@ -0,0 +1,119 @@ +"""Radio integration test for the TlmArchive component.""" + +import os +import time +from pathlib import Path + +import pytest +from common import proves_send_and_assert_command +from fprime_gds.common.testing_fw.api import IntegrationTestAPI + +pytestmark = [pytest.mark.radio_only] + +TLM_ARCHIVE = "ReferenceDeployment.tlmArchive" +TELEMETRY_DELAY = "ReferenceDeployment.telemetryDelay" +FILE_MANAGER = "FileHandling.fileManager" +FILE_DOWNLINK = "FileHandling.fileDownlink" + +DEFAULT_TELEMETRY_DIVIDER = 29 +RECORD_TIMEOUT_S = 180 +FILE_RECEIVE_TIMEOUT_S = 180 + + +def _listed_tlm_files(fprime_test_api: IntegrationTestAPI) -> list[tuple[str, int]]: + """List //tlm and return the telemetry filenames and sizes received as events.""" + # Directory listing entries are events sent separately from the command + # response. Retry the listing when the lossy RF link drops all matching + # entries even though the command response arrived. + for _ in range(3): + proves_send_and_assert_command( + fprime_test_api, + f"{FILE_MANAGER}.ListDirectory", + ["//tlm"], + ) + entries = [] + for event in fprime_test_api.get_event_test_history().retrieve(): + if ( + event.get_template().get_full_name() + != f"{FILE_MANAGER}.DirectoryListing" + ): + continue + directory, filename, size = (arg.val for arg in event.get_args()) + if directory == "//tlm" and filename.endswith(".tlm"): + entries.append((filename, int(size))) + if entries: + return entries + time.sleep(2) + return [] + + +def _await_complete_local_file(path: Path, expected_size: int) -> None: + """Wait until GDS has received and closed the complete downlinked file.""" + deadline = time.monotonic() + FILE_RECEIVE_TIMEOUT_S + while time.monotonic() < deadline: + if path.is_file() and path.stat().st_size == expected_size: + return + time.sleep(1) + actual_size = path.stat().st_size if path.exists() else "missing" + pytest.fail( + f"GDS did not save {path} at the expected size of {expected_size} bytes " + f"(actual: {actual_size})" + ) + + +def test_tlm_archive_downlinks_record_over_radio( + fprime_test_api: IntegrationTestAPI, start_gds +): + """Create a telemetry record, discover it, and downlink it over LoRa.""" + try: + proves_send_and_assert_command( + fprime_test_api, + f"{TLM_ARCHIVE}.RECORDING_STATUS", + ["ENABLED"], + ) + proves_send_and_assert_command( + fprime_test_api, + f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", + [1], + ) + + writing = fprime_test_api.await_event( + f"{TLM_ARCHIVE}.Debug", + args=["Writing record."], + timeout=RECORD_TIMEOUT_S, + ) + assert writing is not None, "TlmArchive did not start writing a record" + + # TlmArchive writes synchronously, but leave time for the filesystem + # close and the associated events to clear the radio link. + time.sleep(5) + + entries = _listed_tlm_files(fprime_test_api) + assert entries, "ListDirectory(//tlm) returned no telemetry archive files" + source_name, expected_size = max(entries, key=lambda entry: entry[0]) + assert expected_size > 0, f"Telemetry archive {source_name} is empty" + + source_path = f"//tlm/{source_name}" + destination_name = f"tlm_archive_{time.time_ns()}.tlm" + artifact_dir = Path( + os.environ.get( + "TLM_ARCHIVE_ARTIFACT_DIR", + "build-artifacts/radio-downlinked-tlm", + ) + ) + local_path = artifact_dir / destination_name + + proves_send_and_assert_command( + fprime_test_api, + f"{FILE_DOWNLINK}.SendFile", + [source_path, destination_name], + ) + # The completed local file is the end-to-end assertion: unlike the + # FileSent event, it proves every radio packet reached GDS. + _await_complete_local_file(local_path, expected_size) + finally: + proves_send_and_assert_command( + fprime_test_api, + f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", + [DEFAULT_TELEMETRY_DIVIDER], + ) diff --git a/fprime-gds.yml b/fprime-gds.yml index 9d086a79..b6dd3e19 100644 --- a/fprime-gds.yml +++ b/fprime-gds.yml @@ -6,5 +6,6 @@ command-line-options: output-unframed-data: "-" frame-size: 248 framing-selection: authenticate-space-data-link + file-storage-directory: build-artifacts/radio-downlinked-tlm file-uplink-cooldown: 0.400 file-uplink-chunk-size: 204 diff --git a/pytest.ini b/pytest.ini index 07ca9309..c12f79bf 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,6 +1,7 @@ [pytest] markers = uart_only: marks tests that sever the RF link (resets, TRANSMIT toggle) and should only be run when connected via UART + radio_only: marks tests that specifically exercise the RF link and should only be run with --with-radio sync_sequence_number: marks the test that synchronizes the sequence number between GDS and flight software; should be run before any other tests to avoid sequence number mismatches format_filesystem: marks the test that formats the filesystem; should be run before any other tests to ensure a clean state requires_face: marks tests that require a face board (TMP112 / VEML6031 / DRV2605 sensors) to be plugged in; skip on a bare flight controller From ad90a15ba536525c4e225f7cf9ef5e1b2c41cec8 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 21 Jul 2026 15:35:14 -0500 Subject: [PATCH 04/38] fix packet --- .../ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi index 0eff6b31..cc8c0cb0 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi @@ -313,6 +313,6 @@ telemetry packets ReferenceDeploymentPackets { watchdog.WatchdogTransitions - ReferenceDeployment.tlmArchive.RecordingEnabled + tlmArchive.RecordingEnabled } From c9d5171d76f29c1c4dcabb49b163ac1f7e06227c Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 21 Jul 2026 18:05:39 -0500 Subject: [PATCH 05/38] test update --- .../Components/TlmArchive/TlmArchive.cpp | 2 -- .../test/int/tlm_archive_radio_test.py | 25 ++++++++++++++----- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index cdf64de7..de28ebf5 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -27,8 +27,6 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con return; } - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Got a packet from comIn")); - if (this->m_packetCount == MAX_STORED_PACKETS) { this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Writing record.")); bool status = this->writeRecord(); diff --git a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py index ec3a0556..2de94ee1 100644 --- a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py +++ b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py @@ -65,6 +65,7 @@ def test_tlm_archive_downlinks_record_over_radio( fprime_test_api: IntegrationTestAPI, start_gds ): """Create a telemetry record, discover it, and downlink it over LoRa.""" + telemetry_delay_restored = False try: proves_send_and_assert_command( fprime_test_api, @@ -84,8 +85,19 @@ def test_tlm_archive_downlinks_record_over_radio( ) assert writing is not None, "TlmArchive did not start writing a record" + # Stop the high-rate telemetry before sending filesystem commands or + # file packets over the bandwidth-constrained, half-duplex radio link. + # Waiting for this command's acknowledgement also gives the queued + # telemetry generated at the fast rate time to drain. + proves_send_and_assert_command( + fprime_test_api, + f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", + [DEFAULT_TELEMETRY_DIVIDER], + ) + telemetry_delay_restored = True + # TlmArchive writes synchronously, but leave time for the filesystem - # close and the associated events to clear the radio link. + # close and the remaining radio backlog to clear. time.sleep(5) entries = _listed_tlm_files(fprime_test_api) @@ -112,8 +124,9 @@ def test_tlm_archive_downlinks_record_over_radio( # FileSent event, it proves every radio packet reached GDS. _await_complete_local_file(local_path, expected_size) finally: - proves_send_and_assert_command( - fprime_test_api, - f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", - [DEFAULT_TELEMETRY_DIVIDER], - ) + if not telemetry_delay_restored: + proves_send_and_assert_command( + fprime_test_api, + f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", + [DEFAULT_TELEMETRY_DIVIDER], + ) From d7e0a49afe0561d90eb93b5370d5b53e35176624 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 21 Jul 2026 18:35:33 -0500 Subject: [PATCH 06/38] higher tlm delay pleaes don't die --- .../test/int/tlm_archive_radio_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py index 2de94ee1..1b58be81 100644 --- a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py +++ b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py @@ -16,7 +16,7 @@ FILE_DOWNLINK = "FileHandling.fileDownlink" DEFAULT_TELEMETRY_DIVIDER = 29 -RECORD_TIMEOUT_S = 180 +RECORD_TIMEOUT_S = 250 FILE_RECEIVE_TIMEOUT_S = 180 @@ -75,7 +75,7 @@ def test_tlm_archive_downlinks_record_over_radio( proves_send_and_assert_command( fprime_test_api, f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", - [1], + [3], ) writing = fprime_test_api.await_event( From 5d66a29aa39638d0667ce6f9273ef227978016d8 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 21 Jul 2026 19:10:45 -0500 Subject: [PATCH 07/38] decrease downlink delay during test --- .github/workflows/ci.yaml | 2 +- .../test/int/tlm_archive_radio_test.py | 39 +++++++++++++++---- 2 files changed, 33 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f9eb0335..4b65f776 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -570,7 +570,7 @@ jobs: path: build-artifacts/radio-downlinked-tlm/*.tlm if-no-files-found: warn retention-days: 14 - + - name: Upload GDS logs if: always() uses: actions/upload-artifact@v4 diff --git a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py index 1b58be81..39884123 100644 --- a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py +++ b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py @@ -12,10 +12,14 @@ TLM_ARCHIVE = "ReferenceDeployment.tlmArchive" TELEMETRY_DELAY = "ReferenceDeployment.telemetryDelay" +DOWNLINK_DELAY = "ReferenceDeployment.downlinkDelay" FILE_MANAGER = "FileHandling.fileManager" FILE_DOWNLINK = "FileHandling.fileDownlink" DEFAULT_TELEMETRY_DIVIDER = 29 +DEFAULT_DOWNLINK_DIVIDER = 20 +ARCHIVE_TELEMETRY_DIVIDER = 4 +ARCHIVE_DOWNLINK_DIVIDER = 3 RECORD_TIMEOUT_S = 250 FILE_RECEIVE_TIMEOUT_S = 180 @@ -66,16 +70,22 @@ def test_tlm_archive_downlinks_record_over_radio( ): """Create a telemetry record, discover it, and downlink it over LoRa.""" telemetry_delay_restored = False + downlink_delay_restored = False try: proves_send_and_assert_command( fprime_test_api, f"{TLM_ARCHIVE}.RECORDING_STATUS", ["ENABLED"], ) + proves_send_and_assert_command( + fprime_test_api, + f"{DOWNLINK_DELAY}.DIVIDER_PRM_SET", + [ARCHIVE_DOWNLINK_DIVIDER], + ) proves_send_and_assert_command( fprime_test_api, f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", - [3], + [ARCHIVE_TELEMETRY_DIVIDER], ) writing = fprime_test_api.await_event( @@ -123,10 +133,25 @@ def test_tlm_archive_downlinks_record_over_radio( # The completed local file is the end-to-end assertion: unlike the # FileSent event, it proves every radio packet reached GDS. _await_complete_local_file(local_path, expected_size) + + proves_send_and_assert_command( + fprime_test_api, + f"{DOWNLINK_DELAY}.DIVIDER_PRM_SET", + [DEFAULT_DOWNLINK_DIVIDER], + ) + downlink_delay_restored = True finally: - if not telemetry_delay_restored: - proves_send_and_assert_command( - fprime_test_api, - f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", - [DEFAULT_TELEMETRY_DIVIDER], - ) + try: + if not telemetry_delay_restored: + proves_send_and_assert_command( + fprime_test_api, + f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", + [DEFAULT_TELEMETRY_DIVIDER], + ) + finally: + if not downlink_delay_restored: + proves_send_and_assert_command( + fprime_test_api, + f"{DOWNLINK_DELAY}.DIVIDER_PRM_SET", + [DEFAULT_DOWNLINK_DIVIDER], + ) From f132ee3f22cb9181a6dd0bc47d23388b48505a4f Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Fri, 24 Jul 2026 17:29:10 -0500 Subject: [PATCH 08/38] file count and pruning --- .../AntennaDeployer/AntennaDeployer.cpp | 19 +- .../AntennaDeployer/AntennaDeployer.fpp | 6 + .../AntennaDeployer/AntennaDeployer.hpp | 2 + .../Components/TlmArchive/TlmArchive.cpp | 237 +++++++++++++++++- .../Components/TlmArchive/TlmArchive.fpp | 3 + .../Components/TlmArchive/TlmArchive.hpp | 15 +- .../ReferenceDeployment/Top/topology.fpp | 1 + .../test/int/tlm_archive_radio_test.py | 51 ++-- 8 files changed, 295 insertions(+), 39 deletions(-) diff --git a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp index cd7e7a66..1e561b9e 100644 --- a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp +++ b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp @@ -63,6 +63,8 @@ void AntennaDeployer ::init(FwEnumStoreType instance) { this->log_WARNING_HI_FileOperationError(logFilePath, logOperation); } } + + this->m_deployed = this->readDeploymentState(); } // ---------------------------------------------------------------------- @@ -86,14 +88,18 @@ void AntennaDeployer ::schedIn_handler(FwIndexType portNum, U32 context) { } } +bool AntennaDeployer ::deploymentStateGet_handler(FwIndexType portNum) { + (void)portNum; + return this->m_deployed; +} + // ---------------------------------------------------------------------- // Command handler implementations // ---------------------------------------------------------------------- void AntennaDeployer ::DEPLOY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq) { // Check if antenna has already been deployed - bool isDeployed = this->readDeploymentState(); - if (isDeployed) { + if (this->m_deployed) { this->log_ACTIVITY_HI_DeploymentAlreadyComplete(); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); return; @@ -273,14 +279,14 @@ bool AntennaDeployer ::readDeploymentState() { Os::File::Status read_status = file.read(&value, size); (void)file.close(); - if (read_status != Os::File::OP_OK) { + if ((read_status != Os::File::OP_OK) || (size != sizeof(value))) { Fw::LogStringArg logFilePath(path_str); Fw::LogStringArg logOperation("read"); this->log_WARNING_HI_FileOperationError(logFilePath, logOperation); } // Return true if file contains 1, false otherwise - return (read_status == Os::File::OP_OK && value == 1); + return (read_status == Os::File::OP_OK && size == sizeof(value) && value == 1); } void AntennaDeployer ::writeDeploymentState(bool deployed) { @@ -313,11 +319,14 @@ void AntennaDeployer ::writeDeploymentState(bool deployed) { Os::File::Status write_status = file.write(&value, size); (void)file.close(); - if (write_status != Os::File::OP_OK) { + if ((write_status != Os::File::OP_OK) || (size != sizeof(value))) { Fw::LogStringArg logFilePath(path_str); Fw::LogStringArg logOperation("write"); this->log_WARNING_HI_FileOperationError(logFilePath, logOperation); + return; } + + this->m_deployed = deployed; } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.fpp b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.fpp index 46dcc148..ea773b9a 100644 --- a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.fpp +++ b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.fpp @@ -4,6 +4,9 @@ module Components { DEPLOY_RESULT_ABORT @< Deployment aborted via command DEPLOY_RESULT_FAILED @< Deployment failed after exhausting retries } + + @ Port for querying the persistent antenna deployment state + port GetDeploymentState -> bool } module Components { @@ -79,6 +82,9 @@ module Components { @ Port receiving calls from the rate group sync input port schedIn: Svc.Sched + @ Port returning whether antenna deployment has completed + sync input port deploymentStateGet: Components.GetDeploymentState + @ Port signaling the burnwire component to start heating output port burnStart: Fw.Signal diff --git a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp index c915b898..183dceec 100644 --- a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp +++ b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp @@ -33,6 +33,7 @@ class AntennaDeployer final : public AntennaDeployerComponentBase { // Handler implementations // ---------------------------------------------------------------------- void schedIn_handler(FwIndexType portNum, U32 context) override; + bool deploymentStateGet_handler(FwIndexType portNum) override; // ---------------------------------------------------------------------- // Command handlers @@ -63,6 +64,7 @@ class AntennaDeployer final : public AntennaDeployerComponentBase { U32 m_totalAttempts = 0; bool m_stopRequested = false; U32 m_burnTicksThisAttempt = 0; + bool m_deployed = false; }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index de28ebf5..909afc27 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -8,12 +8,67 @@ #include #include +#include #include "Os/Directory.hpp" #include "Os/FileSystem.hpp" namespace Components { +namespace { + +constexpr const char* TLM_DIRECTORY = "//tlm"; +constexpr const char* FIRST_BOOT_DIRECTORY = "//tlm/firstboot"; +constexpr const char* TLM_COUNT_PATH = "//tlm/file_count.txt"; +constexpr const char* FIRST_BOOT_COUNT_PATH = "//tlm/firstboot/file_count.txt"; +constexpr FwSizeType FILE_NAME_BUFFER_SIZE = 48U; +constexpr FwSizeType FILE_COUNT_BUFFER_SIZE = 12U; +constexpr U32 MAX_FILE_ID = 99999999U; + +bool parseFileName(const char* fileName, U32& fileId) { + if ((fileName == nullptr) || (std::strlen(fileName) != 16U) || (std::strncmp(fileName, "tlm_", 4U) != 0) || + (std::strcmp(&fileName[12], ".tlm") != 0)) { + return false; + } + + U32 parsedId = 0U; + for (U32 i = 4U; i < 12U; i++) { + if ((fileName[i] < '0') || (fileName[i] > '9')) { + return false; + } + parsedId = (parsedId * 10U) + static_cast(fileName[i] - '0'); + } + fileId = parsedId; + return true; +} + +bool parseFileCount(const char* text, U32& count) { + if ((text == nullptr) || (*text < '0') || (*text > '9')) { + return false; + } + + U32 parsedCount = 0U; + const char* cursor = text; + while ((*cursor >= '0') && (*cursor <= '9')) { + const U32 digit = static_cast(*cursor - '0'); + if (parsedCount > ((std::numeric_limits::max() - digit) / 10U)) { + return false; + } + parsedCount = (parsedCount * 10U) + digit; + cursor++; + } + if (*cursor == '\n') { + cursor++; + } + if (*cursor != '\0') { + return false; + } + count = parsedCount; + return true; +} + +} // namespace + TlmArchive ::TlmArchive(const char* const compName) : TlmArchiveComponentBase(compName) {} TlmArchive ::~TlmArchive() {} @@ -62,11 +117,33 @@ void TlmArchive::RECORDING_STATUS_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, Co } bool TlmArchive::writeRecord() { - // TODO: Ensure capacity & prune old if at capacity - // TODO: Check boot status for if packet should be permanent - // TODO: Add record to db.txt file + if (!this->m_directoriesReady) { + this->m_directoriesReady = this->ensureDirectories(); + if (!this->m_directoriesReady) { + return false; + } + } - if (!this->openFile()) { + const bool deployed = this->deploymentStateGet_out(0); + const char* const directory = deployed ? TLM_DIRECTORY : FIRST_BOOT_DIRECTORY; + const char* const countPath = deployed ? TLM_COUNT_PATH : FIRST_BOOT_COUNT_PATH; + U32& fileCount = deployed ? this->m_regularFileCount : this->m_firstBootFileCount; + bool& fileCountLoaded = deployed ? this->m_regularFileCountLoaded : this->m_firstBootFileCountLoaded; + + if (!fileCountLoaded) { + if (!this->loadFileCount(directory, countPath, fileCount)) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to load telemetry file count")); + return false; + } + fileCountLoaded = true; + } + + if (deployed && !this->pruneOldFiles(fileCount)) { + return false; + } + + char fileName[FILE_NAME_BUFFER_SIZE] = {}; + if (!this->formatFileName(directory, fileCount, fileName, sizeof(fileName)) || !this->openFile(fileName)) { return false; } @@ -81,25 +158,29 @@ bool TlmArchive::writeRecord() { } if (!this->writeToFile(fileDataSize)) { + (void)Os::FileSystem::removeFile(fileName); this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry data to file")); return false; } if (!this->closeFile()) { + (void)Os::FileSystem::removeFile(fileName); this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to close telemetry file")); return false; } this->m_packetCount = 0; + fileCount++; + if (!this->writeFileCount(countPath, fileCount)) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to update telemetry file count")); + } return true; } -bool TlmArchive::openFile() { - if (this->m_fileOpen || !this->ensureDirectories()) { +bool TlmArchive::openFile(const char* fileName) { + if (this->m_fileOpen) { return false; } - char fileName[96U] = {}; - std::snprintf(fileName, sizeof(fileName), "//tlm/tlm_%08u.tlm", this->m_fileCount++); Os::FileInterface::Status status = this->m_file.open(fileName, Os::File::OPEN_CREATE, Os::File::NO_OVERWRITE); if (status != Os::File::OP_OK) { this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to open telemetry file")); @@ -136,7 +217,145 @@ bool TlmArchive::writeToFile(FwSizeType fileDataSize) { } bool TlmArchive::ensureDirectories() { - return Os::FileSystem::createDirectory("//tlm", false) == Os::FileSystem::OP_OK; + return (Os::FileSystem::createDirectory(TLM_DIRECTORY, false) == Os::FileSystem::OP_OK) && + (Os::FileSystem::createDirectory(FIRST_BOOT_DIRECTORY, false) == Os::FileSystem::OP_OK); +} + +bool TlmArchive::loadFileCount(const char* directory, const char* countPath, U32& count) { + if (!this->readFileCount(countPath, count)) { + U32 retainedCount = 0U; + U32 lowestId = 0U; + while (true) { + if (!this->scanFileCount(directory, count, retainedCount, lowestId)) { + return false; + } + if ((std::strcmp(directory, TLM_DIRECTORY) != 0) || (retainedCount <= 100U)) { + break; + } + if (!this->removeFileRange(lowestId)) { + return false; + } + } + if (!this->writeFileCount(countPath, count)) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to initialize telemetry file count")); + } + return true; + } + + const U32 savedCount = count; + char fileName[FILE_NAME_BUFFER_SIZE] = {}; + while ((count <= MAX_FILE_ID) && this->formatFileName(directory, count, fileName, sizeof(fileName)) && + Os::FileSystem::exists(fileName)) { + count++; + } + + if ((count != savedCount) && !this->writeFileCount(countPath, count)) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to repair telemetry file count")); + } + return true; +} + +bool TlmArchive::readFileCount(const char* countPath, U32& count) { + Os::File countFile; + if (countFile.open(countPath, Os::File::OPEN_READ) != Os::File::OP_OK) { + return false; + } + + FwSizeType fileSize = 0U; + if ((countFile.size(fileSize) != Os::File::OP_OK) || (fileSize == 0U) || (fileSize >= FILE_COUNT_BUFFER_SIZE)) { + countFile.close(); + return false; + } + + char buffer[FILE_COUNT_BUFFER_SIZE] = {}; + FwSizeType readSize = fileSize; + const Os::File::Status status = countFile.read(reinterpret_cast(buffer), readSize); + countFile.close(); + return (status == Os::File::OP_OK) && (readSize == fileSize) && parseFileCount(buffer, count); +} + +bool TlmArchive::writeFileCount(const char* countPath, U32 count) { + char buffer[FILE_COUNT_BUFFER_SIZE] = {}; + const int countSize = std::snprintf(buffer, sizeof(buffer), "%u\n", count); + if ((countSize <= 0) || (static_cast(countSize) >= sizeof(buffer))) { + return false; + } + + Os::File countFile; + if (countFile.open(countPath, Os::File::OPEN_CREATE, Os::File::OVERWRITE) != Os::File::OP_OK) { + return false; + } + + FwSizeType writeSize = static_cast(std::strlen(buffer)); + const FwSizeType expectedSize = writeSize; + const Os::File::Status status = countFile.write(reinterpret_cast(buffer), writeSize); + countFile.close(); + return (status == Os::File::OP_OK) && (writeSize == expectedSize); +} + +bool TlmArchive::scanFileCount(const char* directory, U32& count, U32& retainedCount, U32& lowestId) { + Os::Directory archiveDirectory; + if (archiveDirectory.open(directory, Os::Directory::READ) != Os::Directory::OP_OK) { + return false; + } + + bool foundFile = false; + U32 highestId = 0U; + retainedCount = 0U; + char fileName[FILE_NAME_BUFFER_SIZE] = {}; + Os::Directory::Status status = Os::Directory::OP_OK; + while ((status = archiveDirectory.read(fileName, sizeof(fileName))) == Os::Directory::OP_OK) { + U32 fileId = 0U; + if (parseFileName(fileName, fileId)) { + if (!foundFile || (fileId < lowestId)) { + lowestId = fileId; + } + if (!foundFile || (fileId > highestId)) { + highestId = fileId; + } + foundFile = true; + retainedCount++; + } + } + archiveDirectory.close(); + + if (status != Os::Directory::NO_MORE_FILES) { + return false; + } + count = foundFile ? highestId + 1U : 0U; + return true; +} + +bool TlmArchive::formatFileName(const char* directory, U32 fileId, char* fileName, FwSizeType fileNameSize) { + if (fileId > MAX_FILE_ID) { + return false; + } + const int size = + std::snprintf(fileName, static_cast(fileNameSize), "%s/tlm_%08u.tlm", directory, fileId); + return (size > 0) && (static_cast(size) < fileNameSize); +} + +bool TlmArchive::pruneOldFiles(U32 nextFileId) { + if ((nextFileId < 100U) || ((nextFileId % 10U) != 0U)) { + return true; + } + + return this->removeFileRange(nextFileId - 100U); +} + +bool TlmArchive::removeFileRange(U32 firstId) { + char fileName[FILE_NAME_BUFFER_SIZE] = {}; + for (U32 id = firstId; id < (firstId + 10U); id++) { + if (!this->formatFileName(TLM_DIRECTORY, id, fileName, sizeof(fileName))) { + return false; + } + const Os::FileSystem::Status status = Os::FileSystem::removeFile(fileName); + if ((status != Os::FileSystem::OP_OK) && (status != Os::FileSystem::DOESNT_EXIST)) { + this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to prune old telemetry file")); + return false; + } + } + return true; } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 07ff5e5c..da8dcc0e 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -15,6 +15,9 @@ module Components { sync input port run: Svc.Sched sync input port comIn: Fw.Com + @ Port for checking whether antenna deployment has completed + output port deploymentStateGet: Components.GetDeploymentState + ############################################################################### # Standard AC Ports: Required for Channels, Events, Commands, and Parameters # ############################################################################### diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 9caec2f7..5c5eb6c3 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -35,20 +35,31 @@ class TlmArchive final : public TlmArchiveComponentBase { void RECORDING_STATUS_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, Components::Status status) override; bool writeRecord(); - bool openFile(); + bool openFile(const char* fileName); bool closeFile(); bool writeToFile(FwSizeType fileDataSize); bool ensureDirectories(); + bool loadFileCount(const char* directory, const char* countPath, U32& count); + bool readFileCount(const char* countPath, U32& count); + bool writeFileCount(const char* countPath, U32 count); + bool scanFileCount(const char* directory, U32& count, U32& retainedCount, U32& lowestId); + bool formatFileName(const char* directory, U32 fileId, char* fileName, FwSizeType fileNameSize); + bool pruneOldFiles(U32 nextFileId); + bool removeFileRange(U32 firstId); Os::Mutex m_mutex; Os::File m_file; bool m_fileOpen = false; + bool m_directoriesReady = false; U8 m_fileData[MAX_FILE_SIZE] = {}; bool m_enabled = true; U32 m_packetCount = 0; - U32 m_fileCount = 0; + U32 m_regularFileCount = 0; + U32 m_firstBootFileCount = 0; + bool m_regularFileCountLoaded = false; + bool m_firstBootFileCountLoaded = false; Fw::ComBuffer m_packetArr[MAX_STORED_PACKETS] = {}; }; diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp index a04645f3..5e1393fb 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp @@ -342,6 +342,7 @@ module ReferenceDeployment { connections AntennaDeployment { antennaDeployer.burnStart -> burnwire.burnStart antennaDeployer.burnStop -> burnwire.burnStop + tlmArchive.deploymentStateGet -> antennaDeployer.deploymentStateGet } connections DetumbleManager { diff --git a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py index 39884123..afdc34f2 100644 --- a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py +++ b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py @@ -24,30 +24,33 @@ FILE_RECEIVE_TIMEOUT_S = 180 -def _listed_tlm_files(fprime_test_api: IntegrationTestAPI) -> list[tuple[str, int]]: - """List //tlm and return the telemetry filenames and sizes received as events.""" +def _listed_tlm_files( + fprime_test_api: IntegrationTestAPI, +) -> list[tuple[str, str, int]]: + """Return telemetry archive directories, filenames, and sizes.""" # Directory listing entries are events sent separately from the command # response. Retry the listing when the lossy RF link drops all matching # entries even though the command response arrived. - for _ in range(3): - proves_send_and_assert_command( - fprime_test_api, - f"{FILE_MANAGER}.ListDirectory", - ["//tlm"], - ) - entries = [] - for event in fprime_test_api.get_event_test_history().retrieve(): - if ( - event.get_template().get_full_name() - != f"{FILE_MANAGER}.DirectoryListing" - ): - continue - directory, filename, size = (arg.val for arg in event.get_args()) - if directory == "//tlm" and filename.endswith(".tlm"): - entries.append((filename, int(size))) - if entries: - return entries - time.sleep(2) + for archive_directory in ("//tlm", "//tlm/firstboot"): + for _ in range(3): + proves_send_and_assert_command( + fprime_test_api, + f"{FILE_MANAGER}.ListDirectory", + [archive_directory], + ) + entries = [] + for event in fprime_test_api.get_event_test_history().retrieve(): + if ( + event.get_template().get_full_name() + != f"{FILE_MANAGER}.DirectoryListing" + ): + continue + directory, filename, size = (arg.val for arg in event.get_args()) + if directory == archive_directory and filename.endswith(".tlm"): + entries.append((directory, filename, int(size))) + if entries: + return entries + time.sleep(2) return [] @@ -112,10 +115,12 @@ def test_tlm_archive_downlinks_record_over_radio( entries = _listed_tlm_files(fprime_test_api) assert entries, "ListDirectory(//tlm) returned no telemetry archive files" - source_name, expected_size = max(entries, key=lambda entry: entry[0]) + source_directory, source_name, expected_size = max( + entries, key=lambda entry: entry[1] + ) assert expected_size > 0, f"Telemetry archive {source_name} is empty" - source_path = f"//tlm/{source_name}" + source_path = f"{source_directory}/{source_name}" destination_name = f"tlm_archive_{time.time_ns()}.tlm" artifact_dir = Path( os.environ.get( From 3746d9db90ade26459a963863ea94d24a64c5e3a Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Fri, 24 Jul 2026 18:10:45 -0500 Subject: [PATCH 09/38] add configurable timeout to send and assert --- PROVESFlightControllerReference/test/int/common.py | 8 +++++--- .../test/int/tlm_archive_radio_test.py | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/PROVESFlightControllerReference/test/int/common.py b/PROVESFlightControllerReference/test/int/common.py index 88ae3f59..ad3c171d 100644 --- a/PROVESFlightControllerReference/test/int/common.py +++ b/PROVESFlightControllerReference/test/int/common.py @@ -59,6 +59,7 @@ def proves_send_and_assert_command( args: list[str] = [], events: list[event_predicate] = [], retries: int | None = None, + timeout: float = 10, ): """Send command and assert completion @@ -67,7 +68,8 @@ def proves_send_and_assert_command( take longer to complete. This function clears histories before sending the command, sets a longer timeout for command completion, and retries up to `retries` times if command assertion fails (default: module-level - _DEFAULT_RETRIES, bumped to 5 for radio runs via --with-radio). + _DEFAULT_RETRIES, bumped to 5 for radio runs via --with-radio). The + completion timeout can be increased for long-running commands. """ attempts = retries if retries is not None else _DEFAULT_RETRIES for attempt in range(attempts): @@ -76,8 +78,8 @@ def proves_send_and_assert_command( fprime_test_api.send_and_assert_command( command, args, - timeout=10, - max_delay=10, + timeout=timeout, + max_delay=timeout, events=[], ) if events: diff --git a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py index afdc34f2..1cb19ffb 100644 --- a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py +++ b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py @@ -22,6 +22,7 @@ ARCHIVE_DOWNLINK_DIVIDER = 3 RECORD_TIMEOUT_S = 250 FILE_RECEIVE_TIMEOUT_S = 180 +FILE_DOWNLINK_COMMAND_TIMEOUT_S = 30 def _listed_tlm_files( @@ -134,6 +135,7 @@ def test_tlm_archive_downlinks_record_over_radio( fprime_test_api, f"{FILE_DOWNLINK}.SendFile", [source_path, destination_name], + timeout=FILE_DOWNLINK_COMMAND_TIMEOUT_S, ) # The completed local file is the end-to-end assertion: unlike the # FileSent event, it proves every radio packet reached GDS. From a80e238f3dfada197d7ca7baff233fb330c1c476 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Fri, 24 Jul 2026 22:29:57 -0500 Subject: [PATCH 10/38] nuke everything --- .github/workflows/ci.yaml | 9 - .../AntennaDeployer/AntennaDeployer.cpp | 16 +- .../AntennaDeployer/AntennaDeployer.hpp | 1 - .../Components/TlmArchive/CMakeLists.txt | 30 -- .../Components/TlmArchive/TlmArchive.cpp | 336 +----------------- .../Components/TlmArchive/TlmArchive.fpp | 37 +- .../Components/TlmArchive/TlmArchive.hpp | 40 --- .../Components/TlmArchive/docs/sdd.md | 24 +- .../Top/ReferenceDeploymentPackets.fppi | 2 - .../test/int/common.py | 8 +- .../test/int/conftest.py | 6 +- .../test/int/tlm_archive_radio_test.py | 164 --------- fprime-gds.yml | 1 - lib/fprime-zephyr | 2 +- pytest.ini | 1 - 15 files changed, 28 insertions(+), 649 deletions(-) delete mode 100644 PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4b65f776..335bd032 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -562,15 +562,6 @@ jobs: if-no-files-found: ignore retention-days: 14 - - name: Upload downlinked telemetry archives - if: always() - uses: actions/upload-artifact@v4 - with: - name: radio-downlinked-tlm-archives - path: build-artifacts/radio-downlinked-tlm/*.tlm - if-no-files-found: warn - retention-days: 14 - - name: Upload GDS logs if: always() uses: actions/upload-artifact@v4 diff --git a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp index 1e561b9e..3cd1695e 100644 --- a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp +++ b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.cpp @@ -63,8 +63,6 @@ void AntennaDeployer ::init(FwEnumStoreType instance) { this->log_WARNING_HI_FileOperationError(logFilePath, logOperation); } } - - this->m_deployed = this->readDeploymentState(); } // ---------------------------------------------------------------------- @@ -90,7 +88,7 @@ void AntennaDeployer ::schedIn_handler(FwIndexType portNum, U32 context) { bool AntennaDeployer ::deploymentStateGet_handler(FwIndexType portNum) { (void)portNum; - return this->m_deployed; + return this->readDeploymentState(); } // ---------------------------------------------------------------------- @@ -99,7 +97,8 @@ bool AntennaDeployer ::deploymentStateGet_handler(FwIndexType portNum) { void AntennaDeployer ::DEPLOY_cmdHandler(FwOpcodeType opCode, U32 cmdSeq) { // Check if antenna has already been deployed - if (this->m_deployed) { + bool isDeployed = this->readDeploymentState(); + if (isDeployed) { this->log_ACTIVITY_HI_DeploymentAlreadyComplete(); this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); return; @@ -279,14 +278,14 @@ bool AntennaDeployer ::readDeploymentState() { Os::File::Status read_status = file.read(&value, size); (void)file.close(); - if ((read_status != Os::File::OP_OK) || (size != sizeof(value))) { + if (read_status != Os::File::OP_OK) { Fw::LogStringArg logFilePath(path_str); Fw::LogStringArg logOperation("read"); this->log_WARNING_HI_FileOperationError(logFilePath, logOperation); } // Return true if file contains 1, false otherwise - return (read_status == Os::File::OP_OK && size == sizeof(value) && value == 1); + return (read_status == Os::File::OP_OK && value == 1); } void AntennaDeployer ::writeDeploymentState(bool deployed) { @@ -319,14 +318,11 @@ void AntennaDeployer ::writeDeploymentState(bool deployed) { Os::File::Status write_status = file.write(&value, size); (void)file.close(); - if ((write_status != Os::File::OP_OK) || (size != sizeof(value))) { + if (write_status != Os::File::OP_OK) { Fw::LogStringArg logFilePath(path_str); Fw::LogStringArg logOperation("write"); this->log_WARNING_HI_FileOperationError(logFilePath, logOperation); - return; } - - this->m_deployed = deployed; } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp index 183dceec..6df17afd 100644 --- a/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp +++ b/PROVESFlightControllerReference/Components/AntennaDeployer/AntennaDeployer.hpp @@ -64,7 +64,6 @@ class AntennaDeployer final : public AntennaDeployerComponentBase { U32 m_totalAttempts = 0; bool m_stopRequested = false; U32 m_burnTicksThisAttempt = 0; - bool m_deployed = false; }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt b/PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt index 52477a89..5dba32d1 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt +++ b/PROVESFlightControllerReference/Components/TlmArchive/CMakeLists.txt @@ -1,36 +1,6 @@ -#### -# F Prime CMakeLists.txt: -# -# SOURCES: list of source files (to be compiled) -# AUTOCODER_INPUTS: list of files to be passed to the autocoders -# DEPENDS: list of libraries that this module depends on -# -# More information in the F´ CMake API documentation: -# https://fprime.jpl.nasa.gov/latest/docs/reference/api/cmake/API/ -# -#### - -# Module names are derived from the path from the nearest project/library/framework -# root when not specifically overridden by the developer, i.e. the module defined by -# `MyProj/Some/Path/CMakeLists.txt` will be named `MyProj_Some_Path`. - register_fprime_library( AUTOCODER_INPUTS "${CMAKE_CURRENT_LIST_DIR}/TlmArchive.fpp" SOURCES "${CMAKE_CURRENT_LIST_DIR}/TlmArchive.cpp" -# DEPENDS -# MyPackage_MyOtherModule ) - -### Unit Tests ### -# register_fprime_ut( -# AUTOCODER_INPUTS -# "${CMAKE_CURRENT_LIST_DIR}/TlmArchive.fpp" -# SOURCES -# "${CMAKE_CURRENT_LIST_DIR}/test/ut/TlmArchiveTestMain.cpp" -# "${CMAKE_CURRENT_LIST_DIR}/test/ut/TlmArchiveTester.cpp" -# DEPENDS -# STest # For rules-based testing -# UT_AUTO_HELPERS -# ) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 909afc27..1a6c0b0c 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -6,11 +6,7 @@ #include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp" -#include -#include -#include - -#include "Os/Directory.hpp" +#include "Os/File.hpp" #include "Os/FileSystem.hpp" namespace Components { @@ -18,54 +14,7 @@ namespace Components { namespace { constexpr const char* TLM_DIRECTORY = "//tlm"; -constexpr const char* FIRST_BOOT_DIRECTORY = "//tlm/firstboot"; -constexpr const char* TLM_COUNT_PATH = "//tlm/file_count.txt"; -constexpr const char* FIRST_BOOT_COUNT_PATH = "//tlm/firstboot/file_count.txt"; -constexpr FwSizeType FILE_NAME_BUFFER_SIZE = 48U; -constexpr FwSizeType FILE_COUNT_BUFFER_SIZE = 12U; -constexpr U32 MAX_FILE_ID = 99999999U; - -bool parseFileName(const char* fileName, U32& fileId) { - if ((fileName == nullptr) || (std::strlen(fileName) != 16U) || (std::strncmp(fileName, "tlm_", 4U) != 0) || - (std::strcmp(&fileName[12], ".tlm") != 0)) { - return false; - } - - U32 parsedId = 0U; - for (U32 i = 4U; i < 12U; i++) { - if ((fileName[i] < '0') || (fileName[i] > '9')) { - return false; - } - parsedId = (parsedId * 10U) + static_cast(fileName[i] - '0'); - } - fileId = parsedId; - return true; -} - -bool parseFileCount(const char* text, U32& count) { - if ((text == nullptr) || (*text < '0') || (*text > '9')) { - return false; - } - - U32 parsedCount = 0U; - const char* cursor = text; - while ((*cursor >= '0') && (*cursor <= '9')) { - const U32 digit = static_cast(*cursor - '0'); - if (parsedCount > ((std::numeric_limits::max() - digit) / 10U)) { - return false; - } - parsedCount = (parsedCount * 10U) + digit; - cursor++; - } - if (*cursor == '\n') { - cursor++; - } - if (*cursor != '\0') { - return false; - } - count = parsedCount; - return true; -} +constexpr const char* PRE_DEPLOYMENT_TLM_PATH = "//tlm/pre_deployment.tlm"; } // namespace @@ -77,285 +26,22 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; - Os::ScopeLock lock(this->m_mutex); - if (!this->m_enabled) { + if (this->deploymentStateGet_out(0)) { return; } - if (this->m_packetCount == MAX_STORED_PACKETS) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Writing record.")); - bool status = this->writeRecord(); - if (!status) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry record")); - } - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Done writing record.")); - } - - if (this->m_packetCount < MAX_STORED_PACKETS) { - this->m_packetArr[this->m_packetCount] = data; - this->m_packetCount++; - } -} - -void TlmArchive::run_handler(FwIndexType portNum, U32 context) { - (void)portNum; - (void)context; - - Os::ScopeLock lock(this->m_mutex); - if (this->m_enabled && (this->m_packetCount > 0U) && !this->writeRecord()) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry record")); - } -} - -void TlmArchive::RECORDING_STATUS_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, Components::Status status) { - { - Os::ScopeLock lock(this->m_mutex); - this->m_enabled = (status == Components::Status::ENABLED); - this->tlmWrite_RecordingEnabled(this->m_enabled ? Fw::On::ON : Fw::On::OFF); - } - this->cmdResponse_out(opCode, cmdSeq, Fw::CmdResponse::OK); -} - -bool TlmArchive::writeRecord() { - if (!this->m_directoriesReady) { - this->m_directoriesReady = this->ensureDirectories(); - if (!this->m_directoriesReady) { - return false; - } - } - - const bool deployed = this->deploymentStateGet_out(0); - const char* const directory = deployed ? TLM_DIRECTORY : FIRST_BOOT_DIRECTORY; - const char* const countPath = deployed ? TLM_COUNT_PATH : FIRST_BOOT_COUNT_PATH; - U32& fileCount = deployed ? this->m_regularFileCount : this->m_firstBootFileCount; - bool& fileCountLoaded = deployed ? this->m_regularFileCountLoaded : this->m_firstBootFileCountLoaded; - - if (!fileCountLoaded) { - if (!this->loadFileCount(directory, countPath, fileCount)) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to load telemetry file count")); - return false; - } - fileCountLoaded = true; - } - - if (deployed && !this->pruneOldFiles(fileCount)) { - return false; - } - - char fileName[FILE_NAME_BUFFER_SIZE] = {}; - if (!this->formatFileName(directory, fileCount, fileName, sizeof(fileName)) || !this->openFile(fileName)) { - return false; - } - - FwSizeType fileDataSize = 0; - std::memset(this->m_fileData, 0, sizeof(this->m_fileData)); - - for (U32 i = 0; i < this->m_packetCount; i++) { - const FwSizeType pktSize = this->m_packetArr[i].getSize(); - std::memcpy(&this->m_fileData[fileDataSize], this->m_packetArr[i].getBuffAddr(), - static_cast(pktSize)); - fileDataSize += pktSize; - } - - if (!this->writeToFile(fileDataSize)) { - (void)Os::FileSystem::removeFile(fileName); - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write telemetry data to file")); - return false; - } - if (!this->closeFile()) { - (void)Os::FileSystem::removeFile(fileName); - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to close telemetry file")); - return false; - } - - this->m_packetCount = 0; - fileCount++; - if (!this->writeFileCount(countPath, fileCount)) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to update telemetry file count")); - } - return true; -} - -bool TlmArchive::openFile(const char* fileName) { - if (this->m_fileOpen) { - return false; - } - - Os::FileInterface::Status status = this->m_file.open(fileName, Os::File::OPEN_CREATE, Os::File::NO_OVERWRITE); - if (status != Os::File::OP_OK) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to open telemetry file")); - return false; - } - - this->m_fileOpen = true; - return true; -} - -bool TlmArchive::closeFile() { - if (!this->m_fileOpen) { - return false; - } - - this->m_file.close(); - this->m_fileOpen = false; - return true; -} - -bool TlmArchive::writeToFile(FwSizeType fileDataSize) { - if (!this->m_fileOpen) { - return false; - } - - const FwSizeType expectedSize = fileDataSize; - if ((this->m_file.write(this->m_fileData, fileDataSize) != Os::File::OP_OK) || (fileDataSize != expectedSize)) { - this->closeFile(); - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to write to telemetry file")); - return false; - } - - return true; -} - -bool TlmArchive::ensureDirectories() { - return (Os::FileSystem::createDirectory(TLM_DIRECTORY, false) == Os::FileSystem::OP_OK) && - (Os::FileSystem::createDirectory(FIRST_BOOT_DIRECTORY, false) == Os::FileSystem::OP_OK); -} - -bool TlmArchive::loadFileCount(const char* directory, const char* countPath, U32& count) { - if (!this->readFileCount(countPath, count)) { - U32 retainedCount = 0U; - U32 lowestId = 0U; - while (true) { - if (!this->scanFileCount(directory, count, retainedCount, lowestId)) { - return false; - } - if ((std::strcmp(directory, TLM_DIRECTORY) != 0) || (retainedCount <= 100U)) { - break; - } - if (!this->removeFileRange(lowestId)) { - return false; - } - } - if (!this->writeFileCount(countPath, count)) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to initialize telemetry file count")); - } - return true; - } - - const U32 savedCount = count; - char fileName[FILE_NAME_BUFFER_SIZE] = {}; - while ((count <= MAX_FILE_ID) && this->formatFileName(directory, count, fileName, sizeof(fileName)) && - Os::FileSystem::exists(fileName)) { - count++; - } - - if ((count != savedCount) && !this->writeFileCount(countPath, count)) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to repair telemetry file count")); - } - return true; -} - -bool TlmArchive::readFileCount(const char* countPath, U32& count) { - Os::File countFile; - if (countFile.open(countPath, Os::File::OPEN_READ) != Os::File::OP_OK) { - return false; - } - - FwSizeType fileSize = 0U; - if ((countFile.size(fileSize) != Os::File::OP_OK) || (fileSize == 0U) || (fileSize >= FILE_COUNT_BUFFER_SIZE)) { - countFile.close(); - return false; - } - - char buffer[FILE_COUNT_BUFFER_SIZE] = {}; - FwSizeType readSize = fileSize; - const Os::File::Status status = countFile.read(reinterpret_cast(buffer), readSize); - countFile.close(); - return (status == Os::File::OP_OK) && (readSize == fileSize) && parseFileCount(buffer, count); -} - -bool TlmArchive::writeFileCount(const char* countPath, U32 count) { - char buffer[FILE_COUNT_BUFFER_SIZE] = {}; - const int countSize = std::snprintf(buffer, sizeof(buffer), "%u\n", count); - if ((countSize <= 0) || (static_cast(countSize) >= sizeof(buffer))) { - return false; - } - - Os::File countFile; - if (countFile.open(countPath, Os::File::OPEN_CREATE, Os::File::OVERWRITE) != Os::File::OP_OK) { - return false; - } - - FwSizeType writeSize = static_cast(std::strlen(buffer)); - const FwSizeType expectedSize = writeSize; - const Os::File::Status status = countFile.write(reinterpret_cast(buffer), writeSize); - countFile.close(); - return (status == Os::File::OP_OK) && (writeSize == expectedSize); -} - -bool TlmArchive::scanFileCount(const char* directory, U32& count, U32& retainedCount, U32& lowestId) { - Os::Directory archiveDirectory; - if (archiveDirectory.open(directory, Os::Directory::READ) != Os::Directory::OP_OK) { - return false; - } - - bool foundFile = false; - U32 highestId = 0U; - retainedCount = 0U; - char fileName[FILE_NAME_BUFFER_SIZE] = {}; - Os::Directory::Status status = Os::Directory::OP_OK; - while ((status = archiveDirectory.read(fileName, sizeof(fileName))) == Os::Directory::OP_OK) { - U32 fileId = 0U; - if (parseFileName(fileName, fileId)) { - if (!foundFile || (fileId < lowestId)) { - lowestId = fileId; - } - if (!foundFile || (fileId > highestId)) { - highestId = fileId; - } - foundFile = true; - retainedCount++; - } - } - archiveDirectory.close(); - - if (status != Os::Directory::NO_MORE_FILES) { - return false; + if (Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { + return; } - count = foundFile ? highestId + 1U : 0U; - return true; -} -bool TlmArchive::formatFileName(const char* directory, U32 fileId, char* fileName, FwSizeType fileNameSize) { - if (fileId > MAX_FILE_ID) { - return false; - } - const int size = - std::snprintf(fileName, static_cast(fileNameSize), "%s/tlm_%08u.tlm", directory, fileId); - return (size > 0) && (static_cast(size) < fileNameSize); -} - -bool TlmArchive::pruneOldFiles(U32 nextFileId) { - if ((nextFileId < 100U) || ((nextFileId % 10U) != 0U)) { - return true; + Os::File file; + if (file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND) != Os::File::OP_OK) { + return; } - return this->removeFileRange(nextFileId - 100U); -} - -bool TlmArchive::removeFileRange(U32 firstId) { - char fileName[FILE_NAME_BUFFER_SIZE] = {}; - for (U32 id = firstId; id < (firstId + 10U); id++) { - if (!this->formatFileName(TLM_DIRECTORY, id, fileName, sizeof(fileName))) { - return false; - } - const Os::FileSystem::Status status = Os::FileSystem::removeFile(fileName); - if ((status != Os::FileSystem::OP_OK) && (status != Os::FileSystem::DOESNT_EXIST)) { - this->log_ACTIVITY_LO_Debug(Fw::LogStringArg("Failed to prune old telemetry file")); - return false; - } - } - return true; + FwSizeType size = data.getSize(); + (void)file.write(data.getBuffAddr(), size); + file.close(); } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index da8dcc0e..f353b605 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -1,43 +1,10 @@ module Components { - @ Component for F Prime FSW framework. - enum Status { - ENABLED - DISABLED - } - + @ Stores telemetry generated before antenna deployment passive component TlmArchive { - sync command RECORDING_STATUS(status: Status) - - telemetry RecordingEnabled: Fw.On - - event Debug(message: string) severity activity low format "{}" - - sync input port run: Svc.Sched + @ Telemetry packet to archive sync input port comIn: Fw.Com @ Port for checking whether antenna deployment has completed output port deploymentStateGet: Components.GetDeploymentState - - ############################################################################### - # Standard AC Ports: Required for Channels, Events, Commands, and Parameters # - ############################################################################### - @ Port for requesting the current time - time get port timeCaller - - @ Enables command handling - import Fw.Command - - @ Enables event handling - import Fw.Event - - @ Enables telemetry channels handling - import Fw.Channel - - @ Port to return the value of a parameter - param get port prmGetOut - - @Port to set the value of a parameter - param set port prmSetOut - } } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 5c5eb6c3..8b30c7a6 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -7,18 +7,12 @@ #ifndef Components_TlmArchive_HPP #define Components_TlmArchive_HPP -#include "Os/File.hpp" -#include "Os/Mutex.hpp" #include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchiveComponentAc.hpp" namespace Components { class TlmArchive final : public TlmArchiveComponentBase { public: - // ---------------------------------------------------------------------- - // Component construction and destruction - // ---------------------------------------------------------------------- - //! Construct TlmArchive object TlmArchive(const char* const compName //!< The component name ); @@ -27,41 +21,7 @@ class TlmArchive final : public TlmArchiveComponentBase { ~TlmArchive(); private: - static constexpr U32 MAX_STORED_PACKETS = 32; - static constexpr FwSizeType MAX_FILE_SIZE = 233 * MAX_STORED_PACKETS; - void comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 context) override; - void run_handler(FwIndexType portNum, U32 context) override; - void RECORDING_STATUS_cmdHandler(FwOpcodeType opCode, U32 cmdSeq, Components::Status status) override; - - bool writeRecord(); - bool openFile(const char* fileName); - bool closeFile(); - bool writeToFile(FwSizeType fileDataSize); - bool ensureDirectories(); - bool loadFileCount(const char* directory, const char* countPath, U32& count); - bool readFileCount(const char* countPath, U32& count); - bool writeFileCount(const char* countPath, U32 count); - bool scanFileCount(const char* directory, U32& count, U32& retainedCount, U32& lowestId); - bool formatFileName(const char* directory, U32 fileId, char* fileName, FwSizeType fileNameSize); - bool pruneOldFiles(U32 nextFileId); - bool removeFileRange(U32 firstId); - - Os::Mutex m_mutex; - Os::File m_file; - bool m_fileOpen = false; - bool m_directoriesReady = false; - U8 m_fileData[MAX_FILE_SIZE] = {}; - - bool m_enabled = true; - - U32 m_packetCount = 0; - U32 m_regularFileCount = 0; - U32 m_firstBootFileCount = 0; - bool m_regularFileCountLoaded = false; - bool m_firstBootFileCountLoaded = false; - - Fw::ComBuffer m_packetArr[MAX_STORED_PACKETS] = {}; }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index 04f98031..a6aa0b44 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -1,21 +1,5 @@ -# PROVESFlightControllerReference::TlmArchive +# TlmArchive -Component for F Prime FSW framework. - -## Introduction - - - -## Requirements - -| Name | Description | Rationale | Validation | -|---|---|---|---| -| | | | | - -## Design - - - -## Configuration - - +`TlmArchive` appends telemetry packets to `//tlm/pre_deployment.tlm` +while the antenna deployment state is false. Once the antenna is marked +deployed, incoming telemetry is no longer written. diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi index cc8c0cb0..5ca221ec 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/ReferenceDeploymentPackets.fppi @@ -313,6 +313,4 @@ telemetry packets ReferenceDeploymentPackets { watchdog.WatchdogTransitions - tlmArchive.RecordingEnabled - } diff --git a/PROVESFlightControllerReference/test/int/common.py b/PROVESFlightControllerReference/test/int/common.py index ad3c171d..88ae3f59 100644 --- a/PROVESFlightControllerReference/test/int/common.py +++ b/PROVESFlightControllerReference/test/int/common.py @@ -59,7 +59,6 @@ def proves_send_and_assert_command( args: list[str] = [], events: list[event_predicate] = [], retries: int | None = None, - timeout: float = 10, ): """Send command and assert completion @@ -68,8 +67,7 @@ def proves_send_and_assert_command( take longer to complete. This function clears histories before sending the command, sets a longer timeout for command completion, and retries up to `retries` times if command assertion fails (default: module-level - _DEFAULT_RETRIES, bumped to 5 for radio runs via --with-radio). The - completion timeout can be increased for long-running commands. + _DEFAULT_RETRIES, bumped to 5 for radio runs via --with-radio). """ attempts = retries if retries is not None else _DEFAULT_RETRIES for attempt in range(attempts): @@ -78,8 +76,8 @@ def proves_send_and_assert_command( fprime_test_api.send_and_assert_command( command, args, - timeout=timeout, - max_delay=timeout, + timeout=10, + max_delay=10, events=[], ) if events: diff --git a/PROVESFlightControllerReference/test/int/conftest.py b/PROVESFlightControllerReference/test/int/conftest.py index b2d37bbd..933ebcee 100644 --- a/PROVESFlightControllerReference/test/int/conftest.py +++ b/PROVESFlightControllerReference/test/int/conftest.py @@ -59,11 +59,7 @@ def pytest_collection_modifyitems( ) ) - with_radio = config.getoption("--with-radio", default=False) - if not with_radio: - for item in items: - if item.get_closest_marker("radio_only") is not None: - item.add_marker(pytest.mark.skip(reason="requires --with-radio")) + if not config.getoption("--with-radio", default=False): return rtc_items = [i for i in items if "rtc_test" in i.nodeid] diff --git a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py b/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py deleted file mode 100644 index 1cb19ffb..00000000 --- a/PROVESFlightControllerReference/test/int/tlm_archive_radio_test.py +++ /dev/null @@ -1,164 +0,0 @@ -"""Radio integration test for the TlmArchive component.""" - -import os -import time -from pathlib import Path - -import pytest -from common import proves_send_and_assert_command -from fprime_gds.common.testing_fw.api import IntegrationTestAPI - -pytestmark = [pytest.mark.radio_only] - -TLM_ARCHIVE = "ReferenceDeployment.tlmArchive" -TELEMETRY_DELAY = "ReferenceDeployment.telemetryDelay" -DOWNLINK_DELAY = "ReferenceDeployment.downlinkDelay" -FILE_MANAGER = "FileHandling.fileManager" -FILE_DOWNLINK = "FileHandling.fileDownlink" - -DEFAULT_TELEMETRY_DIVIDER = 29 -DEFAULT_DOWNLINK_DIVIDER = 20 -ARCHIVE_TELEMETRY_DIVIDER = 4 -ARCHIVE_DOWNLINK_DIVIDER = 3 -RECORD_TIMEOUT_S = 250 -FILE_RECEIVE_TIMEOUT_S = 180 -FILE_DOWNLINK_COMMAND_TIMEOUT_S = 30 - - -def _listed_tlm_files( - fprime_test_api: IntegrationTestAPI, -) -> list[tuple[str, str, int]]: - """Return telemetry archive directories, filenames, and sizes.""" - # Directory listing entries are events sent separately from the command - # response. Retry the listing when the lossy RF link drops all matching - # entries even though the command response arrived. - for archive_directory in ("//tlm", "//tlm/firstboot"): - for _ in range(3): - proves_send_and_assert_command( - fprime_test_api, - f"{FILE_MANAGER}.ListDirectory", - [archive_directory], - ) - entries = [] - for event in fprime_test_api.get_event_test_history().retrieve(): - if ( - event.get_template().get_full_name() - != f"{FILE_MANAGER}.DirectoryListing" - ): - continue - directory, filename, size = (arg.val for arg in event.get_args()) - if directory == archive_directory and filename.endswith(".tlm"): - entries.append((directory, filename, int(size))) - if entries: - return entries - time.sleep(2) - return [] - - -def _await_complete_local_file(path: Path, expected_size: int) -> None: - """Wait until GDS has received and closed the complete downlinked file.""" - deadline = time.monotonic() + FILE_RECEIVE_TIMEOUT_S - while time.monotonic() < deadline: - if path.is_file() and path.stat().st_size == expected_size: - return - time.sleep(1) - actual_size = path.stat().st_size if path.exists() else "missing" - pytest.fail( - f"GDS did not save {path} at the expected size of {expected_size} bytes " - f"(actual: {actual_size})" - ) - - -def test_tlm_archive_downlinks_record_over_radio( - fprime_test_api: IntegrationTestAPI, start_gds -): - """Create a telemetry record, discover it, and downlink it over LoRa.""" - telemetry_delay_restored = False - downlink_delay_restored = False - try: - proves_send_and_assert_command( - fprime_test_api, - f"{TLM_ARCHIVE}.RECORDING_STATUS", - ["ENABLED"], - ) - proves_send_and_assert_command( - fprime_test_api, - f"{DOWNLINK_DELAY}.DIVIDER_PRM_SET", - [ARCHIVE_DOWNLINK_DIVIDER], - ) - proves_send_and_assert_command( - fprime_test_api, - f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", - [ARCHIVE_TELEMETRY_DIVIDER], - ) - - writing = fprime_test_api.await_event( - f"{TLM_ARCHIVE}.Debug", - args=["Writing record."], - timeout=RECORD_TIMEOUT_S, - ) - assert writing is not None, "TlmArchive did not start writing a record" - - # Stop the high-rate telemetry before sending filesystem commands or - # file packets over the bandwidth-constrained, half-duplex radio link. - # Waiting for this command's acknowledgement also gives the queued - # telemetry generated at the fast rate time to drain. - proves_send_and_assert_command( - fprime_test_api, - f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", - [DEFAULT_TELEMETRY_DIVIDER], - ) - telemetry_delay_restored = True - - # TlmArchive writes synchronously, but leave time for the filesystem - # close and the remaining radio backlog to clear. - time.sleep(5) - - entries = _listed_tlm_files(fprime_test_api) - assert entries, "ListDirectory(//tlm) returned no telemetry archive files" - source_directory, source_name, expected_size = max( - entries, key=lambda entry: entry[1] - ) - assert expected_size > 0, f"Telemetry archive {source_name} is empty" - - source_path = f"{source_directory}/{source_name}" - destination_name = f"tlm_archive_{time.time_ns()}.tlm" - artifact_dir = Path( - os.environ.get( - "TLM_ARCHIVE_ARTIFACT_DIR", - "build-artifacts/radio-downlinked-tlm", - ) - ) - local_path = artifact_dir / destination_name - - proves_send_and_assert_command( - fprime_test_api, - f"{FILE_DOWNLINK}.SendFile", - [source_path, destination_name], - timeout=FILE_DOWNLINK_COMMAND_TIMEOUT_S, - ) - # The completed local file is the end-to-end assertion: unlike the - # FileSent event, it proves every radio packet reached GDS. - _await_complete_local_file(local_path, expected_size) - - proves_send_and_assert_command( - fprime_test_api, - f"{DOWNLINK_DELAY}.DIVIDER_PRM_SET", - [DEFAULT_DOWNLINK_DIVIDER], - ) - downlink_delay_restored = True - finally: - try: - if not telemetry_delay_restored: - proves_send_and_assert_command( - fprime_test_api, - f"{TELEMETRY_DELAY}.DIVIDER_PRM_SET", - [DEFAULT_TELEMETRY_DIVIDER], - ) - finally: - if not downlink_delay_restored: - proves_send_and_assert_command( - fprime_test_api, - f"{DOWNLINK_DELAY}.DIVIDER_PRM_SET", - [DEFAULT_DOWNLINK_DIVIDER], - ) diff --git a/fprime-gds.yml b/fprime-gds.yml index b6dd3e19..9d086a79 100644 --- a/fprime-gds.yml +++ b/fprime-gds.yml @@ -6,6 +6,5 @@ command-line-options: output-unframed-data: "-" frame-size: 248 framing-selection: authenticate-space-data-link - file-storage-directory: build-artifacts/radio-downlinked-tlm file-uplink-cooldown: 0.400 file-uplink-chunk-size: 204 diff --git a/lib/fprime-zephyr b/lib/fprime-zephyr index 7fd74e4e..bc81a9fb 160000 --- a/lib/fprime-zephyr +++ b/lib/fprime-zephyr @@ -1 +1 @@ -Subproject commit 7fd74e4e60069a3c924589698cfdd51927c697b9 +Subproject commit bc81a9fbef2a68eb8027fae51a1bc51c9fc8d9de diff --git a/pytest.ini b/pytest.ini index c12f79bf..07ca9309 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,7 +1,6 @@ [pytest] markers = uart_only: marks tests that sever the RF link (resets, TRANSMIT toggle) and should only be run when connected via UART - radio_only: marks tests that specifically exercise the RF link and should only be run with --with-radio sync_sequence_number: marks the test that synchronizes the sequence number between GDS and flight software; should be run before any other tests to avoid sequence number mismatches format_filesystem: marks the test that formats the filesystem; should be run before any other tests to ensure a clean state requires_face: marks tests that require a face board (TMP112 / VEML6031 / DRV2605 sensors) to be plugged in; skip on a bare flight controller From 6b82a52994e43c307478372100ae7019f4b588a6 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Fri, 24 Jul 2026 23:29:36 -0500 Subject: [PATCH 11/38] coderabbit recommendations --- .../Components/TlmArchive/TlmArchive.cpp | 42 +++++++++++++++++-- .../Components/TlmArchive/TlmArchive.fpp | 28 ++++++++++++- .../Components/TlmArchive/TlmArchive.hpp | 7 ++++ .../ReferenceDeployment/Top/topology.fpp | 1 + 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 1a6c0b0c..3a8d6152 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -8,6 +8,7 @@ #include "Os/File.hpp" #include "Os/FileSystem.hpp" +#include "Os/Models/FileStatusEnumAc.hpp" namespace Components { @@ -26,21 +27,54 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; - if (this->deploymentStateGet_out(0)) { + Os::ScopeLock lock(this->m_queueMutex); + if (!this->m_packetPending) { + this->m_pendingPacket = data; + this->m_packetPending = true; + } +} + +void TlmArchive::run_handler(FwIndexType portNum, U32 context) { + (void)portNum; + (void)context; + + Fw::ComBuffer packet; + { + Os::ScopeLock lock(this->m_queueMutex); + if (!this->m_packetPending) { + return; + } + packet = this->m_pendingPacket; + this->m_packetPending = false; + } + + if (!this->m_antennasDeployed) { + this->m_antennasDeployed = this->deploymentStateGet_out(0); + } + + if (this->m_antennasDeployed) { return; } if (Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { + this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("create_directory")); return; } Os::File file; - if (file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND) != Os::File::OP_OK) { + const Os::File::Status openStatus = file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND); + if (openStatus != Os::File::OP_OK) { + this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("open_append")); return; } - FwSizeType size = data.getSize(); - (void)file.write(data.getBuffAddr(), size); + const FwSizeType requestedSize = packet.getSize(); + FwSizeType writtenSize = requestedSize; + const Os::File::Status writeStatus = file.write(packet.getBuffAddr(), writtenSize); + if ((writeStatus != Os::File::OP_OK) || (writtenSize != requestedSize)) { + this->log_WARNING_HI_ArchiveWriteError(Os::FileStatus(static_cast(writeStatus)), + requestedSize, writtenSize); + } file.close(); } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index f353b605..0c99e576 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -1,10 +1,36 @@ module Components { @ Stores telemetry generated before antenna deployment passive component TlmArchive { - @ Telemetry packet to archive + @ Telemetry packet to buffer. Drop new packets while the one-packet mailbox is full. sync input port comIn: Fw.Com + @ Drains the telemetry mailbox and performs filesystem work + sync input port run: Svc.Sched + @ Port for checking whether antenna deployment has completed output port deploymentStateGet: Components.GetDeploymentState + + @ Reports archive directory and open failures + event ArchiveFileError( + operation: string @< Filesystem operation that failed + ) severity warning high \ + format "Pre-deployment telemetry archive operation failed: {}" + + @ Reports failed and incomplete archive writes + event ArchiveWriteError( + status: Os.FileStatus @< File write status + requested: FwSizeType @< Requested byte count + written: FwSizeType @< Reported byte count + ) severity warning high \ + format "Pre-deployment telemetry archive write failed: status {}, requested {}, wrote {}" + + @ Port for requesting the current time + time get port timeCaller + + @ Port for sending textual representation of events + text event port logTextOut + + @ Port for sending events to downlink + event port logOut } } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 8b30c7a6..d4ddd78c 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -7,6 +7,7 @@ #ifndef Components_TlmArchive_HPP #define Components_TlmArchive_HPP +#include "Os/Mutex.hpp" #include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchiveComponentAc.hpp" namespace Components { @@ -22,6 +23,12 @@ class TlmArchive final : public TlmArchiveComponentBase { private: void comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 context) override; + void run_handler(FwIndexType portNum, U32 context) override; + + Os::Mutex m_queueMutex; + Fw::ComBuffer m_pendingPacket; + bool m_packetPending = false; + bool m_antennasDeployed = false; }; } // namespace Components diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp index 5e1393fb..ecd738f2 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp @@ -285,6 +285,7 @@ module ReferenceDeployment { rateGroup1Hz.RateGroupMemberOut[16] -> modeManager.run rateGroup1Hz.RateGroupMemberOut[17] -> adcs.run rateGroup1Hz.RateGroupMemberOut[18] -> thermalManager.run + rateGroup1Hz.RateGroupMemberOut[19] -> tlmArchive.run } From d8b3c1a7a95dfb972e50d4bbc219dd427a001e45 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 26 Jul 2026 15:00:16 -0500 Subject: [PATCH 12/38] add archive write events --- .../Components/TlmArchive/TlmArchive.cpp | 2 ++ .../Components/TlmArchive/TlmArchive.fpp | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 3a8d6152..b8d172e0 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -61,6 +61,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { return; } + this->log_ACTIVITY_LO_ArchiveWriteStart(); Os::File file; const Os::File::Status openStatus = file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND); if (openStatus != Os::File::OP_OK) { @@ -76,6 +77,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { requestedSize, writtenSize); } file.close(); + this->log_ACTIVITY_LO_ArchiveWriteFinish(); } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 0c99e576..8ff8dc25 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -10,6 +10,12 @@ module Components { @ Port for checking whether antenna deployment has completed output port deploymentStateGet: Components.GetDeploymentState + @ Report when file write has started + event ArchiveWriteStart severity activity low format "Beginning write to pre_deployment.tlm" + + @ Report when file write has finished + event ArchiveWriteFinish severity activity low format "Write to pre_deployment.tlm successful" + @ Reports archive directory and open failures event ArchiveFileError( operation: string @< Filesystem operation that failed From 6218823631d43d0aa9b420cc6cc5c4fca881a14b Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 26 Jul 2026 15:03:27 -0500 Subject: [PATCH 13/38] stop attempting directory creation once successful --- .../Components/TlmArchive/TlmArchive.cpp | 4 +++- .../Components/TlmArchive/TlmArchive.hpp | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index b8d172e0..eca14e24 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -56,10 +56,12 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { return; } - if (Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { + if (!this->m_directoryInitialized && + Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("create_directory")); return; } + this->m_directoryInitialized = true; this->log_ACTIVITY_LO_ArchiveWriteStart(); Os::File file; diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index d4ddd78c..b0776853 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -27,6 +27,7 @@ class TlmArchive final : public TlmArchiveComponentBase { Os::Mutex m_queueMutex; Fw::ComBuffer m_pendingPacket; + bool m_directoryInitialized = false; bool m_packetPending = false; bool m_antennasDeployed = false; }; From 4e6d07e137edba843c51bb956f7f84eaff7cd586 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 26 Jul 2026 15:10:57 -0500 Subject: [PATCH 14/38] add filesystem failure limit --- .../Components/TlmArchive/TlmArchive.cpp | 14 +++++++++++++- .../Components/TlmArchive/TlmArchive.fpp | 3 +++ .../Components/TlmArchive/TlmArchive.hpp | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index eca14e24..8e19afcd 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -27,6 +27,10 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; + if (this->m_failures >= 3) { + return; + } + Os::ScopeLock lock(this->m_queueMutex); if (!this->m_packetPending) { this->m_pendingPacket = data; @@ -59,6 +63,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { if (!this->m_directoryInitialized && Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("create_directory")); + this->m_failures++; return; } this->m_directoryInitialized = true; @@ -68,6 +73,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { const Os::File::Status openStatus = file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND); if (openStatus != Os::File::OP_OK) { this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("open_append")); + this->m_failures++; return; } @@ -77,9 +83,15 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { if ((writeStatus != Os::File::OP_OK) || (writtenSize != requestedSize)) { this->log_WARNING_HI_ArchiveWriteError(Os::FileStatus(static_cast(writeStatus)), requestedSize, writtenSize); + this->m_failures++; + } else { + this->log_ACTIVITY_LO_ArchiveWriteFinish(); } file.close(); - this->log_ACTIVITY_LO_ArchiveWriteFinish(); + + if (this->m_failures >= 3) { + this->log_WARNING_HI_ArchiveWriteDisabled(); + } } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 8ff8dc25..2f22fc9b 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -30,6 +30,9 @@ module Components { ) severity warning high \ format "Pre-deployment telemetry archive write failed: status {}, requested {}, wrote {}" + @ Reports when telemetry archiving is disabled due to hitting the failure limit + event ArchiveWriteDisabled severity warning high format "Three filesystem failures counted; disabling further telemetry writes." + @ Port for requesting the current time time get port timeCaller diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index b0776853..a5130eff 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -27,6 +27,7 @@ class TlmArchive final : public TlmArchiveComponentBase { Os::Mutex m_queueMutex; Fw::ComBuffer m_pendingPacket; + int m_failures = 0; bool m_directoryInitialized = false; bool m_packetPending = false; bool m_antennasDeployed = false; From b4e359c1eac9693944e894c08a7f983bfe2aa321 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 26 Jul 2026 15:41:43 -0500 Subject: [PATCH 15/38] add max file size limit to component --- .../Components/TlmArchive/TlmArchive.cpp | 20 ++++++++++++++++--- .../Components/TlmArchive/TlmArchive.fpp | 5 ++++- .../Components/TlmArchive/TlmArchive.hpp | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 8e19afcd..cd7f3eff 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -16,6 +16,8 @@ namespace { constexpr const char* TLM_DIRECTORY = "//tlm"; constexpr const char* PRE_DEPLOYMENT_TLM_PATH = "//tlm/pre_deployment.tlm"; +constexpr const int MAX_FAILURES = 3; +constexpr const FwSizeType MAX_FILE_SIZE = 10000; } // namespace @@ -27,7 +29,8 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; - if (this->m_failures >= 3) { + if (this->m_failures >= MAX_FAILURES || this->m_fileSize >= MAX_FILE_SIZE) { + this->log_WARNING_HI_ArchiveWriteDisabled(MAX_FAILURES, MAX_FILE_SIZE); return; } @@ -89,9 +92,20 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { } file.close(); - if (this->m_failures >= 3) { - this->log_WARNING_HI_ArchiveWriteDisabled(); + // File size not initialized yet + if (this->m_fileSize == 0) { + FwSizeType size_arg; + const Os::FileSystem::Status status = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, size_arg); + if (status != Os::FileSystem::OP_OK || size_arg == 0) { + this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("file size still 0 after write")); + return; + } + + this->m_fileSize = size_arg; + return; } + + this->m_fileSize += requestedSize; } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 2f22fc9b..a38c3793 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -31,7 +31,10 @@ module Components { format "Pre-deployment telemetry archive write failed: status {}, requested {}, wrote {}" @ Reports when telemetry archiving is disabled due to hitting the failure limit - event ArchiveWriteDisabled severity warning high format "Three filesystem failures counted; disabling further telemetry writes." + event ArchiveWriteDisabled( + count: I8 + maxSize: FwSizeType + ) severity warning high format "{} filesystem failures counted and/or {}b file size limit reached; disabling further telemetry writes." throttle 1 @ Port for requesting the current time time get port timeCaller diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index a5130eff..76503b31 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -27,6 +27,7 @@ class TlmArchive final : public TlmArchiveComponentBase { Os::Mutex m_queueMutex; Fw::ComBuffer m_pendingPacket; + FwSizeType m_fileSize = 0; int m_failures = 0; bool m_directoryInitialized = false; bool m_packetPending = false; From fc06f3518dd53b021a6d028a44a0f8ed21925c4f Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 26 Jul 2026 15:52:19 -0500 Subject: [PATCH 16/38] ensure state changes are scoped with mutex --- .../Components/TlmArchive/TlmArchive.cpp | 49 ++++++++++++++----- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index cd7f3eff..64d29d0a 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -29,15 +29,18 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; - if (this->m_failures >= MAX_FAILURES || this->m_fileSize >= MAX_FILE_SIZE) { - this->log_WARNING_HI_ArchiveWriteDisabled(MAX_FAILURES, MAX_FILE_SIZE); - return; + bool writeDisabled = false; + { + Os::ScopeLock lock(this->m_queueMutex); + writeDisabled = this->m_failures >= MAX_FAILURES || this->m_fileSize >= MAX_FILE_SIZE; + if (!writeDisabled && !this->m_packetPending) { + this->m_pendingPacket = data; + this->m_packetPending = true; + } } - Os::ScopeLock lock(this->m_queueMutex); - if (!this->m_packetPending) { - this->m_pendingPacket = data; - this->m_packetPending = true; + if (writeDisabled) { + this->log_WARNING_HI_ArchiveWriteDisabled(MAX_FAILURES, MAX_FILE_SIZE); } } @@ -66,7 +69,10 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { if (!this->m_directoryInitialized && Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("create_directory")); - this->m_failures++; + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } return; } this->m_directoryInitialized = true; @@ -76,7 +82,10 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { const Os::File::Status openStatus = file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND); if (openStatus != Os::File::OP_OK) { this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("open_append")); - this->m_failures++; + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } return; } @@ -86,14 +95,22 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { if ((writeStatus != Os::File::OP_OK) || (writtenSize != requestedSize)) { this->log_WARNING_HI_ArchiveWriteError(Os::FileStatus(static_cast(writeStatus)), requestedSize, writtenSize); - this->m_failures++; + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } } else { this->log_ACTIVITY_LO_ArchiveWriteFinish(); } file.close(); // File size not initialized yet - if (this->m_fileSize == 0) { + bool fileSizeUninitialized = false; + { + Os::ScopeLock lock(this->m_queueMutex); + fileSizeUninitialized = this->m_fileSize == 0; + } + if (fileSizeUninitialized) { FwSizeType size_arg; const Os::FileSystem::Status status = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, size_arg); if (status != Os::FileSystem::OP_OK || size_arg == 0) { @@ -101,11 +118,17 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { return; } - this->m_fileSize = size_arg; + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_fileSize = size_arg; + } return; } - this->m_fileSize += requestedSize; + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_fileSize += requestedSize; + } } } // namespace Components From 44a37e4cb878f92d23d5bc7ac859c7a86f122671 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 26 Jul 2026 21:33:37 -0500 Subject: [PATCH 17/38] only send one event when starting telemetry archival --- .../Components/TlmArchive/TlmArchive.cpp | 2 -- .../Components/TlmArchive/TlmArchive.fpp | 5 +---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 64d29d0a..df8a7dfe 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -99,8 +99,6 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; } - } else { - this->log_ACTIVITY_LO_ArchiveWriteFinish(); } file.close(); diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index a38c3793..69da4ab2 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -11,10 +11,7 @@ module Components { output port deploymentStateGet: Components.GetDeploymentState @ Report when file write has started - event ArchiveWriteStart severity activity low format "Beginning write to pre_deployment.tlm" - - @ Report when file write has finished - event ArchiveWriteFinish severity activity low format "Write to pre_deployment.tlm successful" + event ArchiveWriteStart severity activity low format "Beginning telemetry archival to pre_deployment.tlm" throttle 1 @ Reports archive directory and open failures event ArchiveFileError( From c5b3349ead7b3947ab74945928724421d9505df3 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 26 Jul 2026 23:40:34 -0500 Subject: [PATCH 18/38] updates --- .../Components/TlmArchive/TlmArchive.cpp | 51 ++-- .../Components/TlmArchive/docs/sdd.md | 257 +++++++++++++++++- 2 files changed, 281 insertions(+), 27 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index df8a7dfe..aa959b26 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -77,6 +77,31 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { } this->m_directoryInitialized = true; + const FwSizeType requestedSize = packet.getSize(); + FwSizeType currentSize = 0; + const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); + if ((sizeStatus != Os::FileSystem::OP_OK) && (sizeStatus != Os::FileSystem::DOESNT_EXIST)) { + this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("get_file_size")); + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } + return; + } + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_fileSize = currentSize; + } + + if ((currentSize > MAX_FILE_SIZE) || (requestedSize > (MAX_FILE_SIZE - currentSize))) { + this->log_WARNING_HI_ArchiveWriteDisabled(MAX_FAILURES, MAX_FILE_SIZE); + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } + return; + } + this->log_ACTIVITY_LO_ArchiveWriteStart(); Os::File file; const Os::File::Status openStatus = file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND); @@ -89,7 +114,6 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { return; } - const FwSizeType requestedSize = packet.getSize(); FwSizeType writtenSize = requestedSize; const Os::File::Status writeStatus = file.write(packet.getBuffAddr(), writtenSize); if ((writeStatus != Os::File::OP_OK) || (writtenSize != requestedSize)) { @@ -99,33 +123,14 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; } - } - file.close(); - - // File size not initialized yet - bool fileSizeUninitialized = false; - { - Os::ScopeLock lock(this->m_queueMutex); - fileSizeUninitialized = this->m_fileSize == 0; - } - if (fileSizeUninitialized) { - FwSizeType size_arg; - const Os::FileSystem::Status status = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, size_arg); - if (status != Os::FileSystem::OP_OK || size_arg == 0) { - this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("file size still 0 after write")); - return; - } - - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_fileSize = size_arg; - } + file.close(); return; } + file.close(); { Os::ScopeLock lock(this->m_queueMutex); - this->m_fileSize += requestedSize; + this->m_fileSize += writtenSize; } } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index a6aa0b44..ba81c022 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -1,5 +1,254 @@ -# TlmArchive +# Components::TlmArchive -`TlmArchive` appends telemetry packets to `//tlm/pre_deployment.tlm` -while the antenna deployment state is false. Once the antenna is marked -deployed, incoming telemetry is no longer written. +The TlmArchive component preserves telemetry generated before antenna +deployment by appending serialized telemetry packets to +`//tlm/pre_deployment.tlm`. It uses a one-packet mailbox to move filesystem +work out of the telemetry path and into the 1 Hz rate group. + +## Overview + +TlmArchive is a passive component with two execution paths. The synchronous +`comIn` handler receives telemetry and copies at most one packet into an +in-memory mailbox. The synchronous `run` handler drains that mailbox and +performs the filesystem work. A mutex protects state shared by the two calling +contexts. + +The reference topology connects the component as follows: + +- `CdhCore.tlmSend.PktSend` sends telemetry through + `comSplitterTelemetry.comOut` to `TlmArchive.comIn`. +- `rateGroup1Hz.RateGroupMemberOut[19]` invokes `TlmArchive.run`. +- `TlmArchive.deploymentStateGet` queries + `AntennaDeployer.deploymentStateGet`. + +Archiving stops for the remainder of the component's lifetime after it +observes a deployed antenna state. New packets are disabled after three +counted failures. An individual write is rejected when the current archive +size plus the pending packet would exceed 10,000 bytes. + +## Usage Examples + +### Typical Usage + +The component is instantiated and connected during topology setup; it does not +require an explicit configuration call. + +1. A serialized telemetry packet arrives on `comIn`. +2. If writing is enabled and the mailbox is empty, the packet is copied into + the mailbox. A new packet arriving while the mailbox is full is silently + dropped. +3. On the next 1 Hz `run` call, the component removes the pending packet and + queries the antenna deployment state. +4. If the antenna is not deployed, the component creates `//tlm` when needed, + reads the current archive size, verifies that the pending packet fits, opens + `//tlm/pre_deployment.tlm` in append mode, writes the raw `Fw.ComBuffer` + bytes, and closes the file. +5. If the antenna is deployed, the pending packet is discarded without + filesystem access. The true deployment state is latched so it is not queried + again. + +The one-packet mailbox prevents filesystem access from blocking the telemetry +producer. Because `run` executes at 1 Hz, at most one buffered packet is +processed per rate-group tick. + +## Class Diagram + +```mermaid +classDiagram + namespace Components { + class TlmArchiveComponentBase { + <> + } + + class TlmArchive { + +TlmArchive(compName: const char*) + +~TlmArchive() + -comIn_handler(portNum: FwIndexType, data: Fw.ComBuffer, context: U32) + -run_handler(portNum: FwIndexType, context: U32) + -m_queueMutex: Os.Mutex + -m_pendingPacket: Fw.ComBuffer + -m_fileSize: FwSizeType + -m_failures: int + -m_directoryInitialized: bool + -m_packetPending: bool + -m_antennasDeployed: bool + } + } + + TlmArchiveComponentBase <|-- TlmArchive : inherits +``` + +## Port Descriptions + +| Name | Type | Direction | Description | +|---|---|---|---| +| `comIn` | `Fw.Com` | sync input | Receives serialized telemetry packets. Copies a packet into the mailbox when writing is enabled and the mailbox is empty. | +| `run` | `Svc.Sched` | sync input | Drains one pending packet, checks deployment state, and performs archive filesystem work. Connected to the 1 Hz rate group. | +| `deploymentStateGet` | `Components.GetDeploymentState` | output | Queries AntennaDeployer for its persistent deployed state. | +| `timeCaller` | time get | time get | Supplies timestamps for emitted events. | +| `logOut` | event | output | Sends binary event records. | +| `logTextOut` | text event | output | Sends text-formatted event records. | + +## Component Behavior and States + +TlmArchive has no explicit state enumeration. Its behavior is determined by +the following internal state: + +| Name | Initial value | Description | +|---|---:|---| +| `m_pendingPacket` | Empty buffer | Storage for the single pending telemetry packet. | +| `m_packetPending` | `false` | Indicates whether the mailbox contains a packet. | +| `m_fileSize` | `0` | Tracked archive size. Refreshed from the filesystem before each append attempt and advanced by the actual bytes written after a successful write. | +| `m_failures` | `0` | Cumulative count of directory, stat, size-limit, open, and write failures. | +| `m_directoryInitialized` | `false` | Becomes true after `//tlm` is successfully initialized, preventing repeated directory creation attempts. | +| `m_antennasDeployed` | `false` | In-memory latch set when AntennaDeployer first reports a deployed state. | +| `m_queueMutex` | Unlocked | Protects the mailbox, tracked size, and failure count shared by the input and rate-group contexts. | + +Conceptually, the component operates in these states: + +| Name | Description | +|---|---| +| `WAITING` | Writing is enabled and no packet is pending. | +| `PACKET_PENDING` | One packet is waiting for the next `run` invocation. Additional packets are dropped. | +| `DEPLOYED` | A true antenna deployment state has been latched. Pending packets are discarded and no more archive writes occur. | +| `WRITE_DISABLED` | The failure count has reached its limit or the tracked file size has reached 10,000 bytes. New packets are rejected by `comIn`. | + +After deployment, `comIn` can still place a packet in the mailbox because the +deployment latch is evaluated by `run`. The next `run` invocation removes and +discards that packet. No telemetry is written after the deployed state has been +observed. + +### Archive Size Tracking + +The archive is opened in append mode, so data from an existing archive is +preserved. Before each append attempt, the component reads the current on-disk +size. A missing archive is treated as an empty archive; other stat failures are +reported and counted. The pending packet is rejected when its requested size +would make the archive exceed 10,000 bytes. + +After a successful write, `m_fileSize` is incremented by the actual byte count +reported by the file API. The archive is not truncated or deleted when the +limit is reached. + +### Failure Handling + +Filesystem failures are cumulative and are not reset after a successful write. +The following failures increment `m_failures`: + +- failure to create `//tlm`; +- failure to read the size of an existing archive; +- rejection of a packet that would exceed the archive size limit; +- failure to open the archive for append; and +- a failed or short file write. + +A missing archive during the size check is expected and is treated as a +zero-byte file. Once three counted failures have occurred, subsequent `comIn` +calls reject new packets. The packet that encountered an error is not retried. + +## Sequence Diagrams + +### Telemetry Archival + +```mermaid +sequenceDiagram + participant Producer as Telemetry Producer + participant Archive as TlmArchive + participant Rate as 1 Hz Rate Group + participant Deploy as AntennaDeployer + participant FS as Filesystem + + Producer->>Archive: comIn(packet) + Archive->>Archive: Lock and check failure/size limits + alt Writing disabled + Archive->>Archive: Emit ArchiveWriteDisabled + else Mailbox empty + Archive->>Archive: Copy packet into mailbox + else Mailbox full + Archive->>Archive: Drop new packet + end + + Rate->>Archive: run() + alt No packet pending + Archive-->>Rate: Return + else Packet pending + Archive->>Archive: Remove packet from mailbox + opt Deployment state not yet latched + Archive->>Deploy: deploymentStateGet() + Deploy-->>Archive: deployed + end + alt Antenna deployed + Archive->>Archive: Discard packet + else Antenna not deployed + opt Directory not initialized + Archive->>FS: createDirectory("//tlm") + end + Archive->>FS: getFileSize() + alt Stat fails or packet exceeds limit + Archive->>Archive: Count failure and discard packet + else Packet fits + Archive->>Archive: Emit ArchiveWriteStart + Archive->>FS: open("//tlm/pre_deployment.tlm", append) + Archive->>FS: write(packet bytes) + Archive->>FS: close() + Archive->>Archive: Add actual bytes written to tracked size + end + end + end +``` + +## Parameters + +The component defines no runtime F Prime parameters. The following compile-time +implementation constants control its behavior: + +| Name | Value | Description | +|---|---:|---| +| `TLM_DIRECTORY` | `//tlm` | Directory containing the archive. | +| `PRE_DEPLOYMENT_TLM_PATH` | `//tlm/pre_deployment.tlm` | Append-only pre-deployment telemetry archive. | +| `MAX_FAILURES` | `3` | Counted stat, limit, directory, open, or write failures after which new packets are rejected. | +| `MAX_FILE_SIZE` | `10000` bytes | Maximum permitted on-disk archive size after an append. | + +## Commands + +| Name | Description | +|---|---| +| N/A | The component defines no commands. | + +## Events + +| Name | Severity | Throttle | Parameters | Description | +|---|---|---:|---|---| +| `ArchiveWriteStart` | Activity Low | 1 | None | Emitted immediately before each attempt to open the archive. | +| `ArchiveFileError` | Warning High | None | `operation: string` | Reports directory creation, pre-write file-size lookup, or archive open errors. Current operation strings are `create_directory`, `get_file_size`, and `open_append`. | +| `ArchiveWriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | +| `ArchiveWriteDisabled` | Warning High | 1 | `count: I8`, `maxSize: FwSizeType` | Reports that the failure or size cutoff has disabled writes. The implementation supplies the configured limits (`3` and `10000`). | + +The throttled events have no corresponding throttle-clear calls, so only their +first occurrence is reported during the component's lifetime. + +## Telemetry + +| Name | Description | +|---|---| +| N/A | The component defines no telemetry channels. Operational status is reported through events. | + +## Unit Tests + +There are currently no component-specific unit tests for TlmArchive. + +## Requirements + +| Name | Description | Validation | +|---|---|---| +| `TLM_ARCHIVE_001` | The component shall buffer at most one telemetry packet outside the telemetry producer's filesystem path. | Inspection | +| `TLM_ARCHIVE_002` | The component shall append buffered telemetry packet bytes to `//tlm/pre_deployment.tlm` while the antenna deployment state is false. | Inspection | +| `TLM_ARCHIVE_003` | The component shall stop writing telemetry after observing a deployed antenna state. | Inspection | +| `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | +| `TLM_ARCHIVE_005` | The component shall reject a write when the current on-disk archive size plus the pending packet size would exceed 10,000 bytes. | Inspection | +| `TLM_ARCHIVE_006` | The component shall report archive start, filesystem error, write error, and disabled conditions through events. | Inspection | + +## Change Log + +| Date | Description | +|---|---| +| 2026-07-26 | Documented the mailbox, deployment-state handling, archive workflow, limits, events, topology connections, and failure behavior. | From 46a5269355f8e31a189b6c3c02f923dc52423289 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 27 Jul 2026 11:00:01 -0500 Subject: [PATCH 19/38] check file size only once --- .../Components/TlmArchive/TlmArchive.cpp | 27 ++++++++++++------- .../Components/TlmArchive/TlmArchive.hpp | 1 + .../Components/TlmArchive/docs/sdd.md | 23 ++++++++++------ 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index aa959b26..4e54e1c7 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -79,18 +79,27 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { const FwSizeType requestedSize = packet.getSize(); FwSizeType currentSize = 0; - const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); - if ((sizeStatus != Os::FileSystem::OP_OK) && (sizeStatus != Os::FileSystem::DOESNT_EXIST)) { - this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("get_file_size")); + bool fileSizeInitialized = false; + { + Os::ScopeLock lock(this->m_queueMutex); + currentSize = this->m_fileSize; + fileSizeInitialized = this->m_fileSizeInitialized; + } + if (!fileSizeInitialized) { + const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); + if ((sizeStatus != Os::FileSystem::OP_OK) && (sizeStatus != Os::FileSystem::DOESNT_EXIST)) { + this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("get_file_size")); + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } + return; + } { Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; + this->m_fileSize = currentSize; + this->m_fileSizeInitialized = true; } - return; - } - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_fileSize = currentSize; } if ((currentSize > MAX_FILE_SIZE) || (requestedSize > (MAX_FILE_SIZE - currentSize))) { diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 76503b31..84acad58 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -30,6 +30,7 @@ class TlmArchive final : public TlmArchiveComponentBase { FwSizeType m_fileSize = 0; int m_failures = 0; bool m_directoryInitialized = false; + bool m_fileSizeInitialized = false; bool m_packetPending = false; bool m_antennasDeployed = false; }; diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index ba81c022..ce0db511 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -40,7 +40,8 @@ require an explicit configuration call. 3. On the next 1 Hz `run` call, the component removes the pending packet and queries the antenna deployment state. 4. If the antenna is not deployed, the component creates `//tlm` when needed, - reads the current archive size, verifies that the pending packet fits, opens + reads the existing archive size if it has not already been initialized, + verifies that the pending packet fits, opens `//tlm/pre_deployment.tlm` in append mode, writes the raw `Fw.ComBuffer` bytes, and closes the file. 5. If the antenna is deployed, the pending packet is discarded without @@ -70,6 +71,7 @@ classDiagram -m_fileSize: FwSizeType -m_failures: int -m_directoryInitialized: bool + -m_fileSizeInitialized: bool -m_packetPending: bool -m_antennasDeployed: bool } @@ -98,9 +100,10 @@ the following internal state: |---|---:|---| | `m_pendingPacket` | Empty buffer | Storage for the single pending telemetry packet. | | `m_packetPending` | `false` | Indicates whether the mailbox contains a packet. | -| `m_fileSize` | `0` | Tracked archive size. Refreshed from the filesystem before each append attempt and advanced by the actual bytes written after a successful write. | +| `m_fileSize` | `0` | Cached archive size. Initialized once from the filesystem and advanced by the actual bytes written after each successful write. | | `m_failures` | `0` | Cumulative count of directory, stat, size-limit, open, and write failures. | | `m_directoryInitialized` | `false` | Becomes true after `//tlm` is successfully initialized, preventing repeated directory creation attempts. | +| `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read or the archive is confirmed missing, preventing later size queries. | | `m_antennasDeployed` | `false` | In-memory latch set when AntennaDeployer first reports a deployed state. | | `m_queueMutex` | Unlocked | Protects the mailbox, tracked size, and failure count shared by the input and rate-group contexts. | @@ -121,10 +124,12 @@ observed. ### Archive Size Tracking The archive is opened in append mode, so data from an existing archive is -preserved. Before each append attempt, the component reads the current on-disk -size. A missing archive is treated as an empty archive; other stat failures are -reported and counted. The pending packet is rejected when its requested size -would make the archive exceed 10,000 bytes. +preserved. Before the first append attempt, the component reads the existing +on-disk size. A missing archive is treated as an empty archive; other stat +failures are reported and counted. After a successful initialization, the +component uses the cached size rather than querying the filesystem again. The +pending packet is rejected when its requested size would make the cached +archive size exceed 10,000 bytes. After a successful write, `m_fileSize` is incremented by the actual byte count reported by the file API. The archive is not truncated or deleted when the @@ -182,7 +187,9 @@ sequenceDiagram opt Directory not initialized Archive->>FS: createDirectory("//tlm") end - Archive->>FS: getFileSize() + opt File size not initialized + Archive->>FS: getFileSize() + end alt Stat fails or packet exceeds limit Archive->>Archive: Count failure and discard packet else Packet fits @@ -244,7 +251,7 @@ There are currently no component-specific unit tests for TlmArchive. | `TLM_ARCHIVE_002` | The component shall append buffered telemetry packet bytes to `//tlm/pre_deployment.tlm` while the antenna deployment state is false. | Inspection | | `TLM_ARCHIVE_003` | The component shall stop writing telemetry after observing a deployed antenna state. | Inspection | | `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | -| `TLM_ARCHIVE_005` | The component shall reject a write when the current on-disk archive size plus the pending packet size would exceed 10,000 bytes. | Inspection | +| `TLM_ARCHIVE_005` | The component shall reject a write when the cached archive size plus the pending packet size would exceed 10,000 bytes. | Inspection | | `TLM_ARCHIVE_006` | The component shall report archive start, filesystem error, write error, and disabled conditions through events. | Inspection | ## Change Log From 46d00cab799193ddcda395177f82cf1f597cd830 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 27 Jul 2026 11:17:09 -0500 Subject: [PATCH 20/38] update events --- .../Components/TlmArchive/TlmArchive.cpp | 33 ++++++++++--------- .../Components/TlmArchive/TlmArchive.fpp | 17 +++++++--- 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 4e54e1c7..bf3e3083 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -29,18 +29,22 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; - bool writeDisabled = false; { Os::ScopeLock lock(this->m_queueMutex); - writeDisabled = this->m_failures >= MAX_FAILURES || this->m_fileSize >= MAX_FILE_SIZE; - if (!writeDisabled && !this->m_packetPending) { - this->m_pendingPacket = data; - this->m_packetPending = true; + + if (this->m_failures >= MAX_FAILURES) { + this->log_WARNING_HI_FailureLimitReached(MAX_FAILURES); + return; + } else if (this->m_fileSize >= MAX_FILE_SIZE) { + this->log_WARNING_LO_SizeLimitReached(MAX_FILE_SIZE); + return; + } else if (this->m_antennasDeployed) { + this->log_WARNING_LO_AntennasDeployed(); + return; } - } - if (writeDisabled) { - this->log_WARNING_HI_ArchiveWriteDisabled(MAX_FAILURES, MAX_FILE_SIZE); + this->m_pendingPacket = data; + this->m_packetPending = true; } } @@ -68,7 +72,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { if (!this->m_directoryInitialized && Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { - this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("create_directory")); + this->log_WARNING_HI_FileError(Fw::LogStringArg("create_directory")); { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; @@ -88,7 +92,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { if (!fileSizeInitialized) { const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); if ((sizeStatus != Os::FileSystem::OP_OK) && (sizeStatus != Os::FileSystem::DOESNT_EXIST)) { - this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("get_file_size")); + this->log_WARNING_HI_FileError(Fw::LogStringArg("get_file_size")); { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; @@ -103,7 +107,6 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { } if ((currentSize > MAX_FILE_SIZE) || (requestedSize > (MAX_FILE_SIZE - currentSize))) { - this->log_WARNING_HI_ArchiveWriteDisabled(MAX_FAILURES, MAX_FILE_SIZE); { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; @@ -111,11 +114,11 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { return; } - this->log_ACTIVITY_LO_ArchiveWriteStart(); + this->log_ACTIVITY_LO_WriteStart(); Os::File file; const Os::File::Status openStatus = file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND); if (openStatus != Os::File::OP_OK) { - this->log_WARNING_HI_ArchiveFileError(Fw::LogStringArg("open_append")); + this->log_WARNING_HI_FileError(Fw::LogStringArg("open_append")); { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; @@ -126,8 +129,8 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { FwSizeType writtenSize = requestedSize; const Os::File::Status writeStatus = file.write(packet.getBuffAddr(), writtenSize); if ((writeStatus != Os::File::OP_OK) || (writtenSize != requestedSize)) { - this->log_WARNING_HI_ArchiveWriteError(Os::FileStatus(static_cast(writeStatus)), - requestedSize, writtenSize); + this->log_WARNING_HI_WriteError(Os::FileStatus(static_cast(writeStatus)), requestedSize, + writtenSize); { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 69da4ab2..59a44444 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -11,16 +11,16 @@ module Components { output port deploymentStateGet: Components.GetDeploymentState @ Report when file write has started - event ArchiveWriteStart severity activity low format "Beginning telemetry archival to pre_deployment.tlm" throttle 1 + event WriteStart severity activity low format "Beginning telemetry archival to pre_deployment.tlm" throttle 1 @ Reports archive directory and open failures - event ArchiveFileError( + event FileError( operation: string @< Filesystem operation that failed ) severity warning high \ format "Pre-deployment telemetry archive operation failed: {}" @ Reports failed and incomplete archive writes - event ArchiveWriteError( + event WriteError( status: Os.FileStatus @< File write status requested: FwSizeType @< Requested byte count written: FwSizeType @< Reported byte count @@ -28,10 +28,17 @@ module Components { format "Pre-deployment telemetry archive write failed: status {}, requested {}, wrote {}" @ Reports when telemetry archiving is disabled due to hitting the failure limit - event ArchiveWriteDisabled( + event FailureLimitReached( count: I8 + ) severity warning high format "{} filesystem failures counted; disabling further telemetry writes." throttle 1 + + @ Reports when telemetry archiving is disabled due to antennas being deployed + event AntennasDeployed() severity warning low format "Antennas deployed; disabling further telemetry writes." throttle 1 + + @ Reports when telemetry archiving is disabled due to pre_deployment.tlm hitting the size limit + event SizeLimitReached( maxSize: FwSizeType - ) severity warning high format "{} filesystem failures counted and/or {}b file size limit reached; disabling further telemetry writes." throttle 1 + ) severity warning low format "pre_deployment.tlm file size limit of {}b reached; disabling further telemetry writes." throttle 1 @ Port for requesting the current time time get port timeCaller From 44be94f54372c66c36250b4ee5152ed56b290a3d Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 27 Jul 2026 11:24:23 -0500 Subject: [PATCH 21/38] update sdd --- .../Components/TlmArchive/docs/sdd.md | 90 +++++++++++-------- 1 file changed, 51 insertions(+), 39 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index ce0db511..3984bd07 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -2,16 +2,16 @@ The TlmArchive component preserves telemetry generated before antenna deployment by appending serialized telemetry packets to -`//tlm/pre_deployment.tlm`. It uses a one-packet mailbox to move filesystem -work out of the telemetry path and into the 1 Hz rate group. +`//tlm/pre_deployment.tlm`. It uses a one-packet, latest-value mailbox to move +filesystem work out of the telemetry path and into the 1 Hz rate group. ## Overview TlmArchive is a passive component with two execution paths. The synchronous -`comIn` handler receives telemetry and copies at most one packet into an +`comIn` handler receives telemetry and stores the most recent packet in an in-memory mailbox. The synchronous `run` handler drains that mailbox and -performs the filesystem work. A mutex protects state shared by the two calling -contexts. +performs the filesystem work. A mutex protects the mailbox, cached size, and +failure count shared by the two calling contexts. The reference topology connects the component as follows: @@ -22,9 +22,10 @@ The reference topology connects the component as follows: `AntennaDeployer.deploymentStateGet`. Archiving stops for the remainder of the component's lifetime after it -observes a deployed antenna state. New packets are disabled after three -counted failures. An individual write is rejected when the current archive -size plus the pending packet would exceed 10,000 bytes. +observes a deployed antenna state. `comIn` also rejects new packets after three +counted failures or when the cached archive size reaches 10,000 bytes. A +pending packet is rejected and counted as a failure when appending it would +exceed the size limit. ## Usage Examples @@ -34,9 +35,8 @@ The component is instantiated and connected during topology setup; it does not require an explicit configuration call. 1. A serialized telemetry packet arrives on `comIn`. -2. If writing is enabled and the mailbox is empty, the packet is copied into - the mailbox. A new packet arriving while the mailbox is full is silently - dropped. +2. If writing is enabled, the packet is copied into the mailbox. A new packet + arriving while another packet is pending replaces the older packet. 3. On the next 1 Hz `run` call, the component removes the pending packet and queries the antenna deployment state. 4. If the antenna is not deployed, the component creates `//tlm` when needed, @@ -46,7 +46,7 @@ require an explicit configuration call. bytes, and closes the file. 5. If the antenna is deployed, the pending packet is discarded without filesystem access. The true deployment state is latched so it is not queried - again. + again, and subsequent `comIn` calls reject new packets. The one-packet mailbox prevents filesystem access from blocking the telemetry producer. Because `run` executes at 1 Hz, at most one buffered packet is @@ -84,7 +84,7 @@ classDiagram | Name | Type | Direction | Description | |---|---|---|---| -| `comIn` | `Fw.Com` | sync input | Receives serialized telemetry packets. Copies a packet into the mailbox when writing is enabled and the mailbox is empty. | +| `comIn` | `Fw.Com` | sync input | Receives serialized telemetry packets. When writing is enabled, stores the packet in the mailbox, replacing any packet already pending. | | `run` | `Svc.Sched` | sync input | Drains one pending packet, checks deployment state, and performs archive filesystem work. Connected to the 1 Hz rate group. | | `deploymentStateGet` | `Components.GetDeploymentState` | output | Queries AntennaDeployer for its persistent deployed state. | | `timeCaller` | time get | time get | Supplies timestamps for emitted events. | @@ -105,21 +105,20 @@ the following internal state: | `m_directoryInitialized` | `false` | Becomes true after `//tlm` is successfully initialized, preventing repeated directory creation attempts. | | `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read or the archive is confirmed missing, preventing later size queries. | | `m_antennasDeployed` | `false` | In-memory latch set when AntennaDeployer first reports a deployed state. | -| `m_queueMutex` | Unlocked | Protects the mailbox, tracked size, and failure count shared by the input and rate-group contexts. | +| `m_queueMutex` | Unlocked | Protects the mailbox, cached size, and failure count shared by the input and rate-group contexts. | Conceptually, the component operates in these states: | Name | Description | |---|---| | `WAITING` | Writing is enabled and no packet is pending. | -| `PACKET_PENDING` | One packet is waiting for the next `run` invocation. Additional packets are dropped. | -| `DEPLOYED` | A true antenna deployment state has been latched. Pending packets are discarded and no more archive writes occur. | -| `WRITE_DISABLED` | The failure count has reached its limit or the tracked file size has reached 10,000 bytes. New packets are rejected by `comIn`. | +| `PACKET_PENDING` | One packet is waiting for the next `run` invocation. A newer packet replaces the pending packet. | +| `DEPLOYED` | A true antenna deployment state has been latched. The packet that observed deployment is discarded, and subsequent packets are rejected by `comIn`. | +| `WRITE_DISABLED` | The failure count has reached its limit or the cached file size has reached 10,000 bytes. New packets are rejected by `comIn`. | -After deployment, `comIn` can still place a packet in the mailbox because the -deployment latch is evaluated by `run`. The next `run` invocation removes and -discards that packet. No telemetry is written after the deployed state has been -observed. +The `run` handler does not emit an event when it first observes deployment. +The throttled `AntennasDeployed` event is emitted if another packet later +arrives on `comIn`. ### Archive Size Tracking @@ -133,11 +132,12 @@ archive size exceed 10,000 bytes. After a successful write, `m_fileSize` is incremented by the actual byte count reported by the file API. The archive is not truncated or deleted when the -limit is reached. +limit is reached. Changes made to the archive by another component after size +initialization are not reflected in the cache. ### Failure Handling -Filesystem failures are cumulative and are not reset after a successful write. +Counted failures are cumulative and are not reset after a successful write. The following failures increment `m_failures`: - failure to create `//tlm`; @@ -149,6 +149,9 @@ The following failures increment `m_failures`: A missing archive during the size check is expected and is treated as a zero-byte file. Once three counted failures have occurred, subsequent `comIn` calls reject new packets. The packet that encountered an error is not retried. +A size-limit rejection in `run` increments the failure count without emitting +an event; `FailureLimitReached` is emitted if a later `comIn` call observes the +failure cutoff. ## Sequence Diagrams @@ -164,12 +167,14 @@ sequenceDiagram Producer->>Archive: comIn(packet) Archive->>Archive: Lock and check failure/size limits - alt Writing disabled - Archive->>Archive: Emit ArchiveWriteDisabled - else Mailbox empty - Archive->>Archive: Copy packet into mailbox - else Mailbox full - Archive->>Archive: Drop new packet + alt Failure limit reached + Archive->>Archive: Emit FailureLimitReached + else Size limit reached + Archive->>Archive: Emit SizeLimitReached + else Deployment latched + Archive->>Archive: Emit AntennasDeployed + else Writing enabled + Archive->>Archive: Store packet, replacing pending packet end Rate->>Archive: run() @@ -190,10 +195,12 @@ sequenceDiagram opt File size not initialized Archive->>FS: getFileSize() end - alt Stat fails or packet exceeds limit + alt Stat fails + Archive->>Archive: Count failure and discard packet + else Packet exceeds size limit Archive->>Archive: Count failure and discard packet else Packet fits - Archive->>Archive: Emit ArchiveWriteStart + Archive->>Archive: Emit WriteStart Archive->>FS: open("//tlm/pre_deployment.tlm", append) Archive->>FS: write(packet bytes) Archive->>FS: close() @@ -213,7 +220,7 @@ implementation constants control its behavior: | `TLM_DIRECTORY` | `//tlm` | Directory containing the archive. | | `PRE_DEPLOYMENT_TLM_PATH` | `//tlm/pre_deployment.tlm` | Append-only pre-deployment telemetry archive. | | `MAX_FAILURES` | `3` | Counted stat, limit, directory, open, or write failures after which new packets are rejected. | -| `MAX_FILE_SIZE` | `10000` bytes | Maximum permitted on-disk archive size after an append. | +| `MAX_FILE_SIZE` | `10000` bytes | Maximum cached archive size permitted after a component-managed append. | ## Commands @@ -225,13 +232,17 @@ implementation constants control its behavior: | Name | Severity | Throttle | Parameters | Description | |---|---|---:|---|---| -| `ArchiveWriteStart` | Activity Low | 1 | None | Emitted immediately before each attempt to open the archive. | -| `ArchiveFileError` | Warning High | None | `operation: string` | Reports directory creation, pre-write file-size lookup, or archive open errors. Current operation strings are `create_directory`, `get_file_size`, and `open_append`. | -| `ArchiveWriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | -| `ArchiveWriteDisabled` | Warning High | 1 | `count: I8`, `maxSize: FwSizeType` | Reports that the failure or size cutoff has disabled writes. The implementation supplies the configured limits (`3` and `10000`). | +| `WriteStart` | Activity Low | 1 | None | Emitted immediately before each attempt to open the archive. | +| `FileError` | Warning High | None | `operation: string` | Reports directory creation, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `get_file_size`, and `open_append`. | +| `WriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | +| `FailureLimitReached` | Warning High | 1 | `count: I8` | Emitted by `comIn` when the cumulative failure count is at least three. The implementation supplies `3`. | +| `AntennasDeployed` | Warning Low | 1 | None | Emitted by `comIn` when deployment has been latched and a new packet is rejected. | +| `SizeLimitReached` | Warning Low | 1 | `maxSize: FwSizeType` | Emitted by `comIn` when the cached archive size is at least 10,000 bytes. The implementation supplies `10000`. | The throttled events have no corresponding throttle-clear calls, so only their -first occurrence is reported during the component's lifetime. +first occurrence is reported during the component's lifetime. The `comIn` +checks are ordered failure limit, size limit, then antenna deployment; if more +than one condition is true, only the first applicable event is invoked. ## Telemetry @@ -247,15 +258,16 @@ There are currently no component-specific unit tests for TlmArchive. | Name | Description | Validation | |---|---|---| -| `TLM_ARCHIVE_001` | The component shall buffer at most one telemetry packet outside the telemetry producer's filesystem path. | Inspection | +| `TLM_ARCHIVE_001` | The component shall buffer at most one telemetry packet outside the telemetry producer's filesystem path, replacing a pending packet when newer telemetry arrives. | Inspection | | `TLM_ARCHIVE_002` | The component shall append buffered telemetry packet bytes to `//tlm/pre_deployment.tlm` while the antenna deployment state is false. | Inspection | | `TLM_ARCHIVE_003` | The component shall stop writing telemetry after observing a deployed antenna state. | Inspection | | `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | | `TLM_ARCHIVE_005` | The component shall reject a write when the cached archive size plus the pending packet size would exceed 10,000 bytes. | Inspection | -| `TLM_ARCHIVE_006` | The component shall report archive start, filesystem error, write error, and disabled conditions through events. | Inspection | +| `TLM_ARCHIVE_006` | The component shall report archive start, filesystem errors, write errors, failure-limit rejection, size-limit rejection, and deployed-state rejection through events. | Inspection | ## Change Log | Date | Description | |---|---| | 2026-07-26 | Documented the mailbox, deployment-state handling, archive workflow, limits, events, topology connections, and failure behavior. | +| 2026-07-27 | Updated mailbox replacement behavior, cached-size handling, rejection paths, and the current event interface. | From aff88a573cb6445ad4484373d9c14ea803ab3237 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 27 Jul 2026 12:37:16 -0500 Subject: [PATCH 22/38] ensure file is created before checking size --- .../Components/TlmArchive/TlmArchive.cpp | 27 +++++++---- .../Components/TlmArchive/TlmArchive.fpp | 2 +- .../Components/TlmArchive/docs/sdd.md | 46 +++++++++++-------- 3 files changed, 45 insertions(+), 30 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index bf3e3083..58036e27 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -70,16 +70,25 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { return; } - if (!this->m_directoryInitialized && - Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { - this->log_WARNING_HI_FileError(Fw::LogStringArg("create_directory")); - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; + if (!this->m_directoryInitialized) { + if (Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { + this->log_WARNING_HI_FileError(Fw::LogStringArg("create_directory")); + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } + return; } - return; + if (Os::FileSystem::touch(PRE_DEPLOYMENT_TLM_PATH) != Os::FileSystem::OP_OK) { + this->log_WARNING_HI_FileError(Fw::LogStringArg("create_file")); + { + Os::ScopeLock lock(this->m_queueMutex); + this->m_failures++; + } + return; + } + this->m_directoryInitialized = true; } - this->m_directoryInitialized = true; const FwSizeType requestedSize = packet.getSize(); FwSizeType currentSize = 0; @@ -91,7 +100,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { } if (!fileSizeInitialized) { const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); - if ((sizeStatus != Os::FileSystem::OP_OK) && (sizeStatus != Os::FileSystem::DOESNT_EXIST)) { + if (sizeStatus != Os::FileSystem::OP_OK) { this->log_WARNING_HI_FileError(Fw::LogStringArg("get_file_size")); { Os::ScopeLock lock(this->m_queueMutex); diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 59a44444..1ffee197 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -13,7 +13,7 @@ module Components { @ Report when file write has started event WriteStart severity activity low format "Beginning telemetry archival to pre_deployment.tlm" throttle 1 - @ Reports archive directory and open failures + @ Reports archive initialization, size, and open failures event FileError( operation: string @< Filesystem operation that failed ) severity warning high \ diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index 3984bd07..3caa0a08 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -39,9 +39,9 @@ require an explicit configuration call. arriving while another packet is pending replaces the older packet. 3. On the next 1 Hz `run` call, the component removes the pending packet and queries the antenna deployment state. -4. If the antenna is not deployed, the component creates `//tlm` when needed, - reads the existing archive size if it has not already been initialized, - verifies that the pending packet fits, opens +4. If the antenna is not deployed, the component creates `//tlm` and + `//tlm/pre_deployment.tlm` when needed, reads the existing archive size if it + has not already been initialized, verifies that the pending packet fits, opens `//tlm/pre_deployment.tlm` in append mode, writes the raw `Fw.ComBuffer` bytes, and closes the file. 5. If the antenna is deployed, the pending packet is discarded without @@ -102,8 +102,8 @@ the following internal state: | `m_packetPending` | `false` | Indicates whether the mailbox contains a packet. | | `m_fileSize` | `0` | Cached archive size. Initialized once from the filesystem and advanced by the actual bytes written after each successful write. | | `m_failures` | `0` | Cumulative count of directory, stat, size-limit, open, and write failures. | -| `m_directoryInitialized` | `false` | Becomes true after `//tlm` is successfully initialized, preventing repeated directory creation attempts. | -| `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read or the archive is confirmed missing, preventing later size queries. | +| `m_directoryInitialized` | `false` | Becomes true after both `//tlm` and `//tlm/pre_deployment.tlm` are successfully initialized, preventing repeated creation attempts. | +| `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read, preventing later size queries. | | `m_antennasDeployed` | `false` | In-memory latch set when AntennaDeployer first reports a deployed state. | | `m_queueMutex` | Unlocked | Protects the mailbox, cached size, and failure count shared by the input and rate-group contexts. | @@ -123,12 +123,16 @@ arrives on `comIn`. ### Archive Size Tracking The archive is opened in append mode, so data from an existing archive is -preserved. Before the first append attempt, the component reads the existing -on-disk size. A missing archive is treated as an empty archive; other stat -failures are reported and counted. After a successful initialization, the -component uses the cached size rather than querying the filesystem again. The -pending packet is rejected when its requested size would make the cached -archive size exceed 10,000 bytes. +preserved. During one-time storage initialization, `createDirectory` ensures +that `//tlm` exists and `FileSystem::touch` ensures that +`//tlm/pre_deployment.tlm` exists. `touch` creates a missing archive without +truncating an existing archive. + +Before the first append attempt, the component reads the existing on-disk +size. Stat failures are reported and counted. After a successful size +initialization, the component uses the cached size rather than querying the +filesystem again. The pending packet is rejected when its requested size would +make the cached archive size exceed 10,000 bytes. After a successful write, `m_fileSize` is incremented by the actual byte count reported by the file API. The archive is not truncated or deleted when the @@ -141,17 +145,17 @@ Counted failures are cumulative and are not reset after a successful write. The following failures increment `m_failures`: - failure to create `//tlm`; +- failure to create or open `//tlm/pre_deployment.tlm` during initialization; - failure to read the size of an existing archive; - rejection of a packet that would exceed the archive size limit; - failure to open the archive for append; and - a failed or short file write. -A missing archive during the size check is expected and is treated as a -zero-byte file. Once three counted failures have occurred, subsequent `comIn` -calls reject new packets. The packet that encountered an error is not retried. -A size-limit rejection in `run` increments the failure count without emitting -an event; `FailureLimitReached` is emitted if a later `comIn` call observes the -failure cutoff. +Once three counted failures have occurred, subsequent `comIn` calls reject new +packets. The packet that encountered an error is not retried. A size-limit +rejection in `run` increments the failure count without emitting an event; +`FailureLimitReached` is emitted if a later `comIn` call observes the failure +cutoff. ## Sequence Diagrams @@ -189,8 +193,9 @@ sequenceDiagram alt Antenna deployed Archive->>Archive: Discard packet else Antenna not deployed - opt Directory not initialized + opt Storage not initialized Archive->>FS: createDirectory("//tlm") + Archive->>FS: touch("//tlm/pre_deployment.tlm") end opt File size not initialized Archive->>FS: getFileSize() @@ -233,7 +238,7 @@ implementation constants control its behavior: | Name | Severity | Throttle | Parameters | Description | |---|---|---:|---|---| | `WriteStart` | Activity Low | 1 | None | Emitted immediately before each attempt to open the archive. | -| `FileError` | Warning High | None | `operation: string` | Reports directory creation, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `get_file_size`, and `open_append`. | +| `FileError` | Warning High | None | `operation: string` | Reports directory creation, archive creation, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `create_file`, `get_file_size`, and `open_append`. | | `WriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | | `FailureLimitReached` | Warning High | 1 | `count: I8` | Emitted by `comIn` when the cumulative failure count is at least three. The implementation supplies `3`. | | `AntennasDeployed` | Warning Low | 1 | None | Emitted by `comIn` when deployment has been latched and a new packet is rejected. | @@ -264,10 +269,11 @@ There are currently no component-specific unit tests for TlmArchive. | `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | | `TLM_ARCHIVE_005` | The component shall reject a write when the cached archive size plus the pending packet size would exceed 10,000 bytes. | Inspection | | `TLM_ARCHIVE_006` | The component shall report archive start, filesystem errors, write errors, failure-limit rejection, size-limit rejection, and deployed-state rejection through events. | Inspection | +| `TLM_ARCHIVE_007` | The component shall create the archive when missing without truncating an existing archive. | Inspection | ## Change Log | Date | Description | |---|---| | 2026-07-26 | Documented the mailbox, deployment-state handling, archive workflow, limits, events, topology connections, and failure behavior. | -| 2026-07-27 | Updated mailbox replacement behavior, cached-size handling, rejection paths, and the current event interface. | +| 2026-07-27 | Updated mailbox replacement behavior, non-destructive storage initialization, cached-size handling, rejection paths, and the current event interface. | From 2571b1a5f0cd427f8fb801ce4e60f84c4e368c3a Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Tue, 28 Jul 2026 12:25:26 -0500 Subject: [PATCH 23/38] get rid of size remaining check --- .../Components/TlmArchive/TlmArchive.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 58036e27..e0d3a350 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -115,7 +115,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { } } - if ((currentSize > MAX_FILE_SIZE) || (requestedSize > (MAX_FILE_SIZE - currentSize))) { + if ((currentSize > MAX_FILE_SIZE)) { { Os::ScopeLock lock(this->m_queueMutex); this->m_failures++; From 2e04e37f5039af8e38d9e4b4cc9d1d744142f668 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Thu, 30 Jul 2026 09:08:32 -0500 Subject: [PATCH 24/38] use atomics for telemetry archive state - Retain the mutex only for packet mailbox access - Update the design documentation for atomic state synchronization --- .../Components/TlmArchive/TlmArchive.cpp | 83 ++++++------------- .../Components/TlmArchive/TlmArchive.hpp | 8 +- .../Components/TlmArchive/docs/sdd.md | 26 +++--- 3 files changed, 48 insertions(+), 69 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index e0d3a350..de7c9195 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -29,20 +29,19 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; + if (this->m_failures.load() >= MAX_FAILURES) { + this->log_WARNING_HI_FailureLimitReached(MAX_FAILURES); + return; + } else if (this->m_fileSize.load() >= MAX_FILE_SIZE) { + this->log_WARNING_LO_SizeLimitReached(MAX_FILE_SIZE); + return; + } else if (this->m_antennasDeployed.load()) { + this->log_WARNING_LO_AntennasDeployed(); + return; + } + { Os::ScopeLock lock(this->m_queueMutex); - - if (this->m_failures >= MAX_FAILURES) { - this->log_WARNING_HI_FailureLimitReached(MAX_FAILURES); - return; - } else if (this->m_fileSize >= MAX_FILE_SIZE) { - this->log_WARNING_LO_SizeLimitReached(MAX_FILE_SIZE); - return; - } else if (this->m_antennasDeployed) { - this->log_WARNING_LO_AntennasDeployed(); - return; - } - this->m_pendingPacket = data; this->m_packetPending = true; } @@ -62,64 +61,45 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { this->m_packetPending = false; } - if (!this->m_antennasDeployed) { - this->m_antennasDeployed = this->deploymentStateGet_out(0); + if (!this->m_antennasDeployed.load()) { + this->m_antennasDeployed.store(this->deploymentStateGet_out(0)); } - if (this->m_antennasDeployed) { + if (this->m_antennasDeployed.load()) { return; } if (!this->m_directoryInitialized) { if (Os::FileSystem::createDirectory(TLM_DIRECTORY, false) != Os::FileSystem::OP_OK) { this->log_WARNING_HI_FileError(Fw::LogStringArg("create_directory")); - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; - } + this->m_failures.fetch_add(1); return; } if (Os::FileSystem::touch(PRE_DEPLOYMENT_TLM_PATH) != Os::FileSystem::OP_OK) { this->log_WARNING_HI_FileError(Fw::LogStringArg("create_file")); - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; - } + this->m_failures.fetch_add(1); return; } this->m_directoryInitialized = true; } const FwSizeType requestedSize = packet.getSize(); - FwSizeType currentSize = 0; - bool fileSizeInitialized = false; - { - Os::ScopeLock lock(this->m_queueMutex); - currentSize = this->m_fileSize; - fileSizeInitialized = this->m_fileSizeInitialized; - } - if (!fileSizeInitialized) { + FwSizeType currentSize = this->m_fileSize.load(); + if (!this->m_fileSizeInitialized) { const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); if (sizeStatus != Os::FileSystem::OP_OK) { this->log_WARNING_HI_FileError(Fw::LogStringArg("get_file_size")); - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; - } + this->m_failures.fetch_add(1); return; } - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_fileSize = currentSize; - this->m_fileSizeInitialized = true; - } + const U32 cachedSize = + (currentSize > MAX_FILE_SIZE) ? static_cast(MAX_FILE_SIZE) : static_cast(currentSize); + this->m_fileSize.store(cachedSize); + this->m_fileSizeInitialized = true; } if ((currentSize > MAX_FILE_SIZE)) { - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; - } + this->m_failures.fetch_add(1); return; } @@ -128,10 +108,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { const Os::File::Status openStatus = file.open(PRE_DEPLOYMENT_TLM_PATH, Os::File::OPEN_APPEND); if (openStatus != Os::File::OP_OK) { this->log_WARNING_HI_FileError(Fw::LogStringArg("open_append")); - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; - } + this->m_failures.fetch_add(1); return; } @@ -140,19 +117,13 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { if ((writeStatus != Os::File::OP_OK) || (writtenSize != requestedSize)) { this->log_WARNING_HI_WriteError(Os::FileStatus(static_cast(writeStatus)), requestedSize, writtenSize); - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_failures++; - } + this->m_failures.fetch_add(1); file.close(); return; } file.close(); - { - Os::ScopeLock lock(this->m_queueMutex); - this->m_fileSize += writtenSize; - } + this->m_fileSize.fetch_add(static_cast(writtenSize)); } } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 84acad58..74ed373f 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -7,6 +7,8 @@ #ifndef Components_TlmArchive_HPP #define Components_TlmArchive_HPP +#include + #include "Os/Mutex.hpp" #include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchiveComponentAc.hpp" @@ -27,12 +29,12 @@ class TlmArchive final : public TlmArchiveComponentBase { Os::Mutex m_queueMutex; Fw::ComBuffer m_pendingPacket; - FwSizeType m_fileSize = 0; - int m_failures = 0; + std::atomic m_fileSize{0}; + std::atomic m_failures{0}; + std::atomic m_antennasDeployed{false}; bool m_directoryInitialized = false; bool m_fileSizeInitialized = false; bool m_packetPending = false; - bool m_antennasDeployed = false; }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index 3caa0a08..1597c363 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -10,8 +10,9 @@ filesystem work out of the telemetry path and into the 1 Hz rate group. TlmArchive is a passive component with two execution paths. The synchronous `comIn` handler receives telemetry and stores the most recent packet in an in-memory mailbox. The synchronous `run` handler drains that mailbox and -performs the filesystem work. A mutex protects the mailbox, cached size, and -failure count shared by the two calling contexts. +performs the filesystem work. A mutex protects the non-atomic packet mailbox, +while atomics synchronize the cached size, failure count, and deployment latch +shared by the two calling contexts. The reference topology connects the component as follows: @@ -68,12 +69,12 @@ classDiagram -run_handler(portNum: FwIndexType, context: U32) -m_queueMutex: Os.Mutex -m_pendingPacket: Fw.ComBuffer - -m_fileSize: FwSizeType - -m_failures: int + -m_fileSize: atomic~U32~ + -m_failures: atomic~int~ + -m_antennasDeployed: atomic~bool~ -m_directoryInitialized: bool -m_fileSizeInitialized: bool -m_packetPending: bool - -m_antennasDeployed: bool } } @@ -100,12 +101,12 @@ the following internal state: |---|---:|---| | `m_pendingPacket` | Empty buffer | Storage for the single pending telemetry packet. | | `m_packetPending` | `false` | Indicates whether the mailbox contains a packet. | -| `m_fileSize` | `0` | Cached archive size. Initialized once from the filesystem and advanced by the actual bytes written after each successful write. | -| `m_failures` | `0` | Cumulative count of directory, stat, size-limit, open, and write failures. | +| `m_fileSize` | `0` | Atomic cached archive size. Initialized once from the filesystem and advanced by the actual bytes written after each successful write. | +| `m_failures` | `0` | Atomic cumulative count of directory, stat, size-limit, open, and write failures. | | `m_directoryInitialized` | `false` | Becomes true after both `//tlm` and `//tlm/pre_deployment.tlm` are successfully initialized, preventing repeated creation attempts. | | `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read, preventing later size queries. | -| `m_antennasDeployed` | `false` | In-memory latch set when AntennaDeployer first reports a deployed state. | -| `m_queueMutex` | Unlocked | Protects the mailbox, cached size, and failure count shared by the input and rate-group contexts. | +| `m_antennasDeployed` | `false` | Atomic in-memory latch set when AntennaDeployer first reports a deployed state. | +| `m_queueMutex` | Unlocked | Protects the pending packet and its availability flag while the packet is copied between calling contexts. | Conceptually, the component operates in these states: @@ -139,6 +140,10 @@ reported by the file API. The archive is not truncated or deleted when the limit is reached. Changes made to the archive by another component after size initialization are not reflected in the cache. +The atomic cache uses the target's native 32-bit `U32` width. This is lossless +for all permitted archive sizes; an existing on-disk size above the 10,000-byte +limit is represented by the limit value so packet admission remains disabled. + ### Failure Handling Counted failures are cumulative and are not reset after a successful write. @@ -170,7 +175,7 @@ sequenceDiagram participant FS as Filesystem Producer->>Archive: comIn(packet) - Archive->>Archive: Lock and check failure/size limits + Archive->>Archive: Atomically check failure/size/deployment state alt Failure limit reached Archive->>Archive: Emit FailureLimitReached else Size limit reached @@ -277,3 +282,4 @@ There are currently no component-specific unit tests for TlmArchive. |---|---| | 2026-07-26 | Documented the mailbox, deployment-state handling, archive workflow, limits, events, topology connections, and failure behavior. | | 2026-07-27 | Updated mailbox replacement behavior, non-destructive storage initialization, cached-size handling, rejection paths, and the current event interface. | +| 2026-07-30 | Replaced mutex-protected scalar state updates with atomics; retained the mutex only for packet mailbox copies. | From 1e4dfe004837ab7cb0db84d948ddc63ae27f339b Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sat, 1 Aug 2026 21:55:45 -0500 Subject: [PATCH 25/38] switch to using a csv format --- .../Components/TlmArchive/TlmArchive.cpp | 45 ++- .../Components/TlmArchive/TlmArchive.fpp | 6 +- .../Components/TlmArchive/docs/sdd.md | 49 ++- .../test/tools/test_decode_tlm_archive.py | 75 ++++ tools/README.md | 32 ++ tools/decode_tlm_archive.py | 341 ++++++++++++++++++ 6 files changed, 522 insertions(+), 26 deletions(-) create mode 100644 PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py create mode 100644 tools/decode_tlm_archive.py diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index de7c9195..5790422c 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -6,6 +6,9 @@ #include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp" +#include +#include + #include "Os/File.hpp" #include "Os/FileSystem.hpp" #include "Os/Models/FileStatusEnumAc.hpp" @@ -15,7 +18,10 @@ namespace Components { namespace { constexpr const char* TLM_DIRECTORY = "//tlm"; -constexpr const char* PRE_DEPLOYMENT_TLM_PATH = "//tlm/pre_deployment.tlm"; +constexpr const char* PRE_DEPLOYMENT_TLM_PATH = "//tlm/pre_deployment.csv"; +constexpr char CSV_HEADER[] = "format_version,packet_size_bytes,packet_hex\n"; +constexpr char HEX_DIGITS[] = "0123456789ABCDEF"; +constexpr FwSizeType CSV_RECORD_BUFFER_SIZE = (FW_COM_BUFFER_MAX_SIZE * 2) + sizeof(CSV_HEADER) + 32; constexpr const int MAX_FAILURES = 3; constexpr const FwSizeType MAX_FILE_SIZE = 10000; @@ -83,7 +89,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { this->m_directoryInitialized = true; } - const FwSizeType requestedSize = packet.getSize(); + const FwSizeType packetSize = packet.getSize(); FwSizeType currentSize = this->m_fileSize.load(); if (!this->m_fileSizeInitialized) { const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); @@ -98,7 +104,31 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { this->m_fileSizeInitialized = true; } - if ((currentSize > MAX_FILE_SIZE)) { + char csvRecord[CSV_RECORD_BUFFER_SIZE]; + FwSizeType recordSize = 0; + if (currentSize == 0) { + static_assert(sizeof(CSV_HEADER) > 1, "CSV header must not be empty"); + (void)std::memcpy(csvRecord, CSV_HEADER, sizeof(CSV_HEADER) - 1); + recordSize = sizeof(CSV_HEADER) - 1; + } + + const int prefixSize = std::snprintf(&csvRecord[recordSize], sizeof(csvRecord) - recordSize, "1,%llu,", + static_cast(packetSize)); + if ((prefixSize < 0) || (static_cast(prefixSize) >= (sizeof(csvRecord) - recordSize))) { + this->log_WARNING_HI_FileError(Fw::LogStringArg("format_record")); + this->m_failures.fetch_add(1); + return; + } + recordSize += static_cast(prefixSize); + + const U8* const packetBytes = packet.getBuffAddr(); + for (FwSizeType index = 0; index < packetSize; index++) { + csvRecord[recordSize++] = HEX_DIGITS[(packetBytes[index] >> 4) & 0x0F]; + csvRecord[recordSize++] = HEX_DIGITS[packetBytes[index] & 0x0F]; + } + csvRecord[recordSize++] = '\n'; + + if ((currentSize > MAX_FILE_SIZE) || (recordSize > (MAX_FILE_SIZE - currentSize))) { this->m_failures.fetch_add(1); return; } @@ -112,10 +142,11 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { return; } - FwSizeType writtenSize = requestedSize; - const Os::File::Status writeStatus = file.write(packet.getBuffAddr(), writtenSize); - if ((writeStatus != Os::File::OP_OK) || (writtenSize != requestedSize)) { - this->log_WARNING_HI_WriteError(Os::FileStatus(static_cast(writeStatus)), requestedSize, + FwSizeType writtenSize = recordSize; + const Os::File::Status writeStatus = + file.write(reinterpret_cast(csvRecord), writtenSize, Os::File::WaitType::WAIT); + if ((writeStatus != Os::File::OP_OK) || (writtenSize != recordSize)) { + this->log_WARNING_HI_WriteError(Os::FileStatus(static_cast(writeStatus)), recordSize, writtenSize); this->m_failures.fetch_add(1); file.close(); diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 1ffee197..21cccb67 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -11,7 +11,7 @@ module Components { output port deploymentStateGet: Components.GetDeploymentState @ Report when file write has started - event WriteStart severity activity low format "Beginning telemetry archival to pre_deployment.tlm" throttle 1 + event WriteStart severity activity low format "Beginning telemetry archival to pre_deployment.csv" throttle 1 @ Reports archive initialization, size, and open failures event FileError( @@ -35,10 +35,10 @@ module Components { @ Reports when telemetry archiving is disabled due to antennas being deployed event AntennasDeployed() severity warning low format "Antennas deployed; disabling further telemetry writes." throttle 1 - @ Reports when telemetry archiving is disabled due to pre_deployment.tlm hitting the size limit + @ Reports when telemetry archiving is disabled due to pre_deployment.csv hitting the size limit event SizeLimitReached( maxSize: FwSizeType - ) severity warning low format "pre_deployment.tlm file size limit of {}b reached; disabling further telemetry writes." throttle 1 + ) severity warning low format "pre_deployment.csv file size limit of {}b reached; disabling further telemetry writes." throttle 1 @ Port for requesting the current time time get port timeCaller diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index 1597c363..8109cbc9 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -2,7 +2,9 @@ The TlmArchive component preserves telemetry generated before antenna deployment by appending serialized telemetry packets to -`//tlm/pre_deployment.tlm`. It uses a one-packet, latest-value mailbox to move +`//tlm/pre_deployment.csv`. Each packet is stored as one self-delimiting CSV +record containing the archive format version, packet byte count, and uppercase +hexadecimal packet payload. It uses a one-packet, latest-value mailbox to move filesystem work out of the telemetry path and into the 1 Hz rate group. ## Overview @@ -41,14 +43,27 @@ require an explicit configuration call. 3. On the next 1 Hz `run` call, the component removes the pending packet and queries the antenna deployment state. 4. If the antenna is not deployed, the component creates `//tlm` and - `//tlm/pre_deployment.tlm` when needed, reads the existing archive size if it + `//tlm/pre_deployment.csv` when needed, reads the existing archive size if it has not already been initialized, verifies that the pending packet fits, opens - `//tlm/pre_deployment.tlm` in append mode, writes the raw `Fw.ComBuffer` - bytes, and closes the file. + `//tlm/pre_deployment.csv` in append mode, writes one CSV record, and closes + the file. The CSV header is included with the first record in an empty file. 5. If the antenna is deployed, the pending packet is discarded without filesystem access. The true deployment state is latched so it is not queried again, and subsequent `comIn` calls reject new packets. +The archive schema is: + +```csv +format_version,packet_size_bytes,packet_hex +1,4,00040102 +``` + +`packet_hex` contains the complete `Fw.ComBuffer`, including its F Prime packet +descriptor, packet ID, timestamp, and serialized channel values. One line is +one packet; `packet_size_bytes` independently detects truncated or malformed +rows. `tools/decode_tlm_archive.py` combines these records with the generated F +Prime topology dictionary to produce named, typed telemetry values. + The one-packet mailbox prevents filesystem access from blocking the telemetry producer. Because `run` executes at 1 Hz, at most one buffered packet is processed per rate-group tick. @@ -103,7 +118,7 @@ the following internal state: | `m_packetPending` | `false` | Indicates whether the mailbox contains a packet. | | `m_fileSize` | `0` | Atomic cached archive size. Initialized once from the filesystem and advanced by the actual bytes written after each successful write. | | `m_failures` | `0` | Atomic cumulative count of directory, stat, size-limit, open, and write failures. | -| `m_directoryInitialized` | `false` | Becomes true after both `//tlm` and `//tlm/pre_deployment.tlm` are successfully initialized, preventing repeated creation attempts. | +| `m_directoryInitialized` | `false` | Becomes true after both `//tlm` and `//tlm/pre_deployment.csv` are successfully initialized, preventing repeated creation attempts. | | `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read, preventing later size queries. | | `m_antennasDeployed` | `false` | Atomic in-memory latch set when AntennaDeployer first reports a deployed state. | | `m_queueMutex` | Unlocked | Protects the pending packet and its availability flag while the packet is copied between calling contexts. | @@ -126,14 +141,15 @@ arrives on `comIn`. The archive is opened in append mode, so data from an existing archive is preserved. During one-time storage initialization, `createDirectory` ensures that `//tlm` exists and `FileSystem::touch` ensures that -`//tlm/pre_deployment.tlm` exists. `touch` creates a missing archive without +`//tlm/pre_deployment.csv` exists. `touch` creates a missing archive without truncating an existing archive. Before the first append attempt, the component reads the existing on-disk size. Stat failures are reported and counted. After a successful size initialization, the component uses the cached size rather than querying the -filesystem again. The pending packet is rejected when its requested size would -make the cached archive size exceed 10,000 bytes. +filesystem again. The pending packet is rejected when its encoded CSV record, +including the header when necessary, would make the cached archive size exceed +10,000 bytes. After a successful write, `m_fileSize` is incremented by the actual byte count reported by the file API. The archive is not truncated or deleted when the @@ -150,7 +166,7 @@ Counted failures are cumulative and are not reset after a successful write. The following failures increment `m_failures`: - failure to create `//tlm`; -- failure to create or open `//tlm/pre_deployment.tlm` during initialization; +- failure to create or open `//tlm/pre_deployment.csv` during initialization; - failure to read the size of an existing archive; - rejection of a packet that would exceed the archive size limit; - failure to open the archive for append; and @@ -200,7 +216,7 @@ sequenceDiagram else Antenna not deployed opt Storage not initialized Archive->>FS: createDirectory("//tlm") - Archive->>FS: touch("//tlm/pre_deployment.tlm") + Archive->>FS: touch("//tlm/pre_deployment.csv") end opt File size not initialized Archive->>FS: getFileSize() @@ -211,8 +227,8 @@ sequenceDiagram Archive->>Archive: Count failure and discard packet else Packet fits Archive->>Archive: Emit WriteStart - Archive->>FS: open("//tlm/pre_deployment.tlm", append) - Archive->>FS: write(packet bytes) + Archive->>FS: open("//tlm/pre_deployment.csv", append) + Archive->>FS: write(CSV header if empty + one packet record) Archive->>FS: close() Archive->>Archive: Add actual bytes written to tracked size end @@ -228,7 +244,7 @@ implementation constants control its behavior: | Name | Value | Description | |---|---:|---| | `TLM_DIRECTORY` | `//tlm` | Directory containing the archive. | -| `PRE_DEPLOYMENT_TLM_PATH` | `//tlm/pre_deployment.tlm` | Append-only pre-deployment telemetry archive. | +| `PRE_DEPLOYMENT_TLM_PATH` | `//tlm/pre_deployment.csv` | Append-only, one-packet-per-row pre-deployment telemetry archive. | | `MAX_FAILURES` | `3` | Counted stat, limit, directory, open, or write failures after which new packets are rejected. | | `MAX_FILE_SIZE` | `10000` bytes | Maximum cached archive size permitted after a component-managed append. | @@ -243,7 +259,7 @@ implementation constants control its behavior: | Name | Severity | Throttle | Parameters | Description | |---|---|---:|---|---| | `WriteStart` | Activity Low | 1 | None | Emitted immediately before each attempt to open the archive. | -| `FileError` | Warning High | None | `operation: string` | Reports directory creation, archive creation, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `create_file`, `get_file_size`, and `open_append`. | +| `FileError` | Warning High | None | `operation: string` | Reports directory creation, archive creation, record formatting, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `create_file`, `format_record`, `get_file_size`, and `open_append`. | | `WriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | | `FailureLimitReached` | Warning High | 1 | `count: I8` | Emitted by `comIn` when the cumulative failure count is at least three. The implementation supplies `3`. | | `AntennasDeployed` | Warning Low | 1 | None | Emitted by `comIn` when deployment has been latched and a new packet is rejected. | @@ -269,10 +285,10 @@ There are currently no component-specific unit tests for TlmArchive. | Name | Description | Validation | |---|---|---| | `TLM_ARCHIVE_001` | The component shall buffer at most one telemetry packet outside the telemetry producer's filesystem path, replacing a pending packet when newer telemetry arrives. | Inspection | -| `TLM_ARCHIVE_002` | The component shall append buffered telemetry packet bytes to `//tlm/pre_deployment.tlm` while the antenna deployment state is false. | Inspection | +| `TLM_ARCHIVE_002` | The component shall append buffered telemetry as versioned, one-packet-per-row CSV records to `//tlm/pre_deployment.csv` while the antenna deployment state is false. | Inspection | | `TLM_ARCHIVE_003` | The component shall stop writing telemetry after observing a deployed antenna state. | Inspection | | `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | -| `TLM_ARCHIVE_005` | The component shall reject a write when the cached archive size plus the pending packet size would exceed 10,000 bytes. | Inspection | +| `TLM_ARCHIVE_005` | The component shall reject a write when the cached archive size plus the encoded CSV record would exceed 10,000 bytes. | Inspection | | `TLM_ARCHIVE_006` | The component shall report archive start, filesystem errors, write errors, failure-limit rejection, size-limit rejection, and deployed-state rejection through events. | Inspection | | `TLM_ARCHIVE_007` | The component shall create the archive when missing without truncating an existing archive. | Inspection | @@ -283,3 +299,4 @@ There are currently no component-specific unit tests for TlmArchive. | 2026-07-26 | Documented the mailbox, deployment-state handling, archive workflow, limits, events, topology connections, and failure behavior. | | 2026-07-27 | Updated mailbox replacement behavior, non-destructive storage initialization, cached-size handling, rejection paths, and the current event interface. | | 2026-07-30 | Replaced mutex-protected scalar state updates with atomics; retained the mutex only for packet mailbox copies. | +| 2026-08-01 | Replaced the concatenated binary archive with a versioned, one-packet-per-row CSV archive and documented the dictionary-backed ground decoder. | diff --git a/PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py b/PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py new file mode 100644 index 00000000..238ee650 --- /dev/null +++ b/PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py @@ -0,0 +1,75 @@ +"""Tests for the ground-side telemetry archive decoder.""" + +from __future__ import annotations + +import io +from pathlib import Path + +import pytest + +from tools.decode_tlm_archive import ( + ArchiveDecodeError, + TelemetryDecoder, + read_archive, +) + +PROJECT_ROOT = Path(__file__).parents[3] +DICTIONARY = ( + PROJECT_ROOT + / "build-artifacts/zephyr/fprime-zephyr-deployment/dict" + / "ReferenceDeploymentTopologyDictionary.json" +) + + +def make_filesystem_packet() -> bytes: + """Create a real packet ID 5 using types loaded from the flight dictionary.""" + + from fprime_gds.common.models.dictionaries import Dictionaries + from fprime_gds.common.models.serialize.time_type import TimeType + from fprime_gds.common.utils.config_manager import ConfigManager + + dictionaries = Dictionaries.load_dictionaries_into_config(str(DICTIONARY)) + config = ConfigManager() + packet = dictionaries.packet[5] + return b"".join( + ( + config.get_type("FwPacketDescriptorType")(4).serialize(), + config.get_type("FwTlmPacketizeIdType")(5).serialize(), + TimeType(TimeType.TimeBase("TB_PROC_TIME"), 2, 123, 456).serialize(), + packet.get_ch_list()[0].get_type_obj()(4096).serialize(), + packet.get_ch_list()[1].get_type_obj()(8192).serialize(), + ) + ) + + +@pytest.mark.skipif(not DICTIONARY.is_file(), reason="generated dictionary unavailable") +def test_decode_packet_uses_dictionary_names_and_types() -> None: + """Decode packet boundaries, metadata, and typed channel values.""" + + payload = make_filesystem_packet() + archive = io.StringIO( + "format_version,packet_size_bytes,packet_hex\n" + f"1,{len(payload)},{payload.hex().upper()}\n" + ) + record = next(iter(read_archive(archive))) + decoded = TelemetryDecoder(DICTIONARY).decode(record) + + assert decoded.packet_name == "FileSystem" + assert decoded.packet_id == 5 + assert decoded.time_base == "TB_PROC_TIME" + assert decoded.time_context == 2 + assert decoded.seconds == 123 + assert decoded.microseconds == 456 + assert [channel["name"] for channel in decoded.channels] == [ + "ReferenceDeployment.fsSpace.FreeSpace", + "ReferenceDeployment.fsSpace.TotalSpace", + ] + assert [channel["value"] for channel in decoded.channels] == [4096, 8192] + + +def test_archive_rejects_size_mismatch() -> None: + """Reject truncated rows before asking F Prime to deserialize them.""" + + archive = io.StringIO("format_version,packet_size_bytes,packet_hex\n1,3,0001\n") + with pytest.raises(ArchiveDecodeError, match="declared 3 packet bytes, decoded 2"): + list(read_archive(archive)) diff --git a/tools/README.md b/tools/README.md index 8251cb8c..47f8389a 100644 --- a/tools/README.md +++ b/tools/README.md @@ -2,6 +2,38 @@ This directory contains development and analysis tools for the PROVES Core Reference project. +## Telemetry Archive Decoder + +`decode_tlm_archive.py` decodes the CSV file written by the flight +`TlmArchive` component. Use the topology dictionary from the same firmware +build as the archive: + +```bash +fprime-venv/bin/python tools/decode_tlm_archive.py \ + pre_deployment.csv \ + build-artifacts/zephyr/fprime-zephyr-deployment/dict/ReferenceDeploymentTopologyDictionary.json +``` + +The default output identifies each packet and prints its timestamp and named, +typed channel values. Machine-readable output is also available: + +```bash +# One row per telemetry channel +fprime-venv/bin/python tools/decode_tlm_archive.py \ + pre_deployment.csv ReferenceDeploymentTopologyDictionary.json \ + --format csv --output decoded.csv + +# One object per packet +fprime-venv/bin/python tools/decode_tlm_archive.py \ + pre_deployment.csv ReferenceDeploymentTopologyDictionary.json \ + --format json --output decoded.json +``` + +The script checks the archive version, declared packet byte count, F Prime +packet descriptor, packet ID, and dictionary-derived packet size. Always keep +the dictionary artifact with its firmware image: a dictionary from a different +build may assign different channel layouts or types. + ## Data Budget Tool The Data Budget Tool (`data_budget.py`) analyzes F Prime telemetry definitions to calculate the serialized byte size of telemetry channels and packets. diff --git a/tools/decode_tlm_archive.py b/tools/decode_tlm_archive.py new file mode 100644 index 00000000..89f8113d --- /dev/null +++ b/tools/decode_tlm_archive.py @@ -0,0 +1,341 @@ +#!/usr/bin/env python3 +"""Decode a TlmArchive CSV file with an F Prime topology dictionary.""" + +from __future__ import annotations + +import argparse +import csv +import json +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Iterable, TextIO + +ARCHIVE_FIELDS = ("format_version", "packet_size_bytes", "packet_hex") +ARCHIVE_FORMAT_VERSION = 1 +PACKETIZED_TLM_DESCRIPTOR = 0x0004 + + +class ArchiveDecodeError(ValueError): + """Report malformed archive records and undecodable telemetry packets.""" + + +@dataclass(frozen=True) +class ArchiveRecord: + """One packet record read from the telemetry archive.""" + + record_number: int + payload: bytes + + +@dataclass(frozen=True) +class DecodedPacket: + """Human-readable metadata and channel values for one packet.""" + + record_number: int + packet_id: int + packet_name: str + packet_size_bytes: int + time_base: str + time_context: int + seconds: int + microseconds: int + channels: tuple[dict[str, Any], ...] + + +def read_archive(stream: TextIO) -> Iterable[ArchiveRecord]: + """Yield validated packets from a TlmArchive CSV stream.""" + + reader = csv.DictReader(stream) + if reader.fieldnames != list(ARCHIVE_FIELDS): + actual = ",".join(reader.fieldnames or ()) or "" + expected = ",".join(ARCHIVE_FIELDS) + raise ArchiveDecodeError(f"archive header is {actual!r}; expected {expected!r}") + + for record_number, row in enumerate(reader, start=1): + line_number = reader.line_num + try: + version = int(row["format_version"]) + declared_size = int(row["packet_size_bytes"]) + except (TypeError, ValueError) as exc: + raise ArchiveDecodeError( + f"line {line_number}: version and packet size must be integers" + ) from exc + + if version != ARCHIVE_FORMAT_VERSION: + raise ArchiveDecodeError( + f"line {line_number}: unsupported archive format version {version}" + ) + if declared_size < 0: + raise ArchiveDecodeError( + f"line {line_number}: packet size cannot be negative" + ) + + packet_hex = row["packet_hex"] + try: + payload = bytes.fromhex(packet_hex) + except (TypeError, ValueError) as exc: + raise ArchiveDecodeError( + f"line {line_number}: packet_hex is not valid hexadecimal" + ) from exc + if len(payload) != declared_size: + raise ArchiveDecodeError( + f"line {line_number}: declared {declared_size} packet bytes, " + f"decoded {len(payload)}" + ) + + yield ArchiveRecord(record_number, payload) + + +class TelemetryDecoder: + """Decode packetized telemetry using the project's generated dictionary.""" + + def __init__(self, dictionary_path: Path, packet_set_name: str | None = None): + """Load the dictionary and configure the standard F Prime decoder.""" + + try: + from fprime_gds.common.decoders.pkt_decoder import PktDecoder + from fprime_gds.common.models.dictionaries import Dictionaries + from fprime_gds.common.models.serialize.time_type import TimeType + from fprime_gds.common.utils.config_manager import ConfigManager + except ImportError as exc: + raise ArchiveDecodeError( + "fprime-gds is unavailable; run 'make fprime-venv' and invoke this " + "script with 'fprime-venv/bin/python'" + ) from exc + + if not dictionary_path.is_file(): + raise ArchiveDecodeError(f"dictionary does not exist: {dictionary_path}") + + try: + self._dictionaries = Dictionaries.load_dictionaries_into_config( + str(dictionary_path), packet_set_name=packet_set_name + ) + except Exception as exc: + raise ArchiveDecodeError(f"could not load dictionary: {exc}") from exc + + if not self._dictionaries.packet: + raise ArchiveDecodeError("dictionary contains no telemetry packet set") + + self._config = ConfigManager() + self._time_type = TimeType + self._decoder = PktDecoder( + self._dictionaries.packet, self._dictionaries.channel_id + ) + + def decode(self, record: ArchiveRecord) -> DecodedPacket: + """Decode one validated archive record.""" + + payload = record.payload + descriptor_type = self._config.get_type("FwPacketDescriptorType")() + packet_id_type = self._config.get_type("FwTlmPacketizeIdType")() + + minimum_size = ( + descriptor_type.getSize() + + packet_id_type.getSize() + + self._time_type.getSize() + ) + if len(payload) < minimum_size: + raise ArchiveDecodeError( + f"record {record.record_number}: packet is {len(payload)} bytes; " + f"the packetized telemetry header requires {minimum_size}" + ) + + try: + descriptor_type.deserialize(payload, 0) + descriptor = descriptor_type.val + descriptor_size = descriptor_type.getSize() + packet_id_type.deserialize(payload, descriptor_size) + packet_id = packet_id_type.val + except Exception as exc: + raise ArchiveDecodeError( + f"record {record.record_number}: invalid packet header: {exc}" + ) from exc + + if descriptor != PACKETIZED_TLM_DESCRIPTOR: + raise ArchiveDecodeError( + f"record {record.record_number}: descriptor 0x{descriptor:04X} is not " + "packetized telemetry (0x0004)" + ) + if packet_id not in self._dictionaries.packet: + raise ArchiveDecodeError( + f"record {record.record_number}: packet ID {packet_id} is not in the dictionary" + ) + + packet_template = self._dictionaries.packet[packet_id] + expected_size = minimum_size + sum( + channel.get_type_obj().getMaxSize() + for channel in packet_template.get_ch_list() + ) + if len(payload) != expected_size: + raise ArchiveDecodeError( + f"record {record.record_number}: packet ID {packet_id} is {len(payload)} " + f"bytes; dictionary expects {expected_size}" + ) + + packet_time = self._time_type() + try: + packet_time.deserialize(payload, descriptor_size + packet_id_type.getSize()) + channel_data = self._decoder.decode_api(payload[descriptor_size:]) + except Exception as exc: + raise ArchiveDecodeError( + f"record {record.record_number}: packet ID {packet_id} failed to decode: {exc}" + ) from exc + + channels = tuple( + { + "name": channel.template.get_full_name(), + "id": channel.id, + "value": channel.get_val(), + "display": str(channel.get_display_text()), + } + for channel in channel_data + ) + return DecodedPacket( + record_number=record.record_number, + packet_id=packet_id, + packet_name=packet_template.get_name(), + packet_size_bytes=len(payload), + time_base=str(packet_time.timeBase.val), + time_context=packet_time.timeContext, + seconds=packet_time.seconds, + microseconds=packet_time.useconds, + channels=channels, + ) + + +def packet_to_dict(packet: DecodedPacket) -> dict[str, Any]: + """Convert a decoded packet to JSON-compatible primitives.""" + + return { + "record_number": packet.record_number, + "packet_id": packet.packet_id, + "packet_name": packet.packet_name, + "packet_size_bytes": packet.packet_size_bytes, + "timestamp": { + "time_base": packet.time_base, + "context": packet.time_context, + "seconds": packet.seconds, + "microseconds": packet.microseconds, + }, + "channels": list(packet.channels), + } + + +def write_text(packets: Iterable[DecodedPacket], stream: TextIO) -> None: + """Write decoded packets in a compact human-readable form.""" + + for packet_index, packet in enumerate(packets): + if packet_index: + stream.write("\n") + stream.write( + f"Packet {packet.record_number}: {packet.packet_name} " + f"(ID {packet.packet_id}, {packet.packet_size_bytes} bytes)\n" + ) + stream.write( + f" Time: {packet.time_base}, context {packet.time_context}, " + f"{packet.seconds}.{packet.microseconds:06d}\n" + ) + for channel in packet.channels: + stream.write(f" {channel['name']} = {channel['display']}\n") + + +def write_csv(packets: Iterable[DecodedPacket], stream: TextIO) -> None: + """Write one long-form CSV row per decoded telemetry channel.""" + + writer = csv.writer(stream, lineterminator="\n") + writer.writerow( + ( + "record_number", + "packet_name", + "packet_id", + "time_base", + "time_context", + "seconds", + "microseconds", + "channel_name", + "channel_id", + "value", + ) + ) + for packet in packets: + for channel in packet.channels: + writer.writerow( + ( + packet.record_number, + packet.packet_name, + packet.packet_id, + packet.time_base, + packet.time_context, + packet.seconds, + packet.microseconds, + channel["name"], + channel["id"], + json.dumps(channel["value"], separators=(",", ":")), + ) + ) + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + """Parse command-line arguments.""" + + parser = argparse.ArgumentParser( + description="Decode pre_deployment.csv into named telemetry values." + ) + parser.add_argument("archive", type=Path, help="downloaded TlmArchive CSV file") + parser.add_argument( + "dictionary", type=Path, help="matching F Prime topology dictionary JSON" + ) + parser.add_argument( + "--packet-set", + help="telemetry packet set name (only needed when the dictionary has several)", + ) + parser.add_argument( + "--format", + choices=("text", "csv", "json"), + default="text", + help="output format (default: text)", + ) + parser.add_argument("--output", type=Path, help="write output to this file") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + """Run the archive decoder command.""" + + args = parse_args(argv) + try: + decoder = TelemetryDecoder(args.dictionary, args.packet_set) + with args.archive.open("r", encoding="ascii", newline="") as archive_stream: + packets = [ + decoder.decode(record) for record in read_archive(archive_stream) + ] + + output_stream = ( + args.output.open("w", encoding="utf-8", newline="") + if args.output + else sys.stdout + ) + try: + if args.format == "text": + write_text(packets, output_stream) + elif args.format == "csv": + write_csv(packets, output_stream) + else: + json.dump( + [packet_to_dict(packet) for packet in packets], + output_stream, + indent=2, + ) + output_stream.write("\n") + finally: + if args.output: + output_stream.close() + except (ArchiveDecodeError, OSError) as exc: + print(f"error: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From b2ef32709f83f2598f25588f97cb5883fafc0ca6 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sat, 1 Aug 2026 21:59:30 -0500 Subject: [PATCH 26/38] bump max size to 25000 --- .../Components/TlmArchive/TlmArchive.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 5790422c..173f8505 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -23,7 +23,7 @@ constexpr char CSV_HEADER[] = "format_version,packet_size_bytes,packet_hex\n"; constexpr char HEX_DIGITS[] = "0123456789ABCDEF"; constexpr FwSizeType CSV_RECORD_BUFFER_SIZE = (FW_COM_BUFFER_MAX_SIZE * 2) + sizeof(CSV_HEADER) + 32; constexpr const int MAX_FAILURES = 3; -constexpr const FwSizeType MAX_FILE_SIZE = 10000; +constexpr const FwSizeType MAX_FILE_SIZE = 25000; } // namespace From 3a72539ff038ed4d5da8c93be66e0248310efffb Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sat, 1 Aug 2026 22:26:45 -0500 Subject: [PATCH 27/38] delete random test --- .../Components/TlmArchive/TlmArchive.cpp | 2 +- .../Components/TlmArchive/docs/sdd.md | 29 ++++--- .../test/tools/test_decode_tlm_archive.py | 75 ------------------- 3 files changed, 15 insertions(+), 91 deletions(-) delete mode 100644 PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 173f8505..922f27a1 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -128,7 +128,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { } csvRecord[recordSize++] = '\n'; - if ((currentSize > MAX_FILE_SIZE) || (recordSize > (MAX_FILE_SIZE - currentSize))) { + if (currentSize > MAX_FILE_SIZE) { this->m_failures.fetch_add(1); return; } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index 8109cbc9..5898d4b4 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -27,8 +27,8 @@ The reference topology connects the component as follows: Archiving stops for the remainder of the component's lifetime after it observes a deployed antenna state. `comIn` also rejects new packets after three counted failures or when the cached archive size reaches 10,000 bytes. A -pending packet is rejected and counted as a failure when appending it would -exceed the size limit. +packet that brings the archive to or above the threshold is written; subsequent +packets are rejected by `comIn` and report the size-limit event. ## Usage Examples @@ -117,7 +117,7 @@ the following internal state: | `m_pendingPacket` | Empty buffer | Storage for the single pending telemetry packet. | | `m_packetPending` | `false` | Indicates whether the mailbox contains a packet. | | `m_fileSize` | `0` | Atomic cached archive size. Initialized once from the filesystem and advanced by the actual bytes written after each successful write. | -| `m_failures` | `0` | Atomic cumulative count of directory, stat, size-limit, open, and write failures. | +| `m_failures` | `0` | Atomic cumulative count of directory, stat, oversized-existing-archive, open, and write failures. | | `m_directoryInitialized` | `false` | Becomes true after both `//tlm` and `//tlm/pre_deployment.csv` are successfully initialized, preventing repeated creation attempts. | | `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read, preventing later size queries. | | `m_antennasDeployed` | `false` | Atomic in-memory latch set when AntennaDeployer first reports a deployed state. | @@ -147,9 +147,9 @@ truncating an existing archive. Before the first append attempt, the component reads the existing on-disk size. Stat failures are reported and counted. After a successful size initialization, the component uses the cached size rather than querying the -filesystem again. The pending packet is rejected when its encoded CSV record, -including the header when necessary, would make the cached archive size exceed -10,000 bytes. +filesystem again. Once a successful write brings the cached archive size to or +above 10,000 bytes, subsequent packets are rejected by `comIn`. The final write +may therefore make the archive larger than the threshold by one CSV record. After a successful write, `m_fileSize` is incremented by the actual byte count reported by the file API. The archive is not truncated or deleted when the @@ -168,15 +168,14 @@ The following failures increment `m_failures`: - failure to create `//tlm`; - failure to create or open `//tlm/pre_deployment.csv` during initialization; - failure to read the size of an existing archive; -- rejection of a packet that would exceed the archive size limit; +- rejection when the archive was already larger than the size threshold at initialization; - failure to open the archive for append; and - a failed or short file write. Once three counted failures have occurred, subsequent `comIn` calls reject new -packets. The packet that encountered an error is not retried. A size-limit -rejection in `run` increments the failure count without emitting an event; -`FailureLimitReached` is emitted if a later `comIn` call observes the failure -cutoff. +packets. The packet that encountered an error is not retried. An archive that +is already larger than the threshold when its size is first read is rejected in +`run` and increments the failure count without emitting an event. ## Sequence Diagrams @@ -223,9 +222,9 @@ sequenceDiagram end alt Stat fails Archive->>Archive: Count failure and discard packet - else Packet exceeds size limit + else Existing archive exceeds size threshold Archive->>Archive: Count failure and discard packet - else Packet fits + else Packet is ready Archive->>Archive: Emit WriteStart Archive->>FS: open("//tlm/pre_deployment.csv", append) Archive->>FS: write(CSV header if empty + one packet record) @@ -246,7 +245,7 @@ implementation constants control its behavior: | `TLM_DIRECTORY` | `//tlm` | Directory containing the archive. | | `PRE_DEPLOYMENT_TLM_PATH` | `//tlm/pre_deployment.csv` | Append-only, one-packet-per-row pre-deployment telemetry archive. | | `MAX_FAILURES` | `3` | Counted stat, limit, directory, open, or write failures after which new packets are rejected. | -| `MAX_FILE_SIZE` | `10000` bytes | Maximum cached archive size permitted after a component-managed append. | +| `MAX_FILE_SIZE` | `10000` bytes | Cached archive-size threshold after which subsequent packets are rejected. One final record may cross the threshold. | ## Commands @@ -288,7 +287,7 @@ There are currently no component-specific unit tests for TlmArchive. | `TLM_ARCHIVE_002` | The component shall append buffered telemetry as versioned, one-packet-per-row CSV records to `//tlm/pre_deployment.csv` while the antenna deployment state is false. | Inspection | | `TLM_ARCHIVE_003` | The component shall stop writing telemetry after observing a deployed antenna state. | Inspection | | `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | -| `TLM_ARCHIVE_005` | The component shall reject a write when the cached archive size plus the encoded CSV record would exceed 10,000 bytes. | Inspection | +| `TLM_ARCHIVE_005` | The component shall reject new packets after the cached archive size reaches 10,000 bytes. The write that crosses the threshold is permitted. | Inspection | | `TLM_ARCHIVE_006` | The component shall report archive start, filesystem errors, write errors, failure-limit rejection, size-limit rejection, and deployed-state rejection through events. | Inspection | | `TLM_ARCHIVE_007` | The component shall create the archive when missing without truncating an existing archive. | Inspection | diff --git a/PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py b/PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py deleted file mode 100644 index 238ee650..00000000 --- a/PROVESFlightControllerReference/test/tools/test_decode_tlm_archive.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Tests for the ground-side telemetry archive decoder.""" - -from __future__ import annotations - -import io -from pathlib import Path - -import pytest - -from tools.decode_tlm_archive import ( - ArchiveDecodeError, - TelemetryDecoder, - read_archive, -) - -PROJECT_ROOT = Path(__file__).parents[3] -DICTIONARY = ( - PROJECT_ROOT - / "build-artifacts/zephyr/fprime-zephyr-deployment/dict" - / "ReferenceDeploymentTopologyDictionary.json" -) - - -def make_filesystem_packet() -> bytes: - """Create a real packet ID 5 using types loaded from the flight dictionary.""" - - from fprime_gds.common.models.dictionaries import Dictionaries - from fprime_gds.common.models.serialize.time_type import TimeType - from fprime_gds.common.utils.config_manager import ConfigManager - - dictionaries = Dictionaries.load_dictionaries_into_config(str(DICTIONARY)) - config = ConfigManager() - packet = dictionaries.packet[5] - return b"".join( - ( - config.get_type("FwPacketDescriptorType")(4).serialize(), - config.get_type("FwTlmPacketizeIdType")(5).serialize(), - TimeType(TimeType.TimeBase("TB_PROC_TIME"), 2, 123, 456).serialize(), - packet.get_ch_list()[0].get_type_obj()(4096).serialize(), - packet.get_ch_list()[1].get_type_obj()(8192).serialize(), - ) - ) - - -@pytest.mark.skipif(not DICTIONARY.is_file(), reason="generated dictionary unavailable") -def test_decode_packet_uses_dictionary_names_and_types() -> None: - """Decode packet boundaries, metadata, and typed channel values.""" - - payload = make_filesystem_packet() - archive = io.StringIO( - "format_version,packet_size_bytes,packet_hex\n" - f"1,{len(payload)},{payload.hex().upper()}\n" - ) - record = next(iter(read_archive(archive))) - decoded = TelemetryDecoder(DICTIONARY).decode(record) - - assert decoded.packet_name == "FileSystem" - assert decoded.packet_id == 5 - assert decoded.time_base == "TB_PROC_TIME" - assert decoded.time_context == 2 - assert decoded.seconds == 123 - assert decoded.microseconds == 456 - assert [channel["name"] for channel in decoded.channels] == [ - "ReferenceDeployment.fsSpace.FreeSpace", - "ReferenceDeployment.fsSpace.TotalSpace", - ] - assert [channel["value"] for channel in decoded.channels] == [4096, 8192] - - -def test_archive_rejects_size_mismatch() -> None: - """Reject truncated rows before asking F Prime to deserialize them.""" - - archive = io.StringIO("format_version,packet_size_bytes,packet_hex\n1,3,0001\n") - with pytest.raises(ArchiveDecodeError, match="declared 3 packet bytes, decoded 2"): - list(read_archive(archive)) From 4100a6cceecebe7a41326a2863c74f80ad0068f6 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sat, 1 Aug 2026 23:22:45 -0500 Subject: [PATCH 28/38] queue and filter updates --- .../Components/TlmArchive/TlmArchive.cpp | 69 ++++++++++-- .../Components/TlmArchive/TlmArchive.fpp | 12 ++- .../Components/TlmArchive/TlmArchive.hpp | 11 +- .../Components/TlmArchive/docs/sdd.md | 102 +++++++++++------- 4 files changed, 145 insertions(+), 49 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 922f27a1..c97449f7 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -9,6 +9,7 @@ #include #include +#include "Fw/Com/ComPacket.hpp" #include "Os/File.hpp" #include "Os/FileSystem.hpp" #include "Os/Models/FileStatusEnumAc.hpp" @@ -25,6 +26,31 @@ constexpr FwSizeType CSV_RECORD_BUFFER_SIZE = (FW_COM_BUFFER_MAX_SIZE * 2) + siz constexpr const int MAX_FAILURES = 3; constexpr const FwSizeType MAX_FILE_SIZE = 25000; +// Only packetized telemetry IDs in this list are stored in the archive. +constexpr FwTlmPacketizeIdType STORED_PACKET_IDS[] = { + 1, // Beacon + 7, // Imu +}; + +bool shouldStorePacket(const Fw::ComBuffer& data) { + Fw::ComBuffer packet = data; + FwPacketDescriptorType descriptor = 0; + FwTlmPacketizeIdType packetId = 0; + + if ((packet.deserializeTo(descriptor) != Fw::FW_SERIALIZE_OK) || + (descriptor != static_cast(Fw::ComPacketType::FW_PACKET_PACKETIZED_TLM)) || + (packet.deserializeTo(packetId) != Fw::FW_SERIALIZE_OK)) { + return false; + } + + for (const FwTlmPacketizeIdType storedPacketId : STORED_PACKET_IDS) { + if (packetId == storedPacketId) { + return true; + } + } + return false; +} + } // namespace TlmArchive ::TlmArchive(const char* const compName) : TlmArchiveComponentBase(compName) {} @@ -46,10 +72,26 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con return; } + if (!shouldStorePacket(data)) { + return; + } + + bool queueFull = false; { Os::ScopeLock lock(this->m_queueMutex); - this->m_pendingPacket = data; - this->m_packetPending = true; + if (this->m_queueSize >= PACKET_QUEUE_CAPACITY) { + queueFull = true; + } else { + this->m_packetQueue[this->m_queueTail] = data; + this->m_queueTail = (this->m_queueTail + 1) % PACKET_QUEUE_CAPACITY; + this->m_queueSize++; + } + } + + if (queueFull) { + this->log_WARNING_HI_QueueFull(PACKET_QUEUE_CAPACITY); + } else { + this->log_WARNING_HI_QueueFull_ThrottleClear(); } } @@ -58,13 +100,28 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { (void)context; Fw::ComBuffer packet; + bool queueEmpty = false; + bool clearQueueEmptyThrottle = false; { Os::ScopeLock lock(this->m_queueMutex); - if (!this->m_packetPending) { - return; + if (this->m_queueSize == 0) { + this->m_queueWasEmpty = true; + queueEmpty = true; + } else { + packet = this->m_packetQueue[this->m_queueHead]; + this->m_queueHead = (this->m_queueHead + 1) % PACKET_QUEUE_CAPACITY; + this->m_queueSize--; + clearQueueEmptyThrottle = this->m_queueWasEmpty; + this->m_queueWasEmpty = false; } - packet = this->m_pendingPacket; - this->m_packetPending = false; + } + + if (queueEmpty) { + this->log_ACTIVITY_LO_QueueEmpty(); + return; + } + if (clearQueueEmptyThrottle) { + this->log_ACTIVITY_LO_QueueEmpty_ThrottleClear(); } if (!this->m_antennasDeployed.load()) { diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 21cccb67..1d7a39aa 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -1,10 +1,10 @@ module Components { @ Stores telemetry generated before antenna deployment passive component TlmArchive { - @ Telemetry packet to buffer. Drop new packets while the one-packet mailbox is full. + @ Telemetry packet to enqueue for deferred archival sync input port comIn: Fw.Com - @ Drains the telemetry mailbox and performs filesystem work + @ Drains one queued telemetry packet and performs filesystem work sync input port run: Svc.Sched @ Port for checking whether antenna deployment has completed @@ -13,6 +13,14 @@ module Components { @ Report when file write has started event WriteStart severity activity low format "Beginning telemetry archival to pre_deployment.csv" throttle 1 + @ Reports an empty packet queue. The throttle is cleared after a later dequeue succeeds. + event QueueEmpty() severity activity low format "Telemetry archive packet queue is empty" throttle 1 + + @ Reports that a packet could not be enqueued because the queue is full + event QueueFull( + capacity: FwSizeType @< Maximum number of queued telemetry packets + ) severity warning high format "Telemetry archive packet queue is full at {} packets" throttle 1 + @ Reports archive initialization, size, and open failures event FileError( operation: string @< Filesystem operation that failed diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 74ed373f..15651c14 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -11,6 +11,7 @@ #include "Os/Mutex.hpp" #include "PROVESFlightControllerReference/Components/TlmArchive/TlmArchiveComponentAc.hpp" +#include "PROVESFlightControllerReference/project/config/TlmPacketizerCfg.hpp" namespace Components { @@ -24,17 +25,23 @@ class TlmArchive final : public TlmArchiveComponentBase { ~TlmArchive(); private: + static constexpr FwSizeType PACKET_QUEUE_CAPACITY = Svc::MAX_PACKETIZER_PACKETS; + static_assert(PACKET_QUEUE_CAPACITY > 0, "Telemetry archive packet queue must have storage"); + void comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 context) override; void run_handler(FwIndexType portNum, U32 context) override; Os::Mutex m_queueMutex; - Fw::ComBuffer m_pendingPacket; + Fw::ComBuffer m_packetQueue[PACKET_QUEUE_CAPACITY]; + FwSizeType m_queueHead = 0; + FwSizeType m_queueTail = 0; + FwSizeType m_queueSize = 0; std::atomic m_fileSize{0}; std::atomic m_failures{0}; std::atomic m_antennasDeployed{false}; bool m_directoryInitialized = false; bool m_fileSizeInitialized = false; - bool m_packetPending = false; + bool m_queueWasEmpty = false; }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index 5898d4b4..bd11a2e3 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -4,15 +4,17 @@ The TlmArchive component preserves telemetry generated before antenna deployment by appending serialized telemetry packets to `//tlm/pre_deployment.csv`. Each packet is stored as one self-delimiting CSV record containing the archive format version, packet byte count, and uppercase -hexadecimal packet payload. It uses a one-packet, latest-value mailbox to move -filesystem work out of the telemetry path and into the 1 Hz rate group. +hexadecimal packet payload. It uses a fixed-capacity FIFO to move filesystem +work out of the telemetry path and into the 1 Hz rate group without dropping +the other packets in a packetizer burst. A compile-time packet ID allowlist +filters packets before they consume queue space. ## Overview TlmArchive is a passive component with two execution paths. The synchronous -`comIn` handler receives telemetry and stores the most recent packet in an -in-memory mailbox. The synchronous `run` handler drains that mailbox and -performs the filesystem work. A mutex protects the non-atomic packet mailbox, +`comIn` handler receives telemetry and appends it to an in-memory ring queue. +The synchronous `run` handler removes one packet from the queue and performs +the filesystem work. A mutex protects the queue storage and indices, while atomics synchronize the cached size, failure count, and deployment latch shared by the two calling contexts. @@ -26,7 +28,7 @@ The reference topology connects the component as follows: Archiving stops for the remainder of the component's lifetime after it observes a deployed antenna state. `comIn` also rejects new packets after three -counted failures or when the cached archive size reaches 10,000 bytes. A +counted failures or when the cached archive size reaches 25,000 bytes. A packet that brings the archive to or above the threshold is written; subsequent packets are rejected by `comIn` and report the size-limit event. @@ -38,16 +40,20 @@ The component is instantiated and connected during topology setup; it does not require an explicit configuration call. 1. A serialized telemetry packet arrives on `comIn`. -2. If writing is enabled, the packet is copied into the mailbox. A new packet - arriving while another packet is pending replaces the older packet. -3. On the next 1 Hz `run` call, the component removes the pending packet and +2. If writing is enabled, the component reads the F Prime packet descriptor and + packet ID. Malformed packets, non-packetized telemetry, and packet IDs absent + from `STORED_PACKET_IDS` are discarded. +3. If the packet passes the filter and the queue has room, it is appended to the + FIFO. The queue is sized to `Svc::MAX_PACKETIZER_PACKETS`, allowing one + complete packetizer send cycle to be buffered. +4. On the next 1 Hz `run` call, the component removes the oldest packet and queries the antenna deployment state. -4. If the antenna is not deployed, the component creates `//tlm` and +5. If the antenna is not deployed, the component creates `//tlm` and `//tlm/pre_deployment.csv` when needed, reads the existing archive size if it - has not already been initialized, verifies that the pending packet fits, opens + has not already been initialized, verifies that the queued packet fits, opens `//tlm/pre_deployment.csv` in append mode, writes one CSV record, and closes the file. The CSV header is included with the first record in an empty file. -5. If the antenna is deployed, the pending packet is discarded without +6. If the antenna is deployed, the dequeued packet is discarded without filesystem access. The true deployment state is latched so it is not queried again, and subsequent `comIn` calls reject new packets. @@ -64,9 +70,11 @@ one packet; `packet_size_bytes` independently detects truncated or malformed rows. `tools/decode_tlm_archive.py` combines these records with the generated F Prime topology dictionary to produce named, typed telemetry values. -The one-packet mailbox prevents filesystem access from blocking the telemetry -producer. Because `run` executes at 1 Hz, at most one buffered packet is -processed per rate-group tick. +The queue prevents filesystem access from blocking the telemetry producer. +Because `run` executes at 1 Hz, one buffered packet is processed per rate-group +tick. `QueueEmpty` reports the transition into an empty period once; its +throttle is cleared after a later dequeue succeeds so the next empty period is +reported again. ## Class Diagram @@ -83,13 +91,16 @@ classDiagram -comIn_handler(portNum: FwIndexType, data: Fw.ComBuffer, context: U32) -run_handler(portNum: FwIndexType, context: U32) -m_queueMutex: Os.Mutex - -m_pendingPacket: Fw.ComBuffer + -m_packetQueue: Fw.ComBuffer[] + -m_queueHead: FwSizeType + -m_queueTail: FwSizeType + -m_queueSize: FwSizeType -m_fileSize: atomic~U32~ -m_failures: atomic~int~ -m_antennasDeployed: atomic~bool~ -m_directoryInitialized: bool -m_fileSizeInitialized: bool - -m_packetPending: bool + -m_queueWasEmpty: bool } } @@ -100,8 +111,8 @@ classDiagram | Name | Type | Direction | Description | |---|---|---|---| -| `comIn` | `Fw.Com` | sync input | Receives serialized telemetry packets. When writing is enabled, stores the packet in the mailbox, replacing any packet already pending. | -| `run` | `Svc.Sched` | sync input | Drains one pending packet, checks deployment state, and performs archive filesystem work. Connected to the 1 Hz rate group. | +| `comIn` | `Fw.Com` | sync input | Receives serialized telemetry packets and appends allowlisted packet IDs to the bounded FIFO when writing is enabled. | +| `run` | `Svc.Sched` | sync input | Drains the oldest queued packet, checks deployment state, and performs archive filesystem work. Connected to the 1 Hz rate group. | | `deploymentStateGet` | `Components.GetDeploymentState` | output | Queries AntennaDeployer for its persistent deployed state. | | `timeCaller` | time get | time get | Supplies timestamps for emitted events. | | `logOut` | event | output | Sends binary event records. | @@ -114,23 +125,26 @@ the following internal state: | Name | Initial value | Description | |---|---:|---| -| `m_pendingPacket` | Empty buffer | Storage for the single pending telemetry packet. | -| `m_packetPending` | `false` | Indicates whether the mailbox contains a packet. | +| `m_packetQueue` | Empty buffers | Ring storage sized to `Svc::MAX_PACKETIZER_PACKETS` packets. | +| `m_queueHead` | `0` | Index of the next packet to dequeue. | +| `m_queueTail` | `0` | Index at which the next packet is enqueued. | +| `m_queueSize` | `0` | Number of packets currently waiting in the FIFO. | | `m_fileSize` | `0` | Atomic cached archive size. Initialized once from the filesystem and advanced by the actual bytes written after each successful write. | | `m_failures` | `0` | Atomic cumulative count of directory, stat, oversized-existing-archive, open, and write failures. | | `m_directoryInitialized` | `false` | Becomes true after both `//tlm` and `//tlm/pre_deployment.csv` are successfully initialized, preventing repeated creation attempts. | | `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read, preventing later size queries. | | `m_antennasDeployed` | `false` | Atomic in-memory latch set when AntennaDeployer first reports a deployed state. | -| `m_queueMutex` | Unlocked | Protects the pending packet and its availability flag while the packet is copied between calling contexts. | +| `m_queueMutex` | Unlocked | Protects the ring storage, indices, count, and empty-period state. | +| `m_queueWasEmpty` | `false` | Records that an empty period was observed so the `QueueEmpty` throttle can be cleared after the next successful dequeue. | Conceptually, the component operates in these states: | Name | Description | |---|---| -| `WAITING` | Writing is enabled and no packet is pending. | -| `PACKET_PENDING` | One packet is waiting for the next `run` invocation. A newer packet replaces the pending packet. | +| `WAITING` | Writing is enabled and the packet queue is empty. | +| `PACKETS_QUEUED` | One or more packets are waiting and are processed oldest-first, one per `run` invocation. | | `DEPLOYED` | A true antenna deployment state has been latched. The packet that observed deployment is discarded, and subsequent packets are rejected by `comIn`. | -| `WRITE_DISABLED` | The failure count has reached its limit or the cached file size has reached 10,000 bytes. New packets are rejected by `comIn`. | +| `WRITE_DISABLED` | The failure count has reached its limit or the cached file size has reached 25,000 bytes. New packets are rejected by `comIn`. | The `run` handler does not emit an event when it first observes deployment. The throttled `AntennasDeployed` event is emitted if another packet later @@ -148,7 +162,7 @@ Before the first append attempt, the component reads the existing on-disk size. Stat failures are reported and counted. After a successful size initialization, the component uses the cached size rather than querying the filesystem again. Once a successful write brings the cached archive size to or -above 10,000 bytes, subsequent packets are rejected by `comIn`. The final write +above 25,000 bytes, subsequent packets are rejected by `comIn`. The final write may therefore make the archive larger than the threshold by one CSV record. After a successful write, `m_fileSize` is incremented by the actual byte count @@ -157,7 +171,7 @@ limit is reached. Changes made to the archive by another component after size initialization are not reflected in the cache. The atomic cache uses the target's native 32-bit `U32` width. This is lossless -for all permitted archive sizes; an existing on-disk size above the 10,000-byte +for all permitted archive sizes; an existing on-disk size above the 25,000-byte limit is represented by the limit value so packet admission remains disabled. ### Failure Handling @@ -198,14 +212,16 @@ sequenceDiagram else Deployment latched Archive->>Archive: Emit AntennasDeployed else Writing enabled - Archive->>Archive: Store packet, replacing pending packet + Archive->>Archive: Append packet to FIFO end Rate->>Archive: run() - alt No packet pending + alt Queue empty + Archive->>Archive: Emit QueueEmpty (throttled) Archive-->>Rate: Return - else Packet pending - Archive->>Archive: Remove packet from mailbox + else Packet queued + Archive->>Archive: Remove oldest packet from FIFO + Archive->>Archive: Clear QueueEmpty throttle after an empty period opt Deployment state not yet latched Archive->>Deploy: deploymentStateGet() Deploy-->>Archive: deployed @@ -244,8 +260,10 @@ implementation constants control its behavior: |---|---:|---| | `TLM_DIRECTORY` | `//tlm` | Directory containing the archive. | | `PRE_DEPLOYMENT_TLM_PATH` | `//tlm/pre_deployment.csv` | Append-only, one-packet-per-row pre-deployment telemetry archive. | +| `STORED_PACKET_IDS` | All currently configured telemetry packet IDs | Compile-time allowlist in `TlmArchive.cpp`. Packets whose IDs are absent are discarded before enqueueing. | +| `PACKET_QUEUE_CAPACITY` | `Svc::MAX_PACKETIZER_PACKETS` (currently 22) | Maximum packets held between the telemetry producer and 1 Hz filesystem worker. | | `MAX_FAILURES` | `3` | Counted stat, limit, directory, open, or write failures after which new packets are rejected. | -| `MAX_FILE_SIZE` | `10000` bytes | Cached archive-size threshold after which subsequent packets are rejected. One final record may cross the threshold. | +| `MAX_FILE_SIZE` | `25000` bytes | Cached archive-size threshold after which subsequent packets are rejected. One final record may cross the threshold. | ## Commands @@ -258,16 +276,19 @@ implementation constants control its behavior: | Name | Severity | Throttle | Parameters | Description | |---|---|---:|---|---| | `WriteStart` | Activity Low | 1 | None | Emitted immediately before each attempt to open the archive. | +| `QueueEmpty` | Activity Low | 1 | None | Emitted once per empty period. Its throttle is cleared when a later dequeue succeeds. | +| `QueueFull` | Warning High | 1 | `capacity: FwSizeType` | Emitted when an incoming packet cannot be enqueued. Its throttle is cleared after a later enqueue succeeds. | | `FileError` | Warning High | None | `operation: string` | Reports directory creation, archive creation, record formatting, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `create_file`, `format_record`, `get_file_size`, and `open_append`. | | `WriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | | `FailureLimitReached` | Warning High | 1 | `count: I8` | Emitted by `comIn` when the cumulative failure count is at least three. The implementation supplies `3`. | | `AntennasDeployed` | Warning Low | 1 | None | Emitted by `comIn` when deployment has been latched and a new packet is rejected. | -| `SizeLimitReached` | Warning Low | 1 | `maxSize: FwSizeType` | Emitted by `comIn` when the cached archive size is at least 10,000 bytes. The implementation supplies `10000`. | +| `SizeLimitReached` | Warning Low | 1 | `maxSize: FwSizeType` | Emitted by `comIn` when the cached archive size is at least 25,000 bytes. The implementation supplies `25000`. | -The throttled events have no corresponding throttle-clear calls, so only their -first occurrence is reported during the component's lifetime. The `comIn` -checks are ordered failure limit, size limit, then antenna deployment; if more -than one condition is true, only the first applicable event is invoked. +`QueueEmpty` and `QueueFull` have explicit throttle-clear calls so each distinct +empty or full period can be reported. The other throttled events report only +their first occurrence during the component's lifetime. The `comIn` checks are +ordered failure limit, size limit, then antenna deployment; if more than one +condition is true, only the first applicable event is invoked. ## Telemetry @@ -283,13 +304,14 @@ There are currently no component-specific unit tests for TlmArchive. | Name | Description | Validation | |---|---|---| -| `TLM_ARCHIVE_001` | The component shall buffer at most one telemetry packet outside the telemetry producer's filesystem path, replacing a pending packet when newer telemetry arrives. | Inspection | +| `TLM_ARCHIVE_001` | The component shall enqueue telemetry packets in arrival order in a bounded FIFO sized for one complete packetizer send cycle. | Inspection | | `TLM_ARCHIVE_002` | The component shall append buffered telemetry as versioned, one-packet-per-row CSV records to `//tlm/pre_deployment.csv` while the antenna deployment state is false. | Inspection | | `TLM_ARCHIVE_003` | The component shall stop writing telemetry after observing a deployed antenna state. | Inspection | | `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | -| `TLM_ARCHIVE_005` | The component shall reject new packets after the cached archive size reaches 10,000 bytes. The write that crosses the threshold is permitted. | Inspection | +| `TLM_ARCHIVE_005` | The component shall reject new packets after the cached archive size reaches 25,000 bytes. The write that crosses the threshold is permitted. | Inspection | | `TLM_ARCHIVE_006` | The component shall report archive start, filesystem errors, write errors, failure-limit rejection, size-limit rejection, and deployed-state rejection through events. | Inspection | | `TLM_ARCHIVE_007` | The component shall create the archive when missing without truncating an existing archive. | Inspection | +| `TLM_ARCHIVE_008` | The component shall enqueue only valid packetized telemetry whose packet ID appears in the compile-time `STORED_PACKET_IDS` allowlist. | Inspection | ## Change Log @@ -299,3 +321,5 @@ There are currently no component-specific unit tests for TlmArchive. | 2026-07-27 | Updated mailbox replacement behavior, non-destructive storage initialization, cached-size handling, rejection paths, and the current event interface. | | 2026-07-30 | Replaced mutex-protected scalar state updates with atomics; retained the mutex only for packet mailbox copies. | | 2026-08-01 | Replaced the concatenated binary archive with a versioned, one-packet-per-row CSV archive and documented the dictionary-backed ground decoder. | +| 2026-08-01 | Replaced the latest-value mailbox with a bounded FIFO and added repeatable empty/full queue diagnostics. | +| 2026-08-01 | Added a compile-time packet ID allowlist that filters telemetry before enqueueing. | From 2eb632b82af75bec07e65a777a3d54355f20faf4 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 2 Aug 2026 00:09:57 -0500 Subject: [PATCH 29/38] increase size --- .../Components/TlmArchive/TlmArchive.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index c97449f7..587927ad 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -24,7 +24,7 @@ constexpr char CSV_HEADER[] = "format_version,packet_size_bytes,packet_hex\n"; constexpr char HEX_DIGITS[] = "0123456789ABCDEF"; constexpr FwSizeType CSV_RECORD_BUFFER_SIZE = (FW_COM_BUFFER_MAX_SIZE * 2) + sizeof(CSV_HEADER) + 32; constexpr const int MAX_FAILURES = 3; -constexpr const FwSizeType MAX_FILE_SIZE = 25000; +constexpr const FwSizeType MAX_FILE_SIZE = 50000; // Only packetized telemetry IDs in this list are stored in the archive. constexpr FwTlmPacketizeIdType STORED_PACKET_IDS[] = { From 8165f2833a186e053772772e92adf277500389e9 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 2 Aug 2026 00:13:11 -0500 Subject: [PATCH 30/38] remove queue empty event --- .../Components/TlmArchive/TlmArchive.cpp | 22 ++++--------------- .../Components/TlmArchive/TlmArchive.fpp | 3 --- .../Components/TlmArchive/TlmArchive.hpp | 1 - .../Components/TlmArchive/docs/sdd.md | 22 +++++++------------ 4 files changed, 12 insertions(+), 36 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 587927ad..1db8570e 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -100,28 +100,14 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { (void)context; Fw::ComBuffer packet; - bool queueEmpty = false; - bool clearQueueEmptyThrottle = false; { Os::ScopeLock lock(this->m_queueMutex); if (this->m_queueSize == 0) { - this->m_queueWasEmpty = true; - queueEmpty = true; - } else { - packet = this->m_packetQueue[this->m_queueHead]; - this->m_queueHead = (this->m_queueHead + 1) % PACKET_QUEUE_CAPACITY; - this->m_queueSize--; - clearQueueEmptyThrottle = this->m_queueWasEmpty; - this->m_queueWasEmpty = false; + return; } - } - - if (queueEmpty) { - this->log_ACTIVITY_LO_QueueEmpty(); - return; - } - if (clearQueueEmptyThrottle) { - this->log_ACTIVITY_LO_QueueEmpty_ThrottleClear(); + packet = this->m_packetQueue[this->m_queueHead]; + this->m_queueHead = (this->m_queueHead + 1) % PACKET_QUEUE_CAPACITY; + this->m_queueSize--; } if (!this->m_antennasDeployed.load()) { diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 1d7a39aa..4c1ae78e 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -13,9 +13,6 @@ module Components { @ Report when file write has started event WriteStart severity activity low format "Beginning telemetry archival to pre_deployment.csv" throttle 1 - @ Reports an empty packet queue. The throttle is cleared after a later dequeue succeeds. - event QueueEmpty() severity activity low format "Telemetry archive packet queue is empty" throttle 1 - @ Reports that a packet could not be enqueued because the queue is full event QueueFull( capacity: FwSizeType @< Maximum number of queued telemetry packets diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 15651c14..4180e9cb 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -41,7 +41,6 @@ class TlmArchive final : public TlmArchiveComponentBase { std::atomic m_antennasDeployed{false}; bool m_directoryInitialized = false; bool m_fileSizeInitialized = false; - bool m_queueWasEmpty = false; }; } // namespace Components diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index bd11a2e3..df091eb0 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -72,9 +72,7 @@ Prime topology dictionary to produce named, typed telemetry values. The queue prevents filesystem access from blocking the telemetry producer. Because `run` executes at 1 Hz, one buffered packet is processed per rate-group -tick. `QueueEmpty` reports the transition into an empty period once; its -throttle is cleared after a later dequeue succeeds so the next empty period is -reported again. +tick. An empty queue causes `run` to return without emitting an event. ## Class Diagram @@ -100,7 +98,6 @@ classDiagram -m_antennasDeployed: atomic~bool~ -m_directoryInitialized: bool -m_fileSizeInitialized: bool - -m_queueWasEmpty: bool } } @@ -134,8 +131,7 @@ the following internal state: | `m_directoryInitialized` | `false` | Becomes true after both `//tlm` and `//tlm/pre_deployment.csv` are successfully initialized, preventing repeated creation attempts. | | `m_fileSizeInitialized` | `false` | Becomes true after the initial archive size is read, preventing later size queries. | | `m_antennasDeployed` | `false` | Atomic in-memory latch set when AntennaDeployer first reports a deployed state. | -| `m_queueMutex` | Unlocked | Protects the ring storage, indices, count, and empty-period state. | -| `m_queueWasEmpty` | `false` | Records that an empty period was observed so the `QueueEmpty` throttle can be cleared after the next successful dequeue. | +| `m_queueMutex` | Unlocked | Protects the ring storage, indices, and count. | Conceptually, the component operates in these states: @@ -217,11 +213,9 @@ sequenceDiagram Rate->>Archive: run() alt Queue empty - Archive->>Archive: Emit QueueEmpty (throttled) Archive-->>Rate: Return else Packet queued Archive->>Archive: Remove oldest packet from FIFO - Archive->>Archive: Clear QueueEmpty throttle after an empty period opt Deployment state not yet latched Archive->>Deploy: deploymentStateGet() Deploy-->>Archive: deployed @@ -276,7 +270,6 @@ implementation constants control its behavior: | Name | Severity | Throttle | Parameters | Description | |---|---|---:|---|---| | `WriteStart` | Activity Low | 1 | None | Emitted immediately before each attempt to open the archive. | -| `QueueEmpty` | Activity Low | 1 | None | Emitted once per empty period. Its throttle is cleared when a later dequeue succeeds. | | `QueueFull` | Warning High | 1 | `capacity: FwSizeType` | Emitted when an incoming packet cannot be enqueued. Its throttle is cleared after a later enqueue succeeds. | | `FileError` | Warning High | None | `operation: string` | Reports directory creation, archive creation, record formatting, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `create_file`, `format_record`, `get_file_size`, and `open_append`. | | `WriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | @@ -284,11 +277,11 @@ implementation constants control its behavior: | `AntennasDeployed` | Warning Low | 1 | None | Emitted by `comIn` when deployment has been latched and a new packet is rejected. | | `SizeLimitReached` | Warning Low | 1 | `maxSize: FwSizeType` | Emitted by `comIn` when the cached archive size is at least 25,000 bytes. The implementation supplies `25000`. | -`QueueEmpty` and `QueueFull` have explicit throttle-clear calls so each distinct -empty or full period can be reported. The other throttled events report only -their first occurrence during the component's lifetime. The `comIn` checks are -ordered failure limit, size limit, then antenna deployment; if more than one -condition is true, only the first applicable event is invoked. +`QueueFull` has an explicit throttle-clear call so each distinct full period +can be reported. The other throttled events report only their first occurrence +during the component's lifetime. The `comIn` checks are ordered failure limit, +size limit, then antenna deployment; if more than one condition is true, only +the first applicable event is invoked. ## Telemetry @@ -323,3 +316,4 @@ There are currently no component-specific unit tests for TlmArchive. | 2026-08-01 | Replaced the concatenated binary archive with a versioned, one-packet-per-row CSV archive and documented the dictionary-backed ground decoder. | | 2026-08-01 | Replaced the latest-value mailbox with a bounded FIFO and added repeatable empty/full queue diagnostics. | | 2026-08-01 | Added a compile-time packet ID allowlist that filters telemetry before enqueueing. | +| 2026-08-02 | Removed the empty-queue event and its throttle state. | From eabfedf0434df85841b1c45e93325a2b86a5ed14 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 2 Aug 2026 09:06:43 -0500 Subject: [PATCH 31/38] add parameters for file size limit and error limit --- .../Components/TlmArchive/TlmArchive.cpp | 27 +++++--- .../Components/TlmArchive/TlmArchive.fpp | 25 ++++++- .../Components/TlmArchive/TlmArchive.hpp | 2 +- .../Components/TlmArchive/docs/sdd.md | 67 ++++++++++++------- 4 files changed, 84 insertions(+), 37 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 1db8570e..351d0e3e 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -23,8 +23,6 @@ constexpr const char* PRE_DEPLOYMENT_TLM_PATH = "//tlm/pre_deployment.csv"; constexpr char CSV_HEADER[] = "format_version,packet_size_bytes,packet_hex\n"; constexpr char HEX_DIGITS[] = "0123456789ABCDEF"; constexpr FwSizeType CSV_RECORD_BUFFER_SIZE = (FW_COM_BUFFER_MAX_SIZE * 2) + sizeof(CSV_HEADER) + 32; -constexpr const int MAX_FAILURES = 3; -constexpr const FwSizeType MAX_FILE_SIZE = 50000; // Only packetized telemetry IDs in this list are stored in the archive. constexpr FwTlmPacketizeIdType STORED_PACKET_IDS[] = { @@ -61,11 +59,19 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con (void)portNum; (void)context; - if (this->m_failures.load() >= MAX_FAILURES) { - this->log_WARNING_HI_FailureLimitReached(MAX_FAILURES); + Fw::ParamValid maxFailuresValid; + const U32 maxFailures = this->paramGet_MAX_FAILURES(maxFailuresValid); + FW_ASSERT(maxFailuresValid == Fw::ParamValid::VALID || maxFailuresValid == Fw::ParamValid::DEFAULT); + + Fw::ParamValid maxFileSizeValid; + const U32 maxFileSize = this->paramGet_MAX_FILE_SIZE(maxFileSizeValid); + FW_ASSERT(maxFileSizeValid == Fw::ParamValid::VALID || maxFileSizeValid == Fw::ParamValid::DEFAULT); + + if (this->m_failures.load() >= maxFailures) { + this->log_WARNING_HI_FailureLimitReached(maxFailures); return; - } else if (this->m_fileSize.load() >= MAX_FILE_SIZE) { - this->log_WARNING_LO_SizeLimitReached(MAX_FILE_SIZE); + } else if (this->m_fileSize.load() >= maxFileSize) { + this->log_WARNING_LO_SizeLimitReached(maxFileSize); return; } else if (this->m_antennasDeployed.load()) { this->log_WARNING_LO_AntennasDeployed(); @@ -134,6 +140,9 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { const FwSizeType packetSize = packet.getSize(); FwSizeType currentSize = this->m_fileSize.load(); + Fw::ParamValid maxFileSizeValid; + const U32 maxFileSize = this->paramGet_MAX_FILE_SIZE(maxFileSizeValid); + FW_ASSERT(maxFileSizeValid == Fw::ParamValid::VALID || maxFileSizeValid == Fw::ParamValid::DEFAULT); if (!this->m_fileSizeInitialized) { const Os::FileSystem::Status sizeStatus = Os::FileSystem::getFileSize(PRE_DEPLOYMENT_TLM_PATH, currentSize); if (sizeStatus != Os::FileSystem::OP_OK) { @@ -141,9 +150,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { this->m_failures.fetch_add(1); return; } - const U32 cachedSize = - (currentSize > MAX_FILE_SIZE) ? static_cast(MAX_FILE_SIZE) : static_cast(currentSize); - this->m_fileSize.store(cachedSize); + this->m_fileSize.store(static_cast(currentSize)); this->m_fileSizeInitialized = true; } @@ -171,7 +178,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { } csvRecord[recordSize++] = '\n'; - if (currentSize > MAX_FILE_SIZE) { + if (currentSize > maxFileSize) { this->m_failures.fetch_add(1); return; } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp index 4c1ae78e..e81f6236 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.fpp @@ -10,6 +10,12 @@ module Components { @ Port for checking whether antenna deployment has completed output port deploymentStateGet: Components.GetDeploymentState + @ Maximum archive size in bytes + param MAX_FILE_SIZE: U32 default 50000 id 0 + + @ Number of counted filesystem failures that disables archiving + param MAX_FAILURES: U32 default 3 id 1 + @ Report when file write has started event WriteStart severity activity low format "Beginning telemetry archival to pre_deployment.csv" throttle 1 @@ -34,7 +40,7 @@ module Components { @ Reports when telemetry archiving is disabled due to hitting the failure limit event FailureLimitReached( - count: I8 + count: U32 ) severity warning high format "{} filesystem failures counted; disabling further telemetry writes." throttle 1 @ Reports when telemetry archiving is disabled due to antennas being deployed @@ -42,16 +48,31 @@ module Components { @ Reports when telemetry archiving is disabled due to pre_deployment.csv hitting the size limit event SizeLimitReached( - maxSize: FwSizeType + maxSize: U32 ) severity warning low format "pre_deployment.csv file size limit of {}b reached; disabling further telemetry writes." throttle 1 @ Port for requesting the current time time get port timeCaller + @ Port for sending command registrations + command reg port cmdRegOut + + @ Port for receiving commands + command recv port cmdIn + + @ Port for sending command responses + command resp port cmdResponseOut + @ Port for sending textual representation of events text event port logTextOut @ Port for sending events to downlink event port logOut + + @ Port for getting parameter values + param get port prmGetOut + + @ Port for setting parameter values + param set port prmSetOut } } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 4180e9cb..05e74e61 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -37,7 +37,7 @@ class TlmArchive final : public TlmArchiveComponentBase { FwSizeType m_queueTail = 0; FwSizeType m_queueSize = 0; std::atomic m_fileSize{0}; - std::atomic m_failures{0}; + std::atomic m_failures{0}; std::atomic m_antennasDeployed{false}; bool m_directoryInitialized = false; bool m_fileSizeInitialized = false; diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index df091eb0..60fbf74f 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -25,11 +25,14 @@ The reference topology connects the component as follows: - `rateGroup1Hz.RateGroupMemberOut[19]` invokes `TlmArchive.run`. - `TlmArchive.deploymentStateGet` queries `AntennaDeployer.deploymentStateGet`. +- The topology command and parameter patterns connect the generated parameter + commands and parameter database ports. Archiving stops for the remainder of the component's lifetime after it -observes a deployed antenna state. `comIn` also rejects new packets after three -counted failures or when the cached archive size reaches 25,000 bytes. A -packet that brings the archive to or above the threshold is written; subsequent +observes a deployed antenna state. `comIn` also rejects new packets when the +counted failures reach `MAX_FAILURES` or the cached archive size reaches +`MAX_FILE_SIZE`. Their defaults are three failures and 50,000 bytes. A packet +that brings the archive to or above the threshold is written; subsequent packets are rejected by `comIn` and report the size-limit event. ## Usage Examples @@ -94,7 +97,7 @@ classDiagram -m_queueTail: FwSizeType -m_queueSize: FwSizeType -m_fileSize: atomic~U32~ - -m_failures: atomic~int~ + -m_failures: atomic~U32~ -m_antennasDeployed: atomic~bool~ -m_directoryInitialized: bool -m_fileSizeInitialized: bool @@ -112,8 +115,13 @@ classDiagram | `run` | `Svc.Sched` | sync input | Drains the oldest queued packet, checks deployment state, and performs archive filesystem work. Connected to the 1 Hz rate group. | | `deploymentStateGet` | `Components.GetDeploymentState` | output | Queries AntennaDeployer for its persistent deployed state. | | `timeCaller` | time get | time get | Supplies timestamps for emitted events. | +| `cmdRegOut` | command reg | output | Registers the generated parameter set/save commands. | +| `cmdIn` | command recv | input | Receives the generated parameter set/save commands. | +| `cmdResponseOut` | command resp | output | Reports generated parameter command completion. | | `logOut` | event | output | Sends binary event records. | | `logTextOut` | text event | output | Sends text-formatted event records. | +| `prmGetOut` | param get | output | Loads parameter values from the parameter database. | +| `prmSetOut` | param set | output | Saves parameter values to the parameter database. | ## Component Behavior and States @@ -140,7 +148,7 @@ Conceptually, the component operates in these states: | `WAITING` | Writing is enabled and the packet queue is empty. | | `PACKETS_QUEUED` | One or more packets are waiting and are processed oldest-first, one per `run` invocation. | | `DEPLOYED` | A true antenna deployment state has been latched. The packet that observed deployment is discarded, and subsequent packets are rejected by `comIn`. | -| `WRITE_DISABLED` | The failure count has reached its limit or the cached file size has reached 25,000 bytes. New packets are rejected by `comIn`. | +| `WRITE_DISABLED` | The failure count has reached `MAX_FAILURES` or the cached file size has reached `MAX_FILE_SIZE`. New packets are rejected by `comIn`. | The `run` handler does not emit an event when it first observes deployment. The throttled `AntennasDeployed` event is emitted if another packet later @@ -158,17 +166,18 @@ Before the first append attempt, the component reads the existing on-disk size. Stat failures are reported and counted. After a successful size initialization, the component uses the cached size rather than querying the filesystem again. Once a successful write brings the cached archive size to or -above 25,000 bytes, subsequent packets are rejected by `comIn`. The final write -may therefore make the archive larger than the threshold by one CSV record. +above the configured `MAX_FILE_SIZE`, subsequent packets are rejected by +`comIn`. The final write may therefore make the archive larger than the +threshold by one CSV record. After a successful write, `m_fileSize` is incremented by the actual byte count reported by the file API. The archive is not truncated or deleted when the limit is reached. Changes made to the archive by another component after size initialization are not reflected in the cache. -The atomic cache uses the target's native 32-bit `U32` width. This is lossless -for all permitted archive sizes; an existing on-disk size above the 25,000-byte -limit is represented by the limit value so packet admission remains disabled. +The atomic cache uses the target's native 32-bit `U32` width and retains the +actual initial file size. This allows increasing `MAX_FILE_SIZE` at runtime +without losing track of bytes already present in the archive. ### Failure Handling @@ -182,10 +191,10 @@ The following failures increment `m_failures`: - failure to open the archive for append; and - a failed or short file write. -Once three counted failures have occurred, subsequent `comIn` calls reject new -packets. The packet that encountered an error is not retried. An archive that -is already larger than the threshold when its size is first read is rejected in -`run` and increments the failure count without emitting an event. +Once the counted failures reach `MAX_FAILURES`, subsequent `comIn` calls reject +new packets. The packet that encountered an error is not retried. An archive +that is already larger than `MAX_FILE_SIZE` when its size is first read is +rejected in `run` and increments the failure count without emitting an event. ## Sequence Diagrams @@ -247,23 +256,31 @@ sequenceDiagram ## Parameters -The component defines no runtime F Prime parameters. The following compile-time -implementation constants control its behavior: +| Name | Type | Default | Description | +|---|---|---:|---| +| `MAX_FILE_SIZE` | `U32` | `50000` | Cached archive-size threshold in bytes after which subsequent packets are rejected. One final record may cross the threshold. | +| `MAX_FAILURES` | `U32` | `3` | Counted stat, limit, directory, open, or write failures after which new packets are rejected. | + +The parameters are loaded during topology startup and may be set or saved at +runtime through their generated F Prime parameter commands. Changes affect the +next packet admission or archive processing check. Each parameter read asserts +that F Prime returned either a valid stored value or the declared default. + +The remaining compile-time implementation constants are: | Name | Value | Description | |---|---:|---| | `TLM_DIRECTORY` | `//tlm` | Directory containing the archive. | | `PRE_DEPLOYMENT_TLM_PATH` | `//tlm/pre_deployment.csv` | Append-only, one-packet-per-row pre-deployment telemetry archive. | -| `STORED_PACKET_IDS` | All currently configured telemetry packet IDs | Compile-time allowlist in `TlmArchive.cpp`. Packets whose IDs are absent are discarded before enqueueing. | +| `STORED_PACKET_IDS` | Beacon (`1`) and Imu (`7`) | Compile-time allowlist in `TlmArchive.cpp`. Packets whose IDs are absent are discarded before enqueueing. | | `PACKET_QUEUE_CAPACITY` | `Svc::MAX_PACKETIZER_PACKETS` (currently 22) | Maximum packets held between the telemetry producer and 1 Hz filesystem worker. | -| `MAX_FAILURES` | `3` | Counted stat, limit, directory, open, or write failures after which new packets are rejected. | -| `MAX_FILE_SIZE` | `25000` bytes | Cached archive-size threshold after which subsequent packets are rejected. One final record may cross the threshold. | ## Commands | Name | Description | |---|---| -| N/A | The component defines no commands. | +| `MAX_FILE_SIZE_PRM_SET` / `MAX_FILE_SIZE_PRM_SAVE` | Generated F Prime commands that update or persist the maximum archive size. | +| `MAX_FAILURES_PRM_SET` / `MAX_FAILURES_PRM_SAVE` | Generated F Prime commands that update or persist the filesystem failure limit. | ## Events @@ -273,9 +290,9 @@ implementation constants control its behavior: | `QueueFull` | Warning High | 1 | `capacity: FwSizeType` | Emitted when an incoming packet cannot be enqueued. Its throttle is cleared after a later enqueue succeeds. | | `FileError` | Warning High | None | `operation: string` | Reports directory creation, archive creation, record formatting, initial file-size lookup, or archive open errors. Current operation strings are `create_directory`, `create_file`, `format_record`, `get_file_size`, and `open_append`. | | `WriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | -| `FailureLimitReached` | Warning High | 1 | `count: I8` | Emitted by `comIn` when the cumulative failure count is at least three. The implementation supplies `3`. | +| `FailureLimitReached` | Warning High | 1 | `count: U32` | Emitted by `comIn` when the cumulative failure count reaches the configured limit. | | `AntennasDeployed` | Warning Low | 1 | None | Emitted by `comIn` when deployment has been latched and a new packet is rejected. | -| `SizeLimitReached` | Warning Low | 1 | `maxSize: FwSizeType` | Emitted by `comIn` when the cached archive size is at least 25,000 bytes. The implementation supplies `25000`. | +| `SizeLimitReached` | Warning Low | 1 | `maxSize: U32` | Emitted by `comIn` when the cached archive size reaches the configured maximum. | `QueueFull` has an explicit throttle-clear call so each distinct full period can be reported. The other throttled events report only their first occurrence @@ -300,8 +317,8 @@ There are currently no component-specific unit tests for TlmArchive. | `TLM_ARCHIVE_001` | The component shall enqueue telemetry packets in arrival order in a bounded FIFO sized for one complete packetizer send cycle. | Inspection | | `TLM_ARCHIVE_002` | The component shall append buffered telemetry as versioned, one-packet-per-row CSV records to `//tlm/pre_deployment.csv` while the antenna deployment state is false. | Inspection | | `TLM_ARCHIVE_003` | The component shall stop writing telemetry after observing a deployed antenna state. | Inspection | -| `TLM_ARCHIVE_004` | The component shall reject new packets after three counted failures. | Inspection | -| `TLM_ARCHIVE_005` | The component shall reject new packets after the cached archive size reaches 25,000 bytes. The write that crosses the threshold is permitted. | Inspection | +| `TLM_ARCHIVE_004` | The component shall reject new packets after the counted failures reach the configurable `MAX_FAILURES` parameter. | Inspection | +| `TLM_ARCHIVE_005` | The component shall reject new packets after the cached archive size reaches the configurable `MAX_FILE_SIZE` parameter. The write that crosses the threshold is permitted. | Inspection | | `TLM_ARCHIVE_006` | The component shall report archive start, filesystem errors, write errors, failure-limit rejection, size-limit rejection, and deployed-state rejection through events. | Inspection | | `TLM_ARCHIVE_007` | The component shall create the archive when missing without truncating an existing archive. | Inspection | | `TLM_ARCHIVE_008` | The component shall enqueue only valid packetized telemetry whose packet ID appears in the compile-time `STORED_PACKET_IDS` allowlist. | Inspection | @@ -317,3 +334,5 @@ There are currently no component-specific unit tests for TlmArchive. | 2026-08-01 | Replaced the latest-value mailbox with a bounded FIFO and added repeatable empty/full queue diagnostics. | | 2026-08-01 | Added a compile-time packet ID allowlist that filters telemetry before enqueueing. | | 2026-08-02 | Removed the empty-queue event and its throttle state. | +| 2026-08-02 | Converted the archive size and filesystem failure limits to runtime F Prime parameters. | +| 2026-08-02 | Added validity assertions after reading runtime parameters. | From 773932021005928117e6aaa254136a16e01ad11f Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Sun, 2 Aug 2026 17:01:40 -0500 Subject: [PATCH 32/38] ensure file limit event is fired and queue is cleared on fileSize limit --- .../Components/TlmArchive/TlmArchive.cpp | 11 ++++++++++- .../Components/TlmArchive/TlmArchive.hpp | 1 + .../Components/TlmArchive/docs/sdd.md | 10 +++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 351d0e3e..9d1f47df 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -55,6 +55,13 @@ TlmArchive ::TlmArchive(const char* const compName) : TlmArchiveComponentBase(co TlmArchive ::~TlmArchive() {} +void TlmArchive::clearPacketQueue() { + Os::ScopeLock lock(this->m_queueMutex); + this->m_queueHead = 0; + this->m_queueTail = 0; + this->m_queueSize = 0; +} + void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 context) { (void)portNum; (void)context; @@ -71,6 +78,7 @@ void TlmArchive::comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 con this->log_WARNING_HI_FailureLimitReached(maxFailures); return; } else if (this->m_fileSize.load() >= maxFileSize) { + this->clearPacketQueue(); this->log_WARNING_LO_SizeLimitReached(maxFileSize); return; } else if (this->m_antennasDeployed.load()) { @@ -179,7 +187,8 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { csvRecord[recordSize++] = '\n'; if (currentSize > maxFileSize) { - this->m_failures.fetch_add(1); + this->clearPacketQueue(); + this->log_WARNING_LO_SizeLimitReached(maxFileSize); return; } diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 05e74e61..365e1f84 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -28,6 +28,7 @@ class TlmArchive final : public TlmArchiveComponentBase { static constexpr FwSizeType PACKET_QUEUE_CAPACITY = Svc::MAX_PACKETIZER_PACKETS; static_assert(PACKET_QUEUE_CAPACITY > 0, "Telemetry archive packet queue must have storage"); + void clearPacketQueue(); void comIn_handler(FwIndexType portNum, Fw::ComBuffer& data, U32 context) override; void run_handler(FwIndexType portNum, U32 context) override; diff --git a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md index 60fbf74f..e897bd16 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md +++ b/PROVESFlightControllerReference/Components/TlmArchive/docs/sdd.md @@ -167,8 +167,10 @@ size. Stat failures are reported and counted. After a successful size initialization, the component uses the cached size rather than querying the filesystem again. Once a successful write brings the cached archive size to or above the configured `MAX_FILE_SIZE`, subsequent packets are rejected by -`comIn`. The final write may therefore make the archive larger than the -threshold by one CSV record. +`comIn` and all remaining queued packets are discarded. The worker also clears +the queue if it observes that the archive is already over the limit. The final +write may therefore make the archive larger than the threshold by one CSV +record. After a successful write, `m_fileSize` is incremented by the actual byte count reported by the file API. The archive is not truncated or deleted when the @@ -292,7 +294,7 @@ The remaining compile-time implementation constants are: | `WriteError` | Warning High | None | `status: Os.FileStatus`, `requested: FwSizeType`, `written: FwSizeType` | Reports a failed or incomplete archive write. | | `FailureLimitReached` | Warning High | 1 | `count: U32` | Emitted by `comIn` when the cumulative failure count reaches the configured limit. | | `AntennasDeployed` | Warning Low | 1 | None | Emitted by `comIn` when deployment has been latched and a new packet is rejected. | -| `SizeLimitReached` | Warning Low | 1 | `maxSize: U32` | Emitted by `comIn` when the cached archive size reaches the configured maximum. | +| `SizeLimitReached` | Warning Low | 1 | `maxSize: U32` | Emitted by `comIn` when the cached archive size reaches the configured maximum, or by `run` when a queued packet observes that the size has exceeded it. | `QueueFull` has an explicit throttle-clear call so each distinct full period can be reported. The other throttled events report only their first occurrence @@ -336,3 +338,5 @@ There are currently no component-specific unit tests for TlmArchive. | 2026-08-02 | Removed the empty-queue event and its throttle state. | | 2026-08-02 | Converted the archive size and filesystem failure limits to runtime F Prime parameters. | | 2026-08-02 | Added validity assertions after reading runtime parameters. | +| 2026-08-02 | Reported the size-limit event when the archive worker observes an oversized file. | +| 2026-08-02 | Cleared queued telemetry whenever the archive size limit is observed. | From c1aecfa3aad75ec55edf43963d104c3fc3810528 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 3 Aug 2026 16:29:31 -0500 Subject: [PATCH 33/38] bump command table size --- .../project/config/CommandDispatcherImplCfg.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp b/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp index b28b21a5..aec87a67 100644 --- a/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp +++ b/PROVESFlightControllerReference/project/config/CommandDispatcherImplCfg.hpp @@ -11,7 +11,7 @@ // Define configuration values for dispatcher enum { - CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 350, // !< The size of the table holding opcodes to dispatch + CMD_DISPATCHER_DISPATCH_TABLE_SIZE = 400, // !< The size of the table holding opcodes to dispatch CMD_DISPATCHER_SEQUENCER_TABLE_SIZE = 10, // !< The size of the table holding commands in progress }; From 06bba3b4f5a7d6b52e0ce1252e8520f164b85727 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 3 Aug 2026 16:32:24 -0500 Subject: [PATCH 34/38] wait 30 seconds to start writing to disk --- .../Components/TlmArchive/TlmArchive.cpp | 7 +++++++ .../Components/TlmArchive/TlmArchive.hpp | 1 + 2 files changed, 8 insertions(+) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 9d1f47df..5055cc5c 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -113,6 +113,13 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { (void)portNum; (void)context; + // Don't write for the first 30s of boot to attempt to mitigate + // any conflicts with other filesystem writes/actions + if (!(this->m_ticks <= 30)) { + this->m_ticks++; + return; + } + Fw::ComBuffer packet; { Os::ScopeLock lock(this->m_queueMutex); diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp index 365e1f84..2219ad52 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.hpp @@ -42,6 +42,7 @@ class TlmArchive final : public TlmArchiveComponentBase { std::atomic m_antennasDeployed{false}; bool m_directoryInitialized = false; bool m_fileSizeInitialized = false; + int m_ticks = 0; }; } // namespace Components From 5dbd15ea58d7a3485c027ec0c055f8746772d089 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 3 Aug 2026 16:37:45 -0500 Subject: [PATCH 35/38] i am a stupid man --- .../Components/TlmArchive/TlmArchive.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp index 5055cc5c..a9aa3ca4 100644 --- a/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp +++ b/PROVESFlightControllerReference/Components/TlmArchive/TlmArchive.cpp @@ -115,7 +115,7 @@ void TlmArchive::run_handler(FwIndexType portNum, U32 context) { // Don't write for the first 30s of boot to attempt to mitigate // any conflicts with other filesystem writes/actions - if (!(this->m_ticks <= 30)) { + if (this->m_ticks <= 30) { this->m_ticks++; return; } From 4ccceba260497bc9f9d38ccdb8d99dfb44f26782 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 3 Aug 2026 17:21:02 -0500 Subject: [PATCH 36/38] test codex recommendation for radio test setup --- .../test/int/conftest.py | 49 ++++++++++++------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/PROVESFlightControllerReference/test/int/conftest.py b/PROVESFlightControllerReference/test/int/conftest.py index 933ebcee..4abf31c7 100644 --- a/PROVESFlightControllerReference/test/int/conftest.py +++ b/PROVESFlightControllerReference/test/int/conftest.py @@ -10,7 +10,11 @@ import time import pytest -from common import cmdDispatch, set_radio_recover_fn +from common import ( + cmdDispatch, + proves_send_and_assert_command, + set_radio_recover_fn, +) from fprime_gds.common.testing_fw.api import IntegrationTestAPI # After TRANSMIT is first enabled the satellite flushes the event backlog that @@ -118,28 +122,35 @@ def start_gds( GDS is used to send commands and receive telemetry/events. """ - gds_working = False - timeout_time = time.time() + 30 - while time.time() < timeout_time: - try: - if request.config.getoption("--with-radio"): - _enable_radio(fprime_test_api_session) - fprime_test_api_session.send_and_assert_command( - command=f"{cmdDispatch}.CMD_NO_OP" - ) - gds_working = True - break - except Exception: - time.sleep(1) - assert gds_working - if request.config.getoption("--with-radio"): - # Allow the boot-time event backlog to drain before any test commands - # are issued. Without this wait the initial burst of queued events can - # swamp command-ack events and cause the first test assertions to fail. + # Enable transmission once, then allow the boot-time event backlog to + # drain before probing the half-duplex link. Sending NO_OP immediately + # after TRANSMIT collides with that initial downlink burst, and + # re-sending the setup commands on every retry only adds more traffic. + _enable_radio(fprime_test_api_session) time.sleep(RADIO_STABILIZE_S) fprime_test_api_session.clear_histories() + # Use the shared jittered retry path so attempts do not remain locked + # to the periodic LoRa downlink cadence. + proves_send_and_assert_command( + fprime_test_api_session, + command=f"{cmdDispatch}.CMD_NO_OP", + ) + else: + gds_working = False + timeout_time = time.time() + 30 + while time.time() < timeout_time: + try: + fprime_test_api_session.send_and_assert_command( + command=f"{cmdDispatch}.CMD_NO_OP" + ) + gds_working = True + break + except Exception: + time.sleep(1) + assert gds_working + yield From f2a1a9d53ab18ba92f33a5699469c281a6a5c801 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 3 Aug 2026 17:51:46 -0500 Subject: [PATCH 37/38] Revert "test codex recommendation for radio test setup" This reverts commit 4ccceba260497bc9f9d38ccdb8d99dfb44f26782. --- .../test/int/conftest.py | 49 +++++++------------ 1 file changed, 19 insertions(+), 30 deletions(-) diff --git a/PROVESFlightControllerReference/test/int/conftest.py b/PROVESFlightControllerReference/test/int/conftest.py index 4abf31c7..933ebcee 100644 --- a/PROVESFlightControllerReference/test/int/conftest.py +++ b/PROVESFlightControllerReference/test/int/conftest.py @@ -10,11 +10,7 @@ import time import pytest -from common import ( - cmdDispatch, - proves_send_and_assert_command, - set_radio_recover_fn, -) +from common import cmdDispatch, set_radio_recover_fn from fprime_gds.common.testing_fw.api import IntegrationTestAPI # After TRANSMIT is first enabled the satellite flushes the event backlog that @@ -122,35 +118,28 @@ def start_gds( GDS is used to send commands and receive telemetry/events. """ + gds_working = False + timeout_time = time.time() + 30 + while time.time() < timeout_time: + try: + if request.config.getoption("--with-radio"): + _enable_radio(fprime_test_api_session) + fprime_test_api_session.send_and_assert_command( + command=f"{cmdDispatch}.CMD_NO_OP" + ) + gds_working = True + break + except Exception: + time.sleep(1) + assert gds_working + if request.config.getoption("--with-radio"): - # Enable transmission once, then allow the boot-time event backlog to - # drain before probing the half-duplex link. Sending NO_OP immediately - # after TRANSMIT collides with that initial downlink burst, and - # re-sending the setup commands on every retry only adds more traffic. - _enable_radio(fprime_test_api_session) + # Allow the boot-time event backlog to drain before any test commands + # are issued. Without this wait the initial burst of queued events can + # swamp command-ack events and cause the first test assertions to fail. time.sleep(RADIO_STABILIZE_S) fprime_test_api_session.clear_histories() - # Use the shared jittered retry path so attempts do not remain locked - # to the periodic LoRa downlink cadence. - proves_send_and_assert_command( - fprime_test_api_session, - command=f"{cmdDispatch}.CMD_NO_OP", - ) - else: - gds_working = False - timeout_time = time.time() + 30 - while time.time() < timeout_time: - try: - fprime_test_api_session.send_and_assert_command( - command=f"{cmdDispatch}.CMD_NO_OP" - ) - gds_working = True - break - except Exception: - time.sleep(1) - assert gds_working - yield From 21309ec412cdb759e633b933d29beb183edcd991 Mon Sep 17 00:00:00 2001 From: hrfarmer Date: Mon, 3 Aug 2026 18:09:45 -0500 Subject: [PATCH 38/38] test i guess --- .../ReferenceDeployment/Top/topology.fpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp index 3938ca95..902260e1 100644 --- a/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp +++ b/PROVESFlightControllerReference/ReferenceDeployment/Top/topology.fpp @@ -303,7 +303,7 @@ module ReferenceDeployment { rateGroup1Hz.RateGroupMemberOut[16] -> modeManager.run rateGroup1Hz.RateGroupMemberOut[17] -> adcs.run rateGroup1Hz.RateGroupMemberOut[18] -> thermalManager.run - rateGroup1Hz.RateGroupMemberOut[19] -> tlmArchive.run + # rateGroup1Hz.RateGroupMemberOut[19] -> tlmArchive.run }