feat(ota): fix update failure reporting, add progress telemetry and delta patching

PROVESFlightControllerReference/Components/FlashWorker/CMakeLists.txt

@@ -20,6 +20,10 @@ register_fprime_library(
         "${CMAKE_CURRENT_LIST_DIR}/FlashWorker.fpp"
     SOURCES
         "${CMAKE_CURRENT_LIST_DIR}/FlashWorker.cpp"
+        "${CMAKE_CURRENT_LIST_DIR}/UpdateSequencer.cpp"
+        "${CMAKE_CURRENT_LIST_DIR}/SegmentPlan.cpp"
+        "${CMAKE_CURRENT_LIST_DIR}/PatchApplier.cpp"
+        "${CMAKE_CURRENT_LIST_DIR}/LzssDecoder.cpp"
 #   DEPENDS
 #       MyPackage_MyOtherModule
 )

PROVESFlightControllerReference/Components/FlashWorker/FlashWorker.fpp

@@ -1,8 +1,90 @@
 module Components {
+    @ Stage of the flight software update sequence, reported as telemetry so that an operator can
+    @ see where an update stands without replaying the event history from an earlier pass.
+    enum FlashUpdateStage {
+        IDLE, @< No update in progress
+        PREPARING, @< Erasing the staging slot
+        PREPARED, @< Staging slot erased and ready to receive an image
+        WRITING, @< Writing an image into the staging slot
+        UPDATED, @< Image written and verified in the staging slot
+        FAILED @< The last operation failed; see LastUpdateStatus
+    }
+
     @ Performs long-running operations for the flash subsystem
     active component FlashWorker {
         import Update.UpdateWorker
 
+        @ Microseconds to pause after each buffered flash write, giving the flash time to settle.
+        @ Exposed as a parameter so that it can be tuned against real hardware rather than rebuilt.
+        param CHUNK_DELAY_US: U32 default 5000
+
+        @ Percent of the image to advance between progress reports. Larger values mean fewer
+        @ progress events and less downlink spent reporting on an update in flight.
+        param PROGRESS_STEP_PERCENT: U8 default 10
+
+        @ Whether the flight software may confirm a test-booted image without the ground.
+        @
+        @ An image booted in TEST mode reverts unless it is confirmed before the next reboot. If the
+        @ confirming pass is missed, a working image is thrown away along with the uplink that
+        @ delivered it. Arming this lets the spacecraft keep an image that has demonstrated it can
+        @ run. Defaults to disabled, so confirmation stays operator-in-the-loop until armed.
+        @
+        @ Set with AUTO_CONFIRM_ENABLED_PRM_SET followed by PRM_SAVE: the decision is made after the
+        @ reboot into the test image, so an unsaved value would be lost exactly when it is needed.
+        param AUTO_CONFIRM_ENABLED: bool default false
+
+        @ Seconds the test-booted image must run continuously before it confirms itself. Counted
+        @ from the start of the image that is pending confirmation, and reset by any reboot.
+        param AUTO_CONFIRM_DELAY_SECONDS: U32 default 1800
+
+        @ Concatenate numbered uplink segments into a single image file.
+        @
+        @ A full image does not fit in one pass and file uplink cannot resume across passes, so an
+        @ image is uplinked as "<prefix>.000", "<prefix>.001", and so on. This joins them back
+        @ together and validates the result before it is written to flash.
+        async command ASSEMBLE_IMAGE(
+            prefix: string size FileNameStringSize @< Segment file name prefix
+            segments: U16 @< Number of segments to join
+            destination: string size FileNameStringSize @< Image file to write
+            crc32: U32 @< Expected CRC32 of the assembled image
+        )
+
+        @ Reconstruct an image from a delta patch applied to the running image.
+        @
+        @ Uplinking a patch rather than a whole image is the difference between an update that fits
+        @ a pass and one that does not. The patch names the reference it was built from and is
+        @ refused if the running image is not it.
+        async command APPLY_PATCH(
+            patch: string size FileNameStringSize @< Patch file to apply
+            destination: string size FileNameStringSize @< Image file to write
+            crc32: U32 @< Expected CRC32 of the reconstructed image
+        )
+
+        @ Stage of the update sequence
+        telemetry UpdateStage: FlashUpdateStage
+
+        @ Bytes of the image written into the staging slot so far
+        telemetry BytesWritten: U32
+
+        @ Total size in bytes of the image currently being written
+        telemetry ImageTotalBytes: U32
+
+        @ Status reported by the most recent preparation or update operation
+        telemetry LastUpdateStatus: Update.UpdateStatus
+
+        @ Whether the running image has been confirmed. False means this is a test boot that will
+        @ revert on the next reboot unless it is confirmed.
+        telemetry RunningImageConfirmed: bool
+
+        @ Seconds the running image has been up while pending confirmation
+        telemetry PendingConfirmSeconds: U32
+
+        @ Scheduled input used to age a test-booted image toward self-confirmation
+        sync input port run: Svc.Sched
+
+        event UpdateProgress(written: U32, total: U32, percent: U8) severity activity low \
+            format "Update progress: {}/{} bytes ({}%)"
+
         event NoImagePrepared() severity warning low \
             format "No image has been prepared for update"
 
@@ -25,17 +107,75 @@ module Components {
             severity warning low \
             format "Failed CRC validation of {} with status {} expected 0x{x} and actual 0x{x}"
 
+        @ Emitted when the bytes written to flash do not match the bytes that were validated. This
+        @ indicates the image file changed underneath the update or the filesystem read is unstable.
+        event ImageWriteCrcMismatch(expected: U32, actual: U32) severity warning high \
+            format "Image written to flash failed verification: expected 0x{x} and actual 0x{x}"
+
+        @ The running image confirmed itself after demonstrating it can run
+        event AutoConfirmed(seconds: U32) severity activity high \
+            format "Test image confirmed automatically after {} seconds of operation"
+
+        event AutoConfirmFailed(error_number: I32) severity warning high \
+            format "Automatic confirmation failed (errno: {})"
+
+        event AssembleStarted(segments: U16, destination: string) severity activity high \
+            format "Assembling {} segments into {}"
+
+        event AssembleSucceeded(bytes: U32) severity activity high \
+            format "Assembled image of {} bytes"
+
+        event AssembleFailed(segment: U16, status: Update.UpdateStatus) severity warning high \
+            format "Assembly failed at segment {} with status {}"
+
+        event InvalidSegmentCount(segments: U16) severity warning high \
+            format "Segment count {} is out of range"
+
+        event PatchStarted(patch: string, destination: string) severity activity high \
+            format "Applying patch {} to produce {}"
+
+        event PatchSucceeded(bytes: U32) severity activity high \
+            format "Reconstructed image of {} bytes from patch"
+
+        @ The error code is Components::PatchApplier::Error
+        event PatchFailed(error_code: U8) severity warning high \
+            format "Patch application failed with error {}"
+
+        @ Emitted when the running image is not the one the patch was built against. Applying the
+        @ patch anyway would produce a corrupt image, so it is refused.
+        event PatchReferenceMismatch(expected_crc: U32, expected_size: U32, actual_crc: U32) \
+            severity warning high \
+            format "Patch reference mismatch: expected CRC 0x{x} over {} bytes but running image has CRC 0x{x}"
+
         ###############################################################################
         # Standard AC Ports: Required for Channels, Events, Commands, and Parameters  #
         ###############################################################################
         @ Port for requesting the current time
         time get port timeCaller
 
+        @ Port to return the value of a parameter
+        param get port prmGetOut
+
+        @ Port to set the value of a parameter
+        param set port prmSetOut
+
+        @ Port for sending command registrations
+        command reg port cmdRegOut
+
+        @ Port for receiving commands
+        command recv port cmdIn
+
+        @ Port for sending command responses
+        command resp port cmdResponseOut
+
         @ Port for sending textual representation of events
         text event port logTextOut
 
         @ Port for sending events to downlink
         event port logOut
 
+        @ Port for sending telemetry channels to downlink
+        telemetry port tlmOut
+
     }
 }

PROVESFlightControllerReference/Components/FlashWorker/FlashWorker.hpp

@@ -8,13 +8,22 @@
 #define Update_FlashWorker_HPP
 #include "Os/File.hpp"
 #include "PROVESFlightControllerReference/Components/FlashWorker/FlashWorkerComponentAc.hpp"
+#include "PROVESFlightControllerReference/Components/FlashWorker/LzssDecoder.hpp"
+#include "PROVESFlightControllerReference/Components/FlashWorker/PatchApplier.hpp"
+#include "PROVESFlightControllerReference/Components/FlashWorker/SegmentPlan.hpp"
+#include "PROVESFlightControllerReference/Components/FlashWorker/UpdateSequencer.hpp"
 #include <zephyr/dfu/flash_img.h>
+#include <zephyr/storage/flash_map.h>
 namespace Components {
 
 class FlashWorker final : public FlashWorkerComponentBase {
   public:
-    constexpr static U8 REGION_NUMBER = 2;  // 0: bootloader, 1: slot0, **2: slot1**
-    enum Step { IDLE, PREPARE, UPDATE };
+    //! Flash area holding the MCUBoot staging slot that updates are written into.
+    //!
+    //! Derived from the device tree rather than hard coded: fixed partition IDs follow the
+    //! declaration order of the partitions node, so a literal would silently point at the wrong
+    //! region if a partition were ever added above slot1_partition.
+    constexpr static U8 REGION_NUMBER = FIXED_PARTITION_ID(slot1_partition);
     // ----------------------------------------------------------------------
     // Component construction and destruction
     // ----------------------------------------------------------------------
@@ -27,7 +36,48 @@ class FlashWorker final : public FlashWorkerComponentBase {
     ~FlashWorker();
 
   private:
-    Update::UpdateStatus writeImage(const Fw::StringBase& file_name, Os::File& image_file, U32 crc32);
+    //! Stream an image file into the staging slot, reporting what went wrong if anything did
+    UpdateSequencer::WriteOutcome writeImage(const Fw::StringBase& file_name, Os::File& image_file, U32 crc32);
+
+    //! Convert a sequencer status into the autocoded status reported over the update ports
+    static Update::UpdateStatus toUpdateStatus(UpdateSequencer::Status status);
+
+    //! Convert a sequencer stage into the autocoded stage reported as telemetry
+    static Components::FlashUpdateStage toUpdateStage(UpdateSequencer::Stage stage);
+
+    //! Report the stage of the sequence, along with the status of the operation that produced it
+    void reportStage(Components::FlashUpdateStage stage, Update::UpdateStatus status);
+
+    //! Report write progress, emitting an event only when the configured step has been passed
+    void reportProgress(U32 written, U32 total);
+
+    //! Sources and sink used when applying a delta patch
+    class PatchIo;
+
+    //! Compressed patch input and decoded output, used when a patch carries a codec
+    class PatchSource;
+    class PatchSink;
+
+    //! Decompress a patch payload into a scratch file, leaving the apply to operate on plain
+    //! streams. Returns true on success; the caller reports the failure.
+    bool decompressPatch(const Fw::StringBase& patch, const char* scratch_path, U32 expected_size);
+
+    //! Handler implementation for command ASSEMBLE_IMAGE
+    void ASSEMBLE_IMAGE_cmdHandler(FwOpcodeType opCode,                  //!< The opcode
+                                   U32 cmdSeq,                           //!< The command sequence number
+                                   const Fw::CmdStringArg& prefix,       //!< Segment file name prefix
+                                   U16 segments,                         //!< Number of segments to join
+                                   const Fw::CmdStringArg& destination,  //!< Image file to write
+                                   U32 crc32                             //!< Expected CRC32 of the image
+                                   ) override;
+
+    //! Handler implementation for command APPLY_PATCH
+    void APPLY_PATCH_cmdHandler(FwOpcodeType opCode,                  //!< The opcode
+                                U32 cmdSeq,                           //!< The command sequence number
+                                const Fw::CmdStringArg& patch,        //!< Patch file to apply
+                                const Fw::CmdStringArg& destination,  //!< Image file to write
+                                U32 crc32                             //!< Expected CRC32 of the image
+                                ) override;
 
   private:
     // ----------------------------------------------------------------------
@@ -56,9 +106,19 @@ class FlashWorker final : public FlashWorkerComponentBase {
                              U32 crc32                    //!< Expected CRC32 of the file used to verify file integrity
                              ) override;
 
+    //! Handler implementation for run
+    //!
+    //! Ages a test-booted image toward self-confirmation
+    void run_handler(FwIndexType portNum,  //!< The port number
+                     U32 context           //!< The call order
+                     ) override;
+
   private:
-    Step m_last_successful;
+    UpdateSequencer m_sequencer;
+    U32 m_pending_confirm_seconds;  //!< Seconds the running image has been up while unconfirmed
+    U8 m_last_reported_percent;     //!< Percent at the most recent progress event, for step throttling
     U8 m_data[CONFIG_IMG_BLOCK_BUF_SIZE];
+    U8 m_window[LzssDecoder::WINDOW_SIZE];  //!< History buffer for decoding a compressed patch
     struct flash_img_context m_flash_context;
 };
 

PROVESFlightControllerReference/Components/FlashWorker/LzssDecoder.cpp

@@ -0,0 +1,85 @@
+// ======================================================================
+// \title  LzssDecoder.cpp
+// \brief  cpp file for decoding the LZ77 stream a delta patch is carried in
+// ======================================================================
+
+#include "LzssDecoder.hpp"
+
+namespace Components {
+
+// Out of line definitions so these can be odr-used by callers; not every translation unit here is
+// compiled as C++17, where static constexpr members would be implicitly inline.
+constexpr size_t LzssDecoder::WINDOW_SIZE;
+constexpr uint8_t LzssDecoder::MIN_MATCH;
+
+LzssDecoder::Error LzssDecoder ::decode(Source& source, Sink& sink, uint32_t expected_size, uint8_t* window) {
+    if (window == nullptr) {
+        return Error::OUTPUT_WRITE_FAILED;
+    }
+
+    uint32_t produced = 0;
+    // Position in the ring buffer where the next produced byte goes
+    size_t cursor = 0;
+
+    while (produced < expected_size) {
+        uint8_t tag = 0;
+        if (!source.read(&tag, 1)) {
+            return Error::TRUNCATED_INPUT;
+        }
+
+        for (uint8_t bit = 0; bit < 8; bit++) {
+            if (produced >= expected_size) {
+                // The final group is padded; ignore whatever follows the last real token
+                break;
+            }
+
+            if ((tag & (1U << bit)) != 0) {
+                // Literal
+                uint8_t value = 0;
+                if (!source.read(&value, 1)) {
+                    return Error::TRUNCATED_INPUT;
+                }
+                if (!sink.write(&value, 1)) {
+                    return Error::OUTPUT_WRITE_FAILED;
+                }
+                window[cursor] = value;
+                cursor = (cursor + 1) % LzssDecoder::WINDOW_SIZE;
+                produced++;
+                continue;
+            }
+
+            // Match: distance back, then length
+            uint8_t encoded[3] = {0, 0, 0};
+            if (!source.read(encoded, sizeof(encoded))) {
+                return Error::TRUNCATED_INPUT;
+            }
+            const size_t distance = static_cast<size_t>(encoded[0]) | (static_cast<size_t>(encoded[1]) << 8);
+            const size_t length = static_cast<size_t>(encoded[2]) + LzssDecoder::MIN_MATCH;
+
+            if ((distance == 0) || (distance > LzssDecoder::WINDOW_SIZE) ||
+                (distance > static_cast<size_t>(produced))) {
+                return Error::BAD_DISTANCE;
+            }
+            if (length > (expected_size - produced)) {
+                return Error::OUTPUT_OVERRUN;
+            }
+
+            // Copied one byte at a time on purpose: a match is allowed to overlap the bytes it is
+            // producing, which is how a run is encoded, so the source advances with the output
+            size_t from = (cursor + LzssDecoder::WINDOW_SIZE - distance) % LzssDecoder::WINDOW_SIZE;
+            for (size_t i = 0; i < length; i++) {
+                const uint8_t value = window[from];
+                if (!sink.write(&value, 1)) {
+                    return Error::OUTPUT_WRITE_FAILED;
+                }
+                window[cursor] = value;
+                cursor = (cursor + 1) % LzssDecoder::WINDOW_SIZE;
+                from = (from + 1) % LzssDecoder::WINDOW_SIZE;
+            }
+            produced += static_cast<uint32_t>(length);
+        }
+    }
+    return Error::NONE;
+}
+
+}  // namespace Components