fix(nomadnet): bound cache transient-stall so a failing SD seam cannot freeze the UI

lib/tdeck_ui/UI/LXMF/NomadNetCache.cpp

@@ -928,7 +928,48 @@ CacheResult NomadNetCache::invalidate(const CacheKey& key) {
     return result_;
 }
 
-void NomadNetCache::service() {
+void NomadNetCache::service(std::uint64_t now_ms) {
+    // Transient-stall guard (cross-call): compare this call's entry state to
+    // the previous call's. A transient (BUSY/UNAVAILABLE) retry changes none
+    // of the tracked bits, so an unchanged entry across consecutive service()
+    // calls is a stall; any advance resets the counter. The bail is gated on
+    // BOTH a tick floor and an elapsed-time window: a not-ready SD seam
+    // returns UNAVAILABLE immediately (no bus wait), so a pure tick budget
+    // could expire during a legitimate mount window and disable caching for
+    // the whole session. Past both bounds, bail so a persistently unhealthy
+    // SD seam can never pin the cache (and the NomadNet UI at "Checking SD
+    // page cache..."). The baseline is refreshed before the switch so every
+    // path — including early returns — leaves a correct entry for the next
+    // call.
+    if (operation_ != Operation::NONE &&
+        operation_ == transient_prev_op_ &&
+        offset_ == transient_prev_offset_ &&
+        scan_index_ == transient_prev_scan_index_ &&
+        cleanup_index_ == transient_prev_cleanup_index_ &&
+        scan_seen_ == transient_prev_scan_seen_ &&
+        read_open_ == transient_prev_read_open_ &&
+        write_open_ == transient_prev_write_open_) {
+        if (transient_stall_count_ < MAX_TRANSIENT_STALL_TICKS) {
+            ++transient_stall_count_;
+            if (transient_stall_count_ == 1) transient_stall_start_ms_ = now_ms;
+        } else if (static_cast<std::uint32_t>(now_ms - transient_stall_start_ms_) >=
+                   static_cast<std::uint32_t>(MAX_TRANSIENT_STALL_MS)) {
+            // 32-bit unsigned subtraction is wrap-safe across the millis()
+            // wraparound, so the 10s window measures true elapsed time even
+            // when the stall spans the 49.7-day counter rollover.
+            transient_stall_count_ = 0;
+            transient_bail();
+        }
+    } else if (operation_ != Operation::NONE) {
+        transient_stall_count_ = 0;
+    }
+    transient_prev_op_ = operation_;
+    transient_prev_offset_ = offset_;
+    transient_prev_scan_index_ = scan_index_;
+    transient_prev_cleanup_index_ = cleanup_index_;
+    transient_prev_scan_seen_ = scan_seen_;
+    transient_prev_read_open_ = read_open_;
+    transient_prev_write_open_ = write_open_;
     switch (operation_) {
         case Operation::NONE:
             return;
@@ -1428,6 +1469,65 @@ void NomadNetCache::service() {
     }
 }
 
+void NomadNetCache::transient_bail() {
+    // The storage seam has been stalled for far longer than any real SPI
+    // contention or SD mount window. Stop retrying: release any in-flight
+    // storage resources (read handle, partial write, directory enumeration)
+    // so nothing leaks, drop all namespace authority (lookups and commits now
+    // bypass), and clear the in-flight op so the caller's flow falls through
+    // to a live fetch, restoring the pre-cache page-load behavior instead of
+    // a frozen UI. Each release is a bounded best effort: transient
+    // (BUSY/UNAVAILABLE) retries are re-tried a few times, and a still-open
+    // handle is accepted rather than re-pinning the op — the storage seam
+    // stays abandoned for the rest of the session, and its destructor closes
+    // whatever is left.
+    transient_stall_count_ = 0;
+    namespace_authoritative_ = false;
+    const bool list_was_active =
+        operation_ == Operation::RECOVERY_BEGIN ||
+        operation_ == Operation::RECOVERY_NEXT ||
+        operation_ == Operation::RECOVERY_END;
+    if (list_was_active) {
+        for (int attempt = 0; attempt < 3; ++attempt) {
+            const auto list_result = storage_.endList();
+            if (!storage_result_is_transient(list_result)) break;
+        }
+    }
+    if (read_open_) {
+        for (int attempt = 0; attempt < 3; ++attempt) {
+            const auto read_result = storage_.endRead();
+            if (!storage_result_is_transient(read_result)) {
+                read_open_ = false;
+                break;
+            }
+        }
+    }
+    if (write_open_) {
+        for (int attempt = 0; attempt < 3; ++attempt) {
+            const auto write_result = storage_.abortWrite();
+            if (!storage_result_is_transient(write_result)) {
+                write_open_ = false;
+                break;
+            }
+        }
+    }
+    commit_job_ = false;
+    cleanup_stages_after_failure_ = false;
+    eviction_pending_ = false;
+    eviction_generation_ = -1;
+    quota_recovery_ = false;
+    recovery_complete_ = false;
+    offset_ = 0;
+    io_.clear();
+    metadata_bytes_.clear();
+    ExternalVector<std::uint8_t>().swap(body_);
+    operation_ = Operation::NONE;
+    // MISS (not BYPASS): for an admitted reload invalidation the flow accepts
+    // MISS as "the stale generation will not be served", which correctly falls
+    // through to a live fetch; a failed bail must never report a hard error.
+    result_ = CacheResult::MISS;
+}
+
 void NomadNetCache::cancel() {
     if (!busy() || eviction_pending_ || quota_recovery_) {
         return;

lib/tdeck_ui/UI/LXMF/NomadNetCache.h

@@ -93,7 +93,11 @@ class NomadNetCache {
     CacheResult beginRecovery(std::uint64_t now, bool cleanup_stages = false);
     CacheResult invalidate(const CacheKey& key);
 
-    void service();
+    // now_ms is a monotonic millisecond clock used to bound the transient-stall
+    // bail in wall-time (see the stall-guard note below). Production passes
+    // millis(); 0 is a safe default for tests that never expect a stall (the
+    // time window is only reached after 500 consecutive no-progress ticks).
+    void service(std::uint64_t now_ms = 0);
     void cancel();
     bool busy() const { return operation_ != Operation::NONE; }
     bool recoveryComplete() const { return recovery_complete_; }
@@ -226,6 +230,40 @@ class NomadNetCache {
     CacheResult quota_result_ = CacheResult::STORED;
     bool quota_recovery_ = false;
 
+    // Transient-stall guard. A storage op that returns BUSY/UNAVAILABLE (SPI
+    // mutex starved, card not mounted) is retried forever by the step machine;
+    // on a persistently unhealthy seam that pins operation_ != NONE and the
+    // NomadNet UI freezes at "Checking SD page cache...". service() compares
+    // this call's entry state to the previous call's: any advance (op change,
+    // offset, scan/cleanup index, scan count, open-flag) resets the stall
+    // counter, so slow-but-progressing steps (chunked reads/writes, directory
+    // scans) never false-trip; a tick with no advance is a transient stall.
+    //
+    // The bail is time-bounded, not tick-bounded: a not-ready SD seam returns
+    // UNAVAILABLE immediately (no bus wait), so the main loop can spin many
+    // ticks in a second and a pure tick budget would expire during a
+    // legitimate mount window and disable caching for the whole session.
+    // Both a tick floor (protects a pathological fast tick loop) AND an elapsed
+    // wall-time window must pass before the cache bails. 10s comfortably
+    // exceeds the one-time boot mount window (~2.5s, and recovery starts after
+    // UI-ready with the card already mounted) and any SPI contention burst.
+    // On bail the namespace is marked non-authoritative for the session
+    // (lookups/commits then bypass) and the in-flight op is cleared, so the
+    // flow falls through to a live fetch (the pre-cache behavior) instead of
+    // a frozen UI.
+    static constexpr std::uint32_t MAX_TRANSIENT_STALL_TICKS = 500;
+    static constexpr std::uint64_t MAX_TRANSIENT_STALL_MS = 10000;
+    Operation transient_prev_op_ = Operation::NONE;
+    std::size_t transient_prev_offset_ = 0;
+    std::size_t transient_prev_scan_index_ = 0;
+    std::size_t transient_prev_cleanup_index_ = 0;
+    std::size_t transient_prev_scan_seen_ = 0;
+    bool transient_prev_read_open_ = false;
+    bool transient_prev_write_open_ = false;
+    std::uint32_t transient_stall_count_ = 0;
+    std::uint64_t transient_stall_start_ms_ = 0;
+    void transient_bail();
+
     static constexpr std::size_t VERIFY_SCRATCH_BYTES = 1024;
     std::array<std::uint8_t, VERIFY_SCRATCH_BYTES> verify_scratch_{};
     std::uint64_t verify_hash_ = 1469598103934665603ULL;

lib/tdeck_ui/UI/LXMF/NomadNetCacheFlow.cpp

@@ -31,10 +31,10 @@ CacheFlowState NomadNetCacheFlow::begin(const CacheKey& key, std::uint64_t now,
     return state_;
 }
 
-void NomadNetCacheFlow::service() {
+void NomadNetCacheFlow::service(std::uint64_t now_ms) {
     if (state_ == CacheFlowState::CANCELLED || state_ == CacheFlowState::FAILED)
         return;
-    if (cache_.busy()) cache_.service();
+    if (cache_.busy()) cache_.service(now_ms);
     if (state_ == CacheFlowState::INVALIDATE) {
         if (cache_.busy()) return;
         if (!invalidation_admitted_) {

lib/tdeck_ui/UI/LXMF/NomadNetCacheFlow.h

@@ -22,7 +22,7 @@ class NomadNetCacheFlow {
     explicit NomadNetCacheFlow(NomadNetCache& cache) : cache_(cache) {}
 
     CacheFlowState begin(const CacheKey&, std::uint64_t now, bool reload);
-    void service();
+    void service(std::uint64_t now_ms = 0);
     bool acceptLive(const std::vector<std::uint8_t>&, const CacheEligibility&,
                     std::uint64_t now);
     void cancel();

lib/tdeck_ui/UI/LXMF/UIManager.cpp

@@ -3101,7 +3101,7 @@ void UIManager::nomad_update() {
         _nomad_cache_pending_now = 0;
         _nomad_cache_pending_ttl = 0;
     }
-    _nomad_cache_flow.service();
+    _nomad_cache_flow.service(millis());
     if (_nomad_state == NomadState::CACHE) {
         if (_nomad_cache_generation != _nomad_navigation_generation) {
             _nomad_cache_flow.cancel();

tests/native/test_nomadnet_cache.cpp

@@ -23,9 +23,28 @@ struct MemoryStorage final:NomadNetStorage{
  StorageResult stat(const char*n,uint32_t&s)override{++operations;if(n==stat_fail_path&&stat_fail_result!=StorageResult::OK)return stat_fail_result;auto i=files.find(n);if(i==files.end())return StorageResult::MISS;s=i->second.size();return StorageResult::OK;}
  StorageResult beginList(const char*d)override{++operations;list.clear();for(auto&f:files)if(f.first.rfind(std::string(d)+"/",0)==0)list.push_back(f.first);li=0;return available?list_result:StorageResult::UNAVAILABLE;}
  StorageResult nextList(char*n,size_t c,bool&done)override{++operations;if(next_list_fail_at&&static_cast<int>(li+1)==next_list_fail_at)return StorageResult::IO_ERROR;if(li==list.size()){done=true;return StorageResult::OK;}done=false;if(list[li].size()+1>c){++li;return StorageResult::TOO_LARGE;}std::memcpy(n,list[li].c_str(),list[li].size()+1);++li;return StorageResult::OK;}
- StorageResult endList()override{++operations;return StorageResult::OK;}std::vector<std::string>list;size_t li=0;
+ StorageResult endList()override{++operations;if(list_close_busy)return StorageResult::BUSY;return StorageResult::OK;}std::vector<std::string>list;size_t li=0;bool list_close_busy=false;
 };
 static CacheKey key(const char*path="/page/index.mu"){return CacheKey{"0123456789abcdef0123456789abcdef",path,RequestDataClass::NIL};}
+// A seam whose directory enumeration is permanently BUSY (SPI mutex starved
+// for longer than any real contention window) — the exact condition that
+// pinned the boot-time recovery and froze the UI at "Checking SD page cache...".
+struct HangListStorage final:NomadNetStorage{
+ StorageResult hang = StorageResult::OK;
+ bool isAvailable()const override{return true;}
+ StorageResult beginRead(const char*,uint32_t&s)override{s=0;return StorageResult::MISS;}
+ StorageResult readChunk(uint8_t*,size_t,size_t&n)override{n=0;return StorageResult::OK;}
+ StorageResult endRead()override{return StorageResult::OK;}
+ StorageResult beginWrite(const char*)override{return StorageResult::OK;}
+ StorageResult writeChunk(const uint8_t*,size_t z,size_t&n)override{n=z;return StorageResult::OK;}
+ StorageResult commitWrite()override{return StorageResult::OK;}
+ StorageResult abortWrite()override{return StorageResult::OK;}
+ StorageResult remove(const char*)override{return StorageResult::MISS;}
+ StorageResult rename(const char*,const char*)override{return StorageResult::MISS;}
+ StorageResult stat(const char*,uint32_t&)override{return StorageResult::MISS;}
+ StorageResult beginList(const char*)override{return hang;}
+ StorageResult nextList(char*,size_t,bool&d)override{d=true;return StorageResult::OK;}
+ StorageResult endList()override{return StorageResult::OK;}};
 static void drain(NomadNetCache&c,MemoryStorage*s=nullptr){for(int i=0;i<1000&&c.busy();++i){const auto before=s?s->operations:0;c.service();if(s&&s->operations-before>1)throw std::runtime_error("more than one storage operation per service tick");}}
 int main(){int f=0;auto ck=[&](bool x,const char*n){if(!x){++f;std::cerr<<"FAIL "<<n<<"\n";}};MemoryStorage s;CacheConfig cfg;cfg.max_entries=2;cfg.max_bytes=4096;cfg.max_scan_records=8;NomadNetCache c(s,cfg);drain(c,&s);const std::vector<uint8_t> body={'h','e','l','l','o'};
  ck(canonical_cache_key(key())=="0123456789abcdef0123456789abcdef\n/page/index.mu\nnil","canonical key");
@@ -45,7 +64,60 @@ int main(){int f=0;auto ck=[&](bool x,const char*n){if(!x){++f;std::cerr<<"FAIL
  // Interrupted promotion keeps a valid prior generation.
  MemoryStorage s2;NomadNetCache c2(s2,cfg);drain(c2,&s2);ck(c2.beginCommit(key(),body,300,60)==CacheResult::PENDING,"seed");drain(c2);s2.fail_rename_at=2;ck(c2.beginCommit(key(),std::vector<uint8_t>{'x'},301,60)==CacheResult::PENDING,"crash commit");drain(c2);ck(c2.lastResult()==CacheResult::STORAGE_ERROR,"rename crash reported");s2.fail_rename_at=0;ck(c2.beginLookup(key(),302)==CacheResult::PENDING,"fallback after crash");drain(c2);ck(c2.takeBody(got)&&std::equal(got.begin(),got.end(),body.begin(),body.end()),"prior generation survives");
  // SD faults are cache misses to caller, never page failures.
- s2.available=false;ck(c2.beginLookup(key(),302)==CacheResult::PENDING,"unavailable begins");drain(c2);ck(c2.lastResult()==CacheResult::BYPASS,"unavailable bypass");s2.available=true;s2.busy=true;ck(c2.beginLookup(key(),302)==CacheResult::PENDING,"busy begins");drain(c2);ck(c2.lastResult()==CacheResult::BYPASS,"busy bypass");
+ s2.available=false;ck(c2.beginLookup(key(),302)==CacheResult::PENDING,"unavailable begins");drain(c2);ck(c2.lastResult()==CacheResult::BYPASS,"unavailable bypass");s2.available=true;s2.busy=true;ck(c2.beginLookup(key(),302)==CacheResult::PENDING,"busy begins");drain(c2);ck(c2.lastResult()==CacheResult::BYPASS,"busy bypass");s2.busy=false;
+ // A persistently transient seam during boot-time recovery must not pin the
+ // cache (and the UI). Before this fix the recovery retried beginList forever;
+ // now the stall budget expires, the namespace is marked non-authoritative,
+ // and the op clears so the flow can fall through to a live fetch. The bail is
+ // gated on wall-time, so the driver advances a fake monotonic clock (1ms per
+ // tick) so the time window elapses in a bounded number of ticks.
+ auto stall_run=[](NomadNetCache&c){uint64_t now=0;for(int i=0;i<200000&&c.busy();++i){now+=1;c.service(now);}};
+ MemoryStorage busy_boot;busy_boot.available=false;NomadNetCache bc(busy_boot,cfg);
+ stall_run(bc);
+ ck(!bc.busy(),"persistent transient recovery bails instead of pinning");
+ ck(!bc.recoveryComplete(),"bailed recovery is not authoritative");
+ ck(bc.beginLookup(key(),100)==CacheResult::BYPASS,"post-bail lookup bypasses to live");
+ ck(busy_boot.active.empty(),"bail does not leak a read handle");
+ // F3: 500 fast no-progress ticks are NOT enough to bail; the wall-time window
+ // (10s) must also elapse. With a flat (slow) clock the stall must survive well
+ // past the tick floor so a healthy SD that stays briefly quiet keeps its cache
+ // authority for the session.
+ MemoryStorage slow_boot;slow_boot.available=false;NomadNetCache sc(slow_boot,cfg);
+ uint64_t flat=0;for(int i=0;i<600&&sc.busy();++i){flat+=1;sc.service(flat);}
+ ck(sc.busy(),"fast ticks alone do not disable the session cache");
+ // F3/wrap: the wall-time window must measure true elapsed time across the
+ // 32-bit millis() rollover. millis() is a 32-bit counter zero-extended at the
+ // call site, so model it that way: a stall that starts 7.8s below the counter
+ // max and wraps must still bail after 10s of elapsed ticks, not wait ~49.7
+ // days for the counter to lap the pre-wrap start value.
+ MemoryStorage wrap_boot;wrap_boot.available=false;NomadNetCache wc(wrap_boot,cfg);
+ const uint64_t wrap_pre=4294959500ULL;
+ for(int i=0;i<500&&wc.busy();++i)wc.service(wrap_pre+i);
+ ck(wc.busy(),"stall still active at tick floor before wrap");
+ uint64_t now=4294960000ULL;int wcap=0;
+ while(wc.busy()&&wcap<200000){wc.service(static_cast<std::uint32_t>(now));++now;++wcap;}
+ ck(!wc.busy(),"stall spanning the millis wrap still bails");
+ ck(wcap<200000,"wrap-spanning stall bails in bounded ticks, not ~49.7 days");
+ // A seam that heals within the window terminates recovery cleanly. Boot
+ // recovery transients do not degrade the namespace (the RECOVERY_BEGIN/END
+ // paths keep it authoritative), so a healable transient still completes
+ // recovery authoritatively and the session cache remains usable.
+ MemoryStorage recover;recover.available=false;NomadNetCache hr(recover,cfg);
+ uint64_t hrnow=0;int guard=0;for(;hr.busy()&&guard<100000;++guard,++hrnow){if(guard==500)recover.available=true;hr.service(hrnow);}
+ ck(!hr.busy(),"seam that heals within the window terminates recovery");
+ ck(hr.recoveryComplete(),"healable transient keeps the namespace authoritative");
+ ck(hr.beginLookup(key(),100)==CacheResult::PENDING,"post-heal lookup uses the cache, not a bypass");
+ // F2: a bail while the directory enumeration handle is open (RECOVERY_END
+ // endList stuck BUSY) must release it rather than leak it. A fresh boot
+ // recovery opens the list in RECOVERY_BEGIN and closes it in RECOVERY_END, so
+ // a seam whose endList is persistently BUSY reaches the guard with the list
+ // handle open — exactly the device case a stale SPI mutex can produce.
+ MemoryStorage list_seam;list_seam.list_close_busy=true;
+ NomadNetCache list_recover(list_seam,cfg);
+ uint64_t lnow=0;for(int i=0;i<200000&&list_recover.busy();++i){lnow+=1;list_recover.service(lnow);}
+ ck(!list_recover.busy(),"list-open stall bails instead of pinning");
+ ck(!list_recover.recoveryComplete(),"bailed recovery is not authoritative");
+ ck(list_recover.beginLookup(key(),100)==CacheResult::BYPASS,"post list-bail lookup bypasses to live");
  // Deterministic expired-first then oldest quota eviction.
  MemoryStorage s3;NomadNetCache c3(s3,cfg);drain(c3,&s3);for(int i=0;i<3;i++){auto k=key((std::string("/page/")+char('a'+i)).c_str());ck(c3.beginCommit(k,body,400+i,i==0?1:100)==CacheResult::PENDING,"quota commit");drain(c3);}ck(c3.entryCount()<=2&&c3.totalBytes()<=cfg.max_bytes,"quotas bounded");ck(c3.beginLookup(key("/page/a"),500)==CacheResult::PENDING,"evicted lookup");drain(c3);ck(c3.lastResult()!=CacheResult::HIT,"expired evicted first");