From 87d2020fdcdd8f397aae181b8ec2047aaf690c22 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Fri, 12 Jun 2026 02:08:40 +0100 Subject: [PATCH 01/13] NodeDB: 3-tier node store with persistent warm tier (long-tail identity retention) Introduces a tiered NodeDB so the device retains identity (public key, last_heard) for far more nodes than fit in the full-record hot store, without growing heap or the persisted nodes.proto unboundedly. - Hot store: full NodeInfoLite, MAX_NUM_NODES (120 on nRF52). - Satellite maps: position/telemetry/environment/status capped at MAX_SATELLITE_NODES (40 freshest); eviction via enforceSatelliteCaps / evictSatelliteOverCap. - Warm tier (WarmNodeStore): 40 B {num,last_heard,public_key} records for evicted nodes so DMs to/from long-tail nodes keep encrypting/decrypting. Persisted to /prefs/warm.dat, or on nRF52840 a dedicated 12 KB raw-flash record-ring below LittleFS (3x4 KB pages; see linker scripts + the nrf52_warm_region.py post-link guard). NodeDB::getOrCreateMeshNode now demotes evicted nodes into the warm tier and re-admits them (restoring key/last_heard). Router PKI decrypt/encode resolve the peer key via NodeDB::copyPublicKey (hot store, then warm tier). NodeInfoLite gains snr_q4 (sint32, Q4-encoded dB); the float snr is zeroed on disk. NodeInfoLite grows 105 -> 112 B; backup 2432 -> 2468 B. Note: the snr_q4 .proto change still needs to land in the protobufs submodule (generated header is updated here; submodule pointer left at upstream). Co-Authored-By: Claude Opus 4.8 (1M context) --- extra_scripts/nrf52_warm_region.py | 73 +++ src/mesh/NodeDB.cpp | 202 ++++++- src/mesh/NodeDB.h | 46 +- src/mesh/Router.cpp | 30 +- src/mesh/WarmNodeStore.cpp | 503 ++++++++++++++++++ src/mesh/WarmNodeStore.h | 122 +++++ src/mesh/generated/meshtastic/deviceonly.pb.h | 2 +- src/mesh/mesh-pb-constants.h | 71 ++- src/platform/nrf52/nrf52840_s140_v6.ld | 46 ++ src/platform/nrf52/nrf52840_s140_v7.ld | 7 +- test/test_warm_store/test_main.cpp | 211 ++++++++ variants/nrf52840/nrf52.ini | 1 + variants/nrf52840/nrf52840.ini | 4 + 13 files changed, 1274 insertions(+), 44 deletions(-) create mode 100644 extra_scripts/nrf52_warm_region.py create mode 100644 src/mesh/WarmNodeStore.cpp create mode 100644 src/mesh/WarmNodeStore.h create mode 100644 src/platform/nrf52/nrf52840_s140_v6.ld create mode 100644 test/test_warm_store/test_main.cpp diff --git a/extra_scripts/nrf52_warm_region.py b/extra_scripts/nrf52_warm_region.py new file mode 100644 index 00000000000..adc09a18e1e --- /dev/null +++ b/extra_scripts/nrf52_warm_region.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports +# +# Post-link guard for the warm-node-store raw-flash region on nrf52840. +# +# The top 3 app-region flash pages below LittleFS (0xEA000-0xED000, reclaimed by +# whole-image LTO; LittleFS itself stays stock at 0xED000) are reserved for the WarmNodeStore record-ring +# (src/mesh/WarmNodeStore.h, 3-page record-ring). Our own linker script +# (src/platform/nrf52/nrf52840_s140_v7.ld) already caps the image at 0xEA000, +# but many boards (e.g. rak4631) link with the framework-default script whose +# FLASH region still ends at 0xED000 -- those would silently place code in the +# reserved pages if the image ever grew that large, and the first warm-store +# save would then brick the firmware. This guard turns that into a red build. +# +# Image flash end = __etext (end of code/rodata in flash) + sizeof(.data) +# (which is loaded at LMA __etext). Symbols come from the framework's +# nrf52_common.ld. +import os + +Import("env") + +WARM_REGION_BASE = 0xEA000 # keep in sync with WARM_FLASH_REGION_BASE in WarmNodeStore.h (3 x 4 KB record-ring) + +_tc = env.PioPlatform().get_package_dir("toolchain-gccarmnoneeabi") or "" +_NM = os.path.join(_tc, "bin", "arm-none-eabi-nm") +if not os.path.isfile(_NM): + _NM = "arm-none-eabi-nm" # fall back to PATH + + +def _assert_warm_region_clear(source, target, env): + import subprocess + import sys + + try: + elf = env.subst("$BUILD_DIR/${PROGNAME}.elf") + out = subprocess.check_output([_NM, elf], universal_newlines=True) + except Exception as exc: # tooling hiccup: warn loudly, don't wedge the build + print("nrf52_warm_region: WARNING - guard skipped (nm failed: %s)" % exc) + return + + syms = {} + for line in out.split("\n"): + f = line.split() + if len(f) >= 3 and f[-1] in ("__etext", "__data_start__", "__data_end__"): + syms[f[-1]] = int(f[0], 16) + if len(syms) != 3: + print("nrf52_warm_region: WARNING - guard skipped (linker symbols not found)") + return + + flash_end = syms["__etext"] + (syms["__data_end__"] - syms["__data_start__"]) + if flash_end > WARM_REGION_BASE: + sys.stderr.write( + "\n*** nrf52 warm-region guard: image ends at 0x%X, past the reserved " + "warm-store region at 0x%X ***\n" + "The 12 KB region at 0xEA000 holds the WarmNodeStore record-ring; a warm-store\n" + "save would overwrite this firmware's tail. Shrink the image, or shrink/move\n" + "the region (WARM_FLASH_REGION_BASE in src/mesh/WarmNodeStore.h, the FLASH\n" + "LENGTH in src/platform/nrf52/nrf52840_s140_v7.ld, and this guard).\n\n" + % (flash_end, WARM_REGION_BASE) + ) + from SCons.Script import Exit + + Exit(1) + print( + "nrf52_warm_region: guard OK -- image ends at 0x%X, %d KB clear of the warm region" + % (flash_end, (WARM_REGION_BASE - flash_end) // 1024) + ) + + +# Attach to the phony "buildprog" alias (not the .elf node) so the guard runs +# on incremental relinks too -- same reasoning as nrf52_lto.py's guard. +env.AddPostAction("buildprog", _assert_warm_region_clear) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 2481eb8ca6b..11604f33541 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -196,6 +196,8 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre if (ostream) { const auto *vec = static_cast *>(iter->pData); for (auto item : *vec) { + item.snr_q4 = (int32_t)(item.snr * 4.0f); + item.snr = 0.0f; if (!pb_encode_tag_for_field(ostream, iter)) return false; if (!pb_encode_submessage(ostream, meshtastic_NodeInfoLite_fields, &item)) @@ -205,8 +207,12 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre if (istream && istream->bytes_left) { meshtastic_NodeInfoLite node = meshtastic_NodeInfoLite_init_zero; auto *vec = static_cast *>(iter->pData); - if (pb_decode(istream, meshtastic_NodeInfoLite_fields, &node)) + if (pb_decode(istream, meshtastic_NodeInfoLite_fields, &node)) { + if (node.snr_q4) + node.snr = node.snr_q4 / 4.0f; + node.snr_q4 = 0; vec->push_back(node); + } } return true; } @@ -688,6 +694,39 @@ template std::vector snapshotSatelliteNodeNums(const Map } return result; } + +// Drop the stalest entry of `map` (staleness proxied via the owner's +// last_heard; 0 = owner evicted, i.e. an orphan — first out). Never evicts our +// own node's entry. Caller holds satelliteMutex. Returns false if nothing +// could be evicted. +template bool evictStalestSatellite(NodeDB &db, Map &map) +{ + auto victim = map.end(); + uint32_t victimTs = UINT32_MAX; + for (auto it = map.begin(); it != map.end(); ++it) { + if (it->first == db.getNodeNum()) + continue; + uint32_t ts = db.hotNodeLastHeard(it->first); + if (ts < victimTs) { + victimTs = ts; + victim = it; + } + } + if (victim == map.end()) + return false; + map.erase(victim); + return true; +} + +// Keep `map` within MAX_SATELLITE_NODES ahead of inserting `incoming` (the +// tier-1/tier-2 split: only the freshest MAX_SATELLITE_NODES nodes carry +// satellite payloads). Caller holds satelliteMutex. +template void evictSatelliteOverCap(NodeDB &db, Map &map, NodeNum incoming) +{ + if (map.size() < MAX_SATELLITE_NODES || map.count(incoming)) + return; + evictStalestSatellite(db, map); +} } // namespace void NodeDB::resetRadioConfig(bool is_fresh_install) @@ -727,6 +766,12 @@ bool NodeDB::factoryReset(bool eraseBleBonds) if (transmitHistory) { transmitHistory->clear(); } +#if WARM_NODE_COUNT > 0 + // On nRF52840 the warm tier lives in raw flash outside /prefs, so rmDir + // didn't touch it; clear it and persist the empty store. + warmStore.clear(); + warmStore.saveIfDirty(); +#endif // second, install default state (this will deal with the duplicate mac address issue) installDefaultNodeDatabase(); installDefaultDeviceState(); @@ -1298,6 +1343,9 @@ void NodeDB::resetNodes(bool keepFavorites) std::fill(nodeDatabase.nodes.begin() + 1, nodeDatabase.nodes.end(), meshtastic_NodeInfoLite()); } (void)ourNum; +#if WARM_NODE_COUNT > 0 + warmStore.clear(); // warm entries are never favorites; a DB reset clears them too +#endif devicestate.has_rx_waypoint = false; saveNodeDatabaseToDisk(); saveDeviceStateToDisk(); @@ -1319,6 +1367,10 @@ void NodeDB::removeNodeByNum(NodeNum nodeNum) meshtastic_NodeInfoLite()); if (removed) eraseNodeSatellites(nodeNum); +#if WARM_NODE_COUNT > 0 + // Explicit user removal: don't let the warm tier resurrect the node + warmStore.remove(nodeNum); +#endif LOG_DEBUG("NodeDB::removeNodeByNum purged %d entries. Save changes", removed); saveNodeDatabaseToDisk(); } @@ -1432,6 +1484,7 @@ void NodeDB::setNodeStatus(NodeNum n, const meshtastic_StatusMessage &status) (void)status; #else concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeStatus, n); nodeStatus[n] = status; #endif } @@ -1443,6 +1496,7 @@ void NodeDB::touchNodePositionTime(NodeNum n, uint32_t time) (void)time; #else concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodePositions, n); nodePositions[n].time = time; #endif } @@ -1464,6 +1518,34 @@ void NodeDB::eraseNodeSatellites(NodeNum n) #endif } +void NodeDB::enforceSatelliteCaps() +{ + concurrency::LockGuard guard(&satelliteMutex); + auto trim = [this](auto &map, const char *name) { + const size_t before = map.size(); + while (map.size() > MAX_SATELLITE_NODES) { + if (!evictStalestSatellite(*this, map)) + break; + } + if (map.size() != before) + LOG_INFO("Trimmed %s satellites %u -> %u (cap %d)", name, (unsigned)before, (unsigned)map.size(), + MAX_SATELLITE_NODES); + }; +#if !MESHTASTIC_EXCLUDE_POSITIONDB + trim(nodePositions, "position"); +#endif +#if !MESHTASTIC_EXCLUDE_TELEMETRYDB + trim(nodeTelemetry, "telemetry"); +#endif +#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB + trim(nodeEnvironment, "environment"); +#endif +#if !MESHTASTIC_EXCLUDE_STATUSDB + trim(nodeStatus, "status"); +#endif + (void)trim; // all four maps may be compiled out +} + void NodeDB::cleanupMeshDB() { int newPos = 0, removed = 0; @@ -1482,8 +1564,15 @@ void NodeDB::cleanupMeshDB() } else { // No user info - drop this node and its satellites const NodeNum gone = n.num; - if (gone) + if (gone) { +#if WARM_NODE_COUNT > 0 + // Keep any key we learned (e.g. via a DM before the NodeInfo + // exchange completed) rather than losing it with the purge. + if (n.public_key.size == 32) + warmStore.absorb(gone, n.last_heard, n.public_key.bytes); +#endif eraseNodeSatellites(gone); + } removed++; } } @@ -1633,6 +1722,38 @@ LoadFileResult NodeDB::loadProto(const char *filename, size_t protoSize, size_t return state; } +#if WARM_NODE_COUNT > 0 +void NodeDB::demoteOldestHotNodesToWarm() +{ + if (numMeshNodes <= MAX_NUM_NODES) + return; + + // Rank: favorites ahead of recency (a user-marked node is never demoted in + // favour of a more-recently-heard stranger), then most-recently-heard first. + // After the sort, [0, MAX_NUM_NODES) are the keepers and the tail is overflow. + std::sort(meshNodes->begin(), meshNodes->begin() + numMeshNodes, + [](const meshtastic_NodeInfoLite &a, const meshtastic_NodeInfoLite &b) { + const bool fa = nodeInfoLiteIsFavorite(&a); + const bool fb = nodeInfoLiteIsFavorite(&b); + if (fa != fb) + return fa; + return a.last_heard > b.last_heard; + }); + + int demoted = 0; + for (int i = MAX_NUM_NODES; i < numMeshNodes; i++) { + const meshtastic_NodeInfoLite &n = (*meshNodes)[i]; + if (n.num == 0) + continue; + // Keep the public key if we have one (40 B warm record); keyless nodes + // still get a placeholder so re-admission restores last_heard. + warmStore.absorb(n.num, n.last_heard, n.public_key.size > 0 ? n.public_key.bytes : nullptr); + demoted++; + } + LOG_INFO("NodeDB migration: demoted %d oldest node(s) above MAX_NUM_NODES %d into the warm tier", demoted, MAX_NUM_NODES); +} +#endif + void NodeDB::loadFromDisk() { // Mark the current device state as completely unusable, so that if we fail reading the entire file from @@ -1776,12 +1897,32 @@ void NodeDB::loadFromDisk() nodeDatabase.nodes.size(), posCount, telCount, envCount, statusCount); } +#if WARM_NODE_COUNT > 0 + // Load the warm tier first so a hot-store migration below merges into the + // persisted set instead of being clobbered by the on-disk warm snapshot. + warmStore.load(); + // A database from a larger-cap build (the pre-fork 150-node nRF52 hot store) + // can exceed MAX_NUM_NODES. Demote the oldest overflow into the warm tier + // before we truncate, so their public keys survive for PKI DMs. + demoteOldestHotNodesToWarm(); +#endif + if (numMeshNodes > MAX_NUM_NODES) { LOG_WARN("Node count %d exceeds MAX_NUM_NODES %d, truncating", numMeshNodes, MAX_NUM_NODES); numMeshNodes = MAX_NUM_NODES; } meshNodes->resize(MAX_NUM_NODES); + // Files written before the satellite cap (or by a larger-cap build) can + // carry more satellite entries than we keep; trim to the freshest. + enforceSatelliteCaps(); + +#if WARM_NODE_COUNT > 0 + // Persist the migrated entries now so they survive a reboot before the next + // node-DB save cadence. + warmStore.saveIfDirty(); +#endif + // static DeviceState scratch; We no longer read into a tempbuf because this structure is 15KB of valuable RAM state = loadProto(deviceStateFileName, meshtastic_DeviceState_size, sizeof(meshtastic_DeviceState), &meshtastic_DeviceState_msg, &devicestate); @@ -2302,7 +2443,6 @@ bool NodeDB::saveNodeDatabaseToDisk() #else nodeDatabase.status.clear(); #endif - size_t nodeDatabaseSize; pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &nodeDatabase); bool ok = saveProto(nodeDatabaseFileName, nodeDatabaseSize, &meshtastic_NodeDatabase_msg, &nodeDatabase, false); @@ -2315,6 +2455,11 @@ bool NodeDB::saveNodeDatabaseToDisk() nodeDatabase.environment.shrink_to_fit(); nodeDatabase.status.clear(); nodeDatabase.status.shrink_to_fit(); +#if WARM_NODE_COUNT > 0 + // Same cadence as the node DB; failure is logged but must not propagate — + // a false return from here would trigger saveToDisk()'s fsFormat() path. + warmStore.saveIfDirty(); +#endif return ok; } @@ -2551,6 +2696,7 @@ void NodeDB::updatePosition(uint32_t nodeId, const meshtastic_Position &p, RxSou #else { concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodePositions, nodeId); meshtastic_PositionLite &slot = nodePositions[nodeId]; // creates default-zero entry if missing if (src == RX_SRC_LOCAL) { @@ -2608,6 +2754,7 @@ void NodeDB::updateTelemetry(uint32_t nodeId, const meshtastic_Telemetry &t, RxS } #if !MESHTASTIC_EXCLUDE_TELEMETRYDB concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeTelemetry, nodeId); nodeTelemetry[nodeId] = t.variant.device_metrics; #endif } else if (t.which_variant == meshtastic_Telemetry_environment_metrics_tag) { @@ -2618,6 +2765,7 @@ void NodeDB::updateTelemetry(uint32_t nodeId, const meshtastic_Telemetry &t, RxS } #if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeEnvironment, nodeId); nodeEnvironment[nodeId] = t.variant.environment_metrics; #endif } else { @@ -2653,6 +2801,9 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, false); eraseNodeSatellites(contact.node_num); +#if WARM_NODE_COUNT > 0 + warmStore.remove(contact.node_num); // ignored: drop the retained key too +#endif info->public_key.size = 0; memset(info->public_key.bytes, 0, sizeof(info->public_key.bytes)); } else { @@ -2948,6 +3099,30 @@ bool NodeDB::isFull() return (numMeshNodes >= MAX_NUM_NODES) || (memGet.getFreeHeap() < MINIMUM_SAFE_FREE_HEAP); } +uint32_t NodeDB::hotNodeLastHeard(NodeNum n) const +{ + for (int i = 0; i < numMeshNodes; i++) + if (meshNodes->at(i).num == n) + return meshNodes->at(i).last_heard; + return 0; +} + +bool NodeDB::copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out) +{ + const meshtastic_NodeInfoLite *info = getMeshNode(n); + if (info && info->public_key.size == 32) { + out = info->public_key; + return true; + } +#if WARM_NODE_COUNT > 0 + if (warmStore.copyKey(n, out.bytes)) { + out.size = 32; + return true; + } +#endif + return false; +} + /// Find a node in our DB, create an empty NodeInfo if missing meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) { @@ -2984,7 +3159,14 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) } if (oldestIndex != -1) { - eraseNodeSatellites(meshNodes->at(oldestIndex).num); + const meshtastic_NodeInfoLite &evicted = meshNodes->at(oldestIndex); +#if WARM_NODE_COUNT > 0 + // Demote to the warm tier so the identity (and crucially the + // PKI key) outlives the hot-store slot. + warmStore.absorb(evicted.num, evicted.last_heard, + evicted.public_key.size == 32 ? evicted.public_key.bytes : NULL); +#endif + eraseNodeSatellites(evicted.num); // Shove the remaining nodes down the chain for (int i = oldestIndex; i < numMeshNodes - 1; i++) { meshNodes->at(i) = meshNodes->at(i + 1); @@ -2998,6 +3180,18 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) // everything is missing except the nodenum memset(lite, 0, sizeof(*lite)); lite->num = n; +#if WARM_NODE_COUNT > 0 + // Re-admission: restore what the warm tier kept for this node + WarmNodeEntry warm; + if (warmStore.take(n, warm)) { + lite->last_heard = warm.last_heard; + if (!memfll(warm.public_key, 0, sizeof(warm.public_key))) { + lite->public_key.size = 32; + memcpy(lite->public_key.bytes, warm.public_key, 32); + } + LOG_INFO("Rehydrated node 0x%x from warm tier (key=%d)", n, lite->public_key.size == 32); + } +#endif LOG_INFO("Adding node to database with %i nodes and %u bytes free!", numMeshNodes, memGet.getFreeHeap()); } diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index 3fe24429406..3c35e379f89 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -11,6 +11,7 @@ #include "MeshTypes.h" #include "NodeStatus.h" +#include "WarmNodeStore.h" #include "concurrency/Lock.h" #include "configuration.h" #include "mesh-pb-constants.h" @@ -296,6 +297,20 @@ class NodeDB virtual meshtastic_NodeInfoLite *getMeshNode(NodeNum n); size_t getNumMeshNodes() { return numMeshNodes; } +#if WARM_NODE_COUNT > 0 + // Warm ("long-tail") tier: minimal {num, last_heard, public_key} records + // for nodes evicted from the hot store. See WarmNodeStore.h. + WarmNodeStore warmStore; +#endif + + /// Copy the 32-byte public key for node n — hot store first, then the warm + /// tier. Returns false if we don't know a key for n. + bool copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out); + + /// last_heard of a hot-store node, or 0 if absent. Plain scan of meshNodes + /// with no allocation side effects (unlike getOrCreateMeshNode). + uint32_t hotNodeLastHeard(NodeNum n) const; + // Thread-safe satellite-map accessors. Return false if absent or the // corresponding DB is compiled out. bool copyNodePosition(NodeNum n, meshtastic_PositionLite &out) const; @@ -341,11 +356,17 @@ class NodeDB emptyNodeDatabase.version = DEVICESTATE_CUR_VER; size_t nodeDatabaseSize; pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &emptyNodeDatabase); - // Always include satellite slots so backups from higher-cap peers - // decode without truncation, even when our build excludes the DBs. - return nodeDatabaseSize + (MAX_NUM_NODES * meshtastic_NodeInfoLite_size) + - (MAX_NUM_NODES * meshtastic_NodePositionEntry_size) + (MAX_NUM_NODES * meshtastic_NodeTelemetryEntry_size) + - (MAX_NUM_NODES * meshtastic_NodeEnvironmentEntry_size) + (MAX_NUM_NODES * meshtastic_NodeStatusEntry_size); + // Decode-stream size ceiling only — no buffer of this size is ever + // allocated (load streams from the file). Sized for the largest file + // any previous firmware could have written (250-node ESP32-S3 builds, + // satellites uncapped), not the current MAX_NUM_NODES, so capacity + // downgrades and backups from higher-cap peers still decode; excess + // entries are trimmed after load. + // (not constexpr: portduino resolves MAX_NUM_NODES from runtime config) + const size_t loadCeiling = ((size_t)MAX_NUM_NODES > 250) ? (size_t)MAX_NUM_NODES : 250; + return nodeDatabaseSize + (loadCeiling * meshtastic_NodeInfoLite_size) + + (loadCeiling * meshtastic_NodePositionEntry_size) + (loadCeiling * meshtastic_NodeTelemetryEntry_size) + + (loadCeiling * meshtastic_NodeEnvironmentEntry_size) + (loadCeiling * meshtastic_NodeStatusEntry_size); } // returns true if the maximum number of nodes is reached or we are running low on memory @@ -450,6 +471,21 @@ class NodeDB /// purge db entries without user info void cleanupMeshDB(); + /// Trim each satellite map down to MAX_SATELLITE_NODES, dropping the + /// stalest entries (used after loading files written before the cap, or by + /// a build with a larger cap). + void enforceSatelliteCaps(); + +#if WARM_NODE_COUNT > 0 + /// On load, a database from a larger-cap build (notably the pre-fork 150-node + /// nRF52 hot store) can exceed MAX_NUM_NODES. Rank the hot store so the + /// freshest MAX_NUM_NODES survive and demote the oldest overflow into the + /// warm tier, preserving {num, last_heard, public_key} so PKI DMs keep + /// working instead of dropping those nodes on truncation. Caller truncates + /// the hot store to MAX_NUM_NODES afterwards. + void demoteOldestHotNodesToWarm(); +#endif + /// Reinit device state from scratch (not loading from disk) void installDefaultDeviceState(), installDefaultNodeDatabase(), installDefaultChannels(), installDefaultConfig(bool preserveKey), installDefaultModuleConfig(); diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index f176f60e63b..be4d5e550f8 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -485,14 +485,16 @@ DecodeState perhapsDecode(meshtastic_MeshPacket *p) bool decrypted = false; ChannelIndex chIndex = 0; #if !(MESHTASTIC_EXCLUDE_PKI) - // Attempt PKI decryption first - if (p->channel == 0 && isToUs(p) && p->to > 0 && !isBroadcast(p->to) && nodeDB->getMeshNode(p->from) != nullptr && - nodeDB->getMeshNode(p->from)->public_key.size > 0 && nodeDB->getMeshNode(p->to) != nullptr && - nodeDB->getMeshNode(p->to)->public_key.size > 0 && rawSize > MESHTASTIC_PKC_OVERHEAD) { + // Attempt PKI decryption first. The sender's key may come from the hot + // store or the warm tier (nodes evicted from the hot store keep their key + // there), so DMs from long-tail nodes still decrypt. + meshtastic_NodeInfoLite_public_key_t fromKey = {0, {0}}; + if (p->channel == 0 && isToUs(p) && p->to > 0 && !isBroadcast(p->to) && nodeDB->copyPublicKey(p->from, fromKey) && + nodeDB->getMeshNode(p->to) != nullptr && nodeDB->getMeshNode(p->to)->public_key.size > 0 && + rawSize > MESHTASTIC_PKC_OVERHEAD) { LOG_DEBUG("Attempt PKI decryption"); - if (crypto->decryptCurve25519(p->from, nodeDB->getMeshNode(p->from)->public_key, p->id, rawSize, p->encrypted.bytes, - bytes)) { + if (crypto->decryptCurve25519(p->from, fromKey, p->id, rawSize, p->encrypted.bytes, bytes)) { LOG_INFO("PKI Decryption worked!"); meshtastic_Data decodedtmp; @@ -503,7 +505,7 @@ DecodeState perhapsDecode(meshtastic_MeshPacket *p) decrypted = true; LOG_INFO("Packet decrypted using PKI!"); p->pki_encrypted = true; - memcpy(&p->public_key.bytes, nodeDB->getMeshNode(p->from)->public_key.bytes, 32); + memcpy(&p->public_key.bytes, fromKey.bytes, 32); p->public_key.size = 32; p->decoded = decodedtmp; p->which_payload_variant = meshtastic_MeshPacket_decoded_tag; // change type to decoded @@ -720,7 +722,10 @@ meshtastic_Routing_Error perhapsEncode(meshtastic_MeshPacket *p) ChannelIndex chIndex = p->channel; // keep as a local because we are about to change it #if !(MESHTASTIC_EXCLUDE_PKI) - meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(p->to); + // Destination key from the hot store or the warm tier (evicted + // long-tail nodes keep their key there) + meshtastic_NodeInfoLite_public_key_t destKey = {0, {0}}; + bool haveDestKey = nodeDB->copyPublicKey(p->to, destKey); // We may want to retool things so we can send a PKC packet when the client specifies a key and nodenum, even if the node // is not in the local nodedb // First, only PKC encrypt packets we are originating @@ -743,18 +748,17 @@ meshtastic_Routing_Error perhapsEncode(meshtastic_MeshPacket *p) if (numbytes + MESHTASTIC_HEADER_LENGTH + MESHTASTIC_PKC_OVERHEAD > MAX_LORA_PAYLOAD_LEN) return meshtastic_Routing_Error_TOO_LARGE; // Check for a known public key for the destination - if (node == nullptr || node->public_key.size != 32) { + if (!haveDestKey) { LOG_WARN("Unknown public key for destination node 0x%08x (portnum %d), refusing to send legacy DM", p->to, p->decoded.portnum); return meshtastic_Routing_Error_PKI_SEND_FAIL_PUBLIC_KEY; } - if (p->pki_encrypted && !memfll(p->public_key.bytes, 0, 32) && - memcmp(p->public_key.bytes, node->public_key.bytes, 32) != 0) { + if (p->pki_encrypted && !memfll(p->public_key.bytes, 0, 32) && memcmp(p->public_key.bytes, destKey.bytes, 32) != 0) { LOG_WARN("Client public key differs from requested: 0x%02x, stored key begins 0x%02x", *p->public_key.bytes, - *node->public_key.bytes); + *destKey.bytes); return meshtastic_Routing_Error_PKI_FAILED; } - crypto->encryptCurve25519(p->to, getFrom(p), node->public_key, p->id, numbytes, bytes, p->encrypted.bytes); + crypto->encryptCurve25519(p->to, getFrom(p), destKey, p->id, numbytes, bytes, p->encrypted.bytes); numbytes += MESHTASTIC_PKC_OVERHEAD; p->channel = 0; p->pki_encrypted = true; diff --git a/src/mesh/WarmNodeStore.cpp b/src/mesh/WarmNodeStore.cpp new file mode 100644 index 00000000000..e39327a3657 --- /dev/null +++ b/src/mesh/WarmNodeStore.cpp @@ -0,0 +1,503 @@ +#include "WarmNodeStore.h" + +#if WARM_NODE_COUNT > 0 + +#include "FSCommon.h" +#include "SPILock.h" +#include "SafeFile.h" +#include "configuration.h" +#include "power/PowerHAL.h" +#include +#include + +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) +#include "flash/flash_nrf5x.h" +#define WARM_RING_MAGIC 0x474E5257u // "WRNG" +// A tombstone is an entry record whose last_heard is all-ones — getTime() +// (unix seconds) cannot reach 0xFFFFFFFF until 2106, and erased flash is +// detected via num == 0xFFFFFFFF before last_heard is ever inspected. +#define WARM_RING_TOMBSTONE 0xFFFFFFFFu +#else +// warm.dat layout: this header followed by count packed WarmNodeEntry records. +struct WarmStoreHeader { + uint32_t magic; // WARM_STORE_MAGIC + uint32_t reserved; // 0; kept so the header stays 16 B + uint16_t count; // entries persisted + uint16_t entrySize; // sizeof(WarmNodeEntry), format guard + uint32_t crc; // crc32 over count * entrySize bytes +}; +static_assert(sizeof(WarmStoreHeader) == 16, "header layout is part of the persistence format"); + +#define WARM_STORE_MAGIC 0x314D5257u // "WRM1" + +#ifdef FSCom +static const char *warmFileName = "/prefs/warm.dat"; +#endif +#endif // defined(ARCH_NRF52) && defined(NRF52840_XXAA) + +static inline bool keyIsSet(const uint8_t key[32]) +{ + for (int i = 0; i < 32; i++) + if (key[i]) + return true; + return false; +} + +WarmNodeStore::WarmNodeStore() +{ +#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) + entries = static_cast(ps_calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); + if (entries) { + entriesFromPsram = true; + } else { + LOG_WARN("WarmStore: PSRAM allocation failed, falling back to heap"); + entries = new WarmNodeEntry[WARM_NODE_COUNT](); + } +#else + entries = new WarmNodeEntry[WARM_NODE_COUNT](); +#endif +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + memset(pageOf, 0xFF, sizeof(pageOf)); +#endif +} + +WarmNodeStore::~WarmNodeStore() +{ + if (!entries) + return; + if (entriesFromPsram) + free(entries); + else + delete[] entries; + entries = nullptr; +} + +WarmNodeEntry *WarmNodeStore::find(NodeNum num) const +{ + if (!entries || !num) + return nullptr; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (entries[i].num == num) + return &entries[i]; + return nullptr; +} + +// Slot placement with the keyed-first admission policy. Shared by absorb() +// and the ring replay, so the policy is applied identically in both paths. +WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8_t *key32) +{ + if (!entries || !num) + return nullptr; + + const bool candidateKeyed = key32 && keyIsSet(key32); + + WarmNodeEntry *slot = find(num); + if (!slot) { + // Pick a victim: any empty slot, else the oldest keyless entry, else + // (only for keyed candidates) the oldest keyed entry. + WarmNodeEntry *oldestKeyless = nullptr, *oldestKeyed = nullptr; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) { + WarmNodeEntry &e = entries[i]; + if (!e.num) { + slot = &e; + break; + } + if (keyIsSet(e.public_key)) { + if (!oldestKeyed || e.last_heard < oldestKeyed->last_heard) + oldestKeyed = &e; + } else { + if (!oldestKeyless || e.last_heard < oldestKeyless->last_heard) + oldestKeyless = &e; + } + } + if (!slot) + slot = oldestKeyless ? oldestKeyless : (candidateKeyed ? oldestKeyed : nullptr); + if (!slot) + return nullptr; // store full of keyed entries and the candidate has no key + } + + slot->num = num; + slot->last_heard = lastHeard; + if (candidateKeyed) + memcpy(slot->public_key, key32, 32); + else + memset(slot->public_key, 0, 32); + return slot; +} + +bool WarmNodeStore::absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32) +{ + WarmNodeEntry *slot = place(num, lastHeard, key32); + if (!slot) + return false; + persistEntry(*slot); + return true; +} + +bool WarmNodeStore::take(NodeNum num, WarmNodeEntry &out) +{ + WarmNodeEntry *e = find(num); + if (!e) + return false; + out = *e; + const int idx = static_cast(e - entries); + memset(e, 0, sizeof(*e)); + persistRemove(num, idx); + return true; +} + +bool WarmNodeStore::copyKey(NodeNum num, uint8_t out[32]) const +{ + const WarmNodeEntry *e = find(num); + if (!e || !keyIsSet(e->public_key)) + return false; + memcpy(out, e->public_key, 32); + return true; +} + +bool WarmNodeStore::contains(NodeNum num) const +{ + return find(num) != nullptr; +} + +void WarmNodeStore::remove(NodeNum num) +{ + WarmNodeEntry *e = find(num); + if (e) { + const int idx = static_cast(e - entries); + memset(e, 0, sizeof(*e)); + persistRemove(num, idx); + } +} + +void WarmNodeStore::clear() +{ + if (!entries) + return; + memset(entries, 0, WARM_NODE_COUNT * sizeof(WarmNodeEntry)); +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + memset(pageOf, 0xFF, sizeof(pageOf)); +#endif + persistClear(); +} + +size_t WarmNodeStore::count() const +{ + size_t n = 0; + if (entries) + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (entries[i].num) + n++; + return n; +} + +bool WarmNodeStore::saveIfDirty() +{ + if (!dirty) + return true; + bool ok = save(); + if (ok) + dirty = false; + return ok; +} + +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + +// ---- Raw-flash record-ring backend (nRF52840 only; nRF52832 uses file backend) --- +// 3 × 4 KB pages directly below LittleFS. Mutations append 40 B records (an +// entry snapshot, or a tombstone with last_heard == 0xFFFFFFFF) through the +// shared flash_nrf5x page cache; saveIfDirty() is the durability point. When +// the active page fills, the oldest page is reclaimed: live entries whose +// newest record is stranded there get re-appended first, then the page is +// erased and becomes the new head. All flash access happens under spiLock +// because the page cache is shared with InternalFS/LittleFS. + +bool WarmNodeStore::ringReadHeader(uint8_t page, WarmPageHeader &h) const +{ + flash_nrf5x_read(&h, WARM_FLASH_PAGE_ADDR(page), sizeof(h)); + return h.magic == WARM_RING_MAGIC && h.seq != 0xFFFFFFFFu; +} + +// Erase `page` and stamp a fresh header. Caller holds spiLock. +void WarmNodeStore::ringOpenPage(uint8_t page) +{ + // Drop any cached state for the page before the real erase, so a later + // cache flush can't resurrect stale bytes. + flash_nrf5x_flush(); + flash_nrf5x_erase(WARM_FLASH_PAGE_ADDR(page)); + WarmPageHeader h; + h.magic = WARM_RING_MAGIC; + h.seq = nextSeq++; + flash_nrf5x_write(WARM_FLASH_PAGE_ADDR(page), &h, sizeof(h)); + activePage = page; + writeSlot = 0; +} + +// Reclaim the oldest (or an unused) page and compact stranded live entries +// into it. Caller holds spiLock. May recurse once via ringAppend when the +// stranded set fills the fresh page exactly — bounded by the page count and +// the WARM_NODE_COUNT <= 2 * kRecordsPerPage static_assert. +void WarmNodeStore::ringRotate() +{ + uint8_t target = 0; + if (activePage != 0xFF) { + // Lowest-seq valid page, preferring erased pages; never the active one + uint32_t bestSeq = 0; + bool found = false; + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { + if (p == activePage) + continue; + WarmPageHeader h; + if (!ringReadHeader(p, h)) { + target = p; // erased/invalid page: free real estate, take it + found = true; + break; + } + if (!found || static_cast(h.seq - bestSeq) < 0) { + target = p; + bestSeq = h.seq; + found = true; + } + } + } + + // Capture live entries stranded in the page we're about to erase + int stranded[WARM_NODE_COUNT]; + int nStranded = 0; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) { + if (entries[i].num && pageOf[i] == target) + stranded[nStranded++] = static_cast(i); + if (pageOf[i] == target) + pageOf[i] = 0xFF; + } + + ringOpenPage(target); + + for (int k = 0; k < nStranded; k++) + ringAppend(entries[stranded[k]], stranded[k]); +} + +// Append one record. Caller holds spiLock. +void WarmNodeStore::ringAppend(const WarmNodeEntry &rec, int storeSlot) +{ + if (activePage == 0xFF || writeSlot >= kRecordsPerPage) + ringRotate(); + const uint32_t addr = + WARM_FLASH_PAGE_ADDR(activePage) + sizeof(WarmPageHeader) + static_cast(writeSlot) * sizeof(WarmNodeEntry); + flash_nrf5x_write(addr, &rec, sizeof(rec)); + writeSlot++; + if (storeSlot >= 0) + pageOf[storeSlot] = activePage; + dirty = true; +} + +void WarmNodeStore::persistEntry(const WarmNodeEntry &e) +{ + concurrency::LockGuard g(spiLock); + ringAppend(e, static_cast(&e - entries)); +} + +void WarmNodeStore::persistRemove(NodeNum num, int storeSlot) +{ + if (storeSlot >= 0 && storeSlot < static_cast(WARM_NODE_COUNT)) + pageOf[storeSlot] = 0xFF; + WarmNodeEntry tomb; + memset(&tomb, 0, sizeof(tomb)); + tomb.num = num; + tomb.last_heard = WARM_RING_TOMBSTONE; + concurrency::LockGuard g(spiLock); + ringAppend(tomb, -1); +} + +void WarmNodeStore::persistClear() +{ + concurrency::LockGuard g(spiLock); + flash_nrf5x_flush(); + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) + flash_nrf5x_erase(WARM_FLASH_PAGE_ADDR(p)); + activePage = 0xFF; + writeSlot = 0; + nextSeq = 1; + dirty = false; // the erased ring already reflects the empty store +} + +void WarmNodeStore::load() +{ + if (!entries) + return; + concurrency::LockGuard g(spiLock); + + // Order valid pages by ascending seq so replay applies oldest first + uint8_t order[WARM_FLASH_PAGES]; + uint32_t seqs[WARM_FLASH_PAGES]; + uint8_t nValid = 0; + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { + WarmPageHeader h; + if (!ringReadHeader(p, h)) + continue; + uint8_t pos = nValid; + while (pos > 0 && static_cast(h.seq - seqs[pos - 1]) < 0) { + order[pos] = order[pos - 1]; + seqs[pos] = seqs[pos - 1]; + pos--; + } + order[pos] = p; + seqs[pos] = h.seq; + nValid++; + } + + if (nValid == 0) { + activePage = 0xFF; + writeSlot = 0; + nextSeq = 1; + LOG_INFO("WarmStore: ring empty, starting fresh"); + return; + } + + uint32_t replayed = 0; + for (uint8_t k = 0; k < nValid; k++) { + const uint8_t p = order[k]; + uint16_t slot = 0; + for (; slot < kRecordsPerPage; slot++) { + WarmNodeEntry rec; + flash_nrf5x_read(&rec, WARM_FLASH_PAGE_ADDR(p) + sizeof(WarmPageHeader) + (uint32_t)slot * sizeof(rec), sizeof(rec)); + if (rec.num == 0xFFFFFFFFu) + break; // erased space: end of this page's records (append-only) + if (rec.num == 0) + continue; // unexpected; skip defensively + replayed++; + if (rec.last_heard == WARM_RING_TOMBSTONE) { + WarmNodeEntry *e = find(rec.num); + if (e) { + pageOf[e - entries] = 0xFF; + memset(e, 0, sizeof(*e)); + } + } else { + WarmNodeEntry *e = place(rec.num, rec.last_heard, rec.public_key); + if (e) + pageOf[e - entries] = p; + } + } + if (k == nValid - 1) { // newest page becomes the active head + activePage = p; + writeSlot = slot; + nextSeq = seqs[k] + 1; + } + } + LOG_INFO("WarmStore: replayed %u ring records -> %u live nodes (page %u, slot %u)", (unsigned)replayed, (unsigned)count(), + activePage, writeSlot); +} + +bool WarmNodeStore::save() +{ + if (!powerHAL_isPowerLevelSafe()) { + LOG_ERROR("Error: trying to save WarmStore on unsafe device power level."); + return false; + } + concurrency::LockGuard g(spiLock); + flash_nrf5x_flush(); + return true; +} + +#else // !(defined(ARCH_NRF52) && defined(NRF52840_XXAA)) -------------------- + +void WarmNodeStore::persistEntry(const WarmNodeEntry &e) +{ + (void)e; + dirty = true; +} + +void WarmNodeStore::persistRemove(NodeNum num, int storeSlot) +{ + (void)num; + (void)storeSlot; + dirty = true; +} + +void WarmNodeStore::persistClear() +{ + dirty = true; +} + +#ifdef FSCom + +// ---- File persistence: /prefs/warm.dat snapshots ---------------------------- + +// Compact occupied slots to the front of `dst`; returns the count. +static uint16_t packEntries(const WarmNodeEntry *src, WarmNodeEntry *dst) +{ + uint16_t n = 0; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (src[i].num) + dst[n++] = src[i]; + return n; +} + +void WarmNodeStore::load() +{ + if (!entries) + return; + concurrency::LockGuard g(spiLock); + auto f = FSCom.open(warmFileName, FILE_O_READ); + if (!f) + return; + WarmStoreHeader h; + bool ok = (size_t)f.read((uint8_t *)&h, sizeof(h)) == sizeof(h) && h.magic == WARM_STORE_MAGIC && + h.entrySize == sizeof(WarmNodeEntry) && h.count <= WARM_NODE_COUNT; + if (ok && h.count) { + const size_t len = (size_t)h.count * sizeof(WarmNodeEntry); + ok = (size_t)f.read((uint8_t *)entries, len) == len && crc32Buffer(entries, len) == h.crc; + if (!ok) + memset(entries, 0, WARM_NODE_COUNT * sizeof(WarmNodeEntry)); + } + f.close(); + if (ok) + LOG_INFO("WarmStore: loaded %u warm nodes from %s", h.count, warmFileName); + else + LOG_WARN("WarmStore: %s invalid, starting empty", warmFileName); +} + +bool WarmNodeStore::save() +{ + if (!entries) + return false; + if (!powerHAL_isPowerLevelSafe()) { + LOG_ERROR("Error: trying to save WarmStore on unsafe device power level."); + return false; + } + + std::vector packed(WARM_NODE_COUNT); + WarmStoreHeader h; + h.magic = WARM_STORE_MAGIC; + h.reserved = 0; + h.count = packEntries(entries, packed.data()); + h.entrySize = sizeof(WarmNodeEntry); + h.crc = crc32Buffer(packed.data(), h.count * sizeof(WarmNodeEntry)); + + spiLock->lock(); + FSCom.mkdir("/prefs"); + spiLock->unlock(); + + auto f = SafeFile(warmFileName, false); + f.write((const uint8_t *)&h, sizeof(h)); + f.write((const uint8_t *)packed.data(), h.count * sizeof(WarmNodeEntry)); + bool ok = f.close(); + if (!ok) + LOG_ERROR("WarmStore: can't write %s", warmFileName); + else + LOG_DEBUG("WarmStore: saved %u warm nodes to %s", h.count, warmFileName); + return ok; +} + +#else + +void WarmNodeStore::load() {} +bool WarmNodeStore::save() +{ + return true; +} + +#endif // FSCom +#endif // defined(ARCH_NRF52) && defined(NRF52840_XXAA) + +#endif // WARM_NODE_COUNT > 0 diff --git a/src/mesh/WarmNodeStore.h b/src/mesh/WarmNodeStore.h new file mode 100644 index 00000000000..6ea9286d874 --- /dev/null +++ b/src/mesh/WarmNodeStore.h @@ -0,0 +1,122 @@ +#pragma once + +#include "MeshTypes.h" +#include "mesh-pb-constants.h" +#include +#include + +#if WARM_NODE_COUNT > 0 + +/** + * Warm ("long-tail") node tier + * + * Holds a minimal identity record for nodes evicted from the hot NodeInfoLite + * store: NodeNum, last_heard and the Curve25519 public key. The tier exists + * primarily so DMs to/from an evicted node keep encrypting/decrypting — the + * public key is expensive to re-learn (requires a NodeInfo exchange) while + * everything else in NodeInfoLite is rebuilt from traffic within seconds. + * + * Flat fixed-capacity array, linear scan (lookups happen only on hot-store + * misses), LRU eviction by last_heard with key-bearing entries outranking + * keyless ones. + * + * Persistence: + * - nRF52840: a dedicated 12 KB raw-flash record-ring (3 × 4 KB pages at + * 0xEA000, directly below the stock LittleFS partition). Mutations append + * 40 B upsert/tombstone records via the same SoftDevice-safe flash HAL + * InternalFS uses; boot replays the pages in sequence order. When the + * active page fills, the oldest page is reclaimed: live entries stranded + * in it are re-appended first (compaction), then it is erased and becomes + * the new head. The app image must end below the region — + * nrf52840_s140_v7.ld enforces this at link time and + * extra_scripts/nrf52_warm_region.py guards boards on the framework + * default linker script. + * - everywhere else: /prefs/warm.dat snapshot on the node-DB save cadence. + */ +struct WarmNodeEntry { + NodeNum num; // 0 = empty slot + uint32_t last_heard; // recency for LRU ordering + uint8_t public_key[32]; // all-zero = no key (a real key is never all-zero) +}; +static_assert(sizeof(WarmNodeEntry) == 40, "WarmNodeEntry must stay 40 B — persistence format depends on it"); + +// NRF52840_XXAA is explicit: nRF52832 also defines ARCH_NRF52 but only has +// 512 KB flash (top at 0x80000), so 0xEA000 would be past its flash end. +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) +#define WARM_FLASH_PAGE_SIZE 4096u +#define WARM_FLASH_PAGES 3u +#define WARM_FLASH_REGION_BASE (0xED000u - WARM_FLASH_PAGES * WARM_FLASH_PAGE_SIZE) // 0xEA000 +#define WARM_FLASH_PAGE_ADDR(i) (WARM_FLASH_REGION_BASE + (i)*WARM_FLASH_PAGE_SIZE) +#endif + +class WarmNodeStore +{ + public: + WarmNodeStore(); + ~WarmNodeStore(); + + /// Remember an evicted hot node. Keyless candidates never displace keyed + /// entries; otherwise the oldest (keyless-first) entry is replaced. + /// @return true if the node was stored or updated + bool absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32 /* may be NULL */); + + /// Find and remove an entry (used when the node is re-admitted to the hot store). + bool take(NodeNum num, WarmNodeEntry &out); + + /// Copy the 32-byte public key for a node, if we have one. + bool copyKey(NodeNum num, uint8_t out[32]) const; + + bool contains(NodeNum num) const; + void remove(NodeNum num); + void clear(); + size_t count() const; + size_t capacity() const { return entries ? WARM_NODE_COUNT : 0; } + + /// Load persisted entries (called once at boot, after the node DB loads). + void load(); + /// Durability point, piggybacked on the node-database save cadence. On the + /// ring backend this flushes the shared flash page cache; on the file + /// backend it writes the warm.dat snapshot. + bool saveIfDirty(); + + private: + WarmNodeEntry *entries = nullptr; // WARM_NODE_COUNT slots; PSRAM on ESP32 when available + bool entriesFromPsram = false; + bool dirty = false; + + WarmNodeEntry *find(NodeNum num) const; + // Internal slot-placement shared by absorb() and ring replay: applies the + // keyed-first admission policy without touching persistence. + WarmNodeEntry *place(NodeNum num, uint32_t lastHeard, const uint8_t *key32); + + // Persistence hooks called from the mutation paths. File backend: mark + // dirty. Ring backend: append an upsert/tombstone record (+ mark dirty). + void persistEntry(const WarmNodeEntry &e); // e must point into entries[] + void persistRemove(NodeNum num, int storeSlot); + void persistClear(); + +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + // ---- raw-flash record-ring state ---- + struct WarmPageHeader { + uint32_t magic; // WARM_RING_MAGIC + uint32_t seq; // page generation; 0xFFFFFFFF = erased/unused + }; + static_assert(sizeof(WarmPageHeader) == 8, "page header is part of the flash format"); + static constexpr uint16_t kRecordsPerPage = (WARM_FLASH_PAGE_SIZE - sizeof(WarmPageHeader)) / sizeof(WarmNodeEntry); // 102 + static_assert(WARM_NODE_COUNT <= 2 * ((WARM_FLASH_PAGE_SIZE - 8) / 40), "live set must fit the ring with one page reclaimed"); + + uint8_t activePage = 0xFF; // 0xFF = no page opened yet (fresh/erased ring) + uint16_t writeSlot = 0; // next free record slot in the active page + uint32_t nextSeq = 1; // seq for the next page opened + uint8_t pageOf[WARM_NODE_COUNT]; // flash page holding each RAM slot's newest record; 0xFF = none + + void ringAppend(const WarmNodeEntry &rec, int storeSlot /* -1 for tombstones */); + void ringRotate(); // reclaim oldest page, compacting stranded live entries + void ringOpenPage(uint8_t page); // erase + write header (seq = nextSeq++) + bool ringReadHeader(uint8_t page, WarmPageHeader &h) const; +#endif + + bool save(); +}; + +#endif // WARM_NODE_COUNT > 0 diff --git a/src/mesh/generated/meshtastic/deviceonly.pb.h b/src/mesh/generated/meshtastic/deviceonly.pb.h index 474b332a8da..ed4b0792db1 100644 --- a/src/mesh/generated/meshtastic/deviceonly.pb.h +++ b/src/mesh/generated/meshtastic/deviceonly.pb.h @@ -452,7 +452,7 @@ extern const pb_msgdesc_t meshtastic_BackupPreferences_msg; /* Maximum encoded size of messages (where known) */ /* meshtastic_NodeDatabase_size depends on runtime parameters */ #define MESHTASTIC_MESHTASTIC_DEVICEONLY_PB_H_MAX_SIZE meshtastic_BackupPreferences_size -#define meshtastic_BackupPreferences_size 2432 +#define meshtastic_BackupPreferences_size 2468 #define meshtastic_ChannelFile_size 718 #define meshtastic_DeviceState_size 1944 #define meshtastic_NodeEnvironmentEntry_size 170 diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index b4d99901110..99f5dcbb86e 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -48,7 +48,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_POSITIONDB 1 #else #define MESHTASTIC_EXCLUDE_POSITIONDB 0 -#endif +#endif // STM32WL #endif #ifndef MESHTASTIC_EXCLUDE_TELEMETRYDB @@ -56,7 +56,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_TELEMETRYDB 1 #else #define MESHTASTIC_EXCLUDE_TELEMETRYDB 0 -#endif +#endif // STM32WL #endif #ifndef MESHTASTIC_EXCLUDE_ENVIRONMENTDB @@ -64,7 +64,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_ENVIRONMENTDB 1 #else #define MESHTASTIC_EXCLUDE_ENVIRONMENTDB 0 -#endif +#endif // STM32WL #endif #ifndef MESHTASTIC_EXCLUDE_STATUSDB @@ -72,31 +72,62 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_STATUSDB 1 #else #define MESHTASTIC_EXCLUDE_STATUSDB 0 -#endif +#endif // STM32WL #endif -/// max number of nodes allowed in the nodeDB +/// max number of nodes allowed in the nodeDB (the full-NodeInfoLite "hot + +/// short-tail" store). +/// Long-tail identity retention for evicted nodes is handled by the warm tier +/// (WARM_NODE_COUNT). 120 keeps nodes.proto (~14 KB typical / ~27 KB worst) +/// comfortably inside the stock 28 KB nRF52 LittleFS alongside prefs and the +/// message store — the warm tier persists outside LittleFS on nRF52840. #ifndef MAX_NUM_NODES #if defined(ARCH_STM32WL) #define MAX_NUM_NODES 10 +#else +#define MAX_NUM_NODES 120 +#endif // STM32WL +#endif + +/// Cap on each satellite map (position/telemetry/environment/status). Only the +/// MAX_SATELLITE_NODES most-recently-heard nodes keep satellite payloads; the +/// rest of the hot store carries just the 96 B NodeInfoLite header. This is +/// what bounds both heap (the maps are ~408 B/node worst case) and the +/// satellite share of nodes.proto. Deliberately NOT scaled with MAX_NUM_NODES: +/// the tier-1 "rich data" population stays at 40 regardless of how long the +/// short tail grows. +#ifndef MAX_SATELLITE_NODES +#define MAX_SATELLITE_NODES 40 +#endif + +/// Warm tier: number of 40 B {num, last_heard, public_key} records retained +/// for nodes evicted from the hot store, primarily so DMs to/from them keep +/// decrypting. 0 disables the tier entirely. Persisted to /prefs/warm.dat, +/// so the count is bounded by the platform's filesystem budget. +#ifndef WARM_NODE_COUNT +#if defined(ARCH_STM32WL) +#define WARM_NODE_COUNT 0 +#elif defined(NRF52840_XXAA) +// nRF52840: persisted in the dedicated 12 KB raw-flash record-ring below +// LittleFS (3 × 4 KB pages, append + replay + compact-on-rotate; see +// WarmNodeStore.h). 200 live entries across 306 record slots. +// +// Keyed on the NRF52840_XXAA build flag (a command-line -D from the board +// definition, defined before any header is processed) rather than the +// header-defined ARCH_NRF52. ARCH_NRF52 comes from platform/nrf52/architecture.h +// via configuration.h, which some include chains (MeshTypes.h -> WarmNodeStore.h +// -> here) reach AFTER mesh-pb-constants.h — so keying on it would let +// WARM_NODE_COUNT resolve to the generic fallback and then fail the ring-sizing +// static_assert in WarmNodeStore.h (which is itself gated on NRF52840_XXAA). +#define WARM_NODE_COUNT 200 #elif defined(ARCH_NRF52) -#define MAX_NUM_NODES 150 +// Other nRF52 (e.g. nRF52832): no raw-flash warm ring is reserved for these, so +// keep the warm tier off rather than risk a file-backed default on tight flash. +#define WARM_NODE_COUNT 0 #elif defined(CONFIG_IDF_TARGET_ESP32S3) -#include "Esp.h" -static inline int get_max_num_nodes() -{ - uint32_t flash_size = ESP.getFlashChipSize() / (1024 * 1024); // Convert Bytes to MB - if (flash_size >= 15) { - return 250; - } else if (flash_size >= 7) { - return 200; - } else { - return 100; - } -} -#define MAX_NUM_NODES get_max_num_nodes() +#define WARM_NODE_COUNT 2000 // PSRAM-backed when available; warm.dat ~80 KB #else -#define MAX_NUM_NODES 100 +#define WARM_NODE_COUNT 320 #endif #endif diff --git a/src/platform/nrf52/nrf52840_s140_v6.ld b/src/platform/nrf52/nrf52840_s140_v6.ld new file mode 100644 index 00000000000..c8dac4e5545 --- /dev/null +++ b/src/platform/nrf52/nrf52840_s140_v6.ld @@ -0,0 +1,46 @@ +/* Linker script to configure memory regions. */ + +SEARCH_DIR(.) +GROUP(-lgcc -lc -lnosys) + +MEMORY +{ + /* App region ends at 0xEA000, not 0xED000: the 12 KB warm-node-store + * record-ring (3 x 4 KB pages, src/mesh/WarmNodeStore.h) occupies + * 0xEA000-0xED000, directly below the stock LittleFS partition. Boards on + * the framework-default linker script are covered by the post-link guard + * in extra_scripts/nrf52_warm_region.py instead. + * + * S140 v6.x app region starts at 0x26000 (152 KB SoftDevice); v7.x uses + * 0x27000. All other boundaries are identical — see nrf52840_s140_v7.ld. */ + FLASH (rx) : ORIGIN = 0x26000, LENGTH = 0xEA000 - 0x26000 + + /* SRAM required by Softdevice depend on + * - Attribute Table Size (Number of Services and Characteristics) + * - Vendor UUID count + * - Max ATT MTU + * - Concurrent connection peripheral + central + secure links + * - Event Len, HVN queue, Write CMD queue + */ + RAM (rwx) : ORIGIN = 0x20006000, LENGTH = 0x20040000 - 0x20006000 +} + +SECTIONS +{ + . = ALIGN(4); + .svc_data : + { + PROVIDE(__start_svc_data = .); + KEEP(*(.svc_data)) + PROVIDE(__stop_svc_data = .); + } > RAM + + .fs_data : + { + PROVIDE(__start_fs_data = .); + KEEP(*(.fs_data)) + PROVIDE(__stop_fs_data = .); + } > RAM +} INSERT AFTER .data; + +INCLUDE "nrf52_common.ld" diff --git a/src/platform/nrf52/nrf52840_s140_v7.ld b/src/platform/nrf52/nrf52840_s140_v7.ld index 6aaeb4034fe..4546b4a7a3f 100644 --- a/src/platform/nrf52/nrf52840_s140_v7.ld +++ b/src/platform/nrf52/nrf52840_s140_v7.ld @@ -5,7 +5,12 @@ GROUP(-lgcc -lc -lnosys) MEMORY { - FLASH (rx) : ORIGIN = 0x27000, LENGTH = 0xED000 - 0x27000 + /* App region ends at 0xEA000, not 0xED000: the 12 KB warm-node-store + * record-ring (3 x 4 KB pages, src/mesh/WarmNodeStore.h) occupies + * 0xEA000-0xED000, directly below the stock LittleFS partition. Boards on + * the framework-default linker script are covered by the post-link guard + * in extra_scripts/nrf52_warm_region.py instead. */ + FLASH (rx) : ORIGIN = 0x27000, LENGTH = 0xEA000 - 0x27000 /* SRAM required by Softdevice depend on * - Attribute Table Size (Number of Services and Characteristics) diff --git a/test/test_warm_store/test_main.cpp b/test/test_warm_store/test_main.cpp new file mode 100644 index 00000000000..3c845588363 --- /dev/null +++ b/test/test_warm_store/test_main.cpp @@ -0,0 +1,211 @@ +// Unit tests for the warm ("long-tail") node tier — src/mesh/WarmNodeStore.cpp. +// Covers admission/eviction policy (keyed entries outrank keyless), take() +// rehydration semantics, and a tolerant persistence round trip. +#include "MeshTypes.h" // BEFORE TestUtil.h — provides WARM_NODE_COUNT via mesh-pb-constants.h +#include "TestUtil.h" +#include + +#if defined(ARCH_PORTDUINO) +#define WS_TEST_ENTRY extern "C" +#else +#define WS_TEST_ENTRY +#endif + +#if WARM_NODE_COUNT > 0 + +#include "mesh/WarmNodeStore.h" +#include + +namespace +{ + +void makeKey(uint8_t out[32], uint8_t seed) +{ + memset(out, 0, 32); + out[0] = seed; + out[31] = seed ^ 0xA5; +} + +} // namespace + +void setUp(void) {} +void tearDown(void) {} + +void test_ws_absorb_and_copyKey_roundTrip() +{ + WarmNodeStore ws; + uint8_t key[32], got[32]; + makeKey(key, 7); + TEST_ASSERT_TRUE(ws.absorb(0x100, 1000, key)); + TEST_ASSERT_TRUE(ws.contains(0x100)); + TEST_ASSERT_TRUE(ws.copyKey(0x100, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); + TEST_ASSERT_EQUAL(1, ws.count()); +} + +void test_ws_keylessEntry_hasNoKey() +{ + WarmNodeStore ws; + uint8_t got[32]; + TEST_ASSERT_TRUE(ws.absorb(0x200, 1000, NULL)); + TEST_ASSERT_TRUE(ws.contains(0x200)); + TEST_ASSERT_FALSE(ws.copyKey(0x200, got)); +} + +void test_ws_absorb_rejectsNodeNumZero() +{ + WarmNodeStore ws; + TEST_ASSERT_FALSE(ws.absorb(0, 1000, NULL)); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_absorb_updatesExistingEntry() +{ + WarmNodeStore ws; + uint8_t key[32], got[32]; + makeKey(key, 9); + TEST_ASSERT_TRUE(ws.absorb(0x300, 1000, NULL)); + TEST_ASSERT_TRUE(ws.absorb(0x300, 2000, key)); // later eviction learned a key + TEST_ASSERT_EQUAL(1, ws.count()); + TEST_ASSERT_TRUE(ws.copyKey(0x300, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); +} + +void test_ws_take_removesEntry() +{ + WarmNodeStore ws; + uint8_t key[32]; + makeKey(key, 3); + ws.absorb(0x400, 1234, key); + + WarmNodeEntry e; + TEST_ASSERT_TRUE(ws.take(0x400, e)); + TEST_ASSERT_EQUAL(0x400, e.num); + TEST_ASSERT_EQUAL(1234, e.last_heard); + TEST_ASSERT_EQUAL_MEMORY(key, e.public_key, 32); + TEST_ASSERT_FALSE(ws.contains(0x400)); + TEST_ASSERT_FALSE(ws.take(0x400, e)); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_keylessCandidate_neverEvictsKeyedEntries() +{ + WarmNodeStore ws; + uint8_t key[32]; + // Fill the store entirely with keyed entries + for (size_t i = 0; i < ws.capacity(); i++) { + makeKey(key, (uint8_t)i); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, 100 + i, key)); + } + TEST_ASSERT_EQUAL(ws.capacity(), ws.count()); + // A keyless candidate (even a fresh one) must be rejected + TEST_ASSERT_FALSE(ws.absorb(0x9999, 999999, NULL)); + TEST_ASSERT_FALSE(ws.contains(0x9999)); +} + +void test_ws_keyedCandidate_evictsOldestKeylessFirst() +{ + WarmNodeStore ws; + uint8_t key[32]; + makeKey(key, 0x42); + // Fill with keyed entries except two keyless ones in the middle + for (size_t i = 0; i < ws.capacity(); i++) { + const bool keyless = (i == 5 || i == 10); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, keyless ? (i == 10 ? 50 : 60) : 10, keyless ? NULL : key)); + } + // Keyed candidate must displace the OLDEST KEYLESS entry (0x100A, ts=50), + // even though every keyed entry is older (ts=10) + uint8_t k2[32]; + makeKey(k2, 0x43); + TEST_ASSERT_TRUE(ws.absorb(0x8888, 70, k2)); + TEST_ASSERT_FALSE(ws.contains(0x1000 + 10)); + TEST_ASSERT_TRUE(ws.contains(0x1000 + 5)); + TEST_ASSERT_TRUE(ws.contains(0x8888)); +} + +void test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless() +{ + WarmNodeStore ws; + uint8_t key[32]; + for (size_t i = 0; i < ws.capacity(); i++) { + makeKey(key, (uint8_t)i); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, 1000 + i, key)); // 0x1000 is the oldest + } + uint8_t k2[32]; + makeKey(k2, 0x44); + TEST_ASSERT_TRUE(ws.absorb(0x7777, 999999, k2)); + TEST_ASSERT_TRUE(ws.contains(0x7777)); + TEST_ASSERT_FALSE(ws.contains(0x1000)); // oldest keyed evicted + TEST_ASSERT_EQUAL(ws.capacity(), ws.count()); +} + +void test_ws_remove_and_clear() +{ + WarmNodeStore ws; + ws.absorb(0x500, 1, NULL); + ws.absorb(0x501, 2, NULL); + ws.remove(0x500); + TEST_ASSERT_FALSE(ws.contains(0x500)); + TEST_ASSERT_EQUAL(1, ws.count()); + ws.clear(); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_persistence_roundTrip() +{ + WarmNodeStore a; + uint8_t key[32], got[32]; + makeKey(key, 0x55); + a.absorb(0x600, 4242, key); + a.absorb(0x601, 4243, NULL); + if (!a.saveIfDirty()) { + TEST_IGNORE_MESSAGE("Filesystem not available in this test environment"); + return; + } + + WarmNodeStore b; + b.load(); + TEST_ASSERT_TRUE(b.contains(0x600)); + TEST_ASSERT_TRUE(b.contains(0x601)); + TEST_ASSERT_TRUE(b.copyKey(0x600, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); + + // Cleanup so reruns start fresh + b.clear(); + b.saveIfDirty(); +} + +WS_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + RUN_TEST(test_ws_absorb_and_copyKey_roundTrip); + RUN_TEST(test_ws_keylessEntry_hasNoKey); + RUN_TEST(test_ws_absorb_rejectsNodeNumZero); + RUN_TEST(test_ws_absorb_updatesExistingEntry); + RUN_TEST(test_ws_take_removesEntry); + RUN_TEST(test_ws_keylessCandidate_neverEvictsKeyedEntries); + RUN_TEST(test_ws_keyedCandidate_evictsOldestKeylessFirst); + RUN_TEST(test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless); + RUN_TEST(test_ws_remove_and_clear); + RUN_TEST(test_ws_persistence_roundTrip); + exit(UNITY_END()); +} + +WS_TEST_ENTRY void loop() {} + +#else + +void setUp(void) {} +void tearDown(void) {} + +WS_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + exit(UNITY_END()); +} + +WS_TEST_ENTRY void loop() {} + +#endif diff --git a/variants/nrf52840/nrf52.ini b/variants/nrf52840/nrf52.ini index f99c78d8a3e..999ea6f9f1d 100644 --- a/variants/nrf52840/nrf52.ini +++ b/variants/nrf52840/nrf52.ini @@ -15,6 +15,7 @@ extra_scripts = ${env.extra_scripts} extra_scripts/nrf52_extra.py pre:extra_scripts/nrf52_lto.py + extra_scripts/nrf52_warm_region.py ; post-link guard: image must end below the 16 KB warm-store raw-flash region at 0xE9000 build_type = release build_flags = diff --git a/variants/nrf52840/nrf52840.ini b/variants/nrf52840/nrf52840.ini index c5590cbc3f6..610b5b53030 100644 --- a/variants/nrf52840/nrf52840.ini +++ b/variants/nrf52840/nrf52840.ini @@ -1,6 +1,10 @@ [nrf52840_base] extends = nrf52_base +; Cap the app image at 0xEA000 (below the WarmNodeStore raw-flash region). +; Boards that have upgraded to S140 v7 override this in their own platformio.ini. +board_build.ldscript = src/platform/nrf52/nrf52840_s140_v6.ld + build_flags = ${nrf52_base.build_flags} -DSERIAL_BUFFER_SIZE=4096 From d0f19c171f292f2b7e45ad643fcc62c8d1fd19ad Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sat, 13 Jun 2026 16:25:49 +0100 Subject: [PATCH 02/13] NodeDB: robust receive + retention for blocked (ignored) nodes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hardens how ignored/favourite nodes are received over admin and retained, closing paths where a block could be lost or accidentally cleared. - Blocking keeps the node's public key (admin set_ignored_node and addFromContact no longer zero it / drop the warm-tier key), so a blocked peer stays a verifiable identity. - set_ignored_node creates the node if absent, so a block by node ID sticks even for a node we've never heard from (e.g. pushed by a remote admin) with no NodeInfo or key. - Eviction protection (favourite/ignored/manually-verified) now also applies to the load-time hot-store migration and is never undone by cleanupMeshDB, which previously purged ignored nodes that lacked user info. - The hot-store migration leaves our own node (index 0) in place and prefers to demote non-protected nodes, like the runtime eviction scan. Caps the protected set (favourite + ignored + verified) at MAX_NUM_NODES-2 via NodeDB::setProtectedFlag(), so at least two evictable slots always remain and getOrCreateMeshNode can always make room — replacing the previous unconditional append that could run off the end of the node vector when every node was protected. A locally-set favourite/ignore that hits the cap reports back to the phone via a ClientNotification. Adds test_nodedb_blocked covering the migration, favourite/ignored eviction protection, ignored-survives-cleanup, and the protected-node cap. The maintenance methods stay private in production; the test reaches them through a PIO_UNIT_TESTING-guarded friend shim. Co-Authored-By: Claude Opus 4.8 (1M context) # Conflicts: # src/mesh/NodeDB.h --- src/graphics/draw/MenuHandler.cpp | 5 +- src/mesh/NodeDB.cpp | 88 +++++++---- src/mesh/NodeDB.h | 36 ++++- src/modules/AdminModule.cpp | 29 ++-- test/test_nodedb_blocked/test_main.cpp | 193 +++++++++++++++++++++++++ 5 files changed, 307 insertions(+), 44 deletions(-) create mode 100644 test/test_nodedb_blocked/test_main.cpp diff --git a/src/graphics/draw/MenuHandler.cpp b/src/graphics/draw/MenuHandler.cpp index 365739112d5..02fbba7a38d 100644 --- a/src/graphics/draw/MenuHandler.cpp +++ b/src/graphics/draw/MenuHandler.cpp @@ -1618,9 +1618,10 @@ void menuHandler::manageNodeMenu() if (nodeInfoLiteIsIgnored(n)) { nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, false); LOG_INFO("Unignoring node %08X", menuHandler::pickedNodeNum); - } else { - nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + } else if (nodeDB->setProtectedFlag(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) { LOG_INFO("Ignoring node %08X", menuHandler::pickedNodeNum); + } else { + LOG_WARN("Can't ignore %08X: protected-node limit reached", menuHandler::pickedNodeNum); } nodeDB->notifyObservers(true); nodeDB->saveToDisk(); diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 11604f33541..d2c36569d7d 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -1551,7 +1551,10 @@ void NodeDB::cleanupMeshDB() int newPos = 0, removed = 0; for (int i = 0; i < numMeshNodes; i++) { meshtastic_NodeInfoLite &n = meshNodes->at(i); - if (nodeInfoLiteHasUser(&n)) { + // Keep ignored (blocked) nodes even without user info: a block set by + // bare node ID has no NodeInfo and would otherwise be purged here, + // silently dropping the block. + if (nodeInfoLiteHasUser(&n) || nodeInfoLiteIsIgnored(&n)) { if (n.public_key.size > 0) { if (memfll(n.public_key.bytes, 0, n.public_key.size)) { n.public_key.size = 0; @@ -1725,23 +1728,26 @@ LoadFileResult NodeDB::loadProto(const char *filename, size_t protoSize, size_t #if WARM_NODE_COUNT > 0 void NodeDB::demoteOldestHotNodesToWarm() { - if (numMeshNodes <= MAX_NUM_NODES) + const int keep = MAX_NUM_NODES; + if (numMeshNodes <= keep) return; - // Rank: favorites ahead of recency (a user-marked node is never demoted in - // favour of a more-recently-heard stranger), then most-recently-heard first. - // After the sort, [0, MAX_NUM_NODES) are the keepers and the tail is overflow. - std::sort(meshNodes->begin(), meshNodes->begin() + numMeshNodes, + // Favorites, ignored (blocked) and manually-verified nodes rank ahead of + // recency — the same protection getOrCreateMeshNode applies on eviction — so + // they are demoted only when the store is full of them. Within a class, + // most-recently-heard first. Index 0 is our own node and is left in place + // (sort begins at +1), as in the runtime eviction scan. + std::sort(meshNodes->begin() + 1, meshNodes->begin() + numMeshNodes, [](const meshtastic_NodeInfoLite &a, const meshtastic_NodeInfoLite &b) { - const bool fa = nodeInfoLiteIsFavorite(&a); - const bool fb = nodeInfoLiteIsFavorite(&b); - if (fa != fb) - return fa; + const bool ka = nodeInfoLiteIsProtected(&a); + const bool kb = nodeInfoLiteIsProtected(&b); + if (ka != kb) + return ka; return a.last_heard > b.last_heard; }); int demoted = 0; - for (int i = MAX_NUM_NODES; i < numMeshNodes; i++) { + for (int i = keep; i < numMeshNodes; i++) { const meshtastic_NodeInfoLite &n = (*meshNodes)[i]; if (n.num == 0) continue; @@ -1750,7 +1756,8 @@ void NodeDB::demoteOldestHotNodesToWarm() warmStore.absorb(n.num, n.last_heard, n.public_key.size > 0 ? n.public_key.bytes : nullptr); demoted++; } - LOG_INFO("NodeDB migration: demoted %d oldest node(s) above MAX_NUM_NODES %d into the warm tier", demoted, MAX_NUM_NODES); + numMeshNodes = keep; // the resize() in loadFromDisk reclaims the demoted tail + LOG_INFO("NodeDB migration: demoted %d node(s) over %d into the warm tier (keepers preferred)", demoted, keep); } #endif @@ -2796,16 +2803,12 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) info->num = contact.node_num; TypeConversions::CopyUserToNodeInfoLite(info, contact.user); if (contact.should_ignore) { - // If should_ignore is set, - // we need to clear the public key and other cruft, in addition to setting the node as ignored - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + // Block the contact and drop its rich satellite data, but keep the + // public key copied above — an ignored peer keeps a usable identity + // (a verifiable target) rather than a bare node number. + setProtectedFlag(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, false); eraseNodeSatellites(contact.node_num); -#if WARM_NODE_COUNT > 0 - warmStore.remove(contact.node_num); // ignored: drop the retained key too -#endif - info->public_key.size = 0; - memset(info->public_key.bytes, 0, sizeof(info->public_key.bytes)); } else { /* Clients are sending add_contact before every text message DM (because clients may hold a larger node database with * public keys than the radio holds). However, we don't want to update last_heard just because we sent someone a DM! @@ -2824,12 +2827,12 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) } else { // Normal case: set is_favorite to prevent expiration. // last_heard will remain as-is (or remain 0 if this entry wasn't in the nodeDB). - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + setProtectedFlag(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); } // As the clients will begin sending the contact with DMs, we want to strictly check if the node is manually verified if (contact.manually_verified) { - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true); + setProtectedFlag(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true); } // Mark the node's key as manually verified to indicate trustworthiness. updateGUIforNode = info; @@ -2962,13 +2965,42 @@ void NodeDB::updateFrom(const meshtastic_MeshPacket &mp) } } +int NodeDB::numProtectedNodes() const +{ + int count = 0; + for (int i = 0; i < numMeshNodes; i++) + if (nodeInfoLiteIsProtected(&meshNodes->at(i))) + count++; + return count; +} + +bool NodeDB::setProtectedFlag(meshtastic_NodeInfoLite *node, uint32_t mask, bool on) +{ + if (!node) + return false; + if (!on) { + nodeInfoLiteSetBit(node, mask, false); + return true; + } + // Adding a flag to a node that is already protected doesn't grow the + // protected set, so it's always allowed. A newly-protected node is refused + // once the protected set has reached MAX_NUM_NODES-2, leaving two evictable + // slots so getOrCreateMeshNode can always make room. + if (nodeInfoLiteIsProtected(node) || numProtectedNodes() < MAX_NUM_NODES - 2) { + nodeInfoLiteSetBit(node, mask, true); + return true; + } + return false; +} + void NodeDB::set_favorite(bool is_favorite, uint32_t nodeId) { meshtastic_NodeInfoLite *lite = getMeshNode(nodeId); if (lite && nodeInfoLiteIsFavorite(lite) != is_favorite) { - nodeInfoLiteSetBit(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite); - sortMeshDB(); - saveNodeDatabaseToDisk(); + if (setProtectedFlag(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite)) { + sortMeshDB(); + saveNodeDatabaseToDisk(); + } } } @@ -3174,6 +3206,12 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) (numMeshNodes)--; } } + // Don't append past the end of the vector. The protected-node cap + // (numProtectedNodes() <= MAX_NUM_NODES-2) means the eviction above frees + // a slot in normal operation; this guards the legacy case of a pre-cap + // database that is full of protected nodes — refuse rather than overrun. + if (numMeshNodes >= MAX_NUM_NODES) + return NULL; // add the node at the end lite = &meshNodes->at((numMeshNodes)++); diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index 3c35e379f89..2cbf3e42939 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -231,6 +231,17 @@ class NodeDB */ void set_favorite(bool is_favorite, uint32_t nodeId); + /// Count of eviction-protected (favourite/ignored/manually-verified) nodes. + int numProtectedNodes() const; + + /// Turn an eviction-protection flag (favourite/ignored/verified) on or off, + /// capped so the protected set never exceeds MAX_NUM_NODES-2 — keeping at + /// least two always-evictable slots. Turning a flag off always succeeds; + /// turning one on for a node that isn't already protected returns false + /// (and changes nothing) once the cap is reached. Callers surface that to + /// the user where appropriate. + bool setProtectedFlag(meshtastic_NodeInfoLite *node, uint32_t mask, bool on); + /* * Returns true if the node is in the NodeDB and marked as favorite */ @@ -296,6 +307,10 @@ class NodeDB virtual meshtastic_NodeInfoLite *getMeshNode(NodeNum n); size_t getNumMeshNodes() { return numMeshNodes; } + /// Find a node in our DB, create an empty NodeInfoLite if missing (evicting + /// the oldest non-protected node when full). Public so admin handlers can + /// register a node we have not heard from yet (e.g. to block it by ID). + meshtastic_NodeInfoLite *getOrCreateMeshNode(NodeNum n); #if WARM_NODE_COUNT > 0 // Warm ("long-tail") tier: minimal {num, last_heard, public_key} records @@ -454,8 +469,6 @@ class NodeDB uint32_t lastNodeDbSave = 0; // when we last saved our db to flash uint32_t lastBackupAttempt = 0; // when we last tried a backup automatically or manually uint32_t lastSort = 0; // When last sorted the nodeDB - /// Find a node in our DB, create an empty NodeInfoLite if missing - meshtastic_NodeInfoLite *getOrCreateMeshNode(NodeNum n); /* * Internal boolean to track sorting paused @@ -468,6 +481,12 @@ class NodeDB /// read our db from flash void loadFromDisk(); +#ifdef PIO_UNIT_TESTING + // Grant the unit-test shim access to the private maintenance paths below + // (migration / cleanup / eviction) without relaxing production access. + friend class NodeDBTestShim; +#endif + /// purge db entries without user info void cleanupMeshDB(); @@ -479,10 +498,9 @@ class NodeDB #if WARM_NODE_COUNT > 0 /// On load, a database from a larger-cap build (notably the pre-fork 150-node /// nRF52 hot store) can exceed MAX_NUM_NODES. Rank the hot store so the - /// freshest MAX_NUM_NODES survive and demote the oldest overflow into the - /// warm tier, preserving {num, last_heard, public_key} so PKI DMs keep - /// working instead of dropping those nodes on truncation. Caller truncates - /// the hot store to MAX_NUM_NODES afterwards. + /// freshest survive and demote the oldest overflow into the warm tier, + /// preserving {num, last_heard, public_key} so PKI DMs keep working instead + /// of dropping those nodes on truncation. void demoteOldestHotNodesToWarm(); #endif @@ -606,6 +624,12 @@ inline bool nodeInfoLiteHasXeddsaSigned(const meshtastic_NodeInfoLite *n) { return n && (n->bitfield & NODEINFO_BITFIELD_HAS_XEDDSA_SIGNED_MASK); } +/// A node that the eviction/migration paths must not drop: a favourite, an +/// ignored (blocked) node, or a manually-verified key. +inline bool nodeInfoLiteIsProtected(const meshtastic_NodeInfoLite *n) +{ + return nodeInfoLiteIsFavorite(n) || nodeInfoLiteIsIgnored(n) || nodeInfoLiteIsKeyManuallyVerified(n); +} inline void nodeInfoLiteSetBit(meshtastic_NodeInfoLite *n, uint32_t mask, bool value) { diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index 8473930e780..2eff7976a81 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -168,7 +168,7 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta LOG_INFO("PKC admin valid, but not auto-favoriting node %x because role==CLIENT_BASE", mp.from); } else { LOG_INFO("PKC admin valid. Auto-favoriting node %x", mp.from); - nodeInfoLiteSetBit(remoteNode, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + nodeDB->setProtectedFlag(remoteNode, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); } } } else { @@ -458,10 +458,13 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta LOG_INFO("Client received set_favorite_node command"); meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(r->set_favorite_node); if (node != NULL) { - nodeInfoLiteSetBit(node, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); - saveChanges(SEGMENT_NODEDATABASE, false); - if (screen) - screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens + if (nodeDB->setProtectedFlag(node, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true)) { + saveChanges(SEGMENT_NODEDATABASE, false); + if (screen) + screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens + } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take + sendWarning("Can't favorite 0x%08x: protected-node limit (%d) reached", r->set_favorite_node, MAX_NUM_NODES - 2); + } } break; } @@ -478,13 +481,17 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta } case meshtastic_AdminMessage_set_ignored_node_tag: { LOG_INFO("Client received set_ignored_node command"); - meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(r->set_ignored_node); + // Unlike the sibling node-targeted admin commands, create the entry if + // it's absent so the block sticks for a node we've not heard from yet + // (e.g. one a remote admin asks us to block) with no NodeInfo or key. + meshtastic_NodeInfoLite *node = nodeDB->getOrCreateMeshNode(r->set_ignored_node); if (node != NULL) { - nodeInfoLiteSetBit(node, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); - nodeDB->eraseNodeSatellites(node->num); - node->public_key.size = 0; - memset(node->public_key.bytes, 0, sizeof(node->public_key.bytes)); - saveChanges(SEGMENT_NODEDATABASE, false); + if (nodeDB->setProtectedFlag(node, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) { + nodeDB->eraseNodeSatellites(node->num); + saveChanges(SEGMENT_NODEDATABASE, false); + } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take + sendWarning("Can't ignore 0x%08x: protected-node limit (%d) reached", r->set_ignored_node, MAX_NUM_NODES - 2); + } } break; } diff --git a/test/test_nodedb_blocked/test_main.cpp b/test/test_nodedb_blocked/test_main.cpp new file mode 100644 index 00000000000..2bbe9fb6b36 --- /dev/null +++ b/test/test_nodedb_blocked/test_main.cpp @@ -0,0 +1,193 @@ +// Tests for the NodeDB hot-store migration and favourite/ignored (blocked) +// retention paths — src/mesh/NodeDB.cpp. +#include "MeshTypes.h" // BEFORE TestUtil.h — provides WARM_NODE_COUNT / MAX_NUM_NODES via mesh-pb-constants.h +#include "TestUtil.h" +#include + +#if defined(ARCH_PORTDUINO) +#define NDB_TEST_ENTRY extern "C" +#else +#define NDB_TEST_ENTRY +#endif + +// The migration demotes overflow into the warm tier, so these tests need it. +#if WARM_NODE_COUNT > 0 + +#include "mesh/NodeDB.h" +#include + +// Subclass shim: exposes the private maintenance paths (via the friend +// declaration in NodeDB.h) and lets a test own the hot store directly +// (meshNodes/numMeshNodes are public). Declared at global scope so it matches +// `friend class NodeDBTestShim` — an anonymous-namespace class would not. +class NodeDBTestShim : public NodeDB +{ + public: + void runDemote() { demoteOldestHotNodesToWarm(); } + void runCleanup() { cleanupMeshDB(); } + + void clearHot() + { + meshNodes->clear(); + numMeshNodes = 0; + } + + void push(NodeNum num, uint32_t lastHeard, bool favorite, bool ignored, bool withUser, bool withKey) + { + meshtastic_NodeInfoLite n = meshtastic_NodeInfoLite_init_zero; + n.num = num; + n.last_heard = lastHeard; + if (favorite) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + if (ignored) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + if (withUser) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_HAS_USER_MASK, true); + if (withKey) { + n.public_key.size = 32; + memset(n.public_key.bytes, static_cast(num & 0xff), 32); + n.public_key.bytes[0] = 0x01; // ensure non-zero (all-zero == "no key") + } + meshNodes->push_back(n); + numMeshNodes = meshNodes->size(); + } + + // Index 0 is our own node; the eviction/migration scans treat it as self. + void seedSelf() { push(0x0BADF00D, 0xFFFFFFFFu, false, false, /*withUser=*/true, /*withKey=*/false); } +}; + +namespace +{ + +NodeDBTestShim *db = nullptr; + +bool warmHasKey(NodeNum n) +{ + meshtastic_NodeInfoLite_public_key_t k = {0, {0}}; + return db->copyPublicKey(n, k) && k.size == 32; +} + +} // namespace + +void setUp(void) +{ + db->clearHot(); +} +void tearDown(void) {} + +// Migration: a database from a larger-cap build trims to MAX_NUM_NODES; the +// oldest non-protected nodes are demoted into the warm tier (keys preserved), +// while self, favourites and ignored survive even when they are the oldest. +static void test_migration_demotesOldestKeepsKeepersAndSelf(void) +{ + db->seedSelf(); + const int extra = MAX_NUM_NODES + 30; // overflow well past the MAX-2 cap + for (int i = 1; i <= extra; i++) { + const bool fav = (i == 1); // oldest, but a favourite + const bool ign = (i == 2); // 2nd-oldest, but blocked + db->push(2000 + i, /*last_heard=*/i, fav, ign, /*withUser=*/true, /*withKey=*/true); + } + + db->runDemote(); + + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES, (int)db->getNumMeshNodes()); + TEST_ASSERT_NOT_NULL(db->getMeshNode(0x0BADF00D)); // self retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + 1)); // oldest favourite retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + 2)); // oldest ignored retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + extra)); // freshest retained + TEST_ASSERT_NULL(db->getMeshNode(2000 + 3)); // oldest non-protected demoted out of hot + TEST_ASSERT_TRUE(warmHasKey(2000 + 3)); // ...but its key kept in the warm tier +} + +// Favourite handling: a favourite is never the eviction victim, even when it is +// the oldest node in a full hot store. +static void test_eviction_preservesFavorite(void) +{ + db->seedSelf(); + for (int i = 1; i < MAX_NUM_NODES; i++) { // fill to MAX_NUM_NODES total (incl. self) + const bool fav = (i == 1); // oldest non-self, favourite + db->push(3000 + i, /*last_heard=*/i, fav, false, /*withUser=*/true, /*withKey=*/true); + } + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES, (int)db->getNumMeshNodes()); // full + + TEST_ASSERT_NOT_NULL(db->getOrCreateMeshNode(0x99990000)); // forces an eviction + + TEST_ASSERT_NOT_NULL(db->getMeshNode(3000 + 1)); // favourite survived despite being oldest + TEST_ASSERT_NULL(db->getMeshNode(3000 + 2)); // oldest non-favourite evicted + TEST_ASSERT_NOT_NULL(db->getMeshNode(0x99990000)); +} + +// Ignored handling: an ignored node survives eviction (like a favourite), and is +// never purged by cleanupMeshDB even with no user info (a block set by bare ID). +static void test_ignored_survivesEvictionAndCleanup(void) +{ + // (a) eviction protection + db->clearHot(); + db->seedSelf(); + for (int i = 1; i < MAX_NUM_NODES; i++) { + const bool ign = (i == 1); // oldest non-self, blocked + db->push(4000 + i, /*last_heard=*/i, false, ign, /*withUser=*/true, /*withKey=*/true); + } + TEST_ASSERT_NOT_NULL(db->getOrCreateMeshNode(0x88880000)); + TEST_ASSERT_NOT_NULL(db->getMeshNode(4000 + 1)); // blocked node survived + TEST_ASSERT_NULL(db->getMeshNode(4000 + 2)); // oldest non-blocked evicted + + // (b) cleanup protection — ignored kept without user info, plain no-user purged + db->clearHot(); + db->seedSelf(); + db->push(5000, 100, false, /*ignored=*/true, /*withUser=*/false, false); + db->push(5001, 100, false, false, /*withUser=*/false, false); + db->runCleanup(); + TEST_ASSERT_NOT_NULL(db->getMeshNode(5000)); // blocked-by-ID kept despite no user info + TEST_ASSERT_NULL(db->getMeshNode(5001)); // ordinary no-user node purged +} + +// Protected-node cap: at most MAX_NUM_NODES-2 nodes may be protected, so >=2 +// evictable slots always remain. setProtectedFlag refuses once the cap is hit. +static void test_protectedCap_refusesBeyondLimit(void) +{ + db->seedSelf(); + for (int i = 0; i < MAX_NUM_NODES - 2; i++) + db->push(6000 + i, 100, /*favorite=*/true, false, /*withUser=*/true, false); + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES - 2, db->numProtectedNodes()); + + db->push(7000, 100, false, false, /*withUser=*/true, false); + meshtastic_NodeInfoLite *fresh = db->getMeshNode(7000); + TEST_ASSERT_NOT_NULL(fresh); + TEST_ASSERT_FALSE(db->setProtectedFlag(fresh, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)); // refused at cap + TEST_ASSERT_FALSE(nodeInfoLiteIsIgnored(fresh)); // unchanged + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES - 2, db->numProtectedNodes()); + + // Adding another flag to an already-protected node doesn't grow the set, so + // it's still allowed at the cap. + meshtastic_NodeInfoLite *already = db->getMeshNode(6000); + TEST_ASSERT_TRUE(db->setProtectedFlag(already, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)); +} + +NDB_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + db = new NodeDBTestShim(); + nodeDB = db; + + UNITY_BEGIN(); + RUN_TEST(test_migration_demotesOldestKeepsKeepersAndSelf); + RUN_TEST(test_eviction_preservesFavorite); + RUN_TEST(test_ignored_survivesEvictionAndCleanup); + RUN_TEST(test_protectedCap_refusesBeyondLimit); + exit(UNITY_END()); +} +NDB_TEST_ENTRY void loop() {} + +#else // WARM_NODE_COUNT == 0 — nothing to exercise here + +void setUp(void) {} +void tearDown(void) {} +NDB_TEST_ENTRY void setup() +{ + UNITY_BEGIN(); + exit(UNITY_END()); +} +NDB_TEST_ENTRY void loop() {} + +#endif From 2ce298edb8be7bb8a13b5483a84d0e9e8ff1008f Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sat, 13 Jun 2026 00:54:58 +0100 Subject: [PATCH 03/13] fix copilot comments --- src/graphics/draw/MenuHandler.cpp | 16 ++++++++++++---- src/mesh/NodeDB.cpp | 17 +++++++++++------ src/mesh/NodeDB.h | 11 +++++++++-- src/mesh/WarmNodeStore.cpp | 21 ++++++++++++++++++--- src/modules/AdminModule.cpp | 4 ++-- variants/nrf52840/nrf52.ini | 2 +- 6 files changed, 53 insertions(+), 18 deletions(-) diff --git a/src/graphics/draw/MenuHandler.cpp b/src/graphics/draw/MenuHandler.cpp index 02fbba7a38d..afa17a29bdc 100644 --- a/src/graphics/draw/MenuHandler.cpp +++ b/src/graphics/draw/MenuHandler.cpp @@ -1572,7 +1572,8 @@ void menuHandler::manageNodeMenu() nodeDB->set_favorite(false, menuHandler::pickedNodeNum); } else { LOG_INFO("Adding node %08X to favorites", menuHandler::pickedNodeNum); - nodeDB->set_favorite(true, menuHandler::pickedNodeNum); + if (!nodeDB->set_favorite(true, menuHandler::pickedNodeNum)) + LOG_WARN(NodeDB::PROTECTED_CAP_WARN_FMT, "favorite", menuHandler::pickedNodeNum, MAX_NUM_NODES - 2); } screen->setFrames(graphics::Screen::FOCUS_PRESERVE); return; @@ -1615,16 +1616,23 @@ void menuHandler::manageNodeMenu() return; } + bool changed = false; if (nodeInfoLiteIsIgnored(n)) { nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, false); LOG_INFO("Unignoring node %08X", menuHandler::pickedNodeNum); + changed = true; } else if (nodeDB->setProtectedFlag(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) { LOG_INFO("Ignoring node %08X", menuHandler::pickedNodeNum); + changed = true; } else { - LOG_WARN("Can't ignore %08X: protected-node limit reached", menuHandler::pickedNodeNum); + LOG_WARN(NodeDB::PROTECTED_CAP_WARN_FMT, "ignore", menuHandler::pickedNodeNum, MAX_NUM_NODES - 2); + } + // Only persist/notify when the ignore bit actually moved; a cap + // refusal changed nothing and shouldn't trigger a prefs save. + if (changed) { + nodeDB->notifyObservers(true); + nodeDB->saveToDisk(); } - nodeDB->notifyObservers(true); - nodeDB->saveToDisk(); screen->setFrames(graphics::Screen::FOCUS_PRESERVE); return; } diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index d2c36569d7d..aaeaf968d6e 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -2993,15 +2993,20 @@ bool NodeDB::setProtectedFlag(meshtastic_NodeInfoLite *node, uint32_t mask, bool return false; } -void NodeDB::set_favorite(bool is_favorite, uint32_t nodeId) +bool NodeDB::set_favorite(bool is_favorite, uint32_t nodeId) { meshtastic_NodeInfoLite *lite = getMeshNode(nodeId); - if (lite && nodeInfoLiteIsFavorite(lite) != is_favorite) { - if (setProtectedFlag(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite)) { - sortMeshDB(); - saveNodeDatabaseToDisk(); - } + if (!lite) + return false; + if (nodeInfoLiteIsFavorite(lite) == is_favorite) + return true; // already in the requested state + if (setProtectedFlag(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite)) { + sortMeshDB(); + saveNodeDatabaseToDisk(); + return true; } + LOG_WARN(PROTECTED_CAP_WARN_FMT, "favorite", nodeId, MAX_NUM_NODES - 2); + return false; } bool NodeDB::isFavorite(uint32_t nodeId) diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index 2cbf3e42939..5fc202e5b39 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -227,13 +227,20 @@ class NodeDB bool updateUser(uint32_t nodeId, meshtastic_User &p, uint8_t channelIndex = 0); /* - * Sets a node either favorite or unfavorite + * Sets a node either favorite or unfavorite. Returns true if the node ends + * up in the requested state; false if the node is unknown or favouriting + * was refused by the protected-node cap (MAX_NUM_NODES - 2). */ - void set_favorite(bool is_favorite, uint32_t nodeId); + bool set_favorite(bool is_favorite, uint32_t nodeId); /// Count of eviction-protected (favourite/ignored/manually-verified) nodes. int numProtectedNodes() const; + /// printf-style warning emitted when setProtectedFlag() refuses a node at + /// the cap. %s = verb (favorite/ignore), 0x%08x = node, %d = cap. Shared by + /// LOG_WARN here and AdminModule::sendWarning so the wording stays in sync. + static constexpr const char *PROTECTED_CAP_WARN_FMT = "Can't %s 0x%08x: protected-node limit (%d) reached"; + /// Turn an eviction-protection flag (favourite/ignored/verified) on or off, /// capped so the protected set never exceeds MAX_NUM_NODES-2 — keeping at /// least two always-evictable slots. Turning a flag off always succeeds; diff --git a/src/mesh/WarmNodeStore.cpp b/src/mesh/WarmNodeStore.cpp index e39327a3657..933584ef39d 100644 --- a/src/mesh/WarmNodeStore.cpp +++ b/src/mesh/WarmNodeStore.cpp @@ -92,6 +92,7 @@ WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8 const bool candidateKeyed = key32 && keyIsSet(key32); WarmNodeEntry *slot = find(num); + const bool sameNode = slot != nullptr; if (!slot) { // Pick a victim: any empty slot, else the oldest keyless entry, else // (only for keyed candidates) the oldest keyed entry. @@ -120,7 +121,9 @@ WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8 slot->last_heard = lastHeard; if (candidateKeyed) memcpy(slot->public_key, key32, 32); - else + else if (!sameNode) + // Repurposing a victim slot for a different node: clear its stale key. + // A keyless refresh of a node already here keeps the key we learned. memset(slot->public_key, 0, 32); return slot; } @@ -331,10 +334,17 @@ void WarmNodeStore::load() uint8_t order[WARM_FLASH_PAGES]; uint32_t seqs[WARM_FLASH_PAGES]; uint8_t nValid = 0; + uint8_t nCorrupt = 0; for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { WarmPageHeader h; - if (!ringReadHeader(p, h)) + if (!ringReadHeader(p, h)) { + // An erased page reads back all-ones; any other magic is a + // partially-written or bit-rotted header we're dropping, so flag it + // rather than silently treating the loss as a clean empty ring. + if (h.magic != 0xFFFFFFFFu) + nCorrupt++; continue; + } uint8_t pos = nValid; while (pos > 0 && static_cast(h.seq - seqs[pos - 1]) < 0) { order[pos] = order[pos - 1]; @@ -350,7 +360,10 @@ void WarmNodeStore::load() activePage = 0xFF; writeSlot = 0; nextSeq = 1; - LOG_INFO("WarmStore: ring empty, starting fresh"); + if (nCorrupt) + LOG_WARN("WarmStore: ring unreadable (%u corrupt page(s)), starting empty", nCorrupt); + else + LOG_INFO("WarmStore: ring empty, starting fresh"); return; } @@ -384,6 +397,8 @@ void WarmNodeStore::load() nextSeq = seqs[k] + 1; } } + if (nCorrupt) + LOG_WARN("WarmStore: dropped %u corrupt ring page(s) during replay; some warm nodes may be lost", nCorrupt); LOG_INFO("WarmStore: replayed %u ring records -> %u live nodes (page %u, slot %u)", (unsigned)replayed, (unsigned)count(), activePage, writeSlot); } diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index 2eff7976a81..b512fc47872 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -463,7 +463,7 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta if (screen) screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take - sendWarning("Can't favorite 0x%08x: protected-node limit (%d) reached", r->set_favorite_node, MAX_NUM_NODES - 2); + sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "favorite", r->set_favorite_node, MAX_NUM_NODES - 2); } } break; @@ -490,7 +490,7 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta nodeDB->eraseNodeSatellites(node->num); saveChanges(SEGMENT_NODEDATABASE, false); } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take - sendWarning("Can't ignore 0x%08x: protected-node limit (%d) reached", r->set_ignored_node, MAX_NUM_NODES - 2); + sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "ignore", r->set_ignored_node, MAX_NUM_NODES - 2); } } break; diff --git a/variants/nrf52840/nrf52.ini b/variants/nrf52840/nrf52.ini index 999ea6f9f1d..5a308796bdc 100644 --- a/variants/nrf52840/nrf52.ini +++ b/variants/nrf52840/nrf52.ini @@ -15,7 +15,7 @@ extra_scripts = ${env.extra_scripts} extra_scripts/nrf52_extra.py pre:extra_scripts/nrf52_lto.py - extra_scripts/nrf52_warm_region.py ; post-link guard: image must end below the 16 KB warm-store raw-flash region at 0xE9000 + extra_scripts/nrf52_warm_region.py ; post-link guard: image must end below the 12 KB warm-store raw-flash region at 0xEA000-0xED000 build_type = release build_flags = From 48edd5a8bcb8d6123fc3464c7caa5116f51ea08e Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sat, 13 Jun 2026 01:46:11 +0100 Subject: [PATCH 04/13] once again --- src/mesh/WarmNodeStore.cpp | 17 +++++------------ src/mesh/WarmNodeStore.h | 3 ++- 2 files changed, 7 insertions(+), 13 deletions(-) diff --git a/src/mesh/WarmNodeStore.cpp b/src/mesh/WarmNodeStore.cpp index 933584ef39d..c50d1f5e624 100644 --- a/src/mesh/WarmNodeStore.cpp +++ b/src/mesh/WarmNodeStore.cpp @@ -47,14 +47,12 @@ WarmNodeStore::WarmNodeStore() { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) entries = static_cast(ps_calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); - if (entries) { - entriesFromPsram = true; - } else { + if (!entries) { LOG_WARN("WarmStore: PSRAM allocation failed, falling back to heap"); - entries = new WarmNodeEntry[WARM_NODE_COUNT](); + entries = static_cast(calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); } #else - entries = new WarmNodeEntry[WARM_NODE_COUNT](); + entries = static_cast(calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); #endif #if defined(ARCH_NRF52) && defined(NRF52840_XXAA) memset(pageOf, 0xFF, sizeof(pageOf)); @@ -63,12 +61,7 @@ WarmNodeStore::WarmNodeStore() WarmNodeStore::~WarmNodeStore() { - if (!entries) - return; - if (entriesFromPsram) - free(entries); - else - delete[] entries; + free(entries); // always malloc-family (calloc / ps_calloc) entries = nullptr; } @@ -130,7 +123,7 @@ WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8 bool WarmNodeStore::absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32) { - WarmNodeEntry *slot = place(num, lastHeard, key32); + const WarmNodeEntry *slot = place(num, lastHeard, key32); if (!slot) return false; persistEntry(*slot); diff --git a/src/mesh/WarmNodeStore.h b/src/mesh/WarmNodeStore.h index 6ea9286d874..a91d6cd4aca 100644 --- a/src/mesh/WarmNodeStore.h +++ b/src/mesh/WarmNodeStore.h @@ -54,6 +54,8 @@ class WarmNodeStore public: WarmNodeStore(); ~WarmNodeStore(); + WarmNodeStore(const WarmNodeStore &) = delete; + WarmNodeStore &operator=(const WarmNodeStore &) = delete; /// Remember an evicted hot node. Keyless candidates never displace keyed /// entries; otherwise the oldest (keyless-first) entry is replaced. @@ -81,7 +83,6 @@ class WarmNodeStore private: WarmNodeEntry *entries = nullptr; // WARM_NODE_COUNT slots; PSRAM on ESP32 when available - bool entriesFromPsram = false; bool dirty = false; WarmNodeEntry *find(NodeNum num) const; From 17ada5ba9bb6b587dd1c58974bb5c4404bac8ba0 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sat, 13 Jun 2026 02:08:44 +0100 Subject: [PATCH 05/13] WarmNodeStore: fix cppcheck warnings (uninitvar, constVariablePointer) Zero-initialise `stranded[]` and `seqs[]/order[]` VLAs so cppcheck can verify there are no unguarded reads of uninitialised memory (the guards exist but are not visible to static analysis). Mark two local pointers `const` where the pointed-to entry is never mutated after assignment. Co-Authored-By: Claude Sonnet 4.6 --- src/mesh/WarmNodeStore.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mesh/WarmNodeStore.cpp b/src/mesh/WarmNodeStore.cpp index c50d1f5e624..a53c6531c4a 100644 --- a/src/mesh/WarmNodeStore.cpp +++ b/src/mesh/WarmNodeStore.cpp @@ -258,7 +258,7 @@ void WarmNodeStore::ringRotate() } // Capture live entries stranded in the page we're about to erase - int stranded[WARM_NODE_COUNT]; + int stranded[WARM_NODE_COUNT] = {}; int nStranded = 0; for (size_t i = 0; i < WARM_NODE_COUNT; i++) { if (entries[i].num && pageOf[i] == target) @@ -324,8 +324,8 @@ void WarmNodeStore::load() concurrency::LockGuard g(spiLock); // Order valid pages by ascending seq so replay applies oldest first - uint8_t order[WARM_FLASH_PAGES]; - uint32_t seqs[WARM_FLASH_PAGES]; + uint8_t order[WARM_FLASH_PAGES] = {}; + uint32_t seqs[WARM_FLASH_PAGES] = {}; uint8_t nValid = 0; uint8_t nCorrupt = 0; for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { @@ -379,7 +379,7 @@ void WarmNodeStore::load() memset(e, 0, sizeof(*e)); } } else { - WarmNodeEntry *e = place(rec.num, rec.last_heard, rec.public_key); + const WarmNodeEntry *e = place(rec.num, rec.last_heard, rec.public_key); if (e) pageOf[e - entries] = p; } From 542fd49c0adb6c683645e2a800a8e53a6f2a9320 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sat, 13 Jun 2026 19:49:10 +0100 Subject: [PATCH 06/13] bufs --- protobufs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protobufs b/protobufs index 1df6c115424..36251667179 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit 1df6c1154248c82abb21b73eb06c203f595780db +Subproject commit 36251667179496c1e1261f4e4e5b349a51e5e861 From 7a46764c2e2089e92c13196dc48afd9fd41ba6f1 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Fri, 12 Jun 2026 02:48:34 +0100 Subject: [PATCH 07/13] TrafficManagement: flat unified cache + persistent next-hop overflow store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks the TrafficManagementModule cache layer (policing behaviour unchanged from upstream) and adds a routing-hint overflow store: - Flatten the ring: replace the cuckoo-hashed unified cache and the bucketed PSRAM NodeInfo index with plain flat arrays + linear scan (same idiom as WarmNodeStore). At LoRa packet rates an O(n) scan of the cache is negligible, and it removes a large amount of hashing/displacement complexity. The cache entry is 11 B; timestamps use a uniform +1 presence-offset so a 0 byte always means "empty" across every sub-store. Adds rebaseEpoch() so cached state survives the ~19 h relative-timestamp horizon instead of being flushed. - Next-hop overflow cache: setNextHop/getNextHopHint store a confirmed last-byte relay for a destination, written only from NextHopRouter's ACK-confirmed decision (and mirrored from TraceRoute). NextHopRouter::getNextHop falls back to this cache when the hot NodeDB has no hint, so DMs/relays to long-tail nodes keep routing after the node ages out of NodeInfoLite. - Persistence: preloadNextHopsFromNodeDB warm-starts the cache from persisted NodeInfoLite hints on first maintenance pass; next_hop entries are kept alive across the maintenance sweep (no TTL) and never clobbered by a stale preload. All packet-policing logic (rate limit, position dedup, unknown-packet drop, NodeInfo direct response, hop exhaustion) is the existing upstream behaviour, untouched. HAS_TRAFFIC_MANAGEMENT defaults on so the module is compiled in. (see note). Tests: upstream policing suite now actually runs (adds the MeshTypes.h include that gates HAS_TRAFFIC_MANAGEMENT) plus 4 next-hop tests. Role-aware throttles, politeness, precision clamp, port-interval and mesh-radius gating — and the rate-limit >255 saturation fix — are deferred to the advanced-TMM branch. Note: default dedup movement grid moves to ~91m, which also means 1.5km required to end up with the same signature position - coarser and therefore further than before. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mesh/Default.h | 2 +- src/mesh/NextHopRouter.cpp | 49 +- src/mesh/NodeDB.cpp | 26 + src/mesh/mesh-pb-constants.h | 2 +- src/modules/TraceRouteModule.cpp | 12 + src/modules/TrafficManagementModule.cpp | 684 ++++++++------------- src/modules/TrafficManagementModule.h | 257 +++----- test/test_traffic_management/test_main.cpp | 269 ++++++-- 8 files changed, 616 insertions(+), 685 deletions(-) diff --git a/src/mesh/Default.h b/src/mesh/Default.h index 97f87fc7bfc..d638ca2a936 100644 --- a/src/mesh/Default.h +++ b/src/mesh/Default.h @@ -34,7 +34,7 @@ enum class TrafficType { POSITION, TELEMETRY }; // Traffic management defaults -#define default_traffic_mgmt_position_precision_bits 24 // ~10m grid cells +#define default_traffic_mgmt_position_precision_bits 19 // ~90m grid cells (±45m) #define default_traffic_mgmt_position_min_interval_secs (ONE_DAY / 2) // 12 hours between identical positions // Hop scaling defaults diff --git a/src/mesh/NextHopRouter.cpp b/src/mesh/NextHopRouter.cpp index e8613d45729..4708faf0c15 100644 --- a/src/mesh/NextHopRouter.cpp +++ b/src/mesh/NextHopRouter.cpp @@ -98,22 +98,27 @@ void NextHopRouter::sniffReceived(const meshtastic_MeshPacket *p, const meshtast // destination if (p->from != 0) { meshtastic_NodeInfoLite *origTx = nodeDB->getMeshNode(p->from); - if (origTx) { - // Either relayer of ACK was also a relayer of the packet, or we were the *only* relayer and the ACK came - // directly from the destination - // Single lookup for both relayer checks on the same (request_id, to) pair - bool wasAlreadyRelayer = false; - bool weWereSoleRelayer = false; - bool weWereRelayer = false; - checkRelayers(p->relay_node, ourRelayID, p->decoded.request_id, p->to, &wasAlreadyRelayer, &weWereRelayer, - &weWereSoleRelayer); - if ((weWereRelayer && wasAlreadyRelayer) || (getHopsAway(*p) == 0 && weWereSoleRelayer)) { - if (origTx->next_hop != p->relay_node) { // Not already set - LOG_INFO("Update next hop of 0x%x to 0x%x based on ACK/reply (was relayer %d we were sole %d)", p->from, - p->relay_node, wasAlreadyRelayer, weWereSoleRelayer); - origTx->next_hop = p->relay_node; - } + // Either relayer of ACK was also a relayer of the packet, or we were the *only* relayer and the ACK came + // directly from the destination. checkRelayers is read-only on PacketHistory and O(1), so we run it even + // when origTx is absent — that lets us still capture the confirmed hop into the TMM overflow cache below. + // Single lookup for both relayer checks on the same (request_id, to) pair + bool wasAlreadyRelayer = false; + bool weWereSoleRelayer = false; + bool weWereRelayer = false; + checkRelayers(p->relay_node, ourRelayID, p->decoded.request_id, p->to, &wasAlreadyRelayer, &weWereRelayer, + &weWereSoleRelayer); + if ((weWereRelayer && wasAlreadyRelayer) || (getHopsAway(*p) == 0 && weWereSoleRelayer)) { + if (origTx && origTx->next_hop != p->relay_node) { // Not already set + LOG_INFO("Update next hop of 0x%x to 0x%x based on ACK/reply (was relayer %d we were sole %d)", p->from, + p->relay_node, wasAlreadyRelayer, weWereSoleRelayer); + origTx->next_hop = p->relay_node; } +#if HAS_TRAFFIC_MANAGEMENT + // Mirror the confirmed hop into the TMM overflow cache so it survives even when the source + // isn't (or is no longer) in the hot NodeDB. Same bidirectional confirmation gate as above. + if (trafficManagementModule) + trafficManagementModule->setNextHop(p->from, p->relay_node); +#endif } } if (!isToUs(p)) { @@ -194,6 +199,7 @@ std::optional NextHopRouter::getNextHop(NodeNum to, uint8_t relay_node) if (isBroadcast(to)) return std::nullopt; + // Hot store first: a direct array hit on the live NodeDB entry. meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(to); if (node && node->next_hop) { // We are careful not to return the relay node as the next hop @@ -203,6 +209,19 @@ std::optional NextHopRouter::getNextHop(NodeNum to, uint8_t relay_node) } else LOG_WARN("Next hop for 0x%x is 0x%x, same as relayer; set no pref", to, node->next_hop); } + +#if HAS_TRAFFIC_MANAGEMENT + // Fallback: TMM overflow cache holds confirmed hops for nodes that have aged + // out of the hot store. Same byte source/confidence as NodeInfoLite.next_hop. + if (trafficManagementModule) { + uint8_t hint = trafficManagementModule->getNextHopHint(to); + if (hint && hint != relay_node) { + // LOG_DEBUG("Next hop for 0x%x is 0x%x (TMM cache)", to, hint); + return hint; + } + } +#endif + return std::nullopt; } diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index aaeaf968d6e..5877c6cbff1 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -1107,6 +1107,20 @@ void NodeDB::initConfigIntervals() #endif } +// Always-on traffic management defaults. Only booleans are written; every +// numeric field stays 0 and resolves to its default_traffic_mgmt_* macro at +// use (e.g. position dedup precision/interval), so fork-wide tuning changes +// take effect without another migration. Rate limiting and the features that +// exhaust or reshape relayed traffic (exhaust_hop_*, drop_unknown_enabled, +// nodeinfo_direct_response) stay opt-in. +static void installTrafficManagementDefaults(meshtastic_LocalModuleConfig &mc) +{ + mc.has_traffic_management = true; + mc.traffic_management = meshtastic_ModuleConfig_TrafficManagementConfig_init_zero; + mc.traffic_management.enabled = true; + mc.traffic_management.position_dedup_enabled = true; +} + void NodeDB::installDefaultModuleConfig() { LOG_INFO("Install default ModuleConfig"); @@ -1210,6 +1224,8 @@ void NodeDB::installDefaultModuleConfig() moduleConfig.has_neighbor_info = true; moduleConfig.neighbor_info.enabled = false; + installTrafficManagementDefaults(moduleConfig); + moduleConfig.has_detection_sensor = true; moduleConfig.detection_sensor.enabled = false; moduleConfig.detection_sensor.detection_trigger_type = meshtastic_ModuleConfig_DetectionSensorConfig_TriggerType_LOGIC_HIGH; @@ -2086,6 +2102,16 @@ void NodeDB::loadFromDisk() } } + // Always-on traffic management: a device that has NEVER configured TMM + // (has_traffic_management false — AdminModule always sets the has_ flag on + // write, even when disabling) gets the fork defaults. Explicitly configured + // devices keep their exact settings. + if (!moduleConfig.has_traffic_management) { + LOG_INFO("Traffic management never configured, installing always-on defaults"); + installTrafficManagementDefaults(moduleConfig); + saveToDisk(SEGMENT_MODULECONFIG); + } + state = loadProto(channelFileName, meshtastic_ChannelFile_size, sizeof(meshtastic_ChannelFile), &meshtastic_ChannelFile_msg, &channelFile); if (state != LoadFileResult::LOAD_SUCCESS) { diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index 99f5dcbb86e..9857d72277c 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -137,7 +137,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas // Traffic Management module configuration // Enable per-variant by defining HAS_TRAFFIC_MANAGEMENT=1 in variant.h #ifndef HAS_TRAFFIC_MANAGEMENT -#define HAS_TRAFFIC_MANAGEMENT 0 +#define HAS_TRAFFIC_MANAGEMENT 1 #endif // HopScalingModule - variable hop module: dynamically adjusts broadcast hop_limit based on mesh density diff --git a/src/modules/TraceRouteModule.cpp b/src/modules/TraceRouteModule.cpp index b6cb5d04181..915203ed16b 100644 --- a/src/modules/TraceRouteModule.cpp +++ b/src/modules/TraceRouteModule.cpp @@ -8,6 +8,10 @@ #include "meshUtils.h" #include +#if HAS_TRAFFIC_MANAGEMENT +#include "modules/TrafficManagementModule.h" +#endif + extern graphics::Screen *screen; TraceRouteModule *traceRouteModule; @@ -323,6 +327,14 @@ void TraceRouteModule::maybeSetNextHop(NodeNum target, uint8_t nextHopByte) LOG_INFO("Updating next-hop for 0x%08x to 0x%02x based on traceroute", target, nextHopByte); node->next_hop = nextHopByte; } + +#if HAS_TRAFFIC_MANAGEMENT + // Mirror into the TMM overflow cache. Traceroute is the highest-confidence + // source (full known route), and this captures the target even when it isn't + // in the hot NodeDB — same rationale as the ACK-confirmed path in NextHopRouter. + if (trafficManagementModule) + trafficManagementModule->setNextHop(target, nextHopByte); +#endif } void TraceRouteModule::processUpgradedPacket(const meshtastic_MeshPacket &mp) diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index 4c73c2947c5..318921f2241 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -28,7 +28,6 @@ namespace constexpr uint32_t kMaintenanceIntervalMs = 60 * 1000UL; // Cache cleanup interval constexpr uint32_t kUnknownResetMs = 60 * 1000UL; // Unknown packet window -constexpr uint8_t kMaxCuckooKicks = 16; // Max displacement chain length // NodeInfo direct response: enforced maximum hops by device role // Both use maxHops logic (respond when hopsAway <= threshold) @@ -76,6 +75,21 @@ bool isWithinWindow(uint32_t nowMs, uint32_t startMs, uint32_t intervalMs) return (nowMs - startMs) < intervalMs; } +/** + * Slide an 8-bit relative timestamp back by a wall-clock slab during epoch rebase. + * + * Entries older than the slab clamp to 0 (then reclaimed by the maintenance sweep); + * live entries keep their reconstructed age minus a sub-tick remainder. Each field + * slides by its own resolution's worth of ticks, so a single slab covers all three. + */ +inline void slideRelativeTime(uint8_t &ticks, uint32_t slabMs, uint16_t resolutionSecs) +{ + if (ticks == 0 || resolutionSecs == 0) + return; + uint32_t dec = slabMs / (static_cast(resolutionSecs) * 1000UL); + ticks = (ticks > dec) ? static_cast(ticks - dec) : 0; +} + /** * Truncate lat/lon to specified precision for position deduplication. * @@ -203,27 +217,16 @@ TrafficManagementModule::TrafficManagementModule() : MeshModule("TrafficManageme #endif // TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - TM_LOG_INFO("Allocating NodeInfo cache: target=%u occupancy=%u%% payload=%u bytes (PSRAM) tags=%u bytes (%u-bit, %u slots, " - "%u buckets x %u)", - static_cast(nodeInfoTargetEntries()), static_cast(nodeInfoTargetOccupancyPercent()), - static_cast(nodeInfoTargetEntries() * sizeof(NodeInfoPayloadEntry)), - static_cast(nodeInfoIndexMetadataBudgetBytes()), static_cast(nodeInfoTagBits()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoBucketCount()), - static_cast(nodeInfoBucketSize())); - - nodeInfoIndex = static_cast(calloc(nodeInfoIndexMetadataBudgetBytes(), sizeof(uint8_t))); - if (!nodeInfoIndex) { - TM_LOG_WARN("NodeInfo index allocation failed; direct responses will fall back to NodeDB"); + TM_LOG_INFO("Allocating NodeInfo cache: %u entries, %u bytes (PSRAM flat array)", + static_cast(nodeInfoTargetEntries()), + static_cast(nodeInfoTargetEntries() * sizeof(NodeInfoPayloadEntry))); + + nodeInfoPayload = static_cast(ps_calloc(nodeInfoTargetEntries(), sizeof(NodeInfoPayloadEntry))); + if (nodeInfoPayload) { + nodeInfoPayloadFromPsram = true; + TM_LOG_INFO("NodeInfo PSRAM cache ready"); } else { - nodeInfoPayload = static_cast(ps_calloc(nodeInfoTargetEntries(), sizeof(NodeInfoPayloadEntry))); - if (nodeInfoPayload) { - nodeInfoPayloadFromPsram = true; - TM_LOG_INFO("NodeInfo bucketed cuckoo cache ready"); - } else { - TM_LOG_WARN("NodeInfo PSRAM payload allocation failed; direct responses will fall back to NodeDB"); - free(nodeInfoIndex); - nodeInfoIndex = nullptr; - } + TM_LOG_WARN("NodeInfo PSRAM payload allocation failed; direct responses will fall back to NodeDB"); } #else TM_LOG_DEBUG("NodeInfo PSRAM cache not available on this target"); @@ -255,11 +258,6 @@ TrafficManagementModule::~TrafficManagementModule() delete[] nodeInfoPayload; nodeInfoPayload = nullptr; } - - if (nodeInfoIndex) { - free(nodeInfoIndex); - nodeInfoIndex = nullptr; - } } // ============================================================================= @@ -292,17 +290,11 @@ void TrafficManagementModule::incrementStat(uint32_t *field) } // ============================================================================= -// Cuckoo Hash Table Operations +// Flat Unified Cache Operations // ============================================================================= /** - * Find an existing entry for the given node. - * - * Cuckoo hashing guarantees that if an entry exists, it's in one of exactly - * two locations: hash1(node) or hash2(node). This provides O(1) lookup. - * - * @param node NodeNum to search for - * @return Pointer to entry if found, nullptr otherwise + * Find an existing entry for the given node (linear scan). */ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(NodeNum node) { @@ -313,35 +305,26 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(N if (!cache || node == 0) return nullptr; - // Check primary location - uint16_t h1 = cuckooHash1(node); - if (cache[h1].node == node) - return &cache[h1]; - - // Check alternate location - uint16_t h2 = cuckooHash2(node); - if (cache[h2].node == node) - return &cache[h2]; - + for (uint16_t i = 0; i < cacheSize(); i++) { + if (cache[i].node == node) + return &cache[i]; + } return nullptr; #endif } /** - * Find or create an entry for the given node using cuckoo hashing. - * - * If the node exists, returns the existing entry. Otherwise, attempts to - * insert a new entry using cuckoo displacement: + * Find or create an entry for the given node. * - * 1. Try to insert at h1(node) - if empty, done - * 2. Try to insert at h2(node) - if empty, done - * 3. Kick existing entry from h1 to its alternate location - * 4. Repeat up to kMaxCuckooKicks times - * 5. If cycle detected or max kicks exceeded, evict oldest entry + * One linear pass tracks the match, the first empty slot, and the eviction + * victim. When the cache is full, the victim is the stalest entry (largest + * of its three relative timestamps is smallest), preferring entries without + * a next_hop hint — those hints are the long-tail routing state the cache + * exists to keep, and the maintenance sweep never ages them out. * * @param node NodeNum to find or create * @param isNew Set to true if a new entry was created - * @return Pointer to entry, or nullptr if allocation failed + * @return Pointer to entry, or nullptr if the cache is unavailable */ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreateEntry(NodeNum node, bool *isNew) { @@ -351,304 +334,76 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreat *isNew = false; return nullptr; #else - if (!cache || node == 0) { - if (isNew) - *isNew = false; + if (isNew) + *isNew = false; + if (!cache || node == 0) return nullptr; - } - - // Check if entry already exists (O(1) lookup) - uint16_t h1 = cuckooHash1(node); - if (cache[h1].node == node) { - if (isNew) - *isNew = false; - return &cache[h1]; - } - uint16_t h2 = cuckooHash2(node); - if (cache[h2].node == node) { - if (isNew) - *isNew = false; - return &cache[h2]; - } - - // Entry doesn't exist - try to insert - - // Prefer empty slot at h1 - if (cache[h1].node == 0) { - memset(&cache[h1], 0, sizeof(UnifiedCacheEntry)); - cache[h1].node = node; - if (isNew) - *isNew = true; - return &cache[h1]; - } + UnifiedCacheEntry *empty = nullptr; + UnifiedCacheEntry *victim = nullptr; + bool victimHasHop = true; + uint8_t victimRecency = UINT8_MAX; - // Try empty slot at h2 - if (cache[h2].node == 0) { - memset(&cache[h2], 0, sizeof(UnifiedCacheEntry)); - cache[h2].node = node; - if (isNew) - *isNew = true; - return &cache[h2]; - } - - // Both slots occupied - perform cuckoo displacement - // Start by kicking entry at h1 to its alternate location - UnifiedCacheEntry displaced = cache[h1]; - memset(&cache[h1], 0, sizeof(UnifiedCacheEntry)); - cache[h1].node = node; - - for (uint8_t kicks = 0; kicks < kMaxCuckooKicks; kicks++) { - // Find alternate location for displaced entry - uint16_t altH1 = cuckooHash1(displaced.node); - uint16_t altH2 = cuckooHash2(displaced.node); - uint16_t altSlot = (altH1 == h1) ? altH2 : altH1; - - if (cache[altSlot].node == 0) { - // Found empty slot - insert displaced entry - cache[altSlot] = displaced; - if (isNew) - *isNew = true; - return &cache[h1]; + for (uint16_t i = 0; i < cacheSize(); i++) { + UnifiedCacheEntry &e = cache[i]; + if (e.node == node) + return &e; + if (e.node == 0) { + if (!empty) + empty = &e; + continue; + } + if (empty) + continue; // an empty slot beats any victim; stop scoring + const bool hasHop = e.next_hop != 0; + uint8_t recency = e.pos_time; + if (e.rate_time > recency) + recency = e.rate_time; + if (e.unknown_time > recency) + recency = e.unknown_time; + if (!victim || (hasHop == victimHasHop ? recency < victimRecency : !hasHop)) { + victim = &e; + victimHasHop = hasHop; + victimRecency = recency; } - - // Kick entry from alternate slot - UnifiedCacheEntry temp = cache[altSlot]; - cache[altSlot] = displaced; - displaced = temp; - h1 = altSlot; } - // Cuckoo cycle detected or max kicks exceeded. - // The displaced entry has no valid cuckoo slot — drop it to preserve cache integrity. - // Placing it at an arbitrary slot would make it unreachable by findEntry(). - TM_LOG_DEBUG("Cuckoo cycle, evicting node 0x%08x", displaced.node); - + UnifiedCacheEntry *slot = empty ? empty : victim; + if (!slot) + return nullptr; + if (!empty) + TM_LOG_DEBUG("Unified cache full, evicting node 0x%08x", slot->node); + memset(slot, 0, sizeof(UnifiedCacheEntry)); + slot->node = node; if (isNew) *isNew = true; - return &cache[cuckooHash1(node)]; + return slot; #endif } const TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findNodeInfoEntry(NodeNum node) const { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) - return nullptr; - - uint16_t payloadIndex = findNodeInfoPayloadIndex(node); - if (payloadIndex >= nodeInfoTargetEntries()) + if (!nodeInfoPayload || node == 0) return nullptr; - return &nodeInfoPayload[payloadIndex]; -#else - (void)node; - return nullptr; -#endif -} - -uint16_t TrafficManagementModule::encodeNodeInfoTag(uint16_t payloadIndex) const -{ - if (payloadIndex >= nodeInfoTargetEntries()) - return 0; - return static_cast(payloadIndex + 1u); -} - -uint16_t TrafficManagementModule::decodeNodeInfoPayloadIndex(uint16_t tag) const -{ - if (tag == 0 || tag > nodeInfoTargetEntries()) - return UINT16_MAX; - return static_cast(tag - 1u); -} - -uint16_t TrafficManagementModule::getNodeInfoTag(uint16_t slot) const -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || slot >= nodeInfoIndexSlots()) - return 0; - - const uint32_t bitOffset = static_cast(slot) * nodeInfoTagBits(); - const uint16_t byteOffset = static_cast(bitOffset >> 3); - const uint8_t shift = static_cast(bitOffset & 7u); - uint32_t packed = 0; - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset]); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 1u]) << 8; - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 2u]) << 16; - - return static_cast((packed >> shift) & nodeInfoTagMask()); -#else - (void)slot; - return 0; -#endif -} - -void TrafficManagementModule::setNodeInfoTag(uint16_t slot, uint16_t tag) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || slot >= nodeInfoIndexSlots()) - return; - - const uint16_t normalizedTag = static_cast(tag & nodeInfoTagMask()); - const uint32_t bitOffset = static_cast(slot) * nodeInfoTagBits(); - const uint16_t byteOffset = static_cast(bitOffset >> 3); - const uint8_t shift = static_cast(bitOffset & 7u); - uint32_t packed = 0; - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset]); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 1u]) << 8; - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 2u]) << 16; - - const uint32_t mask = static_cast(nodeInfoTagMask()) << shift; - packed = (packed & ~mask) | ((static_cast(normalizedTag) << shift) & mask); - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset] = static_cast(packed & 0xFFu); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset + 1u] = static_cast((packed >> 8) & 0xFFu); - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset + 2u] = static_cast((packed >> 16) & 0xFFu); -#else - (void)slot; - (void)tag; -#endif -} - -uint16_t TrafficManagementModule::findNodeInfoPayloadIndex(NodeNum node) const -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) - return UINT16_MAX; - - const uint16_t buckets[2] = {nodeInfoHash1(node), nodeInfoHash2(node)}; - - for (uint8_t b = 0; b < 2; b++) { - const uint16_t base = static_cast(buckets[b] * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - uint16_t tag = getNodeInfoTag(static_cast(base + slot)); - if (tag == 0) - continue; - - uint16_t payloadIndex = decodeNodeInfoPayloadIndex(tag); - if (payloadIndex >= nodeInfoTargetEntries()) - continue; - - if (nodeInfoPayload[payloadIndex].node == node) - return payloadIndex; - } - } - - return UINT16_MAX; -#else - (void)node; - return UINT16_MAX; -#endif -} - -bool TrafficManagementModule::removeNodeInfoIndexEntry(NodeNum node, uint16_t payloadIndex) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || node == 0 || payloadIndex >= nodeInfoTargetEntries()) - return false; - - const uint16_t payloadTag = encodeNodeInfoTag(payloadIndex); - if (payloadTag == 0) - return false; - const uint16_t buckets[2] = {nodeInfoHash1(node), nodeInfoHash2(node)}; - - for (uint8_t b = 0; b < 2; b++) { - const uint16_t base = static_cast(buckets[b] * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - const uint16_t indexSlot = static_cast(base + slot); - if (getNodeInfoTag(indexSlot) == payloadTag) { - setNodeInfoTag(indexSlot, 0); - return true; - } - } + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + if (nodeInfoPayload[i].node == node) + return &nodeInfoPayload[i]; } - - return false; + return nullptr; #else (void)node; - (void)payloadIndex; - return false; -#endif -} - -uint16_t TrafficManagementModule::allocateNodeInfoPayloadSlot() -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload) - return UINT16_MAX; - - for (uint16_t tries = 0; tries < nodeInfoTargetEntries(); tries++) { - uint16_t idx = static_cast((nodeInfoAllocHint + tries) % nodeInfoTargetEntries()); - if (nodeInfoPayload[idx].node == 0) { - nodeInfoAllocHint = static_cast((idx + 1u) % nodeInfoTargetEntries()); - return idx; - } - } -#endif - return UINT16_MAX; -} - -uint16_t TrafficManagementModule::evictNodeInfoPayloadSlot() -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex) - return UINT16_MAX; - - for (uint16_t tries = 0; tries < nodeInfoTargetEntries(); tries++) { - uint16_t idx = static_cast(nodeInfoEvictCursor % nodeInfoTargetEntries()); - nodeInfoEvictCursor = static_cast((nodeInfoEvictCursor + 1u) % nodeInfoTargetEntries()); - - NodeNum oldNode = nodeInfoPayload[idx].node; - if (oldNode == 0) - continue; - - removeNodeInfoIndexEntry(oldNode, idx); // best effort; cache tolerates occasional stale miss - nodeInfoPayload[idx].node = 0; - return idx; - } -#endif - return UINT16_MAX; -} - -bool TrafficManagementModule::tryInsertNodeInfoEntryInBucket(uint16_t bucket, uint16_t tag) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || !nodeInfoPayload || bucket >= nodeInfoBucketCount() || tag == 0) - return false; - - const uint16_t base = static_cast(bucket * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - const uint16_t indexSlot = static_cast(base + slot); - const uint16_t existingTag = getNodeInfoTag(indexSlot); - if (existingTag == 0) { - setNodeInfoTag(indexSlot, tag); - return true; - } - - // Opportunistically reuse stale tags that point at empty/invalid payload slots. - const uint16_t payloadIndex = decodeNodeInfoPayloadIndex(existingTag); - if (payloadIndex >= nodeInfoTargetEntries() || nodeInfoPayload[payloadIndex].node == 0) { - setNodeInfoTag(indexSlot, tag); - return true; - } - } -#else - (void)bucket; - (void)tag; + return nullptr; #endif - return false; } +/** + * Find or create a NodeInfo payload entry (linear scan of the flat PSRAM + * array). One pass tracks the match, the first empty slot, and the LRU + * victim by lastObservedMs (wrap-safe age). NodeInfo traffic is low-rate, + * so the O(n) scan is negligible. + */ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCreateNodeInfoEntry(NodeNum node, bool *usedEmptySlot) { @@ -656,88 +411,40 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr *usedEmptySlot = false; #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) + if (!nodeInfoPayload || node == 0) return nullptr; - uint16_t existing = findNodeInfoPayloadIndex(node); - if (existing < nodeInfoTargetEntries()) - return &nodeInfoPayload[existing]; - - const uint16_t beforeCount = countNodeInfoEntriesLocked(); - - uint16_t payloadIndex = allocateNodeInfoPayloadSlot(); - if (payloadIndex == UINT16_MAX) { - payloadIndex = evictNodeInfoPayloadSlot(); - if (payloadIndex == UINT16_MAX) - return nullptr; - } - - nodeInfoPayload[payloadIndex].node = node; - - // 4-way bucketed cuckoo insertion mirrors Cuckoo Filter practice from - // Fan et al. (CoNEXT 2014): high occupancy with short relocation chains. - uint16_t pending = encodeNodeInfoTag(payloadIndex); - uint16_t h1 = nodeInfoHash1(node); - uint16_t h2 = nodeInfoHash2(node); - - if (!tryInsertNodeInfoEntryInBucket(h1, pending) && !tryInsertNodeInfoEntryInBucket(h2, pending)) { - uint16_t currentBucket = h1; - for (uint8_t kicks = 0; kicks < kMaxCuckooKicks; kicks++) { - const uint16_t base = static_cast(currentBucket * nodeInfoBucketSize()); - const uint16_t kickSlot = static_cast((node + kicks) & (nodeInfoBucketSize() - 1u)); - const uint16_t pos = static_cast(base + kickSlot); - - uint16_t displaced = getNodeInfoTag(pos); - setNodeInfoTag(pos, pending); - pending = displaced; - - uint16_t displacedPayload = decodeNodeInfoPayloadIndex(pending); - if (displacedPayload >= nodeInfoTargetEntries()) { - pending = 0; - break; - } - - NodeNum displacedNode = nodeInfoPayload[displacedPayload].node; - if (displacedNode == 0) { - pending = 0; - break; - } - - uint16_t altH1 = nodeInfoHash1(displacedNode); - uint16_t altH2 = nodeInfoHash2(displacedNode); - uint16_t altBucket = (altH1 == currentBucket) ? altH2 : altH1; - - if (tryInsertNodeInfoEntryInBucket(altBucket, pending)) { - pending = 0; - break; - } - - currentBucket = altBucket; + NodeInfoPayloadEntry *empty = nullptr; + NodeInfoPayloadEntry *lru = nullptr; + uint32_t lruAge = 0; + const uint32_t now = millis(); + + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + NodeInfoPayloadEntry &e = nodeInfoPayload[i]; + if (e.node == node) + return &e; + if (e.node == 0) { + if (!empty) + empty = &e; + continue; } - - if (pending != 0) { - uint16_t droppedPayload = decodeNodeInfoPayloadIndex(pending); - if (droppedPayload < nodeInfoTargetEntries()) - nodeInfoPayload[droppedPayload].node = 0; - TM_LOG_DEBUG("NodeInfo bucketed cuckoo overflow, dropped payload idx=%u", - static_cast(droppedPayload < nodeInfoTargetEntries() ? droppedPayload : UINT16_MAX)); + if (empty) + continue; // an empty slot beats any victim; stop scoring + const uint32_t age = now - e.lastObservedMs; // unsigned subtraction is wrap-safe + if (!lru || age > lruAge) { + lru = &e; + lruAge = age; } } - uint16_t finalIndex = findNodeInfoPayloadIndex(node); - if (finalIndex >= nodeInfoTargetEntries()) { - // New entry did not survive insertion chain. - if (payloadIndex < nodeInfoTargetEntries() && nodeInfoPayload[payloadIndex].node == node) - nodeInfoPayload[payloadIndex].node = 0; + NodeInfoPayloadEntry *slot = empty ? empty : lru; + if (!slot) return nullptr; - } - - if (usedEmptySlot) { - const uint16_t afterCount = countNodeInfoEntriesLocked(); - *usedEmptySlot = afterCount > beforeCount; - } - - return &nodeInfoPayload[finalIndex]; + memset(slot, 0, sizeof(NodeInfoPayloadEntry)); + slot->node = node; + if (usedEmptySlot) + *usedEmptySlot = (slot == empty); + return slot; #else (void)node; return nullptr; @@ -747,12 +454,12 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr uint16_t TrafficManagementModule::countNodeInfoEntriesLocked() const { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex) + if (!nodeInfoPayload) return 0; uint16_t count = 0; - for (uint16_t i = 0; i < nodeInfoIndexSlots(); i++) { - if (getNodeInfoTag(i) != 0) + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + if (nodeInfoPayload[i].node != 0) count++; } return count; @@ -764,7 +471,7 @@ uint16_t TrafficManagementModule::countNodeInfoEntriesLocked() const void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &mp) { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || mp.decoded.payload.size == 0) + if (!nodeInfoPayload || mp.decoded.payload.size == 0) return; meshtastic_User user = meshtastic_User_init_zero; @@ -797,16 +504,84 @@ void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &m } if (usedEmptySlot) { - TM_LOG_INFO("NodeInfo PSRAM cache entries: %u/%u target (%u packed slots, %u-bit tags, %u-byte DRAM index)", - static_cast(cachedCount), static_cast(nodeInfoTargetEntries()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoTagBits()), - static_cast(nodeInfoIndexMetadataBudgetBytes())); + TM_LOG_INFO("NodeInfo PSRAM cache entries: %u/%u", static_cast(cachedCount), + static_cast(nodeInfoTargetEntries())); } #else (void)mp; #endif } +// ============================================================================= +// Next-Hop Overflow Cache +// ============================================================================= +// +// A routing hint store. The byte is the last byte of the NodeNum to use as next +// hop to reach `dest`. It is written ONLY from NextHopRouter's ACK-confirmed +// decision (a bidirectionally-verified relay) — never inferred one-way from +// relayed traffic. The TMM cache holds confirmed next-hops that have aged out of +// the hot NodeDB (NodeInfoLite), and NextHopRouter::getNextHop() consults it as a +// fallback after the hot store. + +void TrafficManagementModule::setNextHop(NodeNum dest, uint8_t nextHopByte) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || dest == 0 || nextHopByte == 0) + return; + + concurrency::LockGuard guard(&cacheLock); + bool isNew = false; + UnifiedCacheEntry *entry = findOrCreateEntry(dest, &isNew); + if (entry) + entry->next_hop = nextHopByte; // last-write-wins; only confirmed bytes reach here +#else + (void)dest; + (void)nextHopByte; +#endif +} + +uint8_t TrafficManagementModule::getNextHopHint(NodeNum dest) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || dest == 0) + return 0; + + concurrency::LockGuard guard(&cacheLock); + UnifiedCacheEntry *entry = findEntry(dest); + return entry ? entry->next_hop : 0; +#else + (void)dest; + return 0; +#endif +} + +void TrafficManagementModule::preloadNextHopsFromNodeDB() +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || !nodeDB) + return; + + uint16_t seeded = 0; + concurrency::LockGuard guard(&cacheLock); + const size_t count = nodeDB->getNumMeshNodes(); + for (size_t i = 0; i < count; i++) { + meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); + if (!node || node->num == 0 || node->next_hop == 0) + continue; + + bool isNew = false; + UnifiedCacheEntry *entry = findOrCreateEntry(node->num, &isNew); + // Don't clobber a freshly-learned confirmed hop with a (possibly stale) persisted one. + if (entry && entry->next_hop == 0) { + entry->next_hop = node->next_hop; + seeded++; + } + } + + TM_LOG_INFO("Preloaded %u next-hop hints from NodeDB", static_cast(seeded)); +#endif +} + // ============================================================================= // Epoch Management // ============================================================================= @@ -830,6 +605,43 @@ void TrafficManagementModule::resetEpoch(uint32_t nowMs) #endif } +/** + * Sliding-epoch rebase — preserve cached state past the 8-bit timestamp horizon. + * + * Instead of flushing the whole cache when offsets approach overflow, advance the + * epoch by a fixed slab and shift every live entry's relative timestamps back by + * the same wall-clock amount. A valid entry's window is only a handful of ticks + * wide (TTL auto-scales with resolution), so live entries comfortably survive; + * already-expired entries clamp to 0 and are reclaimed by the maintenance sweep in + * the same locked pass. Reconstructed absolute time is preserved (minus a sub-tick + * remainder), so in-flight TTL checks remain correct across the rebase. + * + * Caller must hold cacheLock. + */ +void TrafficManagementModule::rebaseEpoch(uint32_t nowMs) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + (void)nowMs; + + // Slab stays well below the 200-tick reset threshold so a single rebase drops + // the offset back into range (~200 -> ~72 ticks) while live entries survive. + const uint32_t slabMs = 128UL * maxResolution() * 1000UL; + cacheEpochMs += slabMs; + + TM_LOG_DEBUG("Rebasing cache epoch by %lus", static_cast(slabMs / 1000UL)); + + for (uint16_t i = 0; i < cacheSize(); i++) { + if (cache[i].node == 0) + continue; + slideRelativeTime(cache[i].pos_time, slabMs, posTimeResolution); + slideRelativeTime(cache[i].rate_time, slabMs, rateTimeResolution); + slideRelativeTime(cache[i].unknown_time, slabMs, unknownTimeResolution); + } +#else + (void)nowMs; +#endif +} + // ============================================================================= // Position Hash (Compact Mode) // ============================================================================= @@ -1042,11 +854,12 @@ int32_t TrafficManagementModule::runOnce() #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 const uint32_t nowMs = millis(); - // Check if epoch reset needed (~3.5 hours approaching 8-bit minute overflow) - if (needsEpochReset(nowMs)) { - concurrency::LockGuard guard(&cacheLock); - resetEpoch(nowMs); - return kMaintenanceIntervalMs; + // Warm-start the next-hop cache from persisted NodeInfoLite hints once nodeDB + // is populated. Done here (not in the constructor) so nodeDB has finished + // loading. Takes its own lock, so call before acquiring the sweep guard below. + if (!nextHopPreloaded) { + preloadNextHopsFromNodeDB(); + nextHopPreloaded = true; } // Calculate TTLs for cache expiration @@ -1065,6 +878,13 @@ int32_t TrafficManagementModule::runOnce() const uint32_t sweepStartMs = millis(); concurrency::LockGuard guard(&cacheLock); + + // Slide the epoch instead of flushing when offsets approach 8-bit overflow. + // Rebase preserves live entries; only already-expired ones clamp to 0 and are + // reclaimed by the sweep below in this same locked pass. + if (needsEpochReset(nowMs)) + rebaseEpoch(nowMs); + for (uint16_t i = 0; i < cacheSize(); i++) { if (cache[i].node == 0) continue; @@ -1104,6 +924,11 @@ int32_t TrafficManagementModule::runOnce() } } + // A confirmed next-hop hint has no TTL of its own and keeps the slot alive, + // so an aged-out routing hint outlives the dedup/rate/unknown state. + if (cache[i].next_hop != 0) + anyValid = true; + // If all data expired, free the slot entirely if (!anyValid) { memset(&cache[i], 0, sizeof(UnifiedCacheEntry)); @@ -1118,11 +943,9 @@ int32_t TrafficManagementModule::runOnce() static_cast(millis() - sweepStartMs)); #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (nodeInfoPayload && nodeInfoIndex) { - TM_LOG_DEBUG("NodeInfo PSRAM cache: %u/%u target (%u packed slots, %u buckets, %u-bit tags, %u-byte index)", - static_cast(countNodeInfoEntriesLocked()), static_cast(nodeInfoTargetEntries()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoBucketCount()), - static_cast(nodeInfoTagBits()), static_cast(nodeInfoIndexMetadataBudgetBytes())); + if (nodeInfoPayload) { + TM_LOG_DEBUG("NodeInfo PSRAM cache: %u/%u", static_cast(countNodeInfoEntriesLocked()), + static_cast(nodeInfoTargetEntries())); } #endif @@ -1153,8 +976,11 @@ bool TrafficManagementModule::shouldDropPosition(const meshtastic_MeshPacket *p, const int32_t lat_truncated = truncateLatLon(pos->latitude_i, precision); const int32_t lon_truncated = truncateLatLon(pos->longitude_i, precision); const uint8_t fingerprint = computePositionFingerprint(lat_truncated, lon_truncated, precision); - const uint32_t minIntervalMs = secsToMs(Default::getConfiguredOrDefault( - moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); + // Drop gate uses the RAW configured interval: 0 means "dedup disabled" (the + // contract documented below). The 12h default is only for resolution/TTL + // sizing (constructor / runOnce), not for deciding whether to drop — feeding + // the default here would silently turn the 0-disables-dedup contract off. + const uint32_t minIntervalMs = secsToMs(moduleConfig.traffic_management.position_min_interval_secs); bool isNew = false; concurrency::LockGuard guard(&cacheLock); @@ -1218,7 +1044,7 @@ bool TrafficManagementModule::shouldRespondToNodeInfo(const meshtastic_MeshPacke // If the PSRAM cache exists but misses, we intentionally do not fall back // to the node-wide table. This keeps the PSRAM direct-reply path separate // from NodeInfoModule/NodeDB behavior when PSRAM is available. - if (nodeInfoPayload && nodeInfoIndex) { + if (nodeInfoPayload) { TM_LOG_DEBUG("NodeInfo PSRAM cache miss for node=0x%08x", p->to); return false; } @@ -1378,7 +1204,9 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, entry->unknown_count = 0; } - // Increment counter (saturates at 255) + // Increment counter (saturates at 255). Same saturation handling as + // isRateLimited: without it, a clamped threshold of 255 can never fire. + const bool alreadySaturated = (entry->unknown_count == UINT8_MAX); saturatingIncrement(entry->unknown_count); // Check against threshold @@ -1386,7 +1214,7 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (threshold > 255) threshold = 255; - bool drop = entry->unknown_count > threshold; + bool drop = entry->unknown_count > threshold || (alreadySaturated && threshold == 255); if (drop || entry->unknown_count == threshold) { TM_LOG_DEBUG("Unknown packets 0x%08x: count=%u threshold=%u -> %s", p->from, entry->unknown_count, threshold, drop ? "DROP" : "at-limit"); diff --git a/src/modules/TrafficManagementModule.h b/src/modules/TrafficManagementModule.h index fe3483a8e51..22d1df5fe46 100644 --- a/src/modules/TrafficManagementModule.h +++ b/src/modules/TrafficManagementModule.h @@ -20,9 +20,11 @@ * - Router hop preservation (maintain hop_limit for router-to-router traffic) * * Memory Optimization: - * Uses a unified cache with cuckoo hashing for O(1) lookups and 56% memory reduction - * compared to separate per-feature caches. Timestamps are stored as 8-bit relative - * offsets from a rolling epoch to further reduce memory footprint. + * Uses one flat unified cache (plain array, linear scan) shared by all + * per-node features instead of separate per-feature caches. Timestamps are + * stored as 8-bit relative offsets from a rolling epoch to further reduce + * memory footprint. LoRa packet rates are low enough that an O(n) scan of + * ~1000 11-byte entries is negligible next to packet processing. */ class TrafficManagementModule : public MeshModule, private concurrency::OSThread { @@ -38,6 +40,19 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread void resetStats(); void recordRouterHopPreserved(); + // Next-hop overflow cache (routing hint). + // setNextHop: store a confirmed last-byte next hop for `dest`. Called by + // NextHopRouter from its ACK-confirmed decision (see sniffReceived). The + // byte must come from a bidirectionally-verified relay, not one-way inference. + // getNextHopHint: return the cached next-hop byte for `dest`, 0 if unknown. + void setNextHop(NodeNum dest, uint8_t nextHopByte); + uint8_t getNextHopHint(NodeNum dest); + + // Warm-start the next-hop cache from persisted NodeInfoLite hints so confirmed + // hops survive later hot-store (NodeDB) eviction. Idempotent; runs once after + // nodeDB is populated (lazily on first maintenance pass). + void preloadNextHopsFromNodeDB(); + /** * Check if this packet should have its hops exhausted. * Called from perhapsRebroadcast() to force hop_limit = 0 regardless of @@ -55,14 +70,18 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread int32_t runOnce() override; // Protected so test shims can force epoch rollover behavior. void resetEpoch(uint32_t nowMs); + // Sliding-epoch rebase: advance the epoch and shift live entries back by the + // same wall-clock amount instead of flushing, so cached state survives past the + // ~19h horizon. Caller must hold cacheLock. + void rebaseEpoch(uint32_t nowMs); private: // ========================================================================= - // Unified Cache Entry (10 bytes) - Same for ALL platforms + // Unified Cache Entry (11 bytes) - Same for ALL platforms // ========================================================================= // // A single compact structure used across ESP32, NRF52, and all other platforms. - // Memory: 10 bytes × 2048 entries = 20KB + // Memory: 11 bytes × 2048 entries = 22KB // // Position Fingerprinting: // Instead of storing full coordinates (8 bytes) or a computed hash, @@ -90,6 +109,16 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // [7] pos_time - Position timestamp (1 byte, adaptive resolution) // [8] rate_time - Rate window start (1 byte, adaptive resolution) // [9] unknown_time - Unknown tracking start (1 byte, adaptive resolution) + // [10] next_hop - Last-byte relay to reach `node` (1 byte, 0 = none) + // + // next_hop semantics: + // A routing hint: the last byte of the NodeNum to use as next hop to reach + // `node`. Written ONLY from NextHopRouter's ACK-confirmed decision (a + // bidirectionally-verified relay), never inferred one-way from relayed + // traffic. The TMM cache acts as an overflow store for confirmed next-hops + // that have aged out of the hot NodeDB (NodeInfoLite). Unlike the other + // fields it has no TTL of its own — it keeps its slot alive (see runOnce) + // and is refreshed only on the next confirmed exchange. // struct __attribute__((packed)) UnifiedCacheEntry { NodeNum node; // 4 bytes - Node identifier (0 = empty slot) @@ -99,66 +128,27 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread uint8_t pos_time; // 1 byte - Position timestamp (adaptive resolution) uint8_t rate_time; // 1 byte - Rate window start (adaptive resolution) uint8_t unknown_time; // 1 byte - Unknown tracking start (adaptive resolution) + uint8_t next_hop; // 1 byte - Last-byte relay to reach `node` (0 = none). See note below. }; - static_assert(sizeof(UnifiedCacheEntry) == 10, "UnifiedCacheEntry should be 10 bytes"); + static_assert(sizeof(UnifiedCacheEntry) == 11, "UnifiedCacheEntry should be 11 bytes"); // ========================================================================= - // Cuckoo Hash Table Implementation + // Flat unified cache // ========================================================================= // - // Cuckoo hashing provides O(1) worst-case lookup time using two hash functions. - // Each key can be in one of two possible locations (h1 or h2). On collision, - // the existing entry is "kicked" to its alternate location. + // Plain array, linear scan (same idiom as WarmNodeStore). A lookup walks at + // most cacheSize() × 11 B — microseconds at LoRa packet rates, not worth a + // hash table. Insertion on a full cache evicts the stalest entry, + // preferring entries without a next_hop hint (those are the long-tail + // routing state this cache exists to keep). // - // Benefits over linear scan: - // - O(1) lookup vs O(n) - critical at packet processing rates - // - O(1) insertion (amortized) with simple eviction on cycles - // - ~95% load factor achievable - // - // Cache size rounds to power-of-2 for fast modulo via bitmask. - // TRAFFIC_MANAGEMENT_CACHE_SIZE=2000 → cacheSize()=2048 - // - static constexpr uint16_t cacheSize(); - static constexpr uint16_t cacheMask(); - - // Hash functions for cuckoo hashing - inline uint16_t cuckooHash1(NodeNum node) const { return node & cacheMask(); } - inline uint16_t cuckooHash2(NodeNum node) const { return ((node * 2654435769u) >> (32 - cuckooHashBits())) & cacheMask(); } - static constexpr uint8_t cuckooHashBits(); - - // NodeInfo cache configuration (PSRAM path): - // - Payload lives in PSRAM - // - DRAM keeps packed 12-bit tags with 4-way bucketed cuckoo hashing - // (Fan et al., CoNEXT 2014). Tag value 0 is reserved as "empty". - static constexpr uint16_t kNodeInfoIndexMetadataBudgetBytes = 3072; // 3KB DRAM tag store - static constexpr uint8_t kNodeInfoTargetOccupancyPercent = 95; - static constexpr uint8_t kNodeInfoBucketSize = 4; - static constexpr uint8_t kNodeInfoTagBits = 12; - static constexpr uint16_t kNodeInfoTagMask = static_cast((1u << kNodeInfoTagBits) - 1u); - static constexpr uint16_t kNodeInfoIndexSlotsRaw = - static_cast((kNodeInfoIndexMetadataBudgetBytes * 8u) / kNodeInfoTagBits); - static constexpr uint16_t kNodeInfoIndexSlots = - static_cast(kNodeInfoIndexSlotsRaw - (kNodeInfoIndexSlotsRaw % kNodeInfoBucketSize)); - static constexpr uint16_t kNodeInfoTargetEntries = - static_cast((kNodeInfoIndexSlots * kNodeInfoTargetOccupancyPercent) / 100u); - static_assert((kNodeInfoIndexSlots % kNodeInfoBucketSize) == 0, "NodeInfo slot count must align to bucket size"); - static_assert(kNodeInfoTargetEntries < (1u << kNodeInfoTagBits), "NodeInfo tag bits must encode payload index"); - - static constexpr uint16_t nodeInfoTargetEntries(); - static constexpr uint16_t nodeInfoIndexMetadataBudgetBytes(); - static constexpr uint8_t nodeInfoTargetOccupancyPercent(); - static constexpr uint8_t nodeInfoBucketSize(); - static constexpr uint8_t nodeInfoTagBits(); - static constexpr uint16_t nodeInfoTagMask(); - static constexpr uint16_t nodeInfoIndexSlots(); - static constexpr uint16_t nodeInfoBucketCount(); - static constexpr uint16_t nodeInfoBucketMask(); - static constexpr uint8_t nodeInfoBucketHashBits(); - inline uint16_t nodeInfoHash1(NodeNum node) const { return node & nodeInfoBucketMask(); } - inline uint16_t nodeInfoHash2(NodeNum node) const - { - return ((node * 2246822519u) >> (32 - nodeInfoBucketHashBits())) & nodeInfoBucketMask(); - } + static constexpr uint16_t cacheSize() { return TRAFFIC_MANAGEMENT_CACHE_SIZE; } + + // NodeInfo cache configuration (PSRAM path): a flat PSRAM array of payload + // entries, linear scan keyed by `node`, LRU eviction by lastObservedMs. + // NodeInfo traffic is low-rate, so a full scan per lookup/insert is fine. + static constexpr uint16_t kNodeInfoCacheEntries = 2000; + static constexpr uint16_t nodeInfoTargetEntries() { return kNodeInfoCacheEntries; } // ========================================================================= // Adaptive Timestamp Resolution @@ -192,18 +182,28 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread return static_cast(res); } - // Convert to/from 8-bit relative timestamps with given resolution + // Convert to/from 8-bit relative timestamps with given resolution. + // + // All stored timestamps carry a uniform +1 "presence" offset: a value of 0 is + // reserved for "no timestamp recorded" (which is also the zero-initialized + // state), and stored values 1..255 encode raw ticks 0..254. This keeps the + // 0-means-empty sentinel consistent with memset/calloc zeroing across every + // sub-store, so the maintenance sweep's `_time != 0` presence checks are + // unambiguous (a timestamp recorded in the first tick after the epoch is no + // longer mistaken for an empty slot). The offset is applied here and removed + // on read, so it cancels out in all window math. uint8_t toRelativeTime(uint32_t nowMs, uint16_t resolutionSecs) const { uint32_t ticks = (nowMs - cacheEpochMs) / (resolutionSecs * 1000UL); - return (ticks > UINT8_MAX) ? UINT8_MAX : static_cast(ticks); + return (ticks >= UINT8_MAX) ? UINT8_MAX : static_cast(ticks + 1); } uint32_t fromRelativeTime(uint8_t ticks, uint16_t resolutionSecs) const { - return cacheEpochMs + (static_cast(ticks) * resolutionSecs * 1000UL); + return (ticks == 0) ? cacheEpochMs : cacheEpochMs + (static_cast(ticks - 1) * resolutionSecs * 1000UL); } - // Convenience wrappers for each timestamp type + // Convenience wrappers for each timestamp type (the +1 presence offset lives + // in the shared converters above, so these are plain pass-throughs). uint8_t toRelativePosTime(uint32_t nowMs) const { return toRelativeTime(nowMs, posTimeResolution); } uint32_t fromRelativePosTime(uint8_t t) const { return fromRelativeTime(t, posTimeResolution); } @@ -213,17 +213,20 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread uint8_t toRelativeUnknownTime(uint32_t nowMs) const { return toRelativeTime(nowMs, unknownTimeResolution); } uint32_t fromRelativeUnknownTime(uint8_t t) const { return fromRelativeTime(t, unknownTimeResolution); } - // Epoch reset when any timestamp approaches overflow - // With max resolution of 339 sec, 200 ticks = ~19 hours (safe margin for 24h max) - bool needsEpochReset(uint32_t nowMs) const + // Coarsest of the per-feature resolutions (seconds per tick). + uint16_t maxResolution() const { uint16_t maxRes = posTimeResolution; if (rateTimeResolution > maxRes) maxRes = rateTimeResolution; if (unknownTimeResolution > maxRes) maxRes = unknownTimeResolution; - return (nowMs - cacheEpochMs) > (200UL * maxRes * 1000UL); + return maxRes; } + + // True when relative offsets approach 8-bit overflow. + // With max resolution of 339 sec, 200 ticks = ~19 hours (safe margin for 24h max). + bool needsEpochReset(uint32_t nowMs) const { return (nowMs - cacheEpochMs) > (200UL * maxResolution() * 1000UL); } // ========================================================================= // Position Fingerprint // ========================================================================= @@ -246,7 +249,7 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // ========================================================================= mutable concurrency::Lock cacheLock; // Protects all cache access - UnifiedCacheEntry *cache = nullptr; // Cuckoo hash table (unified for all platforms) + UnifiedCacheEntry *cache = nullptr; // Flat unified cache (linear scan; all platforms) bool cacheFromPsram = false; // Tracks allocator for correct deallocation struct NodeInfoPayloadEntry { @@ -278,11 +281,8 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread uint8_t decodedBitfield; }; - NodeInfoPayloadEntry *nodeInfoPayload = nullptr; // NodeInfo payloads in PSRAM + NodeInfoPayloadEntry *nodeInfoPayload = nullptr; // NodeInfo payloads in PSRAM (flat array, linear scan) bool nodeInfoPayloadFromPsram = false; // Tracks allocator for correct deallocation - uint8_t *nodeInfoIndex = nullptr; // Packed 12-bit NodeInfo tags in DRAM - uint16_t nodeInfoAllocHint = 0; - uint16_t nodeInfoEvictCursor = 0; meshtastic_TrafficManagementStats stats; @@ -293,29 +293,22 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread NodeNum exhaustRequestedFrom = 0; PacketId exhaustRequestedId = 0; + // One-shot guard: warm-start next-hop cache from NodeDB on first maintenance pass. + bool nextHopPreloaded = false; + // ========================================================================= // Cache Operations // ========================================================================= - // Find or create entry for node using cuckoo hashing - // Returns nullptr if cache is full and eviction fails + // Find or create entry for node (linear scan; stalest-first eviction when full) UnifiedCacheEntry *findOrCreateEntry(NodeNum node, bool *isNew); // Find existing entry (no creation) UnifiedCacheEntry *findEntry(NodeNum node); - // NodeInfo cache operations (bucketed cuckoo index + PSRAM payloads) + // NodeInfo cache operations (flat PSRAM payload array, linear scan) const NodeInfoPayloadEntry *findNodeInfoEntry(NodeNum node) const; NodeInfoPayloadEntry *findOrCreateNodeInfoEntry(NodeNum node, bool *usedEmptySlot); - uint16_t findNodeInfoPayloadIndex(NodeNum node) const; - bool removeNodeInfoIndexEntry(NodeNum node, uint16_t payloadIndex); - uint16_t allocateNodeInfoPayloadSlot(); - uint16_t evictNodeInfoPayloadSlot(); - bool tryInsertNodeInfoEntryInBucket(uint16_t bucket, uint16_t tag); - uint16_t encodeNodeInfoTag(uint16_t payloadIndex) const; - uint16_t decodeNodeInfoPayloadIndex(uint16_t tag) const; - uint16_t getNodeInfoTag(uint16_t slot) const; - void setNodeInfoTag(uint16_t slot, uint16_t tag); uint16_t countNodeInfoEntriesLocked() const; void cacheNodeInfoPacket(const meshtastic_MeshPacket &mp); @@ -333,101 +326,7 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread void incrementStat(uint32_t *field); }; -// ========================================================================= -// Compile-time Cache Size Calculations -// ========================================================================= -// -// Round TRAFFIC_MANAGEMENT_CACHE_SIZE up to next power of 2 for efficient -// cuckoo hash indexing (allows bitmask instead of modulo). -// -// These use C++11-compatible constexpr (single return statement). -// - -namespace detail -{ -// Helper: round up to next power of 2 using bit manipulation -constexpr uint16_t nextPow2(uint16_t n) -{ - return n == 0 ? 0 : (((n - 1) | ((n - 1) >> 1) | ((n - 1) >> 2) | ((n - 1) >> 4) | ((n - 1) >> 8)) + 1); -} - -// Helper: floor(log2(n)) for n >= 0, C++11-compatible constexpr. -constexpr uint8_t log2Floor(uint16_t n) -{ - return n <= 1 ? 0 : static_cast(1 + log2Floor(static_cast(n >> 1))); -} - -// Helper: ceil(log2(n)) for n >= 1, C++11-compatible constexpr. -constexpr uint8_t log2Ceil(uint16_t n) -{ - return n <= 1 ? 0 : static_cast(1 + log2Floor(static_cast(n - 1))); -} -} // namespace detail - -constexpr uint16_t TrafficManagementModule::cacheSize() -{ - return detail::nextPow2(TRAFFIC_MANAGEMENT_CACHE_SIZE); -} - -constexpr uint16_t TrafficManagementModule::cacheMask() -{ - return cacheSize() > 0 ? cacheSize() - 1 : 0; -} - -constexpr uint8_t TrafficManagementModule::cuckooHashBits() -{ - return detail::log2Floor(cacheSize()); -} - -constexpr uint16_t TrafficManagementModule::nodeInfoTargetEntries() -{ - return kNodeInfoTargetEntries; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoIndexMetadataBudgetBytes() -{ - return kNodeInfoIndexMetadataBudgetBytes; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoTargetOccupancyPercent() -{ - return kNodeInfoTargetOccupancyPercent; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoBucketSize() -{ - return kNodeInfoBucketSize; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoTagBits() -{ - return kNodeInfoTagBits; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoTagMask() -{ - return kNodeInfoTagMask; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoIndexSlots() -{ - return kNodeInfoIndexSlots; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoBucketCount() -{ - return static_cast(nodeInfoIndexSlots() / nodeInfoBucketSize()); -} - -constexpr uint16_t TrafficManagementModule::nodeInfoBucketMask() -{ - return nodeInfoBucketCount() > 0 ? nodeInfoBucketCount() - 1 : 0; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoBucketHashBits() -{ - return detail::log2Floor(nodeInfoBucketCount()); -} +static_assert(TRAFFIC_MANAGEMENT_CACHE_SIZE <= UINT16_MAX, "cacheSize() returns uint16_t"); extern TrafficManagementModule *trafficManagementModule; diff --git a/test/test_traffic_management/test_main.cpp b/test/test_traffic_management/test_main.cpp index 7631999b319..c75e0da9000 100644 --- a/test/test_traffic_management/test_main.cpp +++ b/test/test_traffic_management/test_main.cpp @@ -1,3 +1,4 @@ +#include "MeshTypes.h" // Include BEFORE TestUtil.h — provides HAS_TRAFFIC_MANAGEMENT (via mesh-pb-constants.h) #include "TestUtil.h" #include #include @@ -10,6 +11,9 @@ #if HAS_TRAFFIC_MANAGEMENT +#include "airtime.h" +#if HAS_VARIABLE_HOPS +#endif #include "mesh/CryptoEngine.h" #include "mesh/MeshService.h" #include "mesh/NodeDB.h" @@ -28,6 +32,25 @@ constexpr NodeNum kLocalNode = 0x11111111; constexpr NodeNum kRemoteNode = 0x22222222; constexpr NodeNum kTargetNode = 0x33333333; +// Telemetry hop exhaustion is gated on channel congestion (alterReceived checks +// airTime->isTxAllowedChannelUtil/isTxAllowedAirUtil). Installs a global +// airTime reporting 100% channel utilization for the enclosing scope. +class ScopedBusyAirTime +{ + public: + ScopedBusyAirTime() : previous(airTime) + { + for (uint32_t i = 0; i < CHANNEL_UTILIZATION_PERIODS; i++) + busy.channelUtilization[i] = 10000; // 10 s of airtime per 10 s period + airTime = &busy; + } + ~ScopedBusyAirTime() { airTime = previous; } + + private: + AirTime busy; + AirTime *previous; +}; + class MockNodeDB : public NodeDB { public: @@ -54,6 +77,32 @@ class MockNodeDB : public NodeDB cachedNode.bitfield |= NODEINFO_BITFIELD_HAS_USER_MASK; } + // Role the TMM should see for the cached node (sender-role-aware throttles). + void setCachedNodeRole(meshtastic_Config_DeviceConfig_Role role) { cachedNode.role = role; } + + // Seed a node into the hot-store buffer at index 1 (index 0 is reserved for + // "self"). Respects the fixed-buffer invariant: `meshNodes` is a buffer of + // MAX_NUM_NODES slots with `numMeshNodes` as the logical count — we grow the + // buffer if needed and bump the count, never clear()/push_back() (which would + // shrink it and break NodeDB::resetNodes()'s begin()+1..end() fill). + void setHotNode(NodeNum n, uint8_t nextHop) + { + if (meshNodes->size() < 2) + meshNodes->resize(2); + (*meshNodes)[1] = meshtastic_NodeInfoLite_init_zero; + (*meshNodes)[1].num = n; + (*meshNodes)[1].next_hop = nextHop; + numMeshNodes = 2; + } + + // Evict everything but "self" — simulates the hot DB rolling over. Logical + // count only; the buffer is left intact so the invariant holds. + void rollHotStore() + { + numMeshNodes = 1; + clearCachedNode(); + } + private: bool hasCachedNode = false; NodeNum cachedNodeNum = 0; @@ -121,6 +170,9 @@ static void resetTrafficConfig() config = meshtastic_LocalConfig_init_zero; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; + channelFile = meshtastic_ChannelFile_init_zero; + owner.is_licensed = false; + myNodeInfo.my_node_num = kLocalNode; router = nullptr; @@ -175,6 +227,42 @@ static meshtastic_MeshPacket makePositionPacket(NodeNum from, int32_t lat, int32 return packet; } +static meshtastic_MeshPacket makePositionPacketWithPrecision(NodeNum from, int32_t lat, int32_t lon, uint32_t precisionBits) +{ + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_POSITION_APP, from, NODENUM_BROADCAST); + meshtastic_Position pos = meshtastic_Position_init_zero; + pos.has_latitude_i = true; + pos.has_longitude_i = true; + pos.latitude_i = lat; + pos.longitude_i = lon; + pos.precision_bits = precisionBits; + + packet.decoded.payload.size = + pb_encode_to_bytes(packet.decoded.payload.bytes, sizeof(packet.decoded.payload.bytes), &meshtastic_Position_msg, &pos); + return packet; +} + +static bool decodePositionPayload(const meshtastic_MeshPacket &packet, meshtastic_Position &out) +{ + out = meshtastic_Position_init_zero; + return pb_decode_from_bytes(packet.decoded.payload.bytes, packet.decoded.payload.size, &meshtastic_Position_msg, &out); +} + +// Primary channel with a well-known single-byte PSK and the (empty -> preset) +// default name, so Channels::isWellKnownChannel(0) is true. +static void installWellKnownPrimaryChannel() +{ + channelFile = meshtastic_ChannelFile_init_zero; + channelFile.channels_count = 1; + channelFile.channels[0].index = 0; + channelFile.channels[0].has_settings = true; + channelFile.channels[0].role = meshtastic_Channel_Role_PRIMARY; + channelFile.channels[0].settings.psk.size = 1; + channelFile.channels[0].settings.psk.bytes[0] = 1; + config.lora.use_preset = true; + config.lora.modem_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; +} + static meshtastic_MeshPacket makeNodeInfoPacket(NodeNum from, const char *longName, const char *shortName) { meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_NODEINFO_APP, from, NODENUM_BROADCAST); @@ -290,7 +378,7 @@ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 3; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); @@ -305,31 +393,6 @@ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); TEST_ASSERT_TRUE(module.ignoreRequestFlag()); } - -/** - * Verify routing/admin traffic is exempt from rate limiting. - * Important because throttling control traffic can destabilize the mesh. - */ -static void test_tm_rateLimit_skipsRoutingAndAdminPorts(void) -{ - moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 60; - moduleConfig.traffic_management.rate_limit_max_packets = 1; - TrafficManagementModuleTestShim module; - meshtastic_MeshPacket routingPacket = makeDecodedPacket(meshtastic_PortNum_ROUTING_APP, kRemoteNode); - meshtastic_MeshPacket adminPacket = makeDecodedPacket(meshtastic_PortNum_ADMIN_APP, kRemoteNode); - - for (int i = 0; i < 4; i++) { - ProcessMessage rr = module.handleReceived(routingPacket); - ProcessMessage ar = module.handleReceived(adminPacket); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(rr)); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(ar)); - } - - meshtastic_TrafficManagementStats stats = module.getStats(); - TEST_ASSERT_EQUAL_UINT32(0, stats.rate_limit_drops); -} - /** * Verify packets sourced from this node bypass dedup and rate limiting. * Important so local transmissions are not accidentally self-throttled. @@ -650,12 +713,15 @@ static void test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb(voi #endif /** - * Verify relayed telemetry broadcasts are hop-exhausted when enabled. + * Verify relayed telemetry broadcasts are hop-exhausted when enabled AND the + * channel is congested (telemetry exhaustion is gated on channel utilization, + * unlike position exhaustion). * Important to prevent further mesh propagation while still allowing one relay step. */ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -670,6 +736,8 @@ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); } +#if HAS_VARIABLE_HOPS +#endif // HAS_VARIABLE_HOPS /** * Verify hop exhaustion skips unicast and local-origin packets. * Important to avoid mutating traffic that should retain normal forwarding behavior. @@ -677,6 +745,7 @@ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) static void test_tm_alterReceived_skipsLocalAndUnicast(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // congestion satisfied, so only the skip conditions are under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket unicast = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, kTargetNode); @@ -794,8 +863,11 @@ static void test_tm_positionDedup_precision32_allowsDistinctPositions(void) } /** - * Verify invalid precision=0 is treated as full precision. - * Important so invalid config does not collapse all positions into one fingerprint. + * Verify precision=0 falls back to the default precision (same contract as + * >32: getConfiguredOrDefault + sanitizePositionPrecision treat 0 as unset). + * Important so invalid config does not collapse all positions into one + * fingerprint — positions in different default-precision grid cells must + * still be distinct. */ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) { @@ -805,7 +877,7 @@ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); - meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 374221235, -1220845677); + meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 384221234, -1210845678); ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); @@ -857,11 +929,11 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero moduleConfig.traffic_management.rate_limit_max_packets = 10; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket text = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 0x12300000, 0x45600000); meshtastic_MeshPacket duplicate = makePositionPacket(kRemoteNode, 0x12300000, 0x45600000); - ProcessMessage seeded = module.handleReceived(text); + ProcessMessage seeded = module.handleReceived(telemetry); ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(duplicate); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -882,7 +954,7 @@ static void test_tm_rateLimit_resetsAfterWindowExpires(void) moduleConfig.traffic_management.rate_limit_window_secs = 1; moduleConfig.traffic_management.rate_limit_max_packets = 1; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); @@ -895,30 +967,6 @@ static void test_tm_rateLimit_resetsAfterWindowExpires(void) TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); } - -/** - * Verify rate-limit thresholds above 255 effectively clamp to 255. - * Important because counters are uint8_t and must not overflow behavior. - */ -static void test_tm_rateLimit_thresholdAbove255_clamps(void) -{ - moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 60; - moduleConfig.traffic_management.rate_limit_max_packets = 300; - TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); - - for (int i = 0; i < 255; i++) { - ProcessMessage result = module.handleReceived(packet); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(result)); - } - ProcessMessage dropped = module.handleReceived(packet); - meshtastic_TrafficManagementStats stats = module.getStats(); - - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(dropped)); - TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); -} - /** * Verify unknown-packet tracking resets after its active window expires. * Important so old unknown traffic does not trigger delayed drops. @@ -966,12 +1014,14 @@ static void test_tm_unknownPackets_thresholdAbove255_clamps(void) } /** - * Verify relayed position broadcasts can also be hop-exhausted. + * Verify relayed position broadcasts can also be hop-exhausted — under the + * same pressure gate as telemetry (here: channel congestion). * Important because telemetry and position use separate exhaust flags. */ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) { moduleConfig.traffic_management.exhaust_hop_position = true; + ScopedBusyAirTime busyChannel; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makePositionPacket(kRemoteNode, 374221234, -1220845678, NODENUM_BROADCAST); packet.hop_start = 5; @@ -985,7 +1035,6 @@ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) TEST_ASSERT_TRUE(module.shouldExhaustHops(packet)); TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); } - /** * Verify hop exhaustion ignores undecoded/encrypted packets. * Important so we never mutate packets that were not decoded by this module. @@ -993,6 +1042,7 @@ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) static void test_tm_alterReceived_skipsUndecodedPackets(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // congestion satisfied, so only the undecoded skip is under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -1014,6 +1064,7 @@ static void test_tm_alterReceived_skipsUndecodedPackets(void) static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); @@ -1039,6 +1090,7 @@ static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) static void test_tm_alterReceived_exhaustFlag_isPacketScoped(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; meshtastic_MeshPacket exhausted = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); @@ -1083,6 +1135,95 @@ static void test_tm_runOnce_enabledReturnsMaintenanceInterval(void) TEST_ASSERT_EQUAL_INT32(60 * 1000, interval); } +// --------------------------------------------------------------------------- +// Next-hop overflow cache +// --------------------------------------------------------------------------- + +/** + * Round-trip set/get of a confirmed next hop, plus the input guards. + */ +static void test_tm_nextHop_setAndGetRoundTrip(void) +{ + TrafficManagementModuleTestShim module; + + // Unknown node yields no hint. + TEST_ASSERT_EQUAL_UINT8(0, module.getNextHopHint(kTargetNode)); + + // Store a confirmed hop and read it back. + module.setNextHop(kTargetNode, 0x42); + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); + + // Zero dest and zero byte are rejected (no spurious entry created). + module.setNextHop(0, 0x42); + module.setNextHop(kRemoteNode, 0); + TEST_ASSERT_EQUAL_UINT8(0, module.getNextHopHint(kRemoteNode)); + + // Last-write-wins on re-confirmation. + module.setNextHop(kTargetNode, 0x99); + TEST_ASSERT_EQUAL_UINT8(0x99, module.getNextHopHint(kTargetNode)); +} + +/** + * The headline scenario: a node carrying a next hop in the hot NodeInfoLite DB + * is warm-loaded into the TMM cache, then the hot DB is "rolled" (the node ages + * out entirely). The hint must still be served — now exclusively from TMM. + */ +static void test_tm_nextHop_servedAfterNodeDbRoll(void) +{ + TrafficManagementModuleTestShim module; + + // Seed the hot NodeInfoLite DB with a node that has a confirmed next hop. + mockNodeDB->setHotNode(kTargetNode, 0x42); + + // Warm-start the overflow cache from the hot DB. + module.preloadNextHopsFromNodeDB(); + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); + + // Roll the main NodeInfoLite DB: the node is evicted from the hot store. + mockNodeDB->rollHotStore(); + TEST_ASSERT_NULL(nodeDB->getMeshNode(kTargetNode)); // gone from the hot store + + // Hit is still served — proving it now comes from the TMM overflow cache. + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); +} + +/** + * Preload must not clobber a freshly-learned (confirmed) hop with a possibly + * stale persisted one from NodeInfoLite. + */ +static void test_tm_nextHop_preloadDoesNotClobberLearned(void) +{ + TrafficManagementModuleTestShim module; + + // A fresher confirmed hop is already cached. + module.setNextHop(kTargetNode, 0x99); + + // The hot DB carries an older next hop for the same node. + mockNodeDB->setHotNode(kTargetNode, 0x42); + + module.preloadNextHopsFromNodeDB(); + + // The freshly-learned hop survives. + TEST_ASSERT_EQUAL_UINT8(0x99, module.getNextHopHint(kTargetNode)); +} + +/** + * A pure routing hint (no dedup/rate/unknown state) must survive the maintenance + * sweep — next_hop != 0 keeps the slot alive even though it has no TTL. + */ +static void test_tm_nextHop_keptAliveAcrossMaintenanceSweep(void) +{ + TrafficManagementModuleTestShim module; + + module.setNextHop(kTargetNode, 0x42); + + // The sweep frees slots whose sub-stores are all empty; next_hop must veto that. + module.runOnce(); + + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); +} +#if HAS_VARIABLE_HOPS +#endif // HAS_VARIABLE_HOPS } // namespace void setUp(void) @@ -1106,7 +1247,6 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_positionDedup_dropsDuplicateWithinWindow); RUN_TEST(test_tm_positionDedup_allowsMovedPosition); RUN_TEST(test_tm_rateLimit_dropsOnlyAfterThreshold); - RUN_TEST(test_tm_rateLimit_skipsRoutingAndAdminPorts); RUN_TEST(test_tm_fromUs_bypassesPositionAndRateFilters); RUN_TEST(test_tm_localDestination_bypassesTransitFilters); RUN_TEST(test_tm_nodeinfo_routerClamp_skipsWhenTooManyHops); @@ -1121,6 +1261,8 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb); #endif RUN_TEST(test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast); +#if HAS_VARIABLE_HOPS +#endif RUN_TEST(test_tm_alterReceived_skipsLocalAndUnicast); RUN_TEST(test_tm_positionDedup_allowsDuplicateAfterIntervalExpires); RUN_TEST(test_tm_positionDedup_intervalZero_neverDrops); @@ -1130,7 +1272,6 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset); RUN_TEST(test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero); RUN_TEST(test_tm_rateLimit_resetsAfterWindowExpires); - RUN_TEST(test_tm_rateLimit_thresholdAbove255_clamps); RUN_TEST(test_tm_unknownPackets_resetAfterWindowExpires); RUN_TEST(test_tm_unknownPackets_thresholdAbove255_clamps); RUN_TEST(test_tm_alterReceived_exhaustsRelayedPositionBroadcast); @@ -1139,6 +1280,12 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_alterReceived_exhaustFlag_isPacketScoped); RUN_TEST(test_tm_runOnce_disabledReturnsMaxInterval); RUN_TEST(test_tm_runOnce_enabledReturnsMaintenanceInterval); + RUN_TEST(test_tm_nextHop_setAndGetRoundTrip); + RUN_TEST(test_tm_nextHop_servedAfterNodeDbRoll); + RUN_TEST(test_tm_nextHop_preloadDoesNotClobberLearned); + RUN_TEST(test_tm_nextHop_keptAliveAcrossMaintenanceSweep); +#if HAS_VARIABLE_HOPS +#endif exit(UNITY_END()); } From ecccd1f73fb92a31273a415612b407856926eb83 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sat, 13 Jun 2026 08:05:46 +0100 Subject: [PATCH 08/13] TrafficManagement: fix cppcheck constVariablePointer warning MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `node` in preloadNextHopsFromNodeDB() is never written through — mark it const to satisfy cppcheck's constVariablePointer check in CI. Co-Authored-By: Claude Sonnet 4.6 --- src/modules/TrafficManagementModule.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index 318921f2241..661e1019dd7 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -565,7 +565,7 @@ void TrafficManagementModule::preloadNextHopsFromNodeDB() concurrency::LockGuard guard(&cacheLock); const size_t count = nodeDB->getNumMeshNodes(); for (size_t i = 0; i < count; i++) { - meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); + const meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); if (!node || node->num == 0 || node->next_hop == 0) continue; From 554de9e3ac68b9f0435d566514dca0f6bbdf1241 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sat, 13 Jun 2026 23:46:33 +0100 Subject: [PATCH 09/13] housekeeping and interference limiting --- src/mesh/Channels.cpp | 11 +++++++++++ src/mesh/Channels.h | 4 ++++ src/mesh/NextHopRouter.cpp | 4 ++-- src/mesh/NodeDB.cpp | 13 ++++++++++--- src/modules/TrafficManagementModule.cpp | 4 +++- 5 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/mesh/Channels.cpp b/src/mesh/Channels.cpp index be75e3d4214..5c04588c4bb 100644 --- a/src/mesh/Channels.cpp +++ b/src/mesh/Channels.cpp @@ -404,6 +404,17 @@ bool Channels::isDefaultChannel(ChannelIndex chIndex) return false; } +bool Channels::isPublicChannel(ChannelIndex chIndex) +{ + const auto &ch = getByIndex(chIndex); + if (ch.settings.psk.size > 1) + return false; + const char *name = getName(chIndex); + const char *presetName = + DisplayFormatters::getModemPresetDisplayName(config.lora.modem_preset, false, config.lora.use_preset); + return strcmp(name, presetName) == 0; +} + bool Channels::hasDefaultChannel() { // If we don't use a preset or the default frequency slot, or we override the frequency, we don't have a default channel diff --git a/src/mesh/Channels.h b/src/mesh/Channels.h index a3cc7791c7c..ed6638c06c7 100644 --- a/src/mesh/Channels.h +++ b/src/mesh/Channels.h @@ -86,6 +86,10 @@ class Channels // Returns true if the channel has the default name and PSK bool isDefaultChannel(ChannelIndex chIndex); + // Returns true if the channel is public: PSK is 0 or 1 bytes (no key or a well-known + // default-index shorthand) AND the name matches the modem-preset name. + bool isPublicChannel(ChannelIndex chIndex); + // Returns true if we can be reached via a channel with the default settings given a region and modem preset bool hasDefaultChannel(); diff --git a/src/mesh/NextHopRouter.cpp b/src/mesh/NextHopRouter.cpp index 4708faf0c15..c3cb29377a0 100644 --- a/src/mesh/NextHopRouter.cpp +++ b/src/mesh/NextHopRouter.cpp @@ -204,7 +204,7 @@ std::optional NextHopRouter::getNextHop(NodeNum to, uint8_t relay_node) if (node && node->next_hop) { // We are careful not to return the relay node as the next hop if (node->next_hop != relay_node) { - // LOG_DEBUG("Next hop for 0x%x is 0x%x", to, node->next_hop); + LOG_DEBUG("Next hop for 0x%x is 0x%x", to, node->next_hop); return node->next_hop; } else LOG_WARN("Next hop for 0x%x is 0x%x, same as relayer; set no pref", to, node->next_hop); @@ -216,7 +216,7 @@ std::optional NextHopRouter::getNextHop(NodeNum to, uint8_t relay_node) if (trafficManagementModule) { uint8_t hint = trafficManagementModule->getNextHopHint(to); if (hint && hint != relay_node) { - // LOG_DEBUG("Next hop for 0x%x is 0x%x (TMM cache)", to, hint); + LOG_DEBUG("Next hop for 0x%x is 0x%x (TMM cache)", to, hint); return hint; } } diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 5877c6cbff1..e90b91a1786 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -489,6 +489,8 @@ NodeDB::NodeDB() #endif // Include our owner in the node db under our nodenum meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); + if (!info) + return; TypeConversions::CopyUserToNodeInfoLite(info, owner); // If node database has not been saved for the first time, save it now @@ -2832,7 +2834,8 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) // Block the contact and drop its rich satellite data, but keep the // public key copied above — an ignored peer keeps a usable identity // (a verifiable target) rather than a bare node number. - setProtectedFlag(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "ignore", contact.node_num, MAX_NUM_NODES - 2); nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, false); eraseNodeSatellites(contact.node_num); } else { @@ -2853,12 +2856,14 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) } else { // Normal case: set is_favorite to prevent expiration. // last_heard will remain as-is (or remain 0 if this entry wasn't in the nodeDB). - setProtectedFlag(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "favorite", contact.node_num, MAX_NUM_NODES - 2); } // As the clients will begin sending the contact with DMs, we want to strictly check if the node is manually verified if (contact.manually_verified) { - setProtectedFlag(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true); + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "verify", contact.node_num, MAX_NUM_NODES - 2); } // Mark the node's key as manually verified to indicate trustworthiness. updateGUIforNode = info; @@ -3395,6 +3400,8 @@ bool NodeDB::createNewIdentity() myNodeInfo.my_node_num = newNodeNum; meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); + if (!info) + return false; TypeConversions::CopyUserToNodeInfoLite(info, owner); return true; diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index 661e1019dd7..ac120066716 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -2,6 +2,7 @@ #if HAS_TRAFFIC_MANAGEMENT +#include "Channels.h" #include "Default.h" #include "MeshService.h" #include "NodeDB.h" @@ -761,7 +762,8 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack // GPS jitter within the configured precision. if (!isFromUs(&mp) && !isToUs(&mp)) { - if (cfg.position_dedup_enabled && mp.decoded.portnum == meshtastic_PortNum_POSITION_APP) { + if (cfg.position_dedup_enabled && channels.isPublicChannel(mp.channel) && + mp.decoded.portnum == meshtastic_PortNum_POSITION_APP) { meshtastic_Position pos = meshtastic_Position_init_zero; if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_Position_msg, &pos)) { if (shouldDropPosition(&mp, &pos, nowMs)) { From b02585f46ad2009bedd3e5d7ccb561faba5df586 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sun, 14 Jun 2026 00:08:49 +0100 Subject: [PATCH 10/13] betterer --- src/mesh/Channels.cpp | 15 +++++++++++---- src/mesh/Channels.h | 9 ++++++--- src/modules/TrafficManagementModule.cpp | 2 +- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/mesh/Channels.cpp b/src/mesh/Channels.cpp index 5c04588c4bb..34d56e0f7c4 100644 --- a/src/mesh/Channels.cpp +++ b/src/mesh/Channels.cpp @@ -404,15 +404,22 @@ bool Channels::isDefaultChannel(ChannelIndex chIndex) return false; } -bool Channels::isPublicChannel(ChannelIndex chIndex) +bool Channels::isWellKnownChannel(ChannelIndex chIndex) { const auto &ch = getByIndex(chIndex); + // Absent (unencrypted) or single-byte PSK — all the well-known key indexes if (ch.settings.psk.size > 1) return false; + const char *name = getName(chIndex); - const char *presetName = - DisplayFormatters::getModemPresetDisplayName(config.lora.modem_preset, false, config.lora.use_preset); - return strcmp(name, presetName) == 0; + for (int p = _meshtastic_Config_LoRaConfig_ModemPreset_MIN; p <= _meshtastic_Config_LoRaConfig_ModemPreset_MAX; p++) { + const char *presetName = + DisplayFormatters::getModemPresetDisplayName(static_cast(p), false, true); + // Presets without a display name fall through to "Invalid" — never a match + if (strcmp(presetName, "Invalid") != 0 && strcmp(name, presetName) == 0) + return true; + } + return false; } bool Channels::hasDefaultChannel() diff --git a/src/mesh/Channels.h b/src/mesh/Channels.h index ed6638c06c7..2f30d2299fd 100644 --- a/src/mesh/Channels.h +++ b/src/mesh/Channels.h @@ -86,9 +86,12 @@ class Channels // Returns true if the channel has the default name and PSK bool isDefaultChannel(ChannelIndex chIndex); - // Returns true if the channel is public: PSK is 0 or 1 bytes (no key or a well-known - // default-index shorthand) AND the name matches the modem-preset name. - bool isPublicChannel(ChannelIndex chIndex); + // Returns true if the channel is "well known": its PSK is absent or a + // single-byte well-known key index, AND its name is any modem-preset + // display name (e.g. a channel named "LongFast" counts even while the + // radio runs MediumFast). Broader than isDefaultChannel, which only + // matches the current preset's name and PSK byte 1. + bool isWellKnownChannel(ChannelIndex chIndex); // Returns true if we can be reached via a channel with the default settings given a region and modem preset bool hasDefaultChannel(); diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index ac120066716..859af3cf240 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -762,7 +762,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack // GPS jitter within the configured precision. if (!isFromUs(&mp) && !isToUs(&mp)) { - if (cfg.position_dedup_enabled && channels.isPublicChannel(mp.channel) && + if (cfg.position_dedup_enabled && channels.isWellKnownChannel(mp.channel) && mp.decoded.portnum == meshtastic_PortNum_POSITION_APP) { meshtastic_Position pos = meshtastic_Position_init_zero; if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_Position_msg, &pos)) { From e27fa47858c6bdfebeb1681d4a7730d556d81166 Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sun, 14 Jun 2026 03:42:58 +0100 Subject: [PATCH 11/13] simplify --- src/mesh/Default.h | 4 +- src/modules/TrafficManagementModule.cpp | 240 +++++++++------------ src/modules/TrafficManagementModule.h | 177 +++++---------- test/test_traffic_management/test_main.cpp | 25 +-- userPrefs.jsonc | 1 + 5 files changed, 173 insertions(+), 274 deletions(-) diff --git a/src/mesh/Default.h b/src/mesh/Default.h index d638ca2a936..38842ad751b 100644 --- a/src/mesh/Default.h +++ b/src/mesh/Default.h @@ -34,8 +34,8 @@ enum class TrafficType { POSITION, TELEMETRY }; // Traffic management defaults -#define default_traffic_mgmt_position_precision_bits 19 // ~90m grid cells (±45m) -#define default_traffic_mgmt_position_min_interval_secs (ONE_DAY / 2) // 12 hours between identical positions +#define default_traffic_mgmt_position_precision_bits 19 // ~90m grid cells (±45m) +#define default_traffic_mgmt_position_min_interval_secs (11 * 60 * 60) // 11 hours between identical positions // Hop scaling defaults #define default_hop_scaling_min_target_nodes 40 // walk threshold: first hop reaching this cumulative count diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index 859af3cf240..1edc07b11c8 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -6,6 +6,7 @@ #include "Default.h" #include "MeshService.h" #include "NodeDB.h" +#include "PositionPrecision.h" #include "Router.h" #include "TypeConversions.h" #include "airtime.h" @@ -14,6 +15,7 @@ #include "mesh-pb-constants.h" #include "meshUtils.h" #include +#include #include #define TM_LOG_DEBUG(fmt, ...) LOG_DEBUG("[TM] " fmt, ##__VA_ARGS__) @@ -28,7 +30,6 @@ namespace { constexpr uint32_t kMaintenanceIntervalMs = 60 * 1000UL; // Cache cleanup interval -constexpr uint32_t kUnknownResetMs = 60 * 1000UL; // Unknown packet window // NodeInfo direct response: enforced maximum hops by device role // Both use maxHops logic (respond when hopsAway <= threshold) @@ -65,32 +66,6 @@ uint8_t sanitizePositionPrecision(uint8_t precision) return 32; } -/** - * Check if a timestamp is within a time window. - * Handles wrap-around correctly using unsigned subtraction. - */ -bool isWithinWindow(uint32_t nowMs, uint32_t startMs, uint32_t intervalMs) -{ - if (intervalMs == 0 || startMs == 0) - return false; - return (nowMs - startMs) < intervalMs; -} - -/** - * Slide an 8-bit relative timestamp back by a wall-clock slab during epoch rebase. - * - * Entries older than the slab clamp to 0 (then reclaimed by the maintenance sweep); - * live entries keep their reconstructed age minus a sub-tick remainder. Each field - * slides by its own resolution's worth of ticks, so a single slab covers all three. - */ -inline void slideRelativeTime(uint8_t &ticks, uint32_t slabMs, uint16_t resolutionSecs) -{ - if (ticks == 0 || resolutionSecs == 0) - return; - uint32_t dec = slabMs / (static_cast(resolutionSecs) * 1000UL); - ticks = (ticks > dec) ? static_cast(ticks - dec) : 0; -} - /** * Truncate lat/lon to specified precision for position deduplication. * @@ -177,23 +152,11 @@ TrafficManagementModule::TrafficManagementModule() : MeshModule("TrafficManageme encryptedOk = true; // Can process encrypted packets stats = meshtastic_TrafficManagementStats_init_zero; - // Initialize rolling epoch for relative timestamps - cacheEpochMs = millis(); - - // Calculate adaptive time resolutions from config (config changes require reboot) - // Resolution = max(60, min(339, interval/2)) for ~24 hour range with good precision - posTimeResolution = calcTimeResolution(Default::getConfiguredOrDefault( - moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); - rateTimeResolution = calcTimeResolution(moduleConfig.traffic_management.rate_limit_window_secs); - unknownTimeResolution = calcTimeResolution(kUnknownResetMs / 1000); // ~5 min default - const auto &cfg = moduleConfig.traffic_management; TM_LOG_INFO("Enabled: pos_dedup=%d nodeinfo_resp=%d rate_limit=%d drop_unknown=%d exhaust_telem=%d exhaust_pos=%d " "preserve_hops=%d", cfg.position_dedup_enabled, cfg.nodeinfo_direct_response, cfg.rate_limit_enabled, cfg.drop_unknown_enabled, cfg.exhaust_hop_telemetry, cfg.exhaust_hop_position, cfg.router_preserve_hops); - TM_LOG_DEBUG("Time resolutions: pos=%us, rate=%us, unknown=%us", posTimeResolution, rateTimeResolution, - unknownTimeResolution); // Allocate unified cache (10 bytes/entry for all platforms) #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 @@ -357,11 +320,20 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreat if (empty) continue; // an empty slot beats any victim; stop scoring const bool hasHop = e.next_hop != 0; - uint8_t recency = e.pos_time; - if (e.rate_time > recency) - recency = e.rate_time; - if (e.unknown_time > recency) - recency = e.unknown_time; + // Age in pos-ticks (8-bit modular, wraps correctly). Entries with no + // pos state (pos_time==0) score as maximally old (age=currentPosTick()). + const uint8_t nowPosTick = currentPosTick(); + const uint8_t posAge = static_cast(nowPosTick - e.pos_time); + // Blend in rate/unknown ages scaled to pos-tick units (coarser = conservative). + const uint8_t rateAgePosScale = + static_cast(static_cast((currentRateTick() - e.getRateTime()) & 0x0F) * 5 / 3); + const uint8_t unknownAgePosScale = + static_cast(static_cast((currentUnknownTick() - e.getUnknownTime()) & 0x0F) / 6); + uint8_t recency = posAge; + if (e.rate_count != 0 && rateAgePosScale > recency) + recency = rateAgePosScale; + if (e.unknown_count != 0 && unknownAgePosScale > recency) + recency = unknownAgePosScale; if (!victim || (hasHop == victimHasHop ? recency < victimRecency : !hasHop)) { victim = &e; victimHasHop = hasHop; @@ -587,59 +559,11 @@ void TrafficManagementModule::preloadNextHopsFromNodeDB() // Epoch Management // ============================================================================= -/** - * Reset the timestamp epoch when relative offsets approach overflow. - * - * Called when epoch age exceeds ~19 hours (approaching 8-bit minute overflow). - * Invalidates all cached per-node traffic state. - */ -void TrafficManagementModule::resetEpoch(uint32_t nowMs) +void TrafficManagementModule::flushCache() { #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - TM_LOG_DEBUG("Resetting cache epoch"); - cacheEpochMs = nowMs; - - // Full flush avoids stale dedup identity/counters surviving epoch rollover. + TM_LOG_DEBUG("Flushing cache"); memset(cache, 0, static_cast(cacheSize()) * sizeof(UnifiedCacheEntry)); -#else - (void)nowMs; -#endif -} - -/** - * Sliding-epoch rebase — preserve cached state past the 8-bit timestamp horizon. - * - * Instead of flushing the whole cache when offsets approach overflow, advance the - * epoch by a fixed slab and shift every live entry's relative timestamps back by - * the same wall-clock amount. A valid entry's window is only a handful of ticks - * wide (TTL auto-scales with resolution), so live entries comfortably survive; - * already-expired entries clamp to 0 and are reclaimed by the maintenance sweep in - * the same locked pass. Reconstructed absolute time is preserved (minus a sub-tick - * remainder), so in-flight TTL checks remain correct across the rebase. - * - * Caller must hold cacheLock. - */ -void TrafficManagementModule::rebaseEpoch(uint32_t nowMs) -{ -#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - (void)nowMs; - - // Slab stays well below the 200-tick reset threshold so a single rebase drops - // the offset back into range (~200 -> ~72 ticks) while live entries survive. - const uint32_t slabMs = 128UL * maxResolution() * 1000UL; - cacheEpochMs += slabMs; - - TM_LOG_DEBUG("Rebasing cache epoch by %lus", static_cast(slabMs / 1000UL)); - - for (uint16_t i = 0; i < cacheSize(); i++) { - if (cache[i].node == 0) - continue; - slideRelativeTime(cache[i].pos_time, slabMs, posTimeResolution); - slideRelativeTime(cache[i].rate_time, slabMs, rateTimeResolution); - slideRelativeTime(cache[i].unknown_time, slabMs, unknownTimeResolution); - } -#else - (void)nowMs; #endif } @@ -822,6 +746,39 @@ void TrafficManagementModule::alterReceived(meshtastic_MeshPacket &mp) const bool shouldExhaust = ((channelBusy && isTelemetry && cfg.exhaust_hop_telemetry) || (isPosition && cfg.exhaust_hop_position)); + // ------------------------------------------------------------------------- + // Relayed Position Precision Clamp + // ------------------------------------------------------------------------- + // Clamp relayed position broadcasts to the channel's configured precision + // ceiling. Guards against forwarding more-precise coordinates than the + // channel is intended to carry (e.g. a LongFast channel set to 13-bit / + // ~1.5 km). chanPrec==0 means position sharing is disabled on the channel; + // skip — not our job to zero positions on relay. + // Ham mode (owner.is_licensed) is exempt. + // Compile USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS to extend to private channels. + if (!owner.is_licensed && isPosition && isBroadcast(mp.to)) { +#ifdef USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS + const bool shouldClamp = true; +#else + const bool shouldClamp = channels.isWellKnownChannel(mp.channel); +#endif + if (shouldClamp) { + const uint32_t chanPrec = getPositionPrecisionForChannel(mp.channel); + if (chanPrec > 0) { + meshtastic_Position pos = meshtastic_Position_init_default; + if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_Position_msg, &pos)) { + const uint32_t packetPrec = pos.precision_bits > 0 ? pos.precision_bits : 32u; + if (packetPrec > chanPrec) { + applyPositionPrecision(pos, chanPrec); + mp.decoded.payload.size = pb_encode_to_bytes(mp.decoded.payload.bytes, sizeof(mp.decoded.payload.bytes), + &meshtastic_Position_msg, &pos); + logAction("clamp", &mp, "precision"); + } + } + } + } + } + if (!shouldExhaust || !isBroadcast(mp.to)) return; @@ -864,39 +821,42 @@ int32_t TrafficManagementModule::runOnce() nextHopPreloaded = true; } - // Calculate TTLs for cache expiration + // Free-running tick counters (no epoch needed). + // TTL expressed in ticks: pos 4× interval (default 44 ticks ≈ 4.4h), + // rate 2× window (default 24 ticks = 2h), unknown fixed 12 ticks (12 min). const uint32_t positionIntervalMs = secsToMs(Default::getConfiguredOrDefault( moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); - const uint32_t positionTtlMs = positionIntervalMs * 4; + const uint8_t posTtlTicks = + static_cast(std::min(static_cast(255), (positionIntervalMs * 4) / kPosTimeTickMs)); - const uint32_t rateIntervalMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); - const uint32_t rateTtlMs = (rateIntervalMs > 0) ? rateIntervalMs * 2 : (10 * 60 * 1000UL); + const uint32_t rateWindowMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); + const uint8_t rateTtlTicks = static_cast( + std::min(static_cast(15), (rateWindowMs > 0 ? rateWindowMs * 2 : 24 * kRateTimeTickMs) / kRateTimeTickMs)); - const uint32_t unknownTtlMs = kUnknownResetMs * 5; + // unknown: fixed 12-tick TTL (12 min — 4 ticks past the 5-min default window) + const uint8_t unknownTtlTicks = 12; + + const uint8_t nowPosTick = currentPosTick(); + const uint8_t nowRateTick = currentRateTick(); + const uint8_t nowUnknownTick = currentUnknownTick(); // Sweep cache and clear expired entries uint16_t activeEntries = 0; uint16_t expiredEntries = 0; const uint32_t sweepStartMs = millis(); + const auto &cfg = moduleConfig.traffic_management; concurrency::LockGuard guard(&cacheLock); - // Slide the epoch instead of flushing when offsets approach 8-bit overflow. - // Rebase preserves live entries; only already-expired ones clamp to 0 and are - // reclaimed by the sweep below in this same locked pass. - if (needsEpochReset(nowMs)) - rebaseEpoch(nowMs); - for (uint16_t i = 0; i < cacheSize(); i++) { if (cache[i].node == 0) continue; bool anyValid = false; - // Check and clear expired position data - if (cache[i].pos_time != 0) { - uint32_t posTimeMs = fromRelativePosTime(cache[i].pos_time); - if (!isWithinWindow(nowMs, posTimeMs, positionTtlMs)) { + // Check and clear expired position data (presence: pos_fingerprint != 0) + if (cache[i].pos_fingerprint != 0) { + if (static_cast(nowPosTick - cache[i].pos_time) >= posTtlTicks) { cache[i].pos_fingerprint = 0; cache[i].pos_time = 0; } else { @@ -904,23 +864,21 @@ int32_t TrafficManagementModule::runOnce() } } - // Check and clear expired rate limit data - if (cache[i].rate_time != 0) { - uint32_t rateTimeMs = fromRelativeRateTime(cache[i].rate_time); - if (!isWithinWindow(nowMs, rateTimeMs, rateTtlMs)) { + // Check and clear expired rate limit data (gated on feature; presence: rate_count != 0) + if (cfg.rate_limit_enabled && cache[i].rate_count != 0) { + if ((static_cast(nowRateTick - cache[i].getRateTime()) & 0x0F) >= rateTtlTicks) { cache[i].rate_count = 0; - cache[i].rate_time = 0; + cache[i].setRateTime(0); } else { anyValid = true; } } - // Check and clear expired unknown tracking data - if (cache[i].unknown_time != 0) { - uint32_t unknownTimeMs = fromRelativeUnknownTime(cache[i].unknown_time); - if (!isWithinWindow(nowMs, unknownTimeMs, unknownTtlMs)) { + // Check and clear expired unknown tracking data (gated on feature; presence: unknown_count != 0) + if (cfg.drop_unknown_enabled && cache[i].unknown_count != 0) { + if ((static_cast(nowUnknownTick - cache[i].getUnknownTime()) & 0x0F) >= unknownTtlTicks) { cache[i].unknown_count = 0; - cache[i].unknown_time = 0; + cache[i].setUnknownTime(0); } else { anyValid = true; } @@ -990,19 +948,26 @@ bool TrafficManagementModule::shouldDropPosition(const meshtastic_MeshPacket *p, if (!entry) return false; - // Compare fingerprint and check time window - // When minIntervalMs == 0, deduplication is disabled (withinInterval = false means never drop) - const bool hasPositionState = !isNew && entry->pos_time != 0; + // Compare fingerprint and check time window. + // When minIntervalMs == 0, deduplication is disabled (withinInterval = false means never drop). + // Presence: pos_fingerprint != 0 (zero fingerprint from real coordinates is astronomically unlikely). + const bool hasPositionState = !isNew && entry->pos_fingerprint != 0; const bool samePosition = hasPositionState && entry->pos_fingerprint == fingerprint; + const uint8_t nowPosTick = currentPosTick(); + // Clamp to [1, 255]: intervals shorter than one tick still dedup within the same tick. + const uint8_t windowTicks = + (minIntervalMs == 0) ? 0 + : static_cast(std::min(static_cast(UINT8_MAX), + std::max(static_cast(1), minIntervalMs / kPosTimeTickMs))); const bool withinInterval = - hasPositionState && (minIntervalMs != 0) && isWithinWindow(nowMs, fromRelativePosTime(entry->pos_time), minIntervalMs); + hasPositionState && (windowTicks != 0) && (static_cast(nowPosTick - entry->pos_time) < windowTicks); TM_LOG_DEBUG("Position dedup 0x%08x: fp=0x%02x prev=0x%02x same=%d within=%d new=%d", p->from, fingerprint, entry->pos_fingerprint, samePosition, withinInterval, isNew); - // Update cache entry + // Update cache entry (raw tick; 0 is a valid tick value) entry->pos_fingerprint = fingerprint; - entry->pos_time = toRelativePosTime(nowMs); + entry->pos_time = nowPosTick; // Drop only if same position AND within the minimum interval return samePosition && withinInterval; @@ -1156,9 +1121,14 @@ bool TrafficManagementModule::isRateLimited(NodeNum from, uint32_t nowMs) if (!entry) return false; - // Check if window has expired - if (isNew || !isWithinWindow(nowMs, fromRelativeRateTime(entry->rate_time), windowMs)) { - entry->rate_time = toRelativeRateTime(nowMs); + // Window ticks: clamp to [1,15] so zero windowMs (config error) opens a new window. + const uint8_t windowTicks = static_cast(std::min(static_cast(15), windowMs / kRateTimeTickMs)); + const uint8_t nowRateTick = currentRateTick(); + const bool windowExpired = + isNew || entry->rate_count == 0 || + ((static_cast(nowRateTick - entry->getRateTime()) & 0x0F) >= std::max(static_cast(1), windowTicks)); + if (windowExpired) { + entry->setRateTime(nowRateTick); entry->rate_count = 1; return false; } @@ -1190,9 +1160,8 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (!moduleConfig.traffic_management.drop_unknown_enabled || moduleConfig.traffic_management.unknown_packet_threshold == 0) return false; - uint32_t windowMs = kUnknownResetMs; - if (moduleConfig.traffic_management.rate_limit_window_secs > 0) - windowMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); + // Fixed 5-tick (5 min) unknown window; capped at 12 ticks (12 min max). + static constexpr uint8_t kUnknownWindowTicks = 5; bool isNew = false; concurrency::LockGuard guard(&cacheLock); @@ -1200,9 +1169,12 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (!entry) return false; - // Check if window has expired - if (isNew || !isWithinWindow(nowMs, fromRelativeUnknownTime(entry->unknown_time), windowMs)) { - entry->unknown_time = toRelativeUnknownTime(nowMs); + // Check if window has expired (presence: unknown_count != 0) + const uint8_t nowUnknownTick = currentUnknownTick(); + const bool windowExpired = isNew || entry->unknown_count == 0 || + ((static_cast(nowUnknownTick - entry->getUnknownTime()) & 0x0F) >= kUnknownWindowTicks); + if (windowExpired) { + entry->setUnknownTime(nowUnknownTick); entry->unknown_count = 0; } diff --git a/src/modules/TrafficManagementModule.h b/src/modules/TrafficManagementModule.h index 22d1df5fe46..a38f7115200 100644 --- a/src/modules/TrafficManagementModule.h +++ b/src/modules/TrafficManagementModule.h @@ -22,9 +22,10 @@ * Memory Optimization: * Uses one flat unified cache (plain array, linear scan) shared by all * per-node features instead of separate per-feature caches. Timestamps are - * stored as 8-bit relative offsets from a rolling epoch to further reduce - * memory footprint. LoRa packet rates are low enough that an O(n) scan of - * ~1000 11-byte entries is negligible next to packet processing. + * stored as free-running modular tick counters (pos: 8-bit 360 s/tick; + * rate+unknown: paired 4-bit nibbles in one byte) for a 10-byte entry. + * LoRa packet rates are low enough that an O(n) scan of ~1000 entries is + * negligible next to packet processing. */ class TrafficManagementModule : public MeshModule, private concurrency::OSThread { @@ -68,69 +69,46 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread bool wantPacket(const meshtastic_MeshPacket *p) override { return true; } void alterReceived(meshtastic_MeshPacket &mp) override; int32_t runOnce() override; - // Protected so test shims can force epoch rollover behavior. - void resetEpoch(uint32_t nowMs); - // Sliding-epoch rebase: advance the epoch and shift live entries back by the - // same wall-clock amount instead of flushing, so cached state survives past the - // ~19h horizon. Caller must hold cacheLock. - void rebaseEpoch(uint32_t nowMs); + // Protected so test shims can flush per-node traffic state. + void flushCache(); private: // ========================================================================= - // Unified Cache Entry (11 bytes) - Same for ALL platforms + // Unified Cache Entry (10 bytes) - Same for ALL platforms // ========================================================================= // - // A single compact structure used across ESP32, NRF52, and all other platforms. - // Memory: 11 bytes × 2048 entries = 22KB - // - // Position Fingerprinting: - // Instead of storing full coordinates (8 bytes) or a computed hash, - // we store an 8-bit fingerprint derived deterministically from the - // truncated lat/lon. This extracts the lower 4 significant bits from - // each coordinate: fingerprint = (lat_low4 << 4) | lon_low4 - // - // Benefits over hash: - // - Adjacent grid cells have sequential fingerprints (no collision) - // - Two positions only collide if 16+ grid cells apart in BOTH dimensions - // - Deterministic: same input always produces same output - // - // Adaptive Timestamp Resolution: - // All timestamps use 8-bit values with adaptive resolution calculated - // from config at startup. Resolution = max(60, min(339, interval/2)). - // - Min 60 seconds ensures reasonable precision - // - Max 339 seconds allows ~24 hour range (255 * 339 = 86445 sec) - // - interval/2 ensures at least 2 ticks per configured interval - // // Layout: - // [0-3] node - NodeNum (4 bytes) - // [4] pos_fingerprint - 4 bits lat + 4 bits lon (1 byte) - // [5] rate_count - Packets in current window (1 byte) - // [6] unknown_count - Unknown packets count (1 byte) - // [7] pos_time - Position timestamp (1 byte, adaptive resolution) - // [8] rate_time - Rate window start (1 byte, adaptive resolution) - // [9] unknown_time - Unknown tracking start (1 byte, adaptive resolution) - // [10] next_hop - Last-byte relay to reach `node` (1 byte, 0 = none) + // [0-3] node - NodeNum (4 bytes, 0 = empty slot) + // [4] pos_fingerprint - 4 bits lat + 4 bits lon (0 = no position seen) + // [5] rate_count - Packets in current rate window (0 = no window active) + // [6] unknown_count - Unknown packets in window (0 = no window active) + // [7] pos_time - Position tick (uint8, free-running 360 s/tick) + // [8] rate_unknown_time - [7:4] rate nibble (300 s/tick) | [3:0] unknown nibble (60 s/tick) + // [9] next_hop - Last-byte relay to reach `node` (0 = none) + // + // Presence sentinels (no epoch, no +1 offset needed): + // pos active: pos_fingerprint != 0 + // rate active: rate_count != 0 + // unknown active: unknown_count != 0 // - // next_hop semantics: - // A routing hint: the last byte of the NodeNum to use as next hop to reach - // `node`. Written ONLY from NextHopRouter's ACK-confirmed decision (a - // bidirectionally-verified relay), never inferred one-way from relayed - // traffic. The TMM cache acts as an overflow store for confirmed next-hops - // that have aged out of the hot NodeDB (NodeInfoLite). Unlike the other - // fields it has no TTL of its own — it keeps its slot alive (see runOnce) - // and is refreshed only on the next confirmed exchange. + // next_hop: routing hint written only from ACK-confirmed NextHopRouter decisions. + // No TTL — keeps the slot alive across maintenance sweeps. // struct __attribute__((packed)) UnifiedCacheEntry { - NodeNum node; // 4 bytes - Node identifier (0 = empty slot) - uint8_t pos_fingerprint; // 1 byte - Lower 4 bits of lat + lon - uint8_t rate_count; // 1 byte - Packet count (saturates at 255) - uint8_t unknown_count; // 1 byte - Unknown packet count (saturates at 255) - uint8_t pos_time; // 1 byte - Position timestamp (adaptive resolution) - uint8_t rate_time; // 1 byte - Rate window start (adaptive resolution) - uint8_t unknown_time; // 1 byte - Unknown tracking start (adaptive resolution) - uint8_t next_hop; // 1 byte - Last-byte relay to reach `node` (0 = none). See note below. + NodeNum node; + uint8_t pos_fingerprint; + uint8_t rate_count; + uint8_t unknown_count; + uint8_t pos_time; + uint8_t rate_unknown_time; + uint8_t next_hop; + + uint8_t getRateTime() const { return (rate_unknown_time >> 4) & 0x0F; } + uint8_t getUnknownTime() const { return rate_unknown_time & 0x0F; } + void setRateTime(uint8_t t) { rate_unknown_time = static_cast((rate_unknown_time & 0x0F) | ((t & 0x0F) << 4)); } + void setUnknownTime(uint8_t t) { rate_unknown_time = static_cast((rate_unknown_time & 0xF0) | (t & 0x0F)); } }; - static_assert(sizeof(UnifiedCacheEntry) == 11, "UnifiedCacheEntry should be 11 bytes"); + static_assert(sizeof(UnifiedCacheEntry) == 10, "UnifiedCacheEntry should be 10 bytes"); // ========================================================================= // Flat unified cache @@ -151,82 +129,29 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread static constexpr uint16_t nodeInfoTargetEntries() { return kNodeInfoCacheEntries; } // ========================================================================= - // Adaptive Timestamp Resolution + // Free-Running Tick Counters // ========================================================================= // - // All timestamps use 8-bit values with adaptive resolution calculated from - // config at startup. This allows ~24 hour range while maintaining precision. + // Timestamps are stored as free-running modular tick counters derived from + // millis(). No epoch anchor needed: modular subtraction gives correct age + // as long as the true age stays below the counter period. // - // Resolution formula: max(60, min(339, interval/2)) - // - 60 sec minimum ensures reasonable precision - // - 339 sec maximum allows 24 hour range (255 * 339 ≈ 86400 sec) - // - interval/2 ensures at least 2 ticks per configured interval + // pos_time : uint8 (256 ticks × 360 s = 25.6 h period; max window 12 h = 120 ticks) + // rate_time : nibble (16 ticks × 300 s = 80 min period; max window 1 h = 12 ticks) + // unknown_time: nibble (16 ticks × 60 s = 16 min period; max window 12 min = 12 ticks) // - // Since config changes require reboot, resolution is calculated once. - // - uint32_t cacheEpochMs = 0; - uint16_t posTimeResolution = 60; // Seconds per tick for position - uint16_t rateTimeResolution = 60; // Seconds per tick for rate limiting - uint16_t unknownTimeResolution = 60; // Seconds per tick for unknown tracking - - // Calculate resolution from configured interval (called once at startup) - static uint16_t calcTimeResolution(uint32_t intervalSecs) - { - // Resolution = interval/2 to ensure at least 2 ticks per interval - // Clamped to [60, 339] for min precision and max 24h range - uint32_t res = (intervalSecs > 0) ? (intervalSecs / 2) : 60; - if (res < 60) - res = 60; - if (res > 339) - res = 339; - return static_cast(res); - } - - // Convert to/from 8-bit relative timestamps with given resolution. + // Presence sentinels (no +1 offset needed; count fields serve as guards): + // pos active: pos_fingerprint != 0 (zero fingerprint astronomically unlikely) + // rate active: rate_count != 0 + // unknown active: unknown_count != 0 // - // All stored timestamps carry a uniform +1 "presence" offset: a value of 0 is - // reserved for "no timestamp recorded" (which is also the zero-initialized - // state), and stored values 1..255 encode raw ticks 0..254. This keeps the - // 0-means-empty sentinel consistent with memset/calloc zeroing across every - // sub-store, so the maintenance sweep's `_time != 0` presence checks are - // unambiguous (a timestamp recorded in the first tick after the epoch is no - // longer mistaken for an empty slot). The offset is applied here and removed - // on read, so it cancels out in all window math. - uint8_t toRelativeTime(uint32_t nowMs, uint16_t resolutionSecs) const - { - uint32_t ticks = (nowMs - cacheEpochMs) / (resolutionSecs * 1000UL); - return (ticks >= UINT8_MAX) ? UINT8_MAX : static_cast(ticks + 1); - } - uint32_t fromRelativeTime(uint8_t ticks, uint16_t resolutionSecs) const - { - return (ticks == 0) ? cacheEpochMs : cacheEpochMs + (static_cast(ticks - 1) * resolutionSecs * 1000UL); - } - - // Convenience wrappers for each timestamp type (the +1 presence offset lives - // in the shared converters above, so these are plain pass-throughs). - uint8_t toRelativePosTime(uint32_t nowMs) const { return toRelativeTime(nowMs, posTimeResolution); } - uint32_t fromRelativePosTime(uint8_t t) const { return fromRelativeTime(t, posTimeResolution); } - - uint8_t toRelativeRateTime(uint32_t nowMs) const { return toRelativeTime(nowMs, rateTimeResolution); } - uint32_t fromRelativeRateTime(uint8_t t) const { return fromRelativeTime(t, rateTimeResolution); } - - uint8_t toRelativeUnknownTime(uint32_t nowMs) const { return toRelativeTime(nowMs, unknownTimeResolution); } - uint32_t fromRelativeUnknownTime(uint8_t t) const { return fromRelativeTime(t, unknownTimeResolution); } - - // Coarsest of the per-feature resolutions (seconds per tick). - uint16_t maxResolution() const - { - uint16_t maxRes = posTimeResolution; - if (rateTimeResolution > maxRes) - maxRes = rateTimeResolution; - if (unknownTimeResolution > maxRes) - maxRes = unknownTimeResolution; - return maxRes; - } + static constexpr uint32_t kPosTimeTickMs = 360'000UL; // 6 min/tick + static constexpr uint32_t kRateTimeTickMs = 300'000UL; // 5 min/tick + static constexpr uint32_t kUnknownTimeTickMs = 60'000UL; // 1 min/tick - // True when relative offsets approach 8-bit overflow. - // With max resolution of 339 sec, 200 ticks = ~19 hours (safe margin for 24h max). - bool needsEpochReset(uint32_t nowMs) const { return (nowMs - cacheEpochMs) > (200UL * maxResolution() * 1000UL); } + static uint8_t currentPosTick() { return static_cast(millis() / kPosTimeTickMs); } + static uint8_t currentRateTick() { return static_cast((millis() / kRateTimeTickMs) & 0x0F); } + static uint8_t currentUnknownTick() { return static_cast((millis() / kUnknownTimeTickMs) & 0x0F); } // ========================================================================= // Position Fingerprint // ========================================================================= diff --git a/test/test_traffic_management/test_main.cpp b/test/test_traffic_management/test_main.cpp index c75e0da9000..30da8bcfad5 100644 --- a/test/test_traffic_management/test_main.cpp +++ b/test/test_traffic_management/test_main.cpp @@ -151,8 +151,8 @@ class TrafficManagementModuleTestShim : public TrafficManagementModule { public: using TrafficManagementModule::alterReceived; + using TrafficManagementModule::flushCache; using TrafficManagementModule::handleReceived; - using TrafficManagementModule::resetEpoch; using TrafficManagementModule::runOnce; bool ignoreRequestFlag() const { return ignoreRequest; } @@ -774,7 +774,8 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) { moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; - moduleConfig.traffic_management.position_min_interval_secs = 1; + // 360 s = 1 pos-tick (kPosTimeTickMs); testDelay advances past one tick period. + moduleConfig.traffic_management.position_min_interval_secs = 360; TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -783,7 +784,7 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); - testDelay(1200); + testDelay(360001); // advance past one 6-min pos-tick ProcessMessage r3 = module.handleReceived(third); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -892,7 +893,7 @@ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) * Verify epoch reset invalidates stale position identity for dedup. * Important so reset paths cannot leak prior packet identity into new windows. */ -static void test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset(void) +static void test_tm_positionDedup_cacheFlush_doesNotDropFirstPacketAfterFlush(void) { moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; @@ -900,12 +901,12 @@ static void test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset(vo TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); - meshtastic_MeshPacket afterReset = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterFlush = makePositionPacket(kRemoteNode, 374221234, -1220845678); meshtastic_MeshPacket duplicate = makePositionPacket(kRemoteNode, 374221234, -1220845678); ProcessMessage r1 = module.handleReceived(first); - module.resetEpoch(millis()); - ProcessMessage r2 = module.handleReceived(afterReset); + module.flushCache(); + ProcessMessage r2 = module.handleReceived(afterFlush); ProcessMessage r3 = module.handleReceived(duplicate); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -951,14 +952,15 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero static void test_tm_rateLimit_resetsAfterWindowExpires(void) { moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 1; + // 300 s = 1 rate-tick (kRateTimeTickMs); testDelay advances past one tick period. + moduleConfig.traffic_management.rate_limit_window_secs = 300; moduleConfig.traffic_management.rate_limit_max_packets = 1; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(1200); + testDelay(300001); // advance past one 5-min rate-tick ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -975,13 +977,12 @@ static void test_tm_unknownPackets_resetAfterWindowExpires(void) { moduleConfig.traffic_management.drop_unknown_enabled = true; moduleConfig.traffic_management.unknown_packet_threshold = 1; - moduleConfig.traffic_management.rate_limit_window_secs = 1; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(1200); + testDelay(300001); // advance past 5 unknown-ticks (kUnknownWindowTicks=5 × 60s) ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -1269,7 +1270,7 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_positionDedup_precisionAbove32_usesDefaultPrecision); RUN_TEST(test_tm_positionDedup_precision32_allowsDistinctPositions); RUN_TEST(test_tm_positionDedup_precisionZero_allowsDistinctPositions); - RUN_TEST(test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset); + RUN_TEST(test_tm_positionDedup_cacheFlush_doesNotDropFirstPacketAfterFlush); RUN_TEST(test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero); RUN_TEST(test_tm_rateLimit_resetsAfterWindowExpires); RUN_TEST(test_tm_unknownPackets_resetAfterWindowExpires); diff --git a/userPrefs.jsonc b/userPrefs.jsonc index 57ede8bbf91..118588e195c 100644 --- a/userPrefs.jsonc +++ b/userPrefs.jsonc @@ -24,6 +24,7 @@ // "USERPREFS_CONFIG_OWNER_SHORT_NAME": "MLN", // "USERPREFS_CONFIG_DEVICE_ROLE": "meshtastic_Config_DeviceConfig_Role_CLIENT", // Defaults to CLIENT. ROUTER*, and LOST AND FOUND roles are restricted. // "USERPREFS_EVENT_MODE": "1", + // "USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS": "1", // Extend TMM position dedup and precision clamping to private/custom-key channels (default: well-known channels only) // "USERPREFS_FIRMWARE_EDITION": "meshtastic_FirmwareEdition_BURNING_MAN", // "USERPREFS_FIXED_BLUETOOTH": "121212", // "USERPREFS_FIXED_GPS": "", From c99668095c78aad8c49bc3b78834e7953cefaa0a Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sun, 14 Jun 2026 09:11:35 +0100 Subject: [PATCH 12/13] test(traffic_management): exercise dedup path in position-dedup tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Position dedup in TrafficManagementModule::handleReceived is gated on channels.isWellKnownChannel(mp.channel). The test helper installWellKnownPrimaryChannel() sets up channelFile/config.lora so that gate is true, but it was defined and never called — so the dedup path was never reached. test_tm_positionDedup_dropsDuplicateWithinWindow therefore failed (duplicate forwarded -> CONTINUE instead of STOP), and test_tm_positionDedup_allowsMovedPosition passed only vacuously. Call installWellKnownPrimaryChannel() in both dedup tests so the dedup path is genuinely exercised. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_traffic_management/test_main.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_traffic_management/test_main.cpp b/test/test_traffic_management/test_main.cpp index 30da8bcfad5..1b02a9c3ed4 100644 --- a/test/test_traffic_management/test_main.cpp +++ b/test/test_traffic_management/test_main.cpp @@ -329,6 +329,7 @@ static void test_tm_positionDedup_dropsDuplicateWithinWindow(void) moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -354,6 +355,7 @@ static void test_tm_positionDedup_allowsMovedPosition(void) moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); From ab56f453a9d6c0d2233a93e7beb780aaf956265e Mon Sep 17 00:00:00 2001 From: nomdetom Date: Sun, 14 Jun 2026 11:33:59 +0100 Subject: [PATCH 13/13] TMM: address review, fix dedup tests, inject fake time, fix fingerprint-0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Copilot review (PR #10706): - preloadNextHopsFromNodeDB() now returns bool; runOnce only latches nextHopPreloaded once the preload actually ran (retries if nodeDB wasn't ready), instead of skipping it forever. - Remove the empty `#if HAS_VARIABLE_HOPS` blocks in the test. Test correctness: - Three more position-dedup tests were missing installWellKnownPrimaryChannel() (dropsDuplicate/allowsMoved were fixed earlier; allowsDuplicateAfterInterval, cacheFlush, priorRateState were not) — without the well-known-channel gate the dedup path never runs, so their STOP assertions failed. Fake-time injection (no more real sleeps): - Add TrafficManagementModule::s_testNowMs + nowMs(), mirroring HopScalingModule; route all TMM tick/time reads through nowMs(). Tests advance a virtual clock via s_testNowMs instead of testDelay() sleeping real 5-6 min across a tick — the suite drops from ~15 min to ~30 s. Production behaviour is unchanged (nowMs() inlines to millis()). Fingerprint-0 fix: - computePositionFingerprint() never returns 0 now (remap 0 -> 0xFF, mirroring getLastByteOfNodeNum), so a real position that hashes to 0 doesn't collide with the "no position seen" sentinel and its duplicates dedup correctly. test_traffic_management: 34/34 green. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/modules/TrafficManagementModule.cpp | 34 +++++++++++++--------- src/modules/TrafficManagementModule.h | 21 ++++++++++--- test/test_traffic_management/test_main.cpp | 27 ++++++++--------- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index 1edc07b11c8..07a570ec8b3 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -390,7 +390,7 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr NodeInfoPayloadEntry *empty = nullptr; NodeInfoPayloadEntry *lru = nullptr; uint32_t lruAge = 0; - const uint32_t now = millis(); + const uint32_t now = clockMs(); for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { NodeInfoPayloadEntry &e = nodeInfoPayload[i]; @@ -466,7 +466,7 @@ void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &m // richer context than "just the user protobuf" when PSRAM is present. // This path is intentionally independent from NodeInfoModule/NodeDB. entry->user = user; - entry->lastObservedMs = millis(); + entry->lastObservedMs = clockMs(); entry->lastObservedRxTime = mp.rx_time; entry->sourceChannel = mp.channel; entry->hasDecodedBitfield = mp.decoded.has_bitfield; @@ -528,11 +528,11 @@ uint8_t TrafficManagementModule::getNextHopHint(NodeNum dest) #endif } -void TrafficManagementModule::preloadNextHopsFromNodeDB() +bool TrafficManagementModule::preloadNextHopsFromNodeDB() { #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 if (!cache || !nodeDB) - return; + return false; // prerequisites not ready yet — caller should retry on a later pass uint16_t seeded = 0; concurrency::LockGuard guard(&cacheLock); @@ -552,6 +552,9 @@ void TrafficManagementModule::preloadNextHopsFromNodeDB() } TM_LOG_INFO("Preloaded %u next-hop hints from NodeDB", static_cast(seeded)); + return true; +#else + return true; // nothing to preload on a cache-less build; don't keep retrying #endif } @@ -607,7 +610,12 @@ uint8_t TrafficManagementModule::computePositionFingerprint(int32_t lat_truncate uint8_t latBits = (static_cast(lat_truncated) >> shift) & ((1u << bitsToTake) - 1); uint8_t lonBits = (static_cast(lon_truncated) >> shift) & ((1u << bitsToTake) - 1); - return static_cast((latBits << 4) | lonBits); + const uint8_t fp = static_cast((latBits << 4) | lonBits); + // 0 is the "no position seen" sentinel for pos_fingerprint, so a real position that happens to + // hash to 0 must not collide with it (otherwise its duplicates would never dedup). Remap 0 -> 0xFF, + // mirroring NodeDB::getLastByteOfNodeNum()'s 0 -> 0xFF idiom. Cost: the 0x00 bucket merges into + // 0xFF (one extra collision in 256 — negligible; the fingerprint already collides every 16 cells). + return fp ? fp : 0xFF; } // ============================================================================= @@ -632,7 +640,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack incrementStat(&stats.packets_inspected); const auto &cfg = moduleConfig.traffic_management; - const uint32_t nowMs = millis(); + const uint32_t nowMs = TrafficManagementModule::clockMs(); // ------------------------------------------------------------------------- // Undecoded Packet Handling @@ -811,15 +819,15 @@ int32_t TrafficManagementModule::runOnce() return INT32_MAX; #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - const uint32_t nowMs = millis(); + const uint32_t nowMs = TrafficManagementModule::clockMs(); // Warm-start the next-hop cache from persisted NodeInfoLite hints once nodeDB // is populated. Done here (not in the constructor) so nodeDB has finished // loading. Takes its own lock, so call before acquiring the sweep guard below. - if (!nextHopPreloaded) { - preloadNextHopsFromNodeDB(); + // Only latch the one-shot guard once the preload actually ran; if nodeDB wasn't + // ready yet, retry on the next maintenance pass instead of skipping it forever. + if (!nextHopPreloaded && preloadNextHopsFromNodeDB()) nextHopPreloaded = true; - } // Free-running tick counters (no epoch needed). // TTL expressed in ticks: pos 4× interval (default 44 ticks ≈ 4.4h), @@ -843,7 +851,7 @@ int32_t TrafficManagementModule::runOnce() // Sweep cache and clear expired entries uint16_t activeEntries = 0; uint16_t expiredEntries = 0; - const uint32_t sweepStartMs = millis(); + const uint32_t sweepStartMs = TrafficManagementModule::clockMs(); const auto &cfg = moduleConfig.traffic_management; concurrency::LockGuard guard(&cacheLock); @@ -900,7 +908,7 @@ int32_t TrafficManagementModule::runOnce() TM_LOG_DEBUG("Maintenance: %u active, %u expired, %u/%u slots, %lums elapsed", activeEntries, expiredEntries, static_cast(activeEntries), static_cast(cacheSize()), - static_cast(millis() - sweepStartMs)); + static_cast(TrafficManagementModule::clockMs() - sweepStartMs)); #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) if (nodeInfoPayload) { @@ -1056,7 +1064,7 @@ bool TrafficManagementModule::shouldRespondToNodeInfo(const meshtastic_MeshPacke reply->decoded.bitfield |= BITFIELD_OK_TO_MQTT_MASK; if (hasCachedUser && cachedLastObservedMs != 0) { - uint32_t ageMs = millis() - cachedLastObservedMs; + uint32_t ageMs = clockMs() - cachedLastObservedMs; TM_LOG_DEBUG("NodeInfo PSRAM hit node=0x%08x age=%lu ms src_ch=%u req_ch=%u rx_time=%lu", p->to, static_cast(ageMs), static_cast(cachedSourceChannel), static_cast(p->channel), static_cast(cachedLastObservedRxTime)); diff --git a/src/modules/TrafficManagementModule.h b/src/modules/TrafficManagementModule.h index a38f7115200..eb32e6cfdc6 100644 --- a/src/modules/TrafficManagementModule.h +++ b/src/modules/TrafficManagementModule.h @@ -52,7 +52,9 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // Warm-start the next-hop cache from persisted NodeInfoLite hints so confirmed // hops survive later hot-store (NodeDB) eviction. Idempotent; runs once after // nodeDB is populated (lazily on first maintenance pass). - void preloadNextHopsFromNodeDB(); + // @return true if it actually ran (prereqs met / nothing to do); false if + // prerequisites (cache, nodeDB) weren't ready yet, so the caller should retry. + bool preloadNextHopsFromNodeDB(); /** * Check if this packet should have its hops exhausted. @@ -64,6 +66,17 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread return exhaustRequested && exhaustRequestedFrom == getFrom(&mp) && exhaustRequestedId == mp.id; } + // Injectable monotonic clock (ms). All TMM time reads go through clockMs() so unit tests can + // advance a virtual timebase instead of sleeping real seconds across the 6 min/360 s tick. + // Mirrors HopScalingModule::s_testNowMs. Writable from tests as TrafficManagementModule::s_testNowMs; + // ignored in production (clockMs() returns millis()). + inline static uint32_t s_testNowMs = 0; +#ifdef PIO_UNIT_TESTING + static uint32_t clockMs() { return s_testNowMs; } +#else + static uint32_t clockMs() { return millis(); } +#endif + protected: ProcessMessage handleReceived(const meshtastic_MeshPacket &mp) override; bool wantPacket(const meshtastic_MeshPacket *p) override { return true; } @@ -149,9 +162,9 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread static constexpr uint32_t kRateTimeTickMs = 300'000UL; // 5 min/tick static constexpr uint32_t kUnknownTimeTickMs = 60'000UL; // 1 min/tick - static uint8_t currentPosTick() { return static_cast(millis() / kPosTimeTickMs); } - static uint8_t currentRateTick() { return static_cast((millis() / kRateTimeTickMs) & 0x0F); } - static uint8_t currentUnknownTick() { return static_cast((millis() / kUnknownTimeTickMs) & 0x0F); } + static uint8_t currentPosTick() { return static_cast(clockMs() / kPosTimeTickMs); } + static uint8_t currentRateTick() { return static_cast((clockMs() / kRateTimeTickMs) & 0x0F); } + static uint8_t currentUnknownTick() { return static_cast((clockMs() / kUnknownTimeTickMs) & 0x0F); } // ========================================================================= // Position Fingerprint // ========================================================================= diff --git a/test/test_traffic_management/test_main.cpp b/test/test_traffic_management/test_main.cpp index 1b02a9c3ed4..c7ae12e5a11 100644 --- a/test/test_traffic_management/test_main.cpp +++ b/test/test_traffic_management/test_main.cpp @@ -12,8 +12,6 @@ #if HAS_TRAFFIC_MANAGEMENT #include "airtime.h" -#if HAS_VARIABLE_HOPS -#endif #include "mesh/CryptoEngine.h" #include "mesh/MeshService.h" #include "mesh/NodeDB.h" @@ -181,6 +179,10 @@ static void resetTrafficConfig() mockNodeDB->resetNodes(); mockNodeDB->clearCachedNode(); nodeDB = mockNodeDB; + + // Virtual clock base (1 h in, so tick subtraction never underflows). Tests advance time by + // bumping TrafficManagementModule::s_testNowMs instead of sleeping real seconds across a tick. + TrafficManagementModule::s_testNowMs = 3600000; } static meshtastic_MeshPacket makeDecodedPacket(meshtastic_PortNum port, NodeNum from, NodeNum to = NODENUM_BROADCAST) @@ -738,8 +740,6 @@ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); } -#if HAS_VARIABLE_HOPS -#endif // HAS_VARIABLE_HOPS /** * Verify hop exhaustion skips unicast and local-origin packets. * Important to avoid mutating traffic that should retain normal forwarding behavior. @@ -776,8 +776,9 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) { moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; - // 360 s = 1 pos-tick (kPosTimeTickMs); testDelay advances past one tick period. + // 360 s = 1 pos-tick (kPosTimeTickMs); advance the virtual clock past one tick period. moduleConfig.traffic_management.position_min_interval_secs = 360; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -786,7 +787,7 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); - testDelay(360001); // advance past one 6-min pos-tick + TrafficManagementModule::s_testNowMs += 360001; // advance past one 6-min pos-tick (virtual clock) ProcessMessage r3 = module.handleReceived(third); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -900,6 +901,7 @@ static void test_tm_positionDedup_cacheFlush_doesNotDropFirstPacketAfterFlush(vo moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -930,6 +932,7 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero moduleConfig.traffic_management.rate_limit_enabled = true; moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 10; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); @@ -954,7 +957,7 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero static void test_tm_rateLimit_resetsAfterWindowExpires(void) { moduleConfig.traffic_management.rate_limit_enabled = true; - // 300 s = 1 rate-tick (kRateTimeTickMs); testDelay advances past one tick period. + // 300 s = 1 rate-tick (kRateTimeTickMs); advance the virtual clock past one tick period. moduleConfig.traffic_management.rate_limit_window_secs = 300; moduleConfig.traffic_management.rate_limit_max_packets = 1; TrafficManagementModuleTestShim module; @@ -962,7 +965,7 @@ static void test_tm_rateLimit_resetsAfterWindowExpires(void) ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(300001); // advance past one 5-min rate-tick + TrafficManagementModule::s_testNowMs += 300001; // advance past one 5-min rate-tick (virtual clock) ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -984,7 +987,7 @@ static void test_tm_unknownPackets_resetAfterWindowExpires(void) ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(300001); // advance past 5 unknown-ticks (kUnknownWindowTicks=5 × 60s) + TrafficManagementModule::s_testNowMs += 300001; // advance past 5 unknown-ticks (5 × 60s) (virtual clock) ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -1225,8 +1228,6 @@ static void test_tm_nextHop_keptAliveAcrossMaintenanceSweep(void) TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); } -#if HAS_VARIABLE_HOPS -#endif // HAS_VARIABLE_HOPS } // namespace void setUp(void) @@ -1264,8 +1265,6 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb); #endif RUN_TEST(test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast); -#if HAS_VARIABLE_HOPS -#endif RUN_TEST(test_tm_alterReceived_skipsLocalAndUnicast); RUN_TEST(test_tm_positionDedup_allowsDuplicateAfterIntervalExpires); RUN_TEST(test_tm_positionDedup_intervalZero_neverDrops); @@ -1287,8 +1286,6 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_nextHop_servedAfterNodeDbRoll); RUN_TEST(test_tm_nextHop_preloadDoesNotClobberLearned); RUN_TEST(test_tm_nextHop_keptAliveAcrossMaintenanceSweep); -#if HAS_VARIABLE_HOPS -#endif exit(UNITY_END()); }