diff --git a/extra_scripts/nrf52_warm_region.py b/extra_scripts/nrf52_warm_region.py new file mode 100644 index 00000000000..adc09a18e1e --- /dev/null +++ b/extra_scripts/nrf52_warm_region.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +# trunk-ignore-all(ruff/F821) +# trunk-ignore-all(flake8/F821): For SConstruct imports +# +# Post-link guard for the warm-node-store raw-flash region on nrf52840. +# +# The top 3 app-region flash pages below LittleFS (0xEA000-0xED000, reclaimed by +# whole-image LTO; LittleFS itself stays stock at 0xED000) are reserved for the WarmNodeStore record-ring +# (src/mesh/WarmNodeStore.h, 3-page record-ring). Our own linker script +# (src/platform/nrf52/nrf52840_s140_v7.ld) already caps the image at 0xEA000, +# but many boards (e.g. rak4631) link with the framework-default script whose +# FLASH region still ends at 0xED000 -- those would silently place code in the +# reserved pages if the image ever grew that large, and the first warm-store +# save would then brick the firmware. This guard turns that into a red build. +# +# Image flash end = __etext (end of code/rodata in flash) + sizeof(.data) +# (which is loaded at LMA __etext). Symbols come from the framework's +# nrf52_common.ld. +import os + +Import("env") + +WARM_REGION_BASE = 0xEA000 # keep in sync with WARM_FLASH_REGION_BASE in WarmNodeStore.h (3 x 4 KB record-ring) + +_tc = env.PioPlatform().get_package_dir("toolchain-gccarmnoneeabi") or "" +_NM = os.path.join(_tc, "bin", "arm-none-eabi-nm") +if not os.path.isfile(_NM): + _NM = "arm-none-eabi-nm" # fall back to PATH + + +def _assert_warm_region_clear(source, target, env): + import subprocess + import sys + + try: + elf = env.subst("$BUILD_DIR/${PROGNAME}.elf") + out = subprocess.check_output([_NM, elf], universal_newlines=True) + except Exception as exc: # tooling hiccup: warn loudly, don't wedge the build + print("nrf52_warm_region: WARNING - guard skipped (nm failed: %s)" % exc) + return + + syms = {} + for line in out.split("\n"): + f = line.split() + if len(f) >= 3 and f[-1] in ("__etext", "__data_start__", "__data_end__"): + syms[f[-1]] = int(f[0], 16) + if len(syms) != 3: + print("nrf52_warm_region: WARNING - guard skipped (linker symbols not found)") + return + + flash_end = syms["__etext"] + (syms["__data_end__"] - syms["__data_start__"]) + if flash_end > WARM_REGION_BASE: + sys.stderr.write( + "\n*** nrf52 warm-region guard: image ends at 0x%X, past the reserved " + "warm-store region at 0x%X ***\n" + "The 12 KB region at 0xEA000 holds the WarmNodeStore record-ring; a warm-store\n" + "save would overwrite this firmware's tail. Shrink the image, or shrink/move\n" + "the region (WARM_FLASH_REGION_BASE in src/mesh/WarmNodeStore.h, the FLASH\n" + "LENGTH in src/platform/nrf52/nrf52840_s140_v7.ld, and this guard).\n\n" + % (flash_end, WARM_REGION_BASE) + ) + from SCons.Script import Exit + + Exit(1) + print( + "nrf52_warm_region: guard OK -- image ends at 0x%X, %d KB clear of the warm region" + % (flash_end, (WARM_REGION_BASE - flash_end) // 1024) + ) + + +# Attach to the phony "buildprog" alias (not the .elf node) so the guard runs +# on incremental relinks too -- same reasoning as nrf52_lto.py's guard. +env.AddPostAction("buildprog", _assert_warm_region_clear) diff --git a/protobufs b/protobufs index 1df6c115424..36251667179 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit 1df6c1154248c82abb21b73eb06c203f595780db +Subproject commit 36251667179496c1e1261f4e4e5b349a51e5e861 diff --git a/src/graphics/draw/MenuHandler.cpp b/src/graphics/draw/MenuHandler.cpp index 365739112d5..afa17a29bdc 100644 --- a/src/graphics/draw/MenuHandler.cpp +++ b/src/graphics/draw/MenuHandler.cpp @@ -1572,7 +1572,8 @@ void menuHandler::manageNodeMenu() nodeDB->set_favorite(false, menuHandler::pickedNodeNum); } else { LOG_INFO("Adding node %08X to favorites", menuHandler::pickedNodeNum); - nodeDB->set_favorite(true, menuHandler::pickedNodeNum); + if (!nodeDB->set_favorite(true, menuHandler::pickedNodeNum)) + LOG_WARN(NodeDB::PROTECTED_CAP_WARN_FMT, "favorite", menuHandler::pickedNodeNum, MAX_NUM_NODES - 2); } screen->setFrames(graphics::Screen::FOCUS_PRESERVE); return; @@ -1615,15 +1616,23 @@ void menuHandler::manageNodeMenu() return; } + bool changed = false; if (nodeInfoLiteIsIgnored(n)) { nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, false); LOG_INFO("Unignoring node %08X", menuHandler::pickedNodeNum); - } else { - nodeInfoLiteSetBit(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + changed = true; + } else if (nodeDB->setProtectedFlag(n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) { LOG_INFO("Ignoring node %08X", menuHandler::pickedNodeNum); + changed = true; + } else { + LOG_WARN(NodeDB::PROTECTED_CAP_WARN_FMT, "ignore", menuHandler::pickedNodeNum, MAX_NUM_NODES - 2); + } + // Only persist/notify when the ignore bit actually moved; a cap + // refusal changed nothing and shouldn't trigger a prefs save. + if (changed) { + nodeDB->notifyObservers(true); + nodeDB->saveToDisk(); } - nodeDB->notifyObservers(true); - nodeDB->saveToDisk(); screen->setFrames(graphics::Screen::FOCUS_PRESERVE); return; } diff --git a/src/mesh/Channels.cpp b/src/mesh/Channels.cpp index be75e3d4214..34d56e0f7c4 100644 --- a/src/mesh/Channels.cpp +++ b/src/mesh/Channels.cpp @@ -404,6 +404,24 @@ bool Channels::isDefaultChannel(ChannelIndex chIndex) return false; } +bool Channels::isWellKnownChannel(ChannelIndex chIndex) +{ + const auto &ch = getByIndex(chIndex); + // Absent (unencrypted) or single-byte PSK — all the well-known key indexes + if (ch.settings.psk.size > 1) + return false; + + const char *name = getName(chIndex); + for (int p = _meshtastic_Config_LoRaConfig_ModemPreset_MIN; p <= _meshtastic_Config_LoRaConfig_ModemPreset_MAX; p++) { + const char *presetName = + DisplayFormatters::getModemPresetDisplayName(static_cast(p), false, true); + // Presets without a display name fall through to "Invalid" — never a match + if (strcmp(presetName, "Invalid") != 0 && strcmp(name, presetName) == 0) + return true; + } + return false; +} + bool Channels::hasDefaultChannel() { // If we don't use a preset or the default frequency slot, or we override the frequency, we don't have a default channel diff --git a/src/mesh/Channels.h b/src/mesh/Channels.h index a3cc7791c7c..2f30d2299fd 100644 --- a/src/mesh/Channels.h +++ b/src/mesh/Channels.h @@ -86,6 +86,13 @@ class Channels // Returns true if the channel has the default name and PSK bool isDefaultChannel(ChannelIndex chIndex); + // Returns true if the channel is "well known": its PSK is absent or a + // single-byte well-known key index, AND its name is any modem-preset + // display name (e.g. a channel named "LongFast" counts even while the + // radio runs MediumFast). Broader than isDefaultChannel, which only + // matches the current preset's name and PSK byte 1. + bool isWellKnownChannel(ChannelIndex chIndex); + // Returns true if we can be reached via a channel with the default settings given a region and modem preset bool hasDefaultChannel(); diff --git a/src/mesh/Default.h b/src/mesh/Default.h index 97f87fc7bfc..38842ad751b 100644 --- a/src/mesh/Default.h +++ b/src/mesh/Default.h @@ -34,8 +34,8 @@ enum class TrafficType { POSITION, TELEMETRY }; // Traffic management defaults -#define default_traffic_mgmt_position_precision_bits 24 // ~10m grid cells -#define default_traffic_mgmt_position_min_interval_secs (ONE_DAY / 2) // 12 hours between identical positions +#define default_traffic_mgmt_position_precision_bits 19 // ~90m grid cells (±45m) +#define default_traffic_mgmt_position_min_interval_secs (11 * 60 * 60) // 11 hours between identical positions // Hop scaling defaults #define default_hop_scaling_min_target_nodes 40 // walk threshold: first hop reaching this cumulative count diff --git a/src/mesh/NextHopRouter.cpp b/src/mesh/NextHopRouter.cpp index e8613d45729..c3cb29377a0 100644 --- a/src/mesh/NextHopRouter.cpp +++ b/src/mesh/NextHopRouter.cpp @@ -98,22 +98,27 @@ void NextHopRouter::sniffReceived(const meshtastic_MeshPacket *p, const meshtast // destination if (p->from != 0) { meshtastic_NodeInfoLite *origTx = nodeDB->getMeshNode(p->from); - if (origTx) { - // Either relayer of ACK was also a relayer of the packet, or we were the *only* relayer and the ACK came - // directly from the destination - // Single lookup for both relayer checks on the same (request_id, to) pair - bool wasAlreadyRelayer = false; - bool weWereSoleRelayer = false; - bool weWereRelayer = false; - checkRelayers(p->relay_node, ourRelayID, p->decoded.request_id, p->to, &wasAlreadyRelayer, &weWereRelayer, - &weWereSoleRelayer); - if ((weWereRelayer && wasAlreadyRelayer) || (getHopsAway(*p) == 0 && weWereSoleRelayer)) { - if (origTx->next_hop != p->relay_node) { // Not already set - LOG_INFO("Update next hop of 0x%x to 0x%x based on ACK/reply (was relayer %d we were sole %d)", p->from, - p->relay_node, wasAlreadyRelayer, weWereSoleRelayer); - origTx->next_hop = p->relay_node; - } + // Either relayer of ACK was also a relayer of the packet, or we were the *only* relayer and the ACK came + // directly from the destination. checkRelayers is read-only on PacketHistory and O(1), so we run it even + // when origTx is absent — that lets us still capture the confirmed hop into the TMM overflow cache below. + // Single lookup for both relayer checks on the same (request_id, to) pair + bool wasAlreadyRelayer = false; + bool weWereSoleRelayer = false; + bool weWereRelayer = false; + checkRelayers(p->relay_node, ourRelayID, p->decoded.request_id, p->to, &wasAlreadyRelayer, &weWereRelayer, + &weWereSoleRelayer); + if ((weWereRelayer && wasAlreadyRelayer) || (getHopsAway(*p) == 0 && weWereSoleRelayer)) { + if (origTx && origTx->next_hop != p->relay_node) { // Not already set + LOG_INFO("Update next hop of 0x%x to 0x%x based on ACK/reply (was relayer %d we were sole %d)", p->from, + p->relay_node, wasAlreadyRelayer, weWereSoleRelayer); + origTx->next_hop = p->relay_node; } +#if HAS_TRAFFIC_MANAGEMENT + // Mirror the confirmed hop into the TMM overflow cache so it survives even when the source + // isn't (or is no longer) in the hot NodeDB. Same bidirectional confirmation gate as above. + if (trafficManagementModule) + trafficManagementModule->setNextHop(p->from, p->relay_node); +#endif } } if (!isToUs(p)) { @@ -194,15 +199,29 @@ std::optional NextHopRouter::getNextHop(NodeNum to, uint8_t relay_node) if (isBroadcast(to)) return std::nullopt; + // Hot store first: a direct array hit on the live NodeDB entry. meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(to); if (node && node->next_hop) { // We are careful not to return the relay node as the next hop if (node->next_hop != relay_node) { - // LOG_DEBUG("Next hop for 0x%x is 0x%x", to, node->next_hop); + LOG_DEBUG("Next hop for 0x%x is 0x%x", to, node->next_hop); return node->next_hop; } else LOG_WARN("Next hop for 0x%x is 0x%x, same as relayer; set no pref", to, node->next_hop); } + +#if HAS_TRAFFIC_MANAGEMENT + // Fallback: TMM overflow cache holds confirmed hops for nodes that have aged + // out of the hot store. Same byte source/confidence as NodeInfoLite.next_hop. + if (trafficManagementModule) { + uint8_t hint = trafficManagementModule->getNextHopHint(to); + if (hint && hint != relay_node) { + LOG_DEBUG("Next hop for 0x%x is 0x%x (TMM cache)", to, hint); + return hint; + } + } +#endif + return std::nullopt; } diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index 2481eb8ca6b..e90b91a1786 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -196,6 +196,8 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre if (ostream) { const auto *vec = static_cast *>(iter->pData); for (auto item : *vec) { + item.snr_q4 = (int32_t)(item.snr * 4.0f); + item.snr = 0.0f; if (!pb_encode_tag_for_field(ostream, iter)) return false; if (!pb_encode_submessage(ostream, meshtastic_NodeInfoLite_fields, &item)) @@ -205,8 +207,12 @@ bool meshtastic_NodeDatabase_callback(pb_istream_t *istream, pb_ostream_t *ostre if (istream && istream->bytes_left) { meshtastic_NodeInfoLite node = meshtastic_NodeInfoLite_init_zero; auto *vec = static_cast *>(iter->pData); - if (pb_decode(istream, meshtastic_NodeInfoLite_fields, &node)) + if (pb_decode(istream, meshtastic_NodeInfoLite_fields, &node)) { + if (node.snr_q4) + node.snr = node.snr_q4 / 4.0f; + node.snr_q4 = 0; vec->push_back(node); + } } return true; } @@ -483,6 +489,8 @@ NodeDB::NodeDB() #endif // Include our owner in the node db under our nodenum meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); + if (!info) + return; TypeConversions::CopyUserToNodeInfoLite(info, owner); // If node database has not been saved for the first time, save it now @@ -688,6 +696,39 @@ template std::vector snapshotSatelliteNodeNums(const Map } return result; } + +// Drop the stalest entry of `map` (staleness proxied via the owner's +// last_heard; 0 = owner evicted, i.e. an orphan — first out). Never evicts our +// own node's entry. Caller holds satelliteMutex. Returns false if nothing +// could be evicted. +template bool evictStalestSatellite(NodeDB &db, Map &map) +{ + auto victim = map.end(); + uint32_t victimTs = UINT32_MAX; + for (auto it = map.begin(); it != map.end(); ++it) { + if (it->first == db.getNodeNum()) + continue; + uint32_t ts = db.hotNodeLastHeard(it->first); + if (ts < victimTs) { + victimTs = ts; + victim = it; + } + } + if (victim == map.end()) + return false; + map.erase(victim); + return true; +} + +// Keep `map` within MAX_SATELLITE_NODES ahead of inserting `incoming` (the +// tier-1/tier-2 split: only the freshest MAX_SATELLITE_NODES nodes carry +// satellite payloads). Caller holds satelliteMutex. +template void evictSatelliteOverCap(NodeDB &db, Map &map, NodeNum incoming) +{ + if (map.size() < MAX_SATELLITE_NODES || map.count(incoming)) + return; + evictStalestSatellite(db, map); +} } // namespace void NodeDB::resetRadioConfig(bool is_fresh_install) @@ -727,6 +768,12 @@ bool NodeDB::factoryReset(bool eraseBleBonds) if (transmitHistory) { transmitHistory->clear(); } +#if WARM_NODE_COUNT > 0 + // On nRF52840 the warm tier lives in raw flash outside /prefs, so rmDir + // didn't touch it; clear it and persist the empty store. + warmStore.clear(); + warmStore.saveIfDirty(); +#endif // second, install default state (this will deal with the duplicate mac address issue) installDefaultNodeDatabase(); installDefaultDeviceState(); @@ -1062,6 +1109,20 @@ void NodeDB::initConfigIntervals() #endif } +// Always-on traffic management defaults. Only booleans are written; every +// numeric field stays 0 and resolves to its default_traffic_mgmt_* macro at +// use (e.g. position dedup precision/interval), so fork-wide tuning changes +// take effect without another migration. Rate limiting and the features that +// exhaust or reshape relayed traffic (exhaust_hop_*, drop_unknown_enabled, +// nodeinfo_direct_response) stay opt-in. +static void installTrafficManagementDefaults(meshtastic_LocalModuleConfig &mc) +{ + mc.has_traffic_management = true; + mc.traffic_management = meshtastic_ModuleConfig_TrafficManagementConfig_init_zero; + mc.traffic_management.enabled = true; + mc.traffic_management.position_dedup_enabled = true; +} + void NodeDB::installDefaultModuleConfig() { LOG_INFO("Install default ModuleConfig"); @@ -1165,6 +1226,8 @@ void NodeDB::installDefaultModuleConfig() moduleConfig.has_neighbor_info = true; moduleConfig.neighbor_info.enabled = false; + installTrafficManagementDefaults(moduleConfig); + moduleConfig.has_detection_sensor = true; moduleConfig.detection_sensor.enabled = false; moduleConfig.detection_sensor.detection_trigger_type = meshtastic_ModuleConfig_DetectionSensorConfig_TriggerType_LOGIC_HIGH; @@ -1298,6 +1361,9 @@ void NodeDB::resetNodes(bool keepFavorites) std::fill(nodeDatabase.nodes.begin() + 1, nodeDatabase.nodes.end(), meshtastic_NodeInfoLite()); } (void)ourNum; +#if WARM_NODE_COUNT > 0 + warmStore.clear(); // warm entries are never favorites; a DB reset clears them too +#endif devicestate.has_rx_waypoint = false; saveNodeDatabaseToDisk(); saveDeviceStateToDisk(); @@ -1319,6 +1385,10 @@ void NodeDB::removeNodeByNum(NodeNum nodeNum) meshtastic_NodeInfoLite()); if (removed) eraseNodeSatellites(nodeNum); +#if WARM_NODE_COUNT > 0 + // Explicit user removal: don't let the warm tier resurrect the node + warmStore.remove(nodeNum); +#endif LOG_DEBUG("NodeDB::removeNodeByNum purged %d entries. Save changes", removed); saveNodeDatabaseToDisk(); } @@ -1432,6 +1502,7 @@ void NodeDB::setNodeStatus(NodeNum n, const meshtastic_StatusMessage &status) (void)status; #else concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeStatus, n); nodeStatus[n] = status; #endif } @@ -1443,6 +1514,7 @@ void NodeDB::touchNodePositionTime(NodeNum n, uint32_t time) (void)time; #else concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodePositions, n); nodePositions[n].time = time; #endif } @@ -1464,12 +1536,43 @@ void NodeDB::eraseNodeSatellites(NodeNum n) #endif } +void NodeDB::enforceSatelliteCaps() +{ + concurrency::LockGuard guard(&satelliteMutex); + auto trim = [this](auto &map, const char *name) { + const size_t before = map.size(); + while (map.size() > MAX_SATELLITE_NODES) { + if (!evictStalestSatellite(*this, map)) + break; + } + if (map.size() != before) + LOG_INFO("Trimmed %s satellites %u -> %u (cap %d)", name, (unsigned)before, (unsigned)map.size(), + MAX_SATELLITE_NODES); + }; +#if !MESHTASTIC_EXCLUDE_POSITIONDB + trim(nodePositions, "position"); +#endif +#if !MESHTASTIC_EXCLUDE_TELEMETRYDB + trim(nodeTelemetry, "telemetry"); +#endif +#if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB + trim(nodeEnvironment, "environment"); +#endif +#if !MESHTASTIC_EXCLUDE_STATUSDB + trim(nodeStatus, "status"); +#endif + (void)trim; // all four maps may be compiled out +} + void NodeDB::cleanupMeshDB() { int newPos = 0, removed = 0; for (int i = 0; i < numMeshNodes; i++) { meshtastic_NodeInfoLite &n = meshNodes->at(i); - if (nodeInfoLiteHasUser(&n)) { + // Keep ignored (blocked) nodes even without user info: a block set by + // bare node ID has no NodeInfo and would otherwise be purged here, + // silently dropping the block. + if (nodeInfoLiteHasUser(&n) || nodeInfoLiteIsIgnored(&n)) { if (n.public_key.size > 0) { if (memfll(n.public_key.bytes, 0, n.public_key.size)) { n.public_key.size = 0; @@ -1482,8 +1585,15 @@ void NodeDB::cleanupMeshDB() } else { // No user info - drop this node and its satellites const NodeNum gone = n.num; - if (gone) + if (gone) { +#if WARM_NODE_COUNT > 0 + // Keep any key we learned (e.g. via a DM before the NodeInfo + // exchange completed) rather than losing it with the purge. + if (n.public_key.size == 32) + warmStore.absorb(gone, n.last_heard, n.public_key.bytes); +#endif eraseNodeSatellites(gone); + } removed++; } } @@ -1633,6 +1743,42 @@ LoadFileResult NodeDB::loadProto(const char *filename, size_t protoSize, size_t return state; } +#if WARM_NODE_COUNT > 0 +void NodeDB::demoteOldestHotNodesToWarm() +{ + const int keep = MAX_NUM_NODES; + if (numMeshNodes <= keep) + return; + + // Favorites, ignored (blocked) and manually-verified nodes rank ahead of + // recency — the same protection getOrCreateMeshNode applies on eviction — so + // they are demoted only when the store is full of them. Within a class, + // most-recently-heard first. Index 0 is our own node and is left in place + // (sort begins at +1), as in the runtime eviction scan. + std::sort(meshNodes->begin() + 1, meshNodes->begin() + numMeshNodes, + [](const meshtastic_NodeInfoLite &a, const meshtastic_NodeInfoLite &b) { + const bool ka = nodeInfoLiteIsProtected(&a); + const bool kb = nodeInfoLiteIsProtected(&b); + if (ka != kb) + return ka; + return a.last_heard > b.last_heard; + }); + + int demoted = 0; + for (int i = keep; i < numMeshNodes; i++) { + const meshtastic_NodeInfoLite &n = (*meshNodes)[i]; + if (n.num == 0) + continue; + // Keep the public key if we have one (40 B warm record); keyless nodes + // still get a placeholder so re-admission restores last_heard. + warmStore.absorb(n.num, n.last_heard, n.public_key.size > 0 ? n.public_key.bytes : nullptr); + demoted++; + } + numMeshNodes = keep; // the resize() in loadFromDisk reclaims the demoted tail + LOG_INFO("NodeDB migration: demoted %d node(s) over %d into the warm tier (keepers preferred)", demoted, keep); +} +#endif + void NodeDB::loadFromDisk() { // Mark the current device state as completely unusable, so that if we fail reading the entire file from @@ -1776,12 +1922,32 @@ void NodeDB::loadFromDisk() nodeDatabase.nodes.size(), posCount, telCount, envCount, statusCount); } +#if WARM_NODE_COUNT > 0 + // Load the warm tier first so a hot-store migration below merges into the + // persisted set instead of being clobbered by the on-disk warm snapshot. + warmStore.load(); + // A database from a larger-cap build (the pre-fork 150-node nRF52 hot store) + // can exceed MAX_NUM_NODES. Demote the oldest overflow into the warm tier + // before we truncate, so their public keys survive for PKI DMs. + demoteOldestHotNodesToWarm(); +#endif + if (numMeshNodes > MAX_NUM_NODES) { LOG_WARN("Node count %d exceeds MAX_NUM_NODES %d, truncating", numMeshNodes, MAX_NUM_NODES); numMeshNodes = MAX_NUM_NODES; } meshNodes->resize(MAX_NUM_NODES); + // Files written before the satellite cap (or by a larger-cap build) can + // carry more satellite entries than we keep; trim to the freshest. + enforceSatelliteCaps(); + +#if WARM_NODE_COUNT > 0 + // Persist the migrated entries now so they survive a reboot before the next + // node-DB save cadence. + warmStore.saveIfDirty(); +#endif + // static DeviceState scratch; We no longer read into a tempbuf because this structure is 15KB of valuable RAM state = loadProto(deviceStateFileName, meshtastic_DeviceState_size, sizeof(meshtastic_DeviceState), &meshtastic_DeviceState_msg, &devicestate); @@ -1938,6 +2104,16 @@ void NodeDB::loadFromDisk() } } + // Always-on traffic management: a device that has NEVER configured TMM + // (has_traffic_management false — AdminModule always sets the has_ flag on + // write, even when disabling) gets the fork defaults. Explicitly configured + // devices keep their exact settings. + if (!moduleConfig.has_traffic_management) { + LOG_INFO("Traffic management never configured, installing always-on defaults"); + installTrafficManagementDefaults(moduleConfig); + saveToDisk(SEGMENT_MODULECONFIG); + } + state = loadProto(channelFileName, meshtastic_ChannelFile_size, sizeof(meshtastic_ChannelFile), &meshtastic_ChannelFile_msg, &channelFile); if (state != LoadFileResult::LOAD_SUCCESS) { @@ -2302,7 +2478,6 @@ bool NodeDB::saveNodeDatabaseToDisk() #else nodeDatabase.status.clear(); #endif - size_t nodeDatabaseSize; pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &nodeDatabase); bool ok = saveProto(nodeDatabaseFileName, nodeDatabaseSize, &meshtastic_NodeDatabase_msg, &nodeDatabase, false); @@ -2315,6 +2490,11 @@ bool NodeDB::saveNodeDatabaseToDisk() nodeDatabase.environment.shrink_to_fit(); nodeDatabase.status.clear(); nodeDatabase.status.shrink_to_fit(); +#if WARM_NODE_COUNT > 0 + // Same cadence as the node DB; failure is logged but must not propagate — + // a false return from here would trigger saveToDisk()'s fsFormat() path. + warmStore.saveIfDirty(); +#endif return ok; } @@ -2551,6 +2731,7 @@ void NodeDB::updatePosition(uint32_t nodeId, const meshtastic_Position &p, RxSou #else { concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodePositions, nodeId); meshtastic_PositionLite &slot = nodePositions[nodeId]; // creates default-zero entry if missing if (src == RX_SRC_LOCAL) { @@ -2608,6 +2789,7 @@ void NodeDB::updateTelemetry(uint32_t nodeId, const meshtastic_Telemetry &t, RxS } #if !MESHTASTIC_EXCLUDE_TELEMETRYDB concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeTelemetry, nodeId); nodeTelemetry[nodeId] = t.variant.device_metrics; #endif } else if (t.which_variant == meshtastic_Telemetry_environment_metrics_tag) { @@ -2618,6 +2800,7 @@ void NodeDB::updateTelemetry(uint32_t nodeId, const meshtastic_Telemetry &t, RxS } #if !MESHTASTIC_EXCLUDE_ENVIRONMENTDB concurrency::LockGuard guard(&satelliteMutex); + evictSatelliteOverCap(*this, nodeEnvironment, nodeId); nodeEnvironment[nodeId] = t.variant.environment_metrics; #endif } else { @@ -2648,13 +2831,13 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) info->num = contact.node_num; TypeConversions::CopyUserToNodeInfoLite(info, contact.user); if (contact.should_ignore) { - // If should_ignore is set, - // we need to clear the public key and other cruft, in addition to setting the node as ignored - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + // Block the contact and drop its rich satellite data, but keep the + // public key copied above — an ignored peer keeps a usable identity + // (a verifiable target) rather than a bare node number. + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "ignore", contact.node_num, MAX_NUM_NODES - 2); nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, false); eraseNodeSatellites(contact.node_num); - info->public_key.size = 0; - memset(info->public_key.bytes, 0, sizeof(info->public_key.bytes)); } else { /* Clients are sending add_contact before every text message DM (because clients may hold a larger node database with * public keys than the radio holds). However, we don't want to update last_heard just because we sent someone a DM! @@ -2673,12 +2856,14 @@ void NodeDB::addFromContact(meshtastic_SharedContact contact) } else { // Normal case: set is_favorite to prevent expiration. // last_heard will remain as-is (or remain 0 if this entry wasn't in the nodeDB). - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "favorite", contact.node_num, MAX_NUM_NODES - 2); } // As the clients will begin sending the contact with DMs, we want to strictly check if the node is manually verified if (contact.manually_verified) { - nodeInfoLiteSetBit(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true); + if (!setProtectedFlag(info, NODEINFO_BITFIELD_IS_KEY_MANUALLY_VERIFIED_MASK, true)) + LOG_WARN(PROTECTED_CAP_WARN_FMT, "verify", contact.node_num, MAX_NUM_NODES - 2); } // Mark the node's key as manually verified to indicate trustworthiness. updateGUIforNode = info; @@ -2811,14 +2996,48 @@ void NodeDB::updateFrom(const meshtastic_MeshPacket &mp) } } -void NodeDB::set_favorite(bool is_favorite, uint32_t nodeId) +int NodeDB::numProtectedNodes() const +{ + int count = 0; + for (int i = 0; i < numMeshNodes; i++) + if (nodeInfoLiteIsProtected(&meshNodes->at(i))) + count++; + return count; +} + +bool NodeDB::setProtectedFlag(meshtastic_NodeInfoLite *node, uint32_t mask, bool on) +{ + if (!node) + return false; + if (!on) { + nodeInfoLiteSetBit(node, mask, false); + return true; + } + // Adding a flag to a node that is already protected doesn't grow the + // protected set, so it's always allowed. A newly-protected node is refused + // once the protected set has reached MAX_NUM_NODES-2, leaving two evictable + // slots so getOrCreateMeshNode can always make room. + if (nodeInfoLiteIsProtected(node) || numProtectedNodes() < MAX_NUM_NODES - 2) { + nodeInfoLiteSetBit(node, mask, true); + return true; + } + return false; +} + +bool NodeDB::set_favorite(bool is_favorite, uint32_t nodeId) { meshtastic_NodeInfoLite *lite = getMeshNode(nodeId); - if (lite && nodeInfoLiteIsFavorite(lite) != is_favorite) { - nodeInfoLiteSetBit(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite); + if (!lite) + return false; + if (nodeInfoLiteIsFavorite(lite) == is_favorite) + return true; // already in the requested state + if (setProtectedFlag(lite, NODEINFO_BITFIELD_IS_FAVORITE_MASK, is_favorite)) { sortMeshDB(); saveNodeDatabaseToDisk(); + return true; } + LOG_WARN(PROTECTED_CAP_WARN_FMT, "favorite", nodeId, MAX_NUM_NODES - 2); + return false; } bool NodeDB::isFavorite(uint32_t nodeId) @@ -2948,6 +3167,30 @@ bool NodeDB::isFull() return (numMeshNodes >= MAX_NUM_NODES) || (memGet.getFreeHeap() < MINIMUM_SAFE_FREE_HEAP); } +uint32_t NodeDB::hotNodeLastHeard(NodeNum n) const +{ + for (int i = 0; i < numMeshNodes; i++) + if (meshNodes->at(i).num == n) + return meshNodes->at(i).last_heard; + return 0; +} + +bool NodeDB::copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out) +{ + const meshtastic_NodeInfoLite *info = getMeshNode(n); + if (info && info->public_key.size == 32) { + out = info->public_key; + return true; + } +#if WARM_NODE_COUNT > 0 + if (warmStore.copyKey(n, out.bytes)) { + out.size = 32; + return true; + } +#endif + return false; +} + /// Find a node in our DB, create an empty NodeInfo if missing meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) { @@ -2984,7 +3227,14 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) } if (oldestIndex != -1) { - eraseNodeSatellites(meshNodes->at(oldestIndex).num); + const meshtastic_NodeInfoLite &evicted = meshNodes->at(oldestIndex); +#if WARM_NODE_COUNT > 0 + // Demote to the warm tier so the identity (and crucially the + // PKI key) outlives the hot-store slot. + warmStore.absorb(evicted.num, evicted.last_heard, + evicted.public_key.size == 32 ? evicted.public_key.bytes : NULL); +#endif + eraseNodeSatellites(evicted.num); // Shove the remaining nodes down the chain for (int i = oldestIndex; i < numMeshNodes - 1; i++) { meshNodes->at(i) = meshNodes->at(i + 1); @@ -2992,12 +3242,30 @@ meshtastic_NodeInfoLite *NodeDB::getOrCreateMeshNode(NodeNum n) (numMeshNodes)--; } } + // Don't append past the end of the vector. The protected-node cap + // (numProtectedNodes() <= MAX_NUM_NODES-2) means the eviction above frees + // a slot in normal operation; this guards the legacy case of a pre-cap + // database that is full of protected nodes — refuse rather than overrun. + if (numMeshNodes >= MAX_NUM_NODES) + return NULL; // add the node at the end lite = &meshNodes->at((numMeshNodes)++); // everything is missing except the nodenum memset(lite, 0, sizeof(*lite)); lite->num = n; +#if WARM_NODE_COUNT > 0 + // Re-admission: restore what the warm tier kept for this node + WarmNodeEntry warm; + if (warmStore.take(n, warm)) { + lite->last_heard = warm.last_heard; + if (!memfll(warm.public_key, 0, sizeof(warm.public_key))) { + lite->public_key.size = 32; + memcpy(lite->public_key.bytes, warm.public_key, 32); + } + LOG_INFO("Rehydrated node 0x%x from warm tier (key=%d)", n, lite->public_key.size == 32); + } +#endif LOG_INFO("Adding node to database with %i nodes and %u bytes free!", numMeshNodes, memGet.getFreeHeap()); } @@ -3132,6 +3400,8 @@ bool NodeDB::createNewIdentity() myNodeInfo.my_node_num = newNodeNum; meshtastic_NodeInfoLite *info = getOrCreateMeshNode(getNodeNum()); + if (!info) + return false; TypeConversions::CopyUserToNodeInfoLite(info, owner); return true; diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index 3fe24429406..5fc202e5b39 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -11,6 +11,7 @@ #include "MeshTypes.h" #include "NodeStatus.h" +#include "WarmNodeStore.h" #include "concurrency/Lock.h" #include "configuration.h" #include "mesh-pb-constants.h" @@ -226,9 +227,27 @@ class NodeDB bool updateUser(uint32_t nodeId, meshtastic_User &p, uint8_t channelIndex = 0); /* - * Sets a node either favorite or unfavorite + * Sets a node either favorite or unfavorite. Returns true if the node ends + * up in the requested state; false if the node is unknown or favouriting + * was refused by the protected-node cap (MAX_NUM_NODES - 2). */ - void set_favorite(bool is_favorite, uint32_t nodeId); + bool set_favorite(bool is_favorite, uint32_t nodeId); + + /// Count of eviction-protected (favourite/ignored/manually-verified) nodes. + int numProtectedNodes() const; + + /// printf-style warning emitted when setProtectedFlag() refuses a node at + /// the cap. %s = verb (favorite/ignore), 0x%08x = node, %d = cap. Shared by + /// LOG_WARN here and AdminModule::sendWarning so the wording stays in sync. + static constexpr const char *PROTECTED_CAP_WARN_FMT = "Can't %s 0x%08x: protected-node limit (%d) reached"; + + /// Turn an eviction-protection flag (favourite/ignored/verified) on or off, + /// capped so the protected set never exceeds MAX_NUM_NODES-2 — keeping at + /// least two always-evictable slots. Turning a flag off always succeeds; + /// turning one on for a node that isn't already protected returns false + /// (and changes nothing) once the cap is reached. Callers surface that to + /// the user where appropriate. + bool setProtectedFlag(meshtastic_NodeInfoLite *node, uint32_t mask, bool on); /* * Returns true if the node is in the NodeDB and marked as favorite @@ -295,6 +314,24 @@ class NodeDB virtual meshtastic_NodeInfoLite *getMeshNode(NodeNum n); size_t getNumMeshNodes() { return numMeshNodes; } + /// Find a node in our DB, create an empty NodeInfoLite if missing (evicting + /// the oldest non-protected node when full). Public so admin handlers can + /// register a node we have not heard from yet (e.g. to block it by ID). + meshtastic_NodeInfoLite *getOrCreateMeshNode(NodeNum n); + +#if WARM_NODE_COUNT > 0 + // Warm ("long-tail") tier: minimal {num, last_heard, public_key} records + // for nodes evicted from the hot store. See WarmNodeStore.h. + WarmNodeStore warmStore; +#endif + + /// Copy the 32-byte public key for node n — hot store first, then the warm + /// tier. Returns false if we don't know a key for n. + bool copyPublicKey(NodeNum n, meshtastic_NodeInfoLite_public_key_t &out); + + /// last_heard of a hot-store node, or 0 if absent. Plain scan of meshNodes + /// with no allocation side effects (unlike getOrCreateMeshNode). + uint32_t hotNodeLastHeard(NodeNum n) const; // Thread-safe satellite-map accessors. Return false if absent or the // corresponding DB is compiled out. @@ -341,11 +378,17 @@ class NodeDB emptyNodeDatabase.version = DEVICESTATE_CUR_VER; size_t nodeDatabaseSize; pb_get_encoded_size(&nodeDatabaseSize, meshtastic_NodeDatabase_fields, &emptyNodeDatabase); - // Always include satellite slots so backups from higher-cap peers - // decode without truncation, even when our build excludes the DBs. - return nodeDatabaseSize + (MAX_NUM_NODES * meshtastic_NodeInfoLite_size) + - (MAX_NUM_NODES * meshtastic_NodePositionEntry_size) + (MAX_NUM_NODES * meshtastic_NodeTelemetryEntry_size) + - (MAX_NUM_NODES * meshtastic_NodeEnvironmentEntry_size) + (MAX_NUM_NODES * meshtastic_NodeStatusEntry_size); + // Decode-stream size ceiling only — no buffer of this size is ever + // allocated (load streams from the file). Sized for the largest file + // any previous firmware could have written (250-node ESP32-S3 builds, + // satellites uncapped), not the current MAX_NUM_NODES, so capacity + // downgrades and backups from higher-cap peers still decode; excess + // entries are trimmed after load. + // (not constexpr: portduino resolves MAX_NUM_NODES from runtime config) + const size_t loadCeiling = ((size_t)MAX_NUM_NODES > 250) ? (size_t)MAX_NUM_NODES : 250; + return nodeDatabaseSize + (loadCeiling * meshtastic_NodeInfoLite_size) + + (loadCeiling * meshtastic_NodePositionEntry_size) + (loadCeiling * meshtastic_NodeTelemetryEntry_size) + + (loadCeiling * meshtastic_NodeEnvironmentEntry_size) + (loadCeiling * meshtastic_NodeStatusEntry_size); } // returns true if the maximum number of nodes is reached or we are running low on memory @@ -433,8 +476,6 @@ class NodeDB uint32_t lastNodeDbSave = 0; // when we last saved our db to flash uint32_t lastBackupAttempt = 0; // when we last tried a backup automatically or manually uint32_t lastSort = 0; // When last sorted the nodeDB - /// Find a node in our DB, create an empty NodeInfoLite if missing - meshtastic_NodeInfoLite *getOrCreateMeshNode(NodeNum n); /* * Internal boolean to track sorting paused @@ -447,9 +488,29 @@ class NodeDB /// read our db from flash void loadFromDisk(); +#ifdef PIO_UNIT_TESTING + // Grant the unit-test shim access to the private maintenance paths below + // (migration / cleanup / eviction) without relaxing production access. + friend class NodeDBTestShim; +#endif + /// purge db entries without user info void cleanupMeshDB(); + /// Trim each satellite map down to MAX_SATELLITE_NODES, dropping the + /// stalest entries (used after loading files written before the cap, or by + /// a build with a larger cap). + void enforceSatelliteCaps(); + +#if WARM_NODE_COUNT > 0 + /// On load, a database from a larger-cap build (notably the pre-fork 150-node + /// nRF52 hot store) can exceed MAX_NUM_NODES. Rank the hot store so the + /// freshest survive and demote the oldest overflow into the warm tier, + /// preserving {num, last_heard, public_key} so PKI DMs keep working instead + /// of dropping those nodes on truncation. + void demoteOldestHotNodesToWarm(); +#endif + /// Reinit device state from scratch (not loading from disk) void installDefaultDeviceState(), installDefaultNodeDatabase(), installDefaultChannels(), installDefaultConfig(bool preserveKey), installDefaultModuleConfig(); @@ -570,6 +631,12 @@ inline bool nodeInfoLiteHasXeddsaSigned(const meshtastic_NodeInfoLite *n) { return n && (n->bitfield & NODEINFO_BITFIELD_HAS_XEDDSA_SIGNED_MASK); } +/// A node that the eviction/migration paths must not drop: a favourite, an +/// ignored (blocked) node, or a manually-verified key. +inline bool nodeInfoLiteIsProtected(const meshtastic_NodeInfoLite *n) +{ + return nodeInfoLiteIsFavorite(n) || nodeInfoLiteIsIgnored(n) || nodeInfoLiteIsKeyManuallyVerified(n); +} inline void nodeInfoLiteSetBit(meshtastic_NodeInfoLite *n, uint32_t mask, bool value) { diff --git a/src/mesh/Router.cpp b/src/mesh/Router.cpp index f176f60e63b..be4d5e550f8 100644 --- a/src/mesh/Router.cpp +++ b/src/mesh/Router.cpp @@ -485,14 +485,16 @@ DecodeState perhapsDecode(meshtastic_MeshPacket *p) bool decrypted = false; ChannelIndex chIndex = 0; #if !(MESHTASTIC_EXCLUDE_PKI) - // Attempt PKI decryption first - if (p->channel == 0 && isToUs(p) && p->to > 0 && !isBroadcast(p->to) && nodeDB->getMeshNode(p->from) != nullptr && - nodeDB->getMeshNode(p->from)->public_key.size > 0 && nodeDB->getMeshNode(p->to) != nullptr && - nodeDB->getMeshNode(p->to)->public_key.size > 0 && rawSize > MESHTASTIC_PKC_OVERHEAD) { + // Attempt PKI decryption first. The sender's key may come from the hot + // store or the warm tier (nodes evicted from the hot store keep their key + // there), so DMs from long-tail nodes still decrypt. + meshtastic_NodeInfoLite_public_key_t fromKey = {0, {0}}; + if (p->channel == 0 && isToUs(p) && p->to > 0 && !isBroadcast(p->to) && nodeDB->copyPublicKey(p->from, fromKey) && + nodeDB->getMeshNode(p->to) != nullptr && nodeDB->getMeshNode(p->to)->public_key.size > 0 && + rawSize > MESHTASTIC_PKC_OVERHEAD) { LOG_DEBUG("Attempt PKI decryption"); - if (crypto->decryptCurve25519(p->from, nodeDB->getMeshNode(p->from)->public_key, p->id, rawSize, p->encrypted.bytes, - bytes)) { + if (crypto->decryptCurve25519(p->from, fromKey, p->id, rawSize, p->encrypted.bytes, bytes)) { LOG_INFO("PKI Decryption worked!"); meshtastic_Data decodedtmp; @@ -503,7 +505,7 @@ DecodeState perhapsDecode(meshtastic_MeshPacket *p) decrypted = true; LOG_INFO("Packet decrypted using PKI!"); p->pki_encrypted = true; - memcpy(&p->public_key.bytes, nodeDB->getMeshNode(p->from)->public_key.bytes, 32); + memcpy(&p->public_key.bytes, fromKey.bytes, 32); p->public_key.size = 32; p->decoded = decodedtmp; p->which_payload_variant = meshtastic_MeshPacket_decoded_tag; // change type to decoded @@ -720,7 +722,10 @@ meshtastic_Routing_Error perhapsEncode(meshtastic_MeshPacket *p) ChannelIndex chIndex = p->channel; // keep as a local because we are about to change it #if !(MESHTASTIC_EXCLUDE_PKI) - meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(p->to); + // Destination key from the hot store or the warm tier (evicted + // long-tail nodes keep their key there) + meshtastic_NodeInfoLite_public_key_t destKey = {0, {0}}; + bool haveDestKey = nodeDB->copyPublicKey(p->to, destKey); // We may want to retool things so we can send a PKC packet when the client specifies a key and nodenum, even if the node // is not in the local nodedb // First, only PKC encrypt packets we are originating @@ -743,18 +748,17 @@ meshtastic_Routing_Error perhapsEncode(meshtastic_MeshPacket *p) if (numbytes + MESHTASTIC_HEADER_LENGTH + MESHTASTIC_PKC_OVERHEAD > MAX_LORA_PAYLOAD_LEN) return meshtastic_Routing_Error_TOO_LARGE; // Check for a known public key for the destination - if (node == nullptr || node->public_key.size != 32) { + if (!haveDestKey) { LOG_WARN("Unknown public key for destination node 0x%08x (portnum %d), refusing to send legacy DM", p->to, p->decoded.portnum); return meshtastic_Routing_Error_PKI_SEND_FAIL_PUBLIC_KEY; } - if (p->pki_encrypted && !memfll(p->public_key.bytes, 0, 32) && - memcmp(p->public_key.bytes, node->public_key.bytes, 32) != 0) { + if (p->pki_encrypted && !memfll(p->public_key.bytes, 0, 32) && memcmp(p->public_key.bytes, destKey.bytes, 32) != 0) { LOG_WARN("Client public key differs from requested: 0x%02x, stored key begins 0x%02x", *p->public_key.bytes, - *node->public_key.bytes); + *destKey.bytes); return meshtastic_Routing_Error_PKI_FAILED; } - crypto->encryptCurve25519(p->to, getFrom(p), node->public_key, p->id, numbytes, bytes, p->encrypted.bytes); + crypto->encryptCurve25519(p->to, getFrom(p), destKey, p->id, numbytes, bytes, p->encrypted.bytes); numbytes += MESHTASTIC_PKC_OVERHEAD; p->channel = 0; p->pki_encrypted = true; diff --git a/src/mesh/WarmNodeStore.cpp b/src/mesh/WarmNodeStore.cpp new file mode 100644 index 00000000000..a53c6531c4a --- /dev/null +++ b/src/mesh/WarmNodeStore.cpp @@ -0,0 +1,511 @@ +#include "WarmNodeStore.h" + +#if WARM_NODE_COUNT > 0 + +#include "FSCommon.h" +#include "SPILock.h" +#include "SafeFile.h" +#include "configuration.h" +#include "power/PowerHAL.h" +#include +#include + +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) +#include "flash/flash_nrf5x.h" +#define WARM_RING_MAGIC 0x474E5257u // "WRNG" +// A tombstone is an entry record whose last_heard is all-ones — getTime() +// (unix seconds) cannot reach 0xFFFFFFFF until 2106, and erased flash is +// detected via num == 0xFFFFFFFF before last_heard is ever inspected. +#define WARM_RING_TOMBSTONE 0xFFFFFFFFu +#else +// warm.dat layout: this header followed by count packed WarmNodeEntry records. +struct WarmStoreHeader { + uint32_t magic; // WARM_STORE_MAGIC + uint32_t reserved; // 0; kept so the header stays 16 B + uint16_t count; // entries persisted + uint16_t entrySize; // sizeof(WarmNodeEntry), format guard + uint32_t crc; // crc32 over count * entrySize bytes +}; +static_assert(sizeof(WarmStoreHeader) == 16, "header layout is part of the persistence format"); + +#define WARM_STORE_MAGIC 0x314D5257u // "WRM1" + +#ifdef FSCom +static const char *warmFileName = "/prefs/warm.dat"; +#endif +#endif // defined(ARCH_NRF52) && defined(NRF52840_XXAA) + +static inline bool keyIsSet(const uint8_t key[32]) +{ + for (int i = 0; i < 32; i++) + if (key[i]) + return true; + return false; +} + +WarmNodeStore::WarmNodeStore() +{ +#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) + entries = static_cast(ps_calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); + if (!entries) { + LOG_WARN("WarmStore: PSRAM allocation failed, falling back to heap"); + entries = static_cast(calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); + } +#else + entries = static_cast(calloc(WARM_NODE_COUNT, sizeof(WarmNodeEntry))); +#endif +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + memset(pageOf, 0xFF, sizeof(pageOf)); +#endif +} + +WarmNodeStore::~WarmNodeStore() +{ + free(entries); // always malloc-family (calloc / ps_calloc) + entries = nullptr; +} + +WarmNodeEntry *WarmNodeStore::find(NodeNum num) const +{ + if (!entries || !num) + return nullptr; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (entries[i].num == num) + return &entries[i]; + return nullptr; +} + +// Slot placement with the keyed-first admission policy. Shared by absorb() +// and the ring replay, so the policy is applied identically in both paths. +WarmNodeEntry *WarmNodeStore::place(NodeNum num, uint32_t lastHeard, const uint8_t *key32) +{ + if (!entries || !num) + return nullptr; + + const bool candidateKeyed = key32 && keyIsSet(key32); + + WarmNodeEntry *slot = find(num); + const bool sameNode = slot != nullptr; + if (!slot) { + // Pick a victim: any empty slot, else the oldest keyless entry, else + // (only for keyed candidates) the oldest keyed entry. + WarmNodeEntry *oldestKeyless = nullptr, *oldestKeyed = nullptr; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) { + WarmNodeEntry &e = entries[i]; + if (!e.num) { + slot = &e; + break; + } + if (keyIsSet(e.public_key)) { + if (!oldestKeyed || e.last_heard < oldestKeyed->last_heard) + oldestKeyed = &e; + } else { + if (!oldestKeyless || e.last_heard < oldestKeyless->last_heard) + oldestKeyless = &e; + } + } + if (!slot) + slot = oldestKeyless ? oldestKeyless : (candidateKeyed ? oldestKeyed : nullptr); + if (!slot) + return nullptr; // store full of keyed entries and the candidate has no key + } + + slot->num = num; + slot->last_heard = lastHeard; + if (candidateKeyed) + memcpy(slot->public_key, key32, 32); + else if (!sameNode) + // Repurposing a victim slot for a different node: clear its stale key. + // A keyless refresh of a node already here keeps the key we learned. + memset(slot->public_key, 0, 32); + return slot; +} + +bool WarmNodeStore::absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32) +{ + const WarmNodeEntry *slot = place(num, lastHeard, key32); + if (!slot) + return false; + persistEntry(*slot); + return true; +} + +bool WarmNodeStore::take(NodeNum num, WarmNodeEntry &out) +{ + WarmNodeEntry *e = find(num); + if (!e) + return false; + out = *e; + const int idx = static_cast(e - entries); + memset(e, 0, sizeof(*e)); + persistRemove(num, idx); + return true; +} + +bool WarmNodeStore::copyKey(NodeNum num, uint8_t out[32]) const +{ + const WarmNodeEntry *e = find(num); + if (!e || !keyIsSet(e->public_key)) + return false; + memcpy(out, e->public_key, 32); + return true; +} + +bool WarmNodeStore::contains(NodeNum num) const +{ + return find(num) != nullptr; +} + +void WarmNodeStore::remove(NodeNum num) +{ + WarmNodeEntry *e = find(num); + if (e) { + const int idx = static_cast(e - entries); + memset(e, 0, sizeof(*e)); + persistRemove(num, idx); + } +} + +void WarmNodeStore::clear() +{ + if (!entries) + return; + memset(entries, 0, WARM_NODE_COUNT * sizeof(WarmNodeEntry)); +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + memset(pageOf, 0xFF, sizeof(pageOf)); +#endif + persistClear(); +} + +size_t WarmNodeStore::count() const +{ + size_t n = 0; + if (entries) + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (entries[i].num) + n++; + return n; +} + +bool WarmNodeStore::saveIfDirty() +{ + if (!dirty) + return true; + bool ok = save(); + if (ok) + dirty = false; + return ok; +} + +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + +// ---- Raw-flash record-ring backend (nRF52840 only; nRF52832 uses file backend) --- +// 3 × 4 KB pages directly below LittleFS. Mutations append 40 B records (an +// entry snapshot, or a tombstone with last_heard == 0xFFFFFFFF) through the +// shared flash_nrf5x page cache; saveIfDirty() is the durability point. When +// the active page fills, the oldest page is reclaimed: live entries whose +// newest record is stranded there get re-appended first, then the page is +// erased and becomes the new head. All flash access happens under spiLock +// because the page cache is shared with InternalFS/LittleFS. + +bool WarmNodeStore::ringReadHeader(uint8_t page, WarmPageHeader &h) const +{ + flash_nrf5x_read(&h, WARM_FLASH_PAGE_ADDR(page), sizeof(h)); + return h.magic == WARM_RING_MAGIC && h.seq != 0xFFFFFFFFu; +} + +// Erase `page` and stamp a fresh header. Caller holds spiLock. +void WarmNodeStore::ringOpenPage(uint8_t page) +{ + // Drop any cached state for the page before the real erase, so a later + // cache flush can't resurrect stale bytes. + flash_nrf5x_flush(); + flash_nrf5x_erase(WARM_FLASH_PAGE_ADDR(page)); + WarmPageHeader h; + h.magic = WARM_RING_MAGIC; + h.seq = nextSeq++; + flash_nrf5x_write(WARM_FLASH_PAGE_ADDR(page), &h, sizeof(h)); + activePage = page; + writeSlot = 0; +} + +// Reclaim the oldest (or an unused) page and compact stranded live entries +// into it. Caller holds spiLock. May recurse once via ringAppend when the +// stranded set fills the fresh page exactly — bounded by the page count and +// the WARM_NODE_COUNT <= 2 * kRecordsPerPage static_assert. +void WarmNodeStore::ringRotate() +{ + uint8_t target = 0; + if (activePage != 0xFF) { + // Lowest-seq valid page, preferring erased pages; never the active one + uint32_t bestSeq = 0; + bool found = false; + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { + if (p == activePage) + continue; + WarmPageHeader h; + if (!ringReadHeader(p, h)) { + target = p; // erased/invalid page: free real estate, take it + found = true; + break; + } + if (!found || static_cast(h.seq - bestSeq) < 0) { + target = p; + bestSeq = h.seq; + found = true; + } + } + } + + // Capture live entries stranded in the page we're about to erase + int stranded[WARM_NODE_COUNT] = {}; + int nStranded = 0; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) { + if (entries[i].num && pageOf[i] == target) + stranded[nStranded++] = static_cast(i); + if (pageOf[i] == target) + pageOf[i] = 0xFF; + } + + ringOpenPage(target); + + for (int k = 0; k < nStranded; k++) + ringAppend(entries[stranded[k]], stranded[k]); +} + +// Append one record. Caller holds spiLock. +void WarmNodeStore::ringAppend(const WarmNodeEntry &rec, int storeSlot) +{ + if (activePage == 0xFF || writeSlot >= kRecordsPerPage) + ringRotate(); + const uint32_t addr = + WARM_FLASH_PAGE_ADDR(activePage) + sizeof(WarmPageHeader) + static_cast(writeSlot) * sizeof(WarmNodeEntry); + flash_nrf5x_write(addr, &rec, sizeof(rec)); + writeSlot++; + if (storeSlot >= 0) + pageOf[storeSlot] = activePage; + dirty = true; +} + +void WarmNodeStore::persistEntry(const WarmNodeEntry &e) +{ + concurrency::LockGuard g(spiLock); + ringAppend(e, static_cast(&e - entries)); +} + +void WarmNodeStore::persistRemove(NodeNum num, int storeSlot) +{ + if (storeSlot >= 0 && storeSlot < static_cast(WARM_NODE_COUNT)) + pageOf[storeSlot] = 0xFF; + WarmNodeEntry tomb; + memset(&tomb, 0, sizeof(tomb)); + tomb.num = num; + tomb.last_heard = WARM_RING_TOMBSTONE; + concurrency::LockGuard g(spiLock); + ringAppend(tomb, -1); +} + +void WarmNodeStore::persistClear() +{ + concurrency::LockGuard g(spiLock); + flash_nrf5x_flush(); + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) + flash_nrf5x_erase(WARM_FLASH_PAGE_ADDR(p)); + activePage = 0xFF; + writeSlot = 0; + nextSeq = 1; + dirty = false; // the erased ring already reflects the empty store +} + +void WarmNodeStore::load() +{ + if (!entries) + return; + concurrency::LockGuard g(spiLock); + + // Order valid pages by ascending seq so replay applies oldest first + uint8_t order[WARM_FLASH_PAGES] = {}; + uint32_t seqs[WARM_FLASH_PAGES] = {}; + uint8_t nValid = 0; + uint8_t nCorrupt = 0; + for (uint8_t p = 0; p < WARM_FLASH_PAGES; p++) { + WarmPageHeader h; + if (!ringReadHeader(p, h)) { + // An erased page reads back all-ones; any other magic is a + // partially-written or bit-rotted header we're dropping, so flag it + // rather than silently treating the loss as a clean empty ring. + if (h.magic != 0xFFFFFFFFu) + nCorrupt++; + continue; + } + uint8_t pos = nValid; + while (pos > 0 && static_cast(h.seq - seqs[pos - 1]) < 0) { + order[pos] = order[pos - 1]; + seqs[pos] = seqs[pos - 1]; + pos--; + } + order[pos] = p; + seqs[pos] = h.seq; + nValid++; + } + + if (nValid == 0) { + activePage = 0xFF; + writeSlot = 0; + nextSeq = 1; + if (nCorrupt) + LOG_WARN("WarmStore: ring unreadable (%u corrupt page(s)), starting empty", nCorrupt); + else + LOG_INFO("WarmStore: ring empty, starting fresh"); + return; + } + + uint32_t replayed = 0; + for (uint8_t k = 0; k < nValid; k++) { + const uint8_t p = order[k]; + uint16_t slot = 0; + for (; slot < kRecordsPerPage; slot++) { + WarmNodeEntry rec; + flash_nrf5x_read(&rec, WARM_FLASH_PAGE_ADDR(p) + sizeof(WarmPageHeader) + (uint32_t)slot * sizeof(rec), sizeof(rec)); + if (rec.num == 0xFFFFFFFFu) + break; // erased space: end of this page's records (append-only) + if (rec.num == 0) + continue; // unexpected; skip defensively + replayed++; + if (rec.last_heard == WARM_RING_TOMBSTONE) { + WarmNodeEntry *e = find(rec.num); + if (e) { + pageOf[e - entries] = 0xFF; + memset(e, 0, sizeof(*e)); + } + } else { + const WarmNodeEntry *e = place(rec.num, rec.last_heard, rec.public_key); + if (e) + pageOf[e - entries] = p; + } + } + if (k == nValid - 1) { // newest page becomes the active head + activePage = p; + writeSlot = slot; + nextSeq = seqs[k] + 1; + } + } + if (nCorrupt) + LOG_WARN("WarmStore: dropped %u corrupt ring page(s) during replay; some warm nodes may be lost", nCorrupt); + LOG_INFO("WarmStore: replayed %u ring records -> %u live nodes (page %u, slot %u)", (unsigned)replayed, (unsigned)count(), + activePage, writeSlot); +} + +bool WarmNodeStore::save() +{ + if (!powerHAL_isPowerLevelSafe()) { + LOG_ERROR("Error: trying to save WarmStore on unsafe device power level."); + return false; + } + concurrency::LockGuard g(spiLock); + flash_nrf5x_flush(); + return true; +} + +#else // !(defined(ARCH_NRF52) && defined(NRF52840_XXAA)) -------------------- + +void WarmNodeStore::persistEntry(const WarmNodeEntry &e) +{ + (void)e; + dirty = true; +} + +void WarmNodeStore::persistRemove(NodeNum num, int storeSlot) +{ + (void)num; + (void)storeSlot; + dirty = true; +} + +void WarmNodeStore::persistClear() +{ + dirty = true; +} + +#ifdef FSCom + +// ---- File persistence: /prefs/warm.dat snapshots ---------------------------- + +// Compact occupied slots to the front of `dst`; returns the count. +static uint16_t packEntries(const WarmNodeEntry *src, WarmNodeEntry *dst) +{ + uint16_t n = 0; + for (size_t i = 0; i < WARM_NODE_COUNT; i++) + if (src[i].num) + dst[n++] = src[i]; + return n; +} + +void WarmNodeStore::load() +{ + if (!entries) + return; + concurrency::LockGuard g(spiLock); + auto f = FSCom.open(warmFileName, FILE_O_READ); + if (!f) + return; + WarmStoreHeader h; + bool ok = (size_t)f.read((uint8_t *)&h, sizeof(h)) == sizeof(h) && h.magic == WARM_STORE_MAGIC && + h.entrySize == sizeof(WarmNodeEntry) && h.count <= WARM_NODE_COUNT; + if (ok && h.count) { + const size_t len = (size_t)h.count * sizeof(WarmNodeEntry); + ok = (size_t)f.read((uint8_t *)entries, len) == len && crc32Buffer(entries, len) == h.crc; + if (!ok) + memset(entries, 0, WARM_NODE_COUNT * sizeof(WarmNodeEntry)); + } + f.close(); + if (ok) + LOG_INFO("WarmStore: loaded %u warm nodes from %s", h.count, warmFileName); + else + LOG_WARN("WarmStore: %s invalid, starting empty", warmFileName); +} + +bool WarmNodeStore::save() +{ + if (!entries) + return false; + if (!powerHAL_isPowerLevelSafe()) { + LOG_ERROR("Error: trying to save WarmStore on unsafe device power level."); + return false; + } + + std::vector packed(WARM_NODE_COUNT); + WarmStoreHeader h; + h.magic = WARM_STORE_MAGIC; + h.reserved = 0; + h.count = packEntries(entries, packed.data()); + h.entrySize = sizeof(WarmNodeEntry); + h.crc = crc32Buffer(packed.data(), h.count * sizeof(WarmNodeEntry)); + + spiLock->lock(); + FSCom.mkdir("/prefs"); + spiLock->unlock(); + + auto f = SafeFile(warmFileName, false); + f.write((const uint8_t *)&h, sizeof(h)); + f.write((const uint8_t *)packed.data(), h.count * sizeof(WarmNodeEntry)); + bool ok = f.close(); + if (!ok) + LOG_ERROR("WarmStore: can't write %s", warmFileName); + else + LOG_DEBUG("WarmStore: saved %u warm nodes to %s", h.count, warmFileName); + return ok; +} + +#else + +void WarmNodeStore::load() {} +bool WarmNodeStore::save() +{ + return true; +} + +#endif // FSCom +#endif // defined(ARCH_NRF52) && defined(NRF52840_XXAA) + +#endif // WARM_NODE_COUNT > 0 diff --git a/src/mesh/WarmNodeStore.h b/src/mesh/WarmNodeStore.h new file mode 100644 index 00000000000..a91d6cd4aca --- /dev/null +++ b/src/mesh/WarmNodeStore.h @@ -0,0 +1,123 @@ +#pragma once + +#include "MeshTypes.h" +#include "mesh-pb-constants.h" +#include +#include + +#if WARM_NODE_COUNT > 0 + +/** + * Warm ("long-tail") node tier + * + * Holds a minimal identity record for nodes evicted from the hot NodeInfoLite + * store: NodeNum, last_heard and the Curve25519 public key. The tier exists + * primarily so DMs to/from an evicted node keep encrypting/decrypting — the + * public key is expensive to re-learn (requires a NodeInfo exchange) while + * everything else in NodeInfoLite is rebuilt from traffic within seconds. + * + * Flat fixed-capacity array, linear scan (lookups happen only on hot-store + * misses), LRU eviction by last_heard with key-bearing entries outranking + * keyless ones. + * + * Persistence: + * - nRF52840: a dedicated 12 KB raw-flash record-ring (3 × 4 KB pages at + * 0xEA000, directly below the stock LittleFS partition). Mutations append + * 40 B upsert/tombstone records via the same SoftDevice-safe flash HAL + * InternalFS uses; boot replays the pages in sequence order. When the + * active page fills, the oldest page is reclaimed: live entries stranded + * in it are re-appended first (compaction), then it is erased and becomes + * the new head. The app image must end below the region — + * nrf52840_s140_v7.ld enforces this at link time and + * extra_scripts/nrf52_warm_region.py guards boards on the framework + * default linker script. + * - everywhere else: /prefs/warm.dat snapshot on the node-DB save cadence. + */ +struct WarmNodeEntry { + NodeNum num; // 0 = empty slot + uint32_t last_heard; // recency for LRU ordering + uint8_t public_key[32]; // all-zero = no key (a real key is never all-zero) +}; +static_assert(sizeof(WarmNodeEntry) == 40, "WarmNodeEntry must stay 40 B — persistence format depends on it"); + +// NRF52840_XXAA is explicit: nRF52832 also defines ARCH_NRF52 but only has +// 512 KB flash (top at 0x80000), so 0xEA000 would be past its flash end. +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) +#define WARM_FLASH_PAGE_SIZE 4096u +#define WARM_FLASH_PAGES 3u +#define WARM_FLASH_REGION_BASE (0xED000u - WARM_FLASH_PAGES * WARM_FLASH_PAGE_SIZE) // 0xEA000 +#define WARM_FLASH_PAGE_ADDR(i) (WARM_FLASH_REGION_BASE + (i)*WARM_FLASH_PAGE_SIZE) +#endif + +class WarmNodeStore +{ + public: + WarmNodeStore(); + ~WarmNodeStore(); + WarmNodeStore(const WarmNodeStore &) = delete; + WarmNodeStore &operator=(const WarmNodeStore &) = delete; + + /// Remember an evicted hot node. Keyless candidates never displace keyed + /// entries; otherwise the oldest (keyless-first) entry is replaced. + /// @return true if the node was stored or updated + bool absorb(NodeNum num, uint32_t lastHeard, const uint8_t *key32 /* may be NULL */); + + /// Find and remove an entry (used when the node is re-admitted to the hot store). + bool take(NodeNum num, WarmNodeEntry &out); + + /// Copy the 32-byte public key for a node, if we have one. + bool copyKey(NodeNum num, uint8_t out[32]) const; + + bool contains(NodeNum num) const; + void remove(NodeNum num); + void clear(); + size_t count() const; + size_t capacity() const { return entries ? WARM_NODE_COUNT : 0; } + + /// Load persisted entries (called once at boot, after the node DB loads). + void load(); + /// Durability point, piggybacked on the node-database save cadence. On the + /// ring backend this flushes the shared flash page cache; on the file + /// backend it writes the warm.dat snapshot. + bool saveIfDirty(); + + private: + WarmNodeEntry *entries = nullptr; // WARM_NODE_COUNT slots; PSRAM on ESP32 when available + bool dirty = false; + + WarmNodeEntry *find(NodeNum num) const; + // Internal slot-placement shared by absorb() and ring replay: applies the + // keyed-first admission policy without touching persistence. + WarmNodeEntry *place(NodeNum num, uint32_t lastHeard, const uint8_t *key32); + + // Persistence hooks called from the mutation paths. File backend: mark + // dirty. Ring backend: append an upsert/tombstone record (+ mark dirty). + void persistEntry(const WarmNodeEntry &e); // e must point into entries[] + void persistRemove(NodeNum num, int storeSlot); + void persistClear(); + +#if defined(ARCH_NRF52) && defined(NRF52840_XXAA) + // ---- raw-flash record-ring state ---- + struct WarmPageHeader { + uint32_t magic; // WARM_RING_MAGIC + uint32_t seq; // page generation; 0xFFFFFFFF = erased/unused + }; + static_assert(sizeof(WarmPageHeader) == 8, "page header is part of the flash format"); + static constexpr uint16_t kRecordsPerPage = (WARM_FLASH_PAGE_SIZE - sizeof(WarmPageHeader)) / sizeof(WarmNodeEntry); // 102 + static_assert(WARM_NODE_COUNT <= 2 * ((WARM_FLASH_PAGE_SIZE - 8) / 40), "live set must fit the ring with one page reclaimed"); + + uint8_t activePage = 0xFF; // 0xFF = no page opened yet (fresh/erased ring) + uint16_t writeSlot = 0; // next free record slot in the active page + uint32_t nextSeq = 1; // seq for the next page opened + uint8_t pageOf[WARM_NODE_COUNT]; // flash page holding each RAM slot's newest record; 0xFF = none + + void ringAppend(const WarmNodeEntry &rec, int storeSlot /* -1 for tombstones */); + void ringRotate(); // reclaim oldest page, compacting stranded live entries + void ringOpenPage(uint8_t page); // erase + write header (seq = nextSeq++) + bool ringReadHeader(uint8_t page, WarmPageHeader &h) const; +#endif + + bool save(); +}; + +#endif // WARM_NODE_COUNT > 0 diff --git a/src/mesh/generated/meshtastic/deviceonly.pb.h b/src/mesh/generated/meshtastic/deviceonly.pb.h index 474b332a8da..ed4b0792db1 100644 --- a/src/mesh/generated/meshtastic/deviceonly.pb.h +++ b/src/mesh/generated/meshtastic/deviceonly.pb.h @@ -452,7 +452,7 @@ extern const pb_msgdesc_t meshtastic_BackupPreferences_msg; /* Maximum encoded size of messages (where known) */ /* meshtastic_NodeDatabase_size depends on runtime parameters */ #define MESHTASTIC_MESHTASTIC_DEVICEONLY_PB_H_MAX_SIZE meshtastic_BackupPreferences_size -#define meshtastic_BackupPreferences_size 2432 +#define meshtastic_BackupPreferences_size 2468 #define meshtastic_ChannelFile_size 718 #define meshtastic_DeviceState_size 1944 #define meshtastic_NodeEnvironmentEntry_size 170 diff --git a/src/mesh/mesh-pb-constants.h b/src/mesh/mesh-pb-constants.h index b4d99901110..9857d72277c 100644 --- a/src/mesh/mesh-pb-constants.h +++ b/src/mesh/mesh-pb-constants.h @@ -48,7 +48,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_POSITIONDB 1 #else #define MESHTASTIC_EXCLUDE_POSITIONDB 0 -#endif +#endif // STM32WL #endif #ifndef MESHTASTIC_EXCLUDE_TELEMETRYDB @@ -56,7 +56,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_TELEMETRYDB 1 #else #define MESHTASTIC_EXCLUDE_TELEMETRYDB 0 -#endif +#endif // STM32WL #endif #ifndef MESHTASTIC_EXCLUDE_ENVIRONMENTDB @@ -64,7 +64,7 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_ENVIRONMENTDB 1 #else #define MESHTASTIC_EXCLUDE_ENVIRONMENTDB 0 -#endif +#endif // STM32WL #endif #ifndef MESHTASTIC_EXCLUDE_STATUSDB @@ -72,31 +72,62 @@ static_assert(sizeof(meshtastic_NodeInfoLite) <= 130, "NodeInfoLite size increas #define MESHTASTIC_EXCLUDE_STATUSDB 1 #else #define MESHTASTIC_EXCLUDE_STATUSDB 0 -#endif +#endif // STM32WL #endif -/// max number of nodes allowed in the nodeDB +/// max number of nodes allowed in the nodeDB (the full-NodeInfoLite "hot + +/// short-tail" store). +/// Long-tail identity retention for evicted nodes is handled by the warm tier +/// (WARM_NODE_COUNT). 120 keeps nodes.proto (~14 KB typical / ~27 KB worst) +/// comfortably inside the stock 28 KB nRF52 LittleFS alongside prefs and the +/// message store — the warm tier persists outside LittleFS on nRF52840. #ifndef MAX_NUM_NODES #if defined(ARCH_STM32WL) #define MAX_NUM_NODES 10 +#else +#define MAX_NUM_NODES 120 +#endif // STM32WL +#endif + +/// Cap on each satellite map (position/telemetry/environment/status). Only the +/// MAX_SATELLITE_NODES most-recently-heard nodes keep satellite payloads; the +/// rest of the hot store carries just the 96 B NodeInfoLite header. This is +/// what bounds both heap (the maps are ~408 B/node worst case) and the +/// satellite share of nodes.proto. Deliberately NOT scaled with MAX_NUM_NODES: +/// the tier-1 "rich data" population stays at 40 regardless of how long the +/// short tail grows. +#ifndef MAX_SATELLITE_NODES +#define MAX_SATELLITE_NODES 40 +#endif + +/// Warm tier: number of 40 B {num, last_heard, public_key} records retained +/// for nodes evicted from the hot store, primarily so DMs to/from them keep +/// decrypting. 0 disables the tier entirely. Persisted to /prefs/warm.dat, +/// so the count is bounded by the platform's filesystem budget. +#ifndef WARM_NODE_COUNT +#if defined(ARCH_STM32WL) +#define WARM_NODE_COUNT 0 +#elif defined(NRF52840_XXAA) +// nRF52840: persisted in the dedicated 12 KB raw-flash record-ring below +// LittleFS (3 × 4 KB pages, append + replay + compact-on-rotate; see +// WarmNodeStore.h). 200 live entries across 306 record slots. +// +// Keyed on the NRF52840_XXAA build flag (a command-line -D from the board +// definition, defined before any header is processed) rather than the +// header-defined ARCH_NRF52. ARCH_NRF52 comes from platform/nrf52/architecture.h +// via configuration.h, which some include chains (MeshTypes.h -> WarmNodeStore.h +// -> here) reach AFTER mesh-pb-constants.h — so keying on it would let +// WARM_NODE_COUNT resolve to the generic fallback and then fail the ring-sizing +// static_assert in WarmNodeStore.h (which is itself gated on NRF52840_XXAA). +#define WARM_NODE_COUNT 200 #elif defined(ARCH_NRF52) -#define MAX_NUM_NODES 150 +// Other nRF52 (e.g. nRF52832): no raw-flash warm ring is reserved for these, so +// keep the warm tier off rather than risk a file-backed default on tight flash. +#define WARM_NODE_COUNT 0 #elif defined(CONFIG_IDF_TARGET_ESP32S3) -#include "Esp.h" -static inline int get_max_num_nodes() -{ - uint32_t flash_size = ESP.getFlashChipSize() / (1024 * 1024); // Convert Bytes to MB - if (flash_size >= 15) { - return 250; - } else if (flash_size >= 7) { - return 200; - } else { - return 100; - } -} -#define MAX_NUM_NODES get_max_num_nodes() +#define WARM_NODE_COUNT 2000 // PSRAM-backed when available; warm.dat ~80 KB #else -#define MAX_NUM_NODES 100 +#define WARM_NODE_COUNT 320 #endif #endif @@ -106,7 +137,7 @@ static inline int get_max_num_nodes() // Traffic Management module configuration // Enable per-variant by defining HAS_TRAFFIC_MANAGEMENT=1 in variant.h #ifndef HAS_TRAFFIC_MANAGEMENT -#define HAS_TRAFFIC_MANAGEMENT 0 +#define HAS_TRAFFIC_MANAGEMENT 1 #endif // HopScalingModule - variable hop module: dynamically adjusts broadcast hop_limit based on mesh density diff --git a/src/modules/AdminModule.cpp b/src/modules/AdminModule.cpp index 1511a709c58..b45e03db2d2 100644 --- a/src/modules/AdminModule.cpp +++ b/src/modules/AdminModule.cpp @@ -168,7 +168,7 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta LOG_INFO("PKC admin valid, but not auto-favoriting node %x because role==CLIENT_BASE", mp.from); } else { LOG_INFO("PKC admin valid. Auto-favoriting node %x", mp.from); - nodeInfoLiteSetBit(remoteNode, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + nodeDB->setProtectedFlag(remoteNode, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); } } } else { @@ -458,10 +458,13 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta LOG_INFO("Client received set_favorite_node command"); meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(r->set_favorite_node); if (node != NULL) { - nodeInfoLiteSetBit(node, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); - saveChanges(SEGMENT_NODEDATABASE, false); - if (screen) - screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens + if (nodeDB->setProtectedFlag(node, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true)) { + saveChanges(SEGMENT_NODEDATABASE, false); + if (screen) + screen->setFrames(graphics::Screen::FOCUS_PRESERVE); // <-- Rebuild screens + } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take + sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "favorite", r->set_favorite_node, MAX_NUM_NODES - 2); + } } break; } @@ -478,13 +481,17 @@ bool AdminModule::handleReceivedProtobuf(const meshtastic_MeshPacket &mp, meshta } case meshtastic_AdminMessage_set_ignored_node_tag: { LOG_INFO("Client received set_ignored_node command"); - meshtastic_NodeInfoLite *node = nodeDB->getMeshNode(r->set_ignored_node); + // Unlike the sibling node-targeted admin commands, create the entry if + // it's absent so the block sticks for a node we've not heard from yet + // (e.g. one a remote admin asks us to block) with no NodeInfo or key. + meshtastic_NodeInfoLite *node = nodeDB->getOrCreateMeshNode(r->set_ignored_node); if (node != NULL) { - nodeInfoLiteSetBit(node, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); - nodeDB->eraseNodeSatellites(node->num); - node->public_key.size = 0; - memset(node->public_key.bytes, 0, sizeof(node->public_key.bytes)); - saveChanges(SEGMENT_NODEDATABASE, false); + if (nodeDB->setProtectedFlag(node, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)) { + nodeDB->eraseNodeSatellites(node->num); + saveChanges(SEGMENT_NODEDATABASE, false); + } else if (mp.from == 0) { // local request from the phone — tell the user why it didn't take + sendWarning(NodeDB::PROTECTED_CAP_WARN_FMT, "ignore", r->set_ignored_node, MAX_NUM_NODES - 2); + } } break; } diff --git a/src/modules/TraceRouteModule.cpp b/src/modules/TraceRouteModule.cpp index b6cb5d04181..915203ed16b 100644 --- a/src/modules/TraceRouteModule.cpp +++ b/src/modules/TraceRouteModule.cpp @@ -8,6 +8,10 @@ #include "meshUtils.h" #include +#if HAS_TRAFFIC_MANAGEMENT +#include "modules/TrafficManagementModule.h" +#endif + extern graphics::Screen *screen; TraceRouteModule *traceRouteModule; @@ -323,6 +327,14 @@ void TraceRouteModule::maybeSetNextHop(NodeNum target, uint8_t nextHopByte) LOG_INFO("Updating next-hop for 0x%08x to 0x%02x based on traceroute", target, nextHopByte); node->next_hop = nextHopByte; } + +#if HAS_TRAFFIC_MANAGEMENT + // Mirror into the TMM overflow cache. Traceroute is the highest-confidence + // source (full known route), and this captures the target even when it isn't + // in the hot NodeDB — same rationale as the ACK-confirmed path in NextHopRouter. + if (trafficManagementModule) + trafficManagementModule->setNextHop(target, nextHopByte); +#endif } void TraceRouteModule::processUpgradedPacket(const meshtastic_MeshPacket &mp) diff --git a/src/modules/TrafficManagementModule.cpp b/src/modules/TrafficManagementModule.cpp index 4c73c2947c5..07a570ec8b3 100644 --- a/src/modules/TrafficManagementModule.cpp +++ b/src/modules/TrafficManagementModule.cpp @@ -2,9 +2,11 @@ #if HAS_TRAFFIC_MANAGEMENT +#include "Channels.h" #include "Default.h" #include "MeshService.h" #include "NodeDB.h" +#include "PositionPrecision.h" #include "Router.h" #include "TypeConversions.h" #include "airtime.h" @@ -13,6 +15,7 @@ #include "mesh-pb-constants.h" #include "meshUtils.h" #include +#include #include #define TM_LOG_DEBUG(fmt, ...) LOG_DEBUG("[TM] " fmt, ##__VA_ARGS__) @@ -27,8 +30,6 @@ namespace { constexpr uint32_t kMaintenanceIntervalMs = 60 * 1000UL; // Cache cleanup interval -constexpr uint32_t kUnknownResetMs = 60 * 1000UL; // Unknown packet window -constexpr uint8_t kMaxCuckooKicks = 16; // Max displacement chain length // NodeInfo direct response: enforced maximum hops by device role // Both use maxHops logic (respond when hopsAway <= threshold) @@ -65,17 +66,6 @@ uint8_t sanitizePositionPrecision(uint8_t precision) return 32; } -/** - * Check if a timestamp is within a time window. - * Handles wrap-around correctly using unsigned subtraction. - */ -bool isWithinWindow(uint32_t nowMs, uint32_t startMs, uint32_t intervalMs) -{ - if (intervalMs == 0 || startMs == 0) - return false; - return (nowMs - startMs) < intervalMs; -} - /** * Truncate lat/lon to specified precision for position deduplication. * @@ -162,23 +152,11 @@ TrafficManagementModule::TrafficManagementModule() : MeshModule("TrafficManageme encryptedOk = true; // Can process encrypted packets stats = meshtastic_TrafficManagementStats_init_zero; - // Initialize rolling epoch for relative timestamps - cacheEpochMs = millis(); - - // Calculate adaptive time resolutions from config (config changes require reboot) - // Resolution = max(60, min(339, interval/2)) for ~24 hour range with good precision - posTimeResolution = calcTimeResolution(Default::getConfiguredOrDefault( - moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); - rateTimeResolution = calcTimeResolution(moduleConfig.traffic_management.rate_limit_window_secs); - unknownTimeResolution = calcTimeResolution(kUnknownResetMs / 1000); // ~5 min default - const auto &cfg = moduleConfig.traffic_management; TM_LOG_INFO("Enabled: pos_dedup=%d nodeinfo_resp=%d rate_limit=%d drop_unknown=%d exhaust_telem=%d exhaust_pos=%d " "preserve_hops=%d", cfg.position_dedup_enabled, cfg.nodeinfo_direct_response, cfg.rate_limit_enabled, cfg.drop_unknown_enabled, cfg.exhaust_hop_telemetry, cfg.exhaust_hop_position, cfg.router_preserve_hops); - TM_LOG_DEBUG("Time resolutions: pos=%us, rate=%us, unknown=%us", posTimeResolution, rateTimeResolution, - unknownTimeResolution); // Allocate unified cache (10 bytes/entry for all platforms) #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 @@ -203,27 +181,16 @@ TrafficManagementModule::TrafficManagementModule() : MeshModule("TrafficManageme #endif // TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - TM_LOG_INFO("Allocating NodeInfo cache: target=%u occupancy=%u%% payload=%u bytes (PSRAM) tags=%u bytes (%u-bit, %u slots, " - "%u buckets x %u)", - static_cast(nodeInfoTargetEntries()), static_cast(nodeInfoTargetOccupancyPercent()), - static_cast(nodeInfoTargetEntries() * sizeof(NodeInfoPayloadEntry)), - static_cast(nodeInfoIndexMetadataBudgetBytes()), static_cast(nodeInfoTagBits()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoBucketCount()), - static_cast(nodeInfoBucketSize())); - - nodeInfoIndex = static_cast(calloc(nodeInfoIndexMetadataBudgetBytes(), sizeof(uint8_t))); - if (!nodeInfoIndex) { - TM_LOG_WARN("NodeInfo index allocation failed; direct responses will fall back to NodeDB"); + TM_LOG_INFO("Allocating NodeInfo cache: %u entries, %u bytes (PSRAM flat array)", + static_cast(nodeInfoTargetEntries()), + static_cast(nodeInfoTargetEntries() * sizeof(NodeInfoPayloadEntry))); + + nodeInfoPayload = static_cast(ps_calloc(nodeInfoTargetEntries(), sizeof(NodeInfoPayloadEntry))); + if (nodeInfoPayload) { + nodeInfoPayloadFromPsram = true; + TM_LOG_INFO("NodeInfo PSRAM cache ready"); } else { - nodeInfoPayload = static_cast(ps_calloc(nodeInfoTargetEntries(), sizeof(NodeInfoPayloadEntry))); - if (nodeInfoPayload) { - nodeInfoPayloadFromPsram = true; - TM_LOG_INFO("NodeInfo bucketed cuckoo cache ready"); - } else { - TM_LOG_WARN("NodeInfo PSRAM payload allocation failed; direct responses will fall back to NodeDB"); - free(nodeInfoIndex); - nodeInfoIndex = nullptr; - } + TM_LOG_WARN("NodeInfo PSRAM payload allocation failed; direct responses will fall back to NodeDB"); } #else TM_LOG_DEBUG("NodeInfo PSRAM cache not available on this target"); @@ -255,11 +222,6 @@ TrafficManagementModule::~TrafficManagementModule() delete[] nodeInfoPayload; nodeInfoPayload = nullptr; } - - if (nodeInfoIndex) { - free(nodeInfoIndex); - nodeInfoIndex = nullptr; - } } // ============================================================================= @@ -292,17 +254,11 @@ void TrafficManagementModule::incrementStat(uint32_t *field) } // ============================================================================= -// Cuckoo Hash Table Operations +// Flat Unified Cache Operations // ============================================================================= /** - * Find an existing entry for the given node. - * - * Cuckoo hashing guarantees that if an entry exists, it's in one of exactly - * two locations: hash1(node) or hash2(node). This provides O(1) lookup. - * - * @param node NodeNum to search for - * @return Pointer to entry if found, nullptr otherwise + * Find an existing entry for the given node (linear scan). */ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(NodeNum node) { @@ -313,35 +269,26 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findEntry(N if (!cache || node == 0) return nullptr; - // Check primary location - uint16_t h1 = cuckooHash1(node); - if (cache[h1].node == node) - return &cache[h1]; - - // Check alternate location - uint16_t h2 = cuckooHash2(node); - if (cache[h2].node == node) - return &cache[h2]; - + for (uint16_t i = 0; i < cacheSize(); i++) { + if (cache[i].node == node) + return &cache[i]; + } return nullptr; #endif } /** - * Find or create an entry for the given node using cuckoo hashing. - * - * If the node exists, returns the existing entry. Otherwise, attempts to - * insert a new entry using cuckoo displacement: + * Find or create an entry for the given node. * - * 1. Try to insert at h1(node) - if empty, done - * 2. Try to insert at h2(node) - if empty, done - * 3. Kick existing entry from h1 to its alternate location - * 4. Repeat up to kMaxCuckooKicks times - * 5. If cycle detected or max kicks exceeded, evict oldest entry + * One linear pass tracks the match, the first empty slot, and the eviction + * victim. When the cache is full, the victim is the stalest entry (largest + * of its three relative timestamps is smallest), preferring entries without + * a next_hop hint — those hints are the long-tail routing state the cache + * exists to keep, and the maintenance sweep never ages them out. * * @param node NodeNum to find or create * @param isNew Set to true if a new entry was created - * @return Pointer to entry, or nullptr if allocation failed + * @return Pointer to entry, or nullptr if the cache is unavailable */ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreateEntry(NodeNum node, bool *isNew) { @@ -351,304 +298,85 @@ TrafficManagementModule::UnifiedCacheEntry *TrafficManagementModule::findOrCreat *isNew = false; return nullptr; #else - if (!cache || node == 0) { - if (isNew) - *isNew = false; + if (isNew) + *isNew = false; + if (!cache || node == 0) return nullptr; - } - // Check if entry already exists (O(1) lookup) - uint16_t h1 = cuckooHash1(node); - if (cache[h1].node == node) { - if (isNew) - *isNew = false; - return &cache[h1]; - } + UnifiedCacheEntry *empty = nullptr; + UnifiedCacheEntry *victim = nullptr; + bool victimHasHop = true; + uint8_t victimRecency = UINT8_MAX; - uint16_t h2 = cuckooHash2(node); - if (cache[h2].node == node) { - if (isNew) - *isNew = false; - return &cache[h2]; - } - - // Entry doesn't exist - try to insert - - // Prefer empty slot at h1 - if (cache[h1].node == 0) { - memset(&cache[h1], 0, sizeof(UnifiedCacheEntry)); - cache[h1].node = node; - if (isNew) - *isNew = true; - return &cache[h1]; - } - - // Try empty slot at h2 - if (cache[h2].node == 0) { - memset(&cache[h2], 0, sizeof(UnifiedCacheEntry)); - cache[h2].node = node; - if (isNew) - *isNew = true; - return &cache[h2]; - } - - // Both slots occupied - perform cuckoo displacement - // Start by kicking entry at h1 to its alternate location - UnifiedCacheEntry displaced = cache[h1]; - memset(&cache[h1], 0, sizeof(UnifiedCacheEntry)); - cache[h1].node = node; - - for (uint8_t kicks = 0; kicks < kMaxCuckooKicks; kicks++) { - // Find alternate location for displaced entry - uint16_t altH1 = cuckooHash1(displaced.node); - uint16_t altH2 = cuckooHash2(displaced.node); - uint16_t altSlot = (altH1 == h1) ? altH2 : altH1; - - if (cache[altSlot].node == 0) { - // Found empty slot - insert displaced entry - cache[altSlot] = displaced; - if (isNew) - *isNew = true; - return &cache[h1]; + for (uint16_t i = 0; i < cacheSize(); i++) { + UnifiedCacheEntry &e = cache[i]; + if (e.node == node) + return &e; + if (e.node == 0) { + if (!empty) + empty = &e; + continue; + } + if (empty) + continue; // an empty slot beats any victim; stop scoring + const bool hasHop = e.next_hop != 0; + // Age in pos-ticks (8-bit modular, wraps correctly). Entries with no + // pos state (pos_time==0) score as maximally old (age=currentPosTick()). + const uint8_t nowPosTick = currentPosTick(); + const uint8_t posAge = static_cast(nowPosTick - e.pos_time); + // Blend in rate/unknown ages scaled to pos-tick units (coarser = conservative). + const uint8_t rateAgePosScale = + static_cast(static_cast((currentRateTick() - e.getRateTime()) & 0x0F) * 5 / 3); + const uint8_t unknownAgePosScale = + static_cast(static_cast((currentUnknownTick() - e.getUnknownTime()) & 0x0F) / 6); + uint8_t recency = posAge; + if (e.rate_count != 0 && rateAgePosScale > recency) + recency = rateAgePosScale; + if (e.unknown_count != 0 && unknownAgePosScale > recency) + recency = unknownAgePosScale; + if (!victim || (hasHop == victimHasHop ? recency < victimRecency : !hasHop)) { + victim = &e; + victimHasHop = hasHop; + victimRecency = recency; } - - // Kick entry from alternate slot - UnifiedCacheEntry temp = cache[altSlot]; - cache[altSlot] = displaced; - displaced = temp; - h1 = altSlot; } - // Cuckoo cycle detected or max kicks exceeded. - // The displaced entry has no valid cuckoo slot — drop it to preserve cache integrity. - // Placing it at an arbitrary slot would make it unreachable by findEntry(). - TM_LOG_DEBUG("Cuckoo cycle, evicting node 0x%08x", displaced.node); - + UnifiedCacheEntry *slot = empty ? empty : victim; + if (!slot) + return nullptr; + if (!empty) + TM_LOG_DEBUG("Unified cache full, evicting node 0x%08x", slot->node); + memset(slot, 0, sizeof(UnifiedCacheEntry)); + slot->node = node; if (isNew) *isNew = true; - return &cache[cuckooHash1(node)]; + return slot; #endif } const TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findNodeInfoEntry(NodeNum node) const { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) + if (!nodeInfoPayload || node == 0) return nullptr; - uint16_t payloadIndex = findNodeInfoPayloadIndex(node); - if (payloadIndex >= nodeInfoTargetEntries()) - return nullptr; - - return &nodeInfoPayload[payloadIndex]; -#else - (void)node; - return nullptr; -#endif -} - -uint16_t TrafficManagementModule::encodeNodeInfoTag(uint16_t payloadIndex) const -{ - if (payloadIndex >= nodeInfoTargetEntries()) - return 0; - return static_cast(payloadIndex + 1u); -} - -uint16_t TrafficManagementModule::decodeNodeInfoPayloadIndex(uint16_t tag) const -{ - if (tag == 0 || tag > nodeInfoTargetEntries()) - return UINT16_MAX; - return static_cast(tag - 1u); -} - -uint16_t TrafficManagementModule::getNodeInfoTag(uint16_t slot) const -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || slot >= nodeInfoIndexSlots()) - return 0; - - const uint32_t bitOffset = static_cast(slot) * nodeInfoTagBits(); - const uint16_t byteOffset = static_cast(bitOffset >> 3); - const uint8_t shift = static_cast(bitOffset & 7u); - uint32_t packed = 0; - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset]); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 1u]) << 8; - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 2u]) << 16; - - return static_cast((packed >> shift) & nodeInfoTagMask()); -#else - (void)slot; - return 0; -#endif -} - -void TrafficManagementModule::setNodeInfoTag(uint16_t slot, uint16_t tag) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || slot >= nodeInfoIndexSlots()) - return; - - const uint16_t normalizedTag = static_cast(tag & nodeInfoTagMask()); - const uint32_t bitOffset = static_cast(slot) * nodeInfoTagBits(); - const uint16_t byteOffset = static_cast(bitOffset >> 3); - const uint8_t shift = static_cast(bitOffset & 7u); - uint32_t packed = 0; - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset]); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 1u]) << 8; - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - packed |= static_cast(nodeInfoIndex[byteOffset + 2u]) << 16; - - const uint32_t mask = static_cast(nodeInfoTagMask()) << shift; - packed = (packed & ~mask) | ((static_cast(normalizedTag) << shift) & mask); - - if (byteOffset < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset] = static_cast(packed & 0xFFu); - if (static_cast(byteOffset + 1u) < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset + 1u] = static_cast((packed >> 8) & 0xFFu); - if (static_cast(byteOffset + 2u) < nodeInfoIndexMetadataBudgetBytes()) - nodeInfoIndex[byteOffset + 2u] = static_cast((packed >> 16) & 0xFFu); -#else - (void)slot; - (void)tag; -#endif -} - -uint16_t TrafficManagementModule::findNodeInfoPayloadIndex(NodeNum node) const -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) - return UINT16_MAX; - - const uint16_t buckets[2] = {nodeInfoHash1(node), nodeInfoHash2(node)}; - - for (uint8_t b = 0; b < 2; b++) { - const uint16_t base = static_cast(buckets[b] * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - uint16_t tag = getNodeInfoTag(static_cast(base + slot)); - if (tag == 0) - continue; - - uint16_t payloadIndex = decodeNodeInfoPayloadIndex(tag); - if (payloadIndex >= nodeInfoTargetEntries()) - continue; - - if (nodeInfoPayload[payloadIndex].node == node) - return payloadIndex; - } + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + if (nodeInfoPayload[i].node == node) + return &nodeInfoPayload[i]; } - - return UINT16_MAX; -#else - (void)node; - return UINT16_MAX; -#endif -} - -bool TrafficManagementModule::removeNodeInfoIndexEntry(NodeNum node, uint16_t payloadIndex) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || node == 0 || payloadIndex >= nodeInfoTargetEntries()) - return false; - - const uint16_t payloadTag = encodeNodeInfoTag(payloadIndex); - if (payloadTag == 0) - return false; - const uint16_t buckets[2] = {nodeInfoHash1(node), nodeInfoHash2(node)}; - - for (uint8_t b = 0; b < 2; b++) { - const uint16_t base = static_cast(buckets[b] * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - const uint16_t indexSlot = static_cast(base + slot); - if (getNodeInfoTag(indexSlot) == payloadTag) { - setNodeInfoTag(indexSlot, 0); - return true; - } - } - } - - return false; + return nullptr; #else (void)node; - (void)payloadIndex; - return false; -#endif -} - -uint16_t TrafficManagementModule::allocateNodeInfoPayloadSlot() -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload) - return UINT16_MAX; - - for (uint16_t tries = 0; tries < nodeInfoTargetEntries(); tries++) { - uint16_t idx = static_cast((nodeInfoAllocHint + tries) % nodeInfoTargetEntries()); - if (nodeInfoPayload[idx].node == 0) { - nodeInfoAllocHint = static_cast((idx + 1u) % nodeInfoTargetEntries()); - return idx; - } - } -#endif - return UINT16_MAX; -} - -uint16_t TrafficManagementModule::evictNodeInfoPayloadSlot() -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex) - return UINT16_MAX; - - for (uint16_t tries = 0; tries < nodeInfoTargetEntries(); tries++) { - uint16_t idx = static_cast(nodeInfoEvictCursor % nodeInfoTargetEntries()); - nodeInfoEvictCursor = static_cast((nodeInfoEvictCursor + 1u) % nodeInfoTargetEntries()); - - NodeNum oldNode = nodeInfoPayload[idx].node; - if (oldNode == 0) - continue; - - removeNodeInfoIndexEntry(oldNode, idx); // best effort; cache tolerates occasional stale miss - nodeInfoPayload[idx].node = 0; - return idx; - } -#endif - return UINT16_MAX; -} - -bool TrafficManagementModule::tryInsertNodeInfoEntryInBucket(uint16_t bucket, uint16_t tag) -{ -#if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex || !nodeInfoPayload || bucket >= nodeInfoBucketCount() || tag == 0) - return false; - - const uint16_t base = static_cast(bucket * nodeInfoBucketSize()); - for (uint8_t slot = 0; slot < nodeInfoBucketSize(); slot++) { - const uint16_t indexSlot = static_cast(base + slot); - const uint16_t existingTag = getNodeInfoTag(indexSlot); - if (existingTag == 0) { - setNodeInfoTag(indexSlot, tag); - return true; - } - - // Opportunistically reuse stale tags that point at empty/invalid payload slots. - const uint16_t payloadIndex = decodeNodeInfoPayloadIndex(existingTag); - if (payloadIndex >= nodeInfoTargetEntries() || nodeInfoPayload[payloadIndex].node == 0) { - setNodeInfoTag(indexSlot, tag); - return true; - } - } -#else - (void)bucket; - (void)tag; + return nullptr; #endif - return false; } +/** + * Find or create a NodeInfo payload entry (linear scan of the flat PSRAM + * array). One pass tracks the match, the first empty slot, and the LRU + * victim by lastObservedMs (wrap-safe age). NodeInfo traffic is low-rate, + * so the O(n) scan is negligible. + */ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCreateNodeInfoEntry(NodeNum node, bool *usedEmptySlot) { @@ -656,88 +384,40 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr *usedEmptySlot = false; #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || node == 0) + if (!nodeInfoPayload || node == 0) return nullptr; - uint16_t existing = findNodeInfoPayloadIndex(node); - if (existing < nodeInfoTargetEntries()) - return &nodeInfoPayload[existing]; - - const uint16_t beforeCount = countNodeInfoEntriesLocked(); - - uint16_t payloadIndex = allocateNodeInfoPayloadSlot(); - if (payloadIndex == UINT16_MAX) { - payloadIndex = evictNodeInfoPayloadSlot(); - if (payloadIndex == UINT16_MAX) - return nullptr; - } - - nodeInfoPayload[payloadIndex].node = node; - - // 4-way bucketed cuckoo insertion mirrors Cuckoo Filter practice from - // Fan et al. (CoNEXT 2014): high occupancy with short relocation chains. - uint16_t pending = encodeNodeInfoTag(payloadIndex); - uint16_t h1 = nodeInfoHash1(node); - uint16_t h2 = nodeInfoHash2(node); - - if (!tryInsertNodeInfoEntryInBucket(h1, pending) && !tryInsertNodeInfoEntryInBucket(h2, pending)) { - uint16_t currentBucket = h1; - for (uint8_t kicks = 0; kicks < kMaxCuckooKicks; kicks++) { - const uint16_t base = static_cast(currentBucket * nodeInfoBucketSize()); - const uint16_t kickSlot = static_cast((node + kicks) & (nodeInfoBucketSize() - 1u)); - const uint16_t pos = static_cast(base + kickSlot); - - uint16_t displaced = getNodeInfoTag(pos); - setNodeInfoTag(pos, pending); - pending = displaced; - - uint16_t displacedPayload = decodeNodeInfoPayloadIndex(pending); - if (displacedPayload >= nodeInfoTargetEntries()) { - pending = 0; - break; - } - - NodeNum displacedNode = nodeInfoPayload[displacedPayload].node; - if (displacedNode == 0) { - pending = 0; - break; - } - - uint16_t altH1 = nodeInfoHash1(displacedNode); - uint16_t altH2 = nodeInfoHash2(displacedNode); - uint16_t altBucket = (altH1 == currentBucket) ? altH2 : altH1; - - if (tryInsertNodeInfoEntryInBucket(altBucket, pending)) { - pending = 0; - break; - } - - currentBucket = altBucket; + NodeInfoPayloadEntry *empty = nullptr; + NodeInfoPayloadEntry *lru = nullptr; + uint32_t lruAge = 0; + const uint32_t now = clockMs(); + + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + NodeInfoPayloadEntry &e = nodeInfoPayload[i]; + if (e.node == node) + return &e; + if (e.node == 0) { + if (!empty) + empty = &e; + continue; } - - if (pending != 0) { - uint16_t droppedPayload = decodeNodeInfoPayloadIndex(pending); - if (droppedPayload < nodeInfoTargetEntries()) - nodeInfoPayload[droppedPayload].node = 0; - TM_LOG_DEBUG("NodeInfo bucketed cuckoo overflow, dropped payload idx=%u", - static_cast(droppedPayload < nodeInfoTargetEntries() ? droppedPayload : UINT16_MAX)); + if (empty) + continue; // an empty slot beats any victim; stop scoring + const uint32_t age = now - e.lastObservedMs; // unsigned subtraction is wrap-safe + if (!lru || age > lruAge) { + lru = &e; + lruAge = age; } } - uint16_t finalIndex = findNodeInfoPayloadIndex(node); - if (finalIndex >= nodeInfoTargetEntries()) { - // New entry did not survive insertion chain. - if (payloadIndex < nodeInfoTargetEntries() && nodeInfoPayload[payloadIndex].node == node) - nodeInfoPayload[payloadIndex].node = 0; + NodeInfoPayloadEntry *slot = empty ? empty : lru; + if (!slot) return nullptr; - } - - if (usedEmptySlot) { - const uint16_t afterCount = countNodeInfoEntriesLocked(); - *usedEmptySlot = afterCount > beforeCount; - } - - return &nodeInfoPayload[finalIndex]; + memset(slot, 0, sizeof(NodeInfoPayloadEntry)); + slot->node = node; + if (usedEmptySlot) + *usedEmptySlot = (slot == empty); + return slot; #else (void)node; return nullptr; @@ -747,12 +427,12 @@ TrafficManagementModule::NodeInfoPayloadEntry *TrafficManagementModule::findOrCr uint16_t TrafficManagementModule::countNodeInfoEntriesLocked() const { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoIndex) + if (!nodeInfoPayload) return 0; uint16_t count = 0; - for (uint16_t i = 0; i < nodeInfoIndexSlots(); i++) { - if (getNodeInfoTag(i) != 0) + for (uint16_t i = 0; i < nodeInfoTargetEntries(); i++) { + if (nodeInfoPayload[i].node != 0) count++; } return count; @@ -764,7 +444,7 @@ uint16_t TrafficManagementModule::countNodeInfoEntriesLocked() const void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &mp) { #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (!nodeInfoPayload || !nodeInfoIndex || mp.decoded.payload.size == 0) + if (!nodeInfoPayload || mp.decoded.payload.size == 0) return; meshtastic_User user = meshtastic_User_init_zero; @@ -786,7 +466,7 @@ void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &m // richer context than "just the user protobuf" when PSRAM is present. // This path is intentionally independent from NodeInfoModule/NodeDB. entry->user = user; - entry->lastObservedMs = millis(); + entry->lastObservedMs = clockMs(); entry->lastObservedRxTime = mp.rx_time; entry->sourceChannel = mp.channel; entry->hasDecodedBitfield = mp.decoded.has_bitfield; @@ -797,10 +477,8 @@ void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &m } if (usedEmptySlot) { - TM_LOG_INFO("NodeInfo PSRAM cache entries: %u/%u target (%u packed slots, %u-bit tags, %u-byte DRAM index)", - static_cast(cachedCount), static_cast(nodeInfoTargetEntries()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoTagBits()), - static_cast(nodeInfoIndexMetadataBudgetBytes())); + TM_LOG_INFO("NodeInfo PSRAM cache entries: %u/%u", static_cast(cachedCount), + static_cast(nodeInfoTargetEntries())); } #else (void)mp; @@ -808,25 +486,87 @@ void TrafficManagementModule::cacheNodeInfoPacket(const meshtastic_MeshPacket &m } // ============================================================================= -// Epoch Management +// Next-Hop Overflow Cache // ============================================================================= +// +// A routing hint store. The byte is the last byte of the NodeNum to use as next +// hop to reach `dest`. It is written ONLY from NextHopRouter's ACK-confirmed +// decision (a bidirectionally-verified relay) — never inferred one-way from +// relayed traffic. The TMM cache holds confirmed next-hops that have aged out of +// the hot NodeDB (NodeInfoLite), and NextHopRouter::getNextHop() consults it as a +// fallback after the hot store. + +void TrafficManagementModule::setNextHop(NodeNum dest, uint8_t nextHopByte) +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || dest == 0 || nextHopByte == 0) + return; -/** - * Reset the timestamp epoch when relative offsets approach overflow. - * - * Called when epoch age exceeds ~19 hours (approaching 8-bit minute overflow). - * Invalidates all cached per-node traffic state. - */ -void TrafficManagementModule::resetEpoch(uint32_t nowMs) + concurrency::LockGuard guard(&cacheLock); + bool isNew = false; + UnifiedCacheEntry *entry = findOrCreateEntry(dest, &isNew); + if (entry) + entry->next_hop = nextHopByte; // last-write-wins; only confirmed bytes reach here +#else + (void)dest; + (void)nextHopByte; +#endif +} + +uint8_t TrafficManagementModule::getNextHopHint(NodeNum dest) { #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - TM_LOG_DEBUG("Resetting cache epoch"); - cacheEpochMs = nowMs; + if (!cache || dest == 0) + return 0; - // Full flush avoids stale dedup identity/counters surviving epoch rollover. - memset(cache, 0, static_cast(cacheSize()) * sizeof(UnifiedCacheEntry)); + concurrency::LockGuard guard(&cacheLock); + UnifiedCacheEntry *entry = findEntry(dest); + return entry ? entry->next_hop : 0; #else - (void)nowMs; + (void)dest; + return 0; +#endif +} + +bool TrafficManagementModule::preloadNextHopsFromNodeDB() +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + if (!cache || !nodeDB) + return false; // prerequisites not ready yet — caller should retry on a later pass + + uint16_t seeded = 0; + concurrency::LockGuard guard(&cacheLock); + const size_t count = nodeDB->getNumMeshNodes(); + for (size_t i = 0; i < count; i++) { + const meshtastic_NodeInfoLite *node = nodeDB->getMeshNodeByIndex(i); + if (!node || node->num == 0 || node->next_hop == 0) + continue; + + bool isNew = false; + UnifiedCacheEntry *entry = findOrCreateEntry(node->num, &isNew); + // Don't clobber a freshly-learned confirmed hop with a (possibly stale) persisted one. + if (entry && entry->next_hop == 0) { + entry->next_hop = node->next_hop; + seeded++; + } + } + + TM_LOG_INFO("Preloaded %u next-hop hints from NodeDB", static_cast(seeded)); + return true; +#else + return true; // nothing to preload on a cache-less build; don't keep retrying +#endif +} + +// ============================================================================= +// Epoch Management +// ============================================================================= + +void TrafficManagementModule::flushCache() +{ +#if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 + TM_LOG_DEBUG("Flushing cache"); + memset(cache, 0, static_cast(cacheSize()) * sizeof(UnifiedCacheEntry)); #endif } @@ -870,7 +610,12 @@ uint8_t TrafficManagementModule::computePositionFingerprint(int32_t lat_truncate uint8_t latBits = (static_cast(lat_truncated) >> shift) & ((1u << bitsToTake) - 1); uint8_t lonBits = (static_cast(lon_truncated) >> shift) & ((1u << bitsToTake) - 1); - return static_cast((latBits << 4) | lonBits); + const uint8_t fp = static_cast((latBits << 4) | lonBits); + // 0 is the "no position seen" sentinel for pos_fingerprint, so a real position that happens to + // hash to 0 must not collide with it (otherwise its duplicates would never dedup). Remap 0 -> 0xFF, + // mirroring NodeDB::getLastByteOfNodeNum()'s 0 -> 0xFF idiom. Cost: the 0x00 bucket merges into + // 0xFF (one extra collision in 256 — negligible; the fingerprint already collides every 16 cells). + return fp ? fp : 0xFF; } // ============================================================================= @@ -895,7 +640,7 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack incrementStat(&stats.packets_inspected); const auto &cfg = moduleConfig.traffic_management; - const uint32_t nowMs = millis(); + const uint32_t nowMs = TrafficManagementModule::clockMs(); // ------------------------------------------------------------------------- // Undecoded Packet Handling @@ -949,7 +694,8 @@ ProcessMessage TrafficManagementModule::handleReceived(const meshtastic_MeshPack // GPS jitter within the configured precision. if (!isFromUs(&mp) && !isToUs(&mp)) { - if (cfg.position_dedup_enabled && mp.decoded.portnum == meshtastic_PortNum_POSITION_APP) { + if (cfg.position_dedup_enabled && channels.isWellKnownChannel(mp.channel) && + mp.decoded.portnum == meshtastic_PortNum_POSITION_APP) { meshtastic_Position pos = meshtastic_Position_init_zero; if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_Position_msg, &pos)) { if (shouldDropPosition(&mp, &pos, nowMs)) { @@ -1008,6 +754,39 @@ void TrafficManagementModule::alterReceived(meshtastic_MeshPacket &mp) const bool shouldExhaust = ((channelBusy && isTelemetry && cfg.exhaust_hop_telemetry) || (isPosition && cfg.exhaust_hop_position)); + // ------------------------------------------------------------------------- + // Relayed Position Precision Clamp + // ------------------------------------------------------------------------- + // Clamp relayed position broadcasts to the channel's configured precision + // ceiling. Guards against forwarding more-precise coordinates than the + // channel is intended to carry (e.g. a LongFast channel set to 13-bit / + // ~1.5 km). chanPrec==0 means position sharing is disabled on the channel; + // skip — not our job to zero positions on relay. + // Ham mode (owner.is_licensed) is exempt. + // Compile USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS to extend to private channels. + if (!owner.is_licensed && isPosition && isBroadcast(mp.to)) { +#ifdef USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS + const bool shouldClamp = true; +#else + const bool shouldClamp = channels.isWellKnownChannel(mp.channel); +#endif + if (shouldClamp) { + const uint32_t chanPrec = getPositionPrecisionForChannel(mp.channel); + if (chanPrec > 0) { + meshtastic_Position pos = meshtastic_Position_init_default; + if (pb_decode_from_bytes(mp.decoded.payload.bytes, mp.decoded.payload.size, &meshtastic_Position_msg, &pos)) { + const uint32_t packetPrec = pos.precision_bits > 0 ? pos.precision_bits : 32u; + if (packetPrec > chanPrec) { + applyPositionPrecision(pos, chanPrec); + mp.decoded.payload.size = pb_encode_to_bytes(mp.decoded.payload.bytes, sizeof(mp.decoded.payload.bytes), + &meshtastic_Position_msg, &pos); + logAction("clamp", &mp, "precision"); + } + } + } + } + } + if (!shouldExhaust || !isBroadcast(mp.to)) return; @@ -1040,41 +819,52 @@ int32_t TrafficManagementModule::runOnce() return INT32_MAX; #if TRAFFIC_MANAGEMENT_CACHE_SIZE > 0 - const uint32_t nowMs = millis(); - - // Check if epoch reset needed (~3.5 hours approaching 8-bit minute overflow) - if (needsEpochReset(nowMs)) { - concurrency::LockGuard guard(&cacheLock); - resetEpoch(nowMs); - return kMaintenanceIntervalMs; - } - - // Calculate TTLs for cache expiration + const uint32_t nowMs = TrafficManagementModule::clockMs(); + + // Warm-start the next-hop cache from persisted NodeInfoLite hints once nodeDB + // is populated. Done here (not in the constructor) so nodeDB has finished + // loading. Takes its own lock, so call before acquiring the sweep guard below. + // Only latch the one-shot guard once the preload actually ran; if nodeDB wasn't + // ready yet, retry on the next maintenance pass instead of skipping it forever. + if (!nextHopPreloaded && preloadNextHopsFromNodeDB()) + nextHopPreloaded = true; + + // Free-running tick counters (no epoch needed). + // TTL expressed in ticks: pos 4× interval (default 44 ticks ≈ 4.4h), + // rate 2× window (default 24 ticks = 2h), unknown fixed 12 ticks (12 min). const uint32_t positionIntervalMs = secsToMs(Default::getConfiguredOrDefault( moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); - const uint32_t positionTtlMs = positionIntervalMs * 4; + const uint8_t posTtlTicks = + static_cast(std::min(static_cast(255), (positionIntervalMs * 4) / kPosTimeTickMs)); + + const uint32_t rateWindowMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); + const uint8_t rateTtlTicks = static_cast( + std::min(static_cast(15), (rateWindowMs > 0 ? rateWindowMs * 2 : 24 * kRateTimeTickMs) / kRateTimeTickMs)); - const uint32_t rateIntervalMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); - const uint32_t rateTtlMs = (rateIntervalMs > 0) ? rateIntervalMs * 2 : (10 * 60 * 1000UL); + // unknown: fixed 12-tick TTL (12 min — 4 ticks past the 5-min default window) + const uint8_t unknownTtlTicks = 12; - const uint32_t unknownTtlMs = kUnknownResetMs * 5; + const uint8_t nowPosTick = currentPosTick(); + const uint8_t nowRateTick = currentRateTick(); + const uint8_t nowUnknownTick = currentUnknownTick(); // Sweep cache and clear expired entries uint16_t activeEntries = 0; uint16_t expiredEntries = 0; - const uint32_t sweepStartMs = millis(); + const uint32_t sweepStartMs = TrafficManagementModule::clockMs(); + const auto &cfg = moduleConfig.traffic_management; concurrency::LockGuard guard(&cacheLock); + for (uint16_t i = 0; i < cacheSize(); i++) { if (cache[i].node == 0) continue; bool anyValid = false; - // Check and clear expired position data - if (cache[i].pos_time != 0) { - uint32_t posTimeMs = fromRelativePosTime(cache[i].pos_time); - if (!isWithinWindow(nowMs, posTimeMs, positionTtlMs)) { + // Check and clear expired position data (presence: pos_fingerprint != 0) + if (cache[i].pos_fingerprint != 0) { + if (static_cast(nowPosTick - cache[i].pos_time) >= posTtlTicks) { cache[i].pos_fingerprint = 0; cache[i].pos_time = 0; } else { @@ -1082,28 +872,31 @@ int32_t TrafficManagementModule::runOnce() } } - // Check and clear expired rate limit data - if (cache[i].rate_time != 0) { - uint32_t rateTimeMs = fromRelativeRateTime(cache[i].rate_time); - if (!isWithinWindow(nowMs, rateTimeMs, rateTtlMs)) { + // Check and clear expired rate limit data (gated on feature; presence: rate_count != 0) + if (cfg.rate_limit_enabled && cache[i].rate_count != 0) { + if ((static_cast(nowRateTick - cache[i].getRateTime()) & 0x0F) >= rateTtlTicks) { cache[i].rate_count = 0; - cache[i].rate_time = 0; + cache[i].setRateTime(0); } else { anyValid = true; } } - // Check and clear expired unknown tracking data - if (cache[i].unknown_time != 0) { - uint32_t unknownTimeMs = fromRelativeUnknownTime(cache[i].unknown_time); - if (!isWithinWindow(nowMs, unknownTimeMs, unknownTtlMs)) { + // Check and clear expired unknown tracking data (gated on feature; presence: unknown_count != 0) + if (cfg.drop_unknown_enabled && cache[i].unknown_count != 0) { + if ((static_cast(nowUnknownTick - cache[i].getUnknownTime()) & 0x0F) >= unknownTtlTicks) { cache[i].unknown_count = 0; - cache[i].unknown_time = 0; + cache[i].setUnknownTime(0); } else { anyValid = true; } } + // A confirmed next-hop hint has no TTL of its own and keeps the slot alive, + // so an aged-out routing hint outlives the dedup/rate/unknown state. + if (cache[i].next_hop != 0) + anyValid = true; + // If all data expired, free the slot entirely if (!anyValid) { memset(&cache[i], 0, sizeof(UnifiedCacheEntry)); @@ -1115,14 +908,12 @@ int32_t TrafficManagementModule::runOnce() TM_LOG_DEBUG("Maintenance: %u active, %u expired, %u/%u slots, %lums elapsed", activeEntries, expiredEntries, static_cast(activeEntries), static_cast(cacheSize()), - static_cast(millis() - sweepStartMs)); + static_cast(TrafficManagementModule::clockMs() - sweepStartMs)); #if defined(ARCH_ESP32) && defined(BOARD_HAS_PSRAM) - if (nodeInfoPayload && nodeInfoIndex) { - TM_LOG_DEBUG("NodeInfo PSRAM cache: %u/%u target (%u packed slots, %u buckets, %u-bit tags, %u-byte index)", - static_cast(countNodeInfoEntriesLocked()), static_cast(nodeInfoTargetEntries()), - static_cast(nodeInfoIndexSlots()), static_cast(nodeInfoBucketCount()), - static_cast(nodeInfoTagBits()), static_cast(nodeInfoIndexMetadataBudgetBytes())); + if (nodeInfoPayload) { + TM_LOG_DEBUG("NodeInfo PSRAM cache: %u/%u", static_cast(countNodeInfoEntriesLocked()), + static_cast(nodeInfoTargetEntries())); } #endif @@ -1153,8 +944,11 @@ bool TrafficManagementModule::shouldDropPosition(const meshtastic_MeshPacket *p, const int32_t lat_truncated = truncateLatLon(pos->latitude_i, precision); const int32_t lon_truncated = truncateLatLon(pos->longitude_i, precision); const uint8_t fingerprint = computePositionFingerprint(lat_truncated, lon_truncated, precision); - const uint32_t minIntervalMs = secsToMs(Default::getConfiguredOrDefault( - moduleConfig.traffic_management.position_min_interval_secs, default_traffic_mgmt_position_min_interval_secs)); + // Drop gate uses the RAW configured interval: 0 means "dedup disabled" (the + // contract documented below). The 12h default is only for resolution/TTL + // sizing (constructor / runOnce), not for deciding whether to drop — feeding + // the default here would silently turn the 0-disables-dedup contract off. + const uint32_t minIntervalMs = secsToMs(moduleConfig.traffic_management.position_min_interval_secs); bool isNew = false; concurrency::LockGuard guard(&cacheLock); @@ -1162,19 +956,26 @@ bool TrafficManagementModule::shouldDropPosition(const meshtastic_MeshPacket *p, if (!entry) return false; - // Compare fingerprint and check time window - // When minIntervalMs == 0, deduplication is disabled (withinInterval = false means never drop) - const bool hasPositionState = !isNew && entry->pos_time != 0; + // Compare fingerprint and check time window. + // When minIntervalMs == 0, deduplication is disabled (withinInterval = false means never drop). + // Presence: pos_fingerprint != 0 (zero fingerprint from real coordinates is astronomically unlikely). + const bool hasPositionState = !isNew && entry->pos_fingerprint != 0; const bool samePosition = hasPositionState && entry->pos_fingerprint == fingerprint; + const uint8_t nowPosTick = currentPosTick(); + // Clamp to [1, 255]: intervals shorter than one tick still dedup within the same tick. + const uint8_t windowTicks = + (minIntervalMs == 0) ? 0 + : static_cast(std::min(static_cast(UINT8_MAX), + std::max(static_cast(1), minIntervalMs / kPosTimeTickMs))); const bool withinInterval = - hasPositionState && (minIntervalMs != 0) && isWithinWindow(nowMs, fromRelativePosTime(entry->pos_time), minIntervalMs); + hasPositionState && (windowTicks != 0) && (static_cast(nowPosTick - entry->pos_time) < windowTicks); TM_LOG_DEBUG("Position dedup 0x%08x: fp=0x%02x prev=0x%02x same=%d within=%d new=%d", p->from, fingerprint, entry->pos_fingerprint, samePosition, withinInterval, isNew); - // Update cache entry + // Update cache entry (raw tick; 0 is a valid tick value) entry->pos_fingerprint = fingerprint; - entry->pos_time = toRelativePosTime(nowMs); + entry->pos_time = nowPosTick; // Drop only if same position AND within the minimum interval return samePosition && withinInterval; @@ -1218,7 +1019,7 @@ bool TrafficManagementModule::shouldRespondToNodeInfo(const meshtastic_MeshPacke // If the PSRAM cache exists but misses, we intentionally do not fall back // to the node-wide table. This keeps the PSRAM direct-reply path separate // from NodeInfoModule/NodeDB behavior when PSRAM is available. - if (nodeInfoPayload && nodeInfoIndex) { + if (nodeInfoPayload) { TM_LOG_DEBUG("NodeInfo PSRAM cache miss for node=0x%08x", p->to); return false; } @@ -1263,7 +1064,7 @@ bool TrafficManagementModule::shouldRespondToNodeInfo(const meshtastic_MeshPacke reply->decoded.bitfield |= BITFIELD_OK_TO_MQTT_MASK; if (hasCachedUser && cachedLastObservedMs != 0) { - uint32_t ageMs = millis() - cachedLastObservedMs; + uint32_t ageMs = clockMs() - cachedLastObservedMs; TM_LOG_DEBUG("NodeInfo PSRAM hit node=0x%08x age=%lu ms src_ch=%u req_ch=%u rx_time=%lu", p->to, static_cast(ageMs), static_cast(cachedSourceChannel), static_cast(p->channel), static_cast(cachedLastObservedRxTime)); @@ -1328,9 +1129,14 @@ bool TrafficManagementModule::isRateLimited(NodeNum from, uint32_t nowMs) if (!entry) return false; - // Check if window has expired - if (isNew || !isWithinWindow(nowMs, fromRelativeRateTime(entry->rate_time), windowMs)) { - entry->rate_time = toRelativeRateTime(nowMs); + // Window ticks: clamp to [1,15] so zero windowMs (config error) opens a new window. + const uint8_t windowTicks = static_cast(std::min(static_cast(15), windowMs / kRateTimeTickMs)); + const uint8_t nowRateTick = currentRateTick(); + const bool windowExpired = + isNew || entry->rate_count == 0 || + ((static_cast(nowRateTick - entry->getRateTime()) & 0x0F) >= std::max(static_cast(1), windowTicks)); + if (windowExpired) { + entry->setRateTime(nowRateTick); entry->rate_count = 1; return false; } @@ -1362,9 +1168,8 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (!moduleConfig.traffic_management.drop_unknown_enabled || moduleConfig.traffic_management.unknown_packet_threshold == 0) return false; - uint32_t windowMs = kUnknownResetMs; - if (moduleConfig.traffic_management.rate_limit_window_secs > 0) - windowMs = secsToMs(moduleConfig.traffic_management.rate_limit_window_secs); + // Fixed 5-tick (5 min) unknown window; capped at 12 ticks (12 min max). + static constexpr uint8_t kUnknownWindowTicks = 5; bool isNew = false; concurrency::LockGuard guard(&cacheLock); @@ -1372,13 +1177,18 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (!entry) return false; - // Check if window has expired - if (isNew || !isWithinWindow(nowMs, fromRelativeUnknownTime(entry->unknown_time), windowMs)) { - entry->unknown_time = toRelativeUnknownTime(nowMs); + // Check if window has expired (presence: unknown_count != 0) + const uint8_t nowUnknownTick = currentUnknownTick(); + const bool windowExpired = isNew || entry->unknown_count == 0 || + ((static_cast(nowUnknownTick - entry->getUnknownTime()) & 0x0F) >= kUnknownWindowTicks); + if (windowExpired) { + entry->setUnknownTime(nowUnknownTick); entry->unknown_count = 0; } - // Increment counter (saturates at 255) + // Increment counter (saturates at 255). Same saturation handling as + // isRateLimited: without it, a clamped threshold of 255 can never fire. + const bool alreadySaturated = (entry->unknown_count == UINT8_MAX); saturatingIncrement(entry->unknown_count); // Check against threshold @@ -1386,7 +1196,7 @@ bool TrafficManagementModule::shouldDropUnknown(const meshtastic_MeshPacket *p, if (threshold > 255) threshold = 255; - bool drop = entry->unknown_count > threshold; + bool drop = entry->unknown_count > threshold || (alreadySaturated && threshold == 255); if (drop || entry->unknown_count == threshold) { TM_LOG_DEBUG("Unknown packets 0x%08x: count=%u threshold=%u -> %s", p->from, entry->unknown_count, threshold, drop ? "DROP" : "at-limit"); diff --git a/src/modules/TrafficManagementModule.h b/src/modules/TrafficManagementModule.h index fe3483a8e51..eb32e6cfdc6 100644 --- a/src/modules/TrafficManagementModule.h +++ b/src/modules/TrafficManagementModule.h @@ -20,9 +20,12 @@ * - Router hop preservation (maintain hop_limit for router-to-router traffic) * * Memory Optimization: - * Uses a unified cache with cuckoo hashing for O(1) lookups and 56% memory reduction - * compared to separate per-feature caches. Timestamps are stored as 8-bit relative - * offsets from a rolling epoch to further reduce memory footprint. + * Uses one flat unified cache (plain array, linear scan) shared by all + * per-node features instead of separate per-feature caches. Timestamps are + * stored as free-running modular tick counters (pos: 8-bit 360 s/tick; + * rate+unknown: paired 4-bit nibbles in one byte) for a 10-byte entry. + * LoRa packet rates are low enough that an O(n) scan of ~1000 entries is + * negligible next to packet processing. */ class TrafficManagementModule : public MeshModule, private concurrency::OSThread { @@ -38,6 +41,21 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread void resetStats(); void recordRouterHopPreserved(); + // Next-hop overflow cache (routing hint). + // setNextHop: store a confirmed last-byte next hop for `dest`. Called by + // NextHopRouter from its ACK-confirmed decision (see sniffReceived). The + // byte must come from a bidirectionally-verified relay, not one-way inference. + // getNextHopHint: return the cached next-hop byte for `dest`, 0 if unknown. + void setNextHop(NodeNum dest, uint8_t nextHopByte); + uint8_t getNextHopHint(NodeNum dest); + + // Warm-start the next-hop cache from persisted NodeInfoLite hints so confirmed + // hops survive later hot-store (NodeDB) eviction. Idempotent; runs once after + // nodeDB is populated (lazily on first maintenance pass). + // @return true if it actually ran (prereqs met / nothing to do); false if + // prerequisites (cache, nodeDB) weren't ready yet, so the caller should retry. + bool preloadNextHopsFromNodeDB(); + /** * Check if this packet should have its hops exhausted. * Called from perhapsRebroadcast() to force hop_limit = 0 regardless of @@ -48,182 +66,105 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread return exhaustRequested && exhaustRequestedFrom == getFrom(&mp) && exhaustRequestedId == mp.id; } + // Injectable monotonic clock (ms). All TMM time reads go through clockMs() so unit tests can + // advance a virtual timebase instead of sleeping real seconds across the 6 min/360 s tick. + // Mirrors HopScalingModule::s_testNowMs. Writable from tests as TrafficManagementModule::s_testNowMs; + // ignored in production (clockMs() returns millis()). + inline static uint32_t s_testNowMs = 0; +#ifdef PIO_UNIT_TESTING + static uint32_t clockMs() { return s_testNowMs; } +#else + static uint32_t clockMs() { return millis(); } +#endif + protected: ProcessMessage handleReceived(const meshtastic_MeshPacket &mp) override; bool wantPacket(const meshtastic_MeshPacket *p) override { return true; } void alterReceived(meshtastic_MeshPacket &mp) override; int32_t runOnce() override; - // Protected so test shims can force epoch rollover behavior. - void resetEpoch(uint32_t nowMs); + // Protected so test shims can flush per-node traffic state. + void flushCache(); private: // ========================================================================= // Unified Cache Entry (10 bytes) - Same for ALL platforms // ========================================================================= // - // A single compact structure used across ESP32, NRF52, and all other platforms. - // Memory: 10 bytes × 2048 entries = 20KB - // - // Position Fingerprinting: - // Instead of storing full coordinates (8 bytes) or a computed hash, - // we store an 8-bit fingerprint derived deterministically from the - // truncated lat/lon. This extracts the lower 4 significant bits from - // each coordinate: fingerprint = (lat_low4 << 4) | lon_low4 - // - // Benefits over hash: - // - Adjacent grid cells have sequential fingerprints (no collision) - // - Two positions only collide if 16+ grid cells apart in BOTH dimensions - // - Deterministic: same input always produces same output + // Layout: + // [0-3] node - NodeNum (4 bytes, 0 = empty slot) + // [4] pos_fingerprint - 4 bits lat + 4 bits lon (0 = no position seen) + // [5] rate_count - Packets in current rate window (0 = no window active) + // [6] unknown_count - Unknown packets in window (0 = no window active) + // [7] pos_time - Position tick (uint8, free-running 360 s/tick) + // [8] rate_unknown_time - [7:4] rate nibble (300 s/tick) | [3:0] unknown nibble (60 s/tick) + // [9] next_hop - Last-byte relay to reach `node` (0 = none) // - // Adaptive Timestamp Resolution: - // All timestamps use 8-bit values with adaptive resolution calculated - // from config at startup. Resolution = max(60, min(339, interval/2)). - // - Min 60 seconds ensures reasonable precision - // - Max 339 seconds allows ~24 hour range (255 * 339 = 86445 sec) - // - interval/2 ensures at least 2 ticks per configured interval + // Presence sentinels (no epoch, no +1 offset needed): + // pos active: pos_fingerprint != 0 + // rate active: rate_count != 0 + // unknown active: unknown_count != 0 // - // Layout: - // [0-3] node - NodeNum (4 bytes) - // [4] pos_fingerprint - 4 bits lat + 4 bits lon (1 byte) - // [5] rate_count - Packets in current window (1 byte) - // [6] unknown_count - Unknown packets count (1 byte) - // [7] pos_time - Position timestamp (1 byte, adaptive resolution) - // [8] rate_time - Rate window start (1 byte, adaptive resolution) - // [9] unknown_time - Unknown tracking start (1 byte, adaptive resolution) + // next_hop: routing hint written only from ACK-confirmed NextHopRouter decisions. + // No TTL — keeps the slot alive across maintenance sweeps. // struct __attribute__((packed)) UnifiedCacheEntry { - NodeNum node; // 4 bytes - Node identifier (0 = empty slot) - uint8_t pos_fingerprint; // 1 byte - Lower 4 bits of lat + lon - uint8_t rate_count; // 1 byte - Packet count (saturates at 255) - uint8_t unknown_count; // 1 byte - Unknown packet count (saturates at 255) - uint8_t pos_time; // 1 byte - Position timestamp (adaptive resolution) - uint8_t rate_time; // 1 byte - Rate window start (adaptive resolution) - uint8_t unknown_time; // 1 byte - Unknown tracking start (adaptive resolution) + NodeNum node; + uint8_t pos_fingerprint; + uint8_t rate_count; + uint8_t unknown_count; + uint8_t pos_time; + uint8_t rate_unknown_time; + uint8_t next_hop; + + uint8_t getRateTime() const { return (rate_unknown_time >> 4) & 0x0F; } + uint8_t getUnknownTime() const { return rate_unknown_time & 0x0F; } + void setRateTime(uint8_t t) { rate_unknown_time = static_cast((rate_unknown_time & 0x0F) | ((t & 0x0F) << 4)); } + void setUnknownTime(uint8_t t) { rate_unknown_time = static_cast((rate_unknown_time & 0xF0) | (t & 0x0F)); } }; static_assert(sizeof(UnifiedCacheEntry) == 10, "UnifiedCacheEntry should be 10 bytes"); // ========================================================================= - // Cuckoo Hash Table Implementation + // Flat unified cache // ========================================================================= // - // Cuckoo hashing provides O(1) worst-case lookup time using two hash functions. - // Each key can be in one of two possible locations (h1 or h2). On collision, - // the existing entry is "kicked" to its alternate location. + // Plain array, linear scan (same idiom as WarmNodeStore). A lookup walks at + // most cacheSize() × 11 B — microseconds at LoRa packet rates, not worth a + // hash table. Insertion on a full cache evicts the stalest entry, + // preferring entries without a next_hop hint (those are the long-tail + // routing state this cache exists to keep). // - // Benefits over linear scan: - // - O(1) lookup vs O(n) - critical at packet processing rates - // - O(1) insertion (amortized) with simple eviction on cycles - // - ~95% load factor achievable - // - // Cache size rounds to power-of-2 for fast modulo via bitmask. - // TRAFFIC_MANAGEMENT_CACHE_SIZE=2000 → cacheSize()=2048 - // - static constexpr uint16_t cacheSize(); - static constexpr uint16_t cacheMask(); - - // Hash functions for cuckoo hashing - inline uint16_t cuckooHash1(NodeNum node) const { return node & cacheMask(); } - inline uint16_t cuckooHash2(NodeNum node) const { return ((node * 2654435769u) >> (32 - cuckooHashBits())) & cacheMask(); } - static constexpr uint8_t cuckooHashBits(); - - // NodeInfo cache configuration (PSRAM path): - // - Payload lives in PSRAM - // - DRAM keeps packed 12-bit tags with 4-way bucketed cuckoo hashing - // (Fan et al., CoNEXT 2014). Tag value 0 is reserved as "empty". - static constexpr uint16_t kNodeInfoIndexMetadataBudgetBytes = 3072; // 3KB DRAM tag store - static constexpr uint8_t kNodeInfoTargetOccupancyPercent = 95; - static constexpr uint8_t kNodeInfoBucketSize = 4; - static constexpr uint8_t kNodeInfoTagBits = 12; - static constexpr uint16_t kNodeInfoTagMask = static_cast((1u << kNodeInfoTagBits) - 1u); - static constexpr uint16_t kNodeInfoIndexSlotsRaw = - static_cast((kNodeInfoIndexMetadataBudgetBytes * 8u) / kNodeInfoTagBits); - static constexpr uint16_t kNodeInfoIndexSlots = - static_cast(kNodeInfoIndexSlotsRaw - (kNodeInfoIndexSlotsRaw % kNodeInfoBucketSize)); - static constexpr uint16_t kNodeInfoTargetEntries = - static_cast((kNodeInfoIndexSlots * kNodeInfoTargetOccupancyPercent) / 100u); - static_assert((kNodeInfoIndexSlots % kNodeInfoBucketSize) == 0, "NodeInfo slot count must align to bucket size"); - static_assert(kNodeInfoTargetEntries < (1u << kNodeInfoTagBits), "NodeInfo tag bits must encode payload index"); - - static constexpr uint16_t nodeInfoTargetEntries(); - static constexpr uint16_t nodeInfoIndexMetadataBudgetBytes(); - static constexpr uint8_t nodeInfoTargetOccupancyPercent(); - static constexpr uint8_t nodeInfoBucketSize(); - static constexpr uint8_t nodeInfoTagBits(); - static constexpr uint16_t nodeInfoTagMask(); - static constexpr uint16_t nodeInfoIndexSlots(); - static constexpr uint16_t nodeInfoBucketCount(); - static constexpr uint16_t nodeInfoBucketMask(); - static constexpr uint8_t nodeInfoBucketHashBits(); - inline uint16_t nodeInfoHash1(NodeNum node) const { return node & nodeInfoBucketMask(); } - inline uint16_t nodeInfoHash2(NodeNum node) const - { - return ((node * 2246822519u) >> (32 - nodeInfoBucketHashBits())) & nodeInfoBucketMask(); - } + static constexpr uint16_t cacheSize() { return TRAFFIC_MANAGEMENT_CACHE_SIZE; } + + // NodeInfo cache configuration (PSRAM path): a flat PSRAM array of payload + // entries, linear scan keyed by `node`, LRU eviction by lastObservedMs. + // NodeInfo traffic is low-rate, so a full scan per lookup/insert is fine. + static constexpr uint16_t kNodeInfoCacheEntries = 2000; + static constexpr uint16_t nodeInfoTargetEntries() { return kNodeInfoCacheEntries; } // ========================================================================= - // Adaptive Timestamp Resolution + // Free-Running Tick Counters // ========================================================================= // - // All timestamps use 8-bit values with adaptive resolution calculated from - // config at startup. This allows ~24 hour range while maintaining precision. + // Timestamps are stored as free-running modular tick counters derived from + // millis(). No epoch anchor needed: modular subtraction gives correct age + // as long as the true age stays below the counter period. // - // Resolution formula: max(60, min(339, interval/2)) - // - 60 sec minimum ensures reasonable precision - // - 339 sec maximum allows 24 hour range (255 * 339 ≈ 86400 sec) - // - interval/2 ensures at least 2 ticks per configured interval + // pos_time : uint8 (256 ticks × 360 s = 25.6 h period; max window 12 h = 120 ticks) + // rate_time : nibble (16 ticks × 300 s = 80 min period; max window 1 h = 12 ticks) + // unknown_time: nibble (16 ticks × 60 s = 16 min period; max window 12 min = 12 ticks) // - // Since config changes require reboot, resolution is calculated once. + // Presence sentinels (no +1 offset needed; count fields serve as guards): + // pos active: pos_fingerprint != 0 (zero fingerprint astronomically unlikely) + // rate active: rate_count != 0 + // unknown active: unknown_count != 0 // - uint32_t cacheEpochMs = 0; - uint16_t posTimeResolution = 60; // Seconds per tick for position - uint16_t rateTimeResolution = 60; // Seconds per tick for rate limiting - uint16_t unknownTimeResolution = 60; // Seconds per tick for unknown tracking - - // Calculate resolution from configured interval (called once at startup) - static uint16_t calcTimeResolution(uint32_t intervalSecs) - { - // Resolution = interval/2 to ensure at least 2 ticks per interval - // Clamped to [60, 339] for min precision and max 24h range - uint32_t res = (intervalSecs > 0) ? (intervalSecs / 2) : 60; - if (res < 60) - res = 60; - if (res > 339) - res = 339; - return static_cast(res); - } - - // Convert to/from 8-bit relative timestamps with given resolution - uint8_t toRelativeTime(uint32_t nowMs, uint16_t resolutionSecs) const - { - uint32_t ticks = (nowMs - cacheEpochMs) / (resolutionSecs * 1000UL); - return (ticks > UINT8_MAX) ? UINT8_MAX : static_cast(ticks); - } - uint32_t fromRelativeTime(uint8_t ticks, uint16_t resolutionSecs) const - { - return cacheEpochMs + (static_cast(ticks) * resolutionSecs * 1000UL); - } - - // Convenience wrappers for each timestamp type - uint8_t toRelativePosTime(uint32_t nowMs) const { return toRelativeTime(nowMs, posTimeResolution); } - uint32_t fromRelativePosTime(uint8_t t) const { return fromRelativeTime(t, posTimeResolution); } - - uint8_t toRelativeRateTime(uint32_t nowMs) const { return toRelativeTime(nowMs, rateTimeResolution); } - uint32_t fromRelativeRateTime(uint8_t t) const { return fromRelativeTime(t, rateTimeResolution); } + static constexpr uint32_t kPosTimeTickMs = 360'000UL; // 6 min/tick + static constexpr uint32_t kRateTimeTickMs = 300'000UL; // 5 min/tick + static constexpr uint32_t kUnknownTimeTickMs = 60'000UL; // 1 min/tick - uint8_t toRelativeUnknownTime(uint32_t nowMs) const { return toRelativeTime(nowMs, unknownTimeResolution); } - uint32_t fromRelativeUnknownTime(uint8_t t) const { return fromRelativeTime(t, unknownTimeResolution); } - - // Epoch reset when any timestamp approaches overflow - // With max resolution of 339 sec, 200 ticks = ~19 hours (safe margin for 24h max) - bool needsEpochReset(uint32_t nowMs) const - { - uint16_t maxRes = posTimeResolution; - if (rateTimeResolution > maxRes) - maxRes = rateTimeResolution; - if (unknownTimeResolution > maxRes) - maxRes = unknownTimeResolution; - return (nowMs - cacheEpochMs) > (200UL * maxRes * 1000UL); - } + static uint8_t currentPosTick() { return static_cast(clockMs() / kPosTimeTickMs); } + static uint8_t currentRateTick() { return static_cast((clockMs() / kRateTimeTickMs) & 0x0F); } + static uint8_t currentUnknownTick() { return static_cast((clockMs() / kUnknownTimeTickMs) & 0x0F); } // ========================================================================= // Position Fingerprint // ========================================================================= @@ -246,7 +187,7 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread // ========================================================================= mutable concurrency::Lock cacheLock; // Protects all cache access - UnifiedCacheEntry *cache = nullptr; // Cuckoo hash table (unified for all platforms) + UnifiedCacheEntry *cache = nullptr; // Flat unified cache (linear scan; all platforms) bool cacheFromPsram = false; // Tracks allocator for correct deallocation struct NodeInfoPayloadEntry { @@ -278,11 +219,8 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread uint8_t decodedBitfield; }; - NodeInfoPayloadEntry *nodeInfoPayload = nullptr; // NodeInfo payloads in PSRAM + NodeInfoPayloadEntry *nodeInfoPayload = nullptr; // NodeInfo payloads in PSRAM (flat array, linear scan) bool nodeInfoPayloadFromPsram = false; // Tracks allocator for correct deallocation - uint8_t *nodeInfoIndex = nullptr; // Packed 12-bit NodeInfo tags in DRAM - uint16_t nodeInfoAllocHint = 0; - uint16_t nodeInfoEvictCursor = 0; meshtastic_TrafficManagementStats stats; @@ -293,29 +231,22 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread NodeNum exhaustRequestedFrom = 0; PacketId exhaustRequestedId = 0; + // One-shot guard: warm-start next-hop cache from NodeDB on first maintenance pass. + bool nextHopPreloaded = false; + // ========================================================================= // Cache Operations // ========================================================================= - // Find or create entry for node using cuckoo hashing - // Returns nullptr if cache is full and eviction fails + // Find or create entry for node (linear scan; stalest-first eviction when full) UnifiedCacheEntry *findOrCreateEntry(NodeNum node, bool *isNew); // Find existing entry (no creation) UnifiedCacheEntry *findEntry(NodeNum node); - // NodeInfo cache operations (bucketed cuckoo index + PSRAM payloads) + // NodeInfo cache operations (flat PSRAM payload array, linear scan) const NodeInfoPayloadEntry *findNodeInfoEntry(NodeNum node) const; NodeInfoPayloadEntry *findOrCreateNodeInfoEntry(NodeNum node, bool *usedEmptySlot); - uint16_t findNodeInfoPayloadIndex(NodeNum node) const; - bool removeNodeInfoIndexEntry(NodeNum node, uint16_t payloadIndex); - uint16_t allocateNodeInfoPayloadSlot(); - uint16_t evictNodeInfoPayloadSlot(); - bool tryInsertNodeInfoEntryInBucket(uint16_t bucket, uint16_t tag); - uint16_t encodeNodeInfoTag(uint16_t payloadIndex) const; - uint16_t decodeNodeInfoPayloadIndex(uint16_t tag) const; - uint16_t getNodeInfoTag(uint16_t slot) const; - void setNodeInfoTag(uint16_t slot, uint16_t tag); uint16_t countNodeInfoEntriesLocked() const; void cacheNodeInfoPacket(const meshtastic_MeshPacket &mp); @@ -333,101 +264,7 @@ class TrafficManagementModule : public MeshModule, private concurrency::OSThread void incrementStat(uint32_t *field); }; -// ========================================================================= -// Compile-time Cache Size Calculations -// ========================================================================= -// -// Round TRAFFIC_MANAGEMENT_CACHE_SIZE up to next power of 2 for efficient -// cuckoo hash indexing (allows bitmask instead of modulo). -// -// These use C++11-compatible constexpr (single return statement). -// - -namespace detail -{ -// Helper: round up to next power of 2 using bit manipulation -constexpr uint16_t nextPow2(uint16_t n) -{ - return n == 0 ? 0 : (((n - 1) | ((n - 1) >> 1) | ((n - 1) >> 2) | ((n - 1) >> 4) | ((n - 1) >> 8)) + 1); -} - -// Helper: floor(log2(n)) for n >= 0, C++11-compatible constexpr. -constexpr uint8_t log2Floor(uint16_t n) -{ - return n <= 1 ? 0 : static_cast(1 + log2Floor(static_cast(n >> 1))); -} - -// Helper: ceil(log2(n)) for n >= 1, C++11-compatible constexpr. -constexpr uint8_t log2Ceil(uint16_t n) -{ - return n <= 1 ? 0 : static_cast(1 + log2Floor(static_cast(n - 1))); -} -} // namespace detail - -constexpr uint16_t TrafficManagementModule::cacheSize() -{ - return detail::nextPow2(TRAFFIC_MANAGEMENT_CACHE_SIZE); -} - -constexpr uint16_t TrafficManagementModule::cacheMask() -{ - return cacheSize() > 0 ? cacheSize() - 1 : 0; -} - -constexpr uint8_t TrafficManagementModule::cuckooHashBits() -{ - return detail::log2Floor(cacheSize()); -} - -constexpr uint16_t TrafficManagementModule::nodeInfoTargetEntries() -{ - return kNodeInfoTargetEntries; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoIndexMetadataBudgetBytes() -{ - return kNodeInfoIndexMetadataBudgetBytes; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoTargetOccupancyPercent() -{ - return kNodeInfoTargetOccupancyPercent; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoBucketSize() -{ - return kNodeInfoBucketSize; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoTagBits() -{ - return kNodeInfoTagBits; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoTagMask() -{ - return kNodeInfoTagMask; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoIndexSlots() -{ - return kNodeInfoIndexSlots; -} - -constexpr uint16_t TrafficManagementModule::nodeInfoBucketCount() -{ - return static_cast(nodeInfoIndexSlots() / nodeInfoBucketSize()); -} - -constexpr uint16_t TrafficManagementModule::nodeInfoBucketMask() -{ - return nodeInfoBucketCount() > 0 ? nodeInfoBucketCount() - 1 : 0; -} - -constexpr uint8_t TrafficManagementModule::nodeInfoBucketHashBits() -{ - return detail::log2Floor(nodeInfoBucketCount()); -} +static_assert(TRAFFIC_MANAGEMENT_CACHE_SIZE <= UINT16_MAX, "cacheSize() returns uint16_t"); extern TrafficManagementModule *trafficManagementModule; diff --git a/src/platform/nrf52/nrf52840_s140_v6.ld b/src/platform/nrf52/nrf52840_s140_v6.ld new file mode 100644 index 00000000000..c8dac4e5545 --- /dev/null +++ b/src/platform/nrf52/nrf52840_s140_v6.ld @@ -0,0 +1,46 @@ +/* Linker script to configure memory regions. */ + +SEARCH_DIR(.) +GROUP(-lgcc -lc -lnosys) + +MEMORY +{ + /* App region ends at 0xEA000, not 0xED000: the 12 KB warm-node-store + * record-ring (3 x 4 KB pages, src/mesh/WarmNodeStore.h) occupies + * 0xEA000-0xED000, directly below the stock LittleFS partition. Boards on + * the framework-default linker script are covered by the post-link guard + * in extra_scripts/nrf52_warm_region.py instead. + * + * S140 v6.x app region starts at 0x26000 (152 KB SoftDevice); v7.x uses + * 0x27000. All other boundaries are identical — see nrf52840_s140_v7.ld. */ + FLASH (rx) : ORIGIN = 0x26000, LENGTH = 0xEA000 - 0x26000 + + /* SRAM required by Softdevice depend on + * - Attribute Table Size (Number of Services and Characteristics) + * - Vendor UUID count + * - Max ATT MTU + * - Concurrent connection peripheral + central + secure links + * - Event Len, HVN queue, Write CMD queue + */ + RAM (rwx) : ORIGIN = 0x20006000, LENGTH = 0x20040000 - 0x20006000 +} + +SECTIONS +{ + . = ALIGN(4); + .svc_data : + { + PROVIDE(__start_svc_data = .); + KEEP(*(.svc_data)) + PROVIDE(__stop_svc_data = .); + } > RAM + + .fs_data : + { + PROVIDE(__start_fs_data = .); + KEEP(*(.fs_data)) + PROVIDE(__stop_fs_data = .); + } > RAM +} INSERT AFTER .data; + +INCLUDE "nrf52_common.ld" diff --git a/src/platform/nrf52/nrf52840_s140_v7.ld b/src/platform/nrf52/nrf52840_s140_v7.ld index 6aaeb4034fe..4546b4a7a3f 100644 --- a/src/platform/nrf52/nrf52840_s140_v7.ld +++ b/src/platform/nrf52/nrf52840_s140_v7.ld @@ -5,7 +5,12 @@ GROUP(-lgcc -lc -lnosys) MEMORY { - FLASH (rx) : ORIGIN = 0x27000, LENGTH = 0xED000 - 0x27000 + /* App region ends at 0xEA000, not 0xED000: the 12 KB warm-node-store + * record-ring (3 x 4 KB pages, src/mesh/WarmNodeStore.h) occupies + * 0xEA000-0xED000, directly below the stock LittleFS partition. Boards on + * the framework-default linker script are covered by the post-link guard + * in extra_scripts/nrf52_warm_region.py instead. */ + FLASH (rx) : ORIGIN = 0x27000, LENGTH = 0xEA000 - 0x27000 /* SRAM required by Softdevice depend on * - Attribute Table Size (Number of Services and Characteristics) diff --git a/test/test_nodedb_blocked/test_main.cpp b/test/test_nodedb_blocked/test_main.cpp new file mode 100644 index 00000000000..2bbe9fb6b36 --- /dev/null +++ b/test/test_nodedb_blocked/test_main.cpp @@ -0,0 +1,193 @@ +// Tests for the NodeDB hot-store migration and favourite/ignored (blocked) +// retention paths — src/mesh/NodeDB.cpp. +#include "MeshTypes.h" // BEFORE TestUtil.h — provides WARM_NODE_COUNT / MAX_NUM_NODES via mesh-pb-constants.h +#include "TestUtil.h" +#include + +#if defined(ARCH_PORTDUINO) +#define NDB_TEST_ENTRY extern "C" +#else +#define NDB_TEST_ENTRY +#endif + +// The migration demotes overflow into the warm tier, so these tests need it. +#if WARM_NODE_COUNT > 0 + +#include "mesh/NodeDB.h" +#include + +// Subclass shim: exposes the private maintenance paths (via the friend +// declaration in NodeDB.h) and lets a test own the hot store directly +// (meshNodes/numMeshNodes are public). Declared at global scope so it matches +// `friend class NodeDBTestShim` — an anonymous-namespace class would not. +class NodeDBTestShim : public NodeDB +{ + public: + void runDemote() { demoteOldestHotNodesToWarm(); } + void runCleanup() { cleanupMeshDB(); } + + void clearHot() + { + meshNodes->clear(); + numMeshNodes = 0; + } + + void push(NodeNum num, uint32_t lastHeard, bool favorite, bool ignored, bool withUser, bool withKey) + { + meshtastic_NodeInfoLite n = meshtastic_NodeInfoLite_init_zero; + n.num = num; + n.last_heard = lastHeard; + if (favorite) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_IS_FAVORITE_MASK, true); + if (ignored) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_IS_IGNORED_MASK, true); + if (withUser) + nodeInfoLiteSetBit(&n, NODEINFO_BITFIELD_HAS_USER_MASK, true); + if (withKey) { + n.public_key.size = 32; + memset(n.public_key.bytes, static_cast(num & 0xff), 32); + n.public_key.bytes[0] = 0x01; // ensure non-zero (all-zero == "no key") + } + meshNodes->push_back(n); + numMeshNodes = meshNodes->size(); + } + + // Index 0 is our own node; the eviction/migration scans treat it as self. + void seedSelf() { push(0x0BADF00D, 0xFFFFFFFFu, false, false, /*withUser=*/true, /*withKey=*/false); } +}; + +namespace +{ + +NodeDBTestShim *db = nullptr; + +bool warmHasKey(NodeNum n) +{ + meshtastic_NodeInfoLite_public_key_t k = {0, {0}}; + return db->copyPublicKey(n, k) && k.size == 32; +} + +} // namespace + +void setUp(void) +{ + db->clearHot(); +} +void tearDown(void) {} + +// Migration: a database from a larger-cap build trims to MAX_NUM_NODES; the +// oldest non-protected nodes are demoted into the warm tier (keys preserved), +// while self, favourites and ignored survive even when they are the oldest. +static void test_migration_demotesOldestKeepsKeepersAndSelf(void) +{ + db->seedSelf(); + const int extra = MAX_NUM_NODES + 30; // overflow well past the MAX-2 cap + for (int i = 1; i <= extra; i++) { + const bool fav = (i == 1); // oldest, but a favourite + const bool ign = (i == 2); // 2nd-oldest, but blocked + db->push(2000 + i, /*last_heard=*/i, fav, ign, /*withUser=*/true, /*withKey=*/true); + } + + db->runDemote(); + + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES, (int)db->getNumMeshNodes()); + TEST_ASSERT_NOT_NULL(db->getMeshNode(0x0BADF00D)); // self retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + 1)); // oldest favourite retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + 2)); // oldest ignored retained + TEST_ASSERT_NOT_NULL(db->getMeshNode(2000 + extra)); // freshest retained + TEST_ASSERT_NULL(db->getMeshNode(2000 + 3)); // oldest non-protected demoted out of hot + TEST_ASSERT_TRUE(warmHasKey(2000 + 3)); // ...but its key kept in the warm tier +} + +// Favourite handling: a favourite is never the eviction victim, even when it is +// the oldest node in a full hot store. +static void test_eviction_preservesFavorite(void) +{ + db->seedSelf(); + for (int i = 1; i < MAX_NUM_NODES; i++) { // fill to MAX_NUM_NODES total (incl. self) + const bool fav = (i == 1); // oldest non-self, favourite + db->push(3000 + i, /*last_heard=*/i, fav, false, /*withUser=*/true, /*withKey=*/true); + } + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES, (int)db->getNumMeshNodes()); // full + + TEST_ASSERT_NOT_NULL(db->getOrCreateMeshNode(0x99990000)); // forces an eviction + + TEST_ASSERT_NOT_NULL(db->getMeshNode(3000 + 1)); // favourite survived despite being oldest + TEST_ASSERT_NULL(db->getMeshNode(3000 + 2)); // oldest non-favourite evicted + TEST_ASSERT_NOT_NULL(db->getMeshNode(0x99990000)); +} + +// Ignored handling: an ignored node survives eviction (like a favourite), and is +// never purged by cleanupMeshDB even with no user info (a block set by bare ID). +static void test_ignored_survivesEvictionAndCleanup(void) +{ + // (a) eviction protection + db->clearHot(); + db->seedSelf(); + for (int i = 1; i < MAX_NUM_NODES; i++) { + const bool ign = (i == 1); // oldest non-self, blocked + db->push(4000 + i, /*last_heard=*/i, false, ign, /*withUser=*/true, /*withKey=*/true); + } + TEST_ASSERT_NOT_NULL(db->getOrCreateMeshNode(0x88880000)); + TEST_ASSERT_NOT_NULL(db->getMeshNode(4000 + 1)); // blocked node survived + TEST_ASSERT_NULL(db->getMeshNode(4000 + 2)); // oldest non-blocked evicted + + // (b) cleanup protection — ignored kept without user info, plain no-user purged + db->clearHot(); + db->seedSelf(); + db->push(5000, 100, false, /*ignored=*/true, /*withUser=*/false, false); + db->push(5001, 100, false, false, /*withUser=*/false, false); + db->runCleanup(); + TEST_ASSERT_NOT_NULL(db->getMeshNode(5000)); // blocked-by-ID kept despite no user info + TEST_ASSERT_NULL(db->getMeshNode(5001)); // ordinary no-user node purged +} + +// Protected-node cap: at most MAX_NUM_NODES-2 nodes may be protected, so >=2 +// evictable slots always remain. setProtectedFlag refuses once the cap is hit. +static void test_protectedCap_refusesBeyondLimit(void) +{ + db->seedSelf(); + for (int i = 0; i < MAX_NUM_NODES - 2; i++) + db->push(6000 + i, 100, /*favorite=*/true, false, /*withUser=*/true, false); + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES - 2, db->numProtectedNodes()); + + db->push(7000, 100, false, false, /*withUser=*/true, false); + meshtastic_NodeInfoLite *fresh = db->getMeshNode(7000); + TEST_ASSERT_NOT_NULL(fresh); + TEST_ASSERT_FALSE(db->setProtectedFlag(fresh, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)); // refused at cap + TEST_ASSERT_FALSE(nodeInfoLiteIsIgnored(fresh)); // unchanged + TEST_ASSERT_EQUAL_INT(MAX_NUM_NODES - 2, db->numProtectedNodes()); + + // Adding another flag to an already-protected node doesn't grow the set, so + // it's still allowed at the cap. + meshtastic_NodeInfoLite *already = db->getMeshNode(6000); + TEST_ASSERT_TRUE(db->setProtectedFlag(already, NODEINFO_BITFIELD_IS_IGNORED_MASK, true)); +} + +NDB_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + db = new NodeDBTestShim(); + nodeDB = db; + + UNITY_BEGIN(); + RUN_TEST(test_migration_demotesOldestKeepsKeepersAndSelf); + RUN_TEST(test_eviction_preservesFavorite); + RUN_TEST(test_ignored_survivesEvictionAndCleanup); + RUN_TEST(test_protectedCap_refusesBeyondLimit); + exit(UNITY_END()); +} +NDB_TEST_ENTRY void loop() {} + +#else // WARM_NODE_COUNT == 0 — nothing to exercise here + +void setUp(void) {} +void tearDown(void) {} +NDB_TEST_ENTRY void setup() +{ + UNITY_BEGIN(); + exit(UNITY_END()); +} +NDB_TEST_ENTRY void loop() {} + +#endif diff --git a/test/test_traffic_management/test_main.cpp b/test/test_traffic_management/test_main.cpp index 7631999b319..c7ae12e5a11 100644 --- a/test/test_traffic_management/test_main.cpp +++ b/test/test_traffic_management/test_main.cpp @@ -1,3 +1,4 @@ +#include "MeshTypes.h" // Include BEFORE TestUtil.h — provides HAS_TRAFFIC_MANAGEMENT (via mesh-pb-constants.h) #include "TestUtil.h" #include #include @@ -10,6 +11,7 @@ #if HAS_TRAFFIC_MANAGEMENT +#include "airtime.h" #include "mesh/CryptoEngine.h" #include "mesh/MeshService.h" #include "mesh/NodeDB.h" @@ -28,6 +30,25 @@ constexpr NodeNum kLocalNode = 0x11111111; constexpr NodeNum kRemoteNode = 0x22222222; constexpr NodeNum kTargetNode = 0x33333333; +// Telemetry hop exhaustion is gated on channel congestion (alterReceived checks +// airTime->isTxAllowedChannelUtil/isTxAllowedAirUtil). Installs a global +// airTime reporting 100% channel utilization for the enclosing scope. +class ScopedBusyAirTime +{ + public: + ScopedBusyAirTime() : previous(airTime) + { + for (uint32_t i = 0; i < CHANNEL_UTILIZATION_PERIODS; i++) + busy.channelUtilization[i] = 10000; // 10 s of airtime per 10 s period + airTime = &busy; + } + ~ScopedBusyAirTime() { airTime = previous; } + + private: + AirTime busy; + AirTime *previous; +}; + class MockNodeDB : public NodeDB { public: @@ -54,6 +75,32 @@ class MockNodeDB : public NodeDB cachedNode.bitfield |= NODEINFO_BITFIELD_HAS_USER_MASK; } + // Role the TMM should see for the cached node (sender-role-aware throttles). + void setCachedNodeRole(meshtastic_Config_DeviceConfig_Role role) { cachedNode.role = role; } + + // Seed a node into the hot-store buffer at index 1 (index 0 is reserved for + // "self"). Respects the fixed-buffer invariant: `meshNodes` is a buffer of + // MAX_NUM_NODES slots with `numMeshNodes` as the logical count — we grow the + // buffer if needed and bump the count, never clear()/push_back() (which would + // shrink it and break NodeDB::resetNodes()'s begin()+1..end() fill). + void setHotNode(NodeNum n, uint8_t nextHop) + { + if (meshNodes->size() < 2) + meshNodes->resize(2); + (*meshNodes)[1] = meshtastic_NodeInfoLite_init_zero; + (*meshNodes)[1].num = n; + (*meshNodes)[1].next_hop = nextHop; + numMeshNodes = 2; + } + + // Evict everything but "self" — simulates the hot DB rolling over. Logical + // count only; the buffer is left intact so the invariant holds. + void rollHotStore() + { + numMeshNodes = 1; + clearCachedNode(); + } + private: bool hasCachedNode = false; NodeNum cachedNodeNum = 0; @@ -102,8 +149,8 @@ class TrafficManagementModuleTestShim : public TrafficManagementModule { public: using TrafficManagementModule::alterReceived; + using TrafficManagementModule::flushCache; using TrafficManagementModule::handleReceived; - using TrafficManagementModule::resetEpoch; using TrafficManagementModule::runOnce; bool ignoreRequestFlag() const { return ignoreRequest; } @@ -121,6 +168,9 @@ static void resetTrafficConfig() config = meshtastic_LocalConfig_init_zero; config.device.role = meshtastic_Config_DeviceConfig_Role_CLIENT; + channelFile = meshtastic_ChannelFile_init_zero; + owner.is_licensed = false; + myNodeInfo.my_node_num = kLocalNode; router = nullptr; @@ -129,6 +179,10 @@ static void resetTrafficConfig() mockNodeDB->resetNodes(); mockNodeDB->clearCachedNode(); nodeDB = mockNodeDB; + + // Virtual clock base (1 h in, so tick subtraction never underflows). Tests advance time by + // bumping TrafficManagementModule::s_testNowMs instead of sleeping real seconds across a tick. + TrafficManagementModule::s_testNowMs = 3600000; } static meshtastic_MeshPacket makeDecodedPacket(meshtastic_PortNum port, NodeNum from, NodeNum to = NODENUM_BROADCAST) @@ -175,6 +229,42 @@ static meshtastic_MeshPacket makePositionPacket(NodeNum from, int32_t lat, int32 return packet; } +static meshtastic_MeshPacket makePositionPacketWithPrecision(NodeNum from, int32_t lat, int32_t lon, uint32_t precisionBits) +{ + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_POSITION_APP, from, NODENUM_BROADCAST); + meshtastic_Position pos = meshtastic_Position_init_zero; + pos.has_latitude_i = true; + pos.has_longitude_i = true; + pos.latitude_i = lat; + pos.longitude_i = lon; + pos.precision_bits = precisionBits; + + packet.decoded.payload.size = + pb_encode_to_bytes(packet.decoded.payload.bytes, sizeof(packet.decoded.payload.bytes), &meshtastic_Position_msg, &pos); + return packet; +} + +static bool decodePositionPayload(const meshtastic_MeshPacket &packet, meshtastic_Position &out) +{ + out = meshtastic_Position_init_zero; + return pb_decode_from_bytes(packet.decoded.payload.bytes, packet.decoded.payload.size, &meshtastic_Position_msg, &out); +} + +// Primary channel with a well-known single-byte PSK and the (empty -> preset) +// default name, so Channels::isWellKnownChannel(0) is true. +static void installWellKnownPrimaryChannel() +{ + channelFile = meshtastic_ChannelFile_init_zero; + channelFile.channels_count = 1; + channelFile.channels[0].index = 0; + channelFile.channels[0].has_settings = true; + channelFile.channels[0].role = meshtastic_Channel_Role_PRIMARY; + channelFile.channels[0].settings.psk.size = 1; + channelFile.channels[0].settings.psk.bytes[0] = 1; + config.lora.use_preset = true; + config.lora.modem_preset = meshtastic_Config_LoRaConfig_ModemPreset_LONG_FAST; +} + static meshtastic_MeshPacket makeNodeInfoPacket(NodeNum from, const char *longName, const char *shortName) { meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_NODEINFO_APP, from, NODENUM_BROADCAST); @@ -241,6 +331,7 @@ static void test_tm_positionDedup_dropsDuplicateWithinWindow(void) moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -266,6 +357,7 @@ static void test_tm_positionDedup_allowsMovedPosition(void) moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -290,7 +382,7 @@ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 3; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); @@ -305,31 +397,6 @@ static void test_tm_rateLimit_dropsOnlyAfterThreshold(void) TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); TEST_ASSERT_TRUE(module.ignoreRequestFlag()); } - -/** - * Verify routing/admin traffic is exempt from rate limiting. - * Important because throttling control traffic can destabilize the mesh. - */ -static void test_tm_rateLimit_skipsRoutingAndAdminPorts(void) -{ - moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 60; - moduleConfig.traffic_management.rate_limit_max_packets = 1; - TrafficManagementModuleTestShim module; - meshtastic_MeshPacket routingPacket = makeDecodedPacket(meshtastic_PortNum_ROUTING_APP, kRemoteNode); - meshtastic_MeshPacket adminPacket = makeDecodedPacket(meshtastic_PortNum_ADMIN_APP, kRemoteNode); - - for (int i = 0; i < 4; i++) { - ProcessMessage rr = module.handleReceived(routingPacket); - ProcessMessage ar = module.handleReceived(adminPacket); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(rr)); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(ar)); - } - - meshtastic_TrafficManagementStats stats = module.getStats(); - TEST_ASSERT_EQUAL_UINT32(0, stats.rate_limit_drops); -} - /** * Verify packets sourced from this node bypass dedup and rate limiting. * Important so local transmissions are not accidentally self-throttled. @@ -650,12 +717,15 @@ static void test_tm_nodeinfo_directResponse_psramMissDoesNotFallbackToNodeDb(voi #endif /** - * Verify relayed telemetry broadcasts are hop-exhausted when enabled. + * Verify relayed telemetry broadcasts are hop-exhausted when enabled AND the + * channel is congested (telemetry exhaustion is gated on channel utilization, + * unlike position exhaustion). * Important to prevent further mesh propagation while still allowing one relay step. */ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -677,6 +747,7 @@ static void test_tm_alterReceived_exhaustsRelayedTelemetryBroadcast(void) static void test_tm_alterReceived_skipsLocalAndUnicast(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // congestion satisfied, so only the skip conditions are under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket unicast = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, kTargetNode); @@ -705,7 +776,9 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) { moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; - moduleConfig.traffic_management.position_min_interval_secs = 1; + // 360 s = 1 pos-tick (kPosTimeTickMs); advance the virtual clock past one tick period. + moduleConfig.traffic_management.position_min_interval_secs = 360; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); @@ -714,7 +787,7 @@ static void test_tm_positionDedup_allowsDuplicateAfterIntervalExpires(void) ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); - testDelay(1200); + TrafficManagementModule::s_testNowMs += 360001; // advance past one 6-min pos-tick (virtual clock) ProcessMessage r3 = module.handleReceived(third); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -794,8 +867,11 @@ static void test_tm_positionDedup_precision32_allowsDistinctPositions(void) } /** - * Verify invalid precision=0 is treated as full precision. - * Important so invalid config does not collapse all positions into one fingerprint. + * Verify precision=0 falls back to the default precision (same contract as + * >32: getConfiguredOrDefault + sanitizePositionPrecision treat 0 as unset). + * Important so invalid config does not collapse all positions into one + * fingerprint — positions in different default-precision grid cells must + * still be distinct. */ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) { @@ -805,7 +881,7 @@ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); - meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 374221235, -1220845677); + meshtastic_MeshPacket second = makePositionPacket(kRemoteNode, 384221234, -1210845678); ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(second); @@ -820,20 +896,21 @@ static void test_tm_positionDedup_precisionZero_allowsDistinctPositions(void) * Verify epoch reset invalidates stale position identity for dedup. * Important so reset paths cannot leak prior packet identity into new windows. */ -static void test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset(void) +static void test_tm_positionDedup_cacheFlush_doesNotDropFirstPacketAfterFlush(void) { moduleConfig.traffic_management.position_dedup_enabled = true; moduleConfig.traffic_management.position_precision_bits = 16; moduleConfig.traffic_management.position_min_interval_secs = 300; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 374221234, -1220845678); - meshtastic_MeshPacket afterReset = makePositionPacket(kRemoteNode, 374221234, -1220845678); + meshtastic_MeshPacket afterFlush = makePositionPacket(kRemoteNode, 374221234, -1220845678); meshtastic_MeshPacket duplicate = makePositionPacket(kRemoteNode, 374221234, -1220845678); ProcessMessage r1 = module.handleReceived(first); - module.resetEpoch(millis()); - ProcessMessage r2 = module.handleReceived(afterReset); + module.flushCache(); + ProcessMessage r2 = module.handleReceived(afterFlush); ProcessMessage r3 = module.handleReceived(duplicate); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -855,13 +932,14 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero moduleConfig.traffic_management.rate_limit_enabled = true; moduleConfig.traffic_management.rate_limit_window_secs = 60; moduleConfig.traffic_management.rate_limit_max_packets = 10; + installWellKnownPrimaryChannel(); // dedup only runs on a well-known channel (gate in handleReceived) TrafficManagementModuleTestShim module; - meshtastic_MeshPacket text = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); meshtastic_MeshPacket first = makePositionPacket(kRemoteNode, 0x12300000, 0x45600000); meshtastic_MeshPacket duplicate = makePositionPacket(kRemoteNode, 0x12300000, 0x45600000); - ProcessMessage seeded = module.handleReceived(text); + ProcessMessage seeded = module.handleReceived(telemetry); ProcessMessage r1 = module.handleReceived(first); ProcessMessage r2 = module.handleReceived(duplicate); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -879,14 +957,15 @@ static void test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero static void test_tm_rateLimit_resetsAfterWindowExpires(void) { moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 1; + // 300 s = 1 rate-tick (kRateTimeTickMs); advance the virtual clock past one tick period. + moduleConfig.traffic_management.rate_limit_window_secs = 300; moduleConfig.traffic_management.rate_limit_max_packets = 1; TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); + meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(1200); + TrafficManagementModule::s_testNowMs += 300001; // advance past one 5-min rate-tick (virtual clock) ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -895,30 +974,6 @@ static void test_tm_rateLimit_resetsAfterWindowExpires(void) TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(r3)); TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); } - -/** - * Verify rate-limit thresholds above 255 effectively clamp to 255. - * Important because counters are uint8_t and must not overflow behavior. - */ -static void test_tm_rateLimit_thresholdAbove255_clamps(void) -{ - moduleConfig.traffic_management.rate_limit_enabled = true; - moduleConfig.traffic_management.rate_limit_window_secs = 60; - moduleConfig.traffic_management.rate_limit_max_packets = 300; - TrafficManagementModuleTestShim module; - meshtastic_MeshPacket packet = makeDecodedPacket(meshtastic_PortNum_TEXT_MESSAGE_APP, kRemoteNode); - - for (int i = 0; i < 255; i++) { - ProcessMessage result = module.handleReceived(packet); - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::CONTINUE), static_cast(result)); - } - ProcessMessage dropped = module.handleReceived(packet); - meshtastic_TrafficManagementStats stats = module.getStats(); - - TEST_ASSERT_EQUAL_INT(static_cast(ProcessMessage::STOP), static_cast(dropped)); - TEST_ASSERT_EQUAL_UINT32(1, stats.rate_limit_drops); -} - /** * Verify unknown-packet tracking resets after its active window expires. * Important so old unknown traffic does not trigger delayed drops. @@ -927,13 +982,12 @@ static void test_tm_unknownPackets_resetAfterWindowExpires(void) { moduleConfig.traffic_management.drop_unknown_enabled = true; moduleConfig.traffic_management.unknown_packet_threshold = 1; - moduleConfig.traffic_management.rate_limit_window_secs = 1; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode); ProcessMessage r1 = module.handleReceived(packet); ProcessMessage r2 = module.handleReceived(packet); - testDelay(1200); + TrafficManagementModule::s_testNowMs += 300001; // advance past 5 unknown-ticks (5 × 60s) (virtual clock) ProcessMessage r3 = module.handleReceived(packet); meshtastic_TrafficManagementStats stats = module.getStats(); @@ -966,12 +1020,14 @@ static void test_tm_unknownPackets_thresholdAbove255_clamps(void) } /** - * Verify relayed position broadcasts can also be hop-exhausted. + * Verify relayed position broadcasts can also be hop-exhausted — under the + * same pressure gate as telemetry (here: channel congestion). * Important because telemetry and position use separate exhaust flags. */ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) { moduleConfig.traffic_management.exhaust_hop_position = true; + ScopedBusyAirTime busyChannel; TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makePositionPacket(kRemoteNode, 374221234, -1220845678, NODENUM_BROADCAST); packet.hop_start = 5; @@ -985,7 +1041,6 @@ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) TEST_ASSERT_TRUE(module.shouldExhaustHops(packet)); TEST_ASSERT_EQUAL_UINT32(1, stats.hop_exhausted_packets); } - /** * Verify hop exhaustion ignores undecoded/encrypted packets. * Important so we never mutate packets that were not decoded by this module. @@ -993,6 +1048,7 @@ static void test_tm_alterReceived_exhaustsRelayedPositionBroadcast(void) static void test_tm_alterReceived_skipsUndecodedPackets(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // congestion satisfied, so only the undecoded skip is under test TrafficManagementModuleTestShim module; meshtastic_MeshPacket packet = makeUnknownPacket(kRemoteNode, NODENUM_BROADCAST); packet.hop_start = 5; @@ -1014,6 +1070,7 @@ static void test_tm_alterReceived_skipsUndecodedPackets(void) static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; meshtastic_MeshPacket telemetry = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); @@ -1039,6 +1096,7 @@ static void test_tm_alterReceived_resetExhaustFlagOnNextPacket(void) static void test_tm_alterReceived_exhaustFlag_isPacketScoped(void) { moduleConfig.traffic_management.exhaust_hop_telemetry = true; + ScopedBusyAirTime busyChannel; // telemetry exhaust only fires under congestion TrafficManagementModuleTestShim module; meshtastic_MeshPacket exhausted = makeDecodedPacket(meshtastic_PortNum_TELEMETRY_APP, kRemoteNode, NODENUM_BROADCAST); @@ -1083,6 +1141,93 @@ static void test_tm_runOnce_enabledReturnsMaintenanceInterval(void) TEST_ASSERT_EQUAL_INT32(60 * 1000, interval); } +// --------------------------------------------------------------------------- +// Next-hop overflow cache +// --------------------------------------------------------------------------- + +/** + * Round-trip set/get of a confirmed next hop, plus the input guards. + */ +static void test_tm_nextHop_setAndGetRoundTrip(void) +{ + TrafficManagementModuleTestShim module; + + // Unknown node yields no hint. + TEST_ASSERT_EQUAL_UINT8(0, module.getNextHopHint(kTargetNode)); + + // Store a confirmed hop and read it back. + module.setNextHop(kTargetNode, 0x42); + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); + + // Zero dest and zero byte are rejected (no spurious entry created). + module.setNextHop(0, 0x42); + module.setNextHop(kRemoteNode, 0); + TEST_ASSERT_EQUAL_UINT8(0, module.getNextHopHint(kRemoteNode)); + + // Last-write-wins on re-confirmation. + module.setNextHop(kTargetNode, 0x99); + TEST_ASSERT_EQUAL_UINT8(0x99, module.getNextHopHint(kTargetNode)); +} + +/** + * The headline scenario: a node carrying a next hop in the hot NodeInfoLite DB + * is warm-loaded into the TMM cache, then the hot DB is "rolled" (the node ages + * out entirely). The hint must still be served — now exclusively from TMM. + */ +static void test_tm_nextHop_servedAfterNodeDbRoll(void) +{ + TrafficManagementModuleTestShim module; + + // Seed the hot NodeInfoLite DB with a node that has a confirmed next hop. + mockNodeDB->setHotNode(kTargetNode, 0x42); + + // Warm-start the overflow cache from the hot DB. + module.preloadNextHopsFromNodeDB(); + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); + + // Roll the main NodeInfoLite DB: the node is evicted from the hot store. + mockNodeDB->rollHotStore(); + TEST_ASSERT_NULL(nodeDB->getMeshNode(kTargetNode)); // gone from the hot store + + // Hit is still served — proving it now comes from the TMM overflow cache. + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); +} + +/** + * Preload must not clobber a freshly-learned (confirmed) hop with a possibly + * stale persisted one from NodeInfoLite. + */ +static void test_tm_nextHop_preloadDoesNotClobberLearned(void) +{ + TrafficManagementModuleTestShim module; + + // A fresher confirmed hop is already cached. + module.setNextHop(kTargetNode, 0x99); + + // The hot DB carries an older next hop for the same node. + mockNodeDB->setHotNode(kTargetNode, 0x42); + + module.preloadNextHopsFromNodeDB(); + + // The freshly-learned hop survives. + TEST_ASSERT_EQUAL_UINT8(0x99, module.getNextHopHint(kTargetNode)); +} + +/** + * A pure routing hint (no dedup/rate/unknown state) must survive the maintenance + * sweep — next_hop != 0 keeps the slot alive even though it has no TTL. + */ +static void test_tm_nextHop_keptAliveAcrossMaintenanceSweep(void) +{ + TrafficManagementModuleTestShim module; + + module.setNextHop(kTargetNode, 0x42); + + // The sweep frees slots whose sub-stores are all empty; next_hop must veto that. + module.runOnce(); + + TEST_ASSERT_EQUAL_UINT8(0x42, module.getNextHopHint(kTargetNode)); +} } // namespace void setUp(void) @@ -1106,7 +1251,6 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_positionDedup_dropsDuplicateWithinWindow); RUN_TEST(test_tm_positionDedup_allowsMovedPosition); RUN_TEST(test_tm_rateLimit_dropsOnlyAfterThreshold); - RUN_TEST(test_tm_rateLimit_skipsRoutingAndAdminPorts); RUN_TEST(test_tm_fromUs_bypassesPositionAndRateFilters); RUN_TEST(test_tm_localDestination_bypassesTransitFilters); RUN_TEST(test_tm_nodeinfo_routerClamp_skipsWhenTooManyHops); @@ -1127,10 +1271,9 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_positionDedup_precisionAbove32_usesDefaultPrecision); RUN_TEST(test_tm_positionDedup_precision32_allowsDistinctPositions); RUN_TEST(test_tm_positionDedup_precisionZero_allowsDistinctPositions); - RUN_TEST(test_tm_positionDedup_epochReset_doesNotDropFirstPacketAfterReset); + RUN_TEST(test_tm_positionDedup_cacheFlush_doesNotDropFirstPacketAfterFlush); RUN_TEST(test_tm_positionDedup_priorRateState_doesNotDropFirstFingerprintZero); RUN_TEST(test_tm_rateLimit_resetsAfterWindowExpires); - RUN_TEST(test_tm_rateLimit_thresholdAbove255_clamps); RUN_TEST(test_tm_unknownPackets_resetAfterWindowExpires); RUN_TEST(test_tm_unknownPackets_thresholdAbove255_clamps); RUN_TEST(test_tm_alterReceived_exhaustsRelayedPositionBroadcast); @@ -1139,6 +1282,10 @@ TM_TEST_ENTRY void setup() RUN_TEST(test_tm_alterReceived_exhaustFlag_isPacketScoped); RUN_TEST(test_tm_runOnce_disabledReturnsMaxInterval); RUN_TEST(test_tm_runOnce_enabledReturnsMaintenanceInterval); + RUN_TEST(test_tm_nextHop_setAndGetRoundTrip); + RUN_TEST(test_tm_nextHop_servedAfterNodeDbRoll); + RUN_TEST(test_tm_nextHop_preloadDoesNotClobberLearned); + RUN_TEST(test_tm_nextHop_keptAliveAcrossMaintenanceSweep); exit(UNITY_END()); } diff --git a/test/test_warm_store/test_main.cpp b/test/test_warm_store/test_main.cpp new file mode 100644 index 00000000000..3c845588363 --- /dev/null +++ b/test/test_warm_store/test_main.cpp @@ -0,0 +1,211 @@ +// Unit tests for the warm ("long-tail") node tier — src/mesh/WarmNodeStore.cpp. +// Covers admission/eviction policy (keyed entries outrank keyless), take() +// rehydration semantics, and a tolerant persistence round trip. +#include "MeshTypes.h" // BEFORE TestUtil.h — provides WARM_NODE_COUNT via mesh-pb-constants.h +#include "TestUtil.h" +#include + +#if defined(ARCH_PORTDUINO) +#define WS_TEST_ENTRY extern "C" +#else +#define WS_TEST_ENTRY +#endif + +#if WARM_NODE_COUNT > 0 + +#include "mesh/WarmNodeStore.h" +#include + +namespace +{ + +void makeKey(uint8_t out[32], uint8_t seed) +{ + memset(out, 0, 32); + out[0] = seed; + out[31] = seed ^ 0xA5; +} + +} // namespace + +void setUp(void) {} +void tearDown(void) {} + +void test_ws_absorb_and_copyKey_roundTrip() +{ + WarmNodeStore ws; + uint8_t key[32], got[32]; + makeKey(key, 7); + TEST_ASSERT_TRUE(ws.absorb(0x100, 1000, key)); + TEST_ASSERT_TRUE(ws.contains(0x100)); + TEST_ASSERT_TRUE(ws.copyKey(0x100, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); + TEST_ASSERT_EQUAL(1, ws.count()); +} + +void test_ws_keylessEntry_hasNoKey() +{ + WarmNodeStore ws; + uint8_t got[32]; + TEST_ASSERT_TRUE(ws.absorb(0x200, 1000, NULL)); + TEST_ASSERT_TRUE(ws.contains(0x200)); + TEST_ASSERT_FALSE(ws.copyKey(0x200, got)); +} + +void test_ws_absorb_rejectsNodeNumZero() +{ + WarmNodeStore ws; + TEST_ASSERT_FALSE(ws.absorb(0, 1000, NULL)); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_absorb_updatesExistingEntry() +{ + WarmNodeStore ws; + uint8_t key[32], got[32]; + makeKey(key, 9); + TEST_ASSERT_TRUE(ws.absorb(0x300, 1000, NULL)); + TEST_ASSERT_TRUE(ws.absorb(0x300, 2000, key)); // later eviction learned a key + TEST_ASSERT_EQUAL(1, ws.count()); + TEST_ASSERT_TRUE(ws.copyKey(0x300, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); +} + +void test_ws_take_removesEntry() +{ + WarmNodeStore ws; + uint8_t key[32]; + makeKey(key, 3); + ws.absorb(0x400, 1234, key); + + WarmNodeEntry e; + TEST_ASSERT_TRUE(ws.take(0x400, e)); + TEST_ASSERT_EQUAL(0x400, e.num); + TEST_ASSERT_EQUAL(1234, e.last_heard); + TEST_ASSERT_EQUAL_MEMORY(key, e.public_key, 32); + TEST_ASSERT_FALSE(ws.contains(0x400)); + TEST_ASSERT_FALSE(ws.take(0x400, e)); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_keylessCandidate_neverEvictsKeyedEntries() +{ + WarmNodeStore ws; + uint8_t key[32]; + // Fill the store entirely with keyed entries + for (size_t i = 0; i < ws.capacity(); i++) { + makeKey(key, (uint8_t)i); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, 100 + i, key)); + } + TEST_ASSERT_EQUAL(ws.capacity(), ws.count()); + // A keyless candidate (even a fresh one) must be rejected + TEST_ASSERT_FALSE(ws.absorb(0x9999, 999999, NULL)); + TEST_ASSERT_FALSE(ws.contains(0x9999)); +} + +void test_ws_keyedCandidate_evictsOldestKeylessFirst() +{ + WarmNodeStore ws; + uint8_t key[32]; + makeKey(key, 0x42); + // Fill with keyed entries except two keyless ones in the middle + for (size_t i = 0; i < ws.capacity(); i++) { + const bool keyless = (i == 5 || i == 10); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, keyless ? (i == 10 ? 50 : 60) : 10, keyless ? NULL : key)); + } + // Keyed candidate must displace the OLDEST KEYLESS entry (0x100A, ts=50), + // even though every keyed entry is older (ts=10) + uint8_t k2[32]; + makeKey(k2, 0x43); + TEST_ASSERT_TRUE(ws.absorb(0x8888, 70, k2)); + TEST_ASSERT_FALSE(ws.contains(0x1000 + 10)); + TEST_ASSERT_TRUE(ws.contains(0x1000 + 5)); + TEST_ASSERT_TRUE(ws.contains(0x8888)); +} + +void test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless() +{ + WarmNodeStore ws; + uint8_t key[32]; + for (size_t i = 0; i < ws.capacity(); i++) { + makeKey(key, (uint8_t)i); + TEST_ASSERT_TRUE(ws.absorb(0x1000 + i, 1000 + i, key)); // 0x1000 is the oldest + } + uint8_t k2[32]; + makeKey(k2, 0x44); + TEST_ASSERT_TRUE(ws.absorb(0x7777, 999999, k2)); + TEST_ASSERT_TRUE(ws.contains(0x7777)); + TEST_ASSERT_FALSE(ws.contains(0x1000)); // oldest keyed evicted + TEST_ASSERT_EQUAL(ws.capacity(), ws.count()); +} + +void test_ws_remove_and_clear() +{ + WarmNodeStore ws; + ws.absorb(0x500, 1, NULL); + ws.absorb(0x501, 2, NULL); + ws.remove(0x500); + TEST_ASSERT_FALSE(ws.contains(0x500)); + TEST_ASSERT_EQUAL(1, ws.count()); + ws.clear(); + TEST_ASSERT_EQUAL(0, ws.count()); +} + +void test_ws_persistence_roundTrip() +{ + WarmNodeStore a; + uint8_t key[32], got[32]; + makeKey(key, 0x55); + a.absorb(0x600, 4242, key); + a.absorb(0x601, 4243, NULL); + if (!a.saveIfDirty()) { + TEST_IGNORE_MESSAGE("Filesystem not available in this test environment"); + return; + } + + WarmNodeStore b; + b.load(); + TEST_ASSERT_TRUE(b.contains(0x600)); + TEST_ASSERT_TRUE(b.contains(0x601)); + TEST_ASSERT_TRUE(b.copyKey(0x600, got)); + TEST_ASSERT_EQUAL_MEMORY(key, got, 32); + + // Cleanup so reruns start fresh + b.clear(); + b.saveIfDirty(); +} + +WS_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + RUN_TEST(test_ws_absorb_and_copyKey_roundTrip); + RUN_TEST(test_ws_keylessEntry_hasNoKey); + RUN_TEST(test_ws_absorb_rejectsNodeNumZero); + RUN_TEST(test_ws_absorb_updatesExistingEntry); + RUN_TEST(test_ws_take_removesEntry); + RUN_TEST(test_ws_keylessCandidate_neverEvictsKeyedEntries); + RUN_TEST(test_ws_keyedCandidate_evictsOldestKeylessFirst); + RUN_TEST(test_ws_keyedCandidate_evictsOldestKeyedWhenNoKeyless); + RUN_TEST(test_ws_remove_and_clear); + RUN_TEST(test_ws_persistence_roundTrip); + exit(UNITY_END()); +} + +WS_TEST_ENTRY void loop() {} + +#else + +void setUp(void) {} +void tearDown(void) {} + +WS_TEST_ENTRY void setup() +{ + initializeTestEnvironment(); + UNITY_BEGIN(); + exit(UNITY_END()); +} + +WS_TEST_ENTRY void loop() {} + +#endif diff --git a/userPrefs.jsonc b/userPrefs.jsonc index 57ede8bbf91..118588e195c 100644 --- a/userPrefs.jsonc +++ b/userPrefs.jsonc @@ -24,6 +24,7 @@ // "USERPREFS_CONFIG_OWNER_SHORT_NAME": "MLN", // "USERPREFS_CONFIG_DEVICE_ROLE": "meshtastic_Config_DeviceConfig_Role_CLIENT", // Defaults to CLIENT. ROUTER*, and LOST AND FOUND roles are restricted. // "USERPREFS_EVENT_MODE": "1", + // "USERPREFS_TMM_APPLY_TO_PRIVATE_CHANNELS": "1", // Extend TMM position dedup and precision clamping to private/custom-key channels (default: well-known channels only) // "USERPREFS_FIRMWARE_EDITION": "meshtastic_FirmwareEdition_BURNING_MAN", // "USERPREFS_FIXED_BLUETOOTH": "121212", // "USERPREFS_FIXED_GPS": "", diff --git a/variants/nrf52840/nrf52.ini b/variants/nrf52840/nrf52.ini index f99c78d8a3e..5a308796bdc 100644 --- a/variants/nrf52840/nrf52.ini +++ b/variants/nrf52840/nrf52.ini @@ -15,6 +15,7 @@ extra_scripts = ${env.extra_scripts} extra_scripts/nrf52_extra.py pre:extra_scripts/nrf52_lto.py + extra_scripts/nrf52_warm_region.py ; post-link guard: image must end below the 12 KB warm-store raw-flash region at 0xEA000-0xED000 build_type = release build_flags = diff --git a/variants/nrf52840/nrf52840.ini b/variants/nrf52840/nrf52840.ini index c5590cbc3f6..610b5b53030 100644 --- a/variants/nrf52840/nrf52840.ini +++ b/variants/nrf52840/nrf52840.ini @@ -1,6 +1,10 @@ [nrf52840_base] extends = nrf52_base +; Cap the app image at 0xEA000 (below the WarmNodeStore raw-flash region). +; Boards that have upgraded to S140 v7 override this in their own platformio.ini. +board_build.ldscript = src/platform/nrf52/nrf52840_s140_v6.ld + build_flags = ${nrf52_base.build_flags} -DSERIAL_BUFFER_SIZE=4096