From 7656338eca3ce7f61e2d754daf7170af81cfce9f Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:06:44 +0200 Subject: [PATCH 1/5] refactor(filter): share malformed scan and list editing as headers Move the repeater filter's malformed-text scan (validMessageContent, isValidUTF8) into the header-only MessageContent.h, and its sender/text rule add/remove, path-prefix add/remove and `filter age` argument parsing into SenderRules.h, PathBlock.h and MessageAge.h. Filter.cpp now delegates to them with unchanged signatures, replies and behaviour. This makes the logic usable by the companion receive filter and testable on the native env. --- examples/simple_repeater/Filter.cpp | 172 ++-------------------- examples/simple_repeater/Filter.h | 1 + examples/simple_repeater/MessageAge.h | 19 +++ examples/simple_repeater/MessageContent.h | 96 ++++++++++++ examples/simple_repeater/PathBlock.h | 39 +++++ examples/simple_repeater/SenderRules.h | 57 +++++++ 6 files changed, 225 insertions(+), 159 deletions(-) create mode 100644 examples/simple_repeater/MessageContent.h diff --git a/examples/simple_repeater/Filter.cpp b/examples/simple_repeater/Filter.cpp index d3c5a2d40a8..51cb756856a 100644 --- a/examples/simple_repeater/Filter.cpp +++ b/examples/simple_repeater/Filter.cpp @@ -425,17 +425,11 @@ void Filter::handleCommand(FILESYSTEM* fs, char* command, char* reply) { // message age } else if (strcmp(parts[1], "age") == 0) { - const char* v = parts[2]; - bool digits = v[0] != '\0'; - for (const char* c = v; *c; c++) { - if (*c < '0' || *c > '9') { digits = false; break; } - } - long mins = digits ? atol(v) : -1; - if (strcmp(v, "off") == 0) mins = 0; - if (mins < 0 || mins > FILTER_AGE_MAX_MINS) { + uint16_t mins; + if (!MessageAge::parseMins(parts[2], &mins)) { strcpy(reply, "> Filter: error range is 1-10080, or off"); } else { - _prefs.age_mins = (uint16_t)mins; + _prefs.age_mins = mins; save(fs); strcpy(reply, "> Filter: OK"); } @@ -530,71 +524,31 @@ void Filter::handleCommand(FILESYSTEM* fs, char* command, char* reply) { // ---- sender / text rules ------------------------------------------------------- -template -static bool addRule(Rule* rules, char* (*field)(Rule&), const char* pattern, size_t max_len, uint16_t secs, uint8_t prob) { - if (pattern == nullptr || pattern[0] == '\0' || strlen(pattern) > max_len) return false; - - int free_slot = -1; - for (int i = 0; i < FILTER_RULE_COUNT; i++) { - char* pat = field(rules[i]); - if (pat[0] == '\0') { - if (free_slot < 0) free_slot = i; - } else if (strcmp(pat, pattern) == 0) { - return false; // already present - } - } - if (free_slot < 0) return false; - - Rule& r = rules[free_slot]; - memset(&r, 0, sizeof(r)); - strncpy(field(r), pattern, max_len); - r.secs = secs; - r.prob = prob; - return true; -} - -template -static int findRule(Rule* rules, char* (*field)(Rule&), const char* pattern) { - if (pattern == nullptr || pattern[0] == '\0') return -1; - for (int i = 0; i < FILTER_RULE_COUNT; i++) { - if (strcmp(field(rules[i]), pattern) == 0) return i; - } - return -1; -} - -static char* senderField(SenderRule& r) { return r.name; } -static char* textField(TextRule& r) { return r.text; } - bool Filter::addSenderRule(const char* name, uint16_t secs, uint8_t prob) { - return addRule(_prefs.sender_rules, senderField, name, FILTER_SENDER_LEN - 1, secs, prob); + return FilterRules::addRule(_prefs.sender_rules, FILTER_RULE_COUNT, name, secs, prob); } bool Filter::removeSenderRule(const char* name) { - int i = findRule(_prefs.sender_rules, senderField, name); + int i = FilterRules::removeRule(_prefs.sender_rules, FILTER_RULE_COUNT, name); if (i < 0) return false; - memset(&_prefs.sender_rules[i], 0, sizeof(SenderRule)); _sender_last[i] = 0; return true; } bool Filter::addTextRule(const char* text, uint16_t secs, uint8_t prob) { - return addRule(_prefs.text_rules, textField, text, FILTER_TEXT_LEN - 1, secs, prob); + return FilterRules::addRule(_prefs.text_rules, FILTER_RULE_COUNT, text, secs, prob); } bool Filter::removeTextRule(const char* text) { - int i = findRule(_prefs.text_rules, textField, text); + int i = FilterRules::removeRule(_prefs.text_rules, FILTER_RULE_COUNT, text); if (i < 0) return false; - memset(&_prefs.text_rules[i], 0, sizeof(TextRule)); _text_last[i] = 0; return true; } bool Filter::hasContentRules(void) const { - for (int i = 0; i < FILTER_RULE_COUNT; i++) { - if (_prefs.sender_rules[i].name[0] != '\0') return true; - if (_prefs.text_rules[i].text[0] != '\0') return true; - } - return false; + return FilterRules::anyRule(_prefs.sender_rules, FILTER_RULE_COUNT) || + FilterRules::anyRule(_prefs.text_rules, FILTER_RULE_COUNT); } // The key to decrypt a group text with, when the rules may read it: Public is @@ -711,39 +665,11 @@ bool Filter::removeChannel(const char* name) { // `formatted` receives the canonical upper-case prefix, or "" when `hex` did // not parse, so the caller can tell a bad prefix from a full or missing slot. bool Filter::addPath(const char* hex, char* formatted) { - formatted[0] = '\0'; - PathPrefix p; - if (!FilterPath::parse(hex, &p)) return false; - FilterPath::format(formatted, 2 * FILTER_PATH_MAX_LEN + 1, p); - - int free_slot = -1; - for (int i = 0; i < FILTER_PATH_COUNT; i++) { - PathPrefix& cur = _prefs.path_block[i]; - if (cur.len == 0) { - if (free_slot < 0) free_slot = i; - } else if (cur.len == p.len && memcmp(cur.bytes, p.bytes, p.len) == 0) { - return false; // already blocked - } - } - if (free_slot < 0) return false; - _prefs.path_block[free_slot] = p; - return true; + return FilterPath::add(_prefs.path_block, FILTER_PATH_COUNT, hex, formatted); } bool Filter::removePath(const char* hex, char* formatted) { - formatted[0] = '\0'; - PathPrefix p; - if (!FilterPath::parse(hex, &p)) return false; - FilterPath::format(formatted, 2 * FILTER_PATH_MAX_LEN + 1, p); - - for (int i = 0; i < FILTER_PATH_COUNT; i++) { - PathPrefix& cur = _prefs.path_block[i]; - if (cur.len == p.len && memcmp(cur.bytes, p.bytes, p.len) == 0) { - memset(&cur, 0, sizeof(cur)); - return true; - } - } - return false; + return FilterPath::remove(_prefs.path_block, FILTER_PATH_COUNT, hex, formatted); } bool Filter::getChannelHash(const char* name, mesh::GroupChannel* gc) { @@ -792,83 +718,11 @@ void Filter::listChannelNames(char *out_buf, size_t out_size, bool with_counts) } bool Filter::validMessageContent(const uint8_t* data, uint8_t len, uint8_t* reason) { - if (data == nullptr || len <= 5) { - *reason = MALFORMED_SHORT; - return false; - } - - // check timestamp - uint32_t timestamp; - memcpy(×tamp, &data[0], 4); - if (MessageAge::implausible(timestamp, _rtc->getCurrentTime())) { - *reason = MALFORMED_TIMESTAMP; - return false; - } - - // check message type - uint8_t txt_type = data[4] >> 2; - if (txt_type != TXT_TYPE_PLAIN) return true; - - // calculate text length - uint8_t txt_len = 5; - while (txt_len < len && data[txt_len] != 0) txt_len++; - txt_len -= 5; - if (!txt_len) { - *reason = MALFORMED_EMPTY; - return false; - } - - // valid UTF8 - if (!isValidUTF8(&data[5], txt_len)) { - *reason = MALFORMED_UTF8; - return false; - } - - // valid - return true; + return MessageContent::validate(data, len, _rtc->getCurrentTime(), reason); } bool Filter::isValidUTF8(const uint8_t* data, uint8_t len) { - if (data == nullptr || len == 0) return false; - - uint8_t i = 0; - while (i < len) { - uint8_t c = data[i++]; - if (c == 0) break; - if (c < 0x80) continue; - - uint32_t codepoint; - uint8_t needed; - if ((c & 0xE0) == 0xC0) { - codepoint = c & 0x1F; - needed = 1; - if (codepoint == 0) return false; - } else if ((c & 0xF0) == 0xE0) { - codepoint = c & 0x0F; - needed = 2; - } else if ((c & 0xF8) == 0xF0) { - codepoint = c & 0x07; - needed = 3; - } else { - return false; - } - - if (i + needed > len) return false; - for (uint8_t j = 0; j < needed; j++) { - uint8_t cc = data[i++]; - if ((cc & 0xC0) != 0x80) return false; - codepoint = (codepoint << 6) | (cc & 0x3F); - } - - if (needed == 1 && codepoint < 0x80) return false; - if (needed == 2 && codepoint < 0x0800) return false; - if (needed == 3 && codepoint < 0x10000) return false; - if (codepoint > 0x10FFFF) return false; - if (codepoint >= 0xD800 && codepoint <= 0xDFFF) return false; - if (codepoint >= 0xFDD0 && codepoint <= 0xFDEF) return false; - if ((codepoint & 0xFFFE) == 0xFFFE) return false; - } - return true; + return MessageContent::isValidUTF8(data, len); } bool Filter::load(FILESYSTEM* fs) { diff --git a/examples/simple_repeater/Filter.h b/examples/simple_repeater/Filter.h index cdfc01e8386..fc7695b4c60 100644 --- a/examples/simple_repeater/Filter.h +++ b/examples/simple_repeater/Filter.h @@ -10,6 +10,7 @@ #include "FilterStats.h" #include "Limiter.h" #include "MessageAge.h" +#include "MessageContent.h" #include "PathBlock.h" #include "SenderRules.h" diff --git a/examples/simple_repeater/MessageAge.h b/examples/simple_repeater/MessageAge.h index 189b17861e1..56b08417953 100644 --- a/examples/simple_repeater/MessageAge.h +++ b/examples/simple_repeater/MessageAge.h @@ -1,6 +1,8 @@ #pragma once #include +#include +#include // Timestamp checks on decrypted group text: the malformed scan's sanity window // and the configurable `filter age` limit. Header-only and clock-free (the @@ -37,4 +39,21 @@ namespace MessageAge { if (max_mins == 0 || !clockSet(now)) return false; return ts < now && now - ts > (uint32_t)max_mins * 60UL; } + + // CLI argument of `filter age`: minutes (0 also means off) or "off". + inline bool parseMins(const char* v, uint16_t* out) { + if (v == nullptr || out == nullptr) return false; + if (strcmp(v, "off") == 0) { + *out = 0; + return true; + } + if (v[0] == '\0') return false; + for (const char* c = v; *c; c++) { + if (*c < '0' || *c > '9') return false; + } + long mins = atol(v); + if (mins < 0 || mins > FILTER_AGE_MAX_MINS) return false; + *out = (uint16_t)mins; + return true; + } } diff --git a/examples/simple_repeater/MessageContent.h b/examples/simple_repeater/MessageContent.h new file mode 100644 index 00000000000..cbad79e172f --- /dev/null +++ b/examples/simple_repeater/MessageContent.h @@ -0,0 +1,96 @@ +#pragma once + +#include +#include + +#include "FilterStats.h" +#include "MessageAge.h" + +// The malformed scan on decrypted group text: a plausible timestamp, and for +// plain text a non-empty, valid UTF-8 body. Shared by the repeater filter and +// the companion filter. Header-only and clock-free (the caller passes `now`) +// so it runs on `native`. + +namespace MessageContent { + + // Strict UTF-8: no overlong forms, surrogates, non-characters or code points + // past U+10FFFF. A NUL ends the scan early. + inline bool isValidUTF8(const uint8_t* data, uint8_t len) { + if (data == nullptr || len == 0) return false; + + uint8_t i = 0; + while (i < len) { + uint8_t c = data[i++]; + if (c == 0) break; + if (c < 0x80) continue; + + uint32_t codepoint; + uint8_t needed; + if ((c & 0xE0) == 0xC0) { + codepoint = c & 0x1F; + needed = 1; + if (codepoint == 0) return false; + } else if ((c & 0xF0) == 0xE0) { + codepoint = c & 0x0F; + needed = 2; + } else if ((c & 0xF8) == 0xF0) { + codepoint = c & 0x07; + needed = 3; + } else { + return false; + } + + if (i + needed > len) return false; + for (uint8_t j = 0; j < needed; j++) { + uint8_t cc = data[i++]; + if ((cc & 0xC0) != 0x80) return false; + codepoint = (codepoint << 6) | (cc & 0x3F); + } + + if (needed == 1 && codepoint < 0x80) return false; + if (needed == 2 && codepoint < 0x0800) return false; + if (needed == 3 && codepoint < 0x10000) return false; + if (codepoint > 0x10FFFF) return false; + if (codepoint >= 0xD800 && codepoint <= 0xDFFF) return false; + if (codepoint >= 0xFDD0 && codepoint <= 0xFDEF) return false; + if ((codepoint & 0xFFFE) == 0xFFFE) return false; + } + return true; + } + + // `data` is a decrypted GRP_TXT plaintext: 4-byte timestamp, flags byte + // (txt_type << 2), then the text. On failure `reason` gets a MalformedReason. + inline bool validate(const uint8_t* data, uint8_t len, uint32_t now, uint8_t* reason) { + if (data == nullptr || len <= 5) { + *reason = MALFORMED_SHORT; + return false; + } + + // check timestamp + uint32_t timestamp; + memcpy(×tamp, &data[0], 4); + if (MessageAge::implausible(timestamp, now)) { + *reason = MALFORMED_TIMESTAMP; + return false; + } + + // only plain text (txt_type 0) has a body to check + uint8_t txt_type = data[4] >> 2; + if (txt_type != 0) return true; + + // calculate text length + uint8_t txt_len = 5; + while (txt_len < len && data[txt_len] != 0) txt_len++; + txt_len -= 5; + if (!txt_len) { + *reason = MALFORMED_EMPTY; + return false; + } + + if (!isValidUTF8(&data[5], txt_len)) { + *reason = MALFORMED_UTF8; + return false; + } + return true; + } +} diff --git a/examples/simple_repeater/PathBlock.h b/examples/simple_repeater/PathBlock.h index beb47942375..7684ac58d97 100644 --- a/examples/simple_repeater/PathBlock.h +++ b/examples/simple_repeater/PathBlock.h @@ -59,6 +59,45 @@ namespace FilterPath { } } + // CLI add / remove. `formatted` (2 * FILTER_PATH_MAX_LEN + 1 bytes) receives + // the canonical upper-case prefix, or "" when `hex` did not parse, so the + // caller can tell a bad prefix from a duplicate, a full list or a missing one. + inline bool add(PathPrefix* list, int count, const char* hex, char* formatted) { + formatted[0] = '\0'; + PathPrefix p; + if (!parse(hex, &p)) return false; + format(formatted, 2 * FILTER_PATH_MAX_LEN + 1, p); + + int free_slot = -1; + for (int i = 0; i < count; i++) { + PathPrefix& cur = list[i]; + if (cur.len == 0) { + if (free_slot < 0) free_slot = i; + } else if (cur.len == p.len && memcmp(cur.bytes, p.bytes, p.len) == 0) { + return false; // already blocked + } + } + if (free_slot < 0) return false; + list[free_slot] = p; + return true; + } + + inline bool remove(PathPrefix* list, int count, const char* hex, char* formatted) { + formatted[0] = '\0'; + PathPrefix p; + if (!parse(hex, &p)) return false; + format(formatted, 2 * FILTER_PATH_MAX_LEN + 1, p); + + for (int i = 0; i < count; i++) { + PathPrefix& cur = list[i]; + if (cur.len == p.len && memcmp(cur.bytes, p.bytes, p.len) == 0) { + memset(&cur, 0, sizeof(cur)); + return true; + } + } + return false; + } + // Index of the first prefix that matches the start of any path entry, or -1. // A prefix longer than the packet's hash size never matches: A1B2 is one // 2-byte ID, not the two 1-byte IDs A1 then B2. diff --git a/examples/simple_repeater/SenderRules.h b/examples/simple_repeater/SenderRules.h index 49d1092cb90..91056012de0 100644 --- a/examples/simple_repeater/SenderRules.h +++ b/examples/simple_repeater/SenderRules.h @@ -126,6 +126,63 @@ namespace FilterRules { return -1; } + // ---- list editing (CLI add / remove) ---------------------------------------- + + inline char* patternBuf(SenderRule& r) { return r.name; } + inline char* patternBuf(TextRule& r) { return r.text; } + inline size_t patternMax(const SenderRule&) { return FILTER_SENDER_LEN - 1; } + inline size_t patternMax(const TextRule&) { return FILTER_TEXT_LEN - 1; } + + // Stores a rule in the first free slot. Fails on an empty or over-long + // pattern, a pattern already present, or a full list. + template + inline bool addRule(Rule* rules, int count, const char* pattern, uint16_t secs, uint8_t prob) { + if (rules == nullptr || count <= 0) return false; + size_t max_len = patternMax(rules[0]); + if (pattern == nullptr || pattern[0] == '\0' || strlen(pattern) > max_len) return false; + + int free_slot = -1; + for (int i = 0; i < count; i++) { + char* pat = patternBuf(rules[i]); + if (pat[0] == '\0') { + if (free_slot < 0) free_slot = i; + } else if (strcmp(pat, pattern) == 0) { + return false; // already present + } + } + if (free_slot < 0) return false; + + Rule& r = rules[free_slot]; + memset(&r, 0, sizeof(r)); + strncpy(patternBuf(r), pattern, max_len); + r.secs = secs; + r.prob = prob; + return true; + } + + // Clears the rule with exactly this pattern. Returns its slot, so the caller + // can reset per-slot state, or -1 when there is none. + template + inline int removeRule(Rule* rules, int count, const char* pattern) { + if (rules == nullptr || pattern == nullptr || pattern[0] == '\0') return -1; + for (int i = 0; i < count; i++) { + if (strcmp(patternBuf(rules[i]), pattern) == 0) { + memset(&rules[i], 0, sizeof(Rule)); + return i; + } + } + return -1; + } + + template + inline bool anyRule(const Rule* rules, int count) { + if (rules == nullptr) return false; + for (int i = 0; i < count; i++) { + if (patternOf(rules[i])[0] != '\0') return true; + } + return false; + } + inline bool parseUint(const char* s, long max, long* out) { if (s == nullptr || s[0] == '\0') return false; for (const char* p = s; *p; p++) { From e8b852929fffdadb68d1d8d97952e4c1c88956b9 Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:06:44 +0200 Subject: [PATCH 2/5] feat(companion): add DMC receive filter A `filter` CLI on the companion firmware that keeps junk and unwanted channel traffic out of the app: malformed group text, group text older than N minutes, floods relayed by a blocked repeater prefix, and sender or text rules (block, throttle, dose) reusing the repeater rule engine. Junk (malformed, age, path) is dropped before the mesh handles it, so a companion with repeat enabled does not repeat it either. Sender and text rules only hide: the message is still repeated, so one user's mute list never censors the mesh. Direct and addressed traffic is never filtered. Settings persist as JSON in /cfilter.json; off by default. The shared MyMesh files only gain one-line calls into CompanionFilterHook.cpp. --- examples/companion_radio/CompanionFilter.cpp | 280 +++++++++++ examples/companion_radio/CompanionFilter.h | 154 ++++++ .../companion_radio/CompanionFilterHook.cpp | 101 ++++ examples/companion_radio/MyMesh.cpp | 6 +- examples/companion_radio/MyMesh.h | 9 + platformio.ini | 1 + .../test_companion_filter.cpp | 454 ++++++++++++++++++ 7 files changed, 1004 insertions(+), 1 deletion(-) create mode 100644 examples/companion_radio/CompanionFilter.cpp create mode 100644 examples/companion_radio/CompanionFilter.h create mode 100644 examples/companion_radio/CompanionFilterHook.cpp create mode 100644 test/test_companion_filter/test_companion_filter.cpp diff --git a/examples/companion_radio/CompanionFilter.cpp b/examples/companion_radio/CompanionFilter.cpp new file mode 100644 index 00000000000..1580ec6c547 --- /dev/null +++ b/examples/companion_radio/CompanionFilter.cpp @@ -0,0 +1,280 @@ +#include "CompanionFilter.h" + +#include + +// Copied into the reply verbatim; guard them at compile time so a future +// subcommand cannot overflow the reply buffer silently. +static const char CFILTER_HELP[] = + "> filter [ help | on | off | reset | stats | malformed | age | path | sender | text ]"; +static const char CFILTER_STATS_HELP[] = + "> filter stats [ malformed | age | path | sender | text ]"; +static const char CFILTER_PATH_HELP[] = + "> filter path [ list | add | remove ] "; +static const char CFILTER_RULE_HELP[] = + "> filter sender|text [ list | add [secs] [prob] | remove ]"; +static const char CFILTER_RULE_ARGS_ERR[] = + "> Filter: error range is 0-65535, range is 1-100"; + +static_assert(sizeof(CFILTER_HELP) <= CFILTER_REPLY_SIZE, "filter help text no longer fits the reply buffer"); +static_assert(sizeof(CFILTER_STATS_HELP) <= CFILTER_REPLY_SIZE, "filter stats help text no longer fits the reply buffer"); +static_assert(sizeof(CFILTER_PATH_HELP) <= CFILTER_REPLY_SIZE, "filter path help text no longer fits the reply buffer"); +static_assert(sizeof(CFILTER_RULE_HELP) <= CFILTER_REPLY_SIZE, "filter rule help text no longer fits the reply buffer"); + +// ---- preferences --------------------------------------------------------------- + +void CompanionFilterPrefs::reset() { + enabled = false; + malformed = false; + age_mins = 0; + memset(path_block, 0, sizeof(path_block)); + memset(sender_rules, 0, sizeof(sender_rules)); + memset(text_rules, 0, sizeof(text_rules)); +} + +void CompanionFilterPrefs::sanitize() { + if (age_mins > FILTER_AGE_MAX_MINS) age_mins = FILTER_AGE_MAX_MINS; + for (int i = 0; i < FILTER_PATH_COUNT; i++) { + if (path_block[i].len > FILTER_PATH_MAX_LEN) memset(&path_block[i], 0, sizeof(PathPrefix)); + } + for (int i = 0; i < FILTER_RULE_COUNT; i++) { + SenderRule& sr = sender_rules[i]; + sr.name[FILTER_SENDER_LEN - 1] = '\0'; + if (sr.prob == 0 || sr.prob > 100) sr.prob = 100; + TextRule& tr = text_rules[i]; + tr.text[FILTER_TEXT_LEN - 1] = '\0'; + if (tr.prob == 0 || tr.prob > 100) tr.prob = 100; + } +} + +// ---- verdicts ------------------------------------------------------------------ + +bool CompanionFilter::wantsGroupText() const { + if (!_prefs.enabled) return false; + return _prefs.malformed || _prefs.age_mins > 0 || + FilterRules::anyRule(_prefs.sender_rules, FILTER_RULE_COUNT) || + FilterRules::anyRule(_prefs.text_rules, FILTER_RULE_COUNT); +} + +CFilterVerdict CompanionFilter::checkPath(const mesh::Packet* packet) { + if (!_prefs.enabled || packet == nullptr || !packet->isRouteFlood()) return CFilterVerdict::PASS; + + uint8_t type = packet->getPayloadType(); + if (type != PAYLOAD_TYPE_ADVERT && type != PAYLOAD_TYPE_GRP_TXT && type != PAYLOAD_TYPE_GRP_DATA) { + return CFilterVerdict::PASS; + } + + int slot = FilterPath::findMatch(packet->path, packet->getPathHashSize(), packet->getPathHashCount(), + _prefs.path_block, FILTER_PATH_COUNT); + if (slot < 0) return CFilterVerdict::PASS; + FilterStat::recordPath(_cnt, slot); + return CFilterVerdict::DROP; +} + +CFilterVerdict CompanionFilter::checkGroupText(const uint8_t* data, int len, uint32_t now_secs, uint32_t now_ms, uint8_t roll) { + if (!_prefs.enabled || data == nullptr || len <= 0) return CFilterVerdict::PASS; + if (len > 255) len = 255; // a LoRa payload never is; keeps the uint8_t length below honest + + // junk first: it is dropped outright, not merely hidden + if (_prefs.malformed) { + uint8_t reason; + if (!MessageContent::validate(data, (uint8_t)len, now_secs, &reason)) { + FilterStat::recordMalformed(_cnt, reason); + return CFilterVerdict::DROP; + } + } + + if (_prefs.age_mins > 0 && len >= 4) { + uint32_t ts; + memcpy(&ts, data, 4); + if (MessageAge::tooOld(ts, now_secs, _prefs.age_mins)) { + FilterStat::recordAge(_cnt); + return CFilterVerdict::DROP; + } + } + + // then the user's own rules + FilterRules::GroupText gt; + if (!FilterRules::parseGroupText(data, len, >)) return CFilterVerdict::PASS; + + int slot = FilterRules::evaluate(_prefs.sender_rules, FILTER_RULE_COUNT, now_ms, roll, _sender_last, _cnt.sender_pass, + [&](const SenderRule& r) { return FilterRules::matchName(r.name, gt.sender, gt.sender_len); }); + if (slot >= 0) { + FilterStat::recordSender(_cnt, slot); + return CFilterVerdict::HIDE; + } + slot = FilterRules::evaluate(_prefs.text_rules, FILTER_RULE_COUNT, now_ms, roll, _text_last, _cnt.text_pass, + [&](const TextRule& r) { return FilterRules::matchText(r.text, gt.text, gt.text_len); }); + if (slot >= 0) { + FilterStat::recordText(_cnt, slot); + return CFilterVerdict::HIDE; + } + return CFilterVerdict::PASS; +} + +bool CompanionFilter::alreadyJudged(const uint8_t hash[MAX_HASH_SIZE]) { + for (int i = 0; i < _judged_count; i++) { + if (memcmp(_judged[i], hash, MAX_HASH_SIZE) == 0) return true; + } + memcpy(_judged[_judged_next], hash, MAX_HASH_SIZE); + _judged_next = (uint8_t)((_judged_next + 1) % CFILTER_JUDGED_COUNT); + if (_judged_count < CFILTER_JUDGED_COUNT) _judged_count++; + return false; +} + +// ---- CLI ----------------------------------------------------------------------- + +bool CompanionFilter::isCommand(const char* command) { + return command != nullptr && strncmp(command, "filter", 6) == 0 && (command[6] == '\0' || command[6] == ' '); +} + +static void formatSummary(char* out, const Counters& c, bool enabled) { + FilterStat::Buf b(out, CFILTER_REPLY_SIZE); + b.add("> Filter %s: Hidden [ Malformed: %lu | Age: %lu | Path: %lu | Sender: %lu | Text: %lu ]", + enabled ? "on" : "off", + (unsigned long)c.malformed, (unsigned long)c.age, (unsigned long)c.path, + (unsigned long)c.sender, (unsigned long)c.text); + b.markTruncated(".."); +} + +bool CompanionFilter::handleCommand(const char* command, char* reply, uint32_t now_secs) { + char buf[CFILTER_REPLY_SIZE]; + strncpy(buf, command, sizeof(buf) - 1); + buf[sizeof(buf) - 1] = '\0'; + + const char* parts[6]; + int n = mesh::Utils::parseTextParts(buf, parts, 6, ' '); + bool changed = false; + + if (n <= 1) { + formatSummary(reply, _cnt, _prefs.enabled); + return false; + } + + const char* sub = parts[1]; + + if (n == 2) { + if (strcmp(sub, "help") == 0) { + strcpy(reply, CFILTER_HELP); + } else if (strcmp(sub, "on") == 0 || strcmp(sub, "off") == 0) { + _prefs.enabled = strcmp(sub, "on") == 0; + strcpy(reply, _prefs.enabled ? "> Filter: on" : "> Filter: off"); + changed = true; + } else if (strcmp(sub, "reset") == 0) { + _prefs.reset(); + clearThrottles(); + strcpy(reply, "> Filter: preferences reset"); + changed = true; + } else if (strcmp(sub, "stats") == 0) { + strcpy(reply, CFILTER_STATS_HELP); + } else if (strcmp(sub, "malformed") == 0) { + sprintf(reply, "> Filter: malformed text scan %s", _prefs.malformed ? "on" : "off"); + } else if (strcmp(sub, "age") == 0) { + FilterStat::formatStatsAge(reply, CFILTER_REPLY_SIZE, _cnt, _prefs.age_mins, MessageAge::clockSet(now_secs)); + } else if (strcmp(sub, "path") == 0) { + strcpy(reply, CFILTER_PATH_HELP); + } else if (strcmp(sub, "sender") == 0 || strcmp(sub, "text") == 0) { + strcpy(reply, CFILTER_RULE_HELP); + } else { + strcpy(reply, "> Filter: command error"); + } + return changed; + } + + const char* arg = parts[2]; + + if (strcmp(sub, "stats") == 0) { + if (strcmp(arg, "malformed") == 0) { + FilterStat::formatStatsMalformed(reply, CFILTER_REPLY_SIZE, _cnt); + } else if (strcmp(arg, "age") == 0) { + FilterStat::formatStatsAge(reply, CFILTER_REPLY_SIZE, _cnt, _prefs.age_mins, MessageAge::clockSet(now_secs)); + } else if (strcmp(arg, "path") == 0) { + FilterStat::formatPathList(reply, CFILTER_REPLY_SIZE, _prefs.path_block, &_cnt); + } else if (strcmp(arg, "sender") == 0) { + FilterStat::formatRuleList(reply, CFILTER_REPLY_SIZE, _prefs.sender_rules, FILTER_RULE_COUNT, _cnt.sender_slot, _cnt.sender_pass); + } else if (strcmp(arg, "text") == 0) { + FilterStat::formatRuleList(reply, CFILTER_REPLY_SIZE, _prefs.text_rules, FILTER_RULE_COUNT, _cnt.text_slot, _cnt.text_pass); + } else { + strcpy(reply, CFILTER_STATS_HELP); + } + + } else if (strcmp(sub, "malformed") == 0) { + if (strcmp(arg, "on") == 0 || strcmp(arg, "off") == 0) { + _prefs.malformed = strcmp(arg, "on") == 0; + strcpy(reply, _prefs.malformed ? "> Filter: malformed scan on" : "> Filter: malformed scan off"); + changed = true; + } else { + strcpy(reply, "> Filter: syntax error 'filter malformed '"); + } + + } else if (strcmp(sub, "age") == 0) { + uint16_t mins; + if (MessageAge::parseMins(arg, &mins)) { + _prefs.age_mins = mins; + strcpy(reply, "> Filter: OK"); + changed = true; + } else { + strcpy(reply, "> Filter: error range is 1-10080, or off"); + } + + } else if (strcmp(sub, "path") == 0) { + char hex[2 * FILTER_PATH_MAX_LEN + 1]; + bool is_add = strcmp(arg, "add") == 0; + if (strcmp(arg, "list") == 0) { + FilterStat::formatPathList(reply, CFILTER_REPLY_SIZE, _prefs.path_block, nullptr); + } else if (n >= 4 && (is_add || strcmp(arg, "remove") == 0)) { + bool ok = is_add ? FilterPath::add(_prefs.path_block, FILTER_PATH_COUNT, parts[3], hex) + : FilterPath::remove(_prefs.path_block, FILTER_PATH_COUNT, parts[3], hex); + if (ok) { + sprintf(reply, "> Filter: path %s %s", hex, is_add ? "added" : "removed"); + changed = true; + } else if (hex[0] == '\0') { + strcpy(reply, "> Filter: error path prefix is 2-8 hex digits"); + } else { + strcpy(reply, "Failed"); + } + } else { + strcpy(reply, CFILTER_PATH_HELP); + } + + } else if (strcmp(sub, "sender") == 0 || strcmp(sub, "text") == 0) { + bool is_sender = strcmp(sub, "sender") == 0; + if (strcmp(arg, "list") == 0) { + if (is_sender) { + FilterStat::formatRuleList(reply, CFILTER_REPLY_SIZE, _prefs.sender_rules, FILTER_RULE_COUNT, nullptr, nullptr); + } else { + FilterStat::formatRuleList(reply, CFILTER_REPLY_SIZE, _prefs.text_rules, FILTER_RULE_COUNT, nullptr, nullptr); + } + } else if (n >= 4 && strcmp(arg, "add") == 0) { + uint16_t secs; + uint8_t prob; + size_t max_len = is_sender ? FILTER_SENDER_LEN - 1 : FILTER_TEXT_LEN - 1; + if (strlen(parts[3]) > max_len) { + sprintf(reply, "> Filter: error max %u chars", (unsigned)max_len); + } else if (!FilterRules::parseArgs(n >= 5 ? parts[4] : nullptr, n >= 6 ? parts[5] : nullptr, &secs, &prob)) { + strcpy(reply, CFILTER_RULE_ARGS_ERR); + } else if (is_sender ? FilterRules::addRule(_prefs.sender_rules, FILTER_RULE_COUNT, parts[3], secs, prob) + : FilterRules::addRule(_prefs.text_rules, FILTER_RULE_COUNT, parts[3], secs, prob)) { + snprintf(reply, CFILTER_REPLY_SIZE, "> Filter: %s %s added", sub, parts[3]); + changed = true; + } else { + strcpy(reply, "Failed"); + } + } else if (n >= 4 && strcmp(arg, "remove") == 0) { + int slot = is_sender ? FilterRules::removeRule(_prefs.sender_rules, FILTER_RULE_COUNT, parts[3]) + : FilterRules::removeRule(_prefs.text_rules, FILTER_RULE_COUNT, parts[3]); + if (slot >= 0) { + (is_sender ? _sender_last : _text_last)[slot] = 0; + snprintf(reply, CFILTER_REPLY_SIZE, "> Filter: %s %s removed", sub, parts[3]); + changed = true; + } else { + strcpy(reply, "Failed"); + } + } else { + strcpy(reply, CFILTER_RULE_HELP); + } + + } else { + strcpy(reply, "> Filter: command error"); + } + return changed; +} diff --git a/examples/companion_radio/CompanionFilter.h b/examples/companion_radio/CompanionFilter.h new file mode 100644 index 00000000000..311ef3d2f73 --- /dev/null +++ b/examples/companion_radio/CompanionFilter.h @@ -0,0 +1,154 @@ +#pragma once + +#include +#include + +#include +#include + +// Rule engine, list editing, malformed scan and reply formatting are shared +// with the repeater packet filter. All of these are header-only and free of +// radio, filesystem and clock, so this filter runs on `native` too. +#include "../simple_repeater/FilterStats.h" +#include "../simple_repeater/MessageAge.h" +#include "../simple_repeater/MessageContent.h" +#include "../simple_repeater/PathBlock.h" +#include "../simple_repeater/SenderRules.h" + +// DMC receive-side filter for the companion firmware. A companion does not +// normally repeat, so this exists to keep junk and unwanted channel traffic +// away from the user. Two verdicts besides PASS: +// +// DROP junk (malformed text, too old, via a blocked repeater): not shown, +// and not repeated either when the companion has repeat enabled. +// HIDE personal taste (sender / text rules): not shown, but still repeated, +// so one user's mute list never censors the mesh for others. +// +// Settings persist as JSON in CFILTER_PREFS_FILE and are edited with the +// `filter ...` CLI, reachable over the companion protocol (CMD_RUN_CLI_COMMAND). + +#define CFILTER_PREFS_FILE "/cfilter.json" + +// The tightest reply buffer is remote CLI over the mesh: the reply sits at +// offset 5 of a 166-byte buffer (161 bytes), after an optional 3-byte "xx|" +// prefix echoed from the command. 158 fits every caller. +#define CFILTER_REPLY_SIZE 158 + +// Recently judged packets remembered, so the copies every nearby repeater +// relays are not judged and counted again. +#define CFILTER_JUDGED_COUNT 16 + +enum class CFilterVerdict : uint8_t { PASS, HIDE, DROP }; + +class CompanionFilterPrefs : public ConfigSerializer { +public: + bool enabled = false; + bool malformed = false; + uint16_t age_mins = 0; // 0 = off + PathPrefix path_block[FILTER_PATH_COUNT] = {}; + SenderRule sender_rules[FILTER_RULE_COUNT] = {}; + TextRule text_rules[FILTER_RULE_COUNT] = {}; + + CompanionFilterPrefs() : _sender(sender_rules), _text(text_rules) { } + CompanionFilterPrefs(const CompanionFilterPrefs&) = delete; // the list serializers point into this object + CompanionFilterPrefs& operator=(const CompanionFilterPrefs&) = delete; + + void reset(); + void sanitize(); // after a load: clamp anything a hand-edited file got wrong + +private: + // One rule as {"p":"pattern","secs":N,"prob":N}. + template + class RuleSer : public ConfigSerializer { + public: + Rule* rule = nullptr; + protected: + void structure() override { + def("p", FilterRules::patternBuf(*rule), FilterRules::patternMax(*rule) + 1); + def("secs", rule->secs); + def("prob", rule->prob); + } + }; + + // A rule list as {"0":{...},"3":{...}}, keyed by slot; empty slots are not written. + template + class RuleListSer : public ConfigSerializer { + Rule* _rules; + RuleSer _items[FILTER_RULE_COUNT]; + public: + explicit RuleListSer(Rule* rules) : _rules(rules) { + for (int i = 0; i < FILTER_RULE_COUNT; i++) _items[i].rule = &_rules[i]; + } + protected: + void structure() override { + static const char* const KEYS[] = { "0", "1", "2", "3", "4", "5", "6", "7" }; + static_assert(FILTER_RULE_COUNT <= sizeof(KEYS) / sizeof(KEYS[0]), "add keys for the extra rule slots"); + for (int i = 0; i < FILTER_RULE_COUNT; i++) { + if (_context->op() == OP::WRITE && FilterRules::patternOf(_rules[i])[0] == '\0') continue; + def(KEYS[i], _items[i]); + } + } + }; + + RuleListSer _sender; + RuleListSer _text; + +protected: + void structure() override { + def("on", enabled); + def("malformed", malformed); + def("age", age_mins); + def("path", (void*)path_block, sizeof(path_block)); + def("sender", _sender); + def("text", _text); + } +}; + +class CompanionFilter { + CompanionFilterPrefs _prefs; + Counters _cnt; + uint32_t _sender_last[FILTER_RULE_COUNT]; // throttle state per sender rule (millis, 0 = never) + uint32_t _text_last[FILTER_RULE_COUNT]; // throttle state per text rule + uint8_t _judged[CFILTER_JUDGED_COUNT][MAX_HASH_SIZE]; + uint8_t _judged_count = 0; + uint8_t _judged_next = 0; + +public: + CompanionFilter() { clearThrottles(); } + + CompanionFilterPrefs& prefs() { return _prefs; } + const Counters& counters() const { return _cnt; } + bool isEnabled() const { return _prefs.enabled; } + + // Whether checkGroupText() can say anything, i.e. whether decrypting a + // flood group text up front is worth it. + bool wantsGroupText() const; + + // Flood ADVERT / GRP_TXT / GRP_DATA whose path holds a blocked repeater + // prefix: DROP. Traffic addressed to this node is never path-blocked. + CFilterVerdict checkPath(const mesh::Packet* packet); + + // `data` is a decrypted GRP_TXT plaintext (timestamp, flags, "Sender: text"). + // `roll` is a random 0..99 for the rules' probability. + CFilterVerdict checkGroupText(const uint8_t* data, int len, uint32_t now_secs, uint32_t now_ms, uint8_t roll); + + // True when this packet hash was judged recently; otherwise records it. + // The caller lets such a copy through to the mesh's own duplicate check. + bool alreadyJudged(const uint8_t hash[MAX_HASH_SIZE]); + + static bool isCommand(const char* command); + + // `filter ...`. `reply` must hold CFILTER_REPLY_SIZE bytes. Returns true when + // the preferences changed and should be saved. + bool handleCommand(const char* command, char* reply, uint32_t now_secs); + + void clearThrottles() { + memset(_sender_last, 0, sizeof(_sender_last)); + memset(_text_last, 0, sizeof(_text_last)); + } +}; + +// Firmware-side persistence (CompanionFilterHook.cpp); not built on `native`. +class DataStore; +bool loadCompanionFilter(CompanionFilter& filter, DataStore& store); +bool saveCompanionFilter(CompanionFilter& filter, DataStore& store); diff --git a/examples/companion_radio/CompanionFilterHook.cpp b/examples/companion_radio/CompanionFilterHook.cpp new file mode 100644 index 00000000000..b8d4a326989 --- /dev/null +++ b/examples/companion_radio/CompanionFilterHook.cpp @@ -0,0 +1,101 @@ +// DMC companion filter: the MyMesh side. Lives in its own file so the shared +// upstream MyMesh.{cpp,h} only carry one-line calls into it. + +#include "MyMesh.h" + +bool loadCompanionFilter(CompanionFilter& filter, DataStore& store) { + FILESYSTEM* fs = store.getPrimaryFS(); + if (fs == nullptr || !fs->exists(CFILTER_PREFS_FILE)) return true; // nothing saved yet: defaults (off) + + File file = store.openRead(fs, CFILTER_PREFS_FILE); + if (!file) return false; + filter.prefs().reset(); + bool ok = filter.prefs().loadSerial(file); + file.close(); + filter.prefs().sanitize(); + return ok; +} + +bool saveCompanionFilter(CompanionFilter& filter, DataStore& store) { + FILESYSTEM* fs = store.getPrimaryFS(); + if (fs == nullptr) return false; + +#if defined(NRF52_PLATFORM) || defined(STM32_PLATFORM) + fs->remove(CFILTER_PREFS_FILE); + File file = fs->open(CFILTER_PREFS_FILE, FILE_O_WRITE); +#elif defined(RP2040_PLATFORM) + File file = fs->open(CFILTER_PREFS_FILE, "w"); +#else + File file = fs->open(CFILTER_PREFS_FILE, "w", true); +#endif + if (!file) return false; + bool ok = filter.prefs().saveSerial(file); + file.close(); + return ok; +} + +// From filterRecvFloodPacket(): true drops the packet before the mesh handles +// it, so it is neither shown nor repeated. +bool MyMesh::dmcFilterFlood(mesh::Packet* packet) { + _cfilter_hidden_set = false; + if (!_cfilter.isEnabled()) return false; + + uint8_t hash[MAX_HASH_SIZE]; + packet->calculatePacketHash(hash); + // A relayed copy of a packet already judged: the mesh's duplicate check + // handles it (a dropped one was marked seen below). + if (_cfilter.alreadyJudged(hash)) return false; + + CFilterVerdict verdict = _cfilter.checkPath(packet); + + // Group text is decrypted here with this node's own channel keys, the same + // way Mesh::onRecvPacket() is about to. Only channels the user holds can be + // read, so only those are filtered on content. + if (verdict == CFilterVerdict::PASS && packet->getPayloadType() == PAYLOAD_TYPE_GRP_TXT && + _cfilter.wantsGroupText() && packet->payload_len > PATH_HASH_SIZE + CIPHER_MAC_SIZE) { + mesh::GroupChannel channels[4]; + int num = searchChannelsByHash(packet->payload, channels, 4); + for (int i = 0; i < num; i++) { + uint8_t data[MAX_PACKET_PAYLOAD + 1]; + int len = mesh::Utils::MACThenDecrypt(channels[i].secret, data, &packet->payload[PATH_HASH_SIZE], + packet->payload_len - PATH_HASH_SIZE); + if (len > 0) { + verdict = _cfilter.checkGroupText(data, len, getRTCClock()->getCurrentTime(), millis(), (uint8_t)random(100)); + break; + } + } + } + + if (verdict == CFilterVerdict::PASS) return false; + + if (verdict == CFilterVerdict::HIDE && _prefs.isRepeatEn()) { + // A personal rule: still repeated for everyone else, only not shown here. + memcpy(_cfilter_hidden, hash, sizeof(_cfilter_hidden)); + _cfilter_hidden_set = true; + return false; + } + + getTables()->markSeen(packet); // later copies then die in the mesh's duplicate check + return true; +} + +// From onChannelMessageRecv(): true when this is the message dmcFilterFlood() +// let through only so it could be repeated. +bool MyMesh::dmcFilterHidden(const mesh::Packet* packet) { + if (!_cfilter_hidden_set) return false; + + uint8_t hash[MAX_HASH_SIZE]; + packet->calculatePacketHash(hash); + if (memcmp(hash, _cfilter_hidden, sizeof(_cfilter_hidden)) != 0) return false; + _cfilter_hidden_set = false; + return true; +} + +// From handleCommand(): `filter ...`. +bool MyMesh::dmcFilterCommand(const char* command, char* reply) { + if (!CompanionFilter::isCommand(command)) return false; + if (_cfilter.handleCommand(command, reply, getRTCClock()->getCurrentTime())) { + saveCompanionFilter(_cfilter, *_store); + } + return true; +} diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index 7b38adb7a3b..fc4df40ceaf 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -482,7 +482,7 @@ void MyMesh::queueMessage(const ContactInfo &from, uint8_t txt_type, mesh::Packe bool MyMesh::filterRecvFloodPacket(mesh::Packet* packet) { // REVISIT: try to determine which Region (from transport_codes[1]) that Sender is indicating for replies/responses // if unknown, fallback to finding Region from transport_codes[0], the 'scope' used by Sender - return false; + return dmcFilterFlood(packet); // DMC companion filter } bool MyMesh::allowPacketForward(const mesh::Packet* packet) { @@ -558,6 +558,7 @@ void MyMesh::onSignedMessageRecv(const ContactInfo &from, mesh::Packet *pkt, uin void MyMesh::onChannelMessageRecv(const mesh::GroupChannel &channel, mesh::Packet *pkt, uint32_t timestamp, const char *text) { + if (dmcFilterHidden(pkt)) return; // DMC companion filter: hidden, only let through to be repeated int i = 0; if (app_target_ver >= 3) { out_frame[i++] = RESP_CODE_CHANNEL_MSG_RECV_V3; @@ -979,6 +980,7 @@ void MyMesh::begin() { // load persisted prefs _store->loadPrefs(_prefs); + loadCompanionFilter(_cfilter, *_store); // DMC companion filter sensors.node_lat = _prefs.node_lat; sensors.node_lon = _prefs.node_lon; @@ -2210,6 +2212,8 @@ bool MyMesh::handleCommand(const char* command, uint32_t sender_timestamp, char* return true; } + if (dmcFilterCommand(command, reply)) return true; // DMC companion filter + return false; // not handled } diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index 944790df922..fd806ad5885 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -22,6 +22,7 @@ #include #endif +#include "CompanionFilter.h" #include "DataStore.h" #include "NodePrefs.h" @@ -140,6 +141,14 @@ class MyMesh : public BaseChatMesh, public DataStoreHost { bool filterRecvFloodPacket(mesh::Packet* packet) override; bool allowPacketForward(const mesh::Packet* packet) override; + // DMC companion filter (CompanionFilterHook.cpp) + bool dmcFilterFlood(mesh::Packet* packet); + bool dmcFilterHidden(const mesh::Packet* packet); + bool dmcFilterCommand(const char* command, char* reply); + CompanionFilter _cfilter; + uint8_t _cfilter_hidden[MAX_HASH_SIZE]; + bool _cfilter_hidden_set = false; + void sendFloodScoped(const TransportKey& scope, mesh::Packet* pkt, uint32_t delay_millis); void sendFloodScoped(const ContactInfo& recipient, mesh::Packet* pkt, uint32_t delay_millis=0) override; void sendFloodScoped(const mesh::GroupChannel& channel, mesh::Packet* pkt, uint32_t delay_millis=0) override; diff --git a/platformio.ini b/platformio.ini index c2ea771cd54..27e7585a493 100644 --- a/platformio.ini +++ b/platformio.ini @@ -175,6 +175,7 @@ build_src_filter = +<../src/helpers/ConfigSerializer.cpp> +<../src/helpers/DynamicConfigSerializer.cpp> +<../src/helpers/DutyCycleLimits.cpp> + +<../examples/companion_radio/CompanionFilter.cpp> lib_deps = google/googletest @ 1.17.0 diff --git a/test/test_companion_filter/test_companion_filter.cpp b/test/test_companion_filter/test_companion_filter.cpp new file mode 100644 index 00000000000..f0f243a7e99 --- /dev/null +++ b/test/test_companion_filter/test_companion_filter.cpp @@ -0,0 +1,454 @@ +#include + +#include +#include + +// Receive-side filter of the companion firmware, plus the repeater-filter +// helpers it shares (malformed scan, rule and path-list editing, age parsing). +// Everything here is free of radio, filesystem and clock, so it runs on `native`. +#include "../../examples/companion_radio/CompanionFilter.h" + +// A clock that has been set (after MessageAge::CLOCK_SET_AFTER): Sep 2026. +static const uint32_t NOW = 1790000000UL; + +// Decrypted GRP_TXT plaintext: 4-byte timestamp, flags byte (txt_type << 2), body. +static int plaintext(uint8_t* buf, uint32_t ts, const char* body, uint8_t txt_type = 0) { + memcpy(buf, &ts, 4); + buf[4] = (uint8_t)(txt_type << 2); + size_t n = strlen(body); + memcpy(buf + 5, body, n); + return (int)(5 + n); +} + +static std::string cmd(CompanionFilter& f, const char* command, bool* changed = nullptr) { + char reply[CFILTER_REPLY_SIZE]; + memset(reply, 'X', sizeof(reply)); + bool c = f.handleCommand(command, reply, NOW); + if (changed) *changed = c; + return std::string(reply); +} + +static mesh::Packet floodPacket(uint8_t type, const uint8_t* path, uint8_t hash_size, uint8_t hash_count) { + mesh::Packet p; + memset(&p, 0, sizeof(p)); + p.header = (uint8_t)((type << PH_TYPE_SHIFT) | ROUTE_TYPE_FLOOD); + p.setPathHashSizeAndCount(hash_size, hash_count); + memcpy(p.path, path, (size_t)hash_size * hash_count); + p.payload_len = 20; + return p; +} + +// ---- MessageContent (shared malformed scan) ------------------------------------- + +TEST(MessageContent, AcceptsPlainTextWithAPlausibleStamp) { + uint8_t buf[64]; + uint8_t reason = 0xFF; + EXPECT_TRUE(MessageContent::validate(buf, plaintext(buf, NOW, "Alice: hi"), NOW, &reason)); +} + +TEST(MessageContent, RejectsShortZeroStampEmptyAndBadUtf8) { + uint8_t buf[64]; + uint8_t reason; + + EXPECT_FALSE(MessageContent::validate(buf, 5, NOW, &reason)); + EXPECT_EQ(MALFORMED_SHORT, reason); + + EXPECT_FALSE(MessageContent::validate(buf, plaintext(buf, 0, "Alice: hi"), NOW, &reason)); + EXPECT_EQ(MALFORMED_TIMESTAMP, reason); + + int len = plaintext(buf, NOW, ""); + buf[len++] = 0; // a NUL right after the flags byte: no text at all + EXPECT_FALSE(MessageContent::validate(buf, len, NOW, &reason)); + EXPECT_EQ(MALFORMED_EMPTY, reason); + + len = plaintext(buf, NOW, "Alice: "); + buf[len++] = 0xC0; // overlong lead byte + buf[len++] = 0xAF; + EXPECT_FALSE(MessageContent::validate(buf, len, NOW, &reason)); + EXPECT_EQ(MALFORMED_UTF8, reason); +} + +TEST(MessageContent, JudgesStampsOnlyOnceTheClockIsSet) { + uint8_t buf[64]; + uint8_t reason; + uint32_t two_weeks = 14UL * 24UL * 3600UL; + + EXPECT_FALSE(MessageContent::validate(buf, plaintext(buf, NOW - two_weeks, "A: b"), NOW, &reason)); + EXPECT_EQ(MALFORMED_TIMESTAMP, reason); + EXPECT_TRUE(MessageContent::validate(buf, plaintext(buf, NOW - two_weeks, "A: b"), 1000, &reason)) + << "an unset clock judges no stamp"; +} + +TEST(MessageContent, LeavesNonPlainTextTypesAlone) { + uint8_t buf[64]; + uint8_t reason; + int len = plaintext(buf, NOW, "", 1); + buf[len++] = 0xFF; + EXPECT_TRUE(MessageContent::validate(buf, len, NOW, &reason)); +} + +TEST(MessageContent, Utf8RejectsSurrogatesAndAcceptsEmoji) { + const uint8_t surrogate[] = { 0xED, 0xA0, 0x80 }; + const uint8_t emoji[] = { 0xF0, 0x9F, 0x94, 0x8B }; + EXPECT_FALSE(MessageContent::isValidUTF8(surrogate, sizeof(surrogate))); + EXPECT_TRUE(MessageContent::isValidUTF8(emoji, sizeof(emoji))); +} + +// ---- shared list editing ------------------------------------------------------- + +TEST(FilterRulesEdit, AddsIntoTheFirstFreeSlotAndRejectsDuplicates) { + SenderRule rules[FILTER_RULE_COUNT] = {}; + + EXPECT_TRUE(FilterRules::addRule(rules, FILTER_RULE_COUNT, "Bot*", 0, 100)); + EXPECT_FALSE(FilterRules::addRule(rules, FILTER_RULE_COUNT, "Bot*", 0, 100)); + EXPECT_STREQ("Bot*", rules[0].name); + + EXPECT_EQ(0, FilterRules::removeRule(rules, FILTER_RULE_COUNT, "Bot*")); + EXPECT_EQ(-1, FilterRules::removeRule(rules, FILTER_RULE_COUNT, "Bot*")); + EXPECT_EQ('\0', rules[0].name[0]); +} + +TEST(FilterRulesEdit, RejectsOverlongPatternsAndAFullList) { + TextRule rules[FILTER_RULE_COUNT] = {}; + std::string longest(FILTER_TEXT_LEN - 1, 'x'); + std::string too_long(FILTER_TEXT_LEN, 'x'); + + EXPECT_TRUE(FilterRules::addRule(rules, FILTER_RULE_COUNT, longest.c_str(), 30, 50)); + EXPECT_EQ(30, rules[0].secs); + EXPECT_EQ(50, rules[0].prob); + EXPECT_FALSE(FilterRules::addRule(rules, FILTER_RULE_COUNT, too_long.c_str(), 0, 100)); + EXPECT_FALSE(FilterRules::addRule(rules, FILTER_RULE_COUNT, "", 0, 100)); + + for (int i = 1; i < FILTER_RULE_COUNT; i++) { + std::string p = "p" + std::to_string(i); + ASSERT_TRUE(FilterRules::addRule(rules, FILTER_RULE_COUNT, p.c_str(), 0, 100)); + } + EXPECT_FALSE(FilterRules::addRule(rules, FILTER_RULE_COUNT, "one more", 0, 100)); + EXPECT_TRUE(FilterRules::anyRule(rules, FILTER_RULE_COUNT)); +} + +TEST(FilterPathEdit, AddRemoveReportTheCanonicalPrefix) { + PathPrefix list[FILTER_PATH_COUNT] = {}; + char hex[2 * FILTER_PATH_MAX_LEN + 1]; + + EXPECT_TRUE(FilterPath::add(list, FILTER_PATH_COUNT, "a1b2", hex)); + EXPECT_STREQ("A1B2", hex); + EXPECT_FALSE(FilterPath::add(list, FILTER_PATH_COUNT, "A1B2", hex)) << "duplicate"; + EXPECT_STREQ("A1B2", hex) << "a duplicate still parsed, so it is not a syntax error"; + + EXPECT_FALSE(FilterPath::add(list, FILTER_PATH_COUNT, "xyz", hex)); + EXPECT_STREQ("", hex) << "empty marks a bad prefix"; + + EXPECT_TRUE(FilterPath::remove(list, FILTER_PATH_COUNT, "a1b2", hex)); + EXPECT_FALSE(FilterPath::remove(list, FILTER_PATH_COUNT, "a1b2", hex)); +} + +TEST(MessageAgeParse, AcceptsMinutesOrOff) { + uint16_t mins = 99; + EXPECT_TRUE(MessageAge::parseMins("off", &mins)); + EXPECT_EQ(0, mins); + EXPECT_TRUE(MessageAge::parseMins("60", &mins)); + EXPECT_EQ(60, mins); + EXPECT_TRUE(MessageAge::parseMins("10080", &mins)); + EXPECT_FALSE(MessageAge::parseMins("10081", &mins)); + EXPECT_FALSE(MessageAge::parseMins("-1", &mins)); + EXPECT_FALSE(MessageAge::parseMins("1h", &mins)); + EXPECT_FALSE(MessageAge::parseMins("", &mins)); +} + +// ---- CompanionFilter: verdicts ------------------------------------------------- + +TEST(CompanionFilter, IsOffByDefaultAndPassesEverything) { + CompanionFilter f; + uint8_t buf[64]; + int len = plaintext(buf, 0, "Bot: junk"); + f.prefs().malformed = true; + ASSERT_TRUE(FilterRules::addRule(f.prefs().sender_rules, FILTER_RULE_COUNT, "Bot", 0, 100)); + + EXPECT_FALSE(f.wantsGroupText()); + EXPECT_EQ(CFilterVerdict::PASS, f.checkGroupText(buf, len, NOW, 1000, 0)); +} + +TEST(CompanionFilter, DropsMalformedText) { + CompanionFilter f; + f.prefs().enabled = true; + f.prefs().malformed = true; + uint8_t buf[64]; + + EXPECT_TRUE(f.wantsGroupText()); + EXPECT_EQ(CFilterVerdict::DROP, f.checkGroupText(buf, plaintext(buf, 0, "A: b"), NOW, 1000, 0)); + EXPECT_EQ(CFilterVerdict::PASS, f.checkGroupText(buf, plaintext(buf, NOW, "A: b"), NOW, 1000, 0)); + EXPECT_EQ(1u, f.counters().malformed); + EXPECT_EQ(1u, f.counters().malformed_reason[MALFORMED_TIMESTAMP]); +} + +TEST(CompanionFilter, DropsTextOlderThanTheAgeLimit) { + CompanionFilter f; + f.prefs().enabled = true; + f.prefs().age_mins = 30; + uint8_t buf[64]; + + EXPECT_EQ(CFilterVerdict::DROP, f.checkGroupText(buf, plaintext(buf, NOW - 31 * 60, "A: b"), NOW, 1000, 0)); + EXPECT_EQ(CFilterVerdict::PASS, f.checkGroupText(buf, plaintext(buf, NOW - 29 * 60, "A: b"), NOW, 1000, 0)); + EXPECT_EQ(1u, f.counters().age); +} + +TEST(CompanionFilter, HidesBySenderAndTextRules) { + CompanionFilter f; + f.prefs().enabled = true; + ASSERT_TRUE(FilterRules::addRule(f.prefs().sender_rules, FILTER_RULE_COUNT, "Bot*", 0, 100)); + ASSERT_TRUE(FilterRules::addRule(f.prefs().text_rules, FILTER_RULE_COUNT, "^!ping", 0, 100)); + uint8_t buf[64]; + + EXPECT_EQ(CFilterVerdict::HIDE, f.checkGroupText(buf, plaintext(buf, NOW, "BotNL: weather"), NOW, 1000, 0)); + EXPECT_EQ(CFilterVerdict::HIDE, f.checkGroupText(buf, plaintext(buf, NOW, "Alice: !ping"), NOW, 1000, 0)); + EXPECT_EQ(CFilterVerdict::PASS, f.checkGroupText(buf, plaintext(buf, NOW, "Alice: hi !ping"), NOW, 1000, 0)); + EXPECT_EQ(1u, f.counters().sender); + EXPECT_EQ(1u, f.counters().text); +} + +TEST(CompanionFilter, ThrottleRuleLetsOneMessagePerWindowThrough) { + CompanionFilter f; + f.prefs().enabled = true; + ASSERT_TRUE(FilterRules::addRule(f.prefs().sender_rules, FILTER_RULE_COUNT, "Beacon", 60, 100)); + uint8_t buf[64]; + int len = plaintext(buf, NOW, "Beacon: tick"); + + EXPECT_EQ(CFilterVerdict::PASS, f.checkGroupText(buf, len, NOW, 1000, 0)); + EXPECT_EQ(CFilterVerdict::HIDE, f.checkGroupText(buf, len, NOW, 30000, 0)); + EXPECT_EQ(CFilterVerdict::PASS, f.checkGroupText(buf, len, NOW, 61000, 0)); +} + +TEST(CompanionFilter, JunkChecksRunBeforePersonalRules) { + CompanionFilter f; + f.prefs().enabled = true; + f.prefs().malformed = true; + ASSERT_TRUE(FilterRules::addRule(f.prefs().sender_rules, FILTER_RULE_COUNT, "Bot", 0, 100)); + uint8_t buf[64]; + + EXPECT_EQ(CFilterVerdict::DROP, f.checkGroupText(buf, plaintext(buf, 0, "Bot: x"), NOW, 1000, 0)) + << "junk is dropped (not forwarded), not merely hidden"; +} + +TEST(CompanionFilter, PathBlockDropsGroupAndAdvertFloodsOnly) { + CompanionFilter f; + f.prefs().enabled = true; + char hex[2 * FILTER_PATH_MAX_LEN + 1]; + ASSERT_TRUE(FilterPath::add(f.prefs().path_block, FILTER_PATH_COUNT, "A1", hex)); + const uint8_t path[] = { 0x10, 0xA1 }; + + mesh::Packet grp = floodPacket(PAYLOAD_TYPE_GRP_TXT, path, 1, 2); + mesh::Packet adv = floodPacket(PAYLOAD_TYPE_ADVERT, path, 1, 2); + mesh::Packet dat = floodPacket(PAYLOAD_TYPE_GRP_DATA, path, 1, 2); + mesh::Packet dm = floodPacket(PAYLOAD_TYPE_TXT_MSG, path, 1, 2); + + EXPECT_EQ(CFilterVerdict::DROP, f.checkPath(&grp)); + EXPECT_EQ(CFilterVerdict::DROP, f.checkPath(&adv)); + EXPECT_EQ(CFilterVerdict::DROP, f.checkPath(&dat)); + EXPECT_EQ(CFilterVerdict::PASS, f.checkPath(&dm)) << "traffic addressed to us is never path-blocked"; + EXPECT_EQ(3u, f.counters().path); + + const uint8_t clean[] = { 0x10, 0x20 }; + mesh::Packet ok = floodPacket(PAYLOAD_TYPE_GRP_TXT, clean, 1, 2); + EXPECT_EQ(CFilterVerdict::PASS, f.checkPath(&ok)); + + f.prefs().enabled = false; + EXPECT_EQ(CFilterVerdict::PASS, f.checkPath(&grp)); +} + +TEST(CompanionFilter, RemembersRecentlyJudgedPacketsSoRelayedCopiesAreNotRecounted) { + CompanionFilter f; + uint8_t h[MAX_HASH_SIZE] = { 1 }; + + EXPECT_FALSE(f.alreadyJudged(h)); + EXPECT_TRUE(f.alreadyJudged(h)); + + for (int i = 0; i < CFILTER_JUDGED_COUNT; i++) { + uint8_t other[MAX_HASH_SIZE] = { 2, (uint8_t)i }; + EXPECT_FALSE(f.alreadyJudged(other)); + } + EXPECT_FALSE(f.alreadyJudged(h)) << "the oldest entry is evicted"; +} + +// ---- CompanionFilter: CLI ------------------------------------------------------ + +TEST(CompanionFilterCli, RecognisesOnlyTheFilterCommand) { + EXPECT_TRUE(CompanionFilter::isCommand("filter")); + EXPECT_TRUE(CompanionFilter::isCommand("filter on")); + EXPECT_FALSE(CompanionFilter::isCommand("filters")); + EXPECT_FALSE(CompanionFilter::isCommand("get name")); +} + +TEST(CompanionFilterCli, OnOffAndSummary) { + CompanionFilter f; + bool changed; + + EXPECT_EQ("> Filter: on", cmd(f, "filter on", &changed)); + EXPECT_TRUE(changed); + EXPECT_TRUE(f.prefs().enabled); + EXPECT_EQ("> Filter on: Hidden [ Malformed: 0 | Age: 0 | Path: 0 | Sender: 0 | Text: 0 ]", cmd(f, "filter", &changed)); + EXPECT_FALSE(changed); + EXPECT_EQ("> Filter: off", cmd(f, "filter off")); + EXPECT_FALSE(f.prefs().enabled); +} + +TEST(CompanionFilterCli, MalformedAndAge) { + CompanionFilter f; + bool changed; + + EXPECT_EQ("> Filter: malformed scan on", cmd(f, "filter malformed on", &changed)); + EXPECT_TRUE(changed); + EXPECT_EQ("> Filter: malformed text scan on", cmd(f, "filter malformed")); + EXPECT_EQ("> Filter: syntax error 'filter malformed '", cmd(f, "filter malformed maybe", &changed)); + EXPECT_FALSE(changed); + + EXPECT_EQ("> Filter: OK", cmd(f, "filter age 45")); + EXPECT_EQ(45, f.prefs().age_mins); + EXPECT_EQ("> Filter: error range is 1-10080, or off", cmd(f, "filter age 99999", &changed)); + EXPECT_FALSE(changed); + EXPECT_EQ("> Message age: max 45m, dropped 0", cmd(f, "filter age")); + EXPECT_EQ("> Filter: OK", cmd(f, "filter age off")); + EXPECT_EQ("> Filter: message age limit off", cmd(f, "filter age")); +} + +TEST(CompanionFilterCli, PathList) { + CompanionFilter f; + + EXPECT_EQ("None", cmd(f, "filter path list")); + EXPECT_EQ("> Filter: path A1 added", cmd(f, "filter path add a1")); + EXPECT_EQ("Failed", cmd(f, "filter path add A1")); + EXPECT_EQ("> Filter: error path prefix is 2-8 hex digits", cmd(f, "filter path add q")); + EXPECT_EQ("A1", cmd(f, "filter path list")); + EXPECT_EQ("> Filter: path A1 removed", cmd(f, "filter path remove A1")); +} + +TEST(CompanionFilterCli, SenderAndTextRules) { + CompanionFilter f; + + EXPECT_EQ("> Filter: sender Bot* added", cmd(f, "filter sender add Bot*")); + EXPECT_EQ("> Filter: text ^!ping added", cmd(f, "filter text add ^!ping 30 50")); + EXPECT_EQ("Bot* block", cmd(f, "filter sender list")); + EXPECT_EQ("^!ping throttle 30s 50%", cmd(f, "filter text list")); + EXPECT_EQ("> Filter: error range is 0-65535, range is 1-100", cmd(f, "filter text add x 1 0")); + EXPECT_EQ("> Filter: sender Bot* removed", cmd(f, "filter sender remove Bot*")); + EXPECT_EQ("Failed", cmd(f, "filter sender remove Bot*")); +} + +TEST(CompanionFilterCli, StatsTopics) { + CompanionFilter f; + f.prefs().enabled = true; + f.prefs().malformed = true; + uint8_t buf[64]; + f.checkGroupText(buf, plaintext(buf, 0, "A: b"), NOW, 1000, 0); + + EXPECT_EQ("> Blocked 1 [ short:0 time:1 empty:0 utf8:0 ]", cmd(f, "filter stats malformed")); + EXPECT_EQ("> filter stats [ malformed | age | path | sender | text ]", cmd(f, "filter stats")); + EXPECT_EQ("> Filter: command error", cmd(f, "filter bogus")); +} + +TEST(CompanionFilterCli, ResetClearsEveryPreference) { + CompanionFilter f; + cmd(f, "filter on"); + cmd(f, "filter malformed on"); + cmd(f, "filter sender add Bot"); + + EXPECT_EQ("> Filter: preferences reset", cmd(f, "filter reset")); + EXPECT_FALSE(f.prefs().enabled); + EXPECT_FALSE(f.prefs().malformed); + EXPECT_FALSE(FilterRules::anyRule(f.prefs().sender_rules, FILTER_RULE_COUNT)); +} + +TEST(CompanionFilterCli, EveryReplyFitsTheBuffer) { + CompanionFilter f; + for (int i = 0; i < FILTER_RULE_COUNT; i++) { + std::string c = "filter text add " + std::string(FILTER_TEXT_LEN - 3, 'a' + i) + " 65535 99"; + cmd(f, c.c_str()); + } + std::string r = cmd(f, "filter text list"); + EXPECT_LT(r.size(), (size_t)CFILTER_REPLY_SIZE); + EXPECT_EQ("..", r.substr(r.size() - 2)) << "a clipped list must say so"; +} + +// ---- CompanionFilter: persisted preferences ------------------------------------- + +class MemStream : public Stream { + std::string _data; + size_t _pos = 0; + + size_t printNum(const char* fmt, long long v) { + char t[24]; + snprintf(t, sizeof(t), fmt, v); + return Print::print(t); + } + +public: + size_t write(uint8_t b) override { _data.push_back((char)b); return 1; } + size_t print(unsigned char v, int) override { return printNum("%lld", v); } + size_t print(int v, int) override { return printNum("%lld", v); } + size_t print(unsigned int v, int) override { return printNum("%lld", v); } + size_t print(long v, int) override { return printNum("%lld", v); } + size_t print(unsigned long v, int) override { return printNum("%lld", (long long)v); } + int available() override { return (int)(_data.size() - _pos); } + int read() override { return _pos < _data.size() ? (uint8_t)_data[_pos++] : -1; } + int peek() override { return _pos < _data.size() ? (uint8_t)_data[_pos] : -1; } + const std::string& str() const { return _data; } +}; + +TEST(CompanionFilterPrefs, RoundTripsThroughJson) { + CompanionFilter a; + cmd(a, "filter on"); + cmd(a, "filter malformed on"); + cmd(a, "filter age 120"); + cmd(a, "filter path add A1B2"); + cmd(a, "filter path add 3C"); + cmd(a, "filter sender add Bot*"); + cmd(a, "filter sender add Alice 30 50"); + cmd(a, "filter text add \"quoted\\"); + + MemStream s; + ASSERT_TRUE(a.prefs().saveSerial(s)); + + CompanionFilter b; + ASSERT_TRUE(b.prefs().loadSerial(s)) << s.str(); + b.prefs().sanitize(); + + EXPECT_TRUE(b.prefs().enabled); + EXPECT_TRUE(b.prefs().malformed); + EXPECT_EQ(120, b.prefs().age_mins); + EXPECT_EQ(cmd(a, "filter path list"), cmd(b, "filter path list")); + EXPECT_EQ(cmd(a, "filter sender list"), cmd(b, "filter sender list")); + EXPECT_EQ(cmd(a, "filter text list"), cmd(b, "filter text list")); + EXPECT_STREQ("\"quoted\\", b.prefs().text_rules[0].text); +} + +TEST(CompanionFilterPrefs, EmptyListsSaveAndLoad) { + CompanionFilter a; + MemStream s; + ASSERT_TRUE(a.prefs().saveSerial(s)); + + CompanionFilter b; + cmd(b, "filter sender add Stale"); + b.prefs().reset(); + ASSERT_TRUE(b.prefs().loadSerial(s)) << s.str(); + EXPECT_FALSE(b.prefs().enabled); + EXPECT_FALSE(FilterRules::anyRule(b.prefs().sender_rules, FILTER_RULE_COUNT)); +} + +TEST(CompanionFilterPrefs, SanitizeClampsHandEditedValues) { + CompanionFilter f; + f.prefs().age_mins = 60000; + f.prefs().path_block[0].len = 9; + f.prefs().sender_rules[0].name[0] = 'x'; + f.prefs().sender_rules[0].prob = 0; + + f.prefs().sanitize(); + + EXPECT_EQ(FILTER_AGE_MAX_MINS, f.prefs().age_mins); + EXPECT_EQ(0, f.prefs().path_block[0].len); + EXPECT_EQ(100, f.prefs().sender_rules[0].prob); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 22340b1e54a9eeab2938137d57229ca2f50631eb Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:06:45 +0200 Subject: [PATCH 3/5] docs(companion): document the companion receive filter --- AGENTS.md | 3 + dmc-docs/custom-cli.de.md | 19 +++ dmc-docs/custom-cli.en.md | 19 +++ dmc-docs/custom-cli.md | 19 +++ docs/cli_commands.md | 23 ++++ docs/companion_filter_reference.md | 202 +++++++++++++++++++++++++++++ 6 files changed, 285 insertions(+) create mode 100644 docs/companion_filter_reference.md diff --git a/AGENTS.md b/AGENTS.md index 92e109c7839..b050b0b3d13 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -38,6 +38,8 @@ base (currently synced to upstream `v1.18.0`, embedded firmware version `v1.18.0 - **duty-cycle region gating** (`dc.gate.*`) — `test/test_region_gating/`, wired through `RegionMap.*`, `Dispatcher.*`, `MyMesh.*` and the radio-prefs CLI; - a **frequency-derived duty-cycle default** (`set dutycycle auto`) — `src/helpers/DutyCycleLimits.{cpp,h}`. +- a **companion receive filter** (`filter ...` on companions) — `examples/companion_radio/CompanionFilter*`, + reusing the repeater filter's header-only rule engine; see `docs/companion_filter_reference.md`. Most of it lives in DMC-only files (see §2). The rest is two DMC build workflows, `build.sh`, `scripts/publish-release.sh`, and a handful of small variant and radio tweaks. @@ -62,6 +64,7 @@ not a side effect. Grouped by what pulls them in: ``` build / release build.sh, platformio.ini packet filter examples/simple_repeater/MyMesh.{cpp,h}, docs/cli_commands.md +companion filter examples/companion_radio/MyMesh.{cpp,h}, docs/cli_commands.md region gating src/Dispatcher.{cpp,h}, src/helpers/RegionMap.{cpp,h}, src/helpers/CommonCLI.{cpp,h}, test/mocks/{Arduino,SHA256,Stream}.h duty-cycle auto src/helpers/CommonRadioPrefs.{cpp,h}, src/helpers/CommonCLI.{cpp,h}, diff --git a/dmc-docs/custom-cli.de.md b/dmc-docs/custom-cli.de.md index 5b53214eeb4..6282e452fdf 100644 --- a/dmc-docs/custom-cli.de.md +++ b/dmc-docs/custom-cli.de.md @@ -48,6 +48,25 @@ ACL-Kontaktpakete werden übersprungen. Der Zustand wird in `/filter_prefs` gesp | `04` | ADVERT | `10` | MULTIPART | | `05` | GRP_TXT | `11` | CONTROL | +## Companion-Filter + +`filter ...` auf der **Companion**-Firmware: blendet Müll und unerwünschte Kanalnachrichten in der +App aus. Wird über die Befehlskonsole der App, USB-Seriell oder Remote-CLI gesendet. Standardmäßig +**aus**; Direktnachrichten werden nie gefiltert. Gespeichert in `/cfilter.json`. Siehe +[`docs/companion_filter_reference.md`](https://github.com/Dutch-MeshCore/MeshCore/blob/dmc-dev/docs/companion_filter_reference.md). + +- `filter`: Status und Summen pro Grund +- `filter help` / `filter on` / `filter off` / `filter reset` +- `filter malformed on|off` (falscher Zeitstempel, leerer Text oder ungültiges UTF-8, auf jedem eigenen Kanal) +- `filter age |off` (1-10080; Gruppennachrichten, die älter sind) +- `filter path list|add|remove ` (bis zu 8; Adverts und Gruppenverkehr über diesen Repeater) +- `filter sender add [secs] [prob]` / `remove` / `list` (bis zu 8; `Bot*` = Präfix) +- `filter text add [secs] [prob]` / `remove` / `list` (bis zu 8; `^` = Anfang der Nachricht) +- `filter stats malformed|age|path|sender|text` + +Mit aktiviertem Repeat werden malformed/age/path-Nachrichten auch nicht weitergeleitet; +sender/text-Regeln blenden eine Nachricht nur für dich aus, sie wird für andere weiterhin weitergeleitet. + ## Duty-Cycle-Region-Gating `dc.gate.*`. Standardmäßig **aus**. Wirft Inter-Regions-Flood-Verkehr ab, wenn der diff --git a/dmc-docs/custom-cli.en.md b/dmc-docs/custom-cli.en.md index ef4b4ce0985..f30730dbf74 100644 --- a/dmc-docs/custom-cli.en.md +++ b/dmc-docs/custom-cli.en.md @@ -47,6 +47,25 @@ packets bypass it. State is persisted to `/filter_prefs`. | `04` | ADVERT | `10` | MULTIPART | | `05` | GRP_TXT | `11` | CONTROL | +## Companion filter + +`filter ...` on the **companion** firmware: hides junk and unwanted channel messages from the +app. Sent from the app's command console, USB serial or remote CLI. **Off** by default; direct +messages are never filtered. Saved to `/cfilter.json`. See +[`docs/companion_filter_reference.md`](https://github.com/Dutch-MeshCore/MeshCore/blob/dmc-dev/docs/companion_filter_reference.md). + +- `filter`: status and per-reason totals +- `filter help` / `filter on` / `filter off` / `filter reset` +- `filter malformed on|off` (bad timestamp, empty text or invalid UTF-8, on every channel you hold) +- `filter age |off` (1-10080; group text older than this) +- `filter path list|add|remove ` (up to 8; adverts and group traffic relayed by that repeater) +- `filter sender add [secs] [prob]` / `remove` / `list` (up to 8; `Bot*` = prefix) +- `filter text add [secs] [prob]` / `remove` / `list` (up to 8; `^` = start of message) +- `filter stats malformed|age|path|sender|text` + +With repeat enabled, malformed/age/path messages are not repeated either; sender/text rules only +hide a message from you and it is still repeated for others. + ## Duty-cycle region gating `dc.gate.*`. **Off** by default. Sheds inter-region flood traffic when the repeater has diff --git a/dmc-docs/custom-cli.md b/dmc-docs/custom-cli.md index 2544f66af34..04e5777b20c 100644 --- a/dmc-docs/custom-cli.md +++ b/dmc-docs/custom-cli.md @@ -48,6 +48,25 @@ ACL-contactpakketten worden overgeslagen. De status wordt bewaard in `/filter_pr | `04` | ADVERT | `10` | MULTIPART | | `05` | GRP_TXT | `11` | CONTROL | +## Companion filter + +`filter ...` op de **companion**-firmware: verbergt rommel en ongewenste kanaalberichten voor de +app. Te versturen via de commandoconsole van de app, USB-serieel of remote CLI. Standaard **uit**; +directe berichten worden nooit gefilterd. Opgeslagen in `/cfilter.json`. Zie +[`docs/companion_filter_reference.md`](https://github.com/Dutch-MeshCore/MeshCore/blob/dmc-dev/docs/companion_filter_reference.md). + +- `filter`: status en totalen per reden +- `filter help` / `filter on` / `filter off` / `filter reset` +- `filter malformed on|off` (foute tijdstempel, lege tekst of ongeldige UTF-8, op elk kanaal dat je hebt) +- `filter age |off` (1-10080; groepsberichten ouder dan dit) +- `filter path list|add|remove ` (max. 8; adverts en groepsverkeer via die repeater) +- `filter sender add [secs] [prob]` / `remove` / `list` (max. 8; `Bot*` = prefix) +- `filter text add [secs] [prob]` / `remove` / `list` (max. 8; `^` = begin van het bericht) +- `filter stats malformed|age|path|sender|text` + +Met repeat aan worden malformed/age/path-berichten ook niet doorgestuurd; sender/text-regels +verbergen een bericht alleen voor jou en het wordt nog steeds doorgestuurd voor anderen. + ## Duty-cycle region gating `dc.gate.*`. Standaard **uit**. Werpt inter-regio-floodverkeer af wanneer de repeater diff --git a/docs/cli_commands.md b/docs/cli_commands.md index 4a109c48077..be5ccb07d90 100644 --- a/docs/cli_commands.md +++ b/docs/cli_commands.md @@ -10,6 +10,7 @@ This document provides an overview of CLI commands that can be sent to MeshCore - [Logging](#logging) - [Information](#info) - [Packet Filter](#packet-filter-repeater-only) +- [Companion Receive Filter](#companion-receive-filter-companion-only) - [Configuration](#configuration) - [Radio](#radio) - [System](#system) @@ -323,6 +324,28 @@ filter stats hops --- +## Companion Receive Filter (Companion Only) + +The DMC companion firmware has its own `filter` command that keeps junk and unwanted channel messages out of the app. It is sent through the app's command console (`CMD_RUN_CLI_COMMAND`), USB serial, or remote CLI. Settings are persisted to `/cfilter.json`. Disabled by default; direct and addressed traffic is never filtered. + +Find more information in [companion_filter_reference.md](companion_filter_reference.md) + +**Usage:** +- `filter` — status: `> Filter on: Hidden [ Malformed: N | Age: N | Path: N | Sender: N | Text: N ]` +- `filter help` +- `filter on` / `filter off` +- `filter reset` — restore the defaults (counters are not affected) +- `filter malformed` / `filter malformed ` — drop group text with a bad timestamp, empty text or invalid UTF-8, on every channel the companion holds +- `filter age` / `filter age |off` — drop group text older than this (1–10080) +- `filter path list|add|remove ` — drop flood adverts and group traffic relayed by a repeater whose ID starts with the prefix (2–8 hex digits, up to 8) +- `filter sender list|add [secs] [prob]|remove ` — hide group text by sender (up to 8; `Bot*` = prefix) +- `filter text list|add [secs] [prob]|remove ` — hide group text by content (up to 8; `^` = start of message) +- `filter stats malformed|age|path|sender|text` — what each setting filtered, in the same formats as the repeater filter + +**Note:** Malformed, age and path filtering *drop* a packet: with repeat enabled it is not repeated either. Sender and text rules only *hide* it: it is still repeated for others. + +--- + ## Configuration ### Radio diff --git a/docs/companion_filter_reference.md b/docs/companion_filter_reference.md new file mode 100644 index 00000000000..a383ef5d51e --- /dev/null +++ b/docs/companion_filter_reference.md @@ -0,0 +1,202 @@ +# MeshCore Companion Receive Filter + +## Overview + +A DMC-only filter in the companion firmware that keeps junk and unwanted channel +messages from reaching the app. It is the receive-side counterpart of the +[repeater packet filter](packet_filter_reference.md) and uses the same rule +engine, list syntax and reply formats where the two overlap. + +It filters on: + +* Malformed group text (zero or implausible timestamp, empty text, invalid UTF-8) +* Message age (group text older than N minutes) +* Blocked path prefixes (anything relayed by one specific repeater) +* Sender name or message text of a group text (block, throttle, or dose) + +The filter is **disabled by default**. + +Only flood traffic is filtered. Direct messages, and anything else addressed to +this node, are never touched. Content checks run on the channels the companion +holds the key for, which are exactly the channels the user sees. + +--- + +# Hidden or dropped + +A companion normally does not repeat, so for most users every filtered message +is simply not shown. The difference only matters with repeat enabled: + +| Reason | Verdict | Shown in the app | Repeated (repeat on) | +| --- | --- | --- | --- | +| malformed, age, path | **drop** (junk) | no | no | +| sender, text | **hide** (personal) | no | yes | + +Junk helps nobody, so it is not passed on. A sender or text rule is one user's +taste: the message is hidden here, but still repeated for everyone else, so a +mute list on a repeating companion never censors the mesh. + +--- + +# Command Overview + +The commands run through the companion CLI: from the app's command console +(companion protocol `CMD_RUN_CLI_COMMAND`, v14+), over USB serial, or as remote +CLI from a contact that is allowed to use it. + +```text +filter +filter help +filter on +filter off +filter reset + +filter malformed +filter malformed on|off + +filter age +filter age |off + +filter path list +filter path add +filter path remove + +filter sender list +filter sender add [secs] [prob] +filter sender remove + +filter text list +filter text add [secs] [prob] +filter text remove + +filter stats +filter stats malformed|age|path|sender|text +``` + +Every change is saved immediately. + +--- + +# Enabling the Filter + +```text +filter on +> Filter: on +``` + +`filter off` keeps every setting but filters nothing. `filter reset` restores the +defaults (everything off, all lists empty); the counters are not affected. + +--- + +# Malformed Group Text + +```text +filter malformed on +> Filter: malformed scan on +``` + +Drops group text whose timestamp is zero, or more than a week off this node's +clock (only once the clock has been set), and plain text that is empty or not +valid UTF-8. These are the same checks as the repeater filter, but on every +channel the companion holds rather than on Public only. + +--- + +# Message Age Limit + +```text +filter age 60 +> Filter: OK +``` + +Drops group text stamped more than this many minutes ago (1–10080, or `off`). +Stale messages are what a misbehaving node replays hours later. Inactive while +this node's clock is not set; companions usually get the time from the app. + +--- + +# Blocked Path Prefixes + +```text +filter path add A1B2 +> Filter: path A1B2 added +``` + +Drops flood adverts, group text and group data whose path contains a repeater ID +starting with the prefix (2–8 hex digits, up to 8 prefixes). Use it when one +repeater keeps injecting junk. A prefix longer than the packet's path hash size +never matches. Note that a blocked advert also no longer refreshes an existing +contact. + +--- + +# Sender and Text Rules + +```text +filter sender add Bot* +> Filter: sender Bot* added +filter text add ^!ping 300 +> Filter: text ^!ping added +``` + +Up to 8 sender rules and 8 text rules, checked top to bottom; the first rule that +decides hides the message. + +* **Sender**: exact name, or a prefix when it ends in `*`. Case-sensitive. +* **Text**: substring, or the start of the message when it begins with `^`. +* **secs** (optional): `0` (default) hides every match. Otherwise one match per + `secs` seconds is shown and the rest are hidden, which slows a chatty bot down + instead of muting it. +* **prob** (optional, 1–100, default 100): the share of matches the rule decides; + the rest go on to the next rule. + +`filter sender list` / `filter text list` show the rules: + +```text +filter text list +^!ping throttle 300s +``` + +--- + +# Statistics + +```text +filter +> Filter on: Hidden [ Malformed: 3 | Age: 0 | Path: 12 | Sender: 41 | Text: 7 ] +``` + +Per topic: + +| Command | Reply | +| --- | --- | +| `filter stats malformed` | `> Blocked 3 [ short:0 time:2 empty:0 utf8:1 ]` | +| `filter stats age` | `> Message age: max 60m, dropped 0` (or `> Filter: message age limit off`) | +| `filter stats path` | `A1B2: 12` per prefix, or `None` | +| `filter stats sender` / `text` | `Bot*: 41` per rule, with `(pass N)` for throttle passes | + +Counters live in RAM and restart at zero after a reboot. A message is counted +once, even when several repeaters relay it. + +--- + +# Persistent Configuration + +Settings are stored as JSON in `/cfilter.json` on the companion's internal +filesystem, apart from the main `/prefs.json`. A missing file means defaults +(off). Out-of-range values in a hand-edited file are clamped on load. + +--- + +# Error Replies + +| Reply | Cause | +| --- | --- | +| `> Filter: command error` | unknown subcommand | +| `> Filter: syntax error 'filter malformed '` | bad `malformed` argument | +| `> Filter: error range is 1-10080, or off` | bad `age` argument | +| `> Filter: error path prefix is 2-8 hex digits` | bad path prefix | +| `> Filter: error max N chars` | sender pattern over 15, text pattern over 23 characters | +| `> Filter: error range is 0-65535, range is 1-100` | bad rule arguments | +| `Failed` | duplicate entry, list full, or nothing to remove | From b7df41aac9f40d228c7193f096ad4a4ec2ee253b Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:27:33 +0200 Subject: [PATCH 4/5] feat(companion): keep settings when upgrading from meshcomod meshcomod keeps identity, contacts, channels and blobs in the same files and formats, and its binary /new_prefs is already migrated by upstream's loadPrefs(). Three things were still lost or garbled on the first boot: - the hardware-CAD toggle, a trailing byte (141-byte file); - on files from meshcomod v1.16 and older (136 bytes, no gps_interval), auto-add, RX boost and the default flood scope, which upstream's layout reads 4 bytes off; - the Wi-Fi credentials, which meshcomod keeps in NVS ("meshcomod"). importMeshcomodSettings() runs once, only while /prefs.json does not yet exist, lets upstream migrate, then applies those fields. NVS is read-only and Wi-Fi is only imported into an empty SSID. --- examples/companion_radio/MeshcomodImport.cpp | 63 +++++++++++ examples/companion_radio/MeshcomodPrefs.h | 76 +++++++++++++ examples/companion_radio/MyMesh.cpp | 1 + examples/companion_radio/MyMesh.h | 1 + .../test_meshcomod_import.cpp | 106 ++++++++++++++++++ 5 files changed, 247 insertions(+) create mode 100644 examples/companion_radio/MeshcomodImport.cpp create mode 100644 examples/companion_radio/MeshcomodPrefs.h create mode 100644 test/test_meshcomod_import/test_meshcomod_import.cpp diff --git a/examples/companion_radio/MeshcomodImport.cpp b/examples/companion_radio/MeshcomodImport.cpp new file mode 100644 index 00000000000..d7fbdafbf97 --- /dev/null +++ b/examples/companion_radio/MeshcomodImport.cpp @@ -0,0 +1,63 @@ +// Keeps a companion's settings when it is upgraded from meshcomod. See +// MeshcomodPrefs.h for what differs and why. Identity, contacts, channels and +// blobs need nothing: meshcomod keeps them in the same files and formats. + +#include "MyMesh.h" +#include "MeshcomodPrefs.h" + +#if defined(ESP32) && defined(ENABLE_WIFI_INTERFACE) + #include +#endif + +void importMeshcomodSettings(NodePrefs& prefs, DataStore& store) { + FILESYSTEM* fs = store.getPrimaryFS(); + // Only on the first boot after a binary-prefs firmware: once /prefs.json + // exists this firmware owns the settings. + if (fs == nullptr || fs->exists("/prefs.json") || !fs->exists("/new_prefs")) return; + + uint8_t raw[MeshcomodPrefs::CURRENT_LEN + 8]; + size_t len = 0; + File file = store.openRead(fs, "/new_prefs"); + if (file) { + len = file.read(raw, sizeof(raw)); + file.close(); + } + + store.loadPrefs(prefs); // upstream's migration: /new_prefs in upstream's layout -> /prefs.json + + bool changed = false; + MeshcomodPrefs::Fix fix; + if (MeshcomodPrefs::parse(raw, len, &fix)) { + if (fix.has_cad) prefs.cad_enabled = fix.cad; + if (fix.shifted) { + prefs.gps_interval = 0; // never stored by those versions + prefs.autoadd_config = fix.autoadd_config; + prefs.autoadd_max_hops = fix.autoadd_max_hops; + prefs.rx_boosted_gain = fix.rx_boosted_gain; + memcpy(prefs.default_scope_name, fix.scope_name, sizeof(prefs.default_scope_name)); + prefs.default_scope_name[sizeof(prefs.default_scope_name) - 1] = '\0'; + memcpy(prefs.default_scope_key, fix.scope_key, sizeof(prefs.default_scope_key)); + } + changed = true; + } + +#if defined(ESP32) && defined(ENABLE_WIFI_INTERFACE) + // meshcomod's runtime Wi-Fi credentials (WifiRuntimeStore: NVS "meshcomod"). + // NVS survives an app-only flash; read-only, so meshcomod's copy stays intact. + Preferences nvs; + if (nvs.begin("meshcomod", true)) { + char ssid[sizeof(prefs.wifi_ssid)] = { 0 }; + char pwd[sizeof(prefs.wifi_pwd)] = { 0 }; + nvs.getString("wifi_ssid", ssid, sizeof(ssid)); + nvs.getString("wifi_pwd", pwd, sizeof(pwd)); + nvs.end(); + if (MeshcomodPrefs::shouldImportWifi(prefs.wifi_ssid, ssid)) { + memcpy(prefs.wifi_ssid, ssid, sizeof(prefs.wifi_ssid)); + memcpy(prefs.wifi_pwd, pwd, sizeof(prefs.wifi_pwd)); + changed = true; + } + } +#endif + + if (changed) store.savePrefs(prefs); +} diff --git a/examples/companion_radio/MeshcomodPrefs.h b/examples/companion_radio/MeshcomodPrefs.h new file mode 100644 index 00000000000..30c3fd92d22 --- /dev/null +++ b/examples/companion_radio/MeshcomodPrefs.h @@ -0,0 +1,76 @@ +#pragma once + +#include +#include +#include + +// Upgrading a companion from meshcomod (Elektr0Vodka/meshcomod) keeps its +// settings. meshcomod still stores prefs as the binary /new_prefs, which +// upstream's DataStore::loadPrefs() already migrates to /prefs.json, but in +// upstream's layout only. meshcomod files differ in two ways: +// +// 141 bytes current: upstream's 140 plus a trailing hardware-CAD byte. +// 136 bytes v1.16 and older (before June 2026): gps_interval was never +// written, so everything after gps_enabled sits 4 bytes early. +// Read in upstream's layout, auto-add, RX boost and the default +// flood scope come out shifted. +// +// This header picks those fields out of the raw file. Header-only and free of +// filesystem and NVS so it runs on `native`; MeshcomodImport.cpp applies it. + +namespace MeshcomodPrefs { + + static const size_t UPSTREAM_LEN = 140; + static const size_t CURRENT_LEN = 141; // + CAD byte at 140 + static const size_t SHIFTED_LEN = 136; // no gps_interval + + struct Fix { + bool has_cad = false; + uint8_t cad = 0; + + bool shifted = false; // the fields below are valid only when set + uint8_t autoadd_config = 0; + uint8_t autoadd_max_hops = 0; + uint8_t rx_boosted_gain = 0; + char scope_name[31] = {}; + uint8_t scope_key[16] = {}; + }; + + // Returns true when `file` is a meshcomod /new_prefs that needs a fix-up + // after upstream's migration; `out` then says which. + inline bool parse(const uint8_t* file, size_t len, Fix* out) { + *out = Fix(); + if (file == nullptr) return false; + + if (len == CURRENT_LEN) { + out->has_cad = true; + out->cad = file[140] ? 1 : 0; + return true; + } + + if (len == SHIFTED_LEN) { + out->shifted = true; + out->autoadd_config = file[86]; + out->autoadd_max_hops = file[87]; + out->rx_boosted_gain = file[88] ? 1 : 0; + memcpy(out->scope_name, &file[89], sizeof(out->scope_name)); + out->scope_name[sizeof(out->scope_name) - 1] = '\0'; + memcpy(out->scope_key, &file[120], sizeof(out->scope_key)); + return true; + } + return false; + } + + // meshcomod keeps Wi-Fi credentials in NVS (namespace "meshcomod"), not in + // the prefs file. They are copied over only when this firmware has none, so + // a later `set wifi.clear` is never undone. + inline bool shouldImportWifi(const char* current_ssid, const char* nvs_ssid) { + return current_ssid != nullptr && current_ssid[0] == '\0' && nvs_ssid != nullptr && nvs_ssid[0] != '\0'; + } +} + +// Firmware side (MeshcomodImport.cpp). Call before the regular prefs load in +// MyMesh::begin(); it only acts on the first boot after a binary-prefs firmware. +class NodePrefs; +class DataStore; +void importMeshcomodSettings(NodePrefs& prefs, DataStore& store); diff --git a/examples/companion_radio/MyMesh.cpp b/examples/companion_radio/MyMesh.cpp index fc4df40ceaf..2bd3d51d57f 100644 --- a/examples/companion_radio/MyMesh.cpp +++ b/examples/companion_radio/MyMesh.cpp @@ -979,6 +979,7 @@ void MyMesh::begin() { #endif // load persisted prefs + importMeshcomodSettings(_prefs, *_store); // DMC: keep settings when upgrading from meshcomod _store->loadPrefs(_prefs); loadCompanionFilter(_cfilter, *_store); // DMC companion filter sensors.node_lat = _prefs.node_lat; diff --git a/examples/companion_radio/MyMesh.h b/examples/companion_radio/MyMesh.h index fd806ad5885..a425f647d30 100644 --- a/examples/companion_radio/MyMesh.h +++ b/examples/companion_radio/MyMesh.h @@ -23,6 +23,7 @@ #endif #include "CompanionFilter.h" +#include "MeshcomodPrefs.h" #include "DataStore.h" #include "NodePrefs.h" diff --git a/test/test_meshcomod_import/test_meshcomod_import.cpp b/test/test_meshcomod_import/test_meshcomod_import.cpp new file mode 100644 index 00000000000..5e68a481d84 --- /dev/null +++ b/test/test_meshcomod_import/test_meshcomod_import.cpp @@ -0,0 +1,106 @@ +#include + +#include + +// Reading the binary /new_prefs a meshcomod companion leaves behind, for the +// fields upstream's /new_prefs migration does not cover (or misreads). +#include "../../examples/companion_radio/MeshcomodPrefs.h" + +// Upstream /new_prefs layout up to and including multi_acks is 78 bytes; the +// tests only care about what comes after it, so the head is left zero. +static size_t upstreamFile(uint8_t* buf, bool with_cad, uint8_t cad = 1) { + memset(buf, 0, 160); + buf[78] = 1; // path_hash_mode + buf[85] = 1; // gps_enabled + uint32_t gps_interval = 300; + memcpy(&buf[86], &gps_interval, 4); + buf[90] = 0x0E; // autoadd_config + buf[91] = 3; // autoadd_max_hops + buf[92] = 1; // rx_boosted_gain + strcpy((char*)&buf[93], "nl"); // default_scope_name[31] + for (int i = 0; i < 16; i++) buf[124 + i] = (uint8_t)(0xA0 + i); // default_scope_key[16] + if (!with_cad) return 140; + buf[140] = cad; + return 141; +} + +// meshcomod before June 2026 (v1.16 and older) never wrote gps_interval, so +// everything after gps_enabled sits 4 bytes early and the file is 136 bytes. +static size_t shiftedFile(uint8_t* buf) { + memset(buf, 0, 160); + buf[85] = 1; + buf[86] = 0x0E; + buf[87] = 3; + buf[88] = 1; + strcpy((char*)&buf[89], "nl"); + for (int i = 0; i < 16; i++) buf[120 + i] = (uint8_t)(0xA0 + i); + return 136; +} + +TEST(MeshcomodPrefs, CurrentFileCarriesTheCadByte) { + uint8_t buf[160]; + MeshcomodPrefs::Fix fix; + + ASSERT_TRUE(MeshcomodPrefs::parse(buf, upstreamFile(buf, true, 0), &fix)); + EXPECT_TRUE(fix.has_cad); + EXPECT_EQ(0, fix.cad); + EXPECT_FALSE(fix.shifted) << "the rest is upstream's layout, which the stock migration reads"; + + ASSERT_TRUE(MeshcomodPrefs::parse(buf, upstreamFile(buf, true, 7), &fix)); + EXPECT_EQ(1, fix.cad) << "any non-zero byte is on"; +} + +TEST(MeshcomodPrefs, UpstreamSizedFileNeedsNoFix) { + uint8_t buf[160]; + MeshcomodPrefs::Fix fix; + + EXPECT_FALSE(MeshcomodPrefs::parse(buf, upstreamFile(buf, false), &fix)); + EXPECT_FALSE(fix.has_cad); + EXPECT_FALSE(fix.shifted); +} + +TEST(MeshcomodPrefs, ShiftedFileIsReadAtItsOwnOffsets) { + uint8_t buf[160]; + MeshcomodPrefs::Fix fix; + + ASSERT_TRUE(MeshcomodPrefs::parse(buf, shiftedFile(buf), &fix)); + EXPECT_TRUE(fix.shifted); + EXPECT_FALSE(fix.has_cad); + EXPECT_EQ(0x0E, fix.autoadd_config); + EXPECT_EQ(3, fix.autoadd_max_hops); + EXPECT_EQ(1, fix.rx_boosted_gain); + EXPECT_STREQ("nl", fix.scope_name); + EXPECT_EQ(0xA0, fix.scope_key[0]); + EXPECT_EQ(0xAF, fix.scope_key[15]); +} + +TEST(MeshcomodPrefs, ScopeNameIsAlwaysTerminated) { + uint8_t buf[160]; + MeshcomodPrefs::Fix fix; + size_t len = shiftedFile(buf); + memset(&buf[89], 'x', 31); // no NUL inside the 31-byte field + + ASSERT_TRUE(MeshcomodPrefs::parse(buf, len, &fix)); + EXPECT_EQ(30u, strlen(fix.scope_name)); +} + +TEST(MeshcomodPrefs, RejectsOtherSizesAndNull) { + uint8_t buf[160] = {}; + MeshcomodPrefs::Fix fix; + + EXPECT_FALSE(MeshcomodPrefs::parse(buf, 91, &fix)); + EXPECT_FALSE(MeshcomodPrefs::parse(buf, 0, &fix)); + EXPECT_FALSE(MeshcomodPrefs::parse(nullptr, 141, &fix)); +} + +TEST(MeshcomodPrefs, WifiIsImportedOnlyIntoAnEmptySsid) { + EXPECT_TRUE(MeshcomodPrefs::shouldImportWifi("", "HomeNet")); + EXPECT_FALSE(MeshcomodPrefs::shouldImportWifi("Already", "HomeNet")); + EXPECT_FALSE(MeshcomodPrefs::shouldImportWifi("", "")); + EXPECT_FALSE(MeshcomodPrefs::shouldImportWifi("", nullptr)); +} + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} From 60cc7b27c5e57e4e5828647db8fff18d24bb6754 Mon Sep 17 00:00:00 2001 From: Elektr0Vodka <211697683+Elektr0Vodka@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:27:34 +0200 Subject: [PATCH 5/5] docs(migration): upgrading a companion from meshcomod --- dmc-docs/migration.de.md | 19 +++++++++++++++++++ dmc-docs/migration.en.md | 18 ++++++++++++++++++ dmc-docs/migration.md | 18 ++++++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/dmc-docs/migration.de.md b/dmc-docs/migration.de.md index 3dc012c8760..403ab9019a8 100644 --- a/dmc-docs/migration.de.md +++ b/dmc-docs/migration.de.md @@ -40,6 +40,25 @@ get mqtt.status Aktiviere bei Bedarf das [Webpanel](web-panel.md), um den Status zu bestätigen. +## Von meshcomod (Companion) + +Ein Companion, der von meshcomod auf die DMC-Companion-Firmware wechselt, behält Identität, +Kontakte, Kanäle und Einstellungen, einschließlich CAD-Schalter und WLAN-Netz. Beim ersten Start +liest die DMC-Firmware die meshcomod-Einstellungen einmal ein und speichert sie im eigenen Format. + +- Flashe das **Update-Image** (`*.bin`, an `0x10000`), nicht `*-merged.bin`: Das Merged-Image + löscht den NVS-Bereich, in dem meshcomod die WLAN-Zugangsdaten ablegt. Alles andere bleibt so + oder so erhalten. +- Hat sich meshcomod je per OTA aktualisiert, läuft es eventuell aus dem zweiten App-Slot. Lösche + dann auch die Boot-Auswahl, sonst startet das Gerät weiter meshcomod: + +```text +esptool.py --chip esp32s3 erase_region 0xe000 0x2000 +esptool.py --chip esp32s3 write_flash 0x10000 .bin +``` + +- Daten, die meshcomod auf einer SD-Karte ablegt (Launcher-Installationen), werden nicht übernommen. + ## Letzter Ausweg Läuft es nicht stabil, führe eine vollständige Löschung durch und flashe die diff --git a/dmc-docs/migration.en.md b/dmc-docs/migration.en.md index 8925c5eb6b6..85b3dd3e3ff 100644 --- a/dmc-docs/migration.en.md +++ b/dmc-docs/migration.en.md @@ -40,6 +40,24 @@ get mqtt.status Optionally enable the [Web Panel](web-panel.md) to confirm the status. +## From meshcomod (companion) + +A companion moving from meshcomod to the DMC companion firmware keeps its identity, contacts, +channels and settings, including the CAD toggle and the WiFi network. On the first boot the DMC +firmware reads meshcomod's settings once and stores them in its own format. + +- Flash the **update image** (`*.bin`, at `0x10000`), not `*-merged.bin`: the merged image wipes + the NVS area where meshcomod keeps the WiFi credentials. Everything else survives either way. +- If meshcomod ever updated itself over the air, it may be running from the second app slot. Clear + the boot selector as well, or the device keeps booting meshcomod: + +```text +esptool.py --chip esp32s3 erase_region 0xe000 0x2000 +esptool.py --chip esp32s3 write_flash 0x10000 .bin +``` + +- Data that meshcomod keeps on an SD card (Launcher installs) is not picked up. + ## Last resort If it will not run stably, do a full erase and flash the `*-merged.bin` clean from `0x0`, diff --git a/dmc-docs/migration.md b/dmc-docs/migration.md index 36fe8be7fbb..c9607d60a6a 100644 --- a/dmc-docs/migration.md +++ b/dmc-docs/migration.md @@ -40,6 +40,24 @@ get mqtt.status Schakel eventueel het [Webpaneel](web-panel.md) in om de status te bevestigen. +## Vanaf meshcomod (companion) + +Een companion die van meshcomod naar de DMC-companionfirmware gaat, houdt zijn identiteit, +contacten, kanalen en instellingen, inclusief de CAD-schakelaar en het WiFi-netwerk. Bij de eerste +start leest de DMC-firmware de meshcomod-instellingen één keer en slaat ze op in het eigen formaat. + +- Flash het **update-image** (`*.bin`, op `0x10000`), niet `*-merged.bin`: het merged image wist + het NVS-gebied waar meshcomod de WiFi-gegevens bewaart. De rest blijft hoe dan ook staan. +- Heeft meshcomod zichzelf ooit over-the-air bijgewerkt, dan draait het mogelijk vanuit het tweede + app-slot. Wis dan ook de bootkeuze, anders blijft het apparaat meshcomod starten: + +```text +esptool.py --chip esp32s3 erase_region 0xe000 0x2000 +esptool.py --chip esp32s3 write_flash 0x10000 .bin +``` + +- Gegevens die meshcomod op een SD-kaart bewaart (Launcher-installaties) worden niet overgenomen. + ## Laatste redmiddel Werkt het niet stabiel, doe dan een volledige wis en flash de `*-merged.bin` schoon vanaf