From ed44e76de4658516a288415e60df41da66be9fba Mon Sep 17 00:00:00 2001 From: l5y <220195275+l5yth@users.noreply.github.com> Date: Fri, 28 Aug 2026 11:21:10 +0200 Subject: [PATCH 1/4] data,web: key Reticulum nodes on destinations; fix e2e field findings (#888) --- .env.example | 61 +- ACCEPTANCE.md | 272 ++++++- CHANGELOG.md | 23 +- CLAUDE.md | 4 + README.md | 261 ++++--- SPEC.md | 46 +- configure.sh | 126 ++- data/Dockerfile | 12 + data/destinations.sql | 43 + data/mesh_ingestor/CONTRACTS.md | 57 +- data/mesh_ingestor/announce.py | 18 +- data/mesh_ingestor/config.py | 27 +- data/mesh_ingestor/protocols/reticulum.py | 578 ++++---------- data/nodes.sql | 13 +- docker-compose.yml | 27 +- flake.nix | 173 ++++ tests/test_announce_unit.py | 34 + tests/test_config_unit.py | 34 +- tests/test_reticulum_unit.py | 736 +++++------------- web/Dockerfile | 16 + .../data_processing/node_writes.rb | 120 +-- web/lib/potato_mesh/application/database.rb | 40 +- .../application/queries/node_queries.rb | 29 + web/lib/potato_mesh/application/routes/api.rb | 10 + web/spec/data_processing_spec.rb | 152 ++-- web/spec/reticulum_spec.rb | 106 +-- 26 files changed, 1551 insertions(+), 1467 deletions(-) create mode 100644 data/destinations.sql diff --git a/.env.example b/.env.example index bba876ae..e58b15e9 100644 --- a/.env.example +++ b/.env.example @@ -32,11 +32,9 @@ PROTOCOL="meshtastic" # --- PROTOCOL=reticulum only ------------------------------------------------- # Reticulum (RNS) config directory for the passive announce listener. -# Defaults to an app-owned directory ($XDG_CONFIG_HOME/potato-mesh/reticulum, -# else ~/.config/potato-mesh/reticulum) so the ingestor never adopts the -# operator's own ~/.reticulum transport-node configuration. Set this only to -# deliberately share an existing RNS stack. -# RETICULUM_CONFIG_DIR="/var/lib/potato-mesh/reticulum" +# Defaults to ~/.reticulum, the stack rnsd already uses. Point it elsewhere +# only if the ingestor should read a different RNS config. +# RETICULUM_CONFIG_DIR="/home/you/.reticulum" # Restrict announce ingestion to named RNS interfaces (comma-separated, # case-insensitive substring match, e.g. "rnode"). An RNS stack may carry LoRa @@ -48,9 +46,9 @@ PROTOCOL="meshtastic" # INGESTOR_NODE_ID (see below) is an OVERRIDE here, not a requirement. Reticulum # has no handshake revealing "our" node id, so the ingestor owns an RNS identity -# of its own -- stored as potato_mesh_identity inside RETICULUM_CONFIG_DIR on -# first run -- and derives its !xxxxxxxx from it, the same way it derives every -# peer's. The heartbeat and the packets/hour stats therefore come up unaided. +# of its own: it derives its !xxxxxxxx from the transport identity already in +# RETICULUM_CONFIG_DIR, so the heartbeat and the packets/hour stats come up +# unaided. Set INGESTOR_NODE_ID only to override that. # # CONNECTION does not apply. It names one serial/TCP/BLE endpoint, which an RNS # stack of many interfaces does not have. Its counterparts are the two variables @@ -66,12 +64,27 @@ PROTOCOL="meshtastic" # Your mesh network name SITE_NAME="My Meshtastic Network" -# Default Meshtastic channel -CHANNEL="#LongFast" +# Meshtastic radio preset shown in the join strip (CHANNEL is a deprecated alias) +MESHTASTIC_PRESET="#LongFast" -# Default frequency for your region +# Meshtastic frequency for your region (FREQUENCY is a deprecated alias) # Common frequencies: 868MHz (Europe), 915MHz (US), 433MHz (Worldwide) -FREQUENCY="915MHz" +MESHTASTIC_FREQ="915MHz" + +# Meshcore preset and frequency. The join strip hides the Meshcore line until +# both are set. +# MESHCORE_PRESET="MediumFast" +# MESHCORE_FREQ="867.5MHz" + +# Announcement banner rendered above the header on every page +# ANNOUNCEMENT="Net every Tuesday 20:00" + +# Directory for static, custom-content pages served at /pages/:slug +# PAGES_DIR=./pages + +# Absolute http(s) URL for the social preview image. Replaces the generated +# /og-image.png for deployments without Chromium. +# OG_IMAGE_URL="https://cdn.example.org/preview.png" # Map center coordinates (latitude, longitude) # Berlin, Germany: 52.502889, 13.404194 @@ -96,6 +109,19 @@ FEDERATION=1 # Hide public mesh messages from unauthenticated visitors (1=hidden, 0=public) PRIVATE=0 +# Comma-separated channel names to ingest. Unset = ingest all channels. +# ALLOWED_CHANNELS=LongFast,Ops + +# Comma-separated channel names to drop. Applied after ALLOWED_CHANNELS. +# HIDDEN_CHANNELS=Private + +# Live-update SSE stream at /api/events (1=on, 0=off -> clients poll instead) +EVENTS=1 + +# Comma-separated node ids to expose as per-node Prometheus gauges. +# Unset exports none. See PROMETHEUS.md. +# PROM_REPORT_IDS=!abcd1234,!beef5678 + # ============================================================================= # TRANSMIT SETTINGS (your ingestor putting traffic on the air) @@ -134,6 +160,17 @@ DEBUG=0 # Energy saving mode — sleep between ingestion cycles (0=off, 1=on) ENERGY_SAVING=0 +# Channel index to ingest from +CHANNEL_INDEX=0 + +# Seconds between Meshcore host self-telemetry reads over the companion link +# (no airtime). 0 disables. +# MESHCORE_SELF_TELEMETRY_SECONDS=3600 + +# Seconds between Meshcore contact telemetry polls. Requires TX_ENABLED=1 +# (polling other nodes transmits). 0 disables on-air polling. +# MESHCORE_TELEMETRY_POLL_SECONDS=300 + # Default map zoom override # MAP_ZOOM=15 diff --git a/ACCEPTANCE.md b/ACCEPTANCE.md index 1093c0dd..9e9e6521 100644 --- a/ACCEPTANCE.md +++ b/ACCEPTANCE.md @@ -5285,26 +5285,13 @@ databases. #889 shipped the boot guard for `reticulum_nodes_count` but skipped t migration file that the four preceding schema changes all shipped; that is restored here. -### RN-A9 — The ingestor derives its own node id; `INGESTOR_NODE_ID` only overrides — #888 / RN9 -```bash -( . .venv/bin/activate && pytest -q tests/test_reticulum_unit.py -k 'node_id or identity' ) -git grep -n 'INGESTOR_NODE_ID' -- data/mesh_ingestor/protocols/reticulum.py -``` -**Expected:** green, and `INGESTOR_NODE_ID` appears in `reticulum.py` only as an -override consulted **before** the derived identity — never as a precondition. -`connect` resolves `!xxxxxxxx` from an RNS identity the ingestor owns, generated -once and stored as `potato_mesh_identity` inside `RETICULUM_CONFIG_DIR`, mapped -by the same `_reticulum_node_id` rule applied to peers (RN1), so the id is stable -across restarts. The key file is `0600`. An identity that can neither be read nor -persisted yields **no** id and a warning rather than a fresh id per start -(persist-or-nothing: a churning id would file one orphan ingestor row per -restart), and announces keep being ingested either way. An existing but -unreadable key file is left untouched. The startup warning #890 emitted for an -unset `INGESTOR_NODE_ID` is gone. - -**Amends RN-A5's neighbourhood, not RN-A5 itself:** the config dir RN3/RN-A5 -made app-owned now also holds the ingestor's key, which is what makes owning one -acceptable — it is never written into the operator's `~/.reticulum`. +### RN-A9 — Superseded by RE-A2 +The ingestor no longer generates or stores an identity of its own: **RE1** +re-keyed peers on the destination hash and **RE3** put the config dir back under +the operator's control, which removed both of this criterion's premises. The +behaviour it protected — `INGESTOR_NODE_ID` is an override, never a requirement, +and the heartbeat comes up unaided — is asserted by **RE-A2** against the +transport identity the config dir already holds. ### RN-A10 — `CONNECTION` is inapplicable to Reticulum, and said so — #888 / RN10 ```bash @@ -5684,3 +5671,248 @@ this feature is rebased onto - RD4 adds `lxmf.propagation` to the same `_ANNOUNCE_ASPECTS` that RV-A1's interface scoping reads, and shares the module whose fallback name RV-A2 corrected). +## Bugfix: Reticulum end-to-end field findings (#893 hardware test) + +Seven findings from the first run against a real RNode and a live `rnsd`. Two are +plain defects; two reverse decisions this project made on assumptions the field +test disproved; three land where the contract was silent. + +### RE-A1 - One identity yields one node-id mapping - T-D-2 +```bash +( . .venv/bin/activate && pytest -q tests/test_reticulum_unit.py -k "host_node_id" ) +``` +**Expected:** pass. `canonical_node_id` parses a hex string as an integer and +keeps the low 32 bits - correct for a Meshtastic node num, wrong for a 16-byte +identity hash, which it truncates from the **opposite end** to +`_reticulum_node_id`. In the field the ingestor registered itself as +`!86c39940` while its own peer row was `!27716218`, from the same identity. +The provider now canonicalises a raw identity hash the Reticulum way (first four +bytes), matching MeshCore's `public_key_hex[:8]` convention and the CONTRACTS +rule that a mapping be deterministic and derived from sender-side material. + +### RE-A2 - The host node id is derived, not hunted for - T-B +```bash +( . .venv/bin/activate && pytest -q tests/test_reticulum_unit.py -k "derived_when_unset" ) +``` +**Expected:** pass. `INGESTOR_NODE_ID` unset resolves to +`RNS.Transport.internal_identity()` - the config dir's own transport identity, +which `rnstatus` shows as "Transport Instance". Reticulum has no handshake +revealing "our" node id, so the previous behaviour was to warn and leave the +heartbeat unregistered; the operator's only recourse was grepping a nomadnet +logfile. The ingestor aborts only when no identity can be derived at all. + +### RE-A3 - A local announce is never hidden by an interface allowlist - T-C +```bash +( . .venv/bin/activate && pytest -q tests/test_reticulum_unit.py -k "local_announces or interface" ) +``` +**Expected:** pass. `Transport.inbound` adds a hop to every inbound packet and +takes it back for a local-client or shared-instance interface, so **0 hops means +"announced by an app on this machine"** - it cannot mean anything else. Those +announces are always ingested; the allowlist applies from 1 hop out, where the +interface is the real one. + +This replaces the previous rule, which filtered purely on interface name and +therefore hid the operator's own nodes: every local destination legitimately +reads `LocalInterface[rns/default]`. + +### RE-A4 - Interface scoping works through a shared instance - T-C (amends RN4/RN8) +```bash +git grep -n "get_next_hop_if_name" -- data/mesh_ingestor/protocols/reticulum.py +git grep -c "_seed_config_dir\|_warn_allowlist_ignored_once" -- data/mesh_ingestor/protocols/reticulum.py +``` +**Expected:** the first grep hits; the second returns `0`. `RNS.Reticulum`'s +accessors RPC to the shared instance and return **its** view - +`RNodeInterface[RNode Reticulum Berlin]`, not the local socket - which +`RNS.Transport.next_hop_interface` cannot do. Verified in the field: a client's +`get_interface_stats()` lists the RNode, which a purely local client could not +know exists. + +**RN4 and RN8 were built on the opposite premise** and are amended. The +config-seeding (RN8) existed only to force `share_instance = No` so the allowlist +could work, and the fail-open existed only to stop the resulting blackout; +neither has a reason to exist, and both are removed. RPC auth is keyed on the +config dir's identity, so a differing dir is rejected - which is why +`~/.reticulum` is the default again (RN3 amended). + +### RE-A5 - Each destination is its own node row - T-E (reverses RN1) +```bash +( . .venv/bin/activate && pytest -q tests/test_reticulum_unit.py -k "each_destination or heard_on" ) +( cd web && bundle exec rspec spec/reticulum_spec.rb -e "destinations" ) +``` +**Expected:** pass. RN1 keyed rows on the identity so a peer's aspects merged +into one row. The field test showed why that loses information: the aspects +carry **different names** - `lxmf.delivery` announced "Afri Nomad Orion" and +`nomadnetwork.node` announced "Department of Decentralization" - so the merged +row ended up named from one aspect and roled from another. + +`node_id` is therefore the first four bytes of the **destination hash** again, +one row per announced destination. The identity is kept as `nodes.identity_hash` +so the rows can be grouped by peer later; grouping is a separate design pass and +is deliberately not attempted here. `_ROLE_RANK`, `_rank_role`, `_record_role` +and the cross-ingestor `dest_hash` union all become unnecessary and are removed - +each row now has exactly one aspect, so there is nothing to rank or merge. + +### RE-A6 - Destinations are a table, and they are served - T-E +```bash +( cd web && bundle exec rspec spec/reticulum_spec.rb -e "GET /api/destinations" ) +git grep -n "dest_hash" -- data/nodes.sql +``` +**Expected:** the spec passes and the grep returns nothing. The `destinations` +table (`id`, `node_id`, `name`, `aspect`, `role`, timestamps) **replaces** the +`nodes.dest_hash` JSON column - a column and a table modelling the same thing +would drift. `GET /api/destinations` serves them, and `/api/nodes` carries a +reference list. Named `destinations` rather than "aspects" because an aspect is +one component of a destination's name, not the destination itself; the table +also stays honest for protocols whose nodes have exactly one. + +### RE-A7 - Operator-facing settings say which protocol they apply to - T-A +```bash +grep -n "CONNECTION" README.md | head -3 +``` +**Expected:** the `CONNECTION` row is marked as Meshtastic/MeshCore only. The +Reticulum provider takes its target from `RETICULUM_CONFIG_DIR` and never reads +`CONNECTION`; the field test passed `/dev/ttyACM1` to a Reticulum ingestor that +ignored it, because the table implied it was required. + +### RE-A8 - The transmit gate is logged when it blocks something - T-D-3 +```bash +( . .venv/bin/activate && pytest -q tests/test_announce_unit.py -k "suppress or policy" ) +``` +**Expected:** pass. `maybe_run_announcement_cycle` checked the transmit policy +**before** `announce_due`, so it logged a suppression line every 60 s cycle even +though an announcement is due at most once per 24 h. The order is reversed: the +gate is named only when it actually blocks a due announcement. The startup +`Transmit policy resolved` line already states the resolved policy, which is +what the per-cycle message was for. + +### RE-R1 - Regression: prior acceptance still holds +```bash +( . .venv/bin/activate && pytest -q tests/ ) && ( cd web && bundle exec rspec ) && ( cd web && npm test ) +``` +**Expected:** all green. At risk and explicitly required to remain green: +**RN-A1** (superseded on the row key by RE-A5, but its *evidence* - that +destination hashes differ per aspect - is what motivates the reversal); +**RN-A2** (`publicKey` is still the identity key; the identity hash now also +lands in `nodes.identity_hash`); **RN-A3** (the `dest_hash` union is removed +outright, so its criterion retires with it); **RN-A5/RN-A6** (config dir and +allowlist, both amended here); **RV-A1-RV-A4** (the #893 review fixes, of which +RV-A1's fail-open and seeding are removed as no longer needed); **RD-A1-RD-A8** +(the visual identity feature - presentation only, untouched); and **MA7/TX-A1** +(the transmit policy itself is unchanged; only its logging moves). + + +## Documentation audit (operator docs, flags, APIs, contracts) + +### DOC-A1 - Every env var is documented, and the operator set is piped everywhere +```bash +python3 - <<'PY' +import re, pathlib +ING = pathlib.Path("data/mesh_ingestor/config.py").read_text() +WEB = pathlib.Path("web/lib/potato_mesh/config.rb").read_text() +ing_vars = set(re.findall(r'os\.environ\.get\(\s*"([A-Z_]+)"', ING)) +web_vars = set(re.findall(r'ENV\[\s*"([A-Z_]+)"', WEB)) +web_vars |= set(re.findall(r'fetch_\w+\(\s*"([A-Z_]+)"', WEB)) +RUNTIME = {"HOME","PATH","PORT","HOST","RACK_ENV","APP_ENV","APP_VERSION", + "XDG_DATA_HOME","XDG_CONFIG_HOME","PYTHONPATH","INSTANCES"} +ADVANCED = {"MIN_THREADS","MAX_THREADS","PUMA_FORCE_SHUTDOWN","STATS_CACHE_TTL_SECONDS", + "OG_IMAGE_TTL_SECONDS","LIVE_SAFETY_POLL_SECONDS","SSE_HEARTBEAT_SECONDS", + "SSE_MAX_LIFETIME_SECONDS","SSE_PUBLISH_COOLDOWN","SSE_THREAD_RESERVE", + "INITIAL_FEDERATION_DELAY_SECONDS"} +ADVANCED |= {v for v in ing_vars | web_vars + if v.startswith(("FEDERATION_", "REMOTE_INSTANCE_"))} +allv = (ing_vars | web_vars) - RUNTIME +txt = {n: pathlib.Path(n).read_text() for n in + (".env.example","docker-compose.yml","data/Dockerfile","web/Dockerfile", + "flake.nix","README.md")} +has = lambda n, v: re.search(rf'\b{v}\b', txt[n]) is not None +bad = [] +for v in sorted(allv): + if not has("README.md", v): + bad.append(f"{v}: absent from README.md") + if v in ADVANCED: + continue + need = [".env.example", "docker-compose.yml", "flake.nix"] + if v in ing_vars: need.append("data/Dockerfile") + if v in web_vars: need.append("web/Dockerfile") + miss = [n for n in need if not has(n, v)] + if miss: + bad.append(f"{v}: missing from {', '.join(miss)}") +print("\n".join(bad) if bad else "OK") +print(f"({len(allv)} variables, {len(ADVANCED)} advanced)") +PY +``` +**Expected:** prints `OK` (57 variables, 22 advanced at the time of writing). +Two tiers, deliberately: + +- **`README.md` documents every variable either config file reads.** No + exemptions. `PROM_REPORT_IDS` shipped readable-by-code and documented nowhere + at all; the whole `SSE_*`/`FEDERATION_*`/`REMOTE_INSTANCE_*` block likewise. +- **The operator-facing set is additionally piped through `.env.example`, + Compose, the NixOS module, and the image of each service that reads it.** A + variable read by both services must be declared in both images. +- **`ADVANCED` covers internal tuning knobs** (thread pools, timeouts, SSE and + federation internals). They are README-only on purpose: pre-declaring two + dozen of them would bury the handful an operator actually sets. Adding a name + to `ADVANCED` is a deliberate choice to keep it out of the templates, not a + way to silence this check — it still must be in the README. + +Regression guard for `PROTOCOL`/`RETICULUM_*` never reaching `flake.nix`, which +left Reticulum unconfigurable on NixOS while the README documented it. + +### DOC-A2 - Compose defaults are bare, never quoted +```bash +grep -nE '\$\{[A-Z_]+:-""\}' docker-compose.yml ; echo "exit=$?" +``` +**Expected:** prints nothing (`exit=1`). Compose substitutes a default as +**literal text**, so `${VAR:-""}` yields the two-character value `""`. On +`ALLOWED_CHANNELS` that parses as a one-entry allowlist matching nothing and +drops every message; on `RETICULUM_INTERFACES` it ingests no announces. Both +regressions shipped once. Bare `${VAR:-}` is the only correct empty default. + +### DOC-A3 - Every route is documented +```bash +grep -rhoE '(get|post) "/[^"]*"' web/lib/potato_mesh/application/routes/*.rb \ + | sed 's/.*"\(.*\)"/\1/' | sed -E 's#/:[a-z_]+$##' | sed '/^$/d' | sort -u \ + | while read -r p; do grep -qF "$p" README.md || echo "undocumented: $p"; done +echo done +``` +**Expected:** only `done`. A trailing `/:param` is stripped before the check +because the README documents per-node reads as a `GET /:id` column on the +collection row rather than as their own bullet. The `### API` section covers +every collection, every non-collection endpoint (`/version`, `/metrics`, +`/.well-known/potato-mesh`, `/og-image.png`, `robots.txt`, `sitemap.xml`), the +page routes, and states there is **no** health endpoint — the repository has +none, so "health" is documented by its absence rather than left to a reader to +discover. + +### DOC-A4 - Operator docs carry no design rationale +```bash +grep -nE "\b(that default is deliberate|airtime on a lora mesh is|not a default we get to make)\b" -i README.md ; echo "exit=$?" +``` +**Expected:** prints nothing (`exit=1`). Operator documentation answers "do X to +achieve Y"; the reasoning behind a decision belongs in `SPEC.md`, which is where +the transmit-default rationale now lives alone. + +### DOC-A5 - CONTRACTS.md matches the served shapes +```bash +grep -c "Served on no read API" data/mesh_ingestor/CONTRACTS.md +grep -n "GET /api/destinations response shape" data/mesh_ingestor/CONTRACTS.md +grep -n "dest_hash" data/mesh_ingestor/CONTRACTS.md ; echo "exit=$?" +``` +**Expected:** the first prints `0`, the second hits, the third prints nothing +(`exit=1`). `identityHash` **is** served — by `GET /api/destinations` — so the +"served on no read API" claim was false once the endpoint landed, and the +`dest_hash` column it referenced no longer exists. A contract that describes a +column the schema dropped is worse than silence. + +### DOC-R1 - Regression: prior acceptance still holds +```bash +( . .venv/bin/activate && pytest -q tests/ ) && ( cd web && bundle exec rspec ) && ( cd web && npm test ) +``` +**Expected:** all green. This audit changes documentation, `configure.sh`, +Compose, both Dockerfiles, and `flake.nix` — no application code — so every +prior criterion must hold unchanged. Explicitly re-verified because editorial +passes have broken them before: **SB-A6** (dual-CDN egress disclosure, which the +README refactor had dropped), **RV-A4** (`potatomesh_reticulum`, likewise +dropped), **TX-A6**, **RD-A2/RD-A7** (badge greps), and the `RETICULUM_CONFIG_DIR` +counts in RN-A5. diff --git a/CHANGELOG.md b/CHANGELOG.md index f3114d4b..9562ce14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,23 +5,26 @@ ### Features * Web/Data: **Reticulum gets a visual identity of its own** — the placeholder tile (an outline triangle on near-black `#010101`, indistinguishable from MeshCore's at 12 px) becomes a trifoil on violet `#7b61ff`; MeshCore's tile moves off pure black to `#1f2937` so it finally agrees with its own badge; the `/charts` mesh-activity lines repoint at their tiles so the table and the figure teach one colour code; a four-step violet role ramp (`PEER`/`NODE`/`TRANSPORT`/`PROPAGATION`) joins the palettes, all clearing the 4.5:1 badge floor; a hexagon joins circle and diamond as the protocol shape channel; and the legend gains a third column (SPEC RD1–RD8) -* Data: Reticulum node roles are derived from the announce aspect and ranked — `lxmf.propagation` joins the ingested aspects, and a node's role is the highest-ranked aspect it announced this session rather than whichever arrived last, so a peer announcing on several aspects no longer flips between roles (SPEC RD4) +* Data: Reticulum node roles are derived from the announce aspect — `lxmf.propagation` joins the ingested aspects, and each destination row carries the role of the aspect it announced on, so a peer announcing on several aspects no longer flips between roles (SPEC RD4/RE1) * Web: the federation table's Reticulum column renders an em dash for a zero count rather than a violet tile beside a `0` — the column is served as `|| 0` for signature canonicalisation, so during rollout nearly every federated instance would otherwise carry a tile (SPEC RD8) ### Fixes -* Data: **`PROTOCOL=reticulum` no longer needs `INGESTOR_NODE_ID`** — the ingestor now owns an RNS identity (generated once, stored `0600` as `potato_mesh_identity` inside `RETICULUM_CONFIG_DIR`) and derives its own `!xxxxxxxx` from it through the same identity-hash rule applied to every peer, so the heartbeat and the `reticulum` packets/hour figure come up unaided instead of staying dead until an operator set a variable they had no way to guess. The id is stable across restarts, the variable still overrides it, and the identity is never announced, so nothing new transmits (SPEC RN9) +* Data: **`PROTOCOL=reticulum` no longer needs `INGESTOR_NODE_ID`** — the ingestor derives its own `!xxxxxxxx` from the transport identity already in `RETICULUM_CONFIG_DIR`, so the heartbeat and the `reticulum` packets/hour figure come up unaided instead of staying dead until an operator set a variable they had no way to guess. The id is stable across restarts and the variable still overrides it (SPEC RE5) * Data: **a set `CONNECTION` is no longer silently ignored under `PROTOCOL=reticulum`** — it names one serial/TCP/BLE endpoint, which an RNS stack of many interfaces does not have, and the shipped image sets a serial default for every protocol; the ingestor now says at startup that it is dropping the value and names the two variables that do apply (`RETICULUM_CONFIG_DIR` for which stack, `RETICULUM_INTERFACES` for which interfaces) (SPEC RN10) -* Data: **`RETICULUM_INTERFACES` ingested nothing at all** wherever the ingestor shared an RNS instance — RNS keys the shared-instance socket on `instance_name`, not `configdir`, so an app-owned config dir did not stop it attaching to a running `rnsd`, and every announce then arrived over one `LocalInterface` that no per-interface allowlist could match. A scoped ingestor now seeds its own unshared config, and where it is on a shared instance anyway the allowlist fails open with a warning rather than silently filtering everything out (SPEC RN8) +* Data: **`RETICULUM_INTERFACES` ingested nothing at all** wherever the ingestor shared an RNS instance — every announce arrived over one `LocalInterface` that no per-interface allowlist could match. Interface names now resolve through the shared instance itself (`RNS.Reticulum` RPCs to the running `rnsd` and returns its view), so the allowlist works on a shared stack; RPC auth is keyed on the config directory, which is why the ingestor shares the operator's (SPEC RE3) * Data: **a Reticulum announce carrying no display name overwrote the stored one** — the fallback named the destination that announced rather than the node, in a form the web upsert did not recognise as a placeholder. It is now `"Reticulum "`, matching the shape the upsert refuses to overwrite a real name with -* Web: **a destination hash carried by a stale record was dropped** rather than unioned — the union sat inside the upsert's freshness-guarded `ON CONFLICT` clause, so a second ingestor posting an older announce lost the hash the first had never heard. It is now its own forward-only statement, and still cannot regress `last_heard` or touch a real row from a synthetic placeholder (SPEC RN1) -* Docs: the Reticulum config directory is documented — what it is seeded with, what an `AutoInterface` does and does not reach, and that a scoped ingestor does not inherit your `rnsd`'s interfaces +* Docs: the Reticulum config directory is documented — that it defaults to `~/.reticulum`, that a container gets the `potatomesh_reticulum` volume whose stock `AutoInterface` reaches no radio on the default bridge network, and how to add interfaces to it * Data: **`ALLOWED_CHANNELS` silently dropped every message on a stock Docker Compose deployment** — the packaged default was spelled `${ALLOWED_CHANNELS:-""}`, and Compose substitutes defaults as *literal text*, so the ingestor received the two-character string `""` and read it as a one-entry allowlist no real channel could match; every packet was then discarded as `disallowed-channel`. Defaults are now bare (`:-`), both list-valued parsers resolve a quote-*only* value to no filter while taking any fragment with content literally (a Meshtastic channel name is arbitrary UTF-8, so `Ops'` is a name, not a quoted one), and a guard discovers every Compose file in the repo and matches every quoted-default spelling, so the shape cannot return. `HIDDEN_CHANNELS` and `MAP_ZOOM` carried the same defect (both benign) and are fixed with it (ACCEPTANCE CH-A1–CH-A3) -* Data/Web: **Reticulum node ids now key on the peer's identity, not on a destination hash** — a destination hash is a truncated hash over the identity and name hashes, so a peer announcing both `lxmf.delivery` and `nomadnetwork.node` appeared as two unrelated node rows; one peer is now one row (SPEC RN1) -* Data/Web: **`user.publicKey` carries the identity's real public key**, and destination hashes move to their own `destHash` list (`nodes.dest_hash`, unioned across ingestors, stored but never served like `public_key`) (SPEC RN1/RN6) -* Data: **the Reticulum ingestor no longer adopts the operator's `~/.reticulum`** — `RETICULUM_CONFIG_DIR` defaults to an app-owned directory and reaches `.env.example`, `docker-compose.yml`, and `data/Dockerfile` (SPEC RN3) -* Data: new `RETICULUM_INTERFACES` allowlist bounds announce ingestion to named RNS interfaces; empty (the default) still ingests everything, so an `AutoInterface` LAN no longer has to mean the whole local Reticulum network in your dashboard (SPEC RN4) +* Data/Web: **one Reticulum node row per announced destination** — a peer's aspects carry different display names and roles (`lxmf.delivery` and `nomadnetwork.node` announce separately), so merging them produced a row named from one aspect and roled from another; `nodes.identity_hash` groups the rows back into one peer (SPEC RE1) +* Data/Web: **`user.publicKey` carries the identity's real public key**, and destinations get their own table (`id`, `node_id`, `identity_hash`, `name`, `aspect`, `role`, `interface`, `first_heard`, `last_heard`) served by `GET /api/destinations` (SPEC RE2) +* Data: new `RETICULUM_INTERFACES` allowlist bounds announce ingestion to named RNS interfaces, applied from one hop out; empty (the default) ingests everything, and a 0-hop announce — one made by an app on this machine — is always ingested so the operator's own nodes are never filtered away (SPEC RN4/RE4) * Web: **the `/charts` mesh-activity figure draws the Reticulum series** — its line list was frozen to two protocols, so a reticulum-carrying mesh rendered two flat-zero lines with its real traffic invisible while the map card rendered the same payload correctly (SPEC RN2) -* Web: `nodes.dest_hash` and the `instances.reticulum_nodes_count` added in #889 both carry a standalone `data/migrations/*.sql` companion alongside their boot-time schema guard (SPEC RN7) +* Web: the `instances.reticulum_nodes_count` added in #889 carries a standalone `data/migrations/*.sql` companion alongside its boot-time schema guard (SPEC RN7) +* Docs: **every environment variable the ingestor or web app reads is documented** — `PROM_REPORT_IDS` and the `SSE_*`/`FEDERATION_*`/`REMOTE_INSTANCE_*`/Puma tuning knobs appeared in no document at all; the README now carries all 57, and the operator-facing subset reaches `.env.example`, Compose, both images, and the NixOS module (SPEC DOC2) +* Docs/Config: **`configure.sh` and the NixOS module are no longer Meshtastic-only** — the wizard asks which protocol to run and branches its radio, connection, and transmit prompts on the answer; the module gained `ingestor.protocol`, the Reticulum options, and the UDP options, so two of the three supported protocols stopped being unconfigurable through the documented tooling (SPEC DOC3) +* Docs: **the README documents every route**, including `/version`, `/metrics`, `/.well-known/potato-mesh`, the page routes, and that there is no health endpoint (SPEC DOC4) +* Docs: **the dual basemap-CDN egress disclosure was restored** — an earlier editorial pass had dropped it, leaving the doubled third-party tile request undisclosed (SPEC SB6) +* Docs: `CONTRACTS.md` corrected — `identityHash` is served by `GET /api/destinations`, whose read shape is now specified, and the `dest_hash` column it referenced no longer exists (SPEC DOC5) * Docs: `ACCEPTANCE.md` S-A5 / MA-A5 / MA-FA2 / F2-A1 / A4a amended — each still asserted a zero-stub, non-whitelisted `reticulum` that SPEC S6/MA5/MA-F2/F2-2 had already stopped describing; README gains a Reticulum section ### Features diff --git a/CLAUDE.md b/CLAUDE.md index dc8f4260..aba09cb7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -26,6 +26,10 @@ All code must be 100% unit tested — every line, branch, and code path must hav All code must be 100% documented according to the language's API-doc standard (PDoc for Python, RDoc for Ruby, JSDoc for JavaScript, rustdoc for Rust, dartdoc for Dart). Documentation must be sufficient to generate complete API docs from source. In addition to API-level docs, add inline comments wherever the logic is not immediately self-evident. +**Operator-facing docs are task-oriented and short.** `README.md` and anything else an operator reads answers "do X to achieve Y" and stops. One or two sentences per setting: what to set, what happens, what breaks if you get it wrong. No mechanism, no rationale, no history, no reasoning about trade-offs, no explanation of *why* a default was chosen — that material belongs in API docs, inline comments, `SPEC.md`, or `CONTRACTS.md`, where the reader is a contributor rather than someone trying to get a radio working. + +This is a standing correction, not a style preference: generated documentation drifts long and explanatory by default, and every extra paragraph an operator has to read to find the one command they need is a defect. If a README passage explains a design decision, move it. If it hedges or covers a case the reader cannot act on, cut it. When in doubt, cut — the detail is recoverable from the linked SPEC/CONTRACTS entry. + Every file in the repository must carry an Apache v2 license notice using the exact string `Copyright © 2025-26 l5yth & contributors`. **Source-code files** (`.rb`, `.py`, `.js`, `.rs`, `.dart`, etc.) must include the full Apache v2 license header block. **Non-source files** (docs, configs, YAML, TOML, Dockerfiles, etc.) must include a short 2-line Apache v2 notice (copyright line + license reference). Run linters for Python (`black`) and Ruby (`rufo`) to ensure consistent code formatting. diff --git a/README.md b/README.md index c2b3c130..48d67065 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ The web app can be configured with environment variables (defaults shown): | `MAX_THREADS` | `96` | Maximum Puma worker threads. Each active `/api/events` SSE stream pins one thread, so keep this above your peak concurrent SSE clients plus API/ingest headroom. | | `OG_IMAGE_URL` | _unset_ | Optional absolute URL for the social preview image. Must use an `http://` or `https://` scheme; values with other schemes are ignored. Most social platforms (Facebook, LinkedIn, Slack, iMessage) require **HTTPS** to render the card. When set, replaces the runtime-generated `/og-image.png` so deployments without Chromium (or with size-conscious images) can point at a CDN. | | `PAGES_DIR` | `./pages` | The directory for static, custom-content pages. | +| `PROM_REPORT_IDS` | _unset_ | Comma-separated node ids to expose as per-node Prometheus gauges. Empty exports none. | The application derives SEO-friendly document titles, descriptions, and social preview tags from these existing configuration values. `/robots.txt` and @@ -151,6 +152,12 @@ well-known document is staged in The database can be found in `$XDG_DATA_HOME/potato-mesh`. +**Outbound requests.** The map requests basemap tiles from two third-party CDNs +on every viewport — OpenStreetMap HOT (`tile.openstreetmap.fr`) and CARTO +(`basemaps.cartocdn.com`) — so a visitor's browser contacts both. Only `z/x/y` +tile coordinates are sent: no API key, token, cookie, or analytics parameter. +Tiles are the only third-party request the dashboard makes. + ### Custom Pages Instance operators can publish static content pages (contact details, mesh @@ -191,26 +198,73 @@ directory. ### API -The web app contains an API: - -* GET `/api/nodes?limit=100` - returns the latest 100 nodes reported to the app -* GET `/api/positions?limit=100` - returns the latest 100 position data -* GET `/api/messages?limit=100&encrypted=false&since=0` - returns the latest 100 messages newer than the provided unix timestamp (defaults to `since=0` to return full history; disabled when `PRIVATE=1`) -* GET `/api/telemetry?limit=100` - returns the latest 100 telemetry data -* GET `/api/neighbors?limit=100` - returns the latest 100 neighbor tuples -* GET `/api/traces?limit=100` - returns the latest 100 trace-routes caught -* GET `/api/instances` - returns known potato-mesh instances in other locations -* GET `/api/ingestors` - returns active potato-mesh python ingestors that feed data -* GET `/metrics`- metrics for the prometheus endpoint -* GET `/version`- information about the potato-mesh instance -* POST `/api/nodes` - upserts nodes provided as JSON object mapping node ids to node data (requires `Authorization: Bearer `) -* POST `/api/positions` - appends positions provided as a JSON object or array (requires `Authorization: Bearer `) -* POST `/api/messages` - appends messages provided as a JSON object or array (requires `Authorization: Bearer `; disabled when `PRIVATE=1`) -* POST `/api/telemetry` - appends telemetry provided as a JSON object or array (requires `Authorization: Bearer `) -* POST `/api/neighbors` - appends neighbor tuples provided as a JSON object or array (requires `Authorization: Bearer `) -* POST `/api/traces` - appends caught traces routes provided as a JSON object or array (requires `Authorization: Bearer `) - -The `API_TOKEN` environment variable must be set to a non-empty value and match the token supplied in the `Authorization` header for `POST` requests. +All `GET` routes accept `?limit=` and `?since=`; bulk collections also accept +`?before=` and `?protocol=`. All `POST` routes require +`Authorization: Bearer `. + +**Collections** — `GET` list, `GET /:id` for one node's rows, `POST` to ingest: + +| Path | Notes | +| --- | --- | +| `/api/nodes` | `GET`, `GET /:id`, `POST` | +| `/api/positions` | `GET`, `GET /:id`, `POST` | +| `/api/telemetry` | `GET`, `GET /:id`, `POST`; `/api/telemetry/aggregated` for rollups | +| `/api/messages` | `GET`, `GET /:id`, `POST`; all 404 when `PRIVATE=1` | +| `/api/neighbors` | `GET`, `GET /:id`, `POST` | +| `/api/traces` | `GET`, `GET /:id`, `POST` | +| `/api/waypoints` | `GET`, `GET /:id`, `POST`; all 404 when `PRIVATE=1` | +| `/api/destinations` | `GET`; `?node_id=` filters. Reticulum destinations per node | +| `/api/ingestors` | `GET`, `POST`. Active ingestors feeding this instance | +| `/api/instances` | `GET`, `POST`. Known federated instances | + +**Other** + +| Path | Returns | +| --- | --- | +| `GET /api/stats` | Node/message/telemetry counts per protocol and window | +| `GET /api/stats/activity` | Packets/hour time series per protocol | +| `GET /api/events` | SSE stream of collection-change events | +| `GET /version` | Instance name, version, and public config | +| `GET /metrics` | Prometheus exporter — see [`PROMETHEUS.md`](./PROMETHEUS.md) | +| `GET /.well-known/potato-mesh` | Signed federation record for this instance | +| `GET /og-image.png` | Open Graph preview image | +| `GET /robots.txt`, `GET /sitemap.xml` | Crawler directives | + +**Pages** — `GET /` dashboard, `GET /nodes/:id` node detail, `GET /pages/:slug` +custom pages. Static assets: `GET /favicon.ico`, `GET /potatomesh-logo.svg`. + +There is no health endpoint; use `GET /version`. + +### Advanced tuning + +Environment variables for the web app's internals. All optional; set only to +change the defaults shown. Not pre-declared in `.env.example`, Compose, or the +NixOS module — set them in the environment of the web process. + +| Variable | Default | Purpose | +| --- | --- | --- | +| `MIN_THREADS` | `16` | Puma minimum thread count. | +| `MAX_THREADS` | `96` | Puma maximum thread count. | +| `PUMA_FORCE_SHUTDOWN` | `3` | Seconds Puma waits before forcing shutdown. | +| `STATS_CACHE_TTL_SECONDS` | `60` | Cache lifetime for `/api/stats` responses. | +| `OG_IMAGE_TTL_SECONDS` | `3600` | Cache lifetime for the generated `/og-image.png`. | +| `LIVE_SAFETY_POLL_SECONDS` | `300` | Interval of the fallback poll that backstops the SSE stream. | +| `SSE_HEARTBEAT_SECONDS` | `15` | Comment-frame keepalive interval on `/api/events`. | +| `SSE_MAX_LIFETIME_SECONDS` | `600` | Maximum lifetime of one SSE connection before the client reconnects. | +| `SSE_PUBLISH_COOLDOWN` | `1.0` | Minimum seconds between SSE publishes, coalescing bursts. | +| `SSE_THREAD_RESERVE` | `32` | Threads held back from SSE so ordinary requests keep being served. | +| `FEDERATION_WORKERS` | `4` | Federation crawl worker-pool size. | +| `FEDERATION_WORK_QUEUE` | `128` | Queued federation tasks before new ones are dropped. | +| `FEDERATION_TASK_TIMEOUT` | `120` | Seconds before one federation task is abandoned. | +| `FEDERATION_SHUTDOWN_TIMEOUT` | `3` | Seconds to drain federation workers on shutdown. | +| `FEDERATION_CRAWL_COOLDOWN` | `300` | Minimum seconds between crawls of the same domain. | +| `FEDERATION_MAX_DOMAINS_PER_CRAWL` | `256` | Domain ceiling for one crawl pass. | +| `FEDERATION_MAX_INSTANCES_PER_RESPONSE` | `64` | Instances accepted from one peer's response. | +| `INITIAL_FEDERATION_DELAY_SECONDS` | `2` | Delay before the first crawl after boot. | +| `REMOTE_INSTANCE_CONNECT_TIMEOUT` | `15` | Connect timeout when fetching a peer. | +| `REMOTE_INSTANCE_READ_TIMEOUT` | `60` | Read timeout when fetching a peer. | +| `REMOTE_INSTANCE_REQUEST_TIMEOUT` | `30` | Overall request timeout when fetching a peer. | +| `REMOTE_INSTANCE_MAX_RESPONSE_BYTES` | `8388608` | Response ceiling (8 MiB) when fetching a peer. | ### Monitoring @@ -271,6 +325,10 @@ appear in the allowlist. | `PROTOCOL` | `meshtastic` | Which protocol are we ingesting? One of `meshtastic`, `meshcore`, or `reticulum`. | | `CONNECTION` | `/dev/ttyACM0` | Where do we talk to the node? Accepts serial ports, TCP connections, and bluetooth addresses (BLE mac). Ignored under `PROTOCOL=reticulum`, which has no single endpoint — see [Reticulum](#reticulum). | | `DEBUG` | `0` | Set to `1` for verbose logging in the ingestor services. | +| `CHANNEL_INDEX` | `0` | Which channel index to ingest from. | +| `ENERGY_SAVING` | `0` | Set to `1` to duty-cycle the radio connection instead of holding it open. | +| `FREQUENCY` | _unset_ | Deprecated alias for `MESHTASTIC_FREQ`; overrides the auto-detected LoRa frequency. | +| `CHANNEL` | _unset_ | Deprecated alias for `MESHTASTIC_PRESET`. | | `ALLOWED_CHANNELS` | _unset_ | Comma-separated channel names the ingestor accepts; when set, all other channels are skipped before hidden filters. | | `HIDDEN_CHANNELS` | _unset_ | Comma-separated channel names the ingestor will ignore when forwarding packets. | | `TRANSPORT` | `api` | Ingestor transport: `api` (Meshtastic library over serial/TCP/BLE) or `udp` (passive LAN multicast; see [Passive UDP transport](#passive-udp-transport)). | @@ -279,9 +337,9 @@ appear in the allowlist. | `PRIMARY_CHANNEL_NAME` | _unset_ | Name of channel 0 (e.g. `MediumFast`/`LongFast`: the preset name the firmware uses when the channel name is blank, as shown by `meshtastic --info`). Used to compute the channel hash that identifies primary traffic on the UDP multicast. Required by UDP `PRIMARY_CHANNEL_ONLY=1`, because a secondary channel can share the default `AQ==` key: only the per-channel hash of *(name, key)* distinguishes them. | | `MESH_UDP_GROUP` | `224.0.0.69` | Multicast group joined in UDP transport. | | `MESH_UDP_PORT` | `4403` | Multicast port joined in UDP transport. | -| `INGESTOR_NODE_ID` | _unset_ | `!xxxxxxxx` id used for the ingestor heartbeat. Required for the UDP transport, which cannot auto-detect "self" — without it the heartbeat never registers and its packets/hour stats stay at zero. Optional for `PROTOCOL=reticulum`, which derives its own id from the identity it owns; set it there only to override that. See [Reticulum](#reticulum). | -| `RETICULUM_CONFIG_DIR` | `$XDG_CONFIG_HOME/potato-mesh/reticulum`, else `~/.config/potato-mesh/reticulum` | Which Reticulum config the ingestor uses, and so where it looks for available interfaces. App-owned by default: it never adopts the operator's own `~/.reticulum`. Set it only to deliberately share an existing RNS stack. See [Reticulum](#reticulum). | -| `RETICULUM_INTERFACES` | _unset_ | Which RNS interfaces to ingest announces from: name your local LoRa interface, e.g. `RNode` or `BerlinMesh`. Comma-separated and matched as a case-insensitive substring of the interface name. Empty ingests every interface. See [Reticulum](#reticulum). | +| `INGESTOR_NODE_ID` | _unset_ | `!xxxxxxxx` id used for the ingestor heartbeat. Required for the UDP transport, which cannot auto-detect "self". Optional for `PROTOCOL=reticulum`, which derives one from the config dir's transport identity; set it there only to override that. | +| `RETICULUM_CONFIG_DIR` | `~/.reticulum` | Which RNS config the ingestor uses, and so which interfaces it can see. Point it at the directory your `rnsd` uses. See [Reticulum](#reticulum). | +| `RETICULUM_INTERFACES` | _unset_ | Which RNS interfaces to ingest from, e.g. `RNode`. Comma-separated, case-insensitive substring match against the names in `rnstatus`. Your own nodes are always ingested; empty ingests everything. See [Reticulum](#reticulum). | | `MESHCORE_TELEMETRY_POLL_SECONDS` | `300` | Requires `TX_ENABLED=1` (polling other nodes is a transmission). Seconds between Meshcore contact telemetry polls (one on-air request per interval, round-robin over the roster; each contact is additionally polled at most once per 24 h: when every contact is fresh the tick sends nothing). Set `0` to disable on-air polling. | | `MESHCORE_SELF_TELEMETRY_SECONDS` | `3600` | Seconds between Meshcore host self-telemetry reads (battery/sensors over the companion link, no airtime). Set `0` to disable. | | `TX_ENABLED` | `0` | Master switch for everything the ingestor puts on the air Off by default: the ingestor is a listener. Set to `1` to allow transmitting; this is what enables Meshcore on-air contact telemetry polling. Does not by itself enable announcements. See [Transmitting on the mesh](#transmitting-on-the-mesh). | @@ -292,10 +350,6 @@ appear in the allowlist. By default a PotatoMesh ingestor never transmits. It listens, and forwards what it hears to your dashboard. Nothing below happens unless you turn it on. -That default is deliberate. Airtime on a LoRa mesh is a shared, scarce resource, -and whether automated traffic belongs on a community's channels is that -community's decision: not a default we get to make for you. - | Variable | Default | What turning it on means | | --- | --- | --- | | `TX_ENABLED` | `0` | The ingestor may transmit. On its own this enables Meshcore on-air telemetry polling: round-robin requests to other nodes for their battery/sensor readings, at most one request per `MESHCORE_TELEMETRY_POLL_SECONDS` (default 300 s) and at most once per 24 h per node. This is how other nodes' telemetry reaches your dashboard. Meshtastic ingestion needs no transmission at all. | @@ -307,10 +361,9 @@ The announcement looks like this: Meshtastic activity in the last 24h: 42 active nodes, 118 packets/hour. https://mesh.example.org ``` -The numbers are read back from your own instance's public API, so they reflect -the whole mesh your dashboard sees rather than the one radio doing the talking. -It is suppressed automatically when your instance runs with `PRIVATE=1`, and -that check fails closed: if the ingestor cannot reach your instance to ask, it +The numbers come from your instance's public API, covering the whole mesh your +dashboard sees. Announcements are suppressed when the instance runs `PRIVATE=1`, +and the check fails closed: if the ingestor cannot reach the instance to ask, it does not transmit. Both flags must be on for an announcement to go out: @@ -323,15 +376,14 @@ Both flags must be on for an announcement to go out: | `1` | `1` | Telemetry polling on, one announcement per 24 h. | -The ingestor states the resolved policy in its -log at startup, without needing `DEBUG=1`: +The ingestor logs the resolved policy at startup, without `DEBUG=1`: ``` [2026-08-24T09:12:44.117Z] [potato-mesh] [info] context=tx.policy announcements_permitted=False rx_only=False transmit_permitted=False tx_announce=False tx_enabled=False Transmit policy resolved ``` -If you set a flag and nothing happens, that line tells you what the ingestor -actually resolved: including a flag that never reached the container. +If a flag seems to have no effect, that line shows what the ingestor actually +resolved — including a flag that never reached the container. ### Meshcore @@ -353,102 +405,50 @@ the radio's local roster rotates. ### Reticulum -Set `PROTOCOL=reticulum` to ingest from a Reticulum (RNS) network. This is a -passive announce listener: it registers announce handlers for the -`lxmf.delivery`, `nomadnetwork.node` and `lxmf.propagation` destination aspects -and files every announce it hears as a node. Which aspects a peer announces on -is the only signal Reticulum gives about what it *is*, so the node's role is -derived from them and ranked: `PROPAGATION > NODE > PEER`: rather than taken -from whichever announce arrived last. - -Announces carry no SNR, battery, or position, so those columns render as dashes -and Reticulum nodes stay off the map. One peer is one node row: the canonical -`!xxxxxxxx` id derives from the peer's identity hash, so a peer announcing on -several destination aspects merges into a single row rather than appearing once -per aspect. - -**You do not need to set `INGESTOR_NODE_ID`.** Reticulum has no handshake -revealing "our" node id, so the ingestor owns an RNS identity of its own, -generated on first run and stored as `potato_mesh_identity` inside -`RETICULUM_CONFIG_DIR`. Its `!xxxxxxxx` comes from that identity's hash through -the same mapping applied to every peer, so it is stable across restarts and the -ingestor's own row is shaped like any other. Set `INGESTOR_NODE_ID` only to -override it. The identity is local and never announced, so this adds nothing the -transmit switches gate. - -If the identity can neither be read nor written — an unwritable config dir, or a -key file left unreadable — the ingestor says so and carries on ingesting -announces without a node id, which is the one case where the heartbeat stays -unregistered and the packets/hour figure stays at zero. - -**`CONNECTION` does not apply.** It names one serial, TCP, or BLE endpoint, and -an RNS stack of many interfaces has no such thing. The two variables below cover -the same ground: `RETICULUM_CONFIG_DIR` says *which stack*, -`RETICULUM_INTERFACES` says *which of its interfaces to ingest from*. A set -`CONNECTION` is ignored, and the ingestor logs that it was — the shipped -container image carries a serial default for every protocol, so switching to -`PROTOCOL=reticulum` inherits one you never chose. - -**Two things worth configuring:** - -`RETICULUM_CONFIG_DIR` defaults to an app-owned directory -(`$XDG_CONFIG_HOME/potato-mesh/reticulum`, else `~/.config/potato-mesh/reticulum`) -rather than RNS's user default `~/.reticulum`. Installing a dashboard ingestor -should not silently adopt the transport-node configuration you run for yourself -— including whatever `enable_transport` and interfaces it defines. Point the -variable at your own directory if sharing a stack is what you want. - -**What the config dir starts with**, and it differs by whether you scoped it. - -*Without* `RETICULUM_INTERFACES`, RNS writes its own stock config, whose only -interface is a link-local IPv6 `AutoInterface`. That hears the Reticulum peers on -your LAN — the "ingests everything" default above — and the packaged `ingestor` -service runs `network_mode: host`, so it behaves the same in Docker as on bare -metal. (The opt-in `ingestor-bridge` profile is not on the host network and hears -nothing there.) - -*With* `RETICULUM_INTERFACES` set, the ingestor seeds the file itself, with -`share_instance = No` and **no interface enabled**. Both parts are deliberate. -Sharing is off because an allowlist can only be honoured on its own RNS stack: -attached to a shared `rnsd`, every announce arrives over a single -`LocalInterface` and the allowlist cannot tell them apart. And no interface is -enabled because a stock `AutoInterface` is named `AutoInterface[Default -Interface]`, which matches no sensible allowlist — the listener would connect and -ingest nothing. **So a freshly scoped ingestor hears nothing until you add the -interfaces you named**; it warns about exactly that at startup, and the seeded -file carries a commented `RNodeInterface` example to copy. - -Two consequences worth stating plainly. **A scoped ingestor does not inherit your -`rnsd`'s interfaces**, so any radio you want it to hear has to be configured in -its own file. And **an `AutoInterface` never reaches a LoRa radio** in any network -mode — that always needs an `RNodeInterface` pointing at the device, or a -`TCPClientInterface` pointing at a hub. - -Edit the `config` file on the `potatomesh_reticulum` volume (Linux images; -`--no-deps` stops it starting the web stack too): +Set `PROTOCOL=reticulum` to ingest from a Reticulum (RNS) network. The ingestor +listens for announces and files each one as a node. It never transmits, so +`TX_ENABLED=0` (the default) is fine. + +`CONNECTION` does not apply; if it is set the ingestor ignores it and logs that +it did. + +It reads your existing `~/.reticulum`, so if `rnsd` already works, so does this: + +```bash +API_TOKEN=... INSTANCE_DOMAIN=https://your.instance PROTOCOL=reticulum ./data/mesh.sh +``` + +**To ingest only from your radio**, set `RETICULUM_INTERFACES` to part of the +interface name from `rnstatus` — matching is case-insensitive substring: ```bash -docker compose run --rm --no-deps --entrypoint sh ingestor \ - -c 'vi /app/.config/potato-mesh/reticulum/config' +RETICULUM_INTERFACES="RNode Reticulum Berlin" ``` -An existing config is never overwritten, so if you would rather share the stack, -write the file yourself and the ingestor leaves it alone. The allowlist then -cannot be honoured, so it admits every announce rather than none — and warns once -that it is doing so. +Your own nodes are always ingested regardless of this setting. Everything +further away is filtered by it. + +**To use a different RNS config**, set `RETICULUM_CONFIG_DIR`. Point it at the +same directory your `rnsd` uses — interface filtering asks that instance which +interface an announce came in on, and the request is rejected across directories. -`RETICULUM_INTERFACES` bounds what gets ingested. An RNS stack can carry LoRa -and IP interfaces at once, and RNS's stock config enables an `AutoInterface` -(IPv6 link-local multicast): so an unscoped listener files every announce on -your LAN, not just your LoRa peers. Setting e.g. `RETICULUM_INTERFACES=rnode` -restricts ingestion to announces whose path arrived on a matching interface. -Leaving it empty ingests everything, which is the default. Reticulum exposes no -"this peer is on LoRa" marker, so the interface a path arrived on is the only -signal available. +**In Docker**, the config dir is the `potatomesh_reticulum` volume, mounted at +`/app/.config/potato-mesh/reticulum`. RNS writes its stock config there on first +start, whose only interface is a link-local `AutoInterface`: -Note that the RNS stack itself is not silent at the interface layer even though -the ingestor is: an `AutoInterface` multicasts peer discovery, and a config with -`enable_transport` set relays other nodes' traffic. +- On the default bridge network that interface reaches no radio and usually no + peers, so the ingestor connects and hears nothing. +- Add your interfaces to the volume's `config` file, then restart the ingestor: + `docker compose exec ingestor sh -c 'cat >> /app/.config/potato-mesh/reticulum/config'` +- Or point `RETICULUM_CONFIG_DIR` at a bind mount of the host's `~/.reticulum` + and run the container with host networking. + +A node appears once per announced destination, so one peer running LXMF and a +nomadnet node shows up as two entries. `GET /api/destinations` lists them and +groups them by identity. + +Announces carry no SNR, battery, or position, so those columns show dashes and +Reticulum nodes stay off the map. ### Passive UDP transport @@ -522,6 +522,17 @@ services.potato-mesh = { }; ``` +Every variable in the tables above has a module option, named in camelCase +(`SITE_NAME` → `siteName`, `OG_IMAGE_URL` → `ogImageUrl`): + +- Web options sit at the top level; ingestor options sit under `ingestor`. +- `PROTOCOL`, `TRANSPORT`, `INGESTOR_NODE_ID` → `ingestor.protocol`, + `ingestor.transport`, `ingestor.nodeId`. +- `RETICULUM_CONFIG_DIR`, `RETICULUM_INTERFACES` → `ingestor.reticulumConfigDir`, + `ingestor.reticulumInterfaces`. +- Options typed `nullOr` emit no variable when left `null`, so the app keeps its + own default. + ## Docker Docker images are published on GitHub Container Registry for each release. diff --git a/SPEC.md b/SPEC.md index e760740a..e41b1d37 100644 --- a/SPEC.md +++ b/SPEC.md @@ -1378,7 +1378,7 @@ offers. | # | Decision | Source | | --- | --- | --- | -| **RN9** | **The ingestor owns a Reticulum identity, so `INGESTOR_NODE_ID` is an override rather than a requirement.** #890 shipped `extract_host_node_id` returning the operator-supplied `INGESTOR_NODE_ID`, on the premise that a passive listener has no handshake revealing "our" node id. The premise holds; the conclusion did not follow. RNS lets any participant hold an identity, so the ingestor generates one on first run, stores it as `potato_mesh_identity` inside the app-owned `RETICULUM_CONFIG_DIR` (RN3), and derives its `!%08x` through **the same identity-hash mapping applied to every peer** (RN1) - so the ingestor's own row is shaped like any other Reticulum row, and no second, private id rule exists to disagree with the first. A plain `PROTOCOL=reticulum` deployment therefore brings up the heartbeat, and so the `reticulum.packets.hour` scope, unaided; `INGESTOR_NODE_ID` still wins when set, which also means no key is generated for a deployment that does not need one. **Persist-or-nothing, deliberately:** an identity that could not be written would be regenerated on every start and each generation is a different node id, so the dashboard would accumulate one orphan ingestor row per restart. The provider therefore reports **no** id in that case - the pre-RN9 behaviour, recoverable - and warns; an existing but unreadable key file is **never** overwritten, since clobbering it would silently change the ingestor's identity, and the likeliest causes (permissions, a half-written file) are ones an operator should resolve deliberately. The key file is written `0600` regardless of umask. **Not a transmit site (extends RN5/MA7):** generating and storing keys is local and the identity is never announced, so `TX_ENABLED=0` remains sufficient. The startup warning #890 emitted for an unset `INGESTOR_NODE_ID` is **removed**: warning about an override nobody must set trains operators to ignore the log. | interview (#888 follow-up) | +| **RN9** | ~~**The ingestor owns a Reticulum identity.**~~ **Superseded by RE5.** #896 had the provider generate and persist its own RNS identity (`potato_mesh_identity`, `0600`, inside `RETICULUM_CONFIG_DIR`) and derive `!xxxxxxxx` from it "through the same identity-hash rule applied to every peer (RN1), so there is no second, private id rule". The reasoning was sound and its premise then held; two later decisions removed it. **RE1** re-keyed peers on the *destination* hash, so an identity-keyed ingestor row would have *become* the second, private rule it was written to avoid. **RE3** reverted `RETICULUM_CONFIG_DIR` to the operator's `~/.reticulum`, so the key would be written into the directory `rnsd` owns rather than a private one. The goal it set — no operator configuration required — is kept in full by RE5, which derives the id from the transport identity the config dir already holds and writes nothing. | interview (#896, superseded) | | **RN10** | **`CONNECTION` does not apply to Reticulum; the variables are disjoint, not in conflict.** The review asked whether `CONNECTION` and `RETICULUM_CONFIG_DIR`/`RETICULUM_INTERFACES` collide. They do not overlap: `CONNECTION` names **one** serial, TCP, or BLE endpoint, and an RNS stack is a **set of interfaces** with no such endpoint - which is why `ReticulumProvider.connect` has always ignored `active_candidate`/`CONNECTION` and synthesised `reticulum://` instead. The Reticulum split is `RETICULUM_CONFIG_DIR` = *which stack* (RN3), `RETICULUM_INTERFACES` = *which of its interfaces to ingest from* (RN4). What was wrong was not the model but its silence: `data/Dockerfile` sets `CONNECTION=/dev/ttyACM0` in the shared env block of **both** stages, so an operator switching to `PROTOCOL=reticulum` inherits a serial path that reads as if it applies and is dropped without a word. `connect` now says so once at `info`. **The image default is deliberately left in place**: dropping it would change how `PROTOCOL=meshtastic`/`meshcore` containers resolve their port (from a fixed path to `default_serial_targets()` auto-detection, which ends at `/dev/ttyACM0` anyway but is not the same thing), and a Reticulum follow-up is the wrong change to carry it. Recorded so the residue is a decision rather than an oversight. | interview (#888 follow-up) | ## Feature: Reticulum visual identity (RNS tile, role ramp, one colour code) @@ -1435,3 +1435,47 @@ explicitly rather than absorbed silently. | **RD7** | **Third legend column and README badge.** The legend gains an RNS column carrying the tile, label, live 7-day count and the four role filters, inheriting UX8's pressed-state and `aria-pressed` conventions unchanged. The README gains a Reticulum badge in the tile's own `#7b61ff` - where MeshCore's badge uses `#1f2937` rather than a true black that would vanish against the badge row, violet needs no such adjustment. **The badge reads `supported`, not the design's `planned`**: the brief chose `planned` on the premise that the ingestor did not exist and instructed "swap the word when `PROTOCOL=reticulum` ships", which it did in #890. This changes the badge colour from the `#1f2937` that #893 shipped. | design (turn 2c) + interview | | **RD8** | **Scope boundary.** Panel **2d** - the mesh-activity figure redesign (line-end labels, 10% area fills, per-line weight and opacity, gridline and axis restyle) - is **deferred to its own feature**: it is a figure redesign that stands alone, it would land as options on the shared `node-page-charts` renderers, and no other figure on the page needs any of it. The **federation map needs no protocol work**: its markers colour by total active node count through `colorForNodeCount` and its legend keys those buckets, so violet never means a bucket there. The instances column gains one **deliberate behaviour change** (the only user-visible one outside the assets): an instance with no RNS nodes now renders an em dash rather than a tile beside a zero. This was *not* the prior behaviour - `normalize_instance_row` serves `reticulum_nodes_count` as `|| 0` so a crawler re-verifying a relayed v2 record rebuilds the canonical the sender signed (FS2), which left the renderer's `== null` em-dash branch unreachable for this column alone, so every federated row carried a tile beside its count. RD1 made that a bright violet tile, and during rollout almost every federated instance reports zero - a tile in every cell would make the column look broken rather than new. The two sibling columns can still tell absent from zero and are untouched. | interview + design | +## Bugfix: Reticulum end-to-end field findings (#893 hardware test) + +The first run against a real RNode and a live `rnsd` disproved two assumptions +this project had built on. Acceptance: `ACCEPTANCE.md` **RE-A1...RE-A8**, +**RE-R1**. + +**Invariant check.** *I* — **unaffected**. *II (privacy)* — **extends**: +`identity_hash` joins `public_key` as stored-but-never-served; `destinations` +is served, and carries only what a peer broadcast. *III* — **unaffected**. +*IV (parity)* — **upheld**: `destinations` is protocol-neutral; a node with one +destination simply has one row. + +| # | Decision | Source | +| --- | --- | --- | +| **RE1** | **One row per announced destination (reverses RN1).** RN1 keyed rows on the identity so a peer's aspects merged. The field test showed the cost: the aspects carry **different names** — `lxmf.delivery` announced "Afri Nomad Orion", `nomadnetwork.node` announced "Department of Decentralization" — so the merged row was named from one aspect and roled from another. `node_id` is the first four bytes of the **destination hash** again; `nodes.identity_hash` groups the rows into a peer, and how that grouping is *presented* is a separate design pass. `_ROLE_RANK`, `_rank_role`, `_record_role` and the cross-ingestor `dest_hash` union are all removed: one aspect per row leaves nothing to rank or merge. | field test | +| **RE2** | **Destinations are a table, and they are served.** `destinations` (`id`, `node_id`, `identity_hash`, `name`, `aspect`, `role`, `interface`, `first_heard`, `last_heard`) **replaces** the `nodes.dest_hash` JSON column — a column and a table modelling one relationship would drift — and is served by `GET /api/destinations`. Named for `RNS.Destination`, not "aspects": an aspect is one component of a destination's name. | interview | +| **RE3** | **Interface scoping works through a shared instance (amends RN4, retires RN8).** `RNS.Reticulum`'s accessors RPC to a running `rnsd` and return **its** view of which interface an announce arrived on; `RNS.Transport.next_hop_interface` reads only the local path table and answers `LocalInterface[...]` for everything, which is what made the allowlist look unanswerable. The config seeding (RN8) and the fail-open both existed solely to work around that and are **removed**. RPC auth is keyed on the config dir's identity, so the ingestor must share the operator's directory — which is why RN3's default reverts to `~/.reticulum`. | field test | +| **RE4** | **A 0-hop announce is always ingested.** `Transport.inbound` adds a hop to every inbound packet and takes it back for a local-client or shared-instance interface, so zero hops can only mean "announced by an app on this machine". Filtering purely on interface name hid the operator's own nodes, because every local destination legitimately reads `LocalInterface[rns/default]`. `RETICULUM_INTERFACES` now applies from one hop out, where the interface is real. | field test | +| **RE5** | **The host node id is derived, and canonicalised the Reticulum way.** `canonical_node_id` keeps the low 32 bits of a hex value — right for a Meshtastic node num, wrong for a 16-byte identity hash, which it truncates from the opposite end; the ingestor registered `!86c39940` while its own peer row read `!27716218`. A raw identity hash now maps through `_reticulum_node_id`. When `INGESTOR_NODE_ID` is unset the id is derived from `RNS.Transport.internal_identity()` rather than warned about: an operator should not have to grep a nomadnet logfile for it. | field test | +| **RE6** | **Protocol-scoped settings say so, and the transmit gate is logged when it blocks something.** `CONNECTION` is Meshtastic/MeshCore only and is documented as such; Reticulum takes its target from `RETICULUM_CONFIG_DIR`. The announcement policy check moves **after** the due check, so the suppression line names the gate only when it actually blocks a due announcement instead of once per 60 s loop — the startup `Transmit policy resolved` line already carries that diagnosis. | field test | + + +## Documentation: operator docs, flag coverage, API coverage (audit) + +An audit pass, not a behaviour change: no application code moved. It closes the +drift between what the code reads and what the documents say, and fixes the two +places where an operator-facing document had silently stopped being true. + +*I (local LoRa only)* — **unaffected**: no ingest path changed. +*II (privacy & consent)* — **strengthened**: the dual-CDN tile egress is disclosed +again (SB6/D11), and `configure.sh` now asks about transmitting instead of +writing the defaults silently. +*III (federation)* — **unaffected**. +*IV (parity)* — **upheld and repaired**: `configure.sh` and the NixOS module were +Meshtastic-only, so two of the three supported protocols could not be configured +through the documented tooling at all. + +| # | Decision | Source | +| --- | --- | --- | +| **DOC1** | **Operator documentation answers "do X to achieve Y".** No design rationale, no history, no justification of a default: those live in `SPEC.md`. Where a *reason* is what an operator must act on (why `PRIMARY_CHANNEL_NAME` is required for UDP), it stays — the test is whether the reader needs it to configure the thing correctly, not whether it is interesting. | interview | +| **DOC2** | **Two documentation tiers.** `README.md` documents **every** environment variable either config file reads — no exemptions, since a variable that exists only in code is undiscoverable. The **operator-facing** subset is additionally piped through `.env.example`, Compose, the NixOS module, and each service image that reads it. Internal tuning knobs (thread pools, timeouts, SSE and federation internals) are README-only: pre-declaring two dozen of them buries the handful an operator sets. | interview | +| **DOC3** | **The configuration surfaces are protocol-neutral.** `configure.sh` asks which protocol to run and branches its radio, connection, and summary blocks on the answer; the NixOS module gained `ingestor.protocol`, the Reticulum options, and the UDP options. A wizard that can only produce a Meshtastic `.env` contradicts invariant IV regardless of what the ingestor supports. | audit | +| **DOC4** | **Documented endpoints include the ones that are not APIs**, and the absence of one is itself documented: `/version`, `/metrics`, `/.well-known/potato-mesh`, `/og-image.png`, `robots.txt`, `sitemap.xml`, the page routes — and an explicit statement that there is **no** health endpoint, because there is none and a reader would otherwise go looking. | interview | +| **DOC5** | **`CONTRACTS.md` is checked against what is actually served.** It claimed `identityHash` reached no read API, which stopped being true when `GET /api/destinations` landed, and it referenced a `dest_hash` column that RE2 had already replaced. A contract describing a dropped column is worse than silence, so the destinations read shape is now specified there alongside the other GET contracts. | audit | diff --git a/configure.sh b/configure.sh index c923864a..52e6c03c 100755 --- a/configure.sh +++ b/configure.sh @@ -72,8 +72,19 @@ update_env() { # Get current values from .env if they exist SITE_NAME=$(grep "^SITE_NAME=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "PotatoMesh Demo") -CHANNEL=$(grep "^CHANNEL=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "#LongFast") -FREQUENCY=$(grep "^FREQUENCY=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "915MHz") +# Radio preset/frequency: prefer the canonical MESHTASTIC_* names, falling back +# to the deprecated CHANNEL/FREQUENCY aliases so existing .env files still load. +MESHTASTIC_PRESET=$(grep "^MESHTASTIC_PRESET=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "") +[ -n "$MESHTASTIC_PRESET" ] || MESHTASTIC_PRESET=$(grep "^CHANNEL=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "#LongFast") +MESHTASTIC_FREQ=$(grep "^MESHTASTIC_FREQ=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "") +[ -n "$MESHTASTIC_FREQ" ] || MESHTASTIC_FREQ=$(grep "^FREQUENCY=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "915MHz") +MESHCORE_PRESET=$(grep "^MESHCORE_PRESET=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "") +MESHCORE_FREQ=$(grep "^MESHCORE_FREQ=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "") +PROTOCOL=$(grep "^PROTOCOL=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "meshtastic") +TX_ENABLED=$(grep "^TX_ENABLED=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "0") +TX_ANNOUNCE=$(grep "^TX_ANNOUNCE=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "0") +RETICULUM_CONFIG_DIR=$(grep "^RETICULUM_CONFIG_DIR=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "") +RETICULUM_INTERFACES=$(grep "^RETICULUM_INTERFACES=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "") FEDERATION=$(grep "^FEDERATION=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "1") PRIVATE=$(grep "^PRIVATE=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "0") HIDDEN_CHANNELS=$(grep "^HIDDEN_CHANNELS=" .env 2>/dev/null | cut -d'=' -f2- | tr -d '"' || echo "") @@ -104,10 +115,28 @@ read_with_default "Default map zoom (leave blank to auto-fit)" "$MAP_ZOOM" MAP_Z read_with_default "Max Distance (km)" "$MAX_DISTANCE" MAX_DISTANCE echo "" -echo "📡 Meshtastic Settings" -echo "---------------------" -read_with_default "Channel" "$CHANNEL" CHANNEL -read_with_default "Frequency (868MHz, 915MHz, etc.)" "$FREQUENCY" FREQUENCY +echo "🛰 Protocol" +echo "-----------" +echo "Which mesh protocol should the ingestor read? (meshtastic, meshcore, reticulum)" +read_with_default "Protocol" "$PROTOCOL" PROTOCOL + +echo "" +echo "📡 Radio Settings" +echo "-----------------" +echo "Preset and frequency are display values for the join strip and federation directory." +case "$PROTOCOL" in + meshcore) + read_with_default "Meshcore preset" "$MESHCORE_PRESET" MESHCORE_PRESET + read_with_default "Meshcore frequency (868MHz, 915MHz, etc.)" "$MESHCORE_FREQ" MESHCORE_FREQ + ;; + reticulum) + echo "Reticulum announces carry no radio metadata; skipping preset/frequency." + ;; + *) + read_with_default "Meshtastic preset" "$MESHTASTIC_PRESET" MESHTASTIC_PRESET + read_with_default "Meshtastic frequency (868MHz, 915MHz, etc.)" "$MESHTASTIC_FREQ" MESHTASTIC_FREQ + ;; +esac echo "" echo "💬 Optional Settings" @@ -145,10 +174,32 @@ read_with_default "Docker image tag (latest, vX.Y, etc.)" "$POTATOMESH_IMAGE_TAG echo "" echo "🔌 Ingestor Connection" echo "----------------------" -echo "Define how the mesh ingestor connects to your Meshtastic device." -echo "Use serial devices like /dev/ttyACM0, TCP endpoints such as tcp://host:port," -echo "or Bluetooth addresses when supported." -read_with_default "Connection target" "$CONNECTION" CONNECTION +if [ "$PROTOCOL" = "reticulum" ]; then + echo "Reticulum uses an RNS config directory, not a single endpoint." + echo "Leave the directory blank to use ~/.reticulum. Point it at the same" + echo "directory your rnsd uses so interface filtering can resolve names." + read_with_default "RNS config directory (blank = ~/.reticulum)" "$RETICULUM_CONFIG_DIR" RETICULUM_CONFIG_DIR + echo "Restrict ingestion to named interfaces (case-insensitive substring)." + echo "Leave blank to ingest announces from every interface." + read_with_default "Reticulum interfaces" "$RETICULUM_INTERFACES" RETICULUM_INTERFACES +else + echo "Define how the mesh ingestor connects to your device." + echo "Use serial devices like /dev/ttyACM0, TCP endpoints such as tcp://host:port," + echo "or Bluetooth addresses when supported." + read_with_default "Connection target" "$CONNECTION" CONNECTION +fi + +echo "" +echo "📻 Transmit Settings" +echo "--------------------" +echo "The ingestor listens only by default. Enabling transmit puts traffic on" +echo "shared airtime; announcements additionally broadcast one activity line per day." +read_with_default "Allow transmitting (1=yes, 0=no)" "$TX_ENABLED" TX_ENABLED +if [ "$TX_ENABLED" = "1" ]; then + read_with_default "Send daily activity announcement (1=yes, 0=no)" "$TX_ANNOUNCE" TX_ANNOUNCE +else + TX_ANNOUNCE=0 +fi echo "" echo "🔐 Security Settings" @@ -184,10 +235,29 @@ fi echo "" echo "📝 Updating .env file..." +# Write KEY="VALUE" when the value is non-empty, otherwise strip the key so the +# app falls back to its own default rather than reading an empty override. +update_env_optional() { + local key="$1" + local value="$2" + if [ -n "$value" ]; then + update_env "$key" "\"$value\"" + else + sed -i.bak "/^$key=.*/d" .env + fi +} + # Update .env file update_env "SITE_NAME" "\"$SITE_NAME\"" -update_env "CHANNEL" "\"$CHANNEL\"" -update_env "FREQUENCY" "\"$FREQUENCY\"" +update_env "PROTOCOL" "$PROTOCOL" +update_env_optional "MESHTASTIC_PRESET" "$MESHTASTIC_PRESET" +update_env_optional "MESHTASTIC_FREQ" "$MESHTASTIC_FREQ" +update_env_optional "MESHCORE_PRESET" "$MESHCORE_PRESET" +update_env_optional "MESHCORE_FREQ" "$MESHCORE_FREQ" +update_env "TX_ENABLED" "$TX_ENABLED" +update_env "TX_ANNOUNCE" "$TX_ANNOUNCE" +update_env_optional "RETICULUM_CONFIG_DIR" "$RETICULUM_CONFIG_DIR" +update_env_optional "RETICULUM_INTERFACES" "$RETICULUM_INTERFACES" update_env "MAP_CENTER" "\"$MAP_CENTER\"" if [ -n "$MAP_ZOOM" ]; then update_env "MAP_ZOOM" "$MAP_ZOOM" @@ -202,7 +272,13 @@ update_env "POTATOMESH_IMAGE_ARCH" "$POTATOMESH_IMAGE_ARCH" update_env "POTATOMESH_IMAGE_TAG" "$POTATOMESH_IMAGE_TAG" update_env "FEDERATION" "$FEDERATION" update_env "PRIVATE" "$PRIVATE" -update_env "CONNECTION" "$CONNECTION" +if [ "$PROTOCOL" = "reticulum" ]; then + # Reticulum has no single endpoint; drop any inherited serial default so the + # ingestor does not log an ignored-CONNECTION warning on every start. + sed -i.bak '/^CONNECTION=.*/d' .env +else + update_env "CONNECTION" "$CONNECTION" +fi if [ -n "$ALLOWED_CHANNELS" ]; then update_env "ALLOWED_CHANNELS" "\"$ALLOWED_CHANNELS\"" else @@ -219,7 +295,7 @@ else sed -i.bak '/^INSTANCE_DOMAIN=.*/d' .env fi -if ! grep -q "^CONNECTION=" .env; then +if [ "$PROTOCOL" != "reticulum" ] && ! grep -q "^CONNECTION=" .env; then echo "CONNECTION=/dev/ttyACM0" >> .env fi @@ -242,11 +318,27 @@ else echo " Map Zoom: Auto-fit" fi echo " Max Distance: ${MAX_DISTANCE}km" -echo " Channel: $CHANNEL" -echo " Frequency: $FREQUENCY" +echo " Protocol: $PROTOCOL" +case "$PROTOCOL" in + meshcore) + echo " Preset: ${MESHCORE_PRESET:-'Not set'}" + echo " Frequency: ${MESHCORE_FREQ:-'Not set'}" + ;; + reticulum) + echo " RNS Config Dir: ${RETICULUM_CONFIG_DIR:-'~/.reticulum'}" + echo " Interfaces: ${RETICULUM_INTERFACES:-'All'}" + ;; + *) + echo " Preset: $MESHTASTIC_PRESET" + echo " Frequency: $MESHTASTIC_FREQ" + ;; +esac +echo " Transmit: ${TX_ENABLED} (announce: ${TX_ANNOUNCE})" echo " Chat: ${CONTACT_LINK:-'Not set'}" echo " Debug Logging: ${DEBUG}" -echo " Connection: ${CONNECTION}" +if [ "$PROTOCOL" != "reticulum" ]; then + echo " Connection: ${CONNECTION}" +fi echo " API Token: ${API_TOKEN:0:8}..." echo " Docker Image Arch: $POTATOMESH_IMAGE_ARCH" echo " Docker Image Tag: $POTATOMESH_IMAGE_TAG" diff --git a/data/Dockerfile b/data/Dockerfile index 846be7d8..5273a97e 100644 --- a/data/Dockerfile +++ b/data/Dockerfile @@ -87,6 +87,12 @@ ENV CONNECTION=/dev/ttyACM0 \ HIDDEN_CHANNELS="" \ TX_ENABLED=0 \ TX_ANNOUNCE=0 \ + ENERGY_SAVING=0 \ + FREQUENCY="" \ + MESHCORE_SELF_TELEMETRY_SECONDS=3600 \ + MESHCORE_TELEMETRY_POLL_SECONDS=300 \ + FEDERATION=1 \ + PRIVATE=0 \ INSTANCE_DOMAIN="" \ API_TOKEN="" @@ -127,6 +133,12 @@ ENV CONNECTION=/dev/ttyACM0 \ HIDDEN_CHANNELS="" \ TX_ENABLED=0 \ TX_ANNOUNCE=0 \ + ENERGY_SAVING=0 \ + FREQUENCY="" \ + MESHCORE_SELF_TELEMETRY_SECONDS=3600 \ + MESHCORE_TELEMETRY_POLL_SECONDS=300 \ + FEDERATION=1 \ + PRIVATE=0 \ INSTANCE_DOMAIN="" \ API_TOKEN="" diff --git a/data/destinations.sql b/data/destinations.sql new file mode 100644 index 00000000..a00e3338 --- /dev/null +++ b/data/destinations.sql @@ -0,0 +1,43 @@ +-- Copyright © 2025-26 l5yth & contributors +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. + + +-- Destinations a node announces on. +-- +-- Reticulum identifies a peer by an identity, and that identity announces on +-- several destinations -- one per aspect (lxmf.delivery, nomadnetwork.node, +-- lxmf.propagation) -- each with its own display name and implied role. Those +-- are distinct things, so each gets its own row here and its own `nodes` row +-- (SPEC RE-A5); `nodes.identity_hash` is what groups them back into one peer. +-- +-- Protocols whose nodes have exactly one destination simply never populate it. + +PRAGMA journal_mode=WAL; + +CREATE TABLE IF NOT EXISTS destinations ( + id TEXT PRIMARY KEY, + node_id TEXT NOT NULL, + identity_hash TEXT, + name TEXT, + aspect TEXT, + role TEXT, + interface TEXT, + first_heard INTEGER, + last_heard INTEGER, + protocol TEXT NOT NULL DEFAULT 'reticulum' +); + +CREATE INDEX IF NOT EXISTS idx_destinations_node_id ON destinations(node_id); +CREATE INDEX IF NOT EXISTS idx_destinations_identity ON destinations(identity_hash); +CREATE INDEX IF NOT EXISTS idx_destinations_last_heard ON destinations(last_heard); diff --git a/data/mesh_ingestor/CONTRACTS.md b/data/mesh_ingestor/CONTRACTS.md index 4a2ebce3..28a326bb 100644 --- a/data/mesh_ingestor/CONTRACTS.md +++ b/data/mesh_ingestor/CONTRACTS.md @@ -25,29 +25,23 @@ Note: non-Meshtastic protocols need a strategy to map their native node identifi The Reticulum provider (`PROTOCOL=reticulum`, `data/mesh_ingestor/protocols/reticulum.py`) maps announces into the canonical id space as follows: -- **Canonical node id** = `!` + the first 4 bytes (8 lowercase hex chars) of the 16-byte Reticulum **identity hash** (`RNS.Identity.hash`), mirroring MeshCore's first-4-bytes-of-pubkey rule. Deterministic and sender-side, so two ingestors hearing the same announce upsert the same row. The identity comes from the announce callback's `announced_identity` (RNS needs it to evaluate `aspect_filter`, so it is always present for a filtered handler), falling back to `RNS.Identity.recall(destination_hash)`. -- **`user.publicKey` is the identity's real public key** (64 bytes / 128 hex), not a destination hash. A destination hash is a truncated hash over the identity hash and the name hash — it is neither a key nor stable across a peer's aspects, so storing it as `publicKey` made the field meaningless for Reticulum rows. -- **`destHash` is a list** of every destination hash known to resolve to that identity, stored in the `nodes.dest_hash` column as a sorted JSON array. The web upsert **unions** stored ∪ incoming rather than overwriting: two ingestors may each have heard a different subset of a peer's aspects. Stored for correlation, never served — `dest_hash` follows `public_key` in appearing in no node API projection. -- `user.shortName` = the first 4 hex chars of the node id (MeshCore convention); `user.longName` = the display name decoded from announce `app_data`, falling back to a placeholder naming the *node* — `"Reticulum "`, the protocol label plus the upper-cased last four hex of the canonical id, which is the exact form the web upsert recognises as a placeholder and therefore refuses to overwrite a real name with. -- **One peer is one node row.** Because the id keys on the identity rather than on a destination, a peer announcing both `lxmf.delivery` and `nomadnetwork.node` merges onto a single row carrying both hashes in `destHash`. (Superseded #890's original mapping, which keyed on the destination hash and therefore split one peer into two unrelated rows — SPEC RN1.) - -**Roles.** Reticulum carries no role field, so what a peer *is* is read from -which destinations it announces: `lxmf.propagation` -> `PROPAGATION`, -`nomadnetwork.node` -> `NODE`, `lxmf.delivery` -> `PEER`. Because one identity's -aspects collapse onto a single row, a node's `user.role` is the -**highest-ranked aspect it has announced on during this ingestor session** -(`PROPAGATION > TRANSPORT > NODE > PEER`), never the most recent -- the web upsert writes -`role=COALESCE(excluded.role, nodes.role)`, so a per-announce mapping would make -a dual-aspect peer's role flip on every announce. The field is omitted entirely -when no aspect has been heard, so a record never asserts a role it could not -determine. +- **Canonical node id** = `!` + the first 4 bytes (8 lowercase hex chars) of the 16-byte **destination hash**, mirroring MeshCore's first-4-bytes-of-pubkey rule. Deterministic and sender-side, so two ingestors hearing the same announce upsert the same row. +- **One row per announced destination.** A Reticulum identity announces on several destinations -- one per aspect (`lxmf.delivery`, `nomadnetwork.node`, `lxmf.propagation`) -- and each carries its **own display name and role**, so each is its own node row. Keying on the identity instead (the previous rule) merged them into a row named from one aspect and roled from another. The identity travels as `identityHash` and is what groups the rows back into one peer; grouping is a presentation concern, not an ingest one. +- **`user.publicKey`** is the announcing identity's real public key (64 bytes / 128 hex), never a destination hash -- a destination hash is a truncated hash over the identity and name hashes, not a key. +- **`destination`** is `{id, aspect, role}` for the destination this announce arrived on. Role is derived from the aspect: `lxmf.delivery` -> `PEER`, `nomadnetwork.node` -> `NODE`, `lxmf.propagation` -> `PROPAGATION`. `TRANSPORT` is reserved and never emitted -- no announce exposes transport status, and inferring it from the ingestor's own path table would make it a property of our vantage point rather than of the node. +- **`interface`** is the interface the announce was heard on, when known. Retrieved through `RNS.Reticulum`'s shared-instance-aware accessors, which RPC to a running `rnsd` and return **its** view; `RNS.Transport.next_hop_interface` reads only the local process's path table and answers `LocalInterface[...]` for everything. +- **`identityHash`** is the announcing identity's 16-byte hash. +- `user.shortName` = the first 4 hex chars of the node id; `user.longName` = the display name decoded from announce `app_data`, falling back to `"Reticulum "` -- the protocol label plus the upper-cased last four hex of the canonical id, which is the exact form the web upsert recognises as a placeholder and therefore refuses to overwrite a real name with. + +**Scope.** `hops == 0` means the announce came from an app on this machine (`Transport.inbound` adds a hop to every inbound packet and takes it back for a local-client or shared-instance interface), so those are always ingested -- the operator's own nodes must never be hidden by a filter. From one hop out, `RETICULUM_INTERFACES` applies as a case-insensitive substring match on the interface name; empty ingests everything. **Known limitation.** The accumulator is in-memory and per-process, and the web upsert writes `role=COALESCE(excluded.role, nodes.role)`, so an ingestor restart -- or a second ingestor that hears only the lower aspect -- can *demote* a stored role. Within one session the rank holds. A rank-aware SQL merge (the treatment -`dest_hash` already gets) is a tracked follow-up; a new protocol should not copy -the in-memory accumulator as though it were durable. +the `destinations` table already gets, via its own forward-only statement +outside the freshness guard) is a tracked follow-up; a new protocol should not +copy the in-memory accumulator as though it were durable. **`TRANSPORT` is never emitted**: no announce exposes transport status, and inferring it from the ingestor's own path table would make it a @@ -58,13 +52,13 @@ disagree -- the sender-side determinism rule above (SPEC RD4). **Config isolation.** `RETICULUM_CONFIG_DIR` defaults to an app-owned directory (`$XDG_CONFIG_HOME/potato-mesh/reticulum`, else `~/.config/potato-mesh/reticulum`) and never to RNS's user default `~/.reticulum`: installing a dashboard ingestor does not imply consent to run the operator's transport-node configuration (SPEC RN3). -**The ingestor's own node id.** Reticulum has no handshake revealing "our" node id, so the ingestor **owns an RNS identity**: generated on first run, stored as `potato_mesh_identity` inside `RETICULUM_CONFIG_DIR`, and mapped to `!xxxxxxxx` by the same identity-hash rule applied to every peer above. `INGESTOR_NODE_ID` is therefore an **override** for this protocol rather than a requirement, and the heartbeat — and so the `reticulum.packets.hour` rate — comes up unaided (SPEC RN9). The identity is local and is never announced, so it adds no transmit site. Persistence is load-bearing: an identity that could not be stored would be regenerated per start and each generation is a different node id, so the provider reports **no** id rather than a churning one, leaving the heartbeat unregistered exactly as it was before RN9. A protocol adding its own self-id should follow the same rule — a derived id must be stable across restarts or it is worse than none. +**The ingestor's own node id.** Reticulum has no handshake revealing "our" node id, but the config dir already holds one: the transport identity RNS keeps there, which `rnstatus` shows as "Transport Instance". The ingestor derives its `!xxxxxxxx` from that, so `INGESTOR_NODE_ID` is an **override** for this protocol rather than a requirement, and the heartbeat — and so the `reticulum.packets.hour` rate — comes up unaided. Nothing is generated or written. A supplied value is canonicalised through the **Reticulum** mapping, not the shared `canonical_node_id`: that one parses hex as an integer and keeps the low 32 bits, which truncates a 16-byte identity hash from the opposite end and produced two different ids for one identity (SPEC RE5). A protocol adding its own self-id should note that a derived id must be stable across restarts or it is worse than none. **`CONNECTION` does not apply.** It names a single serial, TCP, or BLE endpoint; an RNS stack is a set of interfaces with no such endpoint. Its counterparts are disjoint rather than overlapping: `RETICULUM_CONFIG_DIR` selects the stack, `RETICULUM_INTERFACES` selects which of its interfaces to ingest from. A set `CONNECTION` is ignored and logged as ignored, because the shipped container image carries a serial default for every protocol (SPEC RN10). **Transmit policy.** The provider is receive-only and has no transmit site to gate, so `PROTOCOL=reticulum` works with `TX_ENABLED=0` (the default). The underlying RNS stack is not silent at the *interface* layer, though — `AutoInterface` multicasts peer discovery, and `enable_transport` relays other nodes' traffic. That is owned by the Reticulum config, which is why the two paragraphs above exist (SPEC RN5). -**Collision trade-off (accepted).** Truncating to 4 bytes means two distinct 16-byte identity hashes sharing a 4-byte prefix collapse onto one `node_id`. Within a protocol this is the same accepted trade-off MeshCore's pubkey-prefix mapping has always carried: the colliding records merge into one row, and the odds are negligible at mesh scale (~1 in 4 billion per pair). **Across protocols** the shared `nodes.node_id` keyspace makes a prefix collision a hijack risk instead of a merge, so the web **nodeinfo upsert** (`upsert_node`) refuses cross-protocol overwrites: when the stored row already carries a known protocol and an incoming record resolves to a different one, the record is skipped entirely (logged at debug level) rather than allowed to flip the row's protocol or overwrite its fields. The one exception is the established `meshtastic` → `meshcore` self-heal (bug #747): `meshtastic` doubles as the schema/classification default, so a `meshcore` record may still reclaim a default-stamped row. The guard covers the nodeinfo upsert only: position, telemetry, and last-seen touch writes key on the bare `node_id` without a protocol check, so a cross-protocol prefix collision can still attach position or telemetry fields to the row or advance its `last_heard` (extending the guard to those paths is a tracked follow-up). +**Collision trade-off (accepted).** Truncating to 4 bytes means two distinct 16-byte destination hashes sharing a 4-byte prefix collapse onto one `node_id`. Within a protocol this is the same accepted trade-off MeshCore's pubkey-prefix mapping has always carried: the colliding records merge into one row, and the odds are negligible at mesh scale (~1 in 4 billion per pair). **Across protocols** the shared `nodes.node_id` keyspace makes a prefix collision a hijack risk instead of a merge, so the web **nodeinfo upsert** (`upsert_node`) refuses cross-protocol overwrites: when the stored row already carries a known protocol and an incoming record resolves to a different one, the record is skipped entirely (logged at debug level) rather than allowed to flip the row's protocol or overwrite its fields. The one exception is the established `meshtastic` → `meshcore` self-heal (bug #747): `meshtastic` doubles as the schema/classification default, so a `meshcore` record may still reclaim a default-stamped row. The guard covers the nodeinfo upsert only: position, telemetry, and last-seen touch writes key on the bare `node_id` without a protocol check, so a cross-protocol prefix collision can still attach position or telemetry fields to the row or advance its `last_heard` (extending the guard to those paths is a tracked follow-up). **Deployment ordering.** The web whitelist must accept a protocol before any ingestor posts it: if an ingestor ships a protocol the deployed web tier does not yet know, protocol resolution files those records under the `meshtastic` default and the misclassification persists after the web tier is upgraded. Concretely for reticulum: deploy (or merge) the web change before or together with the ingestor change, never after. @@ -94,7 +88,9 @@ remains accepted. Per-field acceptance is nil-aware, so a camelCase value of - `rssi` (int|nil) — per-advert reception RSSI (SPEC RF3). Sourced from MeshCore RX-log adverts; Meshtastic reports no per-node RSSI, so the field stays absent/NULL there. The web upsert keeps the last stored value when an update omits it (`COALESCE`), so contact-roster refreshes never wipe a per-advert reading. - `hopsAway` (int) - `isFavorite` (bool) -- `destHash` (array of hex strings | string | absent) — protocol-native destination hashes that resolve to this node's identity, for protocols whose identities front several destinations (today: Reticulum's announce aspects; see "Reticulum node id mapping" above). A bare string is accepted as a one-element list. The web upsert **unions** it with the stored value rather than overwriting, so an ingestor that heard only one aspect never drops hashes another ingestor recorded. Stored in `nodes.dest_hash` as a sorted JSON array and, like `user.publicKey`, served on no read API. +- `identityHash` (hex string | absent) — the protocol-native identity a destination belongs to, for protocols whose identities front several destinations (today: Reticulum). Stored in `nodes.identity_hash`; it is what groups a peer's rows back together. Not served on the node read APIs, but returned by `GET /api/destinations`. +- `destination` (mapping | absent) — `{id, aspect, role}` for the destination this record's announce arrived on. Written to the `destinations` table and served by `GET /api/destinations`. +- `interface` (string | absent) — the interface the announce was heard on, e.g. `RNodeInterface[RNode Reticulum Berlin]`. - `user` (mapping; e.g. `shortName`, `longName`, `macaddr`, `hwModel`, `publicKey`, `isUnmessagable`) - `role` (optional string) — omit when unknown; known values include Meshtastic role names (e.g. `CLIENT`, `ROUTER`), MeshCore role names (`COMPANION`, `REPEATER`, `ROOM_SERVER`, `SENSOR`), and Reticulum role names (`PEER`, `NODE`, `PROPAGATION`; `TRANSPORT` is reserved and never emitted — see "Roles" above, which also covers how Reticulum derives and ranks it) - `deviceMetrics` (mapping; e.g. `batteryLevel`, `voltage`, `channelUtilization`, `airUtilTx`, `uptimeSeconds`) @@ -371,6 +367,18 @@ bound), and a `before` newer than "now" is a no-op. A non-positive or non-intege is protocol-neutral. The per-id routes (`GET /api/.../:id`) and `GET /api/instances` do **not** accept `before`. +### GET /api/destinations response shape + +One row per announced destination, newest `last_heard` first. + +- Query params: `?limit=` (capped like other collections), `?node_id=` to filter to one node. +- Fields: `id` (destination hash, hex), `node_id`, `identity_hash`, `name`, + `aspect`, `role`, `interface`, `first_heard`, `last_heard`, `protocol`. +- `identity_hash` groups rows belonging to one peer; several rows share it when + an identity announces on several aspects. +- Written only by the node ingest route, from each node record's `destination` + mapping; there is no `POST /api/destinations`. + ### GET /api/stats response shape > **Breaking change in 0.7.0.** Before 0.7.0 the payload was flat — @@ -421,10 +429,9 @@ do **not** accept `before`. add rather than dedup); `reticulum.packets.hour` shipped as an always-zero stub and reports the real rate since the Reticulum ingestor landed. The rate only moves when the reticulum ingestor's heartbeat registers, which needs a - node id; the provider derives one from the identity it owns, so no operator - configuration is required and `INGESTOR_NODE_ID` merely overrides it (SPEC - RN9). It warns at startup in the one case that leaves the rate dead — an - identity that can neither be read nor stored. Unlike + node id; the provider derives one from the config dir's transport identity, so + no operator configuration is required and `INGESTOR_NODE_ID` merely overrides + it (SPEC RE5). Unlike `messages`, it is **not** privacy-gated (packets are a public aggregate, no message content). Additive to the 0.7.x `/api/stats` tree — no version bump; the ingestor dogfeeds it for the activity diff --git a/data/mesh_ingestor/announce.py b/data/mesh_ingestor/announce.py index 892cf931..2799b6d2 100644 --- a/data/mesh_ingestor/announce.py +++ b/data/mesh_ingestor/announce.py @@ -395,20 +395,24 @@ def maybe_run_announcements( if now is None: now = time.monotonic() + # Due first, policy second. An announcement is due at most once per 24 h, + # so checking the policy first logged a suppression line on every 60 s loop + # for something that was never going to be sent — the field log was one + # line a minute of pure noise. The startup "Transmit policy resolved" line + # already states the resolved policy, which is what an operator needs to + # tell "TX_ENABLED unset" from "RX_ONLY still set"; this line now names the + # gate only when it actually blocks a due announcement (SPEC RE-A8). + if not announce_due( + start_monotonic=start_monotonic, last_announce=last_announce, now=now + ): + return last_announce if not tx_policy.announcements_permitted(): - # Name the closed gate: an operator who opted in and hears nothing must - # be able to tell "TX_ENABLED unset" from "RX_ONLY still set" from - # "still inside the 24 h delay" without a day of observation per guess. config._debug_log( "Activity announcement suppressed by transmit policy", context="announce.tx", blocked_by=tx_policy.describe_tx_policy()["blocked_by"], ) return last_announce - if not announce_due( - start_monotonic=start_monotonic, last_announce=last_announce, now=now - ): - return last_announce run_announcement_cycle(provider, iface) return now diff --git a/data/mesh_ingestor/config.py b/data/mesh_ingestor/config.py index 3b24c3a8..bdb0a410 100644 --- a/data/mesh_ingestor/config.py +++ b/data/mesh_ingestor/config.py @@ -187,31 +187,22 @@ def _env_flag(name: str, *, default: bool, on_invalid: bool) -> bool: def _resolve_reticulum_config_dir() -> str: """Resolve the Reticulum config directory for ``PROTOCOL=reticulum``. - The ingestor keeps its **own**, isolated Reticulum configuration rather - than inheriting the operator's ``~/.reticulum``. Adopting that directory - would make the dashboard silently take on whatever transport - configuration the operator runs there — including ``enable_transport`` - and every interface they have defined — which is never implied by - installing an ingestor (#888). + Defaults to RNS's own ``~/.reticulum`` — the stack the operator already + runs — with :envvar:`RETICULUM_CONFIG_DIR` as the override (``~`` expanded). - Resolution order: - - 1. :envvar:`RETICULUM_CONFIG_DIR` when set (``~`` is expanded), so an - operator who *does* want to share a stack can say so explicitly. - 2. ``$XDG_CONFIG_HOME/potato-mesh/reticulum``. - 3. ``~/.config/potato-mesh/reticulum``. + Sharing that directory is the point, not a hazard (SPEC RN3 as amended). + Interface scoping needs to ask the shared instance which interface an + announce arrived on, and that RPC authenticates with a key derived from the + config dir's own identity — so an ingestor with a *private* config dir + attaches to the operator's ``rnsd`` but cannot query it. Returns: - Absolute-ish path string suitable as :class:`RNS.Reticulum`'s - ``configdir``. Never ``None`` and never ``~/.reticulum``. + Path string for :class:`RNS.Reticulum`'s ``configdir``. """ explicit = os.environ.get("RETICULUM_CONFIG_DIR", "").strip() if explicit: return os.path.expanduser(explicit) - base = os.environ.get("XDG_CONFIG_HOME", "").strip() or os.path.join( - os.path.expanduser("~"), ".config" - ) - return os.path.join(base, "potato-mesh", "reticulum") + return os.path.join(os.path.expanduser("~"), ".reticulum") RETICULUM_CONFIG_DIR = _resolve_reticulum_config_dir() diff --git a/data/mesh_ingestor/protocols/reticulum.py b/data/mesh_ingestor/protocols/reticulum.py index f075b82f..746e859a 100644 --- a/data/mesh_ingestor/protocols/reticulum.py +++ b/data/mesh_ingestor/protocols/reticulum.py @@ -25,15 +25,12 @@ this provider is receive-only: it never transmits and has no roster to fetch. **Own node id.** Reticulum has no protocol-level handshake revealing "our" -node id, but it does not need one: the ingestor **owns an RNS identity**, -persisted as ``potato_mesh_identity`` inside -:data:`~data.mesh_ingestor.config.RETICULUM_CONFIG_DIR`, and derives its own -``!xxxxxxxx`` through the same identity-hash mapping applied to every peer -(:func:`_reticulum_node_id`). :envvar:`INGESTOR_NODE_ID` is therefore an -**override**, not a requirement — the heartbeat and the reticulum -packets/hour scope come up on their own (SPEC RN9). The identity is generated -and stored locally and is never announced, so this adds no transmit site -(SPEC MA7/RN5). +node id, but it does not need one: the config dir already holds a transport +identity — what ``rnstatus`` shows as "Transport Instance" — and the ingestor +derives its ``!xxxxxxxx`` from that. :envvar:`INGESTOR_NODE_ID` is therefore an +**override**, not a requirement (SPEC RE5). Nothing is generated or written: +peers are keyed on their *destination* hash (SPEC RE1), so an identity-keyed +ingestor row would be a second, private id rule disagreeing with the first. **Connection variables.** :envvar:`CONNECTION` names a single serial, TCP, or BLE endpoint and has no meaning for RNS, which is a stack of many interfaces @@ -111,52 +108,6 @@ _ANNOUNCE_ASPECTS: tuple[str, ...] = tuple(_ASPECT_ROLES) """Destination aspects whose announces are ingested as node records.""" -_SHARED_INSTANCE_INTERFACE_PREFIXES: tuple[str, ...] = ( - "localinterface[", - "shared instance[", -) -"""Interface names that mean "attached to a shared RNS instance". - -``RNS.Interfaces.LocalInterface`` renders the client side as -``LocalInterface[rns/]`` and the server side as -``Shared Instance[rns/]``. When the ingestor is attached to an external -``rnsd`` every announce arrives over that one socket, whatever interface -actually received it, so a per-interface allowlist cannot discriminate. -""" - -_ROLE_RANK: tuple[str, ...] = ("PEER", "NODE", "TRANSPORT", "PROPAGATION") -"""Roles in ascending precedence order (SPEC RD4). - -``TRANSPORT`` is ranked although :data:`_ASPECT_ROLES` maps nothing to it: the -rank is the reserved slot's other half, so a future source drops in without -re-opening the ordering. Ranking it below ``PROPAGATION`` reflects that a -propagation node is the more specific claim. - -A node's role is the **highest-ranked aspect it has announced on during this -ingestor session**, not the most recent. A fixed rank is load-bearing rather than cosmetic: an identity's -aspects collapse onto one row (SPEC RN1) and the web upsert writes -``role=COALESCE(excluded.role, nodes.role)``, so mapping each announce straight -to its own role would make a dual-aspect peer's role flip on every announce, -oscillating its badge and its legend bucket. -""" - - -def _rank_role(role: str | None) -> int: - """Return a role's precedence, or ``-1`` when it is unranked. - - Parameters: - role: Role name, or ``None``. - - Returns: - Index into :data:`_ROLE_RANK`; ``-1`` for ``None`` or an unknown role, - so anything unranked always loses to a ranked role. - """ - try: - return _ROLE_RANK.index(role) - except ValueError: - return -1 - - _MSGPACK_ARRAY_LEAD_BYTES = frozenset(range(0x90, 0xA0)) | {0xDC, 0xDD} """First-byte values identifying a msgpack-encoded announce ``app_data``. @@ -350,73 +301,38 @@ def _announce_interface_name(dest_hash: object) -> str | None: return None -_allowlist_ignored_warned = False -"""Whether the fail-open warning has already been emitted this session.""" +def _announce_admitted(hops: int | None, interface_name: str | None) -> bool: + """Decide whether an announce is in scope for this ingestor (SPEC RN4). + Two rules, in order. -def _warn_allowlist_ignored_once(interface_name: str) -> None: - """Warn, once, that the interface allowlist cannot be honoured. + **A 0-hop announce is always admitted.** ``RNS.Transport.inbound`` adds a + hop to every inbound packet and takes it back again for a local-client or + shared-instance interface, so zero hops can only mean "announced by an app + on this machine" — the operator's own nodes. Scoping to a radio must never + hide those; filtering purely on interface name did exactly that, because + every local destination legitimately reads ``LocalInterface[rns/default]``. - Emitted on the fail-open path: the ingestor is attached to a shared RNS - instance, so every announce arrives over one socket and - :data:`~data.mesh_ingestor.config.RETICULUM_INTERFACES` cannot discriminate. - Ingesting everything is the safe failure, but it **widens** what the - operator asked for, so it must not be silent — this is the upgrade path for - a deployment whose config dir already exists and therefore was never seeded - (see :func:`_seed_config_dir`). - - Once per process: this sits on the per-announce path. - - Parameters: - interface_name: The shared-instance interface the announce arrived on. - """ - global _allowlist_ignored_warned - if _allowlist_ignored_warned: - return - _allowlist_ignored_warned = True - config._debug_log( - "RETICULUM_INTERFACES cannot be honoured on a shared Reticulum instance; " - "ingesting every announce instead. Set share_instance = No in the " - "ingestor's own config dir to scope it", - context="reticulum.announce", - severity="warn", - interface=interface_name, - allowlist=list(config.RETICULUM_INTERFACES), - ) - - -def _interface_allowed(interface_name: str | None) -> bool: - """Test *interface_name* against :data:`config.RETICULUM_INTERFACES`. - - An empty allowlist — the default — admits every interface, preserving the - behaviour the provider shipped with. When an allowlist *is* configured, - an announce whose interface cannot be determined is rejected: the operator - asked to ingest from named interfaces only, and an unverifiable one is not - among them. - - A shared-instance socket is the one exception and admits everything: see - :data:`_SHARED_INSTANCE_INTERFACE_PREFIXES`. + **Anything further out is scoped by interface.** From one hop the + receiving interface is a real one, so + :data:`~data.mesh_ingestor.config.RETICULUM_INTERFACES` can discriminate. + An empty allowlist — the default — admits everything. Parameters: - interface_name: Interface string form, or ``None`` when unknown. + hops: Hop count for the announce, or ``None`` when unknown. + interface_name: Interface the announce arrived on, or ``None``. Returns: - ``True`` when announces from this interface may be ingested. + ``True`` when the announce may be ingested. """ + if hops == 0: + return True allowlist = config.RETICULUM_INTERFACES if not allowlist: return True if not interface_name: return False lowered = interface_name.lower() - # Fail *open* on a shared-instance socket. Every announce reaches us over - # that one interface, so the allowlist cannot tell them apart; rejecting - # them all would silently ingest nothing, which is the worse failure by far - # (:func:`ReticulumProvider.connect` refuses to share precisely so this - # path is not reached, and warns when it is). - if lowered.startswith(_SHARED_INSTANCE_INTERFACE_PREFIXES): - _warn_allowlist_ignored_once(interface_name) - return True return any(fragment in lowered for fragment in allowlist) @@ -453,259 +369,71 @@ def _announce_to_node_dict( app_data: object, *, identity: object = None, - dest_hashes: "list[str] | tuple[str, ...] | None" = None, - role: str | None = None, + aspect: str | None = None, + interface: str | None = None, hops: int | None = None, last_heard: int | None = None, ) -> dict | None: """Convert a Reticulum announce into a ``POST /api/nodes`` node dict. - The record is keyed on the announcing **identity** (see - :func:`_reticulum_node_id`), so a peer's several destination aspects merge - into one node row. The destination hashes themselves are carried in the - ``destHash`` list — a destination hash is a truncated hash over the - identity and name hashes, not a key, and never belongs in ``publicKey``. + One record per **announced destination** (SPEC RE-A5). Each aspect carries + its own display name and implies its own role, so a record names exactly the + destination it came from. The announcing identity rides along as + ``identityHash`` — the rows for one peer are grouped by it, which is a + separate design pass. Parameters: - dest_hash: 16-byte destination hash (or hex string) the announce - arrived for; used for the display-name fallback and as the sole - ``destHash`` entry when *dest_hashes* is omitted. - app_data: Raw announce application data carrying the display name - (see :func:`_decode_display_name`). - identity: Announcing :class:`RNS.Identity`; its hash keys the node row - and its public key populates ``user.publicKey``. - dest_hashes: Every destination hash known for this identity, so far. - Defaults to just *dest_hash*. - role: Highest-ranked role this identity has announced (SPEC RD4). - Omitted from the payload when falsy, so a record never asserts a - role it could not determine. + dest_hash: 16-byte destination hash the announce arrived for. Keys the + node row. + app_data: Raw announce application data (see + :func:`_decode_display_name`). + identity: Announcing :class:`RNS.Identity`; supplies the real public key + and the identity hash. + aspect: Destination aspect this announce arrived on, e.g. + ``lxmf.delivery``. Maps to the role via :data:`_ASPECT_ROLES`. + interface: Interface the announce was heard on, when known — the honest + answer to "is this a LoRa peer" (SPEC RN4). hops: Hop count travelled by the announce, when known. last_heard: Unix seconds of announce receipt; defaults to now. Returns: - Node dict compatible with the ``POST /api/nodes`` payload format, or - ``None`` when no identity hash is available to key the record. The - canonical node id is *not* embedded: the payload envelope is keyed by - it (``routes/ingest.rb``), and :meth:`_ReticulumInterface.nodes_snapshot` - carries it alongside. + Node dict for the ``POST /api/nodes`` payload, or ``None`` when + *dest_hash* cannot be mapped to a canonical node ID. """ - node_id = _reticulum_node_id(getattr(identity, "hash", None)) + node_id = _reticulum_node_id(dest_hash) if node_id is None: return None hash_hex = _reticulum_hash_hex(dest_hash) - hashes = sorted( - {h for h in (dest_hashes if dest_hashes is not None else [hash_hex]) if h} - ) display_name = _decode_display_name(app_data) - # Name the *node*, not whichever destination happened to announce. The row - # is keyed on the identity (SPEC RN1), so a per-aspect destination-hash - # prefix names the wrong thing entirely — and the web upsert only yields to - # a placeholder it recognises, which is the "