From cf67a5e5ec7b0c553f42ba339c1774f3276977bd Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 15:00:44 -0400 Subject: [PATCH 001/151] fix: remove evicted resolved path hops and preserve relay snapshots (#1966) Eviction removes raw wire hops but leaves resolved full-key entries in `byPathHop`, retaining expired transmissions and stale relay counts/scopes. Filter every hop bucket once per eviction batch using the existing evicted-ID set, remove duplicate references and empty buckets, and clear discarded pointer slots. Bulk relay aggregation now owns its bucket snapshots before releasing the read lock, so eviction and raw-path updates cannot mutate an in-flight reader. Three existing handler test fixtures also wait for index readiness or explicitly simulate not-ready state, preserving their original 200/503 assertions. Fixes #1908. Validation: - Regression commits fail before their corresponding fixes: resolved keys/counts/scopes remain after eviction, and saved relay snapshots change during eviction. - Targeted eviction, relay, scope, cache and concurrent-reader checks pass under `-race`; coverage includes time/cap eviction, missing resolved-path prefetch, disabled membership indexing, duplicate references, retained backing arrays and surviving entries. - Local browser smoke: nodes, node details/path attribution and analytics render using the fixture-backed Go server. - The last full Windows server race run, before the final snapshot-copy correction, had one remaining DB-only timing failure (`TestGetChannelMessagesPerfLargeChannel`: 2.198s against a 1.5s budget). The final correction was checked with focused race tests. The unchanged ingestor suite also cannot create one symlink without Windows privileges. These thresholds/assertions were preserved; full Linux Go/E2E results still require upstream CI approval. Performance tradeoff: cleanup is O(total indexed pointers) per nonempty eviction batch, under the existing write lock. The minute-based ticker pays for one sweep instead of repeated scans of shared raw buckets. No per-transmission string index or dependency is added. Synthetic benchmark medians (three single-iteration runs, shared Windows host): | Transmissions | Evicted | Before | After | |---:|---:|---:|---:| | 30,000 | 1 | 1.07 ms | 14.19 ms | | 30,000 | 3,000 | 56.27 ms | 61.68 ms | | 30,000 | 7,500 | 83.58 ms | 65.54 ms | | 100,000 | 1 | 0.30 ms | 56.95 ms | | 100,000 | 10,000 | 949.93 ms | 190.71 ms | | 100,000 | 25,000 | 1,834.48 ms | 320.38 ms | Fixture: eight raw plus eight resolved hops per transmission, two observations, 2,048 relays; 480,000/1,600,000 hop entries. Timing includes acquiring the store lock and omits unrelated secondary indexes. Small batches now pay for the complete sweep; shared-host timing is noisy. Owning the bulk reader's arrays also has a measured cost on cold/bulk recomputation, rather than cached hits. Snapshot medians from three samples of ten iterations: | Transmissions / relay nodes | Before time / bytes per operation | After time / bytes per operation | |---|---:|---:| | 30,000 / 50 | 0.0068 ms / 5,416 B | 23.65 ms / 4,101,435 B | | 30,000 / 2,000 | 0.1374 ms / 196,768 B | 26.77 ms / 4,274,336 B | | 100,000 / 2,000 | 0.1376 ms / 196,768 B | 27.89 ms / 13,959,337 B | These are total snapshot costs, comparing the unsafe header-only snapshot with owned pointer arrays. Cleanup guarantees here apply to `byPathHop`; other indexes and existing periodic bulk-cache freshness are outside this change. Following #1922, this runtime fix is separate from the release-routing and frontend-runner PRs. Current Go and E2E job results should be assessed separately from workflow-approval or staging-runner state. --- cmd/server/coverage_test.go | 14 +- cmd/server/index_ready_1008_test.go | 8 +- cmd/server/pathhop_eviction_1908_test.go | 185 ++++++++++++++++++++++ cmd/server/repeater_enrich_bulk.go | 30 ++-- cmd/server/repeater_snapshot_1908_test.go | 128 +++++++++++++++ cmd/server/store.go | 31 ++-- 6 files changed, 372 insertions(+), 24 deletions(-) create mode 100644 cmd/server/pathhop_eviction_1908_test.go create mode 100644 cmd/server/repeater_snapshot_1908_test.go diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index de4ba76d6..5e790dbfb 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -2705,7 +2705,12 @@ func TestHandleAnalyticsSubpathsWithStore(t *testing.T) { hub := NewHub() srv := NewServer(db, cfg, hub) store := NewPacketStore(db, nil) - store.Load() + if err := store.Load(); err != nil { + t.Fatalf("store.Load failed: %v", err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("indexes not ready after 5s") + } srv.store = store router := mux.NewRouter() srv.RegisterRoutes(router) @@ -2725,7 +2730,12 @@ func TestHandleAnalyticsSubpathDetailWithStore(t *testing.T) { hub := NewHub() srv := NewServer(db, cfg, hub) store := NewPacketStore(db, nil) - store.Load() + if err := store.Load(); err != nil { + t.Fatalf("store.Load failed: %v", err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("indexes not ready after 5s") + } srv.store = store router := mux.NewRouter() srv.RegisterRoutes(router) diff --git a/cmd/server/index_ready_1008_test.go b/cmd/server/index_ready_1008_test.go index f7c6340f8..96f520c62 100644 --- a/cmd/server/index_ready_1008_test.go +++ b/cmd/server/index_ready_1008_test.go @@ -58,8 +58,12 @@ func TestIssue1008_HandlerReturns503WhileSubpathIndexLoading(t *testing.T) { if err := store.Load(); err != nil { t.Fatalf("Load() error: %v", err) } - // Don't wait for the background build — we want to observe the - // not-ready window. + // Finish the background build before simulating not-ready, so it cannot + // flip the flag back to true before the handler checks it. + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("background builds never finished") + } + store.subpathReady.Store(false) cfg := &Config{} cfg.applyListLimitsDefaults() srv := &Server{store: store, cfg: cfg} diff --git a/cmd/server/pathhop_eviction_1908_test.go b/cmd/server/pathhop_eviction_1908_test.go new file mode 100644 index 000000000..f4b81264b --- /dev/null +++ b/cmd/server/pathhop_eviction_1908_test.go @@ -0,0 +1,185 @@ +package main + +import ( + "fmt" + "reflect" + "strings" + "testing" + "time" +) + +func TestPathHopEviction1908(t *testing.T) { + for _, mode := range []string{"time-direct", "time-run", "memory-empty-batch", "memory-index-disabled"} { + t.Run(mode, func(t *testing.T) { + now := time.Now().UTC() + store := makeTestStore(8, now.Add(-30*time.Minute), 0) + store.byPayloadType[5] = store.byPayloadType[4] + delete(store.byPayloadType, 4) + store.byPathHop = make(map[string][]*StoreTx) + store.useResolvedPathIndex = mode != "memory-index-disabled" + store.initResolvedPathIndex() + store.retentionHours = 1 + if strings.HasPrefix(mode, "memory-") { + store.maxMemoryMB = 1 + store.trackedBytes = 8 * 1048576 // The 25% cap evicts the first two. + } else { + for _, tx := range store.packets[:2] { + tx.FirstSeen = now.Add(-2 * time.Hour).Format(time.RFC3339) + } + } + + shared := strings.Repeat("a", 64) + expiredOnly := strings.Repeat("b", 64) + retainedOnly := strings.Repeat("c", 64) + expiredScope, retainedScope := "expired-scope", "retained-scope" + hopsSeen := make(map[string]bool) + for i, tx := range store.packets { + *tx.PayloadType = 5 // Non-advert traffic contributes to relay stats. + pk := retainedOnly + tx.ScopeName = &retainedScope + if i < 2 { + pk = expiredOnly + tx.ScopeName = &expiredScope + } + addTxToPathHopIndex(store.byPathHop, tx) + store.indexResolvedPathHops(tx, []string{shared, pk}, hopsSeen) + } + // Repeated observations can append the same resolved association again. + store.indexResolvedPathHops(store.packets[0], []string{shared, expiredOnly}, hopsSeen) + store.indexResolvedPathHops(store.packets[7], []string{shared}, hopsSeen) + addTxToPathHopIndex(store.byPathHop, store.packets[0]) + + // Keep the original backing arrays observable: a shortened slice must + // not hide pointers that keep evicted transmissions alive for the GC. + backingArrays := make(map[string][]*StoreTx, len(store.byPathHop)) + for key, list := range store.byPathHop { + backingArrays[key] = list[:cap(list)] + } + before := store.GetRepeaterNodeStatsBatchCached([]string{shared}, 24)[shared].Info + if before.RelayCount24h != 8 || len(before.TransportedScopes) != 2 { + t.Fatalf("fixture must index all transmissions and scopes: %+v", before) + } + + var evicted int + if mode == "time-run" { + evicted = store.RunEviction() // No DB: cleanup cannot depend on SQL. + } else { + store.mu.Lock() + if strings.HasPrefix(mode, "memory-") { + evicted = store.EvictStaleWithRP(map[int][]string{}) + } else { + evicted = store.EvictStale() + } + store.mu.Unlock() + } + if evicted != 2 { + t.Fatalf("evicted %d transmissions, want 2", evicted) + } + + expected := map[string][]int{ + "aa": {3, 4, 5, 6, 7, 8}, + "bb": {3, 4, 5, 6, 7, 8}, + "cc": {3, 4, 5, 6, 7, 8}, + shared: {3, 4, 5, 6, 7, 8, 8}, + retainedOnly: {3, 4, 5, 6, 7, 8}, + } + if len(store.byPathHop) != len(expected) { + t.Errorf("path-hop index has %d keys, want %d (expired-only keys must disappear)", len(store.byPathHop), len(expected)) + } + for key, want := range expected { + var ids []int + for _, tx := range store.byPathHop[key] { + ids = append(ids, tx.ID) + } + if !reflect.DeepEqual(ids, want) { + t.Errorf("path-hop bucket %q = %v, want %v", key, ids, want) + } + } + for key, list := range backingArrays { + for _, tx := range list { + if tx != nil && tx.ID <= 2 { + t.Errorf("backing array for %q still retains evicted tx %d", key, tx.ID) + break + } + } + } + if store.relayStatsCache != nil { + t.Error("eviction must invalidate the relay-stats cache") + } + after := store.GetRepeaterNodeStatsBatchCached([]string{shared}, 24)[shared].Info + if after.RelayCount24h != 6 { + t.Errorf("relay count after eviction = %d, want 6", after.RelayCount24h) + } + if !reflect.DeepEqual(after.TransportedScopes, []string{retainedScope}) { + t.Errorf("transported scopes after eviction = %v, want only retained scope", after.TransportedScopes) + } + + // A later sweep that removes every survivor must also remove every key. + for _, tx := range store.packets { + tx.FirstSeen = now.Add(-2 * time.Hour).Format(time.RFC3339) + } + if got := store.RunEviction(); got != 6 { + t.Fatalf("final eviction removed %d transmissions, want 6", got) + } + if len(store.byPathHop) != 0 { + t.Errorf("empty store retains %d path-hop keys", len(store.byPathHop)) + } + }) + } +} + +// Benchmark the eviction path with 8 raw and 8 resolved hops per transmission, +// 2 observations each, and 2,048 resolved relays sharing 256 raw-prefix buckets. +// Other secondary indexes are omitted to isolate path-hop cleanup cost. Setup +// is outside the timer; the timed operation includes acquiring the store lock. +func BenchmarkPathHopEviction1908(b *testing.B) { + const hops = 8 + pubkeys := make([]string, 2048) + for i := range pubkeys { + pubkeys[i] = fmt.Sprintf("%02x%062x", i%256, i) + } + for _, count := range []int{30000, 100000} { + for _, evictCount := range []int{1, count / 10, count / 4} { + b.Run(fmt.Sprintf("tx=%d/evict=%d", count, evictCount), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + b.StopTimer() + now := time.Now().UTC() + old := now.Add(-2 * time.Hour).Format(time.RFC3339) + recent := now.Add(-30 * time.Minute).Format(time.RFC3339) + store := &PacketStore{ + packets: make([]*StoreTx, count), + byPathHop: make(map[string][]*StoreTx, len(pubkeys)+256), + spIndex: make(map[string]int), + retentionHours: 1, + } + hopsSeen := make(map[string]bool, hops*2) + for j := range store.packets { + tx := &StoreTx{ID: j + 1, FirstSeen: recent, parsedPath: make([]string, hops), pathParsed: true} + if j < evictCount { + tx.FirstSeen = old + } + tx.Observations = []*StoreObs{{ID: j * 2}, {ID: j*2 + 1}} + resolved := make([]string, hops) + for k := range resolved { + resolved[k] = pubkeys[(j*17+k*13)%len(pubkeys)] + tx.parsedPath[k] = resolved[k][:2] + } + store.packets[j] = tx + addTxToPathHopIndex(store.byPathHop, tx) + store.addResolvedPubkeysToPathHopIndex(tx, resolved, hopsSeen) + } + b.StartTimer() + store.mu.Lock() + got := store.EvictStale() + store.mu.Unlock() + b.StopTimer() + if got != evictCount { + b.Fatalf("evicted %d transmissions, want %d", got, evictCount) + } + } + b.ReportMetric(float64(count*hops*2), "entries/batch") + }) + } + } +} diff --git a/cmd/server/repeater_enrich_bulk.go b/cmd/server/repeater_enrich_bulk.go index b3ed7f845..b26c56699 100644 --- a/cmd/server/repeater_enrich_bulk.go +++ b/cmd/server/repeater_enrich_bulk.go @@ -1,6 +1,7 @@ package main import ( + "slices" "strings" "time" ) @@ -52,25 +53,32 @@ func (s *PacketStore) GetRepeaterRelayInfoMap(windowHours float64) map[string]Re return cached } +// snapshotPathHopIndexLocked copies hop buckets for reads after releasing s.mu. +// Eviction and raw-path updates compact their backing arrays in place, so a +// header copy would allow those writers to change an in-flight snapshot. +// The caller must hold s.mu for reading. Cost: O(total path-hop entries). +func (s *PacketStore) snapshotPathHopIndexLocked() map[string][]*StoreTx { + snap := make(map[string][]*StoreTx, len(s.byPathHop)) + for k, list := range s.byPathHop { + snap[k] = slices.Clone(list) + } + return snap +} + // computeRepeaterRelayInfoMap walks byPathHop once under a single RLock, // pre-parses every FirstSeen timestamp once (not once-per-pubkey-bucket), // and emits one RepeaterRelayInfo per hop key. // // Time-complexity invariant: O(unique-tx-in-byPathHop + total-key-bucket -// entries). Memory: one map entry per byPathHop key. Both are bounded by -// the same eviction policy that bounds byPathHop itself. +// entries). Snapshot memory: one pointer per bucket entry, plus a map entry +// per key; both are bounded by the same eviction policy as byPathHop itself. func (s *PacketStore) computeRepeaterRelayInfoMap(windowHours float64) map[string]RepeaterRelayInfo { s.mu.RLock() - // Snapshot the slices (header copy) so we can release the lock before - // the expensive parse pass. Slice headers point at the live underlying - // arrays but those are append-only-by-id; the worst-case race here is - // that ingest grows a slice we already snapshotted (we miss the new - // tail), which is acceptable for a 15s-TTL status read. - snap := make(map[string][]*StoreTx, len(s.byPathHop)) - for k, list := range s.byPathHop { - snap[k] = list - } + // Own the bucket arrays before releasing the lock for aggregation. + // The result represents this snapshot even if eviction removes its txs + // while aggregation runs; the next recomputer pass picks up that change. + snap := s.snapshotPathHopIndexLocked() // Build a tx-id-keyed pre-parsed cache so the inner loop doesn't // re-parse the same FirstSeen N times when the same tx is indexed diff --git a/cmd/server/repeater_snapshot_1908_test.go b/cmd/server/repeater_snapshot_1908_test.go new file mode 100644 index 000000000..4c1ec78ec --- /dev/null +++ b/cmd/server/repeater_snapshot_1908_test.go @@ -0,0 +1,128 @@ +package main + +import ( + "fmt" + "testing" + "time" +) + +// Synthetic traffic: eight resolved relays and their raw prefixes per tx. +func relaySnapshotStore1908(count, nodes, expired int) *PacketStore { + pubkeys := make([]string, nodes) + for i := range pubkeys { + pubkeys[i] = fmt.Sprintf("%02x%062x", i%256, i) + } + now := time.Now().UTC() + old := now.Add(-2 * time.Hour).Format(time.RFC3339) + recent := now.Add(-30 * time.Minute).Format(time.RFC3339) + store := &PacketStore{ + packets: make([]*StoreTx, count), + byPathHop: make(map[string][]*StoreTx, nodes+256), + spIndex: make(map[string]int), + retentionHours: 1, + } + pt := 5 + hopsSeen := make(map[string]bool, 16) + for i := range store.packets { + tx := &StoreTx{ID: i + 1, FirstSeen: recent, PayloadType: &pt, parsedPath: make([]string, 8), pathParsed: true} + if i < expired { + tx.FirstSeen = old + } + resolved := make([]string, 8) + for hop := range resolved { + resolved[hop] = pubkeys[(i*17+hop*13)%nodes] + tx.parsedPath[hop] = resolved[hop][:2] + } + store.packets[i] = tx + addTxToPathHopIndex(store.byPathHop, tx) + store.addResolvedPubkeysToPathHopIndex(tx, resolved, hopsSeen) + } + return store +} + +func TestRelaySnapshotSurvivesEviction1908(t *testing.T) { + store := relaySnapshotStore1908(4, 8, 2) + store.mu.RLock() + snapshot := store.snapshotPathHopIndexLocked() + store.mu.RUnlock() + if len(snapshot) != 16 { + t.Fatalf("snapshot has %d buckets, want eight raw and eight resolved", len(snapshot)) + } + if got := store.RunEviction(); got != 2 { + t.Fatalf("evicted %d transmissions, want 2", got) + } + for key, list := range snapshot { + if len(list) != 4 { + t.Fatalf("snapshot bucket %q has %d transmissions, want 4", key, len(list)) + } + for i, tx := range list { + if tx == nil || tx.ID != i+1 { + t.Errorf("eviction changed snapshot bucket %q at slot %d: got %v, want tx %d", key, i, tx, i+1) + break + } + } + } +} + +func TestRelayComputeConcurrentEviction1908(t *testing.T) { + store := relaySnapshotStore1908(30000, 256, 1) + key := fmt.Sprintf("%064x", 0) + want := len(store.byPathHop[key]) + const readers = 2 + start := make(chan struct{}) + result := make(chan map[string]RepeaterRelayInfo, readers) + for i := 0; i < readers; i++ { + go func() { + <-start + result <- store.computeRepeaterRelayInfoMap(24) + }() + } + evicted := make(chan int, 1) + go func() { + <-start + evicted <- store.RunEviction() + }() + close(start) + + // Either lock order is valid, including on a single CPU. The separate + // snapshot-preservation test deterministically checks bucket ownership; + // this exercises real concurrent callers without assuming a scheduler order. + deadline := time.After(30 * time.Second) + select { + case got := <-evicted: + if got != 1 { + t.Fatalf("evicted %d transmissions, want 1", got) + } + case <-deadline: + t.Fatal("eviction did not finish") + } + for i := 0; i < readers; i++ { + select { + case got := <-result: + if count := got[key].RelayCount24h; count != want && count != want-1 { + t.Errorf("relay snapshot count = %d, want before-eviction %d or after-eviction %d", count, want, want-1) + } + case <-deadline: + t.Fatal("relay computation did not finish") + } + } +} + +func BenchmarkRelaySnapshot1908(b *testing.B) { + for _, size := range []struct{ tx, nodes int }{{30000, 50}, {30000, 2000}, {100000, 2000}} { + store := relaySnapshotStore1908(size.tx, size.nodes, 0) + b.Run(fmt.Sprintf("tx=%d/nodes=%d", size.tx, size.nodes), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + store.mu.RLock() + snapshot := store.snapshotPathHopIndexLocked() + store.mu.RUnlock() + if len(snapshot) != len(store.byPathHop) { + b.Fatal("snapshot lost a hop bucket") + } + } + b.ReportMetric(float64(size.tx*16), "entries/snapshot") + }) + } +} diff --git a/cmd/server/store.go b/cmd/server/store.go index 1a29c2636..959299f72 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -8,6 +8,7 @@ import ( "log" "math" "runtime" + "slices" "sort" "strconv" "strings" @@ -3279,7 +3280,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] tx.parsedPath, tx.pathParsed = saved, savedFlag } // Remove old path-hop index entries using old hops. - // Resolved pubkey entries are managed via resolvedPubkeyIndex, not byPathHop. + // Resolved pubkeys remain indexed independently of the best raw path. if len(oldHops) > 0 { saved, savedFlag := tx.parsedPath, tx.pathParsed tx.parsedPath, tx.pathParsed = oldHops, true @@ -4114,11 +4115,9 @@ func (s *PacketStore) buildPathHopIndex() { // resolved relay attribution, leaving relay counts and transported scopes // empty after each cold load until live ingestion refilled them (#1904). // -// Only transmissions still in s.packets are carried over. This matters: -// eviction's removeTxFromPathHopIndex strips raw hops only (it derives them -// from txGetParsedPath), so evicted transmissions linger in prev under their -// resolved keys. Filtering them here is what keeps the index bounded by the -// eviction policy instead of turning that gap into a permanent leak. +// Only transmissions still in s.packets are carried over. Eviction also +// prunes raw and resolved keys (#1908); this membership check prevents a +// rebuild from reintroducing stale entries from the previous index. // // Cost is O(entries in prev) with one reused scratch map, and it runs only // where buildPathHopIndex already runs — cold load and background-fill @@ -4264,7 +4263,7 @@ func relayMetrics(times []int64, now int64) (count1h, count24h int, lastRelayed } // removeTxFromPathHopIndex removes a transmission from all its raw path-hop index entries. -// Resolved pubkey entries are cleaned up via removeFromResolvedPubkeyIndex. +// Used when the best raw path changes; eviction filters all keys in one batch. func removeTxFromPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) { hops := txGetParsedPath(tx) if len(hops) == 0 { @@ -4814,8 +4813,22 @@ func (s *PacketStore) evictStaleInternal(rpBatch map[int][]string) int { // Remove from subpath index removeTxFromSubpathIndexFull(s.spIndex, s.spTxIndex, tx) - // Remove from path-hop index - removeTxFromPathHopIndex(s.byPathHop, tx) + } + // Sweep raw AND resolved hop keys once per batch (#1908). The hash-only + // membership index cannot recover full pubkey strings. Reuse the evicted + // ID set for O(total path-hop entries) work, with no per-tx string storage + // or repeated scans of shared buckets. DeleteFunc clears discarded pointer + // slots so backing arrays cannot keep evicted transmissions alive. + for key, list := range s.byPathHop { + filtered := slices.DeleteFunc(list, func(tx *StoreTx) bool { + _, evicted := evictedTxIDs[tx.ID] + return evicted + }) + if len(filtered) == 0 { + delete(s.byPathHop, key) + } else { + s.byPathHop[key] = filtered + } } s.invalidateRelayStatsCache() From eb1d733998ac9c2f3254b7a309a67593c78d66d9 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 15:02:14 -0400 Subject: [PATCH 002/151] fix(analytics): preserve selected hash size in links (#1967) Red commit: `5a5ecb6` (local browser: 14 passed, 8 behavior assertion failures before the fix). Hash Issues links now restore `bytes=1|2|3` for the selected control and its matrix/collision data. Missing or malformed values default to one byte. Selector clicks, section/top links, tab-bar changes, filters and theme refreshes retain the chosen view through the existing URL helper. Fixes #1914. - E2E assertion added: `test-issue-1306-collisions-terminology-e2e.js:242`. The existing CI-selected harness passes 23 checks, including distinct nonempty collision rows for each byte size. Its original assertions remain. - Browser verified: `http://127.0.0.1:55634` with the local fixture API, plus reviewed matrix/risk screenshots. Region refresh passed; area coverage skips because the fixture has no areas. - Required frontend checks pass: 99 filter, 18 aging, 666 helpers; URL helpers pass 18. Three independent reviews found no blocking issues; their coverage suggestion is included in `fb482fe`. - Added work parses URL state and updates six links. Rendering and bulk requests are reused; no backend, configuration, dependency or CI-list changes. - A broader smoke run timed out at Live autocomplete (#1110); full-suite success is not established. ## Preflight overrides - The external `run-all.sh` is absent. Corresponding scope, PII, syntax, whitespace and CSS checks passed; no SQL, migration or image changes require those gates. - Red browser evidence is local. Upstream CI execution remains a separate approval gate, as discussed in #1922. --- public/analytics.js | 26 ++- test-issue-1306-collisions-terminology-e2e.js | 164 +++++++++++++++++- 2 files changed, 179 insertions(+), 11 deletions(-) diff --git a/public/analytics.js b/public/analytics.js index eb162e494..538dde856 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -1506,18 +1506,18 @@ async function renderCollisionTab(el, data, collisionData) { el.innerHTML = `

Collisions actually observed in packet traffic — among repeaters grouped by their configured hash size. For theoretical address conflicts that would occur if all repeaters used a given hash size, see the Prefix Tool tab.

-

Inconsistent Hash Sizes

↑ top
+

Inconsistent Hash Sizes

↑ top

Repeaters and room servers sending adverts with varying hash sizes in the last 7 days. Originally caused by a firmware bug where automatic adverts ignored the configured multibyte path setting, fixed in repeater v1.14.1. Companion nodes are excluded.

Loading…
@@ -1525,7 +1525,7 @@

Hash Usage Matrix

- ↑ top + ↑ top
@@ -1539,7 +1539,7 @@
-

Collision Risk

↑ top
+

Collision Risk

↑ top
Loading…
`; @@ -1576,9 +1576,15 @@ // Repeaters and routing nodes no longer needed — collision data is server-computed - let currentBytes = 1; function refreshHashViews(bytes) { - currentBytes = bytes; + // #1914: keep both the bookmark and section links on this byte size. + if (window.URLState) { + const newHash = URLState.updateHashParams({ bytes }, location.hash); + if (newHash !== location.hash) history.replaceState(null, '', newHash); + el.querySelectorAll('[data-hash-section]').forEach(link => { + link.href = URLState.updateHashParams({ section: link.dataset.hashSection }, newHash); + }); + } hideMatrixTip(); // Update selector button states document.querySelectorAll('.hash-byte-btn').forEach(b => { @@ -1607,7 +1613,9 @@ btn.addEventListener('click', () => refreshHashViews(Number(btn.dataset.bytes))); }); - refreshHashViews(1); + // Read on every render so tab, filter and theme refreshes retain the view. + const urlBytes = new URLSearchParams(location.hash.split('?')[1] || '').get('bytes'); + refreshHashViews(['1', '2', '3'].includes(urlBytes) ? Number(urlBytes) : 1); } function renderHashTimeline(hourly) { diff --git a/test-issue-1306-collisions-terminology-e2e.js b/test-issue-1306-collisions-terminology-e2e.js index 1905b25f3..688ce2aca 100644 --- a/test-issue-1306-collisions-terminology-e2e.js +++ b/test-issue-1306-collisions-terminology-e2e.js @@ -15,6 +15,8 @@ * 3. The Collisions (Hash Issues) tab body contains the reverse * cross-reference link back to `#/analytics?tab=prefix-tool` * framed around "actually observed" / packet traffic wording. + * 4. #1914: shared Hash Issues links restore the selected byte size and + * its server data; selection survives navigation and refreshes. * * Usage: BASE_URL=http://localhost:13581 node test-issue-1306-collisions-terminology-e2e.js */ @@ -32,6 +34,7 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } async function openPrefixTool(page) { await page.goto(BASE + '/#/analytics?tab=prefix-tool', { waitUntil: 'domcontentloaded' }); + await page.waitForFunction(() => window.__collisionsThemeReady); await page.waitForSelector('#ptOverview', { timeout: 15000 }); // expand the overview body (collapsed by default) await page.evaluate(() => { @@ -40,9 +43,45 @@ async function openPrefixTool(page) { }); } -async function openCollisions(page) { - await page.goto(BASE + '/#/analytics?tab=collisions', { waitUntil: 'domcontentloaded' }); +async function openCollisions(page, query = '') { + await page.goto('about:blank'); + await page.goto(BASE + '/#/analytics?tab=collisions' + query, { waitUntil: 'domcontentloaded' }); await page.waitForSelector('#hashMatrixSection', { timeout: 15000 }); + await page.waitForFunction(() => document.querySelector('#hashMatrix .analytics-stat-value')); +} + +function hashParams(page) { + return new URLSearchParams(new URL(page.url()).hash.split('?')[1] || ''); +} + +async function refreshCollisions(page, action) { + const previous = await page.locator('#hashMatrix').elementHandle(); + await action(); + await page.waitForFunction(el => !el.isConnected, previous); + await previous.dispose(); +} + +async function assertHashView(page, bytes, collisionData) { + await page.waitForFunction(() => window.__collisionsThemeReady); + await page.waitForFunction(() => document.querySelector('#hashMatrix .analytics-stat-value')); + const selected = page.locator('#hashByteSelector .hash-byte-btn.active'); + assert(await selected.count() === 1, 'Exactly one byte-size button should be selected'); + const actual = await selected.getAttribute('data-bytes'); + assert(actual === String(bytes), `Expected ${bytes}-byte selection, got ${actual}`); + const riskTitle = await page.locator('#collisionRiskTitle').textContent(); + assert(riskTitle.includes(bytes + '-Byte Collision Risk'), 'Risk title does not match selected byte size'); + const usingCard = page.locator('#hashMatrix .analytics-stat-card').nth(1); + assert((await usingCard.textContent()).includes('Using ' + bytes + '-byte ID'), 'Matrix uses the wrong byte-size data'); + const expected = collisionData.by_size[String(bytes)]; + assert(!!expected, 'Fixture must include ' + bytes + '-byte collision data'); + const usingCount = Number((await usingCard.locator('.analytics-stat-value').textContent()).replace(/,/g, '')); + assert(usingCount === (expected.stats.using_this_size || 0), 'Matrix count differs from the selected server data'); + assert(await page.locator('#hashMatrix .hash-matrix-table').count() === (bytes === 3 ? 0 : 1), 'Expected a grid only for 1-byte and 2-byte views'); + const prefixes = await page.locator('#collisionList tbody tr > td:first-child').allTextContents(); + assert(JSON.stringify(prefixes) === JSON.stringify((expected.collisions || []).map(c => c.prefix)), 'Collision risk rows differ from the selected server data'); + if (!prefixes.length) { + assert((await page.locator('#collisionList').textContent()).includes('No ' + bytes + '-byte'), 'Empty collision result should name the selected byte size'); + } } (async () => { @@ -64,6 +103,11 @@ async function openCollisions(page) { } const ctx = await browser.newContext({ viewport: { width: 1400, height: 1200 } }); + // Initial theme loading rebuilds analytics once; wait for it before + // interacting with an expanded panel or asserting the restored view. + await ctx.addInitScript(() => { + window.addEventListener('theme-refresh', () => { window.__collisionsThemeReady = true; }, { once: true }); + }); const page = await ctx.newPage(); page.setDefaultTimeout(15000); page.on('pageerror', (e) => console.error('[pageerror]', e.message)); @@ -108,6 +152,122 @@ async function openCollisions(page) { assert(xref >= 1, 'Collisions tab missing cross-reference link to #/analytics?tab=prefix-tool'); }); + const collisionResponse = await page.request.get(BASE + '/api/analytics/hash-collisions'); + assert(collisionResponse.ok(), 'Local collision API must respond successfully'); + const collisionData = await collisionResponse.json(); + + for (const section of ['hashMatrixSection', 'collisionRiskSection']) { + for (const bytes of [1, 2, 3]) { + await step(`#1914: ${bytes}-byte ${section} deep link restores selection and data`, async () => { + await openCollisions(page, '§ion=' + section + '&bytes=' + bytes); + await assertHashView(page, bytes, collisionData); + assert(hashParams(page).get('section') === section, 'Deep link must retain the target section'); + }); + } + } + + await step('#1914: byte clicks update a reloadable URL without losing other parameters', async () => { + await openCollisions(page, '§ion=collisionRiskSection&window=7d&filter=kept'); + for (const bytes of [2, 3, 1]) { + await page.locator(`.hash-byte-btn[data-bytes="${bytes}"]`).click(); + const params = hashParams(page); + assert(params.get('bytes') === String(bytes), `Byte click should write bytes=${bytes}, got ${params.get('bytes')}`); + assert(params.get('section') === 'collisionRiskSection' && params.get('window') === '7d' && params.get('filter') === 'kept', 'Byte click must preserve other parameters'); + await page.reload({ waitUntil: 'domcontentloaded' }); + await assertHashView(page, bytes, collisionData); + } + }); + + await step('#1914: section and top links preserve byte size and other parameters', async () => { + await openCollisions(page, '&bytes=3&window=7d&filter=kept'); + for (const section of ['hashMatrixSection', 'collisionRiskSection', 'inconsistentHashSection']) { + const link = page.locator(`#hashIssuesToc a[href*="section=${section}"]`); + const params = new URLSearchParams((await link.getAttribute('href')).split('?')[1]); + assert(params.get('bytes') === '3' && params.get('window') === '7d' && params.get('filter') === 'kept', 'Section href must carry the selected view for copying or opening'); + await link.click(); + await assertHashView(page, 3, collisionData); + assert(hashParams(page).get('section') === section, 'Section click must update its URL target'); + } + await page.locator('#collisionRiskSection a', { hasText: 'top' }).click(); + await assertHashView(page, 3, collisionData); + assert(!hashParams(page).has('section') && hashParams(page).get('bytes') === '3', 'Top link should clear only the section'); + }); + + await step('#1914: time-window, tab and theme refreshes retain the chosen byte size', async () => { + await openCollisions(page); + await page.locator('.hash-byte-btn[data-bytes="2"]').click(); + await refreshCollisions(page, () => page.locator('#analyticsTimeWindow').selectOption('24h')); + await page.waitForFunction(() => new URLSearchParams(location.hash.split('?')[1]).get('window') === '24h'); + await assertHashView(page, 2, collisionData); + await page.locator('#analyticsTabs [data-tab="hashsizes"]').click(); + await page.locator('#analyticsTabs [data-tab="collisions"]').click(); + await assertHashView(page, 2, collisionData); + await page.evaluate(() => window.dispatchEvent(new Event('theme-refresh'))); + await assertHashView(page, 2, collisionData); + assert(hashParams(page).get('bytes') === '2' && hashParams(page).get('window') === '24h', 'Refreshes must preserve the view URL'); + }); + + for (const value of [null, '', '0', '4', '-1', '2.5', '2bytes', '02', 'NaN']) { + await step(`#1914: ${value === null ? 'missing' : JSON.stringify(value)} byte size safely defaults to 1`, async () => { + await openCollisions(page, value === null ? '' : '&bytes=' + encodeURIComponent(value)); + await assertHashView(page, 1, collisionData); + }); + } + + for (const filter of ['region', 'area']) { + await openCollisions(page, '&bytes=3'); + const container = page.locator(filter === 'region' ? '#analyticsRegionFilter' : '#analyticsAreaFilter'); + const option = container.locator(`[data-${filter}]:not([data-${filter}="__all__"])`).first(); + const config = await (await page.request.get(BASE + '/api/config/' + filter + 's')).json(); + if (filter === 'region' ? Object.keys(config).length < 2 : config.length === 0) { + console.log(` #1914 ${filter} refresh: SKIP (local fixture has no selectable ${filter} filter)`); + continue; + } + await step(`#1914: ${filter} filter refresh retains byte size and selects its server data`, async () => { + await option.waitFor({ state: 'attached' }); + const selected = await option.getAttribute('data-' + filter); + const dropdown = container.locator('.region-dropdown-trigger'); + if (await dropdown.count()) await dropdown.click(); + await refreshCollisions(page, () => option.click()); + const response = await page.request.get(BASE + '/api/analytics/hash-collisions?' + filter + '=' + encodeURIComponent(selected)); + assert(response.ok(), 'Filtered collision API must respond successfully'); + await assertHashView(page, 3, await response.json()); + assert(hashParams(page).get('bytes') === '3', 'Filter refresh must preserve the byte-size URL'); + if (await dropdown.count()) await dropdown.click(); + await refreshCollisions(page, () => container.locator(`[data-${filter}="__all__"]`).click()); + await assertHashView(page, 3, collisionData); + }); + } + + await step('#1914: nonempty risk rows follow deep-linked and clicked byte sizes', async () => { + const prefixesBySize = { 1: ['A1'], 2: ['B2C3', 'B2D4'], 3: ['D4E5F6'] }; + const routePattern = '**/api/analytics/hash-collisions*'; + await page.route(routePattern, async route => { + const response = await route.fetch(); + const json = await response.json(); + // Keep the real matrix data, but distinguish each risk list even + // when the local fixture has no observed collisions. + for (const [bytes, prefixes] of Object.entries(prefixesBySize)) { + json.by_size[bytes].collisions = prefixes.map(prefix => ({ + prefix, appearances: 1, with_coords: 0, classification: 'unknown', nodes: [] + })); + } + await route.fulfill({ response, json }); + }); + try { + await openCollisions(page, '§ion=collisionRiskSection&bytes=2'); + await page.waitForFunction(() => window.__collisionsThemeReady); + for (const bytes of [2, 3, 1]) { + if (bytes !== 2) await page.locator(`.hash-byte-btn[data-bytes="${bytes}"]`).click(); + const prefixes = await page.locator('#collisionList tbody tr > td:first-child').allTextContents(); + assert(JSON.stringify(prefixes) === JSON.stringify(prefixesBySize[bytes]), + `Expected ${bytes}-byte risk prefixes ${prefixesBySize[bytes]}, got ${prefixes}`); + } + } finally { + await page.unroute(routePattern); + } + }); + await browser.close(); console.log('\n' + passed + ' passed, ' + failed + ' failed'); From 108ea020f74a9d0a460b3e426ef0d20d0e9f0de9 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 15:03:58 -0400 Subject: [PATCH 003/151] fix(nodes): remove misleading aggregate SNR headlines (#1969) Red commit: `2bf9be8` (local Chromium: 3 passed, 3 intended assertion failures). CI: [run](https://github.com/Kpa-clawbot/CoreScope/actions/runs/33988112224) awaits maintainer approval (`action_required`); 0 jobs started. Remove the unqualified aggregate Avg SNR row from node side-panel Overview and full-detail stats, following option 3 in #1149. Heard By retains each observer's SNR reading. Fixes #1149. - E2E assertion added: `test-issue-1281-location-row-e2e.js:224`. Three new browser cases cover desktop side/full and mobile full views with a numeric aggregate and distinct positive/negative observer readings. Existing packet-location assertions remain intact. - Browser verified: local Chromium; 6 cases passed after push. Screenshots: `coverage/issue-1149/issue-1149-desktop-side-panel.png`, `coverage/issue-1149/issue-1149-desktop-full-detail.png`, and `coverage/issue-1149/issue-1149-mobile-full-detail.png`. - Validation: packet filter 99/99, aging 18/18, frontend helpers 666/666; XSS, CSS-variable, syntax, whitespace and PII checks passed. - Independent reviews: adversarial, lifecycle expert and TDD reviewers found no required changes. One initial browser navigation timed out; the unchanged parent rerun passed 6/6. - Performance/config: two production row deletions; no new requests, loops, timers, settings or customizer implications. Backend unchanged; Go suites were not rerun. Fix commit: `d7c68f3`. ## Preflight overrides - External `run-all.sh` is unavailable on this host. Scoped branch, red/green, PII, CSS, XSS and whitespace checks were run directly. The diff adds no migrations, SQL attribution or image markup. --- public/nodes.js | 2 - test-issue-1281-location-row-e2e.js | 79 +++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/public/nodes.js b/public/nodes.js index ed2551b3d..d195ced6b 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -702,7 +702,6 @@ First Seen${renderNodeTimestampHtml(n.first_seen)} Total Packets${stats.totalTransmissions || stats.totalPackets || n.advert_count || 0}${stats.totalObservations && stats.totalObservations !== (stats.totalTransmissions || stats.totalPackets) ? ' (seen ' + stats.totalObservations + '×)' : ''} Packets Today${stats.packetsToday || 0} - ${stats.avgSnr != null ? `Avg SNR${Number(stats.avgSnr).toFixed(1)} dB` : ''} ${stats.avgHops ? `Avg Hops${stats.avgHops}` : ''} ${hasLoc ? `Location${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}` : ''} Hash Prefix${n.hash_size ? '' + n.public_key.slice(0, n.hash_size * 2).toUpperCase() + ' (' + n.hash_size + '-byte)' : 'Unknown'}${n.hash_size_inconsistent ? ' varies' : ''} @@ -1647,7 +1646,6 @@
First Seen
${renderNodeTimestampHtml(n.first_seen)}
Total Packets
${totalPackets}
Packets Today
${stats.packetsToday || 0}
- ${stats.avgSnr != null ? `
Avg SNR
${Number(stats.avgSnr).toFixed(1)} dB
` : ''} ${stats.avgHops ? `
Avg Hops
${stats.avgHops}
` : ''} ${hasLoc ? `
Location
${Number(n.lat).toFixed(5)}, ${Number(n.lon).toFixed(5)}
` : ''} diff --git a/test-issue-1281-location-row-e2e.js b/test-issue-1281-location-row-e2e.js index a92be9a4d..ffde5cc6f 100644 --- a/test-issue-1281-location-row-e2e.js +++ b/test-issue-1281-location-row-e2e.js @@ -14,6 +14,8 @@ * 3. The 📍map link uses class="loc-map-link" with a themed link color * (was --accent pre-M5; now --link-color after #1668/PR #1696 AA-contrast * fix). The hard guarantee is: NOT the default UA blue rgb(0,0,238). + * 4. #1149: node side/full stats omit aggregate SNR, retaining Heard By + * readings on desktop and mobile. Node API fixtures keep SNR non-null. * * Usage: BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js */ @@ -157,6 +159,83 @@ async function findPacketDetailByType(page, predicate, maxRows = 40) { 'Link color is UA-default blue — class is missing or CSS rule does not match'); }); + const pubkey = 'ab'.repeat(32); + const now = new Date().toISOString(); + const node = { + public_key: pubkey, name: 'SNR fixture node', role: 'repeater', + first_seen: now, last_seen: now, advert_count: 1005, + }; + const health = { + stats: { totalPackets: 1005, packetsToday: 7, avgSnr: 6.3, avgHops: 2, lastHeard: now }, + observers: [ + { observer_id: 'observer-a', observer_name: 'Observer A', packetCount: 1000, avgSnr: 12.5, avgRssi: -45 }, + { observer_id: 'observer-b', observer_name: 'Observer B', packetCount: 5, avgSnr: -8.5, avgRssi: -110 }, + ], + recentPackets: [], + }; + + for (const view of [ + { name: 'desktop side panel', width: 1280, full: false }, + { name: 'desktop full detail', width: 1280, full: true }, + { name: 'mobile full detail', width: 390, full: true }, + ]) { + await step('#1149 ' + view.name + ' retains observer SNR without an aggregate headline', async () => { + const nodePage = await browser.newPage({ viewport: { width: view.width, height: 900 } }); + try { + await nodePage.route('**/api/nodes**', async (route) => { + const pathname = new URL(route.request().url()).pathname; + const fixtures = { + '/api/nodes': { nodes: [node], total: 1, counts: { repeaters: 1 } }, + '/api/nodes/clock-skew': [], + ['/api/nodes/' + pubkey]: { node, recentAdverts: [] }, + ['/api/nodes/' + pubkey + '/health']: health, + ['/api/nodes/' + pubkey + '/neighbors']: { neighbors: [] }, + ['/api/nodes/' + pubkey + '/paths']: { paths: [] }, + ['/api/nodes/' + pubkey + '/clock-skew']: {}, + }; + if (Object.prototype.hasOwnProperty.call(fixtures, pathname)) { + await route.fulfill({ json: fixtures[pathname] }); + } else { + await route.continue(); + } + }); + await nodePage.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); + await nodePage.locator('#nodesBody tr[data-key="' + pubkey + '"]').click(); + if (view.full && view.width > 640) { + await nodePage.locator('#nodesRight .node-detail-btn').click(); + } + + const stats = nodePage.locator(view.full ? '#node-stats' : '#nodesRight .detail-meta'); + const readings = nodePage.locator(view.full ? '#node-observers tbody tr' : '#nodesRight .observer-row'); + await readings.first().waitFor({ state: 'visible' }); + assert(await readings.count() === 2, 'Heard By must retain both observers'); + for (const observer of health.observers) { + const row = readings.filter({ hasText: observer.observer_name }); + const text = (await row.innerText()).replace(/\s+/g, ' '); + const snr = observer.avgSnr.toFixed(1) + (view.full ? ' dB' : 'dB'); + assert(text.includes(snr), observer.observer_name + ' must retain SNR ' + snr + ', got: ' + text); + } + for (const [label, value] of [['Total Packets', '1005'], ['Packets Today', '7'], ['Avg Hops', '2']]) { + const metric = stats.getByText(label, { exact: true }); + const actual = await metric.evaluate(el => el.nextElementSibling.textContent.trim()); + assert(actual === value, label + ' must remain ' + value + ', got: ' + actual); + } + const headlineCount = await stats.getByText('Avg SNR', { exact: true }).count(); + assert(headlineCount === 0, + 'Expected no aggregate Avg SNR headline with stats.avgSnr=6.3, got ' + headlineCount); + if (process.env.SCREENSHOT_DIR) { + await stats.scrollIntoViewIfNeeded(); + await nodePage.screenshot({ + path: require('path').join(process.env.SCREENSHOT_DIR, 'issue-1149-' + view.name.replace(/ /g, '-') + '.png'), + fullPage: true, + }); + } + } finally { + await nodePage.close(); + } + }); + } + await browser.close(); console.log(`\n${passed} passed, ${failed} failed`); From a938176f831f979ef0442594ab33286e2585724f Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 15:04:03 -0400 Subject: [PATCH 004/151] fix(nodes): clearly mark the selected node in path chains (#1968) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Red commit: `0988bc0` (local browser: 3 passed, 8 behavior assertion failures before the fix). The selected node now has a compact outline in long “Paths Through This Node” chains, in the side panel and full detail page. Matching uses complete public keys without case sensitivity; same-prefix siblings and unresolved hops stay unmarked. Existing links, escaped names, warnings and ambiguity underlines are preserved. Fixes #1153. Its prerequisite #1144 is already merged. - E2E assertion added: `test-issue-1146-path-link-contrast-e2e.js:220`. The existing CI-selected harness passes 11 checks across 18-hop paths, both themes, desktop/mobile, and the renderer fallback. Review follow-up `bd8118f` verifies the marked ambiguous hop's dashed underline. - Browser verified: `http://127.0.0.1:55635`; desktop/mobile path screenshots were inspected. The broader smoke runner exited successfully with fixture-dependent skips. - Required frontend checks pass: 99 filter, 18 aging, 666 helpers. CSS variables, seven CSS self-tests, 31 XSS sink checks, 17 XSS gate self-tests and XSS diff preflight pass. - Three independent reviews found no blocking issues. Traversal remains linear with no new requests, settings, dependencies or cache invalidation; styling uses the existing customizer token. ## Preflight overrides - The external preflight runner is absent; corresponding scoped gates passed. Red browser evidence is local, with upstream CI approval tracked separately under the process in #1922. - Existing rapid-navigation map resize timer errors remain visible in browser logs and are outside this change. --- public/nodes.js | 10 +- public/style.css | 10 +- test-issue-1146-path-link-contrast-e2e.js | 145 ++++++++++++++++++++-- 3 files changed, 149 insertions(+), 16 deletions(-) diff --git a/public/nodes.js b/public/nodes.js index d195ced6b..ea4c20415 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -960,17 +960,18 @@ } document.querySelector('#fullPathsSection h4').textContent = `Paths Through This Node (${pathData.totalPaths} unique, ${pathData.totalTransmissions} transmissions)`; const COLLAPSE_LIMIT = 10; + const currentPubkey = n.public_key.toLowerCase(); function renderPaths(paths) { return paths.map(p => { const chain = p.hops.map(h => { - const isThis = h.pubkey === n.public_key; + const isThis = h.pubkey && h.pubkey.toLowerCase() === currentPubkey; if (window.HopDisplay) { const entry = { name: h.name, pubkey: h.pubkey, ambiguous: h.ambiguous, conflicts: h.conflicts, totalGlobal: h.totalGlobal, totalRegional: h.totalRegional, globalFallback: h.globalFallback, unreliable: h.unreliable }; const html = HopDisplay.renderHop(h.prefix, entry); return isThis ? html.replace('class="', 'class="hop-current ') : html; } const name = escapeHtml(h.name || h.prefix); - const link = h.pubkey ? `${name}` : `${name}`; + const link = h.pubkey ? `${name}` : `${name}`; return link; }).join(' → '); return `
@@ -1777,12 +1778,13 @@ document.querySelector('#pathsSection h4').textContent = `Paths Through This Node (${pathData.totalPaths} unique path${pathData.totalPaths !== 1 ? 's' : ''}, ${pathData.totalTransmissions} transmissions)`; const COLLAPSE_LIMIT = 10; const showAll = pathData.paths.length <= COLLAPSE_LIMIT; + const currentPubkey = n.public_key.toLowerCase(); function renderPaths(paths) { return paths.map(p => { const chain = p.hops.map(h => { - const isThis = h.pubkey === n.public_key; + const isThis = h.pubkey && h.pubkey.toLowerCase() === currentPubkey; const name = escapeHtml(h.name || h.prefix); - const link = h.pubkey ? `${name}` : `${name}`; + const link = h.pubkey ? `${name}` : `${name}`; return link; }).join(' → '); return `
diff --git a/public/style.css b/public/style.css index 71766a5ac..bf71fd88b 100644 --- a/public/style.css +++ b/public/style.css @@ -3071,7 +3071,15 @@ button.ch-item.ch-item-encrypted .ch-badge { filter: grayscale(0.6); } .hop-unreliable-btn { background: none; border: none; color: var(--status-yellow, #f59e0b); font-size: 13px; cursor: help; vertical-align: middle; margin-left: 2px; padding: 0 2px; line-height: 1; } .hop-global-fallback { border-bottom: 1px dashed var(--status-red); } -.hop-current { font-weight: 700 !important; color: var(--link-color) !important; } +.hop-current { + display: inline-block; + padding: 0 4px; + /* Outline preserves the shared renderer's dashed ambiguity border. */ + outline: 1px solid var(--link-color); + border-radius: 4px; + font-weight: 700 !important; + color: var(--link-color) !important; +} /* Self-loop subpath rows */ .subpath-selfloop { opacity: 0.6; } diff --git a/test-issue-1146-path-link-contrast-e2e.js b/test-issue-1146-path-link-contrast-e2e.js index 77a465fa6..7c496611b 100644 --- a/test-issue-1146-path-link-contrast-e2e.js +++ b/test-issue-1146-path-link-contrast-e2e.js @@ -14,15 +14,21 @@ * theme, then asserts the computed link colour vs. its background * yields a contrast ratio ≥ 4.5:1. * - * Currently FAILS (link color resolves to rgb(0,0,238)). - * After the style.css fix it PASSES. + * #1153 also checks that the selected node has a visible enclosing marker + * in an 18-hop chain, without marking a different node with the same prefix. + * Covers the desktop side/full views and mobile full view in both themes. * * Usage: BASE_URL=http://localhost:13581 node test-issue-1146-path-link-contrast-e2e.js + * Optional SCREENSHOT_DIR saves PNG evidence; use an ignored output directory. */ 'use strict'; const { chromium } = require('playwright'); const BASE = process.env.BASE_URL || 'http://localhost:13581'; +assert(['localhost', '127.0.0.1', '[::1]'].includes(new URL(BASE).hostname), + 'Browser regression tests must use a local server'); +const SCREENSHOT_DIR = process.env.SCREENSHOT_DIR; +if (SCREENSHOT_DIR) require('fs').mkdirSync(SCREENSHOT_DIR, { recursive: true }); let passed = 0, failed = 0; async function step(name, fn) { @@ -83,20 +89,41 @@ async function effectiveBgFor(page, selector) { console.log(`\n=== #1146 path-link contrast E2E against ${BASE} ===`); - const hopPubkey = 'a1b2c3d4e5f60718293a4b5c6d7e8f9001122334455667788990aabbccddeeff'; + const hopPubkey = 'a1'.repeat(32); + const targetName = 'Selected & relay'; + let selectedPubkey; + let targetHopKey; + let siblingKey; // Mock paths API for ANY node so test is deterministic. await page.route('**/api/nodes/*/paths*', (route) => { - route.fulfill({ + selectedPubkey = decodeURIComponent(new URL(route.request().url()).pathname.split('/')[3]); + // Full public-key identity must survive hex casing; a shared prefix alone + // must never qualify. Neither fixture node requires resolver changes. + targetHopKey = selectedPubkey === selectedPubkey.toUpperCase() + ? selectedPubkey.toLowerCase() : selectedPubkey.toUpperCase(); + siblingKey = targetHopKey.slice(0, -1) + (targetHopKey.endsWith('0') ? '1' : '0'); + const hops = Array.from({ length: 18 }, (_, i) => ({ + pubkey: hopPubkey.slice(0, -2) + i.toString(16).padStart(2, '0'), + prefix: 'a1', name: 'Relay ' + (i + 1), + })); + hops[8] = { pubkey: targetHopKey, prefix: targetHopKey.slice(0, 2), name: targetName, unreliable: true, ambiguous: true }; + hops[9] = { + pubkey: siblingKey, prefix: targetHopKey.slice(0, 2), name: 'Same-prefix sibling', ambiguous: true, + conflicts: [ + { pubkey: targetHopKey, name: targetName, regional: true }, + { pubkey: siblingKey, name: 'Same-prefix sibling', regional: true }, + ], + }; + hops[10] = { prefix: targetHopKey.slice(0, 2), name: 'Unresolved prefix' }; + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ totalPaths: 1, totalTransmissions: 5, paths: [{ - hops: [ - { pubkey: hopPubkey, prefix: 'a1', name: 'TestHop' }, - ], + hops, count: 5, lastSeen: new Date().toISOString(), sampleHash: 'deadbeef00', @@ -107,10 +134,10 @@ async function effectiveBgFor(page, selector) { await step('Load nodes page and force dark theme', async () => { await page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); - await page.evaluate(() => { - document.documentElement.setAttribute('data-theme', 'dark'); - }); await page.waitForSelector('#nodesBody tr[data-key]', { timeout: 15000 }); + if (await page.locator('html').getAttribute('data-theme') !== 'dark') { + await page.locator('#darkModeToggle').click(); + } }); await step('Open side panel for first node and wait for paths', async () => { @@ -121,7 +148,7 @@ async function effectiveBgFor(page, selector) { const el = document.getElementById('pathsContent'); return el && el.querySelector('a[href^="#/nodes/"]'); }, - { timeout: 15000 } + null, { timeout: 15000 } ); }); @@ -145,6 +172,102 @@ async function effectiveBgFor(page, selector) { `covered by the .data-table td a rule and inherit UA blue.`); }); + async function assertCurrentHop(selector, hasHopDisplay) { + await page.waitForFunction((sel) => document.querySelector(sel)?.textContent.includes('Relay 18'), selector); + const targetSelector = selector + ' a[href="#/nodes/' + targetHopKey + '"]'; + const result = await page.evaluate(({ selector, targetHopKey, siblingKey }) => { + const container = document.querySelector(selector); + const chain = container.firstElementChild.firstElementChild; + const target = chain.querySelector('a[href="#/nodes/' + targetHopKey + '"]'); + const sibling = chain.querySelector('a[href="#/nodes/' + siblingKey + '"]'); + target.scrollIntoView({ block: 'center' }); + const style = getComputedStyle(target); + const rect = target.getBoundingClientRect(); + const chainRect = chain.getBoundingClientRect(); + const outline = parseFloat(style.outlineWidth) >= 1 && style.outlineStyle === 'solid'; + const border = ['Top', 'Right', 'Bottom', 'Left'].every(side => + parseFloat(style['border' + side + 'Width']) >= 1 && style['border' + side + 'Style'] === 'solid'); + return { + marked: Array.from(chain.querySelectorAll('.hop-current')).map(el => el.getAttribute('href')), + ring: outline || border, ringColor: outline ? style.outlineColor : style.borderTopColor, + outline, dashedBottom: style.borderBottomStyle === 'dashed' && parseFloat(style.borderBottomWidth) >= 1, + textColor: style.color, padding: parseFloat(style.paddingLeft), + text: target.textContent, injectedElements: target.childElementCount, + siblingMarked: sibling.classList.contains('hop-current'), + fits: rect.left >= chainRect.left - 1 && rect.right <= chainRect.right + 1, + fragments: target.getClientRects().length, + warning: !!chain.querySelector('.hop-unreliable-btn'), + conflict: !!chain.querySelector('.hop-conflict-btn'), + siblingAmbiguous: sibling.classList.contains('hop-ambiguous'), + hopCount: chain.children.length - chain.querySelectorAll('button').length, + }; + }, { selector, targetHopKey, siblingKey }); + assert(result.hopCount === 18, 'Expected all 18 hops in the rendered chain'); + assert(result.ring, 'Selected hop must have a visible enclosing border or outline, beyond bold/color'); + assert(result.marked.length === 1 && result.marked[0] === '#/nodes/' + targetHopKey, + 'Exactly the selected full public key must be marked, including mixed-case hex'); + assert(!result.siblingMarked, 'Same-prefix sibling must not be marked as this node'); + assert(result.padding > 0 && result.fits && result.fragments === 1, + 'Selected hop marker must enclose the name and fit within the wrapping chain'); + assert(result.text === targetName && result.injectedElements === 0, 'Selected hop name must stay escaped'); + const bg = parseRgb(await effectiveBgFor(page, targetSelector)); + assert(contrast(parseRgb(result.ringColor), bg) >= 3, 'Selected hop ring must contrast with the active theme'); + assert(contrast(parseRgb(result.textColor), bg) >= 4.5, 'Selected hop text must remain readable'); + if (hasHopDisplay) { + assert(result.warning && result.conflict && result.siblingAmbiguous, + 'Shared hop rendering must preserve unreliable and ambiguous indicators'); + assert(result.outline && result.dashedBottom, + 'The selected ambiguous hop must keep its dashed bottom border alongside the enclosing outline'); + } + if (SCREENSHOT_DIR) { + const theme = await page.locator('html').getAttribute('data-theme'); + const filename = `${selector.slice(1)}-${page.viewportSize().width}-${theme}${hasHopDisplay ? '' : '-plain'}.png`; + await page.locator(selector).screenshot({ path: require('path').join(SCREENSHOT_DIR, filename) }); + } + } + + await step('#1153 desktop side panel: selected hop stands out in an 18-hop dark chain', + () => assertCurrentHop('#pathsContent', false)); + + for (const theme of ['light', 'dark']) { + if (await page.locator('html').getAttribute('data-theme') !== theme) { + await page.locator('#darkModeToggle').click(); + } + await step('#1153 desktop side panel: ' + theme + ' theme', async () => { + await page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); + assert(await page.locator('html').getAttribute('data-theme') === theme, 'Expected the selected theme'); + await page.locator('#nodesBody tr[data-key]').first().click(); + await assertCurrentHop('#pathsContent', false); + }); + await step('#1153 desktop full view: ' + theme + ' theme', async () => { + await page.locator('#nodesRight .node-detail-btn').click(); + await assertCurrentHop('#fullPathsContent', true); + }); + } + + for (const theme of ['light', 'dark']) { + // The theme toggle lives in the desktop header; exercise the same theme + // at mobile width without depending on the unrelated navigation drawer. + await page.setViewportSize({ width: 1280, height: 900 }); + if (await page.locator('html').getAttribute('data-theme') !== theme) { + await page.locator('#darkModeToggle').click(); + } + await page.setViewportSize({ width: 390, height: 844 }); + await step('#1153 mobile row click opens a readable full chain: ' + theme + ' theme', async () => { + await page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded' }); + assert(await page.locator('html').getAttribute('data-theme') === theme, 'Expected the selected theme'); + await page.locator('#nodesBody tr[data-key]').first().click(); + await assertCurrentHop('#fullPathsContent', true); + }); + } + + await step('#1153 full-view fallback also marks only the selected node', async () => { + await page.route('**/hop-display.js*', route => route.fulfill({ contentType: 'text/javascript', body: '' })); + await page.reload({ waitUntil: 'domcontentloaded' }); + await assertCurrentHop('#fullPathsContent', false); + assert(await page.evaluate(() => !window.HopDisplay), 'Expected the real full-view fallback'); + }); + await browser.close(); console.log(`\n${passed} passed, ${failed} failed`); From 43d83ee8cade64b113822244a11b3d51e1c2a9dd Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 15:04:09 -0400 Subject: [PATCH 005/151] fix(nodes): dispose map timers with their owning view (#1970) Red commit: `2f1a50e` (local Chromium: 2 passed, 7 assertion failures). Ownership regression: `059ab49` (9 passed, 4 assertion failures). CI: [run](https://github.com/Kpa-clawbot/CoreScope/actions/runs/33988139479) awaits maintainer approval (`action_required`); 0 jobs started. Rapid navigation or closing node detail could leave a delayed resize targeting a removed or replacement map. Disposal now cancels its timer, each resize captures its own map, and delayed responses respect the view owning the current map. Stale side-pane responses are ignored before rendering. Fixes #1972. - E2E assertion added: `test-issue-1206-resize-observer-leak-e2e.js:216` and `:292`. This existing CI-selected suite covers navigation, replacement deadlines, close/Escape, no-location rendering, and late error/success responses. Existing observer-growth assertions remain intact; readiness waits replace fixed sleeps. - Browser verified: local fixture with real Chromium and Leaflet; 13 browser checks passed after push. Evidence: `data/node-map-validation/post-push-browser.log` and `data/node-map-validation/evidence.md`. - Validation: frontend unit suites 99/18/666 passed; JavaScript syntax, CSS variables, whitespace, PII and XSS checks passed. Backend unchanged; Go suites not rerun. - Independent adversarial, expert and TDD reviews found no required changes. Two original navigation checks initially timed out; the unchanged parent rerun passed 13/13 (`parent-browser-confirm.log`). - Performance/config: one timer handle, no packet/node loops or new requests. Tests enforce zero stale invalidations and one resize at the surviving map's deadline. Existing 100ms delay retained; no new settings or throughput claim. Fix commits: `2492a65`, `1dc090d`. ## Preflight overrides - External `run-all.sh` is unavailable. Scoped branch, red/green, PII, CSS, XSS and whitespace checks were run directly; no migrations, SQL attribution or image markup are added. --- public/nodes.js | 33 +++- test-issue-1206-resize-observer-leak-e2e.js | 191 +++++++++++++++++++- 2 files changed, 214 insertions(+), 10 deletions(-) diff --git a/public/nodes.js b/public/nodes.js index ea4c20415..0ced86bf3 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -84,6 +84,23 @@ let statusFilter = localStorage.getItem('meshcore-nodes-status-filter') || 'all'; let wsHandler = null; let detailMap = null; + let detailMapResizeTimer = null; + + function removeDetailMap(owner) { + // Async responses may still hold a detached view while another map is active. + if (owner && detailMap && !owner.contains(detailMap.getContainer())) return; + clearTimeout(detailMapResizeTimer); + detailMapResizeTimer = null; + if (detailMap) { detailMap.remove(); detailMap = null; } + } + + function resizeDetailMapAfterLayout() { + const map = detailMap; + detailMapResizeTimer = setTimeout(() => { + detailMapResizeTimer = null; + map.invalidateSize(); + }, 100); + } // #1461 followup: node-detail inset map tile layer that honors the // customizer dark-tile-provider pick (#1420/#1430). Falls back to @@ -630,6 +647,7 @@ const dupKeys = n.name && dupMap[n.name.toLowerCase()] ? dupMap[n.name.toLowerCase()].filter(function(k) { return k !== n.public_key; }) : []; const dupSection = dupKeys.length ? '
Also known as: ' + dupKeys.map(function(k) { return '' + escapeHtml(k.slice(0, 12)) + '…'; }).join(', ') + '
' : ''; + removeDetailMap(body); body.innerHTML = `
${escapeHtml(n.name || '(unnamed)')}${dupBadge}
@@ -786,11 +804,10 @@ // Map if (hasLoc) { try { - if (detailMap) { detailMap.remove(); detailMap = null; } detailMap = L.map('nodeFullMap', { zoomControl: true, attributionControl: false }).setView([n.lat, n.lon], 13); _applyTilesToNodeMap(detailMap); L.marker([n.lat, n.lon]).addTo(detailMap).bindPopup(escapeHtml(n.name || n.public_key.slice(0, 12))); - setTimeout(() => detailMap.invalidateSize(), 100); + resizeDetailMapAfterLayout(); } catch {} } @@ -1011,6 +1028,7 @@ const detail = is404 ? 'No node matched the requested public key on this instance. It may exist on another deployment, or it may have been evicted/blacklisted here.' : 'The node detail API call failed: ' + escapeHtml(msg); + removeDetailMap(body); body.innerHTML = '
' + '
' + headline + '
' + @@ -1035,7 +1053,7 @@ function destroy() { if (wsHandler) offWS(wsHandler); wsHandler = null; - if (detailMap) { detailMap.remove(); detailMap = null; } + removeDetailMap(); if (regionChangeHandler) RegionFilter.offChange(regionChangeHandler); regionChangeHandler = null; nodes = []; @@ -1413,6 +1431,7 @@ if (e.key === 'Escape') { const panel = document.getElementById('nodesRight'); if (panel && !panel.classList.contains('empty')) { + removeDetailMap(); panel.classList.add('empty'); panel.innerHTML = 'Select a node to view details'; selectedKey = null; @@ -1439,6 +1458,7 @@ } if (e.target.closest('.panel-close-btn')) { const panel = document.getElementById('nodesRight'); + removeDetailMap(); panel.classList.add('empty'); panel.innerHTML = 'Select a node to view details'; selectedKey = null; @@ -1590,12 +1610,15 @@ renderRows(); const panel = document.getElementById('nodesRight'); panel.classList.remove('empty'); + removeDetailMap(); panel.innerHTML = '
Loading…
'; try { const data = await fetchNodeDetail(pubkey); + if (selectedKey !== pubkey || !panel.isConnected) return; renderDetail(panel, data); } catch (e) { + if (selectedKey !== pubkey || !panel.isConnected) return; panel.innerHTML = `
Error: ${e.message}
`; } } @@ -1620,6 +1643,7 @@ const dupMap = buildDupNameMap(_allNodes); const dupBadge = dupNameBadge(n.name, n.public_key, dupMap); + removeDetailMap(panel); panel.innerHTML = `
@@ -1710,11 +1734,10 @@ // Init map if (hasLoc) { try { - if (detailMap) { detailMap.remove(); detailMap = null; } detailMap = L.map('nodeMap', { zoomControl: false, attributionControl: false }).setView([n.lat, n.lon], 13); _applyTilesToNodeMap(detailMap); L.marker([n.lat, n.lon]).addTo(detailMap).bindPopup(escapeHtml(n.name || n.public_key.slice(0, 12))); - setTimeout(() => detailMap.invalidateSize(), 100); + resizeDetailMapAfterLayout(); } catch {} } diff --git a/test-issue-1206-resize-observer-leak-e2e.js b/test-issue-1206-resize-observer-leak-e2e.js index 307ddd041..38fa65de9 100644 --- a/test-issue-1206-resize-observer-leak-e2e.js +++ b/test-issue-1206-resize-observer-leak-e2e.js @@ -12,6 +12,9 @@ * outstanding (constructed but not disconnected) observers and assert it * does NOT grow with each /live mount. * + * Also exercises node-map disposal with a paused browser clock: navigation, + * pane closure and replacement must cancel the old map's delayed resize. + * * Run: BASE_URL=http://localhost:13581 node test-issue-1206-resize-observer-leak-e2e.js */ 'use strict'; @@ -28,7 +31,83 @@ function assert(c, m) { if (!c) throw new Error(m || 'assertion failed'); } async function gotoHash(page, hash) { await page.evaluate((h) => { window.location.hash = h; }, hash); - await page.waitForTimeout(150); + if (hash === '#/live') await waitForVCRTracker(page); + else await page.locator('#nodesLeft[data-loaded="true"]').waitFor(); +} + +async function waitForVCRTracker(page) { + // The tracker publishes this only after live's async node loading finishes. + await page.locator('.live-page[style*="--vcr-bar-height"]').waitFor({ state: 'attached' }); +} + +// Synthetic nodes keep map lifecycle coverage independent of fixture locations. +const mapNodes = [ + { public_key: 'a1'.repeat(32), name: 'Map fixture A', lat: 0, lon: 0 }, + { public_key: 'b2'.repeat(32), name: 'Map fixture B', lat: 1, lon: 1 }, + { public_key: 'c3'.repeat(32), name: 'No location fixture', lat: null, lon: null }, +].map((node) => ({ ...node, role: 'repeater', advert_count: 1 })); + +async function withNodeMaps(browser, fn, allowedErrors = []) { + const ctx = await browser.newContext({ viewport: { width: 1280, height: 800 } }); + const page = await ctx.newPage(); + page.setDefaultTimeout(8000); + const errors = []; + page.on('pageerror', (error) => errors.push(error.message)); + try { + await page.route('**/api/nodes**', (route) => { + const path = new URL(route.request().url()).pathname; + const node = mapNodes.find((n) => path === '/api/nodes/' + n.public_key); + const body = path === '/api/nodes' + ? { nodes: mapNodes, total: mapNodes.length, counts: { repeater: mapNodes.length } } + : node ? { node, recentAdverts: [] } + : path === '/api/nodes/clock-skew' ? [] + : /\/health$/.test(path) ? { stats: {}, observers: [], recentPackets: [] } + : /\/neighbors$/.test(path) ? { neighbors: [] } + : /\/paths$/.test(path) ? { paths: [] } : {}; + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(body) }); + }); + await page.clock.install({ time: new Date('2026-01-01T00:00:00Z') }); + await page.goto(BASE + '/#/nodes', { waitUntil: 'domcontentloaded', timeout: 30000 }); + await page.locator('#nodesBody tr[data-key="' + mapNodes[0].public_key + '"]').waitFor(); + await page.clock.pauseAt(new Date('2026-01-01T00:02:00Z')); + // Observe real Leaflet instances through public lifecycle APIs. No app hooks. + await page.evaluate(() => { + window.__nodeMaps = []; + L.Map.addInitHook(function() { + const container = this.getContainer(); + if (container.id !== 'nodeMap' && container.id !== 'nodeFullMap') return; + const record = { removed: false, resizes: [] }; + window.__nodeMaps.push(record); + this.on('unload', () => { record.removed = true; }); + const invalidateSize = this.invalidateSize; + this.invalidateSize = function(...args) { + record.resizes.push({ removed: record.removed, connected: container.isConnected }); + return invalidateSize.apply(this, args); + }; + }); + }); + const open = async (view, node) => { + if (view === 'side') { + await page.locator('#nodesBody tr[data-key="' + node.public_key + '"]').click(); + } else { + await page.evaluate((key) => { location.hash = '#/nodes/' + key; }, node.public_key); + } + const root = view === 'side' ? '#nodesRight' : '#nodeFullBody'; + await page.locator(root + ' .node-detail-name').filter({ hasText: node.name }).waitFor(); + if (node.lat != null) await page.locator(root + ' .leaflet-map-pane').waitFor({ state: 'attached' }); + }; + await fn(page, open); + const unexpectedErrors = errors.filter((error) => !allowedErrors.includes(error)); + assert(unexpectedErrors.length === 0, 'unexpected pageerror: ' + unexpectedErrors.join('; ')); + } finally { + await ctx.close(); + } +} + +async function advanceMapClock(page, ms) { + // Playwright also reports exceptions from fake-timer callbacks via runFor. + const error = await page.clock.runFor(ms).then(() => null, (error) => error); + assert(!error, 'timer callback must not throw: ' + (error && error.message)); } (async () => { @@ -78,7 +157,7 @@ async function gotoHash(page, hash) { await step('initial /#/live mount constructs at most 1 VCR ResizeObserver', async () => { await page.goto(BASE + '/#/live', { waitUntil: 'domcontentloaded' }); await page.waitForSelector('#vcrBar', { timeout: 8000 }); - await page.waitForTimeout(300); + await waitForVCRTracker(page); // Baseline snapshot — record outstanding right after first /live mount. const snap = await page.evaluate(() => ({ outstanding: window.__roOutstanding, @@ -93,10 +172,8 @@ async function gotoHash(page, hash) { await step('3 SPA round-trips /live<->/nodes do NOT grow outstanding observer count', async () => { for (let i = 0; i < 3; i++) { await gotoHash(page, '#/nodes'); - await page.waitForTimeout(150); await gotoHash(page, '#/live'); await page.waitForSelector('#vcrBar', { timeout: 8000 }); - await page.waitForTimeout(200); } const after = await page.evaluate(() => ({ outstanding: window.__roOutstanding, @@ -117,7 +194,111 @@ async function gotoHash(page, hash) { }); await ctx.close(); + + for (const view of ['side', 'full']) { + await step(view + ' node map: navigation cancels the pending resize', () => withNodeMaps(browser, async (page, open) => { + await open(view, mapNodes[0]); + await page.evaluate(() => { location.hash = '#/live'; }); + await page.locator('#vcrBar').waitFor(); + await advanceMapClock(page, 100); + const maps = await page.evaluate(() => window.__nodeMaps); + assert(maps.length === 1 && maps[0].removed, 'navigation must remove the node map'); + assert(maps[0].resizes.length === 0, 'removed map must not be resized'); + })); + + await step(view + ' node map: replacement only resizes at its own deadline', () => withNodeMaps(browser, async (page, open) => { + await open(view, mapNodes[0]); + await advanceMapClock(page, 50); + await open(view, mapNodes[1]); + await advanceMapClock(page, 50); // First map's deadline; replacement is only 50ms old. + let maps = await page.evaluate(() => window.__nodeMaps); + assert(maps.length === 2 && maps[0].removed && !maps[1].removed, 'replacement must own the surviving map'); + assert(maps[0].resizes.length === 0 && maps[1].resizes.length === 0, 'old timer must not resize either map'); + await advanceMapClock(page, 49); + maps = await page.evaluate(() => window.__nodeMaps); + assert(maps[1].resizes.length === 0, 'replacement must wait its full 100ms'); + await advanceMapClock(page, 1); + maps = await page.evaluate(() => window.__nodeMaps); + assert(maps[1].resizes.length === 1 && !maps[1].resizes[0].removed && maps[1].resizes[0].connected, + 'surviving map must resize once while connected'); + })); + } + + for (const close of ['button', 'Escape']) { + await step('side node map: ' + close + ' disposes the map before resize', () => withNodeMaps(browser, async (page, open) => { + await open('side', mapNodes[0]); + if (close === 'button') await page.locator('#nodesRight .panel-close-btn').click(); + else await page.keyboard.press('Escape'); + await page.locator('#nodesRight.empty').waitFor(); + await advanceMapClock(page, 100); + const maps = await page.evaluate(() => window.__nodeMaps); + assert(maps.length === 1 && maps[0].removed, 'closing the pane must remove its map'); + assert(maps[0].resizes.length === 0, 'closed pane map must not be resized'); + })); + } + + await step('side node map: no-location replacement disposes the pending map', () => withNodeMaps(browser, async (page, open) => { + await open('side', mapNodes[0]); + await open('side', mapNodes[2]); + await advanceMapClock(page, 100); + const maps = await page.evaluate(() => window.__nodeMaps); + assert(maps.length === 1 && maps[0].removed, 'no-location replacement must remove the previous map'); + assert(maps[0].resizes.length === 0, 'no-location replacement must cancel the old resize'); + assert(await page.locator('#nodeMap').count() === 0, 'no-location node must not create a map'); + })); + + for (const scenario of [ + { view: 'full', fails: true }, + { view: 'full', fails: false }, + { view: 'side', fails: true }, + { view: 'side', fails: false }, + ]) { + const expectedError = scenario.fails ? 'API 500: /nodes/' + mapNodes[0].public_key : null; + await step(scenario.view + ' node map: stale ' + (scenario.fails ? 'failure' : 'no-location response') + ' preserves the replacement', + () => withNodeMaps(browser, async (page, open) => { + const requestUrl = '**/api/nodes/' + mapNodes[0].public_key; + let releaseResponse; + const holdResponse = new Promise((resolve) => { releaseResponse = resolve; }); + await page.route(requestUrl, async (route) => { + await holdResponse; + await route.fulfill({ status: scenario.fails ? 500 : 200, contentType: 'application/json', body: JSON.stringify({ + node: { ...mapNodes[0], lat: null, lon: null }, recentAdverts: [], + }) }); + }); + const requested = page.waitForRequest(requestUrl); + if (scenario.view === 'full') { + await page.evaluate((key) => { location.hash = '#/nodes/' + key; }, mapNodes[0].public_key); + } else { + await page.locator('#nodesBody tr[data-key="' + mapNodes[0].public_key + '"]').click(); + } + await requested; + // Join the existing in-flight API promise so assertions run after the + // client has decoded/rejected the held response, not merely sent it. + const requestCompleted = page.evaluate((key) => api('/nodes/' + key).then(() => null, (error) => error.message), mapNodes[0].public_key); + await open(scenario.view, mapNodes[1]); + await advanceMapClock(page, 50); + // api() may report its rejected finally() derivative as a pageerror. + // Only this deliberately injected API failure is allowed by the helper. + releaseResponse(); + let requestTimeout; + const result = await Promise.race([ + requestCompleted, + new Promise((_, reject) => { + requestTimeout = setTimeout(() => reject(new Error('held response was not handled')), 8000); + }), + ]).finally(() => clearTimeout(requestTimeout)); + assert(result === expectedError, 'held response must complete with the intended result'); + let maps = await page.evaluate(() => window.__nodeMaps); + assert(maps.length === 1 && !maps[0].removed, 'stale response must not remove the replacement map'); + assert(maps[0].resizes.length === 0, 'replacement must wait for its own resize deadline'); + await advanceMapClock(page, 50); + maps = await page.evaluate(() => window.__nodeMaps); + assert(maps[0].resizes.length === 1 && maps[0].resizes[0].connected && !maps[0].resizes[0].removed, + 'replacement must remain connected and receive its own resize'); + }, expectedError ? [expectedError] : [])); + } + await browser.close(); - console.log('\n#1206 ResizeObserver leak: ' + passed + ' passed, ' + failed + ' failed'); + console.log('\nSPA observer and node-map lifecycle: ' + passed + ' passed, ' + failed + ' failed'); process.exit(failed ? 1 : 0); })().catch((e) => { console.error(e); process.exit(1); }); From 9d6f08c14486ee01d8c1ab8e60ebc4ac221226c5 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 6 Sep 2026 21:06:12 +0200 Subject: [PATCH 006/151] feat(#1865): ingest observer /neighbors as confirmed scope evidence (#1971) Carries SaarMesh-Bot's implementation from the closed #1867 forward onto current master, 67 commits later, and surfaces the result on the per-node Reach report. The declared region list a repeater answers with is now stored on the node as configured_scope, normalised to the same leading-# syntax default_scope already uses so the two are directly comparable. That normalisation is the point @cwichura raised on #1865 and @dborup agreed with before the original PR closed. Co-authored-by: SaarMesh-Bot <300107934+SaarMesh-Bot@users.noreply.github.com> --- cmd/ingestor/db.go | 103 ++++++++++++++ cmd/ingestor/issue1865_test.go | 244 +++++++++++++++++++++++++++++++++ cmd/ingestor/main.go | 62 +++++++++ cmd/server/db.go | 14 ++ cmd/server/node_reach.go | 58 +++++++- internal/dbschema/dbschema.go | 39 ++++++ public/live.js | 1 + public/node-reach.css | 9 ++ public/node-reach.js | 32 +++++ public/nodes.js | 1 + test-frontend-helpers.js | 60 ++++++++ 11 files changed, 622 insertions(+), 1 deletion(-) create mode 100644 cmd/ingestor/issue1865_test.go diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index f1eb92f3f..729e6d8ff 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -2044,6 +2044,109 @@ func (s *Store) UpdateNodeDefaultScope(pubkey, scope string) error { return err } +// normalizeReportTS parses an observer report timestamp and returns it in +// canonical UTC RFC3339 form ("2006-01-02T15:04:05Z"). It accepts both the +// firmware's fractional/offset form (e.g. "2026-07-26T09:43:48.000000+00:00") +// and plain "Z"/offset variants. An empty or unparseable input returns "" so +// the caller writes an empty configured_scope_at and skips the ordering guard; +// this keeps every stored timestamp either canonical or empty, never a mix of +// offset/precision formats that would break lexicographic last-write-wins. +func normalizeReportTS(raw string) string { + if raw == "" { + return "" + } + for _, layout := range []string{time.RFC3339Nano, time.RFC3339} { + if t, err := time.Parse(layout, raw); err == nil { + return t.UTC().Format(time.RFC3339) + } + } + return "" +} + +// normalizeScopeList canonicalises the comma-separated region-scope string an +// observer reports for a node, so configured_scope is directly comparable with +// default_scope instead of only looking similar (#1865, raised by @cwichura). +// +// The OTA scope query returns bare region names ("dk,eu,dk-aarhus"), while +// everything else in CoreScope carries the leading "#": every stored +// default_scope value on a live 1425-node instance starts with one (361 of +// 361), and the hashRegions/hashChannels config paths already prefix a missing +// "#" before deriving the channel key from it (main.go). The "#" is therefore +// part of the region's name here, not decoration, so a bare "dk" and a stored +// "#dk" are the same region and must be written the same way. +// +// Rules, in order: entries are trimmed; empty entries are dropped; "*" is a +// wildcard rather than a region name and is passed through untouched; anything +// else gains a leading "#" if it lacks one. Order is preserved and duplicates +// are NOT collapsed, because the observer's ordering carries meaning we do not +// interpret here. Case is left alone: region keys are derived as +// sha256("#name"), so folding case would silently retarget a scope. +// +// An empty or all-empty input returns "", which UpdateNodeConfiguredScope +// stores as a valid "responded, no scopes configured" statement. +func normalizeScopeList(raw string) string { + if strings.TrimSpace(raw) == "" { + return "" + } + parts := strings.Split(raw, ",") + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimSpace(p) + if p == "" { + continue + } + if p != "*" && !strings.HasPrefix(p, "#") { + p = "#" + p + } + out = append(out, p) + } + return strings.Join(out, ",") +} + +// UpdateNodeConfiguredScope records the region scopes a node has CONFIGURED, +// as concrete evidence from an observer /neighbors report (#1865). Unlike +// UpdateNodeDefaultScope (inferred, overwritten on every observation), this is +// only called for the observer's own `self` scopes and for neighbors whose OTA +// scope query returned status="responded" — so the caller, not this method, +// gates on status. An empty scope IS a valid "responded, no scopes configured" +// statement and is stored; a timeout must simply not reach this method. +// +// reportedAt is the report envelope timestamp (ISO-8601). It is stored in +// configured_scope_at and used for last-write-wins: an out-of-order older +// report must not clobber a newer confirmed value. A blank reportedAt skips +// the ordering guard (always writes). +func (s *Store) UpdateNodeConfiguredScope(pubkey, scope, reportedAt string) error { + if pubkey == "" { + return nil + } + // Normalize to canonical UTC RFC3339 so the last-write-wins comparison is + // chronological, not lexicographic (see normalizeReportTS). Stored values + // are therefore always canonical or empty. + reportedAt = normalizeReportTS(reportedAt) + // Canonicalise the scope syntax for the same reason: a value that is stored + // differently from default_scope cannot be compared against it (see + // normalizeScopeList). + scope = normalizeScopeList(scope) + // Last-write-wins: skip if the stored confirmation is newer-or-equal. + if reportedAt != "" { + var curAt sql.NullString + row := s.db.QueryRow(`SELECT configured_scope_at FROM nodes WHERE public_key = ?`, pubkey) + if row.Scan(&curAt) == nil && curAt.Valid && curAt.String != "" && curAt.String >= reportedAt { + return nil + } + } + if _, err := s.db.Exec( + `UPDATE nodes SET configured_scope = ?, configured_scope_at = ? WHERE public_key = ?`, + scope, reportedAt, pubkey); err != nil { + return err + } + // Mirror to inactive_nodes (node may be there if recently moved by retention). + _, err := s.db.Exec( + `UPDATE inactive_nodes SET configured_scope = ?, configured_scope_at = ? WHERE public_key = ?`, + scope, reportedAt, pubkey) + return err +} + // RecordNaiveSkew is called when resolveRxTime() clamps a packet's envelope // timestamp because the observer is emitting a zone-less local-time string // off from UTC by more than 15 min (issue #1478). Stamps the observer's diff --git a/cmd/ingestor/issue1865_test.go b/cmd/ingestor/issue1865_test.go new file mode 100644 index 000000000..4bf3e241e --- /dev/null +++ b/cmd/ingestor/issue1865_test.go @@ -0,0 +1,244 @@ +package main + +// Tests for #1865: ingest the observer /neighbors report as concrete evidence +// of configured region scopes. Covers handleNeighborsReport dispatch semantics +// and the UpdateNodeConfiguredScope store method (status gating, case folding, +// last-write-wins, and the "absence/timeout is not a signal" contract). + +import ( + "database/sql" + "path/filepath" + "testing" +) + +// seedNode inserts a node into both nodes and inactive_nodes (lowercase key). +func seedNode(t *testing.T, store *Store, pubkey string) { + t.Helper() + if _, err := store.db.Exec(`INSERT INTO nodes (public_key, name) VALUES (?, ?)`, pubkey, "n_"+pubkey[:4]); err != nil { + t.Fatalf("seed node %s: %v", pubkey, err) + } + if _, err := store.db.Exec(`INSERT INTO inactive_nodes (public_key, name) VALUES (?, ?)`, pubkey, "n_"+pubkey[:4]); err != nil { + t.Fatalf("seed inactive node %s: %v", pubkey, err) + } +} + +func configuredScope(t *testing.T, store *Store, pubkey string) (sql.NullString, sql.NullString) { + t.Helper() + var sc, at sql.NullString + if err := store.db.QueryRow( + `SELECT configured_scope, configured_scope_at FROM nodes WHERE public_key = ?`, pubkey, + ).Scan(&sc, &at); err != nil { + t.Fatalf("read configured_scope for %s: %v", pubkey, err) + } + return sc, at +} + +func openNeighborsStore(t *testing.T) *Store { + t.Helper() + store, err := OpenStore(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +func TestHandleNeighborsReportWritesSelfAndResponded(t *testing.T) { + store := openNeighborsStore(t) + + // Report keys are UPPERCASE; nodes.public_key is lowercase hex. + const originUpper = "FEEDCA4AD4E2AE615AAAB3CB73FAEC6EF0C7AF4D410F5C58A70FC0F724B7C933" + const respUpper = "B0D17C59FCF580592F8FB78B67D2F0CE9E9187EF3483A765BDFF1D7947A5109C" + const timeoutUpper = "0CE5EA7CFA3AB01D11810EF56B73DD899CD6C58644D6A6832A5C1AE89AFC5E25" + originLower := "feedca4ad4e2ae615aaab3cb73faec6ef0c7af4d410f5c58a70fc0f724b7c933" + respLower := "b0d17c59fcf580592f8fb78b67d2f0ce9e9187ef3483a765bdff1d7947a5109c" + timeoutLower := "0ce5ea7cfa3ab01d11810ef56b73dd899cd6c58644d6a6832a5c1ae89afc5e25" + + seedNode(t, store, originLower) + seedNode(t, store, respLower) + seedNode(t, store, timeoutLower) + // Pre-existing confirmed scope on the timeout node — a timeout must NOT clobber it. + if err := store.UpdateNodeConfiguredScope(timeoutLower, "eu", "2026-07-24T00:00:00Z"); err != nil { + t.Fatal(err) + } + + report := map[string]interface{}{ + "timestamp": "2026-07-25T10:46:14.000000+00:00", + "origin_id": originUpper, + "self": map[string]interface{}{"scopes": "*"}, + "neighbors": []interface{}{ + map[string]interface{}{"pubkey": timeoutUpper, "scopes": "", "status": "timeout"}, + map[string]interface{}{"pubkey": respUpper, "scopes": "de,eu", "status": "responded"}, + }, + } + handleNeighborsReport(store, "test", "obs-topic-id", report) + + // self scopes written under the lowercased origin_id. + if sc, _ := configuredScope(t, store, originLower); !sc.Valid || sc.String != "*" { + t.Errorf("self configured_scope = %v, want '*'", sc) + } + // responded neighbor written, lowercased. + if sc, _ := configuredScope(t, store, respLower); !sc.Valid || sc.String != "#de,#eu" { + t.Errorf("responded configured_scope = %v, want '#de,#eu'", sc) + } + // timeout neighbor untouched — prior confirmed value survives. + if sc, _ := configuredScope(t, store, timeoutLower); !sc.Valid || sc.String != "#eu" { + t.Errorf("timeout node configured_scope = %v, want prior '#eu' (must not be cleared)", sc) + } +} + +func TestHandleNeighborsReportUnknownNeighborIsNoop(t *testing.T) { + store := openNeighborsStore(t) + // No node seeded for this pubkey — UPDATE must match no row (no insert, no error). + report := map[string]interface{}{ + "timestamp": "2026-07-25T10:46:14Z", + "neighbors": []interface{}{ + map[string]interface{}{"pubkey": "aa" + "00000000000000000000000000000000000000000000000000000000000000"[2:], "scopes": "de", "status": "responded"}, + }, + } + handleNeighborsReport(store, "test", "obs", report) + var n int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM nodes`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("nodes count = %d, want 0 (report must not create nodes)", n) + } +} + +func TestHandleNeighborsReportRespondedEmptyScopeIsStored(t *testing.T) { + store := openNeighborsStore(t) + pk := "cc00000000000000000000000000000000000000000000000000000000000001" + seedNode(t, store, pk) + // A responded query with empty scopes is a valid "no scopes configured". + report := map[string]interface{}{ + "timestamp": "2026-07-25T10:46:14Z", + "neighbors": []interface{}{ + map[string]interface{}{"pubkey": pk, "scopes": "", "status": "responded"}, + }, + } + handleNeighborsReport(store, "test", "obs", report) + sc, at := configuredScope(t, store, pk) + if !sc.Valid || sc.String != "" { + t.Errorf("responded-empty configured_scope = %v, want stored empty string", sc) + } + if !at.Valid || at.String == "" { + t.Errorf("configured_scope_at = %v, want the report timestamp", at) + } +} + +func TestUpdateNodeConfiguredScopeLastWriteWins(t *testing.T) { + store := openNeighborsStore(t) + pk := "dd00000000000000000000000000000000000000000000000000000000000001" + seedNode(t, store, pk) + + if err := store.UpdateNodeConfiguredScope(pk, "eu", "2026-07-25T12:00:00Z"); err != nil { + t.Fatal(err) + } + // Older report must NOT clobber the newer confirmed value. + if err := store.UpdateNodeConfiguredScope(pk, "stale", "2026-07-24T00:00:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#eu" { + t.Errorf("configured_scope = %q, want '#eu' (older report must not overwrite)", sc.String) + } + // Newer report updates. + if err := store.UpdateNodeConfiguredScope(pk, "de", "2026-07-26T00:00:00Z"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#de" { + t.Errorf("configured_scope = %q, want '#de' (newer report should update)", sc.String) + } + // inactive_nodes mirrored. + var inactive sql.NullString + if err := store.db.QueryRow(`SELECT configured_scope FROM inactive_nodes WHERE public_key = ?`, pk).Scan(&inactive); err != nil { + t.Fatal(err) + } + if inactive.String != "#de" { + t.Errorf("inactive_nodes.configured_scope = %q, want '#de'", inactive.String) + } +} + +// TestUpdateNodeConfiguredScopeNormalizesAndOrdersByInstant proves the +// last-write-wins guard orders by chronological instant, not by raw string. +// A "+02:00" report that is lexicographically "greater" but chronologically +// EARLIER than the stored UTC value must not win, and stored timestamps are +// canonicalized to UTC "Z" form regardless of the incoming offset/precision. +func TestUpdateNodeConfiguredScopeNormalizesAndOrdersByInstant(t *testing.T) { + store := openNeighborsStore(t) + pk := "ee00000000000000000000000000000000000000000000000000000000000001" + seedNode(t, store, pk) + + // Baseline: noon UTC. + if err := store.UpdateNodeConfiguredScope(pk, "eu", "2026-07-25T12:00:00Z"); err != nil { + t.Fatal(err) + } + if _, at := configuredScope(t, store, pk); at.String != "2026-07-25T12:00:00Z" { + t.Fatalf("stored at = %q, want canonical 'Z' form", at.String) + } + + // "2026-07-25T13:30:00+02:00" == 11:30Z, chronologically EARLIER than 12:00Z, + // but lexicographically GREATER ("13:30..." > "12:00...Z"). Must be skipped. + if err := store.UpdateNodeConfiguredScope(pk, "stale", "2026-07-25T13:30:00+02:00"); err != nil { + t.Fatal(err) + } + if sc, _ := configuredScope(t, store, pk); sc.String != "#eu" { + t.Errorf("configured_scope = %q, want '#eu' (earlier +02:00 report must not win)", sc.String) + } + + // "2026-07-25T15:00:00+02:00" == 13:00Z, chronologically LATER. Must update, + // and be stored canonicalized to UTC. + if err := store.UpdateNodeConfiguredScope(pk, "de", "2026-07-25T15:00:00+02:00"); err != nil { + t.Fatal(err) + } + sc, at := configuredScope(t, store, pk) + if sc.String != "#de" { + t.Errorf("configured_scope = %q, want '#de' (later +02:00 report should win)", sc.String) + } + if at.String != "2026-07-25T13:00:00Z" { + t.Errorf("stored at = %q, want canonical '2026-07-25T13:00:00Z'", at.String) + } + + // Firmware's real format (microseconds + "+00:00") canonicalizes to "Z". + if err := store.UpdateNodeConfiguredScope(pk, "dk", "2026-07-26T09:43:48.000000+00:00"); err != nil { + t.Fatal(err) + } + if _, at := configuredScope(t, store, pk); at.String != "2026-07-26T09:43:48Z" { + t.Errorf("stored at = %q, want canonical '2026-07-26T09:43:48Z'", at.String) + } +} + +// #1865: the OTA scope query returns bare region names while every other +// stored scope in CoreScope carries the leading "#". normalizeScopeList makes +// configured_scope comparable with default_scope; these cases pin the parts +// that are easy to get wrong when someone later "simplifies" it. +func TestNormalizeScopeList(t *testing.T) { + cases := []struct{ name, in, want string }{ + {"empty stays empty (responded, no scopes)", "", ""}, + {"whitespace only is empty, not \"#\"", " ", ""}, + {"bare region gains the hash", "dk", "#dk"}, + {"already hashed is untouched", "#dk", "#dk"}, + {"wildcard is not a region name", "*", "*"}, + {"real observer report, mixed", "*,dk,eu,europe,dk-aarhus", "*,#dk,#eu,#europe,#dk-aarhus"}, + {"already-hashed report is idempotent", "*,#de,#de-sl", "*,#de,#de-sl"}, + {"mixed hashed and bare in one report", "#de,sl", "#de,#sl"}, + {"surrounding spaces are trimmed", " dk , eu ", "#dk,#eu"}, + {"empty entries are dropped, not turned into \"#\"", "dk,,eu,", "#dk,#eu"}, + {"order is preserved", "eu,dk", "#eu,#dk"}, + {"duplicates are kept, not collapsed", "dk,dk", "#dk,#dk"}, + {"case is left alone (keys are sha256 of the name)", "DK", "#DK"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := normalizeScopeList(c.in); got != c.want { + t.Errorf("normalizeScopeList(%q) = %q, want %q", c.in, got, c.want) + } + }) + } + // Applying it twice must not add a second "#": the ingest path can see the + // same report again after a reconnect. + once := normalizeScopeList("*,dk,#eu") + if twice := normalizeScopeList(once); twice != once { + t.Errorf("not idempotent: %q then %q", once, twice) + } +} diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 6149afebd..ec4ef406c 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -726,6 +726,17 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, return } + // Neighbors report topic: meshcore///neighbors (#1865). + // The ESP32 observer firmware emits a periodic neighbor report carrying its + // own configured region scopes (`self`) plus, for each zero-hop neighbor, + // the scopes fetched via an OTA scope query. Like /status this is observer + // metadata (region-independent), so the per-source packet IATA filter below + // does not apply. + if len(parts) >= 4 && parts[3] == "neighbors" { + handleNeighborsReport(store, tag, parts[2], msg) + return + } + // Status topic: meshcore///status // Per-source IATA filter does NOT apply here — observer metadata (noise_floor, battery, etc.) // is region-independent and should be accepted from all observers regardless of @@ -1659,6 +1670,57 @@ func init() { } } +// handleNeighborsReport ingests an observer /neighbors report (#1865) and +// records CONFIRMED region scopes into nodes.configured_scope: +// - the observer's own scopes from `self`, keyed by origin_id (the observer +// node pubkey), which need no OTA query and are always trusted; and +// - each neighbor whose OTA scope query returned status=="responded". +// +// Per the report contract: neighbors with any other status (e.g. "timeout") +// are skipped — a failed query is NOT evidence the scopes were cleared — and a +// missing neighbor is never a signal (the report is 10 KB-capped and truncates +// by ordering, so absent != gone). Report pubkeys are uppercase; nodes.public_key +// is lowercase hex, so keys are lowercased before the UPDATE. Unknown neighbors +// are a no-op (the UPDATE matches no row) until a later advert creates the node. +func handleNeighborsReport(store *Store, tag string, observerID string, msg map[string]interface{}) { + reportedAt, _ := msg["timestamp"].(string) + + // self: the observer's own configured scopes. + originID, _ := msg["origin_id"].(string) + if originID == "" { + originID = observerID + } + originID = strings.ToLower(originID) + if self, ok := msg["self"].(map[string]interface{}); ok && originID != "" { + if sc, ok := self["scopes"].(string); ok { + if err := store.UpdateNodeConfiguredScope(originID, sc, reportedAt); err != nil { + log.Printf("MQTT [%s] neighbors self scope error: %v", tag, err) + } + } + } + + // neighbors[]: only status=="responded" carries usable scope evidence. + neighbors, _ := msg["neighbors"].([]interface{}) + for _, raw := range neighbors { + n, ok := raw.(map[string]interface{}) + if !ok { + continue + } + if status, _ := n["status"].(string); status != "responded" { + continue + } + pubkey, _ := n["pubkey"].(string) + pubkey = strings.ToLower(pubkey) + if pubkey == "" { + continue + } + scopes, _ := n["scopes"].(string) + if err := store.UpdateNodeConfiguredScope(pubkey, scopes, reportedAt); err != nil { + log.Printf("MQTT [%s] neighbors scope error for %.8s: %v", tag, pubkey, err) + } + } +} + // shouldUpdateDefaultScope returns true when the packet carries a transport // scope whose region key matched (#1534). Without the ScopeName non-empty // guard, transport-scoped adverts from non-matching regions would overwrite diff --git a/cmd/server/db.go b/cmd/server/db.go index 980099459..70c10f111 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -42,6 +42,7 @@ type DB struct { hasObsRawHex bool // observations table has raw_hex column (#881) hasScopeName bool // transmissions.scope_name column exists (#899) hasDefaultScope bool // nodes.default_scope column exists (#899) + hasConfiguredScope bool // nodes.configured_scope column exists (#1865) hasMultibyteSupCols bool // nodes/inactive_nodes have multibyte_sup/multibyte_evidence (#903) hasLastSeen bool // transmissions.last_seen column exists (#1690) @@ -234,6 +235,7 @@ func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { } db.hasDefaultScope = nodes["default_scope"] db.hasMultibyteSupCols = nodes["multibyte_sup"] + db.hasConfiguredScope = nodes["configured_scope"] if db.isV3 { log.Printf("[db] schema mode: v3 (observer_idx)") @@ -282,6 +284,10 @@ func (db *DB) nodeSelectCols() string { if db.hasDefaultScope { cols += ", default_scope" } + // #1865: confirmed scopes appended after default_scope; scan order must match. + if db.hasConfiguredScope { + cols += ", configured_scope, configured_scope_at" + } return cols } @@ -2465,11 +2471,15 @@ func (db *DB) scanNodeRow(rows *sql.Rows) map[string]interface{} { var temperatureC sql.NullFloat64 var foreign sql.NullInt64 var defaultScope sql.NullString + var configuredScope, configuredScopeAt sql.NullString scanArgs := []interface{}{&pk, &name, &role, &lat, &lon, &lastSeen, &firstSeen, &advertCount, &batteryMv, &temperatureC, &foreign} if db.hasDefaultScope { scanArgs = append(scanArgs, &defaultScope) } + if db.hasConfiguredScope { + scanArgs = append(scanArgs, &configuredScope, &configuredScopeAt) + } if err := rows.Scan(scanArgs...); err != nil { return nil } @@ -2500,6 +2510,10 @@ func (db *DB) scanNodeRow(rows *sql.Rows) map[string]interface{} { if db.hasDefaultScope { m["default_scope"] = nullStr(defaultScope) } + if db.hasConfiguredScope { + m["configured_scope"] = nullStr(configuredScope) + m["configured_scope_at"] = nullStr(configuredScopeAt) + } return m } diff --git a/cmd/server/node_reach.go b/cmd/server/node_reach.go index 8cb1d9283..08195726a 100644 --- a/cmd/server/node_reach.go +++ b/cmd/server/node_reach.go @@ -187,6 +187,48 @@ func newResolver(pm *prefixMap) func(string) string { } } +// reachNodeScopes returns the node's inferred default_scope and, when an +// observer /neighbors report has confirmed it, the configured_scope and the +// instant that confirmation was reported (#1865). +// +// Both columns are optional: they are added by migration and an older database +// simply lacks them, so each is gated on the schema flag the handle probed at +// open time. Every return value is best-effort — a missing row, a missing +// column or a scan error yields empty strings, because a reach report must +// still render for a node we know nothing else about. +func (s *Server) reachNodeScopes(ctx context.Context, pubkey string) (string, string, string) { + if s.db == nil || s.db.conn == nil { + return "", "", "" + } + cols := make([]string, 0, 3) + if s.db.hasDefaultScope { + cols = append(cols, "default_scope") + } + if s.db.hasConfiguredScope { + cols = append(cols, "configured_scope", "configured_scope_at") + } + if len(cols) == 0 { + return "", "", "" + } + vals := make([]sql.NullString, len(cols)) + args := make([]interface{}, len(cols)) + for i := range vals { + args[i] = &vals[i] + } + row := s.db.conn.QueryRowContext(ctx, + "SELECT "+strings.Join(cols, ", ")+" FROM nodes WHERE public_key = ?", pubkey) + if err := row.Scan(args...); err != nil { + return "", "", "" + } + out := map[string]string{} + for i, c := range cols { + if vals[i].Valid { + out[c] = vals[i].String + } + } + return out["default_scope"], out["configured_scope"], out["configured_scope_at"] +} + type NodeReachInfo struct { Pubkey string `json:"pubkey"` Name string `json:"name"` @@ -194,6 +236,15 @@ type NodeReachInfo struct { Lat *float64 `json:"lat"` Lon *float64 `json:"lon"` FirstSeen string `json:"first_seen"` + // #1865: region scopes, so a printed reach report says which region the + // node actually serves. DefaultScope is INFERRED from the transport scope + // of observed adverts; ConfiguredScope is CONFIRMED, read back off the node + // itself via an observer /neighbors report. Both omitempty: an instance + // whose schema predates either column, or a node that has never been + // confirmed, sends neither field and the UI shows nothing. + DefaultScope string `json:"default_scope,omitempty"` + ConfiguredScope string `json:"configured_scope,omitempty"` + ConfiguredScopeAt string `json:"configured_scope_at,omitempty"` } type NodeReachWindow struct { Days int `json:"days"` @@ -512,6 +563,10 @@ func (s *Server) computeNodeReach(ctx context.Context, pubkey string, days int) // observer-only or pre-first_seen-schema). firstSeen := self.FirstSeen + // #1865: scopes for the report header. Single-row lookup rather than widening + // buildNodeInfoMap's bulk SELECT, which every other consumer would pay for. + defScope, confScope, confAt := s.reachNodeScopes(ctx, pubkey) + // assemble links links := make([]NodeReachLink, 0, len(d.we)+len(d.they)) bidir := 0 @@ -576,7 +631,8 @@ func (s *Server) computeNodeReach(ctx context.Context, pubkey string, days int) selfLat, selfLon := gpsPtrs(self) return NodeReachResponse{ Node: NodeReachInfo{Pubkey: pubkey, Name: self.Name, Role: self.Role, - Lat: selfLat, Lon: selfLon, FirstSeen: firstSeen}, + Lat: selfLat, Lon: selfLon, FirstSeen: firstSeen, + DefaultScope: defScope, ConfiguredScope: confScope, ConfiguredScopeAt: confAt}, Window: NodeReachWindow{Days: days, Since: since.Format(time.RFC3339)}, ReliableTokens: toks, Importance: NodeReachImportance{ diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index 58e378743..8518be281 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -78,6 +78,9 @@ func Apply(rw *sql.DB, logf Logger) error { if err := ensureDefaultScopeColumns(rw, logf); err != nil { return fmt.Errorf("ensure default_scope: %w", err) } + if err := ensureConfiguredScopeColumns(rw, logf); err != nil { + return fmt.Errorf("ensure configured_scope: %w", err) + } if err := ensureObservationsRawHexColumn(rw, logf); err != nil { return fmt.Errorf("ensure observations.raw_hex: %w", err) } @@ -149,6 +152,9 @@ func AssertReady(ro *sql.DB) error { mustCol("transmissions", "scope_name") mustCol("nodes", "default_scope") mustCol("inactive_nodes", "default_scope") + // #1865: confirmed region scopes from the observer /neighbors report. + mustCol("nodes", "configured_scope") + mustCol("inactive_nodes", "configured_scope") mustCol("observations", "raw_hex") // Multi-byte capability cache (#1324 follow-up; PR #903 surface). // Owned by ingestor — server reads these for O(1) /api/nodes @@ -469,6 +475,39 @@ func ensureDefaultScopeColumns(rw *sql.DB, logf Logger) error { return nil } +// ensureConfiguredScopeColumns adds nodes.configured_scope + +// nodes.configured_scope_at (and mirrors on inactive_nodes) for #1865. +// Unlike default_scope (inferred from observed advert transport scope, and +// overwritten on every observation), configured_scope holds the region scopes +// a node has CONFIGURED, taken as concrete evidence from the observer +// /neighbors report — written only for the observer's own `self` scopes and +// for neighbors whose OTA scope query returned status="responded". The +// server PRAGMA-detects configured_scope as hasConfiguredScope. +func ensureConfiguredScopeColumns(rw *sql.DB, logf Logger) error { + if err := ensureMigrationsTable(rw); err != nil { + return err + } + for _, table := range []string{"nodes", "inactive_nodes"} { + for _, col := range []string{"configured_scope", "configured_scope_at"} { + has, err := TableHasColumn(rw, table, col) + if err != nil { + return fmt.Errorf("inspect %s.%s: %w", table, col, err) + } + if has { + continue + } + if _, err := rw.Exec(fmt.Sprintf(`ALTER TABLE %s ADD COLUMN %s TEXT DEFAULT NULL`, table, col)); err != nil { + return fmt.Errorf("alter %s add %s: %w", table, col, err) + } + logf("[dbschema] added %s column to %s (#1865)", col, table) + } + } + if _, err := rw.Exec(`INSERT OR IGNORE INTO _migrations (name) VALUES ('nodes_configured_scope_v1')`); err != nil { + return fmt.Errorf("record nodes_configured_scope_v1: %w", err) + } + return nil +} + // ensureObservationsRawHexColumn adds observations.raw_hex (#881). // Source of truth lives here per #1321 (was previously cmd/ingestor/db.go only): // the server PRAGMA-detects this column as hasObsRawHex. diff --git a/public/live.js b/public/live.js index f5f2923de..8d8e7b28f 100644 --- a/public/live.js +++ b/public/live.js @@ -2629,6 +2629,7 @@ + ${'configured_scope' in n && n.configured_scope !== null ? `` : ''} ${'default_scope' in n ? ``; })() : ''} ${(n.role === 'repeater' || n.role === 'room') && Array.isArray(n.transported_scopes) && n.transported_scopes.length ? `` : ''} + ${'configured_scope' in n && n.configured_scope !== null ? `` : ''} diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index bca9b6a28..324cbcb27 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -7007,3 +7007,63 @@ console.log('\n=== observers.js: healthStatus (configurable thresholds) ==='); assert.strictEqual(r.label, 'Unknown'); }); } + +// ===== node-reach.js: scopeLineHtml (#1865) ===== +// The reach report states which region a node serves. Two claims that must +// never be conflated: "Scope" is INFERRED from observed advert transport +// scopes, "Configured scope" is CONFIRMED by reading it back off the node via +// an observer /neighbors report. Asserting the rendered markup, not the source. +console.log('\n=== node-reach.js: scopeLineHtml (#1865) ==='); +{ + const ctx = makeSandbox(); + ctx.registerPage = () => {}; + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/node-reach.js'); + const scopeLineHtml = ctx.__meshcoreReachInternals.scopeLineHtml; + + test('no scope data at all renders nothing', () => { + assert.strictEqual(scopeLineHtml({}), ''); + }); + + test('inferred-only says "Scope" and marks it observed', () => { + const h = scopeLineHtml({ default_scope: '#be' }); + assert.ok(h.includes('>Scope <'), 'should label it Scope'); + assert.ok(h.includes('#be'), 'should show the value'); + assert.ok(h.includes('observed'), 'should mark provenance as observed'); + assert.ok(!h.includes('nq-scope-ok'), 'inferred data must not get the confirmed tick'); + }); + + test('confirmed wins over inferred and gets the tick', () => { + const h = scopeLineHtml({ default_scope: '#be', configured_scope: '#be,#eu' }); + assert.ok(h.includes('Configured scope'), 'should label it Configured scope'); + assert.ok(h.includes('nq-scope-ok'), 'confirmed data gets the tick'); + assert.ok(h.includes('#be,#eu'), 'should show the confirmed value'); + assert.ok(!h.includes('>Scope <'), 'must not also render the inferred line'); + }); + + test('confirmed-but-empty is a statement, not missing data', () => { + const h = scopeLineHtml({ configured_scope: '' }); + assert.ok(h.includes('none configured'), 'empty confirmed value must render explicitly'); + assert.ok(h.includes('nq-scope-ok'), 'it is still a confirmation'); + }); + + test('empty confirmed value does not fall back to the inferred one', () => { + // A node that answered "I have no scopes" must not be shown its old + // inferred guess instead: that would silently contradict the node. + const h = scopeLineHtml({ default_scope: '#be', configured_scope: '' }); + assert.ok(h.includes('none configured')); + assert.ok(!h.includes('#be'), 'inferred value must not leak back in'); + }); + + test('confirmation instant is carried in the title, not the visible line', () => { + const h = scopeLineHtml({ configured_scope: '#dk', configured_scope_at: '2026-07-26T09:43:48Z' }); + assert.ok(h.includes('2026-07-26T09:43:48Z'), 'timestamp should be present'); + assert.ok(h.includes('title='), 'and it should live in a title attribute'); + }); + + test('scope values are HTML-escaped', () => { + const h = scopeLineHtml({ configured_scope: '' }); + assert.ok(!h.includes(' Date: Sun, 6 Sep 2026 21:11:29 +0200 Subject: [PATCH 007/151] feat(#1794): per-IP limits and a deny list on the /ws upgrade (#1974) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1794. Follow-up to #1793, decided **before** the upgrade because the handshake is the resource being protected. - Deny list of addresses and CIDRs → 403 - Per-IP concurrent connection cap → 403 - Per-IP upgrade rate limit over a rolling minute → **429**, not 403: a temporary refusal should not read as "never come back" - Rejection counters split by cause in `/api/stats` under `websocket` ### The decision this feature lives or dies on Most CoreScope installs sit behind nginx, Caddy, Traefik or an ingress. `cdn_detection.go` says so in as many words: it deliberately excludes `X-Forwarded-For` from its CDN signals precisely because *every* reverse-proxied install sets it. For those deployments `r.RemoteAddr` is the proxy, `127.0.0.1` for every visitor on earth. A per-IP cap keyed on that address protects nobody and hands the sixth legitimate browser tab a 403. That is a self-inflicted outage wearing the costume of hardening. So: - **`X-Forwarded-For` is believed only from an address listed in `webSocket.trustedProxies`.** From anywhere else it is attacker-supplied, and trusting it would let anyone mint a fresh source IP per connection, which is strictly worse than having no limit at all. - **When the peer looks like a local reverse proxy and no `trustedProxies` is set, the per-IP limits are skipped**, and one warning names the setting that fixes it. Silently refusing real users is the worse failure. - **The deny list still applies there**, because it is the operator's explicit instruction rather than an inference. That is the answer to @mcode6726's question on the thread: it is neither "always the socket address" nor "always the header", and the operator decides which by naming their proxy. ### Two deliberate departures from the issue body **`maxConnsPerIP` ships as 0 (off), not 5.** Carrier-grade NAT puts thousands of unrelated mobile subscribers behind a single public IPv4. A cap of 5 refuses real visitors on phones while a scraper simply rents more addresses: all of the cost, none of the benefit. `upgradesPerMinPerIP` ships at **30 and on**, because that one *is* safe under CGNAT: a real client upgrades a handful of times per minute even while reconnecting, so 30 leaves ordinary traffic untouched while flattening a reconnect loop. A pointer type distinguishes "unset" from an explicit `0` that turns it off. **The default deny list is not shipped.** The thread proposed seeding 44 CIDRs for one VPS provider after a single scraper was seen at `23.111.177.6`. I have left it out: blanket-blocking a hosting provider by default breaks legitimate operators who host there, is undiscoverable by the person locked out (they see a bare 403), and ages badly as ranges get reassigned. The mechanism is here and `config.example.json` shows exactly how to configure it, so any operator who wants that list can have it in one line. If you want it shipped as a default anyway, that is your call as maintainer and it is a one-line change. ### Verification 19 tests, including all five the issue specifies as TDD requirements, each marked with the issue's own wording. Beyond those five: - a **bare address** in the deny list works, not just CIDR form. Operators write `1.2.3.4`, and silently ignoring that would be the worst possible failure for a deny list: it looks configured and blocks nothing - an unparseable deny entry is skipped and logged, not fatal. One typo must not take the server down - one client behind a trusted proxy does **not** exhaust another client's budget behind the same proxy, which is the entire point of honouring XFF - changing a forged XFF from an untrusted peer buys no fresh budget - `release` frees a slot and is **idempotent**, because `Unregister` can run twice for one client and double-crediting would leak slots - a **rejected** upgrade does not consume rate budget, or a retrying client could never recover once its window cleared - limits skipped for loopback and private peers; deny list applies anyway - a nil limiter allows everything, so a `Hub` built without `ConfigureLimits` behaves exactly as before - idle per-IP state is collected, while a record with a live connection never is Full `cmd/server` suite green, `gofmt` clean. ### Not done - No runtime config reload; restart required. Listed as optional in the issue. - No `WS_DENY_IPS` env override. Also listed as optional. - From the OWASP expansion in the first comment: `maxPayload` and the idle/read timeout are **already in master** (`SetReadLimit`, `SetReadDeadline`). The ping/pong heartbeat is not, and is not in this PR either; it is a separate change to the read/write pumps and belongs in its own review. Co-authored-by: Claude Opus 5 (1M context) --- cmd/server/config.go | 79 ++++++++++ cmd/server/main.go | 2 + cmd/server/routes.go | 7 +- cmd/server/types.go | 7 + cmd/server/websocket.go | 57 ++++++- cmd/server/ws_limits.go | 274 ++++++++++++++++++++++++++++++++ cmd/server/ws_limits_test.go | 296 +++++++++++++++++++++++++++++++++++ config.example.json | 10 ++ 8 files changed, 727 insertions(+), 5 deletions(-) create mode 100644 cmd/server/ws_limits.go create mode 100644 cmd/server/ws_limits_test.go diff --git a/cmd/server/config.go b/cmd/server/config.go index 28605c2bc..f6e22ee9e 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -147,6 +147,10 @@ type Config struct { // so browsers enforce same-origin policy. Set to ["*"] to allow all origins. CORSAllowedOrigins []string `json:"corsAllowedOrigins,omitempty"` + // WebSocket carries the /ws transport limits from #1794. Omitted entirely + // means: rate limit at its default, connection cap off, no deny list. + WebSocket *WebSocketConfig `json:"webSocket,omitempty"` + DebugAffinity bool `json:"debugAffinity,omitempty"` // MapDarkTileProvider selects the default dark-mode basemap provider for @@ -474,6 +478,81 @@ func LoadConfig(baseDirs ...string) (*Config, error) { return cfg, nil // defaults } +// WebSocketConfig holds the /ws transport limits (#1794). These sit behind +// the CheckOrigin allowlist (#1793) and catch non-browser clients, which can +// omit or forge Origin. +type WebSocketConfig struct { + // MaxConnsPerIP caps concurrent /ws connections from one client address. + // DEFAULT 0, meaning OFF, and that is a deliberate departure from the + // value floated on #1794. + // + // A cap of 5 is safe only when one address means one household. It does + // not on mobile: carrier-grade NAT puts thousands of unrelated subscribers + // behind a single public IPv4, so a low cap would refuse real visitors on + // phones while barely inconveniencing a scraper that can rent more + // addresses. Operators who know their audience can set it; we must not + // pick it for them. + MaxConnsPerIP int `json:"maxConnsPerIP,omitempty"` + + // UpgradesPerMinPerIP caps handshakes per minute from one client address. + // DEFAULT 30, on. Unlike the connection cap this is safe under CGNAT: a + // real client upgrades a handful of times per minute even while + // reconnecting, so 30 leaves ordinary traffic untouched while flattening + // the reconnect loop that makes scrapers expensive. + UpgradesPerMinPerIP *int `json:"upgradesPerMinPerIP,omitempty"` + + // TrustedProxies lists the addresses or CIDRs of reverse proxies whose + // X-Forwarded-For may be believed. REQUIRED for the limits to do anything + // behind nginx/Caddy/Traefik/ingress: without it every visitor shares the + // proxy's address, so the limits are skipped and a warning is logged. + // Never list an address you do not control; anyone reaching the server + // from a trusted address can name any client IP they like. + TrustedProxies []string `json:"trustedProxies,omitempty"` + + // Deny blocks addresses outright at the upgrade, before the handshake. + // Accepts both bare addresses ("1.2.3.4") and CIDRs ("1.2.3.0/24"). + // Applies even when clients cannot be told apart, because it is an + // explicit instruction rather than an inference. + Deny []string `json:"deny,omitempty"` +} + +// WSMaxConnsPerIP returns the configured concurrent-connection cap, 0 = off. +func (c *Config) WSMaxConnsPerIP() int { + if c.WebSocket == nil || c.WebSocket.MaxConnsPerIP < 0 { + return 0 + } + return c.WebSocket.MaxConnsPerIP +} + +// WSUpgradesPerMinPerIP returns the upgrade-rate cap, defaulting to 30 when +// unset. A pointer distinguishes "not configured" (use the default) from an +// explicit 0, which an operator uses to turn the limit off. +func (c *Config) WSUpgradesPerMinPerIP() int { + if c.WebSocket == nil || c.WebSocket.UpgradesPerMinPerIP == nil { + return 30 + } + if v := *c.WebSocket.UpgradesPerMinPerIP; v > 0 { + return v + } + return 0 +} + +// WSTrustedProxies returns the configured reverse-proxy addresses, if any. +func (c *Config) WSTrustedProxies() []string { + if c.WebSocket == nil { + return nil + } + return c.WebSocket.TrustedProxies +} + +// WSDeny returns the configured deny entries, if any. +func (c *Config) WSDeny() []string { + if c.WebSocket == nil { + return nil + } + return c.WebSocket.Deny +} + func (c *Config) applyListLimitsDefaults() { if c.ListLimits == nil { c.ListLimits = &ListLimitsConfig{} diff --git a/cmd/server/main.go b/cmd/server/main.go index c4166d1e6..00381c4dd 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -358,6 +358,8 @@ func main() { // WebSocket hub hub := NewHub() hub.SetAllowedOrigins(cfg.CORSAllowedOrigins) + hub.ConfigureLimits(cfg.WSMaxConnsPerIP(), cfg.WSUpgradesPerMinPerIP(), + cfg.WSTrustedProxies(), cfg.WSDeny()) // #1794 hub.upgrader.EnableCompression = cfg.WSCompressionEnabled() // HTTP server diff --git a/cmd/server/routes.go b/cmd/server/routes.go index c6b0c0714..ca844a6ab 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -685,8 +685,10 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { uptime := time.Since(s.startedAt).Seconds() wsClients := 0 + var wsDeny, wsRate, wsConnCap int64 if s.hub != nil { wsClients = s.hub.ClientCount() + wsDeny, wsRate, wsConnCap = s.hub.limits.counts() // #1794; nil-safe } // Real packet store stats @@ -758,8 +760,9 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { P95Ms: round(percentile(sortedPauses, 0.95), 1), P99Ms: round(percentile(sortedPauses, 0.99), 1), }, - Cache: cs, - WebSocket: WebSocketStatsResp{Clients: wsClients}, + Cache: cs, + WebSocket: WebSocketStatsResp{Clients: wsClients, + RejectedDeny: wsDeny, RejectedRate: wsRate, RejectedConnCap: wsConnCap}, PacketStore: HealthPacketStoreStats{ Packets: pktCount, EstimatedMB: pktEstMB, diff --git a/cmd/server/types.go b/cmd/server/types.go index 4c5e547b4..4bf02da0f 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -158,6 +158,13 @@ type PerfCacheStats struct { type WebSocketStatsResp struct { Clients int `json:"clients"` + // #1794: upgrade rejections since boot, split by cause so an operator can + // tell a deny-list hit from a client that is merely reconnecting too fast. + // omitempty: an install with no limits configured sends none of these + // rather than three permanent zeroes. + RejectedDeny int64 `json:"rejectedDeny,omitempty"` + RejectedRate int64 `json:"rejectedRate,omitempty"` + RejectedConnCap int64 `json:"rejectedConnCap,omitempty"` } type HealthPacketStoreStats struct { diff --git a/cmd/server/websocket.go b/cmd/server/websocket.go index f40c2b438..923c04c29 100644 --- a/cmd/server/websocket.go +++ b/cmd/server/websocket.go @@ -17,7 +17,8 @@ type Hub struct { mu sync.RWMutex clients map[*Client]bool upgrader websocket.Upgrader - allowedOrigins []string // exact-match allowlist for /ws CheckOrigin (see SetAllowedOrigins) + allowedOrigins []string // exact-match allowlist for /ws CheckOrigin (see SetAllowedOrigins) + limits *wsLimiter // #1794: per-IP caps and deny list; nil allows everything } // SetAllowedOrigins configures the exact-match origin allowlist consulted by @@ -78,6 +79,32 @@ type Client struct { conn *websocket.Conn send chan []byte closeOnce sync.Once + // release returns this connection's slot to the per-IP concurrent cap + // (#1794). Always non-nil so Unregister needs no nil check; it is a no-op + // when limits are disabled. + release func() +} + +// ConfigureLimits installs the #1794 transport limits. Called once at +// startup, before the listener binds, so no upgrade can race a half-built +// limiter. Passing zero values for both caps leaves only the deny list +// active, which is a legitimate configuration. +func (h *Hub) ConfigureLimits(maxConnsPerIP, upgradesPerMin int, trustedProxies, deny []string) { + l := newWSLimiter() + l.maxConnsPerIP = maxConnsPerIP + l.upgradesPerMin = upgradesPerMin + l.trustedProxies = parseCIDRList(trustedProxies, "webSocket.trustedProxies") + l.denyNets = parseCIDRList(deny, "webSocket.deny") + h.mu.Lock() + h.limits = l + h.mu.Unlock() + if len(l.denyNets) > 0 { + log.Printf("[ws] deny list active: %d network(s)", len(l.denyNets)) + } + if len(l.trustedProxies) == 0 && (maxConnsPerIP > 0 || upgradesPerMin > 0) { + log.Printf("[ws] per-IP limits configured with no trustedProxies: they apply only to " + + "directly-connected clients. Behind a reverse proxy, set webSocket.trustedProxies.") + } } func NewHub() *Hub { @@ -112,6 +139,12 @@ func (h *Hub) Unregister(c *Client) { c.closeOnce.Do(func() { close(c.send) }) } h.mu.Unlock() + // #1794: outside the lock — release takes the limiter's own mutex, and + // holding two locks in one order here and the other order elsewhere is how + // deadlocks are made. Idempotent, so a double Unregister cannot over-credit. + if c.release != nil { + c.release() + } log.Printf("[ws] client disconnected (%d total)", h.ClientCount()) } @@ -151,15 +184,33 @@ func (h *Hub) Broadcast(msg interface{}) { // ServeWS handles the WebSocket upgrade and runs the client. func (h *Hub) ServeWS(w http.ResponseWriter, r *http.Request) { + // #1794: decide BEFORE the upgrade. Rejecting afterwards would already + // have allocated the connection and completed the handshake, which is the + // resource this is meant to protect. + ok, reason, key, release := h.limits.allow(r) + if !ok { + status := http.StatusForbidden + if reason == wsRejectRate { + // 429 tells a well-behaved client to back off; 403 would read as + // "never come back" for what is a temporary refusal. + status = http.StatusTooManyRequests + } + log.Printf("[ws] reject ip=%s reason=%s", key, reason) + http.Error(w, "websocket upgrade refused", status) + return + } + conn, err := h.upgrader.Upgrade(w, r, nil) if err != nil { log.Printf("[ws] upgrade error: %v", err) + release() // the slot was charged above and this connection never happened return } client := &Client{ - conn: conn, - send: make(chan []byte, 256), + conn: conn, + send: make(chan []byte, 256), + release: release, } h.Register(client) diff --git a/cmd/server/ws_limits.go b/cmd/server/ws_limits.go new file mode 100644 index 000000000..9638b9e33 --- /dev/null +++ b/cmd/server/ws_limits.go @@ -0,0 +1,274 @@ +package main + +// Issue #1794: per-IP limits and a deny list for the /ws upgrade, as defence +// in depth behind the CheckOrigin allowlist from #1793. +// +// CheckOrigin only stops browsers. A Go, Python or curl client can omit the +// Origin header entirely or forge one, connect, and sit in the hub. These +// limits work on the transport instead: who is connecting, how often, and how +// many at once. +// +// THE THING THAT MAKES THIS DANGEROUS TO SHIP NAIVELY, stated up front: +// most CoreScope installs run behind nginx, Caddy, Traefik or a k8s ingress +// (cdn_detection.go says so in as many words). For those, r.RemoteAddr is the +// proxy, 127.0.0.1 for every visitor on earth. A per-IP cap keyed on that +// address protects nobody; it counts the whole internet as one client and +// hands the sixth legitimate browser tab a 403. That is a self-inflicted +// outage, not hardening. +// +// So enforcement is conditional on being able to tell clients apart: +// - X-Forwarded-For is honoured ONLY when the request arrives from an +// address the operator listed in trustedProxies. Otherwise it is an +// attacker-supplied header, and trusting it would let anyone forge a +// fresh source IP per connection, which is worse than no limit at all. +// - If the peer looks like a local reverse proxy and no trustedProxies is +// configured, clients cannot be distinguished, so the per-IP limits are +// SKIPPED and one warning is logged telling the operator what to set. +// The deny list still applies, because an explicit block is the +// operator's own instruction rather than an inference. +// +// Not shipped: a default deny list of hosting-provider ranges. It was +// proposed on the issue (44 CIDRs for one VPS host, after a single scraper +// was seen). Blanket-blocking a provider by default breaks legitimate +// operators who host there, is undiscoverable by the person locked out, and +// ages badly as ranges are reassigned. Operators who want it can configure it. + +import ( + "log" + "net" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" +) + +// wsUpgradeWindow is the span the upgrade budget is counted over. One minute +// matches the config field name (upgradesPerMinPerIP) so an operator reading +// the config can predict the behaviour without reading this file. +const wsUpgradeWindow = time.Minute + +// wsBucketIdleGC is how long an idle per-IP record is kept before collection. +// Longer than the window, so a client that pauses briefly cannot earn a fresh +// budget by being forgotten. +const wsBucketIdleGC = 10 * time.Minute + +// wsRejectReason is the label used both in the log line and in the +// /api/stats counters, so an operator can match one to the other. +type wsRejectReason string + +const ( + wsRejectDeny wsRejectReason = "deny" + wsRejectRate wsRejectReason = "rate" + wsRejectConnCap wsRejectReason = "conncap" +) + +type wsIPState struct { + conns int + upgrades []time.Time // timestamps inside the current window + lastSeenAt time.Time +} + +// wsLimiter holds the parsed configuration and the live per-IP state. A nil +// *wsLimiter allows everything, so a server that never configures limits +// behaves exactly as it did before this change. +type wsLimiter struct { + mu sync.Mutex + + maxConnsPerIP int // 0 disables the concurrent-connection cap + upgradesPerMin int // 0 disables the upgrade-rate limit + + trustedProxies []*net.IPNet + denyNets []*net.IPNet + + state map[string]*wsIPState + + rejectedDeny atomic.Int64 + rejectedRate atomic.Int64 + rejectedConnCap atomic.Int64 + + // warnedIndistinct fires the "set trustedProxies" warning once, rather + // than on every upgrade from behind an unconfigured proxy. + warnedIndistinct sync.Once +} + +func newWSLimiter() *wsLimiter { + return &wsLimiter{state: make(map[string]*wsIPState)} +} + +// parseCIDRList turns operator strings into networks. A bare address is +// accepted and treated as a single-host network, because "1.2.3.4" is what an +// operator naturally writes and silently ignoring it would be the worst +// possible failure mode for a deny list. An unparseable entry is logged and +// skipped rather than aborting startup: one typo must not take the server +// down, but it must not pass unnoticed either. +func parseCIDRList(entries []string, what string) []*net.IPNet { + out := make([]*net.IPNet, 0, len(entries)) + for _, raw := range entries { + e := strings.TrimSpace(raw) + if e == "" { + continue + } + if _, n, err := net.ParseCIDR(e); err == nil { + out = append(out, n) + continue + } + if ip := net.ParseIP(e); ip != nil { + bits := 32 + if ip.To4() == nil { + bits = 128 + } + out = append(out, &net.IPNet{IP: ip, Mask: net.CIDRMask(bits, bits)}) + continue + } + log.Printf("[ws] WARNING: %s entry %q is neither an IP nor a CIDR, ignored", what, e) + } + return out +} + +func ipInAny(ip net.IP, nets []*net.IPNet) bool { + if ip == nil { + return false + } + for _, n := range nets { + if n.Contains(ip) { + return true + } + } + return false +} + +// clientIP resolves the address the limits are counted against, and reports +// whether that address actually identifies a client. +// +// distinct is false when the peer is a loopback or private address and no +// trustedProxies is configured: the request has almost certainly crossed a +// reverse proxy whose X-Forwarded-For we are not allowed to believe, so every +// visitor looks identical and per-IP limits would punish the wrong people. +func (l *wsLimiter) clientIP(r *http.Request) (ip net.IP, distinct bool) { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + host = r.RemoteAddr + } + peer := net.ParseIP(strings.TrimSpace(host)) + if peer == nil { + return nil, false + } + if ipInAny(peer, l.trustedProxies) { + // First hop is the client; later hops were appended by intermediaries + // we have no reason to trust individually. + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + first := strings.TrimSpace(strings.Split(xff, ",")[0]) + if fwd := net.ParseIP(first); fwd != nil { + return fwd, true + } + } + // A trusted proxy that forwarded no XFF still identifies itself. + return peer, true + } + if peer.IsLoopback() || peer.IsPrivate() || peer.IsLinkLocalUnicast() { + return peer, false + } + return peer, true +} + +// allow decides whether an upgrade may proceed. On rejection it returns the +// reason. On success it returns a release func the caller MUST invoke when +// the connection ends, so the concurrent-connection count comes back down. +func (l *wsLimiter) allow(r *http.Request) (ok bool, reason wsRejectReason, key string, release func()) { + noop := func() {} + if l == nil { + return true, "", "", noop + } + ip, distinct := l.clientIP(r) + + // The deny list is an explicit operator instruction, so it applies even + // when clients cannot be told apart. Behind an unconfigured proxy it + // simply never matches, which is visible rather than silently wrong. + if ipInAny(ip, l.denyNets) { + l.rejectedDeny.Add(1) + return false, wsRejectDeny, ip.String(), noop + } + if l.maxConnsPerIP <= 0 && l.upgradesPerMin <= 0 { + return true, "", "", noop + } + if !distinct { + l.warnedIndistinct.Do(func() { + log.Printf("[ws] WARNING: per-IP limits are configured but every request arrives from %s, "+ + "so clients cannot be told apart and the limits are NOT enforced. "+ + "Set webSocket.trustedProxies to your reverse proxy's address to enable them.", ip) + }) + return true, "", "", noop + } + + key = ip.String() + now := time.Now() + + l.mu.Lock() + defer l.mu.Unlock() + l.gcLocked(now) + + st := l.state[key] + if st == nil { + st = &wsIPState{} + l.state[key] = st + } + st.lastSeenAt = now + + if l.upgradesPerMin > 0 { + cutoff := now.Add(-wsUpgradeWindow) + kept := st.upgrades[:0] + for _, t := range st.upgrades { + if t.After(cutoff) { + kept = append(kept, t) + } + } + st.upgrades = kept + if len(st.upgrades) >= l.upgradesPerMin { + l.rejectedRate.Add(1) + return false, wsRejectRate, key, noop + } + } + if l.maxConnsPerIP > 0 && st.conns >= l.maxConnsPerIP { + l.rejectedConnCap.Add(1) + return false, wsRejectConnCap, key, noop + } + + // Both budgets have room, so charge them together: a rejected upgrade + // must never consume rate budget for a connection it did not get. + if l.upgradesPerMin > 0 { + st.upgrades = append(st.upgrades, now) + } + st.conns++ + + var once sync.Once + return true, "", key, func() { + once.Do(func() { + l.mu.Lock() + defer l.mu.Unlock() + if s := l.state[key]; s != nil && s.conns > 0 { + s.conns-- + s.lastSeenAt = time.Now() + } + }) + } +} + +// gcLocked drops records with no live connections and no recent activity, so +// the map cannot grow without bound when the server is scanned from many +// addresses. Caller must hold l.mu. +func (l *wsLimiter) gcLocked(now time.Time) { + for k, st := range l.state { + if st.conns == 0 && now.Sub(st.lastSeenAt) > wsBucketIdleGC { + delete(l.state, k) + } + } +} + +// counts reports the rejection tallies for /api/stats. +func (l *wsLimiter) counts() (deny, rate, connCap int64) { + if l == nil { + return 0, 0, 0 + } + return l.rejectedDeny.Load(), l.rejectedRate.Load(), l.rejectedConnCap.Load() +} diff --git a/cmd/server/ws_limits_test.go b/cmd/server/ws_limits_test.go new file mode 100644 index 000000000..cf025770f --- /dev/null +++ b/cmd/server/ws_limits_test.go @@ -0,0 +1,296 @@ +package main + +// Tests for the #1794 /ws transport limits. The five cases the issue lists as +// TDD requirements are marked with the exact wording from the issue body, so a +// reviewer can match them off without reading the implementation. + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" +) + +func req(remoteAddr, xff string) *http.Request { + r := httptest.NewRequest(http.MethodGet, "/ws", nil) + r.RemoteAddr = remoteAddr + if xff != "" { + r.Header.Set("X-Forwarded-For", xff) + } + return r +} + +func limiter(maxConns, perMin int, trusted, deny []string) *wsLimiter { + l := newWSLimiter() + l.maxConnsPerIP = maxConns + l.upgradesPerMin = perMin + l.trustedProxies = parseCIDRList(trusted, "trusted") + l.denyNets = parseCIDRList(deny, "deny") + return l +} + +// "Deny CIDR match → 403, hub count unchanged." +func TestWSDeny_CIDRMatchRejects(t *testing.T) { + l := limiter(0, 0, nil, []string{"203.0.113.0/24"}) + ok, reason, _, _ := l.allow(req("203.0.113.7:1234", "")) + if ok || reason != wsRejectDeny { + t.Fatalf("denied CIDR: ok=%v reason=%q, want rejected with %q", ok, reason, wsRejectDeny) + } + if ok, _, _, _ := l.allow(req("203.0.114.7:1234", "")); !ok { + t.Error("an address outside the denied range must still connect") + } + if d, _, _ := l.counts(); d != 1 { + t.Errorf("rejectedDeny = %d, want 1", d) + } +} + +func TestWSDeny_BareAddressIsAcceptedAsSingleHost(t *testing.T) { + // Operators write "1.2.3.4", not "1.2.3.4/32". Silently ignoring the bare + // form would be the worst possible failure for a deny list: it looks + // configured and blocks nothing. + l := limiter(0, 0, nil, []string{"198.51.100.9"}) + if ok, _, _, _ := l.allow(req("198.51.100.9:5000", "")); ok { + t.Error("bare address in the deny list must block") + } + if ok, _, _, _ := l.allow(req("198.51.100.10:5000", "")); !ok { + t.Error("the neighbouring address must not be blocked") + } +} + +func TestWSDeny_UnparseableEntryIsSkippedNotFatal(t *testing.T) { + // One typo must not take the deny list, or the server, down. + nets := parseCIDRList([]string{"not-an-ip", "203.0.113.0/24"}, "deny") + if len(nets) != 1 { + t.Fatalf("parsed %d networks, want 1 (the valid entry)", len(nets)) + } +} + +// "6th conn from same IP with cap=5 → 403." +func TestWSConnCap_SixthConnectionRejected(t *testing.T) { + l := limiter(5, 0, nil, nil) + for i := 1; i <= 5; i++ { + if ok, _, _, _ := l.allow(req("203.0.113.5:1", "")); !ok { + t.Fatalf("connection %d of 5 was rejected", i) + } + } + ok, reason, _, _ := l.allow(req("203.0.113.5:1", "")) + if ok || reason != wsRejectConnCap { + t.Fatalf("6th connection: ok=%v reason=%q, want rejected with %q", ok, reason, wsRejectConnCap) + } + if _, _, c := l.counts(); c != 1 { + t.Errorf("rejectedConnCap = %d, want 1", c) + } +} + +func TestWSConnCap_ReleaseFreesTheSlot(t *testing.T) { + l := limiter(1, 0, nil, nil) + _, _, _, release := l.allow(req("203.0.113.5:1", "")) + if ok, _, _, _ := l.allow(req("203.0.113.5:1", "")); ok { + t.Fatal("cap=1 must refuse the second concurrent connection") + } + release() + if ok, _, _, _ := l.allow(req("203.0.113.5:1", "")); !ok { + t.Error("after release the slot must be reusable") + } +} + +func TestWSConnCap_ReleaseIsIdempotent(t *testing.T) { + // Unregister can run twice for one client; double-crediting would let an + // IP accumulate slots it never gave back. + l := limiter(1, 0, nil, nil) + _, _, _, release := l.allow(req("203.0.113.5:1", "")) + release() + release() + if ok, _, _, _ := l.allow(req("203.0.113.5:1", "")); !ok { + t.Fatal("first reconnect must be allowed") + } + if ok, _, _, _ := l.allow(req("203.0.113.5:1", "")); ok { + t.Error("cap=1 must still hold: the double release must not have credited two slots") + } +} + +// "31 upgrades/min from same IP → 31st rejected." +func TestWSRate_ThirtyFirstUpgradeRejected(t *testing.T) { + l := limiter(0, 30, nil, nil) + for i := 1; i <= 30; i++ { + if ok, _, _, _ := l.allow(req("203.0.113.8:1", "")); !ok { + t.Fatalf("upgrade %d of 30 was rejected", i) + } + } + ok, reason, _, _ := l.allow(req("203.0.113.8:1", "")) + if ok || reason != wsRejectRate { + t.Fatalf("31st upgrade: ok=%v reason=%q, want rejected with %q", ok, reason, wsRejectRate) + } + if _, r, _ := l.counts(); r != 1 { + t.Errorf("rejectedRate = %d, want 1", r) + } +} + +func TestWSRate_BudgetIsPerAddress(t *testing.T) { + l := limiter(0, 2, nil, nil) + l.allow(req("203.0.113.1:1", "")) + l.allow(req("203.0.113.1:1", "")) + if ok, _, _, _ := l.allow(req("203.0.113.1:1", "")); ok { + t.Fatal("the first address should be out of budget") + } + if ok, _, _, _ := l.allow(req("203.0.113.2:1", "")); !ok { + t.Error("a different address must have its own budget") + } +} + +func TestWSRate_OldAttemptsLeaveTheWindow(t *testing.T) { + l := limiter(0, 2, nil, nil) + key := "203.0.113.3" + l.state[key] = &wsIPState{ + upgrades: []time.Time{time.Now().Add(-2 * wsUpgradeWindow), time.Now().Add(-90 * time.Second)}, + lastSeenAt: time.Now(), + } + if ok, _, _, _ := l.allow(req(key+":1", "")); !ok { + t.Error("attempts older than the window must not count against the budget") + } +} + +func TestWSRate_RejectedUpgradeDoesNotConsumeBudget(t *testing.T) { + // A refusal must not push the client further into debt, or a client that + // keeps retrying could never recover even once its window cleared. + l := limiter(1, 5, nil, nil) + l.allow(req("203.0.113.4:1", "")) // takes the only conn slot, 1 upgrade used + for i := 0; i < 3; i++ { + l.allow(req("203.0.113.4:1", "")) // all rejected on the conn cap + } + l.mu.Lock() + used := len(l.state["203.0.113.4"].upgrades) + l.mu.Unlock() + if used != 1 { + t.Errorf("upgrades charged = %d, want 1 (rejections must not be charged)", used) + } +} + +// "Trusted proxy + XFF spoofing → rate counted against the XFF IP, not the proxy." +func TestWSClientIP_TrustedProxyUsesForwardedFor(t *testing.T) { + l := limiter(0, 1, []string{"10.0.0.0/8"}, nil) + if ok, _, key, _ := l.allow(req("10.0.0.2:1", "203.0.113.20, 10.0.0.2")); !ok || key != "203.0.113.20" { + t.Fatalf("key = %q (ok=%v), want the first XFF hop 203.0.113.20", key, ok) + } + // Budget of 1 is now spent for that client, but a different client behind + // the SAME proxy must be unaffected. This is the whole point of honouring + // XFF: one visitor must not exhaust the budget for everyone else. + if ok, _, _, _ := l.allow(req("10.0.0.2:1", "203.0.113.21, 10.0.0.2")); !ok { + t.Error("a second client behind the same trusted proxy must have its own budget") + } + if ok, _, _, _ := l.allow(req("10.0.0.2:1", "203.0.113.20, 10.0.0.2")); ok { + t.Error("the first client's budget should still be spent") + } +} + +// "Untrusted RemoteAddr + XFF spoofing → XFF ignored, rate counted against RemoteAddr." +func TestWSClientIP_UntrustedPeerIgnoresForwardedFor(t *testing.T) { + l := limiter(0, 1, []string{"10.0.0.0/8"}, nil) + // A public peer that is not a configured proxy. If XFF were believed here, + // an attacker would mint a fresh identity per request and the limit would + // be worthless. + if ok, _, key, _ := l.allow(req("203.0.113.30:1", "198.51.100.1")); !ok || key != "203.0.113.30" { + t.Fatalf("key = %q (ok=%v), want the peer address, not the forged header", key, ok) + } + if ok, _, _, _ := l.allow(req("203.0.113.30:1", "198.51.100.2")); ok { + t.Error("changing the forged header must not buy a fresh budget") + } +} + +func TestWSClientIP_TrustedProxyWithoutForwardedForUsesPeer(t *testing.T) { + l := limiter(0, 5, []string{"10.0.0.0/8"}, nil) + if _, _, key, _ := l.allow(req("10.0.0.2:1", "")); key != "10.0.0.2" { + t.Errorf("key = %q, want the proxy's own address when it forwards nothing", key) + } +} + +// The safety property this design exists for: behind an unconfigured reverse +// proxy every visitor shares one address, so enforcing a per-IP cap would +// refuse real users. Limits must be skipped, not applied. +func TestWSLimits_SkippedWhenClientsCannotBeToldApart(t *testing.T) { + l := limiter(1, 1, nil, nil) // caps of 1, and no trustedProxies + for i := 0; i < 10; i++ { + if ok, reason, _, _ := l.allow(req("127.0.0.1:1", "203.0.113.9")); !ok { + t.Fatalf("request %d refused (%s): loopback peer with no trustedProxies must not be limited", i, reason) + } + } + if _, r, c := l.counts(); r != 0 || c != 0 { + t.Errorf("rejections rate=%d conncap=%d, want 0 and 0", r, c) + } +} + +func TestWSLimits_PrivatePeerAlsoCountsAsIndistinct(t *testing.T) { + l := limiter(1, 1, nil, nil) + for i := 0; i < 5; i++ { + if ok, _, _, _ := l.allow(req("192.168.1.10:1", "")); !ok { + t.Fatalf("request %d refused: a private peer with no trustedProxies must not be limited", i) + } + } +} + +// The deny list is an explicit operator instruction, so unlike the caps it +// still applies when clients cannot be told apart. +func TestWSDeny_AppliesEvenWhenIndistinct(t *testing.T) { + l := limiter(0, 0, nil, []string{"127.0.0.1"}) + if ok, reason, _, _ := l.allow(req("127.0.0.1:1", "")); ok || reason != wsRejectDeny { + t.Errorf("ok=%v reason=%q, want the deny list to apply regardless", ok, reason) + } +} + +func TestWSLimits_NilLimiterAllowsEverything(t *testing.T) { + // A Hub built without ConfigureLimits must behave exactly as before. + var l *wsLimiter + ok, _, _, release := l.allow(req("203.0.113.99:1", "")) + if !ok { + t.Fatal("a nil limiter must allow the upgrade") + } + release() // must not panic + if d, r, c := l.counts(); d != 0 || r != 0 || c != 0 { + t.Errorf("nil limiter counts = %d/%d/%d, want zeroes", d, r, c) + } +} + +func TestWSLimits_IdleStateIsCollected(t *testing.T) { + // Without collection, scanning the server from many addresses would grow + // the map forever. + l := limiter(0, 5, nil, nil) + l.state["203.0.113.50"] = &wsIPState{conns: 0, lastSeenAt: time.Now().Add(-2 * wsBucketIdleGC)} + l.state["203.0.113.51"] = &wsIPState{conns: 1, lastSeenAt: time.Now().Add(-2 * wsBucketIdleGC)} + l.mu.Lock() + l.gcLocked(time.Now()) + _, idleKept := l.state["203.0.113.50"] + _, liveKept := l.state["203.0.113.51"] + l.mu.Unlock() + if idleKept { + t.Error("an idle record with no connections should be collected") + } + if !liveKept { + t.Error("a record with a live connection must never be collected") + } +} + +func TestWSConfig_Defaults(t *testing.T) { + // The connection cap is deliberately OFF by default: carrier-grade NAT + // puts many unrelated subscribers behind one address, so a low cap would + // refuse real visitors on mobile networks. + var c Config + if got := c.WSMaxConnsPerIP(); got != 0 { + t.Errorf("default maxConnsPerIP = %d, want 0 (off)", got) + } + if got := c.WSUpgradesPerMinPerIP(); got != 30 { + t.Errorf("default upgradesPerMinPerIP = %d, want 30", got) + } + zero := 0 + c.WebSocket = &WebSocketConfig{UpgradesPerMinPerIP: &zero} + if got := c.WSUpgradesPerMinPerIP(); got != 0 { + t.Errorf("explicit 0 = %d, want 0: an operator must be able to turn the rate limit off", got) + } + five := 5 + c.WebSocket = &WebSocketConfig{UpgradesPerMinPerIP: &five, MaxConnsPerIP: 3} + if got := c.WSUpgradesPerMinPerIP(); got != 5 { + t.Errorf("configured rate = %d, want 5", got) + } + if got := c.WSMaxConnsPerIP(); got != 3 { + t.Errorf("configured cap = %d, want 3", got) + } +} diff --git a/config.example.json b/config.example.json index 7a72d36f8..930a224c7 100644 --- a/config.example.json +++ b/config.example.json @@ -38,6 +38,16 @@ "_comment_ingestorStats": "Ingestor publishes a 1-Hz stats snapshot consumed by the server's /api/perf/io and /api/perf/write-sources endpoints (#1120). Path is configured via the CORESCOPE_INGESTOR_STATS environment variable on the INGESTOR process. Default: /tmp/corescope-ingestor-stats.json. The writer uses O_NOFOLLOW + 0o600, so a pre-planted symlink in /tmp cannot be used to clobber an arbitrary file. SECURITY: in shared-tmp environments (multi-tenant hosts), point CORESCOPE_INGESTOR_STATS at a private directory like /var/lib/corescope/ingestor-stats.json that only the corescope user can write to.", "corsAllowedOrigins": [], "_comment_corsAllowedOrigins": "Cross-origin allowlist for embed scenarios (#1369) AND for /ws WebSocket upgrades (#1793 — CSWSH defense per OWASP WebSocket Security Cheat Sheet). Exact-match origins, e.g. [\"https://blog.example.com\", \"https://embed.example.com\"]. Same-origin requests (Origin host == request Host) and non-browser clients (no Origin header) are always allowed for /ws. When empty (default), no Access-Control-* headers are sent and browsers enforce same-origin; cross-origin /ws upgrades are rejected. When non-empty, only the listed origins receive CORS headers, and Access-Control-Allow-Methods is limited to GET, HEAD, OPTIONS (the cross-domain surface is read-only — same-origin admin writes are unaffected). Use [\"*\"] to allow any origin for CORS XHR (NOT recommended for write-capable deployments); note that \"*\" is deliberately NOT honored for /ws upgrades — list explicit origins to permit cross-origin WebSocket clients (OWASP guidance: avoid wildcards in CSWSH defense). Operators can override per-deployment with the CORS_ALLOWED_ORIGINS environment variable (comma-separated). No credentialed CORS is enabled. To embed the map or channels pages cross-domain, add the embedding origin here and use the URL pattern '/#/map?embed=1' or '/#/channels?embed=1' — embed mode hides the top-nav, bottom-nav, and side drawer for full-bleed iframe rendering.", + "webSocket": { + "maxConnsPerIP": 0, + "_comment_maxConnsPerIP": "#1794 - concurrent /ws connections allowed from one client address. 0 = off, and that is the shipped default ON PURPOSE: carrier-grade NAT puts thousands of unrelated mobile subscribers behind one public IPv4, so a low cap refuses real visitors while a scraper simply rents more addresses. Set it only if you know your audience is not behind shared NAT.", + "upgradesPerMinPerIP": 30, + "_comment_upgradesPerMinPerIP": "#1794 - handshakes per minute from one client address. On by default. Safe under CGNAT: a real client upgrades a handful of times per minute even while reconnecting, so 30 leaves ordinary traffic alone while flattening a scraper reconnect loop. Set 0 to disable.", + "trustedProxies": [], + "_comment_trustedProxies": "#1794 - addresses or CIDRs of YOUR reverse proxy. REQUIRED for the per-IP limits to do anything behind nginx/Caddy/Traefik/ingress: without it every visitor shares the proxy's address, so the limits are skipped and one warning is logged at startup. X-Forwarded-For is believed only from these addresses, because anyone else can forge it. Never list an address you do not control.", + "deny": [], + "_comment_deny": "#1794 - addresses or CIDRs refused at the /ws upgrade, before the handshake. Accepts both \"1.2.3.4\" and \"1.2.3.0/24\". Applies even behind an unconfigured proxy, since it is your explicit instruction. Rejections are counted in /api/stats under websocket.rejectedDeny." + }, "https": { "cert": "/path/to/cert.pem", "key": "/path/to/key.pem", From 2288e28d4ee6a2d97c302ff15d5887bab794d037 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 6 Sep 2026 15:20:16 -0400 Subject: [PATCH 008/151] fix: publish release artifacts after successful image retagging (#1964) Successful release fast paths publish image tags but never dispatch the job that creates the GitHub release and decrypt binaries. Dispatch `deploy.yml` from both image routes. A default-off `images_published` input skips E2E and image rebuilding only for an already-published tag; missing or mismatched images keep the complete fallback without requiring new inputs on older workflow definitions. Go validation still gates the release binaries, checkout and version flags retain the tagged source, and the existing release action uploads both architectures before publication. Missing binary files now fail publication. Fixes #1956. Validation: - `node test-issue-1956-release-routing.js` executes the actual workflow shell steps with registry and dispatch commands stubbed. Covers matching, missing and mismatched images; failed retag and Go validation; branch/PR boundaries; and both tagged binary commands. - The original test commit fails because a matching image dispatches zero artifact workflows; the fix passes the same assertion. - Existing release workflow Go checks, decrypt/channel tests, YAML parsing and actionlint pass. - Both static Linux amd64 and arm64 binaries cross-build with verified architecture and version metadata. Actual registry publication and GitHub release creation were not exercised. Existing immutable releases and old tags that contain older workflow definitions are outside this fix. Following #1922, this is a focused release-routing PR. A separate repair for #1858 rewrites the shared frontend test runner; merging this first lets that repair retain this regression in its authoritative list. Please assess current Go and E2E job results separately from workflow-approval or staging-runner state. --- .github/workflows/deploy.yml | 13 +- .github/workflows/release-fast-path.yml | 31 +++-- test-all.sh | 1 + test-issue-1956-release-routing.js | 175 ++++++++++++++++++++++++ 4 files changed, 205 insertions(+), 15 deletions(-) create mode 100644 test-issue-1956-release-routing.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 6b68ff79c..feeb48b40 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -18,6 +18,11 @@ on: pull_request: branches: [master] workflow_dispatch: + inputs: + images_published: + description: 'Release fast path already published the tag images' + type: boolean + default: false permissions: contents: read @@ -235,6 +240,7 @@ jobs: node test-issue-1705-subpath-contrast.js node test-issue-1770-mobile-row-clamp.js node test-a11y-axe-routes-coverage.js + node test-issue-1956-release-routing.js - name: 🛡️ Preflight XSS gate — actual --diff check (PR only) # The fixture self-test above (test-preflight-xss-gate.js) only @@ -322,7 +328,7 @@ jobs: defaults: run: shell: bash - if: needs.changes.outputs.code == 'true' + if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) steps: - name: Checkout code uses: actions/checkout@v5 @@ -665,7 +671,7 @@ jobs: name: "🏗️ Build & Publish Docker Image" needs: [e2e-test, changes] runs-on: ubuntu-latest - if: needs.changes.outputs.code == 'true' + if: needs.changes.outputs.code == 'true' && !(startsWith(github.ref, 'refs/tags/v') && inputs.images_published) steps: - name: Checkout code uses: actions/checkout@v5 @@ -768,8 +774,11 @@ jobs: CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . - name: Upload release assets + # Standard releases upload both assets to a draft before publishing. + # Keep one writer: published releases are immutable in this repository. uses: softprops/action-gh-release@v2 with: + fail_on_unmatched_files: true files: | corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64 diff --git a/.github/workflows/release-fast-path.yml b/.github/workflows/release-fast-path.yml index b7f1b47fa..e71b9f6a4 100644 --- a/.github/workflows/release-fast-path.yml +++ b/.github/workflows/release-fast-path.yml @@ -1,10 +1,10 @@ name: Release Fast-Path # Issue #1677: re-tag :edge as :vX.Y.Z when the tag SHA matches :edge's -# org.opencontainers.image.revision label. Skips ~30 min of Go test + -# Playwright + Docker rebuild because the bytes are identical — only the -# manifest name changes. Falls back to deploy.yml when SHAs differ so -# tags on older commits still go through full validation. +# org.opencontainers.image.revision label. Skips Playwright + Docker rebuild +# because the application bytes are identical. Both paths dispatch deploy.yml +# for Go validation and release binaries; mismatched SHAs also run the full +# image pipeline so tags on older commits still go through full validation. # # This workflow is the SOLE consumer of push.tags. deploy.yml's tag # trigger has been removed to prevent double-fire. @@ -16,7 +16,7 @@ on: permissions: contents: read packages: write - actions: write # issue #1702: required so the fallback `gh workflow run deploy.yml` dispatch is allowed + actions: write # issue #1702: required for the `gh workflow run deploy.yml` dispatch concurrency: group: release-fast-path-${{ github.ref }} @@ -24,7 +24,7 @@ concurrency: jobs: retag-or-fallback: - name: "🏷️ Re-tag :edge → :vX.Y.Z (fast) or dispatch deploy.yml (fallback)" + name: "🏷️ Re-tag :edge and dispatch release artifacts or full fallback" runs-on: ubuntu-latest steps: - name: Log in to GHCR @@ -115,17 +115,22 @@ jobs: done echo "Fast-path complete — release tags point at :edge plus a one-file version layer." - # ─────────── FALLBACK: SHAs differ, run the full pipeline ─────────── - - name: Dispatch full deploy.yml pipeline (fallback) - if: steps.edge.outputs.no_edge == 'true' || steps.edge.outputs.edge_revision != github.sha + # Both image routes need the single release-artifacts writer in deploy.yml. + - name: Dispatch release artifacts or full deploy.yml pipeline env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail - echo "SHA mismatch (or no :edge) — falling back to full pipeline" - echo " :edge revision = '${{ steps.edge.outputs.edge_revision }}'" - echo " tag SHA = '${{ github.sha }}'" + DISPATCH_ARGS=() + if [[ "${{ steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == github.sha }}" == 'true' ]]; then + echo "Images published — dispatching Go validation and release artifacts" + DISPATCH_ARGS+=(--field images_published=true) + else + # No new inputs: an older tag may have an older deploy.yml schema. + echo "SHA mismatch (or no :edge) — falling back to full pipeline" + fi gh workflow run deploy.yml \ --repo "${{ github.repository }}" \ - --ref "${{ github.ref }}" + --ref "${{ github.ref }}" \ + "${DISPATCH_ARGS[@]}" echo "Dispatched deploy.yml against ${{ github.ref }}" diff --git a/test-all.sh b/test-all.sh index bd3b0b696..e8d692c02 100755 --- a/test-all.sh +++ b/test-all.sh @@ -40,6 +40,7 @@ node test-issue-1648-m6-lint-self.js node test-issue-1890-og-url.js node test-traces.js node test-live-multibyte-filter.js +node test-issue-1956-release-routing.js # #1418 — route-view v2 (Tufte) coverage node test-issue-1418-raw-hex-extraction.js diff --git a/test-issue-1956-release-routing.js b/test-issue-1956-release-routing.js new file mode 100644 index 000000000..0f73a4d74 --- /dev/null +++ b/test-issue-1956-release-routing.js @@ -0,0 +1,175 @@ +// Run the real release shell steps with registry/dispatch commands stubbed out. +// No GitHub writes or container builds: node test-issue-1956-release-routing.js +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const vm = require('node:vm'); +const { spawnSync } = require('node:child_process'); + +const read = name => fs.readFileSync(path.join(__dirname, '.github/workflows', name), 'utf8').replace(/\r/g, ''); +const fast = read('release-fast-path.yml'); +const deploy = read('deploy.yml'); +let bash = process.env.BASH_PATH || 'bash'; +if (!process.env.BASH_PATH && process.platform === 'win32') { + // Prefer Git Bash over Windows' WSL launcher; CI uses the native Linux bash. + const git = spawnSync('git', ['--exec-path'], { encoding: 'utf8' }); + const gitBash = path.resolve((git.stdout || '').trim(), '../../../bin/bash.exe'); + if (git.status === 0 && fs.existsSync(gitBash)) bash = gitBash; +} + +// Extract known YAML blocks, retaining the actual expressions and shell code. +// Full YAML syntax is separately checked by actionlint; no YAML dependency here. +function block(source, key, indent) { + const lines = source.split('\n'); + const prefix = ' '.repeat(indent) + key + ':'; + const start = lines.findIndex(line => line.startsWith(prefix)); + if (start < 0) return ''; + let end = start + 1; + while (end < lines.length && (!lines[end].trim() || lines[end].search(/\S/) > indent)) end++; + return lines.slice(start, end).join('\n'); +} + +function value(source, key, indent) { + const raw = block(source, key, indent); + if (!raw) return ''; + const lines = raw.split('\n'); + const first = lines[0].slice(indent + key.length + 1).trim(); + return first === '|' || first === '>' + ? lines.slice(1).map(line => line.slice(indent + 2)).join('\n').trimEnd() + : first; +} + +const steps = source => source.split(/(?=^ - name:)/m).slice(1); +function evaluate(expression, context) { + if (!expression) return true; + return vm.runInNewContext(expression.replace(/^\$\{\{|\}\}$/g, '').trim(), { + ...context, startsWith: (text, prefix) => text.startsWith(prefix) + }); +} +const expand = (script, context) => script.replace(/\$\{\{(.*?)\}\}/g, (_, expression) => String(evaluate(expression, context))); +const bashPath = file => process.platform === 'win32' ? file.replace(/\\/g, '/').replace(/^([A-Za-z]):/, (_, drive) => '/' + drive.toLowerCase()) : file; + +function runSteps(source, context, edge, mutateFails = false) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'corescope-release-test-')); + const output = path.join(dir, 'output'); + const log = path.join(dir, 'commands'); + fs.writeFileSync(log, ''); + fs.mkdirSync(path.join(dir, 'cmd/decrypt'), { recursive: true }); + const stubs = ` + log() { node -e 'require("fs").appendFileSync(process.env.COMMAND_LOG, JSON.stringify(process.argv.slice(1))+"\\n")' "$@"; } + crane() { + if [ "$1" = config ]; then + [ "$EDGE_CONFIG" != missing ] || return 1 + printf '%s' "$EDGE_CONFIG" + else + log crane "$@" + [ "$1" != mutate ] || [ "$MUTATE_FAILS" != true ] + fi + } + gh() { log gh "$@"; } + tar() { log tar "$@"; } + go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$@"; } + jq() { + node -e 'const fs=require("fs"); const assert=require("assert/strict"); assert.equal(process.argv[1], ".config.Labels[\\"org.opencontainers.image.revision\\"] // \\\"\\\""); console.log(JSON.parse(fs.readFileSync(0,"utf8")).config.Labels["org.opencontainers.image.revision"] || "")' "$2" + } + `; + try { + for (const step of steps(source)) { + const script = value(step, 'run', 8); + if (!script || !evaluate(value(step, 'if', 8), context)) continue; + fs.writeFileSync(output, ''); + const result = spawnSync(bash, ['--noprofile', '--norc', '-e', '-o', 'pipefail'], { + input: stubs + '\n' + expand(script, context), cwd: dir, encoding: 'utf8', timeout: 15000, + env: { + ...process.env, GITHUB_REF: context.github.ref, GITHUB_SHA: context.github.sha, + GITHUB_OUTPUT: bashPath(output), COMMAND_LOG: bashPath(log), TMPDIR: bashPath(dir), + EDGE_CONFIG: edge === null ? 'missing' : JSON.stringify({ config: { Labels: { 'org.opencontainers.image.revision': edge } } }), + MUTATE_FAILS: String(mutateFails) + } + }); + if (mutateFails && result.status !== 0) return { commands: commands(), failed: true }; + assert.equal(result.status, 0, `${value(step, '- name', 6)}: ${result.error || result.stderr}`); + const id = value(step, 'id', 8); + if (id) context.steps[id] = { outputs: Object.fromEntries(fs.readFileSync(output, 'utf8').trim().split('\n').filter(Boolean).map(line => { + const at = line.indexOf('='); + return [line.slice(0, at), line.slice(at + 1)]; + })) }; + } + return { commands: commands(), failed: false }; + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + function commands() { return fs.readFileSync(log, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); } +} + +function context(ref = 'refs/tags/v9.8.7', event = 'workflow_dispatch', inputs = {}) { + return { github: { ref, ref_name: ref.split('/').pop(), sha: 'a'.repeat(40), repository: 'example/corescope', event_name: event }, inputs, steps: {}, needs: {}, vars: {} }; +} + +// Honor the real job conditions AND implicit success() for needs. This catches +// release-artifacts accidentally depending on the skipped image or E2E jobs. +function route(ctx, failedJob) { + for (const name of ['changes', 'go-test', 'e2e-test', 'build-and-publish', 'release-artifacts', 'deploy', 'publish']) { + const job = block(deploy, name, 2); + assert.ok(job, `missing ${name} job`); + const needs = value(job, 'needs', 4).replace(/[\[\]\s]/g, '').split(',').filter(Boolean); + const run = needs.every(need => ctx.needs[need].result === 'success') && evaluate(value(job, 'if', 4), ctx); + ctx.needs[name] = { result: run ? (name === failedJob ? 'failure' : 'success') : 'skipped', outputs: { code: 'true' } }; + } + return ctx.needs; +} + +for (const [name, edge] of [['matching', 'a'.repeat(40)], ['missing', null], ['mismatched', 'b'.repeat(40)]]) { + const ctx = context(); + const { commands } = runSteps(block(fast, 'retag-or-fallback', 2), ctx, edge); + const dispatches = commands.filter(command => command[0] === 'gh'); + assert.equal(dispatches.length, 1, `${name}: tag must dispatch the artifact workflow exactly once`); + const dispatch = dispatches[0]; + assert.deepEqual(dispatch.slice(1, 4), ['workflow', 'run', 'deploy.yml']); + assert.equal(dispatch[dispatch.indexOf('--ref') + 1], ctx.github.ref, `${name}: dispatch must preserve the tag source`); + assert.equal(dispatch[dispatch.indexOf('--repo') + 1], ctx.github.repository); + const matching = name === 'matching'; + assert.equal(dispatch.includes('images_published=true'), matching); + if (!matching) assert.ok(!dispatch.includes('--field') && !dispatch.includes('-f'), 'old-tag fallback must not require new workflow inputs'); + assert.equal(commands.filter(command => command[0] === 'crane' && command[1] === 'mutate').length, matching ? 1 : 0); + assert.deepEqual(commands.filter(command => command[0] === 'crane' && command[1] === 'tag').map(command => command.at(-1)), matching ? ['v9.8', 'v9', 'latest'] : []); + const jobs = route(context(undefined, undefined, { images_published: matching })); + assert.equal(jobs['release-artifacts'].result, 'success', `${name}: release artifacts must run`); + assert.equal(jobs['go-test'].result, 'success', `${name}: release still requires Go validation`); + for (const job of ['e2e-test', 'build-and-publish']) assert.equal(jobs[job].result, matching ? 'skipped' : 'success', `${name}: ${job}`); + assert.equal(jobs.deploy.result, 'skipped'); + assert.equal(jobs.publish.result, 'skipped'); + console.log(`PASS ${name} edge: one artifact dispatch, correct image route`); +} + +const failedRetag = runSteps(block(fast, 'retag-or-fallback', 2), context(), 'a'.repeat(40), true); +assert.equal(failedRetag.failed, true); +assert.equal(failedRetag.commands.filter(command => command[0] === 'gh').length, 0, 'failed retag must not dispatch with images_published=true'); + +for (const [ref, event] of [['refs/heads/master', 'push'], ['refs/heads/master', 'workflow_dispatch'], ['refs/pull/1/merge', 'pull_request']]) { + const jobs = route(context(ref, event, { images_published: true })); + assert.equal(jobs['release-artifacts'].result, 'skipped', `${event}: no GitHub release`); + assert.equal(jobs['build-and-publish'].result, 'success', `${event}: tag-only input must not skip branch/PR checks`); + const publishing = steps(block(deploy, 'build-and-publish', 2)).find(step => step.includes('uses: docker/build-push-action')); + assert.equal(Boolean(evaluate(value(publishing, 'if', 8), context(ref, event))), event === 'push', `${event}: GHCR publishing`); +} +assert.equal(route(context(), 'go-test')['release-artifacts'].result, 'skipped', 'failed Go validation must block release'); +const dispatchInput = block(deploy, 'images_published', 6); +assert.equal(value(dispatchInput, 'type', 8), 'boolean', 'dispatch flag must retain boolean semantics'); +assert.equal(value(dispatchInput, 'default', 8), 'false', 'manual and fallback dispatches must build images by default'); + +const release = block(deploy, 'release-artifacts', 2); +const builds = runSteps(release, context(), null).commands.filter(command => command[0] === 'go'); +assert.deepEqual(builds.map(command => command.slice(1, 4)), [['linux', 'amd64', '0'], ['linux', 'arm64', '0']]); +for (const command of builds) assert.ok(command.includes('-ldflags=-s -w -X main.version=v9.8.7'), 'binary version must come from tag'); +const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); +assert.equal(upload.length, 1, 'publish both architectures together, before the release becomes immutable'); +assert.equal(value(upload[0], 'fail_on_unmatched_files', 10), 'true', 'missing assets must prevent publication'); +assert.deepEqual(value(upload[0], 'files', 10).trim().split('\n').map(line => line.trim()), ['corescope-decrypt-linux-amd64', 'corescope-decrypt-linux-arm64']); +assert.equal(value(upload[0], 'draft', 10), '', 'standard release action must finalize after both uploads'); +assert.equal(value(upload[0], 'prerelease', 10), '', 'standard release action must upload before publishing'); +const checkout = steps(release).find(step => step.includes('uses: actions/checkout@')); +assert.equal(value(checkout, 'ref', 10), '', 'checkout must retain the dispatched tag/SHA'); +assert.ok(!value(block(deploy, 'push', 2), 'tags', 4), 'fast path must remain the sole tag-triggered image writer'); +console.log('PASS failed retag/Go gates, branch/PR routes, and complete tagged release assets'); From 5b689f75fece49ecd80a84028c297f7cc252adfb Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 6 Sep 2026 21:43:20 +0200 Subject: [PATCH 009/151] feat(#1845): filter nodes by how long they have been silent (#1973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1845 for the question in its title. The alerting ask that came later in the thread is deliberately **not** in here; see the bottom. ### The gap @Jonher937 asked to flag repeaters that stopped communicating for x days, to find remote gear that has died. Today the Nodes page has Active/Stale with thresholds fixed at 72h for infra and 24h for everything else, plus a Last Heard filter that selects nodes heard **within** a window. Neither answers "show me what has been quiet for over a week". ### What this adds A `Silent for` select beside Last Heard: 1d, 3d, 7d, 14d, 30d, each labelled with the number of nodes it would select. ``` [ All ] [ Active ] [ Stale ] Last Heard: Any v Silent for: over 7d (23) v ``` Counts are computed **before** the silence filter is applied, so the dropdown keeps showing what the other windows would select instead of collapsing to the one already chosen. The choice persists in `localStorage` like the neighbouring filters and is mirrored into the URL as `?silent=7d`, so the view can be pasted to whoever owns the silent gear. The URL sync is wrapped in try/catch, because it is a convenience and must never stop the filter working. ### The part worth reviewing: one definition of freshness, not two `getNodeStatus` has been relay-aware since #1598, while `nodes.js` separately computed `statusAge` from the ADVERT timestamp alone. Filtering on the latter would have listed a repeater as silent for ten days while its own badge on the same row said active, and it would have done so for **exactly** the nodes #1598 exists to protect. So the freshness rule is extracted into `window.getEffectiveHeardMs` in `roles.js`, and `getNodeStatus` now calls it. Behaviour is unchanged, there is now one source. Reviewers should look hardest at that refactor rather than at the select. A node never heard from at all scores `Infinity`, so it matches every window instead of silently dropping out of the filter. ### Why the thresholds are fixed values and not derived I measured the alternative before writing this, on a 1179-repeater mesh, and posted it on #1611: replacing a fixed threshold with `3 x per-node advert median` fixes 8 false "silent" flags and newly mis-flags **28 currently-active nodes**, because a 3h-median node gets a 9h threshold. Raising the global default to 144h rescues 7 and hides 27 genuinely dead repeaters. Both are net-negative. A user-chosen window sidesteps the whole question: the operator picks what "too long" means for their mesh, which is what @Jonher937 asked for in the first place. ### Verification - `test-frontend-helpers.js`: **666 → 680 passed, 0 failed**. Fourteen cases covering `NaN` rather than `0` when nothing is known (0 is a real timestamp and would sort as very old rather than unknown), the full `_liveSeen > _lastHeard > last_heard > last_seen` precedence, a recent relay beating a stale advert, a stale relay **not** dragging a fresh advert backwards, relay alone sufficing, `room` counting as infra while `companion` does not (a `last_relayed` on a companion is meaningless and must not rescue it), case-insensitive roles, the legacy `(role, ms)` call shape, and the 72h boundary asserted at 71h and 73h. - One of those failed on first run and **the test was wrong, not the code**: `9e7` ms is 25h, which is correctly active for infra. Fixed, and the boundary is now asserted explicitly so nobody repeats it. - `eslint` on the changed files: 0 errors. The warnings present are the same ones master already reports. - No server change, no new API, no new column. No cache-buster bump needed: `__BUST__` is substituted at startup in `cmd/server/main.go:570`. ### Not done **No alerting.** @fokcuk asked on the thread for notification when a repeater they look after goes silent, which is a different product: subscriber identity, an evaluation loop and delivery, none of which exist today. That deserves its own issue and a design call rather than being shimmed into `nodes.js`, which is also what the triage concluded. This PR gives the operator the view; it does not push to them. Sizes and boundaries (1d/3d/7d/14d/30d) are a judgement call. Say the word if a different set fits real operator habits better. Co-authored-by: Claude Opus 5 (1M context) --- public/nodes.js | 46 ++++++++++++++++++++ public/roles.js | 41 +++++++++++------- test-frontend-helpers.js | 90 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 163 insertions(+), 14 deletions(-) diff --git a/public/nodes.js b/public/nodes.js index 61a795ee4..a4d856c82 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -82,6 +82,12 @@ } let lastHeard = localStorage.getItem('meshcore-nodes-last-heard') || ''; let statusFilter = localStorage.getItem('meshcore-nodes-status-filter') || 'all'; + // #1845: "silent longer than N", the inverse of the Last Heard filter. Empty + // means off. Ages are relay-aware (getEffectiveHeardMs), so a repeater that + // forwards traffic is never counted as silent just because its ADVERT is old. + let silentFor = localStorage.getItem('meshcore-nodes-silent-for') || ''; + const SILENT_MS = { '1d': 86400000, '3d': 259200000, '7d': 604800000, '14d': 1209600000, '30d': 2592000000 }; + let _silentCounts = null; let wsHandler = null; let detailMap = null; let detailMapResizeTimer = null; @@ -199,6 +205,15 @@ return 'Stale \u2014 not heard for over ' + threshold + '. This ' + role + ' may be offline or out of range.'; } + // #1845: how long we have heard nothing at all from this node. Infinity when + // we have never heard it, which must count as silent for every window rather + // than dropping out of the filter. Uses the shared relay-aware definition so + // this agrees with the Active/Stale badge instead of contradicting it. + function silenceAgeMs(n) { + const ms = (typeof window.getEffectiveHeardMs === 'function') ? window.getEffectiveHeardMs(n) : NaN; + return (typeof ms === 'number' && !isNaN(ms) && ms > 0) ? (Date.now() - ms) : Infinity; + } + function getStatusInfo(n) { // Single source of truth for all status-related info const role = (n.role || '').toLowerCase(); @@ -514,6 +529,9 @@ const _urlSearch = _listUrlParams.get('search'); if (_urlTab && TABS.some(function(t) { return t.key === _urlTab; })) activeTab = _urlTab; if (_urlSearch) search = _urlSearch; + // #1845: shareable — "here are the repeaters silent over a week". + const _urlSilent = _listUrlParams.get('silent'); + if (_urlSilent !== null) silentFor = SILENT_MS[_urlSilent] ? _urlSilent : ''; // #749 — restore sort from URL (overrides localStorage persistence). var _urlSort = _listUrlParams.get('sort'); if (_urlSort && window.URLState) { @@ -1275,6 +1293,16 @@ if (statusFilter === 'active' || statusFilter === 'stale') { filtered = filtered.filter(n => getNodeStatus(n) === statusFilter); // #1598 } + // #1845: counts are taken BEFORE the silence filter is applied, so the + // dropdown keeps showing what each other window would select instead of + // collapsing to the one already chosen. + _silentCounts = {}; + for (const k of Object.keys(SILENT_MS)) { + _silentCounts[k] = filtered.filter(n => silenceAgeMs(n) >= SILENT_MS[k]).length; + } + if (silentFor && SILENT_MS[silentFor]) { + filtered = filtered.filter(n => silenceAgeMs(n) >= SILENT_MS[silentFor]); + } nodes = filtered; // Defensive filter: hide nodes with obviously corrupted data @@ -1365,6 +1393,10 @@ +
Last Seen${lastSeen}
Adverts${n.advert_count || 0}
Configured scope ✓${n.configured_scope === '' ? 'none configured' : `${escapeHtml(n.configured_scope)}`}
Scope${n.default_scope === null ? '—' : n.default_scope === '' ? 'unknown scope' : `${escapeHtml(n.default_scope)}` diff --git a/public/node-reach.css b/public/node-reach.css index 7c830fcbd..d9daa57ff 100644 --- a/public/node-reach.css +++ b/public/node-reach.css @@ -72,3 +72,12 @@ /* Tighter cell padding so the remaining four columns breathe. */ .nq-table th, .nq-table td { padding:4px 6px; } } + +/* #1865: region scope on the reach report. Muted like .nq-sub so it reads as + report metadata rather than a finding, with the confirmed tick carrying the + only colour, because that tick is the one bit of it that is evidence. */ +.nq-scope { color:var(--text-muted); font-size:11px; margin-top:2px; } +.nq-scope code { font-family:var(--mono); color:var(--link-color); } +.nq-scope-ok { color:var(--success); } +.nq-scope-none { font-style:italic; } +.nq-scope-src { opacity:0.75; } diff --git a/public/node-reach.js b/public/node-reach.js index 2d4f2b1f0..5a758820e 100644 --- a/public/node-reach.js +++ b/public/node-reach.js @@ -65,11 +65,36 @@ (on ? ' class="active"' : '') + '>' + label + ''; } + // #1865: which region scopes this node serves, on the report itself so a + // printed or shared reach page carries it. Two different claims, never + // conflated: "Scope" is INFERRED from the transport scope of observed + // adverts, while "Configured scope" is CONFIRMED, read back off the node by + // an observer /neighbors report. Confirmed wins the line when both exist, + // and an empty confirmed value is itself a statement ("responded, none + // configured") rather than missing data, so it renders too. + function scopeLineHtml(n) { + var hasConfirmed = typeof n.configured_scope === 'string'; + if (!hasConfirmed && !n.default_scope) return ''; + if (hasConfirmed) { + var at = n.configured_scope_at ? ' Last confirmed ' + n.configured_scope_at + '.' : ''; + var val = n.configured_scope === '' + ? 'none configured' + : '' + escapeHtml(n.configured_scope) + ''; + return '
Configured scope ' + + '✓ ' + val + + ' confirmed
'; + } + return '
Scope ' + escapeHtml(n.default_scope) + '' + + ' observed
'; + } + function headerHtml(n, nodeName, days) { return '
' + '← Back to ' + nodeName + '' + '

' + nodeName + ' — Reach

' + '
' + escapeHtml(n.role || 'Unknown role') + ' · two-way RF link reach
' + + scopeLineHtml(n) + '
Reliable by design: built only from unique 2–3 byte path-hash (multibyte) matches. 1-byte hops collide between nodes and are excluded, so the links shown are trustworthy.
' + '
' + dayBtn(1, days, '24h') + dayBtn(7, days, '7d') + dayBtn(14, days, '14d') + dayBtn(30, days, '30d') + @@ -277,5 +302,12 @@ current = null; } + if (typeof window !== 'undefined') { + // #1865: exposed so the helper tests can assert what the scope line RENDERS + // rather than grepping this file, the same reason map.js exposes its label + // builder (#1356/#1933). + window.__meshcoreReachInternals = { scopeLineHtml: scopeLineHtml }; + } + registerPage('node-reach', { init: init, destroy: destroy }); })(); diff --git a/public/nodes.js b/public/nodes.js index 0ced86bf3..61a795ee4 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -717,6 +717,7 @@ return `
Bridge score ⓘ${bpct}% ${blabel}
Transported scopes${n.transported_scopes.map(sc => '' + escapeHtml(String(sc)) + '').join('')}
Configured scope ✓${n.configured_scope === '' ? 'none configured' : `${escapeHtml(n.configured_scope)}`}
First Seen${renderNodeTimestampHtml(n.first_seen)}
Total Packets${stats.totalTransmissions || stats.totalPackets || n.advert_count || 0}${stats.totalObservations && stats.totalObservations !== (stats.totalTransmissions || stats.totalPackets) ? ' (seen ' + stats.totalObservations + '×)' : ''}
Packets Today${stats.packetsToday || 0}
@@ -1389,6 +1421,20 @@ // Filter changes document.getElementById('nodeLastHeard').addEventListener('change', e => { lastHeard = e.target.value; localStorage.setItem('meshcore-nodes-last-heard', lastHeard); loadNodes(); }); + // #1845: mirrors the Last Heard handler, and additionally keeps the URL in + // step so the current view can be pasted to whoever owns the silent gear. + document.getElementById('nodeSilentFor').addEventListener('change', e => { + silentFor = e.target.value; + localStorage.setItem('meshcore-nodes-silent-for', silentFor); + try { + const base = String(location.hash || '#/nodes').split('?')[0]; + const params = getHashParams(); + if (silentFor) { params.set('silent', silentFor); } else { params.delete('silent'); } + const qs = params.toString(); + history.replaceState(null, '', base + (qs ? '?' + qs : '')); + } catch (err) { /* URL sync is a convenience; never block the filter on it */ } + loadNodes(); + }); // Status filter buttons document.querySelectorAll('#nodeStatusFilter .btn').forEach(btn => { diff --git a/public/roles.js b/public/roles.js index e037834a1..a816399e7 100644 --- a/public/roles.js +++ b/public/roles.js @@ -453,23 +453,36 @@ // default trajectory is 47h flood adverts), so an actively-relaying // backbone repeater must not be marked stale just because its last // ADVERT is old. + // #1845: the single definition of "how recently did we hear from this node", + // extracted from getNodeStatus so a caller that needs the AGE gets the same + // answer as the caller that needs the STATUS. Before this, nodes.js carried + // two notions: getNodeStatus (relay-aware) and a local statusAge computed + // from the advert timestamp alone. They disagreed for exactly the nodes + // #1598 exists to protect. Returns NaN when nothing is known. + window.getEffectiveHeardMs = function (n) { + if (!n || typeof n !== 'object') return NaN; + var role = String(n.role || 'companion').toLowerCase(); + // Freshness precedence mirrors the existing call sites: + // _liveSeen (live view, ms) > _lastHeard (health API) > + // last_heard (in-memory packets) > last_seen (DB, ADVERT). + var seenTime = n._lastHeard || n.last_heard || n.last_seen; + var effectiveMs = (typeof n._liveSeen === 'number' && n._liveSeen) || + (seenTime ? new Date(seenTime).getTime() : NaN); + // #1598: an infra node that forwards traffic is alive even when its last + // ADVERT is old, which operators increasingly cause on purpose (the + // firmware default flood advert interval moved to 47h). + if ((role === 'repeater' || role === 'room') && n.last_relayed) { + var relayedMs = new Date(n.last_relayed).getTime(); + if (!(effectiveMs >= relayedMs)) effectiveMs = relayedMs; + } + return effectiveMs; + }; + window.getNodeStatus = function (roleOrNode, lastSeenMs) { var role, effectiveMs; if (roleOrNode && typeof roleOrNode === 'object') { - var n = roleOrNode; - role = n.role || 'companion'; - // Freshness precedence mirrors the existing call sites: - // _liveSeen (live view, ms) > _lastHeard (health API) > - // last_heard (in-memory packets) > last_seen (DB, ADVERT). - var seenTime = n._lastHeard || n.last_heard || n.last_seen; - effectiveMs = (typeof n._liveSeen === 'number' && n._liveSeen) || - (seenTime ? new Date(seenTime).getTime() : NaN); - var infra = String(role).toLowerCase() === 'repeater' || - String(role).toLowerCase() === 'room'; - if (infra && n.last_relayed) { - var relayedMs = new Date(n.last_relayed).getTime(); - if (!(effectiveMs >= relayedMs)) effectiveMs = relayedMs; - } + role = roleOrNode.role || 'companion'; + effectiveMs = window.getEffectiveHeardMs(roleOrNode); if (isNaN(effectiveMs)) effectiveMs = undefined; } else { role = roleOrNode; diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index 324cbcb27..106200e88 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -7067,3 +7067,93 @@ console.log('\n=== node-reach.js: scopeLineHtml (#1865) ==='); assert.ok(h.includes('<img'), 'should be escaped instead'); }); } + +// ===== roles.js: getEffectiveHeardMs (#1845) ===== +// Extracted from getNodeStatus so the "silent longer than N" filter measures +// the SAME freshness the Active/Stale badge does. Two definitions is how a node +// ends up listed as silent for 10 days while its own badge says active. +console.log('\n=== roles.js: getEffectiveHeardMs (#1845) ==='); +{ + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/roles.js'); + const eff = ctx.getEffectiveHeardMs; + const status = ctx.getNodeStatus; + const iso = ms => new Date(ms).toISOString(); + const NOW = Date.now(); + + test('nothing known yields NaN, not 0', () => { + // 0 would read as "heard at the epoch", which is a real timestamp and would + // silently sort/compare as very old rather than as unknown. + assert.ok(Number.isNaN(eff({ role: 'repeater' }))); + }); + + test('non-object input yields NaN', () => { + assert.ok(Number.isNaN(eff(null))); + assert.ok(Number.isNaN(eff('repeater'))); + }); + + test('precedence: _liveSeen beats every stored timestamp', () => { + const r = eff({ role: 'companion', _liveSeen: NOW, _lastHeard: iso(NOW - 9e6), last_seen: iso(NOW - 9e7) }); + assert.strictEqual(r, NOW); + }); + + test('precedence: _lastHeard beats last_heard and last_seen', () => { + const r = eff({ role: 'companion', _lastHeard: iso(NOW - 1000), last_heard: iso(NOW - 9e6), last_seen: iso(NOW - 9e7) }); + assert.strictEqual(r, new Date(iso(NOW - 1000)).getTime()); + }); + + test('infra: a recent relay beats a stale advert', () => { + const r = eff({ role: 'repeater', last_seen: iso(NOW - 9e7), last_relayed: iso(NOW - 1000) }); + assert.strictEqual(r, new Date(iso(NOW - 1000)).getTime()); + }); + + test('infra: a stale relay does NOT drag a fresh advert backwards', () => { + const fresh = iso(NOW - 1000); + const r = eff({ role: 'repeater', last_seen: fresh, last_relayed: iso(NOW - 9e7) }); + assert.strictEqual(r, new Date(fresh).getTime()); + }); + + test('infra: relay alone is enough when nothing else is known', () => { + const r = eff({ role: 'repeater', last_relayed: iso(NOW - 1000) }); + assert.strictEqual(r, new Date(iso(NOW - 1000)).getTime()); + }); + + test('room counts as infra, companion does not', () => { + const relayed = iso(NOW - 1000), old = iso(NOW - 9e7); + assert.strictEqual(eff({ role: 'room', last_seen: old, last_relayed: relayed }), new Date(relayed).getTime()); + // A companion is not a relay, so last_relayed on one is meaningless and + // must not silently rescue it. + assert.strictEqual(eff({ role: 'companion', last_seen: old, last_relayed: relayed }), new Date(old).getTime()); + }); + + test('role matching is case-insensitive', () => { + const relayed = iso(NOW - 1000); + assert.strictEqual(eff({ role: 'Repeater', last_seen: iso(NOW - 9e7), last_relayed: relayed }), new Date(relayed).getTime()); + }); + + // Regression: getNodeStatus was rewritten to call this helper. Its contract + // must be unchanged, including the legacy (role, ms) call shape. + test('getNodeStatus still honours the legacy two-argument form', () => { + assert.strictEqual(status('repeater', NOW), 'active'); + assert.strictEqual(status('repeater', NOW - 4 * 86400000), 'stale'); + }); + + test('getNodeStatus is still relay-aware for infra', () => { + assert.strictEqual(status({ role: 'repeater', last_seen: iso(NOW - 9e7), last_relayed: iso(NOW - 1000) }), 'active'); + }); + + test('getNodeStatus still marks a truly silent repeater stale', () => { + // Past infraSilentMs (72h). 9e7 ms is only 25h, which is correctly ACTIVE + // for infra and would make this assertion test nothing. + assert.strictEqual(status({ role: 'repeater', last_seen: iso(NOW - 4 * 86400000) }), 'stale'); + }); + + test('72h is the infra boundary: 71h active, 73h stale', () => { + assert.strictEqual(status({ role: 'repeater', last_seen: iso(NOW - 71 * 3600000) }), 'active'); + assert.strictEqual(status({ role: 'repeater', last_seen: iso(NOW - 73 * 3600000) }), 'stale'); + }); + + test('getNodeStatus treats an unknown node as stale, not active', () => { + assert.strictEqual(status({ role: 'repeater' }), 'stale'); + }); +} From 2c8c1161b576f18fb170bd9493b837695d1e6dde Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 6 Sep 2026 23:04:42 +0200 Subject: [PATCH 010/151] feat(#1975): network-wide Scope Audit page, fed by confirmed scopes (#1976) One row per repeater whose configured region list is known, answering a question no other view answers: you declare these regions, but were you seen forwarding them? default_scope says what a node's adverts were observed under and transported_scopes (#1751) says what it carried, but nothing lined the declared list up against observed forwarding. The declared side merges every confirmed-scope source the database carries, newest answer per node wins, rather than naming one. On a stock install only nodes.configured_scope (#1865/#1971) exists and it degrades to the one-source case; deployments that collect the same fact another way keep working. Reading a single hard-coded source would have rendered an empty page on the very instance the evidence came from. Declared and observed are compared through normScope, so a leading "#" and a bare region name are one region. Unobserved regions render neutral, not red: absence over a short window is weak evidence, which the page header already states in words. Ported from a long-running fork deployment with its 17 server tests, rewired to the upstream data source, plus 14 frontend cases asserting rendered markup. --- cmd/server/db.go | 28 +- cmd/server/openapi.go | 5 + cmd/server/routes.go | 12 + cmd/server/scope_audit.go | 851 ++++++++++++++++++++++++++++++++ cmd/server/scope_audit_test.go | 855 +++++++++++++++++++++++++++++++++ public/bottom-nav.js | 1 + public/index.html | 3 + public/nav-drawer.js | 1 + public/scope-audit.css | 81 ++++ public/scope-audit.js | 367 ++++++++++++++ test-frontend-helpers.js | 147 ++++++ test-nav-priority-1391-e2e.js | 7 +- 12 files changed, 2347 insertions(+), 11 deletions(-) create mode 100644 cmd/server/scope_audit.go create mode 100644 cmd/server/scope_audit_test.go create mode 100644 public/scope-audit.css create mode 100644 public/scope-audit.js diff --git a/cmd/server/db.go b/cmd/server/db.go index 70c10f111..4e2105a3e 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -35,16 +35,17 @@ const routeTypeNonTransportSQL = "route_type IN (1, 2)" // DB wraps a read-only connection to the MeshCore SQLite database. type DB struct { - conn *sql.DB - path string // filesystem path to the database file - isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2) - hasResolvedPath bool // observations table has resolved_path column - hasObsRawHex bool // observations table has raw_hex column (#881) - hasScopeName bool // transmissions.scope_name column exists (#899) - hasDefaultScope bool // nodes.default_scope column exists (#899) - hasConfiguredScope bool // nodes.configured_scope column exists (#1865) - hasMultibyteSupCols bool // nodes/inactive_nodes have multibyte_sup/multibyte_evidence (#903) - hasLastSeen bool // transmissions.last_seen column exists (#1690) + conn *sql.DB + path string // filesystem path to the database file + isV3 bool // v3 schema: observer_idx in observations (vs observer_id in v2) + hasResolvedPath bool // observations table has resolved_path column + hasObsRawHex bool // observations table has raw_hex column (#881) + hasScopeName bool // transmissions.scope_name column exists (#899) + hasDefaultScope bool // nodes.default_scope column exists (#899) + hasConfiguredScope bool // nodes.configured_scope column exists (#1865) + hasDeclaredRegionsTable bool // node_declared_regions table exists (#1975, optional second scope source) + hasMultibyteSupCols bool // nodes/inactive_nodes have multibyte_sup/multibyte_evidence (#903) + hasLastSeen bool // transmissions.last_seen column exists (#1690) // Channel list caches, keyed by region param — avoids repeated GROUP BY // scans (#762). Keyed per-region (not a single slot) so mixed-region @@ -237,6 +238,13 @@ func (db *DB) detectSchema(ctx context.Context, q rowQuerier) error { db.hasMultibyteSupCols = nodes["multibyte_sup"] db.hasConfiguredScope = nodes["configured_scope"] + // #1975: an optional second confirmed-scope source. Absent on a stock + // install, so schemaColumns returns nothing and the flag stays false; + // present on deployments that collect the same fact by another route. + // A missing table is not an error here. + ndr, ndrErr := schemaColumns(ctx, q, "node_declared_regions") + db.hasDeclaredRegionsTable = ndrErr == nil && len(ndr) > 0 + if db.isV3 { log.Printf("[db] schema mode: v3 (observer_idx)") } else { diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 3644247e6..f73ce84fe 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -89,6 +89,11 @@ func routeDescriptions() map[string]routeMeta { "GET /api/nodes/{pubkey}/analytics": {Summary: "Get node analytics", Description: "Per-node packet counts, timing, and RF stats.", Tag: "nodes"}, "GET /api/nodes/{pubkey}/neighbors": {Summary: "Get node neighbors", Description: "Returns the queried node's first-hop neighbors with affinity scores and observation metadata (count, SNR, distance, observers). Ambiguous edges carry candidate pubkeys.", Tag: "nodes", Response: schemaRef("NodeNeighborsResponse")}, + "GET /api/scope-audit": {Summary: "Network-wide scope audit", Description: "For every repeater that has answered a declared-regions request: the regions it declares, which of those it has NOT been observed forwarding in the window, which scopes it forwards without declaring, and whether it forwards unscoped floods while omitting the '*' wildcard. '*' is never listed as a region — it governs unscoped floods, not a scope. Repeaters never successfully asked are absent rather than shown as declaring nothing. Rows with missing regions sort first; a short window is weak evidence, since a quiet region simply has no traffic.", Tag: "analytics", + QueryParams: []paramMeta{ + {Name: "window", Description: "Time window: 1h, 24h, or 7d (default 24h)", Type: "string"}, + }}, + // Analytics "GET /api/analytics/rf": {Summary: "RF analytics", Description: "SNR/RSSI distributions and statistics.", Tag: "analytics"}, "GET /api/analytics/topology": {Summary: "Network topology", Description: "Hop-count distribution and route analysis.", Tag: "analytics"}, diff --git a/cmd/server/routes.go b/cmd/server/routes.go index ca844a6ab..c83fde8ac 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -70,6 +70,17 @@ type Server struct { scopeStatsCache map[string]*ScopeStatsResponse scopeStatsCachedAt map[string]time.Time + // #1975: cached /api/scope-audit response, per window, recomputed at most + // once every 30s. Mirrors the scopeStats cache directly above it. + scopeAuditMu sync.Mutex + scopeAuditCache map[string]*ScopeAuditResponse + scopeAuditCachedAt map[string]time.Time + + // #1975: /api/scope-audit window cache and its single-flight guard, so a + // burst of viewers on a cold cache recomputes the network-wide scan once + // rather than once per request. Lives in scope_audit.go. + scopes scopesState + // Router reference for OpenAPI spec generation router *mux.Router @@ -235,6 +246,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/health", s.handleHealth).Methods("GET") r.HandleFunc("/api/stats", s.handleStats).Methods("GET") r.HandleFunc("/api/scope-stats", s.handleScopeStats).Methods("GET") + r.HandleFunc("/api/scope-audit", s.handleScopeAudit).Methods("GET") // #1975 r.HandleFunc("/api/perf", s.handlePerf).Methods("GET") r.HandleFunc("/api/perf/io", s.handlePerfIO).Methods("GET") r.HandleFunc("/api/perf/sqlite", s.handlePerfSqlite).Methods("GET") diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go new file mode 100644 index 000000000..d1aa719ef --- /dev/null +++ b/cmd/server/scope_audit.go @@ -0,0 +1,851 @@ +package main + +// Issue #1975: the network-wide Scope Audit, GET /api/scope-audit. +// +// One row per repeater whose configured region list is known, answering a +// question no other view answers: you declare these regions, but were you +// actually seen forwarding them? default_scope says what a node's adverts were +// observed under and transported_scopes (#1751) says what it carried, but +// nothing lines the declared list up against observed forwarding, so a +// repeater configured for eight regions that only ever forwards one looks +// healthy everywhere else. +// +// Declared side: nodes.configured_scope, written by the observer /neighbors +// ingest from #1865/#1971. Observed side: forwarder hops in the window, +// aggregated per target. Both sides are compared through normScope, so the +// leading "#" that configured_scope carries and a bare region name are the +// same region. +// +// Ported from a long-running fork deployment. The measurements quoted in +// #1975 come from a 1179-repeater instance. + +import ( + "database/sql" + "fmt" + "net/http" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "golang.org/x/sync/singleflight" +) + +// ScopeObservation is one region scope this repeater has been observed +// forwarding traffic for, within the query window. +type ScopeObservation struct { + Scope string `json:"scope"` // matched region name (transmissions.scope_name) + Packets int64 `json:"packets"` + FirstSeen string `json:"firstSeen"` + LastSeen string `json:"lastSeen"` +} + +// RouteTypeMix is the route-type breakdown of packets this node was +// observed FORWARDING — i.e. packets on which this pubkey was the last hop +// of a FLOOD-family route (RouteTransportFlood, RouteFlood). It does NOT +// mean "packets in which this node appears anywhere in the path": a DIRECT +// or TRANSPORT_DIRECT packet's last path hop is the route's far end, never +// the transmitter, so crediting it here would attribute forwarding this node +// never did. Direct and TransportDirect are therefore always zero by +// construction — the forwarder join can never match those route types. + +// scopeConformanceForwarderRouteTypesSQL restricts the forwarder join to the +// only route types whose path[last] is the packet's actual transmitter: +// RouteTransportFlood (0) and RouteFlood (1). A DIRECT route consumes hops +// from the front, so its path[last] is the route's far end rather than the +// forwarder — including RouteDirect (2) / RouteTransportDirect (3) here +// would misattribute a scope to a node that never forwarded the packet. +const scopeConformanceForwarderRouteTypesSQL = "t.route_type IN (0, 1)" + +// minForwarderHopHexLen is the shortest path_json hop ScopeConformance will +// treat as an attribution: 4 hex chars = 2 bytes. Matches deriveHeardKey's +// floor (cmd/ingestor/client_reception.go: "exclude 1-byte (collision-prone), +// matching Reach") — a 1-byte/2-hex-char hash collides too often across a +// real fleet, and attributing a scope to the wrong node is worse than +// attributing none. +const minForwarderHopHexLen = 4 + +type DeclaredRegions struct { + Regions []string `json:"regions"` + ObservedAt string `json:"observedAt"` + Truncated bool `json:"truncated"` +} + +// splitRegionsCSV parses regions_csv (comma-separated, "#" prefixes already +// stripped by the firmware) into a slice, always non-nil. +func splitRegionsCSV(csv string) []string { + regions := []string{} + if csv == "" { + return regions + } + for _, part := range strings.Split(csv, ",") { + part = strings.TrimSpace(part) + if part != "" { + regions = append(regions, part) + } + } + return regions +} + +// CurrentDeclaredRegions returns pubkey's most recently declared region +// list, or nil (not an error) when the repeater has never successfully +// answered, or when node_declared_regions is absent (an older database that +// predates this table). +// +// "Most recent" is ordered by the greatest observed_at, NEVER ingested_at — +// mirrors the ingestor's own CurrentDeclaredRegions exactly: a drive +// buffered offline can arrive days late, and ordering by arrival would let + +type DeclaredRegionsRow struct { + Target string + ObservedAt string + RegionsCSV string + Truncated bool +} + +// AllCurrentDeclaredRegions returns the newest confirmed region list per node, +// merged across every source this database happens to carry. +// +// A confirmed list is a repeater's own answer about which regions it is +// configured to forward, read back off the node rather than inferred from +// traffic. Upstream that answer arrives one way, via the observer /neighbors +// report which #1865/#1971 writes to nodes.configured_scope. It is not the +// only possible collector: neighbor reports have also landed in +// meshcore-packet-capture and openHop, and some deployments gather the same +// answer over a companion app. So the lookup merges rather than hard-coding a +// single table, and an instance with only the stock source is simply the +// one-source case. +// +// Newest answer wins, compared on the recorded instant. Both sources store +// that in canonical UTC RFC3339, so a lexicographic compare is chronological; +// a row with no instant loses to any row that has one, and only wins against +// nothing at all. +// +// A node that has never answered is absent, never synthesized as declaring +// nothing: "no answer" and "answered with nothing" are different states and +// the audit distinguishes them. +func (db *DB) AllCurrentDeclaredRegions() ([]DeclaredRegionsRow, error) { + merged := map[string]DeclaredRegionsRow{} + + keep := func(r DeclaredRegionsRow) { + if r.Target == "" { + return + } + if cur, ok := merged[r.Target]; ok && cur.ObservedAt >= r.ObservedAt { + return + } + merged[r.Target] = r + } + + if db.hasConfiguredScope { + rows, err := db.conn.Query(` + SELECT public_key, COALESCE(configured_scope_at, ''), configured_scope + FROM nodes + WHERE configured_scope IS NOT NULL + `) + if err != nil { + return nil, fmt.Errorf("declared regions from configured_scope: %w", err) + } + for rows.Next() { + var d DeclaredRegionsRow + if err := rows.Scan(&d.Target, &d.ObservedAt, &d.RegionsCSV); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from configured_scope scan: %w", err) + } + // Truncated has no equivalent on this source: the /neighbors report + // has a size cap but does not say whether it fired, and claiming + // false would invent a fact. The page omits the caveat instead. + keep(d) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from configured_scope rows: %w", err) + } + rows.Close() + } + + if db.hasDeclaredRegionsTable { + rows, err := db.conn.Query(` + WITH ranked AS ( + SELECT target, observed_at, regions_csv, truncated, + ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn + FROM node_declared_regions + ) + SELECT target, observed_at, regions_csv, truncated + FROM ranked + WHERE rn = 1 + `) + if err != nil { + return nil, fmt.Errorf("declared regions from node_declared_regions: %w", err) + } + for rows.Next() { + var d DeclaredRegionsRow + var truncated int + if err := rows.Scan(&d.Target, &d.ObservedAt, &d.RegionsCSV, &truncated); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from node_declared_regions scan: %w", err) + } + d.Truncated = truncated == 1 + keep(d) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from node_declared_regions rows: %w", err) + } + rows.Close() + } + + if len(merged) == 0 { + return nil, nil + } + out := make([]DeclaredRegionsRow, 0, len(merged)) + for _, r := range merged { + out = append(out, r) + } + // Stable output: the handler ranks rows itself, but a deterministic order + // keeps responses comparable between calls on unchanged data. + sort.Slice(out, func(i, j int) bool { return out[i].Target < out[j].Target }) + return out, nil +} + +// scopeAuditNodeIdentity is the name/role display identity for one +// scope-audit row, resolved in bulk (one IN query for every declared +// target) rather than one GetNodeByPubkey call per repeater. + +type scopeAuditNodeIdentity struct { + Name *string + Role *string +} + +// scopeAuditNodeIdentities resolves name/role for pubkeys in a single query. +// A pubkey with no matching nodes row (deleted, pruned) resolves to the zero +// value rather than being an error — GET /api/scope-audit still has a +// PublicKey to identify and link the row. +func (db *DB) scopeAuditNodeIdentities(pubkeys []string) map[string]scopeAuditNodeIdentity { + result := make(map[string]scopeAuditNodeIdentity, len(pubkeys)) + if len(pubkeys) == 0 { + return result + } + placeholders := make([]string, len(pubkeys)) + args := make([]interface{}, len(pubkeys)) + for i, k := range pubkeys { + placeholders[i] = "?" + args[i] = strings.ToLower(k) + } + rows, err := db.conn.Query( + "SELECT public_key, name, role FROM nodes WHERE public_key IN ("+strings.Join(placeholders, ",")+")", + args...) + if err != nil { + return result + } + defer rows.Close() + for rows.Next() { + var pk string + var name, role sql.NullString + if rows.Scan(&pk, &name, &role) != nil { + continue + } + id := scopeAuditNodeIdentity{} + if name.Valid { + v := name.String + id.Name = &v + } + if role.Valid { + v := role.String + id.Role = &v + } + result[strings.ToLower(pk)] = id + } + return result +} + +// normScope strips a leading '#' so the two sides of the declared/observed +// comparison can be matched at all — mirrors public/node-scopes.js's +// normScope exactly. transmissions.scope_name keeps the '#' (region keys +// are configured as hashRegions: ["#belgium", "#eu"]); regions_csv arrives +// from the firmware with the prefix already stripped. Comparing raw +// silently inverts the whole comparison while looking entirely plausible. + +func normScope(s string) string { + if strings.HasPrefix(s, "#") { + return s[1:] + } + return s +} + +// scopeAuditTargetAgg is one declared target's forwarding aggregate for the +// GET /api/scope-audit scan window: named scopes it forwarded (key = +// normScope'd name) plus a count of the plain-FLOOD (unscoped) packets it +// forwarded — the signal the wildcard-contradiction check needs, since '*' +// governs exactly those packets, not any named scope. + +type scopeAuditTargetAgg struct { + scopes map[string]*ScopeObservation + unscopedPackets int64 + // ambiguousHops counts forwarder-hop observations in the window whose + // truncated hash prefix matched this target AND at least one other + // declared target — see ScopeAuditForwarding's doc comment for why + // those hops are attributed to neither candidate instead of both. + ambiguousHops int64 +} + +// scopeAuditPrefixIndex builds, for every even hex length from +// minForwarderHopHexLen up to a full 64-char pubkey, a lowercase prefix -> +// []target map. A forwarder hop of any valid truncated-hash length then +// resolves to its matching declared target(s) via a single map lookup, +// instead of a per-target SQL join — this is what keeps +// ScopeAuditForwarding's underlying scan a single query independent of how +// many repeaters have declared a region list. +func scopeAuditPrefixIndex(targets []string) map[int]map[string][]string { + byLen := make(map[int]map[string][]string) + for _, t := range targets { + t = strings.ToLower(t) + for l := minForwarderHopHexLen; l <= len(t); l += 2 { + m, ok := byLen[l] + if !ok { + m = map[string][]string{} + byLen[l] = m + } + prefix := t[:l] + m[prefix] = append(m[prefix], t) + } + } + return byLen +} + +// scopeAuditForwarderScanQuery is the single full-window scan behind +// GET /api/scope-audit. Unlike scopeConformanceQuery (one EXISTS-correlated +// call per pubkey — fine for one node, but 37+ repeater-sized loop of them +// would each re-scan the same first_seen index range), this scans the +// FLOOD-family window exactly once and returns every forwarder hop found. +// It applies the SAME three conditions scopeConformanceQuery does — +// minForwarderHopHexLen, scopeConformanceForwarderRouteTypesSQL, and the +// explicit json_valid guard against a single malformed path_json row +// erroring the whole query — but does not join against any target list: +// attribution to a specific declared target happens in Go +// (ScopeAuditForwarding), against the small in-memory prefix index built by +// scopeAuditPrefixIndex, so the SQL cost stays O(rows in window) regardless +// of len(targets). +var scopeAuditForwarderScanQuery = ` + SELECT t.id, je.value, t.scope_name, t.first_seen + FROM transmissions t + JOIN observations o ON o.transmission_id = t.id + JOIN json_each(o.path_json) je ON je.key = json_array_length(o.path_json) - 1 + WHERE t.first_seen >= ? + AND ` + scopeConformanceForwarderRouteTypesSQL + ` + AND o.path_json IS NOT NULL + AND json_valid(o.path_json) + AND json_array_length(o.path_json) > 0 + AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` +` + +// ScopeAuditForwarding runs scopeAuditForwarderScanQuery once for the whole +// window and attributes every forwarder hop it finds to targets, by the same +// truncated-hash prefix match ScopeConformance uses for a single pubkey. +// +// A hop is attributed only when its prefix matches EXACTLY ONE declared +// target. This endpoint exists to find a repeater that declares a region and +// is not actually forwarding it — crediting a hop to every target sharing +// its prefix would let a colliding neighbour's traffic silently paper over a +// real gap, and crediting nobody (the alternative of dropping the hop +// entirely) would invent failures for targets that simply share a collision- +// prone prefix. Instead, an ambiguous hop is credited to NEITHER candidate, +// and every candidate's ambiguousHops counter is incremented instead, so the +// row can say "this notObserved might just be a prefix collision" rather +// than presenting it as a confirmed finding. See scopeAuditTargetAgg's +// ambiguousHops field and ScopeAuditRow.AmbiguousHops. +// +// Each (target, transmission) pair is counted at most once even if seen via +// multiple observations, for both the attributed and the ambiguous count — +// the same de-duplication scopeConformanceQuery gets for free from EXISTS, +// done explicitly here since this scan is not correlated per target. + +func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (map[string]*scopeAuditTargetAgg, error) { + byLen := scopeAuditPrefixIndex(targets) + result := make(map[string]*scopeAuditTargetAgg, len(targets)) + + rows, err := s.db.conn.Query(scopeAuditForwarderScanQuery, sinceISO) + if err != nil { + return nil, fmt.Errorf("scope audit forwarder scan: %w", err) + } + defer rows.Close() + + getAgg := func(target string) *scopeAuditTargetAgg { + agg, ok := result[target] + if !ok { + agg = &scopeAuditTargetAgg{scopes: map[string]*ScopeObservation{}} + result[target] = agg + } + return agg + } + + seen := make(map[string]bool) // "|" already counted (attributed or ambiguous) + for rows.Next() { + var txID int64 + var hop string + var scopeName sql.NullString + var firstSeen string + if err := rows.Scan(&txID, &hop, &scopeName, &firstSeen); err != nil { + return nil, fmt.Errorf("scope audit forwarder scan scan: %w", err) + } + hop = strings.ToLower(hop) + candidates := byLen[len(hop)][hop] + + if len(candidates) > 1 { + for _, target := range candidates { + key := target + "|" + strconv.FormatInt(txID, 10) + if seen[key] { + continue + } + seen[key] = true + getAgg(target).ambiguousHops++ + } + continue + } + for _, target := range candidates { + key := target + "|" + strconv.FormatInt(txID, 10) + if seen[key] { + continue + } + seen[key] = true + + agg := getAgg(target) + if !scopeName.Valid { + agg.unscopedPackets++ + continue + } + if scopeName.String == "" { + continue // unmatched — not part of the declared/observed comparison + } + name := normScope(scopeName.String) + so, ok := agg.scopes[name] + if !ok { + so = &ScopeObservation{Scope: name, FirstSeen: firstSeen, LastSeen: firstSeen} + agg.scopes[name] = so + } + so.Packets++ + if firstSeen < so.FirstSeen { + so.FirstSeen = firstSeen + } + if firstSeen > so.LastSeen { + so.LastSeen = firstSeen + } + } + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("scope audit forwarder scan rows: %w", err) + } + return result, nil +} + +// Scope audit configuration-state values — see ScopeAuditRow.ConfigState and +// scopeAuditConfigState. + +const ( + ScopeConfigFull = "full" // named regions AND '*' + ScopeConfigNoScopes = "no-scopes" // '*' only, no named regions + ScopeConfigNoUnscoped = "no-unscoped" // named regions, no '*' + ScopeConfigNoFlood = "no-flood" // neither named regions nor '*' +) + +// scopeAuditConfigState classifies a repeater's declared-regions answer into +// one of four configuration states, from ONLY the two fields the caller has +// already parsed out of the raw declared list (namedRegions and wildcard) — +// never re-derived from regions_csv, which splitRegionsCSV/normScope have +// already reduced to exactly these two facts. +// +// The firmware exports the FLOOD-allowed set (region_map.exportNamesTo(..., +// REGION_DENY_FLOOD) in examples/simple_repeater/MyMesh.cpp), so "no named +// regions in the export" does not strictly prove "no regions defined": a +// repeater with regions defined but every one of them marked deny-flood +// exports exactly the same list as one with no region tree at all, and the +// two are indistinguishable from this data alone. That caveat lands on +// ScopeConfigNoScopes and ScopeConfigNoFlood, both of which read "zero named +// regions in the export". It does NOT land on the wildcard half of the +// classification: '*' absent from the export is exact, not an inference — +// it means the wildcard denies flooding, i.e. plain unscoped floods are not +// forwarded, full stop. That exactness is why ScopeConfigNoUnscoped needs no +// caveat, and why ScopeConfigNoFlood's "no unscoped forwarding" half is +// exact even though its "no named regions" half is not. +func scopeAuditConfigState(namedRegions []string, wildcard bool) string { + switch { + case len(namedRegions) > 0 && wildcard: + return ScopeConfigFull + case len(namedRegions) == 0 && wildcard: + return ScopeConfigNoScopes + case len(namedRegions) > 0 && !wildcard: + return ScopeConfigNoUnscoped + default: + return ScopeConfigNoFlood + } +} + +// ScopeAuditRow is one repeater's declared-vs-observed comparison for +// GET /api/scope-audit — the network-wide answer to "which repeaters +// declare a region they are not actually forwarding". Every field is +// already normalised (no leading '#') and '*' is never present in +// DeclaredRegions/NotObserved/UndeclaredObserved — see DeclaredWildcard and +// WildcardContradiction for its counterpart. + +type ScopeAuditRow struct { + PublicKey string `json:"publicKey"` + // Name/Role are pointers so "we hold no nodes row for this target" serialises + // as null rather than "". A declared-regions row can name a repeater this + // instance has never recorded, and an empty string would make that + // indistinguishable from a node we DO know that simply has no name — the + // same absent-is-not-empty rule the declared side already follows. + Name *string `json:"name"` + Role *string `json:"role"` + + DeclaredRegions []string `json:"declaredRegions"` // '*' excluded — see DeclaredWildcard + DeclaredWildcard bool `json:"declaredWildcard"` // '*' present in the raw declared list + // ConfigState is scopeAuditConfigState(DeclaredRegions, DeclaredWildcard) + // — one of ScopeConfigFull/NoScopes/NoUnscoped/NoFlood, the config-state + // reading of those same two fields so a caller does not have to + // re-derive it from them. See scopeAuditConfigState's doc comment for + // the caveat on the "no named regions" half of NoScopes/NoFlood. + ConfigState string `json:"configState"` + DeclaredAt string `json:"declaredAt"` // ISO — age of the declared answer, not the window + Truncated bool `json:"truncated"` // declared list may have had entries silently dropped + + // NotObserved is declared regions with zero matched-forwarding observed + // in the window — the headline this endpoint exists to surface. + NotObserved []string `json:"notObserved"` + // UndeclaredObserved is scopes this repeater was observed forwarding + // that are absent from its declared list. + UndeclaredObserved []ScopeObservation `json:"undeclaredObserved"` + + // ObservedUnscopedPackets is plain-FLOOD (no transport scope) packets + // this repeater was observed forwarding in the window — the '*' + // counterpart, per DeclaredWildcard's doc comment. + ObservedUnscopedPackets int64 `json:"observedUnscopedPackets"` + // WildcardContradiction is true when this repeater was observed + // forwarding unscoped floods but its declared list omits '*' — it says + // it does NOT forward them, and the traffic says otherwise. + WildcardContradiction bool `json:"wildcardContradiction"` + + // AmbiguousHops counts forwarder-hop observations in the window whose + // truncated hash prefix matched this target's pubkey AND at least one + // other declared target's — see ScopeAuditForwarding's doc comment for + // why those hops are attributed to neither and instead counted here on + // every candidate. A non-zero value is a caveat, not a finding: any + // NotObserved entry on this row could be explained by a colliding + // neighbour's traffic rather than a real absence, and any + // UndeclaredObserved entry is unaffected by it (ambiguous hops are never + // attributed to a scope at all). + AmbiguousHops int64 `json:"ambiguousHops"` +} + +// ScopeAuditResponse is the payload for GET /api/scope-audit. Only +// repeaters with at least one declared-regions answer are included — a +// repeater never successfully asked is absent, not shown declaring nothing +// (see AllCurrentDeclaredRegions). +type ScopeAuditResponse struct { + Window string `json:"window"` + Since string `json:"since"` // ISO — start of the observed-forwarding window + Repeaters []ScopeAuditRow `json:"repeaters"` +} + +// NodeScopesResponse is the payload for GET /api/nodes/{pubkey}/scopes: the +// observed-forwarding side (ScopeConformance, embedded BY VALUE so its three +// scope states sit at the JSON top level — unmatched and unscoped are +// separate counts and a matched scope never has an empty name) plus the +// declared side (DeclaredRegions), returned together so the UI needs only +// one request per node rather than a second per-item call. +// +// ScopeConformance is embedded by value rather than as *ScopeConformance: +// encoding/json silently skips fields promoted through a nil embedded +// pointer instead of erroring, which would drop observed/unmatched/unscoped +// from the body entirely rather than surfacing the failure. + +type scopesState struct { + cacheMu sync.RWMutex + cache map[string]scopesCacheEntry + sf singleflight.Group + + // lastSeenBlacklistGen mirrors reachState's field: when the live + // blacklist generation advances past this value, the cache is purged + // wholesale on the next request. The 404 path itself is never cached + // (the handler returns before a cache key is even computed), so this + // only guards a pre-existing successful entry from outliving a + // blacklist/hide edit made after it was filled. + lastSeenBlacklistGen atomic.Uint64 +} + +type scopesCacheEntry struct { + at time.Time + raw []byte +} + +const ( + // nodeScopesCacheTTL matches the sibling /api/scope-stats endpoint's + // cache lifetime — also per-window region-scope data recomputed from + // the same transmissions table. + nodeScopesCacheTTL = 30 * time.Second + nodeScopesCacheMax = 256 +) + +// scopesCacheGet returns the cached marshalled JSON for key. The returned +// slice is shared (not copied) and MUST NOT be mutated by callers. +func (s *Server) scopesCacheGet(key string) ([]byte, bool) { + s.scopes.cacheMu.RLock() + defer s.scopes.cacheMu.RUnlock() + e, ok := s.scopes.cache[key] + if !ok || time.Since(e.at) > nodeScopesCacheTTL { + return nil, false + } + return e.raw, true +} + +func (s *Server) scopesCachePut(key string, raw []byte) { + s.scopes.cacheMu.Lock() + defer s.scopes.cacheMu.Unlock() + if s.scopes.cache == nil { + s.scopes.cache = map[string]scopesCacheEntry{} + } + if _, exists := s.scopes.cache[key]; !exists && len(s.scopes.cache) >= nodeScopesCacheMax { + s.evictScopesLocked() + } + s.scopes.cache[key] = scopesCacheEntry{at: time.Now(), raw: raw} +} + +// evictScopesLocked drops expired entries first; if still at the cap it +// evicts the single oldest entry. Caller holds s.scopes.cacheMu (write). +func (s *Server) evictScopesLocked() { + now := time.Now() + for k, e := range s.scopes.cache { + if now.Sub(e.at) > nodeScopesCacheTTL { + delete(s.scopes.cache, k) + } + } + if len(s.scopes.cache) < nodeScopesCacheMax { + return + } + var oldestKey string + var oldestAt time.Time + first := true + for k, e := range s.scopes.cache { + if first || e.at.Before(oldestAt) { + oldestKey, oldestAt, first = k, e.at, false + } + } + if !first { + delete(s.scopes.cache, oldestKey) + } +} + +// scopesPurgeIfBlacklistGenChanged drops every cached entry when the live +// blacklist generation has advanced past the cache's last-seen value, +// mirroring reachPurgeIfBlacklistGenChanged. CAS gates the purge so +// concurrent callers only do the work once per gen bump. +func (s *Server) scopesPurgeIfBlacklistGenChanged(gen uint64) { + seen := s.scopes.lastSeenBlacklistGen.Load() + if gen == seen { + return + } + if !s.scopes.lastSeenBlacklistGen.CompareAndSwap(seen, gen) { + return + } + s.scopes.cacheMu.Lock() + s.scopes.cache = nil + s.scopes.cacheMu.Unlock() +} + +// nodeScopesWindowLookback maps the ?window= vocabulary to a lookback +// duration. Matches the sibling /api/scope-stats endpoint's vocabulary +// exactly (1h, 24h, 7d) rather than the broader ParseTimeWindow alias set +// (which also accepts 1d/3d/1w/30d) used by unrelated analytics endpoints. + +func nodeScopesWindowLookback(window string) (time.Duration, bool) { + switch window { + case "1h": + return time.Hour, true + case "24h": + return 24 * time.Hour, true + case "7d": + return 7 * 24 * time.Hour, true + default: + return 0, false + } +} + +// handleNodeScopes serves GET /api/nodes/{pubkey}/scopes?window=1h|24h|7d. +// +// A pubkey never heard forwarding anything is a valid question with an +// empty answer (200), not a 404 — ScopeConformance already treats it that +// way, and this handler performs no node-existence lookup that would +// override it. + +// handleScopeAudit serves GET /api/scope-audit?window=1h|24h|7d: the +// network-wide answer to "which repeaters declare a region they are not +// actually forwarding". Unlike the per-repeater /api/nodes/{pubkey}/scopes, +// this compares every repeater that has ever declared a region list in one +// pass — see scopes.go's AllCurrentDeclaredRegions and ScopeAuditForwarding +// for why that stays a single scan rather than one query per repeater. +func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { + const scopeAuditTTL = 30 * time.Second + + window := r.URL.Query().Get("window") + if window == "" { + window = "24h" + } + lookback, ok := nodeScopesWindowLookback(window) + if !ok { + writeError(w, 400, "window must be 1h, 24h, or 7d") + return + } + + s.scopeAuditMu.Lock() + if s.scopeAuditCache != nil { + if cached, ok := s.scopeAuditCache[window]; ok && time.Since(s.scopeAuditCachedAt[window]) < scopeAuditTTL { + s.scopeAuditMu.Unlock() + writeJSON(w, cached) + return + } + } + s.scopeAuditMu.Unlock() + + declared, err := s.db.AllCurrentDeclaredRegions() + if err != nil { + writeError(w, 500, err.Error()) + return + } + + targets := make([]string, 0, len(declared)) + for _, d := range declared { + targets = append(targets, strings.ToLower(d.Target)) + } + + sinceISO := time.Now().Add(-lookback).UTC().Format(time.RFC3339) + forwarding := map[string]*scopeAuditTargetAgg{} + if s.store != nil { + forwarding, err = s.store.ScopeAuditForwarding(sinceISO, targets) + if err != nil { + writeError(w, 500, err.Error()) + return + } + } + + identities := s.db.scopeAuditNodeIdentities(targets) + + resp := &ScopeAuditResponse{Window: window, Since: sinceISO, Repeaters: []ScopeAuditRow{}} + for _, d := range declared { + pk := strings.ToLower(d.Target) + if s.cfg != nil && s.cfg.IsBlacklisted(pk) { + continue + } + id := identities[pk] + // A target with no nodes row has no name to match a hidden-prefix rule + // against; it cannot be hidden by name, so only check when we have one. + if id.Name != nil && s.cfg != nil && s.cfg.IsNameHidden(*id.Name) { + continue + } + + allRegions := splitRegionsCSV(d.RegionsCSV) + declaredWildcard := false + declaredNamed := make([]string, 0, len(allRegions)) + declaredSet := make(map[string]bool, len(allRegions)) + for _, rgn := range allRegions { + if rgn == "*" { + declaredWildcard = true + continue + } + // normScope mirrors the observed side (scopes.go: agg.scopes keys + // are already normScope'd) — regions_csv is not guaranteed to + // arrive with '#' already stripped in every case, and comparing + // raw here would reintroduce the exact trap normScope exists to + // prevent, just on the other side of the comparison. + rgn = normScope(rgn) + declaredNamed = append(declaredNamed, rgn) + declaredSet[rgn] = true + } + + agg := forwarding[pk] + + notObserved := []string{} + for _, rgn := range declaredNamed { + if agg == nil || agg.scopes[rgn] == nil { + notObserved = append(notObserved, rgn) + } + } + + undeclared := []ScopeObservation{} + if agg != nil { + names := make([]string, 0, len(agg.scopes)) + for name := range agg.scopes { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if !declaredSet[name] { + undeclared = append(undeclared, *agg.scopes[name]) + } + } + } + + var unscopedPackets, ambiguousHops int64 + if agg != nil { + unscopedPackets = agg.unscopedPackets + ambiguousHops = agg.ambiguousHops + } + + resp.Repeaters = append(resp.Repeaters, ScopeAuditRow{ + PublicKey: pk, + Name: id.Name, + Role: id.Role, + DeclaredRegions: declaredNamed, + DeclaredWildcard: declaredWildcard, + ConfigState: scopeAuditConfigState(declaredNamed, declaredWildcard), + DeclaredAt: d.ObservedAt, + Truncated: d.Truncated, + NotObserved: notObserved, + UndeclaredObserved: undeclared, + ObservedUnscopedPackets: unscopedPackets, + WildcardContradiction: unscopedPackets > 0 && !declaredWildcard, + AmbiguousHops: ambiguousHops, + }) + } + + // Interesting rows first: a repeater silently not forwarding a declared + // region is the headline this endpoint exists to surface, ranked by how + // many declared regions it's missing. The wildcard contradiction and + // undeclared-observed counts are secondary tie-breaks — flags on the + // row, not a separate ranking. Full agreement (no issues at all) sorts + // to the bottom, alphabetically, so it doesn't crowd out the rows that + // matter. + sort.Slice(resp.Repeaters, func(i, j int) bool { + a, b := resp.Repeaters[i], resp.Repeaters[j] + if len(a.NotObserved) != len(b.NotObserved) { + return len(a.NotObserved) > len(b.NotObserved) + } + if a.WildcardContradiction != b.WildcardContradiction { + return a.WildcardContradiction + } + if len(a.UndeclaredObserved) != len(b.UndeclaredObserved) { + return len(a.UndeclaredObserved) > len(b.UndeclaredObserved) + } + // Fall back to the pubkey for an unnamed or unknown node so the order is + // still total and stable rather than grouping every nameless row together. + an, bn := a.PublicKey, b.PublicKey + if a.Name != nil && *a.Name != "" { + an = *a.Name + } + if b.Name != nil && *b.Name != "" { + bn = *b.Name + } + return an < bn + }) + + s.scopeAuditMu.Lock() + if s.scopeAuditCache == nil { + s.scopeAuditCache = make(map[string]*ScopeAuditResponse) + s.scopeAuditCachedAt = make(map[string]time.Time) + } + s.scopeAuditCache[window] = resp + s.scopeAuditCachedAt[window] = time.Now() + s.scopeAuditMu.Unlock() + + writeJSON(w, resp) +} diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go new file mode 100644 index 000000000..00ecb5e5f --- /dev/null +++ b/cmd/server/scope_audit_test.go @@ -0,0 +1,855 @@ +package main + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// Full 64-hex pubkeys for fixtures. path_json hops are truncated hashes of +// 1 to 4 bytes, so the forwarder join matches a hop as a PREFIX of the full +// pubkey rather than by equality; the tests need a real full-length key to +// exercise that. +var ( + testFullPubkeyA = "1a2b" + strings.Repeat("11", 30) // 64 hex chars + testFullPubkeyB = "bbbb" + strings.Repeat("22", 30) // 64 hex chars +) + +// setupScopeConformanceDB builds an in-memory transmissions/observations pair +// carrying exactly the columns ScopeConformance reads: code1/scope_name/ +// first_seen/route_type on transmissions, path_json on observations. Mirrors +// the live ingestor schema (cmd/ingestor/db.go) rather than a convenient +// fiction, so the join behaves the way it does against a real database. +func setupScopeConformanceDB(t *testing.T) *DB { + t.Helper() + conn, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatal(err) + } + conn.SetMaxOpenConns(1) + schema := ` + CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + raw_hex TEXT NOT NULL, + hash TEXT NOT NULL UNIQUE, + first_seen TEXT NOT NULL, + route_type INTEGER, + payload_type INTEGER, + code1 TEXT, + code2 TEXT, + scope_name TEXT + ); + CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL REFERENCES transmissions(id), + path_json TEXT, + timestamp INTEGER NOT NULL + ); + ` + if _, err := conn.Exec(schema); err != nil { + t.Fatal(err) + } + return &DB{conn: conn} +} + +// newScopeTestStore wires a *PacketStore to a freshly seeded scope-conformance +// schema. ScopeConformance is a pure SQL read against s.db, so no other +// PacketStore field needs to be populated. +func newScopeTestStore(t *testing.T) *PacketStore { + t.Helper() + db := setupScopeConformanceDB(t) + return newTestStoreWithDB(t, db, &Config{}) +} + +// scopeSeed carries the code1/scope_name pair for one of the three states a +// seeded transmission can be in. scopeName mirrors scopeNameForDB's own +// encoding: nil means the packet carried no scope at all, a non-nil pointer +// to an empty string means transport-scoped but unmatched, and a non-nil +// pointer to a name means matched. +type scopeSeed struct { + code1 string + scopeName *string +} + +func scopeMatched(name string) scopeSeed { + n := name + return scopeSeed{code1: "1234", scopeName: &n} +} + +func scopeUnmatched() scopeSeed { + empty := "" + return scopeSeed{code1: "1234", scopeName: &empty} +} + +func scopeUnscoped() scopeSeed { + return scopeSeed{code1: "0000", scopeName: nil} +} + +var scopeSeedCounter int + +// seedTransmissionRoute inserts one transmission plus a single observation +// attributing it to forwarder, built the way the ingestor would build it: the +// path_json hop is uppercase (packetpath.DecodePathFromRawHex does +// strings.ToUpper on every hop), so the seed exercises the same case the +// live join has to cope with rather than a lowercase convenience fiction. +func seedTransmissionRoute(t *testing.T, s *PacketStore, forwarder string, seed scopeSeed, routeType int) { + t.Helper() + seedTransmissionRouteAt(t, s, forwarder, seed, routeType, "2026-01-15T12:00:00Z") +} + +// seedTransmissionRouteAt is seedTransmissionRoute with an explicit +// first_seen, for tests (e.g. the handler tests below) that need a +// transmission to fall inside a real-wall-clock ?window= lookback rather +// than the fixed date the ScopeConformance unit tests above use. +func seedTransmissionRouteAt(t *testing.T, s *PacketStore, forwarder string, seed scopeSeed, routeType int, firstSeen string) { + t.Helper() + scopeSeedCounter++ + hash := fmt.Sprintf("scopehash%d", scopeSeedCounter) + + res, err := s.db.conn.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) + VALUES ('AA', ?, ?, ?, 1, ?, '00', ?)`, + hash, firstSeen, routeType, seed.code1, seed.scopeName, + ) + if err != nil { + t.Fatalf("seed transmission: %v", err) + } + txID, err := res.LastInsertId() + if err != nil { + t.Fatalf("seed transmission id: %v", err) + } + + pathJSON := fmt.Sprintf(`["%s"]`, strings.ToUpper(forwarder)) + if _, err := s.db.conn.Exec( + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, + txID, pathJSON, time.Now().Unix(), + ); err != nil { + t.Fatalf("seed observation: %v", err) + } +} + +// seedTransmission seeds a FLOOD packet (route_type=1) — path[last] is the +// actual transmitter, so forwarder is attributable. +func seedTransmission(t *testing.T, s *PacketStore, forwarder string, seed scopeSeed) { + t.Helper() + seedTransmissionRoute(t, s, forwarder, seed, RouteFlood) +} + +// seedDirectTransmission seeds a DIRECT packet (route_type=2) — path[last] is +// the route's far end, never the transmitter, so forwarder must NOT be +// attributed even though it appears in path_json. +func seedDirectTransmission(t *testing.T, s *PacketStore, forwarder string, seed scopeSeed) { + t.Helper() + seedTransmissionRoute(t, s, forwarder, seed, RouteDirect) +} + +func TestScopeAuditForwardingAttributesUnambiguousHop(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, s, hop, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil || agg.scopes["be"] == nil || agg.scopes["be"].Packets != 1 { + t.Fatalf("want the hop attributed to the sole matching target, got %+v", got) + } + if agg.ambiguousHops != 0 { + t.Errorf("ambiguousHops = %d, want 0 — nothing here is ambiguous", agg.ambiguousHops) + } +} + +// TestScopeAuditForwardingAmbiguousHopCreditsNeitherTarget is FIX 1's core +// case: two declared targets share a 2-byte prefix. A hop at that prefix +// must be attributed to NEITHER of them (crediting both would let a +// colliding neighbour's traffic silently paper over a real notObserved +// finding), and BOTH candidates' ambiguousHops counters must increment (so +// the row can surface the caveat regardless of which candidate is being +// looked at). +func TestScopeAuditForwardingAmbiguousHopCreditsNeitherTarget(t *testing.T) { + s := newScopeTestStore(t) + pkA := "1a2b" + strings.Repeat("11", 30) + pkB := "1a2b" + strings.Repeat("22", 30) // shares pkA's first 4 hex chars + hop := pkA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, s, hop, scopeMatched("#be"), RouteFlood, recent) + + targets := []string{pkA, pkB} + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", targets) + if err != nil { + t.Fatal(err) + } + for _, pk := range targets { + agg := got[pk] + if agg == nil { + t.Fatalf("%s: want an agg present to carry the ambiguity counter, got none (result = %+v)", pk, got) + } + if len(agg.scopes) != 0 { + t.Errorf("%s: scopes = %+v, want empty — an ambiguous hop must not be attributed to either candidate", pk, agg.scopes) + } + if agg.unscopedPackets != 0 { + t.Errorf("%s: unscopedPackets = %d, want 0", pk, agg.unscopedPackets) + } + if agg.ambiguousHops != 1 { + t.Errorf("%s: ambiguousHops = %d, want 1", pk, agg.ambiguousHops) + } + } +} + +// --- GET /api/scope-audit handler tests --- + +// setupScopeAuditServer builds a *Server over the ScopeConformance schema +// plus a nodes table carrying the confirmed-scope columns from #1865/#1971, +// which is where this endpoint reads its declared side. detectSchema must run +// after the columns exist so hasConfiguredScope is picked up; without it the +// handler correctly serves an empty audit and every assertion below would pass +// vacuously. +func setupScopeAuditServer(t *testing.T) (*Server, *mux.Router) { + t.Helper() + db := setupScopeConformanceDB(t) + if _, err := db.conn.Exec(`CREATE TABLE nodes ( + public_key TEXT PRIMARY KEY, + name TEXT, + role TEXT, + configured_scope TEXT, + configured_scope_at TEXT + )`); err != nil { + t.Fatal(err) + } + if err := db.detectSchema(context.Background(), db.conn); err != nil { + t.Fatal(err) + } + if !db.hasConfiguredScope { + t.Fatal("hasConfiguredScope is false after creating the column: the fixture would test nothing") + } + cfg := &Config{Port: 3000} + hub := NewHub() + srv := NewServer(db, cfg, hub) + srv.store = newTestStoreWithDB(t, db, cfg) + router := mux.NewRouter() + srv.RegisterRoutes(router) + return srv, router +} + +// insertDeclared records a node's confirmed region list. Keeps the original +// signature so the ported cases read unchanged, but writes to the upstream +// source: nodes.configured_scope, which an observer /neighbors report fills +// (#1865/#1971). +// +// truncated is accepted and ignored. The old source recorded whether an answer +// was cut short; /neighbors has its own size cap but does not report whether it +// fired, so the audit cannot claim either way and the page omits the caveat +// rather than showing a wrong one. Kept in the signature so the ported tests +// still document which fixtures meant "this answer was incomplete". +func insertDeclared(t *testing.T, srv *Server, target, observedAt, regionsCSV string, truncated int) { + t.Helper() + _ = truncated + if _, err := srv.db.conn.Exec( + `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES (?, ?, ?) + ON CONFLICT(public_key) DO UPDATE SET configured_scope = excluded.configured_scope, + configured_scope_at = excluded.configured_scope_at`, + target, regionsCSV, observedAt); err != nil { + t.Fatal(err) + } +} + +func getScopeAudit(t *testing.T, router *mux.Router, query string) ScopeAuditResponse { + t.Helper() + req := httptest.NewRequest("GET", "/api/scope-audit"+query, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var got ScopeAuditResponse + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatalf("decode: %v", err) + } + return got +} + +// TestHandleScopeAuditNormalisesHashPrefix pins trap 1: transmissions.scope_name +// keeps the '#' (hashRegions config), regions_csv arrives from the firmware +// with it already stripped. Declared "be-van" and observed "#be-van" must be +// recognised as the same scope, not reported as both missing and undeclared. +func TestHandleScopeAuditNormalisesHashPrefix(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + hop := pk[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "be-van", 0) + seedTransmissionRouteAt(t, srv.store, hop, scopeMatched("#be-van"), RouteFlood, recent) + + // FIX 2: the DECLARED side must be normalised too, not just observed. + // regions_csv is not guaranteed to arrive with '#' already stripped (a + // firmware variant, an operator-seeded row, or a future collector could + // leave it on) — declaring "#be-van" and observing "#be-van" (which the + // observed side always normalises to "be-van") must still match. Without + // the fix, declaredNamed/declaredSet keep the raw "#be-van", so this + // second repeater's row would show BOTH "#be-van" in notObserved (the + // declared value never matches the normalised agg.scopes key) AND + // "be-van" in undeclaredObserved (the normalised observed name isn't in + // declaredSet) — the exact trap normScope exists to prevent, reappearing + // on the other side of the comparison. Seeded alongside pk (not as a + // separate getScopeAudit call) so both land in one response and neither + // is masked by the 30s response cache. + pk2 := testFullPubkeyB + hop2 := pk2[:4] + insertDeclared(t, srv, pk2, time.Now().UTC().Format(time.RFC3339), "#be-van", 0) + seedTransmissionRouteAt(t, srv.store, hop2, scopeMatched("#be-van"), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 2 { + t.Fatalf("repeaters = %+v, want exactly 2", got.Repeaters) + } + + row := findScopeAuditRow(t, got.Repeaters, pk) + if len(row.NotObserved) != 0 { + t.Errorf("notObserved = %v, want empty — declared \"be-van\" observed as \"#be-van\" must match after normalisation", row.NotObserved) + } + if len(row.UndeclaredObserved) != 0 { + t.Errorf("undeclaredObserved = %+v, want empty", row.UndeclaredObserved) + } + + row2 := findScopeAuditRow(t, got.Repeaters, pk2) + if len(row2.DeclaredRegions) != 1 || row2.DeclaredRegions[0] != "be-van" { + t.Errorf("declaredRegions = %v, want [\"be-van\"] — the leading '#' must be stripped from the declared side too", row2.DeclaredRegions) + } + if len(row2.NotObserved) != 0 { + t.Errorf("notObserved = %v, want empty — declared \"#be-van\" observed as \"#be-van\" must match after normalising BOTH sides", row2.NotObserved) + } + if len(row2.UndeclaredObserved) != 0 { + t.Errorf("undeclaredObserved = %+v, want empty", row2.UndeclaredObserved) + } +} + +// findScopeAuditRow locates the row for pk, failing the test if absent — +// used wherever more than one repeater is seeded in the same response, since +// sort order is not by pubkey and cannot be relied on to pick the right row. +func findScopeAuditRow(t *testing.T, rows []ScopeAuditRow, pk string) ScopeAuditRow { + t.Helper() + for _, r := range rows { + if r.PublicKey == pk { + return r + } + } + t.Fatalf("no row for pubkey %s in %+v", pk, rows) + return ScopeAuditRow{} +} + +// TestHandleScopeAuditExcludesWildcardFromComparison pins trap 2: '*' is the +// root of the region tree (governs plain FLOOD), not a scope. It must never +// appear in declaredRegions (or its count) or in notObserved/undeclaredObserved. +func TestHandleScopeAuditExcludesWildcardFromComparison(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "*,be", 0) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if !row.DeclaredWildcard { + t.Error("declaredWildcard = false, want true") + } + if len(row.DeclaredRegions) != 1 || row.DeclaredRegions[0] != "be" { + t.Errorf("declaredRegions = %v, want [\"be\"] — '*' must be excluded from the region list and its count", row.DeclaredRegions) + } + for _, r := range row.NotObserved { + if r == "*" { + t.Error("notObserved contains '*' — it must never be treated as a scope") + } + } +} + +// TestHandleScopeAuditOmitsRepeaterWithNoDeclaredRow pins trap 3: a repeater +// that was never successfully asked is absent from the response entirely — +// not shown as a row declaring nothing, which is a distinct, meaningful fact. +func TestHandleScopeAuditOmitsRepeaterWithNoDeclaredRow(t *testing.T) { + srv, router := setupScopeAuditServer(t) + declaredPk := testFullPubkeyA + neverAskedPk := testFullPubkeyB + insertDeclared(t, srv, declaredPk, time.Now().UTC().Format(time.RFC3339), "be", 0) + if _, err := srv.db.conn.Exec(`INSERT INTO nodes (public_key, name, role) VALUES (?, 'never-asked', 'repeater')`, neverAskedPk); err != nil { + t.Fatal(err) + } + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 || got.Repeaters[0].PublicKey != declaredPk { + t.Fatalf("repeaters = %+v, want exactly the one repeater that has declared", got.Repeaters) + } +} + +// TestHandleScopeAuditUnknownNodeNameIsNullNotEmpty proves a declared target +// this instance holds no nodes row for serialises name/role as null, not "". +// A declared-regions answer can name a repeater the network has never recorded, +// and "" would make that indistinguishable from a node we DO know that simply +// has no name — the same absent-is-not-empty rule the declared side follows. +func TestHandleScopeAuditUnknownNodeNameIsNullNotEmpty(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "be", 0) + // deliberately NO nodes row for pk + + req := httptest.NewRequest("GET", "/api/scope-audit", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + body := w.Body.String() + if !strings.Contains(body, `"name":null`) { + t.Errorf(`body must carry "name":null for a target with no nodes row, got: %s`, body) + } + if strings.Contains(body, `"name":""`) { + t.Error(`"name":"" collapses "unknown node" into "node with no name"`) + } + + var got ScopeAuditResponse + if err := json.Unmarshal(w.Body.Bytes(), &got); err != nil { + t.Fatal(err) + } + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %d, want 1", len(got.Repeaters)) + } + if got.Repeaters[0].Name != nil { + t.Errorf("Name = %q, want nil", *got.Repeaters[0].Name) + } +} + +// TestHandleScopeAuditWildcardContradiction: observed forwarding unscoped +// (plain-FLOOD) traffic while the declared list omits '*' is the wildcard +// contradiction this endpoint must flag — the repeater says it does NOT +// forward those packets, and the traffic says otherwise. +func TestHandleScopeAuditWildcardContradiction(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + hop := pk[:4] + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "be", 0) // no '*' + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, hop, scopeUnscoped(), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if !row.WildcardContradiction { + t.Error("wildcardContradiction = false, want true — observed unscoped forwarding but '*' not declared") + } + if row.ObservedUnscopedPackets != 1 { + t.Errorf("observedUnscopedPackets = %d, want 1", row.ObservedUnscopedPackets) + } +} + +// TestHandleScopeAuditWildcardDeclaredIsNotAContradiction is the other half: +// the same observed unscoped traffic is expected, not a contradiction, once +// '*' is declared. +func TestHandleScopeAuditWildcardDeclaredIsNotAContradiction(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + hop := pk[:4] + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "*,be", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, hop, scopeUnscoped(), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + if got.Repeaters[0].WildcardContradiction { + t.Error("wildcardContradiction = true, want false — '*' IS declared, so unscoped forwarding is expected") + } +} + +// TestHandleScopeAuditNotObservedAndUndeclaredObserved exercises both +// comparison directions in one repeater: a declared region with zero +// observed forwarding, and an observed scope absent from the declared list. +func TestHandleScopeAuditNotObservedAndUndeclaredObserved(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + hop := pk[:4] + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "be,be-vlg", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, hop, scopeMatched("#be"), RouteFlood, recent) + seedTransmissionRouteAt(t, srv.store, hop, scopeMatched("#de-nw"), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + row := got.Repeaters[0] + if len(row.NotObserved) != 1 || row.NotObserved[0] != "be-vlg" { + t.Errorf("notObserved = %v, want [\"be-vlg\"]", row.NotObserved) + } + if len(row.UndeclaredObserved) != 1 || row.UndeclaredObserved[0].Scope != "de-nw" { + t.Errorf("undeclaredObserved = %+v, want exactly \"de-nw\"", row.UndeclaredObserved) + } +} + +// TestHandleScopeAuditSurfacesAmbiguousHops is FIX 1's handler-level case: +// two repeaters both declare "be" and share a 2-byte pubkey prefix. The one +// hop seen in the window can't be attributed to either, so both rows must +// still show "be" as notObserved (an ambiguous hop invents no attribution, +// so it cannot silently satisfy the declared region) AND both rows must +// carry ambiguousHops=1, the caveat that the notObserved finding might be a +// prefix collision rather than a real gap. +func TestHandleScopeAuditSurfacesAmbiguousHops(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pkA := "1a2b" + strings.Repeat("11", 30) + pkB := "1a2b" + strings.Repeat("22", 30) + hop := pkA[:4] + insertDeclared(t, srv, pkA, time.Now().UTC().Format(time.RFC3339), "be", 0) + insertDeclared(t, srv, pkB, time.Now().UTC().Format(time.RFC3339), "be", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, hop, scopeMatched("#be"), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 2 { + t.Fatalf("repeaters = %+v, want 2", got.Repeaters) + } + for _, row := range got.Repeaters { + if row.AmbiguousHops != 1 { + t.Errorf("%s: ambiguousHops = %d, want 1", row.PublicKey, row.AmbiguousHops) + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "be" { + t.Errorf("%s: notObserved = %v, want [\"be\"] — an ambiguous hop must not silently satisfy the declared region", row.PublicKey, row.NotObserved) + } + } +} + +// TestHandleScopeAuditSortsMissingRegionsFirst: the repeater with a declared +// region it is not forwarding must rank above a repeater in full agreement — +// that's the headline this endpoint exists to surface, not the boring majority. +func TestHandleScopeAuditSortsMissingRegionsFirst(t *testing.T) { + srv, router := setupScopeAuditServer(t) + agreePk := testFullPubkeyA + missingPk := testFullPubkeyB + agreeHop := agreePk[:4] + missingHop := missingPk[:4] + insertDeclared(t, srv, agreePk, time.Now().UTC().Format(time.RFC3339), "be", 0) + insertDeclared(t, srv, missingPk, time.Now().UTC().Format(time.RFC3339), "be,be-vlg", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, agreeHop, scopeMatched("#be"), RouteFlood, recent) + seedTransmissionRouteAt(t, srv.store, missingHop, scopeMatched("#be"), RouteFlood, recent) + // missingPk never forwards be-vlg -> 1 missing declared region; agreePk has 0. + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 2 { + t.Fatalf("repeaters = %+v, want 2", got.Repeaters) + } + if got.Repeaters[0].PublicKey != missingPk { + t.Errorf("repeaters[0] = %s, want %s (the row missing a declared region) ranked first", got.Repeaters[0].PublicKey, missingPk) + } +} + +// TestHandleScopeAuditWindowVocabulary confirms this endpoint matches the +// vocabulary used by the sibling /api/scope-stats and /api/nodes/{pubkey}/scopes +// endpoints (1h, 24h, 7d), not the broader ParseTimeWindow alias set. +func TestHandleScopeAuditWindowVocabulary(t *testing.T) { + _, router := setupScopeAuditServer(t) + req := httptest.NewRequest("GET", "/api/scope-audit?window=30d", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusBadRequest { + t.Errorf("window=30d: status = %d, want 400 (not part of this endpoint's vocabulary)", w.Code) + } +} + +// TestHandleScopeAuditFiltersBlacklistedNode confirms a blacklisted repeater +// is dropped from the audit even though it has declared a region list, +// mirroring the blacklist filtering applied by other multi-node endpoints. +func TestHandleScopeAuditFiltersBlacklistedNode(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "be", 0) + srv.cfg.NodeBlacklist = []string{pk} + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 0 { + t.Errorf("repeaters = %+v, want empty — blacklisted node must be excluded", got.Repeaters) + } +} + +// TestHandleScopeAuditFiltersHiddenNamePrefix is the scope-audit twin of +// TestHandleScopeAuditFiltersBlacklistedNode, covering FIX 5: a repeater +// whose known name matches an operator-configured hidden-name prefix is +// excluded. It also pins the subtler half of the `id.Name != nil` guard in +// handleScopeAudit — a declared target this instance holds NO nodes row for +// has no name to test a hidden-prefix rule against, so it must NOT be +// filtered merely for lacking a name; only a KNOWN, matching name is ever +// grounds for hiding. +func TestHandleScopeAuditFiltersHiddenNamePrefix(t *testing.T) { + srv, router := setupScopeAuditServer(t) + hiddenPk := testFullPubkeyA + unknownPk := testFullPubkeyB + insertDeclared(t, srv, hiddenPk, time.Now().UTC().Format(time.RFC3339), "be", 0) + insertDeclared(t, srv, unknownPk, time.Now().UTC().Format(time.RFC3339), "be", 0) + // Upsert, not insert: upstream the declared list lives ON the nodes row + // (configured_scope), so insertDeclared has already created it. The fork + // kept them in separate tables and a plain INSERT was safe there. + if _, err := srv.db.conn.Exec( + `INSERT INTO nodes (public_key, name, role) VALUES (?, ?, 'repeater') + ON CONFLICT(public_key) DO UPDATE SET name = excluded.name, role = excluded.role`, + hiddenPk, "🚫 ban me"); err != nil { + t.Fatal(err) + } + // unknownPk deliberately gets NO name. On the fork that meant no nodes row + // at all; here the row exists (configured_scope lives on it) but name stays + // NULL. The property under test is unchanged: a node whose name we do not + // know must never be hidden by a name-prefix rule. + srv.cfg.SetHiddenNamePrefixes([]string{"🚫"}) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 || got.Repeaters[0].PublicKey != unknownPk { + t.Fatalf("repeaters = %+v, want exactly the unknown-name target — the hidden-name-prefixed repeater must be excluded, and the nameless one must NOT be excluded merely for having no name", got.Repeaters) + } +} + +// TestHandleScopeAuditNoDeclaredRegionsTable covers the missing-table +// degrade path (mirrors TestHandleNodeScopesNoDeclaredRegionsTable): an +// older database predating the configured_scope column must not fail the +// request: the audit simply has no declared side to report. +func TestHandleScopeAuditWithoutConfiguredScopeColumn(t *testing.T) { + // setupScopeConformanceDB creates no nodes table, so detectSchema leaves + // hasConfiguredScope false: exactly an instance that has not yet ingested a + // /neighbors report or is on a schema predating #1971. + db := setupScopeConformanceDB(t) + cfg := &Config{Port: 3000} + hub := NewHub() + srv := NewServer(db, cfg, hub) + srv.store = newTestStoreWithDB(t, db, cfg) + router := mux.NewRouter() + srv.RegisterRoutes(router) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 0 { + t.Errorf("repeaters = %+v, want empty when configured_scope does not exist", got.Repeaters) + } +} + +// --- Scope audit config-state classification --- + +// TestScopeAuditConfigStateClassification is a table-driven test of the pure +// classification function scopeAuditConfigState — the derivation is a plain +// switch over (namedRegions, wildcard), so each of the four cells is worth +// pinning directly rather than only indirectly via the handler. +func TestScopeAuditConfigStateClassification(t *testing.T) { + tests := []struct { + name string + namedRegions []string + wildcard bool + want string + }{ + {"named and wildcard", []string{"be"}, true, ScopeConfigFull}, + {"wildcard only", nil, true, ScopeConfigNoScopes}, + {"named only", []string{"be"}, false, ScopeConfigNoUnscoped}, + {"neither", nil, false, ScopeConfigNoFlood}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := scopeAuditConfigState(tt.namedRegions, tt.wildcard) + if got != tt.want { + t.Errorf("scopeAuditConfigState(%v, %v) = %q, want %q", tt.namedRegions, tt.wildcard, got, tt.want) + } + }) + } +} + +// TestHandleScopeAuditConfigStateAllFourShapes seeds one repeater per shape +// of the (declaredRegions, declaredWildcard) pair — including the fourth, +// "answered but empty" shape (no named regions AND no '*') that sits outside +// the three named states — and confirms both that each row's ConfigState +// lands correctly AND that tallying ConfigState across the response (the +// same arithmetic the frontend summary line performs over d.repeaters) +// reproduces the exact per-state counts seeded here. That second check is +// what pins "the counts in the summary match the rows": if +// scopeAuditConfigState mis-tags even one row, the tally computed from the +// response no longer matches what was seeded here, and the test fails. +func TestHandleScopeAuditConfigStateAllFourShapes(t *testing.T) { + srv, router := setupScopeAuditServer(t) + now := time.Now().UTC().Format(time.RFC3339) + + fullPk := strings.Repeat("11", 32) + noScopesPk := strings.Repeat("22", 32) + noUnscopedPk := strings.Repeat("33", 32) + noFloodPk := strings.Repeat("44", 32) + + insertDeclared(t, srv, fullPk, now, "*,be", 0) + insertDeclared(t, srv, noScopesPk, now, "*", 0) + insertDeclared(t, srv, noUnscopedPk, now, "be", 0) + insertDeclared(t, srv, noFloodPk, now, "", 0) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 4 { + t.Fatalf("repeaters = %+v, want exactly 4", got.Repeaters) + } + + want := map[string]string{ + fullPk: ScopeConfigFull, + noScopesPk: ScopeConfigNoScopes, + noUnscopedPk: ScopeConfigNoUnscoped, + noFloodPk: ScopeConfigNoFlood, + } + for pk, wantState := range want { + row := findScopeAuditRow(t, got.Repeaters, pk) + if row.ConfigState != wantState { + t.Errorf("pk %s: configState = %q, want %q", pk, row.ConfigState, wantState) + } + } + + counts := map[string]int{} + for _, row := range got.Repeaters { + counts[row.ConfigState]++ + } + for _, state := range []string{ScopeConfigFull, ScopeConfigNoScopes, ScopeConfigNoUnscoped, ScopeConfigNoFlood} { + if counts[state] != 1 { + t.Errorf("count of state %q across repeaters = %d, want 1 (summary tally must match the seeded rows)", state, counts[state]) + } + } +} + +// TestHandleNodeScopesDifferentWindowIsSeparateCacheEntry confirms the cache +// key includes window: a request for a different window must recompute +// rather than reuse another window's cached entry. + +// --- Merging confirmed-scope sources (#1975) --- + +// createDeclaredRegionsTable adds the optional second source and re-probes the +// schema, so hasDeclaredRegionsTable flips. Without the re-probe the merge +// would skip the table and every assertion below would pass vacuously. +func createDeclaredRegionsTable(t *testing.T, srv *Server) { + t.Helper() + if _, err := srv.db.conn.Exec(` + CREATE TABLE node_declared_regions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + UNIQUE(target, rx_pubkey, observed_at) + )`); err != nil { + t.Fatal(err) + } + if err := srv.db.detectSchema(context.Background(), srv.db.conn); err != nil { + t.Fatal(err) + } + if !srv.db.hasDeclaredRegionsTable { + t.Fatal("hasDeclaredRegionsTable is false after creating the table: the fixture would test nothing") + } +} + +func seedSecondSource(t *testing.T, srv *Server, target, observedAt, regionsCSV string, truncated int) { + t.Helper() + if _, err := srv.db.conn.Exec( + `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) + VALUES (?, 'rxpubkeyhex', ?, ?, ?, ?)`, + target, observedAt, observedAt, regionsCSV, truncated); err != nil { + t.Fatal(err) + } +} + +func declaredFor(t *testing.T, srv *Server, target string) (DeclaredRegionsRow, bool) { + t.Helper() + rows, err := srv.db.AllCurrentDeclaredRegions() + if err != nil { + t.Fatal(err) + } + for _, r := range rows { + if r.Target == target { + return r, true + } + } + return DeclaredRegionsRow{}, false +} + +func TestDeclaredRegionsMergeUsesConfiguredScopeAlone(t *testing.T) { + srv, _ := setupScopeAuditServer(t) + insertDeclared(t, srv, testFullPubkeyA, "2026-09-01T00:00:00Z", "#be,#eu", 0) + got, ok := declaredFor(t, srv, testFullPubkeyA) + if !ok || got.RegionsCSV != "#be,#eu" { + t.Fatalf("got %+v (found=%v), want the configured_scope answer", got, ok) + } +} + +func TestDeclaredRegionsMergeUsesSecondSourceAlone(t *testing.T) { + // The case that made this merge necessary: an instance collecting confirmed + // scopes by another route, with no configured_scope written yet. Reading + // only configured_scope would render an empty page on real data. + srv, _ := setupScopeAuditServer(t) + createDeclaredRegionsTable(t, srv) + seedSecondSource(t, srv, testFullPubkeyA, "2026-09-01T00:00:00Z", "be,eu", 0) + got, ok := declaredFor(t, srv, testFullPubkeyA) + if !ok || got.RegionsCSV != "be,eu" { + t.Fatalf("got %+v (found=%v), want the second source's answer", got, ok) + } +} + +func TestDeclaredRegionsMergeNewestAnswerWins(t *testing.T) { + srv, _ := setupScopeAuditServer(t) + createDeclaredRegionsTable(t, srv) + insertDeclared(t, srv, testFullPubkeyA, "2026-09-01T00:00:00Z", "#old", 0) + seedSecondSource(t, srv, testFullPubkeyA, "2026-09-02T00:00:00Z", "newer", 0) + got, _ := declaredFor(t, srv, testFullPubkeyA) + if got.RegionsCSV != "newer" { + t.Errorf("regions = %q, want the later answer regardless of which source it came from", got.RegionsCSV) + } + + // And the other way round, so the rule is "newest wins" and not "one source + // always beats the other". + insertDeclared(t, srv, testFullPubkeyB, "2026-09-03T00:00:00Z", "#newer", 0) + seedSecondSource(t, srv, testFullPubkeyB, "2026-09-02T00:00:00Z", "old", 0) + gotB, _ := declaredFor(t, srv, testFullPubkeyB) + if gotB.RegionsCSV != "#newer" { + t.Errorf("regions = %q, want the later answer from the other source", gotB.RegionsCSV) + } +} + +func TestDeclaredRegionsMergeAnswerWithAnInstantBeatsOneWithout(t *testing.T) { + // An empty instant means "we do not know when this was answered". It must + // not outrank a dated answer, but it must still be used when it is all we + // have. + srv, _ := setupScopeAuditServer(t) + createDeclaredRegionsTable(t, srv) + insertDeclared(t, srv, testFullPubkeyA, "", "#undated", 0) + seedSecondSource(t, srv, testFullPubkeyA, "2026-09-01T00:00:00Z", "dated", 0) + got, _ := declaredFor(t, srv, testFullPubkeyA) + if got.RegionsCSV != "dated" { + t.Errorf("regions = %q, want the dated answer to win", got.RegionsCSV) + } + + insertDeclared(t, srv, testFullPubkeyB, "", "#undated", 0) + gotB, ok := declaredFor(t, srv, testFullPubkeyB) + if !ok || gotB.RegionsCSV != "#undated" { + t.Errorf("got %+v (found=%v), want an undated answer to be used when it is the only one", gotB, ok) + } +} + +func TestDeclaredRegionsMergeCarriesTruncatedFromTheSourceThatHasIt(t *testing.T) { + srv, _ := setupScopeAuditServer(t) + createDeclaredRegionsTable(t, srv) + seedSecondSource(t, srv, testFullPubkeyA, "2026-09-01T00:00:00Z", "be", 1) + got, _ := declaredFor(t, srv, testFullPubkeyA) + if !got.Truncated { + t.Error("truncated must survive the merge from a source that records it") + } +} + +func TestDeclaredRegionsMergeWithNoSourcesIsEmptyNotAnError(t *testing.T) { + db := setupScopeConformanceDB(t) + rows, err := db.AllCurrentDeclaredRegions() + if err != nil { + t.Fatalf("no sources must not be an error: %v", err) + } + if len(rows) != 0 { + t.Errorf("rows = %+v, want empty", rows) + } +} diff --git a/public/bottom-nav.js b/public/bottom-nav.js index 20f055146..ac1d4b0b9 100644 --- a/public/bottom-nav.js +++ b/public/bottom-nav.js @@ -77,6 +77,7 @@ { route: 'tools', hash: '#/tools', label: 'Tools', ph: 'wrench' }, { route: 'observers', hash: '#/observers', label: 'Observers', ph: 'eye' }, { route: 'analytics', hash: '#/analytics', label: 'Analytics', ph: 'chart-bar' }, + { route: 'scope-audit', hash: '#/scope-audit', label: 'Scope Audit', ph: 'clipboard-text' }, { route: 'perf', hash: '#/perf', label: 'Perf', ph: 'lightning' }, { route: 'audio-lab', hash: '#/audio-lab', label: 'Audio Lab', ph: 'music-note' }, ]; diff --git a/public/index.html b/public/index.html index 1806051cc..aeeaa5f64 100644 --- a/public/index.html +++ b/public/index.html @@ -43,6 +43,7 @@ + @@ -130,6 +131,7 @@ Tools Observers Analytics + Scope Audit Perf Lab @@ -226,6 +228,7 @@ + diff --git a/public/nav-drawer.js b/public/nav-drawer.js index cf8a4bc43..1098c63c9 100644 --- a/public/nav-drawer.js +++ b/public/nav-drawer.js @@ -61,6 +61,7 @@ { route: 'tools', hash: '#/tools', label: 'Tools', ph: 'wrench' }, { route: 'observers', hash: '#/observers', label: 'Observers', ph: 'eye' }, { route: 'analytics', hash: '#/analytics', label: 'Analytics', ph: 'chart-bar' }, + { route: 'scope-audit', hash: '#/scope-audit', label: 'Scope Audit', ph: 'clipboard-text' }, { route: 'perf', hash: '#/perf', label: 'Perf', ph: 'lightning' }, { route: 'audio-lab', hash: '#/audio-lab', label: 'Audio Lab', ph: 'music-note' }, ]; diff --git a/public/scope-audit.css b/public/scope-audit.css new file mode 100644 index 000000000..f5074e8e2 --- /dev/null +++ b/public/scope-audit.css @@ -0,0 +1,81 @@ +/* Network-wide scope audit page (#/scope-audit). Reuses .ns-decl / .ns-table / + .ns-empty / .ns-truncated from node-scopes.css (same badge vocabulary, so + the per-node and network-wide views agree visually) plus .analytics-time-range + for the window buttons. No hex/rgb literals — --text-muted / --border / + --status-* / --link-color are defined globally. */ + +.sa-page { max-width: 1200px; margin: 0 auto; padding: 12px 16px; } + +.sa-head { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 8px; margin-bottom: 4px; } +.sa-head h2 { margin: 0; font-size: 18px; } + +.sa-intro { color: var(--text-muted); font-size: 12px; margin: 4px 0 10px; } +.sa-intro a { color: var(--link-color); } + +/* .sa-search reuses .nodes-search (style.css) for the input's colours/border + so this page doesn't invent a second search-box style — only the sizing + below is page-specific. */ +.sa-search-bar { margin: 0 0 10px; } +.sa-search { width: 100%; max-width: 360px; } + +.sa-window-note { + font-size: 12px; color: var(--text-muted); margin: 4px 0 10px; + border-left: 3px solid var(--border); padding: 4px 10px; +} + +.sa-table-wrap { overflow-x: auto; } +.sa-table { min-width: 720px; } +.sa-name a { color: var(--link-color); text-decoration: none; } +.sa-name a:hover { text-decoration: underline; } +.sa-name-unknown { color: var(--text-muted); font-style: italic; } +.sa-role { font-size: 10px; text-transform: uppercase; letter-spacing: .3px; } + +.sa-chip { + display: inline-block; padding: 1px 6px; margin: 1px 2px 1px 0; border-radius: 4px; + font-size: 11px; font-family: var(--mono); white-space: nowrap; +} +.sa-chip-declared { background: var(--section-bg, var(--card-bg)); color: var(--text); border: 1px solid var(--border); } +.sa-chip-undeclared { background: color-mix(in srgb, var(--status-yellow) 18%, transparent); color: var(--status-amber-text); } +.sa-chip-wildcard { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); font-weight: 700; } +.sa-chip-ambiguous { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px dashed var(--border); font-family: inherit; font-style: italic; } + +.sa-count { font-size: 11px; margin-top: 6px; } + +/* sa-summary is the per-configState count line at the top of the page — + "13 no scopes configured" etc — reusing .ns-decl (node-scopes.css) for the + count pill so it shares the exact colour vocabulary the row-level Config + badge (also .ns-decl) uses, rather than a separate summary palette. */ +.sa-summary { display: flex; flex-wrap: wrap; gap: 10px 16px; font-size: 12px; color: var(--text-muted); margin: 2px 0 10px; } +.sa-summary-item { display: inline-flex; align-items: center; gap: 5px; } + +@media (max-width: 640px) { + .sa-table th:nth-child(6), .sa-table td:nth-child(6) { display: none; } +} + +/* Observed forwarding, the green half of the merged Scopes column. Pairs with + .sa-chip-unobserved below, which stays neutral. Only the observed side is + coloured: red on every unobserved region would read as an alarm on rows that + are often just a quiet region over a short window. */ +.sa-chip-observed { background: color-mix(in srgb, var(--status-green) 16%, transparent); color: var(--status-green-text); } + +/* Declared but not observed in this window. Deliberately NOT red: absence over + a short window is weak evidence, which the page header says in words, so it + should not shout in colour. */ +.sa-chip-unobserved { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px solid var(--border); } + +/* Provenance note under the intro. Same muted treatment as .sa-intro; it is + context, not a finding, and must not compete with the table. */ +.sa-sources { color: var(--text-muted); font-size: 12px; margin: 0 0 10px; line-height: 1.5; } +.sa-sources a { color: var(--link-color); } +.sa-sources code { font-family: var(--mono); font-size: 11px; } + +/* Provenance note under the intro, and the empty state that has to carry the + same explanation on its own: on a stock install the table IS empty, so that + is where an operator actually reads it. Muted like .sa-intro; this is + context, not a finding, and must not compete with the table. */ +.sa-sources { color: var(--text-muted); font-size: 12px; margin: 0 0 10px; line-height: 1.5; } +.sa-sources a, .sa-empty a { color: var(--link-color); } +.sa-sources code, .sa-empty code { font-family: var(--mono); font-size: 11px; } +.sa-empty { line-height: 1.6; max-width: 70ch; } +.sa-empty ul { margin: 8px 0; padding-left: 20px; } +.sa-empty li { margin: 4px 0; } diff --git a/public/scope-audit.js b/public/scope-audit.js new file mode 100644 index 000000000..1ae419501 --- /dev/null +++ b/public/scope-audit.js @@ -0,0 +1,367 @@ +/* === CoreScope — scope-audit.js === + Network-wide scope audit (route #/scope-audit): the whole-network answer + to "which repeaters declare a region they are not actually forwarding". + Fetches GET /api/scope-audit once per window and renders one row per + declared repeater, sorted server-side so the interesting rows (missing a + declared region, or contradicting its own wildcard) sit at the top. + Reuses node-scopes.js's vocabulary: scope names are shown without the + leading '#' (the two backends spell them differently — see normScope + there), and '*' is never listed as a scope. */ +'use strict'; +(function () { + var loadGen = 0; // bumped per load; guards against in-flight races (mirrors node-reach.js) + var DEFAULT_WINDOW = '24h'; + var WINDOWS = [ + { key: '1h', label: '1h' }, + { key: '24h', label: '24h' }, + { key: '7d', label: '7d' } + ]; + var win = DEFAULT_WINDOW; + var searchQuery = ''; // free-text filter over name/pubkey/region, applied client-side + var searchIndex = {}; // publicKey -> lowercased searchable haystack, rebuilt every renderBody + var sortCtl = null; // tracked TableSort controller for destroy-before-reinit (mirrors observers.js) + + function windowBtn(key, cur, label) { + var on = key === cur; + return ''; + } + + // windowHonestyNote spells out, in the window's own words, why "declared + // but not observed" is weak evidence over a short window (a quiet region + // simply has no traffic) and strong evidence over a long one — the reader + // must never mistake a 1h result for a 7d one. + function windowHonestyNote(w) { + var strength = w === '7d' ? 'a strong signal — a full week with zero matching traffic is hard to explain as bad luck' + : (w === '24h' ? 'a moderate signal — worth checking, but a quiet region can still look this way for a day' + : 'weak evidence — an hour with no traffic proves very little; treat this as a hint to re-check on 24h or 7d'); + return '
Rows below are ranked by declared regions with zero observed forwarding in the last ' + + escapeHtml(w) + '. At this window, that absence is ' + strength + '.
'; + } + + function pageHtml() { + return '
' + + '
' + + '

Scope audit

' + + '
' + + WINDOWS.map(function (w) { return windowBtn(w.key, win, w.label); }).join('') + + '
' + + '
Network-wide comparison of declared vs. observed region-scope forwarding, across every repeater that has declared a region list over RF. Per-node detail lives on each node\'s page.
' + + sourcesLineHtml() + + '' + + '
Loading scope audit…
' + + '
'; + } + + function ageHtml(row) { + var age = timeAgo(row.declaredAt); + return '' + escapeHtml(age) + ''; + } + + // sourcesLineHtml is the always-visible half of the provenance note. It has + // to carry the links itself: the fuller explanation lives in emptyStateHtml, + // which by definition never renders on an instance that HAS data, so an + // operator with a full table would otherwise never see where it came from. + function sourcesLineHtml() { + return '
The declared side is the repeater’s own answer, read back off the node by an ' + + 'ESP32 observer on the neighbour-report firmware or by ' + + 'CoreDrive RX. The newest answer per repeater wins, whichever collected it. ' + + 'The observed side is forwarding CoreScope already sees in its own traffic.
'; + } + + // mergedScopeChips renders ONE chip per declared region, coloured by whether + // that region was actually observed forwarding in the window. + // + // This replaces the old DECLARED and NOT OBSERVED pair. They were never + // independent: notObserved is a strict subset of declaredRegions, checked + // against a live 197-row response where it held on 197 of 197 rows. The two + // columns printed the same set twice, once whole and once filtered, and left + // the reader to diff them. On a typical mixed row that meant comparing two + // lists of eight to find the one or two entries that differ. Here the + // observed ones are simply the green ones. + // + // No new claim is made about the data: a region present in declaredRegions + // and absent from notObserved is exactly what the server already means by + // "observed forwarding in this window". + // emptyStateHtml is what a stock install sees, so it carries the whole + // explanation rather than deferring to the intro: on a fresh deployment this + // IS the page. Named and returned rather than inlined so it can be asserted. + function emptyStateHtml() { + return '
No repeater has answered with its configured region list yet, so there is nothing to audit here.

That answer has to be collected from the repeater itself; nothing else in CoreScope knows which regions a node is configured for, only which ones its traffic was seen under. Two things can collect it, and neither ships with CoreScope, so an empty table is the normal state until you run one:The newest answer per repeater wins, whichever collected it.
'; + } + + function mergedScopeChips(row) { + var missing = Object.create(null); + row.notObserved.forEach(function (n) { missing[n] = true; }); + var chips = row.declaredRegions.map(function (n) { + var observed = !missing[n]; + return '' + escapeHtml(n) + ''; + }); + if (!chips.length) return '—'; + return chips.join(' '); + } + + // nameHtml renders the repeater identity cell. row.name == null means this + // instance holds NO nodes row for the target at all (a declared-regions + // answer can name a repeater the network has never recorded) — distinct + // from row.name === "" (a known node that simply has no name). A + // truthiness check collapses those two into the same pubkey-stub + // rendering, which defeats the point of the API making the field + // nullable. The unknown case also gets no link (FIX 4): #/nodes/ + // cannot resolve a target we hold no row for. + function nameHtml(row) { + if (row.name == null) { + return '' + + escapeHtml(row.publicKey.slice(0, 10)) + '… (unknown node)'; + } + var name = row.name ? escapeHtml(row.name) : escapeHtml(row.publicKey.slice(0, 10)) + '…'; + return '' + name + ''; + } + + // CONFIG_STATES maps ScopeAuditRow.configState (server-computed from + // declaredRegions + declaredWildcard — see docs/api-spec.md) to a compact + // badge, reusing the same .ns-decl colour vocabulary the Status column + // already uses on this page (node-scopes.css) rather than inventing a new + // one. "no-scopes" gets the warn (red) treatment because it is the + // headline finding this classification exists to surface — roughly a + // third of repeaters answering at all fall in it — but its title text + // carries the caveat that "*" alone does not strictly prove no region is + // configured, only that no region is flood-allowed (see scopeAuditConfigState + // in cmd/server/scopes.go): a repeater with regions defined but every one + // marked deny-flood looks identical from this data. + var CONFIG_STATES = { + 'full': { + label: 'Full', cls: 'ns-decl-yes', summary: 'fully configured', + title: 'Declares named regions and \'*\' — forwards both its declared regions and plain unscoped floods.' + }, + 'no-scopes': { + label: 'No scopes', cls: 'ns-decl-warn', summary: 'no scopes configured', + title: 'Declares only \'*\', no named regions — no region is flood-allowed. Almost always means no scopes are configured at all, but a repeater with regions defined that are ALL set to deny flooding would look identical from this data alone.' + }, + 'no-unscoped': { + label: 'No unscoped', cls: 'ns-decl-quiet', summary: 'not forwarding unscoped floods', + title: 'Declares named regions but not \'*\' — does not forward plain unscoped floods. Exact, not an inference.' + }, + 'no-flood': { + label: 'No flood', cls: 'ns-decl-unknown', summary: 'nothing flood-allowed', + title: 'Declares neither named regions nor \'*\' — an answered-but-empty list. Nothing is flood-allowed by this repeater, not even plain unscoped traffic.' + } + }; + var CONFIG_STATE_ORDER = ['no-scopes', 'no-unscoped', 'full', 'no-flood']; + + function configStateHtml(row) { + var meta = CONFIG_STATES[row.configState]; + return '' + meta.label + ''; + } + + // configStateSummaryHtml gives the reader "13 repeaters have no scopes + // configured" at a glance without reading every row — the counts are + // tallied straight off d.repeaters[].configState, the same array the table + // below renders, so they can never drift out of sync with the rows. + function configStateSummaryHtml(repeaters) { + var counts = { full: 0, 'no-scopes': 0, 'no-unscoped': 0, 'no-flood': 0 }; + repeaters.forEach(function (r) { counts[r.configState]++; }); + return '
' + + CONFIG_STATE_ORDER.map(function (state) { + var meta = CONFIG_STATES[state]; + return '' + + '' + counts[state] + ' ' + escapeHtml(meta.summary) + ''; + }).join('') + + '
'; + } + + // ambiguousCaveat flags rows with a non-zero FIX 1 ambiguity count: hops + // whose truncated hash prefix matched more than one declared target were + // credited to none of them, so a notObserved finding here could be a + // prefix collision rather than a confirmed gap. + function ambiguousCaveat(row) { + if (!row.ambiguousHops) return ''; + return ' possibly ambiguous'; + } + + // statusScore ranks a row's Status column numerically for sorting — a + // simple weighted count (notObserved dominates, matching the server's own + // findings-first ranking) rather than the badge text, which the Status + // column shows as chips, not a single sortable label. + function statusScore(row) { + return row.notObserved.length * 100 + (row.wildcardContradiction ? 10 : 0) + row.undeclaredObserved.length; + } + + function rowHtml(row) { + var issues = []; + if (row.notObserved.length) issues.push('' + row.notObserved.length + ' not observed'); + if (row.wildcardContradiction) issues.push('wildcard contradiction'); + if (row.undeclaredObserved.length) issues.push('' + row.undeclaredObserved.length + ' undeclared'); + var issuesHtml = issues.length ? issues.join(' ') : 'agrees'; + + // declaredAtMs is the underlying declaredAt timestamp in epoch ms, fed to + // the Declared age
' + + '' + + '' + + '' + + '' + + '' + + ''; + } + + // buildSearchIndex maps publicKey -> lowercased haystack (name, pubkey, + // and every region name this row mentions — declared, not-observed, and + // undeclared-observed) so applyFilter can match a row without re-deriving + // it from rendered chip text. + function buildSearchIndex(repeaters) { + var idx = {}; + repeaters.forEach(function (row) { + var parts = [row.publicKey]; + if (row.name) parts.push(row.name); + row.declaredRegions.forEach(function (r) { parts.push(r); }); + row.notObserved.forEach(function (r) { parts.push(r); }); + row.undeclaredObserved.forEach(function (o) { parts.push(o.scope); }); + idx[row.publicKey] = parts.join(' ').toLowerCase(); + }); + return idx; + } + + // applyFilter toggles row visibility against searchQuery and refreshes the + // shown-count line — independent of sort order, since it only sets + // style.display on whatever elements are currently in the tbody, so + // filtering a sorted table keeps the sort and sorting a filtered table + // keeps the filter. + function applyFilter() { + var tbody = document.querySelector('#saTable tbody'); + var countEl = document.getElementById('saCount'); + if (!tbody) return; + var q = searchQuery.trim().toLowerCase(); + var rows = tbody.querySelectorAll('tr'); + var shown = 0; + for (var i = 0; i < rows.length; i++) { + var pk = rows[i].getAttribute('data-pubkey'); + var hay = searchIndex[pk] || ''; + var match = !q || hay.indexOf(q) !== -1; + rows[i].style.display = match ? '' : 'none'; + if (match) shown++; + } + if (!countEl) return; + if (q) { + countEl.textContent = 'Showing ' + shown + ' of ' + rows.length + ' repeater' + (rows.length === 1 ? '' : 's') + ' matching “' + searchQuery.trim() + '”.'; + } else { + countEl.textContent = rows.length + ' repeater' + (rows.length === 1 ? '' : 's') + ' with a declared region list.'; + } + } + + function renderBody(d) { + var el = document.getElementById('saBody'); + if (!el) return; + if (sortCtl && typeof sortCtl.destroy === 'function') { + try { sortCtl.destroy(); } catch (e) { /* ignore */ } + } + sortCtl = null; + if (!d.repeaters.length) { + searchIndex = {}; + el.innerHTML = emptyStateHtml(); + return; + } + searchIndex = buildSearchIndex(d.repeaters); + el.innerHTML = configStateSummaryHtml(d.repeaters) + + windowHonestyNote(d.window) + + '
as data-value (mirroring observers.js's + // _lastSeenMs) so TableSort's numeric comparator sorts on the real + // timestamp instead of the rendered "4h ago" text, which cannot be + // parsed back into a date. + var declaredAtMs = row.declaredAt ? new Date(row.declaredAt).getTime() : NaN; + var nameSortValue = row.name != null ? row.name : row.publicKey; + + return '
' + nameHtml(row) + (row.role != null && row.role !== '' ? ' ' + escapeHtml(row.role) + '' : '') + '' + issuesHtml + '' + configStateHtml(row) + '' + mergedScopeChips(row) + (row.declaredWildcard ? ' *' : '') + ambiguousCaveat(row) + '' + ageHtml(row) + (row.truncated ? ' truncated' : '') + '
' + + '' + + '' + + '' + + '' + + '' + + '' + + d.repeaters.map(rowHtml).join('') + + '
RepeaterStatusConfigScopesDeclared age
' + + '
'; + + var saTbl = document.getElementById('saTable'); + if (saTbl && window.TableSort) { + // No defaultColumn: the server's own findings-first order (most + // notObserved at the top) stays the default until the reader picks a + // column — that ordering is the reason this page exists. + sortCtl = TableSort.init(saTbl, { storageKey: 'meshcore-scope-audit-sort' }); + } else if (saTbl && !window.TableSort) { + console.warn('[scope-audit] window.TableSort missing — table will not be sortable'); + } + applyFilter(); + } + + async function load(w) { + win = w; + var myGen = ++loadGen; + var head = document.getElementById('saWindow'); + if (head) { + WINDOWS.forEach(function (wd) { + var b = head.querySelector('button[data-window="' + wd.key + '"]'); + if (b) { b.classList.toggle('active', wd.key === w); b.setAttribute('aria-pressed', wd.key === w ? 'true' : 'false'); } + }); + } + var body = document.getElementById('saBody'); + if (body) body.innerHTML = '
Loading scope audit…
'; + var d; + try { + d = await api('/scope-audit?window=' + encodeURIComponent(w), { ttl: 30000 }); + } catch (e) { + if (myGen !== loadGen) return; + if (body) body.innerHTML = '
Failed to load scope audit: ' + escapeHtml(e.message) + '
'; + return; + } + if (myGen !== loadGen) return; + renderBody(d); + syncHash(); + } + + function syncHash() { + try { history.replaceState(null, '', '#/scope-audit?window=' + win); } catch (e) {} + } + + function init(container) { + win = DEFAULT_WINDOW; + searchQuery = ''; + try { + var p = (typeof getHashParams === 'function') ? getHashParams() : null; + var qw = p ? p.get('window') : null; + if (qw && WINDOWS.some(function (w) { return w.key === qw; })) win = qw; + } catch (e) {} + container.innerHTML = pageHtml(); + var bar = document.getElementById('saWindow'); + if (bar) bar.addEventListener('click', function (e) { + var b = e.target.closest('button[data-window]'); + if (b) load(b.getAttribute('data-window')); + }); + var search = document.getElementById('saSearch'); + if (search) search.addEventListener('input', debounce(function (e) { + searchQuery = e.target.value; + applyFilter(); + }, 250)); + load(win); + } + + function destroy() { + loadGen++; + if (sortCtl && typeof sortCtl.destroy === 'function') { + try { sortCtl.destroy(); } catch (e) { /* ignore */ } + } + sortCtl = null; + } + + if (typeof window !== 'undefined') { + // Exposed so the helper tests can assert what the Scopes column RENDERS + // rather than grepping this file, the same reason map.js exposes its label + // builder (#1356/#1933). + window.__meshcoreScopeAuditInternals = { mergedScopeChips: mergedScopeChips, emptyStateHtml: emptyStateHtml, sourcesLineHtml: sourcesLineHtml }; + } + + registerPage('scope-audit', { init: init, destroy: destroy }); +})(); diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index 106200e88..65884c7b9 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -7157,3 +7157,150 @@ console.log('\n=== roles.js: getEffectiveHeardMs (#1845) ==='); assert.strictEqual(status({ role: 'repeater' }), 'stale'); }); } + +// ===== scope-audit.js: mergedScopeChips ===== +// DECLARED and NOT OBSERVED were merged into one colour-coded Scopes column. +// They were never independent: notObserved is a strict subset of +// declaredRegions, so the page printed the same set twice and made the reader +// diff it. These assert the rendered markup, not the source. +console.log('\n=== scope-audit.js: mergedScopeChips ==='); +{ + const ctx = makeSandbox(); + ctx.registerPage = () => {}; + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/scope-audit.js'); + const chips = ctx.__meshcoreScopeAuditInternals.mergedScopeChips; + const row = (declared, notObserved) => ({ declaredRegions: declared, notObserved: notObserved }); + + test('a declared region absent from notObserved renders as observed', () => { + const h = chips(row(['be'], [])); + assert.ok(h.includes('sa-chip-observed'), 'should carry the observed class'); + assert.ok(!h.includes('sa-chip-unobserved'), 'and not the unobserved one'); + }); + + test('a declared region present in notObserved renders as unobserved, not as an alarm', () => { + const h = chips(row(['be'], ['be'])); + assert.ok(h.includes('sa-chip-unobserved')); + assert.ok(!h.includes('sa-chip-observed')); + }); + + test('a mixed row renders both colours, one chip per declared region', () => { + // The case the merge exists for: on live data a typical mixed row declares + // 8 regions of which 6 are unobserved, so the observed ones are the needle. + const h = chips(row(['be', 'eu', 'nl'], ['eu', 'nl'])); + assert.strictEqual((h.match(/sa-chip-observed/g) || []).length, 1); + assert.strictEqual((h.match(/sa-chip-unobserved/g) || []).length, 2); + assert.strictEqual((h.match(/ { + // Regrouping would break the operator habit of scanning for a known + // region in the position it always sits. + const h = chips(row(['be', 'eu', 'nl'], ['eu'])); + assert.ok(h.indexOf('>be<') < h.indexOf('>eu<'), 'be before eu'); + assert.ok(h.indexOf('>eu<') < h.indexOf('>nl<'), 'eu before nl'); + }); + + test('no declared regions renders an em dash, not an empty cell', () => { + // 68 of 197 rows on live data declare nothing at all; an empty cell reads + // as a rendering fault rather than as an answer. + assert.ok(chips(row([], [])).includes('—')); + }); + + test('every chip explains its own colour in a title', () => { + const h = chips(row(['be', 'eu'], ['eu'])); + assert.ok(h.includes('observed forwarding in this window')); + assert.ok(h.includes('declared, but no forwarding observed in this window')); + }); + + test('region names are HTML-escaped', () => { + const h = chips(row([''], [])); + assert.ok(!h.includes(' { + // Defensive: the server guarantees notObserved is a subset (197 of 197 + // rows checked), but the column must not grow a phantom chip if that ever + // stops holding. + const h = chips(row(['be'], ['be', 'ghost'])); + assert.strictEqual((h.match(/ {}; + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/scope-audit.js'); + const empty = ctx.__meshcoreScopeAuditInternals.emptyStateHtml(); + + test('names both collectors', () => { + assert.ok(/neighbour-report firmware/i.test(empty), 'should name the observer firmware'); + assert.ok(/CoreDrive RX/i.test(empty), 'should name the companion app'); + }); + + test('links to both so the reader can act on it', () => { + assert.ok(empty.includes('observer.gessaman.com'), 'observer firmware link'); + // Upstream links the repo; a fork may link its own hosted instance. + assert.ok(/rx\.on8ar\.eu|coredrive-rx/.test(empty), 'companion app link'); + }); + + test('says an empty table is normal, not a fault', () => { + assert.ok(/normal state/i.test(empty)); + }); + + test('states the precedence rule, so two collectors are not confusing', () => { + assert.ok(/newest answer per repeater wins/i.test(empty)); + }); + + test('says what nothing else in CoreScope can tell you', () => { + // The reason the page exists at all: observed traffic shows which scopes a + // node carried, never which it is configured for. + assert.ok(/configured/i.test(empty) && /traffic was seen under/i.test(empty)); + }); + + test('does not use the old drive-around wording', () => { + // "fills in as devices drive" was CoreDrive-specific jargon that meant + // nothing to an operator running the observer firmware instead. + assert.ok(!/as devices drive/i.test(empty)); + }); +} + +// The always-visible provenance line. It must carry the links itself: the +// fuller explanation lives in the empty state, which never renders on an +// instance that HAS data, so an operator with a full table would otherwise +// never learn where the declared column came from. That was the actual bug. +console.log('\n=== scope-audit.js: sourcesLineHtml ==='); +{ + const ctx = makeSandbox(); + ctx.registerPage = () => {}; + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/scope-audit.js'); + const line = ctx.__meshcoreScopeAuditInternals.sourcesLineHtml(); + + test('carries a link for each collector, not just their names', () => { + assert.strictEqual((line.match(/ { + assert.strictEqual((line.match(/rel="noopener"/g) || []).length, 2); + }); + + test('separates the two claims the page rests on', () => { + assert.ok(/declared/i.test(line) && /observed/i.test(line)); + assert.ok(/read back off the node/i.test(line), 'declared is the node answering'); + assert.ok(/already sees in its own traffic/i.test(line), 'observed is our own data'); + }); + + test('states the precedence rule where a reader with data will see it', () => { + assert.ok(/newest answer per repeater wins/i.test(line)); + }); +} diff --git a/test-nav-priority-1391-e2e.js b/test-nav-priority-1391-e2e.js index 546ec5880..d266775f5 100644 --- a/test-nav-priority-1391-e2e.js +++ b/test-nav-priority-1391-e2e.js @@ -197,7 +197,12 @@ async function main() { // EXACTLY the non-active non-high routes so the channels link (when // active) stays inline and is not orphaned in the dropdown. if (w <= 1100) { - const ALL_NON_HIGH = ['#/channels', '#/tools', '#/observers', '#/analytics', '#/perf', '#/audio-lab']; + // Every nav-link in index.html WITHOUT data-priority="high". Adding a + // nav entry means updating this list too, or the exactly-equals + // assertion below fails on the new route. That is the fourth place a + // nav link has to be registered, after index.html, bottom-nav.js + // MORE_ROUTES and nav-drawer.js ROUTES. + const ALL_NON_HIGH = ['#/channels', '#/tools', '#/observers', '#/analytics', '#/perf', '#/audio-lab', '#/scope-audit']; const expectedMore = ALL_NON_HIGH.filter(h => h !== expectedActive).sort(); assert.deepStrictEqual( [...data.moreItems].sort(), From 3fbff01f647c192f2fd39b7db28cb83419729d4b Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Wed, 9 Sep 2026 11:49:33 +0200 Subject: [PATCH 011/151] build: upgrade Go toolchain to 1.27 (#1946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Bumps the Go toolchain used to build/test to 1.27: `golang:1.27-alpine` in `Dockerfile` and `Dockerfile.go`, and `go-version: '1.27'` in the three `actions/setup-go` steps in `.github/workflows/deploy.yml`. - Each module's `go.mod` `go` directive is intentionally left at `1.22` — no 1.27-only language features are being adopted, and a 1.27 toolchain builds a `go 1.22`-declared module without issue. ## Test plan - [x] `go build ./...` + `go vet ./...` for all 13 modules (`cmd/server`, `cmd/ingestor`, `cmd/migrate`, `cmd/decrypt`, 10 `internal/*` packages) under Go 1.27.0 - [x] `go test ./...` passes for `cmd/server`, `cmd/ingestor`, `cmd/migrate`, `cmd/decrypt` - [ ] `docker build` against the new `golang:1.27-alpine` base (Docker wasn't available in the sandbox this change was prepared in — needs a check in CI or locally) --- .github/workflows/deploy.yml | 12 ++++++------ Dockerfile | 2 +- Dockerfile.go | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index feeb48b40..49ea5f762 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -103,10 +103,10 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.22 + - name: Set up Go 1.27 uses: actions/setup-go@v6 with: - go-version: '1.22' + go-version: '1.27' cache-dependency-path: | cmd/server/go.sum cmd/ingestor/go.sum @@ -343,10 +343,10 @@ jobs: - name: Clean Go module cache run: rm -rf ~/go/pkg/mod 2>/dev/null || true - - name: Set up Go 1.22 + - name: Set up Go 1.27 uses: actions/setup-go@v6 with: - go-version: '1.22' + go-version: '1.27' cache-dependency-path: cmd/server/go.sum - name: Build Go server @@ -758,10 +758,10 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - - name: Set up Go 1.22 + - name: Set up Go 1.27 uses: actions/setup-go@v6 with: - go-version: '1.22' + go-version: '1.27' - name: Build corescope-decrypt (static, linux/amd64) run: | diff --git a/Dockerfile b/Dockerfile index bfb8fd009..4e70794cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ # BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain # `docker build` (without buildx) doesn't fail on an empty platform string. ARG BUILDPLATFORM=linux/amd64 -FROM --platform=$BUILDPLATFORM golang:1.22-alpine AS builder +FROM --platform=$BUILDPLATFORM golang:1.27-alpine AS builder ARG APP_VERSION=unknown ARG GIT_COMMIT=unknown diff --git a/Dockerfile.go b/Dockerfile.go index 77e0329a8..ec2725f52 100644 --- a/Dockerfile.go +++ b/Dockerfile.go @@ -1,4 +1,4 @@ -FROM golang:1.22-alpine AS builder +FROM golang:1.27-alpine AS builder RUN apk add --no-cache build-base From de237fc29cb1d1b682d8b91519c05c6506148232 Mon Sep 17 00:00:00 2001 From: Anupam Mediratta Date: Wed, 9 Sep 2026 19:07:40 +0530 Subject: [PATCH 012/151] fix(qa): bind TEST_PUBKEY as a SQLite parameter instead of interpolating it (#1982) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #1977. Supersedes #1952. Follow-up filed as #1983. ## What §10.2 did ```bash q="SELECT COUNT(*) FROM transmissions WHERE from_node = '$TEST_PUBKEY';" qq=$(printf %q "$q") if ! count=$(ssh_t "docker exec … sqlite3 … $qq" 2>/dev/null); then count=$(ssh_t "sqlite3 … $qq" 2>/dev/null || echo "") fi ``` The injection is not reachable today — `TEST_PUBKEY` is hex-gated and the script `exit 2`s before the SQL is built. The problem is that the SQL layer's safety rests entirely on that outer gate rather than on the SQL layer itself. #1952 proposed doubling embedded quotes; that is string escaping, not parameterisation, which is why it was withdrawn in favour of this. ## What this does Per the four points in the sign-off on #1977: **1. Bind the value.** A constant `SELECT` and a bound `:pubkey`, fed to sqlite3 on stdin. The SQL no longer crosses the remote shell as a command word, so there is no `printf %q` on the query at all any more. **Why hex rather than `.parameter set :pk ''`.** Dot-command arguments are split on whitespace, so a payload containing a space produces too many arguments — and sqlite3 responds by printing the `.parameter` help to **stdout**, exiting **0**, and leaving `:pk` **unbound**. `COUNT(*)` then returns 0, which reads exactly like a passing security fix. `-bail` does not catch it. Verified on 3.51.0: ``` $ printf ".parameter set :pk '' OR 1=1 --'\nSELECT COUNT(*) FROM transmissions WHERE from_node = :pk;\n" \ | sqlite3 -bail ptest.db .parameter CMD ... Manage SQL parameter bindings # <- help, on stdout … 0 # <- :pk never bound $ echo $? 0 ``` `.parameter set :pk 1+1` also binds the integer `2` — the value is evaluated as an SQL expression and only falls back to a text literal when evaluation fails. So interpolating into the `.parameter set` line trades one hazard for another. Hex-encoding removes the quoting layer instead of adding one: the value is bound as `cast(x'' as text)`, so its contribution to the SQL text is drawn from the alphabet `[0-9a-f]` only. Nothing to quote, no tokenizer arity hazard, and it holds for **arbitrary** input rather than only for hex-gated input — which is the point. Verified against a fixture table holding two rows, one of them `deadbeef`: | value | result | exit | |---|---|---| | `deadbeef`, bound as `cast(x'6465616462656566' as text)` | `1` | 0 | | `' OR 1=1 --`, bound the same way | `0` | 0 | | `' OR 1=1 --`, interpolated the current way | `2` (whole table) | 0 | | query against a DB with no `transmissions` table, `-bail` | `Parse error … no such table` on **stderr** | **1** | **2. Probe the capability, not a version.** `resolve_sqlite_runner` binds `corescope-probe-ok` and asserts it comes back — a round trip, not a bare `.parameter init`, so the positive control runs against the operator's actual binary rather than one we pin. If neither the container nor the host qualifies, it fails loudly and names what is needed: ``` ❌ retain-failed: no sqlite3 able to bind a parameter on the target tried: docker exec -i corescope-prod sqlite3, then sqlite3 on runner@example need: the sqlite3 CLI reachable over ssh, supporting '.parameter set' OCI runtime exec failed: exec: "sqlite3": executable file not found in $PATH bash: line 1: sqlite3: command not found ``` There is deliberately **no** interpolating fallback. That would leave the vulnerable path in place under a nicer name. **3. The hex gate is kept**, with its comment updated to say why: for the SQL layer it is now defence in depth rather than the only guard. Redundant is not the same as wrong. **4. The exit status and stderr survive.** `-batch -bail -init /dev/null -noheader -list` (stop at the first SQL error; ignore the operator's `~/.sqliterc`, where a stray `.mode` would make the count unparseable; stdout is exactly the number). Query stderr is captured and printed on failure rather than sent to `/dev/null`, so a broken query is distinguishable from a legitimately empty result. Probe stderr is collected too, and printed only if *both* probes fail — the container miss is the known-normal case, so surfacing it on every run would be noise. ## Also fixed An existing double-count in §10.2: the `TARGET_DB_PATH unset` branch incremented `$fails` and then left `count=""`, so the generic branch incremented it a **second** time for the same failure. `read_retain_count` now gives §10.2 exactly one increment point. Opportunistic cleanup in a file already being touched (AGENTS.md line 318). ## Tests New `qa/scripts/test-blacklist-sql.sh`, wired into the `go-test` job. 24 assertions, modelled on `scripts/staging/test-disk-monitor.sh`. Both directions are asserted, because a zero from a command that failed proves nothing: - **Positive control** — `deadbeef` still returns its row (`1`, exit 0), and so does `cafebabe`; an absent pubkey returns `0`. - **Negative** — `' OR 1=1 --` returns `0` while the table demonstrably holds 2 rows, and the old interpolated form is asserted to leak all `2`. That last assertion is what makes the `0` above worth something. - **Error surfacing** — the same SQL against a DB with no `transmissions` table exits non-zero with a message on stderr and nothing on stdout. - **Alphabet** — `sql_hex_literal` output matches `^x'[0-9a-f]*'$` for the SQL payloads, a backslash, `$(id)` / backticks, an embedded newline, `héllo`, and a 4096-byte repetitive string. That last one is a regression guard for `od -v`: without the flag `od` collapses repeated identical lines to `*`. - `run_sqlite` with no resolved runner refuses rather than guessing. Group 2 skips loudly (rather than silently) if `sqlite3` is not on PATH; group 1 needs no sqlite3 and always runs. **Mutation-tested** — each of these breaks the suite, so the assertions have teeth: | mutation | caught by | |---|---| | restore full interpolation | `injection payload → 0 rows — expected '0' got '2'` | | naive `.parameter set '%s'` | `expected '0' got '.parameter CMD ...'` | | drop `od -v` | alphabet failure on `*`, plus `expected '8192' got '33'` | Commit 1 is a behaviour-neutral refactor that moves the imperative body into `main()` behind a `BASH_SOURCE` guard, so the test can source the script and exercise individual helpers. Same idiom as `scripts/staging/disk-monitor.sh:99`. ## Verification - `bash qa/scripts/test-blacklist-sql.sh` → 24 passed, 0 failed - `bash -n` on both scripts - All three runtime paths exercised end to end with PATH shims for `ssh`/`docker`/`sqlite3` against a real fixture DB: success (`sqlite3 runner: host`, count 2), query failure (classified message + `Parse error … no such table`, `fails=1`), and no-capability (the loud block above, both probe stderrs, `fails=1` — not 2) - The new step lands inside `go-test`, which runs when `changes.outputs.code == 'true'`; `qa/scripts/*.sh` does not match that job's `^docs/|[.]md$|^LICENSE$` documentation filter, so it is not skipped ## Deliberately out of scope - **The `docker exec` branch is dead on current images** → filed as #1983. The app container has no `sqlite3` at all: `Dockerfile:15` is pure-Go SQLite with no CGO, and the `apk add` installs only `mosquitto mosquitto-clients supervisor caddy wget`. So the host "fallback" is the only path that has ever executed, silently, because both branches discarded stderr. This change keeps both branches and merely makes the outcome visible (`sqlite3 runner: …` on every run). - **`-readonly` on the target DB.** Tempting, and verified compatible with `.parameter` (the binding table lives in the TEMP database), but a WAL database needing journal recovery can refuse a read-only open. Adding it here risks exactly the "trades an unreachable injection for a script that does not run" outcome flagged in the #1952 thread. Worth its own issue. - **The other `2>/dev/null` sites** in this file, which also sit awkwardly with `qa/README.md`'s "Don't silence stderr". Only the §10.2 lines named in the sign-off are touched. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 --- .github/workflows/deploy.yml | 3 + qa/scripts/blacklist-test.sh | 342 +++++++++++++++++++++---------- qa/scripts/test-blacklist-sql.sh | 135 ++++++++++++ 3 files changed, 369 insertions(+), 111 deletions(-) create mode 100755 qa/scripts/test-blacklist-sql.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 49ea5f762..3097bacd9 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -166,6 +166,9 @@ jobs: - name: Staging disk-monitor unit tests (issue #1684) run: bash scripts/staging/test-disk-monitor.sh + - name: QA SQL parameter-binding unit tests (issue #1977) + run: bash qa/scripts/test-blacklist-sql.sh + - name: Lint CSS variables (issue #1128) run: | set -e diff --git a/qa/scripts/blacklist-test.sh b/qa/scripts/blacklist-test.sh index 94b5b4127..535f477f0 100755 --- a/qa/scripts/blacklist-test.sh +++ b/qa/scripts/blacklist-test.sh @@ -25,65 +25,24 @@ # ssh-failed → cannot reach/control target # restart-stuck → /api/stats not 200 within RESTART_WAIT_S # hide-failed → blacklisted pubkey still surfaced via API (§10.1 fail) -# retain-failed → blacklisted pubkey absent from DB (§10.2 fail) +# retain-failed → blacklisted pubkey absent from DB (§10.2 fail), or the +# §10.2 probe could not run at all — no sqlite3 on the target +# able to bind a parameter. The message names what is needed; +# there is no fallback to interpolated SQL. # teardown-failed→ post-test removal did not restore listing # # Exit code = number of failures (0 = pass). # PUBLIC repo: zero PII — no real pubkeys, IPs, or hostnames as defaults. +# +# Structure: helpers live at top level and the imperative body lives in main(), +# so test-blacklist-sql.sh can source this file and exercise individual helpers +# without running the suite. Same idiom as scripts/staging/disk-monitor.sh. set -uo pipefail -BASELINE_URL="${1:-}" -TARGET_URL="${2:-}" -if [[ -z "$BASELINE_URL" || -z "$TARGET_URL" ]]; then - echo "usage: $0 BASELINE_URL TARGET_URL (TEST_NODE_PUBKEY+TARGET_* via env)" >&2 - exit 2 -fi - -TEST_PUBKEY="${TEST_NODE_PUBKEY:-}" -TARGET_SSH_HOST="${TARGET_SSH_HOST:-}" -TARGET_SSH_KEY="${TARGET_SSH_KEY:-/root/.ssh/id_ed25519}" -TARGET_CONFIG_PATH="${TARGET_CONFIG_PATH:-}" -TARGET_CONTAINER="${TARGET_CONTAINER:-}" -TARGET_DB_PATH="${TARGET_DB_PATH:-}" -ADMIN_API_TOKEN="${ADMIN_API_TOKEN:-}" - -if [[ -z "$TEST_PUBKEY" || -z "$TARGET_SSH_HOST" || -z "$TARGET_CONFIG_PATH" || -z "$TARGET_CONTAINER" ]]; then - echo "error: TEST_NODE_PUBKEY, TARGET_SSH_HOST, TARGET_CONFIG_PATH, TARGET_CONTAINER are required" >&2 - exit 2 -fi - -# Hard input validation — these strings are interpolated into remote shell/SQL. -# Pubkey must be hex (MeshCore pubkeys are hex-encoded ed25519 prefixes). -if ! [[ "$TEST_PUBKEY" =~ ^[0-9a-fA-F]+$ ]]; then - echo "error: TEST_NODE_PUBKEY must be hex (got: redacted)" >&2 - exit 2 -fi -# Container name must match docker's allowed chars: [a-zA-Z0-9][a-zA-Z0-9_.-]* -if ! [[ "$TARGET_CONTAINER" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then - echo "error: TARGET_CONTAINER has illegal chars" >&2 - exit 2 -fi -# Config path must be an absolute, sane path (no spaces, quotes, $, ;, etc.). -if ! [[ "$TARGET_CONFIG_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then - echo "error: TARGET_CONFIG_PATH must be a sane absolute path" >&2 - exit 2 -fi -if [[ -n "$TARGET_DB_PATH" ]] && ! [[ "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then - echo "error: TARGET_DB_PATH must be a sane absolute path" >&2 - exit 2 -fi - -CURL_TIMEOUT="${CURL_TIMEOUT:-60}" -RESTART_WAIT_S="${RESTART_WAIT_S:-120}" - -SSH_OPTS=(-i "$TARGET_SSH_KEY" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -o BatchMode=yes) +SSH_OPTS=() # populated by main() from TARGET_SSH_KEY ssh_t() { ssh "${SSH_OPTS[@]}" "$TARGET_SSH_HOST" "$@"; } -TMP=$(mktemp -d) -fails=0 -TEARDOWN_DONE=0 - # ----------------------------------------------------------------------------- # Teardown — MANDATORY in all exit paths. # ----------------------------------------------------------------------------- @@ -106,7 +65,6 @@ teardown() { rm -rf "$TMP" exit "$rc" } -trap teardown EXIT INT TERM # ----------------------------------------------------------------------------- # Helpers @@ -197,75 +155,237 @@ node_visible() { } # ----------------------------------------------------------------------------- -# §10.1 — hide +# §10.2 DB probe — bind the pubkey, do not interpolate it (issue #1977) # ----------------------------------------------------------------------------- -echo "=== §10.1 add $TEST_PUBKEY to nodeBlacklist ===" -if ! add_to_blacklist; then fails=$((fails+1)); exit "$fails"; fi -if ! restart_target; then fails=$((fails+1)); exit "$fails"; fi -if ! wait_for_stats; then fails=$((fails+1)); exit "$fails"; fi +# Batch flags, all in service of "the count is parseable and errors are visible": +# -bail stop at the first SQL error instead of running on +# -init /dev/null ignore the operator's ~/.sqliterc — a stray .mode there +# would make the count unparseable +# -noheader -list stdout is exactly the number, nothing else +SQLITE_ARGS=(-batch -bail -init /dev/null -noheader -list) +# Round-trip probe token. The value is arbitrary; it only has to come back intact. +SQLITE_PROBE_TOKEN="corescope-probe-ok" +SQLITE_RUNNER="" # "container" | "host", set by resolve_sqlite_runner +RETAIN_COUNT="" # set by read_retain_count -detail_code=$(fetch_code "$TARGET_URL/api/nodes/$TEST_PUBKEY" "$TMP/detail.json") -list_code=$(fetch_code "$TARGET_URL/api/nodes?limit=10000" "$TMP/list.json") -in_list=0 -if [[ "$list_code" == "200" ]] && grep -qF -- "\"$TEST_PUBKEY\"" "$TMP/list.json"; then - in_list=1 -fi -if [[ "$detail_code" == "404" || "$in_list" == "0" ]]; then - echo " ✅ hide ok: detail=$detail_code in_list=$in_list" -else - echo " ❌ hide-failed: detail=$detail_code in_list=$in_list — pubkey still surfaced" - fails=$((fails+1)) -fi +# Hex-encode a value for embedding in SQL as a blob literal. +# +# Why hex rather than quoting: the output alphabet is [0-9a-f], so no byte the +# caller passes can terminate a string literal or add a dot-command argument. +# That holds for arbitrary input, which is the point — the SQL layer stops +# depending on main()'s hex gate in order to be safe. +# +# `od -v` is load-bearing: without it od collapses runs of identical lines to +# '*' and long repetitive values encode wrongly. +sql_hex_literal() { + printf "x'%s'" "$(printf '%s' "$1" | od -An -v -tx1 | tr -d ' \n')" +} -topo_code=$(fetch_code "$TARGET_URL/api/topology" "$TMP/topo.json") -if [[ "$topo_code" != "200" ]]; then - echo " ⚠️ /api/topology HTTP $topo_code — skipping topology assertion" -elif grep -qF -- "$TEST_PUBKEY" "$TMP/topo.json"; then - echo " ❌ hide-failed: /api/topology references blacklisted pubkey" - fails=$((fails+1)) -else - echo " ✅ topology clean" -fi +# SQL for the §10.2 count, fed to sqlite3 on stdin. The SELECT text is a +# constant; the pubkey arrives as a bound parameter. +# +# Note the nested cast rather than `.parameter set :pubkey ''`: +# dot-command arguments are split on whitespace, so a value containing a space +# (e.g. "' OR 1=1 --") makes sqlite3 print the .parameter help to STDOUT, exit +# 0, and leave :pubkey unbound. COUNT(*) then returns 0 — which reads exactly +# like a passing security fix. -bail does not catch it either. +transmission_count_sql() { + printf '.parameter init\n' + printf '.parameter set :pubkey "cast(%s as text)"\n' "$(sql_hex_literal "$1")" + printf 'SELECT COUNT(*) FROM transmissions WHERE from_node = :pubkey;\n' +} -# ----------------------------------------------------------------------------- -# §10.2 — DB retain -# ----------------------------------------------------------------------------- -echo "=== §10.2 verify packets retained in DB ===" -count="" -if [[ -n "$ADMIN_API_TOKEN" ]]; then - # Read auth header from stdin so the token never enters argv (ps-safe). - code=$(printf 'header = "Authorization: Bearer %s"\n' "$ADMIN_API_TOKEN" | \ - curl -s -m "$CURL_TIMEOUT" -K - -o "$TMP/admin.json" -w "%{http_code}" \ - "$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo "000") - if [[ "$code" == "200" ]]; then - count=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo "") +# Capability probe: bind a known value and read it back. A version number only +# implies that .parameter works; binding something and getting it back proves it +# on the binary actually in front of us, which is the operator's, not ours. +sqlite_probe_sql() { + printf '.parameter init\n' + printf '.parameter set :probe "cast(%s as text)"\n' "$(sql_hex_literal "$SQLITE_PROBE_TOKEN")" + printf 'SELECT :probe;\n' +} + +# Find a sqlite3 that can bind a parameter — in the container first, then on the +# host. Sets SQLITE_RUNNER; returns 1 if neither qualifies. There is deliberately +# no interpolating fallback: that would leave the vulnerable path in place under +# a nicer name. +# +# Probe stderr is collected rather than discarded, but only printed if BOTH +# probes fail. The container miss is the known-normal case — the app image has +# no sqlite3 (pure-Go driver, no CGO; Dockerfile:15) — so surfacing it on every +# run would be noise. +resolve_sqlite_runner() { + local probe out + probe=$(sqlite_probe_sql) + SQLITE_RUNNER="" + out=$(ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} :memory:" \ + <<<"$probe" 2>>"$TMP/sqlite-probe.err") + if [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then SQLITE_RUNNER="container"; return 0; fi + out=$(ssh_t "sqlite3 ${SQLITE_ARGS[*]} :memory:" <<<"$probe" 2>>"$TMP/sqlite-probe.err") + if [[ "$out" == "$SQLITE_PROBE_TOKEN" ]]; then SQLITE_RUNNER="host"; return 0; fi + return 1 +} + +# Run SQL from stdin against TARGET_DB_PATH via the resolved runner. Stderr is +# left alone so the caller can capture it, and the exit status is sqlite3's. +# The SQL crosses on stdin, so only the container name and db path still need +# printf %q for the remote shell. docker exec needs -i to attach stdin. +run_sqlite() { + case "$SQLITE_RUNNER" in + container) ssh_t "docker exec -i $(printf %q "$TARGET_CONTAINER") sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; + host) ssh_t "sqlite3 ${SQLITE_ARGS[*]} $(printf %q "$TARGET_DB_PATH")" ;; + *) echo "run_sqlite: no runner resolved" >&2; return 127 ;; + esac +} + +# Read the retained-transmission count into RETAIN_COUNT. Prints a classified +# "retain-failed" line and returns 1 on failure, so §10.2 has exactly one place +# that increments $fails. +read_retain_count() { + RETAIN_COUNT="" + local code + if [[ -n "$ADMIN_API_TOKEN" ]]; then + # Read auth header from stdin so the token never enters argv (ps-safe). + code=$(printf 'header = "Authorization: Bearer %s"\n' "$ADMIN_API_TOKEN" | \ + curl -s -m "$CURL_TIMEOUT" -K - -o "$TMP/admin.json" -w "%{http_code}" \ + "$TARGET_URL/api/admin/transmissions?from_node=$TEST_PUBKEY&count=1" 2>/dev/null || echo "000") + if [[ "$code" == "200" ]]; then + RETAIN_COUNT=$(jq -r '.count // ((.transmissions // []) | length)' "$TMP/admin.json" 2>/dev/null || echo "") + fi + if [[ -n "$RETAIN_COUNT" ]]; then return 0; fi fi -fi -if [[ -z "$count" ]]; then + if [[ -z "$TARGET_DB_PATH" ]]; then echo " ❌ retain-failed: TARGET_DB_PATH unset and no ADMIN_API_TOKEN — cannot probe" + return 1 + fi + if ! resolve_sqlite_runner; then + echo " ❌ retain-failed: no sqlite3 able to bind a parameter on the target" + echo " tried: docker exec -i $TARGET_CONTAINER sqlite3, then sqlite3 on $TARGET_SSH_HOST" + echo " need: the sqlite3 CLI reachable over ssh, supporting '.parameter set'" + cat "$TMP/sqlite-probe.err" >&2 + return 1 + fi + echo " sqlite3 runner: $SQLITE_RUNNER" + if ! RETAIN_COUNT=$(run_sqlite <<<"$(transmission_count_sql "$TEST_PUBKEY")" 2>"$TMP/sqlite.err"); then + echo " ❌ retain-failed: sqlite3 query failed via $SQLITE_RUNNER" + cat "$TMP/sqlite.err" >&2 + RETAIN_COUNT="" + return 1 + fi + return 0 +} + +# ----------------------------------------------------------------------------- +# main +# ----------------------------------------------------------------------------- +main() { + BASELINE_URL="${1:-}" + TARGET_URL="${2:-}" + if [[ -z "$BASELINE_URL" || -z "$TARGET_URL" ]]; then + echo "usage: $0 BASELINE_URL TARGET_URL (TEST_NODE_PUBKEY+TARGET_* via env)" >&2 + exit 2 + fi + + TEST_PUBKEY="${TEST_NODE_PUBKEY:-}" + TARGET_SSH_HOST="${TARGET_SSH_HOST:-}" + TARGET_SSH_KEY="${TARGET_SSH_KEY:-/root/.ssh/id_ed25519}" + TARGET_CONFIG_PATH="${TARGET_CONFIG_PATH:-}" + TARGET_CONTAINER="${TARGET_CONTAINER:-}" + TARGET_DB_PATH="${TARGET_DB_PATH:-}" + ADMIN_API_TOKEN="${ADMIN_API_TOKEN:-}" + + if [[ -z "$TEST_PUBKEY" || -z "$TARGET_SSH_HOST" || -z "$TARGET_CONFIG_PATH" || -z "$TARGET_CONTAINER" ]]; then + echo "error: TEST_NODE_PUBKEY, TARGET_SSH_HOST, TARGET_CONFIG_PATH, TARGET_CONTAINER are required" >&2 + exit 2 + fi + + # Hard input validation — these strings are interpolated into the remote shell. + # §10.2's SQL binds TEST_PUBKEY as a parameter rather than interpolating it, so + # for the SQL layer this gate is defence in depth rather than the only guard + # (issue #1977). Keep it: redundant is not the same as wrong. + # Pubkey must be hex (MeshCore pubkeys are hex-encoded ed25519 prefixes). + if ! [[ "$TEST_PUBKEY" =~ ^[0-9a-fA-F]+$ ]]; then + echo "error: TEST_NODE_PUBKEY must be hex (got: redacted)" >&2 + exit 2 + fi + # Container name must match docker's allowed chars: [a-zA-Z0-9][a-zA-Z0-9_.-]* + if ! [[ "$TARGET_CONTAINER" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then + echo "error: TARGET_CONTAINER has illegal chars" >&2 + exit 2 + fi + # Config path must be an absolute, sane path (no spaces, quotes, $, ;, etc.). + if ! [[ "$TARGET_CONFIG_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then + echo "error: TARGET_CONFIG_PATH must be a sane absolute path" >&2 + exit 2 + fi + if [[ -n "$TARGET_DB_PATH" ]] && ! [[ "$TARGET_DB_PATH" =~ ^/[A-Za-z0-9_./-]+$ ]]; then + echo "error: TARGET_DB_PATH must be a sane absolute path" >&2 + exit 2 + fi + + CURL_TIMEOUT="${CURL_TIMEOUT:-60}" + RESTART_WAIT_S="${RESTART_WAIT_S:-120}" + + SSH_OPTS=(-i "$TARGET_SSH_KEY" -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -o BatchMode=yes) + + TMP=$(mktemp -d) + fails=0 + TEARDOWN_DONE=0 + trap teardown EXIT INT TERM + + # --------------------------------------------------------------------------- + # §10.1 — hide + # --------------------------------------------------------------------------- + echo "=== §10.1 add $TEST_PUBKEY to nodeBlacklist ===" + if ! add_to_blacklist; then fails=$((fails+1)); exit "$fails"; fi + if ! restart_target; then fails=$((fails+1)); exit "$fails"; fi + if ! wait_for_stats; then fails=$((fails+1)); exit "$fails"; fi + + detail_code=$(fetch_code "$TARGET_URL/api/nodes/$TEST_PUBKEY" "$TMP/detail.json") + list_code=$(fetch_code "$TARGET_URL/api/nodes?limit=10000" "$TMP/list.json") + in_list=0 + if [[ "$list_code" == "200" ]] && grep -qF -- "\"$TEST_PUBKEY\"" "$TMP/list.json"; then + in_list=1 + fi + if [[ "$detail_code" == "404" || "$in_list" == "0" ]]; then + echo " ✅ hide ok: detail=$detail_code in_list=$in_list" + else + echo " ❌ hide-failed: detail=$detail_code in_list=$in_list — pubkey still surfaced" + fails=$((fails+1)) + fi + + topo_code=$(fetch_code "$TARGET_URL/api/topology" "$TMP/topo.json") + if [[ "$topo_code" != "200" ]]; then + echo " ⚠️ /api/topology HTTP $topo_code — skipping topology assertion" + elif grep -qF -- "$TEST_PUBKEY" "$TMP/topo.json"; then + echo " ❌ hide-failed: /api/topology references blacklisted pubkey" fails=$((fails+1)) else - # TEST_PUBKEY is hex-validated → safe to inline single-quoted in SQL. - # Container/db path also validated; printf %q for defense in depth. - q="SELECT COUNT(*) FROM transmissions WHERE from_node = '$TEST_PUBKEY';" - qq=$(printf %q "$q") - if ! count=$(ssh_t "docker exec $(printf %q "$TARGET_CONTAINER") sqlite3 $(printf %q "$TARGET_DB_PATH") $qq" 2>/dev/null); then - count=$(ssh_t "sqlite3 $(printf %q "$TARGET_DB_PATH") $qq" 2>/dev/null || echo "") - fi + echo " ✅ topology clean" fi -fi -if [[ -z "$count" ]]; then - echo " ❌ retain-failed: could not read transmissions count" - fails=$((fails+1)) -elif [[ "$count" =~ ^[0-9]+$ ]] && (( count > 0 )); then - echo " ✅ DB retains $count packets from $TEST_PUBKEY" -else - echo " ❌ retain-failed: count=$count (expected > 0)" - fails=$((fails+1)) -fi + # --------------------------------------------------------------------------- + # §10.2 — DB retain + # --------------------------------------------------------------------------- + echo "=== §10.2 verify packets retained in DB ===" + if ! read_retain_count; then + # read_retain_count already printed the classified reason. Counting here and + # nowhere else: the old code incremented $fails for the "TARGET_DB_PATH + # unset" case and then again for the empty count it left behind. + fails=$((fails+1)) + elif [[ "$RETAIN_COUNT" =~ ^[0-9]+$ ]] && (( RETAIN_COUNT > 0 )); then + echo " ✅ DB retains $RETAIN_COUNT packets from $TEST_PUBKEY" + else + echo " ❌ retain-failed: count=$RETAIN_COUNT (expected > 0)" + fails=$((fails+1)) + fi + + echo "=== summary: $fails failure(s) before teardown ===" + # trap handles teardown + exit + exit "$fails" +} -echo "=== summary: $fails failure(s) before teardown ===" -# trap handles teardown + exit -exit "$fails" +# Only run main when executed directly (not when sourced by tests). +if [ "${BASH_SOURCE[0]}" = "${0}" ]; then + main "$@" +fi diff --git a/qa/scripts/test-blacklist-sql.sh b/qa/scripts/test-blacklist-sql.sh new file mode 100755 index 000000000..770d6e065 --- /dev/null +++ b/qa/scripts/test-blacklist-sql.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# test-blacklist-sql.sh — unit tests for the §10.2 SQL construction in +# qa/scripts/blacklist-test.sh (issue #1977). Sources the script and exercises +# its pure helpers, plus a real local sqlite3 against a throwaway fixture DB. +# +# Run: bash qa/scripts/test-blacklist-sql.sh +# Exits non-zero if any case fails. +# +# The point of the sqlite3 group is that BOTH directions are asserted. A test +# that only checks "the injection payload returns 0" passes just as happily when +# the query is silently broken and returns 0 for everything, so the legitimate +# pubkey must be shown to still return the row it should. + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=blacklist-test.sh +. "$SCRIPT_DIR/blacklist-test.sh" + +PASS=0 +FAIL=0 + +assert_eq() { + local label="$1" expected="$2" actual="$3" + if [ "$expected" = "$actual" ]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $label — expected '$expected' got '$actual'" >&2 + fi +} + +assert_match() { + local label="$1" pattern="$2" actual="$3" + if [[ "$actual" =~ $pattern ]]; then + PASS=$((PASS + 1)) + else + FAIL=$((FAIL + 1)) + echo "FAIL: $label — '$actual' does not match /$pattern/" >&2 + fi +} + +# ----- sql_hex_literal ------------------------------------------------------ +# The security property: whatever goes in, the SQL text it produces is drawn +# from [0-9a-f] only. No caller-supplied byte can close a string literal or add +# a dot-command argument. Needs no sqlite3, so this group always runs. +assert_eq "hex of deadbeef" "x'6465616462656566'" "$(sql_hex_literal deadbeef)" +assert_eq "hex of empty" "x''" "$(sql_hex_literal "")" + +HEX_ONLY="^x'[0-9a-f]*'\$" +assert_match "alphabet: sql quote payload" "$HEX_ONLY" "$(sql_hex_literal "' OR 1=1 --")" +assert_match "alphabet: drop table" "$HEX_ONLY" "$(sql_hex_literal '"; DROP TABLE transmissions; --')" +assert_match "alphabet: backslash" "$HEX_ONLY" "$(sql_hex_literal 'a\b')" +assert_match "alphabet: dollar and backtick" "$HEX_ONLY" "$(sql_hex_literal '$(id) `id`')" +assert_match "alphabet: embedded newline" "$HEX_ONLY" "$(sql_hex_literal "$(printf 'a\nb')")" +assert_match "alphabet: multibyte" "$HEX_ONLY" "$(sql_hex_literal 'héllo')" + +# `od` without -v collapses runs of identical lines to '*'. A long repetitive +# value is the case that catches losing the flag. +LONG=$(printf 'x%.0s' $(seq 1 4096)) +LONG_HEX=$(sql_hex_literal "$LONG") +assert_match "alphabet: 4096 repeated bytes" "$HEX_ONLY" "$LONG_HEX" +# 4096 bytes → 8192 hex digits, plus the 3 chars of x''. A collapsed run would +# be far shorter and would also fail the alphabet check on '*'. +assert_eq "no od line-collapse in 4096-byte value" "8192" "$(( ${#LONG_HEX} - 3 ))" + +# ----- against a real sqlite3 ---------------------------------------------- +if ! command -v sqlite3 >/dev/null 2>&1; then + echo "SKIP: sqlite3 not on PATH — skipping the ${#SQLITE_ARGS[@]}-flag query group" >&2 + echo " (the alphabet assertions above still ran)" >&2 +else + FIXTURE_DIR=$(mktemp -d) + trap 'rm -rf "$FIXTURE_DIR"' EXIT + DB="$FIXTURE_DIR/fixture.db" + EMPTY_DB="$FIXTURE_DIR/no-table.db" + sqlite3 "$DB" \ + "CREATE TABLE transmissions(from_node TEXT); INSERT INTO transmissions VALUES('deadbeef'),('cafebabe');" + sqlite3 "$EMPTY_DB" "CREATE TABLE unrelated(x);" + + run_local() { sqlite3 "${SQLITE_ARGS[@]}" "$1"; } + + # The capability probe must round-trip on this machine, or the assertions + # below would be testing nothing. + assert_eq "probe round-trips" "$SQLITE_PROBE_TOKEN" "$(sqlite_probe_sql | run_local :memory:)" + + # POSITIVE CONTROL: a legitimate pubkey still returns its row. Without this, + # a silently broken query looks like a passing security fix. + out=$(transmission_count_sql deadbeef | run_local "$DB"); rc=$? + assert_eq "legit pubkey → its row" "1" "$out" + assert_eq "legit pubkey → exit 0" "0" "$rc" + assert_eq "other legit pubkey" "1" "$(transmission_count_sql cafebabe | run_local "$DB")" + assert_eq "absent pubkey → 0" "0" "$(transmission_count_sql abc123 | run_local "$DB")" + + # NEGATIVE: the payload binds as a literal that matches nothing. The table + # holds 2 rows, so a structural injection would return 2, not 0. + out=$(transmission_count_sql "' OR 1=1 --" | run_local "$DB"); rc=$? + assert_eq "injection payload → 0 rows" "0" "$out" + assert_eq "injection payload → exit 0" "0" "$rc" + assert_eq "table really does hold 2 rows" "2" \ + "$(run_local "$DB" <<<'SELECT COUNT(*) FROM transmissions;')" + + # Interpolating the same payload the old way returns the whole table. This is + # the behaviour the change removes; asserting it keeps the test honest about + # what "0" above is worth. + legacy="SELECT COUNT(*) FROM transmissions WHERE from_node = '' OR 1=1 --';" + assert_eq "old interpolated form leaked the table" "2" "$(run_local "$DB" <<<"$legacy")" + + # Multibyte and whitespace values bind as themselves rather than erroring. + sqlite3 "$DB" "INSERT INTO transmissions VALUES('héllo wörld');" + assert_eq "multibyte value with a space binds" "1" \ + "$(transmission_count_sql 'héllo wörld' | run_local "$DB")" + + # ERROR SURFACING: a broken query must be distinguishable from an empty + # result — non-zero exit and something on stderr, not a silent "". + err_file="$FIXTURE_DIR/err" + out=$(transmission_count_sql deadbeef | run_local "$EMPTY_DB" 2>"$err_file"); rc=$? + if [ "$rc" -ne 0 ]; then PASS=$((PASS + 1)); else + FAIL=$((FAIL + 1)); echo "FAIL: missing table — expected non-zero exit, got $rc" >&2 + fi + if [ -s "$err_file" ]; then PASS=$((PASS + 1)); else + FAIL=$((FAIL + 1)); echo "FAIL: missing table — expected a message on stderr" >&2 + fi + assert_eq "missing table → no count on stdout" "" "$out" + + # run_sqlite with no resolved runner must refuse rather than guess. + SQLITE_RUNNER="" + if run_sqlite /dev/null 2>&1; then + FAIL=$((FAIL + 1)); echo "FAIL: run_sqlite with no runner — expected non-zero exit" >&2 + else + PASS=$((PASS + 1)) + fi +fi + +echo "test-blacklist-sql.sh: $PASS passed, $FAIL failed" +[ "$FAIL" -eq 0 ] From 079e73aa4cbbfb753f2f13f6dda8c0b4993243e5 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 9 Sep 2026 16:00:15 +0200 Subject: [PATCH 013/151] fix(scope-audit): attribute forwarding to every hop, and pay for the wider scan (#1986) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Scope Audit credits a transmission to `path[last]` only. On a flood-family route every forwarder appends its own hash to the END of the path (`internal/packetpath/route.go`), so `path[last]` does not mean "forwarded this packet", it means "was the transmission an uplinked observer heard directly". Every earlier hop forwarded the same packet and is discarded. The last-hop rule is genuinely required for DIRECT routes, which consume hops from the front, so their `path[last]` is the route’s far end rather than the transmitter. But `scopeAuditForwarderScanQuery` already restricts to `route_type IN (0, 1)` via `scopeConformanceForwarderRouteTypesSQL`, where that hazard cannot arise, so inside this query the restriction only throws evidence away. ## What it costs the page today Measured on a live-shaped instance, 206 declared repeaters, 965k transmissions, 7d window: | | before | after | |---|---|---| | repeaters with no attributable evidence of any kind | 133 of 205 (65%) | 30 of 206 (15%) | On a 1000-packet flood sample the mean path length is 7.08 hops, so the last-hop rule keeps 394 of 2789 hop observations (14%), and 85% of the nodes seen forwarding never appear as a last hop at all. Those repeaters have every region they declare reported as "declared, not observed", which is the page presenting a gap in our own attribution as a finding about someone else’s repeater. ## Rule 0: what widening it costs, and what pays for it Reading every hop multiplies the rows the scan returns: a 7d window yields **3,470,188 hop rows** from 1,368,761 observations carrying a path. Cold cost before this change was 16.7s for 7d and 4.0s for 24h, of which SQLite accounts for 2.7s. The rest was the Go side reading rows. Three changes, in order of what they bought: 1. **The scan carried `scope_name` and `first_seen` on every hop row.** Both are columns of `transmissions`, and at 43 hop rows per transmission the same two values were re-read that many times. They now come from one query over the same window keyed by transmission id, both inside one read transaction so a transmission arriving between them cannot appear in the hop scan with no metadata to attribute it by. The hop scan carries two columns instead of four. 2. **The hop is lower-cased into a stack buffer** instead of through `strings.ToLower`. 1,026,814 of the 1,284,897 hops in a 24h window are stored uppercase, because `packetpath.DecodePathFromRawHex` writes them that way, and the great majority match no declared target, so that allocation was paid millions of times to answer "no". The `(target, txID)` de-duplication key became a struct for the same reason. 3. **The compute ran outside the cache mutex**, so every request arriving on a cold window ran its own full scan concurrently. It now sits behind a singleflight, the same treatment `/api/observers` and `/api/nodes/{pubkey}/reach` already have, and the 7d window gets a 5 minute TTL while 1h and 24h keep 30s. At 30s a single reader with 7d open keeps the instance recomputing more than half the time, for an aggregate that moves at the pace of a week of traffic. Result, warm process: | window | before | after | |---|---|---| | 1h | 0.155s | 0.140s | | 24h | 4.04s | 2.79-2.89s across six samples | | 7d | 16.7s | 11.6s | | repeat inside TTL | ~1ms | ~1ms | **Rejected alternatives, measured on the same database**, so the next reader does not have to re-derive them: | approach | rows returned | time in SQLite | |---|---|---| | the query as written | 3,470,188 | 2.7s | | pre-filter on the declared targets’ first 4 hex chars | 1,971,126 | 20.9s | | `GROUP BY t.id, hop` | 965,025 | 38.0s | | `SELECT DISTINCT t.id, path_json` | 1,229,966 | 17.7s | The query plan is already index-driven (`idx_transmissions_first_seen`, then `idx_observations_tx_ts`), so there is no missing index behind this: the rows are inherent to the data. Note for anyone attempting a hop comparison in SQL: a case-sensitive comparison silently drops most attributable hops, per the 80% figure above. ## Tests - a mid-path hop is attributed (the case behind the 65% blind spot) - a DIRECT transmission whose `path[last]` **is** the target is still not attributed. With the last-hop rule gone this is the only thing standing between the audit and misattribution, so it gets its own test rather than relying on the route filter being obvious - one transmission counted once per target even when it appears on several hops of the same path, which the `(target, txID)` de-duplication now carries alone - a hop longer than the 4-char floor resolved by its own length, which nothing pinned before: every other test seeds 4-char hops - the per-window TTL, so collapsing it back to one constant has to delete the reason - a second request inside the TTL served from cache rather than recomputed. The cache path had no test at all `cd cmd/server && go test ./...` passes (168s), `go vet` and `gofmt -l` clean. Server-side only, no API shape change, no frontend change. Browser validation: run against a live instance carrying this change, the Scope Audit renders 220 rows matching the API row for row, and the per-node scopes page still answers with its route-type mix. --- cmd/server/routes.go | 1 + cmd/server/scope_audit.go | 337 +++++++++++++++++++++++++++++---- cmd/server/scope_audit_test.go | 231 ++++++++++++++++++++++ 3 files changed, 532 insertions(+), 37 deletions(-) diff --git a/cmd/server/routes.go b/cmd/server/routes.go index c83fde8ac..f9d6f1c59 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -75,6 +75,7 @@ type Server struct { scopeAuditMu sync.Mutex scopeAuditCache map[string]*ScopeAuditResponse scopeAuditCachedAt map[string]time.Time + scopeAuditSF singleflight.Group // #1975: /api/scope-audit window cache and its single-flight guard, so a // burst of viewers on a cold cache recomputes the network-wide scan once diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index d1aa719ef..32249c8fb 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -24,7 +24,6 @@ import ( "fmt" "net/http" "sort" - "strconv" "strings" "sync" "sync/atomic" @@ -320,6 +319,14 @@ func scopeAuditPrefixIndex(targets []string) map[int]map[string][]string { // call per pubkey — fine for one node, but 37+ repeater-sized loop of them // would each re-scan the same first_seen index range), this scans the // FLOOD-family window exactly once and returns every forwarder hop found. +// +// "Every forwarder hop" means every hop of the path, not only path[last] — +// see scopeConformanceQuery's doc comment for why, and note that this query +// returns one ROW PER HOP, so a single transmission now yields as many rows as +// it has hops (mean 7.08 on the live network). ScopeAuditForwarding's +// "|" de-duplication is what keeps that from counting a +// transmission twice for one target, and it is now load-bearing rather than +// belt-and-braces: one path can carry the same target on several hops. // It applies the SAME three conditions scopeConformanceQuery does — // minForwarderHopHexLen, scopeConformanceForwarderRouteTypesSQL, and the // explicit json_valid guard against a single malformed path_json row @@ -329,10 +336,10 @@ func scopeAuditPrefixIndex(targets []string) map[int]map[string][]string { // scopeAuditPrefixIndex, so the SQL cost stays O(rows in window) regardless // of len(targets). var scopeAuditForwarderScanQuery = ` - SELECT t.id, je.value, t.scope_name, t.first_seen + SELECT t.id, je.value FROM transmissions t JOIN observations o ON o.transmission_id = t.id - JOIN json_each(o.path_json) je ON je.key = json_array_length(o.path_json) - 1 + JOIN json_each(o.path_json) je WHERE t.first_seen >= ? AND ` + scopeConformanceForwarderRouteTypesSQL + ` AND o.path_json IS NOT NULL @@ -341,6 +348,26 @@ var scopeAuditForwarderScanQuery = ` AND LENGTH(je.value) >= ` + fmt.Sprint(minForwarderHopHexLen) + ` ` +// scopeAuditWindowMetaQuery reads the two per-TRANSMISSION facts the hop scan +// used to carry on every hop row: the scope name and the timestamp. It applies +// the identical window and route-type filter, so it covers every transmission +// the hop scan can produce, and both run inside one read transaction so the +// two see the same snapshot. +// +// Splitting these out is why the hop scan carries two columns instead of four. +// Measured on the live-shaped staging database on 2026-09-07, a 7d window +// yields 3,470,188 hop rows against 79,652 transmissions: 43 hop rows per +// transmission, each of which was re-reading the same scope_name and +// first_seen. SQLite spends 2.7s of the 16.7s that window cost; the rest was +// the Go side scanning columns it already knew. +// +// Every SQL-side attempt to shrink the hop scan itself measured worse on that +// same database and was rejected: a first-4-hex prefix filter against the +// declared targets takes 20.9s (and needs lower() on both sides, because 80% +// of stored hops are uppercase), GROUP BY t.id, hop takes 38.0s, and +// SELECT DISTINCT t.id, path_json takes 17.7s. The 3.47M rows are inherent: +// 1,368,761 observations carrying a path, ~2.5 usable hops each. + // ScopeAuditForwarding runs scopeAuditForwarderScanQuery once for the whole // window and attributes every forwarder hop it finds to targets, by the same // truncated-hash prefix match ScopeConformance uses for a single pubkey. @@ -362,11 +389,159 @@ var scopeAuditForwarderScanQuery = ` // the same de-duplication scopeConformanceQuery gets for free from EXISTS, // done explicitly here since this scan is not correlated per target. +// scopeAuditWindowMetaQuery reads the two per-TRANSMISSION facts the hop scan +// used to carry on every hop row: the scope name and the timestamp. It applies +// the identical window and route-type filter, so it covers every transmission +// the hop scan can produce, and both run inside one read transaction so the +// two see the same snapshot. +// +// Splitting these out is why the hop scan carries two columns instead of four. +// Measured on the live-shaped staging database on 2026-09-07, a 7d window +// yields 3,470,188 hop rows against 79,652 transmissions: 43 hop rows per +// transmission, each of which was re-reading the same scope_name and +// first_seen. SQLite spends 2.7s of the 16.7s that window cost; the rest was +// the Go side scanning columns it already knew. +// +// Every SQL-side attempt to shrink the hop scan itself measured worse on that +// same database and was rejected: a first-4-hex prefix filter against the +// declared targets takes 20.9s (and needs lower() on both sides, because 80% +// of stored hops are uppercase), GROUP BY t.id, hop takes 38.0s, and +// SELECT DISTINCT t.id, path_json takes 17.7s. The 3.47M rows are inherent: +// 1,368,761 observations carrying a path, ~2.5 usable hops each. +var scopeAuditWindowMetaQuery = ` + SELECT t.id, t.scope_name, t.first_seen + FROM transmissions t + WHERE t.first_seen >= ? + AND ` + scopeConformanceForwarderRouteTypesSQL + ` +` + +// scopeAuditTxMeta is one transmission's contribution to the aggregate, held +// once per transmission rather than once per hop. + +// scopeAuditTxMeta is one transmission's contribution to the aggregate, held +// once per transmission rather than once per hop. +type scopeAuditTxMeta struct { + scopeName sql.NullString + firstSeen string +} + +// scopeAuditWindowMeta loads scopeAuditWindowMetaQuery into a map keyed by +// transmission id. Runs on the caller's transaction so it shares the hop +// scan's snapshot. + +// scopeAuditWindowMeta loads scopeAuditWindowMetaQuery into a map keyed by +// transmission id. Runs on the caller's transaction so it shares the hop +// scan's snapshot. +func scopeAuditWindowMeta(tx *sql.Tx, sinceISO string) (map[int64]scopeAuditTxMeta, error) { + rows, err := tx.Query(scopeAuditWindowMetaQuery, sinceISO) + if err != nil { + return nil, fmt.Errorf("scope audit window meta: %w", err) + } + defer rows.Close() + + meta := map[int64]scopeAuditTxMeta{} + for rows.Next() { + var id int64 + var m scopeAuditTxMeta + if err := rows.Scan(&id, &m.scopeName, &m.firstSeen); err != nil { + return nil, fmt.Errorf("scope audit window meta scan: %w", err) + } + meta[id] = m + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("scope audit window meta rows: %w", err) + } + return meta, nil +} + +// scopeAuditSeenKey identifies one (target, transmission) pair for the +// de-duplication below. A struct key rather than the string it used to be +// built into: the hop scan reaches millions of rows on a 7d window, and every +// candidate hop was allocating a fresh "|" string to ask a +// question that a comparable struct answers without allocating. + +// scopeAuditSeenKey identifies one (target, transmission) pair for the +// de-duplication below. A struct key rather than the string it used to be +// built into: the hop scan reaches millions of rows on a 7d window, and every +// candidate hop was allocating a fresh "|" string to ask a +// question that a comparable struct answers without allocating. +type scopeAuditSeenKey struct { + target string + txID int64 +} + +// ScopeAuditForwarding runs scopeAuditForwarderScanQuery once for the whole +// window and attributes every forwarder hop it finds to targets, by the same +// truncated-hash prefix match ScopeConformance uses for a single pubkey. +// +// A hop is attributed only when its prefix matches EXACTLY ONE declared +// target. This endpoint exists to find a repeater that declares a region and +// is not actually forwarding it — crediting a hop to every target sharing +// its prefix would let a colliding neighbour's traffic silently paper over a +// real gap, and crediting nobody (the alternative of dropping the hop +// entirely) would invent failures for targets that simply share a collision- +// prone prefix. Instead, an ambiguous hop is credited to NEITHER candidate, +// and every candidate's ambiguousHops counter is incremented instead, so the +// row can say "this notObserved might just be a prefix collision" rather +// than presenting it as a confirmed finding. See scopeAuditTargetAgg's +// ambiguousHops field and ScopeAuditRow.AmbiguousHops. +// +// Each (target, transmission) pair is counted at most once even if seen via +// multiple observations OR via several hops of one path (a routing loop, or two +// hops colliding on the same truncated prefix), for both the attributed and the +// ambiguous count — the same de-duplication scopeConformanceQuery gets for free +// from EXISTS, done explicitly here since this scan is not correlated per +// target. Since the scan reads every hop rather than only path[last], this is +// the only thing keeping one transmission from counting several times for the +// same target; TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins +// it. + +// ScopeAuditForwarding runs scopeAuditForwarderScanQuery once for the whole +// window and attributes every forwarder hop it finds to targets, by the same +// truncated-hash prefix match ScopeConformance uses for a single pubkey. +// +// A hop is attributed only when its prefix matches EXACTLY ONE declared +// target. This endpoint exists to find a repeater that declares a region and +// is not actually forwarding it — crediting a hop to every target sharing +// its prefix would let a colliding neighbour's traffic silently paper over a +// real gap, and crediting nobody (the alternative of dropping the hop +// entirely) would invent failures for targets that simply share a collision- +// prone prefix. Instead, an ambiguous hop is credited to NEITHER candidate, +// and every candidate's ambiguousHops counter is incremented instead, so the +// row can say "this notObserved might just be a prefix collision" rather +// than presenting it as a confirmed finding. See scopeAuditTargetAgg's +// ambiguousHops field and ScopeAuditRow.AmbiguousHops. +// +// Each (target, transmission) pair is counted at most once even if seen via +// multiple observations OR via several hops of one path (a routing loop, or two +// hops colliding on the same truncated prefix), for both the attributed and the +// ambiguous count — the same de-duplication scopeConformanceQuery gets for free +// from EXISTS, done explicitly here since this scan is not correlated per +// target. Since the scan reads every hop rather than only path[last], this is +// the only thing keeping one transmission from counting several times for the +// same target; TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins +// it. func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (map[string]*scopeAuditTargetAgg, error) { byLen := scopeAuditPrefixIndex(targets) result := make(map[string]*scopeAuditTargetAgg, len(targets)) - rows, err := s.db.conn.Query(scopeAuditForwarderScanQuery, sinceISO) + // One read transaction for both queries. The hop scan and the per- + // transmission metadata are two passes over the same window, and a + // transmission arriving between them would otherwise appear in the hop scan + // with no metadata to attribute it by — rare, but the fix is a shared + // snapshot rather than a rule about what to do with the leftovers. + tx, err := s.db.conn.Begin() + if err != nil { + return nil, fmt.Errorf("scope audit forwarder scan begin: %w", err) + } + defer tx.Rollback() + + meta, err := scopeAuditWindowMeta(tx, sinceISO) + if err != nil { + return nil, err + } + + rows, err := tx.Query(scopeAuditForwarderScanQuery, sinceISO) if err != nil { return nil, fmt.Errorf("scope audit forwarder scan: %w", err) } @@ -381,21 +556,43 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m return agg } - seen := make(map[string]bool) // "|" already counted (attributed or ambiguous) + seen := make(map[scopeAuditSeenKey]bool) // (target, txID) already counted (attributed or ambiguous) + + // hopBuf lower-cases the hop in place instead of through strings.ToLower. + // 80% of the hops in this database are stored uppercase (1,026,814 of + // 1,284,897 in a 24h window, measured 2026-09-07) because + // packetpath.DecodePathFromRawHex writes them that way, and the great + // majority of them match no declared target at all — so the allocation + // ToLower makes is paid millions of times to answer "no". byLen's keys are + // lowercase, and a map index expression on string(bytes) does not allocate. + var hopBuf [64]byte for rows.Next() { var txID int64 - var hop string - var scopeName sql.NullString - var firstSeen string - if err := rows.Scan(&txID, &hop, &scopeName, &firstSeen); err != nil { + var hopRaw sql.RawBytes + if err := rows.Scan(&txID, &hopRaw); err != nil { return nil, fmt.Errorf("scope audit forwarder scan scan: %w", err) } - hop = strings.ToLower(hop) - candidates := byLen[len(hop)][hop] + n := len(hopRaw) + if n > len(hopBuf) { + // Longer than a full pubkey: cannot be any target's prefix. The + // SQL floor guards the short end, this guards the long one. + continue + } + for i := 0; i < n; i++ { + c := hopRaw[i] + if 'A' <= c && c <= 'Z' { + c += 'a' - 'A' + } + hopBuf[i] = c + } + candidates := byLen[n][string(hopBuf[:n])] + if len(candidates) == 0 { + continue + } if len(candidates) > 1 { for _, target := range candidates { - key := target + "|" + strconv.FormatInt(txID, 10) + key := scopeAuditSeenKey{target: target, txID: txID} if seen[key] { continue } @@ -404,8 +601,16 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m } continue } + txMeta, ok := meta[txID] + if !ok { + // Impossible while both queries share one snapshot and one WHERE + // clause; treated as "nothing to attribute" rather than silently + // counted as unscoped, which is what a zero-valued meta would do. + continue + } + scopeName, firstSeen := txMeta.scopeName, txMeta.firstSeen for _, target := range candidates { - key := target + "|" + strconv.FormatInt(txID, 10) + key := scopeAuditSeenKey{target: target, txID: txID} if seen[key] { continue } @@ -685,8 +890,6 @@ func nodeScopesWindowLookback(window string) (time.Duration, bool) { // pass — see scopes.go's AllCurrentDeclaredRegions and ScopeAuditForwarding // for why that stays a single scan rather than one query per repeater. func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { - const scopeAuditTTL = 30 * time.Second - window := r.URL.Query().Get("window") if window == "" { window = "24h" @@ -697,34 +900,103 @@ func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { return } - s.scopeAuditMu.Lock() - if s.scopeAuditCache != nil { - if cached, ok := s.scopeAuditCache[window]; ok && time.Since(s.scopeAuditCachedAt[window]) < scopeAuditTTL { - s.scopeAuditMu.Unlock() - writeJSON(w, cached) - return - } + sinceISO := time.Now().Add(-lookback).UTC().Format(time.RFC3339) + + if cached, ok := s.scopeAuditCached(window); ok { + writeJSON(w, cached) + return } - s.scopeAuditMu.Unlock() - declared, err := s.db.AllCurrentDeclaredRegions() + // singleflight: the compute below runs outside the cache mutex, so without + // this every request arriving on a cold window ran its own full scan + // concurrently. Reading every hop makes that scan seconds of work over + // millions of rows, which is the shape that turns a thundering herd from + // wasteful into expensive. Same treatment /api/observers and + // /api/nodes/{pubkey}/reach already have. + v, err, _ := s.scopeAuditSF.Do(window, func() (interface{}, error) { + // Waiters that arrive while a scan is in flight are served by that + // scan's result; this second look is for the caller that acquires the + // group right after a winner stored one. + if cached, ok := s.scopeAuditCached(window); ok { + return cached, nil + } + resp, cErr := s.computeScopeAudit(window, sinceISO) + if cErr != nil { + return nil, cErr + } + s.scopeAuditStore(window, resp) + return resp, nil + }) if err != nil { writeError(w, 500, err.Error()) return } + writeJSON(w, v.(*ScopeAuditResponse)) +} + +// scopeAuditTTLFor is how long one window's computed audit stays fresh. +// +// 7d is not 30s because it does not cost what the others cost. Measured on a +// live-shaped database with 206 declared repeaters and 965k transmissions: +// 16.7s cold for 7d against 4.0s for 24h and 0.15s for 1h, with the 7d scan +// reading 3,470,188 hop rows. At a 30s TTL a single reader with that window +// open keeps the instance recomputing more than half the time, for an +// aggregate that moves at the pace of a week of traffic. Five minutes of +// staleness on a seven-day window is not a fact the reader can act on +// differently. +func scopeAuditTTLFor(window string) time.Duration { + if window == "7d" { + return 5 * time.Minute + } + return 30 * time.Second +} + +// scopeAuditCached returns the cached response for a window while it is within +// that window's TTL. +func (s *Server) scopeAuditCached(window string) (*ScopeAuditResponse, bool) { + s.scopeAuditMu.Lock() + defer s.scopeAuditMu.Unlock() + if s.scopeAuditCache == nil { + return nil, false + } + cached, ok := s.scopeAuditCache[window] + if !ok || time.Since(s.scopeAuditCachedAt[window]) >= scopeAuditTTLFor(window) { + return nil, false + } + return cached, true +} + +// scopeAuditStore publishes a freshly computed response for a window. +func (s *Server) scopeAuditStore(window string, resp *ScopeAuditResponse) { + s.scopeAuditMu.Lock() + defer s.scopeAuditMu.Unlock() + if s.scopeAuditCache == nil { + s.scopeAuditCache = make(map[string]*ScopeAuditResponse) + s.scopeAuditCachedAt = make(map[string]time.Time) + } + s.scopeAuditCache[window] = resp + s.scopeAuditCachedAt[window] = time.Now() +} + +// computeScopeAudit builds one window's audit response: the declared lists and +// the forwarding evidence attributed to them. Split out of the handler so the +// cache and its singleflight wrap a plain function instead of a request. +func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse, error) { + declared, err := s.db.AllCurrentDeclaredRegions() + if err != nil { + return nil, err + } targets := make([]string, 0, len(declared)) for _, d := range declared { targets = append(targets, strings.ToLower(d.Target)) } - sinceISO := time.Now().Add(-lookback).UTC().Format(time.RFC3339) forwarding := map[string]*scopeAuditTargetAgg{} if s.store != nil { forwarding, err = s.store.ScopeAuditForwarding(sinceISO, targets) if err != nil { - writeError(w, 500, err.Error()) - return + return nil, err } } @@ -838,14 +1110,5 @@ func (s *Server) handleScopeAudit(w http.ResponseWriter, r *http.Request) { return an < bn }) - s.scopeAuditMu.Lock() - if s.scopeAuditCache == nil { - s.scopeAuditCache = make(map[string]*ScopeAuditResponse) - s.scopeAuditCachedAt = make(map[string]time.Time) - } - s.scopeAuditCache[window] = resp - s.scopeAuditCachedAt[window] = time.Now() - s.scopeAuditMu.Unlock() - - writeJSON(w, resp) + return resp, nil } diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 00ecb5e5f..eaf283cbc 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -853,3 +853,234 @@ func TestDeclaredRegionsMergeWithNoSourcesIsEmptyNotAnError(t *testing.T) { t.Errorf("rows = %+v, want empty", rows) } } + +// --- Every-hop forwarder attribution and the scan that pays for it --- +// +// These cover the change from crediting path[last] to crediting every hop of a +// flood-family route, and the two-column scan plus per-window TTL that keeps +// the wider scan affordable. + +// seedTransmissionPathAt seeds one transmission whose single observation +// carries a MULTI-hop path. A one-hop seed cannot tell the two reasons a node +// gets attributed apart — it is simultaneously path[0] and path[last] — so the +// mid-path cases below need a path with something after the target on it. +// +// Hops are upper-cased for the same reason seedTransmissionRoute does it: the +// decoder writes them that way (packetpath.DecodePathFromRawHex), and the join +// has to cope with that rather than with a lowercase convenience fiction. +func seedTransmissionPathAt(t *testing.T, s *PacketStore, hops []string, seed scopeSeed, routeType int, firstSeen string) { + t.Helper() + scopeSeedCounter++ + hash := fmt.Sprintf("scopehash%d", scopeSeedCounter) + + res, err := s.db.conn.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) + VALUES ('AA', ?, ?, ?, 1, ?, '00', ?)`, + hash, firstSeen, routeType, seed.code1, seed.scopeName, + ) + if err != nil { + t.Fatalf("seed transmission: %v", err) + } + txID, err := res.LastInsertId() + if err != nil { + t.Fatalf("seed transmission id: %v", err) + } + + quoted := make([]string, len(hops)) + for i, h := range hops { + quoted[i] = `"` + strings.ToUpper(h) + `"` + } + pathJSON := "[" + strings.Join(quoted, ",") + "]" + if _, err := s.db.conn.Exec( + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, ?)`, + txID, pathJSON, time.Now().Unix(), + ); err != nil { + t.Fatalf("seed observation: %v", err) + } +} + +// seedTransmission seeds a FLOOD packet (route_type=1) — path[last] is the +// actual transmitter, so forwarder is attributable. + +// TestScopeAuditForwardingAttributesMidPathHop is the fleet-wide half of the +// mid-path attribution fix. The audit runs a different query from +// ScopeConformance — one full-window scan instead of one EXISTS per pubkey — so +// the two share the rule but not the code, and both need pinning. +// +// This is the case behind the audit's 65% blind spot: a declared target that +// forwards steadily but is never the hop an observer hears directly had every +// region it declares reported as notObserved. +func TestScopeAuditForwardingAttributesMidPathHop(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{testFullPubkeyA[:4], "AAAA", "BBBB"}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil || agg.scopes["be"] == nil || agg.scopes["be"].Packets != 1 { + t.Fatalf("want the mid-path hop attributed to its sole matching target, got %+v", got) + } + if agg.ambiguousHops != 0 { + t.Errorf("ambiguousHops = %d, want 0 — one target matches this hop", agg.ambiguousHops) + } +} + +// TestScopeAuditForwardingIgnoresDirectRoutes pins the route-type filter on the +// audit's own query. With the last-hop restriction gone it is the only guard +// against crediting a DIRECT route's remaining path plan as forwarding — and a +// DIRECT packet's hops are frequently the declared targets this audit judges. + +// TestScopeAuditForwardingIgnoresDirectRoutes pins the route-type filter on the +// audit's own query. With the last-hop restriction gone it is the only guard +// against crediting a DIRECT route's remaining path plan as forwarding — and a +// DIRECT packet's hops are frequently the declared targets this audit judges. +func TestScopeAuditForwardingIgnoresDirectRoutes(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{"AAAA", testFullPubkeyA[:4], "BBBB"}, scopeMatched("#be"), RouteDirect, recent) + seedTransmissionPathAt(t, s, []string{"AAAA", "BBBB", testFullPubkeyA[:4]}, scopeMatched("#be"), RouteTransportDirect, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + if agg := got[testFullPubkeyA]; agg != nil && (len(agg.scopes) != 0 || agg.unscopedPackets != 0 || agg.ambiguousHops != 0) { + t.Errorf("agg = %+v, want no attribution from DIRECT routes", agg) + } +} + +// TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins that the +// existing "|" de-duplication also absorbs the same target +// matching several hops of ONE path — which could not happen while only +// path[last] was read, and now can (a routing loop, or two hops colliding on the +// same truncated prefix). Without it a looping packet would inflate a target's +// packet count and quietly make a quiet region look busy. + +// TestScopeAuditForwardingCountsOneTransmissionOncePerTarget pins that the +// existing "|" de-duplication also absorbs the same target +// matching several hops of ONE path — which could not happen while only +// path[last] was read, and now can (a routing loop, or two hops colliding on the +// same truncated prefix). Without it a looping packet would inflate a target's +// packet count and quietly make a quiet region look busy. +func TestScopeAuditForwardingCountsOneTransmissionOncePerTarget(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{testFullPubkeyA[:4], "AAAA", testFullPubkeyA[:4]}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil || agg.scopes["be"] == nil { + t.Fatalf("want #be attributed, got %+v", got) + } + if agg.scopes["be"].Packets != 1 { + t.Errorf("Packets = %d, want 1 — one transmission, matched on two of its hops", agg.scopes["be"].Packets) + } +} + +// TestScopeAuditForwardingAttributesLongerHopByItsOwnLength pins the +// length-indexed half of scopeAuditPrefixIndex, which every other test in this +// file leaves untested: they all seed 4-char hops, so a lookup that ignored hop +// length entirely would still pass them. +// +// pkOther shares the first 4 hex chars with testFullPubkeyA and diverges after +// that, so an 8-char hop has exactly one candidate while a 4-char hop would +// have two. Attribution must therefore key on the hop's OWN length: at 8 chars +// this is an unambiguous attribution, not an ambiguousHops row. + +// TestScopeAuditForwardingAttributesLongerHopByItsOwnLength pins the +// length-indexed half of scopeAuditPrefixIndex, which every other test in this +// file leaves untested: they all seed 4-char hops, so a lookup that ignored hop +// length entirely would still pass them. +// +// pkOther shares the first 4 hex chars with testFullPubkeyA and diverges after +// that, so an 8-char hop has exactly one candidate while a 4-char hop would +// have two. Attribution must therefore key on the hop's OWN length: at 8 chars +// this is an unambiguous attribution, not an ambiguousHops row. +func TestScopeAuditForwardingAttributesLongerHopByItsOwnLength(t *testing.T) { + s := newScopeTestStore(t) + pkOther := testFullPubkeyA[:4] + strings.Repeat("33", 30) + hop := testFullPubkeyA[:8] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{hop, "AAAA"}, scopeMatched("#be"), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA, pkOther}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil || agg.scopes["be"] == nil || agg.scopes["be"].Packets != 1 { + t.Fatalf("want the 8-char hop attributed to its sole matching target, got %+v", got) + } + if agg.ambiguousHops != 0 { + t.Errorf("ambiguousHops = %d, want 0 — the two targets diverge before hop length 8", agg.ambiguousHops) + } + if other := got[pkOther]; other != nil && (len(other.scopes) != 0 || other.ambiguousHops != 0) { + t.Errorf("pkOther = %+v, want no attribution and no ambiguity — the hop is not its prefix", other) + } +} + +// TestScopeAuditForwardingCountsUnmatchedPackets: a transport-scoped packet +// whose code1 matched no configured region key is stored with scope_name = "" +// (scopeNameForDB's "transport-scoped but unnameable" state). It is not a +// named scope, so it must not enter agg.scopes, and it is not an unscoped +// plain flood either, so it must not enter unscopedPackets. It is its own +// fact: this instance saw the target forward traffic it holds no key for. +// +// Without this counter the audit reports the declared region as "not +// observed", which reads as a finding about the repeater when it is really a +// gap in this instance's own hashRegions. + +// TestScopeAuditTTLForSevenDayWindow pins the per-window TTL. The 7d window +// costs a different order of magnitude than the others (16.7s against 4.0s and +// 0.15s, measured on the live-shaped staging database on 2026-09-07), so it is +// deliberately not on the 30s the other two share. A future edit that collapses +// this back to one constant should have to delete a test that says why. +func TestScopeAuditTTLForSevenDayWindow(t *testing.T) { + if got := scopeAuditTTLFor("7d"); got != 5*time.Minute { + t.Errorf("scopeAuditTTLFor(7d) = %s, want 5m", got) + } + for _, w := range []string{"1h", "24h", ""} { + if got := scopeAuditTTLFor(w); got != 30*time.Second { + t.Errorf("scopeAuditTTLFor(%q) = %s, want 30s", w, got) + } + } +} + +// TestHandleScopeAuditServesSecondRequestFromCache pins the cache path itself, +// which the singleflight rewrite moved out of the handler and into +// scopeAuditCached/scopeAuditStore. A declared row inserted between two +// requests inside the TTL must NOT appear in the second response: if it does, +// the response was recomputed and the cache is not being consulted. + +// TestHandleScopeAuditServesSecondRequestFromCache pins the cache path itself, +// which the singleflight rewrite moved out of the handler and into +// scopeAuditCached/scopeAuditStore. A declared row inserted between two +// requests inside the TTL must NOT appear in the second response: if it does, +// the response was recomputed and the cache is not being consulted. +func TestHandleScopeAuditServesSecondRequestFromCache(t *testing.T) { + srv, router := setupScopeAuditServer(t) + now := time.Now().UTC().Format(time.RFC3339) + insertDeclared(t, srv, testFullPubkeyA, now, "be", 0) + + first := getScopeAudit(t, router, "") + if len(first.Repeaters) != 1 { + t.Fatalf("first call repeaters = %d, want 1", len(first.Repeaters)) + } + + insertDeclared(t, srv, testFullPubkeyB, now, "be", 0) + second := getScopeAudit(t, router, "") + if len(second.Repeaters) != 1 { + t.Errorf("second call repeaters = %d, want 1 — the row added after the first call proves the cache was bypassed", len(second.Repeaters)) + } +} + +// TestHandleScopeAuditNormalisesHashPrefix pins trap 1: transmissions.scope_name +// keeps the '#' (hashRegions config), regions_csv arrives from the firmware +// with it already stripped. Declared "be-van" and observed "#be-van" must be +// recognised as the same scope, not reported as both missing and undeclared. From 0605b1703a6133ecdcbaacf8d38ccd9e254cad63 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 9 Sep 2026 17:05:29 +0200 Subject: [PATCH 014/151] feat(scope-audit): count and surface the traffic this instance cannot name (#1987) Follow-up to #1986, and the second half of the same problem. `ScopeAuditForwarding` drops rows whose `scope_name` is the empty string with a bare `continue`. That empty string is the ingestor's "transport-scoped, but no configured region key matched `code1`" state (`scopeNameForDB`), so those packets name no region and can never satisfy a declared one. The consequence is on the page: **a repeater forwarding a region this instance holds no `hashRegions` key for is reported exactly like a repeater forwarding nothing at all.** The audit presents a gap in the reader's own configuration as a finding about someone else's hardware. This counts them per target, exposes the count as `observedUnmatchedPackets`, and renders it as a caveat chip beside the scope chips. ## Why it is not a rare edge Measured on a live instance before this landed: of 613 `notObserved` entries across 205 repeaters, **260 named a region that never appeared under any name in the whole 7-day window**. Two of them (`behss`, `fm-112`) were hash-verified as genuinely forwarded traffic the instance simply could not name: packet `0a065d41d51f1f77` decodes to `code1=9209`, which is exactly the code `#fm-112` derives over that packet's own payload. That instance had 16 region keys configured against 124 distinct region names its repeaters declare. A stock install has fewer. ## What the counter is not It is deliberately **not** folded into `unscopedPackets`. The two are opposites: | | meaning | what governs it | |---|---|---| | `unscopedPackets` | the packet carried no scope at all (`scope_name` SQL NULL) | the `*` wildcard | | `observedUnmatchedPackets` | the packet IS scoped, this instance holds no key for that region | nothing the repeater declares | For the same reason the new count never feeds `wildcardContradiction`, which counts only plain unscoped floods. `scopeNameForDB` in the ingestor is the source of truth for that three-state encoding, and the comments point there rather than restating it. It is also distinct from `ambiguousHops`, and the distinction is the point of the chip: that one is a pubkey-prefix collision between two repeaters and is nobody's fault, this one is a missing entry in the reader's own configuration and they can act on it. Saying which is which is what stops someone investigating an innocent repeater. ## Frontend The chip reuses the muted dashed treatment of `.sa-chip-ambiguous` on purpose: both are caveats on the row's finding rather than findings themselves, and neither may compete visually with the red/green scope chips beside them. It renders nothing for a non-numeric count. The value is server-supplied, and a truthiness check would put the literal string `NaN forwarded packets` on the page if that ever stopped holding. ## Docs `docs/api-spec.md` had **no entry for `GET /api/scope-audit` at all**, so this adds one: query parameter, full response shape, and the notes a client needs (the three traps the per-node endpoint documents apply here identically, `*` is never a scope, and "never asked" is not "declared nothing"). The new field is documented there rather than in isolation. ## Tests - the counter on a last-hop and on a mid-path hop - an unmatched packet enters neither `agg.scopes` nor `unscopedPackets`, which is the confusion this field exists to prevent - the field on the API row - six frontend cases: zero renders nothing, a missing field renders nothing (older server), the chip carries its count and class, singular and plural are both grammatical, the title names the cause and the fix, and a non-numeric count renders nothing rather than `NaN` `cd cmd/server && go test ./...` passes, frontend 712 assertions pass, `go vet` and `gofmt -l` clean. Rule 0: the counter is one increment on a branch that already existed as a `continue`, inside a loop this PR does not change. No new query, no new pass over the data. --- cmd/server/scope_audit.go | 68 +++++++++++++---- cmd/server/scope_audit_test.go | 89 ++++++++++++++++++++++ docs/api-spec.md | 132 +++++++++++++++++++++++++++++++++ public/scope-audit.css | 5 ++ public/scope-audit.js | 29 +++++++- test-frontend-helpers.js | 57 ++++++++++++++ 6 files changed, 363 insertions(+), 17 deletions(-) diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index 32249c8fb..02f3c89dc 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -283,6 +283,20 @@ func normScope(s string) string { type scopeAuditTargetAgg struct { scopes map[string]*ScopeObservation unscopedPackets int64 + // unmatchedPackets counts packets this target was observed forwarding + // that carried a transport scope no configured region key matched + // (transmissions.scope_name = the empty string). Deliberately NOT folded + // into unscopedPackets: those two are opposites. Unscoped means the packet + // carried no scope at all (scope_name SQL NULL) and is what '*' governs; + // unmatched means it IS scoped and this instance simply holds no key for + // that region, so '*' says nothing about it. See scopeNameForDB in the + // ingestor for the encoding. + // + // A non-zero value is a caveat on this target's notObserved entries: any + // of them may be a region this instance cannot name rather than one the + // repeater is not forwarding. + unmatchedPackets int64 + // ambiguousHops counts forwarder-hop observations in the window whose // truncated hash prefix matched this target AND at least one other // declared target — see ScopeAuditForwarding's doc comment for why @@ -622,7 +636,14 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m continue } if scopeName.String == "" { - continue // unmatched — not part of the declared/observed comparison + // Unmatched: transport-scoped, but no configured region key + // matched code1. Still not part of the declared/observed + // comparison — it names no region, so it can never satisfy a + // declaration — but it is the evidence that a notObserved + // finding on this row may be a gap in this instance's + // hashRegions rather than in the repeater's forwarding. + agg.unmatchedPackets++ + continue } name := normScope(scopeName.String) so, ok := agg.scopes[name] @@ -741,6 +762,21 @@ type ScopeAuditRow struct { // UndeclaredObserved entry is unaffected by it (ambiguous hops are never // attributed to a scope at all). AmbiguousHops int64 `json:"ambiguousHops"` + + // ObservedUnmatchedPackets counts packets this repeater was observed + // forwarding whose transport scope matched no region key this instance + // holds. Like AmbiguousHops it is a caveat rather than a finding, but the + // two have different causes and different fixes: AmbiguousHops is a + // pubkey-prefix collision between two repeaters and nobody's fault, + // ObservedUnmatchedPackets is a missing entry in this instance's own + // hashRegions and the reader can act on it. A non-zero value means any + // NotObserved entry on this row may name a region this instance cannot + // name rather than one the repeater is not forwarding. + // + // It says nothing about DeclaredWildcard: unmatched traffic IS scoped, so + // it never feeds WildcardContradiction, which counts only plain unscoped + // floods. + ObservedUnmatchedPackets int64 `json:"observedUnmatchedPackets"` } // ScopeAuditResponse is the payload for GET /api/scope-audit. Only @@ -1057,26 +1093,28 @@ func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse } } - var unscopedPackets, ambiguousHops int64 + var unscopedPackets, ambiguousHops, unmatchedPackets int64 if agg != nil { unscopedPackets = agg.unscopedPackets ambiguousHops = agg.ambiguousHops + unmatchedPackets = agg.unmatchedPackets } resp.Repeaters = append(resp.Repeaters, ScopeAuditRow{ - PublicKey: pk, - Name: id.Name, - Role: id.Role, - DeclaredRegions: declaredNamed, - DeclaredWildcard: declaredWildcard, - ConfigState: scopeAuditConfigState(declaredNamed, declaredWildcard), - DeclaredAt: d.ObservedAt, - Truncated: d.Truncated, - NotObserved: notObserved, - UndeclaredObserved: undeclared, - ObservedUnscopedPackets: unscopedPackets, - WildcardContradiction: unscopedPackets > 0 && !declaredWildcard, - AmbiguousHops: ambiguousHops, + PublicKey: pk, + Name: id.Name, + Role: id.Role, + DeclaredRegions: declaredNamed, + DeclaredWildcard: declaredWildcard, + ConfigState: scopeAuditConfigState(declaredNamed, declaredWildcard), + DeclaredAt: d.ObservedAt, + Truncated: d.Truncated, + NotObserved: notObserved, + UndeclaredObserved: undeclared, + ObservedUnscopedPackets: unscopedPackets, + WildcardContradiction: unscopedPackets > 0 && !declaredWildcard, + AmbiguousHops: ambiguousHops, + ObservedUnmatchedPackets: unmatchedPackets, }) } diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index eaf283cbc..9bb17bc27 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -1084,3 +1084,92 @@ func TestHandleScopeAuditServesSecondRequestFromCache(t *testing.T) { // keeps the '#' (hashRegions config), regions_csv arrives from the firmware // with it already stripped. Declared "be-van" and observed "#be-van" must be // recognised as the same scope, not reported as both missing and undeclared. + +// TestScopeAuditForwardingCountsUnmatchedPackets: a transport-scoped packet +// whose code1 matched no configured region key is stored with scope_name = "" +// (scopeNameForDB's "transport-scoped but unnameable" state). It is not a +// named scope, so it must not enter agg.scopes, and it is not an unscoped +// plain flood either, so it must not enter unscopedPackets. It is its own +// fact: this instance saw the target forward traffic it holds no key for. +// +// Without this counter the audit reports the declared region as "not +// observed", which reads as a finding about the repeater when it is really a +// gap in this instance's own hashRegions. +func TestScopeAuditForwardingCountsUnmatchedPackets(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, s, hop, scopeUnmatched(), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil { + t.Fatalf("want an agg for the target, got none (result = %+v)", got) + } + if agg.unmatchedPackets != 1 { + t.Errorf("unmatchedPackets = %d, want 1", agg.unmatchedPackets) + } + if len(agg.scopes) != 0 { + t.Errorf("scopes = %+v, want empty — an unmatched packet names no region", agg.scopes) + } + if agg.unscopedPackets != 0 { + t.Errorf("unscopedPackets = %d, want 0 — unmatched is not the same as unscoped", agg.unscopedPackets) + } +} + +// TestScopeAuditForwardingCountsUnmatchedOnMidPathHop is the post-M0 case that +// carries almost all of this counter's real volume: before M0 only a last hop +// was attributed, so a repeater deep in a flood path contributed nothing at +// all. Now every hop counts, and the same de-duplication that protects the +// named-scope tally must protect this one — a target appearing twice in one +// path is still one packet, not two. +func TestScopeAuditForwardingCountsUnmatchedOnMidPathHop(t *testing.T) { + s := newScopeTestStore(t) + hop := testFullPubkeyA[:4] + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionPathAt(t, s, []string{"AAAA", hop, "BBBB", hop}, scopeUnmatched(), RouteFlood, recent) + + got, err := s.ScopeAuditForwarding("2026-01-01T00:00:00Z", []string{testFullPubkeyA}) + if err != nil { + t.Fatal(err) + } + agg := got[testFullPubkeyA] + if agg == nil { + t.Fatalf("want an agg for the mid-path target, got none (result = %+v)", got) + } + if agg.unmatchedPackets != 1 { + t.Errorf("unmatchedPackets = %d, want 1 — one transmission, matched on two of its hops", agg.unmatchedPackets) + } +} + +// TestHandleScopeAuditSurfacesUnmatchedPackets: a repeater declares "behss", +// and this instance sees it forward transport-scoped traffic it cannot name. +// The row must still list "behss" as notObserved — an unmatched packet names +// no region, so it cannot satisfy the declaration — but it must also carry +// observedUnmatchedPackets, so a client can say the finding might be a missing +// region key rather than a silent repeater. +func TestHandleScopeAuditSurfacesUnmatchedPackets(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "behss", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, srv.store, pk[:4], scopeUnmatched(), RouteFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if row.ObservedUnmatchedPackets != 1 { + t.Errorf("observedUnmatchedPackets = %d, want 1", row.ObservedUnmatchedPackets) + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "behss" { + t.Errorf("notObserved = %v, want [\"behss\"] — an unmatched packet names no region and cannot satisfy a declaration", row.NotObserved) + } + if row.WildcardContradiction { + t.Error("wildcardContradiction = true, want false — unmatched traffic is scoped, so it says nothing about '*'") + } +} diff --git a/docs/api-spec.md b/docs/api-spec.md index 658679825..59b4d5590 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -40,6 +40,7 @@ - [GET /api/analytics/hash-sizes](#get-apianalyticshash-sizes) - [GET /api/analytics/subpaths](#get-apianalyticssubpaths) - [GET /api/analytics/subpath-detail](#get-apianalyticssubpath-detail) +- [GET /api/scope-audit](#get-apiscope-audit) - [GET /api/scope-stats](#get-apiscope-stats) - [GET /api/resolve-hops](#get-apiresolve-hops) - [GET /api/traces/:hash](#get-apitraceshash) @@ -1570,6 +1571,137 @@ Detailed stats for a specific subpath. --- +## GET /api/scope-audit + +Network-wide declared-vs-observed region-scope comparison: for every repeater that has +ever successfully answered a declared-regions request, which of its declared regions have +no observed forwarding in the window, which scopes it forwards without declaring, and +whether it contradicts its own `'*'` wildcard. This is the whole-network answer to the +question `GET /api/nodes/:pubkey/scopes` answers one repeater at a time — see that +endpoint's notes for the full explanation of the three traps this comparison has to get +right (the `#`-prefix spelling difference, `'*'` not being a scope, and "never asked" +not being the same as "declared nothing"), which apply here identically. + +### Query Parameters + +| Param | Type | Default | Description | +|----------|--------|---------|---------------------------------| +| `window` | string | `24h` | Time window: `1h`, `24h`, `7d` | + +### Response `200` + +```jsonc +{ + "window": string, // echoed window ("1h", "24h", or "7d") + "since": string (ISO), // start of the observed-forwarding window + "repeaters": [ + { + "publicKey": string, + "name": string, // "" if the node row is gone (pruned/deleted) + "role": string, // "" if unknown + "declaredRegions": [string], // '*' excluded — see declaredWildcard + "declaredWildcard": boolean, // '*' present in the raw declared list + "configState": string, // "full" | "no-scopes" | "no-unscoped" | "no-flood" — see note below + "declaredAt": string (ISO),// age of the DECLARED answer, not bounded by window + "truncated": boolean, // declared list may have had entries silently dropped + + "notObserved": [string], // declared regions with zero matched-forwarding observed this window + "undeclaredObserved": [ + { "scope": string, "packets": number, "firstSeen": string (ISO), "lastSeen": string (ISO) } + ], + "observedUnscopedPackets": number, // plain-FLOOD packets forwarded this window + "wildcardContradiction": boolean, // observed unscoped forwarding but '*' not declared + "ambiguousHops": number, // forwarder hops this window that could not be attributed — see note below + "observedUnmatchedPackets": number // forwarded packets whose scope this instance holds no key for — see note below + } + ] +} +``` + +### Response `400` + +```json +{ "error": "window must be 1h, 24h, or 7d" } +``` + +**Notes:** +- Only repeaters with **at least one** declared-regions answer appear in `repeaters`. A + repeater that was never successfully asked is absent, not shown as a row that declares + nothing — those are different facts (see `GET /api/nodes/:pubkey/scopes`'s "never asked" + note). +- `window` bounds `notObserved` / `undeclaredObserved` / `observedUnscopedPackets` only — + `declaredAt` is always the latest declared reading regardless of age, exactly like + `declared.observedAt` on the per-node endpoint. **A "declared but not observed" result + is weak evidence at `window=1h` (a quiet region can simply have had no traffic) and much + stronger at `window=7d`; a client MUST show which window a result belongs to and must + not present a `1h` result as if it were `7d`.** Because `declaredAt` is not bounded by + `window`, the declared answer and the forwarding evidence are not guaranteed to be + temporally aligned — a repeater could have declared its regions well before, or even + after, the window that produced `notObserved` — and the server makes no attempt to + align them; the `declaredAt` column in the UI is the mitigation (so a stale declared + answer is visible to the reader), not a guarantee that the two sides describe the same + period. +- `ambiguousHops` counts forwarder-hop observations in this window whose truncated hash + prefix matched more than one declared target's pubkey — i.e. two or more repeaters that + declared a region list happen to share that prefix. Such a hop is attributed to **none** + of the matching targets (crediting all of them risks papering over a real gap in a + colliding neighbour's row; crediting none of them invents no failure) and this counter + is incremented on **every** matching target instead. A row with a non-zero + `ambiguousHops` carries weaker evidence than one with zero: any entry in that row's + `notObserved` could be explained by a prefix collision rather than a genuine absence of + forwarding, and a client should present it as a caveat rather than a confirmed finding. +- `observedUnmatchedPackets` counts packets this repeater was observed forwarding whose + transport scope matched no region key this instance has configured (`hashRegions`), so + the ingestor stored them with an empty `scope_name`. Those packets name no region and + therefore cannot satisfy a declared one, which means **a repeater forwarding a region + this instance cannot name is reported exactly like one forwarding nothing**. A non-zero + value is a caveat on this row's `notObserved`, in the same spirit as `ambiguousHops` but + with a different cause and a different fix: `ambiguousHops` is a pubkey-prefix collision + between two repeaters and nobody's fault, `observedUnmatchedPackets` is a missing entry + in this instance's own configuration and the operator can act on it. It is **not** + evidence for or against `declaredWildcard` — unmatched traffic is scoped, so it never + affects `wildcardContradiction`, which counts only plain unscoped floods. +- All scope names in `declaredRegions` / `notObserved` / `undeclaredObserved[].scope` are + already normalised (no leading `#`) — the server does the `#`/no-`#` reconciliation + described on the per-node endpoint so this response is directly comparable without a + client-side normalisation step. +- `'*'` is never present in `declaredRegions`, `notObserved`, or `undeclaredObserved` — see + `declaredWildcard` and `wildcardContradiction` for its dedicated (non-scope) treatment, + same rule as the per-node endpoint. +- `configState` is a derived reading of `declaredRegions`/`declaredWildcard` together — + the repeater's overall configuration shape, rather than a per-region detail: + - `"full"` — named regions **and** `'*'` declared: fully configured, forwards both its + declared regions and plain unscoped floods. + - `"no-scopes"` — `'*'` only, no named regions. + - `"no-unscoped"` — named regions declared, `'*'` absent: this repeater does **not** + forward plain unscoped floods. This reading is exact — `'*'` absent from the export + always means the wildcard denies flooding. + - `"no-flood"` — neither named regions nor `'*'`: the repeater answered, but nothing at + all is flood-allowed, not even plain unscoped traffic. The "no unscoped forwarding" + half of this is exact for the same reason as `"no-unscoped"`; the "no named regions" + half carries the same caveat as `"no-scopes"` below. + + **Caveat on `"no-scopes"` and `"no-flood"`:** the firmware exports the FLOOD-*allowed* + set (`region_map.exportNamesTo(..., REGION_DENY_FLOOD)` in + `examples/simple_repeater/MyMesh.cpp`), not "every region this repeater has configured". + A repeater with regions defined but every one of them marked deny-flood exports exactly + the same list as a repeater with no region tree at all — the two are indistinguishable + from this data alone. In practice `"no-scopes"` is almost always "no scopes configured", + but a client MUST NOT present it as proven absence of configuration; word it as "no + region is flood-allowed" rather than "no regions exist". +- `wildcardContradiction` is `true` when the repeater was observed forwarding unscoped + (plain-`FLOOD`) traffic this window but its declared list omits `'*'` — it declares it + will NOT forward those packets, and the traffic says otherwise. +- Rows are sorted with the interesting cases first: most `notObserved` entries first, then + `wildcardContradiction`, then most `undeclaredObserved` entries, then alphabetically by + name. A repeater in full agreement (no `notObserved`, no `wildcardContradiction`, no + `undeclaredObserved`) sorts to the bottom. +- Cached 30 seconds per window, mirroring `/api/scope-stats`. +- Blacklisted nodes and nodes matching an operator-configured hidden-name prefix are + excluded, same as other multi-node endpoints. + +--- + ## GET /api/scope-stats Scope-based packet statistics over a time window. Requires ingestor `scope_name_v1` migration to have run. diff --git a/public/scope-audit.css b/public/scope-audit.css index f5074e8e2..44f8a03d1 100644 --- a/public/scope-audit.css +++ b/public/scope-audit.css @@ -79,3 +79,8 @@ .sa-empty { line-height: 1.6; max-width: 70ch; } .sa-empty ul { margin: 8px 0; padding-left: 20px; } .sa-empty li { margin: 4px 0; } + +/* Same muted, dashed treatment as .sa-chip-ambiguous on purpose: both are + caveats on the row's finding, not findings themselves, and neither may + compete visually with the red/green scope chips beside them. */ +.sa-chip-unmatched { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px dashed var(--border); font-family: inherit; font-style: italic; } diff --git a/public/scope-audit.js b/public/scope-audit.js index 1ae419501..1ffa678c2 100644 --- a/public/scope-audit.js +++ b/public/scope-audit.js @@ -183,6 +183,31 @@ ' in this window matched more than one declared target\'s pubkey prefix and could not be attributed to any of them. Any “not observed” entry on this row may be explained by that prefix collision rather than a real gap.">possibly ambiguous'; } + // unmatchedCaveat flags rows where this instance saw the repeater forward + // transport-scoped traffic it holds no region key for. The ingestor stores + // those packets with an empty scope_name (see scopeNameForDB), so they name + // no region and can never satisfy a declared one — a repeater forwarding a + // region this instance cannot name looks exactly like one forwarding + // nothing. + // + // Distinct from ambiguousCaveat, and the distinction is the whole point: + // that one is a prefix collision between two repeaters and is nobody's + // fault, this one is a missing entry in this instance's own hashRegions and + // the reader can fix it. Saying so is what stops them investigating an + // innocent repeater. + // + // The count is server-supplied, so a non-numeric value renders nothing + // rather than the string "NaN forwarded packets". + function unmatchedCaveat(row) { + var n = row.observedUnmatchedPackets; + if (typeof n !== 'number' || !isFinite(n) || n <= 0) return ''; + var label = escapeHtml(n) + ' forwarded packet' + (n === 1 ? '' : 's'); + return ' ' + + label + ' unnameable'; + } + // statusScore ranks a row's Status column numerically for sorting — a // simple weighted count (notObserved dominates, matching the server's own // findings-first ranking) rather than the badge text, which the Status @@ -210,7 +235,7 @@ '' + nameHtml(row) + (row.role != null && row.role !== '' ? ' ' + escapeHtml(row.role) + '' : '') + '' + '' + issuesHtml + '' + '' + configStateHtml(row) + '' + - '' + mergedScopeChips(row) + (row.declaredWildcard ? ' *' : '') + ambiguousCaveat(row) + '' + + '' + mergedScopeChips(row) + (row.declaredWildcard ? ' *' : '') + ambiguousCaveat(row) + unmatchedCaveat(row) + '' + '' + ageHtml(row) + (row.truncated ? ' truncated' : '') + '' + ''; } @@ -360,7 +385,7 @@ // Exposed so the helper tests can assert what the Scopes column RENDERS // rather than grepping this file, the same reason map.js exposes its label // builder (#1356/#1933). - window.__meshcoreScopeAuditInternals = { mergedScopeChips: mergedScopeChips, emptyStateHtml: emptyStateHtml, sourcesLineHtml: sourcesLineHtml }; + window.__meshcoreScopeAuditInternals = { mergedScopeChips: mergedScopeChips, emptyStateHtml: emptyStateHtml, sourcesLineHtml: sourcesLineHtml, unmatchedCaveat: unmatchedCaveat }; } registerPage('scope-audit', { init: init, destroy: destroy }); diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index 65884c7b9..c961420aa 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -7229,6 +7229,63 @@ console.log('\n=== scope-audit.js: mergedScopeChips ==='); }); } +// ===== scope-audit.js: unmatchedCaveat ===== +// A declared region this instance holds no hashRegions key for can never turn +// green, however much traffic the repeater forwards: the ingestor stores such +// packets with an empty scope_name, so there is no name for the audit to match +// the declaration against. On live data that explains a large share of all +// notObserved entries, so the column must be able to say so instead of +// presenting every grey chip as a confirmed gap. +console.log('\n=== scope-audit.js: unmatchedCaveat ==='); +{ + const ctx = makeSandbox(); + ctx.registerPage = () => {}; + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/scope-audit.js'); + const caveat = ctx.__meshcoreScopeAuditInternals.unmatchedCaveat; + + test('zero unmatched packets renders nothing at all', () => { + assert.strictEqual(caveat({ observedUnmatchedPackets: 0 }), ''); + }); + + test('a missing field renders nothing (older server, field absent)', () => { + assert.strictEqual(caveat({}), ''); + }); + + test('a non-zero count renders a chip carrying the number', () => { + const h = caveat({ observedUnmatchedPackets: 148 }); + assert.ok(h.includes('sa-chip-unmatched'), 'should carry its own class'); + assert.ok(h.includes('148'), 'should state the count, not just that there is one'); + }); + + test('singular and plural are both grammatical', () => { + assert.ok(caveat({ observedUnmatchedPackets: 1 }).includes('1 forwarded packet ')); + assert.ok(caveat({ observedUnmatchedPackets: 2 }).includes('2 forwarded packets ')); + }); + + test('the title names the cause, not just the symptom', () => { + // The operator fix is a hashRegions edit; a caveat that does not say so + // sends them looking at the repeater instead of at their own config. + const h = caveat({ observedUnmatchedPackets: 5 }); + assert.ok(h.includes('hashRegions'), 'must name the config key that fixes it'); + }); + + test('a non-numeric count renders nothing rather than NaN', () => { + // observedUnmatchedPackets is server-supplied. A truthiness check accepts + // a string, and the chip then reads "NaN forwarded packets". + assert.strictEqual(caveat({ observedUnmatchedPackets: '12' }), ''); + assert.strictEqual(caveat({ observedUnmatchedPackets: NaN }), ''); + }); + + test('the count is not injected raw into markup', () => { + // observedUnmatchedPackets is server-supplied. It is a number in every + // sane response, but the chip must not become an injection point if that + // ever stops holding. + const h = caveat({ observedUnmatchedPackets: '1">' }); + assert.ok(!h.includes(' Date: Wed, 9 Sep 2026 17:18:54 +0200 Subject: [PATCH 015/151] fix(ingestor): name a region scope deterministically, or not at all (#1988) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `matchScope` returns the first configured region whose derived code equals the packet's `code1` and stops there. Go randomises map iteration order per `range`, so when two configured regions collide on a payload, the stored region name depends on which key the runtime happened to visit first. **The same packet can be named differently on two runs of the same binary**, and neither answer is evidence of anything. ## How often this actually happens `code1` is two bytes, so any two configured regions collide on a given payload with probability 1/65536. That is a curiosity at 5 configured regions and routine at 150. Measured on a live instance carrying 159 configured regions, over 41.7 hours and 147,535 transport-scoped packets: **374 collisions, 0.253% of decisions.** They concentrate on four key pairs rather than scattering, because `code1` is an HMAC over the payload: a payload that collides collides every time it is seen, and a flooded packet is seen by many observers. The existing comment sizes the function for "≤ 50 regions". A live BE/NL instance declares 126 distinct region names across its repeaters, so operators are already past that. ## The rule `matchingRegions` returns every match; `matchScope` applies one rule: - exactly one match names the packet - several matches name nothing The candidates are equally sourced, there is no principled winner between them, and storing a wrong region name is worse than storing none. `""` is already the ingestor's "transport-scoped but unnameable" state (`scopeNameForDB`), so an ambiguous packet lands in a state the rest of the system already understands rather than in a new one. Nothing downstream needs to learn a new value. The collision is logged, because it is otherwise invisible: such a packet is stored exactly like one whose region this instance holds no key for. An operator watching an unnameable count grow deserves to see which of their own configured regions are colliding, since the fix is theirs to make. ## Rule 0 Cost is unchanged: the same single pass over the same keys, it just no longer stops early. The early exit was worth nothing on the common path, where zero or one key matches and the loop runs to the end either way. Worst case is unchanged at one HMAC per key per transport-scoped packet. There is no indexable shortcut to reach for. `code1` is an HMAC over the packet payload, so nothing is payload-independent to index on, and the old comment suggesting a "pre-indexed lookup table" is removed rather than left as a false lead for the next reader. ## Tests Three, and the fixture matters: - an unambiguous packet still gets its region name - a genuinely colliding payload stores the unmatched state instead of a coin flip - the ambiguous case run 50 times, because a first-match matcher passes a single iteration roughly half the time The collision is **found by searching payloads** (~65k tries, fractions of a second) rather than asserting on a hand-picked `code1`. The case only exists when the matcher genuinely finds two names for one packet, and a fabricated code would only prove the test agrees with itself. `cd cmd/ingestor && go test ./...` passes apart from `TestWriteStatsAtomic_SymlinkAtDestIsReplaced`, which needs `SeCreateSymbolicLinkPrivilege` and fails on Windows on master too. `go vet` and `gofmt -l` clean. --- cmd/ingestor/main.go | 49 +++++++++++++++++++--- cmd/ingestor/main_test.go | 88 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 5 deletions(-) diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index ec4ef406c..d3ed0dda5 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -1635,12 +1635,25 @@ func loadRegionKeys(cfg *Config) map[string][]byte { return keys } -// matchScope performs one HMAC-SHA256 per configured region. Expected -// len(regionKeys) ≤ 50; beyond that, consider a pre-indexed lookup table. -func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { +// matchingRegions returns every configured region whose derived code equals +// the packet's code1, rather than the first one found. +// +// The distinction matters because code1 is two bytes: two configured regions +// collide on a given payload with probability 1/65536, and at 159 keys on a +// live instance that is roughly 0.25% of transport-scoped packets, hundreds a +// week rather than a curiosity. Returning the first match made the stored +// region name depend on Go's randomised map iteration order, so the same +// packet could be named differently on two runs and neither answer was +// evidence of anything. +// +// The cost is unchanged: this is the same single pass over the same keys, it +// just does not stop early. There is no indexable shortcut, because code1 is +// an HMAC over the payload and nothing here is payload-independent. +func matchingRegions(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) []string { if code1 == "0000" || len(regionKeys) == 0 || len(payloadRaw) == 0 { - return "" + return nil } + var matched []string for name, key := range regionKeys { mac := hmac.New(sha256.New, key) mac.Write([]byte{payloadType}) @@ -1654,9 +1667,35 @@ func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byt } codeBytes := [2]byte{byte(code & 0xFF), byte(code >> 8)} if strings.ToUpper(hex.EncodeToString(codeBytes[:])) == code1 { - return name + matched = append(matched, name) } } + return matched +} + +// matchScope names one packet's region scope, or returns "" when it cannot be +// named with confidence. +// +// Exactly one match names the packet. Several matches name nothing: the two +// candidates are equally sourced, there is no principled winner, and storing +// a wrong region name is worse than storing none. "" is already the ingestor's +// "transport-scoped but unnameable" state (scopeNameForDB), so an ambiguous +// packet lands in a state the rest of the system already understands rather +// than in a new one. +// +// The collision is logged because it is otherwise invisible: the packet is +// stored exactly like one whose region this instance holds no key for, and an +// operator looking at a growing unnameable count deserves to see which of +// their configured regions are colliding. +func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { + matched := matchingRegions(regionKeys, payloadType, payloadRaw, code1) + switch len(matched) { + case 0: + return "" + case 1: + return matched[0] + } + log.Printf("[regions] ambiguous collision between %v; storing unmatched", matched) return "" } diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index 9d241f46b..cbc6293c1 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/hmac" + "crypto/sha256" "database/sql" "encoding/hex" "encoding/json" @@ -10,6 +12,7 @@ import ( "os" "path/filepath" "runtime" + "strings" "testing" "time" @@ -1186,3 +1189,88 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { t.Errorf("default_scope after matched-scope advert = %q (valid=%v), want #de", got.String, got.Valid) } } + +// codeForRegion derives the on-wire code1 a sender in this region would emit +// for this payload: the forward direction of what matchScope inverts. +func codeForRegion(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +// findRegionCollision searches for a payload whose code1 is identical under two +// region names. code1 is two bytes, so one turns up after ~65k tries and the +// search costs a fraction of a second. +// +// A hand-built fixture cannot stand in here: the whole point of the case below +// is what happens when the matcher genuinely finds two names for one packet, +// and a fabricated code1 would only prove the test agrees with itself. +func findRegionCollision(t *testing.T, nameA, nameB string, payloadType byte) ([]byte, string) { + t.Helper() + payload := make([]byte, 4) + for i := 0; i < 1<<22; i++ { + payload[0], payload[1] = byte(i), byte(i>>8) + payload[2], payload[3] = byte(i>>16), byte(i>>24) + if a, b := codeForRegion(nameA, payloadType, payload), codeForRegion(nameB, payloadType, payload); a == b { + return append([]byte(nil), payload...), a + } + } + t.Fatalf("no code1 collision between %s and %s in 2^22 payloads", nameA, nameB) + return nil, "" +} + +// TestMatchScopeNamesAnUnambiguousPacket is the ordinary case: one configured +// region derives the packet's code1, so the packet carries that region's name. +func TestMatchScopeNamesAnUnambiguousPacket(t *testing.T) { + keys := loadRegionKeys(&Config{HashRegions: []string{"#be", "#nl"}}) + payload := []byte{0x01, 0x02, 0x03, 0x04} + code1 := codeForRegion("#be", 5, payload) + + if got := matchScope(keys, 5, payload, code1); got != "#be" { + t.Errorf("matchScope = %q, want %q", got, "#be") + } +} + +// TestMatchScopeStoresAmbiguousAsUnmatched pins the reason this changed. Two +// configured regions derive the same code1 for one payload; naming the packet +// after either is a coin flip, and before this the flip was Go's map iteration +// order, so the same packet could be stored under different regions on two +// runs of the same binary. +func TestMatchScopeStoresAmbiguousAsUnmatched(t *testing.T) { + payload, code1 := findRegionCollision(t, "#be", "#zz", 5) + keys := loadRegionKeys(&Config{HashRegions: []string{"#be", "#zz"}}) + + if n := len(matchingRegions(keys, 5, payload, code1)); n != 2 { + t.Fatalf("matchingRegions returned %d names, want 2 — the collision fixture is wrong", n) + } + if got := matchScope(keys, 5, payload, code1); got != "" { + t.Errorf("matchScope = %q, want the unmatched state: two equally-sourced candidates have no principled winner", got) + } +} + +// TestMatchScopeIsOrderIndependent runs the ambiguous case repeatedly. Map +// iteration order is randomised per range in Go, so a first-match matcher +// returns different names across iterations of this loop; the answer must not +// move. +func TestMatchScopeIsOrderIndependent(t *testing.T) { + payload, code1 := findRegionCollision(t, "#be", "#zz", 5) + keys := loadRegionKeys(&Config{HashRegions: []string{"#be", "#zz"}}) + + for i := 0; i < 50; i++ { + if got := matchScope(keys, 5, payload, code1); got != "" { + t.Fatalf("iteration %d: matchScope = %q, want a stable answer across map iteration orders", i, got) + } + } +} From 0e607c1d01b6f19688bc0e57e7f69448914c1bc1 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 9 Sep 2026 17:58:04 +0200 Subject: [PATCH 016/151] feat(ingestor): derive region keys from what nodes declare, opt-in (#1989) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #1988, which made an ambiguous match deterministic. This adds the second tier of keys that ambiguity rule was needed for. A transport-scoped packet can only be named by a region key this instance holds. `hashRegions` is a hand-maintained list, so every region a node forwards that nobody typed into the config is stored unmatched, and everything downstream reports that region as **absent** rather than as **unnameable**. The instance already knows the names, though: `nodes.configured_scope` holds what the observer `/neighbors` ingestion (#1865) confirmed each node is configured for. This derives keys from those names, on top of the explicit list rather than instead of it. **Default off.** An absent config block leaves behaviour byte-for-byte unchanged, asserted by tests rather than argued. ## Sources Two, mirroring what the server's `AllCurrentDeclaredRegions` already merges, so the derived tier sees exactly what the Scope Audit sees: | source | availability | |---|---| | `nodes.configured_scope` | always — the column is part of the schema, written by the `/neighbors` path | | `node_declared_regions` | optional, where a deployment fills it by other means | The optional table is probed via `sqlite_master` before it is read. A stock install does not have it, and its absence must not abort a refresh the first source could answer on its own. ## The two spellings The sources spell the same region differently, and both are accepted: - `configured_scope` carries the leading `#` that `normalizeScopeList` adds, because every other stored scope value has one - an OTA answer in the optional table carries the bare name - `loadRegionKeys` already prefixes a missing `#` before hashing So `regionNameAcceptable` canonicalises before judging, and hands back the bare name the caller re-prefixes. What it rejects is what cannot be a region name at all: `*` (the flood wildcard, not a region), a comma (it would split the name on the next round-trip through a comma-separated column), a second `#`, whitespace, non-ASCII, NUL padding from a stale client, and anything past 32 characters. The rules are deliberately structural rather than about meaning. A real declared set contains entries that look like junk, but a blocklist on string values is unmaintainable, and the cost of one bad name is a single slot out of the cap plus a 1-in-65536 collision chance. `*` is skipped in **two** places on purpose: in the filter, so no key is ever derived for it, and in the source count, because nearly every node declares it and counting it would overstate both the cap arithmetic and the refresh log on every deployment. ## Rule 0 Each derived key costs one HMAC per transport-scoped packet and raises the random 2-byte collision rate by 1/65536. So: - the tier is **capped** (default 256) - over the cap, names are kept by how many distinct nodes declare them, so a one-off local name is dropped before a region half the network uses - benchmarked linear at ~0.65µs per key: at 314 keys that is 217µs per packet, 0.0008% of one core at the 0.037 transport-scoped packets/s this network produces There is no indexable shortcut to reach for, for the same reason as in #1988: `code1` is an HMAC over the payload, so nothing is payload-independent. The set is an `atomic.Pointer` to an immutable snapshot. A refresh builds the replacement off to the side and swaps the pointer, so the ingest path never blocks on a rebuild. `refreshDerived` is a load-then-store rather than a CAS loop, which is safe only because exactly one goroutine calls it: the refresh ticker, plus one synchronous call at startup. The comment says so, rather than leaving the type looking as though it tolerates concurrent refreshers. ## Tier 2, and the counters When several keys match one packet and exactly one of them is explicit operator config, the explicit one wins: an operator who typed a region into `hashRegions` outranks a name overheard on the air. Ambiguity between two equally-sourced keys still stores unmatched, unchanged from #1988. Counters tally how each packet was decided (unique / explicit-over-derived / ambiguous / none) and are logged on the refresh tick. They exist to answer one question with data rather than estimation: whether a third tier that breaks ties on path evidence is worth building at all. Measured on a live instance at 159 keys over 41.7 hours and 147,535 packets: **0.253% ambiguous**, with explicit-over-derived at zero because that instance has exactly one derived key. ## Rule 8 `maxDerived` and `refreshMinutes` are configurable values and belong in the customizer eventually. Documented in `config.example.json` for now, flagged here so it is tracked rather than forgotten. ## Tests - default off, and a refresh that is a no-op while disabled - the name filter across both spellings, the wildcard, and each structural rejection - ranking by declarer count, then recency, then name, so the result is deterministic rather than churning between refreshes - `configured_scope` as a source, including that a node counts once per name and the newest answer wins - both sources merged, neither dropped - the explicit-over-derived tie-break - the derived tier replaced rather than merged on refresh, so a region that stops being declared leaves the key set and the cap keeps meaning something `cd cmd/ingestor && go test ./...` passes apart from `TestWriteStatsAtomic_SymlinkAtDestIsReplaced`, which needs `SeCreateSymbolicLinkPrivilege` and fails on Windows on master too. `go vet` and `gofmt -l` clean, `config.example.json` still parses. --- cmd/ingestor/client_reception.go | 8 +- cmd/ingestor/client_reception_test.go | 4 +- cmd/ingestor/config.go | 56 +++ cmd/ingestor/db.go | 12 +- cmd/ingestor/main.go | 63 ++- cmd/ingestor/main_test.go | 22 +- cmd/ingestor/region_keys.go | 461 ++++++++++++++++++ cmd/ingestor/region_keys_test.go | 646 ++++++++++++++++++++++++++ config.example.json | 2 + 9 files changed, 1219 insertions(+), 55 deletions(-) create mode 100644 cmd/ingestor/region_keys.go create mode 100644 cmd/ingestor/region_keys_test.go diff --git a/cmd/ingestor/client_reception.go b/cmd/ingestor/client_reception.go index c42c3560a..c042799eb 100644 --- a/cmd/ingestor/client_reception.go +++ b/cmd/ingestor/client_reception.go @@ -22,7 +22,7 @@ var clientPubkeyRe = regexp.MustCompile(`^[0-9a-f]{2,64}$`) // companion reports WHERE it directly heard a node, so we write a // client_receptions row and never touch the observers/observations tables. // rxPubkey is the companion pubkey from the topic (ACL-bound by the broker). -func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionKeys map[string][]byte) { +func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map[string]interface{}, channelKeys map[string]string, regionSet *regionKeySet) { // The companion identity IS the (ACL-bound) topic pubkey. Reject non-hex // topic segments so a no-ACL broker can't pollute the coverage tables, and // never fall back to a payload-supplied id (that would defeat the ACL trust @@ -91,7 +91,7 @@ func handleClientPacket(store *Store, cfg *Config, tag, rxPubkey string, msg map // collapse them and ON CONFLICT DO NOTHING would silently drop all but // the first. rxAtMillis := rxTime.Format(rxTimeMillisLayout) - if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionKeys, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil { + if obs := buildClientRxObservation(direction, rxPubkey, rawHex, rxAtMillis, ingestedAt, decoded, regionSet, snrPtr, rssiPtr, lat, lon, accPtr); obs != nil { if _, err := store.InsertClientRxObservation(obs); err != nil { log.Printf("MQTT [%s] client observation insert: %v", tag, err) } @@ -328,7 +328,7 @@ type ClientRxObservation struct { // signal — per-flood row multiplicity meant to measure forwarder // amplification of traffic actually heard over the air. func buildClientRxObservation( - direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionKeys map[string][]byte, + direction, rxPubkey, rawHex, rxAt, ingestedAt string, decoded *DecodedPacket, regionSet *regionKeySet, snr *float64, rssi *int, lat, lon float64, posAccM *float64, ) *ClientRxObservation { if !strings.EqualFold(direction, "rx") { @@ -353,7 +353,7 @@ func buildClientRxObservation( obs.Code1 = &decoded.TransportCodes.Code1 obs.Code2 = &decoded.TransportCodes.Code2 if decoded.TransportCodes.Code1 != "0000" { - sn := matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + sn := regionSet.matchScopeName(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) obs.ScopeName = &sn } } diff --git a/cmd/ingestor/client_reception_test.go b/cmd/ingestor/client_reception_test.go index 8ff513a1f..ceaacb865 100644 --- a/cmd/ingestor/client_reception_test.go +++ b/cmd/ingestor/client_reception_test.go @@ -638,7 +638,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { "timestamp": ts1, "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, } - handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionKeys) + handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg, nil, regionKeySetFromKeys(regionKeys)) // ComputeContentHash deliberately excludes the transport-code bytes (so the // same content dedups across scopes), so raw and raw2 below share one @@ -667,7 +667,7 @@ func TestClientObservationScopeNameFromTransportCode(t *testing.T) { "timestamp": ts2, "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, } - handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeys) + handleClientPacket(s, cfgWithObservations(), "test", "aa11", msg2, nil, regionKeySetFromKeys(regionKeys)) var code1b, scopeNameB sql.NullString if err := s.db.QueryRow(`SELECT code1, scope_name FROM client_rx_observations WHERE rx_at = ?`, ts2). diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index fb1026532..58c4ce7be 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -52,6 +52,7 @@ type Config struct { ChannelKeys map[string]string `json:"channelKeys,omitempty"` HashChannels []string `json:"hashChannels,omitempty"` HashRegions []string `json:"hashRegions,omitempty"` + AutoRegionKeys *AutoRegionKeysConfig `json:"autoRegionKeys,omitempty"` Retention *RetentionConfig `json:"retention,omitempty"` Metrics *MetricsConfig `json:"metrics,omitempty"` Runtime *RuntimeConfig `json:"runtime,omitempty"` @@ -440,3 +441,58 @@ func (c *Config) ResolvedSources() []MQTTSource { } return c.MQTTSources } + +// AutoRegionKeysConfig controls the opt-in derivation of region keys from the +// names repeaters declare over RF (node_declared_regions), on top of the +// explicit hashRegions list. +// +// TOP-LEVEL BLOCK, a sibling of hashRegions — not nested inside it. Config +// loading is plain json.Unmarshal with no DisallowUnknownFields, so a +// mis-nested key is silently ignored and derivation stays off with no error, +// the same trap clientRxObservations documents. +type AutoRegionKeysConfig struct { + Enabled bool `json:"enabled"` + MaxDerived int `json:"maxDerived"` + RefreshMinutes int `json:"refreshMinutes"` +} + +// autoRegionKeysDefaultMaxDerived bounds the derived tier. Every added key +// raises the random ambiguity rate by 1/65536 per scoped packet (code1 is two +// bytes) and costs one more HMAC per transport-scoped packet — the match is +// O(keys) and cannot be indexed, because the code depends on the payload. 256 +// on top of a typical explicit set puts the ambiguity rate near 0.5%, which is +// the ceiling this design accepts. +const autoRegionKeysDefaultMaxDerived = 256 + +// autoRegionKeysDefaultRefreshMinutes is how often the derived tier is rebuilt +// from the database. Declared-region answers arrive at human pace (a drive-by +// with a companion app, or a 24h observer report), so minutes-scale staleness +// is irrelevant and a tighter interval only burns queries. +const autoRegionKeysDefaultRefreshMinutes = 15 + +// AutoRegionKeysEnabled reports whether region keys may be derived from +// declared-region answers. Default false. +func (c *Config) AutoRegionKeysEnabled() bool { + return c.AutoRegionKeys != nil && c.AutoRegionKeys.Enabled +} + +// AutoRegionKeysMaxDerived returns the derived-tier cap, falling back to the +// default for absent, zero, or negative values — a zero is indistinguishable +// from "key omitted" after json.Unmarshal, and neither should silently mean +// "derive nothing" when the operator has switched the feature on. +func (c *Config) AutoRegionKeysMaxDerived() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.MaxDerived <= 0 { + return autoRegionKeysDefaultMaxDerived + } + return c.AutoRegionKeys.MaxDerived +} + +// AutoRegionKeysRefreshMinutes returns the refresh interval in minutes, +// falling back to the default for absent, zero, or negative values — a zero +// here would otherwise panic time.NewTicker. +func (c *Config) AutoRegionKeysRefreshMinutes() int { + if c.AutoRegionKeys == nil || c.AutoRegionKeys.RefreshMinutes <= 0 { + return autoRegionKeysDefaultRefreshMinutes + } + return c.AutoRegionKeys.RefreshMinutes +} diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 729e6d8ff..5a119150b 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -1581,9 +1581,11 @@ func (s *Store) BackfillPathJSONAsync() { // MQTT packet inserts and any concurrent backfill goroutines — serialize through // the single connection pool. busy_timeout(5000) handles transient cross-process // contention with the read-only server process. No additional locking is needed. -func (s *Store) BackfillDefaultScopeAsync(regionKeys map[string][]byte) { - // No region keys configured — all scope_name values will be NULL, nothing to backfill. - if len(regionKeys) == 0 { +func (s *Store) BackfillDefaultScopeAsync(regionSet *regionKeySet) { + // No region keys at all — every scope_name is NULL, nothing to backfill. + // The emptiness gate is read once here rather than per row: the loop below + // uses no keys, it only copies names that are already stored. + if len(regionSet.snapshot().all) == 0 { return } s.backfillWg.Add(1) @@ -2207,7 +2209,7 @@ type MQTTPacketMessage struct { // into the past. Packet ordering is owned by the server clock; client // clocks are untrusted. msg.Timestamp still flows into observer.last_seen // via UpsertObserverAt — that's #1233's MAX/MIN guarded path and is fine. -func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string, regionKeys map[string][]byte) *PacketData { +func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, region string, regionSet *regionKeySet) *PacketData { pathJSON := "[]" // For TRACE packets, path_json must be the payload-decoded route hops // (decoded.Path.Hops), NOT the raw_hex header bytes which are SNR values. @@ -2260,7 +2262,7 @@ func BuildPacketData(msg *MQTTPacketMessage, decoded *DecodedPacket, observerID, pd.Code2 = decoded.TransportCodes.Code2 if decoded.TransportCodes.Code1 != "0000" { pd.IsTransportScoped = true - pd.ScopeName = matchScope(regionKeys, byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) + pd.ScopeName = regionSet.matchScopeName(byte(decoded.Header.PayloadType), decoded.payloadRaw, decoded.TransportCodes.Code1) } } diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index d3ed0dda5..d5a7b82ab 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -101,8 +101,15 @@ func main() { log.Printf("No channel keys loaded — GRP_TXT packets will not be decrypted") } - regionKeys := loadRegionKeys(cfg) - store.BackfillDefaultScopeAsync(regionKeys) + regionSet := newRegionKeySet(cfg) + if cfg.AutoRegionKeysEnabled() { + // Fill the derived tier before the first packet is matched, so a + // restart does not spend a refresh interval naming nothing. + regionSet.refreshFromStore(store) + } else { + log.Printf("[regions] autoRegionKeys disabled — only the %d configured hashRegions key(s) are in force", len(regionSet.snapshot().all)) + } + store.BackfillDefaultScopeAsync(regionSet) store.BackfillTransportCodesAsync() // Subscribe-early + buffer (#1608): the MQTT subscription is brought up @@ -181,7 +188,7 @@ func main() { markReceiptForTag(tag, time.Now()) status.MarkPacket(time.Now()) ingestBuffer.Submit(func() { - handleMessage(store, tag, src, m, channelKeys, regionKeys, cfg) + handleMessage(store, tag, src, m, channelKeys, regionSet, cfg) }) }) @@ -420,6 +427,22 @@ func main() { } } + // Derived region keys refresh on their own ticker rather than the daily + // retention one: declared-region answers arrive continuously (an observer + // report lands, a node is asked again), and waiting up to 24h to name a + // newly-discovered region would defeat the point of deriving them. + if cfg.AutoRegionKeysEnabled() { + interval := time.Duration(cfg.AutoRegionKeysRefreshMinutes()) * time.Minute + regionRefreshTicker := time.NewTicker(interval) + go func() { + for range regionRefreshTicker.C { + regionSet.refreshFromStore(store) + logScopeMatchCounters() + } + }() + log.Printf("[regions] auto-derived region keys enabled: refreshing every %v, cap %d", interval, cfg.AutoRegionKeysMaxDerived()) + } + // Hourly WAL checkpoint to prevent unbounded WAL growth. // TRUNCATE resets the WAL file to zero bytes when all frames are flushed; // if the server's read connection holds frames, remaining pages stay in the @@ -654,7 +677,7 @@ func buildForceReconnectFn(client mqtt.Client, tag string) func() { } } -func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionKeys map[string][]byte, cfg *Config) { +func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, channelKeys map[string]string, regionSet *regionKeySet, cfg *Config) { // Liveness watchdog (#1212): record receipt before any processing so a // slow handler still counts as "source is alive". Cheap atomic store. markLivenessForTag(tag, time.Now()) @@ -698,7 +721,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, switch parts[3] { case "packets": if cfg.ClientRxCoverageEnabled() { - handleClientPacket(store, cfg, tag, parts[2], msg, channelKeys, regionKeys) + handleClientPacket(store, cfg, tag, parts[2], msg, channelKeys, regionSet) } case "rf": if cfg.ClientRfSamplesEnabled() { @@ -946,7 +969,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, log.Printf("MQTT [%s] foreign advert: node=%s name=%s lat=%.4f lon=%.4f observer=%s", tag, truncPK, sanitizeLogString(decoded.Payload.Name), lat, lon, sanitizeLogString(firstNonEmpty(mqttMsg.Origin, observerID))) } - pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys) + pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionSet) pktData.Foreign = foreign isNew, err := store.InsertTransmission(pktData) if err != nil { @@ -981,7 +1004,7 @@ func handleMessage(store *Store, tag string, source MQTTSource, m mqtt.Message, } else { // Non-ADVERT packets: store normally (routing/channel messages from // in-area observers are relevant regardless of relay hop origin). - pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionKeys) + pktData := BuildPacketData(mqttMsg, decoded, observerID, region, regionSet) if _, err := store.InsertTransmission(pktData); err != nil { log.Printf("MQTT [%s] db insert error: %v", tag, err) } @@ -1673,32 +1696,6 @@ func matchingRegions(regionKeys map[string][]byte, payloadType byte, payloadRaw return matched } -// matchScope names one packet's region scope, or returns "" when it cannot be -// named with confidence. -// -// Exactly one match names the packet. Several matches name nothing: the two -// candidates are equally sourced, there is no principled winner, and storing -// a wrong region name is worse than storing none. "" is already the ingestor's -// "transport-scoped but unnameable" state (scopeNameForDB), so an ambiguous -// packet lands in a state the rest of the system already understands rather -// than in a new one. -// -// The collision is logged because it is otherwise invisible: the packet is -// stored exactly like one whose region this instance holds no key for, and an -// operator looking at a growing unnameable count deserves to see which of -// their configured regions are colliding. -func matchScope(regionKeys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { - matched := matchingRegions(regionKeys, payloadType, payloadRaw, code1) - switch len(matched) { - case 0: - return "" - case 1: - return matched[0] - } - log.Printf("[regions] ambiguous collision between %v; storing unmatched", matched) - return "" -} - // Version info (set via ldflags) var version = "dev" diff --git a/cmd/ingestor/main_test.go b/cmd/ingestor/main_test.go index cbc6293c1..1bd817349 100644 --- a/cmd/ingestor/main_test.go +++ b/cmd/ingestor/main_test.go @@ -861,7 +861,7 @@ func TestMatchScope(t *testing.T) { // Key = SHA256("#test")[:16] = 9cd8fcf22a47333b591d96a2b848b73f testKey, _ := hex.DecodeString("9cd8fcf22a47333b591d96a2b848b73f") testKeys := map[string][]byte{"#test": testKey} - if got := matchScope(testKeys, 5, []byte("hello"), "2AB5"); got != "#test" { + if got := regionKeySetFromKeys(testKeys).matchScopeName(5, []byte("hello"), "2AB5"); got != "#test" { t.Errorf("#test vector: matchScope = %q, want #test", got) } @@ -869,17 +869,17 @@ func TestMatchScope(t *testing.T) { // Key = SHA256("#belgium")[:16] = 7085b78ed010599094f8c8e7d1aa0e27 belgiumKey, _ := hex.DecodeString("7085b78ed010599094f8c8e7d1aa0e27") belgiumKeys := map[string][]byte{"#belgium": belgiumKey} - if got := matchScope(belgiumKeys, 5, []byte("hello"), "4A75"); got != "#belgium" { + if got := regionKeySetFromKeys(belgiumKeys).matchScopeName(5, []byte("hello"), "4A75"); got != "#belgium" { t.Errorf("#belgium vector: matchScope = %q, want #belgium", got) } // Code1=0000 (unscoped transport) → no region matched - if got := matchScope(belgiumKeys, 5, []byte("hello"), "0000"); got != "" { + if got := regionKeySetFromKeys(belgiumKeys).matchScopeName(5, []byte("hello"), "0000"); got != "" { t.Errorf("unscoped: matchScope = %q, want empty", got) } // Code1 present but matches no configured region → empty string - if got := matchScope(belgiumKeys, 5, []byte("hello"), "BEEF"); got != "" { + if got := regionKeySetFromKeys(belgiumKeys).matchScopeName(5, []byte("hello"), "BEEF"); got != "" { t.Errorf("no match: matchScope = %q, want empty", got) } } @@ -899,7 +899,7 @@ func TestBuildPacketDataScopeMatching(t *testing.T) { } msg := &MQTTPacketMessage{Raw: rawHex} - pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys) + pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeySetFromKeys(regionKeys)) if pktData.ScopeName != "#test" { t.Errorf("ScopeName = %q, want #test", pktData.ScopeName) } @@ -1078,7 +1078,7 @@ func TestBuildPacketDataScopeMatchingNoMatch(t *testing.T) { t.Fatalf("DecodePacket: %v", err) } msg := &MQTTPacketMessage{Raw: rawHex} - pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeys) + pktData := BuildPacketData(msg, decoded, "obs1", "region1", regionKeySetFromKeys(regionKeys)) if !pktData.IsTransportScoped { t.Fatalf("precondition: IsTransportScoped should be true (Code1 != 0000)") @@ -1127,7 +1127,7 @@ func TestHandleMessageAdvert_EmptyScopeSkipsDefaultScopeUpdate(t *testing.T) { topic: "meshcore/SJC/obs1/packets", payload: []byte(`{"raw":"` + rawHex + `"}`), } - handleMessage(store, "test", source, msg, nil, map[string][]byte{}, &Config{}) + handleMessage(store, "test", source, msg, nil, nil, &Config{}) var got sql.NullString if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { @@ -1179,7 +1179,7 @@ func TestHandleMessageAdvert_MatchedScopeUpdatesDefaultScope(t *testing.T) { topic: "meshcore/SJC/obs1/packets", payload: []byte(`{"raw":"` + rawHex + `"}`), } - handleMessage(store, "test", source, msg, nil, map[string][]byte{"#de": regionKey}, &Config{}) + handleMessage(store, "test", source, msg, nil, regionKeySetFromKeys(map[string][]byte{"#de": regionKey}), &Config{}) var got sql.NullString if err := store.db.QueryRow(`SELECT default_scope FROM nodes WHERE public_key = ?`, pubkey).Scan(&got); err != nil { @@ -1238,7 +1238,7 @@ func TestMatchScopeNamesAnUnambiguousPacket(t *testing.T) { payload := []byte{0x01, 0x02, 0x03, 0x04} code1 := codeForRegion("#be", 5, payload) - if got := matchScope(keys, 5, payload, code1); got != "#be" { + if got := regionKeySetFromKeys(keys).matchScopeName(5, payload, code1); got != "#be" { t.Errorf("matchScope = %q, want %q", got, "#be") } } @@ -1255,7 +1255,7 @@ func TestMatchScopeStoresAmbiguousAsUnmatched(t *testing.T) { if n := len(matchingRegions(keys, 5, payload, code1)); n != 2 { t.Fatalf("matchingRegions returned %d names, want 2 — the collision fixture is wrong", n) } - if got := matchScope(keys, 5, payload, code1); got != "" { + if got := regionKeySetFromKeys(keys).matchScopeName(5, payload, code1); got != "" { t.Errorf("matchScope = %q, want the unmatched state: two equally-sourced candidates have no principled winner", got) } } @@ -1269,7 +1269,7 @@ func TestMatchScopeIsOrderIndependent(t *testing.T) { keys := loadRegionKeys(&Config{HashRegions: []string{"#be", "#zz"}}) for i := 0; i < 50; i++ { - if got := matchScope(keys, 5, payload, code1); got != "" { + if got := regionKeySetFromKeys(keys).matchScopeName(5, payload, code1); got != "" { t.Fatalf("iteration %d: matchScope = %q, want a stable answer across map iteration orders", i, got) } } diff --git a/cmd/ingestor/region_keys.go b/cmd/ingestor/region_keys.go new file mode 100644 index 000000000..45002722a --- /dev/null +++ b/cmd/ingestor/region_keys.go @@ -0,0 +1,461 @@ +package main + +import ( + "crypto/sha256" + "fmt" + "log" + "sort" + "strings" + "sync/atomic" +) + +// declaredRegionStat is one region name as reported over RF, with the two +// facts the cap ranks on: how many distinct repeaters declare it, and how +// recently any of them last answered. +type declaredRegionStat struct { + Name string + Declarers int + LastSeen string // ISO, greatest observed_at across declarers +} + +// maxRegionNameLen bounds a derived region name. Firmware region names are +// short labels; anything longer is a malformed or hostile entry, and each +// accepted name costs an HMAC on every transport-scoped packet. +const maxRegionNameLen = 32 + +// regionNameAcceptable reports whether a declared name may become a derived +// region key, and returns the name in the form the key is derived from. +// +// The two declared-region sources spell a name differently, so this +// canonicalises before judging: nodes.configured_scope carries the leading +// "#" (normalizeScopeList puts it there, matching every other stored scope +// value), while a node_declared_regions row carries the bare name the OTA +// query returned. Both mean the same region, and loadRegionKeys already +// prefixes a missing "#" before hashing, so the "#" is stripped here and put +// back by the caller. +// +// The rules are structural, never about the name's meaning. A declared set +// contains entries that look like junk ("null", "sol3"), but a blocklist on +// string values is unmaintainable, and the cost of one bad name is a single +// slot out of maxDerived plus a 1-in-65536 collision chance. What IS rejected +// is anything that cannot be a region name: +// +// - "*" is the flood wildcard, not a region, and hashing it would invent a +// region nobody declared +// - a comma would split the name on the next round-trip through a +// comma-separated column +// - a second "#" cannot come from either source intact +// - non-ASCII or whitespace would make the key a hash over bytes nobody +// intended, silently mismatching the sender +// - a NUL is block-cipher padding a stale client failed to trim +func regionNameAcceptable(name string) (string, bool) { + name = strings.TrimPrefix(name, "#") + if name == "" || name == "*" || len(name) > maxRegionNameLen { + return "", false + } + for i := 0; i < len(name); i++ { + c := name[i] + if c <= ' ' || c >= 0x7F || c == ',' || c == '#' { + return "", false + } + } + return name, true +} + +// declaredRegionSources reads the declared region names from every source this +// instance has, newest answer per node, and counts how many distinct nodes +// declare each name. +// +// Two sources, mirroring what the server's AllCurrentDeclaredRegions already +// merges, so the derived tier sees exactly what the audit sees: +// +// - nodes.configured_scope, written by the observer /neighbors ingestion +// (#1865). Always present; the column is part of the schema. +// - node_declared_regions, an optional table a deployment may fill by other +// means. Absent on a stock install, so its absence is not an error: the +// probe below asks sqlite_master first rather than letting "no such table" +// abort a refresh that the first source could still answer. +// +// A node counts once per name however many times it declares it, so the +// ranking below reflects how widely a region is claimed rather than how +// chatty one node is. +func (s *Store) declaredRegionSources() ([]declaredRegionStat, error) { + agg := map[string]*declaredRegionStat{} + count := func(csv, at string) { + seenHere := map[string]bool{} // one node counts once per name + for _, name := range splitDeclaredRegionsCSV(csv) { + // '*' is the flood wildcard, not a region. regionNameAcceptable + // rejects it too, but skipping it here as well keeps it out of the + // declared-name count that the cap arithmetic and the refresh log + // are stated in: nearly every node declares it, so counting it + // would overstate both on every deployment. + if name == "*" || name == "#*" || seenHere[name] { + continue + } + seenHere[name] = true + st, ok := agg[name] + if !ok { + st = &declaredRegionStat{Name: name} + agg[name] = st + } + st.Declarers++ + if at > st.LastSeen { + st.LastSeen = at + } + } + } + + rows, err := s.db.Query(` + SELECT COALESCE(configured_scope, ''), COALESCE(configured_scope_at, '') + FROM nodes + WHERE configured_scope IS NOT NULL AND configured_scope != '' + `) + if err != nil { + return nil, fmt.Errorf("declared regions from configured_scope: %w", err) + } + for rows.Next() { + var csv, at string + if err := rows.Scan(&csv, &at); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from configured_scope scan: %w", err) + } + count(csv, at) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, fmt.Errorf("declared regions from configured_scope rows: %w", err) + } + rows.Close() + + var present string + if err := s.db.QueryRow(`SELECT name FROM sqlite_master WHERE type='table' AND name='node_declared_regions'`).Scan(&present); err == nil && present != "" { + ndr, err := s.db.Query(` + WITH ranked AS ( + SELECT target, observed_at, regions_csv, + ROW_NUMBER() OVER (PARTITION BY target ORDER BY observed_at DESC) AS rn + FROM node_declared_regions + ) + SELECT observed_at, regions_csv FROM ranked WHERE rn = 1 + `) + if err != nil { + return nil, fmt.Errorf("declared regions from node_declared_regions: %w", err) + } + defer ndr.Close() + for ndr.Next() { + var at, csv string + if err := ndr.Scan(&at, &csv); err != nil { + return nil, fmt.Errorf("declared regions from node_declared_regions scan: %w", err) + } + count(csv, at) + } + if err := ndr.Err(); err != nil { + return nil, fmt.Errorf("declared regions from node_declared_regions rows: %w", err) + } + } + + out := make([]declaredRegionStat, 0, len(agg)) + for _, st := range agg { + out = append(out, *st) + } + return out, nil +} + +// rankDeclaredRegions filters stats through regionNameAcceptable and returns at +// most max names, most-worth-keeping first: by declarer count descending, then +// by recency, then by name. The name tie-break is what makes the result +// deterministic — without it the derived tier would churn between refreshes on +// equally-ranked names and the add/drop logging would be noise. +func rankDeclaredRegions(stats []declaredRegionStat, max int) []string { + kept := make([]declaredRegionStat, 0, len(stats)) + for _, s := range stats { + if canonical, ok := regionNameAcceptable(s.Name); ok { + s.Name = canonical + kept = append(kept, s) + } + } + sort.Slice(kept, func(i, j int) bool { + if kept[i].Declarers != kept[j].Declarers { + return kept[i].Declarers > kept[j].Declarers + } + if kept[i].LastSeen != kept[j].LastSeen { + return kept[i].LastSeen > kept[j].LastSeen + } + return kept[i].Name < kept[j].Name + }) + if max > 0 && len(kept) > max { + kept = kept[:max] + } + names := make([]string, 0, len(kept)) + for _, s := range kept { + names = append(names, s.Name) + } + return names +} + +// splitDeclaredRegionsCSV parses a regions_csv value into its entries. The +// ingestor writes this column with strings.Join(regions, ","), so this is its +// exact inverse. Mirrors splitRegionsCSV in cmd/server/scopes.go. +func splitDeclaredRegionsCSV(csv string) []string { + out := []string{} + if csv == "" { + return out + } + for _, part := range strings.Split(csv, ",") { + part = strings.TrimSpace(part) + if part != "" { + out = append(out, part) + } + } + return out +} + +// regionKeySnapshot is an immutable view of the region keys in force for one +// packet. `all` is the single map matching iterates - merging at build time +// rather than per packet keeps the hot path free of allocation. `explicit` +// carries membership only, and exists so the ambiguity tie-break can tell an +// operator-configured region from one derived off the air. +type regionKeySnapshot struct { + all map[string][]byte + explicit map[string]bool +} + +func (s *regionKeySnapshot) isExplicit(name string) bool { return s.explicit[name] } + +// regionKeySet holds the live snapshot. Readers take one atomic load; a +// refresh builds the replacement off to the side and swaps the pointer, so the +// ingest hot path never blocks on a rebuild (AGENTS.md rule 0). +type regionKeySet struct { + cur atomic.Pointer[regionKeySnapshot] + enabled bool + max int +} + +// newRegionKeySet builds the explicit tier from hashRegions. The derived tier +// starts empty; refreshDerived fills it, and does nothing at all when +// autoRegionKeys is off. +func newRegionKeySet(cfg *Config) *regionKeySet { + explicitKeys := loadRegionKeys(cfg) + explicitNames := make(map[string]bool, len(explicitKeys)) + all := make(map[string][]byte, len(explicitKeys)) + for name, key := range explicitKeys { + explicitNames[name] = true + all[name] = key + } + s := ®ionKeySet{ + enabled: cfg.AutoRegionKeysEnabled(), + max: cfg.AutoRegionKeysMaxDerived(), + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: explicitNames}) + return s +} + +// regionKeySetFromKeys wraps an explicit key map in a set with no derived +// tier. Tests that predate the derived tier use it to keep expressing "these +// are the configured regions" without building a Config. +func regionKeySetFromKeys(keys map[string][]byte) *regionKeySet { + names := make(map[string]bool, len(keys)) + all := make(map[string][]byte, len(keys)) + for name, key := range keys { + names[name] = true + all[name] = key + } + s := ®ionKeySet{} + s.cur.Store(®ionKeySnapshot{all: all, explicit: names}) + return s +} + +// emptyRegionKeySnapshot backs the nil case below. Shared and never mutated: +// refreshDerived always builds a fresh map rather than writing into one. +var emptyRegionKeySnapshot = ®ionKeySnapshot{all: map[string][]byte{}, explicit: map[string]bool{}} + +// snapshot is nil-safe on purpose. A nil *regionKeySet means "no region keys", +// which is exactly what a nil map[string][]byte meant before M2 — the shape +// several call sites and a good many tests still pass. Panicking there would +// turn an absent key set into a crash on the ingest path, which is a far worse +// failure than naming nothing. +func (s *regionKeySet) snapshot() *regionKeySnapshot { + if s == nil { + return emptyRegionKeySnapshot + } + return s.cur.Load() +} + +// refreshDerived rebuilds the derived tier from names (already ranked and +// capped by the caller) and swaps in a new snapshot. It REPLACES the derived +// tier rather than merging into it, so a region that stops being declared +// leaves the key set and the cap keeps meaning something. +// +// A name that duplicates an explicit key is skipped, not re-added: the +// explicit tier must stay authoritative for the tie-break, and demoting a +// configured region because a repeater also declares it would invert the whole +// rule. +// +// Returns the names actually added, for the caller to log. +func (s *regionKeySet) refreshDerived(names []string) []string { + if !s.enabled { + return nil + } + old := s.cur.Load() + all := make(map[string][]byte, len(old.explicit)+len(names)) + for name := range old.explicit { + all[name] = old.all[name] + } + added := make([]string, 0, len(names)) + for _, raw := range names { + canonical, ok := regionNameAcceptable(raw) + if !ok { + continue + } + name := "#" + canonical + if old.explicit[name] { + continue + } + if _, exists := all[name]; exists { + continue + } + h := sha256.Sum256([]byte(name)) + all[name] = h[:16] + added = append(added, name) + } + s.cur.Store(®ionKeySnapshot{all: all, explicit: old.explicit}) + return added +} + +// scopeReason records how a scope match was decided, so the outcome is +// auditable in logs without a schema change. It is deliberately not stored: +// transmissions.scope_name keeps its existing three-state encoding. +type scopeReason string + +const ( + scopeReasonNone scopeReason = "none" // no key matched + scopeReasonUnique scopeReason = "unique" // exactly one key matched + scopeReasonExplicitOverDerived scopeReason = "explicit-over-derived" // several matched, one was operator config + scopeReasonAmbiguous scopeReason = "ambiguous" // several matched, no principled winner +) + +// scopeMatch is the result of naming one packet's region scope. +type scopeMatch struct { + Name string // empty when unresolved - the caller stores that as the unmatched state + Reason scopeReason + // Candidates is every name whose derived code equals the packet's code1. + // It is always populated, so a caller that needs the match COUNT (scope-repair + // does, to tell "the key set changed" from "several keys now collide") can read + // it from the same result that carries the verdict, instead of re-running the + // match and risking a second, divergent decision rule. + Candidates []string +} + +// match names the region scope of a transport-scoped packet, resolving a +// multi-key collision by evidence rather than by map order. +// +// Tiers, in order: +// +// 1. Exactly one key matched - name it. +// 2. Several matched but exactly one came from hashRegions - name that one. +// The operator's own configuration outranks a name picked up off the air, +// and this covers the bulk of the ambiguity auto-derivation introduces. +// 3. Otherwise abstain, returning an empty name. Two equally-sourced +// candidates offer no principled winner, and naming a packet wrongly is +// worse than leaving it unnamed - the rule #1609 established, unchanged. +// +// (The spec's tier-3 path-evidence tie-break sits between 2 and 3 and is +// deliberately not built here; see +// docs/specs/2026-09-07-auto-region-keys-design.md. The scopeReasonAmbiguous +// counter is what measures whether it is worth building.) +func (s *regionKeySnapshot) match(payloadType byte, payloadRaw []byte, code1 string) scopeMatch { + matched := matchingRegions(s.all, payloadType, payloadRaw, code1) + switch len(matched) { + case 0: + return scopeMatch{Reason: scopeReasonNone, Candidates: matched} + case 1: + return scopeMatch{Name: matched[0], Reason: scopeReasonUnique, Candidates: matched} + } + + var explicitMatches []string + for _, name := range matched { + if s.explicit[name] { + explicitMatches = append(explicitMatches, name) + } + } + if len(explicitMatches) == 1 { + return scopeMatch{Name: explicitMatches[0], Reason: scopeReasonExplicitOverDerived, Candidates: matched} + } + return scopeMatch{Reason: scopeReasonAmbiguous, Candidates: matched} +} + +// matchScopeName is what the ingest path calls: the tiered decision above, +// reduced to the string transmissions.scope_name stores, with the outcome +// tallied for the counters. A nil set (no region keys at all) names nothing, +// which is what an unconfigured instance did before this existed. +func (s *regionKeySet) matchScopeName(payloadType byte, payloadRaw []byte, code1 string) string { + m := s.snapshot().match(payloadType, payloadRaw, code1) + recordScopeMatch(m) + return m.Name +} + +// scopeMatchCounters tallies how each transport-scoped packet's region was +// decided. It exists to answer one question before more machinery is built: +// how often does an ambiguous collision actually happen? The spec gates the +// path-evidence tie-break (tier 3) on this number. +var scopeMatchCounters struct { + unique atomic.Int64 + explicitOverDerived atomic.Int64 + ambiguous atomic.Int64 + none atomic.Int64 +} + +// recordScopeMatch tallies one decision and logs the interesting ones. Unique +// and none are the overwhelming majority and are counted silently; the other +// two are rare by construction and worth a line each. +func recordScopeMatch(m scopeMatch) { + switch m.Reason { + case scopeReasonUnique: + scopeMatchCounters.unique.Add(1) + case scopeReasonNone: + scopeMatchCounters.none.Add(1) + case scopeReasonExplicitOverDerived: + scopeMatchCounters.explicitOverDerived.Add(1) + log.Printf("[regions] collision resolved to explicit %s over derived candidates %v", m.Name, m.Candidates) + case scopeReasonAmbiguous: + scopeMatchCounters.ambiguous.Add(1) + log.Printf("[regions] ambiguous collision between %v; storing unmatched", m.Candidates) + } +} + +// logScopeMatchCounters prints the running tally. Called from the refresh +// ticker so the numbers arrive on the same cadence as the key-set changes that +// move them. +func logScopeMatchCounters() { + log.Printf("[regions] scope matches: unique=%d explicit-over-derived=%d ambiguous=%d none=%d", + scopeMatchCounters.unique.Load(), scopeMatchCounters.explicitOverDerived.Load(), + scopeMatchCounters.ambiguous.Load(), scopeMatchCounters.none.Load()) +} + +// refreshFromStore reads the declared region names, ranks and caps them, and +// swaps in a new snapshot. Shared by startup and the ticker so both apply +// identical rules. +// +// A DB error is logged and the CURRENT snapshot is kept. That matters: an +// empty key set would silently unname all traffic, which looks exactly like +// the bug this feature exists to fix. +func (s *regionKeySet) refreshFromStore(store *Store) { + if s == nil || !s.enabled { + return + } + stats, err := store.declaredRegionSources() + if err != nil { + log.Printf("[regions] derived-key refresh failed, keeping %d existing key(s): %v", len(s.snapshot().all), err) + return + } + ranked := rankDeclaredRegions(stats, s.max) + added := s.refreshDerived(ranked) + snap := s.snapshot() + log.Printf("[regions] derived-key refresh: %d name(s) declared, %d kept after filter+cap(%d), %d total key(s) in force", + len(stats), len(ranked), s.max, len(snap.all)) + if len(added) > 0 { + log.Printf("[regions] derived keys now active: %v", added) + } + if len(stats) > s.max { + log.Printf("[regions] NOTE: %d declared name(s) exceeded maxDerived=%d and were dropped, least-declared first", len(stats)-s.max, s.max) + } +} diff --git a/cmd/ingestor/region_keys_test.go b/cmd/ingestor/region_keys_test.go new file mode 100644 index 000000000..209e47b68 --- /dev/null +++ b/cmd/ingestor/region_keys_test.go @@ -0,0 +1,646 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + "sync" + "testing" +) + +func TestRegionNameAcceptable(t *testing.T) { + cases := []struct { + name string + want string + ok bool + why string + }{ + {"be", "be", true, "a bare name from an OTA answer"}, + {"#be", "be", true, "the same region as nodes.configured_scope spells it — normalizeScopeList adds the '#'"}, + {"nl-li-nth", "nl-li-nth", true, "hyphens are ordinary in region names"}, + {"*", "", false, "the flood wildcard is not a region, and hashing it would invent one nobody declared"}, + {"#*", "", false, "the wildcard stays the wildcard however it is spelled"}, + {"", "", false, "empty"}, + {"#", "", false, "a bare prefix names nothing"}, + {"be,nl", "", false, "a comma would split the name on the next round-trip"}, + {"be#nl", "", false, "a second '#' cannot come from either source intact"}, + {"be nl", "", false, "whitespace would hash bytes nobody intended"}, + {"bé", "", false, "non-ASCII, same reason"}, + {"be" + string(rune(0)), "", false, "NUL is block-cipher padding a stale client failed to trim"}, + {"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "", false, "33 chars, past maxRegionNameLen"}, + } + for _, c := range cases { + got, ok := regionNameAcceptable(c.name) + if ok != c.ok || got != c.want { + t.Errorf("regionNameAcceptable(%q) = (%q, %v), want (%q, %v) — %s", c.name, got, ok, c.want, c.ok, c.why) + } + } +} + +func TestRankDeclaredRegionsPrefersWidelyDeclared(t *testing.T) { + // The cap must drop the long tail of one-off local names, never a region + // half the network declares. On live data "be" is declared by 127 + // repeaters and "behss" by 3. + stats := []declaredRegionStat{ + {Name: "behss", Declarers: 3, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "be", Declarers: 127, LastSeen: "2026-09-01T10:00:00Z"}, + {Name: "sol3", Declarers: 1, LastSeen: "2026-09-07T11:00:00Z"}, + } + got := rankDeclaredRegions(stats, 2) + want := []string{"be", "behss"} + if len(got) != len(want) { + t.Fatalf("rankDeclaredRegions = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("rankDeclaredRegions = %v, want %v — declarer count must dominate recency", got, want) + } + } +} + +func TestRankDeclaredRegionsIsDeterministic(t *testing.T) { + // Equal declarer counts and equal timestamps must still produce a stable + // order, or the derived tier churns between refreshes and the add/drop + // logging becomes noise. + stats := []declaredRegionStat{ + {Name: "zz", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "aa", Declarers: 2, LastSeen: "2026-09-07T10:00:00Z"}, + } + for i := 0; i < 20; i++ { + got := rankDeclaredRegions(stats, 10) + if got[0] != "aa" || got[1] != "zz" { + t.Fatalf("run %d: rankDeclaredRegions = %v, want [aa zz]", i, got) + } + } +} + +func TestRankDeclaredRegionsBreaksTiesOnRecency(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "old", Declarers: 2, LastSeen: "2026-01-01T00:00:00Z"}, + {Name: "new", Declarers: 2, LastSeen: "2026-09-07T00:00:00Z"}, + } + got := rankDeclaredRegions(stats, 1) + if len(got) != 1 || got[0] != "new" { + t.Fatalf("rankDeclaredRegions = %v, want [new] — equal declarers break on recency", got) + } +} + +func TestRankDeclaredRegionsDropsUnacceptableNames(t *testing.T) { + stats := []declaredRegionStat{ + {Name: "be", Declarers: 5, LastSeen: "2026-09-07T10:00:00Z"}, + {Name: "b,ad", Declarers: 99, LastSeen: "2026-09-07T10:00:00Z"}, + } + got := rankDeclaredRegions(stats, 10) + if len(got) != 1 || got[0] != "be" { + t.Fatalf("rankDeclaredRegions = %v, want [be] — a name with a comma cannot survive a comma-separated column, however widely declared", got) + } +} + +func TestSplitDeclaredRegionsCSV(t *testing.T) { + // Exact inverse of the strings.Join the ingestor writes the column with. + got := splitDeclaredRegionsCSV(" be , eu ,, nl ") + want := []string{"be", "eu", "nl"} + if len(got) != len(want) { + t.Fatalf("splitDeclaredRegionsCSV = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("splitDeclaredRegionsCSV = %v, want %v", got, want) + } + } + if n := len(splitDeclaredRegionsCSV("")); n != 0 { + t.Errorf("empty csv produced %d entries, want 0", n) + } +} + +func TestRegionKeySetExplicitOnlyWhenDisabled(t *testing.T) { + // Derivation off: the snapshot must be exactly what loadRegionKeys built, + // and refreshDerived must be a no-op rather than a quiet opt-in. + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "fm-112"}) + + snap := set.snapshot() + if len(snap.all) != 1 { + t.Fatalf("len(all) = %d, want 1 — refreshDerived must not add keys when disabled", len(snap.all)) + } + if _, ok := snap.all["#be"]; !ok { + t.Error("want the explicit #be key present") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true") + } +} + +func TestRegionKeySetMergesDerivedWhenEnabled(t *testing.T) { + cfg := &Config{ + HashRegions: []string{"#be"}, + AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}, + } + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"behss", "be"}) // "be" duplicates the explicit key + + snap := set.snapshot() + if len(snap.all) != 2 { + t.Fatalf("len(all) = %d, want 2 (#be explicit + #behss derived), got keys %v", len(snap.all), keyNames(snap)) + } + if _, ok := snap.all["#behss"]; !ok { + t.Errorf("want the derived #behss key present, got %v", keyNames(snap)) + } + if snap.isExplicit("#behss") { + t.Error("isExplicit(#behss) = true, want false — a derived key is not operator config") + } + if !snap.isExplicit("#be") { + t.Error("isExplicit(#be) = false, want true — an explicit key must not be demoted by a duplicate declaration") + } +} + +func TestRegionKeySetRefreshReplacesRatherThanAccumulates(t *testing.T) { + // A region that stops being declared must leave the derived tier, or the + // key set only ever grows and the cap stops meaning anything. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + set.refreshDerived([]string{"bb"}) + + snap := set.snapshot() + if _, ok := snap.all["#aa"]; ok { + t.Error("want #aa gone after a refresh that no longer lists it") + } + if _, ok := snap.all["#bb"]; !ok { + t.Error("want #bb present after the refresh that lists it") + } +} + +func TestRegionKeySetSnapshotIsStable(t *testing.T) { + // A snapshot handed to a packet must not change under it mid-match. + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa"}) + held := set.snapshot() + set.refreshDerived([]string{"bb"}) + + if _, ok := held.all["#aa"]; !ok { + t.Error("the held snapshot lost #aa — snapshots must be immutable, not aliases of live state") + } +} + +// keyNames is a test helper for readable failure messages. +func keyNames(s *regionKeySnapshot) []string { + out := make([]string, 0, len(s.all)) + for k := range s.all { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// codeFor derives the on-wire code1 a sender in region `name` would emit for +// this payload - the same computation matchingRegions inverts. Used to build +// packets that genuinely belong to a region rather than asserting on a +// hardcoded string. +func codeFor(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +func TestScopeMatchUniqueNamesTheRegion(t *testing.T) { + payload := []byte{0xDE, 0xAD, 0xBE, 0xEF} + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + got := set.snapshot().match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q", got.Name, "#be") + } + if got.Reason != scopeReasonUnique { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonUnique) + } +} + +func TestScopeMatchNoKeyMatches(t *testing.T) { + cfg := &Config{HashRegions: []string{"#be"}} + set := newRegionKeySet(cfg) + + got := set.snapshot().match(5, []byte{1, 2, 3}, "0000") + if got.Name != "" || got.Reason != scopeReasonNone { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonNone) + } +} + +func TestScopeMatchExplicitBeatsDerived(t *testing.T) { + // The ambiguity this feature introduces: a derived key collides with an + // operator-configured one on this payload. Operator config wins - it is + // intent, the derived name is hearsay picked up over RF. + payload := []byte{0x01, 0x02, 0x03, 0x04} + cfg := &Config{HashRegions: []string{"#be"}, AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + // Force the collision rather than searching for a natural one: inject a + // derived key whose bytes are the explicit key's, so both match. + snap := set.snapshot() + collide := make(map[string][]byte, len(snap.all)+1) + for k, v := range snap.all { + collide[k] = v + } + collide["#collider"] = snap.all["#be"] + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "#be" { + t.Errorf("Name = %q, want %q - the explicit key must win", got.Name, "#be") + } + if got.Reason != scopeReasonExplicitOverDerived { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonExplicitOverDerived) + } + if len(got.Candidates) != 2 { + t.Errorf("Candidates = %v, want both names recorded for the log", got.Candidates) + } +} + +func TestScopeMatchTwoExplicitKeysStayAmbiguous(t *testing.T) { + // Two equally-sourced candidates: naming either would be a guess, and + // naming wrongly is worse than not naming. This is #1609's rule, unchanged. + payload := []byte{0x09, 0x08, 0x07} + cfg := &Config{HashRegions: []string{"#be", "#eu"}} + set := newRegionKeySet(cfg) + code := codeFor("#be", 5, payload) + + snap := set.snapshot() + collide := map[string][]byte{"#be": snap.all["#be"], "#eu": snap.all["#be"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + + got := forced.match(5, payload, code) + if got.Name != "" { + t.Errorf("Name = %q, want empty - two explicit candidates must abstain", got.Name) + } + if got.Reason != scopeReasonAmbiguous { + t.Errorf("Reason = %q, want %q", got.Reason, scopeReasonAmbiguous) + } +} + +func TestScopeMatchTwoDerivedKeysStayAmbiguous(t *testing.T) { + // The tier-3 case, deliberately NOT resolved in M2. It must abstain rather + // than pick, and the reason must say ambiguous so the log can measure how + // often this happens before tier 3 is built. + payload := []byte{0x11, 0x22} + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshDerived([]string{"aa", "bb"}) + + snap := set.snapshot() + collide := map[string][]byte{"#aa": snap.all["#aa"], "#bb": snap.all["#aa"]} + forced := ®ionKeySnapshot{all: collide, explicit: snap.explicit} + code := codeFor("#aa", 5, payload) + + got := forced.match(5, payload, code) + if got.Name != "" || got.Reason != scopeReasonAmbiguous { + t.Errorf("got %+v, want an empty name with reason %q", got, scopeReasonAmbiguous) + } +} + +// regionSetFromKeys wraps a raw key map as a *regionKeySet with every key +// treated as explicit. Tests written before M2's two-tier type keep working +// unchanged this way, and "all keys explicit" is precisely what a hashRegions +// map meant back then - so the #1609 ambiguity semantics they assert are +// preserved exactly: two explicit candidates still abstain. +func regionSetFromKeys(keys map[string][]byte) *regionKeySet { + names := make(map[string]bool, len(keys)) + all := make(map[string][]byte, len(keys)) + for n, k := range keys { + names[n] = true + all[n] = k + } + s := ®ionKeySet{} + s.cur.Store(®ionKeySnapshot{all: all, explicit: names}) + return s +} + +// matchScopeName reproduces the removed matchScope's signature over the new +// type, so the tests written against it keep asserting the behaviour they were +// written for rather than being rewritten alongside the change they guard. +func matchScopeName(keys map[string][]byte, payloadType byte, payloadRaw []byte, code1 string) string { + return regionSetFromKeys(keys).snapshot().match(payloadType, payloadRaw, code1).Name +} + +func TestDeclaredRegionSourcesAggregatesLatestAnswerPerTarget(t *testing.T) { + store := newTestStore(t) + // Two answers from the same target: only the newer one counts, exactly as + // CurrentDeclaredRegions orders (by observed_at, never ingested_at - a + // drive buffered offline can arrive days late). + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-01T00:00:00Z", "be,old") + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "be,new") + insertDeclaredRegionsRow(t, store, "bb"+strings.Repeat("22", 31), "2026-09-05T00:00:00Z", "be") + + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + byName := map[string]declaredRegionStat{} + for _, s := range stats { + byName[s.Name] = s + } + if got := byName["be"].Declarers; got != 2 { + t.Errorf("be declarers = %d, want 2", got) + } + if got := byName["be"].LastSeen; got != "2026-09-07T00:00:00Z" { + t.Errorf("be lastSeen = %q, want the greatest observed_at", got) + } + if _, ok := byName["old"]; ok { + t.Error("want the superseded answer's region gone - only the latest answer per target counts") + } + if got := byName["new"].Declarers; got != 1 { + t.Errorf("new declarers = %d, want 1", got) + } +} + +func TestDeclaredRegionSourcesIgnoresWildcard(t *testing.T) { + // '*' is the wildcard, not a region name. Deriving a key for it would add + // a permanent no-op entry to the cap on nearly every deployment. + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "cc"+strings.Repeat("33", 31), "2026-09-07T00:00:00Z", "*,be") + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + for _, s := range stats { + if s.Name == "*" { + t.Error("want '*' excluded - it is the wildcard, not a region") + } + } + if len(stats) != 1 { + t.Errorf("stats = %+v, want just be", stats) + } +} + +// insertDeclaredRegionsRow seeds one node_declared_regions answer, creating +// the table first. That table is optional: this ingestor never creates it, and +// declaredRegionSources probes sqlite_master before reading it. A test that +// wants the optional source has to bring it. +func insertDeclaredRegionsRow(t *testing.T, s *Store, target, observedAt, regionsCSV string) { + t.Helper() + if _, err := s.db.Exec(` + CREATE TABLE IF NOT EXISTS node_declared_regions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + target TEXT NOT NULL, + rx_pubkey TEXT NOT NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + regions_csv TEXT NOT NULL, + truncated INTEGER NOT NULL DEFAULT 0, + UNIQUE(target, rx_pubkey, observed_at) + )`); err != nil { + t.Fatal(err) + } + if _, err := s.db.Exec( + `INSERT INTO node_declared_regions (target, rx_pubkey, observed_at, ingested_at, regions_csv, truncated) + VALUES (?, 'rx', ?, ?, ?, 0)`, target, observedAt, observedAt, regionsCSV); err != nil { + t.Fatal(err) + } +} + +func TestRefreshFromStoreIsNoOpWhenDisabled(t *testing.T) { + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "behss") + + cfg := &Config{HashRegions: []string{"#be"}} // autoRegionKeys absent + set := newRegionKeySet(cfg) + before := len(set.snapshot().all) + set.refreshFromStore(store) + + if got := len(set.snapshot().all); got != before { + t.Errorf("key count %d -> %d with autoRegionKeys off, want unchanged", before, got) + } + if _, ok := set.snapshot().all["#behss"]; ok { + t.Error("a declared name became a key with the feature disabled - this is the safety property the default-off promise rests on") + } +} + +func TestRefreshFromStoreDerivesWhenEnabled(t *testing.T) { + store := newTestStore(t) + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "behss,fm-112") + + cfg := &Config{HashRegions: []string{"#be"}, AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}} + set := newRegionKeySet(cfg) + set.refreshFromStore(store) + + snap := set.snapshot() + for _, want := range []string{"#be", "#behss", "#fm-112"} { + if _, ok := snap.all[want]; !ok { + t.Errorf("want %s in force, got %v", want, keyNames(snap)) + } + } + if !snap.isExplicit("#be") || snap.isExplicit("#behss") { + t.Error("tiers crossed: #be must stay explicit, #behss must be derived") + } +} + +func TestRefreshFromStoreHonoursTheCap(t *testing.T) { + store := newTestStore(t) + // Three names, one declared twice so the ranking is not a coin flip. + insertDeclaredRegionsRow(t, store, "aa"+strings.Repeat("11", 31), "2026-09-07T00:00:00Z", "wide,narrow1") + insertDeclaredRegionsRow(t, store, "bb"+strings.Repeat("22", 31), "2026-09-07T00:00:00Z", "wide,narrow2") + + cfg := &Config{AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true, MaxDerived: 1}} + set := newRegionKeySet(cfg) + set.refreshFromStore(store) + + snap := set.snapshot() + if len(snap.all) != 1 { + t.Fatalf("keys = %v, want exactly 1 under maxDerived=1", keyNames(snap)) + } + if _, ok := snap.all["#wide"]; !ok { + t.Errorf("keys = %v, want the twice-declared name kept, not a one-off", keyNames(snap)) + } +} + +// BenchmarkScopeMatch sweeps key-set size because the cost is linear in it and +// cannot be reduced: code1 is an HMAC over the packet payload, so there is no +// payload-independent lookup key to index on. The sweep is the evidence for +// choosing maxDerived, not a single before/after number - the explicit tier's +// size is operator config and varies per deployment. +func BenchmarkScopeMatch(b *testing.B) { + payload := make([]byte, 51) // a typical GRP_TXT payload + for i := range payload { + payload[i] = byte(i) + } + for _, n := range []int{16, 58, 180, 314} { + b.Run(fmt.Sprintf("keys=%d", n), func(b *testing.B) { + all := make(map[string][]byte, n) + explicit := make(map[string]bool, n) + for i := 0; i < n; i++ { + name := fmt.Sprintf("#r%04d", i) + sum := sha256.Sum256([]byte(name)) + all[name] = sum[:16] + explicit[name] = true + } + snap := ®ionKeySnapshot{all: all, explicit: explicit} + code := codeFor("#r0000", 5, payload) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = snap.match(5, payload, code) + } + }) + } +} + +// TestRegionKeySetConcurrentRefreshAndMatch drives the shape the type exists +// for and that nothing else in this suite produces: ingest goroutines naming +// packets while the refresh ticker swaps the key set underneath them. +// +// It is written for `go test -race`. Run without it, it asserts only that the +// explicit tier keeps naming its packet across 200 swaps, which is worth +// little; run with it, the detector sees every unsynchronised read of the +// snapshot that a real refresh would expose. Until this existed, a race run of +// this package proved nothing about regionKeySet, because refreshDerived and +// match were never called concurrently anywhere in it — the absence of a +// finding was the absence of an experiment. +// +// The final assertion holds regardless of what the churn derives: a derived key +// colliding with #be on this payload resolves to the explicit key by tier 2, +// and #be is the only explicit key here, so there is no second explicit +// candidate that could make the answer ambiguous. +func TestRegionKeySetConcurrentRefreshAndMatch(t *testing.T) { + const explicitName = "#be" + payload := []byte{0x11, 0x22, 0x33, 0x44} + code1 := codeFor(explicitName, 5, payload) + + set := newRegionKeySet(&Config{ + HashRegions: []string{explicitName}, + AutoRegionKeys: &AutoRegionKeysConfig{Enabled: true}, + }) + + var wg sync.WaitGroup + stop := make(chan struct{}) + + for i := 0; i < 8; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + select { + case <-stop: + return + default: + } + // snapshot() then match() is exactly the ingest hot path: + // one atomic load, then reads of the maps it points at. + if got := set.snapshot().match(5, payload, code1); got.Name != explicitName && got.Reason != scopeReasonAmbiguous { + // Reported rather than fataled: t.Fatalf from a + // non-test goroutine is undefined behaviour. + t.Errorf("match during refresh returned %q (%s), want %q", got.Name, got.Reason, explicitName) + return + } + } + }() + } + + wg.Add(1) + go func() { + defer wg.Done() + defer close(stop) + for i := 0; i < 200; i++ { + // A different derived set each time, so every iteration really + // builds and swaps a new snapshot rather than reusing one. + set.refreshDerived([]string{fmt.Sprintf("r%03d", i%7), fmt.Sprintf("q%03d", i)}) + } + }() + + wg.Wait() + + if got := set.snapshot().match(5, payload, code1); got.Name != explicitName { + t.Errorf("after 200 refreshes match = %q (%s), want %q — the explicit tier must survive every swap", + got.Name, got.Reason, explicitName) + } +} + +// TestDeclaredRegionSourcesReadsConfiguredScope is the upstream-native source: +// nodes.configured_scope, written by the observer /neighbors ingestion. It is +// the only source a stock install has, so a derived tier that read only the +// optional table would be permanently empty here. +// +// The stored form carries the leading "#" (normalizeScopeList puts it there), +// which is exactly the spelling regionNameAcceptable has to accept. +func TestDeclaredRegionSourcesReadsConfiguredScope(t *testing.T) { + store := newTestStore(t) + seedNodeWithConfiguredScope(t, store, "aa"+strings.Repeat("11", 31), "#be,#nl-li,*", "2026-09-07T00:00:00Z") + seedNodeWithConfiguredScope(t, store, "bb"+strings.Repeat("22", 31), "#be", "2026-09-08T00:00:00Z") + + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + byName := map[string]declaredRegionStat{} + for _, s := range stats { + byName[s.Name] = s + } + if got := byName["#be"].Declarers; got != 2 { + t.Errorf("#be declarers = %d, want 2 — both nodes declare it", got) + } + if got := byName["#be"].LastSeen; got != "2026-09-08T00:00:00Z" { + t.Errorf("#be lastSeen = %q, want the newer of the two answers", got) + } + if got := byName["#nl-li"].Declarers; got != 1 { + t.Errorf("#nl-li declarers = %d, want 1", got) + } + if _, ok := byName["*"]; ok { + t.Error("want '*' excluded from the declared-name count — it is the wildcard, and nearly every node declares it") + } +} + +// TestDeclaredRegionSourcesMergesBothSources: an instance that has both the +// column and the optional table must see a name declared in either, and count +// a node once per name rather than once per source. +func TestDeclaredRegionSourcesMergesBothSources(t *testing.T) { + store := newTestStore(t) + pk := "cc" + strings.Repeat("33", 31) + seedNodeWithConfiguredScope(t, store, pk, "#be", "2026-09-07T00:00:00Z") + insertDeclaredRegionsRow(t, store, pk, "2026-09-08T00:00:00Z", "fm-112") + + stats, err := store.declaredRegionSources() + if err != nil { + t.Fatal(err) + } + names := map[string]int{} + for _, s := range stats { + names[s.Name] = s.Declarers + } + if names["#be"] != 1 { + t.Errorf("#be declarers = %d, want 1 (from configured_scope)", names["#be"]) + } + if names["fm-112"] != 1 { + t.Errorf("fm-112 declarers = %d, want 1 (from the optional table)", names["fm-112"]) + } + // Both spellings of the same region reach the same derived key, which is + // what regionNameAcceptable's "#"-stripping is for. Asserted there rather + // than here; this only pins that neither source is dropped. +} + +// seedNodeWithConfiguredScope writes one node with a confirmed scope list, the +// shape handleNeighborsReport produces. +func seedNodeWithConfiguredScope(t *testing.T, s *Store, pubkey, scopes, at string) { + t.Helper() + if _, err := s.db.Exec( + `INSERT INTO nodes (public_key, configured_scope, configured_scope_at) VALUES (?, ?, ?)`, + pubkey, scopes, at); err != nil { + t.Fatal(err) + } +} diff --git a/config.example.json b/config.example.json index 930a224c7..f539f006f 100644 --- a/config.example.json +++ b/config.example.json @@ -397,6 +397,8 @@ "#eu" ], "_comment_hashRegions": "Region names for scope matching on transport-route packets. Key = SHA256('#name')[:16]. Add any region names used by nodes in your network.", + "autoRegionKeys": { "enabled": false, "maxDerived": 256, "refreshMinutes": 15 }, + "_comment_autoRegionKeys": "Opt-in: derive region keys from the region names nodes are CONFIRMED to be configured for, on top of the explicit hashRegions list above. Default OFF, and an absent block leaves behaviour unchanged. Sources: nodes.configured_scope (written by the observer /neighbors ingestion) and, where a deployment fills it, the optional node_declared_regions table. Solves the case where a repeater forwards a region this instance holds no key for: its traffic is stored unmatched, and anything comparing declared against observed reports the region as absent rather than unnameable. TOP-LEVEL FLAG, a sibling of hashRegions: config loading is plain json.Unmarshal with no DisallowUnknownFields, so nesting it elsewhere is silently ignored and the feature stays off with no error. maxDerived caps the derived tier (default 256): each key costs one HMAC per transport-scoped packet and raises the random 2-byte collision rate by 1/65536, and the match cannot be indexed because the code is an HMAC over the payload. Over the cap, names are kept by how many distinct nodes declare them, so a one-off local name is dropped before a region half the network uses. refreshMinutes is how often the derived tier is rebuilt (default 15).", "_comment_defaultRegion": "IATA code shown by default in region filters.", "_comment_mapDefaults": "Initial map center [lat, lon] and zoom level.", "_comment_regions": "IATA code → display name mapping for the region filter UI. Each key is a 3-letter IATA code that an observer is tagged with (resolved priority: MQTT payload `region` field > topic-derived region > mqttSources.region). Observers without an IATA tag will not appear under any region filter — only under 'All Regions'. The region filter dropdown shows one entry per code listed here PLUS any extra IATA codes the server discovers from observers at runtime (so you can omit codes here and they will still be selectable, just labelled with the bare IATA code instead of a friendly name). Selecting 'All Regions' (or no region) returns results from every observer including those with no IATA tag; selecting one or more codes restricts results to packets observed by observers tagged with those codes. The reserved value 'All' (case-insensitive) is treated as 'no filter' on the server, so the URL ?region=All behaves identically to omitting the param. Issue #770.", From 8c164c5315f30d222e42b82408f2d1d0c7fc10ff Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 9 Sep 2026 17:58:09 +0200 Subject: [PATCH 017/151] feat(scope-audit): verify a declared region against the repeater's own traffic (#1990) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #1987, and the point of counting that traffic in the first place. A region this instance holds no `hashRegions` key for is **unnameable, not absent**. #1987 says so with a caveat chip. This settles it wherever the evidence allows: derive `SHA256("#region")[:16]` from the repeater's own declaration and HMAC that repeater's own unmatched packets with it. Same computation the ingestor performs at ingest, with the candidate set narrowed from every configured key to this repeater's handful of declarations. Where it fires, a grey "declared but not observed" chip becomes a green one and the caveat count shrinks by the packets it explained. ## Two packets, not one `code1` is two bytes, so an unrelated name matches a given packet with probability 1/65536. Across ~400 unmatched packets and ~124 declared names, chance alone produces roughly one false match per refresh. Two matches on the same region for the same repeater is (1/65536)², about one in four billion. Lowering the threshold to one would not make this noisy, it would make it **unsound**, so the constant carries that arithmetic and a test rather than a comment. A region with exactly one hit stays grey and reports its single hit, so the page can say why it is still shown as not observed instead of leaving the reader to wonder. ## What it deliberately does not do **It writes nothing.** Read-time only. A wrong answer expires with the window instead of sitting in `transmissions.scope_name` until someone runs a repair, and `cmd/server` stays read-only per the invariant in AGENTS.md. **`notObserved` remains the single source of chip colour.** `regionEvidence` says only HOW a region was established. Two fields that can disagree about the same fact is how this column got confusing in the first place. ## Rule 0, including the part that was wrong at first The naive shape is `targets × names × packets` HMACs: 205 × 124 × 400 ≈ 10M. Caching per `(region, transmission)` pair cuts the HMACs to ~50k. **That measured 501ms**, because the HMACs had become a rounding error while the *iteration* stayed cubic at 10.2M map lookups. Re-keyed per region, holding the set of matching transmissions, it is **36ms** at the same worst-case shape: a region is HMACed over every packet once, and a target then asks one question per declared region instead of one per (region, packet). Most declared regions match nothing, so the common case is a single map lookup and no packet loop at all. `hmacCount` exists so a test can assert the first mistake cannot come back; the benchmark exists because only it caught the second. ## Both axes are bounded, because neither is bounded by the data The "~400 packets in a 7 day window" this was sized against is a property of one instance's configuration, not of the feature: the ingestor stores the unnameable state for every transport-scoped packet no configured key names, so an instance with few or no `hashRegions` entries — the stock state, and the one this helps most — has **every** scoped packet in that set. - the window query takes the 4096 most recent candidates and reports truncation, which the handler logs, so a grey chip on a sampled refresh is not read as "not forwarded" - the declared list is capped at 32 names per repeater: it arrives from a collector that validates each entry's shape but never how many entries there are - measured at the cap: **306ms** for 205 targets over 124 names, against 29ms for the shape a real network produces Because both caps make the evidence a sample, the response carries `observedUnmatchedSampled`. Without it a client subtracts a capped numerator from an uncapped total and overstates the unexplained traffic with no way to know it is doing so. The chip subtracts only evidence for regions **absent** from `notObserved` — a single-hit region the server refused to accept is not called explained either — and says "at most N" when the count was sampled. ## Verified on live data Six repeaters clear the threshold in a 7d window on a real instance. One of them: `nl-nb` green with 3 corroborating packets and the tooltip stating the count, `belml` still grey on 1, and the caveat chip reading 31 of 34 packets unexplained rather than 30. ## Tests `scope_verify_test.go` covers the HMAC-input walk against a real transport-flood packet captured from a live instance (a hand-built fixture would only prove the parser agrees with itself), that `regionCode` does not fold case, the threshold in both directions, the memo's HMAC count, both bounds with their truncation flag, and the benchmark at cap size. Handler-level tests cover a region verified into green, a single hit left grey with its count reported, and the sample-size field. `cd cmd/server && go test ./...` passes (77s), frontend 723 assertions pass, `go vet` and `gofmt -l` clean. --- cmd/server/scope_audit.go | 99 +++++++++- cmd/server/scope_audit_test.go | 119 +++++++++++ cmd/server/scope_verify.go | 321 ++++++++++++++++++++++++++++++ cmd/server/scope_verify_test.go | 339 ++++++++++++++++++++++++++++++++ docs/api-spec.md | 32 ++- public/scope-audit.css | 5 + public/scope-audit.js | 69 +++++-- test-frontend-helpers.js | 101 +++++++++- 8 files changed, 1063 insertions(+), 22 deletions(-) create mode 100644 cmd/server/scope_verify.go create mode 100644 cmd/server/scope_verify_test.go diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index 02f3c89dc..d7cf5de06 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -22,6 +22,7 @@ package main import ( "database/sql" "fmt" + "log" "net/http" "sort" "strings" @@ -297,6 +298,16 @@ type scopeAuditTargetAgg struct { // repeater is not forwarding. unmatchedPackets int64 + // unmatchedTxIDs are the transmissions behind unmatchedPackets, kept so + // declared-region verification can test this target's own declarations + // against this target's own unnameable traffic (scope_verify.go). The same + // (target, txID) de-duplication that guards unmatchedPackets guards this, + // so one packet reaching a target by two hops cannot corroborate twice. + // + // Bounded by scopeVerifyMaxPacketsPerTarget, which unmatchedPackets is NOT: + // the count stays the honest total, this is the working set. + unmatchedTxIDs []int64 + // ambiguousHops counts forwarder-hop observations in the window whose // truncated hash prefix matched this target AND at least one other // declared target — see ScopeAuditForwarding's doc comment for why @@ -643,6 +654,9 @@ func (s *PacketStore) ScopeAuditForwarding(sinceISO string, targets []string) (m // finding on this row may be a gap in this instance's // hashRegions rather than in the repeater's forwarding. agg.unmatchedPackets++ + if len(agg.unmatchedTxIDs) < scopeVerifyMaxPacketsPerTarget { + agg.unmatchedTxIDs = append(agg.unmatchedTxIDs, txID) + } continue } name := normScope(scopeName.String) @@ -777,6 +791,32 @@ type ScopeAuditRow struct { // it never feeds WildcardContradiction, which counts only plain unscoped // floods. ObservedUnmatchedPackets int64 `json:"observedUnmatchedPackets"` + + // ObservedUnmatchedSampled is how many of those packets verification could + // actually look at: the per-target list is capped at + // scopeVerifyMaxPacketsPerTarget and the window sample at + // scopeVerifyMaxWindowPackets, while ObservedUnmatchedPackets keeps + // counting past both. + // + // It exists because a client cannot otherwise subtract the two fields + // honestly. RegionEvidence can only ever count packets inside this sample, + // so on a row where this is smaller than ObservedUnmatchedPackets the + // difference between them is an upper bound on the unexplained traffic, not + // a figure. Equal values mean the subtraction is exact. + ObservedUnmatchedSampled int64 `json:"observedUnmatchedSampled"` + + // RegionEvidence maps a declared region to how many of this repeater's own + // unmatched forwarded packets derive to it — see scope_verify.go. A region + // reaching scopeVerifyMinCorroboration is removed from NotObserved, so this + // field is NOT what decides the chip's colour; NotObserved remains the sole + // source of that. This exists so a client can say HOW a region was + // established, and can explain a region that got exactly one hit and + // therefore stayed in NotObserved. + // + // Absent regions simply had no matching traffic. Never nil in the response + // — an empty object and a missing key mean the same thing, and an empty map + // is the cheaper thing for a client to iterate. + RegionEvidence map[string]int `json:"regionEvidence"` } // ScopeAuditResponse is the payload for GET /api/scope-audit. Only @@ -1036,6 +1076,33 @@ func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse } } + // Declared-region verification: a region this instance holds no key for is + // unnameable, not absent, and the audit can settle which by deriving the key + // from the repeater's own declaration and testing it against that + // repeater's own unnameable traffic. One verifier serves every row so each + // (region, transmission) pair is derived at most once — see scope_verify.go + // for why that memo is what keeps this affordable. + // + // A failure here degrades to "no verification" rather than failing the + // request: the audit was useful before this existed and must stay useful if + // the extra query errors. + var verifier *scopeVerifier + if s.store != nil { + unmatchedRows, truncated, uErr := s.store.unmatchedTransmissionsInWindow(sinceISO) + if uErr != nil { + log.Printf("[scope-audit] declared-region verification unavailable: %v", uErr) + } else { + if truncated { + // Said out loud rather than absorbed: past the cap a region can + // hold evidence this refresh did not look at, so a grey chip + // means "not corroborated in this sample", not "not forwarded". + log.Printf("[scope-audit] window %s holds more than %d unnameable packets; verification used the most recent %d and may under-report evidence", + window, scopeVerifyMaxWindowPackets, scopeVerifyMaxWindowPackets) + } + verifier = newScopeVerifier(unmatchedRows) + } + } + identities := s.db.scopeAuditNodeIdentities(targets) resp := &ScopeAuditResponse{Window: window, Since: sinceISO, Repeaters: []ScopeAuditRow{}} @@ -1072,9 +1139,32 @@ func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse agg := forwarding[pk] - notObserved := []string{} + // Verify the declared regions this repeater has no NAMED evidence for, + // against its own unmatched traffic. Regions already observed by name + // need no verification and are not tested — that keeps the candidate + // set to exactly the open questions, which is also what keeps the + // verifier's work proportional to the problem rather than to the fleet. + unnamed := []string{} for _, rgn := range declaredNamed { if agg == nil || agg.scopes[rgn] == nil { + unnamed = append(unnamed, rgn) + } + } + regionEvidence := map[string]int{} + verifiedSet := map[string]bool{} + if verifier != nil && agg != nil && len(unnamed) > 0 { + // capVerifyRegions bounds the per-target half of the verifier's + // work. The declared list arrives from a collector and its LENGTH + // is not validated anywhere on the way in, while each distinct name + // costs a full pass over the packet set. + regionEvidence = verifier.evidence(agg.unmatchedTxIDs, capVerifyRegions(unnamed)) + for _, rgn := range verifier.verified(regionEvidence) { + verifiedSet[rgn] = true + } + } + notObserved := []string{} + for _, rgn := range unnamed { + if !verifiedSet[rgn] { notObserved = append(notObserved, rgn) } } @@ -1093,11 +1183,14 @@ func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse } } - var unscopedPackets, ambiguousHops, unmatchedPackets int64 + var unscopedPackets, ambiguousHops, unmatchedPackets, unmatchedSampled int64 if agg != nil { unscopedPackets = agg.unscopedPackets ambiguousHops = agg.ambiguousHops unmatchedPackets = agg.unmatchedPackets + // What verification could actually see, against what was counted. + // The list is capped; the count is not. + unmatchedSampled = int64(len(agg.unmatchedTxIDs)) } resp.Repeaters = append(resp.Repeaters, ScopeAuditRow{ @@ -1115,6 +1208,8 @@ func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse WildcardContradiction: unscopedPackets > 0 && !declaredWildcard, AmbiguousHops: ambiguousHops, ObservedUnmatchedPackets: unmatchedPackets, + ObservedUnmatchedSampled: unmatchedSampled, + RegionEvidence: regionEvidence, }) } diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 9bb17bc27..749cfec76 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -1173,3 +1173,122 @@ func TestHandleScopeAuditSurfacesUnmatchedPackets(t *testing.T) { t.Error("wildcardContradiction = true, want false — unmatched traffic is scoped, so it says nothing about '*'") } } + +// TestHandleScopeAuditVerifiesDeclaredRegion is the case this milestone exists +// for, built from the real packet that started the investigation. A repeater +// declares "fm-112"; this instance holds no key for it, so both packets it +// forwarded are stored unmatched. Verification derives the key from the +// repeater's own declaration, finds two corroborating packets, and the region +// must leave notObserved with its evidence count reported. +func TestHandleScopeAuditVerifiesDeclaredRegion(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112,behss", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + + got := getScopeAudit(t, router, "") + if len(got.Repeaters) != 1 { + t.Fatalf("repeaters = %+v, want 1", got.Repeaters) + } + row := got.Repeaters[0] + if row.RegionEvidence["fm-112"] != 2 { + t.Errorf("regionEvidence = %v, want fm-112:2", row.RegionEvidence) + } + for _, n := range row.NotObserved { + if n == "fm-112" { + t.Errorf("notObserved = %v, must not contain fm-112 - two corroborating packets prove it is forwarded", row.NotObserved) + } + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "behss" { + t.Errorf("notObserved = %v, want [behss] - that region has no corroborating traffic here", row.NotObserved) + } +} + +// TestHandleScopeAuditReportsTheVerificationSampleSize pins the field a client +// needs to subtract RegionEvidence from ObservedUnmatchedPackets honestly. +// RegionEvidence can only ever count packets inside the sample, so a client +// that subtracts it from an uncapped total overstates what is unexplained. The +// two are equal here, which is the case that says "this subtraction is exact". + +// TestHandleScopeAuditDoesNotVerifyOnOnePacket: a single match is 1-in-65536 +// and must leave the region in notObserved, with its count still reported so a +// client can say "one hit, not enough". +func TestHandleScopeAuditDoesNotVerifyOnOnePacket(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + + got := getScopeAudit(t, router, "") + row := got.Repeaters[0] + if row.RegionEvidence["fm-112"] != 1 { + t.Errorf("regionEvidence = %v, want fm-112:1", row.RegionEvidence) + } + if len(row.NotObserved) != 1 || row.NotObserved[0] != "fm-112" { + t.Errorf("notObserved = %v, want [fm-112] - one corroborating packet is not evidence", row.NotObserved) + } +} + +// TestHandleScopeAuditLeavesCleanRowsAlone: a repeater whose declared regions +// are all observed by name, with no unmatched traffic at all, must be untouched +// by verification - no evidence, no change to notObserved, and an empty (not +// null) regionEvidence so a client can iterate it without a guard. + +// TestHandleScopeAuditReportsTheVerificationSampleSize pins the field a client +// needs to subtract RegionEvidence from ObservedUnmatchedPackets honestly. +// RegionEvidence can only ever count packets inside the sample, so a client +// that subtracts it from an uncapped total overstates what is unexplained. The +// two are equal here, which is the case that says "this subtraction is exact". +func TestHandleScopeAuditReportsTheVerificationSampleSize(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + + row := getScopeAudit(t, router, "").Repeaters[0] + if row.ObservedUnmatchedPackets != 2 { + t.Fatalf("observedUnmatchedPackets = %d, want 2", row.ObservedUnmatchedPackets) + } + if row.ObservedUnmatchedSampled != row.ObservedUnmatchedPackets { + t.Errorf("observedUnmatchedSampled = %d, want %d — nothing was capped away at this size, and a client can only tell from this field", + row.ObservedUnmatchedSampled, row.ObservedUnmatchedPackets) + } +} + +// TestHandleScopeAuditDoesNotVerifyOnOnePacket: a single match is 1-in-65536 +// and must leave the region in notObserved, with its count still reported so a +// client can say "one hit, not enough". + +// seedUnmatchedRawAt seeds one unmatched transmission carrying a real raw_hex, +// attributed to forwarder. Distinct from seedTransmissionRouteAt, which seeds +// raw_hex 'AA' - fine for tests that never parse it, useless here. +func seedUnmatchedRawAt(t *testing.T, s *PacketStore, forwarder, rawHex string, routeType int, firstSeen string) { + t.Helper() + scopeSeedCounter++ + hash := fmt.Sprintf("scoperaw%d", scopeSeedCounter) + res, err := s.db.conn.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, code1, code2, scope_name) + VALUES (?, ?, ?, ?, 5, '9209', '0000', '')`, + rawHex, hash, firstSeen, routeType) + if err != nil { + t.Fatal(err) + } + txID, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + if _, err := s.db.conn.Exec( + `INSERT INTO observations (transmission_id, path_json, timestamp) VALUES (?, ?, 0)`, + txID, `["`+strings.ToUpper(forwarder)+`"]`); err != nil { + t.Fatal(err) + } +} + +// TestHandleNodeScopesDifferentWindowIsSeparateCacheEntry confirms the cache +// key includes window: a request for a different window must recompute +// rather than reuse another window's cached entry. diff --git a/cmd/server/scope_verify.go b/cmd/server/scope_verify.go new file mode 100644 index 000000000..4f460ef26 --- /dev/null +++ b/cmd/server/scope_verify.go @@ -0,0 +1,321 @@ +package main + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" +) + +// scopeVerifyMaxPacketsPerTarget bounds the per-target evidence list. AGENTS.md +// rule 0 forbids unbounded structures, and the corroboration threshold is 2 — +// past a few hundred packets more evidence changes no verdict, it only costs +// memory. scopeAuditTargetAgg.unmatchedPackets keeps counting past this: the +// count is the honest total, the list is the working set. +const scopeVerifyMaxPacketsPerTarget = 512 + +// scopeVerifyMaxWindowPackets bounds the OTHER axis: how many unnameable +// packets one refresh will hold and derive over. +// +// The "~400 packets in a 7 day window" this feature was sized against is a +// property of THIS instance's configuration, not of the feature. The ingestor +// stores an EMPTY scope_name for every transport-scoped packet no configured +// key names (scopeNameForDB) — spelled out in words for the reason the query +// below gives — so the fewer hashRegions an instance has, the more +// rows land here — and an instance with none at all puts every scoped packet +// in this set. That is the stock state, and it is the state this feature was +// built to help. +// +// The cost is one HMAC per distinct region name per packet, measured at ~0.7µs +// (BenchmarkScopeVerifierAudit: 124 names over 400 packets in 36ms). At this +// cap and 124 distinct declared names that is ~500k HMACs, roughly 0.35s of +// worst case on an endpoint that already costs seconds and caches for minutes. +// Ten times this cap would not: it would be seconds of HMAC on every refresh. +const scopeVerifyMaxWindowPackets = 4096 + +// scopeVerifyMaxRegionsPerTarget bounds how many of one repeater's declared +// names are put to the verifier. Every distinct name costs a full pass over +// the packet set, and the list is client-supplied: handleClientRegions +// (cmd/ingestor/client_regions.go) validates the target pubkey and each entry's +// shape, but nothing limits how many entries one companion may report. The +// longest genuine list on this network declares 21 regions, and the firmware +// exports a short label set by construction. +const scopeVerifyMaxRegionsPerTarget = 32 + +// capVerifyRegions applies scopeVerifyMaxRegionsPerTarget, keeping the first +// entries in the order the caller supplied. It does not sort or rank: a +// declared list has no priority order to respect, and reordering here would +// make which regions get verified depend on something the reader cannot see. +func capVerifyRegions(regions []string) []string { + if len(regions) <= scopeVerifyMaxRegionsPerTarget { + return regions + } + return regions[:scopeVerifyMaxRegionsPerTarget] +} + +// scopeHMACInputs pulls the three values needed to test a region hypothesis +// against one packet: the payload type and raw payload bytes the sender HMACed, +// and the resulting two-byte code it put on the wire. +// +// It deliberately does NOT call DecodePacket. That runs decodePayload, which +// attempts channel decryption and signature validation — work this has no use +// for, repeated over every unmatched packet on every audit refresh. Walking the +// offsets is all that is needed, and it reuses decodeHeader/isTransportRoute/ +// decodePath so the offset arithmetic is not duplicated from DecodePacket. +// +// ok is false for anything that cannot carry a region scope: malformed hex, a +// truncated header, an invalid path byte, or a non-transport route. A plain +// FLOOD packet has no transport codes at all, so there is no code1 to compare +// against and HMACing it could only waste time. +func scopeHMACInputs(rawHex string) (payloadType byte, payload []byte, code1 string, ok bool) { + buf, err := hex.DecodeString(strings.TrimSpace(rawHex)) + if err != nil || len(buf) < 2 { + return 0, nil, "", false + } + header := decodeHeader(buf[0]) + if !isTransportRoute(header.RouteType) { + return 0, nil, "", false + } + offset := 1 + if len(buf) < offset+4 { + return 0, nil, "", false + } + code1 = strings.ToUpper(hex.EncodeToString(buf[offset : offset+2])) + offset += 4 // code1 and code2 + + if offset >= len(buf) { + return 0, nil, "", false + } + pathByte := buf[offset] + offset++ + _, consumed, decodeErr := decodePath(pathByte, buf, offset) + if decodeErr != nil { + return 0, nil, "", false + } + offset += consumed + if offset > len(buf) { + return 0, nil, "", false + } + rest := buf[offset:] + if len(rest) == 0 { + return 0, nil, "", false + } + return byte(header.PayloadType), rest, code1, true +} + +// regionCode derives the on-wire code1 a sender in region name would emit for +// this payload — the forward direction of what matchingRegions inverts in the +// ingestor (cmd/ingestor/main.go). The two must stay in step: key is +// SHA256("#name")[:16], the MAC covers payloadType followed by the payload, the +// code is the first two MAC bytes little-endian, and 0x0000/0xFFFF are reserved +// and nudged. Any divergence here silently produces regions that never verify. +// +// The leading '#' is optional because callers hold normScope'd names (the audit +// strips it) while the key is over the '#'-prefixed form. +// +// Case is significant and must stay so: the key is a hash over the raw bytes of +// "#name", so "#BEHSS" and "#behss" are different regions on the wire. +func regionCode(name string, payloadType byte, payload []byte) string { + if !strings.HasPrefix(name, "#") { + name = "#" + name + } + sum := sha256.Sum256([]byte(name)) + mac := hmac.New(sha256.New, sum[:16]) + mac.Write([]byte{payloadType}) + mac.Write(payload) + h := mac.Sum(nil) + code := uint16(h[0]) | uint16(h[1])<<8 + if code == 0 { + code = 1 + } else if code == 0xFFFF { + code = 0xFFFE + } + return strings.ToUpper(hex.EncodeToString([]byte{byte(code & 0xFF), byte(code >> 8)})) +} + +// unmatchedTransmissionRow is one transmission that carried a transport scope +// no configured region key matched, with the raw bytes needed to test a region +// hypothesis against it. +type unmatchedTransmissionRow struct { + txID int64 + rawHex string +} + +// unmatchedTransmissionsInWindow is the SECOND, narrow query behind the audit - +// deliberately not a widening of scopeAuditForwarderScanQuery. +// +// That scan returns one row per hop per flood packet: on a 2,000-packet sample +// after M0 that is 19,049 rows, and carrying raw_hex on every one of them would +// load the hot path to serve a few hundred packets. This selects only the +// transmissions that are actually candidates - an empty scope_name inside the +// window, ~400 over 7 days on the reference deployment - and the main scan is +// left exactly as it is. +// +// An empty scope_name is the "transport-scoped but unnameable" state; NULL +// means the packet carried no scope at all and can never verify against a +// region. The route filter matches the forwarder scan's, so the two agree on +// which packets count as forwarded. +// +// "Empty" is spelled out above rather than written as the two-single-quote +// literal on purpose: gofmt applies the old godoc typographic substitution +// inside doc comments and rewrites that digraph into a closing curly quote, +// which silently misstates the one value this query keys on — and puts it back +// on every gofmt run. +// +// Selection only: a row whose raw_hex cannot be walked is still returned, and +// dropped by newScopeVerifier. Filtering that in SQL is not possible and +// filtering it here would hide how many candidates the window actually held. +// The second return value reports that the window held more than +// scopeVerifyMaxWindowPackets rows and the answer is a sample of the most +// recent ones. Newest-first because a partial answer drawn from the most recent +// traffic is the one that matches what the window claims to describe: a +// repeater still forwarding a region is likelier to have done so recently, and +// verification only ever needs two corroborating packets. +func (s *PacketStore) unmatchedTransmissionsInWindow(sinceISO string) ([]unmatchedTransmissionRow, bool, error) { + rows, err := s.db.conn.Query(` + SELECT t.id, t.raw_hex + FROM transmissions t + WHERE t.first_seen >= ? + AND t.scope_name = '' + AND `+scopeConformanceForwarderRouteTypesSQL+` + ORDER BY t.first_seen DESC + LIMIT ?`, sinceISO, scopeVerifyMaxWindowPackets) + if err != nil { + return nil, false, fmt.Errorf("unmatched transmissions scan: %w", err) + } + defer rows.Close() + + var out []unmatchedTransmissionRow + for rows.Next() { + var r unmatchedTransmissionRow + if err := rows.Scan(&r.txID, &r.rawHex); err != nil { + return nil, false, fmt.Errorf("unmatched transmissions scan row: %w", err) + } + out = append(out, r) + } + if err := rows.Err(); err != nil { + return nil, false, fmt.Errorf("unmatched transmissions rows: %w", err) + } + return out, len(out) == scopeVerifyMaxWindowPackets, nil +} + +// scopeVerifyMinCorroboration is how many of a repeater's own unmatched packets +// must derive to a declared region before that region counts as observed. +// +// One is not enough, and the arithmetic is the whole argument: code1 is two +// bytes, so an unrelated name matches a given packet with probability 1/65536. +// Across ~400 unmatched packets and ~124 distinct declared names, chance alone +// produces roughly one false match per refresh. Two matches on the same region +// for the same repeater is (1/65536)^2 - about one in four billion. Raising +// this costs recall on quiet regions; lowering it to 1 makes the feature +// unsound, not merely noisy. +const scopeVerifyMinCorroboration = 2 + +// scopeVerifier answers "how many of these transmissions are region X" while +// deriving each region's code over each packet at most once. +// +// The cache is not a nicety, and where it has to sit was measured rather than +// guessed. Naively the audit does targets x declaredNames x unmatchedPackets +// HMACs — 205 x 124 x 400 is roughly 10,000,000. Caching per +// (region, transmission) pair cuts the HMACs to names x packets, ~50,000, but +// leaves the ITERATION cubic: a benchmark of that shape spent 501ms on 10.2M +// map lookups at ~49ns each, with the HMACs a rounding error beside it. +// +// So the cache is keyed per REGION, holding the set of transmissions that +// derive to it. A region is HMACed over every packet once, and a target then +// asks one question per declared region instead of one per (region, packet). +// The overwhelmingly common answer is an empty set — most declared regions +// match nothing — which costs a single lookup and no packet loop at all. +// +// Not safe for concurrent use: one verifier is built per audit computation, +// which handleScopeAudit already serialises behind its cache. +type scopeVerifier struct { + packets map[int64]scopeVerifyInputs + // matchesByRegion caches, per region, the transmissions that derive to it. + // Computed once over every packet, never per target. + matchesByRegion map[string]map[int64]bool + // hmacCount is incremented per actual derivation, asserted by the cache + // test so a future refactor cannot quietly reintroduce the naive cost. + hmacCount int +} + +type scopeVerifyInputs struct { + payloadType byte + payload []byte + code1 string + ok bool +} + +// newScopeVerifier parses each row once. A row whose raw_hex cannot be walked +// is kept with ok=false rather than dropped, so its id still resolves and a +// caller asking about it gets "no evidence" instead of a miss. +func newScopeVerifier(rows []unmatchedTransmissionRow) *scopeVerifier { + v := &scopeVerifier{ + packets: make(map[int64]scopeVerifyInputs, len(rows)), + matchesByRegion: map[string]map[int64]bool{}, + } + for _, r := range rows { + pt, payload, code1, ok := scopeHMACInputs(r.rawHex) + v.packets[r.txID] = scopeVerifyInputs{payloadType: pt, payload: payload, code1: code1, ok: ok} + } + return v +} + +// regionMatches returns the transmissions deriving to region, computing the +// whole set on first ask. Unparseable packets are skipped rather than counted +// as misses, so one malformed row in the window cannot blank a region. +func (v *scopeVerifier) regionMatches(region string) map[int64]bool { + if m, ok := v.matchesByRegion[region]; ok { + return m + } + m := map[int64]bool{} + for txID, in := range v.packets { + if !in.ok { + continue + } + v.hmacCount++ + if regionCode(region, in.payloadType, in.payload) == in.code1 { + m[txID] = true + } + } + v.matchesByRegion[region] = m + return m +} + +// evidence counts, for each declared region, how many of txIDs derive to it. +// Regions with zero matches are absent from the result rather than present +// with 0, so the map is directly the "we found something" set. +func (v *scopeVerifier) evidence(txIDs []int64, declaredRegions []string) map[string]int { + out := map[string]int{} + for _, region := range declaredRegions { + m := v.regionMatches(region) + if len(m) == 0 { + continue // the common case: one lookup, no packet loop + } + n := 0 + for _, txID := range txIDs { + if m[txID] { + n++ + } + } + if n > 0 { + out[region] = n + } + } + return out +} + +// verified returns the regions in an evidence map that clear the corroboration +// threshold, sorted so the response is stable across refreshes. +func (v *scopeVerifier) verified(evidence map[string]int) []string { + var out []string + for region, n := range evidence { + if n >= scopeVerifyMinCorroboration { + out = append(out, region) + } + } + sort.Strings(out) + return out +} diff --git a/cmd/server/scope_verify_test.go b/cmd/server/scope_verify_test.go new file mode 100644 index 000000000..f32f1e8ee --- /dev/null +++ b/cmd/server/scope_verify_test.go @@ -0,0 +1,339 @@ +package main + +import ( + "encoding/hex" + "fmt" + "strings" + "testing" + "time" +) + +// realTransportFloodPacket is transmission 0a065d41d51f1f77 from the live +// instance, captured 2026-09-07. Header 0x14 = route_type 0 (TRANSPORT_FLOOD), +// payload_type 5 (GRP_TXT); transport codes 9209/0000; path byte 0x41 = +// hash_size 2, one hop "E3D3"; the rest is payload. +// +// A hand-built fixture would only prove the parser agrees with itself. This +// packet is the one that started the investigation: its code1 is exactly the +// code #fm-112 derives over its own payload, which is why the audit showed +// fm-112 as "not observed" for a repeater that was forwarding it. +const realTransportFloodPacket = "149209000041E3D3EC2D4481DA70893CD71B763958B064A9AAC011D54223FF8A0140CBB4093653BC61D67C960E3ECCE6639CC9FF1147AA6D0F9017" + +func TestScopeHMACInputsParsesRealPacket(t *testing.T) { + payloadType, payload, code1, ok := scopeHMACInputs(realTransportFloodPacket) + if !ok { + t.Fatal("scopeHMACInputs returned ok=false for a valid transport-flood packet") + } + if payloadType != 5 { + t.Errorf("payloadType = %d, want 5 (GRP_TXT)", payloadType) + } + if code1 != "9209" { + t.Errorf("code1 = %q, want %q", code1, "9209") + } + if len(payload) != 51 { + t.Errorf("len(payload) = %d, want 51", len(payload)) + } + if got := strings.ToUpper(hex.EncodeToString(payload[:4])); got != "EC2D4481" { + t.Errorf("payload starts %q, want %q — offset walked wrong", got, "EC2D4481") + } +} + +func TestScopeHMACInputsRejectsNonTransportRoutes(t *testing.T) { + // A plain FLOOD packet carries no transport codes, so it has no code1 to + // verify against. Returning ok=false rather than a zero code1 keeps the + // caller from HMACing packets that can never match anything. + // + // Header 0x15 = route_type 1 (FLOOD), payload_type 5. No transport codes, + // so the path byte follows the header directly. + _, _, _, ok := scopeHMACInputs("15" + "41" + "E3D3" + "AABBCC") + if ok { + t.Error("ok = true for a non-transport route, want false — there is no code1 to verify") + } +} + +func TestScopeHMACInputsRejectsMalformed(t *testing.T) { + for _, c := range []struct{ hex, why string }{ + {"", "empty"}, + {"zz", "not hex"}, + {"14", "header only, no transport codes"}, + {"1492090000", "transport codes but no path byte"}, + {"149209000041", "path byte claims one 2-byte hop, none present"}, + // pathByte 0xC0: upper two bits 11 -> hash_size 4, which firmware + // reserves and isValidPathLen rejects even at hash_count 0 + // (cmd/server/decoder.go, mirroring Packet.cpp:13-18). + {"1492090000C0" + strings.Repeat("00", 8), "hash_size 4 is reserved"}, + } { + if _, _, _, ok := scopeHMACInputs(c.hex); ok { + t.Errorf("ok = true for %q (%s), want false", c.hex, c.why) + } + } +} + +func TestRegionCodeMatchesTheRealPacket(t *testing.T) { + // The end-to-end arithmetic, against a packet whose true region is known. + payloadType, payload, code1, ok := scopeHMACInputs(realTransportFloodPacket) + if !ok { + t.Fatal("setup: scopeHMACInputs failed") + } + if got := regionCode("fm-112", payloadType, payload); got != code1 { + t.Errorf("regionCode(fm-112) = %q, want %q — this packet IS fm-112", got, code1) + } + // Both spellings must agree: the key is SHA256 over "#name", and callers + // hand us names with the '#' already stripped by normScope. + if got := regionCode("#fm-112", payloadType, payload); got != code1 { + t.Errorf("regionCode(#fm-112) = %q, want %q — leading '#' must be optional", got, code1) + } + // A region the repeater also declares, which this packet is NOT. + if got := regionCode("behss", payloadType, payload); got == code1 { + t.Errorf("regionCode(behss) = %q, must not equal fm-112's code1", got) + } +} + +func TestRegionCodeIsCaseSensitive(t *testing.T) { + // The key is SHA256 over the raw bytes of "#name", so "#BEHSS" and + // "#behss" are different regions. Folding case here would silently name + // traffic for a region nobody configured. + payloadType, payload, _, _ := scopeHMACInputs(realTransportFloodPacket) + if regionCode("behss", payloadType, payload) == regionCode("BEHSS", payloadType, payload) { + t.Error("regionCode folded case — the key is a hash over raw bytes and must not") + } +} + +func TestUnmatchedTransmissionsInWindow(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + old := "2020-01-01T00:00:00Z" + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeMatched("#be"), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnscoped(), RouteFlood, recent) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, old) + + since := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + got, _, err := s.unmatchedTransmissionsInWindow(since) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 { + t.Fatalf("got %d rows, want 1 — only the recent scope_name='' row qualifies", len(got)) + } + // scopeUnmatched() seeds raw_hex 'AA', which scopeHMACInputs rejects. The + // query's job is selection; unparseable rows are dropped by the caller, so + // they must still be returned here rather than filtered in SQL. + if got[0].txID == 0 { + t.Error("txID = 0, want the transmission's real id") + } +} + +// buildVerifierFromPackets is a test helper: wraps raw hex strings as rows the +// verifier consumes, with ids 1..N in order. +func buildVerifierFromPackets(t *testing.T, hexes ...string) *scopeVerifier { + t.Helper() + rows := make([]unmatchedTransmissionRow, 0, len(hexes)) + for i, h := range hexes { + rows = append(rows, unmatchedTransmissionRow{txID: int64(i + 1), rawHex: h}) + } + return newScopeVerifier(rows) +} + +func TestScopeVerifierNeedsTwoCorroboratingPackets(t *testing.T) { + // One match is 1-in-65536 and must not be enough; a second makes it + // (1/65536)^2. This threshold is the reason the approach is sound. + v := buildVerifierFromPackets(t, realTransportFloodPacket) + one := v.evidence([]int64{1}, []string{"fm-112"}) + if one["fm-112"] != 1 { + t.Fatalf("evidence = %v, want fm-112:1", one) + } + if got := v.verified(one); len(got) != 0 { + t.Errorf("verified = %v, want none - one corroborating packet is not evidence", got) + } + + // The same packet twice under different ids: two distinct transmissions + // both deriving to fm-112. + v2 := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + two := v2.evidence([]int64{1, 2}, []string{"fm-112"}) + if two["fm-112"] != 2 { + t.Fatalf("evidence = %v, want fm-112:2", two) + } + got := v2.verified(two) + if len(got) != 1 || got[0] != "fm-112" { + t.Errorf("verified = %v, want [fm-112]", got) + } +} + +func TestScopeVerifierIgnoresRegionsThatDoNotMatch(t *testing.T) { + v := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + got := v.evidence([]int64{1, 2}, []string{"behss", "be", "eu"}) + if len(got) != 0 { + t.Errorf("evidence = %v, want empty - none of these regions is this packet", got) + } +} + +func TestScopeVerifierSkipsUnparseablePackets(t *testing.T) { + // A row whose raw_hex cannot be walked contributes nothing and must not + // error the pass: one malformed row in the window would otherwise blank + // the verification for every repeater. + v := buildVerifierFromPackets(t, "AA", realTransportFloodPacket) + got := v.evidence([]int64{1, 2}, []string{"fm-112"}) + if got["fm-112"] != 1 { + t.Errorf("evidence = %v, want fm-112:1 - the malformed row is skipped, the good one still counts", got) + } +} + +func TestScopeVerifierCachesAcrossTargets(t *testing.T) { + // The cost argument: work depends on (region, transmission), not on which + // target asked. Two targets declaring the same region over the same packets + // must not double the HMACs. + v := buildVerifierFromPackets(t, realTransportFloodPacket, realTransportFloodPacket) + v.evidence([]int64{1, 2}, []string{"fm-112"}) + after := v.hmacCount + v.evidence([]int64{1, 2}, []string{"fm-112"}) + if v.hmacCount != after { + t.Errorf("hmacCount %d -> %d on a repeat query, want unchanged - the cache is what keeps this inside rule 0", after, v.hmacCount) + } +} + +func TestScopeVerifierUnknownTxIDIsHarmless(t *testing.T) { + // A target's unmatchedTxIDs come from a different query than the verifier's + // rows. They are taken in the same window, but a row pruned between the two + // must degrade to "no evidence", not panic. + v := buildVerifierFromPackets(t, realTransportFloodPacket) + got := v.evidence([]int64{1, 999}, []string{"fm-112"}) + if got["fm-112"] != 1 { + t.Errorf("evidence = %v, want fm-112:1 - the unknown id contributes nothing", got) + } +} + +// BenchmarkScopeVerifierAudit models a full audit refresh: every declared name +// against every unmatched packet, once, through the memo. The naive shape would +// be targets x names x packets; this asserts the memo keeps it at names x +// packets, which is what makes the feature affordable (AGENTS.md rule 0). +func BenchmarkScopeVerifierAudit(b *testing.B) { + const packets, names, targets = 400, 124, 205 + rows := make([]unmatchedTransmissionRow, 0, packets) + txIDs := make([]int64, 0, packets) + for i := 0; i < packets; i++ { + rows = append(rows, unmatchedTransmissionRow{txID: int64(i + 1), rawHex: realTransportFloodPacket}) + txIDs = append(txIDs, int64(i+1)) + } + declared := make([]string, 0, names) + for i := 0; i < names; i++ { + declared = append(declared, fmt.Sprintf("r%04d", i)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + v := newScopeVerifier(rows) + for t := 0; t < targets; t++ { + v.evidence(txIDs, declared) + } + } +} + +// TestUnmatchedTransmissionsInWindowIsBounded pins the LIMIT and what it keeps. +// +// The "~400 packets in a 7d window" this feature was sized against holds only +// for an instance whose hashRegions covers most of what its repeaters forward. +// The ingestor writes scope_name = ” for EVERY transport-scoped packet it +// cannot name, so a stock instance with no hashRegions at all has every scoped +// packet in this result set, and the verifier's work is (distinct declared +// names x rows). Unbounded on the axis that grows fastest is exactly what +// AGENTS.md rule 0 forbids. +// +// Newest-first is not arbitrary: a partial answer built from the most recent +// traffic matches what the window claims to describe, and a repeater still +// forwarding a region is far likelier to have done so recently. +func TestUnmatchedTransmissionsInWindowIsBounded(t *testing.T) { + s := newScopeTestStore(t) + base := time.Now().UTC().Add(-2 * time.Hour) + total := scopeVerifyMaxWindowPackets + 25 + for i := 0; i < total; i++ { + at := base.Add(time.Duration(i) * time.Second).Format(time.RFC3339) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, at) + } + + since := time.Now().UTC().Add(-3 * time.Hour).Format(time.RFC3339) + got, truncated, err := s.unmatchedTransmissionsInWindow(since) + if err != nil { + t.Fatal(err) + } + if len(got) != scopeVerifyMaxWindowPackets { + t.Fatalf("got %d rows, want the cap of %d", len(got), scopeVerifyMaxWindowPackets) + } + if !truncated { + t.Error("truncated = false, want true — the caller has to be able to say the evidence is partial") + } + + // The 25 oldest rows are the ones that must have been dropped: seeded ids + // ascend with first_seen, so every kept id must be above that boundary. + for _, r := range got { + if r.txID <= 25 { + t.Fatalf("kept txID %d, want only the %d newest rows", r.txID, scopeVerifyMaxWindowPackets) + } + } +} + +// TestUnmatchedTransmissionsInWindowReportsNoTruncationUnderCap: the flag has to +// distinguish "this is everything" from "this is a sample", or the caller +// cannot tell an exact answer from a partial one. +func TestUnmatchedTransmissionsInWindowReportsNoTruncationUnderCap(t *testing.T) { + s := newScopeTestStore(t) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + seedTransmissionRouteAt(t, s, "E3D3", scopeUnmatched(), RouteFlood, recent) + + since := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + got, truncated, err := s.unmatchedTransmissionsInWindow(since) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || truncated { + t.Fatalf("got %d rows truncated=%v, want 1 and false", len(got), truncated) + } +} + +// TestCapVerifyRegions bounds the other axis. A repeater's declared list +// arrives over MQTT from a companion app: handleClientRegions validates the +// target and each entry, but nothing limits how MANY entries a client may +// report, and every distinct name costs one pass over every unmatched packet. +// The longest genuine list on this network declares 21 regions. +func TestCapVerifyRegions(t *testing.T) { + short := []string{"be", "nl", "eu"} + if got := capVerifyRegions(short); len(got) != 3 { + t.Errorf("len = %d, want 3 — a normal list must pass through untouched", len(got)) + } + long := make([]string, scopeVerifyMaxRegionsPerTarget+10) + for i := range long { + long[i] = fmt.Sprintf("r%03d", i) + } + got := capVerifyRegions(long) + if len(got) != scopeVerifyMaxRegionsPerTarget { + t.Fatalf("len = %d, want the cap of %d", len(got), scopeVerifyMaxRegionsPerTarget) + } + if got[0] != long[0] { + t.Errorf("got[0] = %q, want %q — the cap keeps the first entries, it does not reorder", got[0], long[0]) + } +} + +// BenchmarkScopeVerifierStress runs the shape the cap allows, not the shape +// today's network produces. BenchmarkScopeVerifierAudit models 400 packets; +// this one models a full sample, which is what an instance with few configured +// region keys actually hands the verifier. +func BenchmarkScopeVerifierStress(b *testing.B) { + const names, targets = 124, 205 + packets := scopeVerifyMaxWindowPackets + rows := make([]unmatchedTransmissionRow, 0, packets) + txIDs := make([]int64, 0, packets) + for i := 0; i < packets; i++ { + rows = append(rows, unmatchedTransmissionRow{txID: int64(i + 1), rawHex: realTransportFloodPacket}) + txIDs = append(txIDs, int64(i+1)) + } + declared := make([]string, 0, names) + for i := 0; i < names; i++ { + declared = append(declared, fmt.Sprintf("r%04d", i)) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + v := newScopeVerifier(rows) + for t := 0; t < targets; t++ { + v.evidence(txIDs, declared) + } + } +} diff --git a/docs/api-spec.md b/docs/api-spec.md index 59b4d5590..9971689b3 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -1612,7 +1612,9 @@ not being the same as "declared nothing"), which apply here identically. "observedUnscopedPackets": number, // plain-FLOOD packets forwarded this window "wildcardContradiction": boolean, // observed unscoped forwarding but '*' not declared "ambiguousHops": number, // forwarder hops this window that could not be attributed — see note below - "observedUnmatchedPackets": number // forwarded packets whose scope this instance holds no key for — see note below + "observedUnmatchedPackets": number, // forwarded packets whose scope this instance holds no key for — see note below + "observedUnmatchedSampled": number, // how many of those verification could examine — see note below + "regionEvidence": { "": number } // declared regions corroborated by this repeater's own unnameable traffic — see note below } ] } @@ -1661,6 +1663,34 @@ not being the same as "declared nothing"), which apply here identically. in this instance's own configuration and the operator can act on it. It is **not** evidence for or against `declaredWildcard` — unmatched traffic is scoped, so it never affects `wildcardContradiction`, which counts only plain unscoped floods. + Part of this count is explained: packets counted in `regionEvidence` are + attributable to a declared region after all. A client showing this as a caveat should + subtract them and report only the remainder, which carries a sharper meaning — traffic + this repeater forwards for a region it does **not** declare and this instance cannot + name. Two rules on that subtraction: count only evidence for regions **absent** from + `notObserved` (a region with a single hit was deliberately not accepted as evidence, so + its packet is not explained either), and compare against `observedUnmatchedSampled` + first. +- `observedUnmatchedSampled` is how many of those packets verification could actually + examine. Two caps sit between the count and the evidence: the per-repeater working set + (512 packets) and the per-window sample (the 4096 most recent unnameable packets, which + a deployment with few `hashRegions` entries will reach). `regionEvidence` can only ever + count packets inside that sample, so when this field is **smaller** than + `observedUnmatchedPackets` the difference between the count and the evidence is an + **upper bound** on the unexplained traffic rather than a figure, and a client should say + so. Equal values mean the subtraction is exact. +- `regionEvidence` maps a declared region to how many of this repeater's own unmatched + forwarded packets derive to it. The server tests each declared region this repeater has + no *named* evidence for by deriving `SHA256("#region")[:16]` and HMAC-ing that + repeater's own unmatched packets with it — the same computation the ingestor performs at + ingest, with the candidate set narrowed to this repeater's declarations. A region + reaching **2** corroborating packets is removed from `notObserved`: `code1` is two + bytes, so one match happens by chance with probability 1/65536, while two on the same + region is (1/65536)². A region with exactly one hit therefore stays in `notObserved` + **and** appears here with the value 1, so a client can explain why it is still shown as + not observed. `notObserved` remains the single source of truth for whether a region was + observed; this field says only *how* that was established. The object is always present + and may be empty. - All scope names in `declaredRegions` / `notObserved` / `undeclaredObserved[].scope` are already normalised (no leading `#`) — the server does the `#`/no-`#` reconciliation described on the per-node endpoint so this response is directly comparable without a diff --git a/public/scope-audit.css b/public/scope-audit.css index 44f8a03d1..dfb12d7e3 100644 --- a/public/scope-audit.css +++ b/public/scope-audit.css @@ -84,3 +84,8 @@ caveats on the row's finding, not findings themselves, and neither may compete visually with the red/green scope chips beside them. */ .sa-chip-unmatched { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px dashed var(--border); font-family: inherit; font-style: italic; } + +/* Verified-by-declaration: the same green as any observed chip, because the + region IS observed. The dotted underline says how that was established + without introducing a third colour into a column that already carries two. */ +.sa-chip-verified { text-decoration: underline dotted; text-underline-offset: 2px; } diff --git a/public/scope-audit.js b/public/scope-audit.js index 1ffa678c2..de85f3589 100644 --- a/public/scope-audit.js +++ b/public/scope-audit.js @@ -93,12 +93,31 @@ function mergedScopeChips(row) { var missing = Object.create(null); row.notObserved.forEach(function (n) { missing[n] = true; }); + var evidence = row.regionEvidence || {}; var chips = row.declaredRegions.map(function (n) { var observed = !missing[n]; - return '' + escapeHtml(n) + ''; + var hits = evidence[n] || 0; + // A green chip with evidence was established by verifying the repeater's + // own declaration against its own unnameable traffic, not by matching a + // configured region key. Same colour — it is observed either way — with a + // dotted underline, so the reader can tell the two apart without a third + // colour competing for attention in a column that already carries two. + var verified = observed && hits > 0; + var cls = 'sa-chip ' + (observed ? 'sa-chip-observed' : 'sa-chip-unobserved') + (verified ? ' sa-chip-verified' : ''); + var title; + if (verified) { + title = n + ': observed — ' + hits + ' forwarded packet' + (hits === 1 ? '' : 's') + + ' in this window derive to this region, verified against the repeater’s own declared list. ' + + 'This instance holds no hashRegions key for it, so it could not be named directly.'; + } else if (observed) { + title = n + ': observed forwarding in this window'; + } else if (hits === 1) { + title = n + ': declared, and exactly one forwarded packet derives to it — that is one match in 65536 by chance alone, ' + + 'so it is not treated as evidence. Two would be.'; + } else { + title = n + ': declared, but no forwarding observed in this window'; + } + return '' + escapeHtml(n) + ''; }); if (!chips.length) return '—'; return chips.join(' '); @@ -195,17 +214,45 @@ // fault, this one is a missing entry in this instance's own hashRegions and // the reader can fix it. Saying so is what stops them investigating an // innocent repeater. - // - // The count is server-supplied, so a non-numeric value renders nothing - // rather than the string "NaN forwarded packets". function unmatchedCaveat(row) { var n = row.observedUnmatchedPackets; if (typeof n !== 'number' || !isFinite(n) || n <= 0) return ''; - var label = escapeHtml(n) + ' forwarded packet' + (n === 1 ? '' : 's'); + // Traffic already accounted for by verification is explained, not + // mysterious. What is left over is the interesting case: this repeater + // forwards a region it does NOT declare and that this instance also cannot + // name. Reporting the full count here would re-raise a question the Scopes + // column has just answered. + // Only evidence that actually ESTABLISHED a region counts as an + // explanation. A region with a single deriving packet stays in notObserved + // by design — one match in 65536 is chance, not evidence — and subtracting + // it here would call that packet explained while the chip next to it says + // the region was not observed. Keyed on notObserved rather than on the + // count, so the corroboration threshold stays in one place: the server. + var established = Object.create(null); + Object.keys(row.regionEvidence || {}).forEach(function (k) { established[k] = true; }); + (row.notObserved || []).forEach(function (rgn) { delete established[rgn]; }); + + var explained = 0; + var evidence = row.regionEvidence || {}; + Object.keys(established).forEach(function (k) { explained += evidence[k]; }); + // One packet can derive to two of this repeater's declared regions, so the + // values can sum past the number of distinct packets behind them. + if (explained > n) explained = n; + var left = n - explained; + if (left <= 0) return ''; + + // observedUnmatchedPackets counts every unnameable packet; the evidence can + // only come from the ones the verifier actually held (both the per-target + // list and the per-window sample are capped). Subtracting an undercounted + // number from a complete one overstates what is unexplained, so when the + // count was sampled the chip states a bound instead of a figure. + var sampled = typeof row.observedUnmatchedSampled === 'number' ? row.observedUnmatchedSampled : n; + var bounded = sampled < n; + var label = (bounded ? 'at most ' : '') + escapeHtml(left) + ' forwarded packet' + (left === 1 ? '' : 's'); return ' ' + - label + ' unnameable'; + ' in this window carried a region scope this CoreScope instance holds no key for, and match none of this repeater's declared regions. ' + + 'So this repeater forwards at least one region it does not declare, which this instance also cannot name.">' + + label + ' unexplained'; } // statusScore ranks a row's Status column numerically for sorting — a diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index c961420aa..84fd35be0 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -7219,6 +7219,31 @@ console.log('\n=== scope-audit.js: mergedScopeChips ==='); assert.ok(h.includes('<img')); }); + test('a region verified against the repeater own declaration is green, and says so', () => { + const h = chips({ declaredRegions: ['fm-112'], notObserved: [], regionEvidence: { 'fm-112': 23 } }); + assert.ok(h.includes('sa-chip-observed'), 'still green — it is observed'); + assert.ok(h.includes('sa-chip-verified'), 'but marked as established differently'); + assert.ok(h.includes('23'), 'the tooltip states how much evidence there is'); + }); + + test('a region observed by name carries no verified marker', () => { + const h = chips({ declaredRegions: ['be'], notObserved: [], regionEvidence: {} }); + assert.ok(h.includes('sa-chip-observed')); + assert.ok(!h.includes('sa-chip-verified'), 'a normally-named region is not a verification'); + }); + + test('a single-hit region stays grey and its tooltip explains why', () => { + const h = chips({ declaredRegions: ['fm-112'], notObserved: ['fm-112'], regionEvidence: { 'fm-112': 1 } }); + assert.ok(h.includes('sa-chip-unobserved'), 'one hit is not enough to turn it green'); + assert.ok(/one match/i.test(h), 'must say why one hit was not accepted'); + }); + + test('a missing regionEvidence field renders as before (older server)', () => { + const h = chips({ declaredRegions: ['be'], notObserved: ['be'] }); + assert.ok(h.includes('sa-chip-unobserved')); + assert.ok(!h.includes('sa-chip-verified')); + }); + test('a notObserved entry that is not declared cannot invent a chip', () => { // Defensive: the server guarantees notObserved is a subset (197 of 197 // rows checked), but the column must not grow a phantom chip if that ever @@ -7252,10 +7277,11 @@ console.log('\n=== scope-audit.js: unmatchedCaveat ==='); assert.strictEqual(caveat({}), ''); }); - test('a non-zero count renders a chip carrying the number', () => { + test('a non-zero unexplained count renders a chip carrying the number', () => { const h = caveat({ observedUnmatchedPackets: 148 }); assert.ok(h.includes('sa-chip-unmatched'), 'should carry its own class'); - assert.ok(h.includes('148'), 'should state the count, not just that there is one'); + assert.ok(h.includes('148'), 'with no evidence to subtract, the whole count is unexplained'); + assert.ok(h.includes('unexplained'), 'the word changed with the meaning'); }); test('singular and plural are both grammatical', () => { @@ -7263,20 +7289,79 @@ console.log('\n=== scope-audit.js: unmatchedCaveat ==='); assert.ok(caveat({ observedUnmatchedPackets: 2 }).includes('2 forwarded packets ')); }); - test('the title names the cause, not just the symptom', () => { - // The operator fix is a hashRegions edit; a caveat that does not say so - // sends them looking at the repeater instead of at their own config. + test('the title says what unexplained traffic implies', () => { + // The cause is no longer only a hashRegions gap: after verification, what + // is left over is traffic for a region the repeater does not declare. const h = caveat({ observedUnmatchedPackets: 5 }); - assert.ok(h.includes('hashRegions'), 'must name the config key that fixes it'); + assert.ok(/does not declare/i.test(h), 'must state the sharper conclusion'); + }); + + test('traffic fully explained by verification raises no caveat', () => { + assert.strictEqual(caveat({ observedUnmatchedPackets: 23, regionEvidence: { 'fm-112': 23 } }), ''); + }); + + test('only the unexplained remainder is reported', () => { + const h = caveat({ observedUnmatchedPackets: 30, regionEvidence: { 'fm-112': 23 } }); + assert.ok(h.includes('7 forwarded packets '), 'want the remainder, not the total'); + }); + + test('evidence the server refused to count is not subtracted either', () => { + // A region with a single deriving packet stays in notObserved on purpose: + // one match in 65536 is chance, not evidence (scopeVerifyMinCorroboration). + // Subtracting it here would call that packet explained while the chip + // beside it says the opposite. Keyed on notObserved rather than on the + // number, so the threshold lives in one place: the server. + const h = caveat({ + observedUnmatchedPackets: 3, + regionEvidence: { 'fm-112': 1 }, + notObserved: ['fm-112'], + }); + assert.ok(h.includes('3 forwarded packets '), 'all three are still unexplained'); + }); + + test('evidence that did establish a region is still subtracted', () => { + const h = caveat({ + observedUnmatchedPackets: 10, + regionEvidence: { 'nl-nb': 3, belml: 1 }, + notObserved: ['belml'], + }); + assert.ok(h.includes('7 forwarded packets '), 'subtract the verified 3, keep the uncorroborated 1'); + }); + + test('a sampled count is reported as an upper bound', () => { + // observedUnmatchedPackets counts every packet; the evidence can only come + // from the packets the verifier actually held. Subtracting an undercounted + // number from a complete one overstates what is unexplained, so the chip + // says at most rather than pretending to an exact figure. + const h = caveat({ + observedUnmatchedPackets: 900, + observedUnmatchedSampled: 512, + regionEvidence: { 'nl-nb': 40 }, + }); + assert.ok(h.includes('at most 860 forwarded packets '), 'want the bound, stated as one'); + }); + + test('an unsampled count keeps its exact wording', () => { + const h = caveat({ + observedUnmatchedPackets: 30, + observedUnmatchedSampled: 30, + regionEvidence: { 'fm-112': 23 }, + }); + assert.ok(h.includes('7 forwarded packets '), 'exact remainder'); + assert.ok(!/at most/.test(h), 'nothing was sampled away, so do not hedge'); }); test('a non-numeric count renders nothing rather than NaN', () => { - // observedUnmatchedPackets is server-supplied. A truthiness check accepts - // a string, and the chip then reads "NaN forwarded packets". assert.strictEqual(caveat({ observedUnmatchedPackets: '12' }), ''); assert.strictEqual(caveat({ observedUnmatchedPackets: NaN }), ''); }); + test('evidence exceeding the count cannot produce a negative remainder', () => { + // One packet can derive to two of a repeater's declared regions, so the + // evidence values can sum past the number of distinct packets. + assert.strictEqual(caveat({ observedUnmatchedPackets: 4, regionEvidence: { a: 3, b: 3 } }), ''); + }); + test('the count is not injected raw into markup', () => { // observedUnmatchedPackets is server-supplied. It is a number in every // sane response, but the chip must not become an injection point if that From b5c7612166e8e70a7a4075a209a786c12199f464 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 9 Sep 2026 23:16:13 +0200 Subject: [PATCH 018/151] fix(live): use the shared WebSocket instead of opening a second one (#1991) Closes #1980. `app.js` opens a WebSocket on every page load and fans messages out through `onWS()`/`offWS()`. `live.js` ignored that channel and opened its own socket to the same endpoint. `Hub.Broadcast` does no per-client filtering, so both carried the identical full packet stream and **every viewer on the live map pulled it twice**. That page is the one people leave open for hours, so it was a standing multiplier on origin bandwidth rather than a burst. ## Measured, before and after Against a running instance, leaving the live map and returning while counting WebSocket constructions: | | new sockets on re-entry | constructed by | |---|---|---| | before | 1 | `at connectWS (live.js:3315)` | | after | 0 | nothing | And with one viewer on the live map, the server now reports **one** WebSocket client for that viewer, with the live feed counter climbing normally (7 to 31 over one navigation cycle, 71 on a fresh load). ## The change The live map subscribes to the shared channel like every other view, and unsubscribes in `destroy()` rather than closing a socket the rest of the app still needs. `connectWS()` drops any existing registration before adding a fresh one. An early return looks like the natural guard against double subscription, but it keeps the previous visit's closure registered, and re-registering without dropping the old one renders every packet twice. Dropping first is idempotent either way and always binds the current page. Reconnection becomes `app.js`'s business, since it owns the socket. That moved `WS_RECONNECT_MS` out of the only place that honoured it, so `app.js` now uses it too. It comes from `roles.js` and operators set it as `wsReconnectMs`; after this change it applies to the one socket everyone shares, or to nothing at all. ## Tests Three, in the sandbox that already loads `live.js` with a Leaflet mock: - the page registers exactly one listener on the shared channel and constructs no WebSocket of its own - re-entering leaves exactly one listener, and it is the new one rather than the previous visit's - the handler ignores messages that carry no packet `node test-frontend-helpers.js` passes (726 assertions), `node test-packet-filter.js` passes. ## One note on the history The second commit on this branch claims the early-return guard broke rendering on re-entry, citing a real measurement. The measurement happened, the attribution was wrong: the zero counter came from the WebSocket constructor patch I had installed to count sockets, which interfered with the page it was measuring. The third commit records that rather than rewriting it away. The change is kept because dropping the old registration first is the clearer contract, not because the guard was broken. ## Rule 0 Strictly less work than before: one socket per viewer instead of two, one JSON parse instead of two per packet, and no second reconnect loop. Nothing is added to the hot path; a listener already existed for every other view. --- public/app.js | 6 +++- public/live.js | 40 +++++++++++++++++------- test-frontend-helpers.js | 67 ++++++++++++++++++++++++++++++++++++++++ test-live.js | 22 +++++++++++-- 4 files changed, 120 insertions(+), 15 deletions(-) diff --git a/public/app.js b/public/app.js index 28281fa4d..9aad3d31a 100644 --- a/public/app.js +++ b/public/app.js @@ -784,7 +784,11 @@ function connectWS() { ws.onopen = () => Logo.setConnected(true); ws.onclose = () => { Logo.setConnected(false); - setTimeout(connectWS, 3000); + // WS_RECONNECT_MS comes from roles.js and is settable as cacheTTL's + // sibling `wsReconnectMs`. It used to apply only to the live map's own + // socket; now that every view shares this one, the operator's setting + // applies here or nowhere. + setTimeout(connectWS, window.WS_RECONNECT_MS || 3000); }; ws.onerror = () => ws.close(); ws.onmessage = (e) => { diff --git a/public/live.js b/public/live.js index 8d8e7b28f..3e6da6287 100644 --- a/public/live.js +++ b/public/live.js @@ -17,7 +17,7 @@ function cssVar(name) { return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); } function statusGreen() { return cssVar('--status-green') || '#22c55e'; } - let map, ws, nodesLayer, pathsLayer, animLayer, heatLayer, geoFilterLayer, clickablePathsLayer; + let map, wsHandler, nodesLayer, pathsLayer, animLayer, heatLayer, geoFilterLayer, clickablePathsLayer; // New animation canvas let animCanvas, animCtx; let _dprMedia = null; @@ -3215,6 +3215,10 @@ // Expose for testing window._livePruneStaleNodes = pruneStaleNodes; + // Exposed for the "one socket per viewer" test: the live map must reach the + // packet stream through app.js's shared channel and never open its own. + window._liveConnectWS = connectWS; + window._liveWSHandler = function() { return wsHandler; }; window._liveBuildClickablePathPopupHtml = buildClickablePathPopupHtml; window._livePruneClickablePaths = pruneClickablePaths; window._liveClickablePaths = clickablePaths; @@ -3310,17 +3314,29 @@ } catch { } } + // The live map used to open its OWN WebSocket to the same endpoint that + // app.js already holds open on every page. Hub.Broadcast does no per-client + // filtering, so both sockets carried the identical full packet stream and + // every viewer pulled it twice. The live map is the page people leave open + // for hours, which made that a standing multiplier on origin bandwidth + // rather than a burst. + // + // It now subscribes to the shared channel (onWS/offWS in app.js), which is + // what every other view does. Reconnection is app.js's business: it owns the + // socket, and the listener registered here survives a reconnect because the + // listener list does. function connectWS() { - const proto = location.protocol === 'https:' ? 'wss' : 'ws'; - ws = new WebSocket(`${proto}://${location.host}`); - ws.onmessage = (e) => { - try { - const msg = JSON.parse(e.data); - if (msg.type === 'packet') bufferPacket(msg.data); - } catch { } + // Re-subscribe rather than skip when a handler is already registered. + // Returning early here looks like the right guard against double + // subscription, but it keeps the PREVIOUS visit's closure registered, and + // that one renders into a page that no longer exists: packets keep + // arriving and nothing appears. Dropping the old registration first is + // idempotent in the same way and always binds the current page. + if (wsHandler) offWS(wsHandler); + wsHandler = (msg) => { + if (msg && msg.type === 'packet') bufferPacket(msg.data); }; - ws.onclose = () => setTimeout(connectWS, WS_RECONNECT_MS); - ws.onerror = () => {}; + onWS(wsHandler); } // A packet group is multibyte when its path hash size is >= 2 bytes. @@ -4618,7 +4634,9 @@ if (_pruneInterval) { clearInterval(_pruneInterval); _pruneInterval = null; } if (_feedTimestampInterval) { clearInterval(_feedTimestampInterval); _feedTimestampInterval = null; } if (_affinityInterval) { clearInterval(_affinityInterval); _affinityInterval = null; } - if (ws) { ws.onclose = null; ws.close(); ws = null; } + // Unsubscribe rather than close: the socket belongs to app.js and the + // rest of the app still needs it. + if (wsHandler) { offWS(wsHandler); wsHandler = null; } if (regionFilterChangeHandler && window.RegionFilter && typeof RegionFilter.offChange === 'function') { RegionFilter.offChange(regionFilterChangeHandler); regionFilterChangeHandler = null; diff --git a/test-frontend-helpers.js b/test-frontend-helpers.js index 84fd35be0..a0559c48c 100644 --- a/test-frontend-helpers.js +++ b/test-frontend-helpers.js @@ -7446,3 +7446,70 @@ console.log('\n=== scope-audit.js: sourcesLineHtml ==='); assert.ok(/newest answer per repeater wins/i.test(line)); }); } + +// ===== live.js: one WebSocket per viewer ===== +// app.js opens a socket on every page load and fans messages out through +// onWS()/offWS(). live.js used to ignore that and open a second socket to the +// same endpoint, and since the broadcast does no per-client filtering, both +// carried the identical full packet stream. The live map is the page people +// leave open for hours, so it doubled origin bandwidth for as long as the tab +// was open. +console.log('\n=== live.js: one WebSocket per viewer ==='); +{ + function makeWSSandbox() { + const ctx = makeSandbox(); + ctx.registerPage = () => {}; + let constructed = 0; + const registered = []; + ctx.WebSocket = function () { constructed++; }; + ctx.onWS = (fn) => registered.push(fn); + ctx.offWS = (fn) => { + const i = registered.indexOf(fn); + if (i >= 0) registered.splice(i, 1); + }; + ctx.L = { map: () => ({ setView: () => ({}), on: () => {}, remove: () => {} }) }; + ctx.IATA_COORDS_GEO = {}; + ctx.cancelAnimationFrame = () => {}; + ctx.document.createElementNS = () => ctx.document.createElement(); + loadInCtx(ctx, 'public/roles.js'); + try { + loadInCtx(ctx, 'public/live.js'); + } catch (e) { + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; + } + return { ctx, registered, constructedCount: () => constructed }; + } + + test('the live map subscribes to the shared channel instead of opening a socket', () => { + const { ctx, registered, constructedCount } = makeWSSandbox(); + const connect = ctx.window._liveConnectWS; + assert.ok(connect, '_liveConnectWS must be exposed'); + connect(); + assert.strictEqual(constructedCount(), 0, 'live.js must not construct a WebSocket of its own'); + assert.strictEqual(registered.length, 1, 'it must register exactly one listener on the shared channel'); + }); + + test('re-entering the page keeps one listener, and it is the current one', () => { + // Navigating away and back re-runs the page init. Two failure modes sit + // either side of this: registering again without dropping the old one + // doubles every packet, and skipping the registration leaves the previous + // visit's closure subscribed, which renders into a page that is gone. That + // second one is not theoretical: it was measured against a running + // instance, where packets kept arriving and the live counter stayed at 0. + const { ctx, registered } = makeWSSandbox(); + ctx.window._liveConnectWS(); + const first = registered[0]; + ctx.window._liveConnectWS(); + assert.strictEqual(registered.length, 1, 'exactly one listener must remain'); + assert.notStrictEqual(registered[0], first, 'and it must be the new one, bound to the current page'); + }); + + test('the handler only reacts to packet messages', () => { + const { ctx, registered } = makeWSSandbox(); + ctx.window._liveConnectWS(); + const handler = registered[0]; + // Neither of these carries packet data; the handler must not throw on them. + assert.doesNotThrow(() => handler({ type: 'stats' })); + assert.doesNotThrow(() => handler(null)); + }); +} diff --git a/test-live.js b/test-live.js index d7e65764c..5df12ef3f 100644 --- a/test-live.js +++ b/test-live.js @@ -891,9 +891,25 @@ console.log('\n=== live.js: source-level safety checks ==='); 'tab restore should clear propagation buffer'); }); - test('connectWS has reconnect on close', () => { - assert.ok(src.includes('ws.onclose = () => setTimeout(connectWS, WS_RECONNECT_MS)'), - 'WebSocket should auto-reconnect on close'); + test('the live map owns no socket of its own', () => { + // It used to open a second WebSocket to the endpoint app.js already holds + // open, and the broadcast does no per-client filtering, so every viewer on + // this page pulled the full packet stream twice. + assert.ok(!src.includes('new WebSocket'), + 'live.js must not construct a WebSocket; app.js owns the one socket'); + assert.ok(src.includes('onWS(wsHandler)'), + 'it must subscribe to the shared channel instead'); + assert.ok(src.includes('offWS(wsHandler)'), + 'and unsubscribe rather than closing a socket the rest of the app needs'); + }); + + test('reconnect lives with the socket owner and honours wsReconnectMs', () => { + // Moving the subscription took the live map's own reconnect with it. That + // was the only place WS_RECONNECT_MS was honoured, so app.js has to use it + // now or the operator's `wsReconnectMs` setting applies nowhere. + const appSrc = fs.readFileSync('public/app.js', 'utf8'); + assert.ok(appSrc.includes('setTimeout(connectWS, window.WS_RECONNECT_MS || 3000)'), + 'app.js should reconnect on the configured interval, defaulting to 3s'); }); test('addNodeMarker avoids duplicates', () => { From 675c576fead83ea504cc7314abdba550f26028d1 Mon Sep 17 00:00:00 2001 From: efiten Date: Thu, 10 Sep 2026 22:30:24 +0200 Subject: [PATCH 019/151] fix(scope-audit): bound the verifier's payload, and fix two tests that proved less than they claimed (#1993) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three leftovers from reviewing the scope-audit series (#1986, #1987, #1990). None is urgent; all three are the kind of thing that gets harder to explain the longer it sits. ## `scopeHMACInputs` accepted a payload `DecodePacket` rejects Its comment says it walks the same offsets as the decoder, and it does, minus the `maxPacketPayload` bound the decoder enforces. Unreachable in practice: such a packet never reaches the database with an empty `scope_name` in the first place, so the verifier never sees one. Worth closing anyway, because the comment claims the two agree. A verifier that accepts what the decoder refuses is a small divergence today and an hour of confusion on the day it matters. ## The corroboration test seeded the same packet twice The threshold of two rests on `code1` being two bytes: one match is 1/65536 by chance, two on the same region is (1/65536)². That argument needs two **independent** observations. The test seeded `realTransportFloodPacket` twice. Identical payloads derive identical codes, so it was one observation counted twice, and it would have passed just as happily against an implementation that counted rows rather than deriving anything. It now seeds the real captured packet plus a second one built for a different payload, both deriving to `#fm-112` on their own. **The feature was never wrong here.** `transmissions.hash` is unique and `ComputeContentHash` is path-independent, so two rows always mean two distinct payloads in production. Only the test failed to demonstrate the property it is named for. ## Naming at ingest and verifying at read time had no test together They were built separately, in #1990 and #1989, and the interaction between them is not exotic: with derived region keys enabled, a packet that used to be stored unnameable now arrives with a name. The audit must then report that region as observed by the ordinary route: - present in `agg.scopes` - absent from `notObserved` - **not** claimed by `regionEvidence`, which exists to explain regions that could only be established by verification Getting that wrong is quiet. The chip stays green while the reason underneath it is wrong, and a reader asking "how do we know this" gets the wrong story. ## Verification `cd cmd/server && go test ./...` passes (254s), `go vet` and `gofmt -l` clean. No production behaviour changes beyond the payload bound, which rejects input that cannot occur. --- cmd/server/scope_audit_test.go | 58 +++++++++++++++++++++++++++++++++- cmd/server/scope_verify.go | 9 +++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 749cfec76..3ebd811be 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -3,6 +3,7 @@ package main import ( "context" "database/sql" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -1180,13 +1181,32 @@ func TestHandleScopeAuditSurfacesUnmatchedPackets(t *testing.T) { // forwarded are stored unmatched. Verification derives the key from the // repeater's own declaration, finds two corroborating packets, and the region // must leave notObserved with its evidence count reported. +// transportFloodPacketFor builds a TRANSPORT_FLOOD packet whose code1 is the +// code `region` derives over this payload, so the verifier will match it. The +// shape mirrors realTransportFloodPacket: header 0x14 (route 0, payload type +// 5), the two transport codes, path byte 0x41 (hash size 2, one hop), the hop, +// then the payload. +// +// It exists because corroboration has to come from DIFFERENT payloads. Two +// copies of one packet derive the same code by construction, so they are one +// observation counted twice, not the two independent matches the threshold +// argument rests on. +func transportFloodPacketFor(region string, payload []byte) string { + code1 := regionCode(region, 5, payload) + return "14" + code1 + "0000" + "41" + "E3D3" + strings.ToUpper(hex.EncodeToString(payload)) +} + func TestHandleScopeAuditVerifiesDeclaredRegion(t *testing.T) { srv, router := setupScopeAuditServer(t) pk := testFullPubkeyA insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112,behss", 0) recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + // Two DIFFERENT payloads, each deriving to #fm-112 on its own. The first is + // the real packet captured from a live instance; the second is built for + // this test. Seeding the same packet twice would prove only that the + // verifier counts rows. seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) - seedUnmatchedRawAt(t, srv.store, pk[:4], realTransportFloodPacket, RouteTransportFlood, recent) + seedUnmatchedRawAt(t, srv.store, pk[:4], transportFloodPacketFor("fm-112", []byte{0x51, 0x52, 0x53, 0x54, 0x55}), RouteTransportFlood, recent) got := getScopeAudit(t, router, "") if len(got.Repeaters) != 1 { @@ -1292,3 +1312,39 @@ func seedUnmatchedRawAt(t *testing.T, s *PacketStore, forwarder, rawHex string, // TestHandleNodeScopesDifferentWindowIsSeparateCacheEntry confirms the cache // key includes window: a request for a different window must recompute // rather than reuse another window's cached entry. + +// TestHandleScopeAuditNamedRegionNeedsNoVerification pins the interaction +// between naming a packet at ingest and verifying it at read time, which are +// built separately and had no test together. +// +// Deriving region keys from what nodes declare means a packet that used to be +// stored unnameable now arrives with a name. The audit must then report that +// region as observed by the ordinary route: present in agg.scopes, absent from +// notObserved, and NOT claimed by regionEvidence, which exists to explain +// regions that could only be established by verification. +// +// Getting this wrong is not loud. A region would still be green, so the page +// looks right while the reason underneath it is wrong, and a reader chasing +// "how do we know this" is told the wrong story. +func TestHandleScopeAuditNamedRegionNeedsNoVerification(t *testing.T) { + srv, router := setupScopeAuditServer(t) + pk := testFullPubkeyA + insertDeclared(t, srv, pk, time.Now().UTC().Format(time.RFC3339), "fm-112", 0) + recent := time.Now().UTC().Add(-time.Minute).Format(time.RFC3339) + // scopeMatched is the state the ingestor writes once it holds a key for + // the region, whether that key was configured by hand or derived. + seedTransmissionRouteAt(t, srv.store, pk[:4], scopeMatched("#fm-112"), RouteTransportFlood, recent) + + row := getScopeAudit(t, router, "").Repeaters[0] + for _, rgn := range row.NotObserved { + if rgn == "fm-112" { + t.Errorf("notObserved = %v, must not contain fm-112 — it was observed under its own name", row.NotObserved) + } + } + if n, ok := row.RegionEvidence["fm-112"]; ok { + t.Errorf("regionEvidence[fm-112] = %d, want absent — verification explains regions that could not be named, and this one was", n) + } + if row.ObservedUnmatchedPackets != 0 { + t.Errorf("observedUnmatchedPackets = %d, want 0 — a named packet is not unnameable traffic", row.ObservedUnmatchedPackets) + } +} diff --git a/cmd/server/scope_verify.go b/cmd/server/scope_verify.go index 4f460ef26..3988df440 100644 --- a/cmd/server/scope_verify.go +++ b/cmd/server/scope_verify.go @@ -99,7 +99,14 @@ func scopeHMACInputs(rawHex string) (payloadType byte, payload []byte, code1 str return 0, nil, "", false } rest := buf[offset:] - if len(rest) == 0 { + if len(rest) == 0 || len(rest) > maxPacketPayload { + // The upper bound mirrors DecodePacket, which rejects a payload past + // the firmware's MAX_PACKET_PAYLOAD. Unreachable in practice, because + // such a packet never reaches the database with an empty scope_name in + // the first place, but the comment above claims this walks the same + // offsets as the decoder and that should be true rather than nearly + // true. A verifier that accepts what the decoder rejects is a small + // divergence today and a confusing one to debug later. return 0, nil, "", false } return byte(header.PayloadType), rest, code1, true From 6c92a8b6120d0a7b4f77553f09a41a0320611e62 Mon Sep 17 00:00:00 2001 From: efiten Date: Thu, 10 Sep 2026 22:30:27 +0200 Subject: [PATCH 020/151] test(ingestor): make the suite race-clean, and run the detector when it matters (#1994) `go test -race ./...` on `cmd/ingestor` reports **six data races** on master. None is in production logic. All six come from test helpers that outlive the test that started them, which is why the detector blames whichever test happens to be running: two different tests failed on two consecutive runs of the same code. ## What was racing **`StartStatsFileWriter` had no way to stop.** Two tests start it at a 50ms interval, and its goroutine then runs for the rest of the process. It reads the package-level `readProcSelfIOFn` hook, which a later test replaces to inject a fake, so the write and the read race. The same leak explains the stray log lines about writing stats into temp directories that were already cleaned up. It now returns a stop function that closes the goroutine and waits for it to exit. Production ignores the return value and runs for the process lifetime exactly as before; the two tests call it through `t.Cleanup`. **The migration test read a log buffer while a goroutine wrote to it.** `log.Logger` serialises its own writes, but `logContains` read `buf.String()` outside that lock while `RunAsyncMigration` kept logging after the call that started it had returned. The capture helper now uses a mutex-protected buffer. That one is worth calling a real race rather than a test artefact: a concurrent read during a buffer grow can panic outright with "concurrent map read and map write"-class behaviour, not merely trip `-race`. ## Measured `go test -race ./...` on linux/arm64 under go1.27.1: **exit 0, zero races, 704s**. ## The CI job, and why it is shaped this way The server has had `-race` since #1208. This closes the same gap for the ingestor, which carries an `atomic.Pointer` snapshot (the region key set from #1989) whose safety has been an argument rather than a measurement. Two deliberate choices, because a check that costs too much gets switched off: - **Its own job, not a step inside "Go Build & Test".** Appending `-race` there puts its ten-odd minutes on the critical path, taking the pipeline from roughly 20 minutes to roughly 32. As a separate job it runs beside the E2E job (15-17 minutes) and hides inside that window. - **Only when `cmd/ingestor/**.go` changed**, decided by the existing change-scope job, which already gates the heavy jobs on documentation-only PRs. A frontend or docs PR cannot introduce a data race in the ingestor. Pushes to master always run it, as they already do for `code`. Nothing `needs:` the new job. Adding it to `build-and-publish` would mean a skipped race job skips everything downstream, which is the opposite of what a conditional check should do. It reports as its own check; whether that blocks a merge is a repository setting rather than workflow logic. ## Scope Test helpers, one production signature (`StartStatsFileWriter` now returns a stop function), and the workflow. No change to what the ingestor does at runtime. --- .github/workflows/deploy.yml | 46 +++++++++++++++++++ .../multibyte_persist_helpers_test.go | 30 ++++++++++-- cmd/ingestor/stats_file.go | 23 +++++++++- cmd/ingestor/stats_file_test.go | 2 +- cmd/ingestor/stats_file_timestamp_test.go | 2 +- 5 files changed, 96 insertions(+), 7 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3097bacd9..34b777d96 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -51,6 +51,7 @@ jobs: runs-on: ubuntu-latest outputs: code: ${{ steps.scope.outputs.code }} + ingestor: ${{ steps.scope.outputs.ingestor }} steps: - name: Checkout code uses: actions/checkout@v5 @@ -66,6 +67,7 @@ jobs: # never skipped. if [ "${{ github.event_name }}" != "pull_request" ]; then echo "code=true" >> "$GITHUB_OUTPUT" + echo "ingestor=true" >> "$GITHUB_OUTPUT" echo "event=${{ github.event_name }} -> full pipeline" exit 0 fi @@ -73,6 +75,7 @@ jobs: # Empty diff means something is off; run everything rather than guess. if [ -z "$CHANGED" ]; then echo "code=true" >> "$GITHUB_OUTPUT" + echo "ingestor=true" >> "$GITHUB_OUTPUT" echo "empty diff -> full pipeline" exit 0 fi @@ -85,6 +88,49 @@ jobs: echo "code=false" >> "$GITHUB_OUTPUT" echo "-> documentation only, heavy jobs skip (and report Success)" fi + # The race detector is its own job, and it only earns its ten minutes + # when the code it inspects changed. A frontend or docs PR cannot + # introduce a data race in the ingestor. + if echo "$CHANGED" | grep -qE '^cmd/ingestor/.*[.]go$'; then + echo "ingestor=true" >> "$GITHUB_OUTPUT" + echo "-> ingestor Go files changed, race detector runs" + else + echo "ingestor=false" >> "$GITHUB_OUTPUT" + echo "-> no ingestor Go changes, race detector skips" + fi + + # ─────────────────────────────────────────────────────────────── + # 1a. Race detector (ingestor) — its own job on purpose + # ─────────────────────────────────────────────────────────────── + # Runs beside "Go Build & Test" rather than inside it, so its ten-odd + # minutes overlap the E2E job instead of extending the critical path, and + # only when ingestor Go files changed. A frontend or documentation PR cannot + # introduce a data race here, and paying for the check on every PR is what + # gets a check switched off again. + # + # The server has -race in its own step already; this closes the same gap for + # the ingestor, which carries an atomic.Pointer snapshot (the region key + # set) whose safety is an argument until something checks it. + race-test: + name: "🏁 Race detector (ingestor)" + runs-on: ubuntu-latest + needs: [changes] + if: needs.changes.outputs.ingestor == 'true' + steps: + - name: Checkout code + uses: actions/checkout@v5 + + - name: Set up Go 1.27 + uses: actions/setup-go@v6 + with: + go-version: '1.27' + cache-dependency-path: cmd/ingestor/go.sum + + - name: go test -race + run: | + set -e -o pipefail + cd cmd/ingestor + go test -timeout 30m -race ./... # ─────────────────────────────────────────────────────────────── # 1. Go Build & Test diff --git a/cmd/ingestor/multibyte_persist_helpers_test.go b/cmd/ingestor/multibyte_persist_helpers_test.go index aee8078aa..5e57fd8a5 100644 --- a/cmd/ingestor/multibyte_persist_helpers_test.go +++ b/cmd/ingestor/multibyte_persist_helpers_test.go @@ -5,15 +5,16 @@ import ( "database/sql" "log" "strings" + "sync" "testing" ) // captureLogs redirects the standard logger to a buffer for the // duration of the test and returns the buffer. Restores the previous // writer when the test ends. -func captureLogs(t *testing.T) *bytes.Buffer { +func captureLogs(t *testing.T) *syncBuffer { t.Helper() - buf := &bytes.Buffer{} + buf := &syncBuffer{} prevWriter := log.Writer() prevFlags := log.Flags() log.SetOutput(buf) @@ -24,9 +25,32 @@ func captureLogs(t *testing.T) *bytes.Buffer { return buf } +// syncBuffer is a bytes.Buffer that may be read while a background goroutine +// is still logging into it. log.Logger serialises its own writes, but the test +// reading buf.String() sits outside that lock, and RunAsyncMigration keeps +// logging after the call that started it returns. Without this the read and +// the write race, which is a real data race and not a test artefact: it can +// panic on a buffer grow, not merely trip -race. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + // logContains reports whether the captured log buffer contains substr // (case-insensitive). -func logContains(buf *bytes.Buffer, substr string) bool { +func logContains(buf *syncBuffer, substr string) bool { return strings.Contains(strings.ToLower(buf.String()), strings.ToLower(substr)) } diff --git a/cmd/ingestor/stats_file.go b/cmd/ingestor/stats_file.go index cb5ac9e41..449156108 100644 --- a/cmd/ingestor/stats_file.go +++ b/cmd/ingestor/stats_file.go @@ -6,6 +6,7 @@ import ( "encoding/json" "log" "os" + "sync" "time" "github.com/meshcore-analyzer/perfio" @@ -240,11 +241,20 @@ func procIORate(prev, cur procIOSnapshot, stamp string) *PerfIOSample { // The stats file path is resolved via statsFilePath() once at writer-loop // start; the env var (CORESCOPE_INGESTOR_STATS) is only re-read on process // restart, not per tick. -func StartStatsFileWriter(s *Store, interval time.Duration) { +// The returned stop function ends the writer goroutine and returns once it has +// exited. Production ignores it and runs for the process lifetime; tests must +// call it, because this goroutine reads package-level hooks (readProcSelfIOFn) +// that a later test may replace, and a writer left running past its own test +// races with whoever runs next. It is safe to call more than once. +func StartStatsFileWriter(s *Store, interval time.Duration) (stop func()) { if interval <= 0 { interval = time.Second } + done := make(chan struct{}) + stopped := make(chan struct{}) + var once sync.Once go func() { + defer close(stopped) t := time.NewTicker(interval) defer t.Stop() path := statsFilePath() @@ -257,7 +267,12 @@ func StartStatsFileWriter(s *Store, interval time.Duration) { // The buffer grows once and stays. var buf bytes.Buffer enc := json.NewEncoder(&buf) - for range t.C { + for { + select { + case <-done: + return + case <-t.C: + } // Capture time.Now() ONCE per tick (Carmack must-fix #5). // Both snapshot.SampledAt and procIO.SampledAt MUST share the // same string so the freshness guard isn't validating one @@ -305,4 +320,8 @@ func StartStatsFileWriter(s *Store, interval time.Duration) { } } }() + return func() { + once.Do(func() { close(done) }) + <-stopped + } } diff --git a/cmd/ingestor/stats_file_test.go b/cmd/ingestor/stats_file_test.go index 4a488f26a..e96d56bd1 100644 --- a/cmd/ingestor/stats_file_test.go +++ b/cmd/ingestor/stats_file_test.go @@ -56,7 +56,7 @@ func TestStatsFileWriter_PublishesProcIO(t *testing.T) { } defer store.Close() - StartStatsFileWriter(store, 50*time.Millisecond) + t.Cleanup(StartStatsFileWriter(store, 50*time.Millisecond)) // Wait for at least 2 ticks so the writer has had a chance to populate // procIO rates from a delta. diff --git a/cmd/ingestor/stats_file_timestamp_test.go b/cmd/ingestor/stats_file_timestamp_test.go index 900e65606..fd3a66597 100644 --- a/cmd/ingestor/stats_file_timestamp_test.go +++ b/cmd/ingestor/stats_file_timestamp_test.go @@ -69,7 +69,7 @@ func TestStatsFileWriter_SampledAtMatchesProcIOSampledAt(t *testing.T) { } } - StartStatsFileWriter(store, 50*time.Millisecond) + t.Cleanup(StartStatsFileWriter(store, 50*time.Millisecond)) // Wait for the file to land with a populated procIO block. deadline := time.Now().Add(3 * time.Second) From 0c7f2306f6f6a5b70594e4171b1944105ac5da43 Mon Sep 17 00:00:00 2001 From: efiten Date: Fri, 11 Sep 2026 00:00:45 +0200 Subject: [PATCH 021/151] feat(ingestor): keep the scope-match tally across restarts (#2002) ## What `scopeMatchCounters` (unique / explicit-over-derived / ambiguous / none) lives only in the ingestor process, and the only way to read it is the periodic log line. A restart zeroes the counters, and recreating the container removes that log with it, so a measurement in progress cannot be recovered afterwards. That happened here on 2026-09-10: a 24h ambiguity measurement completed, two deploys followed before it was read, and nothing on disk held the number. `/var/lib/docker/containers/*/*-json.log` had no earlier copy. The tally gates a real decision (whether the collision tie-break from the `autoRegionKeys` design is worth building), and that needs days of traffic, so it has to survive the process counting it. ## How A single-row table, `scope_match_totals`: - `OpenStore` restores the counters from it, so counting continues instead of restarting. - The 5-minute stats ticker writes them back, and so does the shutdown path, which is what a deploy triggers. - `since_unix` carries the window start across restarts, so the ratio keeps a denominator. The periodic log line now prints it. Saving rides the **stats** ticker, not the region-refresh ticker: matches are recorded for every transport-scoped packet, including on instances that never enable `autoRegionKeys` and so never start the refresh loop. The table is **not** in `internal/dbschema` on purpose. The server neither reads nor PRAGMA-detects it; putting it in `AssertReady` would make an older DB fail the server's startup check over data the server never looks at. A failed restore is logged and ingestion continues. Losing an observability counter is not a reason to stop ingesting. ## Tests Four, all in `cmd/ingestor/scope_match_tally_test.go`: - totals and `since_unix` restored across a close/reopen - a fresh DB gets its anchor row immediately, so the first window has a start time - recording after a restore adds to the carried total instead of counting from zero - repeated saves keep exactly one row Full `cmd/ingestor` suite green locally except `TestWriteStatsAtomic_SymlinkAtDestIsReplaced`, which fails on Windows only (`os.Symlink` needs a privilege this account lacks) and predates this branch. `go vet` and `gofmt` clean. The race detector was not run locally (no cgo toolchain here); the `race-test` job covers it, since this branch touches `cmd/ingestor`. ## Not done No UI or API surface for the tally. It is still read from the log line or straight from the table. --- cmd/ingestor/db.go | 17 +++ cmd/ingestor/main.go | 13 ++ cmd/ingestor/region_keys.go | 8 +- cmd/ingestor/scope_match_tally.go | 86 ++++++++++++ cmd/ingestor/scope_match_tally_test.go | 177 +++++++++++++++++++++++++ 5 files changed, 300 insertions(+), 1 deletion(-) create mode 100644 cmd/ingestor/scope_match_tally.go create mode 100644 cmd/ingestor/scope_match_tally_test.go diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index 5a119150b..facd529af 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -159,6 +159,13 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) return nil, fmt.Errorf("preparing statements: %w", err) } + // Continue the scope-match tally from where the previous process left + // off. Not fatal: a tally that cannot be read costs an observability + // number, and refusing to ingest over it would be the worse trade. + if err := s.LoadScopeMatchTotals(); err != nil { + log.Printf("[regions] restoring scope-match tally: %v", err) + } + // Schedule async migrations. These must NOT block boot. See // async_migration.go for the convention. // PREFLIGHT: async=true reason="composite index build on observations (1.9M+ rows in prod) — converted from sync after v3.8.3" @@ -228,6 +235,16 @@ func applySchema(db *sql.DB) error { foreign_advert INTEGER DEFAULT 0 ); + CREATE TABLE IF NOT EXISTS scope_match_totals ( + id INTEGER PRIMARY KEY CHECK (id = 1), + since_unix INTEGER NOT NULL, + unique_matches INTEGER NOT NULL, + explicit_over_derived INTEGER NOT NULL, + ambiguous INTEGER NOT NULL, + none_matches INTEGER NOT NULL, + updated_unix INTEGER NOT NULL + ); + CREATE TABLE IF NOT EXISTS observers ( id TEXT PRIMARY KEY, name TEXT, diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index d5a7b82ab..34248721b 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -485,6 +485,13 @@ func main() { go func() { for range statsTicker.C { store.LogStats() + // Persist the scope-match tally on the stats cadence rather + // than the region-refresh one: the counters are recorded for + // every transport-scoped packet, including on instances that + // never enable autoRegionKeys and so never run that ticker. + if err := store.SaveScopeMatchTotals(); err != nil { + log.Printf("[regions] saving scope-match tally: %v", err) + } if d := ingestBuffer.Dropped(); d > 0 || ingestBuffer.Pending() > 0 { log.Printf("[ingest-buffer] pending=%d dropped_total=%d", ingestBuffer.Pending(), d) } @@ -559,6 +566,12 @@ func main() { pruneQueueTicker.Stop() walCheckpointTicker.Stop() stopWatchdog() + // A deploy is a SIGTERM, which is exactly the case that used to lose + // the tally: save before the process goes away rather than leaving up + // to 5 minutes of counting to the next tick that will not come. + if err := store.SaveScopeMatchTotals(); err != nil { + log.Printf("[regions] saving scope-match tally: %v", err) + } store.LogStats() // final stats on shutdown for _, c := range clients { c.Disconnect(5000) // 5s to allow in-flight messages to drain diff --git a/cmd/ingestor/region_keys.go b/cmd/ingestor/region_keys.go index 45002722a..298aba930 100644 --- a/cmd/ingestor/region_keys.go +++ b/cmd/ingestor/region_keys.go @@ -7,6 +7,7 @@ import ( "sort" "strings" "sync/atomic" + "time" ) // declaredRegionStat is one region name as reported over RF, with the two @@ -402,6 +403,10 @@ var scopeMatchCounters struct { explicitOverDerived atomic.Int64 ambiguous atomic.Int64 none atomic.Int64 + // sinceUnix is when this tally started counting, carried across + // restarts with the counters themselves (scope_match_tally.go). A + // ratio without its window is not a measurement. + sinceUnix atomic.Int64 } // recordScopeMatch tallies one decision and logs the interesting ones. Unique @@ -426,7 +431,8 @@ func recordScopeMatch(m scopeMatch) { // ticker so the numbers arrive on the same cadence as the key-set changes that // move them. func logScopeMatchCounters() { - log.Printf("[regions] scope matches: unique=%d explicit-over-derived=%d ambiguous=%d none=%d", + log.Printf("[regions] scope matches since %s: unique=%d explicit-over-derived=%d ambiguous=%d none=%d", + time.Unix(scopeMatchCounters.sinceUnix.Load(), 0).UTC().Format(time.RFC3339), scopeMatchCounters.unique.Load(), scopeMatchCounters.explicitOverDerived.Load(), scopeMatchCounters.ambiguous.Load(), scopeMatchCounters.none.Load()) } diff --git a/cmd/ingestor/scope_match_tally.go b/cmd/ingestor/scope_match_tally.go new file mode 100644 index 000000000..378b563f7 --- /dev/null +++ b/cmd/ingestor/scope_match_tally.go @@ -0,0 +1,86 @@ +package main + +import ( + "database/sql" + "errors" + "time" +) + +// The scope-match tally lives in scopeMatchCounters, four atomics in this +// process and nowhere else. Every restart — a deploy, a crash, an operator +// bouncing the container — sets them back to zero and takes the container's +// log with it, so the previous count is not recoverable afterwards. That is +// how a finished 24h ambiguity measurement was lost. +// +// The numbers only say something over days: "ambiguous is 0.4% of matches" +// needs a denominator large enough to be stable, and the decision they gate +// (whether a collision tie-break is worth building) is not one to take on an +// hour of traffic. So they have to outlive the process doing the counting. +// +// scope_match_totals is one row. The ingestor reads it into the counters at +// startup and writes them back periodically and on shutdown, which makes the +// tally cumulative since `since_unix` instead of since boot. It is deliberately +// NOT in internal/dbschema: the server neither reads this table nor detects it, +// and putting it in AssertReady would make an old DB without the table fail the +// server's startup check for data the server never looks at. + +const scopeMatchTotalsSchema = ` + CREATE TABLE IF NOT EXISTS scope_match_totals ( + id INTEGER PRIMARY KEY CHECK (id = 1), + since_unix INTEGER NOT NULL, + unique_matches INTEGER NOT NULL, + explicit_over_derived INTEGER NOT NULL, + ambiguous INTEGER NOT NULL, + none_matches INTEGER NOT NULL, + updated_unix INTEGER NOT NULL + )` + +// LoadScopeMatchTotals seeds the in-process counters from the stored row so +// counting continues where the previous process stopped. On a DB that has no +// row yet it anchors `since` at now and stores the zeroes, so the very first +// window has a start time too — without it, a tally read after one restart +// would have no honest denominator. +func (s *Store) LoadScopeMatchTotals() error { + var since, uniq, explicit, ambiguous, none int64 + err := s.db.QueryRow(` + SELECT since_unix, unique_matches, explicit_over_derived, ambiguous, none_matches + FROM scope_match_totals WHERE id = 1`).Scan(&since, &uniq, &explicit, &ambiguous, &none) + if errors.Is(err, sql.ErrNoRows) { + scopeMatchCounters.sinceUnix.Store(time.Now().Unix()) + return s.SaveScopeMatchTotals() + } + if err != nil { + return err + } + scopeMatchCounters.sinceUnix.Store(since) + scopeMatchCounters.unique.Store(uniq) + scopeMatchCounters.explicitOverDerived.Store(explicit) + scopeMatchCounters.ambiguous.Store(ambiguous) + scopeMatchCounters.none.Store(none) + return nil +} + +// SaveScopeMatchTotals writes the current counters to the single row. Each +// atomic is loaded independently, so a save taken under live traffic can miss +// a match that lands mid-write; the next save picks it up. The tally is a +// long-window ratio, not an audit trail. +func (s *Store) SaveScopeMatchTotals() error { + _, err := s.db.Exec(` + INSERT INTO scope_match_totals + (id, since_unix, unique_matches, explicit_over_derived, ambiguous, none_matches, updated_unix) + VALUES (1, ?, ?, ?, ?, ?, ?) + ON CONFLICT(id) DO UPDATE SET + since_unix = excluded.since_unix, + unique_matches = excluded.unique_matches, + explicit_over_derived = excluded.explicit_over_derived, + ambiguous = excluded.ambiguous, + none_matches = excluded.none_matches, + updated_unix = excluded.updated_unix`, + scopeMatchCounters.sinceUnix.Load(), + scopeMatchCounters.unique.Load(), + scopeMatchCounters.explicitOverDerived.Load(), + scopeMatchCounters.ambiguous.Load(), + scopeMatchCounters.none.Load(), + time.Now().Unix()) + return err +} diff --git a/cmd/ingestor/scope_match_tally_test.go b/cmd/ingestor/scope_match_tally_test.go new file mode 100644 index 000000000..e62292da1 --- /dev/null +++ b/cmd/ingestor/scope_match_tally_test.go @@ -0,0 +1,177 @@ +package main + +import ( + "testing" + "time" +) + +// resetScopeMatchCounters zeroes the package-level tally and puts it back +// afterwards. The counters are global and OpenStore restores them from the +// DB, so every test here has to start from a known state; no test in this +// package calls t.Parallel, which is what makes that safe. +func resetScopeMatchCounters(t *testing.T) { + t.Helper() + prevUnique := scopeMatchCounters.unique.Load() + prevExplicit := scopeMatchCounters.explicitOverDerived.Load() + prevAmbiguous := scopeMatchCounters.ambiguous.Load() + prevNone := scopeMatchCounters.none.Load() + prevSince := scopeMatchCounters.sinceUnix.Load() + t.Cleanup(func() { + scopeMatchCounters.unique.Store(prevUnique) + scopeMatchCounters.explicitOverDerived.Store(prevExplicit) + scopeMatchCounters.ambiguous.Store(prevAmbiguous) + scopeMatchCounters.none.Store(prevNone) + scopeMatchCounters.sinceUnix.Store(prevSince) + }) + scopeMatchCounters.unique.Store(0) + scopeMatchCounters.explicitOverDerived.Store(0) + scopeMatchCounters.ambiguous.Store(0) + scopeMatchCounters.none.Store(0) + scopeMatchCounters.sinceUnix.Store(0) +} + +// TestScopeMatchTallySurvivesReopen is the whole point of the table: a +// restart used to reset the tally to zero and delete the container log that +// held the previous number, so a measurement in progress was unrecoverable. +func TestScopeMatchTallySurvivesReopen(t *testing.T) { + resetScopeMatchCounters(t) + dbPath := t.TempDir() + "/tally.db" + + s, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + since := scopeMatchCounters.sinceUnix.Load() + scopeMatchCounters.unique.Store(19692) + scopeMatchCounters.ambiguous.Store(80) + scopeMatchCounters.explicitOverDerived.Store(3) + scopeMatchCounters.none.Store(41) + if err := s.SaveScopeMatchTotals(); err != nil { + t.Fatal(err) + } + s.Close() + + // A new process starts with zeroed counters. + scopeMatchCounters.unique.Store(0) + scopeMatchCounters.ambiguous.Store(0) + scopeMatchCounters.explicitOverDerived.Store(0) + scopeMatchCounters.none.Store(0) + scopeMatchCounters.sinceUnix.Store(0) + + s2, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + defer s2.Close() + + if got := scopeMatchCounters.unique.Load(); got != 19692 { + t.Errorf("unique = %d, want 19692 restored from the previous run", got) + } + if got := scopeMatchCounters.ambiguous.Load(); got != 80 { + t.Errorf("ambiguous = %d, want 80 restored from the previous run", got) + } + if got := scopeMatchCounters.explicitOverDerived.Load(); got != 3 { + t.Errorf("explicitOverDerived = %d, want 3", got) + } + if got := scopeMatchCounters.none.Load(); got != 41 { + t.Errorf("none = %d, want 41", got) + } + if got := scopeMatchCounters.sinceUnix.Load(); got != since { + t.Errorf("sinceUnix = %d, want %d — the window start must carry across the restart, or the ratio has no denominator", got, since) + } +} + +// TestScopeMatchTallyAnchorsSinceOnFirstOpen pins that a DB with no row gets +// one immediately. Without it the first window's start would only be written +// once something had already been counted, and a tally read after one restart +// would report a rate over an unknown period. +func TestScopeMatchTallyAnchorsSinceOnFirstOpen(t *testing.T) { + resetScopeMatchCounters(t) + before := time.Now().Unix() + + s, err := OpenStore(t.TempDir() + "/tally.db") + if err != nil { + t.Fatal(err) + } + defer s.Close() + + since := scopeMatchCounters.sinceUnix.Load() + if since < before || since > time.Now().Unix() { + t.Errorf("sinceUnix = %d, want a stamp between %d and now", since, before) + } + var rows int + if err := s.db.QueryRow(`SELECT COUNT(*) FROM scope_match_totals`).Scan(&rows); err != nil { + t.Fatal(err) + } + if rows != 1 { + t.Errorf("scope_match_totals rows = %d, want exactly 1 — the table holds a single running total", rows) + } +} + +// TestScopeMatchTallyContinuesAfterRestore checks the restored values are a +// starting point and not a ceiling: recording after a restart adds to the +// carried total rather than counting from zero again. +func TestScopeMatchTallyContinuesAfterRestore(t *testing.T) { + resetScopeMatchCounters(t) + dbPath := t.TempDir() + "/tally.db" + + s, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + scopeMatchCounters.unique.Store(10) + scopeMatchCounters.ambiguous.Store(2) + if err := s.SaveScopeMatchTotals(); err != nil { + t.Fatal(err) + } + s.Close() + + s2, err := OpenStore(dbPath) + if err != nil { + t.Fatal(err) + } + defer s2.Close() + + recordScopeMatch(scopeMatch{Reason: scopeReasonUnique, Name: "#be"}) + recordScopeMatch(scopeMatch{Reason: scopeReasonAmbiguous, Candidates: []string{"#be", "#nl"}}) + if err := s2.SaveScopeMatchTotals(); err != nil { + t.Fatal(err) + } + + var uniq, ambiguous int64 + if err := s2.db.QueryRow(`SELECT unique_matches, ambiguous FROM scope_match_totals WHERE id = 1`).Scan(&uniq, &ambiguous); err != nil { + t.Fatal(err) + } + if uniq != 11 || ambiguous != 3 { + t.Errorf("stored unique=%d ambiguous=%d, want 11 and 3 — counting must resume on top of the restored totals", uniq, ambiguous) + } +} + +// TestScopeMatchTallySaveIsIdempotent pins the single-row upsert: repeated +// saves must update the row, never accumulate rows. The CHECK(id = 1) makes a +// second row impossible, and this fails loudly if the INSERT is ever changed +// to one without the conflict clause. +func TestScopeMatchTallySaveIsIdempotent(t *testing.T) { + resetScopeMatchCounters(t) + + s, err := OpenStore(t.TempDir() + "/tally.db") + if err != nil { + t.Fatal(err) + } + defer s.Close() + + for i := 0; i < 3; i++ { + scopeMatchCounters.unique.Add(1) + if err := s.SaveScopeMatchTotals(); err != nil { + t.Fatalf("save %d: %v", i, err) + } + } + + var rows, uniq int64 + if err := s.db.QueryRow(`SELECT COUNT(*), MAX(unique_matches) FROM scope_match_totals`).Scan(&rows, &uniq); err != nil { + t.Fatal(err) + } + if rows != 1 || uniq != 3 { + t.Errorf("rows=%d unique=%d, want 1 row holding 3", rows, uniq) + } +} From 02feb2a88ef05284445f022eb87aca78273fbb4c Mon Sep 17 00:00:00 2001 From: efiten Date: Fri, 11 Sep 2026 10:58:56 +0200 Subject: [PATCH 022/151] test(ingestor): join the watchdog loop goroutine instead of only asking it to stop (#2003) Verified before merging: on upstream/master `go test ./cmd/ingestor -run TestMQTTStallWatchdog -count=20` fails; on this branch the same command passes. The flake blocked CI on #2000. --- cmd/ingestor/mqtt_watchdog_1749_test.go | 23 ++++----------- cmd/ingestor/mqtt_watchdog_1810_test.go | 6 ++-- .../mqtt_watchdog_testhelpers_test.go | 29 +++++++++++++++++++ 3 files changed, 37 insertions(+), 21 deletions(-) diff --git a/cmd/ingestor/mqtt_watchdog_1749_test.go b/cmd/ingestor/mqtt_watchdog_1749_test.go index 105c18ad2..43e407829 100644 --- a/cmd/ingestor/mqtt_watchdog_1749_test.go +++ b/cmd/ingestor/mqtt_watchdog_1749_test.go @@ -58,15 +58,8 @@ func TestMQTTStallWatchdog_EscalateOnPersistentDisconnect_1749(t *testing.T) { t.Fatalf("setup: %v", err) } - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - - exited := make(chan struct{}) - go func() { - runLivenessWatchdogLoop(tick, done, threshold, func(args ...any) {}) - close(exited) - }() + tick, stopLoop := startWatchdogTestLoop(t, threshold, func(args ...any) {}) + defer stopLoop() // Feed ticks spanning > (multiplier × threshold) of wall clock so // the escalation path fires. We control the `now` parameter by @@ -114,10 +107,8 @@ func TestMQTTStallWatchdog_DisconnectedEscalationThrottled_1749(t *testing.T) { t.Fatalf("setup: %v", err) } - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - go runLivenessWatchdogLoop(tick, done, threshold, func(args ...any) {}) + tick, stopLoop := startWatchdogTestLoop(t, threshold, func(args ...any) {}) + defer stopLoop() base := time.Now() // Pre-stamp DisconnectedSinceUnix so that the first tick is @@ -259,10 +250,8 @@ func TestMQTTStallWatchdog_LastTickUnixExposed_1749(t *testing.T) { watchdogLastTickUnix.Store(before) }) - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - go runLivenessWatchdogLoop(tick, done, time.Minute, func(args ...any) {}) + tick, stopLoop := startWatchdogTestLoop(t, time.Minute, func(args ...any) {}) + defer stopLoop() stamp := time.Now().Add(48 * time.Hour) // guaranteed > before select { diff --git a/cmd/ingestor/mqtt_watchdog_1810_test.go b/cmd/ingestor/mqtt_watchdog_1810_test.go index aeefd4a0f..b68fa75d3 100644 --- a/cmd/ingestor/mqtt_watchdog_1810_test.go +++ b/cmd/ingestor/mqtt_watchdog_1810_test.go @@ -212,10 +212,8 @@ func TestWatchdog_EscalationWarnThrottled_1810(t *testing.T) { } } - tick := make(chan time.Time) - done := make(chan struct{}) - defer close(done) - go runLivenessWatchdogLoop(tick, done, threshold, emit) + tick, stopLoop := startWatchdogTestLoop(t, threshold, emit) + defer stopLoop() base := time.Now() // First tick: stamps DisconnectedSinceUnix (no escalation yet). diff --git a/cmd/ingestor/mqtt_watchdog_testhelpers_test.go b/cmd/ingestor/mqtt_watchdog_testhelpers_test.go index 348c2f2d1..1db9b99a5 100644 --- a/cmd/ingestor/mqtt_watchdog_testhelpers_test.go +++ b/cmd/ingestor/mqtt_watchdog_testhelpers_test.go @@ -1,6 +1,7 @@ package main import ( + "sync" "testing" "time" ) @@ -45,3 +46,31 @@ func sendTickOrFail(t *testing.T, tick chan<- time.Time, stamp time.Time, timeou t.Fatalf("%s: tick blocked after %s — loop dead?", label, timeout) } } + +// startWatchdogTestLoop is setupWatchdogTestLoop for the common case: a test +// that wants the loop running for its own duration and nothing more. The +// returned stop closes done AND waits for the loop goroutine to return. It is +// safe to call more than once. +// +// Waiting is the part that must not be skipped. Closing done only asks the +// loop to stop; the goroutine can still be inside a scan, and that scan walks +// the package-level livenessRegistry. The next test registers its own source +// there, so a loop that has not returned yet will process that source on a +// tick carrying the previous test's fabricated clock. +// +// That is measured, not theoretical. With four call sites closing done and +// walking away, TestMQTTStallWatchdog_DisconnectedEscalationThrottled_1749 +// failed 2 to 3 times per 20 runs of the watchdog tests and never once in 50 +// runs on its own. Debug tracing showed two loops reaching maybeForceReconnect +// for the same source with tick clocks 420s apart, both reading +// LastForceReconnectUnix as 0 before either wrote it, so the throttle the test +// asserts on let both through. +func startWatchdogTestLoop(t *testing.T, threshold time.Duration, emit func(...any)) (tick chan time.Time, stop func()) { + t.Helper() + tick, done, exited := setupWatchdogTestLoop(t, threshold, emit) + var once sync.Once + return tick, func() { + once.Do(func() { close(done) }) + <-exited + } +} From fe37f1060cdf3f260943ad1cd15d3ed4ad176e48 Mon Sep 17 00:00:00 2001 From: liquidraver <504870+liquidraver@users.noreply.github.com> Date: Fri, 11 Sep 2026 11:29:49 +0200 Subject: [PATCH 023/151] fix(ingestor): delete aged packets in bounded batches so prune stops stalling ingest (#2000) Reviewed at ecf0b371: query plans dumped and confirmed index-driven for all three statements, termination proven against concurrent ingest (first_seen is always time.Now()), FK child-first ordering required and correct, writer-stats assertions non-racy. Two low findings noted on the PR for follow-up: the dropped RowsAffected error now gates the loop, and ~0.53s batches will trip defaultSlowWriterMs=500. --- cmd/ingestor/maintenance.go | 96 ++++++++-- cmd/ingestor/prune_chunked_test.go | 286 +++++++++++++++++++++++++++++ 2 files changed, 362 insertions(+), 20 deletions(-) create mode 100644 cmd/ingestor/prune_chunked_test.go diff --git a/cmd/ingestor/maintenance.go b/cmd/ingestor/maintenance.go index 8159d8402..07d7e1ba1 100644 --- a/cmd/ingestor/maintenance.go +++ b/cmd/ingestor/maintenance.go @@ -10,42 +10,98 @@ import ( "github.com/meshcore-analyzer/dbschema" ) +// pruneBatchTransmissions bounds how many transmissions (and their child +// observations) one prune transaction deletes before it commits and +// releases the writer lock. +// +// Releasing between batches is the entire point. writerMu serialises every +// wrapped writer call, so while the prune holds it the MQTT ingest path is +// blocked. Go's sync.Mutex switches to FIFO handoff once a waiter has been +// blocked for 1ms (starvation mode), so an ingest goroutine queued behind +// the prune is served at the next batch boundary instead of after the whole +// retention day. +// +// The bound is on transmissions, but hold time scales with the rows actually +// deleted, and each transmission carries an unbounded number of observations. +// At ~16 observations per transmission a batch is ~4k row deletes and a few +// hundred milliseconds; an instance with a denser observation ratio gets a +// proportionally longer hold from the same batch size. 250 also keeps the +// commit count low (a 16k-transmission day is 64 transactions, not 16k). +const pruneBatchTransmissions = 250 + +// pruneAgedTransmissionIDs selects the next batch of transmissions older than +// the cutoff. Both statements of a batch embed it, so they resolve the same +// set: nothing modifies `transmissions` between them inside the transaction. +// +// The ORDER BY must be satisfiable from idx_transmissions_first_seen. That +// index carries the rowid as its tiebreaker, so "first_seen, id" is walked +// straight off it and the LIMIT stays deterministic even when timestamps tie. +// Ordering by id alone looks equivalent but makes SQLite abandon the index for +// a rowid SCAN. That is harmless while rows are being deleted — the oldest +// rows have the lowest rowids and match at once — but the batch that finds +// nothing, which is the steady state whenever nothing has aged out, walks the +// whole table under writerMu. TestPruneAgedTransmissionIDsUsesFirstSeenIndex +// pins the plan. +const pruneAgedTransmissionIDs = `SELECT id FROM transmissions WHERE first_seen < ? ORDER BY first_seen, id LIMIT ?` + +// The two statements of one prune batch. Child observations go first (no +// CASCADE in SQLite). +const ( + pruneObservationsBatch = `DELETE FROM observations WHERE transmission_id IN (` + pruneAgedTransmissionIDs + `)` + pruneTransmissionsBatch = `DELETE FROM transmissions WHERE id IN (` + pruneAgedTransmissionIDs + `)` +) + // PruneOldPackets deletes transmissions (and their child observations) // older than `days`. Returns count of transmissions deleted. // // Owned by the ingestor per #1283: the writer process is the only one // allowed to hold the DB write lock; previously this lived in // cmd/server/db.go and raced ingestor INSERTs (SQLITE_BUSY). +// +// Deletion is chunked into bounded transactions so ingest is never blocked +// for longer than a single batch. Deleting a whole retention day in one +// transaction held the writer lock for ~35s on a mesh doing ~260k +// observations/day, stalling MQTT ingest for the same duration. +// +// On error the transmissions deleted by already-committed batches are +// returned alongside it — those rows are gone, so reporting 0 would be +// wrong. func (s *Store) PruneOldPackets(days int) (int64, error) { if days <= 0 { return 0, nil } cutoff := time.Now().UTC().AddDate(0, 0, -days).Format(time.RFC3339) - // Tagged for writer-perf visibility (#1340). - var n int64 - err := s.WriterTx("prune_packets", func(tx *sql.Tx) error { - // Delete child observations first (no CASCADE in SQLite). - if _, err := tx.Exec(`DELETE FROM observations WHERE transmission_id IN ( - SELECT id FROM transmissions WHERE first_seen < ? - )`, cutoff); err != nil { - return fmt.Errorf("prune observations: %w", err) - } - - res, err := tx.Exec(`DELETE FROM transmissions WHERE first_seen < ?`, cutoff) + var total int64 + for { + var batch int64 + // Tagged for writer-perf visibility (#1340). + err := s.WriterTx("prune_packets", func(tx *sql.Tx) error { + if _, err := tx.Exec(pruneObservationsBatch, cutoff, pruneBatchTransmissions); err != nil { + return fmt.Errorf("prune observations: %w", err) + } + res, err := tx.Exec(pruneTransmissionsBatch, cutoff, pruneBatchTransmissions) + if err != nil { + return fmt.Errorf("prune transmissions: %w", err) + } + batch, _ = res.RowsAffected() + return nil + }) if err != nil { - return fmt.Errorf("prune transmissions: %w", err) + return total, err + } + total += batch + // A short batch proves nothing is left below the cutoff: the subquery + // found fewer rows than it was allowed to take. Only a batch that came + // back exactly full needs another pass. + if batch < pruneBatchTransmissions { + break } - n, _ = res.RowsAffected() - return nil - }) - if err != nil { - return 0, err } - if n > 0 { - log.Printf("[prune] deleted %d transmissions older than %d days", n, days) + if total > 0 { + log.Printf("[prune] deleted %d transmissions older than %d days", total, days) } - return n, nil + return total, nil } // PruneOldClientReceptions deletes mobile client-RX coverage rows older than diff --git a/cmd/ingestor/prune_chunked_test.go b/cmd/ingestor/prune_chunked_test.go new file mode 100644 index 000000000..a54130b00 --- /dev/null +++ b/cmd/ingestor/prune_chunked_test.go @@ -0,0 +1,286 @@ +package main + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +// seedAgedTransmissions inserts n transmissions stamped `age` days in the +// past, each with obsPerTx child observations, and returns nothing — the +// caller asserts against the store. Kept local to this file so the batching +// tests own their fixture shape. +func seedAgedTransmissions(t *testing.T, store *Store, n, obsPerTx, ageDays int) { + t.Helper() + ts := time.Now().UTC().AddDate(0, 0, -ageDays).Format(time.RFC3339) + for i := 0; i < n; i++ { + res, err := store.db.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json) + VALUES (?, ?, ?, 0, 1, 1, '{}')`, + "AA", fmt.Sprintf("h%d-%d", ageDays, i), ts, + ) + if err != nil { + t.Fatalf("seed tx %d: %v", i, err) + } + txID, err := res.LastInsertId() + if err != nil { + t.Fatalf("seed tx %d LastInsertId: %v", i, err) + } + for j := 0; j < obsPerTx; j++ { + if _, err := store.db.Exec( + `INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) + VALUES (?, ?, 'rx', 1.0, -100, 0, '[]', ?)`, + txID, j, time.Now().Unix(), + ); err != nil { + t.Fatalf("seed obs %d/%d: %v", i, j, err) + } + } + } +} + +func countRows(t *testing.T, store *Store, table string) int { + t.Helper() + var n int + if err := store.db.QueryRow(`SELECT COUNT(*) FROM ` + table).Scan(&n); err != nil { + t.Fatalf("count %s: %v", table, err) + } + return n +} + +func openPruneStore(t *testing.T, name string) *Store { + t.Helper() + store, err := OpenStore(filepath.Join(t.TempDir(), name)) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + t.Cleanup(func() { store.Close() }) + return store +} + +// TestPruneOldPacketsDeletesInBoundedBatches is the perf assertion for the +// chunked prune: it enforces that a retention day is deleted across many +// bounded transactions rather than one long-held writer lock. +// +// The old implementation issued exactly ONE WriterTx regardless of row +// count, holding writerMu — and therefore blocking MQTT ingest — for the +// whole delete. Measured on a production instance doing ~260k +// observations/day that was prune_packets hold_ms_p50 = 33905, with +// mqtt_handler wait_ms_max = 35797. +// +// Counting prune_packets transactions is the deterministic proxy for that +// characteristic: N transactions means the lock was released N-1 times, so +// the worst-case ingest stall is one batch rather than the full day. +func TestPruneOldPacketsDeletesInBoundedBatches(t *testing.T) { + store := openPruneStore(t, "prune-batched.db") + + // Two full batches plus a partial one. The partial batch is what ends the + // loop, so no terminating empty batch should run. + const aged = pruneBatchTransmissions*2 + 37 + seedAgedTransmissions(t, store, aged, 2, 10) + + ResetWriterStatsForTest() + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != aged { + t.Fatalf("expected %d transmissions pruned, got %d", aged, n) + } + + // 2 full batches + 1 partial batch, which proves nothing is left. + wantTx := int64(3) + got := store.WriterStatsSnapshot()["prune_packets"] + if got.Count != wantTx { + t.Fatalf("expected %d prune_packets transactions for %d rows at batch size %d, got %d "+ + "(1 means the delete was not chunked and holds the writer lock for the whole retention day)", + wantTx, aged, pruneBatchTransmissions, got.Count) + } + + if remaining := countRows(t, store, "transmissions"); remaining != 0 { + t.Fatalf("expected all aged transmissions gone, %d remain", remaining) + } + if remaining := countRows(t, store, "observations"); remaining != 0 { + t.Fatalf("expected all child observations gone, %d remain", remaining) + } +} + +// TestPruneOldPacketsExactMultipleTerminates covers the one case the +// short-batch exit cannot catch on its own: when the aged rows are an exact +// multiple of the batch size, the last batch comes back full, so the loop +// must run one more — empty — batch to learn that nothing is left. +func TestPruneOldPacketsExactMultipleTerminates(t *testing.T) { + store := openPruneStore(t, "prune-exact.db") + + const aged = pruneBatchTransmissions * 2 + seedAgedTransmissions(t, store, aged, 1, 10) + + ResetWriterStatsForTest() + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != aged { + t.Fatalf("expected %d pruned, got %d", aged, n) + } + + // 2 full batches + 1 empty batch that finds nothing and ends the loop. + if got := store.WriterStatsSnapshot()["prune_packets"].Count; got != 3 { + t.Fatalf("expected 3 prune_packets transactions for an exact multiple of the batch size, got %d", got) + } + if remaining := countRows(t, store, "transmissions"); remaining != 0 { + t.Fatalf("expected all aged transmissions gone, %d remain", remaining) + } + if remaining := countRows(t, store, "observations"); remaining != 0 { + t.Fatalf("expected all child observations gone, %d remain", remaining) + } +} + +// TestPruneOldPacketsSpansBatchesAndKeepsFreshRows covers the correctness +// risk the batching introduces: with a LIMIT on both statements, a cutoff +// that straddles several batches must still delete every aged row and no +// fresh one, and must not orphan observations. +func TestPruneOldPacketsSpansBatchesAndKeepsFreshRows(t *testing.T) { + store := openPruneStore(t, "prune-mixed.db") + + const aged = pruneBatchTransmissions + 11 + const fresh = 17 + const obsPerTx = 3 + seedAgedTransmissions(t, store, aged, obsPerTx, 10) + seedAgedTransmissions(t, store, fresh, obsPerTx, 0) + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != aged { + t.Fatalf("expected %d pruned, got %d", aged, n) + } + + if remaining := countRows(t, store, "transmissions"); remaining != fresh { + t.Fatalf("expected %d fresh transmissions kept, got %d", fresh, remaining) + } + if remaining := countRows(t, store, "observations"); remaining != fresh*obsPerTx { + t.Fatalf("expected %d observations kept, got %d", fresh*obsPerTx, remaining) + } + + // No observation may reference a transmission that is gone. + var orphans int + if err := store.db.QueryRow(` + SELECT COUNT(*) FROM observations o + LEFT JOIN transmissions t ON t.id = o.transmission_id + WHERE t.id IS NULL`).Scan(&orphans); err != nil { + t.Fatalf("orphan check: %v", err) + } + if orphans != 0 { + t.Fatalf("expected no orphaned observations, got %d", orphans) + } +} + +// TestPruneOldPacketsDisabledTakesNoWriterLock pins the 0/negative +// short-circuit: retention disabled must not open a transaction at all. +func TestPruneOldPacketsDisabledTakesNoWriterLock(t *testing.T) { + store := openPruneStore(t, "prune-disabled.db") + seedAgedTransmissions(t, store, 5, 1, 10) + + ResetWriterStatsForTest() + + for _, days := range []int{0, -1} { + n, err := store.PruneOldPackets(days) + if err != nil { + t.Fatalf("PruneOldPackets(%d): %v", days, err) + } + if n != 0 { + t.Fatalf("PruneOldPackets(%d) = %d, want 0", days, n) + } + } + + if got := store.WriterStatsSnapshot()["prune_packets"].Count; got != 0 { + t.Fatalf("expected no prune_packets transactions when retention is disabled, got %d", got) + } + if remaining := countRows(t, store, "transmissions"); remaining != 5 { + t.Fatalf("expected 5 transmissions untouched, got %d", remaining) + } +} + +// TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch documents the +// steady-state cost when nothing has aged out yet: a single empty batch, then +// the loop exits. That batch is only cheap because the subquery is walked off +// idx_transmissions_first_seen — see +// TestPruneAgedTransmissionIDsUsesFirstSeenIndex. +func TestPruneOldPacketsNothingToDeleteRunsOneEmptyBatch(t *testing.T) { + store := openPruneStore(t, "prune-noop.db") + seedAgedTransmissions(t, store, 9, 2, 0) + + ResetWriterStatsForTest() + + n, err := store.PruneOldPackets(5) + if err != nil { + t.Fatalf("PruneOldPackets: %v", err) + } + if n != 0 { + t.Fatalf("expected 0 pruned, got %d", n) + } + if got := store.WriterStatsSnapshot()["prune_packets"].Count; got != 1 { + t.Fatalf("expected exactly 1 empty prune_packets transaction, got %d", got) + } + if remaining := countRows(t, store, "transmissions"); remaining != 9 { + t.Fatalf("expected 9 transmissions kept, got %d", remaining) + } +} + +// TestPruneAgedTransmissionIDsUsesFirstSeenIndex pins the query plan of the +// batch subquery and of both statements that embed it. +// +// Ordering the batch by id instead of first_seen makes SQLite drop +// idx_transmissions_first_seen for a rowid SCAN. On a 1M-row table that took +// the terminating, nothing-left batch from ~10µs to ~73ms — once per statement, +// under writerMu, in the state an instance is in whenever nothing has aged out. +// Transaction counts cannot see that regression, so the plan is the assertion. +func TestPruneAgedTransmissionIDsUsesFirstSeenIndex(t *testing.T) { + store := openPruneStore(t, "prune-plan.db") + seedAgedTransmissions(t, store, 20, 2, 10) + + cutoff := time.Now().UTC().AddDate(0, 0, -5).Format(time.RFC3339) + for name, q := range map[string]string{ + "batch subquery": pruneAgedTransmissionIDs, + "observations delete": pruneObservationsBatch, + "transmissions delete": pruneTransmissionsBatch, + } { + rows, err := store.db.Query("EXPLAIN QUERY PLAN "+q, cutoff, pruneBatchTransmissions) + if err != nil { + t.Fatalf("%s: EXPLAIN QUERY PLAN: %v", name, err) + } + var steps []string + for rows.Next() { + var id, parent, notused int + var detail string + if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { + rows.Close() + t.Fatalf("%s: scan plan row: %v", name, err) + } + steps = append(steps, detail) + } + if err := rows.Err(); err != nil { + rows.Close() + t.Fatalf("%s: plan rows: %v", name, err) + } + rows.Close() + plan := strings.Join(steps, " | ") + + if !strings.Contains(plan, "idx_transmissions_first_seen") { + t.Errorf("%s: plan does not use idx_transmissions_first_seen: %s", name, plan) + } + if strings.Contains(plan, "SCAN transmissions") { + t.Errorf("%s: plan scans transmissions, so the empty terminating batch walks the whole table under writerMu: %s", name, plan) + } + if strings.Contains(plan, "TEMP B-TREE") { + t.Errorf("%s: plan sorts in a temp b-tree instead of walking the index in order: %s", name, plan) + } + } +} From 296456f9c14a4de560a3aea86e58195d9306e0aa Mon Sep 17 00:00:00 2001 From: efiten Date: Fri, 11 Sep 2026 15:07:31 +0200 Subject: [PATCH 024/151] feat(map): colour and filter repeaters by scope-configuration state (#2006) Closes #2001. Two review rounds plus a re-review; findings and evidence on the PR. Verified on a deployment against live data: the field over 1653 nodes, marker tints per filter state, the marker title and popup Scope row reaching the DOM, and the colorblind-preset cascade. The audit and the map are held to the same classification by an end-to-end test that fails when either side's wildcard handling drifts. --- .github/workflows/deploy.yml | 1 + cmd/server/routes.go | 38 +++ cmd/server/scope_audit.go | 6 +- cmd/server/scope_config_state.go | 161 +++++++++ cmd/server/scope_config_state_api_test.go | 388 ++++++++++++++++++++++ cmd/server/scope_config_state_test.go | 104 ++++++ docs/api-spec.md | 4 +- public/map.js | 201 ++++++++++- public/style.css | 77 +++++ test-all.sh | 1 + test-issue-2001-map-scope-state.js | 199 +++++++++++ 11 files changed, 1169 insertions(+), 11 deletions(-) create mode 100644 cmd/server/scope_config_state.go create mode 100644 cmd/server/scope_config_state_api_test.go create mode 100644 cmd/server/scope_config_state_test.go create mode 100644 test-issue-2001-map-scope-state.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 34b777d96..de5545517 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -263,6 +263,7 @@ jobs: node test-issue-1420-tile-providers.js node test-issue-1614-tile-url-function.js node test-issue-1438-marker-css-vars.js + node test-issue-2001-map-scope-state.js node test-issue-1846-observers-width.js node test-issue-1562-observers-summary.js node test-issue-1509-nav-active-bg.js diff --git a/cmd/server/routes.go b/cmd/server/routes.go index f9d6f1c59..63565bc40 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -44,6 +44,19 @@ type Server struct { memStatsCache runtime.MemStats memStatsCachedAt time.Time + // #2001: declared region lists for the scope_config_state field, cached + // so /api/nodes does not re-run the merge query pair per request. The + // cached map is read by concurrent requests and replaced, never mutated. + declaredRegionsMu sync.Mutex + declaredRegionsCache map[string]string + declaredRegionsAt time.Time + // Collapses the TTL-boundary herd so the query runs once, not once per + // in-flight request, and never under declaredRegionsMu. + declaredRegionsSF singleflight.Group + // Counts executions of that query. Read by the test that pins the cache: + // the enforceable perf characteristic here is "N requests, one query". + declaredRegionsQueries int64 + // Cached /api/stats response — recomputed at most once every 10s statsMu sync.Mutex statsCache *StatsResponse @@ -1307,6 +1320,16 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { relayMap = s.store.GetRepeaterRelayInfoMap(relayWindow) usefulMap = s.store.GetRepeaterUsefulnessScoreMap() } + // #2001: declared region lists, once per request rather than per + // node, so the map can colour every repeater by its scope-config + // state. Two small queries (232 rows on a live instance) and no + // window scan — the state is a pure function of the declared list, + // see nodeScopeConfigState. + var declaredCSV map[string]string + declaredOK := false + if needsRelay { + declaredCSV, declaredOK = s.declaredRegionsCSV() + } // Bridge axis (#672 axis 2 of 4). Snapshot is an atomic load // — safe to call regardless of needsRelay, and we want the // score on repeater rows specifically. @@ -1344,6 +1367,15 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { if len(info.TransportedScopes) > 0 { node["transported_scopes"] = info.TransportedScopes } + // #2001: how this repeater's region config reads, from + // its own declared answer where it gave one and from + // what it has been observed carrying where it did not. + // Omitted entirely when the declared lookup failed — + // see declaredRegionsCSV. + if declaredOK { + csv, has := declaredCSV[strings.ToLower(pk)] + node["scope_config_state"] = nodeScopeConfigState(csv, has, info.TransportedScopes) + } // #672 4-axis usefulness. traffic_share_score keeps the // raw per-axis Traffic value (#1456); the structural axes // are surfaced individually; the composite uses the @@ -1541,6 +1573,12 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) { if len(info.TransportedScopes) > 0 { node["transported_scopes"] = info.TransportedScopes } + // #2001: same field, same rules as handleNodes — the node page + // and the map must not disagree about a repeater's scope state. + if declaredCSV, ok := s.declaredRegionsCSV(); ok { + csv, has := declaredCSV[strings.ToLower(pubkey)] + node["scope_config_state"] = nodeScopeConfigState(csv, has, info.TransportedScopes) + } // #672 4-axis usefulness (see handleNodes for the field // contract). traffic_share_score keeps the raw per-axis // Traffic value (#1456); the composite uses the diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index d7cf5de06..d6e45ebab 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -1123,7 +1123,11 @@ func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse declaredNamed := make([]string, 0, len(allRegions)) declaredSet := make(map[string]bool, len(allRegions)) for _, rgn := range allRegions { - if rgn == "*" { + // isScopeWildcard, not a bare "*" compare: a collector storing the + // "#*" spelling would otherwise have it counted as a named region + // here and as the wildcard by /api/nodes, and the two pages would + // classify the same repeater differently (#2006 review). + if isScopeWildcard(rgn) { declaredWildcard = true continue } diff --git a/cmd/server/scope_config_state.go b/cmd/server/scope_config_state.go new file mode 100644 index 000000000..de88027da --- /dev/null +++ b/cmd/server/scope_config_state.go @@ -0,0 +1,161 @@ +package main + +import ( + "log" + "strings" + "sync/atomic" + "time" +) + +// The /api/nodes scope_config_state field (#2001). The Scope Audit answers +// "which repeaters declare a region they are not forwarding", and to do that +// it lists only repeaters that have answered a declared-regions request at +// all — 232 of 1289 repeaters on one live instance. /api/nodes has to say +// something about the other 1057, because the map colours every repeater it +// draws and "absent from a different endpoint" is not a colour. +// +// So the field carries the audit's four declared states unchanged, plus two +// that only make sense for a node with no declared answer: +// +// - ScopeConfigObserved: never answered, but we have seen it forward +// scoped traffic, so it demonstrably has a region configured. We just do +// not know which ones, nor whether it also forwards unscoped floods. +// Unlike the four declared states, this one is not a pure function of +// stored data: it reads transported_scopes, which the in-memory packet +// store accumulates over its retention window, so a repeater can move +// between Observed and None after a restart with nothing having changed +// in the network. +// - ScopeConfigNone: never answered, and nothing scoped has been observed +// through it either. +// +// ScopeConfigNone is NOT a finding. Firmware drops scoped floods for regions +// it holds no key for, so a repeater with no region config and a repeater +// nobody has sent scoped traffic past look identical from here. The value +// means "we have not been told", and the UI has to say so. +const ( + ScopeConfigObserved = "observed" // no declared answer; observed forwarding scoped traffic + ScopeConfigNone = "none" // no declared answer; no scoped traffic observed either +) + +// isScopeWildcard recognises both spellings of the flood wildcard. The +// ingestor treats "*" and "#*" as the same thing (cmd/ingestor/region_keys.go), +// and the two readers of a declared list have to agree: counting the prefixed +// form as a named region turns a fully configured repeater into +// ScopeConfigNoUnscoped, which reads as a fault, and does it on one page only. +// +// Shared with the scope audit deliberately. The map and that page classify the +// same declared list, so the rule lives in one place and +// TestScopeConfigStateAgreesWithTheAudit holds them to it. +func isScopeWildcard(region string) bool { + return region == "*" || region == "#*" +} + +// nodeScopeConfigState classifies one node for the scope_config_state field. +// +// declaredCSV is the node's most recent declared-regions answer (the raw +// regions_csv) and hasDeclared says whether such an answer exists at all — +// the two cannot be collapsed into "declaredCSV != \"\"", because answering +// with an empty list is itself a state (ScopeConfigNoFlood: nothing is +// flood-allowed, not even unscoped traffic) and is different from never +// having been asked. +// +// A declared answer always wins over transportedScopes. The answer is the +// repeater's own word about its configuration; traffic is what we happened to +// see. Where the two disagree, reporting that contradiction is the Scope +// Audit's job, and re-labelling the node here would hide it. +func nodeScopeConfigState(declaredCSV string, hasDeclared bool, transportedScopes []string) string { + if hasDeclared { + var named []string + wildcard := false + for _, rgn := range splitRegionsCSV(declaredCSV) { + if isScopeWildcard(rgn) { + wildcard = true + continue + } + named = append(named, rgn) + } + return scopeAuditConfigState(named, wildcard) + } + if len(transportedScopes) > 0 { + return ScopeConfigObserved + } + return ScopeConfigNone +} + +// declaredRegionsTTL bounds how stale the per-request declared map may be. +// A declared answer changes when an observer reports or a drive is uploaded, +// which is hours apart per node, so a few seconds of staleness is invisible — +// while /api/nodes is the busiest endpoint in the server and the map pages +// through it. The scope-audit handler caches the same lookup for 30s +// (scope_audit.go), and using the same figure keeps the two pages from +// disagreeing for longer than either is stale on its own. +const declaredRegionsTTL = 30 * time.Second + +// declaredRegionsCSV maps target pubkey to its newest declared regions_csv, +// merged across every collector this database carries, keyed lowercase. +// +// Keys are lowercased because the two sides of the join disagree by design: +// nodes.public_key is lowercased by migration, while node_declared_regions is +// filled by an external collector whose casing this repo does not control. The +// audit already lowercases the declared target before joining; a map that did +// not would classify the same repeater differently on two pages. +// +// The bool is false when the answer is unknown rather than absent, which +// callers must not confuse with an empty map. Two things make it unknown: the +// query failed, or this database carries neither source, in which case "no +// repeater has answered" is a statement the schema cannot support. An empty map +// with ok=true means the sources exist and nobody has answered yet, which IS a +// finding and is what ScopeConfigNone reports. +func (s *Server) declaredRegionsCSV() (map[string]string, bool) { + if !s.db.hasConfiguredScope && !s.db.hasDeclaredRegionsTable { + return nil, false + } + + s.declaredRegionsMu.Lock() + cached, at := s.declaredRegionsCache, s.declaredRegionsAt + s.declaredRegionsMu.Unlock() + if cached != nil && time.Since(at) < declaredRegionsTTL { + return cached, true + } + + // The query runs outside the mutex, and singleflight collapses the herd at + // the TTL boundary into one execution. Holding the lock across it would + // serialise every concurrent /api/nodes request behind a full scan of nodes + // plus a window function over node_declared_regions — on the busiest + // endpoint in the server, every 30s. AGENTS.md: copy under the lock, + // process outside it. + v, err, _ := s.declaredRegionsSF.Do("declared-regions", func() (interface{}, error) { + // Double-check inside the flight: a previous winner may have stored a + // fresh map between this caller's read above and its arrival here, and + // singleflight frees the key as soon as that winner returns. Without + // this, a TTL boundary under load can run the query a second time for + // nothing. Same idiom as the stats cache in routes.go. + s.declaredRegionsMu.Lock() + fresh, freshAt := s.declaredRegionsCache, s.declaredRegionsAt + s.declaredRegionsMu.Unlock() + if fresh != nil && time.Since(freshAt) < declaredRegionsTTL { + return fresh, nil + } + + rows, qerr := s.db.AllCurrentDeclaredRegions() + if qerr != nil { + return nil, qerr + } + atomic.AddInt64(&s.declaredRegionsQueries, 1) + out := make(map[string]string, len(rows)) + for _, r := range rows { + out[strings.ToLower(r.Target)] = r.RegionsCSV + } + s.declaredRegionsMu.Lock() + // Cached maps are handed to concurrent readers and never written again. + s.declaredRegionsCache = out + s.declaredRegionsAt = time.Now() + s.declaredRegionsMu.Unlock() + return out, nil + }) + if err != nil { + log.Printf("[nodes] declared-regions lookup failed, scope_config_state omitted: %v", err) + return nil, false + } + return v.(map[string]string), true +} diff --git a/cmd/server/scope_config_state_api_test.go b/cmd/server/scope_config_state_api_test.go new file mode 100644 index 000000000..c1b5b0ca4 --- /dev/null +++ b/cmd/server/scope_config_state_api_test.go @@ -0,0 +1,388 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// setupScopeConfigStateServer is setupTestServer plus the declared-regions +// column. The stock test schema predates configured_scope, so without this +// the DB reports hasConfiguredScope == false and every node would classify as +// "never asked" — the fixture would agree with a broken implementation. +func setupScopeConfigStateServer(t *testing.T) (*Server, *mux.Router) { + t.Helper() + srv, router := setupTestServer(t) + for _, stmt := range []string{ + `ALTER TABLE nodes ADD COLUMN configured_scope TEXT`, + `ALTER TABLE nodes ADD COLUMN configured_scope_at TEXT`, + } { + if _, err := srv.db.conn.Exec(stmt); err != nil { + t.Fatal(err) + } + } + if err := srv.db.detectSchema(context.Background(), srv.db.conn); err != nil { + t.Fatal(err) + } + if !srv.db.hasConfiguredScope { + t.Fatal("hasConfiguredScope is false after adding the column: the fixture would test nothing") + } + return srv, router +} + +func nodesByPubkey(t *testing.T, router *mux.Router, query string) map[string]map[string]interface{} { + t.Helper() + req := httptest.NewRequest("GET", "/api/nodes"+query, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var resp struct { + Nodes []map[string]interface{} `json:"nodes"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + out := map[string]map[string]interface{}{} + for _, n := range resp.Nodes { + pk, _ := n["public_key"].(string) + out[pk] = n + } + return out +} + +// TestHandleNodesExposesScopeConfigState pins the field the map colours by +// (#2001): a repeater that has answered a declared-regions request carries +// that answer's state, and one that never answered carries "none" rather +// than being silently indistinguishable from a fully configured node. +func TestHandleNodesExposesScopeConfigState(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES + ('PK_DECLARED', 'declared-rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be,*', '2026-01-01T00:00:00Z'), + ('PK_SILENT', 'silent-rp', 'repeater', 51.1, 4.1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL)`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + if got := nodes["PK_DECLARED"]["scope_config_state"]; got != ScopeConfigFull { + t.Errorf("declared repeater scope_config_state = %v, want %q", got, ScopeConfigFull) + } + if got := nodes["PK_SILENT"]["scope_config_state"]; got != ScopeConfigNone { + t.Errorf("never-asked repeater scope_config_state = %v, want %q", got, ScopeConfigNone) + } +} + +// TestHandleNodesOmitsScopeConfigStateForNonForwarders keeps the field on the +// roles that forward. A companion neither forwards nor answers a +// declared-regions request, so classifying it would state something we have +// no basis for. +func TestHandleNodesOmitsScopeConfigStateForNonForwarders(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES ('PK_COMPANION', 'phone', 'companion', 51.2, 4.2, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + if _, present := nodes["PK_COMPANION"]["scope_config_state"]; present { + t.Errorf("companion carries scope_config_state = %v, want the field absent", + nodes["PK_COMPANION"]["scope_config_state"]) + } +} + +// TestHandleNodeDetailExposesScopeConfigState keeps the single-node endpoint +// from drifting away from the list endpoint: the node page and the map must +// not disagree about a repeater's scope state. +func TestHandleNodeDetailExposesScopeConfigState(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES ('PK_DETAIL', 'detail-rp', 'repeater', 51.3, 4.3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be', '2026-01-01T00:00:00Z')`, + ); err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest("GET", "/api/nodes/PK_DETAIL", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + // The detail endpoint wraps the node: {"node": {...}, "recentAdverts": [...]}. + var resp struct { + Node map[string]interface{} `json:"node"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if got := resp.Node["scope_config_state"]; got != ScopeConfigNoUnscoped { + t.Errorf("scope_config_state = %v, want %q", got, ScopeConfigNoUnscoped) + } +} + +// TestHandleNodesOmitsScopeConfigStateWithoutASource is the difference between +// "nobody has answered" and "this database cannot hold an answer". The stock +// test schema has neither configured_scope nor node_declared_regions, so every +// repeater would otherwise be labelled "none" — a positive claim about a +// network, made from a database that structurally cannot carry the fact. +func TestHandleNodesOmitsScopeConfigStateWithoutASource(t *testing.T) { + srv, router := setupTestServer(t) + if srv.db.hasConfiguredScope || srv.db.hasDeclaredRegionsTable { + t.Fatal("fixture has a declared-regions source: this test would prove nothing") + } + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES ('PK_NOSOURCE', 'rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + if v, present := nodes["PK_NOSOURCE"]["scope_config_state"]; present { + t.Errorf("repeater carries scope_config_state = %v on a schema with no declared-regions source, want the field absent", v) + } +} + +// TestHandleNodesMatchesDeclaredTargetCaseInsensitively pins the join the +// Scope Audit already makes case-insensitively (scope_audit.go lowercases the +// declared target before joining). node_declared_regions is filled by an +// external collector whose casing this repo does not control, and a case-only +// mismatch would put the audit and the map in disagreement about the same node. +func TestHandleNodesMatchesDeclaredTargetCaseInsensitively(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES ('pk_mixedcase', 'rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL)`, + ); err != nil { + t.Fatal(err) + } + // The declared answer names the same node in a different case. + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, configured_scope, configured_scope_at) + VALUES ('PK_MIXEDCASE', 'be,*', '2026-01-01T00:00:00Z')`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + if got := nodes["pk_mixedcase"]["scope_config_state"]; got != ScopeConfigFull { + t.Errorf("scope_config_state = %v for a declared answer in the other case, want %q", got, ScopeConfigFull) + } +} + +// TestDeclaredRegionsLookupRunsOncePerWindow is the perf assertion AGENTS.md +// asks for: the claim is that scope_config_state costs /api/nodes one cached +// DB round-trip, not one per request, and the enforceable form of that claim is +// the query count. Twenty concurrent requests inside the TTL window must +// produce exactly one execution — one for the cold start, none after. +// +// What this pins is the cache, not the lock: with the cache short-circuit +// removed it reports 6 to 9 executions, and it passes either way on the older +// shape that held the mutex across the query. The lock fix is argued from the +// code, not from this test — see the singleflight comment in +// scope_config_state.go. +func TestDeclaredRegionsLookupRunsOncePerWindow(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES ('PK_PERF', 'rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be,*', '2026-01-01T00:00:00Z')`, + ); err != nil { + t.Fatal(err) + } + + atomic.StoreInt64(&srv.declaredRegionsQueries, 0) + + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { + defer wg.Done() + req := httptest.NewRequest("GET", "/api/nodes?limit=200", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Errorf("status = %d, want 200", w.Code) + } + }() + } + wg.Wait() + + if got := atomic.LoadInt64(&srv.declaredRegionsQueries); got != 1 { + t.Errorf("declared-regions query ran %d times for 20 requests inside the TTL, want 1", got) + } + // And the field is still correct, so the cache is not just cheap. + nodes := nodesByPubkey(t, router, "?limit=200") + if got := nodes["PK_PERF"]["scope_config_state"]; got != ScopeConfigFull { + t.Errorf("scope_config_state = %v, want %q", got, ScopeConfigFull) + } +} + +// TestHandleNodesReportsObservedForUndeclaredForwarder pins the half of the +// field that does not come from a declared answer. A repeater that never +// answered but has been seen carrying scoped traffic must read "observed", not +// "none" — the difference between "has a region configured, we just have not +// asked which" and "we have nothing at all", which is the distinction the map +// colours by. +// +// The store is built after the rows are seeded, because TransportedScopes is +// accumulated at load time from transmissions.scope_name joined to the path +// hops, not read per request. +func TestHandleNodesReportsObservedForUndeclaredForwarder(t *testing.T) { + db := setupTestDB(t) + seedTestData(t, db) + + // The columns the two halves of the field read, on a schema old enough to + // have neither. + for _, stmt := range []string{ + `ALTER TABLE nodes ADD COLUMN configured_scope TEXT`, + `ALTER TABLE nodes ADD COLUMN configured_scope_at TEXT`, + `ALTER TABLE transmissions ADD COLUMN scope_name TEXT`, + } { + if _, err := db.conn.Exec(stmt); err != nil { + t.Fatal(err) + } + } + + // 64-hex pubkey: the path-hop join matches a truncated hop against the + // node's own pubkey prefix, so a short fixture key would match nothing. + const pk = "a1b2c3d4e5f60718293a4b5c6d7e8f90a1b2c3d4e5f60718293a4b5c6d7e8f90" + if _, err := db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES (?, 'silent-forwarder', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, pk, + ); err != nil { + t.Fatal(err) + } + res, err := db.conn.Exec( + `INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, payload_version, decoded_json, scope_name) + VALUES ('AA', 'observed-scope-hash', ?, 0, 1, 1, '{}', '#be')`, + time.Now().UTC().Format(time.RFC3339), + ) + if err != nil { + t.Fatal(err) + } + txID, err := res.LastInsertId() + if err != nil { + t.Fatal(err) + } + // The hop is the full pubkey, not a truncated wire hop. TransportedScopes + // is gated on full-pubkey attribution (#1902 — a 1-byte hop cannot prove + // which of the nodes sharing that byte carried the packet), and in + // production the full key is produced by hop resolution rather than read + // off the wire. Seeding the resolved form exercises the same index entry + // that resolution writes. The decoder uppercases hops, so the fixture does + // too, which also pins the lower-casing in addTxToPathHopIndex. + if _, err := db.conn.Exec( + `INSERT INTO observations (transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp) + VALUES (?, 0, 'rx', 1.0, -100, 0, ?, ?)`, + txID, `["`+strings.ToUpper(pk)+`"]`, time.Now().Unix(), + ); err != nil { + t.Fatal(err) + } + + if err := db.detectSchema(context.Background(), db.conn); err != nil { + t.Fatal(err) + } + cfg := &Config{Port: 3000} + srv := NewServer(db, cfg, NewHub()) + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("background indexes never became ready") + } + srv.store = store + router := mux.NewRouter() + srv.RegisterRoutes(router) + + nodes := nodesByPubkey(t, router, "?limit=200") + node := nodes[pk] + if node == nil { + t.Fatalf("seeded repeater missing from the response") + } + if node["transported_scopes"] == nil { + t.Fatalf("fixture did not produce transported_scopes; the test would prove nothing (got %v)", node) + } + if got := node["scope_config_state"]; got != ScopeConfigObserved { + t.Errorf("scope_config_state = %v for a never-asked repeater carrying scoped traffic, want %q", got, ScopeConfigObserved) + } +} + +// TestScopeAuditAndNodesAgreeEndToEnd is the guard the unit-level agreement +// test cannot be: it drives the two real handlers over the same stored answer +// and compares what each page would show. The unit test compares against a +// copy of the audit's parse loop living in the test file, so reverting the +// production fix in scope_audit.go leaves it green — which is the exact +// regression this round was opened for. +func TestScopeAuditAndNodesAgreeEndToEnd(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + // /api/scope-audit reads transmissions.scope_name; the stock test schema + // predates it and the handler fails without it. + if _, err := srv.db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT`); err != nil { + t.Fatal(err) + } + if err := srv.db.detectSchema(context.Background(), srv.db.conn); err != nil { + t.Fatal(err) + } + + cases := map[string]string{ + "PK_AGREE_HASHSTAR": "#be,#*", + "PK_AGREE_BARESTAR": "be,*", + "PK_AGREE_STARONLY": "#*", + "PK_AGREE_NAMEDONLY": "#be", + "PK_AGREE_EMPTY": "", + } + for pk, csv := range cases { + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES (?, ?, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, ?, '2026-01-01T00:00:00Z')`, + pk, "rp-"+pk, csv, + ); err != nil { + t.Fatal(err) + } + } + + audit := getScopeAudit(t, router, "?window=24h") + // The audit lowercases the declared target before joining (scope_audit.go), + // so its PublicKey comes back lowercased whatever the collector stored. + auditState := map[string]string{} + for _, row := range audit.Repeaters { + auditState[strings.ToLower(row.PublicKey)] = row.ConfigState + } + nodes := nodesByPubkey(t, router, "?limit=200") + + for pk, csv := range cases { + got, ok := auditState[strings.ToLower(pk)] + if !ok { + t.Errorf("%s (%q): the scope audit does not list the repeater at all", pk, csv) + continue + } + want := nodes[pk]["scope_config_state"] + if got != want { + t.Errorf("regions_csv %q: the scope audit page shows %q, the map shows %v", csv, got, want) + } + } +} diff --git a/cmd/server/scope_config_state_test.go b/cmd/server/scope_config_state_test.go new file mode 100644 index 000000000..d917a2384 --- /dev/null +++ b/cmd/server/scope_config_state_test.go @@ -0,0 +1,104 @@ +package main + +import "testing" + +// TestNodeScopeConfigState pins the classification behind the /api/nodes +// scope_config_state field (#2001). The four declared states are the audit's +// own vocabulary; the two extra ones exist because /api/nodes has to say +// something about a repeater the audit omits entirely — one that has never +// answered a declared-regions request. +func TestNodeScopeConfigState(t *testing.T) { + tests := []struct { + name string + declaredCSV string + hasDeclared bool + transported []string + want string + }{ + {"named regions and wildcard", "be,*", true, nil, ScopeConfigFull}, + {"wildcard only", "*", true, nil, ScopeConfigNoScopes}, + {"named regions only", "be", true, nil, ScopeConfigNoUnscoped}, + {"answered with an empty list", "", true, nil, ScopeConfigNoFlood}, + {"never asked, but observed carrying a scope", "", false, []string{"#be"}, ScopeConfigObserved}, + {"never asked, nothing scoped observed", "", false, nil, ScopeConfigNone}, + // The declared answer is the repeater's own word about its config and + // outranks what we happened to see it carry: a node that answered with + // an empty list stays no-flood even though traffic says otherwise. + // That contradiction is the audit's finding to report, not a reason to + // re-label the node here. + {"empty answer outranks observed traffic", "", true, []string{"#be"}, ScopeConfigNoFlood}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := nodeScopeConfigState(tt.declaredCSV, tt.hasDeclared, tt.transported) + if got != tt.want { + t.Errorf("nodeScopeConfigState(%q, %v, %v) = %q, want %q", + tt.declaredCSV, tt.hasDeclared, tt.transported, got, tt.want) + } + }) + } +} + +// TestNodeScopeConfigStateIgnoresWhitespaceAndCase pins the two shapes a +// declared list actually arrives in. regions_csv is written by two different +// collectors, and one of them spaces its entries, so " be , * " has to +// classify as full rather than as a pair of unrecognised names. +func TestNodeScopeConfigStateIgnoresWhitespaceAndCase(t *testing.T) { + if got := nodeScopeConfigState(" be , * ", true, nil); got != ScopeConfigFull { + t.Errorf("spaced list = %q, want %q", got, ScopeConfigFull) + } + if got := nodeScopeConfigState("#be", true, nil); got != ScopeConfigNoUnscoped { + t.Errorf("hash-prefixed region = %q, want %q", got, ScopeConfigNoUnscoped) + } +} + +// TestNodeScopeConfigStateAcceptsHashPrefixedWildcard pins the second spelling +// of the wildcard. The ingestor treats "*" and "#*" as the same thing +// (cmd/ingestor/region_keys.go:94), so a collector storing the prefixed form +// must not have it counted as a named region — that would turn a fully +// configured repeater into "no unscoped", which reads as a fault. +func TestNodeScopeConfigStateAcceptsHashPrefixedWildcard(t *testing.T) { + if got := nodeScopeConfigState("#be,#*", true, nil); got != ScopeConfigFull { + t.Errorf("nodeScopeConfigState(\"#be,#*\") = %q, want %q", got, ScopeConfigFull) + } + if got := nodeScopeConfigState("#*", true, nil); got != ScopeConfigNoScopes { + t.Errorf("nodeScopeConfigState(\"#*\") = %q, want %q", got, ScopeConfigNoScopes) + } +} + +// TestScopeConfigStateAgreesWithTheAudit is the one assertion that keeps the +// map and the Scope Audit from telling an operator two different things about +// the same repeater. Both sides classify a declared list; they must classify it +// the same way, on the same input, or one of the two pages is lying. +// +// It compares the real functions rather than two copies of the rules: the +// audit's parse loop and nodeScopeConfigState both run over the same CSV. +func TestScopeConfigStateAgreesWithTheAudit(t *testing.T) { + for _, csv := range []string{ + "be,*", "*", "be", "", "#be", "#be,*", "#be,#*", "#*", "#be,#*,#nl", + " be , * ", "be-vlg,be,*", + } { + audit := auditConfigStateForCSV(csv) + api := nodeScopeConfigState(csv, true, nil) + if audit != api { + t.Errorf("regions_csv %q: scope audit says %q, /api/nodes says %q", csv, audit, api) + } + } +} + +// auditConfigStateForCSV replays the audit's own declared-list parse +// (handleScopeAudit's loop over splitRegionsCSV) and classifies the result, so +// the comparison above is against what that page actually computes. +func auditConfigStateForCSV(csv string) string { + declaredWildcard := false + declaredNamed := make([]string, 0) + for _, rgn := range splitRegionsCSV(csv) { + if isScopeWildcard(rgn) { + declaredWildcard = true + continue + } + declaredNamed = append(declaredNamed, normScope(rgn)) + } + return scopeAuditConfigState(declaredNamed, declaredWildcard) +} diff --git a/docs/api-spec.md b/docs/api-spec.md index 9971689b3..8790696b7 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -317,7 +317,8 @@ Paginated node list with filtering. "hash_size_inconsistent": boolean, // true if flip-flopping "hash_sizes_seen": [number] | undefined, // present only if >1 unique size seen "last_heard": string (ISO) | undefined, // from in-memory packets or path relay - "default_scope": string | null | undefined // Most recently observed transport scope for this node. null = never observed transport-scoped, "" = observed scoped but no configured region matched, "#name" = matched region. Only present when ingestor has applied the nodes_default_scope_v1 migration. + "default_scope": string | null | undefined, // Most recently observed transport scope for this node. null = never observed transport-scoped, "" = observed scoped but no configured region matched, "#name" = matched region. Only present when ingestor has applied the nodes_default_scope_v1 migration. + "scope_config_state": string | undefined // Repeater/room only. How this node's region config reads: "full" | "no-unscoped" | "no-scopes" | "no-flood" from its own declared-regions answer, "observed" when it never answered but has been seen forwarding scoped traffic, "none" when it never answered and nothing scoped was observed. Absent for other roles and when the declared-regions lookup failed. } ], "total": number, // total matching count (before pagination) @@ -333,6 +334,7 @@ Paginated node list with filtering. **Notes:** - `hash_sizes_seen` is only present when more than one hash size has been observed. - `last_heard` is only present when in-memory data provides a more recent timestamp than `last_seen`. +- `scope_config_state` carries the same four declared states as [GET /api/scope-audit](#get-apiscope-audit) and is computed the same way, from the newest declared-regions answer merged across collectors. The audit lists only repeaters that have answered; this field also classifies the ones that have not, which is what the map colours by. `"none"` means the answer is missing, not that the node is misconfigured: firmware drops scoped floods for regions it holds no key for, so a repeater with no region config and one nobody sends scoped traffic past are indistinguishable here. The four declared states are computed from stored data and do not depend on a window; `"observed"` and `"none"` are separated by `transported_scopes`, which the in-memory packet store accumulates over its retention window, so a node can move between them after a restart. The field is absent entirely when this database carries no declared-regions source at all, since "nobody has answered" is then a claim the schema cannot support. --- diff --git a/public/map.js b/public/map.js index 5c6fcba1b..f68a89ae3 100644 --- a/public/map.js +++ b/public/map.js @@ -9,7 +9,7 @@ let nodes = []; let targetNodeKey = null; let observers = []; - let filters = { repeater: true, companion: true, room: true, sensor: true, observer: true, lastHeard: '30d', neighbors: false, clustering: localStorage.getItem('meshcore-map-clustering') !== 'false', hashLabels: localStorage.getItem('meshcore-map-hash-labels') !== 'false', statusFilter: localStorage.getItem('meshcore-map-status-filter') || 'all', byteSize: localStorage.getItem('meshcore-map-byte-filter') || 'all', multiByteOverlay: localStorage.getItem('meshcore-map-multibyte-overlay') === 'true' }; + let filters = { repeater: true, companion: true, room: true, sensor: true, observer: true, lastHeard: '30d', neighbors: false, clustering: localStorage.getItem('meshcore-map-clustering') !== 'false', hashLabels: localStorage.getItem('meshcore-map-hash-labels') !== 'false', statusFilter: localStorage.getItem('meshcore-map-status-filter') || 'all', byteSize: localStorage.getItem('meshcore-map-byte-filter') || 'all', multiByteOverlay: localStorage.getItem('meshcore-map-multibyte-overlay') === 'true', scopeState: localStorage.getItem('meshcore-map-scope-filter') || 'all', scopeOverlay: localStorage.getItem('meshcore-map-scope-overlay') === 'true' }; let selectedReferenceNode = null; // pubkey of the reference node for neighbor filtering let neighborPubkeys = null; // Set of pubkeys that are direct neighbors of selected node let wsHandler = null; @@ -64,6 +64,100 @@ // scope (not loop-local) to avoid per-iteration object allocation. var MB_MARKER_TINT = { confirmed: '#56F0A0', suspected: '#FFD966', unknown: '#FF8888' }; + // #2001 — repeater scope-configuration state, read straight from the + // scope_config_state field /api/nodes carries per forwarding node. The four + // declared states are the Scope Audit's own vocabulary (scope-audit.js + // CONFIG_STATES) and their titles say the same thing, so a repeater does not + // mean two different things on two pages. + // + // The two extra states exist because the audit lists only repeaters that + // have answered a declared-regions request, and the map draws every + // repeater. 'observed' is a node that never answered but has been seen + // forwarding scoped traffic; 'none' is one that never answered and has not + // been seen carrying anything scoped either. + // + // 'none' is not a fault. Firmware drops scoped floods for regions it holds + // no key for, so a repeater with no region config and a repeater nobody has + // sent scoped traffic past look identical from here — the title says so + // rather than leaving the grey to be read as red. + var SCOPE_STATES = [ + { key: 'full', label: 'Full', + title: 'Declares named regions and \'*\': forwards its declared regions and plain unscoped floods.' }, + { key: 'observed', label: 'Observed', + title: 'Never answered a declared-regions request, but has been observed forwarding scoped traffic, so it has a region configured. Which ones, and whether it also forwards unscoped floods, is unknown.' }, + { key: 'no-unscoped', label: 'No unscoped', + title: 'Declares named regions but not \'*\': does not forward plain unscoped floods. Exact, not an inference.' }, + { key: 'no-scopes', label: 'No scopes', + title: 'Declares only \'*\', no named regions: no region is flood-allowed. Almost always means no scopes are configured, but a repeater whose regions are all set to deny flooding looks identical.' }, + { key: 'no-flood', label: 'No flood', + title: 'Declares neither named regions nor \'*\': an answered-but-empty list. Nothing is flood-allowed by this repeater, not even plain unscoped traffic.' }, + { key: 'none', label: 'No data', + title: 'Never answered a declared-regions request and nothing scoped has been observed through it. That is missing information, not a finding: a repeater with no region config and one nobody sends scoped traffic past look the same from here.' }, + ]; + var SCOPE_STATE_KEYS = {}; + SCOPE_STATES.forEach(function (s) { SCOPE_STATE_KEYS[s.key] = true; }); + + // scopeFilterAccepts answers whether a node survives the scope filter. + // A node with no scope_config_state is unclassified, never 'none': the + // server omits the field for roles that do not forward and for every node + // when the declared-regions lookup failed, and folding those into the + // no-data bucket would fill the category operators hunt in with phones. + function scopeFilterAccepts(n, selection) { + if (!selection || selection === 'all') return true; + return n.scope_config_state === selection; + } + + // scopeStateLabel is the short name for a state, or '' for anything this + // build does not know. Used by the marker alt text so the dot-marker path + // carries the state in words too: the label path has a class, an aria-label + // and a title, and a bare dot would otherwise be colour and nothing else — + // which on the achromat preset is six greys within about 1.3:1 of each other. + function scopeStateLabel(state) { + for (var i = 0; i < SCOPE_STATES.length; i++) { + if (SCOPE_STATES[i].key === state) return SCOPE_STATES[i].label; + } + return ''; + } + + // scopeTint returns the marker fill for a node's scope state, or null when + // the node has none — null leaves the role colour in place rather than + // painting an unclassified node as if it had been measured. An unknown + // state string also returns null: emitting var(--mc-scope-) would + // resolve to nothing and draw an invisible marker. + function scopeTint(n) { + var state = n && n.scope_config_state; + if (!state || !SCOPE_STATE_KEYS[state]) return null; + return 'var(--mc-scope-' + state + ')'; + } + + // scopeTintEnabled says whether markers should carry scope colours. The + // overlay checkbox turns them on for the whole map, and picking a state in + // the filter turns them on too: an operator who just asked "show me the + // repeaters with no scope data" should not have to find a second control to + // see the answer in the colour they picked it by. + function scopeTintEnabled(overlayOn, selection) { + return !!overlayOn || (!!selection && selection !== 'all'); + } + + // scopeFilterHtml builds the button group. Same shape as the byte-size and + // status groups above it, so the panel keeps one filter idiom. + function scopeFilterHtml(current) { + var cur = current || 'all'; + // Each state button carries the swatch its markers are painted with: the + // map has no separate colour legend, so this group is the legend. + var btn = function (key, label, title, swatch) { + var dot = swatch ? ' ' : ''; + return ''; + }; + return '
' + + 'Scope Config' + + '
' + + btn('all', 'All', 'Every node, whatever its scope state') + + SCOPE_STATES.map(function (s) { return btn(s.key, s.label, s.title, true); }).join('') + + '
'; + } + function makeMarkerIcon(role, isStale, isAlsoObserver, colorOverride) { const s = ROLE_STYLE[role] || ROLE_STYLE.companion; // #1438: default fill resolves through the live CSS var so existing @@ -140,7 +234,7 @@ }); } - function makeRepeaterLabelIcon(node, isStale, isAlsoObserver, mbStatus) { + function makeRepeaterLabelIcon(node, isStale, isAlsoObserver, mbStatus, scopeState) { // Show the short mesh hash ID (first N bytes of pubkey, uppercased). When // the width is unobserved the label falls back to one byte — it has to draw // something — but says so instead of asserting a 1-byte config. @@ -161,8 +255,21 @@ var shortHash = hashInfo.prefix; var statusClass = status ? (' ' + (MB_STATUS_CLASS[status] || MB_STATUS_CLASS.unknown)) : ''; var ariaWidth = hashInfo.known ? '' : ', hash size unknown'; - var ariaStatus = status ? ('multi-byte ' + status + ', hash ' + shortHash + ariaWidth) - : ('repeater hash ' + shortHash + ariaWidth); + // #2001: the scope-config overlay colours this label's left border, and a + // colour alone says nothing to a screen reader or to an operator who + // cannot separate the hues. The state goes into the aria-label and into a + // hover title so the stripe is never the only carrier. + var scopeMeta = null; + if (scopeState) { + for (var si = 0; si < SCOPE_STATES.length; si++) { + if (SCOPE_STATES[si].key === scopeState) { scopeMeta = SCOPE_STATES[si]; break; } + } + } + var scopeClass = scopeMeta ? (' scope-' + scopeMeta.key) : ''; + var scopeAria = scopeMeta ? (', scope config ' + scopeMeta.label) : ''; + var scopeTitle = scopeMeta ? (' title="' + scopeMeta.label + ' — ' + scopeMeta.title.replace(/"/g, '"') + '"') : ''; + var ariaStatus = status ? ('multi-byte ' + status + ', hash ' + shortHash + ariaWidth + scopeAria) + : ('repeater hash ' + shortHash + ariaWidth + scopeAria); // Observer indicator stays a star — it is an orthogonal signal, not a status color. var obsIndicator = isAlsoObserver ? ' ' @@ -170,7 +277,7 @@ // Glyph + thin-space (U+2009) + hash. Visible content is aria-hidden so AT // reads the aria-label only (avoids "check mark 3 E" literal announcements). var visible = (glyph ? glyph + '\u2009' : '') + shortHash; - var html = ' + ${scopeFilterHtml(filters.scopeState)}
Filters @@ -584,6 +693,15 @@ multiByteEl.checked = filters.multiByteOverlay; multiByteEl.addEventListener('change', e => { filters.multiByteOverlay = e.target.checked; localStorage.setItem('meshcore-map-multibyte-overlay', e.target.checked); renderMarkers(); }); } + // #2001 scope-config colours. Same shape as the multi-byte overlay above, + // and deliberately a separate control from the Scope Config filter: the + // filter answers "show me only these", the overlay answers "colour what + // is on screen", and an operator wants either without the other. + const scopeOverlayEl = document.getElementById('mcScopeOverlay'); + if (scopeOverlayEl) { + scopeOverlayEl.checked = filters.scopeOverlay; + scopeOverlayEl.addEventListener('change', e => { filters.scopeOverlay = e.target.checked; localStorage.setItem('meshcore-map-scope-overlay', e.target.checked); renderMarkers(); }); + } document.getElementById('mcLastHeard').addEventListener('change', e => { filters.lastHeard = e.target.value; loadNodes(); }); AreaFilter.init(document.getElementById('mapAreaFilter')); @@ -599,6 +717,16 @@ }); }); + // Scope config filter buttons (#2001) + document.querySelectorAll('#mcScopeFilter .btn').forEach(btn => { + btn.addEventListener('click', () => { + filters.scopeState = btn.dataset.scope; + localStorage.setItem('meshcore-map-scope-filter', filters.scopeState); + document.querySelectorAll('#mcScopeFilter .btn').forEach(b => b.classList.toggle('active', b.dataset.scope === filters.scopeState)); + renderMarkers(); + }); + }); + // Byte size filter buttons document.querySelectorAll('#mcByteFilter .btn').forEach(btn => { btn.addEventListener('click', () => { @@ -1692,6 +1820,18 @@ return false; } } + // Scope config filter (#2001). Applied to every role, not just repeaters: + // picking a scope state is a question about repeater configuration, and + // the answer should not leave companions and sensors sitting on the map + // as if they were part of it. Unclassified nodes therefore drop out too — + // see scopeFilterAccepts. + // + // This deliberately differs from the byte-size filter three lines up, + // which gates itself on role === 'repeater' and leaves everything else + // visible. Asked and decided (#2006 review): this filter leads. The byte + // filter is not changed here — that is its own behaviour change, for its + // own PR. + if (!scopeFilterAccepts(n, filters.scopeState)) return false; // Status filter if (filters.statusFilter !== 'all') { const status = getNodeStatus(n); // #1598: relay-aware for infra @@ -1730,15 +1870,36 @@ // set kept in sync with --mc-mb-* CSS stripes so label + marker agree. mbColor = MB_MARKER_TINT[mbStatus] || MB_MARKER_TINT.unknown; } - const icon = useLabel ? makeRepeaterLabelIcon(node, isStale, isAlsoObserver, mbStatus) : makeMarkerIcon(node.role || 'companion', isStale, isAlsoObserver, mbColor); + // #2001: scope-config tint. One marker carries one colour, so when both + // overlays are on the older multi-byte tint keeps the marker and the + // scope overlay stands down rather than the two fighting over the fill. + var scopeState = (scopeTintEnabled(filters.scopeOverlay, filters.scopeState) && !mbColor) ? (node.scope_config_state || null) : null; + var scopeColor = scopeState ? scopeTint(node) : null; + const icon = useLabel ? makeRepeaterLabelIcon(node, isStale, isAlsoObserver, mbStatus, scopeState) : makeMarkerIcon(node.role || 'companion', isStale, isAlsoObserver, mbColor || scopeColor); const latLng = L.latLng(node.lat, node.lon); - allMarkers.push({ latLng, node, icon, isLabel: useLabel, popupFn: function() { return buildPopup(node); }, alt: (node.name || 'Unknown') + ' (' + (node.role || 'node') + (isAlsoObserver ? ' + observer' : '') + ')' }); + // The scope state has to leave the fill and reach the DOM, or a dot marker + // is colour and nothing else — which on the achromat preset is six greys + // inside 1.7:1. It goes on `title`, NOT `alt`: Leaflet sets options.alt + // only when the icon element is an (leaflet-src.js, L.Marker + // _initIcon), and every icon on this map is a divIcon, so an alt string + // never lands. options.title is applied to whatever element the icon + // produced. The popup carries it too, for the pointerless case. + var scopeLabel = scopeState ? scopeStateLabel(scopeState) : ''; + var markerAlt = (node.name || 'Unknown') + ' (' + (node.role || 'node') + (isAlsoObserver ? ' + observer' : '') + ')' + + (scopeLabel ? ', scope config ' + scopeLabel : ''); + allMarkers.push({ latLng, node, icon, isLabel: useLabel, popupFn: function() { return buildPopup(node); }, alt: markerAlt, title: scopeLabel ? markerAlt : '' }); } // Add observer markers (skip observers already represented as a node marker) // Build set of node pubkeys that are displayed on the map const displayedNodePubkeys = new Set(filtered.map(n => (n.public_key || '').toLowerCase())); - if (filters.observer) { + // #2001 — while a specific scope state is picked, the observer layer stands + // down. Two reasons: the filter leads (see the note in the predicate above), + // so leaving unrelated pins on the map contradicts the answer; and a + // repeater that is also an observer would otherwise drop out of `filtered`, + // stop matching displayedNodePubkeys, and reappear as a plain observer pin — + // the node the operator just filtered away, back in another guise. + if (filters.observer && filters.scopeState === 'all') { for (const obs of observers) { if (!obs.lat || !obs.lon) continue; // Skip observers whose pubkey matches a displayed node — they're shown as combined markers @@ -1767,7 +1928,7 @@ var clusterMarkers = []; for (const m of allMarkers) { const pos = (useCluster ? m.latLng : (m.adjustedLatLng || m.latLng)); - const marker = L.marker(pos, { icon: m.icon, alt: m.alt }); + const marker = L.marker(pos, { icon: m.icon, alt: m.alt, title: m.title || '' }); marker._nodeKey = m.node.public_key || m.node.id || null; marker._role = (m.node && m.node.role) || 'companion'; marker.bindPopup(m.popupFn(), { maxWidth: 280 }); @@ -1882,6 +2043,20 @@ : `Unknown`; const hashPrefixRow = `
Hash Prefix
${hashPrefixValue}
`; + // #2001 — scope config, whenever the server classified this node. Stated + // here as well as in the marker title because a popup is the one surface + // every path reaches: dot markers, hash labels and clustered markers alike. + var scopeRow = ''; + var scopeMeta = null; + for (var sgi = 0; sgi < SCOPE_STATES.length; sgi++) { + if (SCOPE_STATES[sgi].key === node.scope_config_state) { scopeMeta = SCOPE_STATES[sgi]; break; } + } + if (scopeMeta) { + scopeRow = '
Scope
' + + '
' + + '' + + safeEsc(scopeMeta.label) + '
'; + } // Multi-byte support indicator for repeaters var mbRow = ''; if (node.role === 'repeater' && node.multi_byte_status) { @@ -1898,6 +2073,7 @@
${hashPrefixRow} ${mbRow} + ${scopeRow}
Key
${safeEsc(key)}
Location
@@ -2418,6 +2594,13 @@ window.__meshcoreMapInternals = { createClusterGroup: createClusterGroup, makeClusterIcon: makeClusterIcon, + // #2001: the three pure pieces of the scope-state layer, so the filter + // rule and the tint can be asserted without a browser. + scopeFilterAccepts: scopeFilterAccepts, + scopeTint: scopeTint, + scopeTintEnabled: scopeTintEnabled, + scopeStateLabel: scopeStateLabel, + scopeFilterHtml: scopeFilterHtml, // #1356: exposed so the a11y test can assert what the label RENDERS // instead of grepping map.js for where two identifiers sit. makeRepeaterLabelIcon: makeRepeaterLabelIcon, diff --git a/public/style.css b/public/style.css index bf71fd88b..863e9a95f 100644 --- a/public/style.css +++ b/public/style.css @@ -4541,6 +4541,42 @@ th.sort-active { color: var(--accent, #60a5fa); } --mc-mb-confirmed: #56F0A0; /* audit override of Tol vibrant for fill contrast */ --mc-mb-suspected: #FFD966; --mc-mb-unknown: #FF8888; + + /* #2001 — the scope-state colours are declared on `body`, not here: a custom + * property that references --mc-mb-* has to be declared on the same element + * the presets override those on, or it resolves against :root's value and the + * preset never reaches it. See the body rule below. */ +} + +/* #2001 — scope-state colours, declared on `body` rather than `:root`. + * + * Four of the six derive from the multi-byte accents rather than adding a + * sixth hand-tuned palette, so a colorblind preset retints them with the rest + * of the map. That only works from here: the presets override --mc-mb-* on + * `body[data-cb-preset="..."]`, and a custom property is substituted using the + * value computed on the element it is declared on. Declared on :root these + * resolved against :root's --mc-mb-*, so the CSS half of the preset system + * never reached them — measured with data-cb-preset="deut" set and no inline + * override: --mc-mb-confirmed became #648FFF at body level while + * --mc-scope-full stayed #56F0A0. Declared here, both preset paths work: the + * CSS blocks because they land on the same element, and cb-presets.js's inline + * write on documentElement because body inherits it. + * + * --mc-scope-observed means "carries scoped traffic, but never told us its + * config", a weaker claim than 'full', so it reads as a lighter version of the + * same colour rather than a different one. --mc-scope-no-flood does NOT derive: + * every role colour is operator-customisable and the default --mc-role-room is + * #009E73, a green, which put the rarest state in the same hue as 'full' on a + * live instance. --mc-scope-none is a fixed neutral: absence of information is + * not a state on the good/bad axis and must not compete for attention with the + * states that are. */ +body { + --mc-scope-full: var(--mc-mb-confirmed); + --mc-scope-observed: color-mix(in srgb, var(--mc-mb-confirmed) 55%, #ffffff); + --mc-scope-no-unscoped: var(--mc-mb-suspected); + --mc-scope-no-scopes: var(--mc-mb-unknown); + --mc-scope-no-flood: #B197FC; + --mc-scope-none: #9aa3ad; } /* ───────────────────────────────────────────────────────────────────────── @@ -4633,6 +4669,26 @@ body[data-cb-preset="achromat"] { --mc-mb-confirmed: #b3b3b3; --mc-mb-suspected: #808080; --mc-mb-unknown: #595959; + /* #2001 — the six scope states, spread across this preset's own luminance + ramp. Four of them derive from the --mc-mb-* values above and would come + out at 70/50/35%, but the two literals (no-flood, none) do not derive and + landed at L 0.385 and L 0.361 next to full's 0.451: 1.15:1 and 1.06:1, + which on a monochrome preset is one colour. Ordered by how closed the + state is, so the ramp itself carries meaning: observed lightest, no-flood + darkest, no-data between observed and full. + + This does not make the six distinguishable. Six levels cannot be spread + across a greyscale at 3:1 steps — the worst pair here is observed vs + no-data at 1.25:1, up from 1.06:1 before, and half the pairs sit under + 1.8:1. The ramp is an aid, not the answer. What carries the state on this + preset is the marker title and the Scope row in the popup (map.js), which + is why those are not optional. */ + --mc-scope-observed: #e6e6e6; + --mc-scope-full: #b3b3b3; + --mc-scope-no-unscoped: #808080; + --mc-scope-no-scopes: #595959; + --mc-scope-no-flood: #333333; + --mc-scope-none: #cfcfcf; } .mc-cluster-wrap { background: transparent !important; border: 0 !important; } @@ -4714,6 +4770,27 @@ body[data-cb-preset="achromat"] { .mc-mb-label.status-confirmed { border-left-color: var(--mc-mb-confirmed); } .mc-mb-label.status-suspected { border-left-color: var(--mc-mb-suspected); } .mc-mb-label.status-unknown { border-left-color: var(--mc-mb-unknown); } +/* #2001 — scope-config overlay. Only ever set when the multi-byte overlay is + * off (map.js stands the scope tint down when both are on), so these can share + * the same stripe without a specificity fight. */ +.mc-mb-label.scope-full { border-left-color: var(--mc-scope-full); } +.mc-mb-label.scope-observed { border-left-color: var(--mc-scope-observed); } +.mc-mb-label.scope-no-unscoped { border-left-color: var(--mc-scope-no-unscoped); } +.mc-mb-label.scope-no-scopes { border-left-color: var(--mc-scope-no-scopes); } +.mc-mb-label.scope-no-flood { border-left-color: var(--mc-scope-no-flood); } +.mc-mb-label.scope-none { border-left-color: var(--mc-scope-none); } +/* #2001 — swatch inside a filter button: small enough to sit on the text + * baseline without stretching the button, round so it does not read as a + * second nested button. */ +.scope-swatch { width: 9px; height: 9px; border-radius: 50%; border: none; margin-right: 5px; flex: 0 0 auto; } +/* Seven buttons do not fit the map-controls panel on one line (it is + * clamp(160px, 18vw, 240px) wide), and the flex default of nowrap pushed the + * last ones past the panel edge with "No unscoped" broken mid-label. Wrap the + * group into rows and keep each label whole. Scoped to this group so the + * three-button Status and five-button Byte Size groups keep the layout they + * already have. */ +#mcScopeFilter { flex-wrap: wrap; } +#mcScopeFilter .btn { white-space: nowrap; } /* The width of the prefix itself is a guess when the node has no advert-derived hash_size. Dotted underline marks the digits as unverified — a shape cue, not a color one, so it survives forced-colors and color-vision differences. The diff --git a/test-all.sh b/test-all.sh index e8d692c02..b782da68a 100755 --- a/test-all.sh +++ b/test-all.sh @@ -14,6 +14,7 @@ node test-packet-filter-ux.js node test-aging.js node test-issue-1065-gesture-hints-gates.js node test-frontend-helpers.js +node test-issue-2001-map-scope-state.js node test-fetch-all-nodes-pagination.js node test-my-repeaters-dashboard.js node test-repeater-metric-scatter.js diff --git a/test-issue-2001-map-scope-state.js b/test-issue-2001-map-scope-state.js new file mode 100644 index 000000000..b8471ea93 --- /dev/null +++ b/test-issue-2001-map-scope-state.js @@ -0,0 +1,199 @@ +/* Unit tests for the map's scope-configuration state layer (issue #2001) + * + * Verifies the three pure pieces the layer is built from: + * - scopeFilterAccepts: which nodes survive a given filter selection + * - scopeTint: which CSS variable a node's marker is filled with + * - scopeFilterHtml: the button group, including which button reads active + * + * Runs in the same jsdom-free vm sandbox as test-map-clustering.js, with a + * tiny Leaflet shim, so it exercises map.js and not the library. + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } +} + +function makeSandbox() { + const L = {}; + L.point = (x, y) => ({ x, y }); + L.latLng = (a, b) => ({ lat: a, lng: b }); + L.divIcon = (opts) => ({ _isDivIcon: true, options: opts, html: opts.html, className: opts.className }); + L.layerGroup = () => ({ addLayer(){ return this; }, removeLayer(){ return this; }, clearLayers(){ return this; }, eachLayer(){}, addTo(){ return this; }, hasLayer(){ return false; } }); + L.marker = (latlng, opts) => ({ _isMarker: true, _latlng: latlng, options: opts || {}, getLatLng(){ return this._latlng; }, bindPopup(){ return this; }, bindTooltip(){ return this; } }); + function MarkerClusterGroup(opts) { this.options = opts || {}; this._layers = []; } + MarkerClusterGroup.prototype.addLayer = function () { return this; }; + MarkerClusterGroup.prototype.addLayers = function () { return this; }; + MarkerClusterGroup.prototype.clearLayers = function () { return this; }; + MarkerClusterGroup.prototype.eachLayer = function () {}; + MarkerClusterGroup.prototype.addTo = function () { return this; }; + L.MarkerClusterGroup = MarkerClusterGroup; + L.markerClusterGroup = (opts) => new MarkerClusterGroup(opts); + + const ctx = { + window: {}, + document: { addEventListener(){}, getElementById(){ return null; }, querySelector(){ return null; }, querySelectorAll(){ return []; }, createElement(){ return { id:'', textContent:'', innerHTML:'', appendChild(){}, addEventListener(){}, setAttribute(){}, classList:{add(){},remove(){},toggle(){}} }; }, head: { appendChild(){} }, body: { appendChild(){} } }, + console, Date, Math, Array, Object, String, Number, JSON, RegExp, Error, + parseInt, parseFloat, isFinite, isNaN, Map, Set, Promise, + setTimeout: ()=>{}, clearTimeout: ()=>{}, setInterval: ()=>{}, clearInterval: ()=>{}, + registerPage: () => {}, esc: (s) => s, onWS: () => {}, offWS: () => {}, + localStorage: (() => { const s={}; return { getItem:k=>s[k]||null, setItem:(k,v)=>{s[k]=String(v);}, removeItem:k=>{delete s[k];} }; })(), + fetch: () => Promise.resolve({ json: () => Promise.resolve({}) }), + addEventListener(){}, dispatchEvent(){}, + L: L, + }; + ctx.window.L = ctx.L; + vm.createContext(ctx); + vm.runInContext(fs.readFileSync('public/roles.js', 'utf8'), ctx); + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; + vm.runInContext(fs.readFileSync('public/map.js', 'utf8'), ctx); + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; + return ctx; +} + +console.log('\n=== map.js: scope-configuration state (#2001) ==='); +{ + const internals = makeSandbox().window.__meshcoreMapInternals; + + test('exposes the scope-state hooks', () => { + assert.ok(internals, '__meshcoreMapInternals not exposed'); + assert.strictEqual(typeof internals.scopeFilterAccepts, 'function', 'scopeFilterAccepts not exported'); + assert.strictEqual(typeof internals.scopeTint, 'function', 'scopeTint not exported'); + assert.strictEqual(typeof internals.scopeFilterHtml, 'function', 'scopeFilterHtml not exported'); + }); + + test('"all" accepts every node, classified or not', () => { + assert.strictEqual(internals.scopeFilterAccepts({ scope_config_state: 'full' }, 'all'), true); + assert.strictEqual(internals.scopeFilterAccepts({ role: 'companion' }, 'all'), true); + }); + + test('a selected state accepts only nodes in that state', () => { + const node = { scope_config_state: 'no-scopes' }; + assert.strictEqual(internals.scopeFilterAccepts(node, 'no-scopes'), true); + assert.strictEqual(internals.scopeFilterAccepts(node, 'full'), false); + assert.strictEqual(internals.scopeFilterAccepts(node, 'none'), false); + }); + + // A node with no scope_config_state is unclassified, not "none". The server + // omits the field for roles that do not forward, and for every node when the + // declared-regions lookup failed. Treating that as "no scope data" would put + // companions and phones in the category operators use to hunt for repeaters + // with no region config. + test('an unclassified node is not swept into "none"', () => { + assert.strictEqual(internals.scopeFilterAccepts({ role: 'companion' }, 'none'), false); + assert.strictEqual(internals.scopeFilterAccepts({ role: 'repeater' }, 'full'), false); + }); + + test('each state tints through its own CSS variable', () => { + const want = { + 'full': 'var(--mc-scope-full)', + 'observed': 'var(--mc-scope-observed)', + 'no-unscoped': 'var(--mc-scope-no-unscoped)', + 'no-scopes': 'var(--mc-scope-no-scopes)', + 'no-flood': 'var(--mc-scope-no-flood)', + 'none': 'var(--mc-scope-none)', + }; + for (const state of Object.keys(want)) { + assert.strictEqual(internals.scopeTint({ scope_config_state: state }), want[state], `tint for ${state}`); + } + }); + + test('an unclassified node has no tint, so its role colour stands', () => { + assert.strictEqual(internals.scopeTint({ role: 'companion' }), null); + assert.strictEqual(internals.scopeTint({}), null); + }); + + test('the filter group offers All plus every state', () => { + const html = internals.scopeFilterHtml('all'); + for (const value of ['all', 'full', 'observed', 'no-unscoped', 'no-scopes', 'no-flood', 'none']) { + assert.ok(html.indexOf('data-scope="' + value + '"') >= 0, `missing button for ${value}: ${html}`); + } + }); + + // The map has no separate colour legend, so the filter group is the legend: + // every state button carries the swatch its markers are painted with. A + // colour on the map with nothing to read it against is a puzzle, not a + // signal. + test('every state button carries its own colour swatch', () => { + const html = internals.scopeFilterHtml('all'); + for (const state of ['full', 'observed', 'no-unscoped', 'no-scopes', 'no-flood', 'none']) { + assert.ok(html.indexOf('var(--mc-scope-' + state + ')') >= 0, `no swatch colour for ${state}: ${html}`); + } + }); + + // Picking a state in the filter and getting the map's ordinary role colours + // back is the confusing case: the operator asked about scope config, so the + // markers answer in scope colours without a second control having to be + // found first. The overlay checkbox stays, for colouring the whole map while + // the filter is on All. + test('the tint follows the filter, not only the overlay checkbox', () => { + assert.strictEqual(internals.scopeTintEnabled(false, 'all'), false); + assert.strictEqual(internals.scopeTintEnabled(true, 'all'), true); + assert.strictEqual(internals.scopeTintEnabled(false, 'none'), true); + assert.strictEqual(internals.scopeTintEnabled(false, 'full'), true); + }); + + test('the current selection is the active button, and the only one', () => { + const html = internals.scopeFilterHtml('none'); + const active = html.match(/class="btn active" data-scope="([^"]+)"/g) || []; + assert.strictEqual(active.length, 1, `expected exactly one active button, got ${active.length}: ${html}`); + assert.ok(active[0].indexOf('data-scope="none"') >= 0, `active button is not "none": ${active[0]}`); + }); +} + +// --- the label render path ------------------------------------------------- +// makeRepeaterLabelIcon is where the tint actually lands for a repeater with +// hash labels on, which is the default. The helpers above say what the colour +// should be; these say the label carries it, in a form that is not colour alone. +{ + const internals = makeSandbox().window.__meshcoreMapInternals; + const NODE = { public_key: '3e7a1b9c'.repeat(8), hash_size: 2 }; + + test('the scope state reaches the label as a class', () => { + const html = internals.makeRepeaterLabelIcon(NODE, false, false, null, 'no-flood').html; + assert.ok(/class="[^"]*scope-no-flood/.test(html), 'no scope-no-flood class in: ' + html); + }); + + test('the state is in the aria-label and the hover title, not only the colour', () => { + const html = internals.makeRepeaterLabelIcon(NODE, false, false, null, 'none').html; + assert.ok(/aria-label="[^"]*scope config No data/.test(html), 'state missing from aria-label: ' + html); + assert.ok(/title="No data/.test(html), 'state missing from the hover title: ' + html); + }); + + // Both controls are off by default, so a default map must render exactly what + // it rendered before this feature existed. + test('with no scope state the label is byte-identical to the four-argument call', () => { + const withArg = internals.makeRepeaterLabelIcon(NODE, false, false, 'confirmed', null).html; + const without = internals.makeRepeaterLabelIcon(NODE, false, false, 'confirmed').html; + assert.strictEqual(withArg, without); + assert.ok(!/scope-/.test(without), 'default label carries a scope class: ' + without); + assert.ok(!/title=/.test(without), 'default label carries a title: ' + without); + }); + + test('an unknown state string adds nothing to the label', () => { + const html = internals.makeRepeaterLabelIcon(NODE, false, false, null, 'not-a-state').html; + assert.ok(!/scope-/.test(html), 'unknown state produced a class: ' + html); + }); + + // One marker carries one colour: with the multi-byte overlay on the scope + // tint stands down rather than the two fighting over the same surface. + test('a dot marker carries the state in words, not only in its fill', () => { + assert.strictEqual(internals.scopeStateLabel('no-flood'), 'No flood'); + assert.strictEqual(internals.scopeStateLabel('none'), 'No data'); + assert.strictEqual(internals.scopeStateLabel('not-a-state'), ''); + }); + + test('the multi-byte tint wins when both overlays are on', () => { + const html = internals.makeRepeaterLabelIcon(NODE, false, false, 'confirmed', null).html; + assert.ok(/status-confirmed/.test(html), 'multi-byte class missing: ' + html); + assert.ok(!/scope-/.test(html), 'scope class present alongside the multi-byte tint: ' + html); + }); +} + +console.log(`\n${passed} passed, ${failed} failed`); +process.exit(failed === 0 ? 0 : 1); From 51a2a7dd2f7cfebcfa4e297feb14389764fef84f Mon Sep 17 00:00:00 2001 From: efiten Date: Fri, 11 Sep 2026 15:35:52 +0200 Subject: [PATCH 025/151] fix(scope-audit): ship the stylesheet the page paints its badges with (#2005) Fixes #2004. Two review rounds; findings and evidence on the PR. Verified by injecting the stylesheet into a running deployment and reading computed styles before and after: the badges gain background, size, uppercase and padding; at 430px the Config column stays visible; the sorted column keeps its accent. The new test fails on the eight unstyled classes against the pre-fix tree, on .ns-truncated against the first fix, and on a re-added column-hiding rule. --- .github/workflows/deploy.yml | 1 + public/index.html | 1 + public/node-scopes.css | 69 +++++++++++ public/scope-audit.css | 12 +- test-all.sh | 1 + test-scope-audit-styles-linked.js | 198 ++++++++++++++++++++++++++++++ 6 files changed, 278 insertions(+), 4 deletions(-) create mode 100644 public/node-scopes.css create mode 100644 test-scope-audit-styles-linked.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index de5545517..e813b7dfa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -249,6 +249,7 @@ jobs: node test-issue-1360-pill-letter-count.js node test-issue-1364-pill-no-clamp.js node test-issue-1375-scope-stats-fetch.js + node test-scope-audit-styles-linked.js node test-issue-1361-cb-presets.js node test-issue-1380-cb-sim-overlay.js node test-issue-1380-cb-reset-button.js diff --git a/public/index.html b/public/index.html index aeeaa5f64..2a5f05d95 100644 --- a/public/index.html +++ b/public/index.html @@ -43,6 +43,7 @@ + diff --git a/public/node-scopes.css b/public/node-scopes.css new file mode 100644 index 000000000..4af607d24 --- /dev/null +++ b/public/node-scopes.css @@ -0,0 +1,69 @@ +/* Node detail — Scopes section. Reuses .analytics-stats / .analytics-stat-card / + .analytics-time-range from the analytics page (same tokens node-reach.css + reuses); these are the scopes-specific bits only. No hex/rgb literals — + --text-muted / --border / --status-* are defined globally. --section-bg is + NOT: it exists only in the dark-theme blocks (style.css), never in light + :root, so every use below MUST keep the var(--section-bg, var(--card-bg)) + fallback — the fallback is load-bearing, not decorative. */ + +.ns-head { display: flex; flex-wrap: wrap; align-items: baseline; justify-content: space-between; gap: 8px; margin-bottom: 8px; } +.ns-head h4 { margin: 0; } +.ns-window { margin: 0; } + +.ns-declared-meta { font-size: 11px; color: var(--text-muted); margin: 0 0 10px; } +/* Flattened from `.ns-declared-meta .ns-truncated` when this file was ported: + the Scope Audit renders the marker in its Declared-age cell, with no + .ns-declared-meta ancestor, so the scoped form styled nothing there. The + standalone rule covers both callers. */ +.ns-truncated { color: var(--status-amber-text); font-weight: 600; } +.ns-declared-meta .ns-never-asked { color: var(--text-muted); font-style: italic; } + +.ns-empty { + color: var(--text-muted); font-size: 13px; padding: 14px 12px; + border: 1px dashed var(--border); border-radius: 6px; margin: 6px 0 4px; +} +.ns-empty.ns-empty-config { color: var(--status-amber-text); border-color: var(--status-amber-text); } + +.ns-table { border-collapse: collapse; width: 100%; font-size: 12px; margin: 4px 0 10px; } +.ns-table th, .ns-table td { border: 0; border-bottom: 1px solid var(--border); padding: 5px 8px; } +.ns-table thead th { border-bottom: 2px solid var(--border); background: none; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: .3px; color: var(--text-muted); text-align: left; } +/* The rule above is (0,1,2) and outranks the global th.sort-active at (0,1,1) + whatever the source order, so without this the sorted column label and its + caret (fill: currentColor) both render muted. Measured on a deployment: + rgb(74,158,255) before this file loads, rgb(91,99,112) after. Table-scoped, + the same way .data-table and .analytics-table already solve it. */ +.ns-table thead th.sort-active { color: var(--link-color); } +.ns-n { text-align: right; font-variant-numeric: tabular-nums; } +.ns-scope-name { font-family: var(--mono); font-weight: 600; } + +.ns-decl { + display: inline-block; padding: 2px 7px; border-radius: var(--badge-radius, 4px); + font-size: 10px; font-weight: 700; text-transform: uppercase; letter-spacing: .3px; + white-space: nowrap; +} +.ns-decl-yes { background: color-mix(in srgb, var(--status-green) 16%, transparent); color: var(--status-green-text); } +.ns-decl-quiet { background: color-mix(in srgb, var(--status-yellow) 18%, transparent); color: var(--status-amber-text); } +.ns-decl-warn { background: color-mix(in srgb, var(--status-red) 16%, transparent); color: var(--danger); } +/* The border is load-bearing here and not on its siblings: --section-bg is + defined only in the dark blocks, so in the light theme this falls back to + --card-bg, which is white on a near-white page ground. Without the outline + the "no flood" badge is invisible — the same failure this file is fixing. + An inset shadow rather than a border so the chip keeps the exact box metrics + of .ns-decl-yes/-quiet/-warn beside it in the same column. */ +.ns-decl-unknown { background: var(--section-bg, var(--card-bg)); box-shadow: inset 0 0 0 1px var(--border); color: var(--text-muted); font-weight: 600; text-transform: none; } + +.ns-routes { font-size: 11px; color: var(--text-muted); margin: 0 0 14px; } +.ns-routes b { color: var(--text); font-variant-numeric: tabular-nums; } + +.ns-links { font-size: 12px; margin-top: 4px; } +.ns-links a { color: var(--link-color); } + +/* The fork this file comes from hides the third column of .ns-table at 480px, + where .ns-table is the per-node Scopes panel's four-column table and column + three is "Last seen". That rule is deliberately NOT ported: the only + .ns-table in this repository is the Scope Audit's, whose third column is + Config — the badge column this file exists to make visible. Measured at + 430px with the rule in place: the Config header and every badge under it + computed to display:none. The audit table already scrolls horizontally + (.sa-table-wrap + .sa-table min-width), so it needs no column hiding. When + the panel lands here, it should bring a rule scoped to its own table. */ diff --git a/public/scope-audit.css b/public/scope-audit.css index dfb12d7e3..15a281fad 100644 --- a/public/scope-audit.css +++ b/public/scope-audit.css @@ -36,7 +36,10 @@ } .sa-chip-declared { background: var(--section-bg, var(--card-bg)); color: var(--text); border: 1px solid var(--border); } .sa-chip-undeclared { background: color-mix(in srgb, var(--status-yellow) 18%, transparent); color: var(--status-amber-text); } -.sa-chip-wildcard { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); font-weight: 700; } +/* Same outline, same reason as .ns-decl-unknown: --section-bg is dark-theme + only, so in light this falls back to --card-bg, which is white on a + near-white page ground and leaves the wildcard chip unpainted. */ +.sa-chip-wildcard { background: var(--section-bg, var(--card-bg)); box-shadow: inset 0 0 0 1px var(--border); color: var(--text-muted); font-weight: 700; } .sa-chip-ambiguous { background: var(--section-bg, var(--card-bg)); color: var(--text-muted); border: 1px dashed var(--border); font-family: inherit; font-style: italic; } .sa-count { font-size: 11px; margin-top: 6px; } @@ -48,9 +51,10 @@ .sa-summary { display: flex; flex-wrap: wrap; gap: 10px 16px; font-size: 12px; color: var(--text-muted); margin: 2px 0 10px; } .sa-summary-item { display: inline-flex; align-items: center; gap: 5px; } -@media (max-width: 640px) { - .sa-table th:nth-child(6), .sa-table td:nth-child(6) { display: none; } -} +/* A 640px rule hiding the table's sixth column used to live here. The table + has five (scope-audit.js), so it hid nothing, and the wrapper already scrolls + horizontally. Removed rather than repointed: hiding a column of this table is + what made the Config badge disappear on phones when node-scopes.css landed. */ /* Observed forwarding, the green half of the merged Scopes column. Pairs with .sa-chip-unobserved below, which stays neutral. Only the observed side is diff --git a/test-all.sh b/test-all.sh index b782da68a..8804b3612 100755 --- a/test-all.sh +++ b/test-all.sh @@ -15,6 +15,7 @@ node test-aging.js node test-issue-1065-gesture-hints-gates.js node test-frontend-helpers.js node test-issue-2001-map-scope-state.js +node test-scope-audit-styles-linked.js node test-fetch-all-nodes-pagination.js node test-my-repeaters-dashboard.js node test-repeater-metric-scatter.js diff --git a/test-scope-audit-styles-linked.js b/test-scope-audit-styles-linked.js new file mode 100644 index 000000000..9841681f9 --- /dev/null +++ b/test-scope-audit-styles-linked.js @@ -0,0 +1,198 @@ +/* Every class the Scope Audit page paints with must have a rule that can + * actually apply to the element the page renders, in a stylesheet index.html + * links. + * + * The page shipped referencing .ns-decl / .ns-empty / .ns-table, whose rules + * live in node-scopes.css — a file that was never added, and never linked. The + * page still rendered: correct data, correct DOM, no console error, and every + * badge as unstyled text. Nothing in the suite noticed, because nothing + * compared the classes the page writes against the CSS the page loads. + * + * "Can actually apply" is the part a substring check gets wrong. `.ns-truncated` + * was mentioned in exactly one rule, `.ns-declared-meta .ns-truncated`, and this + * page renders that span with no such ancestor — styled on paper, unstyled on + * screen, which is the same bug one level down. + * + * Deliberately not a whitelist of known-good class names: that would need + * updating every time the page grows a class, which is exactly the moment this + * check has to fire. + */ +'use strict'; +const fs = require('fs'); +const assert = require('assert'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } +} + +// linkedStylesheets returns the local stylesheet paths index.html loads, in +// link order. Vendor CSS is included: it is loaded too, and excluding it would +// be a guess about where a rule is allowed to live. +function linkedStylesheets(html) { + const out = []; + const re = /]+rel=["']stylesheet["'][^>]*>/g; + let m; + while ((m = re.exec(html))) { + const href = /href=["']([^"']+)["']/.exec(m[0]); + if (!href) continue; + const path = href[1].split('?')[0]; + if (/^https?:/.test(path)) continue; // remote sheets are not ours to check + out.push('public/' + path); + } + return out; +} + +// classesUsedBy pulls class names out of the class="..." literals a page +// script writes, plus the ' class-name' fragments it concatenates. Only the +// page's own namespaces are checked; shared vocabulary (btn, active, +// text-muted) belongs to style.css and is not this page's to own. +function classesUsedBy(js, prefixes) { + const found = new Set(); + const re = /class="([^"]*)"|'\s*([a-z][a-z0-9-]*)'/g; + let m; + while ((m = re.exec(js))) { + const chunk = m[1] || m[2] || ''; + for (const raw of chunk.split(/[\s'"+]+/)) { + const name = raw.trim(); + if (!name) continue; + if (prefixes.some(p => name.startsWith(p))) found.add(name); + } + } + return [...found].sort(); +} + +function classRe(className) { + return new RegExp('\\.' + className.replace(/-/g, '\\-') + '(?![a-zA-Z0-9_-])'); +} + +// selectorsFor returns every selector that targets className, comments +// stripped: a class named only inside a comment has no rule. +function selectorsFor(css, className) { + // At-rule preludes are dropped rather than skipped, so the rules INSIDE a + // @media or @supports block are parsed like any other. Skipping them left + // the checker blind to the rule that hid this page's Config column below + // 480px: re-adding that rule kept the suite green. + const clean = css + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/@(?:media|supports|layer|container)[^{]*\{/g, ''); + const re = classRe(className); + const out = []; + for (const chunk of clean.split('}')) { + const brace = chunk.indexOf('{'); + if (brace === -1) continue; + for (const sel of chunk.slice(0, brace).split(',')) { + const t = sel.trim(); + if (t && !t.startsWith('@') && re.test(t)) out.push(t); + } + } + return out; +} + +// requiredAncestors lists the classes a selector needs on an ANCESTOR before +// its own compound can match. ".a .b" requires a; ".a.b" requires nothing, +// since both sit on one element. +function requiredAncestors(selector, className) { + const parts = selector.split(/\s*[>+~]\s*|\s+/).filter(Boolean); + const own = parts.findIndex(p => classRe(className).test(p)); + if (own <= 0) return []; + const need = []; + for (const p of parts.slice(0, own)) { + for (const m of p.match(/\.[a-zA-Z][a-zA-Z0-9_-]*/g) || []) need.push(m.slice(1)); + } + return need; +} + +// A class passes when some rule targets it with no ancestor requirement, or +// when every ancestor that rule requires is a class this page also emits. +function ruleApplies(css, className, emitted) { + const sels = selectorsFor(css, className); + if (!sels.length) return false; + return sels.some(sel => requiredAncestors(sel, className).every(a => emitted.has(a))); +} + +console.log('\n=== scope-audit: every class it paints with is in a linked stylesheet ==='); + +const html = fs.readFileSync('public/index.html', 'utf8'); +const sheets = linkedStylesheets(html); +const css = sheets.map(p => (fs.existsSync(p) ? fs.readFileSync(p, 'utf8') : '')).join('\n'); +const js = fs.readFileSync('public/scope-audit.js', 'utf8'); + +// classAttrClasses reads only class="..." literals, so the ancestor set holds +// classes that are really rendered. An earlier version accepted every +// single-quoted lowercase token in the file, which let 'active', 'true', +// 'number' and 'function' in and made the ancestor gate accept anything. +function classAttrClasses(src) { + const out = new Set(); + const re = /class="([^"]*)"/g; + let m; + while ((m = re.exec(src))) { + for (const raw of m[1].split(/[\s'"+]+/)) { + const t = raw.trim(); + if (/^[a-z][a-z0-9-]*$/.test(t)) out.add(t); + } + } + return out; +} + +// Checked classes are this page's own namespaces. The filter is defensive: +// nothing in scope-audit.js builds a class name by concatenation today, but a +// token like 'ns-decl-' would report as an unstyled class and say nothing. +const used = classesUsedBy(js, ['ns-', 'sa-']).filter(c => /^[a-z][a-z0-9-]*[a-z0-9]$/.test(c)); + +// The ancestor set: classes this page renders in markup, plus the page's own +// namespace tokens it assembles in variables, plus the app shell around it. +// This answers "does the page render this class at all", not "is it an ancestor +// of that element" — a rule like `.active .ns-decl` would pass. That is the +// limit of a static check; the case it exists for is an ancestor the page never +// renders under any arrangement. +const emitted = new Set([...classAttrClasses(js), ...classAttrClasses(html), ...used]); + +test('index.html links the stylesheets it names', () => { + const missing = sheets.filter(p => !fs.existsSync(p)); + assert.deepStrictEqual(missing, [], 'linked but absent from the tree: ' + missing.join(', ')); +}); + +test('scope-audit.js uses a non-trivial number of its own classes', () => { + assert.ok(used.length >= 10, 'expected the page to use its own vocabulary, found: ' + used.join(', ')); +}); + +test('every ns-* / sa-* class it writes has a rule that can apply', () => { + const unstyled = used.filter(c => !ruleApplies(css, c, emitted)); + assert.deepStrictEqual(unstyled, [], 'classes with no applicable rule in any linked stylesheet: ' + unstyled.join(', ')); +}); + +// The regression that started the follow-up commit: a stylesheet hiding a +// column of the table this page renders. The page's own CSS gives the table a +// min-width and a horizontally scrolling wrapper, so hiding columns is never +// the intended answer here, and the column that gets hidden is whichever one +// the fork's panel happened to put third. +test('no linked stylesheet hides a column of the audit table', () => { + const clean = css.replace(/\/\*[\s\S]*?\*\//g, '').replace(/@(?:media|supports|layer|container)[^{]*\{/g, ''); + const offenders = []; + for (const chunk of clean.split('}')) { + const brace = chunk.indexOf('{'); + if (brace === -1) continue; + const head = chunk.slice(0, brace); + const body = chunk.slice(brace + 1); + if (!/\.(ns|sa)-table/.test(head)) continue; + if (!/nth-child|nth-of-type/.test(head)) continue; + if (!/display\s*:\s*none/.test(body)) continue; + offenders.push(head.trim().replace(/\s+/g, ' ')); + } + assert.deepStrictEqual(offenders, [], 'column-hiding rules on the audit table: ' + offenders.join(' | ')); +}); + +// The checker itself, so a broken checker cannot pass the tree. +test('the checker rejects a rule gated on an ancestor the page never renders', () => { + const emit = new Set(['sa-page', 'ns-truncated']); + assert.strictEqual(ruleApplies('.ns-truncated { color: red }', 'ns-truncated', emit), true); + assert.strictEqual(ruleApplies('.sa-page .ns-truncated { color: red }', 'ns-truncated', emit), true); + assert.strictEqual(ruleApplies('.never-rendered .ns-truncated { color: red }', 'ns-truncated', emit), false); + assert.strictEqual(ruleApplies('/* .ns-truncated lives elsewhere */', 'ns-truncated', emit), false); + assert.strictEqual(ruleApplies('.ns-truncated-other { color: red }', 'ns-truncated', emit), false); +}); + +console.log(`\n${passed} passed, ${failed} failed`); +process.exit(failed === 0 ? 0 : 1); From c283f42c1fd1e43fc3dbd78ae6ed4ef4178bd067 Mon Sep 17 00:00:00 2001 From: nullrouten0 Date: Sat, 12 Sep 2026 02:49:27 -0700 Subject: [PATCH 026/151] channels: 23 more names in the rainbow guess list (#2007) Verified: every value is SHA-256(name)[:16] per internal/channel.DeriveKey, 319/320 exact (Public is the fixed firmware default), no duplicate hashes, file parses at 320 entries. --- channel-rainbow.json | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/channel-rainbow.json b/channel-rainbow.json index 73ad3a237..23e1e7829 100644 --- a/channel-rainbow.json +++ b/channel-rainbow.json @@ -295,5 +295,28 @@ "#bogota": "6d0864985b64350ce4cbfebf4979e970", "#peru": "7e6fc347bf29a4c128ac3156865bd521", "#lima": "5f167ce354eca08ab742463df10ef255", - "Public": "8b3387e9c5cdea6ac9e5edbaa115cd72" + "Public": "8b3387e9c5cdea6ac9e5edbaa115cd72", + "#Public": "7534fe292203ccdce93db79377d4b566", + "#newbie": "ff2ecc59f59dbf051a10d880f0b4308d", + "#power": "c4d23a51329aeaee767234644d52cff8", + "#garden": "56ff6fe3500872c2765dbc178350005c", + "#jokes": "abe610d61dee508c0ac18793c36c41c3", + "#space": "5caa27568ba7a216bcc09fcb69c77ea1", + "#path": "29eed6c759d96346b30c869e23fd86f8", + "#hamradio": "83c8b01997654265938da8765cbc7db9", + "#amrron": "68ec397a1dd4f3416911d00211ed0ef6", + "#family": "7d1b805d4bf059c2ef06ed221794663b", + "#testing": "cde5e82cf515647dcb547a79a4f065d1", + "#wicked": "05e990d48943ee03c943c41dea0ddfe1", + "#gay": "3a612fa3bfc9f3b439cc230b01cbbc00", + "#t": "7b5336e85005be805b496bc27c722ff3", + "#doom": "7034910637d6b9b6c2cc3bcd8dfe9bb5", + "#empty": "3351cfa0dad99e1777cce210c983b561", + "#puppy": "c20cf85aab9561e6e65ac4a19a852198", + "#naked": "f423f23ddcf9e472c2ace38dce7591bc", + "#cruising": "550857053360e3492755251f9142e74c", + "#nightowls": "1411c5c3732d3df1b855644f78c47417", + "#worldcup": "d5786cc7bcee5a4828bbb20c3825c5a9", + "#nofilter": "cab31f589959debcd1721b7d252259b1", + "#politics": "64cc9fdaad67f03f23b4cec8c9a4468f" } From 6d7490e7da86b1aae418753316dfd46541e0924b Mon Sep 17 00:00:00 2001 From: nullrouten0 Date: Sat, 12 Sep 2026 03:38:58 -0700 Subject: [PATCH 027/151] fix(tables): stop the trailing action column cropping its own button (#2010) Reproduced on a live instance: 41px column for a 53px button at 1440px, 29px at 1024px, max-width:0 and overflow:hidden on the cell. Injecting the .col-action rule took it to 65px with nothing cropped. The new test fails 13 of 16 checks when only the CSS block is removed. --- .github/workflows/deploy.yml | 1 + public/analytics.js | 8 +- public/nodes.js | 4 +- public/style.css | 24 ++++ test-neighbor-map-btn-clip-e2e.js | 210 ++++++++++++++++++++++++++++++ 5 files changed, 241 insertions(+), 6 deletions(-) create mode 100644 test-neighbor-map-btn-clip-e2e.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index e813b7dfa..10ab45dfa 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -628,6 +628,7 @@ jobs: CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 node test-neighbor-map-btn-clip-e2e.js 2>&1 | tee -a e2e-output.txt # #1616: slide-over focus-restore flake-gate. Runs the slide-over # E2E 20 consecutive times against the SAME backend instance so diff --git a/public/analytics.js b/public/analytics.js index 538dde856..961c2738c 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -2917,7 +2917,7 @@ } // Top hops leaderboard - html += `

Top 20 Longest Hops

`; + html += `

Top 20 Longest Hops

#FromToDistance (${distUnitLabel})TypeObsBest SNRMedian SNRPacket
`; const top20 = data.topHops.slice(0, 20); top20.forEach((h, i) => { const fromLink = h.fromPk ? `${esc(h.fromName)}` : esc(h.fromName || '?'); @@ -2928,13 +2928,13 @@ const pktLink = h.hash ? `${esc(h.hash.slice(0, 12))}…` : '—'; const mapBtn = h.fromPk && h.toPk ? `` : ''; const tsTitle = h.timestamp ? `Best observation: ${h.timestamp}` : ''; - html += ``; + html += ``; }); html += `
#FromToDistance (${distUnitLabel})TypeObsBest SNRMedian SNRPacket
${i+1}${fromLink}${toLink}${formatDistance(h.dist)}${esc(h.type)}${obs}${bestSnr}${medianSnr}${pktLink}${mapBtn}
${i+1}${fromLink}${toLink}${formatDistance(h.dist)}${esc(h.type)}${obs}${bestSnr}${medianSnr}${pktLink}${mapBtn}
`; // Top paths if (data.topPaths.length) { - html += `

Top 10 Longest Multi-Hop Paths

`; + html += `

Top 10 Longest Multi-Hop Paths

#Total Distance (${distUnitLabel})HopsRoutePacket
`; data.topPaths.slice(0, 10).forEach((p, i) => { const route = p.hops.map(h => esc(h.fromName)).concat(esc(p.hops[p.hops.length-1].toName)).join(' → '); const pktLink = p.hash ? `${esc(p.hash.slice(0, 12))}…` : '—'; @@ -2943,7 +2943,7 @@ p.hops.forEach(h => { if (h.fromPk && !pathPks.includes(h.fromPk)) pathPks.push(h.fromPk); }); if (p.hops.length && p.hops[p.hops.length-1].toPk) { const last = p.hops[p.hops.length-1].toPk; if (!pathPks.includes(last)) pathPks.push(last); } const mapBtn = pathPks.length >= 2 ? `` : ''; - html += ``; + html += ``; }); html += `
#Total Distance (${distUnitLabel})HopsRoutePacket
${i+1}${formatDistance(p.totalDist)}${p.hopCount}${route}${pktLink}${mapBtn}
${i+1}${formatDistance(p.totalDist)}${p.hopCount}${route}${pktLink}${mapBtn}
`; } diff --git a/public/nodes.js b/public/nodes.js index a4d856c82..f68b658fd 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -384,7 +384,7 @@ '' + renderNodeTimestampHtml(nb.last_seen) + '' + '' + distanceCell + '' + '' + conf.icon + '' + - '' + showOnMap + '' + + '' + showOnMap + '' + ''; }).join(''); } @@ -398,7 +398,7 @@ 'Obs' + 'Last Seen' + 'Distance' + - 'Conf' + + 'Conf' + '' + '' + renderNeighborRows(neighbors, limit) + ''; } diff --git a/public/style.css b/public/style.css index 863e9a95f..d3e49aeca 100644 --- a/public/style.css +++ b/public/style.css @@ -1104,6 +1104,30 @@ img.brand-logo { padding: 3px 4px; text-align: center; } +/* Trailing action column (row buttons: neighbours "Map", analytics "View on + * map"). These columns carry an empty , so they contribute nothing to the + * width distribution, and the global `.data-table td { max-width: 0 }` above + * zeroes their max-content contribution too — the auto table algorithm then + * hands them whatever is left over, which is routinely narrower than the + * button they contain, and `overflow: hidden` silently clips it. Measured on + * the node Neighbours table: 40px of column for a 53px button at a 1440px + * viewport, 12px in the 420px split panel — clipped on every row at every + * desktop width below ~2000px. + * + * `width: 1%` is the shrink-to-content idiom for auto-layout tables: the + * column takes its min-content width and the data columns absorb the rest. + * `max-width: none` opts back out of the zeroing; `overflow: visible` means a + * future miscalculation shows as a visible overhang rather than a silent crop. + */ +.data-table th.col-action, +.data-table td.col-action { + width: 1%; + max-width: none; + overflow: visible; + white-space: nowrap; + text-align: right; +} + /* #1415 / #1458 / #1461 — mobile packets UX overhaul. * NOTE: hiding the first dt/dd pair below assumes Payload Type is the * first
in .detail-meta (per packets.js renderDetail). Keep that diff --git a/test-neighbor-map-btn-clip-e2e.js b/test-neighbor-map-btn-clip-e2e.js new file mode 100644 index 000000000..59d04cee4 --- /dev/null +++ b/test-neighbor-map-btn-clip-e2e.js @@ -0,0 +1,210 @@ +#!/usr/bin/env node +/* Trailing action-column clip — node Neighbours "Map" button, analytics + * "View on map" button. + * + * Symptom (operator report, 2026-08-30): on the node detail page the + * Neighbours table's "Map" button rendered as a cropped "◎ M" on every row. + * Unaffected by browser, browser version, zoom level or incognito — because + * it is a pure layout bug, not a rendering one. + * + * Root cause: the action column carries an empty , so it contributes + * nothing to the table's width distribution, and the global + * .data-table td { max-width: 0; overflow: hidden; } + * zeroes its max-content contribution as well. The auto table algorithm + * therefore hands that column only whatever space is left over, which is + * narrower than the button it holds, and `overflow: hidden` crops the + * remainder without any scrollbar to reveal it. Measured before the fix: + * + * viewport column button rows clipped + * 1024px 28px 53px all + * 1440px 40px 53px all + * 1920px 54px 53px all (padding pushes it over) + * 420px split panel: 12px column — 47 of 53px cropped + * + * Fix: `.data-table th.col-action, .data-table td.col-action` opts the + * column out of the zeroing (`max-width: none`), sizes it to its content + * (`width: 1%`) and stops it cropping (`overflow: visible`); nodes.js and + * analytics.js tag their action / with it. + * + * This test does not need a running server: the bug lives entirely in + * public/style.css plus the markup contract, so it renders the real + * stylesheet against the real emitted row markup via setContent(). + * + * Mutation guard: dropping the `.col-action` class from either the or + * the , or removing `max-width: none` / `width: 1%` from the rule, must + * make the geometry assertions below fail. + */ +'use strict'; + +const fs = require('node:fs'); +const path = require('node:path'); +const assert = require('node:assert'); +const { chromium } = require('playwright'); + +const CSS = fs.readFileSync(path.join(__dirname, 'public', 'style.css'), 'utf8'); +const NODES_JS = fs.readFileSync(path.join(__dirname, 'public', 'nodes.js'), 'utf8'); +const ANALYTICS_JS = fs.readFileSync(path.join(__dirname, 'public', 'analytics.js'), 'utf8'); + +const HEIGHT = 900; +// The bug's window: below ~640px the table finally overflows and the +// container scrolls (so nothing is cropped); above ~2000px the leftover +// space happens to exceed the button. Everything between was broken. +const WIDTHS = [1024, 1280, 1366, 1440, 1600, 1920]; + +// A neighbour row exactly as renderNeighborRows() emits it (nodes.js). +function neighborRow(name, role, score, count, seen, dist) { + return '' + + `${name}` + + `${role}` + + `${score}` + + `${count}` + + `${seen}` + + `${dist}` + + '●' + + ' ' + + ''; +} + +const ROWS = [ + ['TREX-SBC Room Observer', 'room', '1.00', '12028', '4m ago', '5.5 mi'], + ['SFO-SF-POTRO-RE-8C09', 'repeater', '1.00', '5965', '34m ago', '1.1 mi'], + ['alameda_shoreline_repea', 'repeater', '0.99', '140', '2h ago', '16.0 mi'], + ['Dublin Ridge', 'repeater', '0.97', '138', '2h ago', '17.7 mi'], + ['SLHills Repeater', 'repeater', '0.85', '98', '2h ago', '0.8 mi'], + ['TH-00 San Carlos', 'repeater', '0.80', '86', '3h ago', '9.2 mi'], +].map((r) => neighborRow(...r)).join(''); + +const TABLE = + '' + + '' + + '' + + '' + + '' + ROWS + '
NeighborRoleScoreObsLast SeenDistanceConf
'; + +// Both real hosts: the full-screen node view and the ~420px split panel. +const SHELLS = { + 'node-fullscreen': `
+
+

Neighbors (${ROWS.length})

${TABLE}
+
`, + 'split-panel': `
+
+
+

Neighbors

${TABLE}
+
`, +}; + +function page(shell) { + return ` +${shell}`; +} + +let passes = 0; +let failures = 0; + +function check(fn) { + try { fn(); passes++; } catch (err) { console.error(`FAIL ${err.message}`); failures++; } +} + +// ---- Static guards: the markup contract the CSS rule depends on ---------- + +check(() => { + assert.ok( + /\.data-table th\.col-action,\s*\n\s*\.data-table td\.col-action\s*\{[^}]*max-width:\s*none[^}]*\}/.test(CSS), + '.col-action rule in public/style.css must set `max-width: none` (opts out of the global `.data-table td { max-width: 0 }` that starves the column)', + ); + console.log('PASS style.css: .col-action sets max-width: none'); +}); + +check(() => { + const rule = CSS.match(/\.data-table th\.col-action,\s*\n\s*\.data-table td\.col-action\s*\{([^}]*)\}/); + assert.ok(rule, '.col-action rule not found in public/style.css'); + assert.ok(/width:\s*1%/.test(rule[1]), '.col-action must set `width: 1%` (shrink-to-content for auto-layout tables)'); + assert.ok(/overflow:\s*visible/.test(rule[1]), '.col-action must set `overflow: visible` so a miscalculation overhangs visibly instead of cropping silently'); + console.log('PASS style.css: .col-action sets width: 1% and overflow: visible'); +}); + +check(() => { + assert.ok( + NODES_JS.includes('') && NODES_JS.includes("'' + showOnMap"), + 'public/nodes.js renderNeighborTable/renderNeighborRows must tag BOTH the action and with class="col-action"', + ); + console.log('PASS nodes.js: neighbour action and carry .col-action'); +}); + +check(() => { + const ths = (ANALYTICS_JS.match(//g) || []).length; + const tds = (ANALYTICS_JS.match(/\$\{mapBtn\}<\/td>/g) || []).length; + assert.strictEqual(ths, 2, `public/analytics.js must tag 2 action (Top Hops + Top Paths) with .col-action, found ${ths}`); + assert.strictEqual(tds, 2, `public/analytics.js must tag 2 action with .col-action, found ${tds}`); + console.log('PASS analytics.js: both map-button columns carry .col-action'); +}); + +// ---- Geometry: nothing is cropped at any desktop width ------------------- + +async function main() { + let browser; + try { + browser = await chromium.launch({ + headless: true, + executablePath: process.env.CHROMIUM_PATH || undefined, + args: ['--no-sandbox', '--disable-gpu', '--disable-dev-shm-usage'], + }); + } catch (err) { + if (process.env.CHROMIUM_REQUIRE === '1') { + console.error(`test-neighbor-map-btn-clip-e2e.js: FAIL — Chromium required but unavailable: ${err.message}`); + process.exit(1); + } + console.log(`test-neighbor-map-btn-clip-e2e.js: SKIP geometry (Chromium unavailable: ${err.message.split('\n')[0]})`); + console.log(`test-neighbor-map-btn-clip-e2e.js: ${passes} passed, ${failures} failed`); + process.exit(failures > 0 ? 1 : 0); + } + + const ctx = await browser.newContext(); + const p = await ctx.newPage(); + p.setDefaultTimeout(15000); + + for (const [shellName, shell] of Object.entries(SHELLS)) { + for (const w of WIDTHS) { + await p.setViewportSize({ width: w, height: HEIGHT }); + await p.setContent(page(shell), { waitUntil: 'load' }); + + const probe = await p.evaluate(() => { + const rows = Array.from(document.querySelectorAll('#host tbody tr')); + let worst = 0; + let clipped = 0; + let colWidth = 0; + let btnWidth = 0; + for (const tr of rows) { + const td = tr.querySelector('td.col-action'); + const btn = td && td.querySelector('.neighbor-show-map'); + if (!btn) continue; + colWidth = td.getBoundingClientRect().width; + btnWidth = btn.getBoundingClientRect().width; + const over = btn.getBoundingClientRect().right - td.getBoundingClientRect().right; + if (over > 0.5) { clipped++; if (over > worst) worst = over; } + } + return { rows: rows.length, clipped, worst, colWidth, btnWidth }; + }); + + const tag = `${shellName} vw=${w}`; + check(() => { + assert.strictEqual( + probe.clipped, 0, + `${tag}: ${probe.clipped}/${probe.rows} Map buttons cropped by their cell (worst ${probe.worst.toFixed(1)}px; column ${probe.colWidth.toFixed(1)}px vs button ${probe.btnWidth.toFixed(1)}px)`, + ); + console.log(`PASS ${tag}: column ${probe.colWidth.toFixed(1)}px holds ${probe.btnWidth.toFixed(1)}px button, 0/${probe.rows} cropped`); + }); + } + } + + await browser.close(); + console.log(`\ntest-neighbor-map-btn-clip-e2e.js: ${passes} passed, ${failures} failed`); + if (failures > 0) process.exit(1); +} + +main().catch((err) => { + console.error('test-neighbor-map-btn-clip-e2e.js: ERROR', err); + process.exit(1); +}); From a2f039d4c7b212038a060f3b10412fdd4920b7ba Mon Sep 17 00:00:00 2001 From: nullrouten0 Date: Sat, 12 Sep 2026 03:40:44 -0700 Subject: [PATCH 028/151] fix(packets): show browser-added channels in the channel filter (#2009) Verified against production: ingestor writes enc_ (db.go:2282), store.go:3536 filters on it, live values are uppercase two-digit hex with thousands of packets (enc_28: 2708), /api/channels omits them, and /api/packets?channel=enc_28 returns rows. New test passes at 14 and fails when the dedupe guard is disabled. --- .github/workflows/deploy.yml | 1 + public/packets.js | 118 ++++++++++++++++-- test-all.sh | 1 + test-packets-local-channels.js | 222 +++++++++++++++++++++++++++++++++ 4 files changed, 329 insertions(+), 13 deletions(-) create mode 100644 test-packets-local-channels.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 10ab45dfa..eb6b7db78 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -229,6 +229,7 @@ jobs: node test-confidence-indicator.js node test-1659-analytics-warmup.js node test-channels-merge-1498-unit.js + node test-packets-local-channels.js node test-issue-1518-home-url.js node test-channel-decrypt-insecure-context.js node test-live-region-filter.js diff --git a/public/packets.js b/public/packets.js index d209c38e7..aff30071c 100644 --- a/public/packets.js +++ b/public/packets.js @@ -1042,6 +1042,92 @@ else _docColMenuCloseHandler = handler; } + // --- Locally-added channels in the channel filter --------------------- + // Channels the operator adds in their own browser (Channels page → "Add + // channel") live only in localStorage — the keys never leave the client + // (channel-decrypt.js). The server therefore cannot decrypt their traffic + // and files it under the synthetic channel hash "enc_", which + // /api/channels omits. Result: a channel you just added is invisible in + // the packets channel picker. + // + // Fix: derive the same "enc_" value client-side from the stored key + // (SHA-256(key)[0], exactly what the ingestor writes) and offer those + // channels as extra options. /api/packets?channel=enc_ already + // filters on that value server-side, so no backend change is needed. + + /** + * Read the browser's stored channel keys and map each to the server-side + * channel-hash value its packets are stored under. + * Cost: one SHA-256 per stored key (a handful), once per page init. + * @returns {Promise>} + */ + async function collectLocalChannels() { + const CD = window.ChannelDecrypt; + if (!CD || typeof CD.getStoredKeys !== 'function') return []; + let keys; + try { keys = CD.getStoredKeys() || {}; } catch (e) { return []; } + const out = []; + for (const name of Object.keys(keys)) { + const keyHex = keys[name]; + if (!keyHex || typeof keyHex !== 'string') continue; + let hashByte; + try { + const keyBytes = CD.hexToBytes(keyHex); + if (!keyBytes || keyBytes.length !== 16) continue; + hashByte = await CD.computeChannelHash(keyBytes); + } catch (e) { continue; } + if (typeof hashByte !== 'number') continue; + const label = (typeof CD.getLabel === 'function' && CD.getLabel(name)) || name; + out.push({ + // Uppercase 2-digit hex — matches cmd/ingestor/decoder.go ("%02X"). + value: 'enc_' + hashByte.toString(16).padStart(2, '0').toUpperCase(), + name: name, + label: label + }); + } + return out; + } + + /** + * Merge the server channel list with locally-added ones into the two + * option groups the picker renders. Pure — unit-tested. + * A local channel is dropped when the server already exposes it (same + * hash value, or same name because the server holds the key too). + * @returns {{server: Array<{value:string,label:string}>, local: Array<{value:string,label:string}>}} + */ + function buildChannelOptions(serverChannels, localChannels) { + const byLabel = (a, b) => { + const an = (a.label || '').toLowerCase(); + const bn = (b.label || '').toLowerCase(); + return an < bn ? -1 : an > bn ? 1 : 0; + }; + const server = []; + const seenValues = new Set(); + const seenNames = new Set(); + for (const ch of serverChannels || []) { + const value = (ch && (ch.hash || ch.name)) || ''; + if (!value || seenValues.has(value)) continue; + seenValues.add(value); + seenNames.add(String(ch.name || value).toLowerCase()); + server.push({ value: value, label: ch.name || value }); + } + const local = []; + for (const lc of localChannels || []) { + if (!lc || !lc.value) continue; + if (seenValues.has(lc.value)) continue; + if (seenNames.has(String(lc.name || '').toLowerCase())) continue; + seenValues.add(lc.value); + local.push({ value: lc.value, label: lc.label || lc.name }); + } + return { server: server.sort(byLabel), local: local.sort(byLabel) }; + } + + // Exported for test-packets-local-channels.js (no DOM required). + if (typeof window !== 'undefined') { + window._packetsBuildChannelOptionsForTest = buildChannelOptions; + window._packetsCollectLocalChannelsForTest = collectLocalChannels; + } + function renderTimestampCell(isoString) { if (typeof formatTimestampWithTooltip !== 'function' || typeof getTimestampMode !== 'function') { return escapeHtml(typeof timeAgo === 'function' ? timeAgo(isoString) : '—'); @@ -1815,31 +1901,37 @@ opt.selected = true; channelSel.appendChild(opt); } - api('/channels').then(data => { + Promise.all([ + api('/channels').catch(() => null), + collectLocalChannels() + ]).then(([data, localChannels]) => { const channels = (data && data.channels) || []; // Build options via DOM API: channel names are network-supplied // and must NOT be interpolated into innerHTML (XSS, #812). // Sort alphabetically (case-insensitive) for predictable picker order; // the API returns last-activity order which is unstable for a dropdown. - const sorted = channels.slice().sort((a, b) => { - const an = (a.name || a.hash || '').toLowerCase(); - const bn = (b.name || b.hash || '').toLowerCase(); - return an < bn ? -1 : an > bn ? 1 : 0; - }); + const groups = buildChannelOptions(channels, localChannels); channelSel.textContent = ''; const allOpt = document.createElement('option'); allOpt.value = ''; allOpt.textContent = 'All Channels'; channelSel.appendChild(allOpt); let matched = false; - for (const ch of sorted) { - const v = ch.hash || ch.name || ''; - if (!v) continue; + const addOption = (o, parent) => { const opt = document.createElement('option'); - opt.value = v; - opt.textContent = ch.name || v; - if (v === filters.channel) { opt.selected = true; matched = true; } - channelSel.appendChild(opt); + opt.value = o.value; + opt.textContent = o.label; + if (o.value === filters.channel) { opt.selected = true; matched = true; } + parent.appendChild(opt); + }; + for (const o of groups.server) addOption(o, channelSel); + // Browser-added channels the server can't see, grouped so it's + // obvious they come from keys stored in this browser only. + if (groups.local.length) { + const grp = document.createElement('optgroup'); + grp.label = 'My Channels (this browser)'; + for (const o of groups.local) addOption(o, grp); + channelSel.appendChild(grp); } // If current filter isn't in the list (encrypted hash, stale, or // race with cache), keep it as a selected option so the UI reflects state. diff --git a/test-all.sh b/test-all.sh index 8804b3612..7b65c2931 100755 --- a/test-all.sh +++ b/test-all.sh @@ -11,6 +11,7 @@ echo "" echo "── Unit Tests ──" node test-packet-filter.js node test-packet-filter-ux.js +node test-packets-local-channels.js node test-aging.js node test-issue-1065-gesture-hints-gates.js node test-frontend-helpers.js diff --git a/test-packets-local-channels.js b/test-packets-local-channels.js new file mode 100644 index 000000000..c73914a90 --- /dev/null +++ b/test-packets-local-channels.js @@ -0,0 +1,222 @@ +/** + * Unit tests for the packets channel-picker option merge. + * + * Locally-added channels (keys stored in the browser only) are invisible to + * the server, so /api/channels never lists them. packets.js derives their + * server-side "enc_" channel hash client-side and offers them as an + * extra option group. buildChannelOptions() does that merge; it is pure and + * exported as window._packetsBuildChannelOptionsForTest. + * + * Covers: sorting, server-list passthrough, local append, dedupe by hash + * value, dedupe by name (server holds the key too), and junk input. + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +// packets.js expects a browser at call time; the merge helper touches none +// of it. Load in a tolerant sandbox and grab the exported helper — it is +// assigned before any DOM-dependent code runs. +const noop = () => {}; +const fakeEl = { + addEventListener: noop, removeEventListener: noop, appendChild: noop, remove: noop, + querySelector: () => null, querySelectorAll: () => [], setAttribute: noop, + getAttribute: () => null, classList: { add: noop, remove: noop, toggle: noop, contains: () => false }, + style: {}, dataset: {}, textContent: '', innerHTML: '', +}; +const doc = { + readyState: 'complete', createElement: () => ({ ...fakeEl }), head: fakeEl, body: fakeEl, + getElementById: () => null, querySelector: () => null, querySelectorAll: () => [], + addEventListener: noop, removeEventListener: noop, +}; +const win = { + addEventListener: noop, removeEventListener: noop, + matchMedia: () => ({ matches: false, addEventListener: noop, removeEventListener: noop }), + // packets.js fails loud if this module is missing (#1799). + PayloadLabels: { SHORT_BY_ID: {} }, +}; +const ctx = { + window: win, document: doc, console, Date, Math, JSON, Set, Map, Array, Object, Promise, Error, + setTimeout, clearTimeout, setInterval, clearInterval, + history: { replaceState: noop, pushState: noop }, + location: { hash: '', href: '', pathname: '/' }, + navigator: { userAgent: 'node' }, + localStorage: { getItem: () => null, setItem: noop, removeItem: noop }, + registerPage: noop, + api: () => Promise.resolve({}), + fetch: () => Promise.resolve({ json: () => Promise.resolve({}) }), +}; +ctx.self = ctx; +vm.createContext(ctx); +try { + vm.runInContext(fs.readFileSync('public/packets.js', 'utf8'), ctx); +} catch (e) { + // Downstream init may throw on the stub DOM — fine, the helper is exported + // near the top of the module. +} + +const build = win._packetsBuildChannelOptionsForTest; +assert.strictEqual(typeof build, 'function', 'buildChannelOptions must be exported for tests'); + +// buildChannelOptions returns objects created inside the vm realm, whose +// prototypes differ from this realm's — deepStrictEqual would reject them on +// identity alone. Compare structurally. +const plain = (v) => JSON.parse(JSON.stringify(v)); + +let passed = 0; +function test(name, fn) { + fn(); + passed++; + console.log(' ok - ' + name); +} + +console.log('packets channel-picker option merge'); + +test('empty inputs produce empty groups', () => { + const r = build([], []); + assert.deepStrictEqual(plain(r), { server: [], local: [] }); + const r2 = build(null, undefined); + assert.deepStrictEqual(plain(r2), { server: [], local: [] }); +}); + +test('server channels are sorted case-insensitively by label', () => { + const r = build([{ hash: 'zeta', name: 'zeta' }, { hash: 'Alpha', name: 'Alpha' }, { hash: 'mid', name: 'mid' }], []); + assert.deepStrictEqual(plain(r.server.map(o => o.value)), ['Alpha', 'mid', 'zeta']); + assert.strictEqual(r.local.length, 0); +}); + +test('server channel without a name falls back to its hash', () => { + const r = build([{ hash: 'enc_A5' }], []); + assert.deepStrictEqual(plain(r.server), [{ value: 'enc_A5', label: 'enc_A5' }]); +}); + +test('local channels are appended as their own group', () => { + const r = build( + [{ hash: 'public', name: 'public' }], + [{ value: 'enc_A5', name: '#mytown', label: '#mytown' }] + ); + assert.deepStrictEqual(plain(r.server), [{ value: 'public', label: 'public' }]); + assert.deepStrictEqual(plain(r.local), [{ value: 'enc_A5', label: '#mytown' }]); +}); + +test('local channel is dropped when the server already lists that hash', () => { + const r = build( + [{ hash: 'enc_A5', name: 'enc_A5' }], + [{ value: 'enc_A5', name: '#mytown', label: '#mytown' }] + ); + assert.strictEqual(r.local.length, 0); +}); + +test('local channel is dropped when the server decrypts it under the same name', () => { + const r = build( + [{ hash: '#MyTown', name: '#MyTown' }], + [{ value: 'enc_A5', name: '#mytown', label: '#mytown' }] + ); + assert.strictEqual(r.local.length, 0, 'name match is case-insensitive'); +}); + +test('duplicate local entries collapse to one option', () => { + const r = build([], [ + { value: 'enc_A5', name: '#a', label: '#a' }, + { value: 'enc_A5', name: '#b', label: '#b' }, + ]); + assert.deepStrictEqual(plain(r.local), [{ value: 'enc_A5', label: '#a' }]); +}); + +test('local options are sorted and use their display label', () => { + const r = build([], [ + { value: 'enc_02', name: 'psk:deadbeef', label: 'Work PSK' }, + { value: 'enc_01', name: '#alpha', label: '#alpha' }, + ]); + assert.deepStrictEqual(plain(r.local), [ + { value: 'enc_01', label: '#alpha' }, + { value: 'enc_02', label: 'Work PSK' }, + ]); +}); + +test('valueless / junk entries are skipped', () => { + const r = build([{ name: '' }, null, {}], [null, { name: '#x' }, { value: '' }]); + assert.deepStrictEqual(plain(r), { server: [], local: [] }); +}); + +// --- collectLocalChannels(): stored keys → "enc_" ------------------- +// Stubs ChannelDecrypt with the real crypto (node) so the derived value is +// checked against a genuine SHA-256(key)[0], not a fixture. + +const crypto = require('crypto'); +const collect = win._packetsCollectLocalChannelsForTest; +assert.strictEqual(typeof collect, 'function', 'collectLocalChannels must be exported for tests'); + +function fakeChannelDecrypt(keys, labels) { + return { + getStoredKeys: () => keys, + getLabel: (name) => (labels && labels[name]) || '', + hexToBytes: (hex) => Uint8Array.from(Buffer.from(hex, 'hex')), + computeChannelHash: async (keyBytes) => crypto.createHash('sha256').update(keyBytes).digest()[0], + }; +} + +function deriveKeyHex(channelName) { + return crypto.createHash('sha256').update(channelName).digest().subarray(0, 16).toString('hex'); +} + +function expectedEncValue(channelName) { + const key = crypto.createHash('sha256').update(channelName).digest().subarray(0, 16); + const hashByte = crypto.createHash('sha256').update(key).digest()[0]; + return 'enc_' + hashByte.toString(16).padStart(2, '0').toUpperCase(); +} + +async function asyncTest(name, fn) { + await fn(); + passed++; + console.log(' ok - ' + name); +} + +(async () => { + console.log('\ncollectLocalChannels'); + + await asyncTest('no ChannelDecrypt module → no local channels', async () => { + win.ChannelDecrypt = undefined; + assert.deepStrictEqual(plain(await collect()), []); + }); + + await asyncTest('derives the server-side enc_ value from the stored key', async () => { + // '#alpha' and '#zulu' hash to different bytes; both must round-trip. + const keys = { '#alpha': deriveKeyHex('#alpha'), '#zulu': deriveKeyHex('#zulu') }; + win.ChannelDecrypt = fakeChannelDecrypt(keys); + const out = plain(await collect()); + assert.strictEqual(out.length, 2); + assert.deepStrictEqual(out[0], { value: expectedEncValue('#alpha'), name: '#alpha', label: '#alpha' }); + assert.deepStrictEqual(out[1], { value: expectedEncValue('#zulu'), name: '#zulu', label: '#zulu' }); + // Uppercase, zero-padded — matches cmd/ingestor/decoder.go's "%02X". + for (const o of out) assert.ok(/^enc_[0-9A-F]{2}$/.test(o.value), 'bad value ' + o.value); + }); + + await asyncTest('uses the stored display label when present', async () => { + win.ChannelDecrypt = fakeChannelDecrypt( + { 'psk:deadbeef': deriveKeyHex('#work') }, + { 'psk:deadbeef': 'Work PSK' } + ); + const out = plain(await collect()); + assert.deepStrictEqual(out, [{ value: expectedEncValue('#work'), name: 'psk:deadbeef', label: 'Work PSK' }]); + }); + + await asyncTest('skips malformed keys instead of throwing', async () => { + win.ChannelDecrypt = fakeChannelDecrypt({ + '#short': 'aabb', // not 16 bytes + '#empty': '', + '#null': null, + '#good': deriveKeyHex('#good'), + }); + const out = plain(await collect()); + assert.deepStrictEqual(out, [{ value: expectedEncValue('#good'), name: '#good', label: '#good' }]); + }); + + await asyncTest('a throwing getStoredKeys degrades to no local channels', async () => { + win.ChannelDecrypt = { getStoredKeys: () => { throw new Error('quota'); } }; + assert.deepStrictEqual(plain(await collect()), []); + }); + + console.log(`\n${passed} tests passed`); +})().catch((e) => { console.error(e); process.exit(1); }); From cd9b4c04d0ca3996424ffdb3b3c9b84347ac8c39 Mon Sep 17 00:00:00 2001 From: n30nex Date: Sun, 13 Sep 2026 13:19:39 -0400 Subject: [PATCH 029/151] test: unify frontend test runs and prevent inventory drift (#1965) Make `test-all.sh` the authoritative standalone frontend runner for npm and CI. Restore stale assertions and reject missing, duplicate, removed or undocumented inventory entries. Fixes #1858. Rebased onto `a2f039d4`. Retains release-routing, map scope-state and Scope Audit stylesheet tests, adds `test-packets-local-channels.js` to the sorted runner, and classifies `test-neighbor-map-btn-clip-e2e.js` under browser. Its separate CI browser step is preserved. Inventory: 280 root suites, 167 standalone, 113 requiring separate setup. The icon repair fixes two suites red on master: `test-issue-1648-m2-emoji-scan.js` and `test-issue-1648-m6-final-sweep.js`. Node/live configured-scope confirmations now use the existing accessible Phosphor check sprite. Values, visibility conditions and scanner assertions are preserved. - Red evidence: `89e45a9` inventory assertions; `4e255df` accessible confirmation assertion. The latest rebase also reproduced both unclassified-file failures before adding their entries. This follow-up only changes runner/classification configuration and counts; no test files modified. - Local validation: all 167 standalone suites; 27 M2 browser checks and 16 neighbor geometry checks in Chromium. Syntax, whitespace, PII, CSS-variable and XSS checks passed. - Browser coverage includes populated/empty/null configured scopes in node and live views. - No new dependencies, requests, application settings or Go changes. Workflow outside the unit step matches master. - Windows validation uses process-local UTF-8 settings. Encoding and node-reach confirmation follow-ups remain separate, as requested. ## Preflight override External `run-all.sh` is unavailable; applicable repository checks were run directly. --- .github/workflows/deploy.yml | 74 +--------- docs/frontend-testing.md | 32 +++++ package.json | 2 +- public/live.js | 2 +- public/nodes.js | 2 +- scripts/non-unit-tests.json | 130 +++++++++++++++++ test-all.sh | 207 +++++++++++++++++++-------- test-channel-colors.js | 10 +- test-channel-ux-followup.js | 4 +- test-channel-ux-round2.js | 8 +- test-clear-filters.js | 8 +- test-customizer-v2.js | 7 +- test-drag-manager.js | 6 +- test-fluid-scaffolding.js | 16 ++- test-hop-resolver-affinity.js | 6 +- test-issue-1438-customizer-mcrole.js | 14 +- test-issue-1446-cb-preset-cascade.js | 9 +- test-issue-1470-card-bg-contrast.js | 8 +- test-issue-1646-compare-polish.js | 2 +- test-issue-1648-m2-icons-e2e.js | 49 +++++++ test-live-anims.js | 23 +-- test-map-clustering.js | 8 +- test-packets.js | 26 ++-- test-panel-corner.js | 1 + test-perf-disk-io-1120.js | 64 ++++----- test-preflight-xss-gate.js | 5 +- test-rx-coverage-escape.js | 4 +- test-test-inventory.js | 49 +++++++ 28 files changed, 542 insertions(+), 234 deletions(-) create mode 100644 docs/frontend-testing.md create mode 100644 scripts/non-unit-tests.json create mode 100644 test-test-inventory.js diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index eb6b7db78..8b42e2d3f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -221,78 +221,8 @@ jobs: node scripts/check-css-vars.js node scripts/test-check-css-vars.js - - name: Run JS unit tests (packet-filter) - run: | - set -e - node test-packet-filter.js - node test-packet-filter-time.js - node test-confidence-indicator.js - node test-1659-analytics-warmup.js - node test-channels-merge-1498-unit.js - node test-packets-local-channels.js - node test-issue-1518-home-url.js - node test-channel-decrypt-insecure-context.js - node test-live-region-filter.js - node test-issue-1136-observer-iata-map.js - node test-channel-qr.js - node test-channel-qr-wiring.js - node test-channel-modal-ux.js - node test-channel-issue-1087.js - node test-issue-1409-no-encrypted-flood.js - node test-channel-issue-1101.js - node test-observer-iata-1188.js - node test-pull-to-reconnect-1091.js - node test-channel-fluid-layout.js - node test-issue-1279-p2-code-filter.js - node test-area-filter.js - node test-issue-1293-marker-shapes.js - node test-issue-1356-map-a11y.js - node test-issue-1360-pill-letter-count.js - node test-issue-1364-pill-no-clamp.js - node test-issue-1375-scope-stats-fetch.js - node test-scope-audit-styles-linked.js - node test-issue-1361-cb-presets.js - node test-issue-1380-cb-sim-overlay.js - node test-issue-1380-cb-reset-button.js - node test-issue-1407-cb-preset-propagation.js - node test-issue-1412-customizer-no-override.js - node test-issue-1418-raw-hex-extraction.js - node test-issue-1418-edge-weights.js - node test-issue-1418-cb-preset-ramp.js - node test-issue-1418-spider-fan.js - node test-issue-1418-deeplink-hops-channels.js - node test-issue-1418-polish-review.js - node test-issue-1420-tile-providers.js - node test-issue-1614-tile-url-function.js - node test-issue-1438-marker-css-vars.js - node test-issue-2001-map-scope-state.js - node test-issue-1846-observers-width.js - node test-issue-1562-observers-summary.js - node test-issue-1509-nav-active-bg.js - node test-issue-1509-detect-preset.js - node test-live.js - node test-coverage-gate.js - node test-node-reach-coverage.js - node test-issue-1107-live-layout.js - node test-issue-1532-live-fullscreen.js - node test-issue-1619-feed-detail-card-draggable.js - node test-xss-escape-sinks.js - node test-preflight-xss-gate.js - node test-traces.js - node test-issue-1648-m4-emoji-scan.js - node test-issue-1753-copy-url-slash.js - node test-issue-1668-m3-typography.js - node test-mqtt-status-panel.js - node test-issue-1697-mqtt-mobile-e2e.js - node test-warmup-banner.js - node test-issue-1633-hide-1byte-hops.js - node test-issue-1668-m4-per-route.js - node test-a11y-axe-1668-selftest.js - node test-a11y-1716-rf-range-btn-active.js - node test-issue-1705-subpath-contrast.js - node test-issue-1770-mobile-row-clamp.js - node test-a11y-axe-routes-coverage.js - node test-issue-1956-release-routing.js + - name: Run JS unit tests + run: sh test-all.sh - name: 🛡️ Preflight XSS gate — actual --diff check (PR only) # The fixture self-test above (test-preflight-xss-gate.js) only diff --git a/docs/frontend-testing.md b/docs/frontend-testing.md new file mode 100644 index 000000000..ec2803bff --- /dev/null +++ b/docs/frontend-testing.md @@ -0,0 +1,32 @@ +# Frontend tests + +`test-all.sh` is the authoritative list of standalone frontend suites. Both +`npm run test:unit` and the CI **Run JS unit tests** step invoke it. `npm test` +and `npm run test:coverage` wrap the same runner with coverage reporting. + +The inventory currently covers 280 root suites: 167 standalone suites in the +runner and 113 suites classified separately by their prerequisites. + +Run `npm run test:unit` from a checkout with Node.js, a POSIX shell, Bash, and +Python 3 available. The XSS gate's fixture test uses Bash and Python. On +Windows, use Git Bash and ensure `python3` resolves to Python 3 rather than a +Microsoft Store alias; set `PYTHONUTF8=1` for the gate's Unicode output. +No server or browser is started by this runner. + +When adding a root `test-*.js` file: + +1. Add its `node test-name.js` command to `test-all.sh` if it runs standalone. +2. Otherwise add it to `scripts/non-unit-tests.json`, with a prerequisite or + reason explaining why it belongs outside the unit runner. Browser suites + still need explicit selection in the E2E workflow to become CI gates. +3. Run `node test-test-inventory.js`. The guard rejects unclassified files, + duplicate assignments, removed files left in a list, undocumented groups, + and local/CI entry points that bypass the unit runner. + +The separate inventory distinguishes plain Node Playwright scripts, suites +requiring the `@playwright/test` runner, and the `jsdom` integration suite. +Those last two dependencies are not currently declared in `package.json`; +classification records that limitation rather than installing them implicitly. +The existing E2E workflow retains its browser installation, local fixture +server, coverage collection, and repeat checks. Only use local servers for +browser tests; inventory membership alone does not mean a suite runs in CI. diff --git a/package.json b/package.json index b52d5d892..f9f5b5f67 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "index.js", "scripts": { "test": "npx c8 --reporter=text --reporter=text-summary sh test-all.sh", - "test:unit": "node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js", + "test:unit": "sh test-all.sh", "test:coverage": "npx c8 --reporter=text --reporter=html sh test-all.sh", "test:full-coverage": "sh scripts/combined-coverage.sh", "lint": "eslint public/*.js" diff --git a/public/live.js b/public/live.js index 3e6da6287..f44fb0547 100644 --- a/public/live.js +++ b/public/live.js @@ -2629,7 +2629,7 @@ - ${'configured_scope' in n && n.configured_scope !== null ? `` : ''} + ${'configured_scope' in n && n.configured_scope !== null ? `` : ''} ${'default_scope' in n ? ``; })() : ''} ${(n.role === 'repeater' || n.role === 'room') && Array.isArray(n.transported_scopes) && n.transported_scopes.length ? `` : ''} - ${'configured_scope' in n && n.configured_scope !== null ? `` : ''} + ${'configured_scope' in n && n.configured_scope !== null ? `` : ''} diff --git a/scripts/non-unit-tests.json b/scripts/non-unit-tests.json new file mode 100644 index 000000000..85770ed67 --- /dev/null +++ b/scripts/non-unit-tests.json @@ -0,0 +1,130 @@ +{ + "browser": { + "reason": "Requires Playwright and Chromium; most suites also require a running local fixture server via BASE_URL. Existing E2E workflow steps retain their separate setup and selection. Never point these tests at production.", + "files": [ + "test-1110-live-filter.js", + "test-a11y-axe-1668.js", + "test-analytics-fluid-charts.js", + "test-audio-lab-1297-e2e.js", + "test-audio-live-1297-e2e.js", + "test-bottom-nav-1061-e2e.js", + "test-channel-color-picker-e2e.js", + "test-channel-decrypt-e2e.js", + "test-channel-fluid-e2e.js", + "test-channel-issue-1087-e2e.js", + "test-channel-issue-1111-e2e.js", + "test-channel-modal-e2e.js", + "test-channel-qr-e2e.js", + "test-channels-add-modal-e2e.js", + "test-channels-list-render-e2e.js", + "test-channels-selection-flow-e2e.js", + "test-channels-share-color-e2e.js", + "test-channels-ws-batch-e2e.js", + "test-channels-ws-race-1498-e2e.js", + "test-charts-fluid-1058-e2e.js", + "test-customize-branding-e2e.js", + "test-customize-display-e2e.js", + "test-customize-export-e2e.js", + "test-customize-theme-e2e.js", + "test-drag-manager-e2e.js", + "test-e2e-1267-mobile-vcr.js", + "test-e2e-playwright.js", + "test-filter-ux-e2e.js", + "test-gesture-hints-1065-e2e.js", + "test-gestures-1062-e2e.js", + "test-gestures-1185-scroll-discriminator-e2e.js", + "test-home-coverage-e2e.js", + "test-issue-1109-hamburger-dropdown-visible-e2e.js", + "test-issue-1122-packets-filter-ux-e2e.js", + "test-issue-1128-multi-viewport-e2e.js", + "test-issue-1128-packets-layout-e2e.js", + "test-issue-1136-live-region-e2e.js", + "test-issue-1146-path-link-contrast-e2e.js", + "test-issue-1147-section-order-e2e.js", + "test-issue-1150-404-state-e2e.js", + "test-issue-1151-orphan-separators-e2e.js", + "test-issue-1204-live-panel-structure-e2e.js", + "test-issue-1205-live-controls-anchor-e2e.js", + "test-issue-1206-resize-observer-leak-e2e.js", + "test-issue-1206-vcr-overlap-e2e.js", + "test-issue-1224-channels-mobile-ux-e2e.js", + "test-issue-1234-live-chrome-pass2-e2e.js", + "test-issue-1236-map-mobile-e2e.js", + "test-issue-1244-live-vcr-row-hints-e2e.js", + "test-issue-1273-qr-overlay-height-e2e.js", + "test-issue-1274-legend-coverage-e2e.js", + "test-issue-1279-legend-p2-e2e.js", + "test-issue-1281-location-row-e2e.js", + "test-issue-1306-collisions-terminology-e2e.js", + "test-issue-1329-map-controls-accordion-e2e.js", + "test-issue-1367-channels-chat-app-e2e.js", + "test-issue-1374-route-map-a11y-e2e.js", + "test-issue-1400-nav-vertical-clip.js", + "test-issue-1413-nav-overlap-e2e.js", + "test-issue-1486-collapse-reopens-detail-e2e.js", + "test-issue-1487-byop-modal-layout-e2e.js", + "test-issue-1510-live-nav-pin-e2e.js", + "test-issue-1567-corner-clears-drag-e2e.js", + "test-issue-1599-replay-freeze-e2e.js", + "test-issue-1630-reach-mobile-e2e.js", + "test-issue-1639-observers-sort-e2e.js", + "test-issue-1640-compare-discovery-e2e.js", + "test-issue-1648-m1-icons-e2e.js", + "test-issue-1648-m2-icons-e2e.js", + "test-issue-1648-m3-icons-e2e.js", + "test-issue-1648-m4-icons-e2e.js", + "test-issue-1648-m5-icons-e2e.js", + "test-issue-1657-analytics-channels-group-sprites-e2e.js", + "test-issue-1692-packets-init-parallel-e2e.js", + "test-issue-1705-subpath-contrast-e2e.js", + "test-issue-1758-ng-filter-rerenders-e2e.js", + "test-issue-1799-label-vocab-e2e.js", + "test-live-dedup.js", + "test-live-fullscreen-1572-e2e.js", + "test-live-layout-1178-1179-e2e.js", + "test-live-mql-leak-1180-e2e.js", + "test-live-multibyte-only-e2e.js", + "test-logo-default-sage-teal-e2e.js", + "test-logo-pulse-1173-e2e.js", + "test-logo-rebrand-e2e.js", + "test-logo-theme-e2e.js", + "test-map-modal-fluid-e2e.js", + "test-map-nodes-pagination-e2e.js", + "test-nav-drawer-1064-e2e.js", + "test-nav-fluid-1055-e2e.js", + "test-nav-more-floor-1139-e2e.js", + "test-nav-priority-1102-e2e.js", + "test-nav-priority-1311-e2e.js", + "test-nav-priority-1391-e2e.js", + "test-neighbor-map-btn-clip-e2e.js", + "test-node-reach-coverage-e2e.js", + "test-node-reach-e2e.js", + "test-nodes-export-e2e.js", + "test-observer-iata-1188-e2e.js", + "test-packets-scope-column.js", + "test-path-inspector-coverage-e2e.js", + "test-rx-coverage-mobile-nav-e2e.js", + "test-show-neighbors.js", + "test-slideover-1056-e2e.js", + "test-slideover-1168-munger-e2e.js", + "test-table-fluid-e2e.js", + "test-touch-gestures-coverage-e2e.js", + "test-touch-targets.js" + ] + }, + "playwrightRunner": { + "reason": "Requires the @playwright/test runner (not plain node), its configuration, Chromium, and a local server. @playwright/test is not currently declared in package.json; these suites are inventoried but not part of the standalone unit gate.", + "files": [ + "test-issue-1522-trace-url-sync-e2e.js", + "test-marker-outline-weight.js", + "test-path-inspector-e2e.js", + "test-pr-1490-live-map-gpu-animations-e2e.js" + ] + }, + "domIntegration": { + "reason": "Requires jsdom, which is not currently declared in package.json. This DOM integration suite is inventoried separately; no new dependency is installed by the unit gate.", + "files": [ + "test-table-sort.js" + ] + } +} diff --git a/test-all.sh b/test-all.sh index 7b65c2931..2d59388b8 100755 --- a/test-all.sh +++ b/test-all.sh @@ -1,82 +1,175 @@ #!/bin/sh -# Run all tests with coverage +# Authoritative standalone frontend test list for local runs and CI. +# Every root test is listed here or classified in scripts/non-unit-tests.json. set -e +cd "$(dirname "$0")" -echo "═══════════════════════════════════════" -echo " CoreScope — Test Suite" -echo "═══════════════════════════════════════" -echo "" - -# Unit tests (deterministic, fast) -echo "── Unit Tests ──" -node test-packet-filter.js -node test-packet-filter-ux.js -node test-packets-local-channels.js +node test-test-inventory.js +node test-1108-region-hide-nodes.js +node test-1659-analytics-warmup.js +node test-a11y-1705-subpath-hop-prefix-e2e.js +node test-a11y-1715-dark-role-swatches.js +node test-a11y-1716-rf-range-btn-active.js +node test-a11y-1719-contrast-root-causes-e2e.js +node test-a11y-axe-1668-selftest.js +node test-a11y-axe-routes-coverage.js node test-aging.js -node test-issue-1065-gesture-hints-gates.js -node test-frontend-helpers.js -node test-issue-2001-map-scope-state.js -node test-scope-audit-styles-linked.js -node test-fetch-all-nodes-pagination.js -node test-my-repeaters-dashboard.js -node test-repeater-metric-scatter.js -node test-top-routes-overlay.js -node test-url-state.js -node test-perf-go-runtime.js -node test-channel-psk-ux.js -node test-channel-sidebar-layout.js +node test-analytics-channels-integration.js +node test-anl1-tooltip-render.js +node test-area-filter.js +node test-channel-color-picker.js +node test-channel-colors.js +node test-channel-decrypt-ecb.js +node test-channel-decrypt-insecure-context.js +node test-channel-decrypt-m345.js node test-channel-fluid-layout.js +node test-channel-issue-1087.js +node test-channel-issue-1101.js +node test-channel-live-decrypt-userprefix.js +node test-channel-live-decrypt.js node test-channel-modal-ux.js -node test-channel-decrypt-insecure-context.js -node test-channel-qr.js +node test-channel-psk-ux.js node test-channel-qr-wiring.js -node test-channel-issue-1087.js +node test-channel-qr.js +node test-channel-sidebar-layout.js +node test-channel-ux-followup.js +node test-channel-ux-round2.js +node test-channels-merge-1498-unit.js +node test-clear-filters.js +node test-color-picker-ux.js +node test-compare-flood-filter.js +node test-compare-overlap.js +node test-confidence-indicator.js +node test-coverage-gate.js +node test-customizer-v2.js +node test-drag-manager.js +node test-embed-mode-1369.js +node test-fetch-all-nodes-pagination.js +node test-fluid-scaffolding.js +node test-frontend-helpers.js +node test-geofilter-draft.js +node test-hash-color.js +node test-hop-resolver-affinity.js +node test-issue-1065-gesture-hints-gates.js +node test-issue-1107-live-layout.js +node test-issue-1136-observer-iata-map.js +node test-issue-1166-first-seen-column.js +node test-issue-1189-composed-cell.js +node test-issue-1189-live-iata-badge.js +node test-issue-1279-p2-code-filter.js +node test-issue-1293-marker-shapes.js +node test-issue-1356-map-a11y.js +node test-issue-1360-pill-letter-count.js +node test-issue-1361-cb-presets.js +node test-issue-1364-pill-no-clamp.js +node test-issue-1375-scope-stats-fetch.js +node test-issue-1380-cb-reset-button.js +node test-issue-1380-cb-sim-overlay.js +node test-issue-1407-cb-preset-propagation.js node test-issue-1409-no-encrypted-flood.js -node test-analytics-channels-integration.js -node test-observers-headings.js -node test-issue-1789-observer-firmware-cols.js -node test-issue-1648-m1-emoji-scan.js -node test-issue-1648-m2-emoji-scan.js -node test-issue-1648-m3-emoji-scan.js -node test-issue-1648-m6-final-sweep.js -node test-issue-1648-m6-lint-self.js -node test-issue-1890-og-url.js -node test-traces.js -node test-live-multibyte-filter.js -node test-issue-1956-release-routing.js - -# #1418 — route-view v2 (Tufte) coverage -node test-issue-1418-raw-hex-extraction.js -node test-issue-1418-edge-weights.js +node test-issue-1412-customizer-no-override.js +node test-issue-1415-packets-layout.js node test-issue-1418-cb-preset-ramp.js -node test-issue-1418-spider-fan.js node test-issue-1418-deeplink-hops-channels.js +node test-issue-1418-edge-weights.js node test-issue-1418-polish-review.js +node test-issue-1418-raw-hex-extraction.js +node test-issue-1418-spider-fan.js node test-issue-1420-tile-providers.js -node test-issue-1614-tile-url-function.js -node test-issue-1438-marker-css-vars.js node test-issue-1438-customizer-mcrole.js +node test-issue-1438-marker-css-vars.js node test-issue-1446-cb-preset-cascade.js -node test-issue-1380-cb-sim-overlay.js -node test-issue-1380-cb-reset-button.js node test-issue-1450-logo-aspect.js node test-issue-1454-channels-toggle.js node test-issue-1456-score-labels.js - -# #1461 mobile UX overhaul + #1470 node-detail tile helper (#1468 covered by E2E) node test-issue-1461-mobile-page-actions.js +node test-issue-1470-card-bg-contrast.js node test-issue-1470-node-tile-helper.js +node test-issue-1473-prefix-generator.js +node test-issue-1473-reserved-prefixes.js node test-issue-1485-live-anim-z.js +node test-issue-1488-marker-stroke-vars.js +node test-issue-1496-reset-all-complete.js +node test-issue-1509-detect-preset.js +node test-issue-1509-nav-active-bg.js +node test-issue-1518-home-url.js node test-issue-1532-live-fullscreen.js -node test-naive-banner-tone.js -node test-issue-1473-reserved-prefixes.js -node test-issue-1473-prefix-generator.js +node test-issue-1562-observers-summary.js +node test-issue-1563-aggregate-and-inflight.js +node test-issue-1574-live-map-max-nodes.js +node test-issue-1606-pagination.js +node test-issue-1614-tile-url-function.js +node test-issue-1619-feed-detail-card-draggable.js +node test-issue-1633-hide-1byte-hops.js +node test-issue-1644-redesign.js +node test-issue-1646-compare-polish.js +node test-issue-1648-followup-phosphor-leaks.js +node test-issue-1648-m1-emoji-scan.js +node test-issue-1648-m2-emoji-scan.js +node test-issue-1648-m3-emoji-scan.js +node test-issue-1648-m4-emoji-scan.js +node test-issue-1648-m5-emoji-scan.js +node test-issue-1648-m6-final-sweep.js +node test-issue-1648-m6-lint-self.js +node test-issue-1668-m2-contrast.js +node test-issue-1668-m3-typography.js +node test-issue-1668-m4-per-route.js +node test-issue-1697-mqtt-mobile-e2e.js +node test-issue-1705-subpath-contrast.js +node test-issue-1753-copy-url-slash.js node test-issue-1770-mobile-row-clamp.js +node test-issue-1789-observer-firmware-cols.js +node test-issue-1825-observer-node-cross-links.js +node test-issue-1833-legend-toggle-vcr-offset.js +node test-issue-1836-crossnav-case-normalization.js +node test-issue-1843-node-qr-quiet-zone.js +node test-issue-1846-observers-width.js node test-issue-1849-trace-hashbytes.js -node test-nodes-export.js +node test-issue-1890-og-url.js +node test-issue-1956-release-routing.js +node test-issue-2001-map-scope-state.js +node test-live-anims.js +node test-live-dt-cap-1524.js +node test-live-legend-helper.js +node test-live-multibyte-filter.js +node test-live-region-filter.js +node test-live.js +node test-map-clustering.js +node test-mqtt-status-panel.js +node test-my-repeaters-dashboard.js +node test-naive-banner-tone.js +node test-node-reach-coverage-debounce.js +node test-node-reach-coverage.js node test-nodes-export-wiring.js +node test-nodes-export.js +node test-observer-iata-1188.js +node test-observer-naive-clock-1478.js +node test-observers-headings.js +node test-packet-filter-time.js +node test-packet-filter-ux.js +node test-packet-filter.js +node test-packets-local-channels.js +node test-packets.js +node test-panel-corner.js +node test-path-inspector.js +node test-payload-labels-content.js +node test-payload-labels-namespace.js +node test-perf-anomaly.js +node test-perf-disk-io-1120.js +node test-perf-go-runtime.js +node test-perf-render-1258.js +node test-preflight-xss-gate.js +node test-pull-to-reconnect-1091.js +node test-pull-to-reconnect.js +node test-repeater-metric-scatter.js +node test-rx-coverage-config-race.js +node test-rx-coverage-escape.js +node test-scope-audit-styles-linked.js +node test-slideover-1056-rowsel-strict.js +node test-top-routes-overlay.js +node test-traces.js +node test-url-state.js +node test-warmup-banner.js +node test-xss-escape-sinks.js -echo "" -echo "═══════════════════════════════════════" -echo " All tests passed" -echo "═══════════════════════════════════════" +echo "All standalone frontend suites passed" diff --git a/test-channel-colors.js b/test-channel-colors.js index 06c0e249d..a3e2426b8 100644 --- a/test-channel-colors.js +++ b/test-channel-colors.js @@ -146,20 +146,20 @@ test('getRowStyle returns empty string for null channel', function() { assert.strictEqual(ctx.window.ChannelColors.getRowStyle('GRP_TXT', null), ''); }); -test('getRowStyle returns border + background for assigned GRP_TXT channel', function() { +test('getRowStyle returns the channel accent border without replacing the row background', function() { const ctx = makeSandbox(); ctx.window.ChannelColors.set('#sf', '#ef4444'); const style = ctx.window.ChannelColors.getRowStyle('GRP_TXT', '#sf'); - assert.ok(style.includes('border-left:4px solid #ef4444'), 'should have left border'); - assert.ok(style.includes('background:#ef44441a'), 'should have 10% opacity background'); + assert.strictEqual(style, 'border-left:3px solid #ef4444;'); + assert.ok(!style.includes('background'), 'row state/theme retains control of the background'); }); test('getRowStyle works with CHAN type (alias for GRP_TXT)', function() { const ctx = makeSandbox(); ctx.window.ChannelColors.set('#mesh', '#3b82f6'); const style = ctx.window.ChannelColors.getRowStyle('CHAN', '#mesh'); - assert.ok(style.includes('border-left:4px solid #3b82f6'), 'should have left border'); - assert.ok(style.includes('background:#3b82f61a'), 'should have background tint'); + assert.strictEqual(style, 'border-left:3px solid #3b82f6;'); + assert.ok(!style.includes('background'), 'alias also preserves the row background'); }); test('getRowStyle returns empty when channel has no assigned color', function() { diff --git a/test-channel-ux-followup.js b/test-channel-ux-followup.js index 6e2f284a2..c41a24ee3 100644 --- a/test-channel-ux-followup.js +++ b/test-channel-ux-followup.js @@ -43,8 +43,8 @@ assert(!/\$\{ch\.messageCount\s*\|\|\s*0\}\s*packets/.test(chSrc), console.log('\n=== Fix 3: privacy footer wording ==='); assert(!/Clear browser data to remove stored keys/.test(chSrc), 'old "Clear browser data to remove stored keys" copy is gone'); -assert(/Use\s+✕\s+to remove individual channels/.test(chSrc), - 'new copy points at the ✕ button for individual key removal'); +assert(/Use the close button to remove individual channels/.test(chSrc), + 'copy names the close button for individual key removal'); console.log('\n=== Fix 4: Share/reshare affordance on user-added rows ==='); // Source-level: data attribute and helper exist. Behavior-level checks diff --git a/test-channel-ux-round2.js b/test-channel-ux-round2.js index 4d223b6f9..e18db1235 100644 --- a/test-channel-ux-round2.js +++ b/test-channel-ux-round2.js @@ -64,12 +64,12 @@ assert(/channelDisplayName\(ch\)/.test(chSrc), console.log('\n=== Fix 2: share button has recognizable label ==='); assert(!/'⤴'/.test(chSrc) && !/"⤴"/.test(chSrc), 'bare ⤴ glyph no longer used as the share button content'); -// Tighten: assert the literal '📤 Share' string is the glyph argument -// passed into the iconBtn(...) call for ch-share-btn — this catches the +// Assert the Phosphor icon and visible Share text are passed together +// into the iconBtn(...) call for ch-share-btn — this catches the // case where someone removes the icon from the button content but leaves // "Share" in an aria-label or title. -assert(/iconBtn\(\s*'ch-share-btn'[^)]*'📤 Share'/.test(chSrc), - "iconBtn('ch-share-btn', ...) is called with '📤 Share' as the glyph"); +assert(/iconBtn\(\s*'ch-share-btn'[^)]*#ph-share-network[^)]*<\/svg> Share'/.test(chSrc), + "share button content includes the Phosphor share icon and visible Share label"); console.log('\n=== Fix 3: ✕ delete button is a visibly red destructive button ==='); const removeRule = (cssSrc.match(/\.ch-remove-btn\s*\{[^}]*\}/) || [''])[0]; diff --git a/test-clear-filters.js b/test-clear-filters.js index 1eda5be4f..b8c9184d0 100644 --- a/test-clear-filters.js +++ b/test-clear-filters.js @@ -269,11 +269,11 @@ test('updatePacketsUrl shows clear button when time window != default', () => { const DEFAULT_TIME_WINDOW = 15; const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', - 'document', 'history', 'RegionFilter', 'buildPacketsQuery', + 'document', 'history', 'RegionFilter', 'buildPacketsQuery', 'location', updateUrlBody ); fn(filters, savedTimeWindowMin, DEFAULT_TIME_WINDOW, - ctx.document, ctx.history, ctx.RegionFilter, () => ''); + ctx.document, ctx.history, ctx.RegionFilter, () => '', { hash: '#/packets' }); assert.strictEqual(elements['clearFiltersBtn'].style.display, '', 'clear button should be visible when time window != default'); }); @@ -283,10 +283,10 @@ test('updatePacketsUrl hides clear button when all filters default', () => { const filters = {}; const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', - 'document', 'history', 'RegionFilter', 'buildPacketsQuery', + 'document', 'history', 'RegionFilter', 'buildPacketsQuery', 'location', updateUrlBody ); - fn(filters, 15, 15, ctx.document, ctx.history, ctx.RegionFilter, () => ''); + fn(filters, 15, 15, ctx.document, ctx.history, ctx.RegionFilter, () => '', { hash: '#/packets' }); assert.strictEqual(elements['clearFiltersBtn'].style.display, 'none', 'clear button should be hidden'); }); diff --git a/test-customizer-v2.js b/test-customizer-v2.js index a7163039c..954491ec8 100644 --- a/test-customizer-v2.js +++ b/test-customizer-v2.js @@ -162,11 +162,14 @@ test('accepts valid opacity', () => { // ── computeEffective ── console.log('\ncomputeEffective:'); -test('returns server defaults when overrides is {}', () => { +test('preserves all server sections when overrides is {} and supplies missing home defaults', () => { const { api } = loadCustomizer(); const defaults = { theme: { accent: '#aaa', text: '#bbb' }, nodeColors: { repeater: '#ccc' } }; const result = api.computeEffective(defaults, {}); - assert.deepStrictEqual(result, defaults); + const { home, ...serverSections } = result; + assert.deepStrictEqual(serverSections, defaults); + assert.ok(home && typeof home === 'object', 'missing home config gets built-in defaults'); + assert.strictEqual(defaults.home, undefined, 'server input is not mutated'); }); test('overrides a single key in a section', () => { diff --git a/test-drag-manager.js b/test-drag-manager.js index 1bec20ba5..672851293 100644 --- a/test-drag-manager.js +++ b/test-drag-manager.js @@ -41,7 +41,11 @@ function makePanel(id) { }, getAttribute(k) { return attrs[k] || null; }, setAttribute(k, v) { attrs[k] = v; }, - removeAttribute(k) { delete attrs[k]; }, + removeAttribute(k) { + delete attrs[k]; + // DOMStringMap reflects removal of data-* attributes in real browsers. + if (k.startsWith('data-')) delete dataset[k.slice(5).replace(/-([a-z])/g, (_, c) => c.toUpperCase())]; + }, getBoundingClientRect() { return { left: parseFloat(style.left) || 0, diff --git a/test-fluid-scaffolding.js b/test-fluid-scaffolding.js index 3768f63bb..04b2e77ab 100644 --- a/test-fluid-scaffolding.js +++ b/test-fluid-scaffolding.js @@ -9,6 +9,7 @@ const path = require('path'); const assert = require('assert'); const css = fs.readFileSync(path.join(__dirname, 'public/style.css'), 'utf8'); +const rules = css.replace(/\/\*[\s\S]*?\*\//g, ''); let passed = 0, failed = 0; function test(name, fn) { @@ -18,18 +19,19 @@ function test(name, fn) { // --- Helpers --------------------------------------------------------------- -// Extract the :root { ... } block (first occurrence — the light/default one). +// Base tokens can be split across multiple :root blocks. Theme overrides use +// different selectors; combine the base declarations in stylesheet order. function rootBlock() { - const m = css.match(/:root\s*\{([\s\S]*?)\}/); - if (!m) throw new Error(':root block not found in style.css'); - return m[1]; + const blocks = Array.from(rules.matchAll(/^:root\s*\{([^}]*)\}/gm), m => m[1]); + if (!blocks.length) throw new Error(':root block not found in style.css'); + return blocks.join('\n'); } // Find the value of a custom property declared in :root. function rootVar(name) { - const re = new RegExp(`${name}\\s*:\\s*([^;]+);`); - const m = rootBlock().match(re); - return m ? m[1].trim() : null; + const re = new RegExp(`${name}\\s*:\\s*([^;]+);`, 'g'); + const values = Array.from(rootBlock().matchAll(re), m => m[1].trim()); + return values.length ? values[values.length - 1] : null; } function assertClamp(name) { diff --git a/test-hop-resolver-affinity.js b/test-hop-resolver-affinity.js index a9ce2dda7..14dfe51a7 100644 --- a/test-hop-resolver-affinity.js +++ b/test-hop-resolver-affinity.js @@ -24,7 +24,9 @@ function assert(condition, msg) { // Two nodes share the same 1-byte prefix "ab" const nodeA = { public_key: 'ab1111', name: 'NodeA', role: 'repeater', lat: 37.0, lon: -122.0 }; const nodeB = { public_key: 'ab2222', name: 'NodeB', role: 'repeater', lat: 38.0, lon: -123.0 }; -const nodeC = { public_key: 'cd3333', name: 'NodeC', role: 'repeater', lat: 37.5, lon: -122.5 }; +// Keep the anchor distinctly closer to A. Equal degree offsets are not equal +// great-circle distances: longitude converges toward the poles. +const nodeC = { public_key: 'cd3333', name: 'NodeC', role: 'repeater', lat: 37.25, lon: -122.25 }; console.log('\n=== HopResolver Affinity Tests ===\n'); @@ -50,7 +52,7 @@ HopResolver.setAffinity({}); // No edges // With anchor at NodeC's position, NodeA is closer to NodeC than NodeB const result2 = HopResolver.resolve(['cd33', 'ab'], nodeC.lat, nodeC.lon, null, null, null); -// NodeA (37, -122) is closer to NodeC (37.5, -122.5) than NodeB (38, -123) +// NodeA is 0.25 degrees away on each axis; NodeB is 0.75 degrees away. assert(result2['ab'].name === 'NodeA', 'Should pick NodeA (geo-closest) — got: ' + result2['ab'].name); // Test 3: setAffinity with null/undefined doesn't crash diff --git a/test-issue-1438-customizer-mcrole.js b/test-issue-1438-customizer-mcrole.js index 6689d8ce8..ff8989b34 100644 --- a/test-issue-1438-customizer-mcrole.js +++ b/test-issue-1438-customizer-mcrole.js @@ -40,6 +40,7 @@ function assert(cond, msg) { const cv2Src = fs.readFileSync(path.join(__dirname, 'public', 'customize-v2.js'), 'utf8'); const rolesSrc = fs.readFileSync(path.join(__dirname, 'public', 'roles.js'), 'utf8'); const presetsSrc = fs.readFileSync(path.join(__dirname, 'public', 'cb-presets.js'), 'utf8'); +const styleSrc = fs.readFileSync(path.join(__dirname, 'public', 'style.css'), 'utf8'); // ─── Extract the nodeColors-processing block from customize-v2.js. ─── function extractBlock(src, anchor) { @@ -163,12 +164,15 @@ console.log('\n=== #1438 FINAL C: server-only key does NOT clobber --mc-role-* ( blockA + '\n'; vm.runInContext(setup, env.sandbox); - // --mc-role-companion must remain the preset's value (no clobber from server). + // applyCSS clears inline preset values for server-only keys. The active + // body selector supplies the color; a vm does not load that stylesheet. const got = env.root.style.getPropertyValue('--mc-role-companion').toLowerCase(); - assert(got !== '#2563eb', - '--mc-role-companion is NOT the server-config legacy #2563eb (got ' + got + ')'); - assert(got === '#648fff', - '--mc-role-companion still reflects the active preset #648FFF (got ' + got + ')'); + assert(got === '' && env.body.style.getPropertyValue('--mc-role-companion') === '', + 'server-only companion leaves no inline role color overriding the preset'); + const presetRule = styleSrc.match(/body\[data-cb-preset="deut"\]\s*\{([^}]*)\}/); + assert(env.body.getAttribute('data-cb-preset') === 'deut' && presetRule && + /--mc-role-companion:\s*#648fff\s*;/i.test(presetRule[1]), + 'the selected body preset supplies companion #648FFF from style.css'); // --node-companion CAN take the server value (legacy compat is fine here). assert(env.root.style.getPropertyValue('--node-companion').toLowerCase() === '#2563eb', diff --git a/test-issue-1446-cb-preset-cascade.js b/test-issue-1446-cb-preset-cascade.js index 033a0b30f..20ec27330 100644 --- a/test-issue-1446-cb-preset-cascade.js +++ b/test-issue-1446-cb-preset-cascade.js @@ -186,10 +186,13 @@ console.log('\n=== #1446 Scenario 5: clear preset → reverts to server config / vm.runInContext(presetsSrc, env.sandbox); vm.runInContext(cv2Src, env.sandbox); env.sandbox.window._customizerV2.init({ nodeColors: { repeater: '#aaaaaa' } }); - // Confirm deut is active first. + // The customizer removes inline role colors when a preset is active; + // its body selector in the real stylesheet supplies the selected color. const repWithPreset = env.root.style.getPropertyValue('--mc-role-repeater').toLowerCase(); - assert(repWithPreset === '#fe6100', - 'precondition: deut active → --mc-role-repeater = #FE6100 (got: ' + JSON.stringify(repWithPreset) + ')'); + const presetRule = styleSrc.match(/body\[data-cb-preset="deut"\]\s*\{([^}]*)\}/); + assert(repWithPreset === '' && env.body.getAttribute('data-cb-preset') === 'deut' && + presetRule && /--mc-role-repeater:\s*#fe6100\s*;/i.test(presetRule[1]), + 'precondition: deut selects the CSS repeater color #FE6100 without a root override'); // Now clear the preset. const hasClear = typeof env.sandbox.window.MeshCorePresets.clearPreset === 'function'; assert(hasClear, 'MeshCorePresets.clearPreset() exists'); diff --git a/test-issue-1470-card-bg-contrast.js b/test-issue-1470-card-bg-contrast.js index 9a1bf14c5..14bc7a104 100644 --- a/test-issue-1470-card-bg-contrast.js +++ b/test-issue-1470-card-bg-contrast.js @@ -47,10 +47,10 @@ test('@media dark block sets --card-bg to var(--surface-2)', () => { test('[data-theme="dark"] block sets --card-bg to var(--surface-2)', () => { const css = read('public/style.css'); - const themeIdx = css.indexOf('[data-theme="dark"]'); - assert.ok(themeIdx !== -1, '[data-theme="dark"] block not found'); - const themeBlock = css.slice(themeIdx, themeIdx + 2000); - const m = themeBlock.match(/--card-bg\s*:\s*([^;]+);/); + const rules = css.replace(/\/\*[\s\S]*?\*\//g, ''); + const blocks = Array.from(rules.matchAll(/\[data-theme="dark"\]\s*\{([^}]*)\}/g), m => m[1]); + assert.ok(blocks.length, '[data-theme="dark"] block not found'); + const m = blocks.join('\n').match(/--card-bg\s*:\s*([^;]+);/); assert.ok(m, '--card-bg not found in [data-theme="dark"] block'); assert.strictEqual(m[1].trim(), 'var(--surface-2)', `Expected var(--surface-2), got "${m[1].trim()}"`); diff --git a/test-issue-1646-compare-polish.js b/test-issue-1646-compare-polish.js index 6490e5b90..d77d44490 100644 --- a/test-issue-1646-compare-polish.js +++ b/test-issue-1646-compare-polish.js @@ -31,7 +31,7 @@ const fs = require('fs'); const path = require('path'); -const CSS = fs.readFileSync(path.join(__dirname, 'public/style.css'), 'utf8'); +const CSS = fs.readFileSync(path.join(__dirname, 'public/style.css'), 'utf8').replace(/\/\*[\s\S]*?\*\//g, ''); const COMPARE_JS = fs.readFileSync(path.join(__dirname, 'public/compare.js'), 'utf8'); // Token-rank used by font-size comparisons. Comments-only mirror of the diff --git a/test-issue-1648-m2-icons-e2e.js b/test-issue-1648-m2-icons-e2e.js index 301b041bf..c31ce2069 100644 --- a/test-issue-1648-m2-icons-e2e.js +++ b/test-issue-1648-m2-icons-e2e.js @@ -75,6 +75,53 @@ async function spriteRefsResolve(page, label) { else pass(`${label}: ${r.count} sprite refs rendered (samples: ${r.refs.join(', ')})`); } +async function checkConfiguredScope(page) { + const key = 'c1'.repeat(32); + const node = { public_key: key, name: 'Configured scope fixture', role: 'repeater' }; + await page.route('**/api/nodes/' + key, route => route.fulfill({ + json: { node, recentAdverts: [] }, + })); + await page.route('**/api/nodes/' + key + '/health', route => route.fulfill({ json: {} })); + const nodeListUrl = /\/api\/nodes(?:\?.*)?$/; + await page.route(nodeListUrl, route => route.fulfill({ + json: { nodes: [{ ...node, lat: 1, lon: 1 }], total: 1 }, + })); + try { + for (const live of [false, true]) { + for (const scope of ['#fixture', '', null]) { + node.configured_scope = scope; + // A fresh document also clears the application's node-detail cache. + await page.goto('about:blank'); + const route = live ? '/live?lat=1&lon=1&zoom=10' : '/nodes/' + key; + await page.goto(`${BASE}/#${route}`, { waitUntil: 'domcontentloaded' }); + if (live) await page.locator('#liveMap .live-node-marker').click(); + await page.locator(live ? '#nodeDetailContent table' : '#node-stats').waitFor({ state: 'visible', timeout: 8000 }); + const row = live + ? page.locator('#nodeDetailContent tr').filter({ hasText: 'Configured scope' }) + : page.locator('#row-configured-scope'); + const label = live ? 'live detail' : 'node detail'; + if (scope === null) { + assert.strictEqual(await row.count(), 0, 'unconfirmed scope must not render a confirmation row'); + pass(label + ' configured scope: unconfirmed evidence stays absent'); + continue; + } + assert.ok(await row.isVisible(), 'confirmed scope row must be visible, including an empty scope'); + const confirmation = row.getByRole('img', { name: 'confirmed', exact: true }); + assert.strictEqual(await confirmation.count(), 1, 'confirmation must have an accessible image role and name'); + assert.ok(await confirmation.isVisible(), 'confirmation icon must be visible'); + assert.strictEqual(await confirmation.locator('use').getAttribute('href'), '/icons/phosphor-sprite.svg#ph-check'); + assert.doesNotMatch(await row.textContent(), EMOJI_RE, 'confirmation must follow the Phosphor icon policy'); + assert.strictEqual((await row.locator('td').nth(1).textContent()).trim(), scope || 'none configured'); + pass(label + ' configured scope: accessible confirmation and value for ' + (scope || 'none configured')); + } + } + } finally { + await page.unroute('**/api/nodes/' + key); + await page.unroute('**/api/nodes/' + key + '/health'); + await page.unroute(nodeListUrl); + } +} + async function main() { const requireChromium = process.env.CHROMIUM_REQUIRE === '1'; let browser; @@ -96,6 +143,8 @@ async function main() { const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 } }); const page = await ctx.newPage(); + await checkConfiguredScope(page); + // (a) /analytics await checkHeading(page, '/analytics', /Mesh Analytics/, '(a) /analytics'); await spriteRefsResolve(page, '(a) /analytics sprite'); diff --git a/test-live-anims.js b/test-live-anims.js index 31b4f4e81..91add6237 100644 --- a/test-live-anims.js +++ b/test-live-anims.js @@ -2,6 +2,7 @@ 'use strict'; const fs = require('fs'); const assert = require('assert'); +const vm = require('vm'); const src = fs.readFileSync('public/live.js', 'utf8'); @@ -117,16 +118,22 @@ test2('drawAnimatedLine renderFades() has null guard', () => { assert.ok(fadeBody.includes('!pathsLayer'), 'renderFades() missing pathsLayer null guard'); }); -test2('pulseNode animatePulse() has null guard', () => { - const pulseStart = src2.indexOf('function animatePulse(now)'); - const pulseBody = src2.substring(pulseStart, pulseStart + 200); - assert.ok(pulseBody.includes('!animLayer'), 'animatePulse() missing animLayer null guard'); +function functionSource(name) { + const start = src2.indexOf(' function ' + name + '('); + assert.ok(start >= 0, name + ' exists'); + const next = src2.indexOf('\n function ', start + 1); + assert.ok(next > start, name + ' source boundary exists'); + return src2.slice(start, next); +} + +test2('pulseNode returns safely after either layer is destroyed', () => { + for (const layers of [{ animLayer: null, nodesLayer: {} }, { animLayer: {}, nodesLayer: null }]) { + assert.doesNotThrow(() => vm.runInNewContext(functionSource('pulseNode') + '\npulseNode("test", [0, 0], "ADVERT");', layers)); + } }); -test2('ghostPulse has null guard', () => { - const ghostStart = src2.indexOf('function ghostPulse(now)'); - const ghostBody = src2.substring(ghostStart, ghostStart + 200); - assert.ok(ghostBody.includes('!animLayer'), 'ghostPulse() missing animLayer null guard'); +test2('shared pulse/ghost canvas callback returns safely after the context is destroyed', () => { + assert.doesNotThrow(() => vm.runInNewContext(functionSource('renderAnimations') + '\nrenderAnimations(0);', { animCtx: null })); }); console.log(`\n${p2} passed, ${f2} failed\n`); diff --git a/test-map-clustering.js b/test-map-clustering.js index 25df326b5..b2e5989f1 100644 --- a/test-map-clustering.js +++ b/test-map-clustering.js @@ -115,10 +115,10 @@ console.log('\n=== map.js: clustering ==='); assert.ok(icon && icon._isDivIcon, 'expected an L.divIcon'); const html = icon.html || ''; assert.ok(/>66<') >= 0, `total count 6 not in html: ${html}`); - // Role pill counts should appear - assert.ok(html.indexOf('>3<') >= 0, `repeater pill (3) not in html: ${html}`); - assert.ok(html.indexOf('>2<') >= 0, `companion pill (2) not in html: ${html}`); - assert.ok(html.indexOf('>1<') >= 0, `room pill (1) not in html: ${html}`); + // Role letters disambiguate counts without relying on color alone. + assert.ok(/class="mc-pill role-repeater"[^>]*>R3]*>C2]*>M1= 0, `expected mc-cluster class, got: ${icon.className}`); }); diff --git a/test-packets.js b/test-packets.js index ee8a91007..589a406ea 100644 --- a/test-packets.js +++ b/test-packets.js @@ -211,7 +211,7 @@ console.log('\n=== packets.js: getDetailPreview ==='); test('getDetailPreview handles CHAN type', () => { const result = api.getDetailPreview({ type: 'CHAN', text: 'hello world', channel: 'general' }); - assert(result.includes('💬')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-chat-circle"')); assert(result.includes('hello world')); assert(result.includes('chan-tag')); assert(result.includes('general')); @@ -229,7 +229,7 @@ console.log('\n=== packets.js: getDetailPreview ==='); type: 'ADVERT', name: 'TestNode', pubKey: 'abc123', flags: { repeater: true } }); - assert(result.includes('📡')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-broadcast"')); assert(result.includes('TestNode')); assert(result.includes('hop-link')); }); @@ -239,7 +239,7 @@ console.log('\n=== packets.js: getDetailPreview ==='); type: 'ADVERT', name: 'RoomNode', pubKey: 'abc', flags: { room: true } }); - assert(result.includes('🏠')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-house-line"')); }); test('getDetailPreview handles ADVERT sensor', () => { @@ -247,7 +247,7 @@ console.log('\n=== packets.js: getDetailPreview ==='); type: 'ADVERT', name: 'Sensor1', pubKey: 'abc', flags: { sensor: true } }); - assert(result.includes('🌡')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-thermometer"')); }); test('getDetailPreview handles ADVERT companion (default)', () => { @@ -255,14 +255,14 @@ console.log('\n=== packets.js: getDetailPreview ==='); type: 'ADVERT', name: 'Comp', pubKey: 'abc', flags: {} }); - assert(result.includes('📻')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-radio"')); }); test('getDetailPreview handles GRP_TXT with channelHash (no_key)', () => { const result = api.getDetailPreview({ type: 'GRP_TXT', channelHash: 0xAB, decryptionStatus: 'no_key' }); - assert(result.includes('🔒')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-lock"')); assert(result.includes('0xAB')); assert(result.includes('no key')); }); @@ -446,7 +446,7 @@ console.log('\n=== packets.js: getDetailPreview ==='); const result = api.getDetailPreview({ type: 'TXT_MSG', srcHash: 'abcdef01', destHash: '12345678' }); - assert(result.includes('✉️')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-envelope"')); assert(result.includes('abcdef01')); assert(result.includes('12345678')); }); @@ -455,14 +455,14 @@ console.log('\n=== packets.js: getDetailPreview ==='); const result = api.getDetailPreview({ type: 'PATH', srcHash: 'aabb', destHash: 'ccdd' }); - assert(result.includes('🔀')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-shuffle"')); }); test('getDetailPreview handles REQ', () => { const result = api.getDetailPreview({ type: 'REQ', srcHash: 'aa', destHash: 'bb' }); - assert(result.includes('🔒')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-lock"')); assert(result.includes('aa')); }); @@ -470,7 +470,7 @@ console.log('\n=== packets.js: getDetailPreview ==='); const result = api.getDetailPreview({ type: 'RESPONSE', srcHash: 'aa', destHash: 'bb' }); - assert(result.includes('🔒')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-lock"')); }); test('getDetailPreview handles ANON_REQ', () => { @@ -493,7 +493,7 @@ console.log('\n=== packets.js: getDetailPreview ==='); test('getDetailPreview handles public_key fallback', () => { const result = api.getDetailPreview({ public_key: 'abcdef1234567890abcdef' }); - assert(result.includes('📡')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-broadcast"')); assert(result.includes('abcdef1234567890')); }); @@ -957,7 +957,7 @@ console.log('\n=== packets.js: buildGroupRowHtml ==='); }; const result = api.buildGroupRowHtml(p); assert(result.includes('group-header')); - assert(result.includes('▶')); // collapsed arrow + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-caret-up"')); // collapsed arrow }); test('buildGroupRowHtml shows observation count badge', () => { @@ -969,7 +969,7 @@ console.log('\n=== packets.js: buildGroupRowHtml ==='); }; const result = api.buildGroupRowHtml(p); assert(result.includes('badge-obs')); - assert(result.includes('👁')); + assert(result.includes('href="/icons/phosphor-sprite.svg#ph-eye"')); assert(result.includes('5')); }); diff --git a/test-panel-corner.js b/test-panel-corner.js index 06d4da8e7..22299b276 100644 --- a/test-panel-corner.js +++ b/test-panel-corner.js @@ -147,6 +147,7 @@ function loadLiveModule(ctx) { ctx.globalThis = ctx; vm.createContext(ctx); + vm.runInContext(fs.readFileSync(path.join(__dirname, 'public', 'payload-labels.js'), 'utf8'), ctx); vm.runInContext(src, ctx, { timeout: 3000 }); return ctx.window._panelCorner; } diff --git a/test-perf-disk-io-1120.js b/test-perf-disk-io-1120.js index b9b692319..35509fda4 100644 --- a/test-perf-disk-io-1120.js +++ b/test-perf-disk-io-1120.js @@ -156,7 +156,7 @@ await test('WAL >100 MB fires ⚠️ flag', async () => { const html = sb.getHtml(); // The warning appears in the WAL Size card; assert proximity by extracting // the WAL Size card's text content. - const walSection = html.match(/150\.0MB[^<]*⚠️/); + const walSection = html.match(/150\.0MB\s*]*>]*#ph-warning"/); assert.ok(walSection, 'expected ⚠️ next to 150MB WAL value, html=' + html.slice(html.indexOf('WAL Size') - 200, html.indexOf('WAL Size') + 200)); }); @@ -169,7 +169,7 @@ await test('WAL <100 MB does NOT fire ⚠️ flag', async () => { const html = sb.getHtml(); const walIdx = html.indexOf('WAL Size'); const slice = html.slice(Math.max(0, walIdx - 200), walIdx); - assert.ok(!/12\.3MB[^<]*⚠️/.test(slice), 'expected NO ⚠️ next to 12.3MB WAL value'); + assert.ok(!/12\.3MB\s*]*>]*#ph-warning"/.test(slice), 'expected NO warning icon next to 12.3MB WAL value'); }); await test('Cache hit <90% fires ⚠️ flag', async () => { @@ -179,7 +179,7 @@ await test('Cache hit <90% fires ⚠️ flag', async () => { await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(/85\.0%[^<]*⚠️/.test(html), 'expected ⚠️ next to 85.0% cache hit value'); + assert.ok(/85\.0%\s*]*>]*#ph-warning"/.test(html), 'expected warning icon next to 85.0% cache hit value'); }); await test('Cache hit ≥90% does NOT fire ⚠️ flag', async () => { @@ -189,7 +189,7 @@ await test('Cache hit ≥90% does NOT fire ⚠️ flag', async () => { await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(!/98\.7%[^<]*⚠️/.test(html), 'expected NO ⚠️ next to 98.7% cache hit value'); + assert.ok(!/98\.7%\s*]*>]*#ph-warning"/.test(html), 'expected NO warning icon next to 98.7% cache hit value'); }); // === #1167 must-fix #7: threshold boundary cases === @@ -203,7 +203,7 @@ await test('WAL exactly 100 MB does NOT fire ⚠️ (boundary, strict >)', async const html = sb.getHtml(); const walIdx = html.indexOf('WAL Size'); const slice = html.slice(Math.max(0, walIdx - 200), walIdx); - assert.ok(!/100\.0MB[^<]*⚠️/.test(slice), 'expected NO ⚠️ at exactly 100 MB WAL (boundary), slice=' + slice); + assert.ok(!/100\.0MB\s*]*>]*#ph-warning"/.test(slice), 'expected NO warning icon at exactly 100 MB WAL (boundary), slice=' + slice); }); await test('WAL infinitesimally over 100 MB DOES fire ⚠️', async () => { @@ -213,7 +213,7 @@ await test('WAL infinitesimally over 100 MB DOES fire ⚠️', async () => { await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(/100\.0MB[^<]*⚠️/.test(html), 'expected ⚠️ next to 100.0MB WAL value (just over threshold)'); + assert.ok(/100\.0MB\s*]*>]*#ph-warning"/.test(html), 'expected warning icon next to 100.0MB WAL value (just over threshold)'); }); await test('Cache hit exactly 90% does NOT fire ⚠️ (boundary, strict <)', async () => { @@ -223,7 +223,7 @@ await test('Cache hit exactly 90% does NOT fire ⚠️ (boundary, strict <)', as await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(!/90\.0%[^<]*⚠️/.test(html), 'expected NO ⚠️ at exactly 90.0% cache hit (boundary)'); + assert.ok(!/90\.0%\s*]*>]*#ph-warning"/.test(html), 'expected NO warning icon at exactly 90.0% cache hit (boundary)'); }); await test('Cache hit infinitesimally below 90% DOES fire ⚠️', async () => { @@ -233,40 +233,35 @@ await test('Cache hit infinitesimally below 90% DOES fire ⚠️', async () => { await sb.pages.perf.init({ set innerHTML(v) {} }); await new Promise(r => setTimeout(r, 100)); const html = sb.getHtml(); - assert.ok(/90\.0%[^<]*⚠️/.test(html), 'expected ⚠️ next to 90.0% cache hit value (just under threshold)'); + assert.ok(/90\.0%\s*]*>]*#ph-warning"/.test(html), 'expected warning icon next to 90.0% cache hit value (just under threshold)'); }); -await test('Backfill anomaly: rate >10× tx-rate WITH baseline tx≥100 fires ⚠️', async () => { - // Two-phase: prime the previous-snapshot cache, then tick again with - // a backfill rate >10× the tx rate AND tx_inserted past the baseline gate. +await test('Backfill anomaly: rate >10× its stable rolling baseline shows a warning', async () => { + // The current detector compares a source with its own rolling baseline. + // Prime a 60-second baseline at 1/s, then jump to 1000/s for the next tick. const sb = loadPerf(); - // Reset any previous cached snapshot - sb.ctx.window._perfWriteSourcesPrev = null; - const t0 = '2026-01-01T00:00:00Z'; - const t1 = '2026-01-01T00:00:01Z'; // 1s later - const phase1 = { sources: { tx_inserted: 100, backfill_path_json: 0 }, sampleAt: t0 }; - const phase2 = { sources: { tx_inserted: 105, backfill_path_json: 1000 }, sampleAt: t1 }; - // First render: no prev → no flags possible - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, phase1); - await sb.pages.perf.init({ set innerHTML(v) {} }); - await new Promise(r => setTimeout(r, 50)); - // Second render: simulate tick with delta. Reuse the same fetch wiring. - stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, phase2); - await sb.pages.perf.init({ set innerHTML(v) {} }); - await new Promise(r => setTimeout(r, 50)); + const samples = [ + { sources: { tx_inserted: 100, backfill_path_json: 0 }, sampleAt: '2026-01-01T00:00:00Z' }, + { sources: { tx_inserted: 400, backfill_path_json: 60 }, sampleAt: '2026-01-01T00:01:00Z' }, + { sources: { tx_inserted: 405, backfill_path_json: 1060 }, sampleAt: '2026-01-01T00:01:01Z' }, + ]; + for (const sample of samples) { + stubFetch(sb, { ...basePerf, goRuntime }, goHealth, ioData, sqliteData, sample); + await sb.pages.perf.init({ set innerHTML(v) {} }); + await new Promise(r => setTimeout(r, 50)); + } const html = sb.getHtml(); - // After phase2: tx_rate = 5/s, backfill_rate = 1000/s → ratio = 200x → ⚠️ const idx = html.indexOf('backfill_path_json'); assert.ok(idx >= 0, 'backfill_path_json row missing'); - const row = html.slice(idx, idx + 400); - assert.ok(row.includes('⚠️'), 'expected ⚠️ on backfill row when rate ratio >10×, row=' + row); + const row = html.slice(idx, html.indexOf('', idx)); + assert.ok(row.includes('1000.00') && row.includes('1.00'), 'renders current and baseline rates'); + assert.ok(row.includes('#ph-warning"'), 'expected warning on backfill row when rate ratio >10×, row=' + row); }); -await test('Backfill anomaly: tx_inserted <100 baseline guard SUPPRESSES ⚠️', async () => { - // Same shape but tx_inserted stays well below the 100 floor; even a huge - // backfill rate ratio must NOT fire while we lack a meaningful baseline. +await test('Backfill anomaly: insufficient history suppresses the warning', async () => { + // Two snapshots cannot provide the required stable historical baseline, + // even if the current backfill rate is much larger than the tx rate. const sb = loadPerf(); - sb.ctx.window._perfWriteSourcesPrev = null; const t0 = '2026-01-01T00:00:00Z'; const t1 = '2026-01-01T00:00:01Z'; const phase1 = { sources: { tx_inserted: 5, backfill_path_json: 0 }, sampleAt: t0 }; @@ -279,8 +274,9 @@ await test('Backfill anomaly: tx_inserted <100 baseline guard SUPPRESSES ⚠️' await new Promise(r => setTimeout(r, 50)); const html = sb.getHtml(); const idx = html.indexOf('backfill_path_json'); - const row = html.slice(idx, idx + 400); - assert.ok(!row.includes('⚠️'), 'expected NO ⚠️ on backfill row when tx_inserted<100, row=' + row); + assert.ok(idx >= 0, 'backfill_path_json row missing'); + const row = html.slice(idx, html.indexOf('', idx)); + assert.ok(!row.includes('#ph-warning"'), 'expected NO warning without a stable baseline, row=' + row); }); console.log(`\n${passed} passed, ${failed} failed\n`); diff --git a/test-preflight-xss-gate.js b/test-preflight-xss-gate.js index 0897d74ed..0d7389163 100644 --- a/test-preflight-xss-gate.js +++ b/test-preflight-xss-gate.js @@ -72,7 +72,9 @@ for (const c of cases) { failed++; continue; } - const tests = (c.tests || []).map(t => path.join(FIXTURE_DIR, t)); + // The shell protocol uses ':' separators. Relative paths avoid Windows + // drive-letter colons being mistaken for separators by its Python reader. + const tests = (c.tests || []).map(t => path.relative(__dirname, path.join(FIXTURE_DIR, t))); const env = Object.assign({}, process.env); if (tests.length > 0) { env.PREFLIGHT_TEST_FILES = tests.join(':'); @@ -80,6 +82,7 @@ for (const c of cases) { delete env.PREFLIGHT_TEST_FILES; } const res = spawnSync('bash', [SCRIPT, '--file', target], { + cwd: __dirname, env, encoding: 'utf8', }); diff --git a/test-rx-coverage-escape.js b/test-rx-coverage-escape.js index 3556546c3..746d002f5 100644 --- a/test-rx-coverage-escape.js +++ b/test-rx-coverage-escape.js @@ -17,12 +17,12 @@ const src = fs.readFileSync(path.join(__dirname, 'public', 'rx-coverage.js'), 'u // Slice from `var nm = o.name ...` through the end of the returned row string. const startMarker = 'var nm = o.name ? escapeHtml(o.name)'; -const endMarker = "o.nodes + '';"; +const endMarker = "}).join('');"; const startIdx = src.indexOf(startMarker); assert.ok(startIdx >= 0, 'could not locate row-builder start in rx-coverage.js'); const endIdx = src.indexOf(endMarker, startIdx); assert.ok(endIdx >= 0, 'could not locate row-builder end in rx-coverage.js'); -const block = src.slice(startIdx, endIdx + endMarker.length); +const block = src.slice(startIdx, endIdx); // Canonical escapeHtml (public/app.js). function escapeHtml(s) { diff --git a/test-test-inventory.js b/test-test-inventory.js new file mode 100644 index 000000000..81ff5f0f7 --- /dev/null +++ b/test-test-inventory.js @@ -0,0 +1,49 @@ +/* Every root test must have one explicit home; adding a file cannot skip CI silently. */ +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); + +function checkInventory(files, groups) { + const assigned = new Set(); + const available = new Set(files); + for (const [group, spec] of Object.entries(groups)) { + assert.ok(spec.reason && spec.reason.trim(), group + ' needs a prerequisite/reason'); + for (const file of spec.files) { + assert.ok(available.has(file), group + ' lists a missing test: ' + file); + assert.ok(!assigned.has(file), 'test is listed more than once: ' + file); + assigned.add(file); + } + } + const missing = files.filter(file => !assigned.has(file)); + assert.deepStrictEqual(missing, [], 'unclassified root tests: ' + missing.join(', ')); +} + +// Exercise the guard itself: omissions, stale paths, duplicate classifications, +// and undocumented exclusions must fail, even when the rest of the list is valid. +const unit = files => ({ reason: 'Standalone Node checks', files }); +assert.doesNotThrow(() => checkInventory(['test-a.js'], { unit: unit(['test-a.js']) })); +assert.throws(() => checkInventory(['test-a.js', 'test-new.js'], { unit: unit(['test-a.js']) }), /unclassified root tests/); +assert.throws(() => checkInventory(['test-a.js'], { unit: unit(['test-old.js']) }), /missing test/); +assert.throws(() => checkInventory(['test-a.js'], { unit: unit(['test-a.js', 'test-a.js']) }), /more than once/); +assert.throws(() => checkInventory(['test-a.js'], { unit: unit(['test-a.js']), browser: unit(['test-a.js']) }), /more than once/); +assert.throws(() => checkInventory(['test-a.js'], { browser: { reason: '', files: ['test-a.js'] } }), /prerequisite\/reason/); + +const runner = fs.readFileSync(path.join(__dirname, 'test-all.sh'), 'utf8'); +const unitTests = Array.from(runner.matchAll(/^node (test-[^\s]+\.js)\s*$/gm), match => match[1]); +const groups = JSON.parse(fs.readFileSync(path.join(__dirname, 'scripts', 'non-unit-tests.json'), 'utf8')); +assert.ok(!Object.prototype.hasOwnProperty.call(groups, 'unit'), 'unit tests belong only in test-all.sh'); +const files = fs.readdirSync(__dirname).filter(file => /^test-.*\.js$/.test(file)).sort(); +checkInventory(files, { unit: unit(unitTests), ...groups }); + +const pkg = JSON.parse(fs.readFileSync(path.join(__dirname, 'package.json'), 'utf8')); +assert.strictEqual(pkg.scripts['test:unit'], 'sh test-all.sh', 'npm test:unit must use the authoritative runner'); +assert.ok(pkg.scripts.test.includes('sh test-all.sh'), 'npm test must use the authoritative runner'); +assert.ok(pkg.scripts['test:coverage'].includes('sh test-all.sh'), 'coverage must use the authoritative runner'); +const workflow = fs.readFileSync(path.join(__dirname, '.github', 'workflows', 'deploy.yml'), 'utf8'); +const unitStep = workflow.match(/- name: Run JS unit tests[^\n]*\n([\s\S]*?)(?=\n - name:)/); +assert.ok(unitStep, 'CI must retain its JS unit-test step'); +assert.match(unitStep[1], /\bsh test-all\.sh\b/, 'CI must use the authoritative runner'); +assert.doesNotMatch(unitStep[1], /\bnode test-/, 'CI must not keep a second unit-test list'); +console.log('Test inventory: ' + files.length + ' root suites classified; ' + unitTests.length + ' run by local and CI unit checks.'); From 5efe61eef23383e7a92fc48b2904ab39f5480b9a Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 19:57:37 +0200 Subject: [PATCH 030/151] fix(ingestor): set an explicit MQTT ClientID per source (#2013) (#2016) ## What `buildMQTTOpts` (`cmd/ingestor/main.go:591`) never called `SetClientID`, so with paho.mqtt.golang v1.5.0 every ingestor connected with a zero-length ClientID and `CleanSession=true`. The session identity then depended on the broker. This PR: - adds an optional `clientId` per `mqttSources` entry (`cmd/ingestor/config.go:29`) - when unset, uses `corescope--<6 hex chars>` (`cmd/ingestor/main.go:653`). The name is reduced to `[0-9A-Za-z-]`, with the broker host as fallback when the name is empty. The suffix comes from `crypto/rand` and changes on every ingestor start. - sets the ID once per source in `buildMQTTOpts` (`cmd/ingestor/main.go:616`). paho copies the options into the client and reuses them for every reconnect, and the watchdog force-reconnect reuses the same client, so the ID is stable for the life of the process. - logs the ID on connect: `MQTT [tag] connected to as client ` (`cmd/ingestor/main.go:150`) - documents the key in `config.example.json:210` as a `_comment_clientId` entry rather than a value, because `docker/entrypoint.sh:6` copies that file as a live config and a literal value would give every default deployment the same ID. Also listed in `cmd/ingestor/README.md:94`. ## paho behaviour - No client-side length limit. `SetClientID` only stores the value; the 65535 check in `packets/connect.go:156` is in `Validate()`, which the client never calls. - The default ID is longer than the MQTT 3.1 limit of 23 characters for most source names. paho falls back to MQTT 3.1 after any refused CONNACK when no protocol version is set (`client.go:412`), so on a broker that refuses the first 3.1.1 attempt, the retry may hit that limit. I did not cap the length because paho does not require it and the 3.1.1 path accepts it (see below). ## Tests `cmd/ingestor/mqtt_opts_test.go:53-107`: - default ID is non-empty, has the sanitized name prefix, and contains only `[0-9A-Za-z-]` - broker host is used when the name is empty - configured `clientId` is used verbatim - two unconfigured sources with the same name get different IDs - the client built from the options reports the same ID Mutation checks: removing the random bytes fails the "different IDs" test; removing sanitization fails the prefix and character-set tests. `go test ./...` in `cmd/ingestor` passes except `TestWriteStatsAtomic_SymlinkAtDestIsReplaced`, which fails locally on Windows for a symlink privilege reason. `gofmt` and `go vet` are clean. ## Validation against a real broker On a staging instance (build `e84d2da6`) connecting to a Mosquitto bridge: ``` MQTT [lincomatic] connection attempt #1 to tcp://mosquitto-bridge:1883 MQTT [lincomatic] connected to tcp://mosquitto-bridge:1883 as client corescope-lincomatic-71a6eb MQTT [lincomatic] subscribed to meshcore/# ``` The 27-character default was accepted on the first attempt and packets kept arriving afterwards. ## Not verified - Only one broker type (Mosquitto) was tried. - `-race` was not run locally (no cgo toolchain on the test machine). - The case where both the source name and the broker host are empty (ID becomes `corescope-`) has no test. Fixes #2013 ## Review follow-up (commit `a1d6709e`) An independent review found no bug in the ID handling, but the tests covered less than their names said. Changed, tests only (`cmd/ingestor/mqtt_opts_test.go`): - `TestBuildMQTTOpts_ClientIDSurvivesReconnects` replaces the old stability test, which only checked that paho copies the options. Against a loopback fake broker built on paho's `packets` codec, the first CONNECT, paho's auto-reconnect after the broker drops the socket, and the watchdog force-reconnect (`buildForceReconnectFn`) must all carry the same non-empty ID. It runs in about 0.01 s and passed `-count=30 -cpu 1,2,8`. - `TestBuildMQTTOpts_ClientIDDefaultShape` asserts full IDs: `^corescope-local-feed-1-[0-9a-f]{6}$`, `^corescope-mqtt-example-com-[0-9a-f]{6}$` for the broker host fallback (no port), and `^corescope-[0-9a-f]{6}$` with neither a name nor a host. - Mutations now caught: `SetClientID` removed, `u.Host` instead of `u.Hostname()`, a 1-byte suffix, the name guard dropped, sanitization removed. The "as client" log line has no test because it is logged from a closure inside `main()`. Corrections to the description: - **Fallback to MQTT 3.1.** paho falls back after any failed handshake once the socket is open, not only after a refused CONNACK: also a read error or timeout before any CONNACK, or a first packet that is not a CONNACK (`client.go:401-416`, `net.go:83-97`). A failed dial does not trigger it (`client.go:387-391`), and after the first successful connect the protocol version is locked in (`client.go:422-424`). Without this PR the 3.1 retry sent an empty ID, which MQTT 3.1 forbids as well, so nothing gets worse. - **Broker side.** On the EMQX broker we run, authorization has per-username and all-client rules and no client-ID rules (checked through its REST API). Two per-username rules use `${clientid}` in a topic, but both are publish rules and the ingestor only subscribes, so no rule can match it. A broker that caps IDs at 23 characters but accepted the empty ID before would now reject the default ID for source names of 7 or more characters; I have no evidence such a broker is in use. - The "Not verified" item about the empty name and host case no longer applies. --------- Co-authored-by: Claude Opus 5 (1M context) --- cmd/ingestor/README.md | 1 + cmd/ingestor/config.go | 2 + cmd/ingestor/main.go | 40 +++++++- cmd/ingestor/mqtt_opts_test.go | 171 +++++++++++++++++++++++++++++++++ config.example.json | 3 +- 5 files changed, 214 insertions(+), 3 deletions(-) diff --git a/cmd/ingestor/README.md b/cmd/ingestor/README.md index 5e7bce73a..5cf5536f0 100644 --- a/cmd/ingestor/README.md +++ b/cmd/ingestor/README.md @@ -91,6 +91,7 @@ The ingestor reads these fields from the existing `config.json`: - `username` / `password` — auth credentials - `topics` — array of topic patterns to subscribe - `iataFilter` — optional regional filter + - `clientId`: optional MQTT ClientID. Default `corescope--`, new suffix per ingestor start. Two running ingestors must not share a value on one broker - `mqtt` — legacy single-broker config (auto-converted to `mqttSources`) - `dbPath` — SQLite DB path (default: `data/meshcore.db`) diff --git a/cmd/ingestor/config.go b/cmd/ingestor/config.go index 58c4ce7be..18c82f21f 100644 --- a/cmd/ingestor/config.go +++ b/cmd/ingestor/config.go @@ -25,6 +25,8 @@ type MQTTSource struct { IATAFilter []string `json:"iataFilter,omitempty"` ConnectTimeoutSec int `json:"connectTimeoutSec,omitempty"` Region string `json:"region,omitempty"` + // ClientID is the MQTT ClientID; empty means corescope--. + ClientID string `json:"clientId,omitempty"` } // ConnectTimeoutOrDefault returns the per-source connect timeout in seconds, diff --git a/cmd/ingestor/main.go b/cmd/ingestor/main.go index 34248721b..8269b3b9d 100644 --- a/cmd/ingestor/main.go +++ b/cmd/ingestor/main.go @@ -2,6 +2,7 @@ package main import ( "crypto/hmac" + "crypto/rand" "crypto/sha256" "crypto/tls" "encoding/hex" @@ -146,7 +147,7 @@ func main() { status := RegisterSourceStatus(tag, source.Broker) opts.SetOnConnectHandler(func(c mqtt.Client) { - log.Printf("MQTT [%s] connected to %s", tag, source.Broker) + log.Printf("MQTT [%s] connected to %s as client %s", tag, source.Broker, opts.ClientID) status.MarkConnect(time.Now()) // PR #1216 r1 item 2: clear the stale LastMessageUnix from // before the outage so the watchdog doesn't immediately scream @@ -609,7 +610,10 @@ func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions { // (paho default 30s actually — making this explicit so it can't // drift, and so operators reading the code know it's intentional // per the #1335 RCA). - SetKeepAlive(30 * time.Second) + SetKeepAlive(30 * time.Second). + // #2013: an explicit ClientID, fixed here once per source so every + // reconnect of this client presents the same identity. + SetClientID(mqttClientID(source)) opts.SetConnectionAttemptHandler(func(broker *url.URL, tlsCfg *tls.Config) *tls.Config { // Look up the per-source liveness state (registered in main) so we @@ -642,6 +646,38 @@ func buildMQTTOpts(source MQTTSource) *mqtt.ClientOptions { return opts } +// mqttClientID returns the configured clientId, or corescope-- +// with the source name (or broker host) reduced to [0-9A-Za-z-]. The random +// suffix keeps two instances with the same config from taking over each +// other's session on a broker that does not assign unique IDs itself. +func mqttClientID(source MQTTSource) string { + if source.ClientID != "" { + return source.ClientID + } + name := source.Name + if name == "" { + if u, err := url.Parse(source.Broker); err == nil { + name = u.Hostname() + } + } + var b strings.Builder + b.WriteString("corescope-") + for _, r := range name { + if r >= '0' && r <= '9' || r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z' { + b.WriteRune(r) + } else { + b.WriteByte('-') + } + } + if name != "" { + b.WriteByte('-') + } + suffix := make([]byte, 3) + rand.Read(suffix) + b.WriteString(hex.EncodeToString(suffix)) + return b.String() +} + // buildForceReconnectFn builds the watchdog's forced-reconnect action for a // source (#1335, hardened against a race found while investigating a 100+ // minute reconnect failure). diff --git a/cmd/ingestor/mqtt_opts_test.go b/cmd/ingestor/mqtt_opts_test.go index e8aceec45..8241154a2 100644 --- a/cmd/ingestor/mqtt_opts_test.go +++ b/cmd/ingestor/mqtt_opts_test.go @@ -1,8 +1,14 @@ package main import ( + "net" + "regexp" + "sync" "testing" "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/eclipse/paho.mqtt.golang/packets" ) func TestBuildMQTTOpts_ReconnectSettings(t *testing.T) { @@ -45,6 +51,171 @@ func TestBuildMQTTOpts_Credentials(t *testing.T) { } } +// #2013: without SetClientID paho connects with a zero-length ClientID and +// the ingestor's identity depends on what the broker does with that. +func TestBuildMQTTOpts_ClientIDDefaultShape(t *testing.T) { + cases := []struct { + name string + source MQTTSource + want string + }{ + {"sanitized source name", MQTTSource{Broker: "tcp://broker:1883", Name: "local feed/1"}, `^corescope-local-feed-1-[0-9a-f]{6}$`}, + {"broker host without port", MQTTSource{Broker: "ssl://mqtt.example.com:8883"}, `^corescope-mqtt-example-com-[0-9a-f]{6}$`}, + {"no name and no broker host", MQTTSource{Broker: "tcp://:1883"}, `^corescope-[0-9a-f]{6}$`}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := buildMQTTOpts(tc.source).ClientID + if !regexp.MustCompile(tc.want).MatchString(got) { + t.Errorf("ClientID = %q, want match for %s", got, tc.want) + } + }) + } +} + +func TestBuildMQTTOpts_ClientIDUsesConfiguredValue(t *testing.T) { + opts := buildMQTTOpts(MQTTSource{Broker: "tcp://broker:1883", Name: "local", ClientID: "my-ingestor"}) + + if opts.ClientID != "my-ingestor" { + t.Errorf("ClientID = %q, want %q", opts.ClientID, "my-ingestor") + } +} + +func TestBuildMQTTOpts_ClientIDDiffersBetweenSources(t *testing.T) { + a := buildMQTTOpts(MQTTSource{Broker: "tcp://broker:1883", Name: "same"}) + b := buildMQTTOpts(MQTTSource{Broker: "tcp://broker:1883", Name: "same"}) + + if a.ClientID == b.ClientID { + t.Errorf("two unconfigured sources got the same ClientID %q", a.ClientID) + } +} + +// The ID is chosen once in buildMQTTOpts, so every CONNECT the client sends +// must carry it: the first connect, paho's auto-reconnect after the broker +// drops the socket, and the watchdog's force-reconnect on the same client. +func TestBuildMQTTOpts_ClientIDSurvivesReconnects(t *testing.T) { + const wait = 5 * time.Second + broker, connects, drop := fakeMQTTBroker(t) + + opts := buildMQTTOpts(MQTTSource{Broker: broker, Name: "fake broker"}) + up := make(chan struct{}, 8) + opts.SetOnConnectHandler(func(mqtt.Client) { + select { + case up <- struct{}{}: + default: + } + }) + client := mqtt.NewClient(opts) + t.Cleanup(func() { client.Disconnect(250) }) + + var ids []string + awaitConnect := func(step string) { + t.Helper() + select { + case id := <-connects: + ids = append(ids, id) + case <-time.After(wait): + t.Fatalf("%s: no CONNECT within %v (seen %q)", step, wait, ids) + } + select { + case <-up: + case <-time.After(wait): + t.Fatalf("%s: client not connected within %v", step, wait) + } + } + + if tok := client.Connect(); !tok.WaitTimeout(wait) || tok.Error() != nil { + t.Fatalf("connect: timed out or failed: %v", tok.Error()) + } + awaitConnect("first connect") + + drop() + awaitConnect("auto-reconnect") + + buildForceReconnectFn(client, "fake broker")() + awaitConnect("watchdog force-reconnect") + + for i, id := range ids { + if id == "" { + t.Errorf("CONNECT %d carried an empty ClientID (all: %q)", i+1, ids) + } else if id != opts.ClientID { + t.Errorf("CONNECT %d ClientID = %q, want %q (all: %q)", i+1, id, opts.ClientID, ids) + } + } +} + +// fakeMQTTBroker accepts connections on a loopback port, answers every +// CONNECT with an accepted CONNACK and reports its ClientID on connects. drop +// closes every connection accepted so far, as a broker restart would. +func fakeMQTTBroker(t *testing.T) (broker string, connects <-chan string, drop func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ids := make(chan string, 8) + done := make(chan struct{}) + var mu sync.Mutex + var conns []net.Conn + closed := false + + drop = func() { + mu.Lock() + defer mu.Unlock() + for _, c := range conns { + c.Close() + } + conns = nil + } + t.Cleanup(func() { + close(done) + ln.Close() + mu.Lock() + closed = true + mu.Unlock() + drop() + }) + + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + if closed { + mu.Unlock() + c.Close() + return + } + conns = append(conns, c) + mu.Unlock() + go func() { + for { + cp, err := packets.ReadPacket(c) + if err != nil { + return + } + p, ok := cp.(*packets.ConnectPacket) + if !ok { + continue + } + select { + case ids <- p.ClientIdentifier: + case <-done: + return + } + ack := packets.NewControlPacket(packets.Connack).(*packets.ConnackPacket) + if ack.Write(c) != nil { + return + } + } + }() + } + }() + return "tcp://" + ln.Addr().String(), ids, drop +} + func TestBuildMQTTOpts_TLS_InsecureSkipVerify(t *testing.T) { f := false source := MQTTSource{ diff --git a/config.example.json b/config.example.json index f539f006f..3d713ee1c 100644 --- a/config.example.json +++ b/config.example.json @@ -206,7 +206,8 @@ "MRY" ], "region": "SJC", - "connectTimeoutSec": 45 + "connectTimeoutSec": 45, + "_comment_clientId": "Optional \"clientId\": the MQTT ClientID this source connects with. Unset (the default) means corescope--, with the name reduced to letters, digits and '-' (broker host when name is empty) and a new random suffix on every ingestor start. Set it for a fixed identity in broker logs and ACLs, but never give two running ingestors the same value on one broker: they take over each other's session." }, { "_comment": "WebSocket MQTT broker (e.g. meshcore-mqtt-broker). Use ws:// for plain WebSocket or wss:// for TLS. Username/password supported.", From 3ca176a67623ae556f69aece66c09ce903c69e35 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 19:57:55 +0200 Subject: [PATCH 031/151] fix(packets): decode CONTROL discover fields for humans (#1868) (#2017) ## What CONTROL `DISCOVER_REQ` / `DISCOVER_RESP` packets on the Packets page now show their fields in a readable form: - Node type and the `DISCOVER_REQ` type filter bitmask render as Companion / Repeater / Room Server / Sensor instead of a number or hex. - SNR renders in dB (wire value / 4, sign kept: `-11` shows `-2.75 dB`) instead of the raw wire byte. - The responder pubkey renders as the node name (detail header, and the row preview once the node index is loaded) or a link to the node (detail field table) when the node is known, and as the first 8 hex chars when it is not. The full key no longer appears in the row. - The detail field table has Subtype / Type Filter / Tag / Since and Node Type / SNR / Tag / Public Key rows with byte offsets, instead of one generic `Raw` row. Unknown sub-types keep a `Raw` row. ## Credit Port of @dborup's fix in dborup/CoreScope@ad1680ca (merged on their fork as 601f1e4), adapted to current master. dborup is co-author on the commit. Differences from that commit: - Name lookup uses the existing bulk node index (`HopResolver`) instead of `/api/nodes/{pubkey}` on every opened packet, per AGENTS.md rule 10. New `HopResolver.nodeForKey` (`public/hop-resolver.js:357`) does an O(1) lookup by full key or 8-byte prefix; an ambiguous prefix returns null. - `renderDetail` loads that index for CONTROL packets with a pubkey (`public/packets.js:3293`), because zero-hop packets have no path to trigger it. - The ADVERT app-flags row and CONTROL share one type label map (`advTypeLabel`, `public/packets.js:2992`). ## Firmware references (meshcore-dev/MeshCore @ 0679dbef) - `docs/payloads.md:259-282`: field layout; DISCOVER_RESP `snr` is "signed, SNR*4" - `examples/simple_repeater/MyMesh.cpp:798-799`: sub-type values `0x80` / `0x90` - `examples/simple_repeater/MyMesh.cpp:817`: `filter & (1 << ADV_TYPE_REPEATER)` - `examples/simple_repeater/MyMesh.cpp:820-821`: node type in low nibble, `data[1] = packet->_snr` - `src/Dispatcher.cpp:206`: `_snr = getLastSNR() * 4.0f` - `src/Packet.h:51,92`: `int8_t _snr`, `getSNR()` returns `_snr / 4.0f` - `src/helpers/AdvertDataHelpers.h:7-11`: `ADV_TYPE_*` values `cmd/ingestor/decoder.go:750-803` (`decodeControl`) already emits `ctrlSNR` as `int(int8(buf[1]))` and `ctrlPubKey` as 64 or 16 hex chars, so this is display-only. No backend change. ## Performance Row preview adds one object lookup per CONTROL row (O(1)). The 8-byte prefix index adds one entry per node at `HopResolver.init`. No new API calls. ## Tests - New `test-issue-1868-control-decode.js` (14 cases: type names, filter bitmask, SNR 16 and -21, known full key, known 8-byte prefix, unknown key, escaping of node names). Registered in `test-all.sh` and the unit step of `.github/workflows/deploy.yml`. - Written first: 14/14 failed before the change, 14/14 pass after. - Mutation checks: removing `/ 4` fails 3 cases; treating the byte as unsigned fails 2; disabling the prefix index fails 1. - `test-packets.js`: CONTROL DISCOVER_RESP assertion updated from full pubkey to 8-char prefix. Its other 13 failures are unchanged from master. ## Browser validation On a staging instance with live DISCOVER traffic (build `e84d2da6`), in Chrome, no console errors: - `4d67731357dac2ce`, raw payload `92 39 80 6C B8 41 CC 14 67 5D ...`: the field table shows Subtype `DISCOVER_RESP` (flags `0x92`), Node Type `Repeater`, SNR `14.25 dB` (wire `57 / 4`), Tag `0x41B86C80`, Public Key linked as `BE-JBE-Permeke`. - `99c6121e3fca3782`: SNR `-2.75 dB` (wire `-11 / 4`), Public Key `BE-RIN-SPECTRUM-ESP-01`, link target `#/nodes/6b8c65b7...`, which matches that node's key in `/api/nodes`. ## Not verified - The row preview shows the 8-char prefix until the node index has loaded, which a page listing only zero-hop packets does not trigger. Seen on staging: the row kept `pubkey=cc14675d` while the detail panel showed the name. - `DISCOVER_REQ` packets were not opened in the browser; only the unit test covers them. - eslint and the full `test-all.sh` were not run locally. Fixes #1868 ## Review follow-up (commit `ee1a3887`) An independent review re-derived every DISCOVER field and offset from the firmware and found them correct, but found untested offsets and three small display gaps. Changed: - **Bytes past the last decoded field are kept.** The CONTROL field table now adds a Raw row at the real offset for any payload bytes after the last decoded field (`public/packets.js:3802`, `:3844-3846`). Before, a truncated DISCOVER_REQ such as `80 04 12`, or a DISCOVER_RESP with a partial or oversized key, dropped those bytes, while master showed them in a generic Raw row. UNKNOWN subtypes use the same path now. - **Unknown filter bits are visible.** Type filter bits outside ADV_TYPE 1..4 (ADV_TYPE_NONE and the reserved 5..15 range, `src/helpers/AdvertDataHelpers.h:7-12`) are shown as hex next to the names: `filter=Repeater+0x20` in the row, `Requesting: Repeater +0x21` in the detail. - **prefix_only and since=0.** The DISCOVER_REQ Subtype row decodes `prefix_only` (flags bit 0, `docs/payloads.md:270`, `examples/simple_repeater/MyMesh.cpp:818`), and `since=0` renders as "0 (no filter)" in the detail (`MyMesh.cpp:811-817`). - **Unknown responder key in full.** A key with no known node now appears in full (64 or 16 hex) in the field table; the row preview keeps the 8-character prefix #1868 asked for. - **Tests: 14 to 35.** They check the Offset column of every REQ and RESP row, the since, prefix_only and key-length labels, the UNKNOWN and truncated or oversized payload cases, an ambiguous 8-byte prefix in `HopResolver.nodeForKey`, and that `renderDetail` loads the node index for a zero-hop DISCOVER_RESP. Every mutation listed in the review (offsets, Since row, Raw row, labels, resolver loading, ambiguous prefix) now fails the suite, and so do 7 more on the new code. Firmware references re-checked at `0679dbef`. Not verified: how the wrapped 64-hex key looks in the detail pane. --------- Co-authored-by: dborup <3627142+dborup@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- public/hop-resolver.js | 16 +- public/packets.js | 104 +++++++- test-all.sh | 1 + test-issue-1868-control-decode.js | 411 ++++++++++++++++++++++++++++++ test-packets.js | 4 +- 5 files changed, 528 insertions(+), 8 deletions(-) create mode 100644 test-issue-1868-control-decode.js diff --git a/public/hop-resolver.js b/public/hop-resolver.js index be2437d88..0124a2b6d 100644 --- a/public/hop-resolver.js +++ b/public/hop-resolver.js @@ -17,6 +17,7 @@ window.HopResolver = (function() { } let prefixIdx = {}; // lowercase hex prefix → [node, ...] let pubkeyIdx = {}; // full lowercase pubkey → node (O(1) lookup) + let pubkeyPrefix8Idx = {}; // lowercase 8-byte (16 hex) pubkey prefix → node, null when ambiguous let nodesList = []; let observerIataMap = {}; // observer_id → iata let iataCoords = {}; // iata → {lat, lon} @@ -45,12 +46,15 @@ window.HopResolver = (function() { nodesList = nodes || []; prefixIdx = {}; pubkeyIdx = {}; + pubkeyPrefix8Idx = {}; for (const n of nodesList) { if (!n.public_key) continue; const pk = n.public_key.toLowerCase(); // pubkeyIdx includes ALL nodes — used by resolveFromServer for // server-confirmed full-pubkey lookups (any node type). pubkeyIdx[pk] = n; + const p8 = pk.slice(0, 16); + pubkeyPrefix8Idx[p8] = (p8 in pubkeyPrefix8Idx) ? null : n; // prefixIdx only includes nodes that can appear as path hops. if (!canAppearInPath(n.role)) continue; for (let len = 1; len <= 3; len++) { @@ -347,6 +351,16 @@ window.HopResolver = (function() { return n && n.name ? n.name : null; } + // #1868: O(1) node lookup by FULL pubkey (64 hex) or by the 8-byte prefix + // (16 hex) a CONTROL DISCOVER_RESP carries when the request set prefix_only. + // Returns the node, or null when unknown or when the prefix is ambiguous. + function nodeForKey(key) { + if (!key) return null; + const k = String(key).toLowerCase(); + if (k.length === 16) return pubkeyPrefix8Idx[k] || null; + return pubkeyIdx[k] || null; + } + /** * Resolve hops using server-provided resolved_path (full pubkeys). * Returns the same format as resolve() — { [hop]: { name, pubkey, ... } }. @@ -372,5 +386,5 @@ window.HopResolver = (function() { return result; } - return { init: init, resolve: resolve, resolveFromServer: resolveFromServer, ready: ready, haversineKm: haversineKm, setAffinity: setAffinity, getAffinity: getAffinity, nameForKey: nameForKey }; + return { init: init, resolve: resolve, resolveFromServer: resolveFromServer, ready: ready, haversineKm: haversineKm, setAffinity: setAffinity, getAffinity: getAffinity, nameForKey: nameForKey, nodeForKey: nodeForKey }; })(); diff --git a/public/packets.js b/public/packets.js index aff30071c..19200572c 100644 --- a/public/packets.js +++ b/public/packets.js @@ -2987,6 +2987,34 @@ if (scrollContainer) scrollContainer.scrollTop = savedScrollTop; } + // ADV_TYPE_* values, firmware src/helpers/AdvertDataHelpers.h:7-11. Shared by + // the ADVERT app-flags row and CONTROL DISCOVER node type / filter (#1868). + const ADV_TYPE_LABELS = {1:'Companion',2:'Repeater',3:'Room Server',4:'Sensor'}; + function advTypeLabel(t) { + return ADV_TYPE_LABELS[t] || ('Unknown(' + t + ')'); + } + // DISCOVER_REQ type_filter holds one bit per ADV_TYPE_* (firmware + // examples/simple_repeater/MyMesh.cpp:817 tests filter & (1 << ADV_TYPE_REPEATER)). + function ctrlFilterLabels(filter) { + return Object.keys(ADV_TYPE_LABELS).filter(t => filter & (1 << t)).map(t => ADV_TYPE_LABELS[t]); + } + // Filter bits with no label: ADV_TYPE_NONE (bit 0) and the FUTURE 5..15 range + // (AdvertDataHelpers.h:7,12). Returned as '0x..' so they are not dropped. + function ctrlFilterUnknownHex(filter) { + const extra = filter & 0xFF & ~0x1E; + return extra ? '0x' + extra.toString(16).padStart(2, '0') : ''; + } + // DISCOVER_RESP snr byte is int8 SNR*4 (firmware docs/payloads.md:280, + // examples/simple_repeater/MyMesh.cpp:821, src/Packet.h:92 getSNR() = _snr / 4.0f). + function ctrlSnrDb(raw) { + return (Number(raw) / 4).toFixed(2); + } + // DISCOVER_RESP pubkey is 32 bytes or an 8-byte prefix. Resolve via the + // HopResolver node index (bulk /api/nodes, no per-packet request). + function ctrlPubKeyNode(key) { + return (key && window.HopResolver && HopResolver.nodeForKey) ? HopResolver.nodeForKey(key) : null; + } + function getDetailPreview(decoded) { if (!decoded) return ''; // Channel messages (GRP_TXT) — show channel name and message text @@ -3071,7 +3099,11 @@ const parts = []; if (subtype === 'DISCOVER_REQ') { if (decoded.ctrlFilter != null) { - parts.push(`filter=0x${Number(decoded.ctrlFilter).toString(16).padStart(2, '0')}`); + const filter = Number(decoded.ctrlFilter); + const labels = ctrlFilterLabels(filter); + const unknownBits = ctrlFilterUnknownHex(filter); + if (labels.length && unknownBits) labels.push(unknownBits); + parts.push(`filter=${labels.length ? labels.join('+') : '0x' + filter.toString(16).padStart(2, '0')}`); } if (decoded.ctrlTag != null) { parts.push(`tag=0x${(Number(decoded.ctrlTag) >>> 0).toString(16).padStart(8, '0')}`); @@ -3081,16 +3113,17 @@ } } else if (subtype === 'DISCOVER_RESP') { if (decoded.ctrlNodeType != null) { - parts.push(`type=${Number(decoded.ctrlNodeType)}`); + parts.push(`type=${advTypeLabel(Number(decoded.ctrlNodeType))}`); } if (decoded.ctrlSNR != null) { - parts.push(`snr=${Number(decoded.ctrlSNR)}`); + parts.push(`snr=${ctrlSnrDb(decoded.ctrlSNR)}dB`); } if (decoded.ctrlTag != null) { parts.push(`tag=0x${(Number(decoded.ctrlTag) >>> 0).toString(16).padStart(8, '0')}`); } if (decoded.ctrlPubKey) { - parts.push(`pubkey=${escapeHtml(decoded.ctrlPubKey)}`); + const node = ctrlPubKeyNode(decoded.ctrlPubKey); + parts.push(`pubkey=${escapeHtml(node && node.name ? node.name : decoded.ctrlPubKey.slice(0, 8))}`); } } else if (decoded.ctrlFlags) { parts.push(`flags=0x${escapeHtml(decoded.ctrlFlags)}`); @@ -3264,6 +3297,10 @@ } catch {} } + // #1868: zero-hop CONTROL has no path to trigger the node index load, but + // the DISCOVER_RESP pubkey row resolves its name from that same index. + if (decoded.type === 'CONTROL' && decoded.ctrlPubKey) await ensureHopResolver(); + // Resolve hops: prefer server-side resolved_path, fall back to client-side HopResolver if (pathHops.length) { try { @@ -3706,8 +3743,7 @@ rows += fieldRow(off + 32, 'Timestamp (4B)', decoded.timestampISO || '', 'Unix: ' + (decoded.timestamp || '')); rows += fieldRow(off + 36, 'Signature (64B)', truncate(decoded.signature || '', 24), ''); if (decoded.flags) { - const _typeLabels = {1:'Companion',2:'Repeater',3:'Room Server',4:'Sensor'}; - const _typeName = _typeLabels[decoded.flags.type] || ('Unknown(' + decoded.flags.type + ')'); + const _typeName = advTypeLabel(decoded.flags.type); const _boolFlags = [decoded.flags.hasLocation && 'location', decoded.flags.hasName && 'name'].filter(Boolean); const _flagDesc = _typeName + (_boolFlags.length ? ' + ' + _boolFlags.join(', ') : ''); rows += fieldRow(off + 100, 'App Flags', '0x' + (decoded.flags.raw?.toString(16).padStart(2,'0') || '??'), _flagDesc); @@ -3753,6 +3789,61 @@ rows += fieldRow(off + 1, 'Src Public Key (32B)', anonKeyCell, anonName ? '' : 'sender pubkey (unresolved)'); rows += fieldRow(off + 33, 'MAC (2B)', decoded.mac || '', ''); rows += fieldRow(off + 35, 'Encrypted Data', truncate(decoded.encryptedData || '', 30), ''); + } else if (decoded.type === 'CONTROL') { + // #1868: layout per firmware docs/payloads.md:259-282, decoded by + // cmd/ingestor/decoder.go decodeControl(). Body fields are length-gated + // there, so each row is only added when the field is present. + const subtype = decoded.ctrlSubtype || 'CONTROL'; + let subtypeDesc = decoded.ctrlFlags ? 'flags=0x' + escapeHtml(decoded.ctrlFlags) + ', sub_type in upper 4 bits' : ''; + // prefix_only is flags bit 0 (docs/payloads.md:270, MyMesh.cpp:818): responders send an 8-byte key prefix. + if (subtype === 'DISCOVER_REQ' && decoded.ctrlFlags) subtypeDesc += ', prefix_only=' + (parseInt(decoded.ctrlFlags, 16) & 1); + rows += fieldRow(off, 'Subtype', escapeHtml(subtype), subtypeDesc); + // Payload bytes covered by the rows below; anything past it gets a Raw row. + let ctrlEnd = off + 1; + if (subtype === 'DISCOVER_REQ') { + if (decoded.ctrlFilter != null) { + const filter = Number(decoded.ctrlFilter); + const labels = ctrlFilterLabels(filter); + const unknownBits = ctrlFilterUnknownHex(filter); + rows += fieldRow(off + 1, 'Type Filter (1B)', '0x' + filter.toString(16).padStart(2, '0'), (labels.length ? 'Requesting: ' + labels.join(', ') : 'No known types requested') + (unknownBits ? ' +' + unknownBits : '')); + ctrlEnd = off + 2; + } + if (decoded.ctrlTag != null) { + rows += fieldRow(off + 2, 'Tag (4B)', '0x' + (Number(decoded.ctrlTag) >>> 0).toString(16).toUpperCase().padStart(8, '0'), ''); + ctrlEnd = off + 6; + } + if (decoded.ctrlSince != null) { + // since=0 is the firmware default (MyMesh.cpp:814) and matches every responder (:817). + const since = Number(decoded.ctrlSince) >>> 0; + rows += fieldRow(off + 6, 'Since (4B)', since === 0 ? '0 (no filter)' : String(since), 'Unix epoch'); + ctrlEnd = off + 10; + } + } else if (subtype === 'DISCOVER_RESP') { + if (decoded.ctrlNodeType != null) { + rows += fieldRow(off, 'Node Type', escapeHtml(advTypeLabel(Number(decoded.ctrlNodeType))), 'lower 4 bits of flags'); + } + if (decoded.ctrlSNR != null) { + rows += fieldRow(off + 1, 'SNR (1B)', ctrlSnrDb(decoded.ctrlSNR) + ' dB', 'request SNR as heard by the responder, wire value ' + Number(decoded.ctrlSNR) + ' / 4'); + ctrlEnd = off + 2; + } + if (decoded.ctrlTag != null) { + rows += fieldRow(off + 2, 'Tag (4B)', '0x' + (Number(decoded.ctrlTag) >>> 0).toString(16).toUpperCase().padStart(8, '0'), ''); + ctrlEnd = off + 6; + } + if (decoded.ctrlPubKey) { + const node = ctrlPubKeyNode(decoded.ctrlPubKey); + const pkLen = decoded.ctrlPubKey.length === 64 ? '32B' : '8B prefix'; + // Unknown key: full hex here (the row preview keeps 8 chars), wrappable. + const pkCell = node + ? `${escapeHtml(node.name || node.public_key.slice(0, 8))}` + : `${escapeHtml(decoded.ctrlPubKey)}`; + rows += fieldRow(off + 6, 'Public Key (' + pkLen + ')', pkCell, node ? '' : 'Unknown node'); + ctrlEnd = off + 6 + Math.floor(decoded.ctrlPubKey.length / 2); + } + } + if (size > ctrlEnd) { + rows += fieldRow(ctrlEnd, 'Raw', escapeHtml(truncate(buf.slice(ctrlEnd * 2), 40)), ''); + } } else if (decoded.destHash !== undefined) { rows += fieldRow(off, 'Dest Hash (1B)', decoded.destHash || '', ''); rows += fieldRow(off + 1, 'Src Hash (1B)', decoded.srcHash || '', ''); @@ -4063,6 +4154,7 @@ renderDecodedPacket, kv, buildFieldTable, + renderDetail, sectionRow, fieldRow, renderTimestampCell, diff --git a/test-all.sh b/test-all.sh index 2d59388b8..1ad6d50d2 100755 --- a/test-all.sh +++ b/test-all.sh @@ -125,6 +125,7 @@ node test-issue-1836-crossnav-case-normalization.js node test-issue-1843-node-qr-quiet-zone.js node test-issue-1846-observers-width.js node test-issue-1849-trace-hashbytes.js +node test-issue-1868-control-decode.js node test-issue-1890-og-url.js node test-issue-1956-release-routing.js node test-issue-2001-map-scope-state.js diff --git a/test-issue-1868-control-decode.js b/test-issue-1868-control-decode.js new file mode 100644 index 000000000..faff2ef6d --- /dev/null +++ b/test-issue-1868-control-decode.js @@ -0,0 +1,411 @@ +/** + * #1868: CONTROL DISCOVER_REQ / DISCOVER_RESP fields rendered for humans. + * + * Firmware (meshcore-dev/MeshCore): + * docs/payloads.md:266-282 DISCOVER_REQ/RESP layout, "snr: signed, SNR*4" + * examples/simple_repeater/MyMesh.cpp:820-821 data[0] = RESP | ADV_TYPE_x; data[1] = packet->_snr + * src/Dispatcher.cpp:206 _snr = getLastSNR() * 4.0f + * src/Packet.h:51,92 int8_t _snr; getSNR() = _snr / 4.0f + * src/helpers/AdvertDataHelpers.h:7-11 ADV_TYPE_* values + * + * The ingestor (cmd/ingestor/decoder.go decodeControl) emits ctrlNodeType, + * ctrlFilter, ctrlSNR (raw int8, SNR*4) and ctrlPubKey (64 or 16 hex chars). + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(' ✅ ' + name); } + catch (e) { failed++; console.log(' ❌ ' + name + ': ' + e.message); } +} + +// Sandbox construction mirrors test-issue-1849-trace-hashbytes.js. +function makeSandbox() { + const registeredPages = {}; + const ctx = { + window: { + addEventListener: () => {}, removeEventListener: () => {}, dispatchEvent: () => {}, + innerWidth: 1200, PacketFilter: null, + }, + document: { + readyState: 'complete', + createElement: () => ({ id: '', textContent: '', innerHTML: '', className: '', style: {}, + appendChild: () => {}, setAttribute: () => {}, addEventListener: () => {}, + querySelectorAll: () => [], querySelector: () => null, + classList: { add: () => {}, remove: () => {}, contains: () => false } }), + head: { appendChild: () => {} }, getElementById: () => null, + addEventListener: () => {}, removeEventListener: () => {}, + querySelectorAll: () => [], querySelector: () => null, body: { appendChild: () => {} }, + }, + console, Date, Infinity, Math, Array, Object, String, Number, JSON, RegExp, + Error, TypeError, RangeError, parseInt, parseFloat, isNaN, isFinite, + encodeURIComponent, decodeURIComponent, + setTimeout: () => {}, clearTimeout: () => {}, setInterval: () => {}, clearInterval: () => {}, + fetch: () => Promise.resolve({ ok: true, json: () => Promise.resolve({}) }), + performance: { now: () => Date.now() }, + localStorage: (() => { const s = {}; return { + getItem: k => s[k] || null, setItem: (k, v) => { s[k] = String(v); }, removeItem: k => { delete s[k]; }, + }; })(), + location: { hash: '' }, history: { replaceState: () => {} }, + CustomEvent: class CustomEvent {}, Map, Set, Promise, URLSearchParams, + addEventListener: () => {}, removeEventListener: () => {}, dispatchEvent: () => {}, + requestAnimationFrame: (cb) => setTimeout(cb, 0), + registerPage: (name, handler) => { registeredPages[name] = handler; }, + }; + vm.createContext(ctx); + return ctx; +} + +function loadInCtx(ctx, file) { + vm.runInContext(fs.readFileSync(file, 'utf8'), ctx, { filename: file }); + for (const k of Object.keys(ctx.window)) { ctx[k] = ctx.window[k]; } +} + +function loadPacketsSandbox() { + const ctx = makeSandbox(); + loadInCtx(ctx, 'public/payload-labels.js'); + loadInCtx(ctx, 'public/roles.js'); + loadInCtx(ctx, 'public/app.js'); + loadInCtx(ctx, 'public/packet-helpers.js'); + loadInCtx(ctx, 'public/hop-resolver.js'); + vm.runInContext(` + window.HopDisplay = { + renderHop: function(h, entry, opts) { return '' + h + ''; }, + _showFromBtn: function() {} + }; + `, ctx); + loadInCtx(ctx, 'public/packets.js'); + return ctx; +} + +const KNOWN_KEY = 'ab12cd34'.repeat(8); +const UNKNOWN_KEY = 'fe98dc76'.repeat(8); +const EVIL_KEY = '0badc0de'.repeat(8); +// Zero-hop direct CONTROL: header 0x2E (payload CONTROL 0x0B << 2 | route DIRECT 2), path_len 0x00. +const CTRL_PKT = { raw_hex: '2e00', route_type: 2, payload_type: 11 }; + +console.log('\n=== #1868: CONTROL DISCOVER fields rendered for humans ==='); +{ + const ctx = loadPacketsSandbox(); + const api = ctx._packetsTestAPI; + assert(api, '_packetsTestAPI must be exposed'); + ctx.HopResolver.init([ + { public_key: KNOWN_KEY, name: 'Known Repeater', role: 'repeater' }, + { public_key: EVIL_KEY, name: '', role: 'repeater' }, + ]); + + // --- type by name --- + test('preview: DISCOVER_RESP node type 2 renders as Repeater, not a number', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlNodeType: 2 }); + assert(html.includes('type=Repeater'), 'got: ' + html); + assert(!/type=2\b/.test(html), 'raw number leaked: ' + html); + }); + + test('preview: DISCOVER_RESP node type 4 renders as Sensor', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlNodeType: 4 }); + assert(html.includes('type=Sensor'), 'got: ' + html); + }); + + test('preview: DISCOVER_REQ filter bitmask renders type names', () => { + // (1 << ADV_TYPE_REPEATER) | (1 << ADV_TYPE_SENSOR) = 0x04 | 0x10 + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFilter: 0x14 }); + assert(html.includes('filter=Repeater+Sensor'), 'got: ' + html); + }); + + test('detail: DISCOVER_RESP node type row shows the name', () => { + const html = api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlFlags: '92', ctrlNodeType: 2 }, [], []); + assert(/' + '' + '
Last Seen${lastSeen}
Adverts${n.advert_count || 0}
Configured scope ✓${n.configured_scope === '' ? 'none configured' : `${escapeHtml(n.configured_scope)}`}
Configured scope ${n.configured_scope === '' ? 'none configured' : `${escapeHtml(n.configured_scope)}`}
Scope${n.default_scope === null ? '—' : n.default_scope === '' ? 'unknown scope' : `${escapeHtml(n.default_scope)}` diff --git a/public/nodes.js b/public/nodes.js index f68b658fd..5699bc956 100644 --- a/public/nodes.js +++ b/public/nodes.js @@ -735,7 +735,7 @@ return `
Bridge score ⓘ${bpct}% ${blabel}
Transported scopes${n.transported_scopes.map(sc => '' + escapeHtml(String(sc)) + '').join('')}
Configured scope ✓${n.configured_scope === '' ? 'none configured' : `${escapeHtml(n.configured_scope)}`}
Configured scope ${n.configured_scope === '' ? 'none configured' : `${escapeHtml(n.configured_scope)}`}
First Seen${renderNodeTimestampHtml(n.first_seen)}
Total Packets${stats.totalTransmissions || stats.totalPackets || n.advert_count || 0}${stats.totalObservations && stats.totalObservations !== (stats.totalTransmissions || stats.totalPackets) ? ' (seen ' + stats.totalObservations + '×)' : ''}
Packets Today${stats.packetsToday || 0}
Node Type<\/td>Repeater<\/td>/.test(html), 'got: ' + html); + assert(!html.includes('>Raw<'), 'CONTROL must not fall through to the Raw row: ' + html); + }); + + test('detail: DISCOVER_REQ filter row keeps hex and names the requested types', () => { + const html = api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80', ctrlFilter: 4, ctrlTag: 0xDEADBEEF }, [], []); + assert(html.includes('0x04'), 'got: ' + html); + assert(html.includes('Repeater'), 'got: ' + html); + assert(html.includes('DEADBEEF'), 'got: ' + html); + }); + + // --- SNR from the wire value (int8, SNR*4) --- + test('preview: SNR wire 16 renders as 4.00dB', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlSNR: 16 }); + assert(html.includes('snr=4.00dB'), 'got: ' + html); + assert(!/snr=16\b/.test(html), 'raw wire value leaked: ' + html); + }); + + test('preview: negative SNR wire -21 renders as -5.25dB', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlSNR: -21 }); + assert(html.includes('snr=-5.25dB'), 'got: ' + html); + }); + + test('detail: negative SNR wire -21 renders as -5.25 dB', () => { + const html = api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlSNR: -21 }, [], []); + assert(html.includes('-5.25 dB'), 'got: ' + html); + }); + + // --- pubkey: known node name link; unknown key 8 chars in preview, full in detail --- + test('preview: known full pubkey renders the node name, not the key', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: KNOWN_KEY }); + assert(html.includes('Known Repeater'), 'got: ' + html); + assert(!html.includes(KNOWN_KEY), 'full key leaked: ' + html); + }); + + test('detail: known full pubkey renders a link to the node', () => { + const html = api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: KNOWN_KEY }, [], []); + assert(html.includes('href="#/nodes/' + KNOWN_KEY + '"'), 'got: ' + html); + assert(html.includes('>Known Repeater'), 'got: ' + html); + }); + + test('detail: known 8-byte prefix (prefix_only) links to the full node key', () => { + const html = api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: KNOWN_KEY.slice(0, 16) }, [], []); + assert(html.includes('href="#/nodes/' + KNOWN_KEY + '"'), 'got: ' + html); + assert(html.includes('>Known Repeater'), 'got: ' + html); + }); + + test('preview: unknown pubkey renders the first 8 hex chars only', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: UNKNOWN_KEY }); + assert(html.includes('pubkey=fe98dc76'), 'got: ' + html); + assert(!html.includes(UNKNOWN_KEY.slice(0, 9)), 'more than 8 chars leaked: ' + html); + }); + + test('detail: unknown pubkey renders the full key, no link', () => { + // The row preview stays short (#1868); the field table is the one place + // the responder key is readable outside the hex dump. + const html = api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: UNKNOWN_KEY }, [], []); + assert(html.includes(UNKNOWN_KEY), 'full key missing: ' + html); + assert(!html.includes('#/nodes/'), 'unknown key must not be linked: ' + html); + }); + + test('node names from the node list are escaped in preview and detail', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: EVIL_KEY }; + const preview = api.getDetailPreview(decoded); + const detail = api.buildFieldTable(CTRL_PKT, decoded, [], []); + assert(!preview.includes('(\d+)<\/td>(.*?)<\/td>(.*?)<\/td>(.*?)<\/td><\/tr>/g; + let m; + while ((m = re.exec(html))) rows.push({ off: Number(m[1]), name: m[2], value: m[3], desc: m[4] }); + return rows; +} +function payloadRows(html) { + return fieldRows(html).filter(r => r.name !== 'Header Byte' && r.name !== 'Path Length'); +} +function rowsSummary(rows) { + return rows.map(r => r.off + ':' + r.name).join(' | '); +} + +console.log('\n=== #1868 review: field offsets, trailing bytes, filter bits ==='); +{ + const ctx = loadPacketsSandbox(); + const api = ctx._packetsTestAPI; + ctx.HopResolver.init([{ public_key: KNOWN_KEY, name: 'Known Repeater', role: 'repeater' }]); + const TAG = 'efbeadde'; + + // --- offsets: zero-hop direct packet, payload starts at byte 2 --- + test('detail: DISCOVER_REQ rows sit at offsets 2/3/4/8', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80', ctrlFilter: 4, ctrlTag: 0xDEADBEEF, ctrlSince: 1700000000 }; + const pkt = { raw_hex: '2e00' + '80' + '04' + TAG + '00f15365', route_type: 2, payload_type: 11 }; + const rows = payloadRows(api.buildFieldTable(pkt, decoded, [], [])); + assert.strictEqual(rowsSummary(rows), '2:Subtype | 3:Type Filter (1B) | 4:Tag (4B) | 8:Since (4B)'); + }); + + test('detail: DISCOVER_RESP rows sit at offsets 2/2/3/4/8', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlFlags: '92', ctrlNodeType: 2, ctrlSNR: 16, ctrlTag: 0xDEADBEEF, ctrlPubKey: UNKNOWN_KEY }; + const pkt = { raw_hex: '2e00' + '92' + '10' + TAG + UNKNOWN_KEY, route_type: 2, payload_type: 11 }; + const rows = payloadRows(api.buildFieldTable(pkt, decoded, [], [])); + assert.strictEqual(rowsSummary(rows), '2:Subtype | 2:Node Type | 3:SNR (1B) | 4:Tag (4B) | 8:Public Key (32B)'); + }); + + test('detail: DISCOVER_REQ since row renders the epoch value', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80', ctrlFilter: 4, ctrlTag: 1, ctrlSince: 1700000000 }; + const rows = fieldRows(api.buildFieldTable(CTRL_PKT, decoded, [], [])); + const since = rows.find(r => r.name === 'Since (4B)'); + assert(since, 'Since row missing: ' + rowsSummary(rows)); + assert.strictEqual(since.value, '1700000000'); + }); + + test('detail: DISCOVER_REQ since=0 renders as no filter', () => { + // Firmware examples/simple_repeater/MyMesh.cpp:814 defaults since to 0, + // and :817 answers when discovery_mod_timestamp >= since, so 0 filters nothing. + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80', ctrlFilter: 4, ctrlTag: 1, ctrlSince: 0 }; + const since = fieldRows(api.buildFieldTable(CTRL_PKT, decoded, [], [])).find(r => r.name === 'Since (4B)'); + assert(since, 'Since row missing'); + assert.strictEqual(since.value, '0 (no filter)'); + }); + + test('detail: DISCOVER_REQ subtype row decodes prefix_only (flags bit 0)', () => { + const on = fieldRows(api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '81', ctrlFilter: 4, ctrlTag: 1 }, [], [])) + .find(r => r.name === 'Subtype'); + const off = fieldRows(api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80', ctrlFilter: 4, ctrlTag: 1 }, [], [])) + .find(r => r.name === 'Subtype'); + assert(on.desc.includes('prefix_only=1'), 'got: ' + on.desc); + assert(off.desc.includes('prefix_only=0'), 'got: ' + off.desc); + }); + + // --- pubkey length label --- + test('detail: 64-hex key is labelled 32B, 16-hex key 8B prefix', () => { + const full = fieldRows(api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: UNKNOWN_KEY }, [], [])); + const pre = fieldRows(api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: UNKNOWN_KEY.slice(0, 16) }, [], [])); + assert(full.some(r => r.name === 'Public Key (32B)'), rowsSummary(full)); + assert(pre.some(r => r.name === 'Public Key (8B prefix)'), rowsSummary(pre)); + }); + + // --- UNKNOWN subtype keeps its body --- + test('detail: UNKNOWN subtype shows a Raw row with the body bytes at offset 3', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'UNKNOWN', ctrlFlags: 'a0' }; + const rows = payloadRows(api.buildFieldTable({ raw_hex: '2e00a00102', route_type: 2, payload_type: 11 }, decoded, [], [])); + const raw = rows.find(r => r.name === 'Raw'); + assert(raw, 'Raw row missing: ' + rowsSummary(rows)); + assert.strictEqual(raw.off, 3); + assert.strictEqual(raw.value, '0102'); + }); + + // --- bytes past the last decoded field (decoder.go decodeControl length gates) --- + test('detail: truncated DISCOVER_REQ (80 04 12) keeps 0412 in a Raw row at offset 3', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80' }; + const rows = payloadRows(api.buildFieldTable({ raw_hex: '2e00800412', route_type: 2, payload_type: 11 }, decoded, [], [])); + const raw = rows.find(r => r.name === 'Raw'); + assert(raw, 'Raw row missing: ' + rowsSummary(rows)); + assert.strictEqual(raw.off, 3); + assert.strictEqual(raw.value, '0412'); + }); + + test('detail: DISCOVER_REQ with 2 bytes after the tag (no since) keeps them at offset 8', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80', ctrlFilter: 4, ctrlTag: 0xDEADBEEF }; + const rows = payloadRows(api.buildFieldTable({ raw_hex: '2e00' + '8004' + TAG + 'aabb', route_type: 2, payload_type: 11 }, decoded, [], [])); + const raw = rows.find(r => r.name === 'Raw'); + assert(raw, 'Raw row missing: ' + rowsSummary(rows)); + assert.strictEqual(raw.off, 8); + assert.strictEqual(raw.value, 'aabb'); + }); + + test('detail: truncated DISCOVER_RESP (5 bytes) keeps its body in a Raw row at offset 3', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlFlags: '92', ctrlNodeType: 2 }; + const rows = payloadRows(api.buildFieldTable({ raw_hex: '2e00' + '92' + '10efbead', route_type: 2, payload_type: 11 }, decoded, [], [])); + const raw = rows.find(r => r.name === 'Raw'); + assert(raw, 'Raw row missing: ' + rowsSummary(rows)); + assert.strictEqual(raw.off, 3); + assert.strictEqual(raw.value, '10efbead'); + }); + + test('detail: DISCOVER_RESP with 10 key bytes shows the 8B prefix and the 2 extra bytes at offset 16', () => { + const keyBytes = UNKNOWN_KEY.slice(0, 20); + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlFlags: '92', ctrlNodeType: 2, ctrlSNR: 16, ctrlTag: 0xDEADBEEF, ctrlPubKey: keyBytes.slice(0, 16) }; + const rows = payloadRows(api.buildFieldTable({ raw_hex: '2e00' + '9210' + TAG + keyBytes, route_type: 2, payload_type: 11 }, decoded, [], [])); + const raw = rows.find(r => r.name === 'Raw'); + assert(raw, 'Raw row missing: ' + rowsSummary(rows)); + assert.strictEqual(raw.off, 16); + assert.strictEqual(raw.value, keyBytes.slice(16)); + }); + + test('detail: DISCOVER_RESP with 3 bytes beyond the 32B key keeps them at offset 40', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlFlags: '92', ctrlNodeType: 2, ctrlSNR: 16, ctrlTag: 0xDEADBEEF, ctrlPubKey: UNKNOWN_KEY }; + const rows = payloadRows(api.buildFieldTable({ raw_hex: '2e00' + '9210' + TAG + UNKNOWN_KEY + '010203', route_type: 2, payload_type: 11 }, decoded, [], [])); + const raw = rows.find(r => r.name === 'Raw'); + assert(raw, 'Raw row missing: ' + rowsSummary(rows)); + assert.strictEqual(raw.off, 40); + assert.strictEqual(raw.value, '010203'); + }); + + test('detail: fully decoded DISCOVER_RESP has no Raw row', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlFlags: '92', ctrlNodeType: 2, ctrlSNR: 16, ctrlTag: 0xDEADBEEF, ctrlPubKey: UNKNOWN_KEY }; + const rows = payloadRows(api.buildFieldTable({ raw_hex: '2e00' + '9210' + TAG + UNKNOWN_KEY, route_type: 2, payload_type: 11 }, decoded, [], [])); + assert(!rows.some(r => r.name === 'Raw'), rowsSummary(rows)); + }); + + // --- filter bits outside ADV_TYPE 1..4 (AdvertDataHelpers.h:7 NONE=0, :12 FUTURE 5..15) --- + test('preview: filter 0x24 shows the unknown bit next to Repeater', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFilter: 0x24 }); + assert(html.includes('filter=Repeater+0x20'), 'got: ' + html); + }); + + test('preview: filter 0x04 has no unknown-bit suffix', () => { + const html = api.getDetailPreview({ type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFilter: 0x04 }); + assert(/filter=Repeater(\s|<)/.test(html), 'got: ' + html); + }); + + test('detail: filter 0x25 names Repeater and keeps unknown bits 0x21', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '80', ctrlFilter: 0x25, ctrlTag: 1 }; + const row = fieldRows(api.buildFieldTable(CTRL_PKT, decoded, [], [])).find(r => r.name === 'Type Filter (1B)'); + assert.strictEqual(row.value, '0x25'); + assert.strictEqual(row.desc, 'Requesting: Repeater +0x21'); + }); + + // --- XSS: flags come from decoded_json --- + test('detail: DISCOVER_REQ with non-hex flags stays escaped', () => { + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_REQ', ctrlFlags: '', ctrlFilter: 4, ctrlTag: 1 }; + const html = api.buildFieldTable(CTRL_PKT, decoded, [], []); + assert(!html.includes(''), 'got: ' + html); + }); +} + +console.log('\n=== #1868 review: HopResolver.nodeForKey ambiguous 8-byte prefix ==='); +{ + const ctx = loadPacketsSandbox(); + const api = ctx._packetsTestAPI; + const A = '1122334455667788' + 'aa'.repeat(24); + const B = '1122334455667788' + 'bb'.repeat(24); + ctx.HopResolver.init([ + { public_key: A, name: 'Node A', role: 'repeater' }, + { public_key: B, name: 'Node B', role: 'repeater' }, + ]); + + test('nodeForKey: shared 8-byte prefix returns null', () => { + assert.strictEqual(ctx.HopResolver.nodeForKey('1122334455667788'), null); + }); + + test('nodeForKey: full keys still resolve when their prefix is ambiguous', () => { + assert.strictEqual(ctx.HopResolver.nodeForKey(A).name, 'Node A'); + assert.strictEqual(ctx.HopResolver.nodeForKey(B.toUpperCase()).name, 'Node B'); + }); + + test('detail: ambiguous 8-byte prefix is not linked to either node', () => { + const html = api.buildFieldTable(CTRL_PKT, { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlPubKey: '1122334455667788' }, [], []); + assert(!html.includes('#/nodes/'), 'got: ' + html); + assert(!html.includes('Node A') && !html.includes('Node B'), 'got: ' + html); + }); +} + +console.log('\n=== #1868 review: renderDetail loads the node index for CONTROL ==='); +(async () => { + const ctx = loadPacketsSandbox(); + const api = ctx._packetsTestAPI; + const nodeFetches = []; + ctx.fetchAllNodes = (q) => { nodeFetches.push(q); return Promise.resolve({ nodes: [{ public_key: KNOWN_KEY, name: 'Known Repeater', role: 'repeater' }] }); }; + ctx.api = (path) => Promise.resolve(path === '/observers' ? { observers: [] } : {}); + const panel = { innerHTML: '', querySelectorAll: () => [], querySelector: () => null, addEventListener: () => {} }; + const decoded = { type: 'CONTROL', ctrlSubtype: 'DISCOVER_RESP', ctrlFlags: '92', ctrlNodeType: 2, ctrlSNR: 16, ctrlTag: 0xDEADBEEF, ctrlPubKey: KNOWN_KEY }; + const data = { + packet: { id: 1, hash: 'h1', raw_hex: '2e00' + '9210efbeadde' + KNOWN_KEY, route_type: 2, payload_type: 11, decoded_json: JSON.stringify(decoded), path_json: '[]' }, + observations: [], + }; + try { + assert.strictEqual(typeof api.renderDetail, 'function', 'renderDetail must be exposed on _packetsTestAPI'); + assert.strictEqual(ctx.HopResolver.ready(), false, 'precondition: node index not loaded'); + await api.renderDetail(panel, data); + assert(nodeFetches.length >= 1, 'renderDetail did not load the node index'); + assert(panel.innerHTML.includes('>Known Repeater'), 'responder name not resolved in the field table'); + passed++; console.log(' ✅ renderDetail: zero-hop DISCOVER_RESP loads the node index and names the responder'); + } catch (e) { + failed++; console.log(' ❌ renderDetail: zero-hop DISCOVER_RESP loads the node index and names the responder: ' + e.message); + } + finish(); +})(); + +function finish() { + console.log(''); + if (failed > 0) { + console.error(`❌ ${failed} test(s) failed, ${passed} passed`); + process.exit(1); + } + console.log(`✅ All ${passed} tests passed`); +} diff --git a/test-packets.js b/test-packets.js index 589a406ea..39072f5cf 100644 --- a/test-packets.js +++ b/test-packets.js @@ -528,7 +528,9 @@ console.log('\n=== packets.js: getDetailPreview ==='); }); assert(result.includes('DISCOVER_RESP'), 'should label subtype'); assert(result.includes('snr') || result.includes('SNR'), 'should render snr'); - assert(result.includes('0001020304050607'), 'should render pubkey hex'); + // #1868: unknown pubkey is shortened to its first 8 hex chars. + assert(result.includes('pubkey=00010203'), 'should render pubkey prefix'); + assert(!result.includes('0001020304050607'), 'should not render the full pubkey'); }); test('getDetailPreview handles CONTROL UNKNOWN subtype', () => { From 0fea3f2a75388f7a4907b17b740ef6df774b31b6 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 19:59:00 +0200 Subject: [PATCH 032/151] feat(analytics): break scope adverts down by node role (#1979) (#2019) ## Summary Adds a breakdown of flood adverts by sender role to `/api/scope-stats` and the Scopes tab, in the descriptive shape agreed in #1979: per node role, how many flood adverts were unscoped, scoped with an unnamed region, or scoped with a named region. It reports what was sent, not why. ## Changes - `cmd/server/db.go:3114-3144`: one grouped query in `GetScopeStats`. ADVERT packets on flood routes (TRANSPORT_FLOOD 0, FLOOD 1) in the window, `LEFT JOIN nodes` on `from_pubkey`, split by the three `scope_name` states (NULL, empty string, name). A missing or empty role becomes `"unknown"`. Ordered by total descending, then role. Zero-hop adverts are excluded because firmware sends them as DIRECT/TRANSPORT_DIRECT (`src/Mesh.cpp:717-730`, `Mesh::sendZeroHop`), so they would inflate "unscoped". - `cmd/server/types.go:116-133`: `ScopeAdvertRoleCount` and `ScopeStatsResponse.AdvertsByRole` (`advertsByRole`, always an array). - `public/analytics.js:4760`: `scopeAdvertsByRoleHtml` renders a table under the time-series chart with the count per state and its share of the row. Role text is escaped. It reuses the existing `/scope-stats` response, so there is no extra request. - `docs/api-spec.md:1763-1786`: documents the new field. `/api/scope-stats` is in `openapi_known_gaps.json`, so there is no `openapi.go` entry to update. API addition (existing fields unchanged): "advertsByRole": [ { "role": "repeater", "unscoped": 7741, "unknownScope": 7, "named": 1562 } ] ## Performance The query runs inside `GetScopeStats`, so it shares the existing 30s cache per window. The unary `+` on `payload_type` keeps SQLite on the `first_seen` range index. Without it the planner picked the `payload_type` index and walked every stored advert whatever the window. Read-only timing on a production DB (1,063,345 transmissions, 161,634 adverts, sqlite3 CLI 3.45.1): | Window | payload_type index | first_seen index (this PR) | |---|---|---| | 7d | 0.231s | 0.059s | | 24h | 0.214s | 0.008s | | 1h | 0.213s | 0.001s | ## Tests - `TestGetScopeStatsAdvertsByRole` (`cmd/server/db_test.go:2295`): the three states, flood-only routes, non-advert and out-of-window exclusion, `unknown` for a missing node row, an empty role and a NULL `from_pubkey`, and ordering. Mutation checked: widening to routes 0-3 and dropping the empty-role fallback both fail it. - `TestGetScopeStatsAdvertsByRoleEmpty` (`:2372`): empty result is `[]`, not null. - `test-issue-1979-scope-adverts-by-role.js`: renders the real `analytics.js` helper in a vm sandbox. Covers row order, totals and shares, columns, escaping (mutation checked), the empty state, and the non-causal caption. Registered in `test-all.sh` and the deploy.yml unit step. - `go test ./...` in `cmd/server` passes, gofmt and go vet are clean, `check-css-vars.js` OK, `check-xss-sinks.sh --diff` exits 0. ## Browser validation On a staging instance with live traffic (build `139e484e`, together with #1074's branch), in Chrome, no console errors: `/#/analytics?tab=scopes` shows "Flood adverts by node role" under the time-series chart with its caption, and a table of 6 roles for the default window, for example `repeater 1.361 | 1.109 (81.5%) | 2 (0.1%) | 250 (18.4%)`. Shares in each row add up to 100%. `/api/scope-stats?window=7d` returns `advertsByRole` with the same six roles. ## Not verified - Timings are from the sqlite3 CLI, not the modernc driver in the server process. - Role is the sender's current `nodes.role`. A node whose advert type changed within the window is counted under its latest role. The data also contains a raw `type-13` role, shown as is. - Switching the window in the browser was not exercised; the API was checked for 7d. - No Playwright E2E added. Fixes #1979 ## Review follow-up (commit `43af46b8`) An independent review found no correctness, security or performance problem: counts are per transmission, the window matches the rest of the Scopes tab, zero-hop adverts are DIRECT per firmware, and the `+t.payload_type` hint holds on modernc SQLite 3.46.0. It found two test gaps and a docs gap. Changed: - **Ordering.** The Go fixture gave companion, repeater and unknown 3 adverts each, so ordering by total was never checked (`ORDER BY COUNT(*) ASC` still passed). The fixture now has repeater 4, unknown 3, companion 2, sensor 2, so the expected order differs from alphabetical and the companion/sensor tie checks the role-name tie-break. Reversing the count order, dropping it, or reversing the tie-break now fails the test. - **Column positions.** The JS test only checked that each cell string appeared somewhere in the row, so swapping two columns passed. It now compares each row's cells and the header cells by position; both swaps fail. - **Docs.** `docs/api-spec.md` and the `ScopeAdvertRoleCount` comment now name every source of `unknown`: a NULL `from_pubkey` (legacy rows the #1143 backfill has not reached), a sender with no `nodes` row, including one moved to `inactive_nodes` by node retention (inside the 7d window only with `retention.nodeDays` below 7), and an empty role. Not added: a query-plan test pinning the `+t.payload_type` hint. The SQL is inline in `GetScopeStats`, so the test would have to copy it or the query would have to move into a constant; left for a follow-up if wanted. The raw `type-13` role in the table is the ingestor's placeholder for reserved advert types 5-15 (`cmd/ingestor/decoder.go:1229`, #1279), shown as is. --------- Co-authored-by: Claude Opus 5 (1M context) --- cmd/server/db.go | 32 +++++++ cmd/server/db_test.go | 113 +++++++++++++++++++++- cmd/server/types.go | 23 ++++- docs/api-spec.md | 15 +++ public/analytics.js | 34 ++++++- test-all.sh | 1 + test-issue-1979-scope-adverts-by-role.js | 115 +++++++++++++++++++++++ 7 files changed, 327 insertions(+), 6 deletions(-) create mode 100644 test-issue-1979-scope-adverts-by-role.js diff --git a/cmd/server/db.go b/cmd/server/db.go index 4e2105a3e..dac63be8e 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -3111,6 +3111,38 @@ func (db *DB) GetScopeStats(window string) (*ScopeStatsResponse, error) { resp.TimeSeries = []ScopeTimePoint{} } + // #1979: flood adverts by sender role, split by the three scope_name + // states. Flood routes only (TRANSPORT_FLOOD 0, FLOOD 1): zero-hop adverts + // go out as DIRECT/TRANSPORT_DIRECT (firmware Mesh::sendZeroHop) and are + // not flooded. Role is the sender's current nodes.role. + // The unary + on payload_type keeps the planner on the first_seen range + // index; the payload_type index would walk every advert ever stored. + roleRows, err := db.conn.Query(` + SELECT COALESCE(NULLIF(n.role, ''), 'unknown') AS role, + SUM(CASE WHEN t.scope_name IS NULL THEN 1 ELSE 0 END) AS unscoped, + SUM(CASE WHEN t.scope_name = '' THEN 1 ELSE 0 END) AS unknown_scope, + SUM(CASE WHEN t.scope_name != '' THEN 1 ELSE 0 END) AS named + FROM transmissions t + LEFT JOIN nodes n ON n.public_key = t.from_pubkey + WHERE +t.payload_type = ? AND t.route_type IN (0, 1) AND t.first_seen >= ? + GROUP BY 1 + ORDER BY COUNT(*) DESC, 1 + `, payloadTypeAdvert, since) + if err != nil { + return nil, fmt.Errorf("scope advertsByRole query: %w", err) + } + defer roleRows.Close() + resp.AdvertsByRole = []ScopeAdvertRoleCount{} + for roleRows.Next() { + var rc ScopeAdvertRoleCount + if roleRows.Scan(&rc.Role, &rc.Unscoped, &rc.UnknownScope, &rc.Named) == nil { + resp.AdvertsByRole = append(resp.AdvertsByRole, rc) + } + } + if err := roleRows.Err(); err != nil { + return nil, fmt.Errorf("scope advertsByRole iteration: %w", err) + } + return resp, nil } diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 51bb10ca1..8dc492b0c 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -6,6 +6,7 @@ import ( "errors" "os" "path/filepath" + "reflect" "strings" "testing" "time" @@ -2245,8 +2246,9 @@ func TestGetScopeStats(t *testing.T) { id INTEGER PRIMARY KEY AUTOINCREMENT, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT, - scope_name TEXT DEFAULT NULL + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL )`) + db.conn.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, role TEXT)`) // Manually set hasScopeName since we bypassed the detector db.hasScopeName = true @@ -2286,6 +2288,115 @@ func TestGetScopeStats(t *testing.T) { } } +// TestGetScopeStatsAdvertsByRole covers the #1979 breakdown: flood adverts in +// the window, grouped by the sender's nodes.role, split by the three +// scope_name states (NULL is unscoped, an empty string is unknown scope, a +// name is a named scope). +func TestGetScopeStatsAdvertsByRole(t *testing.T) { + conn, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + conn.SetMaxOpenConns(1) + db := &DB{conn: conn, hasScopeName: true} + defer db.conn.Close() + + for _, stmt := range []string{ + `CREATE TABLE transmissions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, + scope_name TEXT DEFAULT NULL, from_pubkey TEXT DEFAULT NULL + )`, + `CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`, + `INSERT INTO nodes (public_key, role) VALUES ('rpt', 'repeater'), ('cmp', 'companion'), ('sns', 'sensor'), ('norole', '')`, + } { + if _, err := db.conn.Exec(stmt); err != nil { + t.Fatalf("schema: %v", err) + } + } + + now := time.Now().UTC().Format(time.RFC3339) + old := time.Now().UTC().Add(-48 * time.Hour).Format(time.RFC3339) + rows := []struct { + hash string + seen string + route int + ptype int + scope interface{} + fromKey interface{} + }{ + // Totals per role differ from alphabetical order so the test pins + // ORDER BY total: repeater 4, unknown 3, companion 2, sensor 2. + // companion and sensor tie, which pins the role-name tie-break. + // Repeater: every scope state on flood routes, two named. + {"r1", now, 1, 4, nil, "rpt"}, + {"r2", now, 0, 4, "", "rpt"}, + {"r3", now, 0, 4, "#belgium", "rpt"}, + {"r8", now, 1, 4, "#belgium", "rpt"}, + // Excluded: zero-hop adverts are DIRECT/TRANSPORT_DIRECT, not flood. + {"r4", now, 2, 4, nil, "rpt"}, + {"r5", now, 3, 4, "#belgium", "rpt"}, + // Excluded: not an advert. + {"r6", now, 0, 5, "#belgium", "rpt"}, + // Excluded: outside the window. + {"r7", old, 1, 4, nil, "rpt"}, + // Companion: one unscoped, one named. + {"c1", now, 1, 4, nil, "cmp"}, + {"c3", now, 0, 4, "#belgium", "cmp"}, + // Sensor: one unscoped, one unknown scope. + {"s1", now, 1, 4, nil, "sns"}, + {"s2", now, 0, 4, "", "sns"}, + // No node row, and a node row without a role: both "unknown". + {"u1", now, 1, 4, nil, "stranger"}, + {"u2", now, 0, 4, "", "norole"}, + {"u3", now, 0, 4, "#belgium", nil}, + } + for _, r := range rows { + if _, err := db.conn.Exec( + `INSERT INTO transmissions (hash, first_seen, route_type, payload_type, scope_name, from_pubkey) VALUES (?, ?, ?, ?, ?, ?)`, + r.hash, r.seen, r.route, r.ptype, r.scope, r.fromKey, + ); err != nil { + t.Fatalf("seed %s: %v", r.hash, err) + } + } + + stats, err := db.GetScopeStats("24h") + if err != nil { + t.Fatalf("GetScopeStats: %v", err) + } + // Ordered by total adverts descending, then role name. + want := []ScopeAdvertRoleCount{ + {Role: "repeater", Unscoped: 1, UnknownScope: 1, Named: 2}, + {Role: "unknown", Unscoped: 1, UnknownScope: 1, Named: 1}, + {Role: "companion", Unscoped: 1, UnknownScope: 0, Named: 1}, + {Role: "sensor", Unscoped: 1, UnknownScope: 1, Named: 0}, + } + if !reflect.DeepEqual(stats.AdvertsByRole, want) { + t.Errorf("AdvertsByRole = %+v, want %+v", stats.AdvertsByRole, want) + } +} + +func TestGetScopeStatsAdvertsByRoleEmpty(t *testing.T) { + conn, err := sql.Open("sqlite", ":memory:") + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + conn.SetMaxOpenConns(1) + db := &DB{conn: conn, hasScopeName: true} + defer db.conn.Close() + db.conn.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, first_seen TEXT, route_type INTEGER, payload_type INTEGER, scope_name TEXT, from_pubkey TEXT)`) + db.conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, role TEXT)`) + + stats, err := db.GetScopeStats("1h") + if err != nil { + t.Fatalf("GetScopeStats: %v", err) + } + // Array fields are [] when empty, never null (docs/api-spec.md conventions). + if stats.AdvertsByRole == nil || len(stats.AdvertsByRole) != 0 { + t.Errorf("AdvertsByRole = %#v, want empty non-nil slice", stats.AdvertsByRole) + } +} + // TestLoadIndexesRelayHopsFromResolvedPath verifies that after Load(), relay // nodes that appear only in resolved_path (not in decoded_json) are indexed // in byNode. Regression for #692: indexByNode was called before observations diff --git a/cmd/server/types.go b/cmd/server/types.go index 4bf02da0f..e6ab4419f 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -113,11 +113,26 @@ type ScopeTimePoint struct { Unscoped int `json:"unscoped"` } +// ScopeAdvertRoleCount is one row of the adverts-by-role breakdown (#1979): +// flood adverts sent by nodes of one role, split by the three scope_name +// states. Role is the sender's nodes.role, or "unknown" when the advert has +// no from_pubkey (legacy row not yet reached by the #1143 backfill), when the +// sender has no nodes row (also after MoveStaleNodes moved it to +// inactive_nodes, possible inside the 7d window only with +// retention.nodeDays < 7), or when its role is empty. +type ScopeAdvertRoleCount struct { + Role string `json:"role"` + Unscoped int `json:"unscoped"` + UnknownScope int `json:"unknownScope"` + Named int `json:"named"` +} + type ScopeStatsResponse struct { - Window string `json:"window"` - Summary ScopeStatsSummary `json:"summary"` - ByRegion []ScopeRegionCount `json:"byRegion"` - TimeSeries []ScopeTimePoint `json:"timeSeries"` + Window string `json:"window"` + Summary ScopeStatsSummary `json:"summary"` + ByRegion []ScopeRegionCount `json:"byRegion"` + TimeSeries []ScopeTimePoint `json:"timeSeries"` + AdvertsByRole []ScopeAdvertRoleCount `json:"advertsByRole"` } // ─── Health ──────────────────────────────────────────────────────────────────── diff --git a/docs/api-spec.md b/docs/api-spec.md index 8790696b7..0205e8539 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -1760,6 +1760,14 @@ Scope-based packet statistics over a time window. Requires ingestor `scope_name_ ], "timeSeries": [ { "t": string (ISO), "scoped": number, "unscoped": number } // bucket timestamps and counts + ], + "advertsByRole": [ + { + "role": string, // sender's nodes.role, or "unknown" (see notes below) + "unscoped": number, // flood adverts with no scope (scope_name NULL) + "unknownScope": number, // scoped, but no region name could be assigned (scope_name "") + "named": number // scoped with a named region + } ] } ``` @@ -1773,6 +1781,13 @@ Scope-based packet statistics over a time window. Requires ingestor `scope_name_ - `1h` window → 5-minute buckets - `24h` window → 1-hour buckets - `7d` window → 6-hour buckets +- `advertsByRole` (#1979) counts ADVERT packets on flood routes only (TRANSPORT_FLOOD 0, FLOOD 1) in the window, + grouped by the sender's current `nodes.role`. Zero-hop adverts (DIRECT/TRANSPORT_DIRECT) are excluded. + Ordered by total adverts descending, then role. It reports what was sent per role, not why. + `role` is `"unknown"` when the advert row has no `from_pubkey` (legacy rows the ingestor's #1143 + `from_pubkey` backfill has not reached yet), when the sender has no row in `nodes` (including a sender + the ingestor's node retention moved to `inactive_nodes`, which within the 7d window only happens with + `retention.nodeDays` below 7), or when its `nodes.role` is empty. - Cached 30 seconds > **Note:** On deployments with pre-existing data, `unscoped` will be inflated until the async startup backfill completes, because transport-route rows inserted before the `scope_name_v1` migration ran have `scope_name = NULL` and are indistinguishable from Code1=0000 rows. The backfill goroutine populates them at startup but may take several minutes on large databases. diff --git a/public/analytics.js b/public/analytics.js index 961c2738c..48e54aa2b 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -2986,6 +2986,7 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _analyticsData = window._analyticsRenderMultiByteAdopters = renderMultiByteAdopters; window._analyticsHashStatCardsHtml = hashStatCardsHtml; window._analyticsRenderCollisionsFromServer = renderCollisionsFromServer; + window._analyticsScopeAdvertsByRoleHtml = scopeAdvertsByRoleHtml; } // ─── Neighbor Graph Tab ───────────────────────────────────────────────────── @@ -4754,6 +4755,33 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _analyticsData = } // ===================== SCOPES ===================== + // #1979: flood adverts by sender role, split by scope state. Descriptive + // only: it shows what each role sent, not why. + function scopeAdvertsByRoleHtml(rows) { + var html = '

Flood adverts by node role

' + + '

' + + 'Flood adverts (routes 0 and 1) in this window, grouped by the sender\'s role. ' + + 'Shares are of the row. Unknown scope means scoped, but this instance could not name the region. ' + + 'This shows what each role sent; it does not show why.' + + '

'; + if (!rows || !rows.length) { + return html + '

No flood adverts in this window.

'; + } + return html + + '' + + '' + + '' + rows.map(function(r) { + var total = r.unscoped + r.unknownScope + r.named; + function cell(n) { + return ''; + } + return '' + + '' + + cell(r.unscoped) + cell(r.unknownScope) + cell(r.named) + ''; + }).join('') + '' + + '
RoleAdvertsUnscopedUnknown scopeNamed scope
' + n.toLocaleString() + ' (' + (n / total * 100).toFixed(1) + '%)
' + esc(r.role) + '' + total.toLocaleString() + '
'; + } + async function renderScopesTab(el) { var winKey = 'scopes_window'; var selectedWindow = (typeof sessionStorage !== 'undefined' && sessionStorage.getItem(winKey)) || '24h'; @@ -4776,7 +4804,8 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _analyticsData = '
RegionMessages% of Scoped
' + - '
'; + '
' + + '
'; // Attach window-button click listeners (once) el.querySelectorAll('[data-win]').forEach(function(btn) { @@ -4905,6 +4934,9 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _analyticsData = } chartEl.innerHTML = chartHtml; } + + var advertsEl = document.getElementById('scopes-adverts-by-role'); + if (advertsEl) advertsEl.innerHTML = scopeAdvertsByRoleHtml(d.advertsByRole); } load(selectedWindow); diff --git a/test-all.sh b/test-all.sh index 1ad6d50d2..e0063d97b 100755 --- a/test-all.sh +++ b/test-all.sh @@ -128,6 +128,7 @@ node test-issue-1849-trace-hashbytes.js node test-issue-1868-control-decode.js node test-issue-1890-og-url.js node test-issue-1956-release-routing.js +node test-issue-1979-scope-adverts-by-role.js node test-issue-2001-map-scope-state.js node test-live-anims.js node test-live-dt-cap-1524.js diff --git a/test-issue-1979-scope-adverts-by-role.js b/test-issue-1979-scope-adverts-by-role.js new file mode 100644 index 000000000..04714acaa --- /dev/null +++ b/test-issue-1979-scope-adverts-by-role.js @@ -0,0 +1,115 @@ +/* Unit tests for the Scopes tab "flood adverts by node role" section (#1979). + * + * The section renders /api/scope-stats advertsByRole: one row per sender role, + * with the count of flood adverts in each of the three scope states and that + * count's share of the row. It is descriptive only: it reports what each role + * sent and makes no claim about why. + * + * Loads the real public/analytics.js in a vm sandbox and exercises the helper + * it exposes, the same way test-frontend-helpers.js tests hashStatCardsHtml. + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } +} + +function makeSandbox() { + const ctx = { + window: {}, + document: { + documentElement: {}, addEventListener() {}, removeEventListener() {}, + getElementById() { return null; }, querySelector() { return null; }, querySelectorAll() { return []; }, + createElement() { return { style: {}, addEventListener() {} }; }, + }, + console, Date, Math, Array, Object, String, Number, JSON, RegExp, Error, + parseInt, parseFloat, isFinite, isNaN, Map, Set, Promise, + setTimeout: () => {}, clearTimeout: () => {}, setInterval: () => {}, clearInterval: () => {}, + localStorage: { getItem: () => null, setItem() {}, removeItem() {} }, + getComputedStyle: () => ({ getPropertyValue: () => '' }), + registerPage: () => {}, api: async () => ({}), fetch: async () => ({ json: async () => ({}) }), + addEventListener() {}, removeEventListener() {}, + }; + ctx.window = ctx; + vm.createContext(ctx); + vm.runInContext(fs.readFileSync('public/analytics.js', 'utf8'), ctx); + return ctx; +} + +console.log('\n=== analytics.js: scope adverts by role (#1979) ==='); +{ + const ctx = makeSandbox(); + const render = ctx.window._analyticsScopeAdvertsByRoleHtml; + + test('helper is exposed for testing', () => { + assert.strictEqual(typeof render, 'function', '_analyticsScopeAdvertsByRoleHtml must be exposed'); + }); + + const rows = [ + { role: 'repeater', unscoped: 3, unknownScope: 1, named: 0 }, + { role: 'companion', unscoped: 1, unknownScope: 0, named: 1 }, + ]; + const html = typeof render === 'function' ? render(rows) : ''; + + test('renders one body row per role, in server order', () => { + const bodyRows = html.match(//g) || []; + assert.deepStrictEqual(bodyRows, ['', '']); + }); + + // Cell contents of one body row, in column order. + function rowCells(role) { + const start = html.indexOf(''); + assert.ok(start !== -1, 'row for ' + role); + const row = html.slice(start, html.indexOf('', start)); + return row.split('').slice(1).map((c) => c.replace(/<\/td>$/, '')); + } + + test('renders the row total and each state with its share of the row, by column', () => { + assert.deepStrictEqual(rowCells('repeater'), [ + 'repeater', + '4', + '3 (75.0%)', + '1 (25.0%)', + '0 (0.0%)', + ]); + assert.deepStrictEqual(rowCells('companion'), [ + 'companion', + '2', + '1 (50.0%)', + '0 (0.0%)', + '1 (50.0%)', + ]); + }); + + test('header columns are in the same order as the cells', () => { + const heads = (html.match(/[^<]*<\/th>/g) || []).map((th) => th.slice(4, -5)); + assert.deepStrictEqual(heads, ['Role', 'Adverts', 'Unscoped', 'Unknown scope', 'Named scope']); + }); + + test('escapes the role text', () => { + const out = render([{ role: '', unscoped: 1, unknownScope: 0, named: 0 }]); + assert.ok(!out.includes(' { + for (const input of [[], undefined]) { + const out = render(input); + assert.ok(!out.includes(' { + assert.ok(/does not show why/.test(html), 'caption should state the table does not explain causes'); + assert.ok(!/because/i.test(html), 'no causal "because" wording'); + }); +} + +console.log(`\n${passed} passed, ${failed} failed`); +if (failed > 0) process.exit(1); From 7f8f71e7730b61b71ea72b3f8fbc9f6b02dfc7f4 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 19:59:18 +0200 Subject: [PATCH 033/151] fix(live): reconnect when the websocket goes silent (#1074) (#2020) ## Problem #1074 reports that after a proxy dropped the WebSocket, live updates only came back 8 to 10 minutes later. The client only reconnects from `onclose` (`public/app.js:791` on master). A half-open connection (a proxy or NAT dropping state without a FIN reaching the browser, a laptop that slept) can keep a WebSocket OPEN for minutes without `onclose`, and nothing retries in the meantime. The server does ping every 30s (`cmd/server/websocket.go:252` on master), but ping frames are answered by the browser below the page and JS cannot observe them. The only app-level frames are packet broadcasts (`websocket.go:337, 343, 366`), which stop on a quiet mesh. So the client had no signal to tell a quiet mesh from a dead socket. ## Change Server (`cmd/server/websocket.go`): - On the existing ping tick, `writePump` also writes the text frame `{"type":"heartbeat"}` (`:283`, bytes at `:80`). One 20-byte frame per client per 30s, from the goroutine that already writes the ping, no hub lock. - The interval moves to `Hub.pingInterval` (default 30s, `:118`) so a test can shorten it. Client (`public/app.js`): - Every frame refreshes `wsLastMessageAt` (`:841`). One timer (`checkWSLiveness`, `:806`) fires at last message + `WS_STALE_MS` (75s, one late or lost heartbeat of slack) and replaces the socket if it is still silent. It is armed at socket creation, so a stuck handshake is covered too. - `dropWS` (`:796`) detaches the old socket's handlers before `close()`, so a late close event cannot schedule a second connection. - `connectWS` (`:818`) cancels a pending reconnect and drops the previous socket, so the watchdog, resume checks, `onclose` and pull-to-reconnect cannot stack sockets. Before this, `pullReconnect` on a non-open socket left a third socket 3s later. The 3s `WS_RECONNECT_MS` delay after `onclose` is unchanged (`:837`). - `visibilitychange` (to visible) and `online` run the check immediately (`:861`), because a hidden or sleeping tab's timers can run late. - Heartbeat frames are matched by exact bytes (`:842`) and are not pulsed or dispatched to `onWS` listeners. Compatibility: tabs loaded before the deploy dispatch heartbeats to their listeners until reloaded. Every current listener filters on `msg.type`, so the visible effect is a logo pulse and a `/stats` cache refresh every 30s. Perf: one `Date.now()` and one string compare per WS message on the client; one extra 20-byte write per client per 30s on the server. ## Tests - `test-ws-stale-watchdog-1074.js`: real `app.js` in a vm with a fake clock, timers and WebSocket. 12 tests: silence past the threshold replaces the socket exactly once; a handshake that never opens is replaced; heartbeats and packet traffic keep the socket; heartbeats are not dispatched; resume and `online` after silence reconnect immediately, with recent traffic they do not, and hiding does not trigger a check; repeated resume events open one socket; after `onclose` only the reconnect timer is pending; pull-to-reconnect leaves one socket. 9 of 12 fail on master. 12 of 12 source mutations (threshold, reconnect path, detaching, timer cleanup, resume wiring, heartbeat filter) are caught. Registered in `test-all.sh` and the deploy.yml unit step. - `TestWritePumpSendsAppHeartbeat`: fails with a read timeout without the heartbeat, even with pings every 20ms. `TestHubDefaultPingInterval` pins the 30s interval that `WS_STALE_MS` assumes. - `go test ./...` in `cmd/server` passes; gofmt and go vet are clean. ## Browser validation On a staging instance (build `139e484e`, together with #1979's branch), in Chrome, no console errors: - A `{"type":"heartbeat"}` frame arrived on the open socket within the observation window. - Silent socket: after `ws.onmessage = null`, the page replaced the socket after 76.2s (threshold 75s plus a 250ms poll); the old socket ended in CLOSED, the new one OPEN. - Normal close: `ws.close()` led to a new OPEN socket after 4.1s, and exactly one new `WebSocket` was constructed. ## Not verified - The reporter's proxy setup was not reproduced; that their delay was a half-open socket is a hypothesis consistent with the symptom. Hence `Refs`, not `Fixes`. - Laptop sleep and the `visibilitychange` / `online` resume path were only covered by the unit test, not in a browser. - Behaviour under Chrome's intensive background-timer throttling and mobile tab freezing was not measured; a frozen but healthy tab may do one unnecessary reconnect on resume. - Go tests were run without `-race`. Refs #1074 ## Review follow-up (commit `72e5e906`) An independent review found no blocking bug: all data writes stay on the write goroutine, pong-based dead-client detection still works, and no ordering of onclose, watchdog, resume checks and pull ends with two live sockets or none. It reproduced the silent-socket case in headless Chromium through a blackholing TCP proxy (replacement 75.0 s after the last frame). Changed: 1. **Startup wiring tested.** The first resume test now boots through the page's real `DOMContentLoaded` listeners, so removing `setupWSResumeCheck()` from startup makes it fail. 2. **Wall clock stepping back.** If the clock steps back after the last message, the watchdog no longer re-arms for the size of the step (a 1 h step used to delay detection by about an hour). A negative silence reading is treated as stale, so the socket is replaced within `WS_STALE_MS` of the step (`public/app.js:810-814`). A step in either direction costs at most one extra reconnect on a healthy socket. `Date.now()` stays the clock so a tab resumed after sleep is still checked against real elapsed time. 3. **Pull-to-reconnect at once.** On an OPEN socket, pull-to-reconnect now replaces it through `connectWS()` instead of closing it and waiting for onclose, which took 63 s on a half-open connection in the review's measurement (`public/app.js:927-934`). This was slow on master too; it is safe now that `connectWS()` detaches the old socket. Tests: 12 to 16 in `test-ws-stale-watchdog-1074.js`; `test-pull-to-reconnect.js`, `test-pull-to-reconnect-1091.js` and `test-live.js` pass. Correction to the compatibility note: tabs opened before the deploy treat the heartbeat like any other message. Besides the logo pulse, `app.js` runs `updateNavStats` on every message and invalidates the cached `/stats` and `/nodes` responses 5 s later; `packets.js` also pushes every message into `pauseBuffer` unfiltered (~1310-1313), so an old tab with Packets paused sees its counter rise by 2 per minute. Cosmetic: heartbeats are filtered out on replay, and a reload ends it. Not verified: real hidden-tab or mobile freeze behaviour, Firefox and Safari, and the reporter's proxy setup. --------- Co-authored-by: Claude Opus 5 (1M context) --- cmd/server/websocket.go | 21 +- cmd/server/websocket_test.go | 41 ++++ public/app.js | 89 ++++++-- test-all.sh | 1 + test-pull-to-reconnect.js | 4 +- test-ws-stale-watchdog-1074.js | 386 +++++++++++++++++++++++++++++++++ 6 files changed, 520 insertions(+), 22 deletions(-) create mode 100644 test-ws-stale-watchdog-1074.js diff --git a/cmd/server/websocket.go b/cmd/server/websocket.go index 923c04c29..14befa64c 100644 --- a/cmd/server/websocket.go +++ b/cmd/server/websocket.go @@ -19,6 +19,7 @@ type Hub struct { upgrader websocket.Upgrader allowedOrigins []string // exact-match allowlist for /ws CheckOrigin (see SetAllowedOrigins) limits *wsLimiter // #1794: per-IP caps and deny list; nil allows everything + pingInterval time.Duration } // SetAllowedOrigins configures the exact-match origin allowlist consulted by @@ -74,6 +75,10 @@ func (h *Hub) checkOrigin(r *http.Request) bool { return false } +// wsHeartbeat is written to every client on each ping tick. public/app.js +// compares incoming frames against these exact bytes, so keep them in step. +var wsHeartbeat = []byte(`{"type":"heartbeat"}`) + // Client is a single WebSocket connection. type Client struct { conn *websocket.Conn @@ -109,7 +114,8 @@ func (h *Hub) ConfigureLimits(maxConnsPerIP, upgradesPerMin int, trustedProxies, func NewHub() *Hub { h := &Hub{ - clients: make(map[*Client]bool), + clients: make(map[*Client]bool), + pingInterval: 30 * time.Second, } h.upgrader = websocket.Upgrader{ ReadBufferSize: 1024, @@ -214,7 +220,7 @@ func (h *Hub) ServeWS(w http.ResponseWriter, r *http.Request) { } h.Register(client) - go client.writePump() + go client.writePump(h.pingInterval) go client.readPump(h) } @@ -248,8 +254,8 @@ func (c *Client) readPump(hub *Hub) { } } -func (c *Client) writePump() { - ticker := time.NewTicker(30 * time.Second) +func (c *Client) writePump(pingInterval time.Duration) { + ticker := time.NewTicker(pingInterval) defer func() { ticker.Stop() c.conn.Close() @@ -270,6 +276,13 @@ func (c *Client) writePump() { if err := c.conn.WriteMessage(websocket.PingMessage, nil); err != nil { return } + // #1074: the ping above keeps proxies and this server's read + // deadline happy, but browser JS never sees ping frames. Without a + // frame the page receives, a quiet mesh and a dead socket look the + // same to the client, so it could not detect the latter. + if err := c.conn.WriteMessage(websocket.TextMessage, wsHeartbeat); err != nil { + return + } } } } diff --git a/cmd/server/websocket_test.go b/cmd/server/websocket_test.go index 5d5691b85..fbdd62106 100644 --- a/cmd/server/websocket_test.go +++ b/cmd/server/websocket_test.go @@ -413,3 +413,44 @@ func TestHubRegisterUnregister(t *testing.T) { t.Errorf("expected 0 clients, got %d", hub.ClientCount()) } } + +// #1074: browsers cannot see protocol-level pings, so a page cannot tell a +// quiet mesh from a dead socket unless the server also sends something the +// page receives. The heartbeat rides the same ticker as the ping. +func TestWritePumpSendsAppHeartbeat(t *testing.T) { + hub := NewHub() + hub.pingInterval = 20 * time.Millisecond + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hub.ServeWS(w, r) + })) + defer srv.Close() + + conn, _, err := websocket.DefaultDialer.Dial("ws"+srv.URL[4:], nil) + if err != nil { + t.Fatalf("dial error: %v", err) + } + defer conn.Close() + + conn.SetReadDeadline(time.Now().Add(2 * time.Second)) + kind, msg, err := conn.ReadMessage() + if err != nil { + t.Fatalf("expected a heartbeat with no broadcast traffic, got read error: %v", err) + } + if kind != websocket.TextMessage { + t.Fatalf("heartbeat must be a text frame so page JS receives it, got frame type %d", kind) + } + // public/app.js matches these exact bytes to keep heartbeats away from + // page listeners; changing them breaks that match. + if string(msg) != `{"type":"heartbeat"}` { + t.Fatalf("heartbeat bytes changed: %s", msg) + } +} + +// The client's stale threshold (WS_STALE_MS in public/app.js, 75s) assumes +// a heartbeat at least every 30s. +func TestHubDefaultPingInterval(t *testing.T) { + if got := NewHub().pingInterval; got != 30*time.Second { + t.Fatalf("pingInterval = %v, want 30s (public/app.js WS_STALE_MS depends on it)", got) + } +} diff --git a/public/app.js b/public/app.js index 9aad3d31a..5969dbed6 100644 --- a/public/app.js +++ b/public/app.js @@ -635,6 +635,18 @@ function buildHexLegend(ranges) { let ws = null; let wsListeners = []; +// #1074: a half-open connection (a proxy idle timeout, NAT state dropped, a +// laptop that slept) can keep a WebSocket OPEN for minutes without onclose +// ever firing, and the page just stops updating. The server writes +// WS_HEARTBEAT on every 30s ping tick (cmd/server/websocket.go), so a socket +// that has received nothing for WS_STALE_MS is replaced. 75s tolerates one +// late or lost heartbeat. +const WS_STALE_MS = 75000; +const WS_HEARTBEAT = '{"type":"heartbeat"}'; +let wsLastMessageAt = 0; +let wsWatchdogTimer = null; +let wsReconnectTimer = null; + // --- Brand-logo packet-driven pulse (#1173) --- // Replaces the legacy live-dot indicator. Class-toggle only (CSS animations); colors come from // --logo-accent / --logo-accent-hi tokens. Test seam at window.__corescopeLogo. @@ -778,20 +790,60 @@ const Logo = (function () { return api; })(); +// Detach before closing: a half-open socket may not fire onclose until the +// browser gives up on the closing handshake, and a late onclose from a socket +// already replaced would schedule a second connection. +function dropWS() { + clearTimeout(wsWatchdogTimer); + wsWatchdogTimer = null; + if (!ws) return; + const old = ws; + ws = null; + old.onopen = old.onclose = old.onerror = old.onmessage = null; + try { old.close(); } catch (_) {} +} + +function checkWSLiveness() { + if (!ws) return; + clearTimeout(wsWatchdogTimer); + const silentMs = Date.now() - wsLastMessageAt; + // A negative reading means the wall clock stepped back since the last + // message, so the silence can no longer be measured: replace the socket + // like a stale one rather than re-arm for the size of the step. Date.now() + // stays the clock because performance.now() may not count time asleep. + if (silentMs >= 0 && silentMs < WS_STALE_MS) { + wsWatchdogTimer = setTimeout(checkWSLiveness, WS_STALE_MS - silentMs); + return; + } + Logo.setConnected(false); + connectWS(); +} + function connectWS() { + clearTimeout(wsReconnectTimer); + wsReconnectTimer = null; + dropWS(); const proto = location.protocol === 'https:' ? 'wss:' : 'ws:'; - ws = new WebSocket(`${proto}//${location.host}`); - ws.onopen = () => Logo.setConnected(true); - ws.onclose = () => { + const sock = new WebSocket(`${proto}//${location.host}`); + ws = sock; + // Measured from creation, so a handshake that never completes is caught too. + wsLastMessageAt = Date.now(); + wsWatchdogTimer = setTimeout(checkWSLiveness, WS_STALE_MS); + sock.onopen = () => Logo.setConnected(true); + sock.onclose = () => { + clearTimeout(wsWatchdogTimer); + wsWatchdogTimer = null; Logo.setConnected(false); // WS_RECONNECT_MS comes from roles.js and is settable as cacheTTL's // sibling `wsReconnectMs`. It used to apply only to the live map's own // socket; now that every view shares this one, the operator's setting // applies here or nowhere. - setTimeout(connectWS, window.WS_RECONNECT_MS || 3000); + wsReconnectTimer = setTimeout(connectWS, window.WS_RECONNECT_MS || 3000); }; - ws.onerror = () => ws.close(); - ws.onmessage = (e) => { + sock.onerror = () => sock.close(); + sock.onmessage = (e) => { + wsLastMessageAt = Date.now(); + if (e.data === WS_HEARTBEAT) return; Logo.pulse(e); try { const msg = JSON.parse(e.data); @@ -808,6 +860,15 @@ function connectWS() { }; } +// Timers in a hidden or sleeping tab can run late, so check as soon as the +// page is back instead of waiting out a watchdog that may be minutes behind. +function setupWSResumeCheck() { + document.addEventListener('visibilitychange', () => { + if (!document.hidden) checkWSLiveness(); + }); + window.addEventListener('online', checkWSLiveness); +} + function onWS(fn) { wsListeners.push(fn); } function offWS(fn) { wsListeners = wsListeners.filter(f => f !== fn); } @@ -866,16 +927,11 @@ function pullReconnect() { // If WS is connected (readyState OPEN), give a brief "Connected" // confirmation but still cycle so the user sees fresh data. const wasOpen = ws && ws.readyState === 1; - if (wasOpen) { - _showPullToast('Connected', true); - // Fast cycle: close and let onclose reconnect immediately - try { ws.close(); } catch (e) {} - } else { - _showPullToast('Reconnecting…', true); - try { if (ws) ws.close(); } catch (e) {} - // onclose handler schedules reconnect; force one now in case ws was null - try { connectWS(); } catch (e) {} - } + _showPullToast(wasOpen ? 'Connected' : 'Reconnecting…', true); + // Replace the socket now in both cases: an OPEN socket may be half-open, + // and its onclose can take about a minute to fire after close(). + // connectWS() detaches and closes the old socket itself. + try { connectWS(); } catch (e) {} } function _isTouchDevice() { @@ -1201,6 +1257,7 @@ window.addEventListener('timestamp-mode-changed', () => { }); window.addEventListener('DOMContentLoaded', () => { connectWS(); + setupWSResumeCheck(); setupPullToReconnect(); // --- Dark Mode --- diff --git a/test-all.sh b/test-all.sh index e0063d97b..349586c3c 100755 --- a/test-all.sh +++ b/test-all.sh @@ -172,6 +172,7 @@ node test-top-routes-overlay.js node test-traces.js node test-url-state.js node test-warmup-banner.js +node test-ws-stale-watchdog-1074.js node test-xss-escape-sinks.js echo "All standalone frontend suites passed" diff --git a/test-pull-to-reconnect.js b/test-pull-to-reconnect.js index 5f2ccdc39..0a847dfe0 100644 --- a/test-pull-to-reconnect.js +++ b/test-pull-to-reconnect.js @@ -1,8 +1,8 @@ /* test-pull-to-reconnect.js — behavioral tests for pull-to-reconnect (#1063) * Loads app.js in a vm sandbox, stubs WebSocket + DOM, asserts that: * - pullReconnect() exists as a global helper - * - calling it closes the existing WS (which triggers the existing - * auto-reconnect path) + * - calling it replaces the existing WS through connectWS(), which + * detaches and closes the old socket (#1074) * - setupPullToReconnect() exists and wires touchstart/touchmove/touchend * listeners on the document * - a pull-down gesture at scrollTop=0 over the threshold triggers diff --git a/test-ws-stale-watchdog-1074.js b/test-ws-stale-watchdog-1074.js new file mode 100644 index 000000000..3bba3f6bf --- /dev/null +++ b/test-ws-stale-watchdog-1074.js @@ -0,0 +1,386 @@ +/* test-ws-stale-watchdog-1074.js: the shared WebSocket in app.js must notice + * a socket that has gone silent and replace it (#1074). + * + * A half-open TCP connection (a proxy idle timeout, NAT state dropped, a + * laptop that slept) can leave the browser's WebSocket in OPEN state for + * minutes without ever firing onclose, so the page silently stops updating. + * The server sends an app-level heartbeat every 30s; the client treats a + * socket that has received nothing for WS_STALE_MS as dead. + * + * Loads the real public/app.js in a vm sandbox with a fake clock, fake timers + * and a fake WebSocket. + */ +'use strict'; + +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +console.log('--- test-ws-stale-watchdog-1074.js ---'); + +// Mirrors WS_STALE_MS in public/app.js and wsHeartbeat in cmd/server/websocket.go. +const STALE_MS = 75000; +const HEARTBEAT = '{"type":"heartbeat"}'; + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}\n ${e.stack.split('\n').slice(1, 3).join('\n ')}`); } +} + +function inertElement() { + return new Proxy(function () {}, { + get(_, key) { + if (key === Symbol.toPrimitive) return () => ''; + if (key === Symbol.iterator) return function* () {}; + return inertElement(); + }, + set() { return true; }, + apply() { return inertElement(); }, + }); +} + +function makeSandbox() { + // now drives timers and performance.now(); wallOffset is added to Date.now() + // only, so a test can step the wall clock the way NTP or a user can. + const clock = { now: 1700000000000, wallOffset: 0 }; + let nextId = 1; + const timers = new Map(); // id -> { fn, at } + + function setTimeoutFake(fn, ms) { + const id = nextId++; + timers.set(id, { fn, at: clock.now + Math.max(0, ms || 0) }); + return id; + } + function clearTimeoutFake(id) { timers.delete(id); } + // Runs every timer due within the next `ms`, in due order, moving the clock. + function advance(ms) { + const end = clock.now + ms; + for (let fired = 0; ; fired++) { + if (fired > 10000) throw new Error('timer storm: over 10000 timers fired within ' + ms + 'ms'); + let dueId = null, due = null; + for (const [id, t] of timers) { + if (t.at <= end && (due === null || t.at < due.at)) { dueId = id; due = t; } + } + if (dueId === null) break; + timers.delete(dueId); + clock.now = due.at; + due.fn(); + } + clock.now = end; + } + + class FakeDate extends Date {} + FakeDate.now = () => clock.now + clock.wallOffset; + + const sockets = []; + function FakeWS(url) { + this.url = url; + this.readyState = 0; // CONNECTING + this.closed = false; + this.onopen = null; this.onclose = null; this.onerror = null; this.onmessage = null; + sockets.push(this); + } + FakeWS.prototype.open = function () { this.readyState = 1; if (this.onopen) this.onopen({}); }; + FakeWS.prototype.message = function (data) { if (this.onmessage) this.onmessage({ data }); }; + FakeWS.prototype.close = function () { + if (this.closed) return; + this.closed = true; + this.readyState = 3; + if (this.onclose) this.onclose({}); + }; + FakeWS.prototype.send = function () {}; + + const docListeners = {}; + const winListeners = {}; + const document = { + readyState: 'complete', + hidden: false, + documentElement: { scrollTop: 0, style: { setProperty() {} }, setAttribute() {}, getAttribute() { return null; } }, + body: { appendChild() {}, contains() { return true; } }, + head: { appendChild() {} }, + createElement() { return { style: {}, classList: { add() {}, remove() {} }, appendChild() {}, setAttribute() {} }; }, + getElementById() { return null; }, + querySelector() { return null; }, + querySelectorAll() { return []; }, + addEventListener(ev, fn) { (docListeners[ev] = docListeners[ev] || []).push(fn); }, + removeEventListener() {}, + }; + const window = { + addEventListener(ev, fn) { (winListeners[ev] = winListeners[ev] || []).push(fn); }, + removeEventListener() {}, + dispatchEvent() {}, + matchMedia() { return { matches: false, addEventListener() {} }; }, + WS_RECONNECT_MS: 3000, + }; + const ctx = { + console, + setTimeout: setTimeoutFake, clearTimeout: clearTimeoutFake, + setInterval() { return 0; }, clearInterval() {}, + Date: FakeDate, Math, JSON, Object, Array, String, Number, Boolean, + Error, RegExp, Map, Set, Symbol, Promise, + requestAnimationFrame() { return 0; }, + performance: { now: () => clock.now }, + location: { protocol: 'http:', host: 'localhost', hash: '' }, + navigator: { userAgent: 'test' }, + WebSocket: FakeWS, + fetch() { return Promise.resolve({ ok: true, json() { return Promise.resolve({}); } }); }, + localStorage: { getItem() { return null; }, setItem() {}, removeItem() {} }, + document, window, + CustomEvent: function (type, init) { this.type = type; this.detail = (init || {}).detail; }, + }; + window.location = ctx.location; + window.document = document; + ctx.self = window; + ctx.globalThis = ctx; + vm.createContext(ctx); + vm.runInContext(fs.readFileSync('public/app.js', 'utf8'), ctx); + + return { + ctx, clock, timers, advance, sockets, document, + fireDoc(ev) { (docListeners[ev] || []).forEach((fn) => fn({ type: ev })); }, + fireWin(ev) { (winListeners[ev] || []).forEach((fn) => fn({ type: ev })); }, + // Runs app.js's real startup listeners, as the browser does. The startup + // code wires the whole page shell, so every element lookup gets an inert + // stand-in that accepts any property or call. + boot() { + document.getElementById = document.querySelector = () => inertElement(); + ctx.getComputedStyle = () => inertElement(); + (winListeners.DOMContentLoaded || []).forEach((fn) => fn({ type: 'DOMContentLoaded' })); + }, + }; +} + +console.log('\n=== silence past the stale threshold replaces the socket ==='); + +test('an open socket that receives nothing for WS_STALE_MS is closed and replaced exactly once', () => { + const box = makeSandbox(); + box.ctx.connectWS(); + const first = box.sockets[0]; + first.open(); + + box.advance(STALE_MS - 1); + assert.strictEqual(box.sockets.length, 1, 'must not reconnect before the threshold'); + assert.strictEqual(first.closed, false, 'must not close before the threshold'); + + box.advance(1); + assert.strictEqual(first.closed, true, 'the silent socket must be closed'); + assert.strictEqual(box.sockets.length, 2, 'exactly one replacement socket must be opened'); + + // Nothing else is pending that would open a third socket: the replacement + // is still connecting, and the dead socket's handlers are detached. + box.advance(STALE_MS - 1); + assert.strictEqual(box.sockets.length, 2, 'no second replacement'); + assert.strictEqual(first.onclose, null, 'the dropped socket must not be able to schedule a reconnect'); + assert.strictEqual(first.onmessage, null, 'the dropped socket must not dispatch messages'); +}); + +test('a socket that never opens is also replaced after WS_STALE_MS', () => { + const box = makeSandbox(); + box.ctx.connectWS(); + box.advance(STALE_MS); + assert.strictEqual(box.sockets[0].closed, true); + assert.strictEqual(box.sockets.length, 2); +}); + +console.log('\n=== regular traffic keeps the socket ==='); + +test('heartbeats every 30s keep one socket alive for ten minutes', () => { + const box = makeSandbox(); + box.ctx.connectWS(); + const first = box.sockets[0]; + first.open(); + for (let i = 0; i < 20; i++) { + box.advance(30000); + first.message(HEARTBEAT); + } + assert.strictEqual(box.sockets.length, 1, 'no reconnect while heartbeats arrive'); + assert.strictEqual(first.closed, false); +}); + +test('packet traffic without heartbeats also counts as liveness', () => { + const box = makeSandbox(); + box.ctx.connectWS(); + const first = box.sockets[0]; + first.open(); + for (let i = 0; i < 20; i++) { + box.advance(STALE_MS - 1000); + first.message(JSON.stringify({ type: 'packet', data: { id: i } })); + } + assert.strictEqual(box.sockets.length, 1); + assert.strictEqual(first.closed, false); +}); + +test('a heartbeat is not dispatched to page listeners and does not pulse the logo', () => { + const box = makeSandbox(); + const seen = []; + box.ctx.onWS((m) => seen.push(m)); + box.ctx.connectWS(); + const first = box.sockets[0]; + first.open(); + const logo = box.ctx.window.__corescopeLogo; + const before = logo.stats.triggered + logo.stats.dropped; + + first.message(HEARTBEAT); + assert.strictEqual(seen.length, 0, 'heartbeat must not reach onWS listeners'); + assert.strictEqual(logo.stats.triggered + logo.stats.dropped, before, 'heartbeat must not pulse the logo'); + + first.message(JSON.stringify({ type: 'packet', data: { id: 1 } })); + assert.strictEqual(seen.length, 1, 'a packet must still reach listeners'); +}); + +console.log('\n=== resuming a hidden tab or coming back online checks immediately ==='); + +function throttledJump(box, ms) { + // A hidden tab's timers can be held back; move the clock without running them. + box.clock.now += ms; +} + +test('visibilitychange to visible after a long silence reconnects without waiting for the timer', () => { + const box = makeSandbox(); + // The real startup path, so a page that never calls setupWSResumeCheck fails here. + box.boot(); + assert.strictEqual(box.sockets.length, 1, 'startup opens one socket'); + box.sockets[0].open(); + + box.document.hidden = true; + box.fireDoc('visibilitychange'); + throttledJump(box, 5 * 60 * 1000); + box.document.hidden = false; + box.fireDoc('visibilitychange'); + + assert.strictEqual(box.sockets[0].closed, true, 'the silent socket must be dropped on resume'); + assert.strictEqual(box.sockets.length, 2, 'a replacement must be opened on resume'); +}); + +test('visibilitychange to visible with recent traffic keeps the socket', () => { + const box = makeSandbox(); + box.ctx.setupWSResumeCheck(); + box.ctx.connectWS(); + box.sockets[0].open(); + throttledJump(box, 20000); + box.sockets[0].message(HEARTBEAT); + throttledJump(box, 20000); + box.fireDoc('visibilitychange'); + assert.strictEqual(box.sockets.length, 1); + assert.strictEqual(box.sockets[0].closed, false); + assert.strictEqual(box.timers.size, 1, 'the check must replace the watchdog, not add a second one'); +}); + +test('becoming hidden does not trigger a check', () => { + const box = makeSandbox(); + box.ctx.setupWSResumeCheck(); + box.ctx.connectWS(); + box.sockets[0].open(); + throttledJump(box, 5 * 60 * 1000); + box.document.hidden = true; + box.fireDoc('visibilitychange'); + assert.strictEqual(box.sockets.length, 1); +}); + +test('the online event after a long silence reconnects immediately', () => { + const box = makeSandbox(); + box.ctx.setupWSResumeCheck(); + box.ctx.connectWS(); + box.sockets[0].open(); + throttledJump(box, STALE_MS + 1); + box.fireWin('online'); + assert.strictEqual(box.sockets[0].closed, true); + assert.strictEqual(box.sockets.length, 2); +}); + +test('repeated resume events open one replacement, not one each', () => { + const box = makeSandbox(); + box.ctx.setupWSResumeCheck(); + box.ctx.connectWS(); + box.sockets[0].open(); + throttledJump(box, STALE_MS + 1); + box.fireWin('online'); + box.fireDoc('visibilitychange'); + box.fireWin('online'); + assert.strictEqual(box.sockets.length, 2, 'the replacement is fresh, so later checks leave it alone'); +}); + +console.log('\n=== a wall-clock step does not disable the watchdog ==='); + +test('the clock stepping back an hour as the socket goes silent still replaces it within WS_STALE_MS', () => { + const box = makeSandbox(); + box.ctx.connectWS(); + const first = box.sockets[0]; + first.open(); + for (let i = 0; i < 3; i++) { + box.advance(30000); + first.message(HEARTBEAT); + } + box.clock.wallOffset -= 60 * 60 * 1000; // step back, then nothing more arrives + box.advance(STALE_MS); + assert.strictEqual(first.closed, true, 'the silent socket must be dropped'); + assert.strictEqual(box.sockets.length, 2, 'one replacement within WS_STALE_MS of the step'); +}); + +for (const [label, stepMs] of [['forward', 60 * 60 * 1000], ['back', -60 * 60 * 1000]]) { + test(`a clock step ${label} on a healthy socket costs at most one extra reconnect`, () => { + const box = makeSandbox(); + box.ctx.connectWS(); + box.sockets[0].open(); + box.advance(40000); + box.sockets[0].message(HEARTBEAT); + box.advance(10000); + box.clock.wallOffset += stepMs; + for (let i = 0; i < 20; i++) { + box.advance(30000); + const cur = box.sockets[box.sockets.length - 1]; + if (cur.readyState === 0) cur.open(); + cur.message(HEARTBEAT); + } + assert.ok(box.sockets.length <= 2, 'got ' + box.sockets.length + ' sockets over ten minutes'); + }); +} + +console.log('\n=== timers are cleaned up on close ==='); + +test('after onclose only the reconnect timer is pending, and it opens one socket', () => { + const box = makeSandbox(); + box.ctx.connectWS(); + const first = box.sockets[0]; + first.open(); + first.close(); // the browser saw the close + + assert.strictEqual(box.timers.size, 1, 'the watchdog must be cleared; only the reconnect remains'); + box.advance(3000); + assert.strictEqual(box.sockets.length, 2, 'the reconnect opens one socket'); + box.sockets[1].open(); + for (let i = 0; i < 6; i++) { + box.advance(30000); + box.sockets[1].message(HEARTBEAT); + } + assert.strictEqual(box.sockets.length, 2, 'the closed socket\'s watchdog must not fire later'); +}); + +test('pullReconnect on a socket that is not open leaves one socket and no stray reconnect', () => { + const box = makeSandbox(); + box.ctx.connectWS(); // still CONNECTING + box.ctx.window.pullReconnect(); + assert.strictEqual(box.sockets.length, 2, 'pull opens a replacement at once'); + assert.strictEqual(box.sockets[0].closed, true, 'the previous socket is closed'); + box.advance(3000); + assert.strictEqual(box.sockets.length, 2, 'the old socket\'s close must not schedule a third socket'); +}); + +test('pullReconnect on an open socket replaces it at once instead of waiting for onclose', () => { + const box = makeSandbox(); + box.ctx.connectWS(); + const first = box.sockets[0]; + first.open(); + // A half-open socket may not fire onclose for about a minute after close(). + first.close = function () { this.closed = true; this.readyState = 2; }; + box.ctx.window.pullReconnect(); + assert.strictEqual(box.sockets.length, 2, 'the replacement must exist right after the pull'); + assert.strictEqual(first.closed, true, 'the previous socket is closed'); + assert.strictEqual(first.onclose, null, 'the previous socket is detached, so a late onclose cannot reconnect again'); + box.advance(STALE_MS - 1); + assert.strictEqual(box.sockets.length, 2, 'exactly one socket results from the pull'); +}); + +console.log('\n=== Results: ' + passed + ' passed, ' + failed + ' failed ===\n'); +process.exit(failed > 0 ? 1 : 0); From 2d4019f7193efe863a953ff3123440ea39502eb4 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 19:59:37 +0200 Subject: [PATCH 034/151] fix(analytics): recompute once the store has fully loaded (#2025) Refs #2023, #1659, #1724 ### Problem `main.go:258` waits only for the first load chunk, then `main.go:402` starts the analytics recomputers. `Start()` computes immediately on that chunk (`analytics_recomputer.go:86` on master) and the next compute waits a full interval (`:93`, 5 min default). The chunk loader walks by ascending id, so that chunk holds the oldest transmissions. - RF, topology, channels: the #1659 gate checked `LoadComplete()` after the compute (`analytics_warmup_1659.go:122`). `LoadComplete` flips at the end of the hot window (`chunked_load.go:489`), before the background fill (`store.go:1455`), so the gate could open on a snapshot without the background fill, and the 60 s force timeout (`:73`, `:162`) opened it on the first-chunk snapshot. In an end-to-end test on master, `/api/analytics/rf` returned 200 with 8 of 100 packets before the background fill ran. - Distance, hash-collisions, hash-sizes, roles, observers-clock-skew, nodes-clock-skew: no gate, partial snapshot served from the start. - Distance additionally served a snapshot from the previous index for up to one interval after each lazy index build. On a staging instance, a default analytics request returned 5,911 packets with hours-old last buckets until the next recompute (about 74k). ### Change - `StartupLoadDone()` (`chunked_load.go:108`): closed when `RunStartupLoad` returns, on every path (`chunked_load.go:202`). Closing it drops the hash-size info cache (15 s TTL) and the clock-skew engine throttle (30 s, `clock_skew.go:225`), both read by the post-load computes. - `recomputeWhenLoaded` (`analytics_recomputer.go:172`): on that signal, recompute each recomputer once, sequentially, via `RecomputeNow` (`:154`), which runs on the recomputer's own loop and restarts its ticker (`:106`). Order (`:255`): rf, topology, channels, distance, hash-collisions, hash-sizes, observers-clock-skew, nodes-clock-skew, roles (roles reads the nodes-clock-skew snapshot). Logs one line with per-recomputer durations. - Warm-up gate: now the same signal (`:343-354`), sampled before the compute starts (`:129`), so a pass that began on partial data never opens it. 503 + `Retry-After: 5` and the force timeout are unchanged; a forced-open snapshot is replaced by the post-load recompute. - Ungated endpoints: no new 503s (their API has none); snapshot replaced right after the load. - Distance: the lazy index build refreshes the distance recomputer before reporting built (`store.go:4476`). - Recompute intervals and config unchanged. ### Performance One extra compute per recomputer per process start, run sequentially so they do not all hold the store read lock at once. Ticker phases afterwards are offset by the cumulative post-load compute durations instead of all starting within the first-chunk compute window (relevant to #1724; the effect on lock waves is not measured). ### Tests `analytics_recompute_after_load_test.go`: signal open during background fill, closed after success and failure; cache drops; immediate and ordered post-load recompute; gate not opened by a pass started before the load; forced-open snapshot replaced on load; ticker restart; distance refresh before 202 ends; end to end with recomputers started before the background fill (RF 503 until load, then `totalTransmissions` equals the full store; six ungated endpoints 200 during load; all nine recomputed after load). 9 of these failed on master with stubs; 8 single-line mutations each caught. `go test ./...` in `cmd/server` passes. ### Staging validation Deployed together with the review follow-ups of #2015-#2023 (build `c646310f`), container restart: ``` 16:35:20 [store] first chunk ready (chunkSize=10000) 16:35:25 [store] LoadChunked complete ... starting background fill loader 16:36:58 [store] background load complete: 121120/121282 packets in memory (coverage=99.9%) 16:37:03 [analytics-recompute] startup load done: recomputed 10 snapshots in 5.155s (rf=955ms topology=1.684s channels=43ms distance=49ms hash-collisions=30ms hash-sizes=338ms observers-clock-skew=369ms nodes-clock-skew=692ms roles=2ms retransmissions=994ms) ``` Right after that line, `/api/analytics/rf` reported `totalTransmissions` 121,121 against 120,700 packets in memory, and the retransmissions default shape from #2023 covered the full 7 days. Before this change both waited for the next 5 minute tick. ### Merge order with #2023 #2023 adds a tenth recomputer. Whichever of the two merges second has to add `recompRetransmissions` to `analyticsRecomputersLocked`, wire it to `loadedGate` instead of `LoadComplete`, and change 9 to 10 in `TestAnalyticsRecomputers_PostLoadOrder`; the retransmissions gate test then calls `signalStartupLoadDone()` instead of setting `loadComplete`. That resolution is what ran on staging above. ### Not verified - Repeater-enrich recomputer and the region/window TTL caches (hash-collisions region results have a 1 h TTL) may also keep partial results after the load; not changed here. - Recompute order is tested structurally, not with roles/clock-skew data. - Whether this reduces the #1724 stalls; not measured. Co-authored-by: Claude Opus 5 (1M context) --- .../analytics_recompute_after_load_test.go | 454 ++++++++++++++++++ cmd/server/analytics_recomputer.go | 151 ++++-- cmd/server/analytics_warmup_1659.go | 44 +- cmd/server/analytics_warmup_1659_test.go | 11 +- cmd/server/chunked_load.go | 28 ++ cmd/server/clock_skew.go | 8 + cmd/server/store.go | 16 + 7 files changed, 654 insertions(+), 58 deletions(-) create mode 100644 cmd/server/analytics_recompute_after_load_test.go diff --git a/cmd/server/analytics_recompute_after_load_test.go b/cmd/server/analytics_recompute_after_load_test.go new file mode 100644 index 000000000..f18cbb418 --- /dev/null +++ b/cmd/server/analytics_recompute_after_load_test.go @@ -0,0 +1,454 @@ +package main + +// Analytics recomputers after the startup load. +// +// main.go starts the recomputers as soon as the FIRST load chunk is in +// memory. Their initial compute therefore only sees that chunk (the +// oldest ids), and the next compute used to wait a full interval +// (5 min by default). These tests pin the contract: +// +// - the store exposes a signal that fires only after RunStartupLoad +// has finished, background fill included; +// - every recomputer recomputes as soon as that signal fires; +// - a pass that started before the signal never ends the #1659 +// warm-up, and a snapshot served after the force timeout is +// replaced as soon as the load is done. + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "github.com/gorilla/mux" +) + +func isClosedForTest(ch <-chan struct{}) bool { + select { + case <-ch: + return true + default: + return false + } +} + +func waitForTest(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("timed out waiting for: %s", what) +} + +// The startup-load signal must stay open while the background fill is +// still running (LoadComplete is already true at that point) and close +// once RunStartupLoad returns. +func TestStartupLoadDone_ClosesOnlyAfterBackgroundFill(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + createTestDBSpreadOverDays(t, dbPath, 100, 14, time.Now().UTC().Unix()) + db, err := OpenDB(dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + defer db.conn.Close() + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 14 * 24, HotStartupHours: 24}) + + var closedAtBgEntry, loadCompleteAtBgEntry atomic.Bool + store.bgLoaderEntryHook = func() { + closedAtBgEntry.Store(isClosedForTest(store.StartupLoadDone())) + loadCompleteAtBgEntry.Store(store.LoadComplete()) + } + if isClosedForTest(store.StartupLoadDone()) { + t.Fatal("StartupLoadDone closed before RunStartupLoad ran") + } + if err := store.RunStartupLoad(500); err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + if !loadCompleteAtBgEntry.Load() { + t.Fatal("fixture precondition: LoadComplete should already be true when the background fill starts") + } + if closedAtBgEntry.Load() { + t.Fatal("StartupLoadDone closed before the background fill ran") + } + if !isClosedForTest(store.StartupLoadDone()) { + t.Fatal("StartupLoadDone not closed after RunStartupLoad returned") + } +} + +// A failed load is terminal too: nothing more will be loaded, so the +// signal must close instead of leaving the recomputers waiting forever. +func TestStartupLoadDone_ClosesWhenLoadFails(t *testing.T) { + dbPath := filepath.Join(t.TempDir(), "test.db") + createTestDBSpreadOverDays(t, dbPath, 10, 1, time.Now().UTC().Unix()) + db, err := OpenDB(dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 24, HotStartupHours: 1}) + db.conn.Close() + if err := store.RunStartupLoad(500); err == nil { + t.Fatal("fixture precondition: RunStartupLoad on a closed conn should fail") + } + if !isClosedForTest(store.StartupLoadDone()) { + t.Fatal("StartupLoadDone not closed after a failed RunStartupLoad") + } +} + +// Derived caches with their own TTL (node hash-size info 15s, clock-skew +// engine 30s) may hold values computed from the partial store. The +// recomputes that follow the signal read them, so they must be dropped. +func TestStartupLoadDone_DropsDerivedCaches(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + + store.hashSizeInfoMu.Lock() + store.hashSizeInfoCache = map[string]*hashSizeNodeInfo{"partial": {}} + store.hashSizeInfoAt = time.Now() + store.hashSizeInfoMu.Unlock() + store.clockSkew.mu.Lock() + store.clockSkew.lastComputed = time.Now() + store.clockSkew.mu.Unlock() + + store.signalStartupLoadDone() + + store.hashSizeInfoMu.Lock() + hashInfo := store.hashSizeInfoCache + store.hashSizeInfoMu.Unlock() + if hashInfo != nil { + t.Fatal("hash-size info cache survived the startup-load signal") + } + store.clockSkew.mu.RLock() + last := store.clockSkew.lastComputed + store.clockSkew.mu.RUnlock() + if !last.IsZero() { + t.Fatal("clock-skew engine still considers its pre-load result fresh") + } +} + +// The recomputer must not wait a full interval once the store is loaded. +func TestRecomputeWhenLoaded_RunsImmediately(t *testing.T) { + var loadedFlag atomic.Bool + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { + if loadedFlag.Load() { + return "full" + } + return "partial" + }) + rc.Start() + defer rc.Stop() + if got := rc.Load(); got != "partial" { + t.Fatalf("initial snapshot = %v, want partial", got) + } + + loaded := make(chan struct{}) + stop := make(chan struct{}) + defer close(stop) + go recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{rc}) + + loadedFlag.Store(true) + close(loaded) + waitForTest(t, "snapshot recomputed after load", func() bool { return rc.Load() == "full" }) + if runs := rc.ComputeRuns(); runs != 2 { + t.Fatalf("ComputeRuns = %d, want 2 (initial + post-load)", runs) + } +} + +// Recomputers run one after the other, in slice order, so a recomputer +// that reads another one's snapshot (roles reads nodes-clock-skew) sees +// the post-load version. +func TestRecomputeWhenLoaded_RunsInOrder(t *testing.T) { + var loadedFlag atomic.Bool + var skewSnapshot atomic.Value + skew := newAnalyticsRecomputer("skew", time.Hour, func() interface{} { + v := "partial" + if loadedFlag.Load() { + v = "full" + } + time.Sleep(20 * time.Millisecond) + skewSnapshot.Store(v) + return v + }) + roles := newAnalyticsRecomputer("roles", time.Hour, func() interface{} { + return skewSnapshot.Load() + }) + skew.Start() + roles.Start() + defer skew.Stop() + defer roles.Stop() + + loaded := make(chan struct{}) + stop := make(chan struct{}) + defer close(stop) + done := make(chan struct{}) + go func() { + recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{skew, roles}) + close(done) + }() + loadedFlag.Store(true) + close(loaded) + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("recomputeWhenLoaded did not finish") + } + if got := roles.Load(); got != "full" { + t.Fatalf("roles snapshot = %v, want full (it ran before the skew recompute finished)", got) + } +} + +// The post-load order used by StartAnalyticsRecomputers: gated +// recomputers first (their 503 ends soonest), roles after +// nodes-clock-skew (roles reads that snapshot). +func TestAnalyticsRecomputers_PostLoadOrder(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + stop := store.StartAnalyticsRecomputers(time.Hour) + defer stop() + + store.analyticsRecomputerMu.RLock() + list := store.analyticsRecomputersLocked() + store.analyticsRecomputerMu.RUnlock() + pos := map[string]int{} + for i, rc := range list { + pos[rc.name] = i + } + if len(list) != 9 || len(pos) != 9 { + t.Fatalf("want 9 distinct recomputers, got %d (%d distinct)", len(list), len(pos)) + } + for _, name := range []string{"rf", "topology", "channels"} { + if pos[name] > 2 { + t.Errorf("%s at position %d, want among the first three", name, pos[name]) + } + } + if pos["nodes-clock-skew"] > pos["roles"] { + t.Errorf("roles (%d) runs before nodes-clock-skew (%d)", pos["roles"], pos["nodes-clock-skew"]) + } +} + +// A pass that STARTED before the load finished must not end the warm-up, +// even when the load finishes while that pass is still running. +func TestWarmup_PassStartedBeforeLoadDoesNotOpenGate(t *testing.T) { + loaded := make(chan struct{}) + releaseSecond := make(chan struct{}) + var calls atomic.Int32 + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { + switch calls.Add(1) { + case 1: + close(loaded) // load finishes while the first pass is running + return "partial" + default: + <-releaseSecond + return "full" + } + }) + rc.setWarmupReadyGate_1659(func() bool { return isClosedForTest(loaded) }) + rc.Start() + defer rc.Stop() + + if !rc.IsWarmingUp_1659() { + t.Fatal("gate opened on a pass that started before the store was loaded") + } + + stop := make(chan struct{}) + defer close(stop) + go recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{rc}) + waitForTest(t, "post-load pass started", func() bool { return calls.Load() == 2 }) + if !rc.IsWarmingUp_1659() { + t.Fatal("gate opened while the only post-load pass is still running") + } + close(releaseSecond) + waitForTest(t, "gate opens after the post-load pass", func() bool { return !rc.IsWarmingUp_1659() }) + if got := rc.Load(); got != "full" { + t.Fatalf("gate open but snapshot = %v, want full", got) + } +} + +// Force-open timeout followed by the load finishing: the first-chunk +// snapshot that was served after the timeout is replaced right away. +func TestWarmup_ForceOpenedSnapshotReplacedOnLoad(t *testing.T) { + prev := warmupForceTimeout + warmupForceTimeout = 30 * time.Millisecond + defer func() { warmupForceTimeout = prev }() + + loaded := make(chan struct{}) + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { + if isClosedForTest(loaded) { + return "full" + } + return "partial" + }) + rc.setWarmupReadyGate_1659(func() bool { return isClosedForTest(loaded) }) + rc.Start() + defer rc.Stop() + stop := make(chan struct{}) + defer close(stop) + go recomputeWhenLoaded(loaded, stop, []*analyticsRecomputer{rc}) + + waitForTest(t, "force timeout opens the gate", func() bool { return !rc.IsWarmingUp_1659() }) + if got := rc.Load(); got != "partial" { + t.Fatalf("fixture precondition: forced-open snapshot = %v, want partial", got) + } + if !rc.FirstPassDoneAt_1659().IsZero() { + t.Fatal("a pre-load pass was recorded as the first full pass") + } + + close(loaded) + waitForTest(t, "snapshot replaced after load", func() bool { return rc.Load() == "full" }) + waitForTest(t, "first full pass recorded", func() bool { return !rc.FirstPassDoneAt_1659().IsZero() }) +} + +// The periodic ticker restarts from the post-load compute, so the next +// periodic pass is a full interval after it instead of a redundant pass +// on the original phase a moment later. +func TestRecomputeNow_RestartsTicker(t *testing.T) { + const interval = 400 * time.Millisecond + rc := newAnalyticsRecomputer("t", interval, func() interface{} { return 1 }) + rc.Start() // run 1 at t0; unreset ticker would fire at t0+400ms + defer rc.Stop() + + time.Sleep(250 * time.Millisecond) + rc.RecomputeNow() // run 2 at ~250ms; next tick now ~650ms + time.Sleep(300 * time.Millisecond) // ~550ms: past the old phase, before the new one + if runs := rc.ComputeRuns(); runs != 2 { + t.Fatalf("ComputeRuns = %d at ~550ms, want 2 (ticker not restarted by RecomputeNow)", runs) + } +} + +// RecomputeNow must not hang when the recomputer is stopped. +func TestRecomputeNow_ReturnsWhenStopped(t *testing.T) { + rc := newAnalyticsRecomputer("t", time.Hour, func() interface{} { return 1 }) + rc.Start() + rc.Stop() + done := make(chan struct{}) + go func() { + rc.RecomputeNow() + close(done) + }() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("RecomputeNow blocked on a stopped recomputer") + } +} + +// After a lazy distance-index build, the distance recomputer snapshot +// must already reflect the new index when the handler stops answering 202. +func TestDistanceIndexBuild_RefreshesRecomputerBeforeReportingBuilt(t *testing.T) { + db := setupRichTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("Load(): %v", err) + } + stop := store.StartAnalyticsRecomputers(time.Hour) + defer stop() + before := store.recompDistance.ComputeRuns() + + store.TriggerDistanceIndexBuild() + waitForTest(t, "distance index built", store.DistanceIndexBuilt) + if runs := store.recompDistance.ComputeRuns(); runs <= before { + t.Fatalf("distance index reported built while recomputer still serves the pre-build snapshot (runs %d -> %d)", before, runs) + } +} + +// End to end: recomputers started at the first chunk, background fill +// still pending. Gated endpoints answer 503 until the load is done and +// then serve data from the whole store without waiting for the interval. +// Ungated recomputers keep answering (no new 503) and are recomputed +// right after the load. +func TestAnalyticsRecomputers_FullDataRightAfterStartupLoad(t *testing.T) { + const totalRows = 100 + dbPath := filepath.Join(t.TempDir(), "test.db") + createTestDBSpreadOverDays(t, dbPath, totalRows, 14, time.Now().UTC().Unix()) + db, err := OpenDB(dbPath) + if err != nil { + t.Fatalf("OpenDB: %v", err) + } + defer db.conn.Close() + store := NewPacketStore(db, &PacketStoreConfig{RetentionHours: 14 * 24, HotStartupHours: 24}) + + bgEntered := make(chan struct{}) + releaseBg := make(chan struct{}) + store.bgLoaderEntryHook = func() { + close(bgEntered) + <-releaseBg + } + loadErr := make(chan error, 1) + go func() { loadErr <- store.RunStartupLoad(500) }() + <-bgEntered + + stop := store.StartAnalyticsRecomputers(time.Hour) + defer stop() + + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + srv.store = store + router := mux.NewRouter() + srv.RegisterRoutes(router) + get := func(path string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", path, nil)) + return w + } + + for _, p := range []string{"/api/analytics/rf", "/api/analytics/topology", "/api/analytics/channels"} { + if w := get(p); w.Code != http.StatusServiceUnavailable { + t.Fatalf("%s before full load: got %d, want 503", p, w.Code) + } + } + for _, p := range []string{"/api/analytics/hash-sizes", "/api/analytics/hash-collisions", "/api/analytics/roles"} { + if w := get(p); w.Code != http.StatusOK { + t.Fatalf("%s before full load: got %d, want 200 (no new warm-up 503)", p, w.Code) + } + } + + store.analyticsRecomputerMu.RLock() + all := map[string]*analyticsRecomputer{ + "topology": store.recompTopology, "rf": store.recompRF, "distance": store.recompDistance, + "channels": store.recompChannels, "hash-collisions": store.recompHashCollisions, + "hash-sizes": store.recompHashSizes, "roles": store.recompRoles, + "observers-clock-skew": store.recompObserversClockSkew, "nodes-clock-skew": store.recompNodesClockSkew, + } + store.analyticsRecomputerMu.RUnlock() + runsBefore := map[string]int64{} + for name, rc := range all { + runsBefore[name] = rc.ComputeRuns() + } + + close(releaseBg) + if err := <-loadErr; err != nil { + t.Fatalf("RunStartupLoad: %v", err) + } + store.mu.RLock() + inMemory := len(store.packets) + store.mu.RUnlock() + if inMemory != totalRows { + t.Fatalf("fixture precondition: %d packets in memory after load, want %d", inMemory, totalRows) + } + + waitForTest(t, "rf gate opens after full load", func() bool { return get("/api/analytics/rf").Code == http.StatusOK }) + var rf map[string]interface{} + if err := json.Unmarshal(get("/api/analytics/rf").Body.Bytes(), &rf); err != nil { + t.Fatalf("rf body: %v", err) + } + if got, _ := rf["totalTransmissions"].(float64); int(got) != totalRows { + t.Fatalf("rf totalTransmissions = %v right after full load, want %d (first-chunk snapshot still served)", rf["totalTransmissions"], totalRows) + } + for _, p := range []string{"/api/analytics/topology", "/api/analytics/channels"} { + if w := get(p); w.Code != http.StatusOK { + t.Fatalf("%s after full load: got %d, want 200", p, w.Code) + } + } + for name, rc := range all { + name, rc := name, rc + waitForTest(t, name+" recomputed after full load", func() bool { return rc.ComputeRuns() > runsBefore[name] }) + } +} diff --git a/cmd/server/analytics_recomputer.go b/cmd/server/analytics_recomputer.go index 57ff5f764..f0db263c8 100644 --- a/cmd/server/analytics_recomputer.go +++ b/cmd/server/analytics_recomputer.go @@ -10,6 +10,9 @@ package main import ( + "fmt" + "log" + "strings" "sync" "sync/atomic" "time" @@ -34,9 +37,10 @@ type analyticsRecomputer struct { interval time.Duration compute func() interface{} - cache atomic.Value // holds interface{} — the latest snapshot - stop chan struct{} - done chan struct{} + cache atomic.Value // holds interface{} — the latest snapshot + stop chan struct{} + done chan struct{} + recomputeReq chan chan struct{} // RecomputeNow → loop; the loop closes the inner channel when done startOnce sync.Once stopOnce sync.Once @@ -61,11 +65,12 @@ func newAnalyticsRecomputer(name string, interval time.Duration, compute func() interval = 5 * time.Minute } return &analyticsRecomputer{ - name: name, - interval: interval, - compute: compute, - stop: make(chan struct{}), - done: make(chan struct{}), + name: name, + interval: interval, + compute: compute, + stop: make(chan struct{}), + done: make(chan struct{}), + recomputeReq: make(chan chan struct{}), } } @@ -96,6 +101,10 @@ func (r *analyticsRecomputer) loop() { select { case <-t.C: r.runOnce() + case ack := <-r.recomputeReq: + r.runOnce() + t.Reset(r.interval) + close(ack) case <-r.stop: return } @@ -114,6 +123,10 @@ func (r *analyticsRecomputer) runOnce() { // reach markFirstPassDone otherwise). _ = recover() }() + // Sample the #1659 readiness gate BEFORE computing: a pass that + // started on a partially loaded store must not end the warm-up, + // even if the load finishes while it runs. + ready := r.warmupReadyGateOpen_1659() t0 := time.Now() result := r.compute() r.lastComputeNs.Store(int64(time.Since(t0))) @@ -128,9 +141,53 @@ func (r *analyticsRecomputer) runOnce() { // PR #1688 r1: called on EVERY successful pass (even nil // result) so a compute that returns nil but doesn't panic // still lifts the gate — banner-stuck-forever fix (munger #2). - // The markFirstPassDone helper is idempotent and additionally - // consults the chunked-loader readiness gate (munger #5). - r.markFirstPassDone_1659() + if ready { + r.markFirstPassDone_1659() + } +} + +// RecomputeNow has the loop goroutine run a compute that starts after +// this call, and waits for it to finish. The periodic ticker restarts +// from that compute, so the next periodic pass is a full interval +// later. Returns early if the recomputer is stopped; blocks until Start +// if it has not started yet. +func (r *analyticsRecomputer) RecomputeNow() { + ack := make(chan struct{}) + select { + case r.recomputeReq <- ack: + case <-r.stop: + return + } + select { + case <-ack: + case <-r.stop: + } +} + +// recomputeWhenLoaded waits for loaded to close, then recomputes each +// recomputer once, one at a time and in slice order. Sequential so the +// post-load passes do not all hold the store read lock at once, and so +// a recomputer that reads another one's snapshot can be placed after +// it. Returns when done or when stop closes. +func recomputeWhenLoaded(loaded, stop <-chan struct{}, rcs []*analyticsRecomputer) { + select { + case <-loaded: + case <-stop: + return + } + t0 := time.Now() + parts := make([]string, 0, len(rcs)) + for _, rc := range rcs { + select { + case <-stop: + return + default: + } + rc.RecomputeNow() + parts = append(parts, fmt.Sprintf("%s=%s", rc.name, rc.LastComputeDuration().Round(time.Millisecond))) + } + log.Printf("[analytics-recompute] startup load done: recomputed %d snapshots in %s (%s)", + len(rcs), time.Since(t0).Round(time.Millisecond), strings.Join(parts, " ")) } // Load returns the most recently computed snapshot, or nil if Start @@ -190,6 +247,20 @@ func pickInterval(override, def time.Duration) time.Duration { return def } +// analyticsRecomputersLocked lists the analytics recomputers in the +// order they start and recompute after the startup load: the three +// warm-up-gated ones first, and roles after nodes-clock-skew because +// computeAnalyticsRoles reads that recomputer's snapshot +// (GetFleetClockSkew). Caller holds analyticsRecomputerMu. +func (s *PacketStore) analyticsRecomputersLocked() []*analyticsRecomputer { + return []*analyticsRecomputer{ + s.recompRF, s.recompTopology, s.recompChannels, + s.recompDistance, s.recompHashCollisions, s.recompHashSizes, + s.recompObserversClockSkew, s.recompNodesClockSkew, + s.recompRoles, + } +} + // StartAnalyticsRecomputers wires each analytics endpoint to a // background recompute goroutine. Each runs an initial compute // synchronously (so the first read after startup is a cache hit, never @@ -260,34 +331,50 @@ func (s *PacketStore) StartAnalyticsRecomputers(defaultInterval time.Duration, o "nodes-clock-skew", pickInterval(ov.NodesClockSkew, defaultInterval), func() interface{} { return s.computeFleetClockSkew() }, ) - all := []*analyticsRecomputer{ - s.recompTopology, s.recompRF, s.recompDistance, - s.recompChannels, s.recompHashCollisions, s.recompHashSizes, - s.recompRoles, - s.recompObserversClockSkew, s.recompNodesClockSkew, - } + all := s.analyticsRecomputersLocked() s.analyticsRecomputerMu.Unlock() - // Issue #1659 (PR #1688 r1, munger #5): wire the chunked-loader - // readiness gate on the three warmup-gated recomputers (RF, - // Topology, Channels). markFirstPassDone_1659 will refuse to - // flip first-pass-done until s.LoadComplete() reports true — - // i.e. the cold-load has populated all observations. Otherwise - // the FIRST recomputer pass runs against the post-restart in-RAM - // slice and the gate opens on partial data (the original #1659 - // bug class). - loadCompleteGate := s.LoadComplete - s.recompRF.setWarmupReadyGate_1659(loadCompleteGate) - s.recompTopology.setWarmupReadyGate_1659(loadCompleteGate) - s.recompChannels.setWarmupReadyGate_1659(loadCompleteGate) + // Issue #1659 (PR #1688 r1, munger #5): wire the loader readiness + // gate on the three warmup-gated recomputers (RF, Topology, + // Channels). Only a pass that STARTS after the whole startup load + // (hot window AND background fill) ends the warm-up. LoadComplete() + // is not enough: it flips at the end of the hot window, so the gate + // used to open on a snapshot that missed the background fill. + loaded := s.StartupLoadDone() + loadedGate := func() bool { + select { + case <-loaded: + return true + default: + return false + } + } + s.recompRF.setWarmupReadyGate_1659(loadedGate) + s.recompTopology.setWarmupReadyGate_1659(loadedGate) + s.recompChannels.setWarmupReadyGate_1659(loadedGate) for _, rc := range all { rc.Start() } + // main.go starts the recomputers at the first load chunk, so the + // initial computes above only saw part of the data. Recompute as + // soon as the load is done instead of a full interval later. + stopPostLoad := make(chan struct{}) + postLoadDone := make(chan struct{}) + go func() { + defer close(postLoadDone) + recomputeWhenLoaded(loaded, stopPostLoad, all) + }() + + var stopOnce sync.Once return func() { - for _, rc := range all { - rc.Stop() - } + stopOnce.Do(func() { + close(stopPostLoad) + for _, rc := range all { + rc.Stop() + } + <-postLoadDone + }) } } diff --git a/cmd/server/analytics_warmup_1659.go b/cmd/server/analytics_warmup_1659.go index 37585c12c..8812d11fa 100644 --- a/cmd/server/analytics_warmup_1659.go +++ b/cmd/server/analytics_warmup_1659.go @@ -21,12 +21,14 @@ // // 1. (#1688 munger #5) Only mark first-pass-done when BOTH: // a. a recomputer pass has completed, AND -// b. the chunked loader has finished (s.LoadComplete()). +// b. that pass started after the startup load finished, hot +// window and background fill (store.StartupLoadDone()). // The gate's `readyGate` callback is wired by -// StartAnalyticsRecomputers to `store.LoadComplete`. Passes that -// complete while loadComplete is still false leave the gate in -// the warming-up state; the NEXT pass after loadComplete flips -// true is the one that opens the gate. +// StartAnalyticsRecomputers to that signal. Passes that start +// before it leave the gate in the warming-up state; the loop +// recomputes as soon as the signal fires, and that pass opens +// the gate. (The gate used to be store.LoadComplete, which flips +// at the end of the hot window, before the background fill.) // // 2. (#1688 munger #2 + kent-beck #2) The gate MUST lift in bounded // time. If compute() panics on every pass, hangs indefinitely, @@ -40,7 +42,8 @@ // default) elapsed since the recomputer was constructed // forces IsWarmingUp_1659() to false — degraded mode // (serve whatever cache exists, possibly empty) is -// strictly better than a permanent 503. +// strictly better than a permanent 503. The partial snapshot +// served that way is replaced by the post-load recompute. // // Concurrency (#1688 munger #3): // @@ -104,24 +107,23 @@ func (r *analyticsRecomputer) loadWarmupReadyGate_1659() func() bool { return *p } +// warmupReadyGateOpen_1659 reports whether the readyGate (when set) +// is open. runOnce samples it BEFORE computing and only calls +// markFirstPassDone_1659 when it was open: first-pass-done requires a +// pass that started after the store finished loading (munger #5). A +// check after the compute would accept a pass that began on partial +// data and merely finished after the load. +func (r *analyticsRecomputer) warmupReadyGateOpen_1659() bool { + gate := r.loadWarmupReadyGate_1659() + return gate == nil || gate() +} + // markFirstPassDone_1659 is called from analyticsRecomputer.runOnce() -// after every compute attempt (success OR nil result; panics are -// caught upstream and never reach here). +// after a compute attempt (success OR nil result; panics are caught +// upstream and never reach here) that started with the readyGate open. // -// The gate flip is conditional on the readyGate (when set) reporting -// true — this implements the munger #5 fix: first-pass-done must -// require BOTH a recomputer pass complete AND the chunked loader to -// have finished populating the in-RAM observation set. -// -// Idempotent: only the FIRST successful flip wins; subsequent calls -// observe a non-zero firstPassDoneNs and return immediately. +// Idempotent: only the FIRST successful flip wins. func (r *analyticsRecomputer) markFirstPassDone_1659() { - if r.firstPassDoneNs.Load() != 0 { - return - } - if gate := r.loadWarmupReadyGate_1659(); gate != nil && !gate() { - return - } r.firstPassDoneNs.CompareAndSwap(0, time.Now().UnixNano()) } diff --git a/cmd/server/analytics_warmup_1659_test.go b/cmd/server/analytics_warmup_1659_test.go index e78c71d98..3a72130a6 100644 --- a/cmd/server/analytics_warmup_1659_test.go +++ b/cmd/server/analytics_warmup_1659_test.go @@ -82,11 +82,12 @@ func TestAnalyticsRF_AfterFirstPassReturns200(t *testing.T) { db := setupTestDB(t) defer db.Close() store := NewPacketStore(db, nil) - // #1688 r1: the warmup gate now ALSO requires LoadComplete() to be - // true before first-pass-done flips (munger #5). Tests that don't - // exercise the chunked loader must flip it manually to model a - // production server that has finished cold-loading. - store.loadComplete.Store(true) + // #1688 r1: the warmup gate ALSO requires the startup load (hot + // window and background fill) to be done before first-pass-done + // flips (munger #5). Tests that don't exercise the loader must + // signal it manually to model a production server that has + // finished cold-loading. + store.signalStartupLoadDone() stop := store.StartAnalyticsRecomputers(50 * time.Millisecond) defer stop() diff --git a/cmd/server/chunked_load.go b/cmd/server/chunked_load.go index 258061fd0..84d96afc7 100644 --- a/cmd/server/chunked_load.go +++ b/cmd/server/chunked_load.go @@ -96,9 +96,35 @@ func (s *PacketStore) OnChunkLoaded(fn func(rowsThisChunk, totalRows int)) { func (s *PacketStore) chunkedLoadInit() { s.chunkInitOnce.Do(func() { s.firstChunkReady = make(chan struct{}) + s.startupLoadDone = make(chan struct{}) }) } +// StartupLoadDone returns a channel closed once RunStartupLoad has +// returned: LoadChunked AND the background fill loader are finished, +// whether they succeeded or not. Nothing more is loaded from SQLite +// after it closes. LoadComplete() is not a substitute: it flips at the +// end of the hot window, before the background fill starts. +func (s *PacketStore) StartupLoadDone() <-chan struct{} { + s.chunkedLoadInit() + return s.startupLoadDone +} + +func (s *PacketStore) signalStartupLoadDone() { + s.chunkedLoadInit() + if !s.startupLoadSignaled.CompareAndSwap(false, true) { + return + } + // The analytics recomputes this signal triggers read these TTL + // caches. Drop what was computed from the partial store so they see + // the loaded data now rather than after the TTL. + s.hashSizeInfoMu.Lock() + s.hashSizeInfoCache = nil + s.hashSizeInfoMu.Unlock() + s.clockSkew.Invalidate() + close(s.startupLoadDone) +} + func (s *PacketStore) signalFirstChunk() { if s.firstChunkSignaled.CompareAndSwap(false, true) { close(s.firstChunkReady) @@ -153,6 +179,7 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // LoadChunked. // // Steady-state contracts post-return: +// - StartupLoadDone() is closed, on every path below. // - LoadChunked error: backgroundLoadFailed=true, backgroundLoadDone // is also set true (terminal observable state — see dij #1). // backgroundLoadErr non-empty. Returns the error. @@ -172,6 +199,7 @@ func (s *PacketStore) fireChunkCallbacks(rowsThisChunk, totalRows int) { // parallelism while ensuring oldestLoaded has a valid floor when the // bg loader starts. func (s *PacketStore) RunStartupLoad(chunkSize int) error { + defer s.signalStartupLoadDone() // Clear any stale error from a previous invocation (single-call // invariant — see godoc above). Production never re-enters but // test fixtures may construct fresh stores that share no state; diff --git a/cmd/server/clock_skew.go b/cmd/server/clock_skew.go index 51bdf9cdd..b545d4002 100644 --- a/cmd/server/clock_skew.go +++ b/cmd/server/clock_skew.go @@ -220,6 +220,14 @@ func NewClockSkewEngine() *ClockSkewEngine { } } +// Invalidate makes the next Recompute run even if the last one is +// younger than computeInterval. +func (e *ClockSkewEngine) Invalidate() { + e.mu.Lock() + e.lastComputed = time.Time{} + e.mu.Unlock() +} + // Recompute recalculates all clock skew data from the packet store. // Called periodically or on demand. Holds store RLock externally. // Uses read-copy-update: heavy computation runs outside the write lock, diff --git a/cmd/server/store.go b/cmd/server/store.go index 959299f72..6d2f697fd 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -471,6 +471,11 @@ type PacketStore struct { chunkCBMu sync.Mutex chunkCallbacks []func(rowsThisChunk, totalRows int) + // startupLoadDone is closed when RunStartupLoad returns (hot window + // plus background fill); see StartupLoadDone. + startupLoadDone chan struct{} + startupLoadSignaled atomic.Bool + // Eviction config and stats retentionHours float64 // 0 = unlimited maxMemoryMB int // 0 = unlimited (packet store memory budget) @@ -4460,6 +4465,17 @@ func (s *PacketStore) TriggerDistanceIndexBuild() { obsAtBuild := s.totalObs s.mu.Unlock() + // The distance recomputer's snapshot was computed from the index + // as it was before this build. Refresh it before reporting the + // index as built, so the handler does not go from 202 to serving + // that older snapshot for up to one recompute interval. + s.analyticsRecomputerMu.RLock() + rc := s.recompDistance + s.analyticsRecomputerMu.RUnlock() + if rc != nil { + rc.RecomputeNow() + } + s.distLazyMu.Lock() s.distLazyBuilding = false s.distLazyBuilt = true From a059299588b77fe1cb0f5f489e7aca5f6d720f40 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 19:59:56 +0200 Subject: [PATCH 035/151] feat(node-analytics): hop-count statistics per node (#1812) (#2021) ## Summary Adds per-node hop-count statistics so repeater operators can choose `flood.max`, `flood.max.unscoped` and `flood.max.advert` from what their node actually sees. - New endpoint `GET /api/nodes/{pubkey}/hop_analytics?days=N` (`cmd/server/routes.go:299`, `cmd/server/node_hop_analytics.go:312`), separate from `/analytics` as requested in the issue. - New card "Hop Count at This Node" on the node analytics page (`public/node-hop-analytics.js`, wired at `public/node-analytics.js:130,174`): histogram of hop counts with a box plot on the same x axis, filters `flood.max` (default), `flood.max.advert`, `flood.max.unscoped`, driven by the existing range picker. - The existing "Hop Distribution" chart is unchanged: it shows path length at the observer, a different quantity. - No `direct` tag, although the issue lists one: for DIRECT packets the path is the remaining route and no flood limit applies, so there is no hop count to report. ## Hop count definition (firmware 0679dbef) - `src/helpers/RoutingPolicy.h:15-21`: limits compare `getPathHashCount()`; `.unscoped` applies to route type FLOOD, `.advert` to adverts. - `src/Mesh.cpp:344-350`: `routeRecvPacket` checks with n hashes in the path, then writes its own hash at index n. So hops = the node's zero-based index in the path, no +1. - `src/Mesh.cpp:265-285`: a node forwards a flood once; `src/Mesh.cpp:651,680`: an originator never forwards its own flood. - DIRECT packets are excluded: their path is the remaining route (`src/Mesh.cpp:78-103,334-341`). Response: `{timeRange, packets: [{hash, timestamp, hops, tags}], ambiguous}`. Tags: `flood`, `scoped` or `unscoped`, `advert`. Documented in `docs/api-spec.md:679` and `cmd/server/openapi.go:90`. ## Attribution `cmd/server/node_hop_analytics.go:198-309`. The result depends only on the observed paths, the prefix map and the neighbor graph, so it is the same after a restart as after live ingest. - Every observation of every flood packet in the window is read. `byNode` holds the server resolver's pick at ingest and other picks after a cold load; `byPathHop` indexes only each packet's longest path, which for a busy relay often runs through another branch of the flood. - A packet counts when the node's prefix sits at exactly one index across its observations, and either the node is the only relay candidate for that prefix (`prefixMap.relayCandidates`, `cmd/server/store.go:6795`), or the hop resolves to the node under the ingestor's strict rule (`cmd/ingestor/path_resolver.go:143-214`) in at least one observation and to another node in none. Strict rule: earlier hops identified without a tiebreak, exactly one candidate adjacent in `neighbor_edges` to the previous hop (the originator for hop 0 of an advert), nodes already on the path excluded. - The server resolver's tiebreaks (affinity, GPS distance, advert count, pubkey order) are not used. - Everything else with the node's prefix goes to `ambiguous`. In practice that is most packets with a colliding 1-byte path hash. On a read-only 7-day dump of a 1,669-node mesh DB, for one busy repeater: 23,081 packets attributed, 11,437 ambiguous. Taking candidates from `byPathHop` instead gave 9,995 attributed, with the histogram mode moved from 2 to 3-5 hops. ## Performance Scans `s.packets` under the read lock, no SQL per packet. Per observation: one substring test for the node's first prefix byte; the hop scan only for observations containing it; the strict walk only for colliding prefixes, with per-request caches for candidates and adjacency. `BenchmarkNodeHopPackets` models one 7-day request at that scale (73,782 flood packets, 1,430,280 observations): 44-87 ms/op, 13.4 MB, 40 allocs on a throttling laptop. Response size for that repeater over 7 days: about 23k entries, 2.3 MB JSON, 375 KB gzipped. `hash` and `timestamp` are 61% of the raw and 91% of the gzipped bytes; they stay because the issue asks for them so a client can join entries to packets and bin by time. ## Tests - Go: `cmd/server/node_hop_analytics_test.go`: 12 unit tests, a live-ingest test through `IngestNewFromDB` (a colliding prefix without independent attribution goes to `ambiguous`, not to the node the resolver picked), live ingest versus cold load of the same DB, route test, benchmark. 15 mutations of the attribution logic each fail a test. - JS: `test-node-hop-analytics.js` (filters, histogram, quartiles and whiskers with a fixture that separates 1.5 IQR from 3 IQR, render), registered in `test-all.sh` and `.github/workflows/deploy.yml`. - `gofmt`, `go vet ./...`, `go test ./...` in `cmd/server`, `scripts/check-css-vars.js` pass. ## Staging validation Build `c646310f` (this PR's review follow-up together with the other open follow-ups), after a container restart and full load, on a busy Belgian repeater: - `hop_analytics?days=7`: 23,302 packets, 11,548 ambiguous, median 4, adverts never above hop 7 (matching the firmware default `flood_max_advert = 8`, `examples/simple_repeater/MyMesh.cpp:922`), 1.2 s. The first version reported 23,035 packets and 86 ambiguous in 534 ms, because it trusted the resolver's pick for colliding prefixes. - The card rendered on the first version with no console errors; the rework does not touch the frontend beyond a test fixture. ## Not verified - Response time and lock hold for 30 days on the busiest node on a 14-day store. - Server relay candidates exclude companions and listeners while the ingestor's prefix index does not, so a few strict attributions can differ from the ingestor's persisted `resolved_path`. - Identical numbers across a second container restart were shown in a Go test, not repeated on staging. - Dark theme, phone width, and switching the range picker in the browser. - Filter state is not reflected in the URL hash (the range picker is not either). Fixes #1812 --------- Co-authored-by: Claude Opus 5 (1M context) --- .eslintrc.json | 1 + cmd/server/neighbor_persist.go | 13 +- cmd/server/node_hop_analytics.go | 331 +++++++++++++++++ cmd/server/node_hop_analytics_test.go | 504 ++++++++++++++++++++++++++ cmd/server/openapi.go | 4 + cmd/server/routes.go | 32 +- cmd/server/store.go | 73 ++-- cmd/server/types.go | 15 + docs/api-spec.md | 66 ++++ public/index.html | 1 + public/node-analytics.js | 7 + public/node-hop-analytics.js | 178 +++++++++ test-all.sh | 1 + test-node-hop-analytics.js | 84 +++++ 14 files changed, 1260 insertions(+), 50 deletions(-) create mode 100644 cmd/server/node_hop_analytics.go create mode 100644 cmd/server/node_hop_analytics_test.go create mode 100644 public/node-hop-analytics.js create mode 100644 test-node-hop-analytics.js diff --git a/.eslintrc.json b/.eslintrc.json index 33197e2fc..ab6e2ded7 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -35,6 +35,7 @@ "MAX_HOP_DIST": "readonly", "MeshAudio": "readonly", "MeshConfigReady": "readonly", + "NodeHopAnalytics": "readonly", "PAYLOAD_COLORS": "readonly", "PAYLOAD_TYPES": "readonly", "PERF_SLOW_MS": "readonly", diff --git a/cmd/server/neighbor_persist.go b/cmd/server/neighbor_persist.go index 9deea6285..d73b972cf 100644 --- a/cmd/server/neighbor_persist.go +++ b/cmd/server/neighbor_persist.go @@ -183,18 +183,7 @@ func resolvePathForObsColdLoad(pathJSON, observerID string, tx *StoreTx, pm *pre // observation_count_fallback would still pick a winner for // ambiguous prefixes, which is exactly what we must NOT do. // Hence the explicit candidate-count check here. - h := strings.ToLower(hop) - candidates := pm.m[h] - if len(pm.nonRelay) > 0 && len(candidates) > 0 { - filtered := candidates[:0:0] - for j := range candidates { - if _, isListener := pm.nonRelay[strings.ToLower(candidates[j].PublicKey)]; isListener { - continue - } - filtered = append(filtered, candidates[j]) - } - candidates = filtered - } + candidates := pm.relayCandidates(hop) if len(candidates) == 1 { pk := strings.ToLower(candidates[0].PublicKey) resolved[i] = &pk diff --git a/cmd/server/node_hop_analytics.go b/cmd/server/node_hop_analytics.go new file mode 100644 index 000000000..9ae8b407d --- /dev/null +++ b/cmd/server/node_hop_analytics.go @@ -0,0 +1,331 @@ +package main + +import ( + "strings" + "time" +) + +// Per-node hop count statistics (issue #1812). +// +// A repeater decides whether to forward a flood with +// isFloodHopLimitExceeded (firmware src/helpers/RoutingPolicy.h:15-21), +// comparing getPathHashCount() (src/Packet.h:80, path_len & 63) against +// flood.max, flood.max.unscoped (ROUTE_TYPE_FLOOD only) and flood.max.advert +// (PAYLOAD_TYPE_ADVERT only). Mesh::routeRecvPacket (src/Mesh.cpp:344-350) +// runs that check with n hashes in the path and then writes its own hash at +// index n. So the node's zero-based index in an observed flood path is the hop +// count its flood.max check saw. Firmware refs: meshcore-dev/MeshCore 0679dbef. +// +// Hash rules the attribution relies on: +// - A node forwards a given flood once (wasSeen/markSeen before +// routeRecvPacket, e.g. src/Mesh.cpp:265-285 for adverts), so it sits at +// one index in every observation that contains it. A prefix match at two +// different indices means another node shares the prefix. +// - All hops of one packet share one hash size (src/Packet.h:79), so every +// match in a packet is the same prefix string. +// - An originator marks its own flood seen before sending +// (Mesh::sendFlood, src/Mesh.cpp:651 and :680), so it never appears in +// that path. +// - DIRECT paths are the remaining route and shrink at every hop +// (removeSelfFromPath, src/Mesh.cpp:334-341); the flood.max limits only +// apply to floods (examples/simple_repeater/MyMesh.cpp:436-437). DIRECT +// packets carry no hop count for this purpose and are skipped. + +var ( + hopTagsUnscoped = []string{"flood", "unscoped"} + hopTagsScoped = []string{"flood", "scoped"} + hopTagsUnscopedAdvert = []string{"flood", "unscoped", "advert"} + hopTagsScopedAdvert = []string{"flood", "scoped", "advert"} +) + +// nextPathHop returns the hop string starting after offset pos of a path_json +// array and the offset after it, or ok=false when no hop is left. path_json +// is a JSON array of hex hop strings, which carry no escapes, so hops are read +// between quotes without unmarshalling and without allocating: the scan runs +// under s.mu.RLock for every observation in the window. +func nextPathHop(p string, pos int) (hop string, next int, ok bool) { + open := strings.IndexByte(p[pos:], '"') + if open < 0 { + return "", pos, false + } + start := pos + open + 1 + n := strings.IndexByte(p[start:], '"') + if n < 0 { + return "", pos, false + } + return p[start : start+n], start + n + 1, true +} + +// hopAttributor decides whether a path hop is a given node without using the +// server resolver's tiebreaks (affinity score, GPS distance, advert count, +// pubkey order), which always name a winner among colliding prefixes. +// Candidates are prefixMap.relayCandidates; adjacency is the neighbor graph, +// which the server loads from the ingestor's neighbor_edges, the same table +// the ingestor's resolver reads. Caches live for one request. +type hopAttributor struct { + pm *prefixMap + graph *NeighborGraph + cands map[string][]string + adj map[string]map[string]struct{} + seen []string +} + +func newHopAttributor(pm *prefixMap, graph *NeighborGraph) *hopAttributor { + return &hopAttributor{pm: pm, graph: graph, cands: map[string][]string{}, adj: map[string]map[string]struct{}{}} +} + +// candidates returns the lowercase pubkeys of the relay candidates for a hop. +// The cache is keyed by the hop as given, so wire-case hops are looked up +// without lowercasing a copy. +func (a *hopAttributor) candidates(hop string) []string { + c, ok := a.cands[hop] + if !ok { + for _, n := range a.pm.relayCandidates(hop) { + c = append(c, strings.ToLower(n.PublicKey)) + } + a.cands[hop] = c + } + return c +} + +func (a *hopAttributor) adjacent(anchor, pk string) bool { + nbrs, ok := a.adj[anchor] + if !ok { + nbrs = map[string]struct{}{} + if a.graph != nil { + for _, e := range a.graph.Neighbors(anchor) { + if e.Ambiguous || e.NodeA == "" || e.NodeB == "" { + continue + } + other := e.NodeA + if other == anchor { + other = e.NodeB + } + nbrs[other] = struct{}{} + } + } + a.adj[anchor] = nbrs + } + _, ok = nbrs[pk] + return ok +} + +// strictHopAt resolves the hop at index idx of path p the way the ingestor +// does (cmd/ingestor/path_resolver.go resolvePathWithContext): walking from +// hop 0, a hop with one candidate resolves to it, a hop with several resolves +// only when exactly one of them is a graph neighbor of the previous resolved +// hop (the advert originator for hop 0), and nodes already on the path are +// excluded. An unresolved hop breaks the chain for the next one. +// +// ok is false when p has no hop idx or that hop is not prefix. Otherwise pk is +// the lowercase pubkey the hop resolves to, or "" when it does not resolve, +// and end is the offset in p after hop idx. +func (a *hopAttributor) strictHopAt(p string, idx int, prefix, origin string) (pk string, end int, ok bool) { + anchor := origin + a.seen = a.seen[:0] + if anchor != "" { + a.seen = append(a.seen, anchor) + } + pos := 0 + for i := 0; i <= idx; i++ { + hop, next, found := nextPathHop(p, pos) + if !found || (i == idx && !strings.EqualFold(hop, prefix)) { + return "", 0, false + } + pos = next + cands := a.candidates(hop) + match, survivors := "", 0 + switch { + case len(cands) == 1: + if c := cands[0]; !a.onPath(c) { + match, survivors = c, 1 + } + case len(cands) > 1 && anchor != "" && a.graph != nil: + for _, c := range cands { + if !a.onPath(c) && a.adjacent(anchor, c) { + match = c + survivors++ + } + } + } + if survivors != 1 { + match = "" + } + if i == idx { + return match, pos, true + } + anchor = match + if match != "" { + a.seen = append(a.seen, match) + } + } + return "", 0, false +} + +func (a *hopAttributor) onPath(pk string) bool { + for _, s := range a.seen { + if s == pk { + return true + } + } + return false +} + +// computeNodeHopPackets returns one entry per flood packet in txs, first seen +// after fromISO, that the node forwarded, and how many packets carry the node's +// prefix at one index without being attributable to it. +// +// A packet counts when the node's prefix sits at exactly one index across all +// its observations, and that hop is the node independently of the server +// resolver's pick: either the node is the only relay candidate for the prefix, +// or, for a colliding prefix, at least one observation resolves that hop to the +// node under the ingestor's strict neighbor rule (hopAttributor.strictHopAt) +// and none resolves it to another node. Everything else with the node's prefix +// is counted as ambiguous and left out. The answer depends only on the +// observed paths, the prefix map and the neighbor graph, so it is the same +// after a restart as after live ingest. +// +// Every observation is read, not an index: byNode holds the resolver's pick at +// ingest and other picks after a cold load, and byPathHop only indexes each +// packet's longest path, which for a busy relay often runs through another +// branch of the flood (on a live 7-day sample it held 9,995 of the 23,081 +// attributable packets of one repeater, skewed towards higher hop counts). +// +// txs must be deduplicated. Cost is linear in the observations of the flood +// packets in the window: a substring test per observation, the hop scan only +// for observations that contain the node's first prefix byte, and the strict +// walk only for colliding prefixes. +func computeNodeHopPackets(pubkey string, txs []*StoreTx, fromISO string, pm *prefixMap, graph *NeighborGraph) ([]NodeHopPacket, int) { + lowerPK := strings.ToLower(pubkey) + // Every hop hash of the node, whatever its size, starts with its first + // pubkey byte, and path hops are written in upper case + // (internal/packetpath/path.go:50; none of 1,495,712 live observations of + // 7 days had a lower-case hex digit), so an observation without that byte + // after an opening quote holds no hop of the node. + quotedFirstByte := `"` + strings.ToUpper(lowerPK[:2]) + packets := make([]NodeHopPacket, 0) + ambiguous := 0 + attr := newHopAttributor(pm, graph) + onlyCandidate := map[string]bool{} + + for _, tx := range txs { + if tx.RouteType == nil || (*tx.RouteType != RouteFlood && *tx.RouteType != RouteTransportFlood) || tx.FirstSeen <= fromISO { + continue + } + + idx, prefix, conflict := -1, "", false + for _, obs := range tx.Observations { + p := obs.PathJSON + if !strings.Contains(p, quotedFirstByte) { + continue + } + for i, pos := 0, 0; ; i++ { + hop, next, ok := nextPathHop(p, pos) + if !ok { + break + } + pos = next + if len(hop) == 0 || len(hop) > len(lowerPK) || !strings.EqualFold(hop, lowerPK[:len(hop)]) { + continue + } + if idx < 0 { + idx, prefix = i, lowerPK[:len(hop)] + } else if i != idx { + conflict = true + } + } + } + if idx < 0 { + continue + } + origin := "" + if tx.DecodedJSON != "" && strings.Contains(tx.DecodedJSON, "ubKey") { + origin = strings.ToLower(extractFromNode(tx)) + if origin == lowerPK { + continue + } + } + if conflict { + ambiguous++ + continue + } + + advert := tx.PayloadType != nil && *tx.PayloadType == PayloadADVERT + only, known := onlyCandidate[prefix] + if !known { + cands := attr.candidates(prefix) + only = len(cands) == 1 && cands[0] == lowerPK + onlyCandidate[prefix] = only + } + if !only && !attributeByNeighbor(attr, tx, idx, prefix, advert, origin, lowerPK) { + ambiguous++ + continue + } + + tags := hopTagsUnscoped + switch { + case *tx.RouteType == RouteTransportFlood && advert: + tags = hopTagsScopedAdvert + case *tx.RouteType == RouteTransportFlood: + tags = hopTagsScoped + case advert: + tags = hopTagsUnscopedAdvert + } + packets = append(packets, NodeHopPacket{Hash: tx.Hash, Timestamp: tx.FirstSeen, Hops: idx, Tags: tags}) + } + return packets, ambiguous +} + +// attributeByNeighbor reports whether the hop at idx, which carries prefix, is +// the node under the strict neighbor rule in at least one observation and +// another node in none. The ingestor anchors hop 0 on the originator only for +// adverts (cmd/ingestor/db.go FromPubkey), and so does this. An observation +// whose path text equals the previous walked one up to hop idx reuses its +// result. +func attributeByNeighbor(attr *hopAttributor, tx *StoreTx, idx int, prefix string, advert bool, origin, lowerPK string) bool { + if !advert { + origin = "" + } + self := false + walked, pk := "", "" + for _, obs := range tx.Observations { + if walked == "" || !strings.HasPrefix(obs.PathJSON, walked) { + r, end, ok := attr.strictHopAt(obs.PathJSON, idx, prefix, origin) + if !ok { + continue + } + walked, pk = obs.PathJSON[:end], r + } + switch pk { + case "": + case lowerPK: + self = true + default: + return false + } + } + return self +} + +// GetNodeHopAnalytics returns the hop count at this node for every flood packet +// it forwarded in the last days. Returns nil for an unknown node. +func (s *PacketStore) GetNodeHopAnalytics(pubkey string, days int) (*NodeHopAnalyticsResponse, error) { + node, err := s.db.GetNodeByPubkey(pubkey) + if err != nil || node == nil { + return nil, err + } + + now := time.Now() + fromISO := now.Add(-time.Duration(days) * 24 * time.Hour).Format(time.RFC3339) + + s.mu.RLock() + _, pm := s.getCachedNodesAndPM() + packets, ambiguous := computeNodeHopPackets(pubkey, s.packets, fromISO, pm, s.graph.Load()) + s.mu.RUnlock() + + return &NodeHopAnalyticsResponse{ + TimeRange: TimeRangeResp{From: fromISO, To: now.Format(time.RFC3339), Days: days}, + Packets: packets, + Ambiguous: ambiguous, + }, nil +} diff --git a/cmd/server/node_hop_analytics_test.go b/cmd/server/node_hop_analytics_test.go new file mode 100644 index 000000000..5c3dd2c1c --- /dev/null +++ b/cmd/server/node_hop_analytics_test.go @@ -0,0 +1,504 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "sort" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// Issue #1812: per-node hop count, as the repeater's own flood.max check sees +// it. The firmware compares getPathHashCount() (the number of hashes already in +// the path when the packet arrives) against the limits before appending its own +// hash, so the target's zero-based index in an observed flood path is exactly +// that count. + +const ( + hopTarget = "ab12cd34ef567890" + hopOther = "77aa88bb99cc0011" + hopCollider = "ab99ffee00112233" // shares the 1-byte prefix "ab" with hopTarget + hopListener = "ab12ffff00000000" // shares the 2-byte prefix "ab12", listener only + hopBridge = "88cc000000000000" // neighbor of hopCollider only + hopBoth = "55dd000000000000" // neighbor of hopTarget and hopCollider + hopTwinA = "99aa000000000000" // shares the 1-byte prefix "99" with hopTwinB + hopTwinB = "99bb000000000000" + hopTimestamp = "2026-09-10T10:00:00Z" +) + +func hopTx(id int, hash string, routeType, payloadType int, paths ...string) *StoreTx { + rt, pt := routeType, payloadType + tx := &StoreTx{ID: id, Hash: hash, FirstSeen: hopTimestamp, RouteType: &rt, PayloadType: &pt} + for i, p := range paths { + tx.Observations = append(tx.Observations, &StoreObs{ID: id*10 + i, TransmissionID: id, PathJSON: p}) + } + return tx +} + +func hopPM() *prefixMap { + return buildPrefixMap([]nodeInfo{ + {PublicKey: hopTarget, Role: "repeater"}, + {PublicKey: hopOther, Role: "repeater"}, + {PublicKey: hopCollider, Role: "repeater"}, + {PublicKey: hopBridge, Role: "repeater"}, + {PublicKey: hopBoth, Role: "repeater"}, + {PublicKey: hopTwinA, Role: "repeater"}, + {PublicKey: hopTwinB, Role: "repeater"}, + }) +} + +// hopGraph: hopOther and hopTwinB neighbor hopTarget, hopBridge neighbors +// hopCollider, hopBoth neighbors both, and hopOther and hopBoth neighbor both +// twins. +func hopGraph() *NeighborGraph { + g := NewNeighborGraph() + now := time.Now() + for _, e := range [][2]string{ + {hopOther, hopTarget}, + {hopTwinB, hopTarget}, + {hopBridge, hopCollider}, + {hopBoth, hopTarget}, + {hopBoth, hopCollider}, + {hopOther, hopTwinA}, + {hopOther, hopTwinB}, + {hopBoth, hopTwinA}, + {hopBoth, hopTwinB}, + } { + g.upsertEdge(e[0], e[1], "", "obs", nil, now) + } + return g +} + +func hopsOf(packets []NodeHopPacket) []int { + out := make([]int, 0, len(packets)) + for _, p := range packets { + out = append(out, p.Hops) + } + return out +} + +func hashesOf(packets []NodeHopPacket) []string { + out := make([]string, 0, len(packets)) + for _, p := range packets { + out = append(out, p.Hash) + } + return out +} + +func TestNodeHopPackets_IndexIsHopCountAtDifferentPositions(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(2, "h2", RouteFlood, PayloadGRP_TXT, `["77AA","AB12"]`), + hopTx(3, "h3", RouteTransportFlood, PayloadGRP_TXT, `["77AA","77AA","77AA","AB12","77AA"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if got, want := hopsOf(packets), []int{0, 1, 3}; !reflect.DeepEqual(got, want) { + t.Fatalf("hops = %v, want %v (zero-based index of the node in the path, no +1)", got, want) + } + if ambiguous != 0 { + t.Errorf("ambiguous = %d, want 0", ambiguous) + } + if packets[1].Hash != "h2" || packets[1].Timestamp != hopTimestamp { + t.Errorf("packet[1] = %+v, want hash h2 and timestamp %s", packets[1], hopTimestamp) + } +} + +func TestNodeHopPackets_Tags(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "unscoped", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(2, "scoped", RouteTransportFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(3, "advert", RouteFlood, PayloadADVERT, `["AB12"]`), + hopTx(4, "scoped-advert", RouteTransportFlood, PayloadADVERT, `["AB12"]`), + } + packets, _ := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + want := map[string][]string{ + "unscoped": {"flood", "unscoped"}, + "scoped": {"flood", "scoped"}, + "advert": {"flood", "unscoped", "advert"}, + "scoped-advert": {"flood", "scoped", "advert"}, + } + if len(packets) != len(want) { + t.Fatalf("got %d packets, want %d", len(packets), len(want)) + } + for _, p := range packets { + if !reflect.DeepEqual(p.Tags, want[p.Hash]) { + t.Errorf("%s tags = %v, want %v", p.Hash, p.Tags, want[p.Hash]) + } + } +} + +func TestNodeHopPackets_DirectRoutesExcluded(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "direct", RouteDirect, PayloadTXT_MSG, `["77AA","AB12"]`), + hopTx(2, "tdirect", RouteTransportDirect, PayloadTXT_MSG, `["AB12"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if len(packets) != 0 || ambiguous != 0 { + t.Fatalf("direct packets must be skipped: packets=%v ambiguous=%d", packets, ambiguous) + } +} + +func TestNodeHopPackets_OneEventPerPacketAcrossObservations(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["77AA","AB12"]`, `["77AA","AB12","77AA"]`, `["77AA"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if got := hopsOf(packets); !reflect.DeepEqual(got, []int{1}) { + t.Fatalf("hops = %v, want [1] (one event per packet hash)", got) + } + if ambiguous != 0 { + t.Errorf("ambiguous = %d, want 0", ambiguous) + } +} + +func TestNodeHopPackets_NotInPathIsNeitherCountedNorAmbiguous(t *testing.T) { + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["77AA"]`, `[]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if len(packets) != 0 || ambiguous != 0 { + t.Fatalf("packets=%v ambiguous=%d, want none", packets, ambiguous) + } +} + +func TestNodeHopPackets_OriginatorNeverForwardsOwnFlood(t *testing.T) { + tx := hopTx(1, "own-advert", RouteFlood, PayloadADVERT, `["AB12"]`) + tx.DecodedJSON = `{"pubKey":"` + hopTarget + `"}` + packets, ambiguous := computeNodeHopPackets(hopTarget, []*StoreTx{tx}, "", hopPM(), hopGraph()) + if len(packets) != 0 || ambiguous != 0 { + t.Fatalf("own advert: packets=%v ambiguous=%d, want none", packets, ambiguous) + } +} + +// A colliding prefix counts for the node only under the ingestor's strict rule +// (cmd/ingestor/path_resolver.go resolvePathWithContext): the previous hop is +// itself identified without a tiebreak, and exactly one candidate is its +// graph neighbor. The server resolver's pick (affinity, GPS, advert count) +// plays no part. +func TestNodeHopPackets_CollisionNeedsStrictNeighborAttribution(t *testing.T) { + advert := func(id int, hash, origin string, path string) *StoreTx { + tx := hopTx(id, hash, RouteFlood, PayloadADVERT, path) + tx.DecodedJSON = `{"pubKey":"` + origin + `"}` + return tx + } + txs := []*StoreTx{ + hopTx(1, "after-neighbor", RouteFlood, PayloadGRP_TXT, `["77","AB"]`), + hopTx(2, "no-previous-hop", RouteFlood, PayloadGRP_TXT, `["AB","77"]`), + hopTx(3, "after-collider-neighbor", RouteFlood, PayloadGRP_TXT, `["88","AB"]`), + hopTx(4, "after-common-neighbor", RouteFlood, PayloadGRP_TXT, `["55","AB"]`), + hopTx(5, "previous-hop-ambiguous", RouteFlood, PayloadGRP_TXT, `["99","AB"]`), + hopTx(6, "previous-hop-repeats", RouteFlood, PayloadGRP_TXT, `["77","88","77","AB"]`), + advert(7, "advert-from-neighbor", hopOther, `["AB"]`), + advert(8, "advert-from-stranger", hopBridge, `["AB"]`), + hopTx(9, "observations-disagree", RouteFlood, PayloadGRP_TXT, `["77","AB"]`, `["88","AB"]`), + hopTx(10, "one-observation-unresolved", RouteFlood, PayloadGRP_TXT, `["77","AB"]`, `["99","AB"]`), + hopTx(11, "observations-without-node", RouteFlood, PayloadGRP_TXT, `["77","88"]`, `["77"]`, `["77","AB"]`), + hopTx(12, "chain-broken", RouteFlood, PayloadGRP_TXT, `["77","99","AB"]`), + hopTx(13, "origin-of-non-advert", RouteFlood, PayloadTXT_MSG, `["AB"]`), + advert(14, "originator-excluded-later", hopTwinA, `["55","99","AB"]`), + } + txs[12].DecodedJSON = `{"pubKey":"` + hopOther + `"}` + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), hopGraph()) + got := map[string]int{} + for _, p := range packets { + got[p.Hash] = p.Hops + } + want := map[string]int{"after-neighbor": 1, "advert-from-neighbor": 0, "one-observation-unresolved": 1, + "observations-without-node": 1, "originator-excluded-later": 2} + if !reflect.DeepEqual(got, want) { + t.Fatalf("attributed = %v, want %v", got, want) + } + if ambiguous != 9 { + t.Errorf("ambiguous = %d, want 9", ambiguous) + } +} + +func TestNodeHopPackets_CollisionWithoutGraphIsAmbiguous(t *testing.T) { + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["77","AB"]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if len(packets) != 0 || ambiguous != 1 { + t.Fatalf("packets=%v ambiguous=%d, want none and 1", packets, ambiguous) + } +} + +func TestNodeHopPackets_PrefixAtSeveralPositionsIsAmbiguous(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "same-path", RouteFlood, PayloadGRP_TXT, `["AB","77","AB"]`), + hopTx(2, "across-obs", RouteFlood, PayloadGRP_TXT, `["AB"]`, `["77","AB"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), hopGraph()) + if len(packets) != 0 { + t.Fatalf("packets = %+v, want none: a node forwards a flood once, so two positions mean a collision", packets) + } + if ambiguous != 2 { + t.Errorf("ambiguous = %d, want 2", ambiguous) + } +} + +func TestNodeHopPackets_ListenerDoesNotMakePrefixAmbiguous(t *testing.T) { + pm := buildPrefixMap([]nodeInfo{ + {PublicKey: hopTarget, Role: "repeater"}, + {PublicKey: hopListener, Role: "repeater"}, + }) + pm.markNonRelay([]string{hopListener}) + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["AB12","77AA"]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", pm, nil) + if got := hopsOf(packets); !reflect.DeepEqual(got, []int{0}) || ambiguous != 0 { + t.Fatalf("hops=%v ambiguous=%d, want [0] and 0", got, ambiguous) + } +} + +// When the node itself is a listener, the only relay candidate for its prefix +// is another node: nothing may be attributed to it. +func TestNodeHopPackets_OnlyRelayCandidateIsAnotherNode(t *testing.T) { + pm := buildPrefixMap([]nodeInfo{ + {PublicKey: hopTarget, Role: "repeater"}, + {PublicKey: hopCollider, Role: "repeater"}, + }) + pm.markNonRelay([]string{hopTarget}) + txs := []*StoreTx{hopTx(1, "h1", RouteFlood, PayloadGRP_TXT, `["AB"]`)} + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", pm, nil) + if len(packets) != 0 || ambiguous != 1 { + t.Fatalf("packets=%v ambiguous=%d, want none and 1", packets, ambiguous) + } +} + +// Uniqueness is per prefix: "ab" collides with hopCollider, "ab12" does not. +func TestNodeHopPackets_UniquenessIsPerPrefixLength(t *testing.T) { + txs := []*StoreTx{ + hopTx(1, "two-byte", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(2, "one-byte", RouteFlood, PayloadGRP_TXT, `["AB"]`), + hopTx(3, "two-byte-again", RouteFlood, PayloadGRP_TXT, `["AB12"]`), + hopTx(4, "one-byte-again", RouteFlood, PayloadGRP_TXT, `["AB"]`), + } + packets, ambiguous := computeNodeHopPackets(hopTarget, txs, "", hopPM(), nil) + if got, want := hashesOf(packets), []string{"two-byte", "two-byte-again"}; !reflect.DeepEqual(got, want) { + t.Fatalf("attributed = %v, want %v", got, want) + } + if ambiguous != 2 { + t.Errorf("ambiguous = %d, want 2", ambiguous) + } +} + +// ─── Store level: live ingest and cold load ──────────────────────────────── + +func hopInsertNode(t *testing.T, db *DB, pk, name string, adverts int) { + t.Helper() + mustExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) + VALUES (?, ?, 'repeater', ?, '2026-01-01', ?)`, pk, name, time.Now().UTC().Format(time.RFC3339), adverts) +} + +func hopInsertTx(t *testing.T, db *DB, id int, hash string, routeType, payloadType int, paths ...string) { + t.Helper() + ts := time.Now().Add(-1 * time.Hour) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (?, 'AA', ?, ?, ?, ?)`, + id, hash, ts.UTC().Format(time.RFC3339), routeType, payloadType) + for _, p := range paths { + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp, resolved_path) VALUES (?, NULL, ?, ?, NULL)`, + id, p, ts.Unix()) + } +} + +func hopLoadedStore(t *testing.T, db *DB) *PacketStore { + t.Helper() + store := NewPacketStore(db, nil) + store.graph.Store(loadNeighborEdgesFromDB(db.conn)) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + if !store.WaitIndexesReady(10 * time.Second) { + t.Fatal("indexes not ready") + } + return store +} + +func hopAnalytics(t *testing.T, store *PacketStore, pk string) *NodeHopAnalyticsResponse { + t.Helper() + resp, err := store.GetNodeHopAnalytics(pk, 7) + if err != nil || resp == nil { + t.Fatalf("GetNodeHopAnalytics(%s): resp=%v err=%v", pk, resp, err) + } + sort.Slice(resp.Packets, func(i, j int) bool { return resp.Packets[i].Hash < resp.Packets[j].Hash }) + return resp +} + +// Review repro: live ingest resolves a colliding hop with a tiebreak (here the +// advert count). That pick must not become a hop count. +func TestNodeHopAnalytics_LiveIngestDoesNotTrustResolverPick(t *testing.T) { + db := setupTestDB(t) + hopInsertNode(t, db, hopTarget, "Target", 500) + hopInsertNode(t, db, hopCollider, "Collider", 1) + store := hopLoadedStore(t, db) + + hopInsertTx(t, db, 1, "collides", RouteFlood, PayloadGRP_TXT, `["AB"]`) + if _, maxID := store.IngestNewFromDB(0, 100); maxID != 1 { + t.Fatalf("IngestNewFromDB maxID = %d, want 1", maxID) + } + + for _, pk := range []string{hopTarget, hopCollider} { + resp := hopAnalytics(t, store, pk) + if len(resp.Packets) != 0 || resp.Ambiguous != 1 { + t.Errorf("%s: packets=%+v ambiguous=%d, want none and 1", pk, resp.Packets, resp.Ambiguous) + } + } +} + +// Live ingest and a cold load of the same DB must give the same answer: hops +// are read from every observation's raw path and attributed from the prefix +// map and neighbor graph, not from what each load path indexed. +func TestNodeHopAnalytics_SameResultAfterRestart(t *testing.T) { + db := setupTestDB(t) + mustExec(t, db, `CREATE TABLE neighbor_edges (node_a TEXT NOT NULL, node_b TEXT NOT NULL, + count INTEGER DEFAULT 1, last_seen TEXT, PRIMARY KEY (node_a, node_b))`) + mustExec(t, db, `INSERT INTO neighbor_edges (node_a, node_b, count, last_seen) VALUES (?, ?, 50, ?)`, + hopOther, hopTarget, time.Now().UTC().Format(time.RFC3339)) + hopInsertNode(t, db, hopTarget, "Target", 1) + hopInsertNode(t, db, hopCollider, "Collider", 500) + hopInsertNode(t, db, hopOther, "Other", 1) + live := hopLoadedStore(t, db) + + hopInsertTx(t, db, 1, "after-neighbor", RouteFlood, PayloadGRP_TXT, `["77","AB"]`, `["77","AB","99"]`) + hopInsertTx(t, db, 2, "no-previous-hop", RouteFlood, PayloadGRP_TXT, `["AB"]`) + hopInsertTx(t, db, 3, "two-byte", RouteTransportFlood, PayloadGRP_TXT, `["77AA","AB12"]`) + hopInsertTx(t, db, 4, "three-byte", RouteFlood, PayloadGRP_TXT, `["77AA88","77AA88","AB12CD"]`) + // The longest observation runs through another branch of the flood. + hopInsertTx(t, db, 5, "not-on-longest-path", RouteFlood, PayloadGRP_TXT, `["77AA","77AA","77AA"]`, `["77AA","AB12"]`) + live.IngestNewFromDB(0, 100) + fromLive := hopAnalytics(t, live, hopTarget) + + restarted := hopAnalytics(t, hopLoadedStore(t, db), hopTarget) + + want := []NodeHopPacket{ + {Hash: "after-neighbor", Hops: 1, Tags: hopTagsUnscoped}, + {Hash: "not-on-longest-path", Hops: 1, Tags: hopTagsUnscoped}, + {Hash: "three-byte", Hops: 2, Tags: hopTagsUnscoped}, + {Hash: "two-byte", Hops: 1, Tags: hopTagsScoped}, + } + for name, resp := range map[string]*NodeHopAnalyticsResponse{"live ingest": fromLive, "cold load": restarted} { + got := make([]NodeHopPacket, len(resp.Packets)) + for i, p := range resp.Packets { + got[i] = NodeHopPacket{Hash: p.Hash, Hops: p.Hops, Tags: p.Tags} + } + if !reflect.DeepEqual(got, want) || resp.Ambiguous != 1 { + t.Errorf("%s: packets=%+v ambiguous=%d, want %+v and 1", name, got, resp.Ambiguous, want) + } + } + if !reflect.DeepEqual(fromLive.Packets, restarted.Packets) || fromLive.Ambiguous != restarted.Ambiguous { + t.Errorf("live ingest %+v/%d differs from cold load %+v/%d", + fromLive.Packets, fromLive.Ambiguous, restarted.Packets, restarted.Ambiguous) + } +} + +// End to end through the route: days window, response shape, 404. +func TestHandleNodeHopAnalytics(t *testing.T) { + db := setupTestDB(t) + recent := time.Now().Add(-1 * time.Hour).Format(time.RFC3339) + recentEpoch := time.Now().Add(-1 * time.Hour).Unix() + old := time.Now().Add(-3 * 24 * time.Hour).Format(time.RFC3339) + oldEpoch := time.Now().Add(-3 * 24 * time.Hour).Unix() + + hopInsertNode(t, db, hopTarget, "Target", 1) + hopInsertNode(t, db, hopOther, "Other", 1) + hopInsertNode(t, db, hopCollider, "Collider", 1) + + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (1, 'AA', 'hop_recent_2', ?, 1, 5)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, NULL, '["77AA","77AA","AB12"]', ?)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (2, 'BB', 'hop_recent_1byte', ?, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (2, NULL, '["AB"]', ?)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (3, 'CC', 'hop_scoped_advert', ?, 0, 4)`, recent) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (3, NULL, '["AB12"]', ?)`, recentEpoch) + mustExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type) VALUES (4, 'DD', 'hop_old', ?, 1, 5)`, old) + mustExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (4, NULL, '["AB12"]', ?)`, oldEpoch) + + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + router := mux.NewRouter() + srv.RegisterRoutes(router) + get := func(url string) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", url, nil)) + return w + } + + srv.store = hopLoadedStore(t, db) + + w := get("/api/nodes/" + hopTarget + "/hop_analytics?days=1") + if w.Code != http.StatusOK { + t.Fatalf("code=%d body=%s", w.Code, w.Body.String()) + } + var resp NodeHopAnalyticsResponse + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + byHash := map[string]NodeHopPacket{} + for _, p := range resp.Packets { + byHash[p.Hash] = p + } + if len(resp.Packets) != 2 { + t.Fatalf("packets = %+v, want the two attributable packets inside the 1-day window", resp.Packets) + } + if p := byHash["hop_recent_2"]; p.Hops != 2 || p.Timestamp != recent || !reflect.DeepEqual(p.Tags, []string{"flood", "unscoped"}) { + t.Errorf("hop_recent_2 = %+v, want hops 2, timestamp %s, tags [flood unscoped]", p, recent) + } + if p := byHash["hop_scoped_advert"]; p.Hops != 0 || !reflect.DeepEqual(p.Tags, []string{"flood", "scoped", "advert"}) { + t.Errorf("hop_scoped_advert = %+v, want hops 0 tags [flood scoped advert]", p) + } + if resp.TimeRange.Days != 1 || resp.Ambiguous != 1 { + t.Errorf("timeRange.days=%d ambiguous=%d, want 1 and 1 (colliding 1-byte prefix, no neighbor graph)", resp.TimeRange.Days, resp.Ambiguous) + } + + if w := get("/api/nodes/" + hopTarget + "/hop_analytics?days=7"); w.Code != http.StatusOK || !json.Valid(w.Body.Bytes()) { + t.Fatalf("days=7: code=%d", w.Code) + } else { + var wide NodeHopAnalyticsResponse + _ = json.Unmarshal(w.Body.Bytes(), &wide) + if len(wide.Packets) != 3 { + t.Errorf("days=7 packets = %d, want 3", len(wide.Packets)) + } + } + + if w := get("/api/nodes/ffffffffffffffff/hop_analytics"); w.Code != http.StatusNotFound { + t.Errorf("unknown node: code=%d, want 404", w.Code) + } +} + +// BenchmarkNodeHopPackets sizes one request for a busy repeater over 7 days, +// as counted in a 1,669-node mesh DB on 2026-09-13: 73,782 flood packets +// with 1,430,280 observations (19 per packet). A third of the packets pass the +// node under a unique 2-byte prefix, a sixth under a 1-byte prefix shared with +// another relay (strict neighbor walk), the rest do not pass it; in each, half +// of the observations contain the node. Observation slices are shared between +// packets to keep the fixture small; the scan only reads them. +func BenchmarkNodeHopPackets(b *testing.B) { + obsOf := func(self, other string) []*StoreObs { + through := `["` + other + `","` + self + `","` + other + `","` + other + `","` + other + `","` + other + `"]` + elsewhere := `["` + other + `","` + other + `","` + other + `","` + other + `","` + other + `","` + other + `"]` + obs := make([]*StoreObs, 19) + for j := range obs { + obs[j] = &StoreObs{PathJSON: elsewhere} + if j%2 == 0 { + obs[j].PathJSON = through + } + } + return obs + } + unique, colliding, elsewhere := obsOf("AB12", "77AA"), obsOf("AB", "77"), obsOf("88CC", "77AA") + rt, pt := RouteFlood, PayloadGRP_TXT + txs := make([]*StoreTx, 73782) + for i := range txs { + txs[i] = &StoreTx{ID: i, Hash: "h", FirstSeen: hopTimestamp, RouteType: &rt, PayloadType: &pt, Observations: elsewhere} + switch i % 6 { + case 0, 1: + txs[i].Observations = unique + case 2: + txs[i].Observations = colliding + } + } + pm, graph := hopPM(), hopGraph() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + computeNodeHopPackets(hopTarget, txs, "", pm, graph) + } +} diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index f73ce84fe..da6522cc2 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -87,6 +87,10 @@ func routeDescriptions() map[string]routeMeta { "GET /api/nodes/{pubkey}/health": {Summary: "Get node health", Tag: "nodes"}, "GET /api/nodes/{pubkey}/paths": {Summary: "Get node routing paths", Tag: "nodes"}, "GET /api/nodes/{pubkey}/analytics": {Summary: "Get node analytics", Description: "Per-node packet counts, timing, and RF stats.", Tag: "nodes"}, + "GET /api/nodes/{pubkey}/hop_analytics": {Summary: "Get node hop counts", Description: "One entry per flood packet the node forwarded in the window, with the hop count its flood.max check saw (the node's zero-based index in the path) and tags (flood, scoped or unscoped, advert). DIRECT packets are excluded. Every observation in the window is read. A colliding path prefix is attributed only when the previous hop's neighbor_edges neighbors leave this node as the one candidate; the server's resolved-path pick is not used. Packets carrying this node's prefix that cannot be attributed are counted in `ambiguous`.", Tag: "nodes", + QueryParams: []paramMeta{ + {Name: "days", Description: "Lookback window in days (default 7, clamped 1-365)", Type: "integer"}, + }}, "GET /api/nodes/{pubkey}/neighbors": {Summary: "Get node neighbors", Description: "Returns the queried node's first-hop neighbors with affinity scores and observation metadata (count, SNR, distance, observers). Ambiguous edges carry candidate pubkeys.", Tag: "nodes", Response: schemaRef("NodeNeighborsResponse")}, "GET /api/scope-audit": {Summary: "Network-wide scope audit", Description: "For every repeater that has answered a declared-regions request: the regions it declares, which of those it has NOT been observed forwarding in the window, which scopes it forwards without declaring, and whether it forwards unscoped floods while omitting the '*' wildcard. '*' is never listed as a region — it governs unscoped floods, not a scope. Repeaters never successfully asked are absent rather than shown as declaring nothing. Rows with missing regions sort first; a short window is weak evidence, since a quiet region simply has no traffic.", Tag: "analytics", diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 63565bc40..e799b9ecf 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -296,6 +296,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/nodes/{pubkey}/health", s.handleNodeHealth).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}/paths", s.handleNodePaths).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}/analytics", s.handleNodeAnalytics).Methods("GET") + r.HandleFunc("/api/nodes/{pubkey}/hop_analytics", s.handleNodeHopAnalytics).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}/battery", s.handleNodeBattery).Methods("GET") r.HandleFunc("/api/nodes/clock-skew", s.handleFleetClockSkew).Methods("GET") r.HandleFunc("/api/nodes/{pubkey}/clock-skew", s.handleNodeClockSkew).Methods("GET") @@ -2080,13 +2081,7 @@ func (s *Server) handleNodeAnalytics(w http.ResponseWriter, r *http.Request) { writeError(w, 404, "Not found") return } - days := queryInt(r, "days", 7) - if days < 1 { - days = 1 - } - if days > 365 { - days = 365 - } + days := nodeAnalyticsDays(r) if s.store != nil { result, err := s.store.GetNodeAnalytics(pubkey, days) @@ -2101,6 +2096,29 @@ func (s *Server) handleNodeAnalytics(w http.ResponseWriter, r *http.Request) { writeError(w, 404, "Not found") } +// nodeAnalyticsDays reads the node analytics range picker's ?days= (default 7, +// clamped to 1-365). +func nodeAnalyticsDays(r *http.Request) int { + return min(max(queryInt(r, "days", 7), 1), 365) +} + +// handleNodeHopAnalytics serves the hop count at this node for each flood +// packet it forwarded (issue #1812). Separate from /analytics so the hop +// scan does not slow down the main analytics response. +func (s *Server) handleNodeHopAnalytics(w http.ResponseWriter, r *http.Request) { + pubkey := mux.Vars(r)["pubkey"] + if s.cfg.IsBlacklisted(pubkey) || s.isPubkeyHidden(pubkey) || s.store == nil { + writeError(w, 404, "Not found") + return + } + result, err := s.store.GetNodeHopAnalytics(pubkey, nodeAnalyticsDays(r)) + if err != nil || result == nil { + writeError(w, 404, "Not found") + return + } + writeJSON(w, result) +} + func (s *Server) handleNodeClockSkew(w http.ResponseWriter, r *http.Request) { pubkey := mux.Vars(r)["pubkey"] if s.store == nil { diff --git a/cmd/server/store.go b/cmd/server/store.go index 6d2f697fd..6b0b25463 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -6798,6 +6798,31 @@ func (s *PacketStore) InvalidateNodeCache() { s.cacheMu.Unlock() } +// relayCandidates returns the nodes whose pubkey starts with hop and that may +// appear as a path hop. +// +// Issue #1290: observer-known listener-only nodes are dropped from the +// candidate set. By firmware contract a node that advertises `repeat:off` in +// its MQTT /status will never relay a packet, so it cannot legitimately be a +// hop in someone else's path. Filtering shrinks ambiguous candidate sets +// without affecting any upstream caller (only no_match becomes more likely +// when the only matching prefix belonged to a listener). Empty pm.nonRelay +// preserves the pre-#1290 behavior exactly (back-compat). +func (pm *prefixMap) relayCandidates(hop string) []nodeInfo { + candidates := pm.m[strings.ToLower(hop)] + if len(pm.nonRelay) == 0 || len(candidates) == 0 { + return candidates + } + filtered := candidates[:0:0] + for i := range candidates { + if _, isListener := pm.nonRelay[strings.ToLower(candidates[i].PublicKey)]; isListener { + continue + } + filtered = append(filtered, candidates[i]) + } + return filtered +} + func (pm *prefixMap) resolve(hop string) *nodeInfo { h := strings.ToLower(hop) candidates := pm.m[h] @@ -6840,27 +6865,7 @@ func (pm *prefixMap) resolve(hop string) *nodeInfo { // (e.g., the originator, observer, or adjacent hops in the path). // graph may be nil, in which case tier-1 is skipped. func (pm *prefixMap) resolveWithContext(hop string, contextPubkeys []string, graph *NeighborGraph) (*nodeInfo, string, float64) { - h := strings.ToLower(hop) - candidates := pm.m[h] - // Issue #1290: drop observer-known listener-only nodes from the - // candidate set. By firmware contract a node that advertises - // `repeat:off` in its MQTT /status will never relay a packet, so it - // cannot legitimately be a hop in someone else's path. Filtering - // here shrinks ambiguous candidate sets without affecting any - // upstream caller (the returned shape and confidence labels are - // preserved; only no_match becomes more likely when the only - // matching prefix belonged to a listener). Empty pm.nonRelay - // preserves the pre-#1290 behavior exactly (back-compat). - if len(pm.nonRelay) > 0 && len(candidates) > 0 { - filtered := candidates[:0:0] - for i := range candidates { - if _, isListener := pm.nonRelay[strings.ToLower(candidates[i].PublicKey)]; isListener { - continue - } - filtered = append(filtered, candidates[i]) - } - candidates = filtered - } + candidates := pm.relayCandidates(hop) if len(candidates) == 0 { return nil, "no_match", 0 } @@ -9638,6 +9643,21 @@ func (s *PacketStore) GetNodeHealth(pubkey string) (map[string]interface{}, erro }, nil } +// nodeTxsSince returns the node's transmissions first seen after fromISO, +// from the byNode index (sender, recipient and resolved relay hops). +// Raw JSON text search is intentionally avoided: a GRP_TXT packet whose message +// text contains a node's pubkey is not a packet *for* that node. +// Must be called with s.mu held. +func (s *PacketStore) nodeTxsSince(pubkey, fromISO string) []*StoreTx { + var packets []*StoreTx + for _, p := range s.byNode[pubkey] { + if p.FirstSeen > fromISO { + packets = append(packets, p) + } + } + return packets +} + // GetNodeAnalytics computes analytics for a single node using in-memory byNode index. func (s *PacketStore) GetNodeAnalytics(pubkey string, days int) (*NodeAnalyticsResponse, error) { node, err := s.db.GetNodeByPubkey(pubkey) @@ -9652,16 +9672,7 @@ func (s *PacketStore) GetNodeAnalytics(pubkey string, days int) (*NodeAnalyticsR s.mu.RLock() defer s.mu.RUnlock() - // Collect packets from byNode index (time-filtered). - // Raw JSON text search is intentionally avoided: a GRP_TXT packet whose message - // text contains a node's pubkey is not a packet *for* that node. - indexed := s.byNode[pubkey] - var packets []*StoreTx - for _, p := range indexed { - if p.FirstSeen > fromISO { - packets = append(packets, p) - } - } + packets := s.nodeTxsSince(pubkey, fromISO) // Activity timeline (hourly buckets) timelineBuckets := map[string]int{} diff --git a/cmd/server/types.go b/cmd/server/types.go index e6ab4419f..40b3fa0bc 100644 --- a/cmd/server/types.go +++ b/cmd/server/types.go @@ -576,6 +576,21 @@ type NodeAnalyticsResponse struct { ClockSkew *NodeClockSkew `json:"clockSkew,omitempty"` } +// NodeHopPacket is one flood packet this node forwarded, with the hop count +// the node's flood.max check saw for it (issue #1812). +type NodeHopPacket struct { + Hash string `json:"hash"` + Timestamp string `json:"timestamp"` + Hops int `json:"hops"` + Tags []string `json:"tags"` +} + +type NodeHopAnalyticsResponse struct { + TimeRange TimeRangeResp `json:"timeRange"` + Packets []NodeHopPacket `json:"packets"` + Ambiguous int `json:"ambiguous"` +} + // ─── Analytics — RF ──────────────────────────────────────────────────────────── type PayloadTypeSignal struct { diff --git a/docs/api-spec.md b/docs/api-spec.md index 0205e8539..3d2c19cd4 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -23,6 +23,7 @@ - [GET /api/nodes/:pubkey/health](#get-apinodespubkeyhealth) - [GET /api/nodes/:pubkey/paths](#get-apinodespubkeypaths) - [GET /api/nodes/:pubkey/analytics](#get-apinodespubkeyanalytics) +- [GET /api/nodes/:pubkey/hop_analytics](#get-apinodespubkeyhop_analytics) - [GET /api/nodes/:pubkey/reach](#get-apinodespubkeyreach) - [GET /api/packets](#get-apipackets) - [GET /api/packets/timestamps](#get-apipacketstimestamps) @@ -675,6 +676,71 @@ Per-node analytics over a time range. --- +## GET /api/nodes/:pubkey/hop_analytics + +Hop count at this node for every flood packet it forwarded, to help choose +`flood.max`, `flood.max.unscoped` and `flood.max.advert`. A repeater checks +those limits against the number of hashes already in the path, then appends +its own hash, so the hop count is the node's zero-based index in the observed +path (firmware `src/helpers/RoutingPolicy.h`, `src/Mesh.cpp` `routeRecvPacket`). +This is not the `hopDistribution` of `/analytics`, which is the path length at +the observer. + +- One entry per packet hash. Values are raw so the client can filter and bin. +- Only floods (route types 0 and 1). DIRECT packets carry the remaining route, + not a hop count, and are left out. Packets the node originated are left out. +- Every observation of every flood packet in the window is read, not only the + packet's longest path, so a relay on a shorter branch of the flood counts too. +- A packet is attributed when the node's path prefix sits at exactly one index + across its observations, and either no other relay-capable node shares that + prefix, or the hop resolves to the node under the ingestor's strict rule + (every earlier hop identified without a tiebreak, and exactly one candidate + is a `neighbor_edges` neighbor of the previous hop, or of the originator for + an advert) in at least one observation and to another node in none. The + server's resolved-path pick (affinity, GPS distance, advert count) is not + used, so the result is the same before and after a restart. Everything else + with the node's prefix is counted in `ambiguous` and left out; in practice + that is most packets with a colliding 1-byte path hash. +- Size: for a busy repeater on a 1,669-node mesh over 7 days (2026-09-13) + the response held 23,068 entries, 2.3 MB of JSON, 375 KB gzipped. `hash` + and `timestamp` are 61% of the raw and 91% of the gzipped bytes; they stay + so a client can join entries to packets and bin by time (issue #1812). + +### Query Parameters + +| Param | Type | Default | Description | +|--------|--------|---------|--------------------------| +| `days` | number | `7` | Lookback window (1-365) | + +### Response `200` + +```jsonc +{ + "timeRange": { "from": string (ISO), "to": string (ISO), "days": number }, + "packets": [ + { + "hash": string, + "timestamp": string (ISO), // first seen + "hops": number, // 0 = heard straight from the originator + "tags": [string] // "flood", then "scoped" or "unscoped", then "advert" if an ADVERT + } + ], + "ambiguous": number // prefix matched, hop position not attributable to this node +} +``` + +Filters that match the firmware limits: `flood.max` uses all entries, +`flood.max.unscoped` the entries tagged `unscoped`, `flood.max.advert` the +entries tagged `advert`. + +### Response `404` + +```json +{ "error": "Not found" } +``` + +--- + ## GET /api/nodes/:pubkey/reach Per-node RF reach report (two-way link quality). Computes **directional** link counts from raw diff --git a/public/index.html b/public/index.html index 2a5f05d95..715c74c47 100644 --- a/public/index.html +++ b/public/index.html @@ -225,6 +225,7 @@ + diff --git a/public/node-analytics.js b/public/node-analytics.js index 7f209d5ca..3344bab28 100644 --- a/public/node-analytics.js +++ b/public/node-analytics.js @@ -124,6 +124,11 @@
How many repeater hops packets take — 0 means direct
+
+

Hop Count at This Node

+
For each flood packet this node forwarded: how many hops it already had, the number the flood.max, flood.max.unscoped and flood.max.advert limits are checked against. Bars count packets per hop count; the box above spans the middle half of the packets with a line at the median, and the whiskers reach the furthest packet within 1.5 box widths of the box.
+
Loading hop counts...
+

Battery Voltage

Battery voltage over time from observer status reports — flat line means full, downward slope means draining
@@ -166,6 +171,7 @@ buildHopChart(data); buildHeatmap(data); loadBatteryChart(pubkey, currentDays); + NodeHopAnalytics.load(document.getElementById('hopCountSection'), pubkey, currentDays); } function buildActivityChart(data) { @@ -370,6 +376,7 @@ function destroy() { destroyCharts(); + NodeHopAnalytics.destroy(); currentPubkey = null; } diff --git a/public/node-hop-analytics.js b/public/node-hop-analytics.js new file mode 100644 index 000000000..6bd5575ab --- /dev/null +++ b/public/node-hop-analytics.js @@ -0,0 +1,178 @@ +/* === CoreScope: node-hop-analytics.js === */ +'use strict'; +// Hop count at this node (issue #1812): for every flood packet the node +// forwarded, the number of hops the packet already had when the node's +// flood.max check ran. Data: GET /api/nodes/{pubkey}/hop_analytics?days=N. +(function () { + // One filter per repeater CLI limit (firmware src/helpers/RoutingPolicy.h): + // flood.max applies to every flood, flood.max.unscoped to un-scoped floods, + // flood.max.advert to adverts. + const FILTERS = [ + { id: 'flood', label: 'flood.max', tag: null, noun: 'flood packets' }, + { id: 'flood_unscoped', label: 'flood.max.unscoped', tag: 'unscoped', noun: 'un-scoped flood packets' }, + { id: 'flood_adverts', label: 'flood.max.advert', tag: 'advert', noun: 'flood adverts' }, + ]; + + let chart = null; + + function filterById(id) { + return FILTERS.find(f => f.id === id) || FILTERS[0]; + } + + function filterHops(packets, filterId) { + const tag = filterById(filterId).tag; + const out = []; + for (const p of packets || []) { + if (!tag || (p.tags && p.tags.indexOf(tag) >= 0)) out.push(Number(p.hops)); + } + return out; + } + + function hopHistogram(values) { + let max = -1; + for (const v of values) if (v > max) max = v; + const counts = new Array(max + 1).fill(0); + for (const v of values) counts[v]++; + return counts; + } + + // Quartiles by linear interpolation between closest ranks, Tukey whiskers at + // 1.5 IQR. Hop counts are small integers, so sorting a copy stays cheap. + function hopBoxStats(values) { + if (!values.length) return null; + const s = values.slice().sort((a, b) => a - b); + const q = p => { + const h = (s.length - 1) * p; + const lo = Math.floor(h); + return lo + 1 < s.length ? s[lo] + (h - lo) * (s[lo + 1] - s[lo]) : s[lo]; + }; + const q1 = q(0.25), median = q(0.5), q3 = q(0.75); + const lowFence = q1 - 1.5 * (q3 - q1), highFence = q3 + 1.5 * (q3 - q1); + let whiskerLow = q1, whiskerHigh = q3, outliers = 0; + for (const v of s) { + if (v < lowFence || v > highFence) { outliers++; continue; } + if (v < whiskerLow) whiskerLow = v; + if (v > whiskerHigh) whiskerHigh = v; + } + return { n: s.length, min: s[0], q1, median, q3, max: s[s.length - 1], whiskerLow, whiskerHigh, outliers }; + } + + function plural(n, word) { + return n + ' ' + word + (n === 1 ? '' : 's'); + } + + function renderHopSection(data, filterId) { + const filter = filterById(filterId); + const values = filterHops(data.packets, filter.id); + const stats = hopBoxStats(values); + const ambiguous = Number(data.ambiguous) || 0; + const chips = FILTERS.map(f => + `` + ).join(''); + const summary = stats + ? `${plural(stats.n, 'packet')} (${filter.noun}) · median ${stats.median} · middle half ${stats.q1} to ${stats.q3} · max ${stats.max} hops` + : ''; + const body = stats + ? `` + : `
No forwarded ${filter.noun} attributed to this node in this window.
`; + const note = ambiguous + ? `
${plural(ambiguous, 'packet')} left out: the path prefix of this node is shared with another node there, so its hop position is unknown.
` + : ''; + return ` +
${chips}
+
${summary}
+ ${body} + ${note}`; + } + + function cssVar(name) { + return getComputedStyle(document.documentElement).getPropertyValue(name).trim(); + } + + // Box plot drawn in the chart's top padding, on the histogram's own x axis: + // the category scale puts hop count h at index h, so fractional quartiles are + // interpolated between neighbouring category centres. + function boxPlotPlugin(stats) { + return { + id: 'hopBoxPlot', + afterDatasetsDraw(c) { + const x = c.scales.x; + const px = v => { + const lo = Math.floor(v); + const a = x.getPixelForValue(lo); + return v === lo ? a : a + (v - lo) * (x.getPixelForValue(lo + 1) - a); + }; + const ctx = c.ctx; + const mid = c.chartArea.top - 16; + ctx.save(); + ctx.strokeStyle = cssVar('--text'); + ctx.fillStyle = cssVar('--accent-bg'); + ctx.lineWidth = 1.5; + ctx.beginPath(); + ctx.moveTo(px(stats.whiskerLow), mid); ctx.lineTo(px(stats.q1), mid); + ctx.moveTo(px(stats.q3), mid); ctx.lineTo(px(stats.whiskerHigh), mid); + ctx.moveTo(px(stats.whiskerLow), mid - 5); ctx.lineTo(px(stats.whiskerLow), mid + 5); + ctx.moveTo(px(stats.whiskerHigh), mid - 5); ctx.lineTo(px(stats.whiskerHigh), mid + 5); + ctx.stroke(); + ctx.fillRect(px(stats.q1), mid - 8, px(stats.q3) - px(stats.q1), 16); + ctx.strokeRect(px(stats.q1), mid - 8, px(stats.q3) - px(stats.q1), 16); + ctx.beginPath(); + ctx.moveTo(px(stats.median), mid - 8); ctx.lineTo(px(stats.median), mid + 8); + ctx.stroke(); + ctx.restore(); + } + }; + } + + function drawChart(data, filterId) { + if (chart) { chart.destroy(); chart = null; } + const canvas = document.getElementById('hopCountChart'); + if (!canvas) return; + const values = filterHops(data.packets, filterId); + const counts = hopHistogram(values); + chart = new Chart(canvas, { + type: 'bar', + data: { + labels: counts.map((_, h) => String(h)), + datasets: [{ label: 'Packets', data: counts, backgroundColor: cssVar('--accent'), borderWidth: 0 }] + }, + options: { + responsive: true, + layout: { padding: { top: 32 } }, + plugins: { legend: { display: false } }, + scales: { + x: { title: { display: true, text: 'Hops already in the path when this node forwarded' } }, + y: { beginAtZero: true, title: { display: true, text: 'Packets' } } + } + }, + plugins: [boxPlotPlugin(hopBoxStats(values))] + }); + } + + function render(container, data, filterId) { + container.innerHTML = renderHopSection(data, filterId); + container.querySelectorAll('[data-hop-filter]').forEach(btn => { + btn.addEventListener('click', () => render(container, data, btn.dataset.hopFilter)); + }); + drawChart(data, filterId); + } + + async function load(container, pubkey, days) { + destroy(); + let data; + try { + data = await api('/nodes/' + encodeURIComponent(pubkey) + '/hop_analytics?days=' + days, { ttl: CLIENT_TTL.nodeAnalytics }); + } catch (e) { + container.innerHTML = '
Hop counts unavailable: ' + escapeHtml(e.message) + '
'; + return; + } + if (!container.isConnected) return; + render(container, data, 'flood'); + } + + function destroy() { + if (chart) { chart.destroy(); chart = null; } + } + + window.NodeHopAnalytics = { FILTERS, filterHops, hopHistogram, hopBoxStats, renderHopSection, load, destroy }; +})(); diff --git a/test-all.sh b/test-all.sh index 349586c3c..1563a1831 100755 --- a/test-all.sh +++ b/test-all.sh @@ -140,6 +140,7 @@ node test-map-clustering.js node test-mqtt-status-panel.js node test-my-repeaters-dashboard.js node test-naive-banner-tone.js +node test-node-hop-analytics.js node test-node-reach-coverage-debounce.js node test-node-reach-coverage.js node test-nodes-export-wiring.js diff --git a/test-node-hop-analytics.js b/test-node-hop-analytics.js new file mode 100644 index 000000000..b5c7972cc --- /dev/null +++ b/test-node-hop-analytics.js @@ -0,0 +1,84 @@ +'use strict'; +// Issue #1812: hop count at this node, node analytics page. Loads the browser +// IIFE in a vm sandbox (pattern from test-node-reach-coverage.js) and exercises +// the filter, histogram, box statistics and section render. +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +const code = fs.readFileSync(path.join(__dirname, 'public', 'node-hop-analytics.js'), 'utf8'); +const sandbox = { window: {}, console }; +vm.createContext(sandbox); +vm.runInContext(code, sandbox); +const H = sandbox.window.NodeHopAnalytics; + +let passed = 0; +function test(name, fn) { + fn(); + passed++; + console.log(' ok ' + name); +} + +const packets = [ + { hash: 'a', hops: 0, tags: ['flood', 'unscoped'] }, + { hash: 'b', hops: 2, tags: ['flood', 'scoped'] }, + { hash: 'c', hops: 5, tags: ['flood', 'unscoped', 'advert'] }, + { hash: 'd', hops: 1, tags: ['flood', 'scoped', 'advert'] }, +]; + +test('filters follow the firmware limits: flood.max all, .unscoped and .advert by tag', () => { + assert.deepStrictEqual(Array.from(H.filterHops(packets, 'flood')), [0, 2, 5, 1]); + assert.deepStrictEqual(Array.from(H.filterHops(packets, 'flood_unscoped')), [0, 5]); + assert.deepStrictEqual(Array.from(H.filterHops(packets, 'flood_adverts')), [5, 1]); + assert.deepStrictEqual(Array.from(H.filterHops(packets, 'bogus')), [0, 2, 5, 1], 'unknown filter falls back to flood'); +}); + +test('histogram has one bucket per hop count from 0 to max', () => { + assert.deepStrictEqual(Array.from(H.hopHistogram([0, 1, 1, 3])), [1, 2, 0, 1]); + assert.deepStrictEqual(Array.from(H.hopHistogram([])), []); +}); + +test('box stats: interpolated quartiles, 1.5 IQR whiskers, outliers counted', () => { + // 8 lies between the 1.5 IQR fence (6) and a 3 IQR fence (9), so the + // fence factor decides whether it is an outlier. + const s = H.hopBoxStats([8, 0, 1, 3, 1]); + assert.strictEqual(s.n, 5); + assert.strictEqual(s.min, 0); + assert.strictEqual(s.q1, 1); + assert.strictEqual(s.median, 1); + assert.strictEqual(s.q3, 3); + assert.strictEqual(s.max, 8); + assert.strictEqual(s.whiskerLow, 0); + assert.strictEqual(s.whiskerHigh, 3, 'upper fence is q3 + 1.5*IQR = 6, so the whisker stops at 3'); + assert.strictEqual(s.outliers, 1); + + const even = H.hopBoxStats([4, 1, 3, 2]); + assert.strictEqual(even.q1, 1.75); + assert.strictEqual(even.median, 2.5); + assert.strictEqual(even.q3, 3.25); + assert.strictEqual(H.hopBoxStats([]), null); +}); + +test('section render: firmware-named chips, active filter, summary and ambiguous note', () => { + const html = H.renderHopSection({ packets, ambiguous: 2 }, 'flood_adverts'); + assert.ok(html.includes('>flood.max<'), 'flood.max chip'); + assert.ok(html.includes('>flood.max.unscoped<'), 'flood.max.unscoped chip'); + assert.ok(html.includes('>flood.max.advert<'), 'flood.max.advert chip'); + assert.ok(/data-hop-filter="flood_adverts"[^>]*aria-pressed="true"/.test(html), 'selected chip is pressed'); + assert.ok(/data-hop-filter="flood"[^>]*aria-pressed="false"/.test(html), 'other chips are not pressed'); + assert.ok(html.includes('2 packets'), 'packet count for the advert filter'); + assert.ok(html.includes('median 3'), 'median of [5, 1]'); + assert.ok(html.includes('max 5'), 'max of [5, 1]'); + assert.ok(html.includes('id="hopCountChart"'), 'chart canvas'); + assert.ok(html.includes('2 packets left out'), 'ambiguous packets are reported'); +}); + +test('section render: empty filter shows a message instead of a chart', () => { + const html = H.renderHopSection({ packets: [packets[1]], ambiguous: 0 }, 'flood_unscoped'); + assert.ok(!html.includes('id="hopCountChart"'), 'no canvas without data'); + assert.ok(html.includes('No forwarded'), 'empty message'); + assert.ok(!html.includes('left out'), 'no ambiguous note when zero'); +}); + +console.log('node-hop-analytics: ' + passed + ' tests passed'); From 2c6d7bb6ae3d9d4f18ed333df9355db2ed7277b6 Mon Sep 17 00:00:00 2001 From: Jonathan Herlin Date: Sun, 13 Sep 2026 20:36:13 +0200 Subject: [PATCH 036/151] feat(packets): add filter to All Observer dropdown (#1884) Lets users type a prefix to filter the observer checkbox list, instead of scrolling a long list. --------- Co-authored-by: efiten Co-authored-by: Claude Opus 5 (1M context) --- public/packets.js | 47 ++++++++- public/style.css | 10 ++ test-all.sh | 2 + test-clear-filters.js | 5 +- test-observer-menu-interactions.js | 148 +++++++++++++++++++++++++++++ test-observer-search-filter.js | 119 +++++++++++++++++++++++ 6 files changed, 326 insertions(+), 5 deletions(-) create mode 100644 test-observer-menu-interactions.js create mode 100644 test-observer-search-filter.js diff --git a/public/packets.js b/public/packets.js index 19200572c..b27566536 100644 --- a/public/packets.js +++ b/public/packets.js @@ -1678,7 +1678,12 @@
-
+
+
+ +
+
+
@@ -1789,8 +1794,23 @@ // --- Observer multi-select --- const obsMenu = document.getElementById('observerMenu'); + const obsList = document.getElementById('observerList'); + const obsSearchInput = document.getElementById('observerSearchInput'); const obsTrigger = document.getElementById('observerTrigger'); const selectedObservers = new Set(filters.observer ? filters.observer.split(',') : []); + function applyObserverSearchFilter() { + const raw = (obsSearchInput.value || '').trim().toLowerCase(); + // #1884 — default to substring matching so "brussels" finds "ON4XYZ + // Brussels"; a leading ^ opts into prefix-only matching for narrowing + // down a shared prefix like "BE". + const anchored = raw.startsWith('^'); + const term = anchored ? raw.slice(1) : raw; + obsList.querySelectorAll('.multi-select-item[data-obs-name]').forEach((item) => { + const name = item.dataset.obsName; + const matches = !term || (anchored ? name.startsWith(term) : name.includes(term)); + item.style.display = matches ? '' : 'none'; + }); + } function buildObserverMenu() { const allChecked = selectedObservers.size === 0; let html = ``; @@ -1802,11 +1822,15 @@ } else { for (const o of observers) { const checked = selectedObservers.has(String(o.id)) ? 'checked' : ''; - html += ``; + const name = o.name || String(o.id); + html += ``; } } - obsMenu.innerHTML = html; + obsList.innerHTML = html; + applyObserverSearchFilter(); } + obsSearchInput.addEventListener('click', (e) => e.stopPropagation()); + obsSearchInput.addEventListener('input', applyObserverSearchFilter); // #1693 — expose for loadObservers() to refresh on resolve. _rebuildObserverMenu = () => { buildObserverMenu(); updateObsTrigger(); }; function updateObsTrigger() { @@ -1822,9 +1846,22 @@ } buildObserverMenu(); updateObsTrigger(); - obsTrigger.addEventListener('click', (e) => { e.stopPropagation(); obsMenu.classList.toggle('open'); typeMenu.classList.remove('open'); }); + obsTrigger.addEventListener('click', (e) => { + e.stopPropagation(); + obsMenu.classList.toggle('open'); + typeMenu.classList.remove('open'); + // #1884 — don't autofocus on touch devices; it raises the on-screen + // keyboard over the list the user is about to tap. + const isTouch = window.matchMedia('(pointer: coarse)').matches; + if (obsMenu.classList.contains('open') && !isTouch) obsSearchInput.focus(); + }); obsMenu.addEventListener('change', (e) => { const id = e.target.dataset.obsId; + // #1884 — obsSearchInput lives inside obsMenu, so its own change + // events (blur/Enter) bubble here too; without this guard they run + // the else branch below and rebuild the list mid-click, dropping + // whatever checkbox the user just pressed. + if (!id) return; if (id === '__all__') { selectedObservers.clear(); } else { @@ -2000,6 +2037,8 @@ var obMenu = document.getElementById('observerMenu'); if (obMenu) obMenu.querySelectorAll('input[type=checkbox]').forEach(function(cb) { cb.checked = false; }); document.getElementById('observerTrigger').textContent = 'All Observers ▾'; + obsSearchInput.value = ''; + applyObserverSearchFilter(); // Reset type multi-select var typeMenu = document.getElementById('typeMenu'); diff --git a/public/style.css b/public/style.css index d3e49aeca..b541f3f7a 100644 --- a/public/style.css +++ b/public/style.css @@ -3552,6 +3552,16 @@ button.region-pill-active:hover { opacity: 0.85; color: var(--text-on-accent); } box-shadow: 0 4px 16px rgba(0,0,0,0.12); padding: 4px 0; display: none; } .multi-select-menu.open { display: block; } +.multi-select-search-wrap { + position: sticky; top: 0; z-index: 1; background: var(--card-bg, #fff); + padding: 4px 8px 6px; border-bottom: 1px solid var(--border); +} +.multi-select-search { + width: 100%; box-sizing: border-box; padding: 5px 8px; border-radius: 5px; + border: 1px solid var(--border); background: var(--input-bg); color: var(--text); + font-size: 13px; +} +.multi-select-search:focus { border-color: var(--accent); outline: none; } .multi-select-item { display: flex; align-items: center; gap: 6px; padding: 6px 12px; font-size: 13px; cursor: pointer; color: var(--text); white-space: nowrap; diff --git a/test-all.sh b/test-all.sh index 1563a1831..9e15b7285 100755 --- a/test-all.sh +++ b/test-all.sh @@ -146,7 +146,9 @@ node test-node-reach-coverage.js node test-nodes-export-wiring.js node test-nodes-export.js node test-observer-iata-1188.js +node test-observer-menu-interactions.js node test-observer-naive-clock-1478.js +node test-observer-search-filter.js node test-observers-headings.js node test-packet-filter-time.js node test-packet-filter-ux.js diff --git a/test-clear-filters.js b/test-clear-filters.js index b8c9184d0..2340bf9f0 100644 --- a/test-clear-filters.js +++ b/test-clear-filters.js @@ -124,7 +124,10 @@ function extractClearHandler() { else if (src[i] === '}') { depth--; if (depth === 0) { fnEnd = i; break; } } } assert(fnEnd > fnStart, 'could not find end of clear handler'); - return src.substring(fnStart + 1, fnEnd); + // The handler also resets the observer search box (#1884). These cases do + // not test it (test-observer-menu-interactions.js does), so give it stand-ins. + const searchStubs = "const obsSearchInput = { value: '' }; function applyObserverSearchFilter() {}\n"; + return searchStubs + src.substring(fnStart + 1, fnEnd); } /** diff --git a/test-observer-menu-interactions.js b/test-observer-menu-interactions.js new file mode 100644 index 000000000..32d981445 --- /dev/null +++ b/test-observer-menu-interactions.js @@ -0,0 +1,148 @@ +/* test-observer-menu-interactions.js — behavioral tests for the observer + * dropdown's event handlers (#1884 follow-up review). Exercises the actual + * handler bodies extracted from packets.js, not source-grep tautology. + * + * Covers three interaction bugs found after the matcher itself (tested in + * test-observer-search-filter.js) was fixed: + * 1. obsSearchInput's own change/blur events bubble into obsMenu's change + * handler and must not be mistaken for a checkbox toggle. + * 2. Clear Filters must reset the search box and re-apply the filter. + * 3. The dropdown must not autofocus the search box on touch devices. + */ +'use strict'; + +const fs = require('fs'); +const assert = require('assert'); + +console.log('--- test-observer-menu-interactions.js ---'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } +} + +const SRC = fs.readFileSync(__dirname + '/public/packets.js', 'utf-8'); + +/** + * Extract the source between a marker ending in "{" and its matching "}". + */ +function extractBlock(marker) { + const markerIdx = SRC.indexOf(marker); + assert(markerIdx !== -1, `marker not found: ${marker}`); + const fnStart = markerIdx + marker.length - 1; + assert(SRC[fnStart] === '{', `marker must end just before "{": ${marker}`); + let depth = 0, fnEnd = -1; + for (let i = fnStart; i < SRC.length; i++) { + if (SRC[i] === '{') depth++; + else if (SRC[i] === '}') { depth--; if (depth === 0) { fnEnd = i; break; } } + } + assert(fnEnd > fnStart, `could not find end of block: ${marker}`); + return SRC.substring(fnStart + 1, fnEnd); +} + +function stubEl(overrides) { + return Object.assign({ + value: '', + textContent: '', + style: {}, + classList: { add() {}, remove() {}, contains() { return false; }, toggle() {} }, + querySelectorAll: () => [], + }, overrides); +} + +// --- 1. obsMenu change handler must ignore events with no data-obs-id --- +// (the search input's own change/blur events bubble up to this listener) +{ + const body = extractBlock("obsMenu.addEventListener('change', (e) => {"); + + function run(target, { calls, selectedObservers, filters }) { + const localStorage = { setItem: () => calls.push('setItem'), removeItem: () => calls.push('removeItem') }; + const buildObserverMenu = () => calls.push('buildObserverMenu'); + const updateObsTrigger = () => calls.push('updateObsTrigger'); + const updatePacketsUrl = () => calls.push('updatePacketsUrl'); + const renderTableRows = () => calls.push('renderTableRows'); + const fn = new Function( + 'e', 'selectedObservers', 'filters', 'localStorage', + 'buildObserverMenu', 'updateObsTrigger', 'updatePacketsUrl', 'renderTableRows', + body + ); + fn(target, selectedObservers, filters, localStorage, buildObserverMenu, updateObsTrigger, updatePacketsUrl, renderTableRows); + } + + test('change event with no data-obs-id (bubbled from search input) is a no-op', () => { + const calls = []; + const selectedObservers = new Set(['3']); + run({ target: { dataset: {}, checked: false } }, { calls, selectedObservers, filters: {} }); + assert.deepStrictEqual(calls, [], 'no side effects should run for an id-less change event'); + assert.deepStrictEqual([...selectedObservers], ['3'], 'selection must be untouched'); + }); + + test('change event on an actual checkbox still updates the selection', () => { + const calls = []; + const selectedObservers = new Set(); + const filters = {}; + run({ target: { dataset: { obsId: '7' }, checked: true } }, { calls, selectedObservers, filters }); + assert.deepStrictEqual([...selectedObservers], ['7'], 'checking observer 7 should select it'); + assert(calls.includes('buildObserverMenu'), 'buildObserverMenu should run for a real toggle'); + assert(calls.includes('updatePacketsUrl'), 'updatePacketsUrl should run for a real toggle'); + }); +} + +// --- 2. Clear Filters must reset and re-apply the observer search box --- +{ + const body = extractBlock("if (clearBtn) clearBtn.addEventListener('click', function() {"); + + test('Clear Filters resets the search box and re-applies the filter', () => { + const applyCalls = []; + const obsSearchInput = stubEl({ value: 'brussels' }); + const filters = {}; + const localStorage = { removeItem() {}, setItem() {} }; + const RegionFilter = { setSelected() {} }; + const loadPackets = () => {}; + const updatePacketsUrl = () => {}; + const applyObserverSearchFilter = () => applyCalls.push('applied'); + const documentStub = { getElementById: () => stubEl({}) }; + + const fn = new Function( + 'filters', 'localStorage', 'document', 'obsSearchInput', 'applyObserverSearchFilter', + 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', + '_observerFilterSet', + body + ); + fn(filters, localStorage, documentStub, obsSearchInput, applyObserverSearchFilter, + 15, 15, RegionFilter, updatePacketsUrl, loadPackets, null); + + assert.strictEqual(obsSearchInput.value, '', 'search box should be cleared'); + assert.deepStrictEqual(applyCalls, ['applied'], 'the filter should be re-applied after clearing'); + }); +} + +// --- 3. Autofocus must be gated on touch (pointer: coarse) devices --- +{ + const body = extractBlock("obsTrigger.addEventListener('click', (e) => {"); + + function run(isTouch) { + const focusCalls = []; + const obsSearchInput = stubEl({ focus: () => focusCalls.push('focus') }); + const obsMenu = { classList: { toggle() {}, contains: () => true } }; + const typeMenu = { classList: { remove() {} } }; + const window_ = { matchMedia: () => ({ matches: isTouch }) }; + const fn = new Function('e', 'obsMenu', 'typeMenu', 'obsSearchInput', 'window', body); + fn({ stopPropagation() {} }, obsMenu, typeMenu, obsSearchInput, window_); + return focusCalls; + } + + test('touch devices (pointer: coarse) do not get the search box autofocused', () => { + assert.deepStrictEqual(run(true), [], 'focus() should not be called on touch devices'); + }); + + test('non-touch devices still get the search box autofocused', () => { + assert.deepStrictEqual(run(false), ['focus'], 'focus() should still be called on desktop/mouse'); + }); +} + +// Summary +console.log(`\n${passed} passed, ${failed} failed`); +if (failed > 0) process.exit(1); +console.log('All tests passed ✅'); diff --git a/test-observer-search-filter.js b/test-observer-search-filter.js new file mode 100644 index 000000000..ddcdc27dc --- /dev/null +++ b/test-observer-search-filter.js @@ -0,0 +1,119 @@ +/* test-observer-search-filter.js — behavioral tests for the observer dropdown + * search box (#1884). Exercises the actual applyObserverSearchFilter logic, + * not source-grep tautology. + */ +'use strict'; + +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +console.log('--- test-observer-search-filter.js ---'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } +} + +/** + * Extract the applyObserverSearchFilter function body from packets.js. + */ +function extractApplyObserverSearchFilter() { + const src = fs.readFileSync(__dirname + '/public/packets.js', 'utf-8'); + const marker = 'function applyObserverSearchFilter()'; + const idx = src.indexOf(marker); + assert(idx !== -1, 'applyObserverSearchFilter not found in packets.js'); + const fnStart = src.indexOf('{', idx); + let depth = 0, fnEnd = -1; + for (let i = fnStart; i < src.length; i++) { + if (src[i] === '{') depth++; + else if (src[i] === '}') { depth--; if (depth === 0) { fnEnd = i; break; } } + } + assert(fnEnd > fnStart, 'could not find end of applyObserverSearchFilter'); + return src.substring(fnStart + 1, fnEnd); +} + +/** + * Build a minimal sandbox: an obsSearchInput with a settable value, and an + * obsList whose querySelectorAll returns items with a data-obs-name dataset + * and a style.display we can assert on. + */ +function makeItem(name) { + return { dataset: { obsName: name }, style: { display: '' } }; +} + +function runFilter(term, names) { + const items = names.map(makeItem); + const obsSearchInput = { value: term }; + const obsList = { + querySelectorAll: (sel) => { + assert.strictEqual(sel, '.multi-select-item[data-obs-name]'); + return items; + }, + }; + const body = extractApplyObserverSearchFilter(); + const fn = new Function('obsSearchInput', 'obsList', body); + fn(obsSearchInput, obsList); + return items; +} + +const fnBody = extractApplyObserverSearchFilter(); + +test('both includes and startsWith are used (default substring, ^-anchored prefix)', () => { + assert(fnBody.includes('.includes('), 'expected substring includes() in filter body'); + assert(fnBody.includes('startsWith'), 'expected startsWith in filter body for ^-anchored matching'); +}); + +test('empty search term shows every item', () => { + const items = runFilter('', ['on4xyz brussels', 'be1abc', 'be2def']); + for (const it of items) assert.strictEqual(it.style.display, '', `expected visible: ${it.dataset.obsName}`); +}); + +test('default matching is substring, not prefix-only', () => { + const items = runFilter('brussels', ['on4xyz brussels', 'be1abc']); + assert.strictEqual(items[0].style.display, '', 'mid-string "brussels" should match by default (includes)'); + assert.strictEqual(items[1].style.display, 'none', 'be1abc should not match "brussels"'); +}); + +test('default substring match still finds prefix matches too', () => { + const items = runFilter('be1', ['be1abc', 'be2def', 'on4xyz brussels']); + assert.strictEqual(items[0].style.display, '', 'be1abc should match "be1"'); + assert.strictEqual(items[1].style.display, 'none', 'be2def should not match "be1"'); + assert.strictEqual(items[2].style.display, 'none', 'on4xyz brussels should not match "be1"'); +}); + +test('^-anchored term uses prefix-only matching', () => { + const items = runFilter('^be', ['be1abc', 'on4xyz brussels']); + assert.strictEqual(items[0].style.display, '', 'be1abc should match anchored prefix "^be"'); + assert.strictEqual(items[1].style.display, 'none', 'brussels contains "be" but not as a prefix, should not match "^be"'); +}); + +test('bare ^ with no remaining term shows every item', () => { + const items = runFilter('^', ['be1abc', 'on4xyz brussels']); + for (const it of items) assert.strictEqual(it.style.display, '', `expected visible: ${it.dataset.obsName}`); +}); + +test('search term is trimmed before matching', () => { + const items = runFilter(' be1 ', ['be1abc', 'be2def']); + assert.strictEqual(items[0].style.display, '', 'be1abc should match trimmed term "be1"'); + assert.strictEqual(items[1].style.display, 'none', 'be2def should not match trimmed term "be1"'); +}); + +test('matching is case-insensitive relative to the stored lowercase name', () => { + // buildObserverMenu() stores data-obs-name already lowercased; the search + // input itself is lowercased by applyObserverSearchFilter before matching. + const items = runFilter('BE1', ['be1abc']); + assert.strictEqual(items[0].style.display, '', 'uppercase search term should still match lowercase stored name'); +}); + +test('^-anchored search is also case-insensitive', () => { + const items = runFilter('^BE', ['be1abc', 'on4xyz brussels']); + assert.strictEqual(items[0].style.display, '', 'uppercase anchored term should still match lowercase stored name'); + assert.strictEqual(items[1].style.display, 'none', 'brussels should not match anchored "^BE"'); +}); + +// Summary +console.log(`\n${passed} passed, ${failed} failed`); +if (failed > 0) process.exit(1); +console.log('All tests passed ✅'); From 914bd4cf0e01bc120617e30f2eb1b4d57eaca3ac Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 20:36:34 +0200 Subject: [PATCH 037/151] feat(channels): show each message's region (#1851) (#2018) ## Show each channel message's region scope Each message in the Channels view now shows the region scope it was sent with, as a small chip in the meta line: the region name (for example `#be`), `unknown scope`, or nothing. This is the channel-message part of #1852 by @dborup, extracted as a focused change. #1852 was closed unmerged because it had grown to the whole fork diff. The implementation follows dborup's commits c686ae3f, 350bf7ee and a94d57ed on dborup/CoreScope, adapted to current master. dborup is co-author on the commit. ### What changed - `cmd/server/db.go:2099,2195`: `GetChannelMessages` selects `t.scope_name` when the column exists and returns it as `scope_name`. - `cmd/server/store.go:5654`: the in-memory `GetChannelMessages` returns `scope_name`, so the field does not depend on which path serves the endpoint. - `cmd/server/store.go:2966,3244`: both WebSocket broadcast builders carry `scope_name`, so a live message shows its region immediately. - `public/channels.js:342,2278`: `messageScopeChipHtml` renders the chip with the existing `.sa-chip-declared` / `.sa-chip-unmatched` styles from `scope-audit.css`. The name goes through `escapeHtml`. No new CSS. - `public/channels.js:678,695,1435,1487`: the decrypt path and the WebSocket path keep `scope_name` on the message. ### Differences from #1852 - The field is `scope_name`, the name `/api/packets` already uses. - No `routeType` field. `transmissions.scope_name` already tells the states apart: NULL means no transport code, an empty string means a transport code that no configured region key matched. The frontend uses `??`, not `||`, so the empty string is kept. - A chip instead of `scope: ` text. The area label from later #1852 commits is not included. ### Perf One extra column per observation row in the page query (at most `limit` transmissions), and one extra map entry per broadcast observation. No new queries, loops or API calls. ### Tests - `cmd/server/channel_message_scope_name_test.go`: the three states through the DB query, the store, `/api/channels/{hash}/messages` over both paths, a schema without the column, and both broadcast builders. 5 of its 6 tests fail without the change; the sixth guards the missing-column case and passes either way. - `test-issue-1851-channel-message-scope.js`: the REST, WebSocket and client-side decrypt paths, escaping, and the name / unknown / none render. 4/4 fail without the change. Registered in `test-all.sh` and the unit step of `deploy.yml`. - Mutation checks: returning `nil` for `scope_name` in the DB path fails the DB and endpoint tests; `||` instead of `??` in the WebSocket path fails the WebSocket test. - `go test ./...` in `cmd/server`: ok. gofmt and go vet clean. ### Browser validation On a staging instance with live traffic (build `e84d2da6`), in Chrome: - `/api/channels/{hash}/messages` carries the `scope_name` key on every message in the 19 channels whose results I read. `#hamradio`, latest 50: 34 named, 1 empty string, 15 NULL. - Opening `#hamradio` renders 104 chips: `#nl` 53, `#be` 32, `#de` 11, `#eu` 6, `#bx` 1 and `unknown scope` 1, and no chip on unscoped messages. Chip text `rgb(26, 26, 46)` on `rgb(238, 242, 255)` in the light theme. ### Not verified - Dark theme not checked. - The real-decrypt branch of `decryptCandidates` has no test and was not exercised in the browser; the already-decrypted branch is tested. - Messages already in the client decrypt cache show no chip until they are decrypted again. - `go test -race` and the Playwright E2E suite were not run locally. Fixes #1851 ## Review follow-up (commit `50346589`) An independent review found no correctness or XSS problem and confirmed DB, store and WebSocket agree on the value. Changed: - **Real decrypt branch tested.** A new test runs the real AES+HMAC decrypt branch in `decryptCandidates` with one packet per scope state; deleting `scope_name` there now fails 2 of 7 tests. - **Tooltip wording.** The unknown-scope tooltip now says the scope "could not be matched to a single region on this instance" (`public/channels.js:338-347`). The ingestor stores an empty name both when no key matches and when several match without exactly one operator-configured key (`cmd/ingestor/region_keys.go:364-393`), so "matches none of the configured keys" was wrong for the second case. - **Old decrypt cache.** Decrypted messages cached before this change had no `scope_name` key and stayed chipless as long as the candidate count did not change. A cached message missing the key now forces one full decrypt; a cache that has it still takes the delta path. A test covers each case. - **Docs.** `docs/api-spec.md` documents `scope_name` on the channel messages response, with the null / empty string / name semantics. Corrections to the description: - **Test counts.** With the `db.go` and `store.go` changes reverted, 4 of the 5 top-level Go tests fail (6 of 7 counting subtests); only the missing-column test passes. - **Broadcast payload.** `scope_name` is added to `pkt`, which is copied into `broadcastMap` and also nested as `packet` (`store.go` ~2974-2980, ~3252-3257), so the key appears twice per observation: 36 bytes for `null`, 48 bytes for `"#belgium"`. - **Side effect on the Packets page.** The live table reads `m.data.packet` (`packets.js` ~1316-1318), so flat rows and expanded group children now show Scope for live packets. In grouped mode a new group copies a fixed field list without `scope_name` (~1384-1395) and shows the empty placeholder until reload. Before this PR every live row showed that placeholder, so this is not a regression. The three copies of the three-state scope rendering (`app.js`, `packets.js`, `channels.js`) are left as they are. --------- Co-authored-by: dborup <3627142+dborup@users.noreply.github.com> Co-authored-by: Claude Opus 5 (1M context) --- cmd/server/channel_message_scope_name_test.go | 260 ++++++++++++++ cmd/server/db.go | 18 +- cmd/server/store.go | 3 + docs/api-spec.md | 13 +- public/channels.js | 26 +- test-all.sh | 1 + test-issue-1851-channel-message-scope.js | 330 ++++++++++++++++++ 7 files changed, 646 insertions(+), 5 deletions(-) create mode 100644 cmd/server/channel_message_scope_name_test.go create mode 100644 test-issue-1851-channel-message-scope.js diff --git a/cmd/server/channel_message_scope_name_test.go b/cmd/server/channel_message_scope_name_test.go new file mode 100644 index 000000000..3d2af4719 --- /dev/null +++ b/cmd/server/channel_message_scope_name_test.go @@ -0,0 +1,260 @@ +package main + +import ( + "encoding/json" + "net/http/httptest" + "testing" + "time" + + "github.com/gorilla/mux" +) + +// #1851: the Channels view shows the region scope a message was sent under. +// transmissions.scope_name encodes three states (see StoreTx.ScopeName): +// SQL NULL = not transport-scoped, "" = transport-scoped but no configured +// region key matched, "#name" = matched region. /api/channels/{hash}/messages +// is served by the DB query when a DB is attached and by the in-memory store +// otherwise, and live messages arrive over the WebSocket broadcast, so all +// three paths must carry the same three states. + +const ( + chScopeHashUnscoped = "cccccccccccccc01" + chScopeHashUnknown = "cccccccccccccc02" + chScopeHashMatched = "cccccccccccccc03" +) + +var chScopeWant = map[string]interface{}{ + chScopeHashUnscoped: nil, + chScopeHashUnknown: "", + chScopeHashMatched: "#belgium", +} + +// setupChannelScopeDB seeds one channel with three messages, one per scope +// state, each heard by one observation. +func setupChannelScopeDB(t *testing.T) *DB { + t.Helper() + db := setupTestDB(t) + if _, err := db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT DEFAULT NULL`); err != nil { + t.Fatalf("add scope_name column: %v", err) + } + db.hasScopeName = true + if _, err := db.conn.Exec(`INSERT INTO observers (id, name, iata) VALUES ('obs1', 'Observer One', 'BRU')`); err != nil { + t.Fatalf("insert observer: %v", err) + } + now := time.Now().UTC() + rows := []struct { + hash string + routeType int + scope interface{} + text string + }{ + {chScopeHashUnscoped, 1, nil, "Alice: plain flood"}, + {chScopeHashUnknown, 0, "", "Bob: unknown region"}, + {chScopeHashMatched, 0, "#belgium", "Carol: matched region"}, + } + for i, r := range rows { + ts := now.Add(time.Duration(i-3) * time.Minute) + res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', ?, ?, ?, 5, ?, '#scopetest', ?)`, + r.hash, ts.Format(time.RFC3339), r.routeType, + `{"type":"CHAN","channel":"#scopetest","text":"`+r.text+`"}`, r.scope) + if err != nil { + t.Fatalf("insert tx %s: %v", r.hash, err) + } + txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (?, 1, 9.5, -90, '[]', ?)`, txID, ts.Unix()); err != nil { + t.Fatalf("insert obs %s: %v", r.hash, err) + } + } + return db +} + +func assertChannelMessageScopes(t *testing.T, messages []map[string]interface{}) { + t.Helper() + if len(messages) != len(chScopeWant) { + t.Fatalf("expected %d messages, got %d", len(chScopeWant), len(messages)) + } + for _, m := range messages { + h, _ := m["packetHash"].(string) + want, known := chScopeWant[h] + if !known { + t.Errorf("unexpected packetHash %q", h) + continue + } + got, present := m["scope_name"] + if !present { + t.Errorf("%s: scope_name key missing", h) + continue + } + if got != want { + t.Errorf("%s: scope_name = %#v, want %#v", h, got, want) + } + } +} + +func TestDBGetChannelMessagesCarriesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + messages, _, err := db.GetChannelMessages("#scopetest", 100, 0) + if err != nil { + t.Fatal(err) + } + assertChannelMessageScopes(t, messages) +} + +func TestStoreGetChannelMessagesCarriesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + messages, _ := store.GetChannelMessages("#scopetest", 100, 0) + for i := range messages { + // The store keeps the value as a *string; normalise through JSON so + // the comparison is on what the browser receives. + b, err := json.Marshal(messages[i]) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + messages[i] = decoded + } + assertChannelMessageScopes(t, messages) +} + +// A database the ingestor has not migrated yet has no scope_name column. The +// DB query must not select it (the query would fail) and every message reports +// null, the same as a non-transport-scoped message. +func TestDBGetChannelMessagesWithoutScopeNameColumn(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + seedTestData(t, db) + messages, _, err := db.GetChannelMessages("#test", 100, 0) + if err != nil { + t.Fatalf("GetChannelMessages on a schema without scope_name: %v", err) + } + if len(messages) == 0 { + t.Fatal("expected messages for #test") + } + for _, m := range messages { + if got := m["scope_name"]; got != nil { + t.Errorf("scope_name = %#v, want nil without the column", got) + } + } +} + +func TestChannelMessagesEndpointExposesScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + for _, withDB := range []bool{true, false} { + name := "db" + if !withDB { + name = "store" + } + t.Run(name, func(t *testing.T) { + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + srv := NewServer(db, &Config{Port: 3000}, NewHub()) + srv.store = store + if !withDB { + srv.db = nil + } + router := mux.NewRouter() + srv.RegisterRoutes(router) + + req := httptest.NewRequest("GET", "/api/channels/%23scopetest/messages?limit=50", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != 200 { + t.Fatalf("expected 200, got %d (body: %s)", w.Code, w.Body.String()) + } + var body struct { + Messages []map[string]interface{} `json:"messages"` + } + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("unmarshal: %v", err) + } + assertChannelMessageScopes(t, body.Messages) + }) + } +} + +// Live messages in the Channels view are built from the WebSocket broadcast, +// not from the REST response, so both broadcast builders must carry scope_name +// or a new message shows no region until the next REST refresh replaces it. +func TestBroadcastMapsCarryScopeName(t *testing.T) { + db := setupChannelScopeDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatalf("store.Load: %v", err) + } + + assertBroadcast := func(t *testing.T, maps []map[string]interface{}, hash string, want interface{}) { + t.Helper() + found := false + for _, bm := range maps { + if bm["hash"] != hash { + continue + } + found = true + b, err := json.Marshal(bm) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var decoded map[string]interface{} + if err := json.Unmarshal(b, &decoded); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if got, ok := decoded["scope_name"]; !ok || got != want { + t.Errorf("top-level scope_name = %#v (present=%v), want %#v", got, ok, want) + } + pkt, _ := decoded["packet"].(map[string]interface{}) + if got, ok := pkt["scope_name"]; !ok || got != want { + t.Errorf("packet.scope_name = %#v (present=%v), want %#v", got, ok, want) + } + } + if !found { + t.Fatalf("no broadcast map for %s", hash) + } + } + + t.Run("IngestNewFromDB", func(t *testing.T) { + txMax := store.MaxTransmissionID() + now := time.Now().UTC() + res, err := db.conn.Exec(`INSERT INTO transmissions (raw_hex, hash, first_seen, route_type, payload_type, decoded_json, channel_hash, scope_name) + VALUES ('AABB', 'cccccccccccccc04', ?, 0, 5, '{"type":"CHAN","channel":"#scopetest","text":"Dave: live"}', '#scopetest', '#belgium')`, + now.Format(time.RFC3339)) + if err != nil { + t.Fatalf("insert tx: %v", err) + } + txID, _ := res.LastInsertId() + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (?, 1, 7.0, -95, '[]', ?)`, txID, now.Unix()); err != nil { + t.Fatalf("insert obs: %v", err) + } + maps, _ := store.IngestNewFromDB(txMax, 100) + assertBroadcast(t, maps, "cccccccccccccc04", "#belgium") + }) + + t.Run("IngestNewObservations", func(t *testing.T) { + obsMax := db.GetMaxObservationID() + var txID int + if err := db.conn.QueryRow(`SELECT id FROM transmissions WHERE hash = ?`, chScopeHashUnknown).Scan(&txID); err != nil { + t.Fatalf("lookup tx: %v", err) + } + if _, err := db.conn.Exec(`INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp) + VALUES (?, 1, 3.0, -110, '["aa"]', ?)`, txID, time.Now().Unix()); err != nil { + t.Fatalf("insert obs: %v", err) + } + maps := store.IngestNewObservations(obsMax, 100) + assertBroadcast(t, maps, chScopeHashUnknown, "") + }) +} diff --git a/cmd/server/db.go b/cmd/server/db.go index dac63be8e..9ed1b9af1 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -2094,10 +2094,16 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . idPlaceholders[i] = "?" obsArgs[i] = id } + // #1851: scope_name lives on the transmission row, so appending it as the + // last selected column is safe for both query shapes. + scopeNameCol := "" + if db.hasScopeName { + scopeNameCol = ", t.scope_name" + } var obsSQL string if db.isV3 { obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, - obs.id, obs.name, o.snr, o.path_json, o.timestamp + obs.id, obs.name, o.snr, o.path_json, o.timestamp` + scopeNameCol + ` FROM observations o JOIN transmissions t ON t.id = o.transmission_id LEFT JOIN observers obs ON obs.rowid = o.observer_idx @@ -2105,7 +2111,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . ORDER BY o.id ASC` } else { obsSQL = `SELECT o.id, t.id, t.hash, t.decoded_json, t.first_seen, - o.observer_id, o.observer_name, o.snr, o.path_json, o.timestamp + o.observer_id, o.observer_name, o.snr, o.path_json, o.timestamp` + scopeNameCol + ` FROM observations o JOIN transmissions t ON t.id = o.transmission_id WHERE t.id IN (` + strings.Join(idPlaceholders, ",") + `) @@ -2130,7 +2136,12 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . var pktHash, dj, fs, obsID, obsName, pathJSON sql.NullString var snr sql.NullFloat64 var obsTs sql.NullInt64 - rows.Scan(&pktID, &txID, &pktHash, &dj, &fs, &obsID, &obsName, &snr, &pathJSON, &obsTs) + var scopeName sql.NullString + scanArgs := []interface{}{&pktID, &txID, &pktHash, &dj, &fs, &obsID, &obsName, &snr, &pathJSON, &obsTs} + if db.hasScopeName { + scanArgs = append(scanArgs, &scopeName) + } + rows.Scan(scanArgs...) if !dj.Valid { continue } @@ -2181,6 +2192,7 @@ func (db *DB) GetChannelMessages(channelHash string, limit, offset int, region . "observers": []string{}, "hops": hops, "snr": nullFloat(snr), + "scope_name": nullStr(scopeName), }, Repeats: 1, } diff --git a/cmd/server/store.go b/cmd/server/store.go index 6b0b25463..5a0a84e00 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -2968,6 +2968,7 @@ func (s *PacketStore) IngestNewFromDB(sinceID, limit int) ([]map[string]interfac "path_json": strOrNil(obs.PathJSON), "direction": strOrNil(obs.Direction), "observation_count": tx.ObservationCount, + "scope_name": strPtrOrNil(tx.ScopeName), } // Use decode-window resolved path for broadcast (never from struct) if broadcastRP != nil { @@ -3245,6 +3246,7 @@ func (s *PacketStore) IngestNewObservations(sinceObsID, limit int) []map[string] "path_json": strOrNil(obs.PathJSON), "direction": strOrNil(obs.Direction), "observation_count": tx.ObservationCount, + "scope_name": strPtrOrNil(tx.ScopeName), } // Use decode-window resolved path for broadcast if obsRPMap != nil { @@ -5665,6 +5667,7 @@ func (s *PacketStore) GetChannelMessages(channelHash string, limit, offset int, "observers": observers, "hops": hops, "snr": snrVal, + "scope_name": strPtrOrNil(tx.ScopeName), }, Repeats: 1, Observers: observers, diff --git a/docs/api-spec.md b/docs/api-spec.md index 3d2c19cd4..09d38889e 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -1169,13 +1169,24 @@ Messages for a specific channel. "repeats": number, // dedup count "observers": [string], // observer names "hops": number, - "snr": number | null + "snr": number | null, + "scope_name": string | null // region scope, see below } ], "total": number // total deduplicated messages } ``` +`scope_name` is the transmission's region scope (`transmissions.scope_name`), the same field `/api/packets` returns: + +| Value | Meaning | +|-------|---------| +| `null` | No transport code: the message was not region-scoped. Also `null` when the database has no `scope_name` column yet (ingestor migration not run). | +| `""` | Transport-scoped, but the ingestor could not match it to a single region: no region key matched, or several matched with no single operator-configured key among them. | +| `"#name"` | The matched region name. | + +The same field is on the WebSocket `packet` broadcast, both top-level and inside `packet`. + --- ## GET /api/analytics/rf diff --git a/public/channels.js b/public/channels.js index b46889326..964bfa876 100644 --- a/public/channels.js +++ b/public/channels.js @@ -335,6 +335,20 @@ return str.replace(/&/g, '&').replace(//g, '>').replace(/"/g, '"').replace(/'/g, '''); } + // #1851: region scope from the packet's transport code (scope_name). + // null = no transport code, so no chip; '' = a transport code the ingestor + // could not match to a single region (no key matched, or several did with no + // operator-configured winner); otherwise the matched region name. Chip + // classes are the Scope Audit's (scope-audit.css), so a scope looks the same + // everywhere. + function messageScopeChipHtml(scopeName) { + if (scopeName == null) return ''; + if (scopeName === '') { + return 'unknown scope'; + } + return '' + escapeHtml(scopeName) + ''; + } + function truncate(str, len) { if (!str) return ''; return str.length > len ? str.slice(0, len) + '…' : str; @@ -579,7 +593,10 @@ // M5: Cache invalidation — if total candidate count changed, re-decrypt everything var totalCandidates = candidates.length; - var needFullDecrypt = (totalCandidates !== cachedCount) || opts.forceFullDecrypt; + // #1851: a cache written before messages carried scope_name would keep + // those messages chipless on the delta path, so decrypt them again. + var cacheLacksScope = cachedMsgs.some(function (m) { return !('scope_name' in m); }); + var needFullDecrypt = (totalCandidates !== cachedCount) || opts.forceFullDecrypt || cacheLacksScope; // M5: Delta fetch — only decrypt packets newer than lastTs if (!needFullDecrypt && cachedMsgs.length > 0 && lastTs) { @@ -663,6 +680,7 @@ packetHash: c.packet.hash, packetId: c.packet.id, hops: d.path_len || 0, snr: c.packet.snr || null, observers: c.packet.observer_name ? [c.packet.observer_name] : [], + scope_name: c.packet.scope_name ?? null, repeats: 1 }); continue; @@ -679,6 +697,7 @@ packetHash: c.packet.hash, packetId: c.packet.id, hops: 0, snr: c.packet.snr || null, observers: c.packet.observer_name ? [c.packet.observer_name] : [], + scope_name: c.packet.scope_name ?? null, repeats: 1 }); } else { @@ -1417,6 +1436,8 @@ var pktId = m.data?.id || null; var snr = m.data?.snr ?? m.data?.packet?.snr ?? payload.SNR ?? null; var observer = m.data?.packet?.observer_name || m.data?.observer || null; + // ?? not ||: '' (transport-scoped, region unmatched) must survive. + var scopeName = m.data?.scope_name ?? m.data?.packet?.scope_name ?? null; // Update channel list entry — only once per unique packet hash var isFirstObservation = pktHash && !seenHashes.has(pktHash + ':' + channelKey); @@ -1468,6 +1489,7 @@ observers: observer ? [observer] : [], hops: payload.path_len || 0, snr: snr, + scope_name: scopeName, // #1498: mark as WS-pushed so a later REST replacement // (selectChannel / refreshMessages) can merge instead of // stomp. Without this flag the REST response wipes any @@ -2258,6 +2280,8 @@ if (msg.observers?.length > 1) meta.push(`${msg.observers.length} observers`); if (msg.hops > 0) meta.push(`${msg.hops} hops`); if (msg.snr !== null && msg.snr !== undefined) meta.push(`SNR ${msg.snr}`); + const scopeChip = messageScopeChipHtml(msg.scope_name); + if (scopeChip) meta.push(scopeChip); const safeId = btoa(encodeURIComponent(sender)); // #1367: emit BOTH the new chat-app class names (.ch-message / diff --git a/test-all.sh b/test-all.sh index 9e15b7285..2e5950a92 100755 --- a/test-all.sh +++ b/test-all.sh @@ -125,6 +125,7 @@ node test-issue-1836-crossnav-case-normalization.js node test-issue-1843-node-qr-quiet-zone.js node test-issue-1846-observers-width.js node test-issue-1849-trace-hashbytes.js +node test-issue-1851-channel-message-scope.js node test-issue-1868-control-decode.js node test-issue-1890-og-url.js node test-issue-1956-release-routing.js diff --git a/test-issue-1851-channel-message-scope.js b/test-issue-1851-channel-message-scope.js new file mode 100644 index 000000000..4352fa00a --- /dev/null +++ b/test-issue-1851-channel-message-scope.js @@ -0,0 +1,330 @@ +/* #1851: each channel message shows the region scope it was sent with. + * + * scope_name has three states (transmissions.scope_name): null = the packet + * carried no transport code, '' = it carried one the ingestor could not match + * to a single region, '#name' = matched region. A message reaches the Channels view by + * three routes (REST /api/channels/{hash}/messages, the WebSocket broadcast, + * and client-side decryption of /api/packets rows), so each route must keep + * the field, and the render must keep the '' state apart from null. + * + * Loads the real public/channels.js in a vm sandbox and reads the HTML it + * writes into #chMessages. + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); +const { createCipheriv, createHmac, createHash } = require('crypto'); + +let passed = 0, failed = 0; +const pending = []; +function test(name, fn) { + const done = (err) => { + if (err) { failed++; console.log(` ❌ ${name}: ${err.message}`); } + else { passed++; console.log(` ✅ ${name}`); } + }; + try { + const out = fn(); + if (out && typeof out.then === 'function') { pending.push(out.then(() => done(), done)); return; } + done(); + } catch (e) { done(e); } +} + +function makeChannelsSandbox(apiImpl) { + const dom = {}; + function makeEl(id) { + if (dom[id]) return dom[id]; + dom[id] = { + id, innerHTML: '', textContent: '', value: '', + scrollTop: 0, scrollHeight: 100, clientHeight: 80, + style: {}, dataset: {}, + classList: { add() {}, remove() {}, toggle() {}, contains() { return false; } }, + addEventListener() {}, removeEventListener() {}, + querySelector() { return null; }, querySelectorAll() { return []; }, + getBoundingClientRect() { return { left: 0, bottom: 0, width: 0 }; }, + setAttribute() {}, removeAttribute() {}, focus() {}, + }; + return dom[id]; + } + const headerText = { textContent: '' }; + makeEl('chHeader').querySelector = (sel) => (sel === '.ch-header-text' ? headerText : null); + ['chMessages', 'chList', 'chScrollBtn', 'chAriaLive', 'chBackBtn', 'chRegionFilter'].forEach(makeEl); + const layout = { classList: { add() {}, remove() {}, contains() { return false; } } }; + const appEl = { + innerHTML: '', + querySelector(sel) { return sel === '.ch-layout' ? layout : makeEl(sel); }, + addEventListener() {}, + }; + const storage = {}; + const ctx = { + window: { addEventListener() {}, dispatchEvent() {} }, + document: { + readyState: 'complete', + createElement: () => ({ id: '', textContent: '', innerHTML: '' }), + head: { appendChild() {} }, + body: { appendChild() {}, removeChild() {}, contains() { return false; } }, + documentElement: { getAttribute: () => null, setAttribute() {} }, + getElementById: makeEl, + querySelector: (sel) => (sel === '.ch-layout' ? layout : null), + querySelectorAll: () => [], + addEventListener() {}, removeEventListener() {}, + }, + console, Date, Math, Array, Object, String, Number, JSON, RegExp, Error, TypeError, + Map, Set, Promise, URLSearchParams, Infinity, parseInt, parseFloat, isNaN, isFinite, + encodeURIComponent, decodeURIComponent, + setTimeout: () => 0, clearTimeout() {}, setInterval: () => 0, clearInterval() {}, + performance: { now: () => Date.now() }, + localStorage: { + getItem: (k) => (k in storage ? storage[k] : null), + setItem: (k, v) => { storage[k] = String(v); }, + removeItem: (k) => { delete storage[k]; }, + }, + location: { hash: '' }, + getHashParams: () => new URLSearchParams(''), + history: { replaceState() {} }, + matchMedia: () => ({ matches: false }), + MutationObserver: function () { this.observe = () => {}; this.disconnect = () => {}; }, + RegionFilter: { init() {}, onChange() { return () => {}; }, offChange() {}, getRegionParam() { return ''; } }, + debouncedOnWS: (fn) => fn, onWS() {}, offWS() {}, + api: apiImpl, + CLIENT_TTL: { observers: 120000, channels: 15000, channelMessages: 10000, nodeDetail: 10000 }, + ROLE_EMOJI: {}, ROLE_LABELS: {}, + timeAgo: () => '1m ago', + registerPage: (name, handlers) => { ctx._pageHandlers = handlers; }, + btoa: (s) => Buffer.from(String(s), 'utf8').toString('base64'), + atob: (s) => Buffer.from(String(s), 'base64').toString('utf8'), + crypto: { subtle: require('crypto').webcrypto.subtle }, + TextEncoder, TextDecoder, Uint8Array, + }; + ctx.window.matchMedia = ctx.matchMedia; + vm.createContext(ctx); + for (const file of ['public/vendor/aes-ecb.js', 'public/channel-decrypt.js', 'public/channels.js']) { + vm.runInContext(fs.readFileSync(file, 'utf8'), ctx); + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; + } + ctx._pageHandlers.init(appEl); + return { ctx, dom }; +} + +function listApi(extra) { + return (path) => { + if (path.indexOf('/observers') === 0) return Promise.resolve({ observers: [] }); + if (path.indexOf('/channels') === 0 && path.indexOf('/messages') === -1) { + return Promise.resolve({ channels: [{ hash: 'general', name: 'general', messageCount: 4, lastActivity: null }] }); + } + return extra(path); + }; +} + +// Splits the rendered message list into one HTML chunk per message so each +// assertion is about a specific message, not about the page as a whole. +function messageChunks(html) { + return html.split('
').slice(1); +} + +// A GRP_TXT packet ChannelDecrypt can really verify and decrypt: AES-128-ECB +// under SHA-256(channelName)[:16], 2-byte HMAC-SHA256 MAC over the ciphertext +// with key + 16 zero bytes. Same construction as buildEncryptedGrpTxt in +// test-channel-live-decrypt.js, which runs on require and so cannot be imported. +function encryptedGrpTxt(channelName, sender, message) { + const key = createHash('sha256').update(channelName).digest().slice(0, 16); + const channelHash = createHash('sha256').update(key).digest()[0]; + const text = `${sender}: ${message}`; + const pt = Buffer.alloc(Math.ceil((5 + Buffer.byteLength(text, 'utf8') + 1) / 16) * 16); + pt.writeUInt32LE(1788256800, 0); + pt.write(text, 5, 'utf8'); + const cipher = createCipheriv('aes-128-ecb', key, null); + cipher.setAutoPadding(false); + const ct = Buffer.concat([cipher.update(pt), cipher.final()]); + const mac = createHmac('sha256', Buffer.concat([key, Buffer.alloc(16)])).update(ct).digest().slice(0, 2); + return { + keyHex: key.toString('hex'), + channelHash, + decodedJson: JSON.stringify({ type: 'GRP_TXT', channelHash, mac: mac.toString('hex'), encryptedData: ct.toString('hex') }), + }; +} + +// Three encrypted packets on one channel, one per scope_name state. +function encryptedChannelPackets(channelName) { + const rows = [ + { sender: 'Alice', text: 'secret matched', scope: '#belgium' }, + { sender: 'Bob', text: 'secret unmatched', scope: '' }, + { sender: 'Carol', text: 'secret plain', scope: null }, + ]; + let fx; + const packets = rows.map((r, i) => { + fx = encryptedGrpTxt(channelName, r.sender, r.text); + return { id: 10 + i, hash: 'e' + i, first_seen: `2026-09-01T10:0${i}:00Z`, scope_name: r.scope, decoded_json: fx.decodedJson }; + }); + return { packets, keyHex: fx.keyHex, channelHash: fx.channelHash, lastTs: packets[packets.length - 1].first_seen }; +} + +function packetsApi(packets) { + return listApi((path) => Promise.resolve(path.indexOf('/packets?') === 0 ? { packets } : {})); +} + +const SCOPE_CHIP =/]*>([^<]*)<\/span>/; +const UNKNOWN_CHIP = /]*>unknown scope<\/span>/; + +console.log('\n=== #1851: channel message region scope ==='); + +test('REST messages render a scope chip per state: name, unknown, none', async () => { + const { ctx, dom } = makeChannelsSandbox(listApi((path) => { + if (path.indexOf('/channels/general/messages') === 0) { + return Promise.resolve({ messages: [ + { sender: 'Alice', text: 'matched', timestamp: '2026-09-01T10:00:00Z', packetHash: 'h1', scope_name: '#belgium' }, + { sender: 'Bob', text: 'unmatched', timestamp: '2026-09-01T10:01:00Z', packetHash: 'h2', scope_name: '' }, + { sender: 'Carol', text: 'plain flood', timestamp: '2026-09-01T10:02:00Z', packetHash: 'h3', scope_name: null }, + { sender: 'Dave', text: 'older server', timestamp: '2026-09-01T10:03:00Z', packetHash: 'h4' }, + ] }); + } + return Promise.resolve({}); + })); + for (let i = 0; i < 10; i++) await Promise.resolve(); + await ctx.window._channelsSelectChannelForTest('general'); + const chunks = messageChunks(dom.chMessages.innerHTML); + assert.strictEqual(chunks.length, 4, 'expected 4 rendered messages'); + + const named = chunks[0].match(SCOPE_CHIP); + assert.ok(named, 'matched-region message must render a scope chip'); + assert.strictEqual(named[1], '#belgium'); + assert.ok(!UNKNOWN_CHIP.test(chunks[0]), 'matched-region message must not say unknown'); + + assert.ok(UNKNOWN_CHIP.test(chunks[1]), "scope_name '' must render the unknown-scope chip"); + assert.ok(!SCOPE_CHIP.test(chunks[1]), "scope_name '' must not render a named chip"); + // '' is stored both when no region key matched and when several matched + // with no operator-configured winner (cmd/ingestor/region_keys.go match), + // so the tooltip must not claim that no key matched. + const title = (chunks[1].match(/ch-msg-scope" title="([^"]*)"/) || [])[1]; + assert.strictEqual(title, 'Sent with a region scope that could not be matched to a single region on this instance'); + + assert.ok(!/ch-msg-scope/.test(chunks[2]), 'scope_name null must render no chip'); + assert.ok(!/ch-msg-scope/.test(chunks[3]), 'a message without scope_name must render no chip'); +}); + +test('scope name is escaped before it reaches innerHTML', async () => { + const { ctx, dom } = makeChannelsSandbox(listApi((path) => { + if (path.indexOf('/channels/general/messages') === 0) { + return Promise.resolve({ messages: [ + { sender: 'Eve', text: 'x', timestamp: '2026-09-01T10:00:00Z', packetHash: 'h1', scope_name: '' }, + ] }); + } + return Promise.resolve({}); + })); + for (let i = 0; i < 10; i++) await Promise.resolve(); + await ctx.window._channelsSelectChannelForTest('general'); + const html = dom.chMessages.innerHTML; + assert.ok(!html.includes(' { + const { ctx, dom } = makeChannelsSandbox(listApi(() => Promise.resolve({ messages: [] }))); + ctx.window._channelsSetStateForTest({ + selectedHash: 'general', + channels: [{ hash: 'general', name: 'general', messageCount: 0, lastActivityMs: 0 }], + messages: [], + }); + const packet = (hash, text, scope) => ({ + type: 'packet', + data: { + hash, scope_name: scope, + decoded: { header: { payloadTypeName: 'GRP_TXT' }, payload: { channel: 'general', text } }, + packet: { hash, observer_name: 'Obs', scope_name: scope }, + }, + }); + ctx.window._channelsProcessWSBatchForTest([ + packet('w1', 'Alice: live matched', '#belgium'), + packet('w2', 'Bob: live unmatched', ''), + packet('w3', 'Carol: live plain', null), + ], null); + const state = ctx.window._channelsGetStateForTest(); + assert.deepStrictEqual(Array.from(state.messages, (m) => m.scope_name), ['#belgium', '', null]); + const chunks = messageChunks(dom.chMessages.innerHTML); + assert.strictEqual(chunks.length, 3, 'expected 3 rendered messages'); + assert.strictEqual((chunks[0].match(SCOPE_CHIP) || [])[1], '#belgium'); + assert.ok(UNKNOWN_CHIP.test(chunks[1]), 'live message with scope_name "" must render the unknown chip'); + assert.ok(!/ch-msg-scope/.test(chunks[2]), 'live message with scope_name null must render no chip'); +}); + +test('client-side decrypted channel keeps scope_name from the /api/packets row', async () => { + const { ctx, dom } = makeChannelsSandbox(listApi((path) => { + if (path.indexOf('/packets?') === 0) { + return Promise.resolve({ packets: [ + { id: 1, hash: 'd1', first_seen: '2026-09-01T10:00:00Z', scope_name: '#belgium', + decoded_json: JSON.stringify({ type: 'CHAN', channel: '#priv', sender: 'Alice', text: 'Alice: decrypted' }) }, + { id: 2, hash: 'd2', first_seen: '2026-09-01T10:01:00Z', scope_name: '', + decoded_json: JSON.stringify({ type: 'CHAN', channel: '#priv', sender: 'Bob', text: 'Bob: decrypted' }) }, + ] }); + } + return Promise.resolve({}); + })); + for (let i = 0; i < 10; i++) await Promise.resolve(); + await ctx.window._channelsSelectChannelForTest('user:#priv', { + userKey: '00112233445566778899aabbccddeeff', channelHashByte: 1, channelName: '#priv', + }); + const state = ctx.window._channelsGetStateForTest(); + assert.deepStrictEqual(Array.from(state.messages, (m) => m.scope_name), ['#belgium', '']); + const chunks = messageChunks(dom.chMessages.innerHTML); + assert.strictEqual((chunks[0].match(SCOPE_CHIP) || [])[1], '#belgium'); + assert.ok(UNKNOWN_CHIP.test(chunks[1]), 'decrypted message with scope_name "" must render the unknown chip'); +}); + +test('packets decrypted with the stored key keep scope_name from the /api/packets row', async () => { + const fx = encryptedChannelPackets('#secret'); + const { ctx, dom } = makeChannelsSandbox(packetsApi(fx.packets)); + for (let i = 0; i < 10; i++) await Promise.resolve(); + await ctx.window._channelsSelectChannelForTest('user:#secret', { + userKey: fx.keyHex, channelHashByte: fx.channelHash, channelName: '#secret', + }); + const state = ctx.window._channelsGetStateForTest(); + // Sender and text only exist after a MAC check and AES decrypt, so these + // messages came through the encrypted branch, not the already-decrypted one. + assert.deepStrictEqual(Array.from(state.messages, (m) => m.sender + ': ' + m.text), + ['Alice: secret matched', 'Bob: secret unmatched', 'Carol: secret plain']); + assert.deepStrictEqual(Array.from(state.messages, (m) => m.scope_name), ['#belgium', '', null]); + const chunks = messageChunks(dom.chMessages.innerHTML); + assert.strictEqual(chunks.length, 3, 'expected 3 rendered messages'); + assert.strictEqual((chunks[0].match(SCOPE_CHIP) || [])[1], '#belgium'); + assert.ok(UNKNOWN_CHIP.test(chunks[1]), 'decrypted message with scope_name "" must render the unknown chip'); + assert.ok(!/ch-msg-scope/.test(chunks[2]), 'decrypted message with scope_name null must render no chip'); +}); + +test('a decrypt cache written before scope_name existed is decrypted again', async () => { + const fx = encryptedChannelPackets('#secret'); + const { ctx } = makeChannelsSandbox(packetsApi(fx.packets)); + // Same candidate count and last timestamp as the API returns, so without the + // missing key the delta path would find nothing new and serve this cache. + ctx.ChannelDecrypt.setCache('#secret', fx.packets.map((p, i) => ({ + sender: 'Old', text: 'cached ' + i, timestamp: p.first_seen, packetHash: p.hash, packetId: p.id, + hops: 0, snr: null, observers: [], repeats: 1, + })), fx.lastTs, fx.packets.length); + for (let i = 0; i < 10; i++) await Promise.resolve(); + await ctx.window._channelsSelectChannelForTest('user:#secret', { + userKey: fx.keyHex, channelHashByte: fx.channelHash, channelName: '#secret', + }); + const state = ctx.window._channelsGetStateForTest(); + assert.deepStrictEqual(Array.from(state.messages, (m) => m.scope_name), ['#belgium', '', null]); + assert.deepStrictEqual(Array.from(ctx.ChannelDecrypt.getCache('#secret').messages, (m) => m.scope_name), + ['#belgium', '', null], 'the rewritten cache must carry scope_name'); +}); + +test('a decrypt cache that already carries scope_name is still served by the delta path', async () => { + const fx = encryptedChannelPackets('#secret'); + const { ctx } = makeChannelsSandbox(packetsApi(fx.packets)); + ctx.ChannelDecrypt.setCache('#secret', fx.packets.map((p, i) => ({ + sender: 'Old', text: 'cached ' + i, timestamp: p.first_seen, packetHash: p.hash, packetId: p.id, + hops: 0, snr: null, observers: [], scope_name: null, repeats: 1, + })), fx.lastTs, fx.packets.length); + for (let i = 0; i < 10; i++) await Promise.resolve(); + await ctx.window._channelsSelectChannelForTest('user:#secret', { + userKey: fx.keyHex, channelHashByte: fx.channelHash, channelName: '#secret', + }); + const state = ctx.window._channelsGetStateForTest(); + assert.deepStrictEqual(Array.from(state.messages, (m) => m.text), ['cached 0', 'cached 1', 'cached 2']); +}); + +Promise.all(pending).then(() => { + console.log(`\n${passed} passed, ${failed} failed`); + if (failed > 0) process.exit(1); +}); From efdb3ea0b3232a8b363842fdd9f6a84cff4fae9d Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 20:37:14 +0200 Subject: [PATCH 038/151] feat(analytics): retransmission pressure over time (#1699) (#2023) ## Summary Adds `GET /api/analytics/retransmissions` and a "Retransmission Pressure (proxy)" chart on the Analytics Topology tab, implementing the metric agreed in #1699: for each flood, the number of distinct repeaters in the union of the paths of all its observations (`[A]`, `[A,B,C]`, `[A,D]` gives 4), averaged per time bucket. Topology is the tab that already shows hop counts and repeaters in paths, so the chart sits there instead of in a new tab. ## Definition - Flood routes only (`route_type` 0/1); TRACE excluded. Direct routes carry the route still to travel (firmware `src/Mesh.cpp:78-106,334-342`), zero-hop sends are direct (`src/Mesh.cpp:717-737`), TRACE path bytes are SNR values (`src/Mesh.cpp:59-61`, refused by `sendFlood` at `src/Mesh.cpp:637-641`). Firmware commit 0679dbef. - **Flood events, not hashes.** `transmissions.hash` is UNIQUE and the packet hash excludes the path (`src/Packet.cpp:41-50`), so when the same bytes flood again the observations land on the same transmission. Observations are sorted by time and split into events wherever two consecutive observations are more than 5 minutes apart. Each event is counted on its own and bucketed by its first observation. - Why 5 minutes: a node holds a flood for at most 32 s (`src/Dispatcher.cpp:11,243-251`) plus a random retransmit delay. On live over 7 days, 72,806 of 74,347 flood transmissions span 60 s or less, and of 1,372,283 consecutive observation gaps, 52 fall between 60 s and 300 s against 1,823 above 300 s. - Events that start before the store retention floor (now minus `retentionHours`) are left out for every request shape. The store keeps older observations only for hashes heard again recently, so they do not represent that period. Eviction of those transmissions is tracked in #2024. - A flood event heard only with an empty path counts as 0 repeaters. - **Prefixes are not resolved to nodes, and a prefix counts once per event**, whether it repeats across observations or inside one path. On live (7 days), a repeated 2-byte prefix inside one path occurs in 1.08% of flood transmissions and 6,178 of 6,596 such repeats match exactly one known node; for 3-byte it is 0.69% and 104 of 104. That is one node forwarding again after its 160-slot cyclic duplicate filter dropped the hash (`src/helpers/SimpleMeshTables.h:9,52-57`). A repeated 1-byte prefix (44.9% of 1-byte transmissions) is mostly two nodes; counting it once keeps the value a lower bound. `summary.one_byte_packets` reports how many events that affects. - Observations are stored once per observer and path per hash, so a later event of the same hash only holds pairs not stored before; its count is a lower bound too. On live these are 1,466 of 75,356 events (1.9%), and they are kept in the average. - Resolution was not used: on live, 1-byte observations nearly all have `resolved_path` NULL, and cold load refuses context-based resolution of history (`cmd/server/neighbor_persist.go:155-168`). - Buckets `5m|15m|1h|6h|1d`. `region` filters on observers like `/api/analytics/rf`, after the event split; a region with no known observers is not filtered, the same as the other analytics endpoints. `area` is not supported. ## Implementation - `cmd/server/retransmission_pressure.go:255` `addPath`: scans path JSON directly into a generation-stamped hash set, no allocation per observation. - `cmd/server/retransmission_pressure.go:367` `computeRetransmissionPressure`: one pass under `s.mu.RLock`. Per flood transmission it sorts the observations by cached parsed time into a reused scratch slice, splits events and counts each in `addEvent` (`:319`). O(T + O log k + H). - `cmd/server/retransmission_pressure.go:470` `GetRetransmissionPressure`: default shape from the recomputer (#1659 warm-up gate). Other shapes come from a typed TTL cache (max 64 entries) cleared on new paths and eviction (`cmd/server/store.go:2289,2336`); concurrent misses on one key share one compute through singleflight (`store.go:204`). - `cmd/server/retransmission_pressure.go:515` handler, `cmd/server/routes.go:331`, `cmd/server/openapi.go:108`, `docs/api-spec.md:1283`. - `public/analytics.js:761` card, `:855` `renderRetransmissionChart` (CSS variables only, lines break at missing buckets, caption states it is a proxy, names the observer coverage bias, the once-per-flood prefix rule and the 5 minute event split), `:829` loader with stale-response guard. ## Performance - `BenchmarkComputeRetransmissionPressure`, 50k transmissions x 20 observations, `-cpu 1`, i5-1335U: median 161 ms/op (132 ms/op before the event split); first pass after startup with timestamps not yet parsed 192 ms/op. About 22 KB and 281 allocations per op. - On staging the default shape is served from the recomputer in 0.3 s; the post-load recompute of this recomputer took 994 ms on a 121k-transmission store (log line quoted in #2025). A 336h store would be about twice that, every recompute interval, under the store read lock. ## Tests - `cmd/server/retransmission_pressure_test.go`: union counting (reporter example, overlaps, once per event for 1/2/3-byte, width/case, growth); route/TRACE/zero-hop filter, bucketing, window by event start; event split and the 5 minute settle gap (boundary, chained steps, unsorted input), retention floor; region filter, region applied after the split, unknown region, 1-byte share; recomputer read, TTL cache invalidation on new paths and on eviction, cache expiry, singleflight, recomputer gate wiring, handler, warm-up gate. - `test-issue-1699-retransmission-chart.js` (33 tests, registered in `test-all.sh` and `deploy.yml`). - Mutation-checked: 18 mutations of the event split, floor, prefix rule, bucketing, region order, cache clears, expiry, gate wiring and singleflight, all killed. - `go test ./...` in cmd/server passes; `scripts/check-css-vars.js` OK. ## Staging validation Build `c646310f` (this rework plus #2025 and the other review follow-ups), after a container restart and full load: default shape 74,974 flood events, average 27.08 repeaters, 169 hourly buckets from 2026-09-06 16:00 (the 168h floor) to the current hour, highest hourly average 53.3. Before the rework the same instance showed buckets back to 2026-07-18, averages up to 148, and for the first minutes after a restart only 5,911 packets. ## Merge order with #2025 #2025 fixes the recomputer startup for all analytics endpoints (the stale first snapshot seen here). Whichever of the two merges second has to add `recompRetransmissions` to `analyticsRecomputersLocked`, wire it to that PR's `loadedGate` instead of `LoadComplete`, bump the recomputer count in `TestAnalyticsRecomputers_PostLoadOrder` from 9 to 10, and make `TestStartAnalyticsRecomputers_RetransmissionsGatedOnLoadComplete` call `signalStartupLoadDone()`. That resolution is what ran on staging. ## Not verified - Recompute timing on a production-size (336h) store; only extrapolated. - Phone-width layout and dark theme of the reworked chart. - E2E Playwright suite. Fixes #1699 --------- Co-authored-by: Claude Opus 5 (1M context) --- .../analytics_recompute_after_load_test.go | 4 +- cmd/server/analytics_recomputer.go | 8 + cmd/server/openapi.go | 8 + cmd/server/retransmission_pressure.go | 534 ++++++++++++++ cmd/server/retransmission_pressure_test.go | 685 ++++++++++++++++++ cmd/server/routes.go | 1 + cmd/server/store.go | 9 + docs/api-spec.md | 88 +++ public/analytics.js | 143 ++++ test-all.sh | 1 + test-issue-1699-retransmission-chart.js | 130 ++++ 11 files changed, 1609 insertions(+), 2 deletions(-) create mode 100644 cmd/server/retransmission_pressure.go create mode 100644 cmd/server/retransmission_pressure_test.go create mode 100644 test-issue-1699-retransmission-chart.js diff --git a/cmd/server/analytics_recompute_after_load_test.go b/cmd/server/analytics_recompute_after_load_test.go index f18cbb418..0c4eac044 100644 --- a/cmd/server/analytics_recompute_after_load_test.go +++ b/cmd/server/analytics_recompute_after_load_test.go @@ -221,8 +221,8 @@ func TestAnalyticsRecomputers_PostLoadOrder(t *testing.T) { for i, rc := range list { pos[rc.name] = i } - if len(list) != 9 || len(pos) != 9 { - t.Fatalf("want 9 distinct recomputers, got %d (%d distinct)", len(list), len(pos)) + if len(list) != 10 || len(pos) != 10 { + t.Fatalf("want 10 distinct recomputers, got %d (%d distinct)", len(list), len(pos)) } for _, name := range []string{"rf", "topology", "channels"} { if pos[name] > 2 { diff --git a/cmd/server/analytics_recomputer.go b/cmd/server/analytics_recomputer.go index f0db263c8..3cf30c200 100644 --- a/cmd/server/analytics_recomputer.go +++ b/cmd/server/analytics_recomputer.go @@ -258,6 +258,7 @@ func (s *PacketStore) analyticsRecomputersLocked() []*analyticsRecomputer { s.recompDistance, s.recompHashCollisions, s.recompHashSizes, s.recompObserversClockSkew, s.recompNodesClockSkew, s.recompRoles, + s.recompRetransmissions, } } @@ -331,6 +332,12 @@ func (s *PacketStore) StartAnalyticsRecomputers(defaultInterval time.Duration, o "nodes-clock-skew", pickInterval(ov.NodesClockSkew, defaultInterval), func() interface{} { return s.computeFleetClockSkew() }, ) + s.recompRetransmissions = newAnalyticsRecomputer( + "retransmissions", defaultInterval, + func() interface{} { + return s.computeRetransmissionPressure("", TimeWindow{}, retransmissionDefaultBucket) + }, + ) all := s.analyticsRecomputersLocked() s.analyticsRecomputerMu.Unlock() @@ -352,6 +359,7 @@ func (s *PacketStore) StartAnalyticsRecomputers(defaultInterval time.Duration, o s.recompRF.setWarmupReadyGate_1659(loadedGate) s.recompTopology.setWarmupReadyGate_1659(loadedGate) s.recompChannels.setWarmupReadyGate_1659(loadedGate) + s.recompRetransmissions.setWarmupReadyGate_1659(loadedGate) for _, rc := range all { rc.Start() diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index da6522cc2..69aedc389 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -109,6 +109,14 @@ func routeDescriptions() map[string]routeMeta { "GET /api/analytics/subpaths-bulk": {Summary: "Bulk subpath analysis", Tag: "analytics"}, "GET /api/analytics/subpath-detail": {Summary: "Subpath detail", Tag: "analytics"}, "GET /api/analytics/neighbor-graph": {Summary: "Neighbor graph", Description: "Full neighbor affinity graph for visualization.", Tag: "analytics"}, + "GET /api/analytics/retransmissions": {Summary: "Retransmission pressure over time", Description: "Collision-pressure proxy (#1699): per time bucket, the average number of distinct repeaters in the union of all observed paths of each flood event (route types 0/1, TRACE excluded). A transmission's observations are split into flood events at gaps of more than 5 minutes; each event is bucketed by its first observation, and events before the store retention floor are left out. Hop prefixes are not resolved: a prefix counts once per event, so colliding 1-byte prefixes make this a lower bound. Only repeaters some observer heard are counted, so the value also follows observer coverage; each bucket carries its observer count.", Tag: "analytics", + QueryParams: []paramMeta{ + {Name: "region", Description: "Comma-separated IATA codes; only observations from observers in the region are counted. A region with no known observers is not filtered", Type: "string"}, + {Name: "window", Description: "Relative window: 1h, 24h, 3d, 7d or 30d", Type: "string"}, + {Name: "from", Description: "Absolute window start (RFC3339)", Type: "string"}, + {Name: "to", Description: "Absolute window end (RFC3339)", Type: "string"}, + {Name: "bucket", Description: "Bucket size: 5m, 15m, 1h, 6h or 1d (default 1h)", Type: "string"}, + }}, // Channels "GET /api/channels": {Summary: "List channels", Description: "Returns known mesh channels with message counts.", Tag: "channels"}, diff --git a/cmd/server/retransmission_pressure.go b/cmd/server/retransmission_pressure.go new file mode 100644 index 000000000..0ef539eff --- /dev/null +++ b/cmd/server/retransmission_pressure.go @@ -0,0 +1,534 @@ +package main + +// retransmission_pressure.go: issue #1699. +// +// Network-wide time series of how many distinct repeaters took part in +// relaying each flood, as a proxy for collision pressure. Definition agreed +// in the #1699 thread: for one flood, take the union of the paths of ALL its +// observations and count the distinct repeaters in it (paths [A], [A,B,C], +// [A,D] -> 4). Per time bucket we report the average of that count over the +// floods that started in the bucket. +// +// It is a proxy, not a measured collision rate: a repeater that forwarded a +// packet no observer heard is invisible, so the number moves with observer +// coverage too. The response carries the per-bucket observer count so the UI +// can show that. +// +// Protocol facts (MeshCore firmware, commit 0679dbef): +// - Only flood routes build a path of forwarders: routeRecvPacket appends +// the forwarder's own hash to the end of the path (src/Mesh.cpp:344-356). +// ROUTE_TYPE_TRANSPORT_FLOOD (0) and ROUTE_TYPE_FLOOD (1) are both flood +// (src/Packet.h:14-15,64). +// - Direct routes carry the route still to travel; each hop removes itself +// (removeSelfFromPath, src/Mesh.cpp:78-106,334-342), so the observed path +// is not the forwarders. Zero-hop sends are ROUTE_TYPE_DIRECT / +// TRANSPORT_DIRECT with path_len 0 (src/Mesh.cpp:717-737). Both are +// excluded by the route filter. +// - TRACE never floods (sendFlood refuses it, src/Mesh.cpp:637-641) and its +// path bytes are SNR values, not hashes (src/Mesh.cpp:59-61). Excluded +// explicitly as well. +// - A path entry is the first 1-3 bytes of the forwarder's public key; the +// width is chosen by the originator and is the same for every hop of one +// packet (src/Mesh.cpp:649, src/Packet.h:79-83, src/Identity.h:23-25). +// - The duplicate filter (wasSeen/markSeen, e.g. src/Mesh.cpp:121-126) is a +// cyclic buffer of 160 packet hashes (src/helpers/SimpleMeshTables.h:9, +// 52-57). Once a hash is overwritten, the node forwards that packet again +// if it comes back, so the same node can appear twice in one path. +// - A node holds a received flood for at most 32 s before handling it +// (MAX_RX_DELAY_MILLIS, src/Dispatcher.cpp:11,243-251), plus a random +// retransmit delay of a few airtimes (examples/simple_repeater/ +// MyMesh.cpp:547-550). +// +// Flood events. transmissions.hash is UNIQUE and the packet hash excludes the +// path (src/Packet.cpp:41-50), so when the same bytes are flooded again later +// the new observations are appended to the existing transmission. The union +// over all of them would merge separate floods. We sort a transmission's +// observations by time and start a new event when the gap to the previous +// observation exceeds retransmissionEventGap; each event is counted on its +// own, in the bucket of its first observation. +// +// Retention floor. Events that start before now - retentionHours are dropped +// for every request shape: the store keeps observations from before the +// floor only for hashes that were heard again recently, so they are not the +// traffic of that period. +// +// Prefixes are not resolved. A 1-byte prefix is shared by many repeaters, and +// we do NOT resolve hops to public keys here: the resolved-pubkey index is +// empty for observations whose resolved_path is NULL (on live nearly every +// 1-byte observation), and context-based resolution of history is refused on +// purpose elsewhere (resolvePathForObsColdLoad, PR #1643). Counting rule: a +// prefix counts once per event, whether it repeats inside one path or across +// paths. A repeated 2- or 3-byte prefix is one node forwarding twice (see the +// duplicate filter above). A repeated 1-byte prefix can also be two nodes; +// counting it once keeps the value a lower bound, as does merging repeaters +// that share a prefix across paths. The share of 1-byte packets is reported +// so the undercount can be judged. +// +// Region. region filters on the observers of that region, like +// /api/analytics/rf. Events are split on all observations, so the filter +// does not change where an event starts. A region with no known observers is +// not filtered, the same as the other analytics endpoints. +// +// Complexity: one pass over s.packets under s.mu.RLock, O(T + O log k + H) +// for T transmissions, O observations of flood packets (k per transmission, +// sorted by time) and H hop entries. Timestamps are parsed once per +// observation and cached (StoreObs.ParsedTime). Memory: one entry per +// non-empty bucket, a per-pass observer index and scratch reused across +// transmissions. Served from the analytics recomputer for the default shape +// and from a TTL cache otherwise, never computed per request on a warm +// cache; concurrent misses on one key share a single compute. + +import ( + "cmp" + "math/bits" + "net/http" + "slices" + "sort" + "strings" + "time" +) + +// RetransmissionBucket is one time bucket of the series. +type RetransmissionBucket struct { + Start string `json:"start"` // bucket start, RFC3339 UTC + Packets int `json:"packets"` // flood events that started in the bucket + RepeaterSum int `json:"repeater_sum"` // sum of distinct repeaters over those events + AvgRepeaters float64 `json:"avg_repeaters"` // repeater_sum / packets + Observers int `json:"observers"` // distinct observers that heard those events +} + +// RetransmissionSummary aggregates the whole response window. +type RetransmissionSummary struct { + Packets int `json:"packets"` + AvgRepeaters float64 `json:"avg_repeaters"` + Observers int `json:"observers"` + OneBytePackets int `json:"one_byte_packets"` // events whose hops are 1-byte hashes (most ambiguous) + NoRepeaterPackets int `json:"no_repeater_packets"` // events heard with an empty path only +} + +// RetransmissionResponse is the /api/analytics/retransmissions body. +type RetransmissionResponse struct { + BucketSeconds int `json:"bucket_seconds"` + Window string `json:"window"` + Region string `json:"region"` + Summary RetransmissionSummary `json:"summary"` + Buckets []RetransmissionBucket `json:"buckets"` +} + +type retransmissionCacheEntry struct { + data RetransmissionResponse + expiresAt time.Time +} + +// retransmissionCacheMax bounds the TTL cache: ?from=&to= makes the key space +// open-ended, and invalidation only runs when new paths arrive. +const retransmissionCacheMax = 64 + +const retransmissionDefaultBucket = time.Hour + +// retransmissionEventGap is the settle time that separates two flood events +// of one transmission. It is well above the longest per-hop hold in firmware +// (32 s plus a retransmit delay, see the file header), so a flood still +// spreading is not cut. Gaps between re-floods of the same bytes range from +// minutes to weeks; a re-flood within 5 minutes merges into one event. +const retransmissionEventGap = 5 * time.Minute + +// parseRetransmissionBucket maps the ?bucket= value to a duration. Unknown +// values fall back to the default, matching how ParseTimeWindow ignores +// invalid input. +func parseRetransmissionBucket(v string) time.Duration { + switch v { + case "5m": + return 5 * time.Minute + case "15m": + return 15 * time.Minute + case "6h": + return 6 * time.Hour + case "1d": + return 24 * time.Hour + } + return retransmissionDefaultBucket +} + +// repeaterUnion counts the distinct hop prefixes across the observed paths of +// one flood event (see the counting rule in the file header). Hops are keyed +// as (byte width, value) so case does not matter and a 1-byte "AB" differs +// from a 2-byte "AB00". Reuse one value across events via reset(). +// +// The set is an open-addressing hash table whose slots are stamped with a +// generation, so reset() is O(1) instead of clearing the table. +type repeaterUnion struct { + slotKey []uint64 + slotGen []uint32 // slot is live when slotGen[i] == gen + gen uint32 + used int + width int // byte width of the first hop seen, 0 if none +} + +const repeaterUnionMinSlots = 256 + +func (u *repeaterUnion) reset() { + if u.slotKey == nil { + u.allocSlots(repeaterUnionMinSlots) + } + u.gen++ + if u.gen == 0 { + clear(u.slotGen) + u.gen = 1 + } + u.used = 0 + u.width = 0 +} + +func (u *repeaterUnion) allocSlots(n int) { + u.slotKey = make([]uint64, n) + u.slotGen = make([]uint32, n) +} + +// slot returns the index holding k, or the free index where k belongs. +func (u *repeaterUnion) slot(k uint64) int { + mask := len(u.slotKey) - 1 + i := int((k * 0x9E3779B97F4A7C15) >> 40 & uint64(mask)) + for u.slotGen[i] == u.gen && u.slotKey[i] != k { + i = (i + 1) & mask + } + return i +} + +// grow doubles the table, keeping the live entries. Load stays below 1/2, +// so slot() always finds a free index. +func (u *repeaterUnion) grow() { + oldKey, oldGen, gen := u.slotKey, u.slotGen, u.gen + u.allocSlots(2 * len(oldKey)) + u.gen = 1 + for i := range oldKey { + if oldGen[i] == gen { + j := u.slot(oldKey[i]) + u.slotKey[j], u.slotGen[j] = oldKey[i], u.gen + } + } +} + +func (u *repeaterUnion) add(k uint64) { + if u.width == 0 { + u.width = int(k >> 32) + } + i := u.slot(k) + if u.slotGen[i] == u.gen { + return + } + if 2*(u.used+1) > len(u.slotKey) { + u.grow() + i = u.slot(k) + } + u.slotKey[i], u.slotGen[i] = k, u.gen + u.used++ +} + +// hopKey parses a hex hop into (width<<32 | value). ok is false for anything +// that is not 1-4 bytes of hex. +func hopKey(hop string) (uint64, bool) { + if len(hop) == 0 || len(hop) > 8 || len(hop)%2 != 0 { + return 0, false + } + var v uint64 + for i := 0; i < len(hop); i++ { + c := hop[i] + switch { + case c >= '0' && c <= '9': + c -= '0' + case c >= 'a' && c <= 'f': + c = c - 'a' + 10 + case c >= 'A' && c <= 'F': + c = c - 'A' + 10 + default: + return 0, false + } + v = v<<4 | uint64(c) + } + return uint64(len(hop)/2)<<32 | v, true +} + +// addPath folds one observation's path_json (a JSON array of hex strings) +// into the union. Scans the string directly: hop tokens are plain hex, so no +// JSON decoder and no allocation are needed. +func (u *repeaterUnion) addPath(pathJSON string) { + for i := 0; i < len(pathJSON); i++ { + if pathJSON[i] != '"' { + continue + } + end := strings.IndexByte(pathJSON[i+1:], '"') + if end < 0 { + break + } + if k, ok := hopKey(pathJSON[i+1 : i+1+end]); ok { + u.add(k) + } + i += end + 1 + } +} + +func (u *repeaterUnion) count() int { + return u.used +} + +type retransmissionBucketAgg struct { + packets int + repeaterSum int + observers []uint64 // bitset over the pass-local observer index +} + +func setBit(set []uint64, i int) []uint64 { + for len(set) <= i/64 { + set = append(set, 0) + } + set[i/64] |= 1 << uint(i%64) + return set +} + +func popCount(set []uint64) int { + n := 0 + for _, w := range set { + n += bits.OnesCount64(w) + } + return n +} + +// timedObs is an observation with its parsed time, for sorting one +// transmission's observations into flood events. +type timedObs struct { + at int64 // unix nanoseconds + obs *StoreObs +} + +// retransmissionPass accumulates one computeRetransmissionPressure pass. +type retransmissionPass struct { + regionObs map[string]bool + floor time.Time // events starting before it are dropped; zero = none + since, until time.Time + bucketSec int64 + aggs map[int64]*retransmissionBucketAgg + obsIndex map[string]int + allObservers []uint64 + union repeaterUnion + summary RetransmissionSummary + totalRepeaters int +} + +// addEvent counts one flood event: ev holds its observations in time order. +func (p *retransmissionPass) addEvent(ev []timedObs) { + t := time.Unix(0, ev[0].at) + if (!p.floor.IsZero() && t.Before(p.floor)) || + (!p.since.IsZero() && t.Before(p.since)) || (!p.until.IsZero() && t.After(p.until)) { + return + } + start := t.Unix() - t.Unix()%p.bucketSec + agg := p.aggs[start] + u := &p.union + u.reset() + heard := false + for _, e := range ev { + obs := e.obs + if p.regionObs != nil && !p.regionObs[obs.ObserverID] { + continue + } + if agg == nil { + agg = &retransmissionBucketAgg{} + p.aggs[start] = agg + } + heard = true + u.addPath(obs.PathJSON) + idx, ok := p.obsIndex[obs.ObserverID] + if !ok { + idx = len(p.obsIndex) + p.obsIndex[obs.ObserverID] = idx + } + agg.observers = setBit(agg.observers, idx) + p.allObservers = setBit(p.allObservers, idx) + } + if !heard { + return + } + n := u.count() + agg.packets++ + agg.repeaterSum += n + p.summary.Packets++ + p.totalRepeaters += n + if n == 0 { + p.summary.NoRepeaterPackets++ + } + if u.width == 1 { + p.summary.OneBytePackets++ + } +} + +// computeRetransmissionPressure builds the series (see the file header for +// flood events, the retention floor, the counting rule and region). +func (s *PacketStore) computeRetransmissionPressure(region string, window TimeWindow, bucket time.Duration) RetransmissionResponse { + if bucket <= 0 { + bucket = retransmissionDefaultBucket + } + p := retransmissionPass{ + bucketSec: int64(bucket / time.Second), + aggs: make(map[int64]*retransmissionBucketAgg), + obsIndex: make(map[string]int), + } + if region != "" { + p.regionObs = s.resolveRegionObservers(region) + } + if window.Since != "" { + p.since, _ = parseAnyRFC3339(window.Since) + } + if window.Until != "" { + p.until, _ = parseAnyRFC3339(window.Until) + } + + s.mu.RLock() + defer s.mu.RUnlock() + + if s.retentionHours > 0 { + p.floor = time.Now().Add(-time.Duration(s.retentionHours * float64(time.Hour))) + } + gap := int64(retransmissionEventGap) + var timed []timedObs + + for _, tx := range s.packets { + if tx.RouteType == nil || (*tx.RouteType != RouteFlood && *tx.RouteType != RouteTransportFlood) { + continue + } + if tx.PayloadType != nil && *tx.PayloadType == PayloadTRACE { + continue + } + timed = timed[:0] + for _, obs := range tx.Observations { + if at, ok := obs.ParsedTime(); ok { + timed = append(timed, timedObs{at: at.UnixNano(), obs: obs}) + } + } + slices.SortFunc(timed, func(a, b timedObs) int { return cmp.Compare(a.at, b.at) }) + for start := 0; start < len(timed); { + end := start + 1 + for end < len(timed) && timed[end].at-timed[end-1].at <= gap { + end++ + } + p.addEvent(timed[start:end]) + start = end + } + } + + starts := make([]int64, 0, len(p.aggs)) + for st := range p.aggs { + starts = append(starts, st) + } + sort.Slice(starts, func(i, j int) bool { return starts[i] < starts[j] }) + buckets := make([]RetransmissionBucket, 0, len(starts)) + for _, st := range starts { + a := p.aggs[st] + buckets = append(buckets, RetransmissionBucket{ + Start: time.Unix(st, 0).UTC().Format(time.RFC3339), + Packets: a.packets, + RepeaterSum: a.repeaterSum, + AvgRepeaters: float64(a.repeaterSum) / float64(a.packets), + Observers: popCount(a.observers), + }) + } + summary := p.summary + if summary.Packets > 0 { + summary.AvgRepeaters = float64(p.totalRepeaters) / float64(summary.Packets) + } + summary.Observers = popCount(p.allObservers) + + label := window.Label + if label == "" && !window.IsZero() { + label = window.Since + "/" + window.Until + } + return RetransmissionResponse{ + BucketSeconds: int(p.bucketSec), + Window: label, + Region: region, + Summary: summary, + Buckets: buckets, + } +} + +func isDefaultRetransmissionShape(region string, window TimeWindow, bucket time.Duration) bool { + return region == "" && window.IsZero() && bucket == retransmissionDefaultBucket +} + +// retransCacheGet returns a fresh cached result for key. Caller must hold +// s.cacheMu. +func (s *PacketStore) retransCacheGet(key string) (RetransmissionResponse, bool) { + if e, ok := s.retransCache[key]; ok && time.Now().Before(e.expiresAt) { + return e.data, true + } + return RetransmissionResponse{}, false +} + +// GetRetransmissionPressure serves the default shape from the recomputer +// snapshot and every other shape from the TTL cache (compute on miss, +// concurrent misses on one key share the compute). +func (s *PacketStore) GetRetransmissionPressure(region string, window TimeWindow, bucket time.Duration) RetransmissionResponse { + if isDefaultRetransmissionShape(region, window, bucket) { + s.analyticsRecomputerMu.RLock() + rc := s.recompRetransmissions + s.analyticsRecomputerMu.RUnlock() + if rc != nil { + if r, ok := rc.Load().(RetransmissionResponse); ok { + s.cacheMu.Lock() + s.cacheHits++ + s.cacheMu.Unlock() + return r + } + } + } + key := region + "|" + window.CacheKey() + "|" + bucket.String() + s.cacheMu.Lock() + if r, ok := s.retransCacheGet(key); ok { + s.cacheHits++ + s.cacheMu.Unlock() + return r + } + s.cacheMisses++ + s.cacheMu.Unlock() + + v, _, _ := s.retransSF.Do(key, func() (interface{}, error) { + // A caller that joins right after a winner stored its result must + // not start a second pass. + s.cacheMu.Lock() + r, ok := s.retransCacheGet(key) + s.cacheMu.Unlock() + if ok { + return r, nil + } + result := s.computeRetransmissionPressure(region, window, bucket) + s.cacheMu.Lock() + if s.retransCache == nil || len(s.retransCache) >= retransmissionCacheMax { + s.retransCache = make(map[string]*retransmissionCacheEntry) + } + s.retransCache[key] = &retransmissionCacheEntry{data: result, expiresAt: time.Now().Add(s.rfCacheTTL)} + s.cacheMu.Unlock() + return result, nil + }) + return v.(RetransmissionResponse) +} + +func (s *Server) handleAnalyticsRetransmissions(w http.ResponseWriter, r *http.Request) { + region := r.URL.Query().Get("region") + window := ParseTimeWindow(r) + bucket := parseRetransmissionBucket(r.URL.Query().Get("bucket")) + if s.store == nil { + writeJSON(w, RetransmissionResponse{BucketSeconds: int(bucket / time.Second), Buckets: []RetransmissionBucket{}}) + return + } + // #1659 warmup gate (see handleAnalyticsRF for rationale). + if isDefaultRetransmissionShape(region, window, bucket) { + s.store.analyticsRecomputerMu.RLock() + rc := s.store.recompRetransmissions + s.store.analyticsRecomputerMu.RUnlock() + if rc != nil && rc.IsWarmingUp_1659() { + writeAnalyticsWarmup503(w) + return + } + } + writeJSON(w, s.store.GetRetransmissionPressure(region, window, bucket)) +} diff --git a/cmd/server/retransmission_pressure_test.go b/cmd/server/retransmission_pressure_test.go new file mode 100644 index 000000000..9226ba272 --- /dev/null +++ b/cmd/server/retransmission_pressure_test.go @@ -0,0 +1,685 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "runtime" + "strings" + "sync" + "testing" + "time" +) + +// --- repeaterUnion: the per-transmission count (issue #1699) --- + +func unionOf(paths ...string) int { + var u repeaterUnion + u.reset() + for _, p := range paths { + u.addPath(p) + } + return u.count() +} + +// The reporter's worked example from the #1699 thread: observer A saw +// [A] and [A,B,C], observer B saw [A,D]. Four repeaters took part. +func TestRepeaterUnion_ReporterExample(t *testing.T) { + got := unionOf(`["A1"]`, `["A1","B2","C3"]`, `["A1","D4"]`) + if got != 4 { + t.Fatalf("union = %d, want 4", got) + } +} + +func TestRepeaterUnion_OverlappingPathsCountOnce(t *testing.T) { + got := unionOf(`["FD35","95F8","3363"]`, `["FD35","95F8","3363"]`, `["FD35","95F8"]`, `["FD35","95F8","4F47"]`) + if got != 4 { + t.Fatalf("union = %d, want 4 (FD35, 95F8, 3363, 4F47)", got) + } +} + +// A hop prefix counts once per flood, wherever and however often it appears. +// Inside one path a repeated 2- or 3-byte prefix is on live data almost +// always one known node forwarding twice (the firmware dedup table is a +// cyclic 160-slot buffer, SimpleMeshTables.h:9,52-57), so it is one +// repeater. A repeated 1-byte prefix may be two nodes, but counting it once +// keeps the value a lower bound, like the cross-observation merge. +func TestRepeaterUnion_PrefixCountsOncePerFlood(t *testing.T) { + if got := unionOf(`["12","34","12"]`); got != 2 { + t.Errorf("1-byte prefix twice in one path = %d, want 2", got) + } + if got := unionOf(`["AB01","CD02","AB01"]`); got != 2 { + t.Errorf("2-byte prefix twice in one path = %d, want 2", got) + } + if got := unionOf(`["AB01C3","CD02D4","AB01C3"]`); got != 2 { + t.Errorf("3-byte prefix twice in one path = %d, want 2", got) + } + if got := unionOf(`["12","34"]`, `["56","12"]`); got != 3 { + t.Errorf("same prefix in two paths = %d, want 3 (merged, lower bound)", got) + } + if got := unionOf(`["12","34","12"]`, `["12"]`, `["12","12","12"]`); got != 2 { + t.Errorf("repeats within and across paths = %d, want 2 (12, 34)", got) + } +} + +func TestRepeaterUnion_HashSizeAndCaseAreDistinctKeys(t *testing.T) { + if got := unionOf(`["ab"]`, `["AB"]`); got != 1 { + t.Errorf("case variants = %d, want 1", got) + } + // A 1-byte "AB" and a 2-byte "AB00" are different hash widths. + if got := unionOf(`["AB"]`, `["AB00"]`); got != 2 { + t.Errorf("1-byte vs 2-byte = %d, want 2", got) + } +} + +func TestRepeaterUnion_EmptyAndMalformed(t *testing.T) { + if got := unionOf("", `[]`); got != 0 { + t.Errorf("empty paths = %d, want 0", got) + } + if got := unionOf(`["ZZ","A1"]`); got != 1 { + t.Errorf("non-hex hop must be ignored, got %d want 1", got) + } +} + +func TestRepeaterUnion_ResetClearsState(t *testing.T) { + var u repeaterUnion + u.reset() + u.addPath(`["A1","B2"]`) + u.reset() + u.addPath(`["C3"]`) + if got := u.count(); got != 1 { + t.Fatalf("after reset count = %d, want 1", got) + } +} + +func TestRepeaterUnion_GrowsPastInitialTable(t *testing.T) { + var u repeaterUnion + u.reset() + const distinct = 1000 // well past repeaterUnionMinSlots/2 + for i := 0; i < distinct; i += 4 { + u.addPath(fmt.Sprintf(`["%04X","%04X","%04X","%04X"]`, i, i+1, i+2, i+3)) + } + u.addPath(`["0000","0001"]`) // already present, must not add + if got := u.count(); got != distinct { + t.Fatalf("count after growth = %d, want %d", got, distinct) + } + u.reset() + u.addPath(`["0000","ABCD"]`) + if got := u.count(); got != 2 { + t.Fatalf("count after reset on grown table = %d, want 2", got) + } +} + +// --- computeRetransmissionPressure --- + +func rtxInt(v int) *int { return &v } + +func rtxTx(id int, route, payload int, firstSeen string, obs ...*StoreObs) *StoreTx { + tx := &StoreTx{ + ID: id, + Hash: fmt.Sprintf("h%06d", id), + FirstSeen: firstSeen, + RouteType: rtxInt(route), + PayloadType: rtxInt(payload), + } + for _, o := range obs { + o.TransmissionID = id + if o.Timestamp == "" { + o.Timestamp = firstSeen + } + tx.Observations = append(tx.Observations, o) + } + return tx +} + +// rtxObs is an observation heard at the transmission's first_seen. +func rtxObs(observer, path string) *StoreObs { + return &StoreObs{ObserverID: observer, PathJSON: path} +} + +func rtxObsAt(observer, path, ts string) *StoreObs { + return &StoreObs{ObserverID: observer, PathJSON: path, Timestamp: ts} +} + +func rtxStore(packets ...*StoreTx) *PacketStore { + return &PacketStore{packets: packets} +} + +func bucketByStart(t *testing.T, r RetransmissionResponse, start string) RetransmissionBucket { + t.Helper() + for _, b := range r.Buckets { + if b.Start == start { + return b + } + } + t.Fatalf("no bucket %s in %+v", start, r.Buckets) + return RetransmissionBucket{} +} + +func TestComputeRetransmissionPressure_RouteAndPayloadFilter(t *testing.T) { + ts := "2026-09-13T10:10:00Z" + s := rtxStore( + rtxTx(1, RouteFlood, PayloadADVERT, ts, rtxObs("o1", `["A1","B2"]`), rtxObs("o2", `["A1","C3"]`)), + rtxTx(2, RouteTransportFlood, PayloadGRP_TXT, ts, rtxObs("o1", `["D4"]`)), + // Flood heard only straight from the originator: a packet with 0 + // observed repeaters, kept in the denominator. + rtxTx(3, RouteFlood, PayloadGRP_TXT, ts, rtxObs("o1", `[]`)), + // Direct routes carry the remaining route, not the forwarders. + rtxTx(4, RouteDirect, PayloadTXT_MSG, ts, rtxObs("o1", `["E5","F6"]`)), + rtxTx(5, RouteTransportDirect, PayloadTXT_MSG, ts, rtxObs("o1", `["E5"]`)), + // TRACE path bytes are SNR values, never forwarder hashes. + rtxTx(6, RouteFlood, PayloadTRACE, ts, rtxObs("o1", `["E5","F6"]`)), + // Missing route type cannot be classified. + &StoreTx{ID: 7, FirstSeen: ts, Observations: []*StoreObs{rtxObs("o1", `["E5"]`)}}, + ) + r := s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + if r.BucketSeconds != 3600 { + t.Errorf("bucket_seconds = %d, want 3600", r.BucketSeconds) + } + if r.Summary.Packets != 3 { + t.Fatalf("summary packets = %d, want 3 (tx 1,2,3)", r.Summary.Packets) + } + b := bucketByStart(t, r, "2026-09-13T10:00:00Z") + if b.Packets != 3 || b.RepeaterSum != 4 { + t.Fatalf("bucket = %+v, want packets 3 repeater_sum 4 (3+1+0)", b) + } + if want := 4.0 / 3.0; b.AvgRepeaters < want-1e-9 || b.AvgRepeaters > want+1e-9 { + t.Errorf("avg_repeaters = %v, want %v", b.AvgRepeaters, want) + } + if b.Observers != 2 { + t.Errorf("observers = %d, want 2", b.Observers) + } + if r.Summary.NoRepeaterPackets != 1 { + t.Errorf("no_repeater_packets = %d, want 1", r.Summary.NoRepeaterPackets) + } +} + +func TestComputeRetransmissionPressure_Bucketing(t *testing.T) { + s := rtxStore( + rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:05:00Z", rtxObs("o1", `["A1","B2"]`)), + rtxTx(2, RouteFlood, PayloadADVERT, "2026-09-13T10:55:59Z", rtxObs("o2", `["A1"]`)), + rtxTx(3, RouteFlood, PayloadADVERT, "2026-09-13T11:00:00Z", rtxObs("o1", `["A1","B2","C3","D4"]`)), + rtxTx(4, RouteFlood, PayloadADVERT, "not-a-time", rtxObs("o1", `["A1"]`)), + ) + r := s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + if len(r.Buckets) != 2 { + t.Fatalf("buckets = %+v, want 2", r.Buckets) + } + if r.Buckets[0].Start != "2026-09-13T10:00:00Z" || r.Buckets[1].Start != "2026-09-13T11:00:00Z" { + t.Fatalf("bucket order/starts wrong: %+v", r.Buckets) + } + b10 := r.Buckets[0] + if b10.Packets != 2 || b10.RepeaterSum != 3 || b10.AvgRepeaters != 1.5 || b10.Observers != 2 { + t.Errorf("10:00 bucket = %+v, want packets 2 sum 3 avg 1.5 observers 2", b10) + } + b11 := r.Buckets[1] + if b11.Packets != 1 || b11.RepeaterSum != 4 || b11.Observers != 1 { + t.Errorf("11:00 bucket = %+v", b11) + } + if r.Summary.Packets != 3 || r.Summary.Observers != 2 { + t.Errorf("summary = %+v, want packets 3 observers 2", r.Summary) + } + if want := 7.0 / 3.0; r.Summary.AvgRepeaters < want-1e-9 || r.Summary.AvgRepeaters > want+1e-9 { + t.Errorf("summary avg = %v, want %v", r.Summary.AvgRepeaters, want) + } + + r15 := s.computeRetransmissionPressure("", TimeWindow{}, 15*time.Minute) + if len(r15.Buckets) != 3 { + t.Fatalf("15m buckets = %+v, want 3", r15.Buckets) + } + bucketByStart(t, r15, "2026-09-13T10:00:00Z") + bucketByStart(t, r15, "2026-09-13T10:45:00Z") + bucketByStart(t, r15, "2026-09-13T11:00:00Z") + if r15.BucketSeconds != 900 { + t.Errorf("bucket_seconds = %d, want 900", r15.BucketSeconds) + } +} + +func TestComputeRetransmissionPressure_WindowFilter(t *testing.T) { + s := rtxStore( + rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-12T10:00:00Z", rtxObs("o1", `["A1"]`)), + rtxTx(2, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", rtxObs("o1", `["A1","B2"]`)), + rtxTx(3, RouteFlood, PayloadADVERT, "2026-09-14T10:00:00Z", rtxObs("o1", `["A1","B2","C3"]`)), + ) + w := TimeWindow{Since: "2026-09-13T00:00:00Z", Until: "2026-09-13T23:59:59Z"} + r := s.computeRetransmissionPressure("", w, time.Hour) + if r.Summary.Packets != 1 || len(r.Buckets) != 1 || r.Buckets[0].RepeaterSum != 2 { + t.Fatalf("window result = %+v, want only tx 2", r) + } +} + +// transmissions.hash is UNIQUE, so a re-hearing of the same content hours +// later is appended to the existing transmission. Each flood event is +// counted on its own, in the bucket of its first observation. Observations +// are listed newest first, the order the cold load appends them in. +func TestComputeRetransmissionPressure_SplitsFloodEvents(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadGRP_TXT, "2026-09-13T10:00:00Z", + rtxObsAt("o3", `["D4"]`, "2026-09-13T12:00:00.000Z"), + rtxObsAt("o2", `["A1","C3"]`, "2026-09-13T10:00:20.000Z"), + rtxObsAt("o1", `["A1","B2"]`, "2026-09-13T10:00:00.000Z"), + )) + r := s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + if r.Summary.Packets != 2 || len(r.Buckets) != 2 { + t.Fatalf("result = %+v, want 2 flood events in 2 buckets", r) + } + if b := bucketByStart(t, r, "2026-09-13T10:00:00Z"); b.Packets != 1 || b.RepeaterSum != 3 || b.Observers != 2 { + t.Errorf("10:00 bucket = %+v, want packets 1 sum 3 (A1,B2,C3) observers 2", b) + } + if b := bucketByStart(t, r, "2026-09-13T12:00:00Z"); b.Packets != 1 || b.RepeaterSum != 1 || b.Observers != 1 { + t.Errorf("12:00 bucket = %+v, want packets 1 sum 1 (D4) observers 1", b) + } + if r.Summary.AvgRepeaters != 2 { + t.Errorf("summary avg = %v, want 2", r.Summary.AvgRepeaters) + } +} + +// An event ends when the gap to the previous observation exceeds the settle +// time (5 minutes). The gap is measured observation to observation, so a +// slow flood whose steps each stay under it remains one event. +func TestComputeRetransmissionPressure_EventSettleGap(t *testing.T) { + events := func(ts ...string) int { + var obs []*StoreObs + for i, x := range ts { + obs = append(obs, rtxObsAt(fmt.Sprintf("o%d", i), `["A1"]`, x)) + } + return rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, ts[0], obs...)). + computeRetransmissionPressure("", TimeWindow{}, time.Hour).Summary.Packets + } + if got := events("2026-09-13T10:00:00Z", "2026-09-13T10:05:00Z"); got != 1 { + t.Errorf("gap of exactly 5m = %d events, want 1", got) + } + if got := events("2026-09-13T10:00:00Z", "2026-09-13T10:05:01Z"); got != 2 { + t.Errorf("gap of 5m01s = %d events, want 2", got) + } + if got := events("2026-09-13T10:00:00Z", "2026-09-13T10:04:00Z", "2026-09-13T10:08:00Z", "2026-09-13T10:12:00Z"); got != 1 { + t.Errorf("4m steps over 12m = %d events, want 1", got) + } +} + +// The window selects flood events by their first observation, not by the +// transmission's first_seen: a hash first heard weeks ago and flooded again +// today counts today. +func TestComputeRetransmissionPressure_WindowSelectsEvents(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-08-20T10:00:00Z", + rtxObsAt("o1", `["A1","B2"]`, "2026-08-20T10:00:00Z"), + rtxObsAt("o1", `["C3"]`, "2026-09-13T10:00:00Z"), + )) + r := s.computeRetransmissionPressure("", TimeWindow{Since: "2026-09-13T00:00:00Z"}, time.Hour) + if r.Summary.Packets != 1 || len(r.Buckets) != 1 || r.Buckets[0].RepeaterSum != 1 { + t.Fatalf("result = %+v, want only the 2026-09-13 event (C3)", r) + } +} + +// Flood events that start before the store's retention floor are dropped, +// for every request shape: the store keeps old observations only for hashes +// that were heard again recently, so what it holds before the floor is not +// the traffic of that period. +func TestComputeRetransmissionPressure_RetentionFloor(t *testing.T) { + now := time.Now().UTC() + ago := func(d time.Duration) string { return now.Add(-d).Format("2006-01-02T15:04:05.000Z") } + s := rtxStore( + // Weeks-old first event, recent second event: only the second counts. + rtxTx(1, RouteFlood, PayloadADVERT, ago(30*24*time.Hour), + rtxObsAt("o1", `["A1","B2"]`, ago(30*24*time.Hour)), + rtxObsAt("o1", `["C3"]`, ago(time.Hour))), + // An event that starts before the floor is dropped as a whole, not + // counted from its first observation after the floor. + rtxTx(2, RouteFlood, PayloadADVERT, ago(24*time.Hour+time.Minute), + rtxObsAt("o1", `["D4"]`, ago(24*time.Hour+time.Minute)), + rtxObsAt("o2", `["E5"]`, ago(24*time.Hour-time.Minute))), + rtxTx(3, RouteFlood, PayloadADVERT, ago(2*time.Hour), rtxObsAt("o1", `["F6","A7"]`, ago(2*time.Hour))), + ) + s.retentionHours = 24 + r := s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + if r.Summary.Packets != 2 || r.Summary.AvgRepeaters != 1.5 { + t.Fatalf("summary = %+v, want 2 events (C3; F6,A7) avg 1.5", r.Summary) + } + if got := s.computeRetransmissionPressure("", TimeWindow{Since: ago(40 * 24 * time.Hour)}, time.Hour).Summary.Packets; got != 2 { + t.Errorf("window reaching past the floor = %d events, want 2", got) + } + s.retentionHours = 0 + if got := s.computeRetransmissionPressure("", TimeWindow{}, time.Hour).Summary.Packets; got != 4 { + t.Errorf("unlimited retention = %d events, want 4", got) + } +} + +func TestComputeRetransmissionPressure_RegionFilter(t *testing.T) { + s := rtxStore( + rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", + rtxObs("brussels", `["A1","B2"]`), rtxObs("amsterdam", `["A1","C3","D4"]`)), + rtxTx(2, RouteFlood, PayloadADVERT, "2026-09-13T10:10:00Z", + rtxObs("amsterdam", `["E5"]`)), + ) + s.regionObsCache = map[string]map[string]bool{"BRU": {"brussels": true}} + s.regionObsCacheTime = time.Now() + + r := s.computeRetransmissionPressure("BRU", TimeWindow{}, time.Hour) + if r.Region != "BRU" { + t.Errorf("region = %q, want BRU", r.Region) + } + if r.Summary.Packets != 1 { + t.Fatalf("packets = %d, want 1 (tx 2 has no BRU observation)", r.Summary.Packets) + } + if got := r.Buckets[0].RepeaterSum; got != 2 { + t.Errorf("repeater_sum = %d, want 2 (only the brussels path counts)", got) + } + if got := r.Buckets[0].Observers; got != 1 { + t.Errorf("observers = %d, want 1", got) + } +} + +// Events are split on all observations before the region filter, so a flood +// the region heard at its start and end stays one event even when the +// region's own observations are further apart than the settle time. +func TestComputeRetransmissionPressure_RegionDoesNotSplitEvents(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", + rtxObsAt("brussels", `["A1"]`, "2026-09-13T10:00:00Z"), + rtxObsAt("amsterdam", `["B2"]`, "2026-09-13T10:04:00Z"), + rtxObsAt("brussels", `["A1","C3"]`, "2026-09-13T10:08:00Z"), + )) + s.regionObsCache = map[string]map[string]bool{"BRU": {"brussels": true}} + s.regionObsCacheTime = time.Now() + + r := s.computeRetransmissionPressure("BRU", TimeWindow{}, time.Hour) + if r.Summary.Packets != 1 || r.Buckets[0].RepeaterSum != 2 { + t.Fatalf("region result = %+v, want 1 event with A1, C3", r) + } +} + +// A region with no known observers is not filtered, the same as +// /api/analytics/rf and the other analytics endpoints (resolveRegionObservers +// returns nil). Pinned so a change is a deliberate, documented one. +func TestComputeRetransmissionPressure_UnknownRegionIsNotFiltered(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", + rtxObs("brussels", `["A1","B2"]`), rtxObs("amsterdam", `["C3"]`))) + s.regionObsCache = map[string]map[string]bool{"XXX": nil} + s.regionObsCacheTime = time.Now() + + r := s.computeRetransmissionPressure("XXX", TimeWindow{}, time.Hour) + if r.Summary.Packets != 1 || r.Summary.Observers != 2 || r.Buckets[0].RepeaterSum != 3 { + t.Fatalf("unknown region = %+v, want the network-wide result", r) + } +} + +func TestComputeRetransmissionPressure_OneByteShare(t *testing.T) { + ts := "2026-09-13T10:00:00Z" + s := rtxStore( + rtxTx(1, RouteFlood, PayloadADVERT, ts, rtxObs("o1", `["A1","B2"]`)), + rtxTx(2, RouteFlood, PayloadADVERT, ts, rtxObs("o1", `["A1B2","C3D4"]`)), + rtxTx(3, RouteFlood, PayloadADVERT, ts, rtxObs("o1", `[]`), rtxObs("o2", `["A1B2C3"]`)), + rtxTx(4, RouteFlood, PayloadADVERT, ts, rtxObs("o1", `[]`)), + ) + r := s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + if r.Summary.OneBytePackets != 1 { + t.Errorf("one_byte_packets = %d, want 1", r.Summary.OneBytePackets) + } +} + +func TestComputeRetransmissionPressure_EmptyStoreHasNonNilBuckets(t *testing.T) { + r := rtxStore().computeRetransmissionPressure("", TimeWindow{}, time.Hour) + b, _ := json.Marshal(r) + if !strings.Contains(string(b), `"buckets":[]`) { + t.Fatalf("empty result must encode buckets as [], got %s", b) + } +} + +func TestParseRetransmissionBucket(t *testing.T) { + cases := map[string]time.Duration{ + "": time.Hour, + "5m": 5 * time.Minute, + "15m": 15 * time.Minute, + "1h": time.Hour, + "6h": 6 * time.Hour, + "1d": 24 * time.Hour, + "7m": time.Hour, + "abc": time.Hour, + } + for in, want := range cases { + if got := parseRetransmissionBucket(in); got != want { + t.Errorf("parseRetransmissionBucket(%q) = %v, want %v", in, got, want) + } + } +} + +// --- caching --- + +func TestGetRetransmissionPressure_DefaultShapeServedFromRecomputer(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", rtxObs("o1", `["A1"]`))) + sentinel := RetransmissionResponse{BucketSeconds: 3600, Buckets: []RetransmissionBucket{}, Window: "sentinel"} + rc := newAnalyticsRecomputer("retransmissions", time.Hour, func() interface{} { return sentinel }) + rc.runOnce() + s.recompRetransmissions = rc + + if got := s.GetRetransmissionPressure("", TimeWindow{}, time.Hour); got.Window != "sentinel" { + t.Fatalf("default shape must come from the recomputer snapshot, got %+v", got) + } + if got := s.GetRetransmissionPressure("", TimeWindow{}, 15*time.Minute); got.Window == "sentinel" { + t.Fatalf("non-default bucket must not be served from the default snapshot") + } +} + +func TestGetRetransmissionPressure_TTLCacheAndInvalidation(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", rtxObs("o1", `["A1"]`))) + s.rfCacheTTL = time.Hour + w := TimeWindow{Since: "2026-09-13T00:00:00Z", Label: "fixture"} + + first := s.GetRetransmissionPressure("", w, time.Hour) + if first.Summary.Packets != 1 { + t.Fatalf("first = %+v", first) + } + // Mutate the store behind the cache: a cached read must not see it. + s.packets = append(s.packets, rtxTx(2, RouteFlood, PayloadADVERT, "2026-09-13T10:30:00Z", rtxObs("o1", `["B2"]`))) + if got := s.GetRetransmissionPressure("", w, time.Hour); got.Summary.Packets != 1 { + t.Fatalf("expected cache hit with 1 packet, got %d", got.Summary.Packets) + } + s.applyCacheInvalidation(cacheInvalidation{hasNewPaths: true}) + if got := s.GetRetransmissionPressure("", w, time.Hour); got.Summary.Packets != 2 { + t.Fatalf("after hasNewPaths invalidation expected 2 packets, got %d", got.Summary.Packets) + } +} + +func TestGetRetransmissionPressure_EvictionClearsCache(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", rtxObs("o1", `["A1"]`))) + s.rfCacheTTL = time.Hour + w := TimeWindow{Since: "2026-09-13T00:00:00Z", Label: "fixture"} + + s.GetRetransmissionPressure("", w, time.Hour) + s.packets = append(s.packets, rtxTx(2, RouteFlood, PayloadADVERT, "2026-09-13T10:30:00Z", rtxObs("o1", `["B2"]`))) + s.invalidateCachesFor(cacheInvalidation{eviction: true}) + if got := s.GetRetransmissionPressure("", w, time.Hour); got.Summary.Packets != 2 { + t.Fatalf("after eviction invalidation expected 2 packets, got %d", got.Summary.Packets) + } +} + +func TestGetRetransmissionPressure_CacheEntryExpires(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", rtxObs("o1", `["A1"]`))) + s.rfCacheTTL = time.Hour + w := TimeWindow{Since: "2026-09-13T00:00:00Z", Label: "fixture"} + + s.GetRetransmissionPressure("", w, time.Hour) + s.packets = append(s.packets, rtxTx(2, RouteFlood, PayloadADVERT, "2026-09-13T10:30:00Z", rtxObs("o1", `["B2"]`))) + if got := s.GetRetransmissionPressure("", w, time.Hour); got.Summary.Packets != 1 { + t.Fatalf("fresh entry must be served from cache, got %d packets", got.Summary.Packets) + } + if len(s.retransCache) != 1 { + t.Fatalf("cache entries = %d, want 1", len(s.retransCache)) + } + for _, e := range s.retransCache { + e.expiresAt = time.Now().Add(-time.Second) + } + if got := s.GetRetransmissionPressure("", w, time.Hour); got.Summary.Packets != 2 { + t.Fatalf("expired entry must be recomputed, got %d packets", got.Summary.Packets) + } +} + +// Concurrent requests for the same uncached shape share one compute: every +// caller gets the result of that single pass (same bucket backing array). +func TestGetRetransmissionPressure_CollapsesConcurrentMisses(t *testing.T) { + s := rtxStore(rtxTx(1, RouteFlood, PayloadADVERT, "2026-09-13T10:00:00Z", rtxObs("o1", `["A1"]`))) + s.rfCacheTTL = time.Hour + w := TimeWindow{Since: "2026-09-13T00:00:00Z", Label: "fixture"} + + const n = 16 + results := make([]RetransmissionResponse, n) + var wg sync.WaitGroup + s.mu.Lock() // park every compute on the store lock + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + results[i] = s.GetRetransmissionPressure("", w, time.Hour) + }(i) + } + time.Sleep(100 * time.Millisecond) + s.mu.Unlock() + wg.Wait() + for i, r := range results { + if len(r.Buckets) != 1 { + t.Fatalf("result %d = %+v", i, r) + } + if &r.Buckets[0] != &results[0].Buckets[0] { + t.Fatalf("result %d came from a separate compute; concurrent misses must share one", i) + } + } +} + +// StartAnalyticsRecomputers must wire the #1659 readiness gate on the +// retransmissions recomputer: a pass before the cold load completes keeps +// the default shape at 503. +func TestStartAnalyticsRecomputers_RetransmissionsGatedOnLoadComplete(t *testing.T) { + db := setupTestDB(t) + defer db.Close() + store := NewPacketStore(db, nil) + stop := store.StartAnalyticsRecomputers(time.Hour) + defer stop() + + if !store.recompRetransmissions.IsWarmingUp_1659() { + t.Fatal("a pass before LoadComplete must not open the retransmissions gate") + } + store.signalStartupLoadDone() + store.recompRetransmissions.runOnce() + if store.recompRetransmissions.IsWarmingUp_1659() { + t.Fatal("a pass after LoadComplete must open the retransmissions gate") + } +} + +// --- handler --- + +func TestHandleAnalyticsRetransmissions(t *testing.T) { + _, router := setupTestServer(t) + req := httptest.NewRequest("GET", "/api/analytics/retransmissions?bucket=1d", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, body %s", w.Code, w.Body.String()) + } + var body RetransmissionResponse + if err := json.Unmarshal(w.Body.Bytes(), &body); err != nil { + t.Fatalf("decode: %v (%s)", err, w.Body.String()) + } + if body.BucketSeconds != 86400 { + t.Errorf("bucket_seconds = %d, want 86400", body.BucketSeconds) + } + // Seed: tx1 flood ADVERT paths ["aa","bb"] + ["aa"] = 2 repeaters, + // tx2 flood GRP_TXT path [] = 0, tx3 flood ADVERT ["cc"] = 1. + if body.Summary.Packets != 3 { + t.Fatalf("summary.packets = %d, want 3 (%s)", body.Summary.Packets, w.Body.String()) + } + sum := 0 + for _, b := range body.Buckets { + sum += b.RepeaterSum + } + if sum != 3 { + t.Errorf("total repeater_sum = %d, want 3", sum) + } + for _, key := range []string{`"bucket_seconds"`, `"summary"`, `"avg_repeaters"`, `"observers"`, `"one_byte_packets"`, `"no_repeater_packets"`, `"buckets"`} { + if !strings.Contains(w.Body.String(), key) { + t.Errorf("response missing %s: %s", key, w.Body.String()) + } + } +} + +func TestHandleAnalyticsRetransmissions_WarmupGate(t *testing.T) { + srv, router := setupTestServer(t) + rc := newAnalyticsRecomputer("retransmissions", time.Hour, func() interface{} { return nil }) + rc.noteWarmupStart_1659() + rc.setWarmupReadyGate_1659(func() bool { return false }) + srv.store.recompRetransmissions = rc + + w := httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/analytics/retransmissions", nil)) + if w.Code != http.StatusServiceUnavailable { + t.Fatalf("default shape during warmup: status = %d, want 503", w.Code) + } + w = httptest.NewRecorder() + router.ServeHTTP(w, httptest.NewRequest("GET", "/api/analytics/retransmissions?window=24h", nil)) + if w.Code != http.StatusOK { + t.Fatalf("windowed shape bypasses the gate: status = %d, want 200", w.Code) + } +} + +// --- benchmark (perf proof for AGENTS.md rule 0) --- + +// BenchmarkComputeRetransmissionPressure sizes the fixture on live +// magnitudes (2026-09-13): ~15k flood transmissions/day, ~20 observations +// each, ~5.3 hops per path. 50k tx x 20 obs = 1M observations, roughly +// 3.3 days of flood traffic; a 14-day store scales linearly (x4.3). +// +// Observations carry timestamps in the store's format, newest first per +// transmission, which is the order the cold load appends them in +// (ORDER BY o.timestamp DESC). +func retransmissionBenchStore() *PacketStore { + const nTx, obsPerTx = 50000, 20 + base := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) + packets := make([]*StoreTx, 0, nTx) + for i := 0; i < nTx; i++ { + first := base.Add(time.Duration(i) * 6 * time.Second) + obs := make([]*StoreObs, 0, obsPerTx) + for j := 0; j < obsPerTx; j++ { + hops := make([]string, 0, 6) + for h := 0; h < 3+(i+j)%4; h++ { + hops = append(hops, fmt.Sprintf("%04X", (i*7+h*131+j*(h+1))%4096)) + } + pj, _ := json.Marshal(hops) + ts := first.Add(time.Duration(obsPerTx-1-j) * time.Second).Format("2006-01-02T15:04:05.000Z") + obs = append(obs, &StoreObs{ObserverID: fmt.Sprintf("obs%02d", j*3%60), PathJSON: string(pj), Timestamp: ts}) + } + packets = append(packets, rtxTx(i+1, RouteFlood, PayloadADVERT, first.Format(time.RFC3339), obs...)) + } + return rtxStore(packets...) +} + +// BenchmarkComputeRetransmissionPressure measures a recompute pass on a +// store whose observation timestamps were already parsed (the steady state: +// StoreObs.ParsedTime caches per observation). +func BenchmarkComputeRetransmissionPressure(b *testing.B) { + s := retransmissionBenchStore() + s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + runtime.GC() // fixture garbage must not be collected inside the timed loop + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + } +} + +// BenchmarkComputeRetransmissionPressureColdTimestamps measures the first +// pass after startup, when no observation timestamp has been parsed yet. +func BenchmarkComputeRetransmissionPressureColdTimestamps(b *testing.B) { + s := retransmissionBenchStore() + runtime.GC() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + for _, tx := range s.packets { + for _, o := range tx.Observations { + o.tsParseOnce = sync.Once{} + o.tsParsed, o.tsParsedOK = time.Time{}, false + } + } + b.StartTimer() + s.computeRetransmissionPressure("", TimeWindow{}, time.Hour) + } +} diff --git a/cmd/server/routes.go b/cmd/server/routes.go index e799b9ecf..8a755ed23 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -329,6 +329,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/analytics/subpath-detail", s.handleAnalyticsSubpathDetail).Methods("GET") r.HandleFunc("/api/analytics/neighbor-graph", s.handleNeighborGraph).Methods("GET") r.HandleFunc("/api/analytics/relay-airtime-share", s.handleAnalyticsRelayAirtimeShare).Methods("GET") + r.HandleFunc("/api/analytics/retransmissions", s.handleAnalyticsRetransmissions).Methods("GET") // Other endpoints r.HandleFunc("/api/resolve-hops", s.handleResolveHops).Methods("GET") diff --git a/cmd/server/store.go b/cmd/server/store.go index 5a0a84e00..8f24a6ab5 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -197,6 +197,12 @@ type PacketStore struct { subpathCache map[string]*cachedResult // params → cached subpaths result rfCacheTTL time.Duration collisionCacheTTL time.Duration + + // region|window|bucket → retransmission pressure (#1699). Typed, so it + // cannot share the map[string]interface{} caches above; nil until first use. + retransCache map[string]*retransmissionCacheEntry + retransSF singleflight.Group // collapses concurrent misses on one retransCache key + // Steady-state analytics recomputers (issue #1240). Each holds the // latest snapshot for the default region="" / zero-window query of // an analytics endpoint in an atomic.Value, refreshed by a @@ -214,6 +220,7 @@ type PacketStore struct { recompRoles *analyticsRecomputer recompObserversClockSkew *analyticsRecomputer recompNodesClockSkew *analyticsRecomputer + recompRetransmissions *analyticsRecomputer cacheHits int64 cacheMisses int64 // Rate-limited invalidation (fixes #533: caches cleared faster than hit) @@ -2284,6 +2291,7 @@ func (s *PacketStore) invalidateCachesFor(inv cacheInvalidation) { s.chanCache = make(map[string]*cachedResult) s.distCache = make(map[string]*cachedResult) s.subpathCache = make(map[string]*cachedResult) + s.retransCache = nil s.channelsCacheMu.Lock() s.channelsCacheRes = nil s.channelsCacheMu.Unlock() @@ -2330,6 +2338,7 @@ func (s *PacketStore) applyCacheInvalidation(inv cacheInvalidation) { s.topoCache = make(map[string]*cachedResult) s.distCache = make(map[string]*cachedResult) s.subpathCache = make(map[string]*cachedResult) + s.retransCache = nil } if inv.hasNewTransmissions { s.hashCache = make(map[string]*cachedResult) diff --git a/docs/api-spec.md b/docs/api-spec.md index 09d38889e..0de368df5 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -36,6 +36,7 @@ - [GET /api/channels/:hash/messages](#get-apichannelshashmessages) - [GET /api/analytics/rf](#get-apianalyticsrf) - [GET /api/analytics/topology](#get-apianalyticstopology) +- [GET /api/analytics/retransmissions](#get-apianalyticsretransmissions) - [GET /api/analytics/channels](#get-apianalyticschannels) - [GET /api/analytics/distance](#get-apianalyticsdistance) - [GET /api/analytics/hash-sizes](#get-apianalyticshash-sizes) @@ -1356,6 +1357,93 @@ Network topology analytics. --- +## GET /api/analytics/retransmissions + +Retransmission pressure over time (#1699): a collision-pressure **proxy**, not a +measured collision rate. + +For each flood event of a flood-routed packet (`route_type` 0 or 1, TRACE +excluded) the server takes the union of the paths of all its observations and +counts the distinct repeaters in it: paths `[A]`, `[A,B,C]` and `[A,D]` give 4. +Direct routes are excluded because their path is the route still to travel, not +the forwarders; zero-hop sends are direct routes. A flood event heard only with +an empty path counts as 0 repeaters. + +A transmission is one packet hash, and the same bytes can flood again later: +those observations are stored on the same transmission. Its observations are +therefore sorted by time and split into flood events wherever two consecutive +observations are more than 5 minutes apart. Each event is counted on its own and +bucketed by its first observation. A firmware node holds a flood for at most +32 s before forwarding it, plus a random retransmit delay, so a flood that is +still spreading is not split. Every count in the response (`packets`, +`one_byte_packets`, `no_repeater_packets`) counts flood events. Observations are +stored once per observer and path per transmission, so a later event holds only +the observer and path pairs not already stored for that hash, and its count is a +lower bound. + +Events that start before the store's retention floor (now minus +`retentionHours`) are left out for every request shape, including explicit +`window`, `from` and `to`: the store keeps observations older than that only for +hashes heard again recently, so they do not represent that period. + +Hop prefixes are not resolved to nodes. A prefix counts once per flood event, +whether it repeats across observations or inside one path. A 2- or 3-byte prefix +that repeats inside one path is one node forwarding the flood again after its +duplicate filter (a cyclic buffer of 160 hashes) dropped the hash. A repeated +1-byte prefix can also be two nodes; counting it once keeps the value a lower +bound, as does merging repeaters that share a prefix across observations. Only +repeaters that some observer heard are counted, so the value also follows +observer coverage. + +The default shape (no `region`, no window, `bucket=1h`) is served from the +analytics recomputer; other shapes use the TTL cache, and concurrent requests +for the same uncached shape share one computation. During startup the default +shape returns `503` with `Retry-After` until the recomputer completes a pass +after the hot startup window has loaded, or for at most 60 s after the +recomputer started, whichever comes first. The history beyond the hot window +keeps loading in the background after that, so until the first recompute pass +after that load finishes the default shape can cover less than the retention +window. `?area=` is not +supported: the area filter works on resolved node public keys and this metric +does not resolve prefixes. + +### Query Parameters + +| Param | Type | Default | Description | +|----------|--------|---------|-------------------------------------| +| `region` | string | none | Comma-separated IATA codes; only observations from the region's observers feed the union, events none of them heard are skipped. Events are split before this filter. A region with no known observers is not filtered and returns network-wide data, as `/api/analytics/rf` does | +| `window` | string | none | `1h`, `24h`, `3d`, `7d` or `30d` (relative to now) | +| `from`, `to` | string (ISO) | none | Absolute window bounds, take precedence over `window` | +| `bucket` | string | `1h` | `5m`, `15m`, `1h`, `6h` or `1d`; other values fall back to `1h` | + +### Response `200` + +```jsonc +{ + "bucket_seconds": number, + "window": string, // window label, "" for all data + "region": string, + "summary": { + "packets": number, // flood events that started in the window + "avg_repeaters": number, // mean distinct repeaters per event + "observers": number, // distinct observers that heard them + "one_byte_packets": number, // events on 1-byte hop hashes (most ambiguous) + "no_repeater_packets": number // events heard with an empty path only + }, + "buckets": [ // ascending, empty buckets omitted + { + "start": string (ISO), // bucket start, UTC + "packets": number, // flood events that started in the bucket + "repeater_sum": number, + "avg_repeaters": number, + "observers": number + } + ] +} +``` + +--- + ## GET /api/analytics/channels Channel analytics. diff --git a/public/analytics.js b/public/analytics.js index 48e54aa2b..070f9ce49 100644 --- a/public/analytics.js +++ b/public/analytics.js @@ -758,6 +758,12 @@
+
+

Retransmission Pressure (proxy)

+

Average number of distinct repeaters seen in the observed paths of each flood packet, over time.

+
Loading…
+
+

Repeater Pair Heatmap

@@ -808,6 +814,141 @@ obsId === '__all' ? renderAllObserversReach(topo.perObserverReach) : renderPerObserverReach(topo.perObserverReach, obsId); }); } + loadRetransmissionPressure(); + } + + // ===================== RETRANSMISSION PRESSURE (#1699) ===================== + // Bucket per time-window choice. "All data" stays on 1h so it matches the + // server's recomputed default shape instead of a TTL-cache compute. + var RTX_BUCKET_BY_WINDOW = { '1h': '5m', '24h': '1h', '7d': '1h', '30d': '6h' }; + function retransmissionBucketFor(win) { + return Object.prototype.hasOwnProperty.call(RTX_BUCKET_BY_WINDOW, win) ? RTX_BUCKET_BY_WINDOW[win] : '1h'; + } + + var _rtxRequestSeq = 0; + async function loadRetransmissionPressure() { + if (!document.getElementById('rtxPressureChart')) return; + var seq = ++_rtxRequestSeq; + var twEl = document.getElementById('analyticsTimeWindow'); + var tw = twEl ? twEl.value : ''; + var qs = RegionFilter.regionQueryString() + + (tw ? '&window=' + encodeURIComponent(tw) : '') + + '&bucket=' + retransmissionBucketFor(tw); + var html; + try { + var data = await api('/analytics/retransmissions?' + qs.slice(1), { ttl: CLIENT_TTL.analyticsRF }); + html = renderRetransmissionChart(data); + } catch (e) { + html = ''; + } + // A newer request (window/region change) or a tab switch wins. + var host = document.getElementById('rtxPressureChart'); + if (seq !== _rtxRequestSeq || !host) return; + host.innerHTML = html; + } + + // renderRetransmissionChart draws avg distinct repeaters per flood packet + // (the primary line, with a y axis) plus two context series scaled to their + // own maximum: flood packets per bucket (bars) and observers per bucket + // (dashed). Buckets further apart than bucket_seconds break the lines, so a + // gap in data is never drawn as a trend. + function renderRetransmissionChart(data) { + var noData = '
No flood packets in this window.
'; + var raw = data && Array.isArray(data.buckets) ? data.buckets : []; + var pts = []; + for (var i = 0; i < raw.length; i++) { + var t = Date.parse(raw[i].start); + if (!isFinite(t)) continue; + pts.push({ t: t, start: String(raw[i].start), avg: Number(raw[i].avg_repeaters) || 0, + packets: Number(raw[i].packets) || 0, observers: Number(raw[i].observers) || 0 }); + } + if (!pts.length) return noData; + var stepMs = (Number(data.bucket_seconds) || 3600) * 1000; + var w = 800, h = 220, padL = 40, padR = 16, padT = 12, padB = 28; + var plotW = w - padL - padR, plotH = h - padT - padB; + var tMin = pts[0].t, tMax = pts[pts.length - 1].t; + var maxAvg = 1, maxPkts = 1, maxObs = 1; + pts.forEach(function (p) { + if (p.avg > maxAvg) maxAvg = p.avg; + if (p.packets > maxPkts) maxPkts = p.packets; + if (p.observers > maxObs) maxObs = p.observers; + }); + var yMax = Math.ceil(maxAvg); + // The x domain spans whole buckets, so bars and points (at bucket + // centres) stay inside the plot, also for a single bucket. + var span = tMax + stepMs - tMin; + function x(tt) { return padL + (tt + stepMs / 2 - tMin) / span * plotW; } + function yAvg(v) { return padT + plotH - (v / yMax) * plotH; } + function yObs(v) { return padT + plotH - (v / maxObs) * plotH * 0.9; } + function f(n) { return n.toFixed(1); } + + var svg = 'Average distinct repeaters per flood packet over time'; + for (var g = 0; g <= 4; g++) { + var gy = padT + plotH * g / 4; + svg += ''; + svg += '' + esc(String(Math.round(yMax * (4 - g) / 4 * 10) / 10)) + ''; + } + // Context: flood packets per bucket, scaled to 30% of the plot height. + var barW = Math.max(1, stepMs / span * plotW * 0.8); + pts.forEach(function (p) { + var bh = (p.packets / maxPkts) * plotH * 0.3; + svg += ''; + }); + // Split into runs of consecutive buckets. + var runs = [], run = []; + pts.forEach(function (p, idx) { + if (idx > 0 && p.t - pts[idx - 1].t > stepMs) { runs.push(run); run = []; } + run.push(p); + }); + runs.push(run); + runs.forEach(function (r) { + if (r.length === 1) { + svg += ''; + return; + } + svg += ''; + svg += ''; + }); + // Per-bucket hover targets. + pts.forEach(function (p) { + var tip = p.start + '\nAvg distinct repeaters: ' + p.avg.toFixed(1) + '\nFlood packets: ' + p.packets + '\nObservers: ' + p.observers; + svg += '' + esc(tip) + ''; + }); + var multiDay = tMax - tMin > 2 * 86400000; + var labelEvery = Math.max(1, Math.ceil(pts.length / 6)); + for (var li = 0; li < pts.length; li += labelEvery) { + var lbl = multiDay ? pts[li].start.slice(5, 10) : pts[li].start.slice(11, 16); + svg += '' + esc(lbl) + ''; + } + svg += ''; + + var s = (data && data.summary) || {}; + var sPackets = Number(s.packets) || 0; + var oneBytePct = sPackets ? Math.round((Number(s.one_byte_packets) || 0) / sPackets * 100) : 0; + var html = svg; + html += '
' + + 'Avg distinct repeaters per flood packet' + + 'Observers (dashed, scaled)' + + 'Flood packets (bars, scaled)' + + '
'; + html += '
' + + 'Avg: ' + esc((Number(s.avg_repeaters) || 0).toFixed(1)) + ' repeaters/packet' + + 'Flood packets: ' + esc(sPackets.toLocaleString()) + '' + + 'Observers: ' + esc(String(Number(s.observers) || 0)) + '' + + '1-byte hashes: ' + oneBytePct + '%' + + 'Heard without repeaters: ' + esc(String(Number(s.no_repeater_packets) || 0)) + '' + + '
'; + html += '

' + + 'A proxy for collision pressure, not a measured collision rate. ' + + 'It counts only repeaters that at least one observer heard, so observer coverage moves the line too: ' + + 'adding or losing observers (dashed line) changes it without any change on air. ' + + 'Hop prefixes are not resolved to nodes; a prefix counts once per flood, also when two repeaters share it, so the value is a lower bound, ' + + 'most of all for packets on 1-byte hashes. Flood routes only (TRACE and direct routes excluded). ' + + 'A packet heard again more than 5 minutes after its previous observation counts as a new flood; ' + + 'each flood sits in the bucket where it started, so the newest bucket may still be filling. ' + + 'The area filter does not apply to this chart.' + + '

'; + return html; } function renderRepeaterTable(repeaters) { @@ -2986,6 +3127,8 @@ function destroy() { _stopRolesRefresh(); _stopScopesRefresh(); _analyticsData = window._analyticsRenderMultiByteAdopters = renderMultiByteAdopters; window._analyticsHashStatCardsHtml = hashStatCardsHtml; window._analyticsRenderCollisionsFromServer = renderCollisionsFromServer; + window._analyticsRetransmissionBucketFor = retransmissionBucketFor; + window._analyticsRenderRetransmissionChart = renderRetransmissionChart; window._analyticsScopeAdvertsByRoleHtml = scopeAdvertsByRoleHtml; } diff --git a/test-all.sh b/test-all.sh index 2e5950a92..cfd10ce61 100755 --- a/test-all.sh +++ b/test-all.sh @@ -115,6 +115,7 @@ node test-issue-1668-m2-contrast.js node test-issue-1668-m3-typography.js node test-issue-1668-m4-per-route.js node test-issue-1697-mqtt-mobile-e2e.js +node test-issue-1699-retransmission-chart.js node test-issue-1705-subpath-contrast.js node test-issue-1753-copy-url-slash.js node test-issue-1770-mobile-row-clamp.js diff --git a/test-issue-1699-retransmission-chart.js b/test-issue-1699-retransmission-chart.js new file mode 100644 index 000000000..0483a4e55 --- /dev/null +++ b/test-issue-1699-retransmission-chart.js @@ -0,0 +1,130 @@ +/** + * #1699: Topology tab, retransmission pressure chart. + * + * Loads public/analytics.js into a stub browser context and exercises the + * pure render helpers it exposes for testing: + * - _analyticsRetransmissionBucketFor(window) picks a bucket per window + * - _analyticsRenderRetransmissionChart(data) renders the SVG + captions + * + * Usage: node test-issue-1699-retransmission-chart.js + */ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const vm = require('vm'); + +let passed = 0, failed = 0; +function assert(cond, msg) { + if (cond) { passed++; console.log(' ✓ ' + msg); } + else { failed++; console.error(' ✗ ' + msg); } +} + +const ctx = { + console, Math, JSON, Date, Number, String, Array, Object, Set, Map, RegExp, URLSearchParams, + setTimeout, clearTimeout, requestAnimationFrame: () => 0, + getComputedStyle: () => ({ getPropertyValue: () => '' }), + localStorage: { getItem: () => null, setItem() {}, removeItem() {} }, + registerPage: () => {}, + api: async () => ({}), + CLIENT_TTL: {}, + RegionFilter: { regionQueryString: () => '' }, + AreaFilter: { areaQueryString: () => '' }, +}; +ctx.window = ctx; +ctx.document = { + documentElement: {}, + createElement: () => ({ style: {}, addEventListener() {} }), + addEventListener() {}, removeEventListener() {}, + querySelector: () => null, querySelectorAll: () => [], + getElementById: () => null, +}; +vm.createContext(ctx); +vm.runInContext(fs.readFileSync(path.join(__dirname, 'public/analytics.js'), 'utf8'), ctx); + +const bucketFor = ctx._analyticsRetransmissionBucketFor; +const render = ctx._analyticsRenderRetransmissionChart; + +console.log('\n=== #1699 retransmission chart: exports ==='); +assert(typeof bucketFor === 'function', '_analyticsRetransmissionBucketFor exposed'); +assert(typeof render === 'function', '_analyticsRenderRetransmissionChart exposed'); +if (typeof bucketFor !== 'function' || typeof render !== 'function') { + console.log(`\n${passed} passed, ${failed} failed`); + process.exit(1); +} + +console.log('\n=== bucket per window ==='); +assert(bucketFor('') === '1h', 'all data -> 1h (matches the server default shape)'); +assert(bucketFor('1h') === '5m', '1h window -> 5m'); +assert(bucketFor('24h') === '1h', '24h window -> 1h'); +assert(bucketFor('7d') === '1h', '7d window -> 1h'); +assert(bucketFor('30d') === '6h', '30d window -> 6h'); +assert(bucketFor('bogus') === '1h', 'unknown window -> 1h'); + +console.log('\n=== empty data ==='); +const empty = render({ bucket_seconds: 3600, summary: { packets: 0 }, buckets: [] }); +assert(/No flood packets/.test(empty), 'empty buckets render a no-data message'); +assert(!/ s.replace(/.*points="([^"]*)"/, '$1').trim().split(/\s+/).length); +assert(avgPts.join(',') === '2,2', 'each segment carries its 2 points (got ' + avgPts.join(',') + ')'); +assert(/>910 0 && bars.every(m => Number(m[1]) >= 40 && Number(m[1]) + Number(m[2]) <= 784); +} +assert(barsInPlot(html), 'packet bars stay inside the plot area'); + +console.log('\n=== single bucket ==='); +const one = render({ bucket_seconds: H, summary: { packets: 5, avg_repeaters: 3, observers: 2, one_byte_packets: 0 }, + buckets: [{ start: '2026-09-13T10:00:00Z', packets: 5, repeater_sum: 15, avg_repeaters: 3, observers: 2 }] }); +assert(/]*class="rtx-avg-dot"/.test(one), 'a lone bucket is drawn as a dot'); +assert(barsInPlot(one), 'a lone bucket bar stays inside the plot area'); + +console.log('\n=== untrusted server strings ==='); +const evil = render({ bucket_seconds: H, summary: { packets: 1, avg_repeaters: 1, observers: 1, one_byte_packets: 0 }, + buckets: [{ start: '', packets: 1, repeater_sum: 1, avg_repeaters: 1, observers: 1 }] }); +assert(!/2026-09-13T10:00:00Z\nAvg distinct repeaters: 7\.0\nFlood packets: 100\nObservers: 18<\/title>/.test(tipHtml), + 'per-bucket tooltip carries start, average, packets and observers'); + +console.log(`\n${passed} passed, ${failed} failed`); +process.exit(failed ? 1 : 0); From dda04ae9183650d754c4729de366b4b67d488c37 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 21:09:31 +0200 Subject: [PATCH 039/151] fix(packets): empty the observer and type selections on Clear Filters (#2012) (#2015) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What was wrong On the Packets page, Clear Filters reset `filters.observer`, `filters.type`, localStorage and every checkbox in both multi-select menus, but not the Sets that hold the selection: `selectedObservers` (`public/packets.js:1793`) and `selectedTypes` (`public/packets.js:1845`). The next `change` event (`public/packets.js:1826`, `:1876`) added to the stale Set, so select observer A, Clear, select observer B wrote `A,B` to the URL and the trigger read "2 Observers". Types behaved the same way. Clear also unchecked the "All Observers" / "All Types" rows although no filter was active. ## What changed - `public/packets.js:1999-2006`: the Clear handler empties both Sets and rebuilds both menus through `buildObserverMenu()` / `buildTypeMenu()` plus `updateObsTrigger()` / `updateTypeTrigger()`, replacing the hand-written checkbox and trigger resets. All four functions are in the same scope as the handler. - `test-issue-2012-clear-filters-selection.js` (new): runs the real multi-select section and Clear handler from `packets.js` in one function scope against a small fake DOM. For observers and types it drives select A, Clear, select B and asserts that only B is selected (filters, localStorage, trigger text) and that the All row is checked after Clear. - `test-clear-filters.js`: the handler body now uses names this test did not provide, so menu-agnostic cases get empty stand-ins. The old "unchecks every checkbox" case expected the All row to be unchecked (the bug), so it now asserts that the Sets are emptied and both menus rebuilt. - Registered the new test in `test-all.sh` and the unit-test step in `.github/workflows/deploy.yml`. ## Tests - `node test-issue-2012-clear-filters-selection.js`: 4 passed. With the `packets.js` change reverted: 0 passed, 4 failed (`obsA,obsB`, `4,5`, All row `false`). - `node test-clear-filters.js`: 6 passed, 2 failed, the same counts as on master. The two failing `updatePacketsUrl` cases fail on master with `location is not defined`. This file is not run by `test-all.sh` or CI. - Also green: `test-packets-local-channels.js`, `test-issue-1415-packets-layout.js`, `test-frontend-helpers.js`, `test-observer-iata-1188.js`, `test-packet-filter.js`, `test-packet-filter-ux.js`, `test-xss-escape-sinks.js`, `scripts/check-css-vars.js`. ## Browser validation Deployed together with #2013's, #1851's and #1868's branches to a staging instance with live traffic (build `e84d2da6`), in Chrome: - Observers: select BE-BRU-Moris, Clear (URL back to `#/packets`, trigger "All Observers", All row checked), select BE-BRU-Bécodok: URL holds only Bécodok's key, trigger shows Bécodok. - Types: select one type, Clear (All row checked, "All Types"), select another: stored filter holds only the second. ## Not verified - Other ways of resetting filters (navigating back to `#/packets` without params) were not checked for the same stale Sets. - The full `test-all.sh` run was not done locally. Fixes #2012 ## Review follow-up (commit `4b14f54e`) An independent review of this PR reproduced the bug on master and confirmed the fix in headless Chromium against a real server. It found one related pre-existing problem, now fixed here: - The type multi-select change handler never called `updatePacketsUrl()`, which is what shows or hides the Clear button (`public/packets.js` ~779-783). With only a type selected the button stayed hidden, so the Types half of #2012 could not be reached by a click. The handler now makes that call, like the observer handler does (`public/packets.js:1887`). Type is not part of the URL, so the call only toggles the button. - The regression test now runs the real `updatePacketsUrl()` and adds two cases: picking only a type, and only an observer, shows the Clear button, and Clear hides it again. The type case fails without the new call. - The test's fake DOM also provides `#observerList` and `#observerSearchInput`, so it keeps working if #1884 merges first. Checked against a local merge of #1884: 6 of 6 pass. That merge has one conflict in the Clear handler; whichever PR lands second must keep both the Set clear and the search box reset. - Not mentioned before: the fix also resets the type trigger's `title` tooltip through `updateTypeTrigger()`, which the old handler left stale. Validated on staging together with the other follow-ups (build `c646310f`). Not verified: mobile viewport, full Playwright suite. --------- Co-authored-by: Claude Opus 5 (1M context) --- public/packets.js | 18 +- test-all.sh | 1 + test-clear-filters.js | 39 +++- test-issue-2012-clear-filters-selection.js | 208 +++++++++++++++++++++ test-observer-menu-interactions.js | 9 +- 5 files changed, 254 insertions(+), 21 deletions(-) create mode 100644 test-issue-2012-clear-filters-selection.js diff --git a/public/packets.js b/public/packets.js index b27566536..191ebed2c 100644 --- a/public/packets.js +++ b/public/packets.js @@ -1921,6 +1921,7 @@ if (filters.type) localStorage.setItem('meshcore-type-filter', filters.type); else localStorage.removeItem('meshcore-type-filter'); buildTypeMenu(); updateTypeTrigger(); + updatePacketsUrl(); renderTableRows(); }); @@ -2033,17 +2034,16 @@ document.getElementById('fChannel').value = ''; document.getElementById('fMyNodes').classList.remove('active'); - // Reset observer multi-select - var obMenu = document.getElementById('observerMenu'); - if (obMenu) obMenu.querySelectorAll('input[type=checkbox]').forEach(function(cb) { cb.checked = false; }); - document.getElementById('observerTrigger').textContent = 'All Observers ▾'; + // Reset observer and type multi-selects (#2012): empty the selection + // Sets, not only the checkboxes, or the next pick adds to the old one. + selectedObservers.clear(); + buildObserverMenu(); + updateObsTrigger(); obsSearchInput.value = ''; applyObserverSearchFilter(); - - // Reset type multi-select - var typeMenu = document.getElementById('typeMenu'); - if (typeMenu) typeMenu.querySelectorAll('input[type=checkbox]').forEach(function(cb) { cb.checked = false; }); - document.getElementById('typeTrigger').textContent = 'All Types ▾'; + selectedTypes.clear(); + buildTypeMenu(); + updateTypeTrigger(); // Reset time window to default savedTimeWindowMin = DEFAULT_TIME_WINDOW; diff --git a/test-all.sh b/test-all.sh index cfd10ce61..9a327c2a4 100755 --- a/test-all.sh +++ b/test-all.sh @@ -132,6 +132,7 @@ node test-issue-1890-og-url.js node test-issue-1956-release-routing.js node test-issue-1979-scope-adverts-by-role.js node test-issue-2001-map-scope-state.js +node test-issue-2012-clear-filters-selection.js node test-live-anims.js node test-live-dt-cap-1524.js node test-live-legend-helper.js diff --git a/test-clear-filters.js b/test-clear-filters.js index 2340bf9f0..2a4c3baec 100644 --- a/test-clear-filters.js +++ b/test-clear-filters.js @@ -149,6 +149,12 @@ function extractUpdatePacketsUrl() { const clearBody = extractClearHandler(); const updateUrlBody = extractUpdatePacketsUrl(); +// The handler resets the observer/type multi-selects through closure state +// (#2012). Tests that do not look at the menus get inert stand-ins. +const MENU_STUBS = 'var selectedObservers = new Set(), selectedTypes = new Set();' + + ' function buildObserverMenu() {} function updateObsTrigger() {}' + + ' function buildTypeMenu() {} function updateTypeTrigger() {}\n'; +const clearBodyNoMenus = MENU_STUBS + clearBody; // ---- Tests ---- @@ -167,7 +173,7 @@ test('clear handler resets all filter keys to undefined/null/false', () => { const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', '_observerFilterSet', 'localStorage', 'document', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', - `${clearBody}; return { savedTimeWindowMin, _observerFilterSet };` + `${clearBodyNoMenus}; return { savedTimeWindowMin, _observerFilterSet };` ); const result = fn( @@ -198,7 +204,7 @@ test('clear handler resets savedTimeWindowMin to DEFAULT_TIME_WINDOW', () => { const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', '_observerFilterSet', 'localStorage', 'document', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', - `${clearBody}; return { savedTimeWindowMin };` + `${clearBodyNoMenus}; return { savedTimeWindowMin };` ); const result = fn( filters, 120, DEFAULT_TIME_WINDOW, _observerFilterSet, @@ -217,7 +223,7 @@ test('clear handler resets fTimeWindow dropdown value', () => { const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', '_observerFilterSet', 'localStorage', 'document', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', - `${clearBody}; return { savedTimeWindowMin };` + `${clearBodyNoMenus}; return { savedTimeWindowMin };` ); fn(filters, 120, 15, null, ctx.localStorage, ctx.document, ctx.RegionFilter, () => {}, () => {}); assert.strictEqual(elements['fTimeWindow'].value, '15', 'fTimeWindow DOM not reset'); @@ -231,24 +237,37 @@ test('clear handler clears observer and type localStorage', () => { const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', '_observerFilterSet', 'localStorage', 'document', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', - `${clearBody};` + `${clearBodyNoMenus};` ); fn(filters, 15, 15, null, ctx.localStorage, ctx.document, ctx.RegionFilter, () => {}, () => {}); assert.strictEqual(ctx.localStorage.getItem('meshcore-observer-filter'), null); assert.strictEqual(ctx.localStorage.getItem('meshcore-type-filter'), null); }); -test('clear handler unchecks observer/type multi-select checkboxes', () => { - const { ctx, checkboxes } = makeSandbox(); +test('clear handler empties observer/type selections and rebuilds both menus', () => { + // End-to-end menu behaviour is in test-issue-2012-clear-filters-selection.js. + const { ctx } = makeSandbox(); const filters = { myNodes: false }; + const selectedObservers = new Set(['obs1']); + const selectedTypes = new Set(['4']); + const calls = []; const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', '_observerFilterSet', 'localStorage', 'document', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', + 'selectedObservers', 'buildObserverMenu', 'updateObsTrigger', + 'selectedTypes', 'buildTypeMenu', 'updateTypeTrigger', `${clearBody};` ); - fn(filters, 15, 15, null, ctx.localStorage, ctx.document, ctx.RegionFilter, () => {}, () => {}); - for (const cb of checkboxes['observerMenu']) assert.strictEqual(cb.checked, false, 'observer checkbox still checked'); - for (const cb of checkboxes['typeMenu']) assert.strictEqual(cb.checked, false, 'type checkbox still checked'); + fn(filters, 15, 15, null, ctx.localStorage, ctx.document, ctx.RegionFilter, () => {}, () => {}, + selectedObservers, () => calls.push('buildObserverMenu:' + selectedObservers.size), + () => calls.push('updateObsTrigger:' + selectedObservers.size), + selectedTypes, () => calls.push('buildTypeMenu:' + selectedTypes.size), + () => calls.push('updateTypeTrigger:' + selectedTypes.size)); + assert.strictEqual(selectedObservers.size, 0, 'observer selection not emptied'); + assert.strictEqual(selectedTypes.size, 0, 'type selection not emptied'); + assert.deepStrictEqual(calls, [ + 'buildObserverMenu:0', 'updateObsTrigger:0', 'buildTypeMenu:0', 'updateTypeTrigger:0', + ]); }); test('clear handler resets RegionFilter', () => { @@ -258,7 +277,7 @@ test('clear handler resets RegionFilter', () => { const fn = new Function( 'filters', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', '_observerFilterSet', 'localStorage', 'document', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', - `${clearBody};` + `${clearBodyNoMenus};` ); fn(filters, 15, 15, null, ctx.localStorage, ctx.document, ctx.RegionFilter, () => {}, () => {}); assert.deepStrictEqual(regionState.selected, [], 'RegionFilter not cleared'); diff --git a/test-issue-2012-clear-filters-selection.js b/test-issue-2012-clear-filters-selection.js new file mode 100644 index 000000000..06c37df69 --- /dev/null +++ b/test-issue-2012-clear-filters-selection.js @@ -0,0 +1,208 @@ +/** + * Regression test for #2012: Clear Filters must empty the observer and type + * selections, not only filters.observer / filters.type. + * + * Before the fix the Clear handler reset the checkboxes by hand but left the + * closure Sets `selectedObservers` and `selectedTypes` populated, so + * "select A, Clear, select B" produced "A,B". It also unchecked the "All" rows. + * The type change handler also never called updatePacketsUrl(), so with only a + * type selected the Clear button stayed hidden. + * + * The test runs the real packets.js source: the observer/type multi-select + * section and the Clear handler are sliced out and evaluated in one function + * scope (the same scope they share in packets.js), together with the real + * updatePacketsUrl(), against a minimal DOM that parses the menu markup into + * checkbox objects. + */ +'use strict'; +const fs = require('fs'); +const assert = require('assert'); + +console.log('--- test-issue-2012-clear-filters-selection.js ---'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } +} + +const src = fs.readFileSync(__dirname + '/public/packets.js', 'utf-8'); +function slice(startMarker, endMarker) { + const start = src.indexOf(startMarker); + assert(start !== -1, 'marker not found in packets.js: ' + startMarker); + const end = src.indexOf(endMarker, start); + assert(end !== -1, 'marker not found in packets.js: ' + endMarker); + return src.substring(start, end); +} +const multiSelectSrc = slice('// --- Observer multi-select ---', '// --- Channel filter (#812) ---'); +const clearSrc = slice('// --- Clear filters button ---', '// Show clear button if page loaded'); +// The real updatePacketsUrl(), which shows or hides the Clear button. +const urlSrc = slice('function buildPacketsQuery(', 'let filtersBuilt = false;'); + +function makeEl(id) { + const listeners = {}; + const el = { + id, value: '', textContent: '', title: '', style: {}, + classList: { add() {}, remove() {}, toggle() {}, contains: () => false }, + addEventListener(ev, fn) { listeners[ev] = fn; }, + fire(ev, arg) { listeners[ev](arg); }, + _checkboxes: [], + querySelectorAll(sel) { return sel === 'input[type=checkbox]' ? el._checkboxes : []; }, + }; + Object.defineProperty(el, 'innerHTML', { + set(html) { + el._checkboxes = []; + const re = /]*)>/g; + let m; + while ((m = re.exec(html))) { + const attrs = m[1]; + const dataset = {}; + const obs = /data-obs-id="([^"]*)"/.exec(attrs); + const typ = /data-type-id="([^"]*)"/.exec(attrs); + if (obs) dataset.obsId = obs[1]; + if (typ) dataset.typeId = typ[1]; + el._checkboxes.push({ dataset, checked: /\schecked(\s|$)/.test(attrs) }); + } + }, + }); + return el; +} + +function setup() { + const elements = {}; + // #observerList and #observerSearchInput are not in packets.js yet. The + // observer search PR (#1884) renders the observer rows into #observerList + // and wires #observerSearchInput; providing both keeps this test valid + // whichever of the two lands first. + for (const id of [ + 'observerMenu', 'observerList', 'observerSearchInput', 'observerTrigger', + 'typeMenu', 'typeTrigger', 'clearFiltersBtn', + 'fHash', 'fNode', 'fChannel', 'fTimeWindow', 'fMyNodes', + 'packetFilterInput', 'packetFilterError', 'packetFilterCount', + ]) elements[id] = makeEl(id); + elements.clearFiltersBtn.style.display = 'none'; + + const storage = {}; + const localStorage = { + getItem: (k) => (k in storage ? storage[k] : null), + setItem: (k, v) => { storage[k] = String(v); }, + removeItem: (k) => { delete storage[k]; }, + }; + const document = { getElementById: (id) => elements[id] || null }; + const observers = [{ id: 'obsA', name: 'Observer A' }, { id: 'obsB', name: 'Observer B' }]; + const observerMap = new Map(observers.map((o) => [o.id, o])); + const SHORT_BY_ID = { 4: 'ADVERT', 5: 'GRP_TXT' }; + const filters = { myNodes: false }; + const escapeHtml = (s) => String(s); + const RegionFilter = { setSelected() {}, getRegionParam: () => '' }; + const location = { hash: '#/packets' }; + const history = { replaceState() {} }; + const noop = () => {}; + + const run = new Function( + 'filters', 'observers', 'observerMap', 'SHORT_BY_ID', 'escapeHtml', 'document', 'localStorage', + 'RegionFilter', 'renderTableRows', 'loadPackets', + '_rebuildObserverMenu', '_observerFilterSet', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', + 'location', 'history', 'window', '_packetSortColumn', '_packetSortDirection', + urlSrc + '\n' + multiSelectSrc + '\n' + clearSrc + ); + run(filters, observers, observerMap, SHORT_BY_ID, escapeHtml, document, localStorage, + RegionFilter, noop, noop, null, null, 15, 15, + location, history, {}, null, null); + + // Observer rows live in #observerMenu today and in #observerList after + // #1884; the change listener stays on #observerMenu in both layouts. + function boxes(menuId) { + return menuId === 'observerMenu' + ? elements.observerMenu._checkboxes.concat(elements.observerList._checkboxes) + : elements[menuId]._checkboxes; + } + function toggle(menuId, attr, id, checked) { + const cb = boxes(menuId).find((c) => c.dataset[attr] === id); + assert(cb, `checkbox ${id} not found in #${menuId}`); + cb.checked = checked; + elements[menuId].fire('change', { target: cb }); + } + function allRow(menuId, attr) { + return boxes(menuId).find((c) => c.dataset[attr] === '__all__'); + } + return { + filters, elements, storage, boxes, + clear: () => elements.clearFiltersBtn.fire('click'), + pickObserver: (id) => toggle('observerMenu', 'obsId', id, true), + pickType: (id) => toggle('typeMenu', 'typeId', id, true), + observerAllRow: () => allRow('observerMenu', 'obsId'), + typeAllRow: () => allRow('typeMenu', 'typeId'), + }; +} + +test('observer: select A, Clear, select B leaves only B selected', () => { + const s = setup(); + s.pickObserver('obsA'); + assert.strictEqual(s.filters.observer, 'obsA'); + s.clear(); + assert.strictEqual(s.filters.observer, undefined); + s.pickObserver('obsB'); + assert.strictEqual(s.filters.observer, 'obsB', 'stale selection survived Clear'); + assert.strictEqual(s.storage['meshcore-observer-filter'], 'obsB'); + assert.strictEqual(s.elements.observerTrigger.textContent, 'Observer B ▾'); + assert.strictEqual(s.observerAllRow().checked, false, 'All Observers row checked while B is selected'); +}); + +test('observer: All Observers row is checked after Clear', () => { + const s = setup(); + s.pickObserver('obsA'); + assert.strictEqual(s.observerAllRow().checked, false); + s.clear(); + assert.strictEqual(s.observerAllRow().checked, true, 'All Observers row unchecked after Clear'); + const others = s.boxes('observerMenu').filter((c) => c.dataset.obsId !== '__all__'); + assert(others.length === 2 && others.every((c) => !c.checked), 'an observer row is still checked after Clear'); + assert.strictEqual(s.elements.observerTrigger.textContent, 'All Observers ▾'); +}); + +test('type: select A, Clear, select B leaves only B selected', () => { + const s = setup(); + s.pickType('4'); + assert.strictEqual(s.filters.type, '4'); + s.clear(); + assert.strictEqual(s.filters.type, undefined); + s.pickType('5'); + assert.strictEqual(s.filters.type, '5', 'stale selection survived Clear'); + assert.strictEqual(s.storage['meshcore-type-filter'], '5'); + assert.strictEqual(s.elements.typeTrigger.textContent, 'GRP_TXT ▾'); + assert.strictEqual(s.typeAllRow().checked, false, 'All Types row checked while B is selected'); +}); + +test('type: All Types row is checked after Clear', () => { + const s = setup(); + s.pickType('4'); + assert.strictEqual(s.typeAllRow().checked, false); + s.clear(); + assert.strictEqual(s.typeAllRow().checked, true, 'All Types row unchecked after Clear'); + const others = s.elements.typeMenu._checkboxes.filter((c) => c.dataset.typeId !== '__all__'); + assert(others.length === 2 && others.every((c) => !c.checked), 'a type row is still checked after Clear'); + assert.strictEqual(s.elements.typeTrigger.textContent, 'All Types ▾'); +}); + +test('type only: Clear button is shown once a type is picked, hidden after Clear', () => { + const s = setup(); + assert.strictEqual(s.elements.clearFiltersBtn.style.display, 'none'); + s.pickType('4'); + assert.strictEqual(s.filters.type, '4'); + assert.strictEqual(s.elements.clearFiltersBtn.style.display, '', 'Clear button hidden while only a type filter is active'); + s.clear(); + assert.strictEqual(s.elements.clearFiltersBtn.style.display, 'none', 'Clear button still shown after Clear'); +}); + +test('observer only: Clear button is shown once an observer is picked, hidden after Clear', () => { + const s = setup(); + assert.strictEqual(s.elements.clearFiltersBtn.style.display, 'none'); + s.pickObserver('obsA'); + assert.strictEqual(s.elements.clearFiltersBtn.style.display, '', 'Clear button hidden while only an observer filter is active'); + s.clear(); + assert.strictEqual(s.elements.clearFiltersBtn.style.display, 'none', 'Clear button still shown after Clear'); +}); + +console.log(`\n${passed} passed, ${failed} failed`); +if (failed > 0) process.exit(1); +console.log('All tests passed ✅'); diff --git a/test-observer-menu-interactions.js b/test-observer-menu-interactions.js index 32d981445..e8fb2cfc0 100644 --- a/test-observer-menu-interactions.js +++ b/test-observer-menu-interactions.js @@ -103,15 +103,20 @@ function stubEl(overrides) { const updatePacketsUrl = () => {}; const applyObserverSearchFilter = () => applyCalls.push('applied'); const documentStub = { getElementById: () => stubEl({}) }; + // The handler also empties the multi-select Sets and rebuilds both menus + // (#2012); those are tested in test-issue-2012-clear-filters-selection.js. + const noop = () => {}; const fn = new Function( 'filters', 'localStorage', 'document', 'obsSearchInput', 'applyObserverSearchFilter', 'savedTimeWindowMin', 'DEFAULT_TIME_WINDOW', 'RegionFilter', 'updatePacketsUrl', 'loadPackets', - '_observerFilterSet', + '_observerFilterSet', 'selectedObservers', 'selectedTypes', + 'buildObserverMenu', 'updateObsTrigger', 'buildTypeMenu', 'updateTypeTrigger', body ); fn(filters, localStorage, documentStub, obsSearchInput, applyObserverSearchFilter, - 15, 15, RegionFilter, updatePacketsUrl, loadPackets, null); + 15, 15, RegionFilter, updatePacketsUrl, loadPackets, null, new Set(), new Set(), + noop, noop, noop, noop); assert.strictEqual(obsSearchInput.value, '', 'search box should be cleared'); assert.deepStrictEqual(applyCalls, ['applied'], 'the filter should be re-applied after clearing'); From 52b9474d7aafc1251e59f286787958fe69023eb9 Mon Sep 17 00:00:00 2001 From: efiten Date: Sun, 13 Sep 2026 21:09:51 +0200 Subject: [PATCH 040/151] feat(map): filter repeaters by region name (#1862) (#2022) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #1862 ## What Adds a **Region Scope** picker to the map controls: pick `#be` and the map keeps the nodes that declare `#be` or were seen carrying `#be` traffic. It combines with the #2006 scope-state filter and persists in localStorage the same way. While a region is picked, a small "Region: #be · reset" chip sits on the map itself, so the filter stays visible when the controls panel is collapsed (the default on phones) and can be cleared from there. ## API `/api/nodes` and `/api/nodes/{pubkey}` gain two fields on repeater/room rows: - `declared_regions`: named regions from the node's newest declared-regions answer, split by the same function the Scope Audit now uses for `declaredRegions` (`splitDeclaredRegions`, `cmd/server/scope_config_state.go`), so both pages list a repeater under the same names. `[]` means it answered and named no region. Absent means it never answered, other roles, no declared-regions source, or the declared-regions lookup failed. - `declared_regions_truncated`: present, and `true`, only when that answer was flagged as truncated, so the list is partial. Never `false`: the `nodes.configured_scope` source does not record truncation, so absence does not mean the list is complete. The observed side reuses `transported_scopes`. Documented in `docs/api-spec.md` and the served OpenAPI spec. ### Why no `?hashRegion=` query parameter The observed side lives in the in-memory store. Filtering it after the SQL `LIMIT`/`OFFSET` would corrupt `total` and paging, and the map pages through `/api/nodes`. Same reasoning as the Data path section of #2001. ## Map behaviour - Filtering is client-side over the nodes `fetchAllNodes` already loaded: no new request. One pass over the loaded nodes to build the picker counts, one Set lookup per node per render. The marker filter is `nodePassesMapFilters` (`public/map.js:219`) and the observer stand-down `observerLayerShown` (`:212`), both exported and tested. - The picker and hint count only nodes with a map position, the same test the marker filter applies first, so a count never promises markers the map cannot draw. - The observer layer stands down while a region is picked, for the same reason it does for the scope-state filter. - The popup lists declared and observed regions separately. A truncated declared answer carries the same `truncated` badge the Scope Audit shows. - Absence is not read as a finding: the hint under the picker says a node left off the map is not proof it lacks the region. ## Tests - Go: `node_declared_regions_api_test.go` covers `declared_regions` on list and detail endpoints, the no-source case, agreement with `/api/scope-audit`, `declared_regions_truncated` (truncated, truncated-empty, complete, configured_scope-only, newer untruncated answer, companion) and the OpenAPI schema. - JS: `test-issue-1862-map-region-filter.js` (30 tests) covers the pure pieces (evidence, counts, options, hint, popup rows, `nodePassesMapFilters`, `observerLayerShown`) and, at page level, runs the registered map page through `init()` and `loadNodes()` in a vm sandbox: picker built from loaded nodes, markers filtered by a stored region, observer pins standing down, popup rows, the change handler persisting, and the chip showing, resetting and rendering its text as text. - Mutation-checked: removing the region check, the observer stand-down, the picker build in `loadNodes`, the popup rows, the persist on change, the chip reset, or the truncated flag (Go or JS) each fails a test. `test-issue-2001-map-scope-state.js` still passes. - `go test ./...` in `cmd/server` passes; `check-css-vars` and `check-xss-sinks --diff` are clean. ## Staging validation Build `c646310f`, Chrome, no console errors: - Picking `#be`: chip "Region: #be · reset" at the top of the map, hint "221 nodes with a map position have evidence for #be: 129 declare it, 191 seen carrying its traffic. Absence here is not proof: ...". - Clicking reset: picker back to "All regions", stored choice cleared, chip hidden. - First version, same instance: `declared_regions` on 136 of 500 `/api/nodes` rows; returning to "All regions" blocked the main thread 5.4 s against 3.2 s for the existing Status filter returning to "All". The review measured the region filter's own added work at about 0.08 ms per render plus popup rows that are already built for every marker, so most of that time is the existing full re-render. ## Not verified - The chip placement at phone width, and on narrow desktop widths where it may sit under the expanded controls panel. - The truncated badge with real truncated answers (staging has none today). - `test-map-clustering.js` has one failing test on `upstream/master` too; untouched here. - Marker badges from the original issue body are not implemented; the popup rows are the per-node display. --------- Co-authored-by: Claude Opus 5 (1M context) --- cmd/server/node_declared_regions_api_test.go | 293 ++++++++++++ cmd/server/openapi.go | 2 + cmd/server/routes.go | 8 +- cmd/server/scope_audit.go | 30 +- cmd/server/scope_config_state.go | 76 ++- docs/api-spec.md | 6 +- public/map.js | 310 ++++++++++-- public/style.css | 32 ++ test-all.sh | 1 + test-issue-1862-map-region-filter.js | 467 +++++++++++++++++++ 10 files changed, 1140 insertions(+), 85 deletions(-) create mode 100644 cmd/server/node_declared_regions_api_test.go create mode 100644 test-issue-1862-map-region-filter.js diff --git a/cmd/server/node_declared_regions_api_test.go b/cmd/server/node_declared_regions_api_test.go new file mode 100644 index 000000000..480ee6b46 --- /dev/null +++ b/cmd/server/node_declared_regions_api_test.go @@ -0,0 +1,293 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" +) + +// declaredRegionsOf reads the declared_regions field back as a []string, and +// reports whether the field was present at all. Absent and present-but-empty +// are different answers (never asked vs answered with no named region), so +// the helper keeps them apart instead of collapsing both to nil. +func declaredRegionsOf(t *testing.T, node map[string]interface{}) ([]string, bool) { + t.Helper() + raw, present := node["declared_regions"] + if !present { + return nil, false + } + list, ok := raw.([]interface{}) + if !ok { + t.Fatalf("declared_regions is %T (%v), want a JSON array", raw, raw) + } + out := make([]string, 0, len(list)) + for _, v := range list { + s, ok := v.(string) + if !ok { + t.Fatalf("declared_regions entry is %T (%v), want a string", v, v) + } + out = append(out, s) + } + return out, true +} + +// TestHandleNodesExposesDeclaredRegions pins the field the map's region +// filter reads (#1862): the named regions a repeater declared, spelled the way +// the Scope Audit spells them ('#' stripped, the wildcard left out), so the +// map can answer "which repeaters handle #be" from the one bulk fetch it +// already makes. +func TestHandleNodesExposesDeclaredRegions(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES + ('PK_NAMED', 'named-rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, '#be,*,be-bru', '2026-01-01T00:00:00Z'), + ('PK_STARONLY', 'star-rp', 'repeater', 51.1, 4.1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, '*', '2026-01-01T00:00:00Z'), + ('PK_NEVER', 'never-rp', 'repeater', 51.2, 4.2, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('PK_COMPANION', 'phone', 'companion', 51.3, 4.3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be,*', '2026-01-01T00:00:00Z')`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + got, present := declaredRegionsOf(t, nodes["PK_NAMED"]) + if !present { + t.Fatalf("declared repeater has no declared_regions field: %v", nodes["PK_NAMED"]) + } + if want := []string{"be", "be-bru"}; !reflect.DeepEqual(got, want) { + t.Errorf("declared_regions = %v, want %v ('#' stripped, '*' left out, list order kept)", got, want) + } + + // Answered with only the wildcard: the answer exists and names no region. + // That is an empty list, not an absent field. + got, present = declaredRegionsOf(t, nodes["PK_STARONLY"]) + if !present { + t.Errorf("wildcard-only repeater has no declared_regions field, want an empty list") + } else if len(got) != 0 { + t.Errorf("wildcard-only repeater declared_regions = %v, want []", got) + } + + // Never asked: no answer to report. + if v, present := nodes["PK_NEVER"]["declared_regions"]; present { + t.Errorf("never-asked repeater carries declared_regions = %v, want the field absent", v) + } + + // A companion does not forward, so a declared list on it is not something + // the region filter should read, same rule as scope_config_state. + if v, present := nodes["PK_COMPANION"]["declared_regions"]; present { + t.Errorf("companion carries declared_regions = %v, want the field absent", v) + } +} + +// TestHandleNodeDetailExposesDeclaredRegions keeps the node endpoint and the +// list endpoint saying the same thing about one repeater. +func TestHandleNodeDetailExposesDeclaredRegions(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES ('PK_DETAIL_RGN', 'detail-rp', 'repeater', 51.3, 4.3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'nl,#be', '2026-01-01T00:00:00Z')`, + ); err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest("GET", "/api/nodes/PK_DETAIL_RGN", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + var resp struct { + Node map[string]interface{} `json:"node"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + got, present := declaredRegionsOf(t, resp.Node) + if !present { + t.Fatalf("detail endpoint has no declared_regions field: %v", resp.Node) + } + if want := []string{"nl", "be"}; !reflect.DeepEqual(got, want) { + t.Errorf("declared_regions = %v, want %v", got, want) + } +} + +// TestHandleNodesOmitsDeclaredRegionsWithoutASource: on a database that cannot +// hold a declared answer, an empty list would claim "answered, names nothing". +func TestHandleNodesOmitsDeclaredRegionsWithoutASource(t *testing.T) { + srv, router := setupTestServer(t) + if srv.db.hasConfiguredScope || srv.db.hasDeclaredRegionsTable { + t.Fatal("fixture has a declared-regions source: this test would prove nothing") + } + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count) + VALUES ('PK_NOSOURCE_RGN', 'rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1)`, + ); err != nil { + t.Fatal(err) + } + + nodes := nodesByPubkey(t, router, "?limit=200") + + if v, present := nodes["PK_NOSOURCE_RGN"]["declared_regions"]; present { + t.Errorf("repeater carries declared_regions = %v on a schema with no declared-regions source, want the field absent", v) + } +} + +// TestScopeAuditAndNodesAgreeOnDeclaredRegions drives both real handlers over +// the same stored answers. The region filter on the map and the region search +// on the Scope Audit page must list a repeater under the same declared names, +// or searching "be" on one page and picking #be on the other give different +// repeaters. +func TestScopeAuditAndNodesAgreeOnDeclaredRegions(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + if _, err := srv.db.conn.Exec(`ALTER TABLE transmissions ADD COLUMN scope_name TEXT`); err != nil { + t.Fatal(err) + } + if err := srv.db.detectSchema(context.Background(), srv.db.conn); err != nil { + t.Fatal(err) + } + + cases := map[string]string{ + "PK_RGN_HASHSTAR": "#be,#*", + "PK_RGN_BARESTAR": "be,*,nl-li", + "PK_RGN_STARONLY": "#*", + "PK_RGN_NAMEDONLY": "#be, de", + "PK_RGN_EMPTY": "", + } + for pk, csv := range cases { + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES (?, ?, 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, ?, '2026-01-01T00:00:00Z')`, + pk, "rp-"+pk, csv, + ); err != nil { + t.Fatal(err) + } + } + + audit := getScopeAudit(t, router, "?window=24h") + auditRegions := map[string][]string{} + for _, row := range audit.Repeaters { + auditRegions[strings.ToLower(row.PublicKey)] = row.DeclaredRegions + } + nodes := nodesByPubkey(t, router, "?limit=200") + + for pk, csv := range cases { + want, ok := auditRegions[strings.ToLower(pk)] + if !ok { + t.Errorf("%s (%q): the scope audit does not list the repeater at all", pk, csv) + continue + } + got, present := declaredRegionsOf(t, nodes[pk]) + if !present { + t.Errorf("regions_csv %q: the scope audit lists %v, the map has no declared_regions field", csv, want) + continue + } + if !reflect.DeepEqual(got, want) { + t.Errorf("regions_csv %q: the scope audit lists %v, the map lists %v", csv, want, got) + } + } +} + +// TestOpenAPINodeSchemaDocumentsDeclaredRegions keeps the served spec in step +// with the wire. +func TestOpenAPINodeSchemaDocumentsDeclaredRegions(t *testing.T) { + spec := fetchSpec(t) + components := asMap(t, spec["components"], "components") + schemas := asMap(t, components["schemas"], "components.schemas") + props := asMap(t, asMap(t, schemas["Node"], "Node")["properties"], "Node.properties") + + p, ok := props["declared_regions"] + if !ok { + t.Fatal("Node.properties.declared_regions missing") + } + pm := asMap(t, p, "declared_regions") + if pm["type"] != "array" { + t.Errorf("declared_regions: want type array, got %v", pm["type"]) + } + items := asMap(t, pm["items"], "declared_regions.items") + if items["type"] != "string" { + t.Errorf("declared_regions.items: want type string, got %v", items["type"]) + } +} + +// TestNodesCarryDeclaredRegionsTruncated: a repeater whose declared list was +// cut off returns a partial list, and the Scope Audit flags it as such. The +// node row must carry the same caveat, or the map popup shows a partial list +// (or "no named region") as the whole answer (#2022 review). +func TestNodesCarryDeclaredRegionsTruncated(t *testing.T) { + srv, router := setupScopeConfigStateServer(t) + createDeclaredRegionsTable(t, srv) + + if _, err := srv.db.conn.Exec(`INSERT INTO nodes + (public_key, name, role, lat, lon, last_seen, first_seen, advert_count, configured_scope, configured_scope_at) + VALUES + ('pk_trunc', 'trunc-rp', 'repeater', 51.0, 4.0, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('pk_trunc_empty', 'trunc-empty-rp', 'room', 51.1, 4.1, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('pk_complete', 'complete-rp', 'repeater', 51.2, 4.2, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL), + ('pk_cfg_only', 'cfg-rp', 'repeater', 51.3, 4.3, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be', '2026-01-01T00:00:00Z'), + ('pk_newer_cfg', 'newer-rp', 'repeater', 51.4, 4.4, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, 'be,nl', '2026-03-01T00:00:00Z'), + ('pk_companion', 'phone', 'companion', 51.5, 4.5, '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 1, NULL, NULL)`, + ); err != nil { + t.Fatal(err) + } + seedSecondSource(t, srv, "pk_trunc", "2026-02-01T00:00:00Z", "#be", 1) + seedSecondSource(t, srv, "pk_trunc_empty", "2026-02-01T00:00:00Z", "", 1) + seedSecondSource(t, srv, "pk_complete", "2026-02-01T00:00:00Z", "#be", 0) + // Older truncated answer, newer answer from the other source: the newest + // answer decides, caveat included. + seedSecondSource(t, srv, "pk_newer_cfg", "2026-02-01T00:00:00Z", "#be", 1) + seedSecondSource(t, srv, "pk_companion", "2026-02-01T00:00:00Z", "#be", 1) + + nodes := nodesByPubkey(t, router, "?limit=200") + + for _, pk := range []string{"pk_trunc", "pk_trunc_empty"} { + if _, present := declaredRegionsOf(t, nodes[pk]); !present { + t.Errorf("%s: no declared_regions field", pk) + } + if v, present := nodes[pk]["declared_regions_truncated"]; !present || v != true { + t.Errorf("%s: declared_regions_truncated = %v (present %v), want true", pk, v, present) + } + } + for _, pk := range []string{"pk_complete", "pk_cfg_only", "pk_newer_cfg", "pk_companion"} { + if v, present := nodes[pk]["declared_regions_truncated"]; present { + t.Errorf("%s: declared_regions_truncated = %v, want the field absent", pk, v) + } + } + + req := httptest.NewRequest("GET", "/api/nodes/pk_trunc", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + if w.Code != http.StatusOK { + t.Fatalf("detail status = %d, want 200: %s", w.Code, w.Body.String()) + } + var resp struct { + Node map[string]interface{} `json:"node"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode: %v", err) + } + if v := resp.Node["declared_regions_truncated"]; v != true { + t.Errorf("detail endpoint declared_regions_truncated = %v, want true", v) + } +} + +func TestOpenAPINodeSchemaDocumentsDeclaredRegionsTruncated(t *testing.T) { + spec := fetchSpec(t) + components := asMap(t, spec["components"], "components") + schemas := asMap(t, components["schemas"], "components.schemas") + props := asMap(t, asMap(t, schemas["Node"], "Node")["properties"], "Node.properties") + + p, ok := props["declared_regions_truncated"] + if !ok { + t.Fatal("Node.properties.declared_regions_truncated missing") + } + if pm := asMap(t, p, "declared_regions_truncated"); pm["type"] != "boolean" { + t.Errorf("declared_regions_truncated: want type boolean, got %v", pm["type"]) + } +} diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 69aedc389..80053fd9d 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -191,6 +191,7 @@ func componentSchemas() map[string]interface{} { "relay_count_24h": map[string]interface{}{"type": "integer", "description": "Repeater/room only: relay-hop appearances in the last 24 hours."}, "unscoped_relay_count_24h": map[string]interface{}{"type": "integer", "description": "Repeater/room only: subset of relay_count_24h that were unscoped floods (route_type FLOOD). A well-configured repeater sets flood.max.unscoped 0, so a non-trivial count flags a base-config problem."}, "last_relayed": str("Repeater/room only: RFC3339 time this node last appeared as a relay hop."), + "declared_regions": map[string]interface{}{"type": "array", "items": map[string]interface{}{"type": "string"}, "description": "Repeater/room only (#1862): named regions from this node's newest declared-regions answer, spelled as GET /api/scope-audit declaredRegions spells them (leading '#' stripped, '*' wildcard excluded). Empty array: it answered and named no region. Absent: it never answered, this database has no declared-regions source, or the declared-regions lookup failed. Absence is not evidence the node lacks a region."}, "relay_window_hours": map[string]interface{}{"type": "integer", "description": "Repeater/room only, /api/nodes/{pubkey} detail endpoint only: width (hours) of the relay-activity window the relay_count_* values cover."}, "traffic_share_score": score01("#672 Traffic axis: share of non-advert traffic relayed through this repeater. Repeater/room only."), "bridge_score": score01("#672 Bridge axis: normalized betweenness centrality (chokepoint importance). Repeater/room only."), @@ -201,6 +202,7 @@ func componentSchemas() map[string]interface{} { "type": "string", "enum": []string{"A", "B", "C", "D", "F"}, "description": "Letter grade derived from usefulness_score. Repeater/room only.", }, + "declared_regions_truncated": map[string]interface{}{"type": "boolean", "description": "Repeater/room only (#1862): present, and true, only when the answer behind declared_regions was flagged as truncated, so that list is partial. GET /api/scope-audit shows the same flag as truncated. Absent otherwise, including for a source that does not record truncation, so absence does not mean the list is complete."}, }, }, "NodeListResponse": map[string]interface{}{ diff --git a/cmd/server/routes.go b/cmd/server/routes.go index 8a755ed23..a86fc8d26 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -48,7 +48,7 @@ type Server struct { // so /api/nodes does not re-run the merge query pair per request. The // cached map is read by concurrent requests and replaced, never mutated. declaredRegionsMu sync.Mutex - declaredRegionsCache map[string]string + declaredRegionsCache map[string]declaredAnswer declaredRegionsAt time.Time // Collapses the TTL-boundary herd so the query runs once, not once per // in-flight request, and never under declaredRegionsMu. @@ -1327,7 +1327,7 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { // state. Two small queries (232 rows on a live instance) and no // window scan — the state is a pure function of the declared list, // see nodeScopeConfigState. - var declaredCSV map[string]string + var declaredCSV map[string]declaredAnswer declaredOK := false if needsRelay { declaredCSV, declaredOK = s.declaredRegionsCSV() @@ -1376,7 +1376,7 @@ func (s *Server) handleNodes(w http.ResponseWriter, r *http.Request) { // see declaredRegionsCSV. if declaredOK { csv, has := declaredCSV[strings.ToLower(pk)] - node["scope_config_state"] = nodeScopeConfigState(csv, has, info.TransportedScopes) + enrichNodeDeclaredScope(node, csv, has, info.TransportedScopes) } // #672 4-axis usefulness. traffic_share_score keeps the // raw per-axis Traffic value (#1456); the structural axes @@ -1579,7 +1579,7 @@ func (s *Server) handleNodeDetail(w http.ResponseWriter, r *http.Request) { // and the map must not disagree about a repeater's scope state. if declaredCSV, ok := s.declaredRegionsCSV(); ok { csv, has := declaredCSV[strings.ToLower(pubkey)] - node["scope_config_state"] = nodeScopeConfigState(csv, has, info.TransportedScopes) + enrichNodeDeclaredScope(node, csv, has, info.TransportedScopes) } // #672 4-axis usefulness (see handleNodes for the field // contract). traffic_share_score keeps the raw per-axis diff --git a/cmd/server/scope_audit.go b/cmd/server/scope_audit.go index d6e45ebab..963369a26 100644 --- a/cmd/server/scope_audit.go +++ b/cmd/server/scope_audit.go @@ -1118,26 +1118,16 @@ func (s *Server) computeScopeAudit(window, sinceISO string) (*ScopeAuditResponse continue } - allRegions := splitRegionsCSV(d.RegionsCSV) - declaredWildcard := false - declaredNamed := make([]string, 0, len(allRegions)) - declaredSet := make(map[string]bool, len(allRegions)) - for _, rgn := range allRegions { - // isScopeWildcard, not a bare "*" compare: a collector storing the - // "#*" spelling would otherwise have it counted as a named region - // here and as the wildcard by /api/nodes, and the two pages would - // classify the same repeater differently (#2006 review). - if isScopeWildcard(rgn) { - declaredWildcard = true - continue - } - // normScope mirrors the observed side (scopes.go: agg.scopes keys - // are already normScope'd) — regions_csv is not guaranteed to - // arrive with '#' already stripped in every case, and comparing - // raw here would reintroduce the exact trap normScope exists to - // prevent, just on the other side of the comparison. - rgn = normScope(rgn) - declaredNamed = append(declaredNamed, rgn) + // splitDeclaredRegions is shared with /api/nodes (scope_config_state + // and declared_regions), so the map and this page read one declared + // list the same way. It uses isScopeWildcard rather than a bare "*" + // compare (#2006 review), and normScope's the named regions to mirror + // the observed side (scopes.go: agg.scopes keys are already + // normScope'd): regions_csv is not guaranteed to arrive with '#' + // already stripped. + declaredNamed, declaredWildcard := splitDeclaredRegions(d.RegionsCSV) + declaredSet := make(map[string]bool, len(declaredNamed)) + for _, rgn := range declaredNamed { declaredSet[rgn] = true } diff --git a/cmd/server/scope_config_state.go b/cmd/server/scope_config_state.go index de88027da..1f1112d37 100644 --- a/cmd/server/scope_config_state.go +++ b/cmd/server/scope_config_state.go @@ -50,6 +50,52 @@ func isScopeWildcard(region string) bool { return region == "*" || region == "#*" } +// splitDeclaredRegions reads one declared regions_csv into its named regions +// and whether the flood wildcard was present. It is the single reading of a +// declared list: the Scope Audit's declaredRegions, /api/nodes' +// scope_config_state and its declared_regions field (#1862) all come from it, +// so a repeater cannot be listed under one set of names on the audit page and +// another on the map. Named regions are normScope'd, because regions_csv +// arrives with and without the leading '#' depending on the collector, and +// the wildcard is never a named region (see isScopeWildcard). +func splitDeclaredRegions(csv string) (named []string, wildcard bool) { + all := splitRegionsCSV(csv) + named = make([]string, 0, len(all)) + for _, rgn := range all { + if isScopeWildcard(rgn) { + wildcard = true + continue + } + named = append(named, normScope(rgn)) + } + return named, wildcard +} + +// enrichNodeDeclaredScope sets the two declared-regions fields on a +// repeater/room node: scope_config_state (#2001) and declared_regions +// (#1862). Shared by the list and detail handlers so the node page and the +// map read one repeater the same way. +// +// declared_regions is set only when the node has a declared answer. An empty +// list means it answered and named no region; an absent field means it was +// never asked. The map's region filter treats neither as evidence that the +// repeater lacks a region. +// +// declared_regions_truncated is set, to true, only when that answer was +// flagged as cut off: the list is then partial, which the Scope Audit also +// shows. Omitted otherwise, since a source that does not record truncation +// cannot vouch for a complete list either. +func enrichNodeDeclaredScope(node map[string]interface{}, answer declaredAnswer, hasDeclared bool, transportedScopes []string) { + node["scope_config_state"] = nodeScopeConfigState(answer.RegionsCSV, hasDeclared, transportedScopes) + if hasDeclared { + named, _ := splitDeclaredRegions(answer.RegionsCSV) + node["declared_regions"] = named + if answer.Truncated { + node["declared_regions_truncated"] = true + } + } +} + // nodeScopeConfigState classifies one node for the scope_config_state field. // // declaredCSV is the node's most recent declared-regions answer (the raw @@ -65,15 +111,7 @@ func isScopeWildcard(region string) bool { // Audit's job, and re-labelling the node here would hide it. func nodeScopeConfigState(declaredCSV string, hasDeclared bool, transportedScopes []string) string { if hasDeclared { - var named []string - wildcard := false - for _, rgn := range splitRegionsCSV(declaredCSV) { - if isScopeWildcard(rgn) { - wildcard = true - continue - } - named = append(named, rgn) - } + named, wildcard := splitDeclaredRegions(declaredCSV) return scopeAuditConfigState(named, wildcard) } if len(transportedScopes) > 0 { @@ -91,7 +129,17 @@ func nodeScopeConfigState(declaredCSV string, hasDeclared bool, transportedScope // disagreeing for longer than either is stale on its own. const declaredRegionsTTL = 30 * time.Second -// declaredRegionsCSV maps target pubkey to its newest declared regions_csv, +// declaredAnswer is one node's newest declared-regions answer as the node +// endpoints read it: the raw regions_csv, and whether the collector flagged +// the list as cut off. Truncated is only ever true for a source that records +// it (see AllCurrentDeclaredRegions); false means "not flagged", not "known +// complete". +type declaredAnswer struct { + RegionsCSV string + Truncated bool +} + +// declaredRegionsCSV maps target pubkey to its newest declared answer, // merged across every collector this database carries, keyed lowercase. // // Keys are lowercased because the two sides of the join disagree by design: @@ -106,7 +154,7 @@ const declaredRegionsTTL = 30 * time.Second // repeater has answered" is a statement the schema cannot support. An empty map // with ok=true means the sources exist and nobody has answered yet, which IS a // finding and is what ScopeConfigNone reports. -func (s *Server) declaredRegionsCSV() (map[string]string, bool) { +func (s *Server) declaredRegionsCSV() (map[string]declaredAnswer, bool) { if !s.db.hasConfiguredScope && !s.db.hasDeclaredRegionsTable { return nil, false } @@ -142,9 +190,9 @@ func (s *Server) declaredRegionsCSV() (map[string]string, bool) { return nil, qerr } atomic.AddInt64(&s.declaredRegionsQueries, 1) - out := make(map[string]string, len(rows)) + out := make(map[string]declaredAnswer, len(rows)) for _, r := range rows { - out[strings.ToLower(r.Target)] = r.RegionsCSV + out[strings.ToLower(r.Target)] = declaredAnswer{RegionsCSV: r.RegionsCSV, Truncated: r.Truncated} } s.declaredRegionsMu.Lock() // Cached maps are handed to concurrent readers and never written again. @@ -157,5 +205,5 @@ func (s *Server) declaredRegionsCSV() (map[string]string, bool) { log.Printf("[nodes] declared-regions lookup failed, scope_config_state omitted: %v", err) return nil, false } - return v.(map[string]string), true + return v.(map[string]declaredAnswer), true } diff --git a/docs/api-spec.md b/docs/api-spec.md index 0de368df5..21e5bd099 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -320,7 +320,9 @@ Paginated node list with filtering. "hash_sizes_seen": [number] | undefined, // present only if >1 unique size seen "last_heard": string (ISO) | undefined, // from in-memory packets or path relay "default_scope": string | null | undefined, // Most recently observed transport scope for this node. null = never observed transport-scoped, "" = observed scoped but no configured region matched, "#name" = matched region. Only present when ingestor has applied the nodes_default_scope_v1 migration. - "scope_config_state": string | undefined // Repeater/room only. How this node's region config reads: "full" | "no-unscoped" | "no-scopes" | "no-flood" from its own declared-regions answer, "observed" when it never answered but has been seen forwarding scoped traffic, "none" when it never answered and nothing scoped was observed. Absent for other roles and when the declared-regions lookup failed. + "scope_config_state": string | undefined, // Repeater/room only. How this node's region config reads: "full" | "no-unscoped" | "no-scopes" | "no-flood" from its own declared-regions answer, "observed" when it never answered but has been seen forwarding scoped traffic, "none" when it never answered and nothing scoped was observed. Absent for other roles and when the declared-regions lookup failed. + "declared_regions": [string] | undefined, // Repeater/room only. Named regions from this node's newest declared-regions answer, spelled like GET /api/scope-audit `declaredRegions` (leading "#" stripped, "*" excluded). [] = answered with no named region. Absent = never answered, other roles, no declared-regions source, or the declared-regions lookup failed. + "declared_regions_truncated": true | undefined // Repeater/room only. Present, and true, only when the answer behind declared_regions was flagged as truncated: the list is partial. Same flag as GET /api/scope-audit `truncated`. } ], "total": number, // total matching count (before pagination) @@ -336,6 +338,8 @@ Paginated node list with filtering. **Notes:** - `hash_sizes_seen` is only present when more than one hash size has been observed. - `last_heard` is only present when in-memory data provides a more recent timestamp than `last_seen`. +- `declared_regions` is what the map's region-scope filter reads (#1862), together with `transported_scopes`. It is the same list the Scope Audit shows as `declaredRegions`, split and normalised by the same function, so the audit's region search and the map list a repeater under the same names. Its presence follows `scope_config_state`: set whenever that field is one of the four declared states. An absent field or a region missing from the list is not evidence that the repeater lacks that region. +- `declared_regions_truncated` is set only when the newest declared answer was flagged as cut off (`node_declared_regions.truncated`), and the map popup then marks the declared list as truncated, as the Scope Audit does. It is never `false`: the `nodes.configured_scope` source does not record whether its size cap fired, so an absent field does not mean the list is complete. - `scope_config_state` carries the same four declared states as [GET /api/scope-audit](#get-apiscope-audit) and is computed the same way, from the newest declared-regions answer merged across collectors. The audit lists only repeaters that have answered; this field also classifies the ones that have not, which is what the map colours by. `"none"` means the answer is missing, not that the node is misconfigured: firmware drops scoped floods for regions it holds no key for, so a repeater with no region config and one nobody sends scoped traffic past are indistinguishable here. The four declared states are computed from stored data and do not depend on a window; `"observed"` and `"none"` are separated by `transported_scopes`, which the in-memory packet store accumulates over its retention window, so a node can move between them after a restart. The field is absent entirely when this database carries no declared-regions source at all, since "nobody has answered" is then a claim the schema cannot support. --- diff --git a/public/map.js b/public/map.js index f68a89ae3..f7f7e03d9 100644 --- a/public/map.js +++ b/public/map.js @@ -9,7 +9,7 @@ let nodes = []; let targetNodeKey = null; let observers = []; - let filters = { repeater: true, companion: true, room: true, sensor: true, observer: true, lastHeard: '30d', neighbors: false, clustering: localStorage.getItem('meshcore-map-clustering') !== 'false', hashLabels: localStorage.getItem('meshcore-map-hash-labels') !== 'false', statusFilter: localStorage.getItem('meshcore-map-status-filter') || 'all', byteSize: localStorage.getItem('meshcore-map-byte-filter') || 'all', multiByteOverlay: localStorage.getItem('meshcore-map-multibyte-overlay') === 'true', scopeState: localStorage.getItem('meshcore-map-scope-filter') || 'all', scopeOverlay: localStorage.getItem('meshcore-map-scope-overlay') === 'true' }; + let filters = { repeater: true, companion: true, room: true, sensor: true, observer: true, lastHeard: '30d', neighbors: false, clustering: localStorage.getItem('meshcore-map-clustering') !== 'false', hashLabels: localStorage.getItem('meshcore-map-hash-labels') !== 'false', statusFilter: localStorage.getItem('meshcore-map-status-filter') || 'all', byteSize: localStorage.getItem('meshcore-map-byte-filter') || 'all', multiByteOverlay: localStorage.getItem('meshcore-map-multibyte-overlay') === 'true', scopeState: localStorage.getItem('meshcore-map-scope-filter') || 'all', scopeOverlay: localStorage.getItem('meshcore-map-scope-overlay') === 'true', regionScope: localStorage.getItem('meshcore-map-region-filter') || '' }; let selectedReferenceNode = null; // pubkey of the reference node for neighbor filtering let neighborPubkeys = null; // Set of pubkeys that are direct neighbors of selected node let wsHandler = null; @@ -158,6 +158,189 @@ '
'; } + // #1862 region-scope filter: "show me the repeaters that forward #be". + // Reads two fields /api/nodes already carries per repeater/room, so the map + // filters what fetchAllNodes loaded and makes no request of its own: + // - declared_regions: the repeater's own declared answer, the list the + // Scope Audit shows as declaredRegions ('#' stripped, '*' left out); + // - transported_scopes: region scopes of traffic whose path names this + // repeater by its full pubkey, the "observed" side #2006 already uses. + // Either one is evidence. Neither being present is not evidence of the + // opposite: most repeaters have never been asked for their list, and a + // repeater that holds a region shows no traffic for it until some passes + // its way. The hint under the picker says so. + + // normRegion mirrors the server's normScope (scope_audit.go): strip one + // leading '#', nothing else, so the map and the audit spell a region alike. + function normRegion(s) { + s = s == null ? '' : String(s); + return s.charAt(0) === '#' ? s.slice(1) : s; + } + + function nodeRegionEvidence(n, region) { + var want = normRegion(region); + var ev = { declared: false, observed: false }; + if (!want || !n) return ev; + var d = Array.isArray(n.declared_regions) ? n.declared_regions : []; + for (var i = 0; i < d.length; i++) { if (normRegion(d[i]) === want) { ev.declared = true; break; } } + var t = Array.isArray(n.transported_scopes) ? n.transported_scopes : []; + for (var j = 0; j < t.length; j++) { if (normRegion(t[j]) === want) { ev.observed = true; break; } } + return ev; + } + + function regionFilterAccepts(n, region) { + if (!region) return true; + var ev = nodeRegionEvidence(n, region); + return ev.declared || ev.observed; + } + + // nodeFiltersNarrowed says whether a node filter that also removes + // non-forwarding roles is active. The observer layer stands down while it + // is (see observerLayerShown). + function nodeFiltersNarrowed(f) { + return (!!f.scopeState && f.scopeState !== 'all') || !!f.regionScope; + } + + // observerLayerShown decides whether _renderMarkersInner draws the plain + // observer pins. #2001: while a specific scope state is picked, the observer + // layer stands down. Two reasons: the filter leads (see nodePassesMapFilters), + // so leaving unrelated pins on the map contradicts the answer; and a + // repeater that is also an observer would otherwise drop out of the filtered + // nodes, stop matching displayedNodePubkeys, and reappear as a plain observer + // pin: the node the operator just filtered away, back in another guise. + // #1862: the region filter narrows the same way, for the same reasons. + function observerLayerShown(f) { + return !!f.observer && !nodeFiltersNarrowed(f); + } + + // nodePassesMapFilters is the marker filter _renderMarkersInner runs over + // the loaded nodes. ctx carries the neighbor-filter state + // ({ selectedReferenceNode, neighborPubkeys }). + function nodePassesMapFilters(n, f, ctx) { + if (!n.lat || !n.lon) return false; + if (!f[n.role || 'companion']) return false; + // Byte size filter (applies only to repeaters). A node with no observed + // size is its own bucket: folding it into "1-byte" made that bucket a + // mix of measured and merely-unheard nodes. + if (f.byteSize !== 'all' && (n.role || 'companion') === 'repeater') { + const hi = hashPrefixInfo(n); + if (f.byteSize === 'unknown') { + if (hi.known) return false; + } else if (!hi.known || String(hi.bytes) !== f.byteSize) { + return false; + } + } + // Scope config filter (#2001). Applied to every role, not just repeaters: + // picking a scope state is a question about repeater configuration, and + // the answer should not leave companions and sensors sitting on the map + // as if they were part of it. Unclassified nodes therefore drop out too; + // see scopeFilterAccepts. + // + // This deliberately differs from the byte-size filter above, which gates + // itself on role === 'repeater' and leaves everything else visible. Asked + // and decided (#2006 review): this filter leads. The byte filter is not + // changed here; that is its own behaviour change, for its own PR. + if (!scopeFilterAccepts(n, f.scopeState)) return false; + // Region scope filter (#1862). Same rule as the scope filter above: it + // applies to every role, so non-forwarding nodes drop out while a region + // is picked. + if (!regionFilterAccepts(n, f.regionScope)) return false; + // Status filter + if (f.statusFilter !== 'all') { + const status = getNodeStatus(n); // #1598: relay-aware for infra + if (status !== f.statusFilter) return false; + } + // Neighbor filter: show only the reference node and its direct neighbors + if (f.neighbors && ctx.selectedReferenceNode && ctx.neighborPubkeys) { + const pk = n.public_key; + if (pk !== ctx.selectedReferenceNode && !ctx.neighborPubkeys.has(pk)) return false; + } + return true; + } + + // collectRegionCounts builds the picker's option list from the loaded nodes: + // one pass, one Set per node so a region both declared and observed counts + // that node once in total. Only nodes with a map position count, the same + // test nodePassesMapFilters applies first, so a count never promises a + // marker the map cannot draw. + function collectRegionCounts(list) { + var byRegion = new Map(); + function entry(r) { + var e = byRegion.get(r); + if (!e) { e = { region: r, total: 0, declared: 0, observed: 0 }; byRegion.set(r, e); } + return e; + } + (list || []).forEach(function (n) { + if (!n || !n.lat || !n.lon) return; + var seen = new Set(); + var declared = new Set(); + (Array.isArray(n.declared_regions) ? n.declared_regions : []).forEach(function (r) { + r = normRegion(r); + if (r && !declared.has(r)) { declared.add(r); seen.add(r); entry(r).declared++; } + }); + var observed = new Set(); + (Array.isArray(n.transported_scopes) ? n.transported_scopes : []).forEach(function (r) { + r = normRegion(r); + if (r && !observed.has(r)) { observed.add(r); seen.add(r); entry(r).observed++; } + }); + seen.forEach(function (r) { entry(r).total++; }); + }); + return Array.from(byRegion.values()).sort(function (a, b) { return a.region < b.region ? -1 : (a.region > b.region ? 1 : 0); }); + } + + function regionFilterOptionsHtml(counts, current) { + var cur = normRegion(current); + var list = (counts || []).slice(); + if (cur && !list.some(function (c) { return c.region === cur; })) list.push({ region: cur, total: 0 }); + return '' + + list.map(function (c) { + return ''; + }).join(''); + } + + function regionFilterHintHtml(counts, current) { + var cur = normRegion(current); + if (!cur) return ''; + var c = null; + for (var i = 0; i < (counts || []).length; i++) { if (counts[i].region === cur) { c = counts[i]; break; } } + var name = '#' + safeEsc(cur); + var found = c + ? c.total + ' node' + (c.total === 1 ? ' with a map position has' : 's with a map position have') + ' evidence for ' + name + ': ' + + c.declared + ' declare it, ' + c.observed + ' seen carrying its traffic.' + : 'No loaded node with a map position has evidence for ' + name + '.'; + return found + ' Absence here is not proof: most repeaters were never asked for their region list, ' + + 'and a repeater only shows traffic for a region once some passes its way.'; + } + + // regionsPopupRowsHtml is the popup's answer to "why is this node on the + // map for #be": the declared list and the observed scopes, kept apart + // because they are different kinds of evidence. + function regionsPopupRowsHtml(n) { + var dt = function (label, title) { + return '
' + label + '
'; + }; + var names = function (list) { + return list.map(function (r) { return '#' + safeEsc(normRegion(r)); }).join(', '); + }; + var out = ''; + if (Array.isArray(n.declared_regions)) { + // Same badge and wording as the Scope Audit's truncated flag + // (scope-audit.js), so a partial list is not shown as the whole answer. + var truncated = n.declared_regions_truncated === true + ? ' truncated' + : ''; + out += dt('Declared', 'Named regions from this repeater’s own declared-regions answer, the same list the Scope Audit shows.') + + '
' + + (n.declared_regions.length ? names(n.declared_regions) : 'no named region') + truncated + '
'; + } + if (Array.isArray(n.transported_scopes) && n.transported_scopes.length) { + out += dt('Observed', 'Region scopes of traffic whose path names this repeater by its full pubkey, over the server’s in-memory packet window.') + + '
' + names(n.transported_scopes) + '
'; + } + return out; + } + function makeMarkerIcon(role, isStale, isAlsoObserver, colorOverride) { const s = ROLE_STYLE[role] || ROLE_STYLE.companion; // #1438: default fill resolves through the live CSS var so existing @@ -305,6 +488,11 @@
+

Map Controls

@@ -341,6 +529,12 @@
${scopeFilterHtml(filters.scopeState)} +
+ Region Scope + + +
+
Filters @@ -727,6 +921,20 @@ }); }); + // Region scope filter (#1862). Options are rebuilt from the loaded nodes + // in buildRegionFilter; persisted the same way as the scope filter above. + const regionFilterEl = document.getElementById('mcRegionFilter'); + if (regionFilterEl) { + regionFilterEl.addEventListener('change', e => setRegionFilter(e.target.value)); + } + // The on-map chip (#2022 review): on a phone the controls panel starts + // collapsed, so a stored region would thin the map with no visible cause. + const regionChipResetEl = document.getElementById('mcRegionChipReset'); + if (regionChipResetEl) { + regionChipResetEl.addEventListener('click', () => setRegionFilter('')); + } + updateRegionChip(); + // Byte size filter buttons document.querySelectorAll('#mcByteFilter .btn').forEach(btn => { btn.addEventListener('click', () => { @@ -1518,6 +1726,7 @@ observers = obsData.observers || []; buildRoleChecks(data.counts || {}); + buildRegionFilter(); buildJumpButtons(); renderMarkers(); @@ -1632,6 +1841,42 @@ } } + // #1862: region counts for the loaded nodes, kept for the hint so a change + // of selection does not walk the node list again. + let regionCounts = []; + + function buildRegionFilter() { + regionCounts = collectRegionCounts(nodes); + const sel = document.getElementById('mcRegionFilter'); + if (sel) sel.innerHTML = regionFilterOptionsHtml(regionCounts, filters.regionScope); + updateRegionHint(); + } + + function setRegionFilter(region) { + filters.regionScope = region; + localStorage.setItem('meshcore-map-region-filter', filters.regionScope); + const sel = document.getElementById('mcRegionFilter'); + if (sel && sel.value !== region) sel.value = region; + updateRegionHint(); + renderMarkers(); + } + + function updateRegionHint() { + const el = document.getElementById('mcRegionHint'); + if (el) el.innerHTML = regionFilterHintHtml(regionCounts, filters.regionScope); + updateRegionChip(); + } + + // updateRegionChip shows the active region on the map itself, as text, and + // hides the chip while no region is picked. + function updateRegionChip() { + const chip = document.getElementById('mcRegionChip'); + const chipText = document.getElementById('mcRegionChipText'); + const cur = normRegion(filters.regionScope); + if (chipText) chipText.textContent = cur ? 'Region: #' + cur : ''; + if (chip) chip.hidden = !cur; + } + let REGION_NAMES = {}; function buildJumpButtons() { @@ -1806,44 +2051,8 @@ if (clusterGroup) clusterGroup.clearLayers(); _currentMarkerData = []; - const filtered = nodes.filter(n => { - if (!n.lat || !n.lon) return false; - if (!filters[n.role || 'companion']) return false; - // Byte size filter (applies only to repeaters). A node with no observed - // size is its own bucket — folding it into "1-byte" made that bucket a - // mix of measured and merely-unheard nodes. - if (filters.byteSize !== 'all' && (n.role || 'companion') === 'repeater') { - const hi = hashPrefixInfo(n); - if (filters.byteSize === 'unknown') { - if (hi.known) return false; - } else if (!hi.known || String(hi.bytes) !== filters.byteSize) { - return false; - } - } - // Scope config filter (#2001). Applied to every role, not just repeaters: - // picking a scope state is a question about repeater configuration, and - // the answer should not leave companions and sensors sitting on the map - // as if they were part of it. Unclassified nodes therefore drop out too — - // see scopeFilterAccepts. - // - // This deliberately differs from the byte-size filter three lines up, - // which gates itself on role === 'repeater' and leaves everything else - // visible. Asked and decided (#2006 review): this filter leads. The byte - // filter is not changed here — that is its own behaviour change, for its - // own PR. - if (!scopeFilterAccepts(n, filters.scopeState)) return false; - // Status filter - if (filters.statusFilter !== 'all') { - const status = getNodeStatus(n); // #1598: relay-aware for infra - if (status !== filters.statusFilter) return false; - } - // Neighbor filter: show only the reference node and its direct neighbors - if (filters.neighbors && selectedReferenceNode && neighborPubkeys) { - const pk = n.public_key; - if (pk !== selectedReferenceNode && !neighborPubkeys.has(pk)) return false; - } - return true; - }); + const filterCtx = { selectedReferenceNode, neighborPubkeys }; + const filtered = nodes.filter(n => nodePassesMapFilters(n, filters, filterCtx)); const allMarkers = []; @@ -1893,13 +2102,9 @@ // Add observer markers (skip observers already represented as a node marker) // Build set of node pubkeys that are displayed on the map const displayedNodePubkeys = new Set(filtered.map(n => (n.public_key || '').toLowerCase())); - // #2001 — while a specific scope state is picked, the observer layer stands - // down. Two reasons: the filter leads (see the note in the predicate above), - // so leaving unrelated pins on the map contradicts the answer; and a - // repeater that is also an observer would otherwise drop out of `filtered`, - // stop matching displayedNodePubkeys, and reappear as a plain observer pin — - // the node the operator just filtered away, back in another guise. - if (filters.observer && filters.scopeState === 'all') { + // #2001/#1862: the observer layer stands down while a node filter narrows + // the map; see observerLayerShown. + if (observerLayerShown(filters)) { for (const obs of observers) { if (!obs.lat || !obs.lon) continue; // Skip observers whose pubkey matches a displayed node — they're shown as combined markers @@ -2074,6 +2279,7 @@ ${hashPrefixRow} ${mbRow} ${scopeRow} + ${regionsPopupRowsHtml(node)}
Key
${safeEsc(key)}
Location
@@ -2601,6 +2807,18 @@ scopeTintEnabled: scopeTintEnabled, scopeStateLabel: scopeStateLabel, scopeFilterHtml: scopeFilterHtml, + // #1862: the region-scope filter's pure pieces, plus the live filter + // state so persistence can be asserted. + regionFilterAccepts: regionFilterAccepts, + nodeRegionEvidence: nodeRegionEvidence, + collectRegionCounts: collectRegionCounts, + regionFilterOptionsHtml: regionFilterOptionsHtml, + regionFilterHintHtml: regionFilterHintHtml, + regionsPopupRowsHtml: regionsPopupRowsHtml, + nodeFiltersNarrowed: nodeFiltersNarrowed, + observerLayerShown: observerLayerShown, + nodePassesMapFilters: nodePassesMapFilters, + filters: filters, // #1356: exposed so the a11y test can assert what the label RENDERS // instead of grepping map.js for where two identifiers sit. makeRepeaterLabelIcon: makeRepeaterLabelIcon, diff --git a/public/style.css b/public/style.css index b541f3f7a..c258d598c 100644 --- a/public/style.css +++ b/public/style.css @@ -3306,6 +3306,38 @@ tr[data-hops]:hover { background: rgba(59,130,246,0.1); } .map-controls.collapsed { display: none; } +/* #1862: the active region filter, on the map itself, so it stays visible + while the controls panel is collapsed (the default on phones). Centred + between Leaflet's top-left controls and the top-right toggle. */ +.mc-region-chip { + position: absolute; + top: 10px; + left: 50%; + transform: translateX(-50%); + z-index: 1000; + display: flex; + align-items: center; + gap: 6px; + max-width: calc(100% - 128px); + padding: 4px 10px; + border-radius: 999px; + border: 1px solid var(--border); + background: var(--card-bg); + color: var(--text); + font-size: 12px; +} +.mc-region-chip[hidden] { display: none; } +.mc-region-chip-text { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.mc-region-chip-reset { + background: none; + border: 0; + padding: 0; + font: inherit; + color: var(--link-color); + text-decoration: underline; + cursor: pointer; +} +.mc-region-chip-reset:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } .node-detail-map { height: 280px; min-height: 200px; diff --git a/test-all.sh b/test-all.sh index 9a327c2a4..58c8b4f6e 100755 --- a/test-all.sh +++ b/test-all.sh @@ -127,6 +127,7 @@ node test-issue-1843-node-qr-quiet-zone.js node test-issue-1846-observers-width.js node test-issue-1849-trace-hashbytes.js node test-issue-1851-channel-message-scope.js +node test-issue-1862-map-region-filter.js node test-issue-1868-control-decode.js node test-issue-1890-og-url.js node test-issue-1956-release-routing.js diff --git a/test-issue-1862-map-region-filter.js b/test-issue-1862-map-region-filter.js new file mode 100644 index 000000000..58f2e5c25 --- /dev/null +++ b/test-issue-1862-map-region-filter.js @@ -0,0 +1,467 @@ +/* Unit tests for the map's region-scope filter (issue #1862) + * + * "Show me the repeaters that forward #be": the filter reads two per-node + * fields /api/nodes already carries, declared_regions (the repeater's own + * answer, same list as the Scope Audit) and transported_scopes (region scopes + * seen in traffic through it), and never makes a request of its own. + * + * Same vm sandbox as test-issue-2001-map-scope-state.js, so it exercises + * map.js itself. + */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const assert = require('assert'); + +let passed = 0, failed = 0; +function test(name, fn) { + try { fn(); passed++; console.log(` ✅ ${name}`); } + catch (e) { failed++; console.log(` ❌ ${name}: ${e.message}`); } +} + +function makeSandbox(stored) { + const L = {}; + L.point = (x, y) => ({ x, y }); + L.latLng = (a, b) => ({ lat: a, lng: b }); + L.divIcon = (opts) => ({ _isDivIcon: true, options: opts, html: opts.html, className: opts.className }); + L.layerGroup = () => ({ addLayer(){ return this; }, removeLayer(){ return this; }, clearLayers(){ return this; }, eachLayer(){}, addTo(){ return this; }, hasLayer(){ return false; } }); + L.marker = (latlng, opts) => ({ _isMarker: true, _latlng: latlng, options: opts || {}, getLatLng(){ return this._latlng; }, bindPopup(){ return this; }, bindTooltip(){ return this; } }); + function MarkerClusterGroup(opts) { this.options = opts || {}; } + MarkerClusterGroup.prototype.addLayer = function () { return this; }; + MarkerClusterGroup.prototype.addLayers = function () { return this; }; + MarkerClusterGroup.prototype.clearLayers = function () { return this; }; + MarkerClusterGroup.prototype.eachLayer = function () {}; + MarkerClusterGroup.prototype.addTo = function () { return this; }; + L.MarkerClusterGroup = MarkerClusterGroup; + L.markerClusterGroup = (opts) => new MarkerClusterGroup(opts); + + const store = Object.assign({}, stored || {}); + const ctx = { + window: {}, + document: { addEventListener(){}, getElementById(){ return null; }, querySelector(){ return null; }, querySelectorAll(){ return []; }, createElement(){ return { id:'', textContent:'', innerHTML:'', appendChild(){}, addEventListener(){}, setAttribute(){}, classList:{add(){},remove(){},toggle(){}} }; }, head: { appendChild(){} }, body: { appendChild(){} } }, + console, Date, Math, Array, Object, String, Number, JSON, RegExp, Error, + parseInt, parseFloat, isFinite, isNaN, Map, Set, Promise, + setTimeout: ()=>{}, clearTimeout: ()=>{}, setInterval: ()=>{}, clearInterval: ()=>{}, + registerPage: () => {}, esc: (s) => s, onWS: () => {}, offWS: () => {}, + localStorage: { getItem: k => (k in store ? store[k] : null), setItem: (k, v) => { store[k] = String(v); }, removeItem: k => { delete store[k]; } }, + fetch: () => Promise.resolve({ json: () => Promise.resolve({}) }), + addEventListener(){}, dispatchEvent(){}, + L: L, + }; + ctx.window.L = ctx.L; + vm.createContext(ctx); + vm.runInContext(fs.readFileSync('public/roles.js', 'utf8'), ctx); + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; + vm.runInContext(fs.readFileSync('public/map.js', 'utf8'), ctx); + for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; + return ctx; +} + +// Spellings as they arrive on live: transported_scopes keeps the '#' +// (transmissions.scope_name), declared_regions has it stripped by the server. +const DECLARED_ONLY = { public_key: 'aa', role: 'repeater', lat: 51.0, lon: 4.0, declared_regions: ['be', 'be-bru'] }; +const OBSERVED_ONLY = { public_key: 'bb', role: 'repeater', lat: 51.1, lon: 4.1, transported_scopes: ['#be', '#de'] }; +const BOTH = { public_key: 'cc', role: 'repeater', lat: 51.2, lon: 4.2, declared_regions: ['be'], transported_scopes: ['#be'] }; +const DECLARED_BE = { public_key: 'ab', role: 'room', lat: 51.3, lon: 4.3, declared_regions: ['be'] }; +// Declares #be but has no position: it can never become a marker. +const UNPLACED = { public_key: 'ac', role: 'repeater', declared_regions: ['be'], transported_scopes: ['#be'] }; +const ANSWERED_NONE = { public_key: 'dd', role: 'repeater', declared_regions: [] }; +const NEVER_ASKED = { public_key: 'ee', role: 'repeater' }; +const COMPANION = { public_key: 'ff', role: 'companion' }; + +console.log('\n=== map.js: region-scope filter (#1862) ==='); +{ + const m = makeSandbox().window.__meshcoreMapInternals; + + test('exposes the region filter hooks', () => { + assert.ok(m, '__meshcoreMapInternals not exposed'); + for (const fn of ['regionFilterAccepts', 'nodeRegionEvidence', 'collectRegionCounts', 'regionFilterOptionsHtml', 'regionFilterHintHtml', 'regionsPopupRowsHtml', 'nodeFiltersNarrowed', 'nodePassesMapFilters', 'observerLayerShown']) { + assert.strictEqual(typeof m[fn], 'function', fn + ' not exported'); + } + }); + + test('no region picked accepts every node', () => { + for (const n of [DECLARED_ONLY, OBSERVED_ONLY, NEVER_ASKED, COMPANION]) { + assert.strictEqual(m.regionFilterAccepts(n, ''), true); + } + }); + + test('a node that declares the region is accepted, and says why', () => { + assert.strictEqual(m.regionFilterAccepts(DECLARED_ONLY, 'be'), true); + const ev = m.nodeRegionEvidence(DECLARED_ONLY, 'be'); + assert.strictEqual(ev.declared, true); + assert.strictEqual(ev.observed, false); + }); + + // transported_scopes carries the '#', declared_regions does not. Comparing + // raw would split one region into two, the trap normScope exists for. + test('a node seen carrying the region is accepted despite the # spelling', () => { + assert.strictEqual(m.regionFilterAccepts(OBSERVED_ONLY, 'be'), true); + const ev = m.nodeRegionEvidence(OBSERVED_ONLY, 'be'); + assert.strictEqual(ev.declared, false); + assert.strictEqual(ev.observed, true); + assert.strictEqual(m.regionFilterAccepts(OBSERVED_ONLY, '#be'), true, 'a #-prefixed selection must match too'); + }); + + test('declared and observed are reported together when both hold', () => { + const ev = m.nodeRegionEvidence(BOTH, 'be'); + assert.strictEqual(ev.declared, true); + assert.strictEqual(ev.observed, true); + }); + + // A picker, not a search box: #be must not pull in #be-bru repeaters. + test('a region matches exactly, not as a prefix', () => { + assert.strictEqual(m.regionFilterAccepts({ declared_regions: ['be-bru'] }, 'be'), false); + assert.strictEqual(m.regionFilterAccepts({ transported_scopes: ['#be-bru'] }, 'be'), false); + }); + + test('nodes with no evidence for the region drop out while it is picked', () => { + assert.strictEqual(m.regionFilterAccepts(OBSERVED_ONLY, 'nl'), false); + assert.strictEqual(m.regionFilterAccepts(ANSWERED_NONE, 'be'), false); + assert.strictEqual(m.regionFilterAccepts(NEVER_ASKED, 'be'), false); + assert.strictEqual(m.regionFilterAccepts(COMPANION, 'be'), false); + }); + + test('region counts: one entry per region, sorted, a node counted once', () => { + const counts = m.collectRegionCounts([DECLARED_ONLY, OBSERVED_ONLY, BOTH, ANSWERED_NONE, NEVER_ASKED, COMPANION]); + const plain = JSON.parse(JSON.stringify(counts)); + assert.deepStrictEqual(plain, [ + { region: 'be', total: 3, declared: 2, observed: 2 }, + { region: 'be-bru', total: 1, declared: 1, observed: 0 }, + { region: 'de', total: 1, declared: 0, observed: 1 }, + ]); + }); + + test('the picker offers All plus every region, with the selection marked', () => { + const counts = m.collectRegionCounts([DECLARED_ONLY, OBSERVED_ONLY, BOTH]); + const html = m.regionFilterOptionsHtml(counts, 'de'); + assert.ok(/') >= 0, 'no #be option: ' + html); + const selected = html.match(/
${statusMsg}
+
+ +
`; } })); @@ -378,6 +382,7 @@ grid.querySelectorAll('.mnc-btn').forEach(btn => { btn.addEventListener('click', (e) => { e.stopPropagation(); + if (btn.dataset.action === 'node') window.location.hash = '#/nodes/' + encodeURIComponent(btn.dataset.key); if (btn.dataset.action === 'health') loadHealth(btn.dataset.key); if (btn.dataset.action === 'packets') window.location.hash = '#/packets/' + btn.dataset.key; }); From a2ea18f7787c1cdbc783ec42ffabdde729e18fda Mon Sep 17 00:00:00 2001 From: Sylvain Rabot Date: Wed, 16 Sep 2026 09:02:13 +0200 Subject: [PATCH 043/151] perf(sqlite): swap modernc.org/sqlite for mattn/go-sqlite3, cross-built with zig (#1992) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Swaps the SQLite driver from `modernc.org/sqlite` (pure Go, SQLite 3.46.0) to `github.com/mattn/go-sqlite3` (cgo, bundled SQLite 3.53.4), and pays the resulting cross-compilation cost with `zig cc`. Draft because the riskiest part of this deletes rows — see [Please review this part first](#please-review-this-part-first) — and because three things remain unverified at the bottom. `modernc.org/sqlite` is a transpilation of the C amalgamation. This repo is read-heavy: `cmd/server` chunk-loads a graph at startup and fans out neighbour/topology/analytics queries per request, and it pays for that transpilation on exactly those paths. Head-to-head on the same 120k-transmission / 240k-observation database, running our own hot-path SQL under both drivers (Apple M4, `-count=5`, medians): | workload | modernc | mattn | | |---|---:|---:|---| | chunk load (`chunked_load.go` v3 join, 20k tx) | 449ms | 196ms | **2.3×** | | aggregate scan (240k-row join + `GROUP BY`) | 276ms | 137ms | **2.0×** | | 1500 prepared-statement lookups | 512ms | 403ms | **1.3×** | Allocations fall with it: 1.12M vs 1.64M allocs and 21MB vs 30MB on the chunk load. **Superseded by a production run.** @efiten measured both drivers on a real instance — 11,077,038 observations, 9.7GB database, 4-core arm64 — as server-only containers against the same live volume, one at a time, with round 2 reversing the order so the page cache favours the old driver: | | audit 7d | audit 24h | background fill (13 chunks) | start → /api/health | |---|---:|---:|---:|---:| | modernc, round 1 | 16.67s | 2.27s | 130.2s | 16.6s | | mattn, round 1 | 7.87s | 1.34s | 93.8s | 13.5s | | mattn, round 2 | 8.15s | 1.35s | 96.4s | 13.0s | | modernc, round 2 | 13.46s | 2.29s | 137.8s | 15.5s | Warm, the old driver improves to 13.46s on the 7d audit and still loses by ~1.8×. Chunk load is ~1.4×. `/api/nodes?limit=500` is 0.039s against 0.037s — nothing. **So the real gain is ~1.4–1.8× on the paths that matter, not 2–2.3×.** The shape the harness predicted holds — scans and joins gain, small lookups do not — which is more reassuring than the magnitude would have been. Quote these numbers. **The counterweight**, cold and native on that machine: a build goes from **52s to 163s**. An instance that builds its own image pays that per deploy. ## The build is cgo now, and one thing about that is a trap **`CGO_ENABLED=0` still builds.** mattn links a stub, and the binary dies on its first query with `go-sqlite3 requires cgo to work. This is a stub`. A green build is not evidence of anything here, which is why `AGENTS.md` now says so explicitly. `GOOS=linux go build` genuinely cannot cross-compile any more. A new root `Makefile` is the entry point. `make crossbuild` uses `zig cc -target {x86_64,aarch64}-linux-musl` and links static, so each artifact stays a single self-contained file and the `alpine:3.20` runtime no longer depends on the base image's libc at all. `-Wl,-s` is load-bearing: Go's own `-s -w` does not reach the musl objects zig links in, and without it the server binary is 19.8MB instead of 12.1MB. The Dockerfile keeps its single `$BUILDPLATFORM` builder — still no QEMU for compilation — and gains a checksum-pinned zig plus BuildKit cache mounts. The mounts are not a nicety: without them an image build recompiles the amalgamation from cold and takes over half an hour. ## Please review this part first `internal/dbschema/dedup_index.go` **deletes observation rows**. It is the one part of this change that can lose data, and it exists because the migration exposed a real bug rather than causing one. `stmtInsertObservation` resolves its `ON CONFLICT` against `idx_observations_dedup`, which `cmd/ingestor/db.go` only ever created inside the branch that creates the `observations` table for the first time. Any database whose table predates that branch never got one, so the UPSERT had no conflict target. modernc failed on the first insert; mattn fails at `OpenStore`. Same bug, found earlier. Creating the index unconditionally repairs it — but the index is what was supposed to prevent duplicates, so a database that never had it can already hold rows violating it. **`test-fixtures/e2e-fixture.db` in this repo holds one.** So duplicates are collapsed first. Refusing is not the safer option: without the index the ingestor cannot prepare its UPSERT, so it cannot start at all. Replaying that UPSERT faithfully is subtler than it looks, and a first cut of this got it wrong twice: - `COALESCE(excluded.x, x)` means the **incoming** value wins, so down a group in id order the survivor keeps the **last** non-NULL value. Taking the first silently discarded newer readings. - The UPSERT names exactly five columns (`snr`, `rssi`, `score`, `raw_hex`, `resolved_path`). Every other column must keep the surviving row's own value; merging those too invents history the ingestor would never have written. Merge, delete and `CREATE UNIQUE INDEX` now share one transaction. Split apart, a writer inserting a duplicate in the gap fails the index creation while leaving the deletions committed — rows destroyed and no index to show for it. Cost, measured on 2.4M synthetic rows holding 5 duplicates: **4.1s**, holding the write lock throughout, once, at ingestor startup before MQTT subscribe. Materialising the duplicate-group scan once rather than per column took that from 9.7s; the pathological case (400k of 600k rows duplicated) is 5.7s, slightly worse than the 4.2s it was before that change. ## Four more behavioural differences Full detail in `docs/sqlite-driver-migration.md`. Briefly: **Statement preparation is eager.** modernc's `newStmt` stored the SQL and compiled lazily; mattn calls `sqlite3_prepare_v2` inside `Prepare`, so SQL naming a missing table fails at *open*. 59 server tests failed on this alone, all fixtures with partial schemas. `OpenDB` keeps failing loudly (#1901; `main.go` gates on `dbschema.AssertReady` anyway) and the fixtures now declare what they are prepared against via `ensurePreparable`. This also exposed nine `nodes(pubkey …)` declarations across seven files, where production has only ever had `public_key` — lazy compilation had hidden the mismatch for as long as it existed. **`synchronous` silently dropped FULL → NORMAL.** mattn defaults it to NORMAL and executes the pragma unconditionally, where SQLite's own default (what modernc left alone) is FULL. In WAL mode that weakens durability under power loss. Pinned in `dbschema.WriterDSN`, which both writers now share — `cmd/migrate` kept a bare path at first and so quietly wrote at NORMAL, which is what a second copy of a DSN buys you. **The DSN dialects are mutually invisible.** modernc understood only `_pragma=name(value)`, mattn only `_`-prefixed parameters, and neither errors on the other's form — a driver-only rename would have dropped every pragma in silence. `_journal_mode=WAL` is also gone from the server's read handle: modernc ignored it, mattn honours it, and setting `journal_mode` on a read-only connection is a write. Dropping `_busy_timeout` with it costs nothing, since mattn already defaults to 5000ms — which means the read handle finally *gets* the busy timeout it had silently lacked. **`mode=ro` survives for a non-obvious reason.** mattn always passes `READWRITE|CREATE` and its amalgamation has `SQLITE_USE_URI=0`; what makes the URI work is its C wrapper ORing `SQLITE_OPEN_URI` in. So the #1283/#1289 invariant holds with no build flags — but it depends on the `file:` prefix. `cmd/decrypt` had been building its DSN without one, so its `mode=ro` had never applied and a missing path was created read-write. Fixed in passing; never a migration regression. ## What did not change No modernc-specific API was in use: no `RegisterFunction`, no `*sqlite.Conn`, no `sqlite/lib` error constants, no `sql.Register`. No `time.Time` is ever bound as a query argument, so driver time handling is not in play. Both drivers convert declared `DATE`/`DATETIME`/`TIMESTAMP` columns to `time.Time`, so `/api/dropped-packets` keeps emitting `dropped_at` as RFC3339 — an earlier draft "fixed" that with a `CAST` and would have been the regression. ## Tests and CI New regression tests, each written because something got through without it: - `TestEnsureObservationsDedupIndexKeepsLatestValues` — the merge ordering. The original test used complementary NULLs, which passes whichever direction you pick, which is why the bug survived it. - `TestCollapseDuplicatesAndIndexIsAtomic` — a failed index creation must roll the deletions back. - `TestOpenStorePragmas` / `TestWriterDSNPragmas` — every writer pragma, read back through the store's own connection. A separate `sqlite3` session or the startup log line would prove nothing. - `TestOpenDBRefusesMissingDatabase` — the read-only invariant, which now rests on a detail of the driver's C wrapper. - `TestEnsurePreparableMatchesPrepareStatements` — fails when a new prepared statement outgrows the fixture helper. CI gains test execution for `cmd/migrate` and `internal/dbschema`, which had none and both open the database. A PR-time two-arch build plus an arm64 QEMU smoke gate is new: the GHCR push is push/tag-only, so without it nothing on a PR would exercise zig, static musl linking or arm64, and the first signal would arrive on master. `cache-dependency-path` widens from 2 of the 5 tracked `go.sum` files to all of them. `make test` passes across all 14 modules, `cmd/server` also under `-race -count=2` with no failures and no races. `gofmt` and `go vet` clean. Release-routing and Dockerfile COPY-invariant gates pass. ## Verified by running - All 8 cross-builds static and correct-architecture; both arches of the container image built, exported and run under QEMU, serving `/api/health` and `/api/nodes` against a 2.9M-observation production snapshot. - The `migrate` binary repairing that snapshot's duplicate on bare Alpine. - `CGO_ENABLED=0` producing a binary that builds and then fails on first query. ## Not verified - ~~The 2–2.3× figures come from a standalone harness, not this load under the old driver.~~ **Closed** by @efiten's production run above, which also corrected the multiplier. - SQLite 3.46.0 → 3.53.4 query-planner differences on queries with no total `ORDER BY`. - Sustained live ingest through the new writer DSN, and the duplicate collapse against a database an ingestor is actively writing to. Verified against a static snapshot only, and the collapse is measured at 4.1s on 2.4M synthetic rows with 5 duplicates — well short of an 11M-row instance. @efiten has offered a staging instance taking real MQTT traffic; **this is the item to close before the PR leaves draft.** An earlier revision of this branch shipped the dedup merge in the wrong direction with a green test suite, and review then found three more things in the same file: the repair gated on an error string, a non-atomic TEMP table drop aimed at the wrong connection, and a deletion whose only record was a row count. All fixed in ac7e8d38. Passing tests did not establish safety here, which is why the deletion path wanted a second pair of eyes rather than a rubber stamp. --- .github/workflows/deploy.yml | 104 +++- .gitignore | 4 + .squad/decisions/decisions.md | 4 +- AGENTS.md | 4 +- Dockerfile | 91 ++- Makefile | 139 +++++ README.md | 19 + cmd/decrypt/README.md | 2 +- cmd/decrypt/go.mod | 14 +- cmd/decrypt/go.sum | 45 +- cmd/decrypt/main.go | 4 +- cmd/ingestor/README.md | 2 +- cmd/ingestor/db.go | 6 +- cmd/ingestor/db_test.go | 8 +- cmd/ingestor/go.mod | 15 +- cmd/ingestor/go.sum | 45 +- cmd/ingestor/issue1283_test.go | 4 +- cmd/ingestor/pragma_parity_test.go | 55 ++ cmd/ingestor/prune_chunked_test.go | 2 +- cmd/migrate/go.mod | 14 +- cmd/migrate/go.sum | 45 +- cmd/migrate/main.go | 7 +- cmd/migrate/main_test.go | 4 +- cmd/server/bounded_load_test.go | 26 +- cmd/server/chunked_load_id_zero_test.go | 6 +- cmd/server/chunked_load_oldest_test.go | 6 +- cmd/server/coverage_test.go | 4 +- cmd/server/db.go | 23 +- cmd/server/db_test.go | 34 +- cmd/server/go.mod | 14 +- cmd/server/go.sum | 45 +- cmd/server/hexgrid.go | 4 +- cmd/server/hot_startup_test.go | 8 +- cmd/server/issue1690_cold_load_test.go | 2 +- cmd/server/issue1809_bg_load_race_test.go | 6 +- cmd/server/issue673_test.go | 2 +- cmd/server/load_graph_invariant_test.go | 6 +- cmd/server/loadchunk_ambiguous_prefix_test.go | 6 +- .../loadchunk_pathjson_fallback_test.go | 6 +- .../loadchunk_resolved_path_1558_test.go | 8 +- cmd/server/memlimit.go | 15 + cmd/server/memory.go | 19 +- cmd/server/multibyte_capability_test.go | 6 +- cmd/server/neighbor_api_test.go | 21 +- cmd/server/neighbor_persist_legacy_test.go | 6 +- cmd/server/neighbor_recomputer_test.go | 21 +- cmd/server/node_reach_bench_test.go | 10 +- cmd/server/node_reach_endpoint_test.go | 6 +- cmd/server/node_reach_test.go | 6 +- cmd/server/perf_io.go | 2 +- cmd/server/preparable_schema_test.go | 77 +++ cmd/server/readonly_invariant_test.go | 45 +- cmd/server/resolved_index_test.go | 4 +- cmd/server/scope_audit_test.go | 2 +- cmd/server/store_memory_diagnostics_test.go | 6 +- cmd/server/topology_dedup_test.go | 14 +- docs/client-rx-coverage.md | 3 +- docs/sqlite-driver-migration.md | 245 ++++++++ internal/dbschema/dbschema.go | 23 +- internal/dbschema/dbschema_test.go | 4 +- internal/dbschema/dedup_index.go | 314 ++++++++++ internal/dbschema/dedup_index_test.go | 540 ++++++++++++++++++ internal/dbschema/go.mod | 13 +- internal/dbschema/go.sum | 23 +- internal/dbschema/writer_dsn.go | 35 ++ internal/dbschema/writer_dsn_test.go | 43 ++ scripts/allmod.sh | 20 + scripts/migrate-fixture-hashes.go | 4 +- test-issue-1956-release-routing.js | 37 +- 69 files changed, 1982 insertions(+), 425 deletions(-) create mode 100644 Makefile create mode 100644 cmd/ingestor/pragma_parity_test.go create mode 100644 cmd/server/preparable_schema_test.go create mode 100644 docs/sqlite-driver-migration.md create mode 100644 internal/dbschema/dedup_index.go create mode 100644 internal/dbschema/dedup_index_test.go create mode 100644 internal/dbschema/writer_dsn.go create mode 100644 internal/dbschema/writer_dsn_test.go create mode 100644 scripts/allmod.sh diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 8b42e2d3f..737fee938 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -124,7 +124,11 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' - cache-dependency-path: cmd/ingestor/go.sum + # All modules: a -race build of a cgo package is the slowest thing in + # this pipeline, and setup-go caches ~/.cache/go-build as well as the + # module cache, so a key that misses on an unrelated module hurts most + # here. Same reasoning as go-test below. + cache-dependency-path: "**/go.sum" - name: go test -race run: | @@ -153,9 +157,10 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' - cache-dependency-path: | - cmd/server/go.sum - cmd/ingestor/go.sum + # All modules, not just two: the cgo SQLite build is expensive to + # redo, and setup-go caches ~/.cache/go-build as well as the module + # cache, so a key that misses on an unrelated module hurts. + cache-dependency-path: "**/go.sum" - name: Enforce gofmt + go vet (issue #1859) run: | @@ -202,10 +207,22 @@ jobs: go test ./... echo "--- Channel library tests passed ---" cd ../../cmd/decrypt - CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . + go build -ldflags="-s -w" -o corescope-decrypt . go test ./... echo "--- Decrypt CLI tests passed ---" + - name: Test migrate CLI + dbschema + run: | + set -e -o pipefail + # Neither module had CI test execution before the mattn/go-sqlite3 + # migration; both open the database, and dbschema owns the migrations. + cd internal/dbschema + go test ./... + cd ../../cmd/migrate + go build -o /dev/null . + go test ./... + echo "--- migrate + dbschema tests passed ---" + - name: Verify Dockerfile COPY invariants (issue #1316) run: bash scripts/check-dockerfile-internal-pkgs.sh @@ -329,7 +346,7 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' - cache-dependency-path: cmd/server/go.sum + cache-dependency-path: "**/go.sum" - name: Build Go server run: | @@ -682,14 +699,58 @@ jobs: docker compose -f "$STAGING_COMPOSE_FILE" -p corescope-staging build "$STAGING_SERVICE" echo "Built Go staging image ✅" + # Needed by the PR two-arch gate below as well as the GHCR push, so the + # condition covers both rather than push/tag only. - name: Set up Docker Buildx - if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} uses: docker/setup-buildx-action@v3 - - name: Set up QEMU (arm64 runtime stage) - if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} + - name: Set up QEMU (arm64 runtime stage + arm64 smoke) + if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} uses: docker/setup-qemu-action@v3 + # Since the SQLite driver became cgo, the container build depends on a zig + # cross-toolchain and static musl linking. The GHCR step below is + # push/tag-only, so without this a PR never exercises either, and the + # first signal would arrive on master. Builds both architectures and + # actually runs the arm64 binaries under QEMU. + - name: Two-arch build (no publish) + if: ${{ github.event_name == 'pull_request' }} + uses: docker/build-push-action@v6 + with: + context: . + push: false + platforms: linux/amd64,linux/arm64 + tags: corescope:pr-${{ github.event.pull_request.number }} + build-args: | + APP_VERSION=${{ steps.meta.outputs.app_version }} + GIT_COMMIT=${{ steps.meta.outputs.git_commit }} + BUILD_TIME=${{ steps.meta.outputs.build_time }} + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Smoke the arm64 binaries under QEMU + if: ${{ github.event_name == 'pull_request' }} + run: | + set -e + # Export the arm64 binaries so the runner's own `file` can confirm + # they are static, then execute one under QEMU. Cheap: the build above + # already populated the layer cache. + docker buildx build . --platform linux/arm64 \ + --build-arg APP_VERSION=smoke --cache-from type=gha \ + --output type=local,dest=./arm64-image + for b in corescope-server corescope-ingestor corescope-decrypt; do + file "./arm64-image/app/$b" | tee /dev/stderr | grep -q 'statically linked' \ + || { echo "::error::$b is not statically linked"; exit 1; } + file "./arm64-image/app/$b" | grep -q 'ARM aarch64' \ + || { echo "::error::$b is not an arm64 binary"; exit 1; } + done + docker run --rm --platform linux/arm64 \ + -v "$PWD/arm64-image/app:/bin-under-test:ro" alpine:3.20 \ + /bin-under-test/corescope-decrypt --version + rm -rf ./arm64-image + echo "arm64 static binaries run under QEMU ✅" + - name: Log in to GHCR if: ${{ github.event_name == 'push' || startsWith(github.ref, 'refs/tags/v') }} uses: docker/login-action@v3 @@ -745,16 +806,37 @@ jobs: uses: actions/setup-go@v6 with: go-version: '1.27' + cache-dependency-path: | + cmd/decrypt/go.sum + internal/channel/go.sum + + # The SQLite driver is cgo now (github.com/mattn/go-sqlite3), so the Go + # toolchain alone cannot cross-compile these. zig cc is the C compiler + # that can target both architectures; musl makes the result static. + - name: Set up Zig + uses: mlugg/setup-zig@v2 + with: + version: 0.16.0 - name: Build corescope-decrypt (static, linux/amd64) run: | cd cmd/decrypt - CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . + CGO_ENABLED=1 GOOS=linux GOARCH=amd64 CC="zig cc -target x86_64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-amd64 . - name: Build corescope-decrypt (static, linux/arm64) run: | cd cmd/decrypt - CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-s -w -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + CGO_ENABLED=1 GOOS=linux GOARCH=arm64 CC="zig cc -target aarch64-linux-musl" go build -trimpath -tags netgo,osusergo,sqlite_omit_load_extension -ldflags="-s -w -extldflags '-static -Wl,-s' -X main.version=${{ github.ref_name }}" -o ../../corescope-decrypt-linux-arm64 . + + - name: Verify release binaries are static and runnable + run: | + set -e + for f in corescope-decrypt-linux-amd64 corescope-decrypt-linux-arm64; do + file "$f" | grep -q 'statically linked' || { echo "::error::$f is not statically linked"; exit 1; } + done + # amd64 runs natively on the runner; arm64 is only checked structurally + # here (the container image job exercises arm64 under QEMU). + ./corescope-decrypt-linux-amd64 --version - name: Upload release assets # Standard releases upload both assets to a draft before publishing. diff --git a/.gitignore b/.gitignore index 548c25d66..33def5772 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,10 @@ config-lincomatic.json theme.json firmware/ coverage/ +dist/ +# internal/prunequeue writes these next to the database; the server tests leave +# them in cmd/server/. Runtime queue state, never repo content. +prune-requests/ public-instrumented/ .nyc_output/ .setup-state diff --git a/.squad/decisions/decisions.md b/.squad/decisions/decisions.md index d5aef0851..37c8bb32f 100644 --- a/.squad/decisions/decisions.md +++ b/.squad/decisions/decisions.md @@ -211,7 +211,9 @@ Reconciled `manage.sh` and `docker-compose.yml` Docker volume names: Standalone Go MQTT ingestor service. Separate process from Node.js web server that handles MQTT packet ingestion + writes to shared SQLite DB. **Architecture:** -- Single binary, no CGO (uses `modernc.org/sqlite` pure Go) +- Single statically-linked binary. Was CGO-free via `modernc.org/sqlite`; now cgo, + using `github.com/mattn/go-sqlite3` for read performance, cross-compiled with + `zig cc` against musl so the artifact stays a single self-contained file - Reads same `config.json` (mqttSources array) - Shares SQLite DB with Node.js (WAL mode for concurrent access) - Format 1 (raw packet) MQTT only — companion bridge stays in Node.js diff --git a/AGENTS.md b/AGENTS.md index 6fc1db4bc..fb25c6d55 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,7 +90,9 @@ Every change must consider performance impact BEFORE implementation. This codeba No proof = no merge. ### 1. No commit without tests -Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`. For frontend: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. + +The SQLite driver (`github.com/mattn/go-sqlite3`) is cgo. **`CGO_ENABLED=0` still *builds*** — that is the trap. It links a stub, and the binary dies on the first query with `go-sqlite3 requires cgo to work. This is a stub`, so a green build proves nothing. A plain `GOOS=linux go build` likewise cannot cross-compile it. Use `make build` / `make crossbuild` — the latter needs [`zig`](https://ziglang.org/download/) as the cross C compiler and produces static musl binaries. Never re-add `CGO_ENABLED=0`. ### 2. No commit without browser validation After pushing, verify the change works in an actual browser. Use `browser profile=openclaw` against the running instance. Take a screenshot if the change is visual. If you can't validate it, say so — don't claim it works. diff --git a/Dockerfile b/Dockerfile index 4e70794cf..3042081da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,13 @@ -# Build stage always runs natively on the builder's arch ($BUILDPLATFORM) -# and cross-compiles to $TARGETOS/$TARGETARCH via Go toolchain. No QEMU. +# syntax=docker/dockerfile:1 +# Build stage always runs natively on the builder's arch ($BUILDPLATFORM) and +# cross-compiles to $TARGETOS/$TARGETARCH. No QEMU for compilation. +# +# The SQLite driver is github.com/mattn/go-sqlite3, which is cgo, so the Go +# toolchain alone can no longer cross-compile this: it needs a C compiler that +# can target the other architecture. `zig cc` is that compiler. Targeting musl +# makes the result fully static (see -extldflags below), so the runtime stage +# has no libc dependency on the base image at all. +# # BUILDPLATFORM is auto-set by buildx; default to linux/amd64 so plain # `docker build` (without buildx) doesn't fail on an empty platform string. ARG BUILDPLATFORM=linux/amd64 @@ -12,7 +20,40 @@ ARG BUILD_TIME=unknown ARG TARGETOS ARG TARGETARCH -# Build server (pure-Go sqlite — no CGO needed, cross-compiles cleanly) +# Keep these in step with the Makefile: netgo/osusergo preserve the pure-Go +# resolver and user lookup the binaries had under CGO_ENABLED=0, and +# sqlite_omit_load_extension drops the dlopen path so -static links cleanly. +ENV GO_BUILD_TAGS=netgo,osusergo,sqlite_omit_load_extension \ + ZIG_GLOBAL_CACHE_DIR=/tmp/zig-cache + +# Pinned zig, checksum-verified. Arch comes from `uname -m` rather than +# TARGETARCH because this stage is pinned to BUILDPLATFORM — it has to work when +# someone builds on an arm64 machine too. +ARG ZIG_VERSION=0.16.0 +RUN apk add --no-cache curl xz && \ + case "$(uname -m)" in \ + x86_64) ZA=x86_64; ZSHA=70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 ;; \ + aarch64) ZA=aarch64; ZSHA=ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17 ;; \ + *) echo "unsupported builder arch $(uname -m)" >&2; exit 1 ;; \ + esac && \ + curl -sSLo /tmp/zig.tar.xz "https://ziglang.org/download/${ZIG_VERSION}/zig-${ZA}-linux-${ZIG_VERSION}.tar.xz" && \ + echo "${ZSHA} /tmp/zig.tar.xz" | sha256sum -c - && \ + mkdir -p /opt/zig && tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 && \ + ln -s /opt/zig/zig /usr/local/bin/zig && rm /tmp/zig.tar.xz && \ + zig version + +# zigcc resolves TARGETARCH to a zig target triple once, so the three build +# steps below stay readable and cannot disagree with each other. +RUN printf '%s\n' '#!/bin/sh' \ + 'case "$TARGETARCH" in' \ + ' amd64) t=x86_64-linux-musl ;;' \ + ' arm64) t=aarch64-linux-musl ;;' \ + ' *) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;;' \ + 'esac' \ + 'exec zig cc -target "$t" "$@"' > /usr/local/bin/zigcc && chmod +x /usr/local/bin/zigcc +ENV CC=zigcc CGO_ENABLED=1 + +# Build server WORKDIR /build/server COPY cmd/server/go.mod cmd/server/go.sum ./ COPY internal/geofilter/ ../../internal/geofilter/ @@ -24,10 +65,18 @@ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ COPY internal/lora/ ../../internal/lora/ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/server/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -ldflags "-X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" -o /corescope-server . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.Version=${APP_VERSION} -X main.Commit=${GIT_COMMIT} -X main.BuildTime=${BUILD_TIME}" \ + -o /corescope-server . # Build ingestor WORKDIR /build/ingestor @@ -40,19 +89,35 @@ COPY internal/dbschema/ ../../internal/dbschema/ COPY internal/prunequeue/ ../../internal/prunequeue/ COPY internal/perfio/ ../../internal/perfio/ COPY internal/mbcapqueue/ ../../internal/mbcapqueue/ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/ingestor/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -o /corescope-ingestor . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s'" \ + -o /corescope-ingestor . # Build decrypt CLI WORKDIR /build/decrypt COPY cmd/decrypt/go.mod cmd/decrypt/go.sum ./ COPY internal/channel/ ../../internal/channel/ -RUN go mod download +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + go mod download COPY cmd/decrypt/ ./ -RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ - go build -ldflags="-s -w" -o /corescope-decrypt . +RUN --mount=type=cache,target=/root/.cache/go-build \ + --mount=type=cache,target=/go/pkg/mod \ + --mount=type=cache,target=/tmp/zig-cache \ + GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + go build -trimpath -tags ${GO_BUILD_TAGS} \ + -ldflags "-s -w -extldflags '-static -Wl,-s' -X main.version=${APP_VERSION}" \ + -o /corescope-decrypt . # Runtime image FROM alpine:3.20 @@ -61,7 +126,7 @@ RUN apk add --no-cache mosquitto mosquitto-clients supervisor caddy wget WORKDIR /app -# Go binaries +# Go binaries (statically linked; they do not use this image's libc) COPY --from=builder /corescope-server /corescope-ingestor /corescope-decrypt /app/ # Frontend assets + config diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..65c25df4d --- /dev/null +++ b/Makefile @@ -0,0 +1,139 @@ +# corescope build entry point. +# +# This repo is 14 Go modules wired with `replace ../../internal/*` and no +# go.work, so every recipe changes into its module. The build flags live here +# because the SQLite driver is cgo now (github.com/mattn/go-sqlite3) and the +# flags stopped being something you can safely retype at each call site. +# +# Cross-compilation uses `zig cc` as the C toolchain, targeting musl so the +# binaries are fully static and run on the alpine runtime image (or scratch) +# with no libc dependency at all. +# +# Quick reference: +# make build # all four binaries for the host +# make build-server # just one +# make crossbuild # linux/amd64 + linux/arm64, static, into dist/ +# make test / vet / fmt-check # across all modules +# make docker-build # multi-arch image via buildx + +GO ?= $(shell command -v go) +GIT_VERSION ?= $(shell git describe --tags --match "v*" 2>/dev/null || echo unknown) +GIT_COMMIT ?= $(shell git rev-parse --short HEAD 2>/dev/null || echo unknown) +BUILD_TIME ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) + +GOENV_GOOS := $(shell $(GO) env GOOS) +GOENV_GOARCH := $(shell $(GO) env GOARCH) +GOOS ?= $(GOENV_GOOS) +GOARCH ?= $(GOENV_GOARCH) + +# Exported: a bare make variable never reaches the go toolchain. +export CGO_ENABLED ?= 1 + +# netgo/osusergo keep the pure-Go resolver and user lookup the binaries had +# under CGO_ENABLED=0, so enabling cgo for SQLite does not quietly switch DNS +# resolution to the C resolver. sqlite_omit_load_extension drops the dlopen +# path, which is what lets -extldflags -static link cleanly. -Wl,-s on the +# cross builds strips the musl objects zig links in: Go's own -s -w does not +# reach them, and they account for well over half the binary. +GO_BUILD_TAGS ?= netgo,osusergo,sqlite_omit_load_extension +GO_BUILD_FLAGS ?= -trimpath +GO_LDFLAGS_OPTIMS ?= -s -w + +CMDS := server ingestor decrypt migrate +DIST := dist + +# Per-binary version stamping. Only these two read build metadata; ingestor and +# migrate take none. Kept as make variables so crossbuild and the host build +# cannot drift apart. +LDFLAGS_server := -X main.Version=$(GIT_VERSION) -X main.Commit=$(GIT_COMMIT) -X main.BuildTime=$(BUILD_TIME) +LDFLAGS_ingestor := +LDFLAGS_decrypt := -X main.version=$(GIT_VERSION) +LDFLAGS_migrate := + +# zig target triples for the platforms we ship. musl, so the result is static. +ZIG_TARGET_linux_amd64 := x86_64-linux-musl +ZIG_TARGET_linux_arm64 := aarch64-linux-musl +CROSS_PLATFORMS := linux/amd64 linux/arm64 + +DOCKER_IMAGE ?= ghcr.io/kpa-clawbot/corescope +DOCKER_TAG ?= $(GIT_VERSION) +DOCKER_PLATFORMS ?= linux/amd64,linux/arm64 + +.PHONY: all build crossbuild test vet fmt-check tidy clean docker-build docker-push help + +all: build + +help: + @echo "targets: build crossbuild test vet fmt-check tidy clean docker-build docker-push" + @echo " build-{$(shell echo $(CMDS) | tr ' ' ',')}" + +# -- host builds --------------------------------------------------------------- + +build: $(addprefix build-,$(CMDS)) + +build-%: + @mkdir -p $(DIST) + cd cmd/$* && GOOS=$(GOOS) GOARCH=$(GOARCH) $(GO) build \ + -tags $(GO_BUILD_TAGS) $(GO_BUILD_FLAGS) \ + -ldflags "$(GO_LDFLAGS_OPTIMS) $(LDFLAGS_$*)" \ + -o ../../$(DIST)/corescope-$* . + +# -- cross builds -------------------------------------------------------------- +# +# The host build above deliberately leaves CC alone: native builds, `go vet` and +# `-race` must use the host compiler. Only these recipes hand the build to zig. + +crossbuild: $(foreach p,$(CROSS_PLATFORMS),$(foreach c,$(CMDS),crossbuild-$(c)-$(subst /,-,$(p)))) + +define CROSSBUILD_RULE +.PHONY: crossbuild-$(2)-$(3)-$(4) +crossbuild-$(2)-$(3)-$(4): + @mkdir -p $$(DIST) + @command -v zig >/dev/null || { echo "zig not found: needed to cross-compile cgo. See https://ziglang.org/download/"; exit 1; } + cd cmd/$(2) && CGO_ENABLED=1 GOOS=$(3) GOARCH=$(4) CC="zig cc -target $(1)" \ + $$(GO) build -tags $$(GO_BUILD_TAGS) $$(GO_BUILD_FLAGS) \ + -ldflags '$$(GO_LDFLAGS_OPTIMS) -extldflags "-static -Wl,-s" $$(LDFLAGS_$(2))' \ + -o ../../$$(DIST)/corescope-$(2)-$(3)-$(4) . +endef + +$(foreach c,$(CMDS),\ + $(eval $(call CROSSBUILD_RULE,$(ZIG_TARGET_linux_amd64),$(c),linux,amd64))\ + $(eval $(call CROSSBUILD_RULE,$(ZIG_TARGET_linux_arm64),$(c),linux,arm64))) + +# -- checks -------------------------------------------------------------------- + +test: + bash scripts/allmod.sh test ./... + +vet: + bash scripts/allmod.sh vet ./... + +fmt-check: + @unformatted=$$(gofmt -l $$(git ls-files '*.go' | grep -vi 'Dockerfile.go')); \ + if [ -n "$$unformatted" ]; then echo "gofmt required on:"; echo "$$unformatted"; exit 1; fi; \ + echo "gofmt: clean" + +tidy: + bash scripts/allmod.sh mod tidy + +clean: + rm -rf $(DIST) + +# -- docker -------------------------------------------------------------------- + +docker-build: + docker buildx build . \ + --platform=$(DOCKER_PLATFORMS) \ + --build-arg=APP_VERSION=$(GIT_VERSION) \ + --build-arg=GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg=BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE):$(DOCKER_TAG) + +docker-push: DOCKER_PUSH_FLAG := --push +docker-push: + docker buildx build . \ + --platform=$(DOCKER_PLATFORMS) \ + --build-arg=APP_VERSION=$(GIT_VERSION) \ + --build-arg=GIT_COMMIT=$(GIT_COMMIT) \ + --build-arg=BUILD_TIME=$(BUILD_TIME) \ + -t $(DOCKER_IMAGE):$(DOCKER_TAG) $(DOCKER_PUSH_FLAG) diff --git a/README.md b/README.md index 5ba0b3c46..b06d54a43 100644 --- a/README.md +++ b/README.md @@ -234,6 +234,22 @@ corescope/ ## For Developers +### Building + +```bash +make build # all four binaries for your machine, into dist/ +make build-server # just one +make crossbuild # static linux/amd64 + linux/arm64 binaries +``` + +The SQLite driver is [`mattn/go-sqlite3`](https://github.com/mattn/go-sqlite3), which +is cgo, so a plain `GOOS=linux go build` from a Mac will not work: cross-compiling +needs a C compiler that can target the other platform. `make crossbuild` uses +[`zig`](https://ziglang.org/download/) as that compiler (install it and it just +works) and links statically against musl, so the result is one self-contained file +that runs on Alpine or scratch. The container build does the same thing — see +`Dockerfile`. + ### Test Suite **380 Go tests** covering the backend, plus **150+ Node.js tests** for the frontend and legacy logic, plus **49 Playwright E2E tests** for browser validation. @@ -243,6 +259,9 @@ corescope/ cd cmd/server && go test ./... -v cd cmd/ingestor && go test ./... -v +# Or across all 14 modules at once +make test + # Node.js frontend + integration tests npm test diff --git a/cmd/decrypt/README.md b/cmd/decrypt/README.md index 3aa70a482..d6bf5ee78 100644 --- a/cmd/decrypt/README.md +++ b/cmd/decrypt/README.md @@ -37,7 +37,7 @@ chmod +x corescope-decrypt-linux-amd64 ```bash cd cmd/decrypt -CGO_ENABLED=0 go build -ldflags="-s -w" -o corescope-decrypt . +go build -ldflags="-s -w" -o corescope-decrypt . # cgo: the SQLite driver needs it ``` The binary is statically linked — no dependencies, runs on any Linux. diff --git a/cmd/decrypt/go.mod b/cmd/decrypt/go.mod index 185a76aee..226dbcf0e 100644 --- a/cmd/decrypt/go.mod +++ b/cmd/decrypt/go.mod @@ -3,20 +3,8 @@ module github.com/corescope/decrypt go 1.22 require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/channel v0.0.0 - modernc.org/sqlite v1.34.5 -) - -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect ) replace github.com/meshcore-analyzer/channel => ../../internal/channel diff --git a/cmd/decrypt/go.sum b/cmd/decrypt/go.sum index 5424fe41d..75bf6eb53 100644 --- a/cmd/decrypt/go.sum +++ b/cmd/decrypt/go.sum @@ -1,43 +1,2 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/cmd/decrypt/main.go b/cmd/decrypt/main.go index 9a19ca69d..8c897d6ff 100644 --- a/cmd/decrypt/main.go +++ b/cmd/decrypt/main.go @@ -18,8 +18,8 @@ import ( "strings" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/channel" - _ "modernc.org/sqlite" ) // Version info (set via ldflags). @@ -121,7 +121,7 @@ LIMITATIONS key := channel.DeriveKey(ch) chHash := channel.ChannelHash(key) - db, err := sql.Open("sqlite", *dbPath+"?mode=ro") + db, err := sql.Open("sqlite3", "file:"+*dbPath+"?mode=ro") if err != nil { log.Fatalf("Failed to open database: %v", err) } diff --git a/cmd/ingestor/README.md b/cmd/ingestor/README.md index 5cf5536f0..0ad2986b3 100644 --- a/cmd/ingestor/README.md +++ b/cmd/ingestor/README.md @@ -12,7 +12,7 @@ MQTT Broker(s) → Go Ingestor → SQLite DB ← Node.js Web Server ``` - **Single static binary** — no runtime dependencies, no CGO -- **SQLite** via `modernc.org/sqlite` (pure Go) +- **SQLite** via `github.com/mattn/go-sqlite3` (cgo; cross-compiled with `zig cc`, see the root `Makefile`) - **MQTT** via `github.com/eclipse/paho.mqtt.golang` - Runs **alongside** the Node.js server — they share the DB file - Does NOT serve HTTP/WebSocket — that stays in Node.js diff --git a/cmd/ingestor/db.go b/cmd/ingestor/db.go index facd529af..81af4e080 100644 --- a/cmd/ingestor/db.go +++ b/cmd/ingestor/db.go @@ -14,9 +14,9 @@ import ( "sync/atomic" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/packetpath" - _ "modernc.org/sqlite" ) // DBStats tracks operational metrics for the ingestor database. @@ -130,7 +130,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) return nil, fmt.Errorf("creating data dir: %w", err) } - db, err := sql.Open("sqlite", dbPath+"?_pragma=auto_vacuum(INCREMENTAL)&_pragma=journal_mode(WAL)&_pragma=foreign_keys(ON)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", dbschema.WriterDSN(dbPath)) if err != nil { return nil, fmt.Errorf("opening db: %w", err) } @@ -141,7 +141,7 @@ func OpenStoreWithInterval(dbPath string, sampleIntervalSec int) (*Store, error) db.SetMaxOpenConns(1) db.SetMaxIdleConns(1) - log.Printf("SQLite config: busy_timeout=5000ms, max_open_conns=1, max_idle_conns=1, journal=WAL") + log.Printf("SQLite config: busy_timeout=5000ms, max_open_conns=1, max_idle_conns=1, journal=WAL, synchronous=FULL") if err := applySchema(db); err != nil { return nil, fmt.Errorf("applying schema: %w", err) diff --git a/cmd/ingestor/db_test.go b/cmd/ingestor/db_test.go index 1cea8a57e..9cd4fbfef 100644 --- a/cmd/ingestor/db_test.go +++ b/cmd/ingestor/db_test.go @@ -1627,7 +1627,7 @@ func TestObsTimestampIndexMigration(t *testing.T) { // Build a bare-bones DB that mimics an old installation: // observations table exists but idx_observations_timestamp does NOT. - db, err := sql.Open("sqlite", path) + db, err := sql.Open("sqlite3", path) if err != nil { t.Fatal(err) } @@ -2676,7 +2676,7 @@ func TestCleanupLegacyNullHashTimestamp(t *testing.T) { path := tempDBPath(t) // Create a bare-bones DB with legacy bad data - db, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", path+"?_journal_mode=WAL&_busy_timeout=5000") if err != nil { t.Fatal(err) } @@ -2806,7 +2806,7 @@ func TestBackfillPathJSONAsync(t *testing.T) { dbPath := filepath.Join(dir, "async_test.db") // Bootstrap schema manually so we can insert test data BEFORE OpenStore - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") if err != nil { t.Fatal(err) } @@ -2995,7 +2995,7 @@ func TestBackfillPathJSONAsync_BracketRowsTerminate(t *testing.T) { // Bootstrap a minimal schema directly so we can seed pre-existing '[]' rows // before OpenStore runs. - db, err := sql.Open("sqlite", dbPath+"?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)") + db, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL&_busy_timeout=5000") if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/go.mod b/cmd/ingestor/go.mod index 729c8e9d7..1d5cc637b 100644 --- a/cmd/ingestor/go.mod +++ b/cmd/ingestor/go.mod @@ -6,7 +6,6 @@ require ( github.com/eclipse/paho.mqtt.golang v1.5.0 github.com/meshcore-analyzer/geofilter v0.0.0 github.com/meshcore-analyzer/sigvalidate v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter @@ -30,24 +29,18 @@ require github.com/meshcore-analyzer/dbschema v0.0.0 replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect golang.org/x/net v0.27.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect ) require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue -require github.com/meshcore-analyzer/mbcapqueue v0.0.0 +require ( + github.com/mattn/go-sqlite3 v1.14.52 + github.com/meshcore-analyzer/mbcapqueue v0.0.0 +) replace github.com/meshcore-analyzer/mbcapqueue => ../../internal/mbcapqueue diff --git a/cmd/ingestor/go.sum b/cmd/ingestor/go.sum index cc13ef47f..554a48272 100644 --- a/cmd/ingestor/go.sum +++ b/cmd/ingestor/go.sum @@ -1,51 +1,10 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/cmd/ingestor/issue1283_test.go b/cmd/ingestor/issue1283_test.go index f19936bae..4cca01641 100644 --- a/cmd/ingestor/issue1283_test.go +++ b/cmd/ingestor/issue1283_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestIngestorPruneOldPackets enforces #1283: the writer for @@ -64,7 +64,7 @@ func TestIngestorVacuumOnStartupMigratesNONEtoINCREMENTAL(t *testing.T) { path := filepath.Join(dir, "vac.db") // Create a NONE-auto_vacuum DB (simulates an older deployment). - seed, err := sql.Open("sqlite", path+"?_pragma=journal_mode(WAL)") + seed, err := sql.Open("sqlite3", path+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } diff --git a/cmd/ingestor/pragma_parity_test.go b/cmd/ingestor/pragma_parity_test.go new file mode 100644 index 000000000..e90e4d97a --- /dev/null +++ b/cmd/ingestor/pragma_parity_test.go @@ -0,0 +1,55 @@ +package main + +import ( + "path/filepath" + "testing" +) + +// TestOpenStorePragmas pins the pragmas the writer connection runs with. +// +// These were silently at risk during the modernc.org/sqlite → +// github.com/mattn/go-sqlite3 migration in two different ways: +// +// - The DSN dialect changed. modernc understood only `_pragma=name(value)`; +// mattn understands only `_`-prefixed parameters and ignores the other +// form outright. A driver-only rename would have dropped every pragma on +// the floor with no error anywhere. +// +// - mattn runs `PRAGMA synchronous = NORMAL` unconditionally, defaulting the +// mode itself to NORMAL rather than leaving SQLite's own FULL in place. So +// the writer would have quietly moved from FULL (2) to NORMAL (1), +// weakening durability under power loss. `_synchronous=FULL` in the DSN is +// what holds it — this test is what notices if it is ever dropped. +// +// Read through the store's own connection on purpose: a separate connection, or +// the log line in OpenStoreWithInterval, would prove nothing about what the +// writer is actually using. +func TestOpenStorePragmas(t *testing.T) { + store, err := OpenStore(filepath.Join(t.TempDir(), "pragma.db")) + if err != nil { + t.Fatalf("OpenStore: %v", err) + } + defer store.db.Close() + + for _, want := range []struct { + pragma string + value string + why string + }{ + {"journal_mode", "wal", "concurrent reader (cmd/server) requires WAL"}, + {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, + {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, + {"foreign_keys", "1", "schema relies on FK enforcement"}, + {"busy_timeout", "5000", "writer serialises behind the reader"}, + {"cache_size", "-2000", "2 MiB of C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, + } { + var got string + if err := store.db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { + t.Errorf("PRAGMA %s: %v", want.pragma, err) + continue + } + if got != want.value { + t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) + } + } +} diff --git a/cmd/ingestor/prune_chunked_test.go b/cmd/ingestor/prune_chunked_test.go index a54130b00..a3be3261a 100644 --- a/cmd/ingestor/prune_chunked_test.go +++ b/cmd/ingestor/prune_chunked_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // seedAgedTransmissions inserts n transmissions stamped `age` days in the diff --git a/cmd/migrate/go.mod b/cmd/migrate/go.mod index b380d96ca..1d24e0a43 100644 --- a/cmd/migrate/go.mod +++ b/cmd/migrate/go.mod @@ -3,20 +3,8 @@ module github.com/corescope/migrate go 1.22 require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/dbschema v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/dbschema => ../../internal/dbschema - -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect -) diff --git a/cmd/migrate/go.sum b/cmd/migrate/go.sum index 5424fe41d..75bf6eb53 100644 --- a/cmd/migrate/go.sum +++ b/cmd/migrate/go.sum @@ -1,43 +1,2 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/cmd/migrate/main.go b/cmd/migrate/main.go index c10f75a10..d6f7eae09 100644 --- a/cmd/migrate/main.go +++ b/cmd/migrate/main.go @@ -18,8 +18,8 @@ import ( "flag" "log" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" ) func main() { @@ -33,7 +33,10 @@ func main() { log.SetFlags(log.LstdFlags | log.Lmsgprefix) log.SetPrefix("[migrate] ") - db, err := sql.Open("sqlite", *dbPath) + // Same DSN as the ingestor. A bare path here would mean synchronous=NORMAL + // under mattn, quietly weakening durability for a process that writes — and + // this one runs dbschema.Apply, which now also deletes duplicate rows. + db, err := sql.Open("sqlite3", dbschema.WriterDSN(*dbPath)) if err != nil { log.Fatalf("open %s: %v", *dbPath, err) } diff --git a/cmd/migrate/main_test.go b/cmd/migrate/main_test.go index 63956fbcf..b32b5c9b4 100644 --- a/cmd/migrate/main_test.go +++ b/cmd/migrate/main_test.go @@ -12,8 +12,8 @@ import ( "path/filepath" "testing" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" ) // fixtureCandidates lists possible locations of the committed e2e @@ -61,7 +61,7 @@ func TestMigrateBringsFixtureToReady(t *testing.T) { dst := filepath.Join(t.TempDir(), "fixture-copy.db") copyFile(t, src, dst) - db, err := sql.Open("sqlite", dst) + db, err := sql.Open("sqlite3", dst) if err != nil { t.Fatalf("open: %v", err) } diff --git a/cmd/server/bounded_load_test.go b/cmd/server/bounded_load_test.go index 6da1d4586..ef383d549 100644 --- a/cmd/server/bounded_load_test.go +++ b/cmd/server/bounded_load_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDB creates a temporary SQLite database with N transmissions (1 obs each). @@ -149,7 +149,7 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -162,8 +162,8 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { } execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) @@ -171,7 +171,7 @@ func createTestDBWithAgedPackets(t *testing.T, numRecent, numOld int) string { now := time.Now().UTC() id := 1 // Single transaction for all inserts — see createTestDBAt for the rationale - // (modernc.org/sqlite auto-commit per Exec fsyncs per row). numOld+numRecent + // (auto-commit per Exec fsyncs per row). numOld+numRecent // is small here today, but wrapping keeps the fixture robust if callers scale // it up, and is consistent with the other builders. if _, err := conn.Exec("BEGIN"); err != nil { @@ -308,7 +308,7 @@ func BenchmarkLoad_30K_Unlimited(b *testing.B) { // createTestDBAt is like createTestDB but writes to a specific path. func createTestDBAt(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -331,9 +331,9 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) @@ -352,7 +352,7 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { defer obsStmt.Close() base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) - // Wrap the inserts in a single transaction. Without this, modernc.org/sqlite + // Wrap the inserts in a single transaction. Without this, SQLite // (pure-Go driver) auto-commits every Exec → one fsync per row → ~2N fsyncs // for N transmissions (tx + obs). At numTx=5000 that is ~10k fsyncs and the // fixture blows past the test timeout (the #1741 hang). A single @@ -379,7 +379,7 @@ func createTestDBAt(tb testing.TB, dbPath string, numTx int) { // createTestDBWithObs creates a test DB with realistic observation counts (1–5 per tx). func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -398,9 +398,9 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + execOrFail(`CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) execOrFail(`CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`) execOrFail(`CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`) @@ -423,7 +423,7 @@ func createTestDBWithObs(tb testing.TB, dbPath string, numTx int) { obsID := 1 base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) // Single transaction for all inserts — see createTestDBAt for the rationale - // (modernc.org/sqlite auto-commit per Exec would fsync per row; at numTx=30000 + // (auto-commit per Exec would fsync per row; at numTx=30000 // the benchmarks would otherwise stall for minutes). One BEGIN/COMMIT. if _, err := conn.Exec("BEGIN"); err != nil { tb.Fatalf("test DB BEGIN: %v", err) diff --git a/cmd/server/chunked_load_id_zero_test.go b/cmd/server/chunked_load_id_zero_test.go index e6cf74bbe..7b1fb87e7 100644 --- a/cmd/server/chunked_load_id_zero_test.go +++ b/cmd/server/chunked_load_id_zero_test.go @@ -29,7 +29,7 @@ import ( func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -48,9 +48,9 @@ func createTestDBWithIDZero(tb testing.TB, dbPath string, extraTx int) { direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, diff --git a/cmd/server/chunked_load_oldest_test.go b/cmd/server/chunked_load_oldest_test.go index 8b9a5f826..8b3edce7f 100644 --- a/cmd/server/chunked_load_oldest_test.go +++ b/cmd/server/chunked_load_oldest_test.go @@ -31,7 +31,7 @@ import ( // This mirrors the freshen-fixture-shifted e2e DB exactly. func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { tb.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { tb.Fatal(err) } @@ -50,9 +50,9 @@ func createTestDBReverseTime(tb testing.TB, dbPath string, numTx int) { direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + `CREATE TABLE IF NOT EXISTS observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE IF NOT EXISTS nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`, `CREATE TABLE IF NOT EXISTS schema_version (version INTEGER)`, diff --git a/cmd/server/coverage_test.go b/cmd/server/coverage_test.go index 5e790dbfb..907ee448e 100644 --- a/cmd/server/coverage_test.go +++ b/cmd/server/coverage_test.go @@ -15,14 +15,14 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // --- helpers --- func setupTestDBv2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/db.go b/cmd/server/db.go index 9ed1b9af1..450933bf7 100644 --- a/cmd/server/db.go +++ b/cmd/server/db.go @@ -13,9 +13,9 @@ import ( "sync" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" "github.com/meshcore-analyzer/geofilter" - _ "modernc.org/sqlite" ) // routeTypeTransport covers TRANSPORT_FLOOD (0) and TRANSPORT_DIRECT (3) — @@ -103,10 +103,25 @@ type channelMessagesCacheEntry struct { exp time.Time } -// OpenDB opens a read-only SQLite connection with WAL mode. +// OpenDB opens a read-only SQLite connection. +// +// The DSN used to pass _journal_mode=WAL and _busy_timeout=5000, which +// modernc.org/sqlite silently ignored: it understood only the +// _pragma=name(value) form. Under github.com/mattn/go-sqlite3 those parameters +// are honoured, and setting journal_mode on a read-only handle is a write — it +// would succeed only because the database is already WAL. Both are gone: +// mattn's own busy_timeout default is already 5000ms, so the read handle keeps +// the timeout (which it had silently lacked) without the pointless write. +// +// What remains is deliberate. mode=ro is the read-only invariant from +// #1283/#1289 and works because mattn's C wrapper ORs SQLITE_OPEN_URI into the +// open flags — see TestOpenDBRefusesMissingDatabase, which fails if that ever +// stops holding. _cache_size pins SQLite's C-allocated page cache at 2 MiB per +// connection; it sits outside GOMEMLIMIT, so it is bounded here rather than +// left to the driver's default (see memlimit.go). func OpenDB(path string) (*DB, error) { - dsn := fmt.Sprintf("file:%s?mode=ro&_journal_mode=WAL&_busy_timeout=5000", path) - conn, err := sql.Open("sqlite", dsn) + dsn := fmt.Sprintf("file:%s?mode=ro&_cache_size=-2000", path) + conn, err := sql.Open("sqlite3", dsn) if err != nil { return nil, err } diff --git a/cmd/server/db_test.go b/cmd/server/db_test.go index 8dc492b0c..8b08e8e58 100644 --- a/cmd/server/db_test.go +++ b/cmd/server/db_test.go @@ -11,13 +11,13 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // setupTestDB creates an in-memory SQLite database with the v3 schema. func setupTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -1198,7 +1198,7 @@ func TestGetNodesFiltering(t *testing.T) { // where observations use observer_id TEXT instead of observer_idx INTEGER. func setupTestDBV2(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -1226,7 +1226,8 @@ func setupTestDBV2(t *testing.T) *DB { last_seen TEXT, first_seen TEXT, packet_count INTEGER DEFAULT 0, - last_packet_at TEXT DEFAULT NULL + last_packet_at TEXT DEFAULT NULL, + inactive INTEGER DEFAULT 0 ); CREATE TABLE transmissions ( @@ -1454,7 +1455,7 @@ func TestOpenDBValid(t *testing.T) { dbPath := filepath.Join(dir, "test.db") // Create DB with a table using a writable connection first - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -1463,6 +1464,7 @@ func TestOpenDBValid(t *testing.T) { conn.Close() t.Fatal(err) } + ensurePreparable(t, conn) conn.Close() // Now test OpenDB (read-only) @@ -1495,7 +1497,7 @@ func TestDetectSchemaScopeName(t *testing.T) { dbPath := filepath.Join(dir, "detect.db") // Create file-based DB with the scope_name and default_scope columns. - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -1512,6 +1514,7 @@ func TestDetectSchemaScopeName(t *testing.T) { conn.Close() t.Fatalf("create observations: %v", err) } + ensurePreparable(t, conn) conn.Close() db, err := OpenDB(dbPath) @@ -1529,7 +1532,7 @@ func TestDetectSchemaScopeName(t *testing.T) { // Verify the flags stay false when the columns are absent. dbPath2 := filepath.Join(dir, "detect2.db") - conn2, err := sql.Open("sqlite", dbPath2) + conn2, err := sql.Open("sqlite3", dbPath2) if err != nil { t.Fatal(err) } @@ -1537,6 +1540,7 @@ func TestDetectSchemaScopeName(t *testing.T) { conn2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) conn2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY)`) + ensurePreparable(t, conn2) conn2.Close() db2, err := OpenDB(dbPath2) @@ -2233,7 +2237,7 @@ func TestPerObservationRawHexEnrich(t *testing.T) { } func TestGetScopeStats(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2293,7 +2297,7 @@ func TestGetScopeStats(t *testing.T) { // scope_name states (NULL is unscoped, an empty string is unknown scope, a // name is a named scope). func TestGetScopeStatsAdvertsByRole(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2377,7 +2381,7 @@ func TestGetScopeStatsAdvertsByRole(t *testing.T) { } func TestGetScopeStatsAdvertsByRoleEmpty(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("sql.Open: %v", err) } @@ -2463,7 +2467,7 @@ func (f *failingQuerier) QueryContext(ctx context.Context, query string, args .. // a v3 database for the whole process lifetime. func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { dbPath := filepath.Join(t.TempDir(), "v3.db") - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -2477,7 +2481,7 @@ func TestDetectSchemaFailsLoudOnProbeError(t *testing.T) { conn.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) conn.Close() - real, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro") + real, err := sql.Open("sqlite3", "file:"+dbPath+"?mode=ro") if err != nil { t.Fatal(err) } @@ -2499,7 +2503,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { dir := t.TempDir() v3 := filepath.Join(dir, "v3.db") - c, err := sql.Open("sqlite", v3) + c, err := sql.Open("sqlite3", v3) if err != nil { t.Fatal(err) } @@ -2507,6 +2511,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { c.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_idx INTEGER)`) c.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) c.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) + ensurePreparable(t, c) c.Close() db, err := OpenDB(v3) if err != nil { @@ -2518,7 +2523,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { db.Close() v2 := filepath.Join(dir, "v2.db") - c2, err := sql.Open("sqlite", v2) + c2, err := sql.Open("sqlite3", v2) if err != nil { t.Fatal(err) } @@ -2526,6 +2531,7 @@ func TestDetectSchemaV3AndV2(t *testing.T) { c2.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, observer_id INTEGER)`) c2.Exec(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT)`) c2.Exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`) + ensurePreparable(t, c2) c2.Close() db2, err := OpenDB(v2) if err != nil { diff --git a/cmd/server/go.mod b/cmd/server/go.mod index ae93a43fc..7518c6784 100644 --- a/cmd/server/go.mod +++ b/cmd/server/go.mod @@ -7,7 +7,6 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/meshcore-analyzer/geofilter v0.0.0 github.com/meshcore-analyzer/sigvalidate v0.0.0 - modernc.org/sqlite v1.34.5 ) replace github.com/meshcore-analyzer/geofilter => ../../internal/geofilter @@ -34,23 +33,12 @@ require github.com/meshcore-analyzer/lora v0.0.0 replace github.com/meshcore-analyzer/lora => ../../internal/lora -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect -) - require github.com/meshcore-analyzer/prunequeue v0.0.0 replace github.com/meshcore-analyzer/prunequeue => ../../internal/prunequeue require ( + github.com/mattn/go-sqlite3 v1.14.52 github.com/meshcore-analyzer/mbcapqueue v0.0.0 golang.org/x/sync v0.10.0 ) diff --git a/cmd/server/go.sum b/cmd/server/go.sum index 45b74210f..8c505bf84 100644 --- a/cmd/server/go.sum +++ b/cmd/server/go.sum @@ -1,49 +1,8 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo= -github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= -golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= -golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= -modernc.org/cc/v4 v4.21.4 h1:3Be/Rdo1fpr8GrQ7IVw9OHtplU4gWbb+wNgeoBMmGLQ= -modernc.org/cc/v4 v4.21.4/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ= -modernc.org/ccgo/v4 v4.19.2 h1:lwQZgvboKD0jBwdaeVCTouxhxAyN6iawF3STraAal8Y= -modernc.org/ccgo/v4 v4.19.2/go.mod h1:ysS3mxiMV38XGRTTcgo0DQTeTmAO4oCmJl1nX9VFI3s= -modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE= -modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ= -modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw= -modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc= -modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= -modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= -modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/cmd/server/hexgrid.go b/cmd/server/hexgrid.go index 93c25f3b7..689661d8c 100644 --- a/cmd/server/hexgrid.go +++ b/cmd/server/hexgrid.go @@ -8,7 +8,9 @@ import ( ) // Pure-Go hexagonal binning for RX coverage display. We deliberately avoid the -// CGO-based uber/h3-go (this project builds with CGO_ENABLED=0). Points are +// CGO-based uber/h3-go. (That predates the SQLite driver move to cgo; this +// stays pure Go because it needs no C library, not because cgo is unavailable.) +// Points are // projected to Web Mercator and snapped to a pointy-top hex grid whose size // depends on the display resolution. Cell ids are "res:q:r" (axial coords). // At city/region scale this looks like H3/mapme.sh coverage without any deps. diff --git a/cmd/server/hot_startup_test.go b/cmd/server/hot_startup_test.go index d848da8e3..238369c3c 100644 --- a/cmd/server/hot_startup_test.go +++ b/cmd/server/hot_startup_test.go @@ -13,7 +13,7 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBMultiDay creates a test DB with packets spread across numDays days. @@ -24,7 +24,7 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -37,8 +37,8 @@ func createTestDBMultiDay(t *testing.T, numDays, txPerDay int) string { } execOrFail(`CREATE TABLE transmissions (id INTEGER PRIMARY KEY, raw_hex TEXT, hash TEXT, first_seen TEXT, route_type INTEGER, payload_type INTEGER, payload_version INTEGER, decoded_json TEXT)`) execOrFail(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT)`) - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) execOrFail(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) diff --git a/cmd/server/issue1690_cold_load_test.go b/cmd/server/issue1690_cold_load_test.go index 2b1f85002..1668e04c9 100644 --- a/cmd/server/issue1690_cold_load_test.go +++ b/cmd/server/issue1690_cold_load_test.go @@ -19,7 +19,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBWithLastSeen seeds a DB with the post-fix schema (last_seen diff --git a/cmd/server/issue1809_bg_load_race_test.go b/cmd/server/issue1809_bg_load_race_test.go index 14ba1850f..eca19727c 100644 --- a/cmd/server/issue1809_bg_load_race_test.go +++ b/cmd/server/issue1809_bg_load_race_test.go @@ -138,7 +138,7 @@ func createTestDBSpreadOverDays(t *testing.T, dbPath string, numTx, spanDays int // (adv #11: prior code carried a duplicated annotation on every line)." func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes func(i int) (string, int64)) { t.Helper() - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -164,9 +164,9 @@ func seedTestDBRows(t *testing.T, dbPath string, numTx, obsPerTx int, rowTimes f path_json TEXT, timestamp TEXT, raw_hex TEXT )`) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + execOrFail(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" - execOrFail(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + execOrFail(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) // PREFLIGHT: async=true reason="unit-test fixture seeder" execOrFail(`CREATE TABLE schema_version (version INTEGER)`) execOrFail(`INSERT INTO schema_version (version) VALUES (1)`) diff --git a/cmd/server/issue673_test.go b/cmd/server/issue673_test.go index b5ac55173..85337c9d7 100644 --- a/cmd/server/issue673_test.go +++ b/cmd/server/issue673_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) const issue673NodePK = "7502f19f44cad6d7b626e1d811c00a914af452636182ccded3fd019803395ec9" diff --git a/cmd/server/load_graph_invariant_test.go b/cmd/server/load_graph_invariant_test.go index e128756b7..07ae649f0 100644 --- a/cmd/server/load_graph_invariant_test.go +++ b/cmd/server/load_graph_invariant_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist pins the startup-ordering @@ -22,7 +22,7 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -49,7 +49,7 @@ func TestLoad_PanicsWhenGraphNotLoadedAndEdgesExist(t *testing.T) { path_json TEXT, timestamp TEXT, raw_hex TEXT, resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, diff --git a/cmd/server/loadchunk_ambiguous_prefix_test.go b/cmd/server/loadchunk_ambiguous_prefix_test.go index 9a0d4265c..3c0e1596b 100644 --- a/cmd/server/loadchunk_ambiguous_prefix_test.go +++ b/cmd/server/loadchunk_ambiguous_prefix_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBAmbiguousPrefix builds a fixture where TWO repeaters share the @@ -24,7 +24,7 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func createTestDBAmbiguousPrefix(t *testing.T, relayA, relayB, hop, firstSeen st resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, diff --git a/cmd/server/loadchunk_pathjson_fallback_test.go b/cmd/server/loadchunk_pathjson_fallback_test.go index 518fe1611..9d28b6203 100644 --- a/cmd/server/loadchunk_pathjson_fallback_test.go +++ b/cmd/server/loadchunk_pathjson_fallback_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBPathJSONNoResolvedPath builds a fixture that mirrors the LIVE @@ -25,7 +25,7 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -56,7 +56,7 @@ func createTestDBPathJSONNoResolvedPath(t *testing.T, relayPubkey, hopPrefix, fi resolved_path TEXT )`) // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) // Production nodes schema uses public_key (not pubkey) — getAllNodes / // buildPrefixMap reads public_key, role, advert_count, first_seen. // PREFLIGHT: async=true reason="test fixture, in-memory tmpdir DB" diff --git a/cmd/server/loadchunk_resolved_path_1558_test.go b/cmd/server/loadchunk_resolved_path_1558_test.go index b9ede3c4f..9cd83ecd5 100644 --- a/cmd/server/loadchunk_resolved_path_1558_test.go +++ b/cmd/server/loadchunk_resolved_path_1558_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // createTestDBWithResolvedPath creates a fixture DB containing numTx old @@ -20,7 +20,7 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -47,8 +47,8 @@ func createTestDBWithResolvedPath(t *testing.T, numTx int, relayPubkeys []string raw_hex TEXT, resolved_path TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) - exec(`CREATE TABLE nodes (pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) + exec(`CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL)`) exec(`CREATE TABLE schema_version (version INTEGER)`) exec(`INSERT INTO schema_version (version) VALUES (1)`) exec(`CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`) diff --git a/cmd/server/memlimit.go b/cmd/server/memlimit.go index 4ff590600..f8e3df5c8 100644 --- a/cmd/server/memlimit.go +++ b/cmd/server/memlimit.go @@ -36,6 +36,21 @@ func applyMemoryLimit(maxMemoryMB int, envSet bool) (int64, string) { // 1.5x headroom over the steady-state packet store budget covers // transient peaks (cold-load row-scan / decode pipeline, Go's NextGC // trigger at ~2x live heap). See issue #836 heap profile. + // + // GOMEMLIMIT covers memory the Go runtime manages — heap, stacks and + // runtime structures — and nothing else. Since the SQLite driver became + // cgo (github.com/mattn/go-sqlite3), what SQLite allocates in C is + // outside it, so this limit is not an RSS ceiling: actual RSS is this + // plus the native allocations plus whatever the runtime has reserved + // but not returned. + // + // The page cache, at least, is deliberately bounded: both DSNs pin + // _cache_size=-2000, i.e. ~2 MiB per connection, ~8 MiB across + // SetMaxOpenConns(4). That fits inside the headroom above rather than + // eating into the store budget. It does not bound SQLite's other native + // allocations, so if the connection count or _cache_size grows + // materially, or a workload starts holding many prepared statements, + // this derivation needs revisiting against measured RSS. limit := int64(maxMemoryMB) * 1024 * 1024 * 3 / 2 debug.SetMemoryLimit(limit) return limit, "derived" diff --git a/cmd/server/memory.go b/cmd/server/memory.go index 2ffe4d7ce..8ad52b516 100644 --- a/cmd/server/memory.go +++ b/cmd/server/memory.go @@ -32,9 +32,22 @@ import ( // eviction; both can shrink when packets age out. goHeapInuseMB and goSysMB // fluctuate with GC. // -// cgoBytesMB intentionally absent: this build uses the pure-Go -// modernc.org/sqlite driver, so there is no cgo allocator to measure. -// Reintroduce only if we ever switch back to mattn/go-sqlite3. +// There is no cgo-allocation field, but for a different reason than before: +// the driver is github.com/mattn/go-sqlite3 now, so SQLite's page cache IS a C +// allocation — the Go runtime simply exposes no byte counter for it. +// +// processRSSMB does include it, since VmRSS counts every resident page whoever +// allocated it. Resist the temptation to read `processRSSMB - goSysMB` as "the +// C share": goSysMB is address space the runtime has reserved, not what is +// resident, so the subtraction mixes two different things and can even go +// negative. It is a smell test, not a measurement. To size native usage, +// measure RSS against a build with the store disabled, or profile. +// +// What is bounded is SQLite's page cache specifically: both DSNs pin +// _cache_size=-2000, i.e. ~2 MiB per connection, so roughly 8 MiB across +// SetMaxOpenConns(4) here and 2 MiB in the ingestor. That is a cap on the page +// cache, not on everything SQLite allocates — statement and schema memory sit +// outside it. See memlimit.go for why any of it matters to GOMEMLIMIT. type MemorySnapshot struct { ProcessRSSMB float64 `json:"processRSSMB"` GoHeapInuseMB float64 `json:"goHeapInuseMB"` diff --git a/cmd/server/multibyte_capability_test.go b/cmd/server/multibyte_capability_test.go index 1325510a4..53267533f 100644 --- a/cmd/server/multibyte_capability_test.go +++ b/cmd/server/multibyte_capability_test.go @@ -8,7 +8,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // recentTS returns a timestamp string N hours ago, ensuring test data @@ -20,7 +20,7 @@ func recentTS(hoursAgo int) string { // setupCapabilityTestDB creates a minimal in-memory DB with nodes table. func setupCapabilityTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -446,7 +446,7 @@ func TestMultiByteCapability_AdopterEvidenceTakesPrecedence(t *testing.T) { // setupPersistTestDB creates an in-memory DB with multibyte_sup/multibyte_evidence columns. func setupPersistTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/neighbor_api_test.go b/cmd/server/neighbor_api_test.go index 4a146eaf7..e3ab0b96c 100644 --- a/cmd/server/neighbor_api_test.go +++ b/cmd/server/neighbor_api_test.go @@ -9,7 +9,7 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // ─── Helpers ─────────────────────────────────────────────────────────────────── @@ -467,7 +467,7 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { tmpDir := t.TempDir() dbPath := tmpDir + "/test.db" - conn, err := sql.Open("sqlite", dbPath) + conn, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } @@ -476,15 +476,16 @@ func TestBuildNodeInfoMap_ObserverEnrichment(t *testing.T) { // Create tables for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0)", - "INSERT INTO observers VALUES ('BBBB2222', 'Observer-Alpha', '')", - "INSERT INTO observers VALUES ('AAAA1111', 'Obs-also-repeater', '')", + "INSERT INTO observers (id, name, iata) VALUES ('BBBB2222', 'Observer-Alpha', '')", + "INSERT INTO observers (id, name, iata) VALUES ('AAAA1111', 'Obs-also-repeater', '')", } { if _, err := conn.Exec(stmt); err != nil { t.Fatalf("exec %q: %v", stmt, err) } } + ensurePreparable(t, conn) conn.Close() // Open via our DB wrapper @@ -542,20 +543,21 @@ func TestBuildNodeInfoMap_FirstSeenIsCached(t *testing.T) { dbPath := tmpDir + "/test.db" // Seed via rw connection. - rw, err := sql.Open("sqlite", dbPath) + rw, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('AAAA1111', 'Repeater-1', 'repeater', 0, 0, '', '2024-01-01T00:00:00Z', 0)", } { if _, err := rw.Exec(stmt); err != nil { t.Fatalf("seed exec %q: %v", stmt, err) } } + ensurePreparable(t, rw) db, err := OpenDB(dbPath) if err != nil { @@ -608,20 +610,21 @@ func TestGetAllNodes_FirstSeenSchemaFallback(t *testing.T) { dbPath := tmpDir + "/test.db" // Seed a nodes table WITHOUT first_seen (advert_count + last_seen present). - rw, err := sql.Open("sqlite", dbPath) + rw, err := sql.Open("sqlite3", dbPath) if err != nil { t.Fatal(err) } defer rw.Close() for _, stmt := range []string{ "CREATE TABLE nodes (public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, advert_count INTEGER)", - "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT)", + "CREATE TABLE observers (id TEXT, name TEXT, iata TEXT, inactive INTEGER)", "INSERT INTO nodes VALUES ('BBBB2222', 'Repeater-2', 'repeater', 0, 0, '2024-02-02T00:00:00Z', 3)", } { if _, err := rw.Exec(stmt); err != nil { t.Fatalf("seed exec %q: %v", stmt, err) } } + ensurePreparable(t, rw) db, err := OpenDB(dbPath) if err != nil { diff --git a/cmd/server/neighbor_persist_legacy_test.go b/cmd/server/neighbor_persist_legacy_test.go index efb8d6dd1..798b96e4c 100644 --- a/cmd/server/neighbor_persist_legacy_test.go +++ b/cmd/server/neighbor_persist_legacy_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestNeighborPersist_LegacyEdgeInvariant (#1638 adv-#1): edges loaded from @@ -18,7 +18,7 @@ import ( func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "neighbor_legacy.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -79,7 +79,7 @@ func TestNeighborPersist_LegacyEdgeInvariant(t *testing.T) { func TestNeighborPersist_LegacyEdgeMergeOnReload(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "neighbor_legacy_merge.db") - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } diff --git a/cmd/server/neighbor_recomputer_test.go b/cmd/server/neighbor_recomputer_test.go index f78770a43..cdcf7b8f6 100644 --- a/cmd/server/neighbor_recomputer_test.go +++ b/cmd/server/neighbor_recomputer_test.go @@ -6,8 +6,8 @@ import ( "testing" "time" + _ "github.com/mattn/go-sqlite3" "github.com/meshcore-analyzer/dbschema" - _ "modernc.org/sqlite" ) // TestNeighborGraphRecomputerLoadsSnapshot enforces #1287 Option 4: @@ -19,7 +19,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { dbPath := filepath.Join(dir, "neighbor_recomp.db") // Bootstrap a WAL DB with the neighbor_edges table. - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -33,6 +33,7 @@ func TestNeighborGraphRecomputerLoadsSnapshot(t *testing.T) { )`); err != nil { t.Fatal(err) } + ensurePreparable(t, rw) // Stage one edge. now := time.Now().UTC().Format(time.RFC3339) @@ -85,9 +86,15 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { // Bootstrap with ONLY transmissions/observations (the things // server tries to read) but WITHOUT the columns dbschema asserts - // (resolved_path, inactive, last_packet_at, iata, foreign_advert, - // from_pubkey, neighbor_edges). - rw, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + // (resolved_path, last_packet_at, iata, foreign_advert, from_pubkey, + // neighbor_edges). + // + // observers.inactive used to be in that list. It no longer is: OpenDB + // prepares statements eagerly under github.com/mattn/go-sqlite3 and one of + // them selects on inactive, so a fixture without it cannot reach + // AssertReady at all. The other missing surfaces above still make + // AssertReady fail, which is what this test asserts. + rw, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -95,7 +102,7 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { for _, s := range []string{ `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, payload_type INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`, + `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE nodes (public_key TEXT PRIMARY KEY)`, `CREATE TABLE inactive_nodes (public_key TEXT PRIMARY KEY)`, } { @@ -104,6 +111,8 @@ func TestServerStartupRequiresMigratedSchema(t *testing.T) { } } + ensurePreparable(t, rw) + // Open the read-only server handle and call AssertReady directly // (production path: main.go does this before any business logic). d, err := OpenDB(dbPath) diff --git a/cmd/server/node_reach_bench_test.go b/cmd/server/node_reach_bench_test.go index 4159c80ed..bbf24a347 100644 --- a/cmd/server/node_reach_bench_test.go +++ b/cmd/server/node_reach_bench_test.go @@ -6,7 +6,7 @@ import ( "fmt" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // benchReachDB builds an in-memory DB with nObs observations. matchEvery @@ -24,13 +24,13 @@ func benchReachDB(b *testing.B, nObs, matchEvery int, lowerHops bool) *DB { // path_json is uppercase; this measures the worst case Carmack flagged). matchPath, fillerPath = `["aa","01fa","bb"]`, `["aa","cc","bb"]` } - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { b.Fatal(err) } schema := []string{ `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, hash TEXT, first_seen TEXT, payload_type INTEGER, from_pubkey TEXT)`, - `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT)`, + `CREATE TABLE observers (id TEXT PRIMARY KEY, name TEXT, inactive INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, `CREATE INDEX idx_obs_ts ON observations(timestamp)`, } @@ -155,12 +155,12 @@ func BenchmarkNodeReachAttribute(b *testing.B) { // must surface an error, not a swallowed nil. Lives in this file because // the bench callers in the same file rely on the same signature. func TestScanReachRows_ErrorReturn(t *testing.T) { - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatalf("open: %v", err) } // PREFLIGHT: async=true reason="test-only in-memory scratch schema, immediately closed" - if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER); CREATE TABLE transmissions (id INTEGER); CREATE TABLE observers (rowid INTEGER, id TEXT)`); err != nil { + if _, err := conn.Exec(`CREATE TABLE observations (id INTEGER); CREATE TABLE transmissions (id INTEGER); CREATE TABLE observers (rowid INTEGER, id TEXT, inactive INTEGER)`); err != nil { t.Fatalf("schema: %v", err) } conn.Close() // force QueryContext to fail diff --git a/cmd/server/node_reach_endpoint_test.go b/cmd/server/node_reach_endpoint_test.go index 6094077b0..a7fc6d4aa 100644 --- a/cmd/server/node_reach_endpoint_test.go +++ b/cmd/server/node_reach_endpoint_test.go @@ -11,7 +11,7 @@ import ( "time" "github.com/gorilla/mux" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) func serveReach(srv *Server, path string) *httptest.ResponseRecorder { @@ -57,7 +57,7 @@ func resetReachState(t *testing.T, servers ...*Server) { // build the zero-reach case (identifiable node, no matching observations). func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } @@ -68,7 +68,7 @@ func newReachIntegrationDB(t *testing.T, obsPath string) (*DB, string) { stmts := []string{ `CREATE TABLE nodes (public_key TEXT, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, advert_count INTEGER)`, `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT)`, + `CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, `CREATE TABLE neighbor_edges (node_a TEXT, node_b TEXT, count INTEGER)`, } diff --git a/cmd/server/node_reach_test.go b/cmd/server/node_reach_test.go index 606f8e2c7..55938919f 100644 --- a/cmd/server/node_reach_test.go +++ b/cmd/server/node_reach_test.go @@ -6,20 +6,20 @@ import ( "strconv" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // newReachScanTestDB builds a minimal observer_idx-schema DB with two rows whose // path contains "01FA" and one that does not, for scanReachRows coverage. func newReachScanTestDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } stmts := []string{ `CREATE TABLE transmissions (id INTEGER PRIMARY KEY, from_pubkey TEXT, payload_type INTEGER)`, - `CREATE TABLE observers (id TEXT)`, + `CREATE TABLE observers (id TEXT, inactive INTEGER)`, `CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_idx INTEGER, snr REAL, path_json TEXT, timestamp INTEGER)`, `INSERT INTO observers (id) VALUES ('OBS1')`, // rowid 1 `INSERT INTO transmissions (id, from_pubkey, payload_type) VALUES (1,'FF00',4),(2,'',5),(3,'',5)`, diff --git a/cmd/server/perf_io.go b/cmd/server/perf_io.go index 53415e390..dcf1372eb 100644 --- a/cmd/server/perf_io.go +++ b/cmd/server/perf_io.go @@ -260,7 +260,7 @@ func (s *Server) handlePerfSqlite(w http.ResponseWriter, r *http.Request) { resp.CacheSize = cacheSize // Cache hit rate: derived from PacketStore cache (rw_cache). We don't - // have a direct SQLite cache counter via the modernc driver, so we + // have a direct SQLite cache counter through the driver, so we // surface the closest available proxy — the in-process row cache. if s.store != nil { cs := s.store.GetCacheStatsTyped() diff --git a/cmd/server/preparable_schema_test.go b/cmd/server/preparable_schema_test.go new file mode 100644 index 000000000..0540091d7 --- /dev/null +++ b/cmd/server/preparable_schema_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "database/sql" + "strings" + "testing" +) + +// The statements compiled by prepareStatements reference a set of tables and +// columns. Under modernc.org/sqlite that never mattered for test fixtures: +// newStmt only stored the SQL string and compiled it lazily on first use, so a +// fixture could omit half the schema and still open. github.com/mattn/go-sqlite3 +// calls sqlite3_prepare_v2 inside Prepare, so OpenDB now fails on an incomplete +// schema — which is the production behaviour we want (#1901), but it means +// fixtures have to declare what they are prepared against. +// +// ensurePreparable is the seam. Fixtures that deliberately build a partial +// schema (to exercise schema detection, say) call it on the writable connection +// just before OpenDB, and it fills in whatever is missing without disturbing the +// shapes the test actually cares about. +var preparableTables = []string{ + `CREATE TABLE IF NOT EXISTS transmissions (id INTEGER PRIMARY KEY, hash TEXT)`, + `CREATE TABLE IF NOT EXISTS observations (id INTEGER PRIMARY KEY, timestamp TEXT)`, + `CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY)`, + `CREATE TABLE IF NOT EXISTS observers (id TEXT)`, +} + +// preparableColumns are added when absent. A fixture that already declared the +// table keeps its own definition, so only the missing columns get appended. +var preparableColumns = []struct{ table, column, decl string }{ + {"transmissions", "hash", "TEXT"}, + {"observations", "timestamp", "TEXT"}, + {"nodes", "name", "TEXT"}, + {"nodes", "role", "TEXT"}, + {"nodes", "last_seen", "TEXT"}, + {"observers", "inactive", "INTEGER"}, +} + +// ensurePreparable makes conn's schema sufficient for prepareStatements. +// It is idempotent and never overwrites an existing table or column. +func ensurePreparable(tb testing.TB, conn *sql.DB) { + tb.Helper() + for _, ddl := range preparableTables { + if _, err := conn.Exec(ddl); err != nil { + tb.Fatalf("ensurePreparable: %v\nSQL: %s", err, ddl) + } + } + for _, c := range preparableColumns { + _, err := conn.Exec("ALTER TABLE " + c.table + " ADD COLUMN " + c.column + " " + c.decl) + // "duplicate column name" means the fixture already declared it. + if err != nil && !strings.Contains(err.Error(), "duplicate column name") { + tb.Fatalf("ensurePreparable: add %s.%s: %v", c.table, c.column, err) + } + } +} + +// TestEnsurePreparableMatchesPrepareStatements keeps the helper honest. If a new +// prepared statement references a table or column ensurePreparable does not +// create, it fails here rather than as a confusing OpenDB error in whichever +// fixture happens to be thinnest. +func TestEnsurePreparableMatchesPrepareStatements(t *testing.T) { + conn, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + // prepareStatements compiles against a pooled connection; pin the pool so + // the in-memory database the DDL landed in is the one it prepares against. + conn.SetMaxOpenConns(1) + ensurePreparable(t, conn) + + db := &DB{conn: conn} + if err := db.prepareStatements(); err != nil { + t.Fatalf("prepareStatements against the minimal preparable schema: %v\n"+ + "Add the missing table/column to preparableTables or preparableColumns.", err) + } +} diff --git a/cmd/server/readonly_invariant_test.go b/cmd/server/readonly_invariant_test.go index f97baa480..0f671a2a7 100644 --- a/cmd/server/readonly_invariant_test.go +++ b/cmd/server/readonly_invariant_test.go @@ -10,7 +10,7 @@ import ( "strings" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestServerSourceHasNoCachedRWCalls enforces issue #1287: after the @@ -129,7 +129,7 @@ func TestServerDBConnIsReadOnly(t *testing.T) { // Bootstrap a minimal DB with the ingestor-style WAL opener so the // server can attach in read-only mode. - if err := bootstrapMinimalDB(path); err != nil { + if err := bootstrapMinimalDB(t, path); err != nil { t.Fatalf("bootstrap: %v", err) } @@ -149,9 +149,9 @@ func TestServerDBConnIsReadOnly(t *testing.T) { // need, opened with WAL so the read-only opener in OpenDB can attach. // Kept in *_test.go so it does NOT add any write capability to the // production server binary. -func bootstrapMinimalDB(path string) error { +func bootstrapMinimalDB(tb testing.TB, path string) error { dsn := fmt.Sprintf("file:%s?_journal_mode=WAL&_busy_timeout=5000", path) - rw, err := sql.Open("sqlite", dsn) + rw, err := sql.Open("sqlite3", dsn) if err != nil { return err } @@ -159,6 +159,9 @@ func bootstrapMinimalDB(path string) error { if _, err := rw.Exec(`CREATE TABLE IF NOT EXISTS nodes (public_key TEXT PRIMARY KEY, name TEXT)`); err != nil { return err } + // prepareStatements compiles eagerly under mattn; give it the rest of the + // surface it references so OpenDB gets far enough to test read-onlyness. + ensurePreparable(tb, rw) return nil } @@ -211,3 +214,37 @@ func nodeTableWritePattern(verb, trailer string) *regexp.Regexp { } return regexp.MustCompile(expr) } + +// TestOpenDBRefusesMissingDatabase pins the behaviour that makes mode=ro +// meaningful: OpenDB must fail on a path that does not exist rather than +// creating an empty database there. +// +// This is worth a test of its own because the guarantee is not obvious from the +// call site. github.com/mattn/go-sqlite3 always passes +// SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE to sqlite3_open_v2; what restricts it +// is the mode=ro parameter in the URI, which only takes effect because mattn's +// C wrapper ORs SQLITE_OPEN_URI into the flags (sqlite3.go _sqlite3_open_v2). +// Lose the file: prefix, or the URI flag, and this silently degrades into a +// read-write open that manufactures a fresh empty database — which the server +// would then happily serve. cmd/decrypt shipped exactly that bug for a while by +// building its DSN without the file: prefix. +func TestOpenDBRefusesMissingDatabase(t *testing.T) { + dir := t.TempDir() + missing := filepath.Join(dir, "absent.db") + + db, err := OpenDB(missing) + if err == nil { + db.Close() + t.Fatal("OpenDB succeeded against a nonexistent database; mode=ro is not in effect") + } + + // Neither the database itself nor a file literally named after the DSN + // (what happens when URI handling is off) may have been created. + entries, rerr := os.ReadDir(dir) + if rerr != nil { + t.Fatalf("read temp dir: %v", rerr) + } + for _, e := range entries { + t.Errorf("OpenDB created %q; a read-only open must not write anything", e.Name()) + } +} diff --git a/cmd/server/resolved_index_test.go b/cmd/server/resolved_index_test.go index 30f2669dd..90c2bfb89 100644 --- a/cmd/server/resolved_index_test.go +++ b/cmd/server/resolved_index_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // --- Unit tests --- @@ -1096,7 +1096,7 @@ func TestLRU_CapacityAfterBulkDelete(t *testing.T) { // TestConfirmResolvedPathContains_SpecialChars verifies that pubkeys containing // SQL LIKE wildcards (%, _) don't cause false positives with the INSTR approach. func TestConfirmResolvedPathContains_SpecialChars(t *testing.T) { - db, err := sql.Open("sqlite", ":memory:") + db, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/scope_audit_test.go b/cmd/server/scope_audit_test.go index 3ebd811be..83ae72129 100644 --- a/cmd/server/scope_audit_test.go +++ b/cmd/server/scope_audit_test.go @@ -31,7 +31,7 @@ var ( // fiction, so the join behaves the way it does against a real database. func setupScopeConformanceDB(t *testing.T) *DB { t.Helper() - conn, err := sql.Open("sqlite", ":memory:") + conn, err := sql.Open("sqlite3", ":memory:") if err != nil { t.Fatal(err) } diff --git a/cmd/server/store_memory_diagnostics_test.go b/cmd/server/store_memory_diagnostics_test.go index 989a76cc0..e1253d2bf 100644 --- a/cmd/server/store_memory_diagnostics_test.go +++ b/cmd/server/store_memory_diagnostics_test.go @@ -95,7 +95,7 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { const txHex = "deadbeefcafe" const obsHex = "c0ffee0102" // distinct from txHex: proves we DON'T keep it - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -109,9 +109,9 @@ func TestObsRawHexNotRetainedOnLoad(t *testing.T) { observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`, - `CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`, + `CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`, `CREATE TABLE nodes ( - pubkey TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, + public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, first_seen TEXT, frequency REAL )`, `CREATE INDEX idx_tx_first_seen ON transmissions(first_seen)`, diff --git a/cmd/server/topology_dedup_test.go b/cmd/server/topology_dedup_test.go index eac7036c0..b875eb016 100644 --- a/cmd/server/topology_dedup_test.go +++ b/cmd/server/topology_dedup_test.go @@ -7,7 +7,7 @@ import ( "testing" "time" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // TestTopologyDedup_RepeatersMergeByPubkey verifies that topRepeaters @@ -15,7 +15,7 @@ import ( func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -34,7 +34,7 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, frequency REAL @@ -139,7 +139,7 @@ func TestTopologyDedup_RepeatersMergeByPubkey(t *testing.T) { func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -158,7 +158,7 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, frequency REAL @@ -245,7 +245,7 @@ func TestTopologyDedup_AmbiguousPrefixNotMerged(t *testing.T) { func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { dir := t.TempDir() dbPath := filepath.Join(dir, "test.db") - conn, err := sql.Open("sqlite", dbPath+"?_journal_mode=WAL") + conn, err := sql.Open("sqlite3", dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } @@ -264,7 +264,7 @@ func TestTopologyDedup_PairsMergeByPubkey(t *testing.T) { id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT, observer_name TEXT, direction TEXT, snr REAL, rssi REAL, score INTEGER, path_json TEXT, timestamp TEXT, raw_hex TEXT )`) - exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT)`) + exec(`CREATE TABLE observers (rowid INTEGER PRIMARY KEY, id TEXT, name TEXT, iata TEXT, inactive INTEGER)`) exec(`CREATE TABLE nodes ( public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, last_seen TEXT, frequency REAL diff --git a/docs/client-rx-coverage.md b/docs/client-rx-coverage.md index cc8d96aaf..61191a243 100644 --- a/docs/client-rx-coverage.md +++ b/docs/client-rx-coverage.md @@ -203,7 +203,8 @@ server-side (read-only). Each feature: ``` - Hex binning is a pure-Go pointy-top grid over Web Mercator (`cmd/server/hexgrid.go`). We do **not** - use `uber/h3-go` because it is CGO and the project builds with `CGO_ENABLED=0`. Latitude is only + use `uber/h3-go`, which would add a C dependency for no benefit (the SQLite driver + is cgo since the mattn/go-sqlite3 move, but that is a library we need). Latitude is only defined within ±85.05° (Web Mercator limit) and is clamped to that range. - `z` (Leaflet zoom) selects the hex resolution (zoom-adaptive). Raw points never leave the server (privacy: contributors' tracks are not exposed). diff --git a/docs/sqlite-driver-migration.md b/docs/sqlite-driver-migration.md new file mode 100644 index 000000000..f7245164c --- /dev/null +++ b/docs/sqlite-driver-migration.md @@ -0,0 +1,245 @@ +# SQLite driver: modernc.org/sqlite → github.com/mattn/go-sqlite3 + +The database driver changed from `modernc.org/sqlite v1.34.5` (pure Go, SQLite +3.46.0) to `github.com/mattn/go-sqlite3 v1.14.52` (cgo, bundled SQLite 3.53.4). + +This is the record of why, what it measured, and the five behavioural +differences that had to be handled — several of which fail silently if you get +them wrong, so read this before touching the DSNs or the build. + +## Why + +corescope is read-heavy: the server chunk-loads a graph at startup and fans out +neighbor/topology/analytics queries per request. modernc's pure-Go SQLite is a +transpilation of the C amalgamation and pays for it on exactly those paths. + +## What it measured + +Head-to-head on the same 120k-transmission / 240k-observation database, running +corescope's own hot-path SQL under both drivers (Apple M4, `-count=5`, medians): + +| Workload | modernc | mattn | Change | +|---|---:|---:|---| +| Chunk load (the `cmd/server/chunked_load.go` v3 join, 20k transmissions) | 449 ms | 196 ms | **2.3× faster** | +| Aggregate scan (240k-row join + `GROUP BY`, stands in for analytics) | 276 ms | 137 ms | **2.0× faster** | +| 1500 prepared-statement lookups (`prepareStatements` round trips) | 512 ms | 403 ms | **1.3× faster** | + +Allocation counts drop with it: 1.12M vs 1.64M allocs (−32%) and 21 MB vs 30 MB +(−30%) on the chunk load, 20.3k vs 27.3k on the lookups. + +## Confirmed on production + +The table above comes from a standalone harness. @efiten then ran both drivers +against a real instance — 11,077,038 observations, 9.7 GB database, 4-core arm64 +— as server-only containers against the same live volume, one at a time. Round 2 +reverses the order so the page-cache advantage goes to the old driver: + +| | audit 7d | audit 24h | background fill (13 chunks) | start → /api/health | +|---|---:|---:|---:|---:| +| modernc, round 1 | 16.67s | 2.27s | 130.2s | 16.6s | +| mattn, round 1 | 7.87s | 1.34s | 93.8s | 13.5s | +| mattn, round 2 | 8.15s | 1.35s | 96.4s | 13.0s | +| modernc, round 2 | 13.46s | 2.29s | 137.8s | 15.5s | + +Warm, the old driver improves to 13.46s on the 7d audit and still loses by +~1.8×. Chunk load is ~1.4×. `/api/nodes?limit=500` is 0.039s against 0.037s — +i.e. nothing. + +**Real-world gains are smaller than the harness suggests: ~1.4-1.8× on the +paths that matter, not 2-2.3×.** The shape holds — scans and joins gain, small +lookups do not — but quote these numbers, not the harness ones. + +**Build time is the counterweight**, cold and native on that machine: +**52s on master, 163s on this branch**. An instance that builds its own image +pays that on every deploy. + +## The cost: the build is cgo now + +`CGO_ENABLED=0` still *builds*, which is the trap: mattn links a stub, and the +binary dies on its first query with `go-sqlite3 requires cgo to work. This is a +stub`. A green build proves nothing here. `GOOS=linux go build` from a Mac, on +the other hand, genuinely cannot cross-compile any more — cgo needs a C compiler +that can target the other platform. That compiler is +[`zig`](https://ziglang.org/download/): + +```bash +make build # host +make crossbuild # static linux/amd64 + linux/arm64 via `zig cc -target …-linux-musl` +``` + +Targeting musl and linking with `-extldflags "-static -Wl,-s"` keeps the output a +single self-contained binary, so the `alpine:3.20` runtime image no longer +depends on the base image's libc at all. `-Wl,-s` matters: Go's own `-s -w` does +not reach the musl objects zig links in, and without it the server binary is +19.8 MB instead of 12.1 MB. + +The `Dockerfile` builder stage installs a checksum-pinned zig and does the same +thing, still on a single `$BUILDPLATFORM` builder with no QEMU for compilation. +Build-cache mounts are not optional there: compiling the SQLite amalgamation +twice from cold takes over half an hour. + +## The five behavioural differences + +### 1. Statement preparation is eager + +modernc's `newStmt` only stored the SQL and compiled lazily on first use; mattn +calls `sqlite3_prepare_v2` inside `Prepare`. SQL referencing a missing table or +column now fails at **open** time. + +This is the migration's largest single effect: 59 server tests failed on it, +purely from fixtures with partial schemas. `OpenDB` keeps failing loudly (that is +the #1901 behaviour we want in production, and `cmd/server/main.go` also gates on +`dbschema.AssertReady`); the fixtures instead declare what they are prepared +against, via `ensurePreparable` in `cmd/server/preparable_schema_test.go`. If you +add a prepared statement referencing something new, +`TestEnsurePreparableMatchesPrepareStatements` tells you to extend that helper. + +It also surfaced nine `nodes(pubkey …)` declarations across seven files, when +production has only ever had `public_key`. Lazy compilation had hidden the +mismatch. + +### 2. It exposed a real bug in the observation UPSERT + +`stmtInsertObservation` resolves `ON CONFLICT(transmission_id, observer_idx, +COALESCE(path_json, ''))` against the unique expression index +`idx_observations_dedup` — which `cmd/ingestor/db.go` only ever created inside +the branch that creates the `observations` table for the first time. Databases +whose table predates that branch never had one, so the UPSERT had no conflict +target. modernc failed on the first insert; mattn fails at `OpenStore`. Same bug, +found earlier. + +`internal/dbschema` now creates it unconditionally. Because the index is what was +supposed to prevent duplicates, a database that never had it can already hold +rows violating it — the repo's own `test-fixtures/e2e-fixture.db` held one — so +duplicates are collapsed first. Refusing would not have been safer: without the +index the ingestor cannot prepare its UPSERT, so it cannot start at all. + +The collapse has to *replay* the UPSERT, and getting that subtly wrong is easy. +`DO UPDATE SET snr = COALESCE(excluded.snr, snr)` means the **incoming** value +wins when it is non-NULL, so down a group in id order the survivor ends up with +the **last** non-NULL value, not the first. It also names exactly five columns — +`snr`, `rssi`, `score`, `raw_hex`, `resolved_path` — so every other column keeps +the surviving row's own value; merging those too would invent history the +ingestor would never have written. An earlier version of this change took the +first non-NULL value and merged every column, which silently discarded newer +readings. + +Merge, delete and index creation all share one transaction. Split apart, a +writer inserting a duplicate in the gap makes the index creation fail while the +deletions stay committed — rows destroyed and no index to show for it. + +The repair logs what it is about to destroy — group count, rows to remove, and +the first 20 group keys with the id it keeps — before deleting anything, because +a bare row count is not enough to reconstruct from if the merge direction is +ever wrong again. It also says up front that it holds the write lock, since on a +large table that pause at ingestor startup is otherwise unexplained. + +Two hazards worth knowing before an upgrade: + +- **Reads inside the repair must use the transaction, not the pool.** + `cmd/ingestor` runs `SetMaxOpenConns(1)`, so a query issued against the pool + while the repair holds its transaction waits for a connection that + transaction has checked out, forever. It deadlocked a staging ingestor at + boot — logs stop after "Repairing now", the write lock is free, the process + is simply blocked. Anything reading in there takes a `Querier` and is passed + `tx`. +- **NULL is not a duplicate.** `GROUP BY` folds NULLs into one group; a UNIQUE + index treats them as distinct, so a row with a NULL in an indexed column can + never violate it. Grouping without excluding them does not delete the rows — + the `DELETE` joins on `observer_idx = observer_idx`, which NULL never + satisfies — it is the *merge* that does the damage, matching nothing and + writing NULL over the survivor's real readings. The rows stay put and their + measurements disappear. On an 11.2M-row instance 198 of 222 reported groups + were `observer_idx IS NULL`. + +The failing `CREATE UNIQUE INDEX` that triggers all this is itself a stall: on +11.2M rows it held the write lock long enough for a concurrent writer to hit the +full 5s `busy_timeout`. So an instance that needs the repair pauses writers for +seconds *before* the repair starts. + +The repair itself does not hold the write lock throughout, tempting as that is +to assume. The grouping scan (~18s at that size) reads, and a concurrent writer +can proceed during it; the lock is taken when the repair first writes. A writer +that gets in between makes the repair fail and roll back, which is the intended +outcome — but "holds the write lock until it completes" is the wrong mental +model. + +The collapse itself is measured at 4.1s on 2.4M synthetic rows holding 5 +duplicates. That is well short of a real deployment: an 11M-row instance has not +been measured to completion, and the collapse has not been run against a +database an ingestor is actively writing to. + +Note `COALESCE(path_json, '')` makes a NULL path and an empty-string path the +same key, but `NULL` and `'[]'` different keys. See +`internal/dbschema/dedup_index_test.go`, where the last-wins and atomicity +properties each have a test. + +### 3. `synchronous` silently dropped from FULL to NORMAL + +mattn defaults `synchronousMode` to `NORMAL` and runs `PRAGMA synchronous = +NORMAL` unconditionally, where SQLite's own compile default (what modernc left in +place) is FULL. In WAL mode that changes durability under power loss. + +The writer DSN pins `_synchronous=FULL`, and lives in one place — +`dbschema.WriterDSN` — because there are two writers. `cmd/migrate` originally +kept a bare path and so silently wrote at NORMAL, which is exactly what a second +copy of a DSN buys you. `TestOpenStorePragmas` reads every pragma back +**through the store's own connection**, and `TestWriterDSNPragmas` covers the DSN +itself; a separate `sqlite3` session or the startup log line would prove +nothing. + +### 4. The DSN dialect is different, and each driver ignores the other's + +modernc understood only `_pragma=name(value)`; mattn understands only +`_`-prefixed parameters. Neither errors on the other's form, so a driver-only +rename would have dropped every pragma on the floor in silence. All five +`_pragma=` DSNs were rewritten (one production, four test seeds). + +Also removed: `_journal_mode=WAL` on the server's read handle. modernc had been +ignoring it all along; mattn honours it, and setting `journal_mode` on a +read-only connection is a write. Dropping `_busy_timeout` with it costs nothing — +mattn's default is already 5000 ms, so the read handle finally *gets* the busy +timeout it had silently lacked. + +### 5. `mode=ro` still works — but not for the reason you would guess + +mattn always passes `SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE` to +`sqlite3_open_v2`, and the bundled amalgamation has `SQLITE_USE_URI=0`. What +makes `file:…?mode=ro` work anyway is that mattn's C wrapper `_sqlite3_open_v2` +ORs `SQLITE_OPEN_URI` into the flags itself. So the read-only invariant from +#1283/#1289 holds with no build flags — but it depends on the `file:` prefix +being present. `TestOpenDBRefusesMissingDatabase` fails if any of that stops +holding. + +`cmd/decrypt` had been building its DSN *without* the `file:` prefix, so both +drivers stripped the query string and its `mode=ro` had never applied — a missing +path was created read-write. Fixed in passing; it was never a migration +regression. + +## Memory + +`runtime/debug.SetMemoryLimit` (GOMEMLIMIT) covers what the Go runtime manages — +heap, stacks, runtime structures — and nothing else, so it is not an RSS ceiling +now that SQLite allocates in C. + +What is bounded is the page cache specifically: both DSNs pin +`_cache_size=-2000`, i.e. ~2 MiB per connection, so ~8 MiB across the server's +`SetMaxOpenConns(4)` and ~2 MiB in the ingestor, comfortably inside the 1.5× +headroom `applyMemoryLimit` derives. That caps the page cache, not everything +SQLite allocates — statement and schema memory sit outside it — so revisit +against measured RSS if the connection count or `_cache_size` grows, or if a +workload starts holding many prepared statements. + +There is no cgo-bytes metric because Go exposes no counter for one. Do not read +`processRSSMB - goSysMB` as the C share either: `goSysMB` is reserved address +space rather than resident memory, so the subtraction mixes two different +quantities. It is a smell test, not a measurement. + +## Things that did not change + +No modernc-specific API was in use — no `RegisterFunction`, no `*sqlite.Conn`, no +`modernc.org/sqlite/lib` error constants, no `sql.Register`. No `time.Time` is +ever bound as a query argument (every retention cutoff is pre-formatted), so +driver time handling is not in play. Both drivers convert declared +`DATE`/`DATETIME`/`TIMESTAMP` columns to `time.Time`, so `/api/dropped-packets` +keeps emitting `dropped_at` as RFC3339 exactly as before. diff --git a/internal/dbschema/dbschema.go b/internal/dbschema/dbschema.go index 8518be281..433ee8de0 100644 --- a/internal/dbschema/dbschema.go +++ b/internal/dbschema/dbschema.go @@ -43,6 +43,9 @@ func Apply(rw *sql.DB, logf Logger) error { if err := ensureServerIndexes(rw); err != nil { return fmt.Errorf("ensure server indexes: %w", err) } + if err := ensureObservationsDedupIndex(rw, logf); err != nil { + return fmt.Errorf("ensure observations dedup index: %w", err) + } if err := ensureNeighborEdgesTable(rw); err != nil { return fmt.Errorf("ensure neighbor_edges: %w", err) } @@ -189,9 +192,27 @@ func AssertReady(ro *sql.DB) error { return nil } +// Querier is the read surface shared by *sql.DB and *sql.Tx. +// +// Not *sql.Conn: it exposes only QueryContext/QueryRowContext, so it does not +// satisfy this. Widen the interface to the Context variants if a caller ever +// needs one. +// +// It exists so schema probes can run on whichever handle the caller already +// holds. Taking *sql.DB unconditionally is a deadlock waiting to happen: a +// caller inside a transaction has the connection checked out, and on a pool +// capped at one connection — which is what cmd/ingestor runs — a probe against +// the pool waits forever for the connection its own transaction is holding. +type Querier interface { + Query(query string, args ...any) (*sql.Rows, error) + QueryRow(query string, args ...any) *sql.Row +} + // TableHasColumn reports whether the given table has the given column. // Exported because tests and the read-side need it without re-implementing. -func TableHasColumn(db *sql.DB, table, column string) (bool, error) { +// +// Pass the transaction, not the pool, when you are inside one. See Querier. +func TableHasColumn(db Querier, table, column string) (bool, error) { rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%s)", table)) if err != nil { return false, err diff --git a/internal/dbschema/dbschema_test.go b/internal/dbschema/dbschema_test.go index 576c8eec6..5145777b5 100644 --- a/internal/dbschema/dbschema_test.go +++ b/internal/dbschema/dbschema_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) // minimalDB bootstraps a SQLite DB with just enough tables for the @@ -16,7 +16,7 @@ func minimalDB(t *testing.T) *sql.DB { t.Helper() dir := t.TempDir() dbPath := filepath.Join(dir, "schema.db") - db, err := sql.Open("sqlite", "file:"+dbPath+"?_journal_mode=WAL") + db, err := sql.Open("sqlite3", "file:"+dbPath+"?_journal_mode=WAL") if err != nil { t.Fatal(err) } diff --git a/internal/dbschema/dedup_index.go b/internal/dbschema/dedup_index.go new file mode 100644 index 000000000..fb0f728fc --- /dev/null +++ b/internal/dbschema/dedup_index.go @@ -0,0 +1,314 @@ +package dbschema + +import ( + "database/sql" + "errors" + "fmt" + "strconv" + + sqlite3 "github.com/mattn/go-sqlite3" +) + +// upsertMergedColumns are the columns the ingestor's observation UPSERT writes +// on conflict, in cmd/ingestor/db.go: +// +// ON CONFLICT(...) DO UPDATE SET +// snr = COALESCE(excluded.snr, snr), +// rssi = COALESCE(excluded.rssi, rssi), +// score = COALESCE(excluded.score, score), +// raw_hex = COALESCE(excluded.raw_hex, raw_hex), +// resolved_path = COALESCE(excluded.resolved_path, resolved_path) +// +// Two things follow, and collapseDuplicatesAndIndex has to honour both to be a +// faithful replay rather than an approximation of it. +// +// First, `excluded` is the incoming row, so a non-NULL later value REPLACES an +// earlier one. Applied down a group in insertion order the result is the LAST +// non-NULL value, not the first. +// +// Second, every other column is absent from the SET list, so the UPSERT never +// touches it: the surviving row keeps its own direction, timestamp, path_json +// and so on. Merging those too would invent history the ingestor would not have +// written. +var upsertMergedColumns = []string{"snr", "rssi", "score", "raw_hex", "resolved_path"} + +// dupGroupsDDL materialises one row per duplicate group: the group key plus the +// earliest id, which is the row that is kept and merged into. +// +// Materialised rather than left as a repeated subquery on purpose. In the +// subquery form every merged column's UPDATE re-ran this GROUP BY over the whole +// observations table — measured at 9.7s on 2.4M rows holding only 5 duplicates, +// nearly all of it the same scan performed repeatedly. This runs inside a write +// transaction and holds the write lock for its duration, so the repetition is +// worth removing. +// The WHERE clause is the whole correctness of this query, and leaving it out +// destroys data. +// +// GROUP BY folds all NULLs into one group. A UNIQUE index does the opposite: +// SQLite treats NULLs as distinct, so a row with NULL in any indexed column can +// never violate it. Group without excluding them and the repair calls rows +// duplicates that the index would have accepted, then deletes them to build an +// index that did not need them gone. +// +// Not hypothetical: on an 11.2M-row instance, 198 of the 222 groups this +// reported had observer_idx IS NULL — 238 of the 262 rows it planned to delete. +// They were direction='tx' rows, and the index built without complaint once +// they were left alone. +// +// transmission_id is NOT NULL in the schema and COALESCE(path_json, ”) can +// never be NULL, so observer_idx is the only one that needs the guard; it is +// written out in full anyway, because the next person to add a column to this +// index should see the rule rather than infer it. +const dupGroupsDDL = `CREATE TEMP TABLE dedup_groups AS + SELECT transmission_id, observer_idx, COALESCE(path_json, '') AS p, + MIN(id) AS keep, COUNT(*) AS n + FROM observations + WHERE transmission_id IS NOT NULL + AND observer_idx IS NOT NULL + GROUP BY transmission_id, observer_idx, COALESCE(path_json, '') + HAVING COUNT(*) > 1` + +const dedupIndexDDL = `CREATE UNIQUE INDEX IF NOT EXISTS idx_observations_dedup ` + + `ON observations(transmission_id, observer_idx, COALESCE(path_json, ''))` + +// ensureObservationsDedupIndex creates the unique expression index that the +// ingestor's observation UPSERT resolves its ON CONFLICT target against: +// +// ON CONFLICT(transmission_id, observer_idx, COALESCE(path_json, '')) +// +// cmd/ingestor/db.go creates this index, but only inside the branch that +// creates the observations table for the first time. Any database whose +// observations table predates that branch therefore never got one, so the +// UPSERT had no conflict target to resolve against. Under modernc.org/sqlite +// that surfaced late — statements compiled lazily, so it failed on the first +// insert. github.com/mattn/go-sqlite3 compiles inside Prepare, so it now +// surfaces at OpenStore. Same bug either way; creating the index here repairs +// the database rather than just moving the error. +// +// Because the index is what was supposed to prevent duplicates, a database that +// never had it can already hold rows violating it — the repo's own +// test-fixtures/e2e-fixture.db did. Those rows are collapsed first. Refusing +// would not be safer: without the index the ingestor cannot prepare its UPSERT, +// so it cannot run at all. +// +// Skipped on v2 schemas (observer_id rather than observer_idx): the UPSERT does +// not apply there, and indexing a column that does not exist would fail. +func ensureObservationsDedupIndex(rw *sql.DB, logf Logger) error { + hasIdx, err := TableHasColumn(rw, "observations", "observer_idx") + if err != nil { + return err + } + if !hasIdx { + return nil + } + // Fast path: no duplicates, so the index just builds. Atomic on its own. + _, err = rw.Exec(dedupIndexDDL) + if err == nil { + return nil + } + if !isConstraintViolation(err) { + return err + } + + // Deleting rows is not something to do quietly, and on a large table this + // holds the write lock long enough that an operator watching startup + // deserves to know why before it happens rather than after. + logf("[dbschema] idx_observations_dedup cannot be created: duplicate observations exist. " + + "Repairing now — on a large observations table this takes tens of seconds, and " + + "writers are blocked for part of it.") + + removed, err := collapseDuplicatesAndIndex(rw, logf) + if err != nil { + return fmt.Errorf("collapse duplicate observations: %w", err) + } + logf("[dbschema] collapsed %d duplicate observation row(s); idx_observations_dedup created", removed) + return nil +} + +// isConstraintViolation reports whether err is SQLite refusing a constraint. +// +// Deliberately a typed check. This used to match on +// strings.Contains(err, "UNIQUE constraint failed"), which made the entire +// repair path — including the row deletion — hinge on the driver's prose, in +// the same change that swapped the driver. A reworded or wrapped message would +// silently skip the repair and surface later as an OpenStore failure with no +// hint as to why. CREATE UNIQUE INDEX can only hit a constraint error because +// the data violates the uniqueness it asks for, so the primary code is the +// right granularity. +func isConstraintViolation(err error) bool { + var se sqlite3.Error + if errors.As(err, &se) { + return se.Code == sqlite3.ErrConstraint + } + return false +} + +// collapseDuplicatesAndIndex merges duplicate observation rows into the lowest +// id of each group, deletes the rest, and creates the unique index — all in one +// transaction. It returns the number of rows deleted. +// +// The index creation belongs inside the same transaction as the deletion. With +// them separated, a writer inserting a duplicate in the gap makes the index +// creation fail while leaving the deletions committed: rows destroyed and no +// index to show for it. One transaction makes the repair all-or-nothing. +func collapseDuplicatesAndIndex(rw *sql.DB, logf Logger) (int64, error) { + tx, err := rw.Begin() + if err != nil { + return 0, err + } + defer func() { _ = tx.Rollback() }() + + // Every reference to the TEMP table goes through tx, so its whole lifecycle + // sits on one connection: created here, dropped before COMMIT below, and + // removed by ROLLBACK on any error path (a TEMP table created inside a + // transaction does not survive its rollback — verified, not assumed). + // + // It has to be tx, not rw. A TEMP table belongs to a single connection and + // database/sql hands out pooled ones, so an `rw.Exec` drop can land on a + // different connection and leave the table behind on the one that has it. + // The leading DROP is still here because this package cannot prove nothing + // else ever left one on this connection. + if _, err := tx.Exec(`DROP TABLE IF EXISTS temp.dedup_groups`); err != nil { + return 0, fmt.Errorf("drop stale dedup_groups: %w", err) + } + if _, err := tx.Exec(dupGroupsDDL); err != nil { + return 0, fmt.Errorf("build dedup_groups: %w", err) + } + if _, err := tx.Exec(`CREATE INDEX temp.dedup_groups_key + ON dedup_groups(transmission_id, observer_idx, p)`); err != nil { + return 0, fmt.Errorf("index dedup_groups: %w", err) + } + + // The group keys exist right now and cease to after the delete. Record them + // before destroying anything: a row count is not enough to reconstruct from + // if the merge direction is ever wrong again. + if err := logDuplicateGroups(tx, logf); err != nil { + return 0, err + } + + // Only merge columns this database actually has. Apply runs this step + // before ensureResolvedPathColumn and ensureObservationsRawHexColumn, so on + // a database old enough to be missing the dedup index, resolved_path and + // raw_hex may not exist yet either. + cols, err := existingColumns(tx, "observations", upsertMergedColumns) + if err != nil { + return 0, err + } + + // Replay the UPSERT: for each merged column take the LAST non-NULL value in + // the group by id. The subquery spans the whole group, the survivor + // included, so when every row is NULL the result is NULL — which is what + // the UPSERT would also have left behind. + for _, c := range cols { + q := `UPDATE observations SET ` + c + ` = ( + SELECT o2.` + c + ` FROM observations o2 + WHERE o2.transmission_id = observations.transmission_id + AND o2.observer_idx = observations.observer_idx + AND COALESCE(o2.path_json, '') = COALESCE(observations.path_json, '') + AND o2.` + c + ` IS NOT NULL + ORDER BY o2.id DESC LIMIT 1) + WHERE id IN (SELECT keep FROM dedup_groups)` + if _, err := tx.Exec(q); err != nil { + return 0, fmt.Errorf("merge %s: %w", c, err) + } + } + + res, err := tx.Exec(`DELETE FROM observations WHERE id IN ( + SELECT o.id FROM observations o + JOIN dedup_groups d + ON d.transmission_id = o.transmission_id + AND d.observer_idx = o.observer_idx + AND d.p = COALESCE(o.path_json, '') + WHERE o.id <> d.keep)`) + if err != nil { + return 0, err + } + removed, err := res.RowsAffected() + if err != nil { + return 0, err + } + + // Same transaction: if this fails, the deletions above roll back with it. + if _, err := tx.Exec(`DROP TABLE temp.dedup_groups`); err != nil { + return 0, fmt.Errorf("drop dedup_groups: %w", err) + } + + if _, err := tx.Exec(dedupIndexDDL); err != nil { + return 0, fmt.Errorf("create idx_observations_dedup after collapsing %d row(s): %w", removed, err) + } + if err := tx.Commit(); err != nil { + return 0, err + } + return removed, nil +} + +// existingColumns filters want down to the columns table actually has, keeping +// the given order. +// +// Takes a Querier, not *sql.DB, and callers inside a transaction must pass the +// transaction. Probing the pool from inside one deadlocks the ingestor, whose +// pool is capped at a single connection: the probe waits for the connection the +// transaction is holding, and nothing ever releases it. +func existingColumns(rw Querier, table string, want []string) ([]string, error) { + out := make([]string, 0, len(want)) + for _, c := range want { + ok, err := TableHasColumn(rw, table, c) + if err != nil { + return nil, err + } + if ok { + out = append(out, c) + } + } + return out, nil +} + +// maxLoggedDupGroups bounds the audit log. A database missing the index for +// long enough can have a great many duplicate groups, and an unbounded dump at +// startup is its own operational problem. +const maxLoggedDupGroups = 20 + +// logDuplicateGroups records what is about to be collapsed, while dedup_groups +// still holds it. Reads through tx because the TEMP table lives on that +// transaction's connection. +func logDuplicateGroups(tx *sql.Tx, logf Logger) error { + var groups, extra int64 + if err := tx.QueryRow(`SELECT COUNT(*), COALESCE(SUM(n - 1), 0) FROM dedup_groups`). + Scan(&groups, &extra); err != nil { + return fmt.Errorf("count dedup_groups: %w", err) + } + logf("[dbschema] %d duplicate observation group(s), %d row(s) to remove", groups, extra) + + rows, err := tx.Query(`SELECT transmission_id, observer_idx, p, keep, n + FROM dedup_groups ORDER BY keep LIMIT ?`, maxLoggedDupGroups) + if err != nil { + return fmt.Errorf("list dedup_groups: %w", err) + } + defer rows.Close() + var listed int64 + for rows.Next() { + var txID, keep, n int64 + var observerIdx sql.NullInt64 + var pathJSON string + if err := rows.Scan(&txID, &observerIdx, &pathJSON, &keep, &n); err != nil { + return fmt.Errorf("scan dedup_groups: %w", err) + } + // Print NULL as NULL. Rendering observerIdx.Int64 unconditionally shows + // a NULL as 0, which is the one value that would hide the grouping bug + // above from the only person able to notice it. + idx := "NULL" + if observerIdx.Valid { + idx = strconv.FormatInt(observerIdx.Int64, 10) + } + logf("[dbschema] transmission_id=%d observer_idx=%s path_json=%q rows=%d keeping id=%d", + txID, idx, pathJSON, n, keep) + listed++ + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterate dedup_groups: %w", err) + } + if groups > listed { + logf("[dbschema] ... and %d more group(s) not listed", groups-listed) + } + return nil +} diff --git a/internal/dbschema/dedup_index_test.go b/internal/dbschema/dedup_index_test.go new file mode 100644 index 000000000..550e328d9 --- /dev/null +++ b/internal/dbschema/dedup_index_test.go @@ -0,0 +1,540 @@ +package dbschema + +import ( + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + sqlite3 "github.com/mattn/go-sqlite3" +) + +// observationsDB builds a database with an observations table but deliberately +// no idx_observations_dedup — the shape of every database created before +// cmd/ingestor/db.go started making that index, and of any database whose +// observations table it never ran against. +func observationsDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite3", "file:"+filepath.Join(t.TempDir(), "obs.db")+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + // One connection, like cmd/ingestor. An unbounded pool hides any code that + // queries the pool while holding a transaction: it quietly opens a second + // connection instead of deadlocking, so the suite passes and production + // hangs. Every fixture here must match the tightest pool in production. + db.SetMaxOpenConns(1) + // The full observations shape, including all five columns the ingestor's + // UPSERT merges. An earlier fixture omitted score, raw_hex and + // resolved_path, so removing those three from upsertMergedColumns left the + // entire suite green — three fifths of the merge was untested. + if _, err := db.Exec(`CREATE TABLE observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + transmission_id INTEGER NOT NULL, + observer_idx INTEGER, + direction TEXT, + snr REAL, + rssi REAL, + score INTEGER, + path_json TEXT, + timestamp INTEGER, + raw_hex TEXT, + resolved_path TEXT + )`); err != nil { + t.Fatal(err) + } + return db +} + +func dedupIndexExists(t *testing.T, db *sql.DB) bool { + t.Helper() + var n int + if err := db.QueryRow( + `SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name='idx_observations_dedup'`, + ).Scan(&n); err != nil { + t.Fatal(err) + } + return n == 1 +} + +func TestEnsureObservationsDedupIndexOnCleanTable(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec( + `INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, 1, '[]', 10), (1, 2, '[]', 10)`, + ); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + if !dedupIndexExists(t, db) { + t.Error("index was not created") + } +} + +// The in-transaction DROP only matters on the success path: on failure ROLLBACK +// removes the temp table regardless. Replacing that DROP with a no-op survived +// every other test here, because none of them looked at the temp schema after a +// repair that worked. +// +// It matters because the connection goes back to the pool. A leftover +// dedup_groups is stale data the next repair would otherwise inherit. +func TestCollapseLeavesNoTempTableAfterSuccess(t *testing.T) { + db := observationsDB(t) // SetMaxOpenConns(1), so this is the same connection + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 21, 9, '[]', 10), (2, 21, 9, '[]', 10)`); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + var n int + if err := db.QueryRow( + `SELECT COUNT(*) FROM temp.sqlite_master WHERE name LIKE 'dedup_groups%'`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 0 { + t.Errorf("%d temp dedup_groups object(s) left on the connection after a successful repair", n) + } +} + +// The interesting case: duplicates already present, which is what blocked the +// index from being created in the first place. They must be collapsed, and the +// survivor must inherit the non-NULL fields of the rows that went away — the +// same merge the ingestor's ON CONFLICT ... DO UPDATE SET x = COALESCE(...) +// would have performed had the index existed. +func TestEnsureObservationsDedupIndexCollapsesDuplicates(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, path_json, timestamp) VALUES + (1, 5, 3, 'rx', NULL, -90, '[]', 100), + (2, 5, 3, NULL, 7.5, NULL, '[]', 100), + (3, 5, 3, NULL, NULL, NULL, NULL, 100), + (4, 9, 1, 'rx', 1.0, -80, '["AA"]', 200)`); err != nil { + t.Fatal(err) + } + + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + if !dedupIndexExists(t, db) { + t.Fatal("index was not created after collapsing duplicates") + } + + // Rows 1 and 2 share the key (5, 3, '[]') and collapse into id 1. + // + // Row 3 does NOT join them: its path_json is NULL, and COALESCE(path_json,'') + // makes that the empty string, a different key from '[]'. That is the + // ingestor's conflict target verbatim, so an unrecorded path and an + // explicitly empty path are distinct observations here — worth pinning, + // because from the outside it looks like it ought to be one group. + // Row 4 has its own key and is untouched. + var ids string + if err := db.QueryRow(`SELECT GROUP_CONCAT(id) FROM (SELECT id FROM observations ORDER BY id)`).Scan(&ids); err != nil { + t.Fatal(err) + } + if ids != "1,3,4" { + t.Errorf("surviving ids = %q, want \"1,3,4\" (lowest id of each group; NULL path_json is its own group)", ids) + } + + var direction sql.NullString + var snr, rssi sql.NullFloat64 + if err := db.QueryRow(`SELECT direction, snr, rssi FROM observations WHERE id = 1`).Scan(&direction, &snr, &rssi); err != nil { + t.Fatal(err) + } + if direction.String != "rx" { + t.Errorf("direction = %q, want \"rx\" (its own value)", direction.String) + } + if snr.Float64 != 7.5 { + t.Errorf("snr = %v, want 7.5 (merged from the row that was removed)", snr.Float64) + } + if rssi.Float64 != -90 { + t.Errorf("rssi = %v, want -90 (its own value, not overwritten by a NULL)", rssi.Float64) + } +} + +// The merge has to replay the UPSERT, and the UPSERT's +// `COALESCE(excluded.x, x)` means a later non-NULL value REPLACES an earlier +// one. Complementary NULLs (as above) cannot tell "first non-NULL wins" from +// "last non-NULL wins" — both produce the same answer — so this asserts the +// direction explicitly with values that conflict. +func TestEnsureObservationsDedupIndexKeepsLatestValues(t *testing.T) { + db := observationsDB(t) + // One group, three rows, every merged column non-NULL and different. + // path_json is identical so they collide; direction is NOT in the UPSERT's + // SET list, so the survivor must keep its own. + // All five merged columns differ across the group, so each one proves the + // direction independently. direction/timestamp are NOT in the UPSERT's SET + // list and must keep the survivor's own values. + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, score, path_json, timestamp, raw_hex, resolved_path) VALUES + (1, 7, 2, 'first', 1.0, -10, 11, '[]', 100, 'aa', '["a"]'), + (2, 7, 2, 'second', 7.0, -20, 22, '[]', 200, 'bb', '["b"]'), + (3, 7, 2, 'third', 9.0, -30, 33, '[]', 300, 'cc', '["c"]')`); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + + var direction, rawHex, resolvedPath string + var snr, rssi float64 + var score, timestamp int64 + if err := db.QueryRow(`SELECT direction, snr, rssi, score, raw_hex, resolved_path, timestamp + FROM observations WHERE id = 1`).Scan( + &direction, &snr, &rssi, &score, &rawHex, &resolvedPath, ×tamp); err != nil { + t.Fatal(err) + } + // Merged: last non-NULL down the group wins. + for _, c := range []struct { + name string + got, want any + }{ + {"snr", snr, 9.0}, + {"rssi", rssi, -30.0}, + {"score", score, int64(33)}, + {"raw_hex", rawHex, "cc"}, + {"resolved_path", resolvedPath, `["c"]`}, + } { + if c.got != c.want { + t.Errorf("%s = %v, want %v (last non-NULL: COALESCE(excluded.x, x) lets later rows win)", c.name, c.got, c.want) + } + } + // Not merged: the UPSERT never SETs these, so the survivor keeps its own. + if direction != "first" { + t.Errorf("direction = %q, want \"first\": not in the UPSERT's SET list", direction) + } + if timestamp != 100 { + t.Errorf("timestamp = %d, want 100: not in the UPSERT's SET list", timestamp) + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("observations = %d, want 1", n) + } +} + +// The repair must be all-or-nothing. If the index cannot be created, the +// deletions must not survive: rows destroyed with no index to show for it is +// the worst outcome available. +func TestCollapseDuplicatesAndIndexIsAtomic(t *testing.T) { + db := observationsDB(t) + // Conflicting readings, so the merge has something to write. Checking only + // the row count let a version that committed the merge early pass: the rows + // came back but their values did not. + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, snr, rssi, path_json, timestamp) VALUES + (1, 1, 1, 1.0, -10, '[]', 10), + (2, 1, 1, 2.0, -20, '[]', 10)`); err != nil { + t.Fatal(err) + } + // Occupy the index name with a TABLE. `CREATE INDEX IF NOT EXISTS` only + // shrugs when an *index* of that name exists; a table of that name is an + // error, so the CREATE fails after the merge and delete have already run. + if _, err := db.Exec(`CREATE TABLE idx_observations_dedup (x INTEGER)`); err != nil { + t.Fatal(err) + } + + if _, err := collapseDuplicatesAndIndex(db, t.Logf); err == nil { + t.Fatal("expected index creation to fail") + } + + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 2 { + t.Errorf("observations = %d, want 2: a failed index creation must roll the deletions back", n) + } + // The merge must roll back too, not just the delete. + var snr, rssi float64 + if err := db.QueryRow(`SELECT snr, rssi FROM observations WHERE id = 1`).Scan(&snr, &rssi); err != nil { + t.Fatal(err) + } + if snr != 1.0 || rssi != -10 { + t.Errorf("id=1 snr=%v rssi=%v, want 1/-10: the merge must roll back with the delete", snr, rssi) + } + // And nothing may be left in the temp schema for the next user of this + // connection. Replacing the in-transaction DROP with a no-op used to pass. + var temps int + if err := db.QueryRow(`SELECT COUNT(*) FROM temp.sqlite_master WHERE name LIKE 'dedup_groups%'`).Scan(&temps); err != nil { + t.Fatal(err) + } + if temps != 0 { + t.Errorf("temp.dedup_groups survived a failed repair (%d objects)", temps) + } +} + +// Running twice must be a no-op: Apply runs on every ingestor start. +func TestEnsureObservationsDedupIndexIsIdempotent(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (transmission_id, observer_idx, path_json, timestamp) VALUES (1, 1, '[]', 10), (1, 1, '[]', 10)`); err != nil { + t.Fatal(err) + } + for i := range 2 { + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("pass %d: %v", i+1, err) + } + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("observations = %d, want 1", n) + } +} + +// v2 schemas key observations by observer_id, not observer_idx. The ingestor's +// UPSERT does not apply there, and indexing a missing column would error, so the +// step must skip rather than fail. +func TestEnsureObservationsDedupIndexSkipsV2Schema(t *testing.T) { + db, err := sql.Open("sqlite3", "file:"+filepath.Join(t.TempDir(), "v2.db")+"?_journal_mode=WAL") + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE observations (id INTEGER PRIMARY KEY, transmission_id INTEGER, observer_id TEXT)`); err != nil { + t.Fatal(err) + } + if err := ensureObservationsDedupIndex(db, t.Logf); err != nil { + t.Fatalf("expected a skip on a v2 schema, got: %v", err) + } + if dedupIndexExists(t, db) { + t.Error("index must not be created on a v2 schema") + } +} + +// Nothing covered the branch that DECIDES to repair. TestCollapseDuplicates... +// calls collapseDuplicatesAndIndex directly, so a broken error check in +// ensureObservationsDedupIndex would leave every one of those tests green while +// production silently skipped the repair and failed later at OpenStore. +// +// This asserts the decision: duplicates present, the real driver's real error, +// and the repair actually taken. It is the test that would catch the driver +// rewording its constraint message. +func TestEnsureObservationsDedupIndexTakesRepairPathOnRealDriverError(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 4, 4, '[]', 10), (2, 4, 4, '[]', 10)`); err != nil { + t.Fatal(err) + } + + // The error the fast path actually gets. If isConstraintViolation stops + // recognising this, the repair below never runs. + _, createErr := db.Exec(dedupIndexDDL) + if createErr == nil { + t.Fatal("expected CREATE UNIQUE INDEX to fail over duplicates") + } + if !isConstraintViolation(createErr) { + t.Fatalf("isConstraintViolation did not recognise the driver's own error: %v (%T)", createErr, createErr) + } + + // The above passes just as happily with a strings.Contains check, because + // the current driver still says "UNIQUE constraint failed". These pin the + // typed behaviour itself: a constraint error whose text says nothing of the + // sort must still be recognised, a wrapped one must be unwrapped, and an + // unrelated error carrying the magic phrase must NOT trigger a repair that + // deletes rows. + typed := sqlite3.Error{Code: sqlite3.ErrConstraint, ExtendedCode: sqlite3.ErrConstraintUnique} + if !isConstraintViolation(typed) { + t.Error("a bare sqlite3.Error with ErrConstraint was not recognised") + } + if !isConstraintViolation(fmt.Errorf("create index: %w", typed)) { + t.Error("a wrapped constraint error was not recognised; errors.As should unwrap it") + } + if isConstraintViolation(errors.New("UNIQUE constraint failed: index 'x'")) { + t.Error("a plain error carrying the message was treated as a constraint violation — " + + "that is the string match this check exists to replace") + } + if isConstraintViolation(sqlite3.Error{Code: sqlite3.ErrBusy}) { + t.Error("SQLITE_BUSY was treated as a constraint violation; a locked database must not start deleting rows") + } + + var repaired bool + logf := func(format string, args ...interface{}) { + repaired = true + t.Logf(format, args...) + } + if err := ensureObservationsDedupIndex(db, logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + if !repaired { + t.Error("repair path was not taken: no log output, so the constraint error was not recognised") + } + if !dedupIndexExists(t, db) { + t.Error("index missing after repair") + } + var n int + if err := db.QueryRow(`SELECT COUNT(*) FROM observations`).Scan(&n); err != nil { + t.Fatal(err) + } + if n != 1 { + t.Errorf("observations = %d, want 1", n) + } +} + +// The audit log has to name what it destroyed, not just count it. +func TestCollapseLogsGroupKeysBeforeDeleting(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 11, 3, '["AA"]', 10), (2, 11, 3, '["AA"]', 10), (3, 11, 3, '["AA"]', 10)`); err != nil { + t.Fatal(err) + } + var out []string + logf := func(format string, args ...interface{}) { + out = append(out, fmt.Sprintf(format, args...)) + } + if err := ensureObservationsDedupIndex(db, logf); err != nil { + t.Fatal(err) + } + joined := strings.Join(out, "\n") + for _, want := range []string{ + "1 duplicate observation group(s), 2 row(s) to remove", + "transmission_id=11", + `path_json="[\"AA\"]"`, + "keeping id=1", + } { + if !strings.Contains(joined, want) { + t.Errorf("audit log missing %q; got:\n%s", want, joined) + } + } +} + +// Content is not the point — order is. The log exists so that a repair which +// destroys the wrong rows leaves a record of what it destroyed, which requires +// the keys to be written BEFORE the delete, not after. Asserting content alone +// passed with the logging moved below the DELETE. +// +// Forcing the repair to fail after the merge proves the ordering: if the keys +// were logged before the failure, they were logged before the delete. +func TestCollapseLogsGroupKeysEvenWhenTheRepairFails(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 12, 4, '["BB"]', 10), (2, 12, 4, '["BB"]', 10)`); err != nil { + t.Fatal(err) + } + // Fail at the DELETE specifically. Failing later (at CREATE INDEX) does not + // pin the ordering: logging moved to just after the DELETE would still have + // run. A BEFORE DELETE trigger that aborts puts the failure exactly at the + // step the log is supposed to precede. + if _, err := db.Exec(`CREATE TRIGGER block_delete BEFORE DELETE ON observations + BEGIN SELECT RAISE(ABORT, 'no deletes'); END`); err != nil { + t.Fatal(err) + } + + var out []string + logf := func(format string, args ...interface{}) { out = append(out, fmt.Sprintf(format, args...)) } + if _, err := collapseDuplicatesAndIndex(db, logf); err == nil { + t.Fatal("expected the repair to fail") + } + + joined := strings.Join(out, "\n") + if !strings.Contains(joined, "transmission_id=12") || !strings.Contains(joined, "keeping id=1") { + t.Errorf("group keys were not logged before the repair failed, so a failed or wrong "+ + "repair leaves no record of what it touched; got:\n%s", joined) + } +} + +// A pool of one is what cmd/ingestor runs. Anything in the repair that queries +// the pool while holding the transaction waits for a connection the transaction +// itself has checked out, and never gets it: the ingestor hangs at boot, after +// logging that it is repairing, with the database untouched and ingest dead. +// +// Found on staging, not here, because every fixture used an unbounded pool. +// Runs in a goroutine so a regression fails in seconds with a usable message +// rather than hanging until the package timeout. +func TestCollapseDoesNotDeadlockOnSingleConnectionPool(t *testing.T) { + db := observationsDB(t) // SetMaxOpenConns(1) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, path_json, timestamp) VALUES + (1, 3, 1, '[]', 10), (2, 3, 1, '[]', 10)`); err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { done <- ensureObservationsDedupIndex(db, func(string, ...interface{}) {}) }() + + select { + case err := <-done: + if err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + case <-time.After(15 * time.Second): + t.Fatal("deadlock: the repair is querying the pool while holding its own transaction — " + + "pass tx, not rw, to anything that reads inside collapseDuplicatesAndIndex") + } + if !dedupIndexExists(t, db) { + t.Error("index missing") + } +} + +// GROUP BY folds NULLs together; a UNIQUE index keeps them apart. Rows with a +// NULL in an indexed column can never violate the index, so the repair must not +// treat them as duplicates at all. +// +// The damage is not the obvious one. Both the DELETE and the merge's correlated +// subquery join on `observer_idx = observer_idx`, and NULL = NULL is not true, +// so the rows are never actually deleted — the *merge* is what destroys data: +// the subquery matches nothing and writes NULL over the survivor's real +// readings. Measured with the guard removed, a row holding snr=4.5 rssi=-70 +// came back with both NULL and its row still in place, so nothing looks missing +// while the measurements are gone. Assert the values, not just the row count: +// an earlier version of this test checked survival alone and passed against the +// bug. +// +// On an 11.2M-row instance, 198 of 222 reported groups were observer_idx IS +// NULL — direction='tx' rows. +func TestCollapseLeavesNullKeyedRowsAlone(t *testing.T) { + db := observationsDB(t) + if _, err := db.Exec(`INSERT INTO observations (id, transmission_id, observer_idx, direction, snr, rssi, path_json, timestamp) VALUES + (1, 1, NULL, 'tx', 4.5, -70, '[]', 10), + (2, 1, NULL, 'tx', 5.5, -60, '[]', 10), + (3, 2, 7, 'rx', 1.0, -80, '[]', 20), + (4, 2, 7, 'rx', 2.0, -90, '[]', 20)`); err != nil { + t.Fatal(err) + } + + var logged []string + logf := func(format string, args ...interface{}) { logged = append(logged, fmt.Sprintf(format, args...)) } + if err := ensureObservationsDedupIndex(db, logf); err != nil { + t.Fatalf("ensureObservationsDedupIndex: %v", err) + } + + // Only the observer_idx=7 pair was a real violation: one row removed there, + // both NULL rows left entirely alone. + var ids string + if err := db.QueryRow(`SELECT GROUP_CONCAT(id) FROM (SELECT id FROM observations ORDER BY id)`).Scan(&ids); err != nil { + t.Fatal(err) + } + if ids != "1,2,3" { + t.Errorf("surviving ids = %q, want \"1,2,3\": NULL observer_idx rows never violate the unique index", ids) + } + + // The readings on the NULL rows must be exactly as inserted. + for _, want := range []struct { + id int64 + snr, rssi float64 + }{{1, 4.5, -70}, {2, 5.5, -60}} { + var snr, rssi sql.NullFloat64 + if err := db.QueryRow(`SELECT snr, rssi FROM observations WHERE id = ?`, want.id).Scan(&snr, &rssi); err != nil { + t.Fatal(err) + } + if !snr.Valid || !rssi.Valid { + t.Errorf("id=%d: snr/rssi wiped to NULL — the merge matched nothing and overwrote real readings", want.id) + continue + } + if snr.Float64 != want.snr || rssi.Float64 != want.rssi { + t.Errorf("id=%d: snr=%v rssi=%v, want %v/%v", want.id, snr.Float64, rssi.Float64, want.snr, want.rssi) + } + } + + if !dedupIndexExists(t, db) { + t.Error("index missing — proof the NULL rows were never in its way") + } + if strings.Contains(strings.Join(logged, "\n"), "observer_idx=0") { + t.Error("a NULL observer_idx was logged as 0, which hides exactly this class of bug") + } +} diff --git a/internal/dbschema/go.mod b/internal/dbschema/go.mod index dcaac4c6b..3dca3dc9d 100644 --- a/internal/dbschema/go.mod +++ b/internal/dbschema/go.mod @@ -2,15 +2,4 @@ module github.com/meshcore-analyzer/dbschema go 1.22 -require ( - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/mattn/go-isatty v0.0.20 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - golang.org/x/sys v0.22.0 // indirect - modernc.org/libc v1.55.3 // indirect - modernc.org/mathutil v1.6.0 // indirect - modernc.org/memory v1.8.0 // indirect - modernc.org/sqlite v1.34.5 // indirect -) +require github.com/mattn/go-sqlite3 v1.14.52 diff --git a/internal/dbschema/go.sum b/internal/dbschema/go.sum index 2b43db395..75bf6eb53 100644 --- a/internal/dbschema/go.sum +++ b/internal/dbschema/go.sum @@ -1,21 +1,2 @@ -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= -github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= -github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= -golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -modernc.org/libc v1.55.3 h1:AzcW1mhlPNrRtjS5sS+eW2ISCgSOLLNyFzRh/V3Qj/U= -modernc.org/libc v1.55.3/go.mod h1:qFXepLhz+JjFThQ4kzwzOjA/y/artDeg+pcYnY+Q83w= -modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= -modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= -modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E= -modernc.org/memory v1.8.0/go.mod h1:XPZ936zp5OMKGWPqbD3JShgd/ZoQ7899TUuQqxY+peU= -modernc.org/sqlite v1.34.5 h1:Bb6SR13/fjp15jt70CL4f18JIN7p7dnMExd+UFnF15g= -modernc.org/sqlite v1.34.5/go.mod h1:YLuNmX9NKs8wRNK2ko1LW1NGYcc9FkBO69JOt1AR9JE= +github.com/mattn/go-sqlite3 v1.14.52 h1:wVbm2Qnf4OXkqhBTSPuCRZDRnxfbVrrmiCEroVdog8U= +github.com/mattn/go-sqlite3 v1.14.52/go.mod h1:6JTjA44L93a0QCyJef5YvlPoKXntQPjzWv5gtm9sB6w= diff --git a/internal/dbschema/writer_dsn.go b/internal/dbschema/writer_dsn.go new file mode 100644 index 000000000..fb0f7acd2 --- /dev/null +++ b/internal/dbschema/writer_dsn.go @@ -0,0 +1,35 @@ +package dbschema + +// WriterDSN builds the DSN every writer must open the database with. +// +// It lives here because this package is the one thing cmd/ingestor and +// cmd/migrate both import, and they are the only two writers. Keeping the +// string in one place is not tidiness: cmd/migrate previously opened with a +// bare path, which under github.com/mattn/go-sqlite3 silently means +// synchronous=NORMAL, so the durability the ingestor DSN carefully pins was +// lost the moment anyone ran the migrate CLI against the same database. Two +// copies of a DSN is how that happens. +// +// Every parameter is deliberate: +// +// - _journal_mode=WAL — cmd/server reads concurrently and needs WAL. +// - _synchronous=FULL — SQLite's own default, and what modernc.org/sqlite +// left in place. mattn defaults this connection to NORMAL and executes the +// pragma unconditionally, so it must be stated to be kept. NORMAL can lose +// recent transactions on power loss. +// - _auto_vacuum=incremental — cmd/ingestor/maintenance.go drives +// incremental_vacuum. Only takes effect on a database created with it. +// - _foreign_keys=on — the schema relies on FK enforcement. +// - _busy_timeout=5000 — writers serialise behind the reader. +// - _cache_size=-2000 — 2000 KiB of page cache per connection. This is +// SQLite's own default, pinned so it cannot drift: the page cache is a C +// allocation and therefore sits outside GOMEMLIMIT. +func WriterDSN(path string) string { + return path + + "?_journal_mode=WAL" + + "&_synchronous=FULL" + + "&_auto_vacuum=incremental" + + "&_foreign_keys=on" + + "&_busy_timeout=5000" + + "&_cache_size=-2000" +} diff --git a/internal/dbschema/writer_dsn_test.go b/internal/dbschema/writer_dsn_test.go new file mode 100644 index 000000000..1e83fa020 --- /dev/null +++ b/internal/dbschema/writer_dsn_test.go @@ -0,0 +1,43 @@ +package dbschema + +import ( + "database/sql" + "path/filepath" + "testing" + + _ "github.com/mattn/go-sqlite3" +) + +// TestWriterDSNPragmas pins what every writer connection actually gets. +// +// cmd/ingestor has its own end-to-end version of this through the store, but +// this one covers the DSN itself, which is what cmd/migrate also opens with. +// The synchronous line matters most: mattn defaults it to NORMAL and runs the +// pragma unconditionally, so if it ever drops out of the DSN the durability +// change is silent. +func TestWriterDSNPragmas(t *testing.T) { + db, err := sql.Open("sqlite3", WriterDSN(filepath.Join(t.TempDir(), "w.db"))) + if err != nil { + t.Fatal(err) + } + defer db.Close() + db.SetMaxOpenConns(1) + + for _, want := range []struct{ pragma, value, why string }{ + {"journal_mode", "wal", "cmd/server reads concurrently"}, + {"synchronous", "2", "FULL; mattn defaults to 1 (NORMAL) unless the DSN says otherwise"}, + {"auto_vacuum", "2", "INCREMENTAL; maintenance.go drives incremental_vacuum"}, + {"foreign_keys", "1", "the schema relies on FK enforcement"}, + {"busy_timeout", "5000", "writers serialise behind the reader"}, + {"cache_size", "-2000", "C-allocated page cache, pinned: it sits outside GOMEMLIMIT"}, + } { + var got string + if err := db.QueryRow("PRAGMA " + want.pragma).Scan(&got); err != nil { + t.Errorf("PRAGMA %s: %v", want.pragma, err) + continue + } + if got != want.value { + t.Errorf("PRAGMA %s = %q, want %q (%s)", want.pragma, got, want.value, want.why) + } + } +} diff --git a/scripts/allmod.sh b/scripts/allmod.sh new file mode 100644 index 000000000..ad54f93c3 --- /dev/null +++ b/scripts/allmod.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Run a go command in every module of this multi-module repo. +# Usage: bash scripts/allmod.sh vet ./... +# bash scripts/allmod.sh test ./... +set -uo pipefail +cd "$(dirname "$0")/.." +rc=0 +while IFS= read -r mod; do + dir=$(dirname "$mod") + printf '== %-26s ' "$dir" + out=$( (cd "$dir" && go "$@" 2>&1) ) + if [ $? -eq 0 ]; then + echo "OK" + else + echo "FAIL" + echo "$out" | sed 's/^/ /' + rc=1 + fi +done < <(find . -name go.mod -not -path './node_modules/*' | sort) +exit $rc diff --git a/scripts/migrate-fixture-hashes.go b/scripts/migrate-fixture-hashes.go index 04973e119..a0acded93 100644 --- a/scripts/migrate-fixture-hashes.go +++ b/scripts/migrate-fixture-hashes.go @@ -10,7 +10,7 @@ import ( "log" "os" - _ "modernc.org/sqlite" + _ "github.com/mattn/go-sqlite3" ) func computeContentHash(rawHex string) string { @@ -69,7 +69,7 @@ func main() { } dbPath := os.Args[1] - db, err := sql.Open("sqlite", dbPath) + db, err := sql.Open("sqlite3", dbPath) if err != nil { log.Fatal(err) } diff --git a/test-issue-1956-release-routing.js b/test-issue-1956-release-routing.js index 0f73a4d74..97b20fa4b 100644 --- a/test-issue-1956-release-routing.js +++ b/test-issue-1956-release-routing.js @@ -56,6 +56,12 @@ function runSteps(source, context, edge, mutateFails = false) { const log = path.join(dir, 'commands'); fs.writeFileSync(log, ''); fs.mkdirSync(path.join(dir, 'cmd/decrypt'), { recursive: true }); + // go() only logs, so nothing real is produced; the release job's own + // static/runnable verification step still needs these to exist. + for (const arch of ['amd64', 'arm64']) { + fs.writeFileSync(path.join(dir, `corescope-decrypt-linux-${arch}`), + '#!/bin/sh\necho "corescope-decrypt stub"\n', { mode: 0o755 }); + } const stubs = ` log() { node -e 'require("fs").appendFileSync(process.env.COMMAND_LOG, JSON.stringify(process.argv.slice(1))+"\\n")' "$@"; } crane() { @@ -69,7 +75,8 @@ function runSteps(source, context, edge, mutateFails = false) { } gh() { log gh "$@"; } tar() { log tar "$@"; } - go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$@"; } + go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$CC" "$@"; } + file() { log file "$@"; echo "$1: ELF 64-bit LSB executable, statically linked"; } jq() { node -e 'const fs=require("fs"); const assert=require("assert/strict"); assert.equal(process.argv[1], ".config.Labels[\\"org.opencontainers.image.revision\\"] // \\\"\\\""); console.log(JSON.parse(fs.readFileSync(0,"utf8")).config.Labels["org.opencontainers.image.revision"] || "")' "$2" } @@ -151,8 +158,19 @@ for (const [ref, event] of [['refs/heads/master', 'push'], ['refs/heads/master', const jobs = route(context(ref, event, { images_published: true })); assert.equal(jobs['release-artifacts'].result, 'skipped', `${event}: no GitHub release`); assert.equal(jobs['build-and-publish'].result, 'success', `${event}: tag-only input must not skip branch/PR checks`); - const publishing = steps(block(deploy, 'build-and-publish', 2)).find(step => step.includes('uses: docker/build-push-action')); - assert.equal(Boolean(evaluate(value(publishing, 'if', 8), context(ref, event))), event === 'push', `${event}: GHCR publishing`); + // There is more than one build-push-action step now: a PR-only two-arch build + // that must NOT publish, and the GHCR push. Pin the pushing one by `push: true` + // rather than by being first in the job. + const buildSteps = steps(block(deploy, 'build-and-publish', 2)).filter(step => step.includes('uses: docker/build-push-action')); + const publishing = buildSteps.filter(step => value(step, 'push', 10) === 'true'); + assert.equal(publishing.length, 1, 'exactly one step may publish to GHCR'); + assert.equal(Boolean(evaluate(value(publishing[0], 'if', 8), context(ref, event))), event === 'push', `${event}: GHCR publishing`); + // The cross-toolchain gate: since the SQLite driver became cgo, a PR must + // still build both architectures, and must do it without publishing. + const prBuild = buildSteps.filter(step => value(step, 'push', 10) === 'false'); + assert.equal(prBuild.length, 1, 'PRs must get exactly one non-publishing two-arch build'); + assert.equal(value(prBuild[0], 'platforms', 10), 'linux/amd64,linux/arm64', 'the PR gate must cover both shipped architectures'); + assert.equal(Boolean(evaluate(value(prBuild[0], 'if', 8), context(ref, event))), event === 'pull_request', `${event}: PR-only two-arch gate`); } assert.equal(route(context(), 'go-test')['release-artifacts'].result, 'skipped', 'failed Go validation must block release'); const dispatchInput = block(deploy, 'images_published', 6); @@ -161,8 +179,17 @@ assert.equal(value(dispatchInput, 'default', 8), 'false', 'manual and fallback d const release = block(deploy, 'release-artifacts', 2); const builds = runSteps(release, context(), null).commands.filter(command => command[0] === 'go'); -assert.deepEqual(builds.map(command => command.slice(1, 4)), [['linux', 'amd64', '0'], ['linux', 'arm64', '0']]); -for (const command of builds) assert.ok(command.includes('-ldflags=-s -w -X main.version=v9.8.7'), 'binary version must come from tag'); +// CGO_ENABLED=1 since the SQLite driver became github.com/mattn/go-sqlite3, and +// CC must be zig targeting musl — that is what makes the artifact static and +// cross-buildable. A silent revert to the Go-only toolchain fails here. +assert.deepEqual(builds.map(command => command.slice(1, 5)), [ + ['linux', 'amd64', '1', 'zig cc -target x86_64-linux-musl'], + ['linux', 'arm64', '1', 'zig cc -target aarch64-linux-musl'], +]); +for (const command of builds) { + assert.ok(command.includes("-ldflags=-s -w -extldflags '-static -Wl,-s' -X main.version=v9.8.7"), 'binary version must come from tag, and the artifact must stay static'); + assert.ok(command.includes('-tags'), 'netgo/osusergo/sqlite_omit_load_extension must survive'); +} const upload = steps(release).filter(step => step.includes('uses: softprops/action-gh-release@v2')); assert.equal(upload.length, 1, 'publish both architectures together, before the release becomes immutable'); assert.equal(value(upload[0], 'fail_on_unmatched_files', 10), 'true', 'missing assets must prevent publication'); From b8c8d98e618529883ee631233f71802f619a06b1 Mon Sep 17 00:00:00 2001 From: efiten Date: Wed, 16 Sep 2026 11:07:18 +0200 Subject: [PATCH 044/151] fix(release): keep every platform when re-tagging :edge as a release (#2031) ## Problem `crane mutate` works on one image, not on an index. Pointed at the multi-arch `:edge` tag it silently resolves the default platform, so the fast path published v3.11.0 as a single amd64 OCI manifest, and `crane tag` then pointed `v3.11`, `v3` and `latest` at that same manifest. `docker pull` on arm64 against any of those four tags fails. Verified in the registry: | tag | shape | arch | |---|---|---| | `v3.9.2`, `v3.10`, `v3.10.1`, `edge` | index, 4 children | multi-arch | | `v3.11.0`, `v3.11`, `v3`, `latest` | `oci.image.manifest.v1`, 17 layers | amd64 only, revision `a2ea18f7` | Earlier releases are indexes, so this only hit v3.11.0. The GitHub release and both `corescope-decrypt` binaries are unaffected. ## Change - The fast path now reads the `:edge` manifest, mutates each runnable platform child by digest (`/app/.image-version` plus the version label, as #1807 intended) and reassembles an index with `crane index append`. A single-platform `:edge` still takes the old single mutate path. - Attestation manifests (`platform.architecture == "unknown"`) are not carried over: they reference the pre-mutation digests, so copying them would attest the wrong images. - A new verification step compares the platform set of `vX.Y.Z`, `vX.Y`, `vX` and `latest` against `:edge` and fails the run if any of them differs. A release tag that resolves to one platform is worse than a slow release, so this should break the build rather than ship. - Scratch tags (`tmp-vX.Y.Z-linux-amd64`, ...) are deleted best-effort afterwards; the index references the manifests by digest, so leaving them behind is only untidy. - Added `workflow_dispatch` with a `tag` input to republish the images for an existing release. A dispatched run resolves the tagged commit itself, because `github.sha` is then the ref the workflow file came from, and it skips the `deploy.yml` dispatch: that release already exists and releases here are immutable (the trap from #1955/#1956). ## Tests None: this repository has no harness that executes workflow files, and the CI jobs cannot reach a step that pushes to GHCR. What the change is verified against instead: - `crane index append` accepts `-m/--manifest` repeated plus `-t/--tag`, with the base index optional, so building an index from scratch is supported (crane docs for `index append`). - The YAML parses and every `run:` block passes `bash -n`. - The platform comparison was run by hand against the live registry: `:edge` reports `linux/amd64,linux/arm64` and `v3.11.0` reports `single`, which is exactly the case the new step must fail on. - The real test is the dispatch on `v3.11.0` right after merge, which is also the repair. If the verification step fails there, nothing is published and the tags stay as they are. ## Not verified - The scratch-tag delete needs `delete:packages`; `GITHUB_TOKEN` may not have it. It cannot fail the run. - Whether GHCR keeps the attestation manifests attached to `:edge` reachable after the index is rebuilt for a release tag (they stay on `:edge` itself, which is untouched). --------- Co-authored-by: Claude Opus 5 (1M context) --- .github/workflows/release-fast-path.yml | 156 +++++++++++++++++++++--- test-issue-1956-release-routing.js | 92 +++++++++++++- 2 files changed, 230 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release-fast-path.yml b/.github/workflows/release-fast-path.yml index e71b9f6a4..029804112 100644 --- a/.github/workflows/release-fast-path.yml +++ b/.github/workflows/release-fast-path.yml @@ -8,10 +8,21 @@ name: Release Fast-Path # # This workflow is the SOLE consumer of push.tags. deploy.yml's tag # trigger has been removed to prevent double-fire. +# +# It can also be dispatched by hand with an existing tag, to republish the +# images for a release whose tags are wrong (v3.11.0 shipped an amd64-only +# manifest, see below). A manual run never dispatches deploy.yml: the release +# for that tag already exists and this repository has immutable releases. on: push: tags: ['v[0-9]+.[0-9]+.[0-9]+'] + workflow_dispatch: + inputs: + tag: + description: 'Existing vX.Y.Z tag to republish container tags for' + required: true + type: string permissions: contents: read @@ -19,7 +30,7 @@ permissions: actions: write # issue #1702: required for the `gh workflow run deploy.yml` dispatch concurrency: - group: release-fast-path-${{ github.ref }} + group: release-fast-path-${{ inputs.tag || github.ref }} cancel-in-progress: false jobs: @@ -39,22 +50,41 @@ jobs: - name: Parse semver from tag id: semver + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail - TAG="${GITHUB_REF#refs/tags/}" - # Expect vMAJOR.MINOR.PATCH (workflow trigger already enforces this). + if [[ -n "${{ inputs.tag || '' }}" ]]; then + TAG="${{ inputs.tag || '' }}" + else + TAG="${GITHUB_REF#refs/tags/}" + fi + # Expect vMAJOR.MINOR.PATCH (the push trigger already enforces this; + # a dispatched tag has to be checked here). if [[ ! "$TAG" =~ ^v([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then echo "Tag $TAG does not match vMAJOR.MINOR.PATCH" >&2 exit 1 fi MAJOR="${BASH_REMATCH[1]}" MINOR="${BASH_REMATCH[2]}" + # On a dispatched run github.sha is the ref the workflow file came + # from, not the tagged commit, so resolve the tag itself. + if [[ -n "${{ inputs.tag || '' }}" ]]; then + TARGET_SHA="$(gh api "repos/${{ github.repository }}/git/refs/tags/$TAG" --jq '.object.sha')" + OBJ_TYPE="$(gh api "repos/${{ github.repository }}/git/refs/tags/$TAG" --jq '.object.type')" + if [[ "$OBJ_TYPE" == "tag" ]]; then + TARGET_SHA="$(gh api "repos/${{ github.repository }}/git/tags/$TARGET_SHA" --jq '.object.sha')" + fi + else + TARGET_SHA="${{ github.sha }}" + fi { echo "tag=$TAG" echo "vMajor=v$MAJOR" echo "vMajorMinor=v$MAJOR.$MINOR" + echo "targetSha=$TARGET_SHA" } >> "$GITHUB_OUTPUT" - echo "Parsed: $TAG → v$MAJOR / v$MAJOR.$MINOR / $TAG" + echo "Parsed: $TAG → v$MAJOR / v$MAJOR.$MINOR / $TAG at $TARGET_SHA" - name: Inspect :edge revision label id: edge @@ -76,17 +106,22 @@ jobs: echo "edge_revision=$REV" >> "$GITHUB_OUTPUT" echo "no_edge=false" >> "$GITHUB_OUTPUT" echo ":edge org.opencontainers.image.revision = $REV" - echo "tag SHA (github.sha) = ${{ github.sha }}" + echo "tagged commit = ${{ steps.semver.outputs.targetSha }}" # ─────────── FAST PATH: SHAs match, metadata-only retag ─────────── # Issue #1807: a plain `crane tag` keeps the ldflags-baked # Version="edge" inside the binary, so /api/stats reports "edge" on # tagged releases. Instead of a full rebuild we append a single-file # layer (/app/.image-version containing the tag) and set the OCI - # version label, then point the remaining tags at that mutated image. - # The server reads .image-version at startup (see resolveVersion()). + # version label. The server reads .image-version at startup (see + # resolveVersion()). + # + # `crane mutate` works on ONE image, not on an index: pointed at a + # multi-arch tag it silently returns the default platform. v3.11.0 + # shipped amd64-only that way. So mutate each platform child of the + # :edge index and reassemble an index from the results. - name: Re-tag :edge → :vX.Y.Z + :vX.Y + :vX + :latest (fast path) - if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == github.sha + if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha run: | set -euo pipefail IMAGE="ghcr.io/kpa-clawbot/corescope" @@ -100,11 +135,47 @@ jobs: printf '%s' "$TAG" > "${LAYER_DIR}/app/.image-version" tar --owner=0 --group=0 --mtime='UTC 2020-01-01' \ -C "$LAYER_DIR" -cf /tmp/image-version-layer.tar app - echo " crane mutate $SRC (+.image-version layer, +version label) → ${IMAGE}:${TAG}" - crane mutate "$SRC" \ - --append /tmp/image-version-layer.tar \ - --label "org.opencontainers.image.version=${TAG}" \ - --tag "${IMAGE}:${TAG}" + + crane manifest "$SRC" > /tmp/edge-manifest.json + MEDIA="$(jq -r '.mediaType // ""' /tmp/edge-manifest.json)" + + if [[ "$MEDIA" == *".index."* || "$MEDIA" == *".manifest.list."* ]]; then + # Attestation manifests carry platform architecture "unknown" and + # are rebuilt by buildx, not by us; carrying them over would point + # them at the pre-mutation digests, so the release index holds the + # runnable platforms only. + mapfile -t PLATFORMS < <(jq -r ' + .manifests[] + | select((.platform.architecture // "unknown") != "unknown") + | "\(.platform.os)/\(.platform.architecture)\(if .platform.variant then "/" + .platform.variant else "" end)\t\(.digest)" + ' /tmp/edge-manifest.json) + if [[ "${#PLATFORMS[@]}" -eq 0 ]]; then + echo "No runnable platforms in $SRC" >&2 + exit 1 + fi + MANIFEST_ARGS=() + for entry in "${PLATFORMS[@]}"; do + PLATFORM="${entry%%$'\t'*}" + DIGEST="${entry##*$'\t'}" + SAFE="${PLATFORM//\//-}" + TMP_TAG="${IMAGE}:tmp-${TAG}-${SAFE}" + echo " crane mutate ${SRC}@${DIGEST} ($PLATFORM) → $TMP_TAG" + crane mutate "${IMAGE}@${DIGEST}" \ + --append /tmp/image-version-layer.tar \ + --label "org.opencontainers.image.version=${TAG}" \ + --tag "$TMP_TAG" + MANIFEST_ARGS+=(-m "$TMP_TAG") + done + echo " crane index append ${MANIFEST_ARGS[*]} → ${IMAGE}:${TAG}" + crane index append "${MANIFEST_ARGS[@]}" -t "${IMAGE}:${TAG}" + else + # Single-platform :edge (nothing to reassemble). + echo " crane mutate $SRC (single platform) → ${IMAGE}:${TAG}" + crane mutate "$SRC" \ + --append /tmp/image-version-layer.tar \ + --label "org.opencontainers.image.version=${TAG}" \ + --tag "${IMAGE}:${TAG}" + fi for NEW_TAG in \ "${{ steps.semver.outputs.vMajorMinor }}" \ @@ -115,14 +186,69 @@ jobs: done echo "Fast-path complete — release tags point at :edge plus a one-file version layer." - # Both image routes need the single release-artifacts writer in deploy.yml. + # A release tag that resolves to one platform is worse than a slow + # release: `docker pull` on the other architecture fails outright. + - name: Verify the published tags carry every :edge platform + if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha + run: | + set -euo pipefail + IMAGE="ghcr.io/kpa-clawbot/corescope" + TAG="${{ steps.semver.outputs.tag }}" + platforms_of() { + crane manifest "$1" | jq -r ' + if (.mediaType // "") | test("index|manifest.list") then + [ .manifests[] + | select((.platform.architecture // "unknown") != "unknown") + | "\(.platform.os)/\(.platform.architecture)\(if .platform.variant then "/" + .platform.variant else "" end)" ] + | sort | join(",") + else + "single" + end' + } + WANT="$(platforms_of "${IMAGE}:edge")" + echo " :edge platforms = $WANT" + for CHECK in \ + "$TAG" \ + "${{ steps.semver.outputs.vMajorMinor }}" \ + "${{ steps.semver.outputs.vMajor }}" \ + "latest"; do + GOT="$(platforms_of "${IMAGE}:${CHECK}")" + echo " :${CHECK} platforms = $GOT" + if [[ "$GOT" != "$WANT" ]]; then + echo "::error::${IMAGE}:${CHECK} has platforms [$GOT], expected [$WANT]" + exit 1 + fi + done + echo "All release tags carry the same platforms as :edge." + + - name: Remove the per-platform scratch tags + if: steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -uo pipefail + IMAGE="ghcr.io/kpa-clawbot/corescope" + TAG="${{ steps.semver.outputs.tag }}" + # Best effort: the index already references the manifests by digest, + # so deleting the scratch tags does not affect it. Leaving them is + # only untidy, hence no failure here. This step is skipped when an + # earlier one failed, which keeps the scratch tags around to inspect. + for T in $(crane ls "$IMAGE" | grep "^tmp-${TAG}-" || true); do + echo " crane delete ${IMAGE}:${T}" + crane delete "${IMAGE}:${T}" || echo " (could not delete ${T}, leaving it)" + done + + # Both image routes need the single release-artifacts writer in + # deploy.yml. A manual republish skips it: the release exists already and + # releases here are immutable, so action-gh-release would fail. - name: Dispatch release artifacts or full deploy.yml pipeline + if: ${{ !inputs.tag }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -euo pipefail DISPATCH_ARGS=() - if [[ "${{ steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == github.sha }}" == 'true' ]]; then + if [[ "${{ steps.edge.outputs.no_edge == 'false' && steps.edge.outputs.edge_revision == steps.semver.outputs.targetSha }}" == 'true' ]]; then echo "Images published — dispatching Go validation and release artifacts" DISPATCH_ARGS+=(--field images_published=true) else diff --git a/test-issue-1956-release-routing.js b/test-issue-1956-release-routing.js index 97b20fa4b..fc725883e 100644 --- a/test-issue-1956-release-routing.js +++ b/test-issue-1956-release-routing.js @@ -68,17 +68,57 @@ function runSteps(source, context, edge, mutateFails = false) { if [ "$1" = config ]; then [ "$EDGE_CONFIG" != missing ] || return 1 printf '%s' "$EDGE_CONFIG" + elif [ "$1" = manifest ]; then + log crane "$@" + printf '%s' "$EDGE_MANIFEST" + elif [ "$1" = ls ]; then + log crane "$@" + printf 'edge\\ntmp-v9.8.7-linux-amd64\\ntmp-v9.8.7-linux-arm64\\n' else log crane "$@" [ "$1" != mutate ] || [ "$MUTATE_FAILS" != true ] fi } - gh() { log gh "$@"; } + # gh api answers the tag lookup a dispatched republish makes; everything + # else (workflow run) only gets logged. + gh() { + log gh "$@" + if [ "$1" = api ]; then + case "$*" in + *.object.type*) printf 'commit\\n' ;; + *.object.sha*) printf '%s\\n' "$GH_API_SHA" ;; + esac + fi + } tar() { log tar "$@"; } go() { log go "$GOOS" "$GOARCH" "$CGO_ENABLED" "$CC" "$@"; } file() { log file "$@"; echo "$1: ELF 64-bit LSB executable, statically linked"; } + # Stand-in for the jq filters the release steps use. Each branch mirrors one + # filter, so a filter that changes shape without the test knowing fails here + # instead of silently returning nothing. jq() { - node -e 'const fs=require("fs"); const assert=require("assert/strict"); assert.equal(process.argv[1], ".config.Labels[\\"org.opencontainers.image.revision\\"] // \\\"\\\""); console.log(JSON.parse(fs.readFileSync(0,"utf8")).config.Labels["org.opencontainers.image.revision"] || "")' "$2" + node -e ' + const fs = require("fs"); + const filter = process.argv[1]; + const source = process.argv[2]; + const text = source && fs.existsSync(source) ? fs.readFileSync(source, "utf8") : fs.readFileSync(0, "utf8"); + const data = text.trim() ? JSON.parse(text) : {}; + const platform = m => [m.platform.os, m.platform.architecture].join("/") + (m.platform.variant ? "/" + m.platform.variant : ""); + const runnable = () => (data.manifests || []).filter(m => (m.platform && m.platform.architecture || "unknown") !== "unknown"); + if (filter.includes("org.opencontainers.image.revision")) { + console.log((((data.config || {}).Labels || {})["org.opencontainers.image.revision"]) || ""); + } else if (filter.trim() === ".mediaType // \\"\\"") { + console.log(data.mediaType || ""); + } else if (filter.includes("\\\\t\\\\(.digest)")) { + for (const m of runnable()) console.log(platform(m) + "\\t" + m.digest); + } else if (filter.includes("index|manifest.list")) { + const isIndex = /index|manifest.list/.test(data.mediaType || ""); + console.log(isIndex ? runnable().map(platform).sort().join(",") : "single"); + } else { + console.error("unstubbed jq filter: " + filter); + process.exit(3); + } + ' "$2" "\${3:-}" } `; try { @@ -92,6 +132,18 @@ function runSteps(source, context, edge, mutateFails = false) { ...process.env, GITHUB_REF: context.github.ref, GITHUB_SHA: context.github.sha, GITHUB_OUTPUT: bashPath(output), COMMAND_LOG: bashPath(log), TMPDIR: bashPath(dir), EDGE_CONFIG: edge === null ? 'missing' : JSON.stringify({ config: { Labels: { 'org.opencontainers.image.revision': edge } } }), + // :edge is a two-platform index plus the two buildx attestation + // manifests, which is what the registry actually holds. + EDGE_MANIFEST: JSON.stringify({ + mediaType: 'application/vnd.oci.image.index.v1+json', + manifests: [ + { digest: 'sha256:' + '1'.repeat(64), platform: { os: 'linux', architecture: 'amd64' } }, + { digest: 'sha256:' + '2'.repeat(64), platform: { os: 'linux', architecture: 'arm64' } }, + { digest: 'sha256:' + '3'.repeat(64), platform: { os: 'unknown', architecture: 'unknown' } }, + { digest: 'sha256:' + '4'.repeat(64), platform: { os: 'unknown', architecture: 'unknown' } } + ] + }), + GH_API_SHA: edge === null ? 'a'.repeat(40) : edge, MUTATE_FAILS: String(mutateFails) } }); @@ -139,7 +191,22 @@ for (const [name, edge] of [['matching', 'a'.repeat(40)], ['missing', null], ['m const matching = name === 'matching'; assert.equal(dispatch.includes('images_published=true'), matching); if (!matching) assert.ok(!dispatch.includes('--field') && !dispatch.includes('-f'), 'old-tag fallback must not require new workflow inputs'); - assert.equal(commands.filter(command => command[0] === 'crane' && command[1] === 'mutate').length, matching ? 1 : 0); + // One mutate per runnable platform of the :edge index, never one for the + // index itself: `crane mutate` on an index silently drops to one platform, + // which is how v3.11.0 shipped amd64-only. + const mutates = commands.filter(command => command[0] === 'crane' && command[1] === 'mutate'); + assert.equal(mutates.length, matching ? 2 : 0, `${name}: one mutate per platform`); + if (matching) { + assert.deepEqual(mutates.map(command => command.at(-1)).sort(), + ['ghcr.io/kpa-clawbot/corescope:tmp-v9.8.7-linux-amd64', 'ghcr.io/kpa-clawbot/corescope:tmp-v9.8.7-linux-arm64'], + 'each platform is mutated into its own scratch tag'); + assert.ok(mutates.every(command => command[2].includes('@sha256:')), 'mutate must address a platform by digest, not the index tag'); + const indexes = commands.filter(command => command[0] === 'crane' && command[1] === 'index'); + assert.equal(indexes.length, 1, 'the release tag is assembled as one index'); + assert.deepEqual(indexes[0].slice(1, 3), ['index', 'append']); + assert.equal(indexes[0].filter(argument => argument === '-m').length, 2, 'the index carries both platforms'); + assert.equal(indexes[0].at(-1), 'ghcr.io/kpa-clawbot/corescope:v9.8.7'); + } assert.deepEqual(commands.filter(command => command[0] === 'crane' && command[1] === 'tag').map(command => command.at(-1)), matching ? ['v9.8', 'v9', 'latest'] : []); const jobs = route(context(undefined, undefined, { images_published: matching })); assert.equal(jobs['release-artifacts'].result, 'success', `${name}: release artifacts must run`); @@ -200,3 +267,22 @@ const checkout = steps(release).find(step => step.includes('uses: actions/checko assert.equal(value(checkout, 'ref', 10), '', 'checkout must retain the dispatched tag/SHA'); assert.ok(!value(block(deploy, 'push', 2), 'tags', 4), 'fast path must remain the sole tag-triggered image writer'); console.log('PASS failed retag/Go gates, branch/PR routes, and complete tagged release assets'); + +// Republishing the images for a tag that already has a release: the tag comes +// from the dispatch input, the commit is resolved from the tag itself (the +// workflow file's own ref is master there), and deploy.yml must NOT be +// dispatched again — the release exists and releases here are immutable. +{ + const sha = 'c'.repeat(40); + const ctx = context('refs/heads/master', 'workflow_dispatch', { tag: 'v9.8.7' }); + const { commands } = runSteps(block(fast, 'retag-or-fallback', 2), ctx, sha); + assert.equal(ctx.steps.semver.outputs.tag, 'v9.8.7', 'the dispatched tag drives the release tags'); + assert.equal(ctx.steps.semver.outputs.targetSha, sha, 'the tagged commit is resolved from the tag, not from github.sha'); + assert.ok(commands.some(command => command[0] === 'gh' && command[1] === 'api'), 'the tag has to be looked up'); + assert.equal(commands.filter(command => command[0] === 'gh' && command[1] === 'workflow').length, 0, + 'a republish must not dispatch deploy.yml into an immutable release'); + assert.equal(commands.filter(command => command[0] === 'crane' && command[1] === 'mutate').length, 2, 'still one mutate per platform'); + assert.equal(commands.filter(command => command[0] === 'crane' && command[1] === 'index').length, 1, 'still one index'); + assert.deepEqual(commands.filter(command => command[0] === 'crane' && command[1] === 'tag').map(command => command.at(-1)), ['v9.8', 'v9', 'latest']); + console.log('PASS dispatched republish: tags rebuilt, no second release dispatch'); +} From 5430bc7923ab1b184cdbfbceadf3d31b5387ee68 Mon Sep 17 00:00:00 2001 From: efiten Date: Thu, 17 Sep 2026 16:52:20 +0200 Subject: [PATCH 045/151] test(ingestor): anchor the RF-sample fixtures to now, not to a calendar date (#2034) The three ClientRfDeltas tests seeded 2026-08-17T10:00:00.000Z and queried that window back. resolveRxTimeCore (cmd/ingestor/main.go:1527) replaces timestamps older than 30 days with the ingest time, so from 2026-09-16T10:00Z the seeds landed at time.Now() and every delta fell outside the queried window. Master and every open PR went red on it. Fixtures now derive from a package-level base two hours in the past, computed once per test binary so two seeds cannot straddle a second boundary and break the exact WallMillis assertion. Merged by the interim maintainer without a second human reviewer: CI is the only independent check (run 35222316327, ingestor tests ok in 97.033s, race detector ok, no --- FAIL). Fixed dates elsewhere in the ingestor tests are untouched, they assert row counts rather than querying by the seeded date. --- cmd/ingestor/client_rf_sample_test.go | 60 ++++++++++++++++++++------- 1 file changed, 45 insertions(+), 15 deletions(-) diff --git a/cmd/ingestor/client_rf_sample_test.go b/cmd/ingestor/client_rf_sample_test.go index bbc0dc480..1aae0a194 100644 --- a/cmd/ingestor/client_rf_sample_test.go +++ b/cmd/ingestor/client_rf_sample_test.go @@ -5,10 +5,40 @@ import ( "time" ) +// rfFixtureTime returns a sample timestamp offset seconds from a base two +// hours in the past, in the layout the handler round-trips. +// +// The times have to be anchored to now, not to a calendar date. +// resolveRxTimeCore (main.go:1527) replaces any timestamp more than 30 days +// old with the ingest time, so a fixed fixture date stops round-tripping 30 +// days after it is written: the seeded rows land at time.Now() while the query +// window stays in the past, and every delta disappears. The 2026-08-17 +// fixtures this file used to carry did exactly that on 2026-09-16T10:00Z, +// which turned all three delta tests red on master and on every open PR. +// The base is computed once for the whole test binary, not per call: two +// calls straddling a wall-clock second boundary would otherwise sit 16 s +// apart instead of 15 s, and TestRfDeltaBreaksOnReboot asserts the exact +// WallMillis. Truncating to the second keeps the formatted values ending in +// ".000Z", the shape these fixtures have always had. +var rfFixtureBase = time.Now().UTC().Add(-2 * time.Hour).Truncate(time.Second) + +func rfFixtureTime(offset time.Duration) string { + return rfFixtureBase.Add(offset).Format(rxTimeMillisLayout) +} + +// rfFixtureWindow brackets the rfFixtureTime series. Bounds are compared +// lexicographically against millisecond-precision sampled_at values, so they +// must themselves be in the millisecond layout: a second-resolution +// "...:00:00Z" bound would lexicographically exclude "...:00:00.000Z", +// because `.` sorts before `Z`. +func rfFixtureWindow() (from, to string) { + return rfFixtureTime(-time.Hour), rfFixtureTime(time.Hour) +} + func TestHandleClientRfSample(t *testing.T) { s := newTestStore(t) msg := map[string]interface{}{ - "type": "RF_SAMPLE", "timestamp": "2026-08-17T10:00:00.000Z", + "type": "RF_SAMPLE", "timestamp": rfFixtureTime(0), "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4, "acc_m": 8.0}, "stationary": false, "uptime_secs": 84213.0, "noise_floor": -119.0, "rx_air_secs": 20877.0, @@ -46,7 +76,7 @@ func TestHandleClientRfSampleRejects(t *testing.T) { s := newTestStore(t) base := func() map[string]interface{} { return map[string]interface{}{ - "timestamp": "2026-08-17T10:00:00.000Z", + "timestamp": rfFixtureTime(0), "gps": map[string]interface{}{"lat": 51.2, "lon": 4.4}, "uptime_secs": 1.0, } @@ -89,14 +119,12 @@ func TestRfDeltaBreaksOnReboot(t *testing.T) { "rx_air_secs": rxAir, }) } - seed("2026-08-17T10:00:00.000Z", 1000, 500) - seed("2026-08-17T10:00:15.000Z", 1015, 512) // +12 s of RX air over 15 s - seed("2026-08-17T10:00:30.000Z", 10, 3) // rebooted: uptime dropped + seed(rfFixtureTime(0), 1000, 500) + seed(rfFixtureTime(15*time.Second), 1015, 512) // +12 s of RX air over 15 s + seed(rfFixtureTime(30*time.Second), 10, 3) // rebooted: uptime dropped - // Bounds are compared lexicographically against millisecond-precision - // sampled_at values, so they must themselves be in the millisecond layout - // — "10:00:00Z" would lexicographically exclude "10:00:00.000Z" (`.` < `Z`). - deltas, err := s.ClientRfDeltas("aa11", "2026-08-17T00:00:00.000Z", "2026-08-18T00:00:00.000Z") + from, to := rfFixtureWindow() + deltas, err := s.ClientRfDeltas("aa11", from, to) if err != nil { t.Fatalf("deltas: %v", err) } @@ -126,10 +154,11 @@ func TestRfDeltaNilWhenEitherEndpointMissingCounter(t *testing.T) { handleClientRfSample(s, "test", "aa11", msg) } five := int64(5) - seed("2026-08-17T10:00:00.000Z", 1000, nil) // firmware without the counter - seed("2026-08-17T10:00:15.000Z", 1015, &five) + seed(rfFixtureTime(0), 1000, nil) // firmware without the counter + seed(rfFixtureTime(15*time.Second), 1015, &five) - deltas, err := s.ClientRfDeltas("aa11", "2026-08-17T00:00:00.000Z", "2026-08-18T00:00:00.000Z") + from, to := rfFixtureWindow() + deltas, err := s.ClientRfDeltas("aa11", from, to) if err != nil { t.Fatalf("deltas: %v", err) } @@ -156,10 +185,11 @@ func TestClientRfDeltasNormalizesPubkeyCase(t *testing.T) { "rx_air_secs": rxAir, }) } - seed("2026-08-17T10:00:00.000Z", 1000, 500) - seed("2026-08-17T10:00:15.000Z", 1015, 512) + seed(rfFixtureTime(0), 1000, 500) + seed(rfFixtureTime(15*time.Second), 1015, 512) - deltas, err := s.ClientRfDeltas("AA11", "2026-08-17T00:00:00.000Z", "2026-08-18T00:00:00.000Z") + from, to := rfFixtureWindow() + deltas, err := s.ClientRfDeltas("AA11", from, to) if err != nil { t.Fatalf("deltas: %v", err) } From 893773338ee69855e4ff9c78db59c7eef5b8c0e9 Mon Sep 17 00:00:00 2001 From: Alex B <142788550+A13xB0@users.noreply.github.com> Date: Thu, 17 Sep 2026 18:06:07 +0100 Subject: [PATCH 046/151] chore(tests): move root test-*.js into tests/unit and tests/e2e (#2036) Moves 290 root test-*.js into tests/unit (177, listed in test-all.sh) and tests/e2e (113, classified in scripts/non-unit-tests.json), per #1981 and PR-D of #1385. Root goes from 348 entries to 48. test-all.sh and test-fixtures/ stay put. The inventory guard now fails if a test reappears in the root or sits in the wrong folder. Verified independently of the diff: the invoked sets are unchanged (test-all.sh 177 before and after, deploy.yml 96 before and after, both identical as sets), and a full local run of test-all.sh on master and on the branch produced 4702 output lines each whose only differences are absolute paths, stack-trace line numbers shifted by the REPO_ROOT line, the inventory wording and two perf ratios. The guard was mutation-checked: a test back in the root, a unit suite in tests/e2e, and a suite dropped from test-all.sh each make it exit 1. CI run 35246304316 ran 97 suites from tests/e2e and is green. Follow-up 9335c51d finished the instruction files: no bare root test command is left in AGENTS.md, the squad charters, .github or docs, and every tests/ path they name resolves. Merged by the interim maintainer without a second human reviewer: CI and the local runs above are the independent checks. Known and deliberately out of scope: 18 of the 113 files in tests/e2e are invoked by no runner at all, and one of them cannot run anywhere because it requires jsdom, which is not a declared dependency. Tracked separately. --- .github/agents/pr-reviewer.agent.md | 2 +- .github/instructions/copilot.instructions.md | 2 +- .github/workflows/deploy.yml | 192 +++++----- .squad/agents/bishop/charter.md | 14 +- .squad/agents/newt/charter.md | 2 +- AGENTS.md | 30 +- README.md | 5 +- docs/agents/skills/fix-issue.md | 2 +- docs/frontend-testing.md | 12 +- scripts/non-unit-tests.json | 226 +++++------ test-all.sh | 357 +++++++++--------- .../e2e/test-1110-live-filter.js | 0 .../e2e/test-a11y-axe-1668.js | 3 +- .../e2e/test-analytics-fluid-charts.js | 3 +- .../e2e/test-audio-lab-1297-e2e.js | 0 .../e2e/test-audio-live-1297-e2e.js | 0 .../e2e/test-bottom-nav-1061-e2e.js | 0 .../e2e/test-channel-color-picker-e2e.js | 0 .../e2e/test-channel-decrypt-e2e.js | 0 .../e2e/test-channel-fluid-e2e.js | 0 .../e2e/test-channel-issue-1087-e2e.js | 0 .../e2e/test-channel-issue-1111-e2e.js | 0 .../e2e/test-channel-modal-e2e.js | 0 .../e2e/test-channel-qr-e2e.js | 0 .../e2e/test-channels-add-modal-e2e.js | 0 .../e2e/test-channels-list-render-e2e.js | 0 .../e2e/test-channels-selection-flow-e2e.js | 0 .../e2e/test-channels-share-color-e2e.js | 0 .../e2e/test-channels-ws-batch-e2e.js | 0 .../e2e/test-channels-ws-race-1498-e2e.js | 0 .../e2e/test-charts-fluid-1058-e2e.js | 0 .../e2e/test-customize-branding-e2e.js | 0 .../e2e/test-customize-display-e2e.js | 0 .../e2e/test-customize-export-e2e.js | 0 .../e2e/test-customize-theme-e2e.js | 0 .../e2e/test-drag-manager-e2e.js | 0 .../e2e/test-e2e-1267-mobile-vcr.js | 0 .../e2e/test-e2e-playwright.js | 3 +- .../e2e/test-filter-ux-e2e.js | 0 .../e2e/test-gesture-hints-1065-e2e.js | 0 .../e2e/test-gestures-1062-e2e.js | 0 ...-gestures-1185-scroll-discriminator-e2e.js | 0 .../e2e/test-home-coverage-e2e.js | 0 ...sue-1109-hamburger-dropdown-visible-e2e.js | 0 .../test-issue-1122-packets-filter-ux-e2e.js | 0 .../e2e/test-issue-1128-multi-viewport-e2e.js | 0 .../e2e/test-issue-1128-packets-layout-e2e.js | 0 .../e2e/test-issue-1136-live-region-e2e.js | 0 .../test-issue-1146-path-link-contrast-e2e.js | 0 .../e2e/test-issue-1147-section-order-e2e.js | 0 .../e2e/test-issue-1150-404-state-e2e.js | 0 .../test-issue-1151-orphan-separators-e2e.js | 0 ...est-issue-1204-live-panel-structure-e2e.js | 0 ...est-issue-1205-live-controls-anchor-e2e.js | 0 ...est-issue-1206-resize-observer-leak-e2e.js | 0 .../e2e/test-issue-1206-vcr-overlap-e2e.js | 0 .../test-issue-1224-channels-mobile-ux-e2e.js | 0 .../test-issue-1234-live-chrome-pass2-e2e.js | 0 .../e2e/test-issue-1236-map-mobile-e2e.js | 0 .../test-issue-1244-live-vcr-row-hints-e2e.js | 0 .../test-issue-1273-qr-overlay-height-e2e.js | 0 .../test-issue-1274-legend-coverage-e2e.js | 0 .../e2e/test-issue-1279-legend-p2-e2e.js | 0 .../e2e/test-issue-1281-location-row-e2e.js | 0 ...t-issue-1306-collisions-terminology-e2e.js | 0 ...t-issue-1329-map-controls-accordion-e2e.js | 0 .../test-issue-1367-channels-chat-app-e2e.js | 0 .../e2e/test-issue-1374-route-map-a11y-e2e.js | 0 .../e2e/test-issue-1400-nav-vertical-clip.js | 0 .../e2e/test-issue-1413-nav-overlap-e2e.js | 0 ...-issue-1486-collapse-reopens-detail-e2e.js | 0 .../test-issue-1487-byop-modal-layout-e2e.js | 0 .../e2e/test-issue-1510-live-nav-pin-e2e.js | 0 .../e2e/test-issue-1522-trace-url-sync-e2e.js | 0 .../test-issue-1567-corner-clears-drag-e2e.js | 0 .../e2e/test-issue-1599-replay-freeze-e2e.js | 0 .../e2e/test-issue-1630-reach-mobile-e2e.js | 0 .../e2e/test-issue-1639-observers-sort-e2e.js | 0 .../test-issue-1640-compare-discovery-e2e.js | 0 .../e2e/test-issue-1648-m1-icons-e2e.js | 0 .../e2e/test-issue-1648-m2-icons-e2e.js | 0 .../e2e/test-issue-1648-m3-icons-e2e.js | 0 .../e2e/test-issue-1648-m4-icons-e2e.js | 0 .../e2e/test-issue-1648-m5-icons-e2e.js | 0 ...57-analytics-channels-group-sprites-e2e.js | 0 ...st-issue-1692-packets-init-parallel-e2e.js | 0 .../test-issue-1705-subpath-contrast-e2e.js | 3 +- ...test-issue-1758-ng-filter-rerenders-e2e.js | 0 .../e2e/test-issue-1799-label-vocab-e2e.js | 0 .../e2e/test-live-dedup.js | 0 .../e2e/test-live-fullscreen-1572-e2e.js | 0 .../e2e/test-live-layout-1178-1179-e2e.js | 0 .../e2e/test-live-mql-leak-1180-e2e.js | 0 .../e2e/test-live-multibyte-only-e2e.js | 0 .../e2e/test-logo-default-sage-teal-e2e.js | 0 .../e2e/test-logo-pulse-1173-e2e.js | 0 .../e2e/test-logo-rebrand-e2e.js | 0 .../e2e/test-logo-theme-e2e.js | 0 .../e2e/test-map-modal-fluid-e2e.js | 0 .../e2e/test-map-nodes-pagination-e2e.js | 0 .../e2e/test-marker-outline-weight.js | 0 .../e2e/test-nav-drawer-1064-e2e.js | 0 .../e2e/test-nav-fluid-1055-e2e.js | 0 .../e2e/test-nav-more-floor-1139-e2e.js | 0 .../e2e/test-nav-priority-1102-e2e.js | 0 .../e2e/test-nav-priority-1311-e2e.js | 0 .../e2e/test-nav-priority-1391-e2e.js | 0 .../e2e/test-neighbor-map-btn-clip-e2e.js | 7 +- .../e2e/test-node-reach-coverage-e2e.js | 0 .../e2e/test-node-reach-e2e.js | 0 .../e2e/test-nodes-export-e2e.js | 0 .../e2e/test-observer-iata-1188-e2e.js | 0 .../e2e/test-packets-scope-column.js | 0 .../e2e/test-path-inspector-coverage-e2e.js | 0 .../e2e/test-path-inspector-e2e.js | 0 ...est-pr-1490-live-map-gpu-animations-e2e.js | 0 .../e2e/test-rx-coverage-mobile-nav-e2e.js | 0 .../e2e/test-show-neighbors.js | 0 .../e2e/test-slideover-1056-e2e.js | 0 .../e2e/test-slideover-1168-munger-e2e.js | 0 .../e2e/test-table-fluid-e2e.js | 0 .../e2e/test-table-sort.js | 3 +- .../e2e/test-touch-gestures-coverage-e2e.js | 0 .../e2e/test-touch-targets.js | 3 +- .../unit/test-1108-region-hide-nodes.js | 3 +- .../unit/test-1659-analytics-warmup.js | 0 .../test-a11y-1705-subpath-hop-prefix-e2e.js | 3 +- .../unit/test-a11y-1715-dark-role-swatches.js | 3 +- .../test-a11y-1716-rf-range-btn-active.js | 3 +- ...test-a11y-1719-contrast-root-causes-e2e.js | 3 +- .../unit/test-a11y-axe-1668-selftest.js | 9 +- .../unit/test-a11y-axe-routes-coverage.js | 5 +- test-aging.js => tests/unit/test-aging.js | 0 .../test-analytics-channels-integration.js | 3 +- .../unit/test-anl1-tooltip-render.js | 0 .../unit/test-area-filter.js | 0 .../unit/test-channel-color-picker.js | 5 +- .../unit/test-channel-colors.js | 7 +- .../unit/test-channel-decrypt-ecb.js | 5 +- .../test-channel-decrypt-insecure-context.js | 7 +- .../unit/test-channel-decrypt-m345.js | 3 +- .../unit/test-channel-fluid-layout.js | 3 +- .../unit/test-channel-issue-1087.js | 9 +- .../unit/test-channel-issue-1101.js | 5 +- .../test-channel-live-decrypt-userprefix.js | 3 +- .../unit/test-channel-live-decrypt.js | 7 +- .../unit/test-channel-modal-ux.js | 5 +- .../unit/test-channel-psk-ux.js | 5 +- .../unit/test-channel-qr-wiring.js | 3 +- .../unit/test-channel-qr.js | 3 +- .../unit/test-channel-sidebar-layout.js | 5 +- .../unit/test-channel-ux-followup.js | 5 +- .../unit/test-channel-ux-round2.js | 5 +- .../unit/test-channels-merge-1498-unit.js | 0 .../unit/test-clear-filters.js | 5 +- .../unit/test-color-picker-ux.js | 7 +- .../unit/test-compare-flood-filter.js | 3 +- .../unit/test-compare-overlap.js | 3 +- .../unit/test-confidence-indicator.js | 3 +- .../unit/test-coverage-gate.js | 3 +- .../unit/test-customizer-v2.js | 0 .../unit/test-drag-manager.js | 3 +- .../unit/test-e2e-badge-aggregate.sh | 2 +- .../unit/test-embed-mode-1369.js | 3 +- .../unit/test-fetch-all-nodes-pagination.js | 0 .../unit/test-fluid-scaffolding.js | 3 +- .../unit/test-frontend-helpers.js | 31 +- .../unit/test-geofilter-draft.js | 3 +- .../unit/test-hash-color.js | 3 +- .../unit/test-hop-resolver-affinity.js | 3 +- .../test-issue-1065-gesture-hints-gates.js | 5 +- .../unit/test-issue-1107-live-layout.js | 3 +- .../unit/test-issue-1136-observer-iata-map.js | 0 .../unit/test-issue-1166-first-seen-column.js | 9 +- .../unit/test-issue-1189-composed-cell.js | 3 +- .../unit/test-issue-1189-live-iata-badge.js | 3 +- .../unit/test-issue-1279-p2-code-filter.js | 0 .../unit/test-issue-1293-marker-shapes.js | 7 +- .../unit/test-issue-1356-map-a11y.js | 7 +- .../unit/test-issue-1360-pill-letter-count.js | 5 +- .../unit/test-issue-1361-cb-presets.js | 11 +- .../unit/test-issue-1364-pill-no-clamp.js | 3 +- .../unit/test-issue-1375-scope-stats-fetch.js | 3 +- .../unit/test-issue-1380-cb-reset-button.js | 5 +- .../unit/test-issue-1380-cb-sim-overlay.js | 7 +- .../test-issue-1407-cb-preset-propagation.js | 9 +- .../test-issue-1409-no-encrypted-flood.js | 3 +- .../test-issue-1412-customizer-no-override.js | 9 +- .../unit/test-issue-1415-packets-layout.js | 5 +- .../unit/test-issue-1418-cb-preset-ramp.js | 7 +- .../test-issue-1418-deeplink-hops-channels.js | 3 +- .../unit/test-issue-1418-edge-weights.js | 3 +- .../unit/test-issue-1418-polish-review.js | 5 +- .../test-issue-1418-raw-hex-extraction.js | 3 +- .../unit/test-issue-1418-spider-fan.js | 3 +- .../unit/test-issue-1420-tile-providers.js | 3 +- .../unit/test-issue-1438-customizer-mcrole.js | 9 +- .../unit/test-issue-1438-marker-css-vars.js | 9 +- .../unit/test-issue-1446-cb-preset-cascade.js | 3 +- .../unit/test-issue-1450-logo-aspect.js | 3 +- .../unit/test-issue-1454-channels-toggle.js | 5 +- .../unit/test-issue-1456-score-labels.js | 3 +- .../test-issue-1461-mobile-page-actions.js | 3 +- .../unit/test-issue-1470-card-bg-contrast.js | 3 +- .../unit/test-issue-1470-node-tile-helper.js | 5 +- .../unit/test-issue-1473-prefix-generator.js | 7 +- .../unit/test-issue-1473-reserved-prefixes.js | 7 +- .../unit/test-issue-1485-live-anim-z.js | 3 +- .../test-issue-1488-marker-stroke-vars.js | 13 +- .../test-issue-1496-reset-all-complete.js | 3 +- .../unit/test-issue-1509-detect-preset.js | 0 .../unit/test-issue-1509-nav-active-bg.js | 0 .../unit/test-issue-1518-home-url.js | 0 .../unit/test-issue-1532-live-fullscreen.js | 5 +- .../unit/test-issue-1562-observers-summary.js | 7 +- .../test-issue-1563-aggregate-and-inflight.js | 11 +- .../test-issue-1574-live-map-max-nodes.js | 7 +- .../unit/test-issue-1606-pagination.js | 9 +- .../unit/test-issue-1614-tile-url-function.js | 3 +- ...t-issue-1619-feed-detail-card-draggable.js | 5 +- .../unit/test-issue-1633-hide-1byte-hops.js | 29 +- .../unit/test-issue-1644-redesign.js | 7 +- .../unit/test-issue-1646-compare-polish.js | 5 +- ...test-issue-1648-followup-phosphor-leaks.js | 5 +- .../unit/test-issue-1648-m1-emoji-scan.js | 3 +- .../unit/test-issue-1648-m2-emoji-scan.js | 3 +- .../unit/test-issue-1648-m3-emoji-scan.js | 3 +- .../unit/test-issue-1648-m4-emoji-scan.js | 3 +- .../unit/test-issue-1648-m5-emoji-scan.js | 3 +- .../unit/test-issue-1648-m6-final-sweep.js | 3 +- .../unit/test-issue-1648-m6-lint-self.js | 3 +- .../unit/test-issue-1668-m2-contrast.js | 0 .../unit/test-issue-1668-m3-typography.js | 3 +- .../unit/test-issue-1668-m4-per-route.js | 3 +- .../unit/test-issue-1697-mqtt-mobile-e2e.js | 3 +- .../test-issue-1699-retransmission-chart.js | 3 +- .../unit/test-issue-1705-subpath-contrast.js | 0 .../unit/test-issue-1753-copy-url-slash.js | 3 +- .../unit/test-issue-1770-mobile-row-clamp.js | 3 +- .../test-issue-1789-observer-firmware-cols.js | 3 +- ...st-issue-1825-observer-node-cross-links.js | 5 +- ...est-issue-1833-legend-toggle-vcr-offset.js | 3 +- ...-issue-1836-crossnav-case-normalization.js | 5 +- .../test-issue-1843-node-qr-quiet-zone.js | 5 +- .../unit/test-issue-1846-observers-width.js | 3 +- .../unit/test-issue-1849-trace-hashbytes.js | 0 .../test-issue-1851-channel-message-scope.js | 0 .../unit/test-issue-1862-map-region-filter.js | 0 .../unit/test-issue-1868-control-decode.js | 0 .../unit/test-issue-1890-og-url.js | 3 +- .../unit/test-issue-1956-release-routing.js | 3 +- .../test-issue-1979-scope-adverts-by-role.js | 0 .../unit/test-issue-2001-map-scope-state.js | 0 ...test-issue-2012-clear-filters-selection.js | 3 +- .../unit/test-live-anims.js | 0 .../unit/test-live-dt-cap-1524.js | 0 .../unit/test-live-legend-helper.js | 0 .../unit/test-live-multibyte-filter.js | 3 +- .../unit/test-live-region-filter.js | 0 test-live.js => tests/unit/test-live.js | 3 +- .../unit/test-map-clustering.js | 0 .../unit/test-mqtt-status-panel.js | 3 +- .../unit/test-my-repeaters-dashboard.js | 5 +- .../unit/test-naive-banner-tone.js | 3 +- .../unit/test-node-hop-analytics.js | 3 +- .../unit/test-node-reach-coverage-debounce.js | 3 +- .../unit/test-node-reach-coverage.js | 3 +- .../unit/test-nodes-export-wiring.js | 5 +- .../unit/test-nodes-export.js | 3 +- .../unit/test-observer-iata-1188.js | 3 +- .../unit/test-observer-menu-interactions.js | 3 +- .../unit/test-observer-naive-clock-1478.js | 0 .../unit/test-observer-search-filter.js | 3 +- .../unit/test-observers-headings.js | 3 +- .../unit/test-packet-filter-time.js | 0 .../unit/test-packet-filter-ux.js | 0 .../unit/test-packet-filter.js | 0 .../unit/test-packets-local-channels.js | 0 test-packets.js => tests/unit/test-packets.js | 0 .../unit/test-panel-corner.js | 5 +- .../unit/test-path-inspector.js | 3 +- .../unit/test-payload-labels-content.js | 0 .../unit/test-payload-labels-namespace.js | 0 .../unit/test-perf-anomaly.js | 0 .../unit/test-perf-disk-io-1120.js | 0 .../unit/test-perf-go-runtime.js | 0 .../unit/test-perf-render-1258.js | 0 .../unit/test-preflight-xss-gate.js | 9 +- .../unit/test-pull-to-reconnect-1091.js | 0 .../unit/test-pull-to-reconnect.js | 0 .../unit/test-repeater-metric-scatter.js | 3 +- .../unit/test-rx-coverage-config-race.js | 3 +- .../unit/test-rx-coverage-escape.js | 3 +- .../unit/test-scope-audit-styles-linked.js | 0 .../unit/test-slideover-1056-rowsel-strict.js | 3 +- .../unit/test-test-inventory.js | 28 +- .../unit/test-top-routes-overlay.js | 3 +- test-traces.js => tests/unit/test-traces.js | 0 .../unit/test-url-state.js | 2 +- .../unit/test-warmup-banner.js | 7 +- .../unit/test-ws-stale-watchdog-1074.js | 0 .../unit/test-xss-escape-sinks.js | 0 302 files changed, 841 insertions(+), 700 deletions(-) rename test-1110-live-filter.js => tests/e2e/test-1110-live-filter.js (100%) rename test-a11y-axe-1668.js => tests/e2e/test-a11y-axe-1668.js (99%) rename test-analytics-fluid-charts.js => tests/e2e/test-analytics-fluid-charts.js (98%) rename test-audio-lab-1297-e2e.js => tests/e2e/test-audio-lab-1297-e2e.js (100%) rename test-audio-live-1297-e2e.js => tests/e2e/test-audio-live-1297-e2e.js (100%) rename test-bottom-nav-1061-e2e.js => tests/e2e/test-bottom-nav-1061-e2e.js (100%) rename test-channel-color-picker-e2e.js => tests/e2e/test-channel-color-picker-e2e.js (100%) rename test-channel-decrypt-e2e.js => tests/e2e/test-channel-decrypt-e2e.js (100%) rename test-channel-fluid-e2e.js => tests/e2e/test-channel-fluid-e2e.js (100%) rename test-channel-issue-1087-e2e.js => tests/e2e/test-channel-issue-1087-e2e.js (100%) rename test-channel-issue-1111-e2e.js => tests/e2e/test-channel-issue-1111-e2e.js (100%) rename test-channel-modal-e2e.js => tests/e2e/test-channel-modal-e2e.js (100%) rename test-channel-qr-e2e.js => tests/e2e/test-channel-qr-e2e.js (100%) rename test-channels-add-modal-e2e.js => tests/e2e/test-channels-add-modal-e2e.js (100%) rename test-channels-list-render-e2e.js => tests/e2e/test-channels-list-render-e2e.js (100%) rename test-channels-selection-flow-e2e.js => tests/e2e/test-channels-selection-flow-e2e.js (100%) rename test-channels-share-color-e2e.js => tests/e2e/test-channels-share-color-e2e.js (100%) rename test-channels-ws-batch-e2e.js => tests/e2e/test-channels-ws-batch-e2e.js (100%) rename test-channels-ws-race-1498-e2e.js => tests/e2e/test-channels-ws-race-1498-e2e.js (100%) rename test-charts-fluid-1058-e2e.js => tests/e2e/test-charts-fluid-1058-e2e.js (100%) rename test-customize-branding-e2e.js => tests/e2e/test-customize-branding-e2e.js (100%) rename test-customize-display-e2e.js => tests/e2e/test-customize-display-e2e.js (100%) rename test-customize-export-e2e.js => tests/e2e/test-customize-export-e2e.js (100%) rename test-customize-theme-e2e.js => tests/e2e/test-customize-theme-e2e.js (100%) rename test-drag-manager-e2e.js => tests/e2e/test-drag-manager-e2e.js (100%) rename test-e2e-1267-mobile-vcr.js => tests/e2e/test-e2e-1267-mobile-vcr.js (100%) rename test-e2e-playwright.js => tests/e2e/test-e2e-playwright.js (99%) rename test-filter-ux-e2e.js => tests/e2e/test-filter-ux-e2e.js (100%) rename test-gesture-hints-1065-e2e.js => tests/e2e/test-gesture-hints-1065-e2e.js (100%) rename test-gestures-1062-e2e.js => tests/e2e/test-gestures-1062-e2e.js (100%) rename test-gestures-1185-scroll-discriminator-e2e.js => tests/e2e/test-gestures-1185-scroll-discriminator-e2e.js (100%) rename test-home-coverage-e2e.js => tests/e2e/test-home-coverage-e2e.js (100%) rename test-issue-1109-hamburger-dropdown-visible-e2e.js => tests/e2e/test-issue-1109-hamburger-dropdown-visible-e2e.js (100%) rename test-issue-1122-packets-filter-ux-e2e.js => tests/e2e/test-issue-1122-packets-filter-ux-e2e.js (100%) rename test-issue-1128-multi-viewport-e2e.js => tests/e2e/test-issue-1128-multi-viewport-e2e.js (100%) rename test-issue-1128-packets-layout-e2e.js => tests/e2e/test-issue-1128-packets-layout-e2e.js (100%) rename test-issue-1136-live-region-e2e.js => tests/e2e/test-issue-1136-live-region-e2e.js (100%) rename test-issue-1146-path-link-contrast-e2e.js => tests/e2e/test-issue-1146-path-link-contrast-e2e.js (100%) rename test-issue-1147-section-order-e2e.js => tests/e2e/test-issue-1147-section-order-e2e.js (100%) rename test-issue-1150-404-state-e2e.js => tests/e2e/test-issue-1150-404-state-e2e.js (100%) rename test-issue-1151-orphan-separators-e2e.js => tests/e2e/test-issue-1151-orphan-separators-e2e.js (100%) rename test-issue-1204-live-panel-structure-e2e.js => tests/e2e/test-issue-1204-live-panel-structure-e2e.js (100%) rename test-issue-1205-live-controls-anchor-e2e.js => tests/e2e/test-issue-1205-live-controls-anchor-e2e.js (100%) rename test-issue-1206-resize-observer-leak-e2e.js => tests/e2e/test-issue-1206-resize-observer-leak-e2e.js (100%) rename test-issue-1206-vcr-overlap-e2e.js => tests/e2e/test-issue-1206-vcr-overlap-e2e.js (100%) rename test-issue-1224-channels-mobile-ux-e2e.js => tests/e2e/test-issue-1224-channels-mobile-ux-e2e.js (100%) rename test-issue-1234-live-chrome-pass2-e2e.js => tests/e2e/test-issue-1234-live-chrome-pass2-e2e.js (100%) rename test-issue-1236-map-mobile-e2e.js => tests/e2e/test-issue-1236-map-mobile-e2e.js (100%) rename test-issue-1244-live-vcr-row-hints-e2e.js => tests/e2e/test-issue-1244-live-vcr-row-hints-e2e.js (100%) rename test-issue-1273-qr-overlay-height-e2e.js => tests/e2e/test-issue-1273-qr-overlay-height-e2e.js (100%) rename test-issue-1274-legend-coverage-e2e.js => tests/e2e/test-issue-1274-legend-coverage-e2e.js (100%) rename test-issue-1279-legend-p2-e2e.js => tests/e2e/test-issue-1279-legend-p2-e2e.js (100%) rename test-issue-1281-location-row-e2e.js => tests/e2e/test-issue-1281-location-row-e2e.js (100%) rename test-issue-1306-collisions-terminology-e2e.js => tests/e2e/test-issue-1306-collisions-terminology-e2e.js (100%) rename test-issue-1329-map-controls-accordion-e2e.js => tests/e2e/test-issue-1329-map-controls-accordion-e2e.js (100%) rename test-issue-1367-channels-chat-app-e2e.js => tests/e2e/test-issue-1367-channels-chat-app-e2e.js (100%) rename test-issue-1374-route-map-a11y-e2e.js => tests/e2e/test-issue-1374-route-map-a11y-e2e.js (100%) rename test-issue-1400-nav-vertical-clip.js => tests/e2e/test-issue-1400-nav-vertical-clip.js (100%) rename test-issue-1413-nav-overlap-e2e.js => tests/e2e/test-issue-1413-nav-overlap-e2e.js (100%) rename test-issue-1486-collapse-reopens-detail-e2e.js => tests/e2e/test-issue-1486-collapse-reopens-detail-e2e.js (100%) rename test-issue-1487-byop-modal-layout-e2e.js => tests/e2e/test-issue-1487-byop-modal-layout-e2e.js (100%) rename test-issue-1510-live-nav-pin-e2e.js => tests/e2e/test-issue-1510-live-nav-pin-e2e.js (100%) rename test-issue-1522-trace-url-sync-e2e.js => tests/e2e/test-issue-1522-trace-url-sync-e2e.js (100%) rename test-issue-1567-corner-clears-drag-e2e.js => tests/e2e/test-issue-1567-corner-clears-drag-e2e.js (100%) rename test-issue-1599-replay-freeze-e2e.js => tests/e2e/test-issue-1599-replay-freeze-e2e.js (100%) rename test-issue-1630-reach-mobile-e2e.js => tests/e2e/test-issue-1630-reach-mobile-e2e.js (100%) rename test-issue-1639-observers-sort-e2e.js => tests/e2e/test-issue-1639-observers-sort-e2e.js (100%) rename test-issue-1640-compare-discovery-e2e.js => tests/e2e/test-issue-1640-compare-discovery-e2e.js (100%) rename test-issue-1648-m1-icons-e2e.js => tests/e2e/test-issue-1648-m1-icons-e2e.js (100%) rename test-issue-1648-m2-icons-e2e.js => tests/e2e/test-issue-1648-m2-icons-e2e.js (100%) rename test-issue-1648-m3-icons-e2e.js => tests/e2e/test-issue-1648-m3-icons-e2e.js (100%) rename test-issue-1648-m4-icons-e2e.js => tests/e2e/test-issue-1648-m4-icons-e2e.js (100%) rename test-issue-1648-m5-icons-e2e.js => tests/e2e/test-issue-1648-m5-icons-e2e.js (100%) rename test-issue-1657-analytics-channels-group-sprites-e2e.js => tests/e2e/test-issue-1657-analytics-channels-group-sprites-e2e.js (100%) rename test-issue-1692-packets-init-parallel-e2e.js => tests/e2e/test-issue-1692-packets-init-parallel-e2e.js (100%) rename test-issue-1705-subpath-contrast-e2e.js => tests/e2e/test-issue-1705-subpath-contrast-e2e.js (98%) rename test-issue-1758-ng-filter-rerenders-e2e.js => tests/e2e/test-issue-1758-ng-filter-rerenders-e2e.js (100%) rename test-issue-1799-label-vocab-e2e.js => tests/e2e/test-issue-1799-label-vocab-e2e.js (100%) rename test-live-dedup.js => tests/e2e/test-live-dedup.js (100%) rename test-live-fullscreen-1572-e2e.js => tests/e2e/test-live-fullscreen-1572-e2e.js (100%) rename test-live-layout-1178-1179-e2e.js => tests/e2e/test-live-layout-1178-1179-e2e.js (100%) rename test-live-mql-leak-1180-e2e.js => tests/e2e/test-live-mql-leak-1180-e2e.js (100%) rename test-live-multibyte-only-e2e.js => tests/e2e/test-live-multibyte-only-e2e.js (100%) rename test-logo-default-sage-teal-e2e.js => tests/e2e/test-logo-default-sage-teal-e2e.js (100%) rename test-logo-pulse-1173-e2e.js => tests/e2e/test-logo-pulse-1173-e2e.js (100%) rename test-logo-rebrand-e2e.js => tests/e2e/test-logo-rebrand-e2e.js (100%) rename test-logo-theme-e2e.js => tests/e2e/test-logo-theme-e2e.js (100%) rename test-map-modal-fluid-e2e.js => tests/e2e/test-map-modal-fluid-e2e.js (100%) rename test-map-nodes-pagination-e2e.js => tests/e2e/test-map-nodes-pagination-e2e.js (100%) rename test-marker-outline-weight.js => tests/e2e/test-marker-outline-weight.js (100%) rename test-nav-drawer-1064-e2e.js => tests/e2e/test-nav-drawer-1064-e2e.js (100%) rename test-nav-fluid-1055-e2e.js => tests/e2e/test-nav-fluid-1055-e2e.js (100%) rename test-nav-more-floor-1139-e2e.js => tests/e2e/test-nav-more-floor-1139-e2e.js (100%) rename test-nav-priority-1102-e2e.js => tests/e2e/test-nav-priority-1102-e2e.js (100%) rename test-nav-priority-1311-e2e.js => tests/e2e/test-nav-priority-1311-e2e.js (100%) rename test-nav-priority-1391-e2e.js => tests/e2e/test-nav-priority-1391-e2e.js (100%) rename test-neighbor-map-btn-clip-e2e.js => tests/e2e/test-neighbor-map-btn-clip-e2e.js (97%) rename test-node-reach-coverage-e2e.js => tests/e2e/test-node-reach-coverage-e2e.js (100%) rename test-node-reach-e2e.js => tests/e2e/test-node-reach-e2e.js (100%) rename test-nodes-export-e2e.js => tests/e2e/test-nodes-export-e2e.js (100%) rename test-observer-iata-1188-e2e.js => tests/e2e/test-observer-iata-1188-e2e.js (100%) rename test-packets-scope-column.js => tests/e2e/test-packets-scope-column.js (100%) rename test-path-inspector-coverage-e2e.js => tests/e2e/test-path-inspector-coverage-e2e.js (100%) rename test-path-inspector-e2e.js => tests/e2e/test-path-inspector-e2e.js (100%) rename test-pr-1490-live-map-gpu-animations-e2e.js => tests/e2e/test-pr-1490-live-map-gpu-animations-e2e.js (100%) rename test-rx-coverage-mobile-nav-e2e.js => tests/e2e/test-rx-coverage-mobile-nav-e2e.js (100%) rename test-show-neighbors.js => tests/e2e/test-show-neighbors.js (100%) rename test-slideover-1056-e2e.js => tests/e2e/test-slideover-1056-e2e.js (100%) rename test-slideover-1168-munger-e2e.js => tests/e2e/test-slideover-1168-munger-e2e.js (100%) rename test-table-fluid-e2e.js => tests/e2e/test-table-fluid-e2e.js (100%) rename test-table-sort.js => tests/e2e/test-table-sort.js (98%) rename test-touch-gestures-coverage-e2e.js => tests/e2e/test-touch-gestures-coverage-e2e.js (100%) rename test-touch-targets.js => tests/e2e/test-touch-targets.js (99%) rename test-1108-region-hide-nodes.js => tests/unit/test-1108-region-hide-nodes.js (96%) rename test-1659-analytics-warmup.js => tests/unit/test-1659-analytics-warmup.js (100%) rename test-a11y-1705-subpath-hop-prefix-e2e.js => tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js (98%) rename test-a11y-1715-dark-role-swatches.js => tests/unit/test-a11y-1715-dark-role-swatches.js (98%) rename test-a11y-1716-rf-range-btn-active.js => tests/unit/test-a11y-1716-rf-range-btn-active.js (98%) rename test-a11y-1719-contrast-root-causes-e2e.js => tests/unit/test-a11y-1719-contrast-root-causes-e2e.js (99%) rename test-a11y-axe-1668-selftest.js => tests/unit/test-a11y-axe-1668-selftest.js (98%) rename test-a11y-axe-routes-coverage.js => tests/unit/test-a11y-axe-routes-coverage.js (93%) rename test-aging.js => tests/unit/test-aging.js (100%) rename test-analytics-channels-integration.js => tests/unit/test-analytics-channels-integration.js (98%) rename test-anl1-tooltip-render.js => tests/unit/test-anl1-tooltip-render.js (100%) rename test-area-filter.js => tests/unit/test-area-filter.js (100%) rename test-channel-color-picker.js => tests/unit/test-channel-color-picker.js (96%) rename test-channel-colors.js => tests/unit/test-channel-colors.js (97%) rename test-channel-decrypt-ecb.js => tests/unit/test-channel-decrypt-ecb.js (95%) rename test-channel-decrypt-insecure-context.js => tests/unit/test-channel-decrypt-insecure-context.js (96%) rename test-channel-decrypt-m345.js => tests/unit/test-channel-decrypt-m345.js (97%) rename test-channel-fluid-layout.js => tests/unit/test-channel-fluid-layout.js (96%) rename test-channel-issue-1087.js => tests/unit/test-channel-issue-1087.js (95%) rename test-channel-issue-1101.js => tests/unit/test-channel-issue-1101.js (95%) rename test-channel-live-decrypt-userprefix.js => tests/unit/test-channel-live-decrypt-userprefix.js (98%) rename test-channel-live-decrypt.js => tests/unit/test-channel-live-decrypt.js (96%) rename test-channel-modal-ux.js => tests/unit/test-channel-modal-ux.js (96%) rename test-channel-psk-ux.js => tests/unit/test-channel-psk-ux.js (97%) rename test-channel-qr-wiring.js => tests/unit/test-channel-qr-wiring.js (96%) rename test-channel-qr.js => tests/unit/test-channel-qr.js (96%) rename test-channel-sidebar-layout.js => tests/unit/test-channel-sidebar-layout.js (95%) rename test-channel-ux-followup.js => tests/unit/test-channel-ux-followup.js (97%) rename test-channel-ux-round2.js => tests/unit/test-channel-ux-round2.js (95%) rename test-channels-merge-1498-unit.js => tests/unit/test-channels-merge-1498-unit.js (100%) rename test-clear-filters.js => tests/unit/test-clear-filters.js (98%) rename test-color-picker-ux.js => tests/unit/test-color-picker-ux.js (93%) rename test-compare-flood-filter.js => tests/unit/test-compare-flood-filter.js (95%) rename test-compare-overlap.js => tests/unit/test-compare-overlap.js (96%) rename test-confidence-indicator.js => tests/unit/test-confidence-indicator.js (97%) rename test-coverage-gate.js => tests/unit/test-coverage-gate.js (95%) rename test-customizer-v2.js => tests/unit/test-customizer-v2.js (100%) rename test-drag-manager.js => tests/unit/test-drag-manager.js (99%) rename test-e2e-badge-aggregate.sh => tests/unit/test-e2e-badge-aggregate.sh (98%) rename test-embed-mode-1369.js => tests/unit/test-embed-mode-1369.js (97%) rename test-fetch-all-nodes-pagination.js => tests/unit/test-fetch-all-nodes-pagination.js (100%) rename test-fluid-scaffolding.js => tests/unit/test-fluid-scaffolding.js (96%) rename test-frontend-helpers.js => tests/unit/test-frontend-helpers.js (99%) rename test-geofilter-draft.js => tests/unit/test-geofilter-draft.js (96%) rename test-hash-color.js => tests/unit/test-hash-color.js (98%) rename test-hop-resolver-affinity.js => tests/unit/test-hop-resolver-affinity.js (98%) rename test-issue-1065-gesture-hints-gates.js => tests/unit/test-issue-1065-gesture-hints-gates.js (94%) rename test-issue-1107-live-layout.js => tests/unit/test-issue-1107-live-layout.js (98%) rename test-issue-1136-observer-iata-map.js => tests/unit/test-issue-1136-observer-iata-map.js (100%) rename test-issue-1166-first-seen-column.js => tests/unit/test-issue-1166-first-seen-column.js (95%) rename test-issue-1189-composed-cell.js => tests/unit/test-issue-1189-composed-cell.js (97%) rename test-issue-1189-live-iata-badge.js => tests/unit/test-issue-1189-live-iata-badge.js (97%) rename test-issue-1279-p2-code-filter.js => tests/unit/test-issue-1279-p2-code-filter.js (100%) rename test-issue-1293-marker-shapes.js => tests/unit/test-issue-1293-marker-shapes.js (95%) rename test-issue-1356-map-a11y.js => tests/unit/test-issue-1356-map-a11y.js (98%) rename test-issue-1360-pill-letter-count.js => tests/unit/test-issue-1360-pill-letter-count.js (95%) rename test-issue-1361-cb-presets.js => tests/unit/test-issue-1361-cb-presets.js (96%) rename test-issue-1364-pill-no-clamp.js => tests/unit/test-issue-1364-pill-no-clamp.js (93%) rename test-issue-1375-scope-stats-fetch.js => tests/unit/test-issue-1375-scope-stats-fetch.js (93%) rename test-issue-1380-cb-reset-button.js => tests/unit/test-issue-1380-cb-reset-button.js (96%) rename test-issue-1380-cb-sim-overlay.js => tests/unit/test-issue-1380-cb-sim-overlay.js (96%) rename test-issue-1407-cb-preset-propagation.js => tests/unit/test-issue-1407-cb-preset-propagation.js (96%) rename test-issue-1409-no-encrypted-flood.js => tests/unit/test-issue-1409-no-encrypted-flood.js (94%) rename test-issue-1412-customizer-no-override.js => tests/unit/test-issue-1412-customizer-no-override.js (96%) rename test-issue-1415-packets-layout.js => tests/unit/test-issue-1415-packets-layout.js (97%) rename test-issue-1418-cb-preset-ramp.js => tests/unit/test-issue-1418-cb-preset-ramp.js (96%) rename test-issue-1418-deeplink-hops-channels.js => tests/unit/test-issue-1418-deeplink-hops-channels.js (98%) rename test-issue-1418-edge-weights.js => tests/unit/test-issue-1418-edge-weights.js (97%) rename test-issue-1418-polish-review.js => tests/unit/test-issue-1418-polish-review.js (96%) rename test-issue-1418-raw-hex-extraction.js => tests/unit/test-issue-1418-raw-hex-extraction.js (98%) rename test-issue-1418-spider-fan.js => tests/unit/test-issue-1418-spider-fan.js (98%) rename test-issue-1420-tile-providers.js => tests/unit/test-issue-1420-tile-providers.js (99%) rename test-issue-1438-customizer-mcrole.js => tests/unit/test-issue-1438-customizer-mcrole.js (96%) rename test-issue-1438-marker-css-vars.js => tests/unit/test-issue-1438-marker-css-vars.js (95%) rename test-issue-1446-cb-preset-cascade.js => tests/unit/test-issue-1446-cb-preset-cascade.js (99%) rename test-issue-1450-logo-aspect.js => tests/unit/test-issue-1450-logo-aspect.js (98%) rename test-issue-1454-channels-toggle.js => tests/unit/test-issue-1454-channels-toggle.js (94%) rename test-issue-1456-score-labels.js => tests/unit/test-issue-1456-score-labels.js (96%) rename test-issue-1461-mobile-page-actions.js => tests/unit/test-issue-1461-mobile-page-actions.js (99%) rename test-issue-1470-card-bg-contrast.js => tests/unit/test-issue-1470-card-bg-contrast.js (98%) rename test-issue-1470-node-tile-helper.js => tests/unit/test-issue-1470-node-tile-helper.js (97%) rename test-issue-1473-prefix-generator.js => tests/unit/test-issue-1473-prefix-generator.js (94%) rename test-issue-1473-reserved-prefixes.js => tests/unit/test-issue-1473-reserved-prefixes.js (95%) rename test-issue-1485-live-anim-z.js => tests/unit/test-issue-1485-live-anim-z.js (97%) rename test-issue-1488-marker-stroke-vars.js => tests/unit/test-issue-1488-marker-stroke-vars.js (93%) rename test-issue-1496-reset-all-complete.js => tests/unit/test-issue-1496-reset-all-complete.js (98%) rename test-issue-1509-detect-preset.js => tests/unit/test-issue-1509-detect-preset.js (100%) rename test-issue-1509-nav-active-bg.js => tests/unit/test-issue-1509-nav-active-bg.js (100%) rename test-issue-1518-home-url.js => tests/unit/test-issue-1518-home-url.js (100%) rename test-issue-1532-live-fullscreen.js => tests/unit/test-issue-1532-live-fullscreen.js (97%) rename test-issue-1562-observers-summary.js => tests/unit/test-issue-1562-observers-summary.js (96%) rename test-issue-1563-aggregate-and-inflight.js => tests/unit/test-issue-1563-aggregate-and-inflight.js (96%) rename test-issue-1574-live-map-max-nodes.js => tests/unit/test-issue-1574-live-map-max-nodes.js (89%) rename test-issue-1606-pagination.js => tests/unit/test-issue-1606-pagination.js (98%) rename test-issue-1614-tile-url-function.js => tests/unit/test-issue-1614-tile-url-function.js (97%) rename test-issue-1619-feed-detail-card-draggable.js => tests/unit/test-issue-1619-feed-detail-card-draggable.js (95%) rename test-issue-1633-hide-1byte-hops.js => tests/unit/test-issue-1633-hide-1byte-hops.js (96%) rename test-issue-1644-redesign.js => tests/unit/test-issue-1644-redesign.js (97%) rename test-issue-1646-compare-polish.js => tests/unit/test-issue-1646-compare-polish.js (98%) rename test-issue-1648-followup-phosphor-leaks.js => tests/unit/test-issue-1648-followup-phosphor-leaks.js (97%) rename test-issue-1648-m1-emoji-scan.js => tests/unit/test-issue-1648-m1-emoji-scan.js (97%) rename test-issue-1648-m2-emoji-scan.js => tests/unit/test-issue-1648-m2-emoji-scan.js (98%) rename test-issue-1648-m3-emoji-scan.js => tests/unit/test-issue-1648-m3-emoji-scan.js (98%) rename test-issue-1648-m4-emoji-scan.js => tests/unit/test-issue-1648-m4-emoji-scan.js (98%) rename test-issue-1648-m5-emoji-scan.js => tests/unit/test-issue-1648-m5-emoji-scan.js (98%) rename test-issue-1648-m6-final-sweep.js => tests/unit/test-issue-1648-m6-final-sweep.js (98%) rename test-issue-1648-m6-lint-self.js => tests/unit/test-issue-1648-m6-lint-self.js (97%) rename test-issue-1668-m2-contrast.js => tests/unit/test-issue-1668-m2-contrast.js (100%) rename test-issue-1668-m3-typography.js => tests/unit/test-issue-1668-m3-typography.js (98%) rename test-issue-1668-m4-per-route.js => tests/unit/test-issue-1668-m4-per-route.js (99%) rename test-issue-1697-mqtt-mobile-e2e.js => tests/unit/test-issue-1697-mqtt-mobile-e2e.js (97%) rename test-issue-1699-retransmission-chart.js => tests/unit/test-issue-1699-retransmission-chart.js (98%) rename test-issue-1705-subpath-contrast.js => tests/unit/test-issue-1705-subpath-contrast.js (100%) rename test-issue-1753-copy-url-slash.js => tests/unit/test-issue-1753-copy-url-slash.js (94%) rename test-issue-1770-mobile-row-clamp.js => tests/unit/test-issue-1770-mobile-row-clamp.js (96%) rename test-issue-1789-observer-firmware-cols.js => tests/unit/test-issue-1789-observer-firmware-cols.js (97%) rename test-issue-1825-observer-node-cross-links.js => tests/unit/test-issue-1825-observer-node-cross-links.js (94%) rename test-issue-1833-legend-toggle-vcr-offset.js => tests/unit/test-issue-1833-legend-toggle-vcr-offset.js (96%) rename test-issue-1836-crossnav-case-normalization.js => tests/unit/test-issue-1836-crossnav-case-normalization.js (89%) rename test-issue-1843-node-qr-quiet-zone.js => tests/unit/test-issue-1843-node-qr-quiet-zone.js (95%) rename test-issue-1846-observers-width.js => tests/unit/test-issue-1846-observers-width.js (94%) rename test-issue-1849-trace-hashbytes.js => tests/unit/test-issue-1849-trace-hashbytes.js (100%) rename test-issue-1851-channel-message-scope.js => tests/unit/test-issue-1851-channel-message-scope.js (100%) rename test-issue-1862-map-region-filter.js => tests/unit/test-issue-1862-map-region-filter.js (100%) rename test-issue-1868-control-decode.js => tests/unit/test-issue-1868-control-decode.js (100%) rename test-issue-1890-og-url.js => tests/unit/test-issue-1890-og-url.js (94%) rename test-issue-1956-release-routing.js => tests/unit/test-issue-1956-release-routing.js (99%) rename test-issue-1979-scope-adverts-by-role.js => tests/unit/test-issue-1979-scope-adverts-by-role.js (100%) rename test-issue-2001-map-scope-state.js => tests/unit/test-issue-2001-map-scope-state.js (100%) rename test-issue-2012-clear-filters-selection.js => tests/unit/test-issue-2012-clear-filters-selection.js (98%) rename test-live-anims.js => tests/unit/test-live-anims.js (100%) rename test-live-dt-cap-1524.js => tests/unit/test-live-dt-cap-1524.js (100%) rename test-live-legend-helper.js => tests/unit/test-live-legend-helper.js (100%) rename test-live-multibyte-filter.js => tests/unit/test-live-multibyte-filter.js (95%) rename test-live-region-filter.js => tests/unit/test-live-region-filter.js (100%) rename test-live.js => tests/unit/test-live.js (99%) rename test-map-clustering.js => tests/unit/test-map-clustering.js (100%) rename test-mqtt-status-panel.js => tests/unit/test-mqtt-status-panel.js (98%) rename test-my-repeaters-dashboard.js => tests/unit/test-my-repeaters-dashboard.js (98%) rename test-naive-banner-tone.js => tests/unit/test-naive-banner-tone.js (95%) rename test-node-hop-analytics.js => tests/unit/test-node-hop-analytics.js (96%) rename test-node-reach-coverage-debounce.js => tests/unit/test-node-reach-coverage-debounce.js (95%) rename test-node-reach-coverage.js => tests/unit/test-node-reach-coverage.js (94%) rename test-nodes-export-wiring.js => tests/unit/test-nodes-export-wiring.js (92%) rename test-nodes-export.js => tests/unit/test-nodes-export.js (97%) rename test-observer-iata-1188.js => tests/unit/test-observer-iata-1188.js (98%) rename test-observer-menu-interactions.js => tests/unit/test-observer-menu-interactions.js (98%) rename test-observer-naive-clock-1478.js => tests/unit/test-observer-naive-clock-1478.js (100%) rename test-observer-search-filter.js => tests/unit/test-observer-search-filter.js (97%) rename test-observers-headings.js => tests/unit/test-observers-headings.js (95%) rename test-packet-filter-time.js => tests/unit/test-packet-filter-time.js (100%) rename test-packet-filter-ux.js => tests/unit/test-packet-filter-ux.js (100%) rename test-packet-filter.js => tests/unit/test-packet-filter.js (100%) rename test-packets-local-channels.js => tests/unit/test-packets-local-channels.js (100%) rename test-packets.js => tests/unit/test-packets.js (100%) rename test-panel-corner.js => tests/unit/test-panel-corner.js (98%) rename test-path-inspector.js => tests/unit/test-path-inspector.js (96%) rename test-payload-labels-content.js => tests/unit/test-payload-labels-content.js (100%) rename test-payload-labels-namespace.js => tests/unit/test-payload-labels-namespace.js (100%) rename test-perf-anomaly.js => tests/unit/test-perf-anomaly.js (100%) rename test-perf-disk-io-1120.js => tests/unit/test-perf-disk-io-1120.js (100%) rename test-perf-go-runtime.js => tests/unit/test-perf-go-runtime.js (100%) rename test-perf-render-1258.js => tests/unit/test-perf-render-1258.js (100%) rename test-preflight-xss-gate.js => tests/unit/test-preflight-xss-gate.js (94%) rename test-pull-to-reconnect-1091.js => tests/unit/test-pull-to-reconnect-1091.js (100%) rename test-pull-to-reconnect.js => tests/unit/test-pull-to-reconnect.js (100%) rename test-repeater-metric-scatter.js => tests/unit/test-repeater-metric-scatter.js (98%) rename test-rx-coverage-config-race.js => tests/unit/test-rx-coverage-config-race.js (93%) rename test-rx-coverage-escape.js => tests/unit/test-rx-coverage-escape.js (95%) rename test-scope-audit-styles-linked.js => tests/unit/test-scope-audit-styles-linked.js (100%) rename test-slideover-1056-rowsel-strict.js => tests/unit/test-slideover-1056-rowsel-strict.js (94%) rename test-test-inventory.js => tests/unit/test-test-inventory.js (60%) rename test-top-routes-overlay.js => tests/unit/test-top-routes-overlay.js (97%) rename test-traces.js => tests/unit/test-traces.js (100%) rename test-url-state.js => tests/unit/test-url-state.js (98%) rename test-warmup-banner.js => tests/unit/test-warmup-banner.js (97%) rename test-ws-stale-watchdog-1074.js => tests/unit/test-ws-stale-watchdog-1074.js (100%) rename test-xss-escape-sinks.js => tests/unit/test-xss-escape-sinks.js (100%) diff --git a/.github/agents/pr-reviewer.agent.md b/.github/agents/pr-reviewer.agent.md index 357fbca5c..9def873bb 100644 --- a/.github/agents/pr-reviewer.agent.md +++ b/.github/agents/pr-reviewer.agent.md @@ -28,7 +28,7 @@ You will pay special attention to the following areas during your review: - **Project Architecture**: Does the PR respect the single Node.js server + static frontend architecture? Are changes in the right place? ### 2. Testing and Validation -- **No commit without tests**: Is the backend logic change covered by unit tests? Is `test-packet-filter.js` or `test-aging.js` updated if necessary? +- **No commit without tests**: Is the backend logic change covered by unit tests? Is `tests/unit/test-packet-filter.js` or `tests/unit/test-aging.js` updated if necessary? - **Browser Validation**: Has the contributor confirmed the change works in a browser? Is there a screenshot for visual changes? - **Cache Busters**: If any `public/` assets (`.js`, `.css`) were modified, has the cache buster in `public/index.html` been bumped in the *same commit*? This is critical. diff --git a/.github/instructions/copilot.instructions.md b/.github/instructions/copilot.instructions.md index 357fbca5c..9def873bb 100644 --- a/.github/instructions/copilot.instructions.md +++ b/.github/instructions/copilot.instructions.md @@ -28,7 +28,7 @@ You will pay special attention to the following areas during your review: - **Project Architecture**: Does the PR respect the single Node.js server + static frontend architecture? Are changes in the right place? ### 2. Testing and Validation -- **No commit without tests**: Is the backend logic change covered by unit tests? Is `test-packet-filter.js` or `test-aging.js` updated if necessary? +- **No commit without tests**: Is the backend logic change covered by unit tests? Is `tests/unit/test-packet-filter.js` or `tests/unit/test-aging.js` updated if necessary? - **Browser Validation**: Has the contributor confirmed the change works in a browser? Is there a screenshot for visual changes? - **Cache Busters**: If any `public/` assets (`.js`, `.css`) were modified, has the cache buster in `public/index.html` been bumped in the *same commit*? This is critical. diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 737fee938..bd522b19f 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -472,7 +472,7 @@ jobs: - name: Run Playwright E2E tests (fail-fast) run: | - BASE_URL=http://localhost:13581 node test-e2e-playwright.js 2>&1 | tee e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-e2e-playwright.js 2>&1 | tee e2e-output.txt # M5+M6 of #1668 — axe-core CI gate. # M5: color-contrast on desktop dark+light. # M6: expanded ruleset (image-alt, label, aria-required-attr, @@ -483,100 +483,100 @@ jobs: # Allowlist: tests/a11y-allowlist.yaml (0 entries — hard pass policy). # Per-viewport summary printed at the end; any net>0 fails the build. BASE_URL=http://localhost:13581 AXE_SCREENSHOT_DIR=/tmp/axe-1668 \ - node test-a11y-axe-1668.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-issue-1087-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-issue-1111-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-map-modal-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-map-nodes-pagination-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-observer-iata-1188-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1639-observers-sort-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1758-ng-filter-rerenders-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-fluid-1055-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1102-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1311-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-priority-1391-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1413-nav-overlap-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1400-nav-vertical-clip.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-more-floor-1139-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-bottom-nav-1061-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1062-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gestures-1185-scroll-discriminator-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-gesture-hints-1065-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-touch-gestures-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-table-fluid-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-charts-fluid-1058-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1692-packets-init-parallel-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1122-packets-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1128-packets-layout-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1128-multi-viewport-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1136-live-region-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-live-multibyte-only-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1150-404-state-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1146-path-link-contrast-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-issue-1705-subpath-contrast-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1147-section-order-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1151-orphan-separators-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1486-collapse-reopens-detail-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-rebrand-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-theme-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-logo-default-sage-teal-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1109-hamburger-dropdown-visible-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-layout-1178-1179-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1205-live-controls-anchor-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-mql-leak-1180-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1204-live-panel-structure-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1234-live-chrome-pass2-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-vcr-overlap-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1244-live-vcr-row-hints-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1510-live-nav-pin-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-live-fullscreen-1572-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1599-replay-freeze-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m1-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m2-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m3-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1648-m4-icons-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1657-analytics-channels-group-sprites-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1367-channels-chat-app-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1329-map-controls-accordion-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1281-location-row-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1279-legend-p2-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-issue-1799-label-vocab-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-home-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-path-inspector-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1206-resize-observer-leak-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-nav-drawer-1064-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-live-1297-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-audio-lab-1297-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-decrypt-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-qr-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-channel-color-picker-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-theme-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-branding-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-display-e2e.js 2>&1 | tee -a e2e-output.txt - BASE_URL=http://localhost:13581 node test-customize-export-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-drag-manager-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1567-corner-clears-drag-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1306-collisions-terminology-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1374-route-map-a11y-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-list-render-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-selection-flow-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-add-modal-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-share-color-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-batch-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-channels-ws-race-1498-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt - CHROMIUM_REQUIRE=1 node test-neighbor-map-btn-clip-e2e.js 2>&1 | tee -a e2e-output.txt + node tests/e2e/test-a11y-axe-1668.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-channel-issue-1087-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-channel-issue-1111-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-map-modal-fluid-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-map-nodes-pagination-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-observer-iata-1188-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1639-observers-sort-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1758-ng-filter-rerenders-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-nav-fluid-1055-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-nav-priority-1102-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-nav-priority-1311-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-nav-priority-1391-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1413-nav-overlap-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1400-nav-vertical-clip.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-nav-more-floor-1139-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-bottom-nav-1061-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-gestures-1062-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-gestures-1185-scroll-discriminator-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-gesture-hints-1065-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-touch-gestures-coverage-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-channel-fluid-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-table-fluid-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-charts-fluid-1058-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-slideover-1056-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1692-packets-init-parallel-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-slideover-1168-munger-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-logo-pulse-1173-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1122-packets-filter-ux-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1128-packets-layout-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1128-multi-viewport-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1136-live-region-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-live-multibyte-only-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1150-404-state-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1146-path-link-contrast-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 node tests/e2e/test-issue-1705-subpath-contrast-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1147-section-order-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1151-orphan-separators-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1486-collapse-reopens-detail-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-logo-rebrand-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-logo-theme-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-logo-default-sage-teal-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1109-hamburger-dropdown-visible-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-live-layout-1178-1179-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1205-live-controls-anchor-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-live-mql-leak-1180-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1204-live-panel-structure-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1234-live-chrome-pass2-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1206-vcr-overlap-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1244-live-vcr-row-hints-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1510-live-nav-pin-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-live-fullscreen-1572-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1599-replay-freeze-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1648-m1-icons-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1648-m2-icons-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1648-m3-icons-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1648-m4-icons-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1657-analytics-channels-group-sprites-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1224-channels-mobile-ux-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1367-channels-chat-app-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1236-map-mobile-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1329-map-controls-accordion-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1273-qr-overlay-height-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1281-location-row-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1279-legend-p2-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1799-label-vocab-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-home-coverage-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-path-inspector-coverage-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1206-resize-observer-leak-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-nav-drawer-1064-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-audio-live-1297-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-audio-lab-1297-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-channel-decrypt-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-channel-qr-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-channel-color-picker-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-customize-theme-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-customize-branding-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-customize-display-e2e.js 2>&1 | tee -a e2e-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-customize-export-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-drag-manager-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1567-corner-clears-drag-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1306-collisions-terminology-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1374-route-map-a11y-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-channels-list-render-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-channels-selection-flow-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-channels-add-modal-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-channels-share-color-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-channels-ws-batch-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-channels-ws-race-1498-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1487-byop-modal-layout-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1630-reach-mobile-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-node-reach-coverage-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 BASE_URL=http://localhost:13581 node tests/e2e/test-issue-1640-compare-discovery-e2e.js 2>&1 | tee -a e2e-output.txt + CHROMIUM_REQUIRE=1 node tests/e2e/test-neighbor-map-btn-clip-e2e.js 2>&1 | tee -a e2e-output.txt # #1616: slide-over focus-restore flake-gate. Runs the slide-over # E2E 20 consecutive times against the SAME backend instance so @@ -594,7 +594,7 @@ jobs: set -e for i in $(seq 1 3); do echo "--- slide-over E2E run $i/20 ---" - BASE_URL=http://localhost:13581 node test-slideover-1056-e2e.js 2>&1 | tee -a slideover-repeat-output.txt + BASE_URL=http://localhost:13581 node tests/e2e/test-slideover-1056-e2e.js 2>&1 | tee -a slideover-repeat-output.txt done echo "3 passed" diff --git a/.squad/agents/bishop/charter.md b/.squad/agents/bishop/charter.md index 09b0a5d82..5ce7d6fe5 100644 --- a/.squad/agents/bishop/charter.md +++ b/.squad/agents/bishop/charter.md @@ -10,16 +10,15 @@ Unit tests, Playwright E2E, coverage gates, and quality assurance for CoreScope. ## Responsibilities -- Unit tests: test-packet-filter.js, test-aging.js, test-decoder.js, test-decoder-spec.js, test-server-helpers.js, test-server-routes.js, test-packet-store.js, test-db.js, test-frontend-helpers.js, test-regional-filter.js, test-regional-integration.js, test-live-dedup.js -- Playwright E2E: test-e2e-playwright.js (8 browser tests, default localhost:3000) -- E2E tools: tools/e2e-test.js, tools/frontend-test.js +- Unit tests: `tests/unit/`, all listed in `test-all.sh` (e.g. test-packet-filter.js, test-aging.js, test-frontend-helpers.js); Go tests under `cmd/` +- Playwright E2E: `tests/e2e/`, classified in `scripts/non-unit-tests.json` (e.g. test-e2e-playwright.js, default localhost:3000) - Coverage: Backend 85%+ (c8), Frontend 42%+ (Istanbul + nyc). Both only go up. - Review authority: May approve or reject work from Hicks and Newt based on test results ## Boundaries - Test the REAL code — import actual modules, don't copy-paste functions into test files -- Use vm.createContext for frontend helpers (see test-frontend-helpers.js pattern) +- Use vm.createContext for frontend helpers (see tests/unit/test-frontend-helpers.js pattern) - Playwright tests default to localhost:3000 — NEVER run against prod - Every bug fix gets a regression test - Every new feature must add tests — test count only goes up @@ -37,10 +36,9 @@ Unit tests, Playwright E2E, coverage gates, and quality assurance for CoreScope. npm test # all backend tests + coverage summary npm run test:unit # fast: unit tests only npm run test:coverage # all tests + HTML coverage report -node test-packet-filter.js # filter engine -node test-decoder.js # packet decoder -node test-server-routes.js # API routes via supertest -node test-e2e-playwright.js # 8 Playwright browser tests +sh test-all.sh # every suite in tests/unit +node tests/unit/test-packet-filter.js # filter engine +node tests/e2e/test-e2e-playwright.js # Playwright browser tests ``` ## Model diff --git a/.squad/agents/newt/charter.md b/.squad/agents/newt/charter.md index 223f424b2..fc87a6f5a 100644 --- a/.squad/agents/newt/charter.md +++ b/.squad/agents/newt/charter.md @@ -24,7 +24,7 @@ Vanilla JS UI, Leaflet maps, live visualization, theming, and all public/ module - Use shared helpers from roles.js (ROLE_COLORS, TYPE_COLORS, getNodeStatus, getHealthThresholds) - Prefer `n.last_heard || n.last_seen` for display and status - No per-packet API calls from frontend — fetch bulk, filter client-side -- Run `node test-packet-filter.js` and `node test-frontend-helpers.js` after filter/helper changes +- Run `node tests/unit/test-packet-filter.js` and `node tests/unit/test-frontend-helpers.js` after filter/helper changes - Always bump cache busters in the SAME commit as code changes ## Key Files diff --git a/AGENTS.md b/AGENTS.md index fb25c6d55..badc82085 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -90,7 +90,7 @@ Every change must consider performance impact BEFORE implementation. This codeba No proof = no merge. ### 1. No commit without tests -Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node test-packet-filter.js && node test-aging.js && node test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. +Every change that touches logic MUST have tests. For Go backend: `cd cmd/server && go test ./...` and `cd cmd/ingestor && go test ./...`, or `make test` for all 14 modules. For frontend: `node tests/unit/test-packet-filter.js && node tests/unit/test-aging.js && node tests/unit/test-frontend-helpers.js`. If you add new logic, add tests. No exceptions. The SQLite driver (`github.com/mattn/go-sqlite3`) is cgo. **`CGO_ENABLED=0` still *builds*** — that is the trap. It links a stub, and the binary dies on the first query with `go-sqlite3 requires cgo to work. This is a stub`, so a green build proves nothing. A plain `GOOS=linux go build` likewise cannot cross-compile it. Use `make build` / `make crossbuild` — the latter needs [`zig`](https://ziglang.org/download/) as the cross C compiler and produces static musl binaries. Never re-add `CGO_ENABLED=0`. @@ -202,7 +202,7 @@ All colors MUST use CSS variables. Never hardcode `#hex` values outside of `:roo ### Packet Filter (packet-filter.js) Standalone module. No dependencies on app globals (copies what it needs). Testable in Node.js: ```bash -node test-packet-filter.js +node tests/unit/test-packet-filter.js ``` Uses firmware-standard type names (GRP_TXT, TXT_MSG, REQ) with aliases for convenience. @@ -219,21 +219,13 @@ npm run test:full-coverage # backend + instrumented frontend coverage via Playw ### Test Files ```bash # Backend (deterministic, run before every push) -node test-packet-filter.js # filter engine -node test-aging.js # node aging system -node test-regional-filter.js # regional observer filtering -node test-decoder.js # packet decoder -node test-decoder-spec.js # spec-driven + golden fixture tests -node test-server-helpers.js # extracted server functions -node test-server-routes.js # API route tests via supertest -node test-packet-store.js # in-memory packet store -node test-db.js # SQLite operations -node test-frontend-helpers.js # frontend logic (via vm.createContext) -node tools/e2e-test.js # E2E: temp server + synthetic packets -node tools/frontend-test.js # frontend smoke: HTML, JS refs, API shapes +sh test-all.sh # every suite in tests/unit +node tests/unit/test-packet-filter.js # filter engine +node tests/unit/test-aging.js # node aging system +node tests/unit/test-frontend-helpers.js # frontend logic (via vm.createContext) # Frontend E2E (requires running server or Playwright) -node test-e2e-playwright.js # 8 Playwright browser tests (default: localhost:3000) +node tests/e2e/test-e2e-playwright.js # Playwright browser tests (default: localhost:3000) ``` ### Rules @@ -248,14 +240,14 @@ node test-e2e-playwright.js # 8 Playwright browser tests (default: localho 2. Write unit tests for the logic 3. Write/update Playwright tests if it's a UI change 4. Run `npm test` — all tests must pass -5. Run `node test-e2e-playwright.js` against a local server — E2E must pass +5. Run `node tests/e2e/test-e2e-playwright.js` against a local server — E2E must pass 6. THEN push to master ### Testing infrastructure - **Backend coverage**: c8 tracks server-side code in-process - **Frontend coverage**: Istanbul instruments `public/*.js` → Playwright exercises them → `window.__coverage__` extracted → nyc reports. Instrumented files are generated fresh each CI run, never checked in. - **CI pipeline**: backend tests + coverage → instrument frontend → start local server → Playwright E2E + coverage collection → badges update → deploy (only if all pass) -- **Playwright tests default to localhost:3000** — NEVER run against prod. CI sets `BASE_URL=http://localhost:13581`. Running locally: start your server, then `node test-e2e-playwright.js` +- **Playwright tests default to localhost:3000** — NEVER run against prod. CI sets `BASE_URL=http://localhost:13581`. Running locally: start your server, then `node tests/e2e/test-e2e-playwright.js` - **ARM machines**: Basic Playwright tests work with system chromium (`CHROMIUM_PATH=/usr/bin/chromium-browser`). Heavy coverage collection scripts may crash — use CI for those. Tests that need live mesh data can use `https://analyzer.00id.net` — all API endpoints are public, no auth required. @@ -290,7 +282,7 @@ If the same logic exists in two places, it MUST be extracted into a shared funct ### Testability - **Write functions that are easy to test.** Pure functions (input → output, no side effects) are ideal. If a function reads from the DOM, the DB, and localStorage, it's untestable without mocking everything. - **Dependency injection enables testing.** Pass the node list, the map reference, the API function as parameters. Tests can substitute fakes. -- **Test the real code, not copies.** Don't paste a function into a test file and test the copy. Import/require the actual module. If the module isn't importable (IIFE, browser-only), refactor it so it is — or use `vm.createContext` like `test-frontend-helpers.js` does. +- **Test the real code, not copies.** Don't paste a function into a test file and test the copy. Import/require the actual module. If the module isn't importable (IIFE, browser-only), refactor it so it is — or use `vm.createContext` like `tests/unit/test-frontend-helpers.js` does. - **Every bug fix gets a regression test.** If it broke once, it'll break again. The test proves it stays fixed. ### Type Safety (without TypeScript) @@ -372,7 +364,7 @@ One logical change per commit. Each commit is deployable. Each commit has its te | Subagent timed out mid-work | 4 | Give clear scope, don't try to run slow pipelines locally | ## File Naming -- Tests: `test-{feature}.js` in repo root +- Tests: `test-{feature}.js` in `tests/unit/` (standalone, listed in `test-all.sh`) or `tests/e2e/` (browser, listed in `scripts/non-unit-tests.json`) — never in repo root - No build step, no transpilation — write ES2020 for server, ES5/6 for frontend (broad browser support) ### Deep Linking diff --git a/README.md b/README.md index b06d54a43..7345d3d02 100644 --- a/README.md +++ b/README.md @@ -228,7 +228,8 @@ corescope/ │ └── entrypoint-go.sh # Container entrypoint ├── Dockerfile # Multi-stage Go build + Alpine runtime ├── config.example.json # Example configuration -├── test-*.js # Node.js test suite (frontend + legacy) +├── tests/ # Node.js test suite: unit/ (test-all.sh) and e2e/ (Playwright) +├── test-all.sh # Runs every suite in tests/unit └── tools/ # Generators, E2E tests, utilities ``` @@ -266,7 +267,7 @@ make test npm test # Playwright E2E (requires running server on localhost:3000) -node test-e2e-playwright.js +node tests/e2e/test-e2e-playwright.js ``` ### Generate Test Data diff --git a/docs/agents/skills/fix-issue.md b/docs/agents/skills/fix-issue.md index dc2397a2f..6fe5109b7 100644 --- a/docs/agents/skills/fix-issue.md +++ b/docs/agents/skills/fix-issue.md @@ -55,7 +55,7 @@ If the fix touches ANY frontend file (`public/*.js`, `public/*.css`, or HTML) AN as a user-visible UX behavior (click/hover/navigation/rendering), the PR MUST include a browser-level assertion that would have caught the original bug — NOT only unit tests. -- Look for an existing E2E harness first (e.g. `test-e2e-playwright.js`, `e2e/`, `tests/e2e/`, +- Look for an existing E2E harness first (e.g. `tests/e2e/test-e2e-playwright.js`, `e2e/`, `tests/e2e/`, `playwright.config.*`, or whatever the repo uses). Add the assertion there. - If no E2E harness exists, ADD a small one rather than skip. A 30-line Playwright fixture hitting a public staging URL is acceptable. diff --git a/docs/frontend-testing.md b/docs/frontend-testing.md index ec2803bff..a13eaa3cd 100644 --- a/docs/frontend-testing.md +++ b/docs/frontend-testing.md @@ -13,13 +13,17 @@ Windows, use Git Bash and ensure `python3` resolves to Python 3 rather than a Microsoft Store alias; set `PYTHONUTF8=1` for the gate's Unicode output. No server or browser is started by this runner. -When adding a root `test-*.js` file: +Tests live in `tests/unit/` (standalone Node, run by `test-all.sh`) and +`tests/e2e/` (needs a browser, a server or extra tooling). They run from the +repo root, and refer to it as `REPO_ROOT`. When adding a `test-*.js` file: -1. Add its `node test-name.js` command to `test-all.sh` if it runs standalone. -2. Otherwise add it to `scripts/non-unit-tests.json`, with a prerequisite or +1. If it runs standalone, put it in `tests/unit/` and add its + `node tests/unit/test-name.js` command to `test-all.sh`. +2. Otherwise put it in `tests/e2e/` and add it to `scripts/non-unit-tests.json`, with a prerequisite or reason explaining why it belongs outside the unit runner. Browser suites still need explicit selection in the E2E workflow to become CI gates. -3. Run `node test-test-inventory.js`. The guard rejects unclassified files, +3. Run `node tests/unit/test-test-inventory.js`. The guard rejects `test-*.js` + files in the repo root, unclassified files, duplicate assignments, removed files left in a list, undocumented groups, and local/CI entry points that bypass the unit runner. diff --git a/scripts/non-unit-tests.json b/scripts/non-unit-tests.json index 85770ed67..3d7c59fc4 100644 --- a/scripts/non-unit-tests.json +++ b/scripts/non-unit-tests.json @@ -2,129 +2,129 @@ "browser": { "reason": "Requires Playwright and Chromium; most suites also require a running local fixture server via BASE_URL. Existing E2E workflow steps retain their separate setup and selection. Never point these tests at production.", "files": [ - "test-1110-live-filter.js", - "test-a11y-axe-1668.js", - "test-analytics-fluid-charts.js", - "test-audio-lab-1297-e2e.js", - "test-audio-live-1297-e2e.js", - "test-bottom-nav-1061-e2e.js", - "test-channel-color-picker-e2e.js", - "test-channel-decrypt-e2e.js", - "test-channel-fluid-e2e.js", - "test-channel-issue-1087-e2e.js", - "test-channel-issue-1111-e2e.js", - "test-channel-modal-e2e.js", - "test-channel-qr-e2e.js", - "test-channels-add-modal-e2e.js", - "test-channels-list-render-e2e.js", - "test-channels-selection-flow-e2e.js", - "test-channels-share-color-e2e.js", - "test-channels-ws-batch-e2e.js", - "test-channels-ws-race-1498-e2e.js", - "test-charts-fluid-1058-e2e.js", - "test-customize-branding-e2e.js", - "test-customize-display-e2e.js", - "test-customize-export-e2e.js", - "test-customize-theme-e2e.js", - "test-drag-manager-e2e.js", - "test-e2e-1267-mobile-vcr.js", - "test-e2e-playwright.js", - "test-filter-ux-e2e.js", - "test-gesture-hints-1065-e2e.js", - "test-gestures-1062-e2e.js", - "test-gestures-1185-scroll-discriminator-e2e.js", - "test-home-coverage-e2e.js", - "test-issue-1109-hamburger-dropdown-visible-e2e.js", - "test-issue-1122-packets-filter-ux-e2e.js", - "test-issue-1128-multi-viewport-e2e.js", - "test-issue-1128-packets-layout-e2e.js", - "test-issue-1136-live-region-e2e.js", - "test-issue-1146-path-link-contrast-e2e.js", - "test-issue-1147-section-order-e2e.js", - "test-issue-1150-404-state-e2e.js", - "test-issue-1151-orphan-separators-e2e.js", - "test-issue-1204-live-panel-structure-e2e.js", - "test-issue-1205-live-controls-anchor-e2e.js", - "test-issue-1206-resize-observer-leak-e2e.js", - "test-issue-1206-vcr-overlap-e2e.js", - "test-issue-1224-channels-mobile-ux-e2e.js", - "test-issue-1234-live-chrome-pass2-e2e.js", - "test-issue-1236-map-mobile-e2e.js", - "test-issue-1244-live-vcr-row-hints-e2e.js", - "test-issue-1273-qr-overlay-height-e2e.js", - "test-issue-1274-legend-coverage-e2e.js", - "test-issue-1279-legend-p2-e2e.js", - "test-issue-1281-location-row-e2e.js", - "test-issue-1306-collisions-terminology-e2e.js", - "test-issue-1329-map-controls-accordion-e2e.js", - "test-issue-1367-channels-chat-app-e2e.js", - "test-issue-1374-route-map-a11y-e2e.js", - "test-issue-1400-nav-vertical-clip.js", - "test-issue-1413-nav-overlap-e2e.js", - "test-issue-1486-collapse-reopens-detail-e2e.js", - "test-issue-1487-byop-modal-layout-e2e.js", - "test-issue-1510-live-nav-pin-e2e.js", - "test-issue-1567-corner-clears-drag-e2e.js", - "test-issue-1599-replay-freeze-e2e.js", - "test-issue-1630-reach-mobile-e2e.js", - "test-issue-1639-observers-sort-e2e.js", - "test-issue-1640-compare-discovery-e2e.js", - "test-issue-1648-m1-icons-e2e.js", - "test-issue-1648-m2-icons-e2e.js", - "test-issue-1648-m3-icons-e2e.js", - "test-issue-1648-m4-icons-e2e.js", - "test-issue-1648-m5-icons-e2e.js", - "test-issue-1657-analytics-channels-group-sprites-e2e.js", - "test-issue-1692-packets-init-parallel-e2e.js", - "test-issue-1705-subpath-contrast-e2e.js", - "test-issue-1758-ng-filter-rerenders-e2e.js", - "test-issue-1799-label-vocab-e2e.js", - "test-live-dedup.js", - "test-live-fullscreen-1572-e2e.js", - "test-live-layout-1178-1179-e2e.js", - "test-live-mql-leak-1180-e2e.js", - "test-live-multibyte-only-e2e.js", - "test-logo-default-sage-teal-e2e.js", - "test-logo-pulse-1173-e2e.js", - "test-logo-rebrand-e2e.js", - "test-logo-theme-e2e.js", - "test-map-modal-fluid-e2e.js", - "test-map-nodes-pagination-e2e.js", - "test-nav-drawer-1064-e2e.js", - "test-nav-fluid-1055-e2e.js", - "test-nav-more-floor-1139-e2e.js", - "test-nav-priority-1102-e2e.js", - "test-nav-priority-1311-e2e.js", - "test-nav-priority-1391-e2e.js", - "test-neighbor-map-btn-clip-e2e.js", - "test-node-reach-coverage-e2e.js", - "test-node-reach-e2e.js", - "test-nodes-export-e2e.js", - "test-observer-iata-1188-e2e.js", - "test-packets-scope-column.js", - "test-path-inspector-coverage-e2e.js", - "test-rx-coverage-mobile-nav-e2e.js", - "test-show-neighbors.js", - "test-slideover-1056-e2e.js", - "test-slideover-1168-munger-e2e.js", - "test-table-fluid-e2e.js", - "test-touch-gestures-coverage-e2e.js", - "test-touch-targets.js" + "tests/e2e/test-1110-live-filter.js", + "tests/e2e/test-a11y-axe-1668.js", + "tests/e2e/test-analytics-fluid-charts.js", + "tests/e2e/test-audio-lab-1297-e2e.js", + "tests/e2e/test-audio-live-1297-e2e.js", + "tests/e2e/test-bottom-nav-1061-e2e.js", + "tests/e2e/test-channel-color-picker-e2e.js", + "tests/e2e/test-channel-decrypt-e2e.js", + "tests/e2e/test-channel-fluid-e2e.js", + "tests/e2e/test-channel-issue-1087-e2e.js", + "tests/e2e/test-channel-issue-1111-e2e.js", + "tests/e2e/test-channel-modal-e2e.js", + "tests/e2e/test-channel-qr-e2e.js", + "tests/e2e/test-channels-add-modal-e2e.js", + "tests/e2e/test-channels-list-render-e2e.js", + "tests/e2e/test-channels-selection-flow-e2e.js", + "tests/e2e/test-channels-share-color-e2e.js", + "tests/e2e/test-channels-ws-batch-e2e.js", + "tests/e2e/test-channels-ws-race-1498-e2e.js", + "tests/e2e/test-charts-fluid-1058-e2e.js", + "tests/e2e/test-customize-branding-e2e.js", + "tests/e2e/test-customize-display-e2e.js", + "tests/e2e/test-customize-export-e2e.js", + "tests/e2e/test-customize-theme-e2e.js", + "tests/e2e/test-drag-manager-e2e.js", + "tests/e2e/test-e2e-1267-mobile-vcr.js", + "tests/e2e/test-e2e-playwright.js", + "tests/e2e/test-filter-ux-e2e.js", + "tests/e2e/test-gesture-hints-1065-e2e.js", + "tests/e2e/test-gestures-1062-e2e.js", + "tests/e2e/test-gestures-1185-scroll-discriminator-e2e.js", + "tests/e2e/test-home-coverage-e2e.js", + "tests/e2e/test-issue-1109-hamburger-dropdown-visible-e2e.js", + "tests/e2e/test-issue-1122-packets-filter-ux-e2e.js", + "tests/e2e/test-issue-1128-multi-viewport-e2e.js", + "tests/e2e/test-issue-1128-packets-layout-e2e.js", + "tests/e2e/test-issue-1136-live-region-e2e.js", + "tests/e2e/test-issue-1146-path-link-contrast-e2e.js", + "tests/e2e/test-issue-1147-section-order-e2e.js", + "tests/e2e/test-issue-1150-404-state-e2e.js", + "tests/e2e/test-issue-1151-orphan-separators-e2e.js", + "tests/e2e/test-issue-1204-live-panel-structure-e2e.js", + "tests/e2e/test-issue-1205-live-controls-anchor-e2e.js", + "tests/e2e/test-issue-1206-resize-observer-leak-e2e.js", + "tests/e2e/test-issue-1206-vcr-overlap-e2e.js", + "tests/e2e/test-issue-1224-channels-mobile-ux-e2e.js", + "tests/e2e/test-issue-1234-live-chrome-pass2-e2e.js", + "tests/e2e/test-issue-1236-map-mobile-e2e.js", + "tests/e2e/test-issue-1244-live-vcr-row-hints-e2e.js", + "tests/e2e/test-issue-1273-qr-overlay-height-e2e.js", + "tests/e2e/test-issue-1274-legend-coverage-e2e.js", + "tests/e2e/test-issue-1279-legend-p2-e2e.js", + "tests/e2e/test-issue-1281-location-row-e2e.js", + "tests/e2e/test-issue-1306-collisions-terminology-e2e.js", + "tests/e2e/test-issue-1329-map-controls-accordion-e2e.js", + "tests/e2e/test-issue-1367-channels-chat-app-e2e.js", + "tests/e2e/test-issue-1374-route-map-a11y-e2e.js", + "tests/e2e/test-issue-1400-nav-vertical-clip.js", + "tests/e2e/test-issue-1413-nav-overlap-e2e.js", + "tests/e2e/test-issue-1486-collapse-reopens-detail-e2e.js", + "tests/e2e/test-issue-1487-byop-modal-layout-e2e.js", + "tests/e2e/test-issue-1510-live-nav-pin-e2e.js", + "tests/e2e/test-issue-1567-corner-clears-drag-e2e.js", + "tests/e2e/test-issue-1599-replay-freeze-e2e.js", + "tests/e2e/test-issue-1630-reach-mobile-e2e.js", + "tests/e2e/test-issue-1639-observers-sort-e2e.js", + "tests/e2e/test-issue-1640-compare-discovery-e2e.js", + "tests/e2e/test-issue-1648-m1-icons-e2e.js", + "tests/e2e/test-issue-1648-m2-icons-e2e.js", + "tests/e2e/test-issue-1648-m3-icons-e2e.js", + "tests/e2e/test-issue-1648-m4-icons-e2e.js", + "tests/e2e/test-issue-1648-m5-icons-e2e.js", + "tests/e2e/test-issue-1657-analytics-channels-group-sprites-e2e.js", + "tests/e2e/test-issue-1692-packets-init-parallel-e2e.js", + "tests/e2e/test-issue-1705-subpath-contrast-e2e.js", + "tests/e2e/test-issue-1758-ng-filter-rerenders-e2e.js", + "tests/e2e/test-issue-1799-label-vocab-e2e.js", + "tests/e2e/test-live-dedup.js", + "tests/e2e/test-live-fullscreen-1572-e2e.js", + "tests/e2e/test-live-layout-1178-1179-e2e.js", + "tests/e2e/test-live-mql-leak-1180-e2e.js", + "tests/e2e/test-live-multibyte-only-e2e.js", + "tests/e2e/test-logo-default-sage-teal-e2e.js", + "tests/e2e/test-logo-pulse-1173-e2e.js", + "tests/e2e/test-logo-rebrand-e2e.js", + "tests/e2e/test-logo-theme-e2e.js", + "tests/e2e/test-map-modal-fluid-e2e.js", + "tests/e2e/test-map-nodes-pagination-e2e.js", + "tests/e2e/test-nav-drawer-1064-e2e.js", + "tests/e2e/test-nav-fluid-1055-e2e.js", + "tests/e2e/test-nav-more-floor-1139-e2e.js", + "tests/e2e/test-nav-priority-1102-e2e.js", + "tests/e2e/test-nav-priority-1311-e2e.js", + "tests/e2e/test-nav-priority-1391-e2e.js", + "tests/e2e/test-neighbor-map-btn-clip-e2e.js", + "tests/e2e/test-node-reach-coverage-e2e.js", + "tests/e2e/test-node-reach-e2e.js", + "tests/e2e/test-nodes-export-e2e.js", + "tests/e2e/test-observer-iata-1188-e2e.js", + "tests/e2e/test-packets-scope-column.js", + "tests/e2e/test-path-inspector-coverage-e2e.js", + "tests/e2e/test-rx-coverage-mobile-nav-e2e.js", + "tests/e2e/test-show-neighbors.js", + "tests/e2e/test-slideover-1056-e2e.js", + "tests/e2e/test-slideover-1168-munger-e2e.js", + "tests/e2e/test-table-fluid-e2e.js", + "tests/e2e/test-touch-gestures-coverage-e2e.js", + "tests/e2e/test-touch-targets.js" ] }, "playwrightRunner": { "reason": "Requires the @playwright/test runner (not plain node), its configuration, Chromium, and a local server. @playwright/test is not currently declared in package.json; these suites are inventoried but not part of the standalone unit gate.", "files": [ - "test-issue-1522-trace-url-sync-e2e.js", - "test-marker-outline-weight.js", - "test-path-inspector-e2e.js", - "test-pr-1490-live-map-gpu-animations-e2e.js" + "tests/e2e/test-issue-1522-trace-url-sync-e2e.js", + "tests/e2e/test-marker-outline-weight.js", + "tests/e2e/test-path-inspector-e2e.js", + "tests/e2e/test-pr-1490-live-map-gpu-animations-e2e.js" ] }, "domIntegration": { "reason": "Requires jsdom, which is not currently declared in package.json. This DOM integration suite is inventoried separately; no new dependency is installed by the unit gate.", "files": [ - "test-table-sort.js" + "tests/e2e/test-table-sort.js" ] } } diff --git a/test-all.sh b/test-all.sh index 58c8b4f6e..3d6d52454 100755 --- a/test-all.sh +++ b/test-all.sh @@ -1,185 +1,186 @@ #!/bin/sh # Authoritative standalone frontend test list for local runs and CI. -# Every root test is listed here or classified in scripts/non-unit-tests.json. +# Every suite in tests/unit is listed here; every suite in tests/e2e is classified +# in scripts/non-unit-tests.json. set -e cd "$(dirname "$0")" -node test-test-inventory.js -node test-1108-region-hide-nodes.js -node test-1659-analytics-warmup.js -node test-a11y-1705-subpath-hop-prefix-e2e.js -node test-a11y-1715-dark-role-swatches.js -node test-a11y-1716-rf-range-btn-active.js -node test-a11y-1719-contrast-root-causes-e2e.js -node test-a11y-axe-1668-selftest.js -node test-a11y-axe-routes-coverage.js -node test-aging.js -node test-analytics-channels-integration.js -node test-anl1-tooltip-render.js -node test-area-filter.js -node test-channel-color-picker.js -node test-channel-colors.js -node test-channel-decrypt-ecb.js -node test-channel-decrypt-insecure-context.js -node test-channel-decrypt-m345.js -node test-channel-fluid-layout.js -node test-channel-issue-1087.js -node test-channel-issue-1101.js -node test-channel-live-decrypt-userprefix.js -node test-channel-live-decrypt.js -node test-channel-modal-ux.js -node test-channel-psk-ux.js -node test-channel-qr-wiring.js -node test-channel-qr.js -node test-channel-sidebar-layout.js -node test-channel-ux-followup.js -node test-channel-ux-round2.js -node test-channels-merge-1498-unit.js -node test-clear-filters.js -node test-color-picker-ux.js -node test-compare-flood-filter.js -node test-compare-overlap.js -node test-confidence-indicator.js -node test-coverage-gate.js -node test-customizer-v2.js -node test-drag-manager.js -node test-embed-mode-1369.js -node test-fetch-all-nodes-pagination.js -node test-fluid-scaffolding.js -node test-frontend-helpers.js -node test-geofilter-draft.js -node test-hash-color.js -node test-hop-resolver-affinity.js -node test-issue-1065-gesture-hints-gates.js -node test-issue-1107-live-layout.js -node test-issue-1136-observer-iata-map.js -node test-issue-1166-first-seen-column.js -node test-issue-1189-composed-cell.js -node test-issue-1189-live-iata-badge.js -node test-issue-1279-p2-code-filter.js -node test-issue-1293-marker-shapes.js -node test-issue-1356-map-a11y.js -node test-issue-1360-pill-letter-count.js -node test-issue-1361-cb-presets.js -node test-issue-1364-pill-no-clamp.js -node test-issue-1375-scope-stats-fetch.js -node test-issue-1380-cb-reset-button.js -node test-issue-1380-cb-sim-overlay.js -node test-issue-1407-cb-preset-propagation.js -node test-issue-1409-no-encrypted-flood.js -node test-issue-1412-customizer-no-override.js -node test-issue-1415-packets-layout.js -node test-issue-1418-cb-preset-ramp.js -node test-issue-1418-deeplink-hops-channels.js -node test-issue-1418-edge-weights.js -node test-issue-1418-polish-review.js -node test-issue-1418-raw-hex-extraction.js -node test-issue-1418-spider-fan.js -node test-issue-1420-tile-providers.js -node test-issue-1438-customizer-mcrole.js -node test-issue-1438-marker-css-vars.js -node test-issue-1446-cb-preset-cascade.js -node test-issue-1450-logo-aspect.js -node test-issue-1454-channels-toggle.js -node test-issue-1456-score-labels.js -node test-issue-1461-mobile-page-actions.js -node test-issue-1470-card-bg-contrast.js -node test-issue-1470-node-tile-helper.js -node test-issue-1473-prefix-generator.js -node test-issue-1473-reserved-prefixes.js -node test-issue-1485-live-anim-z.js -node test-issue-1488-marker-stroke-vars.js -node test-issue-1496-reset-all-complete.js -node test-issue-1509-detect-preset.js -node test-issue-1509-nav-active-bg.js -node test-issue-1518-home-url.js -node test-issue-1532-live-fullscreen.js -node test-issue-1562-observers-summary.js -node test-issue-1563-aggregate-and-inflight.js -node test-issue-1574-live-map-max-nodes.js -node test-issue-1606-pagination.js -node test-issue-1614-tile-url-function.js -node test-issue-1619-feed-detail-card-draggable.js -node test-issue-1633-hide-1byte-hops.js -node test-issue-1644-redesign.js -node test-issue-1646-compare-polish.js -node test-issue-1648-followup-phosphor-leaks.js -node test-issue-1648-m1-emoji-scan.js -node test-issue-1648-m2-emoji-scan.js -node test-issue-1648-m3-emoji-scan.js -node test-issue-1648-m4-emoji-scan.js -node test-issue-1648-m5-emoji-scan.js -node test-issue-1648-m6-final-sweep.js -node test-issue-1648-m6-lint-self.js -node test-issue-1668-m2-contrast.js -node test-issue-1668-m3-typography.js -node test-issue-1668-m4-per-route.js -node test-issue-1697-mqtt-mobile-e2e.js -node test-issue-1699-retransmission-chart.js -node test-issue-1705-subpath-contrast.js -node test-issue-1753-copy-url-slash.js -node test-issue-1770-mobile-row-clamp.js -node test-issue-1789-observer-firmware-cols.js -node test-issue-1825-observer-node-cross-links.js -node test-issue-1833-legend-toggle-vcr-offset.js -node test-issue-1836-crossnav-case-normalization.js -node test-issue-1843-node-qr-quiet-zone.js -node test-issue-1846-observers-width.js -node test-issue-1849-trace-hashbytes.js -node test-issue-1851-channel-message-scope.js -node test-issue-1862-map-region-filter.js -node test-issue-1868-control-decode.js -node test-issue-1890-og-url.js -node test-issue-1956-release-routing.js -node test-issue-1979-scope-adverts-by-role.js -node test-issue-2001-map-scope-state.js -node test-issue-2012-clear-filters-selection.js -node test-live-anims.js -node test-live-dt-cap-1524.js -node test-live-legend-helper.js -node test-live-multibyte-filter.js -node test-live-region-filter.js -node test-live.js -node test-map-clustering.js -node test-mqtt-status-panel.js -node test-my-repeaters-dashboard.js -node test-naive-banner-tone.js -node test-node-hop-analytics.js -node test-node-reach-coverage-debounce.js -node test-node-reach-coverage.js -node test-nodes-export-wiring.js -node test-nodes-export.js -node test-observer-iata-1188.js -node test-observer-menu-interactions.js -node test-observer-naive-clock-1478.js -node test-observer-search-filter.js -node test-observers-headings.js -node test-packet-filter-time.js -node test-packet-filter-ux.js -node test-packet-filter.js -node test-packets-local-channels.js -node test-packets.js -node test-panel-corner.js -node test-path-inspector.js -node test-payload-labels-content.js -node test-payload-labels-namespace.js -node test-perf-anomaly.js -node test-perf-disk-io-1120.js -node test-perf-go-runtime.js -node test-perf-render-1258.js -node test-preflight-xss-gate.js -node test-pull-to-reconnect-1091.js -node test-pull-to-reconnect.js -node test-repeater-metric-scatter.js -node test-rx-coverage-config-race.js -node test-rx-coverage-escape.js -node test-scope-audit-styles-linked.js -node test-slideover-1056-rowsel-strict.js -node test-top-routes-overlay.js -node test-traces.js -node test-url-state.js -node test-warmup-banner.js -node test-ws-stale-watchdog-1074.js -node test-xss-escape-sinks.js +node tests/unit/test-test-inventory.js +node tests/unit/test-1108-region-hide-nodes.js +node tests/unit/test-1659-analytics-warmup.js +node tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js +node tests/unit/test-a11y-1715-dark-role-swatches.js +node tests/unit/test-a11y-1716-rf-range-btn-active.js +node tests/unit/test-a11y-1719-contrast-root-causes-e2e.js +node tests/unit/test-a11y-axe-1668-selftest.js +node tests/unit/test-a11y-axe-routes-coverage.js +node tests/unit/test-aging.js +node tests/unit/test-analytics-channels-integration.js +node tests/unit/test-anl1-tooltip-render.js +node tests/unit/test-area-filter.js +node tests/unit/test-channel-color-picker.js +node tests/unit/test-channel-colors.js +node tests/unit/test-channel-decrypt-ecb.js +node tests/unit/test-channel-decrypt-insecure-context.js +node tests/unit/test-channel-decrypt-m345.js +node tests/unit/test-channel-fluid-layout.js +node tests/unit/test-channel-issue-1087.js +node tests/unit/test-channel-issue-1101.js +node tests/unit/test-channel-live-decrypt-userprefix.js +node tests/unit/test-channel-live-decrypt.js +node tests/unit/test-channel-modal-ux.js +node tests/unit/test-channel-psk-ux.js +node tests/unit/test-channel-qr-wiring.js +node tests/unit/test-channel-qr.js +node tests/unit/test-channel-sidebar-layout.js +node tests/unit/test-channel-ux-followup.js +node tests/unit/test-channel-ux-round2.js +node tests/unit/test-channels-merge-1498-unit.js +node tests/unit/test-clear-filters.js +node tests/unit/test-color-picker-ux.js +node tests/unit/test-compare-flood-filter.js +node tests/unit/test-compare-overlap.js +node tests/unit/test-confidence-indicator.js +node tests/unit/test-coverage-gate.js +node tests/unit/test-customizer-v2.js +node tests/unit/test-drag-manager.js +node tests/unit/test-embed-mode-1369.js +node tests/unit/test-fetch-all-nodes-pagination.js +node tests/unit/test-fluid-scaffolding.js +node tests/unit/test-frontend-helpers.js +node tests/unit/test-geofilter-draft.js +node tests/unit/test-hash-color.js +node tests/unit/test-hop-resolver-affinity.js +node tests/unit/test-issue-1065-gesture-hints-gates.js +node tests/unit/test-issue-1107-live-layout.js +node tests/unit/test-issue-1136-observer-iata-map.js +node tests/unit/test-issue-1166-first-seen-column.js +node tests/unit/test-issue-1189-composed-cell.js +node tests/unit/test-issue-1189-live-iata-badge.js +node tests/unit/test-issue-1279-p2-code-filter.js +node tests/unit/test-issue-1293-marker-shapes.js +node tests/unit/test-issue-1356-map-a11y.js +node tests/unit/test-issue-1360-pill-letter-count.js +node tests/unit/test-issue-1361-cb-presets.js +node tests/unit/test-issue-1364-pill-no-clamp.js +node tests/unit/test-issue-1375-scope-stats-fetch.js +node tests/unit/test-issue-1380-cb-reset-button.js +node tests/unit/test-issue-1380-cb-sim-overlay.js +node tests/unit/test-issue-1407-cb-preset-propagation.js +node tests/unit/test-issue-1409-no-encrypted-flood.js +node tests/unit/test-issue-1412-customizer-no-override.js +node tests/unit/test-issue-1415-packets-layout.js +node tests/unit/test-issue-1418-cb-preset-ramp.js +node tests/unit/test-issue-1418-deeplink-hops-channels.js +node tests/unit/test-issue-1418-edge-weights.js +node tests/unit/test-issue-1418-polish-review.js +node tests/unit/test-issue-1418-raw-hex-extraction.js +node tests/unit/test-issue-1418-spider-fan.js +node tests/unit/test-issue-1420-tile-providers.js +node tests/unit/test-issue-1438-customizer-mcrole.js +node tests/unit/test-issue-1438-marker-css-vars.js +node tests/unit/test-issue-1446-cb-preset-cascade.js +node tests/unit/test-issue-1450-logo-aspect.js +node tests/unit/test-issue-1454-channels-toggle.js +node tests/unit/test-issue-1456-score-labels.js +node tests/unit/test-issue-1461-mobile-page-actions.js +node tests/unit/test-issue-1470-card-bg-contrast.js +node tests/unit/test-issue-1470-node-tile-helper.js +node tests/unit/test-issue-1473-prefix-generator.js +node tests/unit/test-issue-1473-reserved-prefixes.js +node tests/unit/test-issue-1485-live-anim-z.js +node tests/unit/test-issue-1488-marker-stroke-vars.js +node tests/unit/test-issue-1496-reset-all-complete.js +node tests/unit/test-issue-1509-detect-preset.js +node tests/unit/test-issue-1509-nav-active-bg.js +node tests/unit/test-issue-1518-home-url.js +node tests/unit/test-issue-1532-live-fullscreen.js +node tests/unit/test-issue-1562-observers-summary.js +node tests/unit/test-issue-1563-aggregate-and-inflight.js +node tests/unit/test-issue-1574-live-map-max-nodes.js +node tests/unit/test-issue-1606-pagination.js +node tests/unit/test-issue-1614-tile-url-function.js +node tests/unit/test-issue-1619-feed-detail-card-draggable.js +node tests/unit/test-issue-1633-hide-1byte-hops.js +node tests/unit/test-issue-1644-redesign.js +node tests/unit/test-issue-1646-compare-polish.js +node tests/unit/test-issue-1648-followup-phosphor-leaks.js +node tests/unit/test-issue-1648-m1-emoji-scan.js +node tests/unit/test-issue-1648-m2-emoji-scan.js +node tests/unit/test-issue-1648-m3-emoji-scan.js +node tests/unit/test-issue-1648-m4-emoji-scan.js +node tests/unit/test-issue-1648-m5-emoji-scan.js +node tests/unit/test-issue-1648-m6-final-sweep.js +node tests/unit/test-issue-1648-m6-lint-self.js +node tests/unit/test-issue-1668-m2-contrast.js +node tests/unit/test-issue-1668-m3-typography.js +node tests/unit/test-issue-1668-m4-per-route.js +node tests/unit/test-issue-1697-mqtt-mobile-e2e.js +node tests/unit/test-issue-1699-retransmission-chart.js +node tests/unit/test-issue-1705-subpath-contrast.js +node tests/unit/test-issue-1753-copy-url-slash.js +node tests/unit/test-issue-1770-mobile-row-clamp.js +node tests/unit/test-issue-1789-observer-firmware-cols.js +node tests/unit/test-issue-1825-observer-node-cross-links.js +node tests/unit/test-issue-1833-legend-toggle-vcr-offset.js +node tests/unit/test-issue-1836-crossnav-case-normalization.js +node tests/unit/test-issue-1843-node-qr-quiet-zone.js +node tests/unit/test-issue-1846-observers-width.js +node tests/unit/test-issue-1849-trace-hashbytes.js +node tests/unit/test-issue-1851-channel-message-scope.js +node tests/unit/test-issue-1862-map-region-filter.js +node tests/unit/test-issue-1868-control-decode.js +node tests/unit/test-issue-1890-og-url.js +node tests/unit/test-issue-1956-release-routing.js +node tests/unit/test-issue-1979-scope-adverts-by-role.js +node tests/unit/test-issue-2001-map-scope-state.js +node tests/unit/test-issue-2012-clear-filters-selection.js +node tests/unit/test-live-anims.js +node tests/unit/test-live-dt-cap-1524.js +node tests/unit/test-live-legend-helper.js +node tests/unit/test-live-multibyte-filter.js +node tests/unit/test-live-region-filter.js +node tests/unit/test-live.js +node tests/unit/test-map-clustering.js +node tests/unit/test-mqtt-status-panel.js +node tests/unit/test-my-repeaters-dashboard.js +node tests/unit/test-naive-banner-tone.js +node tests/unit/test-node-hop-analytics.js +node tests/unit/test-node-reach-coverage-debounce.js +node tests/unit/test-node-reach-coverage.js +node tests/unit/test-nodes-export-wiring.js +node tests/unit/test-nodes-export.js +node tests/unit/test-observer-iata-1188.js +node tests/unit/test-observer-menu-interactions.js +node tests/unit/test-observer-naive-clock-1478.js +node tests/unit/test-observer-search-filter.js +node tests/unit/test-observers-headings.js +node tests/unit/test-packet-filter-time.js +node tests/unit/test-packet-filter-ux.js +node tests/unit/test-packet-filter.js +node tests/unit/test-packets-local-channels.js +node tests/unit/test-packets.js +node tests/unit/test-panel-corner.js +node tests/unit/test-path-inspector.js +node tests/unit/test-payload-labels-content.js +node tests/unit/test-payload-labels-namespace.js +node tests/unit/test-perf-anomaly.js +node tests/unit/test-perf-disk-io-1120.js +node tests/unit/test-perf-go-runtime.js +node tests/unit/test-perf-render-1258.js +node tests/unit/test-preflight-xss-gate.js +node tests/unit/test-pull-to-reconnect-1091.js +node tests/unit/test-pull-to-reconnect.js +node tests/unit/test-repeater-metric-scatter.js +node tests/unit/test-rx-coverage-config-race.js +node tests/unit/test-rx-coverage-escape.js +node tests/unit/test-scope-audit-styles-linked.js +node tests/unit/test-slideover-1056-rowsel-strict.js +node tests/unit/test-top-routes-overlay.js +node tests/unit/test-traces.js +node tests/unit/test-url-state.js +node tests/unit/test-warmup-banner.js +node tests/unit/test-ws-stale-watchdog-1074.js +node tests/unit/test-xss-escape-sinks.js echo "All standalone frontend suites passed" diff --git a/test-1110-live-filter.js b/tests/e2e/test-1110-live-filter.js similarity index 100% rename from test-1110-live-filter.js rename to tests/e2e/test-1110-live-filter.js diff --git a/test-a11y-axe-1668.js b/tests/e2e/test-a11y-axe-1668.js similarity index 99% rename from test-a11y-axe-1668.js rename to tests/e2e/test-a11y-axe-1668.js index f260ff5f3..5c38dab69 100644 --- a/test-a11y-axe-1668.js +++ b/tests/e2e/test-a11y-axe-1668.js @@ -31,6 +31,7 @@ */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); @@ -41,7 +42,7 @@ const path = require('path'); const BASE = process.env.BASE_URL || 'http://localhost:13581'; const ROUTES_FILTER = (process.env.AXE_ROUTES_ONLY || '').split(',').filter(Boolean); const SHOT_DIR = process.env.AXE_SCREENSHOT_DIR || '/tmp/axe-1668'; -const ALLOWLIST_PATH = path.join(__dirname, 'tests', 'a11y-allowlist.yaml'); +const ALLOWLIST_PATH = path.join(REPO_ROOT, 'tests', 'a11y-allowlist.yaml'); // Routes: M1 audit baseline (already proven coverage). // Hash routes — CoreScope is a SPA, server returns the same shell for any path. diff --git a/test-analytics-fluid-charts.js b/tests/e2e/test-analytics-fluid-charts.js similarity index 98% rename from test-analytics-fluid-charts.js rename to tests/e2e/test-analytics-fluid-charts.js index b160700b1..ba56f2524 100644 --- a/test-analytics-fluid-charts.js +++ b/tests/e2e/test-analytics-fluid-charts.js @@ -19,12 +19,13 @@ * Usage: node test-analytics-fluid-charts.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const { chromium } = require('playwright'); const fs = require('fs'); const path = require('path'); const os = require('os'); -const CSS_PATH = path.join(__dirname, 'public', 'style.css'); +const CSS_PATH = path.join(REPO_ROOT, 'public', 'style.css'); const cssHref = 'file://' + CSS_PATH; // Minimal harness: a sized wrapper that defines the available width diff --git a/test-audio-lab-1297-e2e.js b/tests/e2e/test-audio-lab-1297-e2e.js similarity index 100% rename from test-audio-lab-1297-e2e.js rename to tests/e2e/test-audio-lab-1297-e2e.js diff --git a/test-audio-live-1297-e2e.js b/tests/e2e/test-audio-live-1297-e2e.js similarity index 100% rename from test-audio-live-1297-e2e.js rename to tests/e2e/test-audio-live-1297-e2e.js diff --git a/test-bottom-nav-1061-e2e.js b/tests/e2e/test-bottom-nav-1061-e2e.js similarity index 100% rename from test-bottom-nav-1061-e2e.js rename to tests/e2e/test-bottom-nav-1061-e2e.js diff --git a/test-channel-color-picker-e2e.js b/tests/e2e/test-channel-color-picker-e2e.js similarity index 100% rename from test-channel-color-picker-e2e.js rename to tests/e2e/test-channel-color-picker-e2e.js diff --git a/test-channel-decrypt-e2e.js b/tests/e2e/test-channel-decrypt-e2e.js similarity index 100% rename from test-channel-decrypt-e2e.js rename to tests/e2e/test-channel-decrypt-e2e.js diff --git a/test-channel-fluid-e2e.js b/tests/e2e/test-channel-fluid-e2e.js similarity index 100% rename from test-channel-fluid-e2e.js rename to tests/e2e/test-channel-fluid-e2e.js diff --git a/test-channel-issue-1087-e2e.js b/tests/e2e/test-channel-issue-1087-e2e.js similarity index 100% rename from test-channel-issue-1087-e2e.js rename to tests/e2e/test-channel-issue-1087-e2e.js diff --git a/test-channel-issue-1111-e2e.js b/tests/e2e/test-channel-issue-1111-e2e.js similarity index 100% rename from test-channel-issue-1111-e2e.js rename to tests/e2e/test-channel-issue-1111-e2e.js diff --git a/test-channel-modal-e2e.js b/tests/e2e/test-channel-modal-e2e.js similarity index 100% rename from test-channel-modal-e2e.js rename to tests/e2e/test-channel-modal-e2e.js diff --git a/test-channel-qr-e2e.js b/tests/e2e/test-channel-qr-e2e.js similarity index 100% rename from test-channel-qr-e2e.js rename to tests/e2e/test-channel-qr-e2e.js diff --git a/test-channels-add-modal-e2e.js b/tests/e2e/test-channels-add-modal-e2e.js similarity index 100% rename from test-channels-add-modal-e2e.js rename to tests/e2e/test-channels-add-modal-e2e.js diff --git a/test-channels-list-render-e2e.js b/tests/e2e/test-channels-list-render-e2e.js similarity index 100% rename from test-channels-list-render-e2e.js rename to tests/e2e/test-channels-list-render-e2e.js diff --git a/test-channels-selection-flow-e2e.js b/tests/e2e/test-channels-selection-flow-e2e.js similarity index 100% rename from test-channels-selection-flow-e2e.js rename to tests/e2e/test-channels-selection-flow-e2e.js diff --git a/test-channels-share-color-e2e.js b/tests/e2e/test-channels-share-color-e2e.js similarity index 100% rename from test-channels-share-color-e2e.js rename to tests/e2e/test-channels-share-color-e2e.js diff --git a/test-channels-ws-batch-e2e.js b/tests/e2e/test-channels-ws-batch-e2e.js similarity index 100% rename from test-channels-ws-batch-e2e.js rename to tests/e2e/test-channels-ws-batch-e2e.js diff --git a/test-channels-ws-race-1498-e2e.js b/tests/e2e/test-channels-ws-race-1498-e2e.js similarity index 100% rename from test-channels-ws-race-1498-e2e.js rename to tests/e2e/test-channels-ws-race-1498-e2e.js diff --git a/test-charts-fluid-1058-e2e.js b/tests/e2e/test-charts-fluid-1058-e2e.js similarity index 100% rename from test-charts-fluid-1058-e2e.js rename to tests/e2e/test-charts-fluid-1058-e2e.js diff --git a/test-customize-branding-e2e.js b/tests/e2e/test-customize-branding-e2e.js similarity index 100% rename from test-customize-branding-e2e.js rename to tests/e2e/test-customize-branding-e2e.js diff --git a/test-customize-display-e2e.js b/tests/e2e/test-customize-display-e2e.js similarity index 100% rename from test-customize-display-e2e.js rename to tests/e2e/test-customize-display-e2e.js diff --git a/test-customize-export-e2e.js b/tests/e2e/test-customize-export-e2e.js similarity index 100% rename from test-customize-export-e2e.js rename to tests/e2e/test-customize-export-e2e.js diff --git a/test-customize-theme-e2e.js b/tests/e2e/test-customize-theme-e2e.js similarity index 100% rename from test-customize-theme-e2e.js rename to tests/e2e/test-customize-theme-e2e.js diff --git a/test-drag-manager-e2e.js b/tests/e2e/test-drag-manager-e2e.js similarity index 100% rename from test-drag-manager-e2e.js rename to tests/e2e/test-drag-manager-e2e.js diff --git a/test-e2e-1267-mobile-vcr.js b/tests/e2e/test-e2e-1267-mobile-vcr.js similarity index 100% rename from test-e2e-1267-mobile-vcr.js rename to tests/e2e/test-e2e-1267-mobile-vcr.js diff --git a/test-e2e-playwright.js b/tests/e2e/test-e2e-playwright.js similarity index 99% rename from test-e2e-playwright.js rename to tests/e2e/test-e2e-playwright.js index a307959af..1c53d079e 100644 --- a/test-e2e-playwright.js +++ b/tests/e2e/test-e2e-playwright.js @@ -1,3 +1,4 @@ +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); /** * Playwright E2E tests — proof of concept * Runs against prod (analyzer.00id.net), read-only. @@ -2313,7 +2314,7 @@ async function run() { if (coverage) { const fs = require('fs'); const path = require('path'); - const outDir = path.join(__dirname, '.nyc_output'); + const outDir = path.join(REPO_ROOT, '.nyc_output'); if (!fs.existsSync(outDir)) fs.mkdirSync(outDir, { recursive: true }); fs.writeFileSync(path.join(outDir, 'e2e-coverage.json'), JSON.stringify(coverage)); console.log(`Frontend coverage from E2E: ${Object.keys(coverage).length} files`); diff --git a/test-filter-ux-e2e.js b/tests/e2e/test-filter-ux-e2e.js similarity index 100% rename from test-filter-ux-e2e.js rename to tests/e2e/test-filter-ux-e2e.js diff --git a/test-gesture-hints-1065-e2e.js b/tests/e2e/test-gesture-hints-1065-e2e.js similarity index 100% rename from test-gesture-hints-1065-e2e.js rename to tests/e2e/test-gesture-hints-1065-e2e.js diff --git a/test-gestures-1062-e2e.js b/tests/e2e/test-gestures-1062-e2e.js similarity index 100% rename from test-gestures-1062-e2e.js rename to tests/e2e/test-gestures-1062-e2e.js diff --git a/test-gestures-1185-scroll-discriminator-e2e.js b/tests/e2e/test-gestures-1185-scroll-discriminator-e2e.js similarity index 100% rename from test-gestures-1185-scroll-discriminator-e2e.js rename to tests/e2e/test-gestures-1185-scroll-discriminator-e2e.js diff --git a/test-home-coverage-e2e.js b/tests/e2e/test-home-coverage-e2e.js similarity index 100% rename from test-home-coverage-e2e.js rename to tests/e2e/test-home-coverage-e2e.js diff --git a/test-issue-1109-hamburger-dropdown-visible-e2e.js b/tests/e2e/test-issue-1109-hamburger-dropdown-visible-e2e.js similarity index 100% rename from test-issue-1109-hamburger-dropdown-visible-e2e.js rename to tests/e2e/test-issue-1109-hamburger-dropdown-visible-e2e.js diff --git a/test-issue-1122-packets-filter-ux-e2e.js b/tests/e2e/test-issue-1122-packets-filter-ux-e2e.js similarity index 100% rename from test-issue-1122-packets-filter-ux-e2e.js rename to tests/e2e/test-issue-1122-packets-filter-ux-e2e.js diff --git a/test-issue-1128-multi-viewport-e2e.js b/tests/e2e/test-issue-1128-multi-viewport-e2e.js similarity index 100% rename from test-issue-1128-multi-viewport-e2e.js rename to tests/e2e/test-issue-1128-multi-viewport-e2e.js diff --git a/test-issue-1128-packets-layout-e2e.js b/tests/e2e/test-issue-1128-packets-layout-e2e.js similarity index 100% rename from test-issue-1128-packets-layout-e2e.js rename to tests/e2e/test-issue-1128-packets-layout-e2e.js diff --git a/test-issue-1136-live-region-e2e.js b/tests/e2e/test-issue-1136-live-region-e2e.js similarity index 100% rename from test-issue-1136-live-region-e2e.js rename to tests/e2e/test-issue-1136-live-region-e2e.js diff --git a/test-issue-1146-path-link-contrast-e2e.js b/tests/e2e/test-issue-1146-path-link-contrast-e2e.js similarity index 100% rename from test-issue-1146-path-link-contrast-e2e.js rename to tests/e2e/test-issue-1146-path-link-contrast-e2e.js diff --git a/test-issue-1147-section-order-e2e.js b/tests/e2e/test-issue-1147-section-order-e2e.js similarity index 100% rename from test-issue-1147-section-order-e2e.js rename to tests/e2e/test-issue-1147-section-order-e2e.js diff --git a/test-issue-1150-404-state-e2e.js b/tests/e2e/test-issue-1150-404-state-e2e.js similarity index 100% rename from test-issue-1150-404-state-e2e.js rename to tests/e2e/test-issue-1150-404-state-e2e.js diff --git a/test-issue-1151-orphan-separators-e2e.js b/tests/e2e/test-issue-1151-orphan-separators-e2e.js similarity index 100% rename from test-issue-1151-orphan-separators-e2e.js rename to tests/e2e/test-issue-1151-orphan-separators-e2e.js diff --git a/test-issue-1204-live-panel-structure-e2e.js b/tests/e2e/test-issue-1204-live-panel-structure-e2e.js similarity index 100% rename from test-issue-1204-live-panel-structure-e2e.js rename to tests/e2e/test-issue-1204-live-panel-structure-e2e.js diff --git a/test-issue-1205-live-controls-anchor-e2e.js b/tests/e2e/test-issue-1205-live-controls-anchor-e2e.js similarity index 100% rename from test-issue-1205-live-controls-anchor-e2e.js rename to tests/e2e/test-issue-1205-live-controls-anchor-e2e.js diff --git a/test-issue-1206-resize-observer-leak-e2e.js b/tests/e2e/test-issue-1206-resize-observer-leak-e2e.js similarity index 100% rename from test-issue-1206-resize-observer-leak-e2e.js rename to tests/e2e/test-issue-1206-resize-observer-leak-e2e.js diff --git a/test-issue-1206-vcr-overlap-e2e.js b/tests/e2e/test-issue-1206-vcr-overlap-e2e.js similarity index 100% rename from test-issue-1206-vcr-overlap-e2e.js rename to tests/e2e/test-issue-1206-vcr-overlap-e2e.js diff --git a/test-issue-1224-channels-mobile-ux-e2e.js b/tests/e2e/test-issue-1224-channels-mobile-ux-e2e.js similarity index 100% rename from test-issue-1224-channels-mobile-ux-e2e.js rename to tests/e2e/test-issue-1224-channels-mobile-ux-e2e.js diff --git a/test-issue-1234-live-chrome-pass2-e2e.js b/tests/e2e/test-issue-1234-live-chrome-pass2-e2e.js similarity index 100% rename from test-issue-1234-live-chrome-pass2-e2e.js rename to tests/e2e/test-issue-1234-live-chrome-pass2-e2e.js diff --git a/test-issue-1236-map-mobile-e2e.js b/tests/e2e/test-issue-1236-map-mobile-e2e.js similarity index 100% rename from test-issue-1236-map-mobile-e2e.js rename to tests/e2e/test-issue-1236-map-mobile-e2e.js diff --git a/test-issue-1244-live-vcr-row-hints-e2e.js b/tests/e2e/test-issue-1244-live-vcr-row-hints-e2e.js similarity index 100% rename from test-issue-1244-live-vcr-row-hints-e2e.js rename to tests/e2e/test-issue-1244-live-vcr-row-hints-e2e.js diff --git a/test-issue-1273-qr-overlay-height-e2e.js b/tests/e2e/test-issue-1273-qr-overlay-height-e2e.js similarity index 100% rename from test-issue-1273-qr-overlay-height-e2e.js rename to tests/e2e/test-issue-1273-qr-overlay-height-e2e.js diff --git a/test-issue-1274-legend-coverage-e2e.js b/tests/e2e/test-issue-1274-legend-coverage-e2e.js similarity index 100% rename from test-issue-1274-legend-coverage-e2e.js rename to tests/e2e/test-issue-1274-legend-coverage-e2e.js diff --git a/test-issue-1279-legend-p2-e2e.js b/tests/e2e/test-issue-1279-legend-p2-e2e.js similarity index 100% rename from test-issue-1279-legend-p2-e2e.js rename to tests/e2e/test-issue-1279-legend-p2-e2e.js diff --git a/test-issue-1281-location-row-e2e.js b/tests/e2e/test-issue-1281-location-row-e2e.js similarity index 100% rename from test-issue-1281-location-row-e2e.js rename to tests/e2e/test-issue-1281-location-row-e2e.js diff --git a/test-issue-1306-collisions-terminology-e2e.js b/tests/e2e/test-issue-1306-collisions-terminology-e2e.js similarity index 100% rename from test-issue-1306-collisions-terminology-e2e.js rename to tests/e2e/test-issue-1306-collisions-terminology-e2e.js diff --git a/test-issue-1329-map-controls-accordion-e2e.js b/tests/e2e/test-issue-1329-map-controls-accordion-e2e.js similarity index 100% rename from test-issue-1329-map-controls-accordion-e2e.js rename to tests/e2e/test-issue-1329-map-controls-accordion-e2e.js diff --git a/test-issue-1367-channels-chat-app-e2e.js b/tests/e2e/test-issue-1367-channels-chat-app-e2e.js similarity index 100% rename from test-issue-1367-channels-chat-app-e2e.js rename to tests/e2e/test-issue-1367-channels-chat-app-e2e.js diff --git a/test-issue-1374-route-map-a11y-e2e.js b/tests/e2e/test-issue-1374-route-map-a11y-e2e.js similarity index 100% rename from test-issue-1374-route-map-a11y-e2e.js rename to tests/e2e/test-issue-1374-route-map-a11y-e2e.js diff --git a/test-issue-1400-nav-vertical-clip.js b/tests/e2e/test-issue-1400-nav-vertical-clip.js similarity index 100% rename from test-issue-1400-nav-vertical-clip.js rename to tests/e2e/test-issue-1400-nav-vertical-clip.js diff --git a/test-issue-1413-nav-overlap-e2e.js b/tests/e2e/test-issue-1413-nav-overlap-e2e.js similarity index 100% rename from test-issue-1413-nav-overlap-e2e.js rename to tests/e2e/test-issue-1413-nav-overlap-e2e.js diff --git a/test-issue-1486-collapse-reopens-detail-e2e.js b/tests/e2e/test-issue-1486-collapse-reopens-detail-e2e.js similarity index 100% rename from test-issue-1486-collapse-reopens-detail-e2e.js rename to tests/e2e/test-issue-1486-collapse-reopens-detail-e2e.js diff --git a/test-issue-1487-byop-modal-layout-e2e.js b/tests/e2e/test-issue-1487-byop-modal-layout-e2e.js similarity index 100% rename from test-issue-1487-byop-modal-layout-e2e.js rename to tests/e2e/test-issue-1487-byop-modal-layout-e2e.js diff --git a/test-issue-1510-live-nav-pin-e2e.js b/tests/e2e/test-issue-1510-live-nav-pin-e2e.js similarity index 100% rename from test-issue-1510-live-nav-pin-e2e.js rename to tests/e2e/test-issue-1510-live-nav-pin-e2e.js diff --git a/test-issue-1522-trace-url-sync-e2e.js b/tests/e2e/test-issue-1522-trace-url-sync-e2e.js similarity index 100% rename from test-issue-1522-trace-url-sync-e2e.js rename to tests/e2e/test-issue-1522-trace-url-sync-e2e.js diff --git a/test-issue-1567-corner-clears-drag-e2e.js b/tests/e2e/test-issue-1567-corner-clears-drag-e2e.js similarity index 100% rename from test-issue-1567-corner-clears-drag-e2e.js rename to tests/e2e/test-issue-1567-corner-clears-drag-e2e.js diff --git a/test-issue-1599-replay-freeze-e2e.js b/tests/e2e/test-issue-1599-replay-freeze-e2e.js similarity index 100% rename from test-issue-1599-replay-freeze-e2e.js rename to tests/e2e/test-issue-1599-replay-freeze-e2e.js diff --git a/test-issue-1630-reach-mobile-e2e.js b/tests/e2e/test-issue-1630-reach-mobile-e2e.js similarity index 100% rename from test-issue-1630-reach-mobile-e2e.js rename to tests/e2e/test-issue-1630-reach-mobile-e2e.js diff --git a/test-issue-1639-observers-sort-e2e.js b/tests/e2e/test-issue-1639-observers-sort-e2e.js similarity index 100% rename from test-issue-1639-observers-sort-e2e.js rename to tests/e2e/test-issue-1639-observers-sort-e2e.js diff --git a/test-issue-1640-compare-discovery-e2e.js b/tests/e2e/test-issue-1640-compare-discovery-e2e.js similarity index 100% rename from test-issue-1640-compare-discovery-e2e.js rename to tests/e2e/test-issue-1640-compare-discovery-e2e.js diff --git a/test-issue-1648-m1-icons-e2e.js b/tests/e2e/test-issue-1648-m1-icons-e2e.js similarity index 100% rename from test-issue-1648-m1-icons-e2e.js rename to tests/e2e/test-issue-1648-m1-icons-e2e.js diff --git a/test-issue-1648-m2-icons-e2e.js b/tests/e2e/test-issue-1648-m2-icons-e2e.js similarity index 100% rename from test-issue-1648-m2-icons-e2e.js rename to tests/e2e/test-issue-1648-m2-icons-e2e.js diff --git a/test-issue-1648-m3-icons-e2e.js b/tests/e2e/test-issue-1648-m3-icons-e2e.js similarity index 100% rename from test-issue-1648-m3-icons-e2e.js rename to tests/e2e/test-issue-1648-m3-icons-e2e.js diff --git a/test-issue-1648-m4-icons-e2e.js b/tests/e2e/test-issue-1648-m4-icons-e2e.js similarity index 100% rename from test-issue-1648-m4-icons-e2e.js rename to tests/e2e/test-issue-1648-m4-icons-e2e.js diff --git a/test-issue-1648-m5-icons-e2e.js b/tests/e2e/test-issue-1648-m5-icons-e2e.js similarity index 100% rename from test-issue-1648-m5-icons-e2e.js rename to tests/e2e/test-issue-1648-m5-icons-e2e.js diff --git a/test-issue-1657-analytics-channels-group-sprites-e2e.js b/tests/e2e/test-issue-1657-analytics-channels-group-sprites-e2e.js similarity index 100% rename from test-issue-1657-analytics-channels-group-sprites-e2e.js rename to tests/e2e/test-issue-1657-analytics-channels-group-sprites-e2e.js diff --git a/test-issue-1692-packets-init-parallel-e2e.js b/tests/e2e/test-issue-1692-packets-init-parallel-e2e.js similarity index 100% rename from test-issue-1692-packets-init-parallel-e2e.js rename to tests/e2e/test-issue-1692-packets-init-parallel-e2e.js diff --git a/test-issue-1705-subpath-contrast-e2e.js b/tests/e2e/test-issue-1705-subpath-contrast-e2e.js similarity index 98% rename from test-issue-1705-subpath-contrast-e2e.js rename to tests/e2e/test-issue-1705-subpath-contrast-e2e.js index 34cf68440..fdf417b9b 100644 --- a/test-issue-1705-subpath-contrast-e2e.js +++ b/tests/e2e/test-issue-1705-subpath-contrast-e2e.js @@ -30,12 +30,13 @@ * to convert SKIP -> FAIL. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); const { chromium } = require('playwright'); -const CSS_PATH = path.resolve(__dirname, 'public', 'style.css'); +const CSS_PATH = path.resolve(REPO_ROOT, 'public', 'style.css'); function fail(msg) { console.error(`test-issue-1705-subpath-contrast-e2e.js: FAIL — ${msg}`); diff --git a/test-issue-1758-ng-filter-rerenders-e2e.js b/tests/e2e/test-issue-1758-ng-filter-rerenders-e2e.js similarity index 100% rename from test-issue-1758-ng-filter-rerenders-e2e.js rename to tests/e2e/test-issue-1758-ng-filter-rerenders-e2e.js diff --git a/test-issue-1799-label-vocab-e2e.js b/tests/e2e/test-issue-1799-label-vocab-e2e.js similarity index 100% rename from test-issue-1799-label-vocab-e2e.js rename to tests/e2e/test-issue-1799-label-vocab-e2e.js diff --git a/test-live-dedup.js b/tests/e2e/test-live-dedup.js similarity index 100% rename from test-live-dedup.js rename to tests/e2e/test-live-dedup.js diff --git a/test-live-fullscreen-1572-e2e.js b/tests/e2e/test-live-fullscreen-1572-e2e.js similarity index 100% rename from test-live-fullscreen-1572-e2e.js rename to tests/e2e/test-live-fullscreen-1572-e2e.js diff --git a/test-live-layout-1178-1179-e2e.js b/tests/e2e/test-live-layout-1178-1179-e2e.js similarity index 100% rename from test-live-layout-1178-1179-e2e.js rename to tests/e2e/test-live-layout-1178-1179-e2e.js diff --git a/test-live-mql-leak-1180-e2e.js b/tests/e2e/test-live-mql-leak-1180-e2e.js similarity index 100% rename from test-live-mql-leak-1180-e2e.js rename to tests/e2e/test-live-mql-leak-1180-e2e.js diff --git a/test-live-multibyte-only-e2e.js b/tests/e2e/test-live-multibyte-only-e2e.js similarity index 100% rename from test-live-multibyte-only-e2e.js rename to tests/e2e/test-live-multibyte-only-e2e.js diff --git a/test-logo-default-sage-teal-e2e.js b/tests/e2e/test-logo-default-sage-teal-e2e.js similarity index 100% rename from test-logo-default-sage-teal-e2e.js rename to tests/e2e/test-logo-default-sage-teal-e2e.js diff --git a/test-logo-pulse-1173-e2e.js b/tests/e2e/test-logo-pulse-1173-e2e.js similarity index 100% rename from test-logo-pulse-1173-e2e.js rename to tests/e2e/test-logo-pulse-1173-e2e.js diff --git a/test-logo-rebrand-e2e.js b/tests/e2e/test-logo-rebrand-e2e.js similarity index 100% rename from test-logo-rebrand-e2e.js rename to tests/e2e/test-logo-rebrand-e2e.js diff --git a/test-logo-theme-e2e.js b/tests/e2e/test-logo-theme-e2e.js similarity index 100% rename from test-logo-theme-e2e.js rename to tests/e2e/test-logo-theme-e2e.js diff --git a/test-map-modal-fluid-e2e.js b/tests/e2e/test-map-modal-fluid-e2e.js similarity index 100% rename from test-map-modal-fluid-e2e.js rename to tests/e2e/test-map-modal-fluid-e2e.js diff --git a/test-map-nodes-pagination-e2e.js b/tests/e2e/test-map-nodes-pagination-e2e.js similarity index 100% rename from test-map-nodes-pagination-e2e.js rename to tests/e2e/test-map-nodes-pagination-e2e.js diff --git a/test-marker-outline-weight.js b/tests/e2e/test-marker-outline-weight.js similarity index 100% rename from test-marker-outline-weight.js rename to tests/e2e/test-marker-outline-weight.js diff --git a/test-nav-drawer-1064-e2e.js b/tests/e2e/test-nav-drawer-1064-e2e.js similarity index 100% rename from test-nav-drawer-1064-e2e.js rename to tests/e2e/test-nav-drawer-1064-e2e.js diff --git a/test-nav-fluid-1055-e2e.js b/tests/e2e/test-nav-fluid-1055-e2e.js similarity index 100% rename from test-nav-fluid-1055-e2e.js rename to tests/e2e/test-nav-fluid-1055-e2e.js diff --git a/test-nav-more-floor-1139-e2e.js b/tests/e2e/test-nav-more-floor-1139-e2e.js similarity index 100% rename from test-nav-more-floor-1139-e2e.js rename to tests/e2e/test-nav-more-floor-1139-e2e.js diff --git a/test-nav-priority-1102-e2e.js b/tests/e2e/test-nav-priority-1102-e2e.js similarity index 100% rename from test-nav-priority-1102-e2e.js rename to tests/e2e/test-nav-priority-1102-e2e.js diff --git a/test-nav-priority-1311-e2e.js b/tests/e2e/test-nav-priority-1311-e2e.js similarity index 100% rename from test-nav-priority-1311-e2e.js rename to tests/e2e/test-nav-priority-1311-e2e.js diff --git a/test-nav-priority-1391-e2e.js b/tests/e2e/test-nav-priority-1391-e2e.js similarity index 100% rename from test-nav-priority-1391-e2e.js rename to tests/e2e/test-nav-priority-1391-e2e.js diff --git a/test-neighbor-map-btn-clip-e2e.js b/tests/e2e/test-neighbor-map-btn-clip-e2e.js similarity index 97% rename from test-neighbor-map-btn-clip-e2e.js rename to tests/e2e/test-neighbor-map-btn-clip-e2e.js index 59d04cee4..a49b80a91 100644 --- a/test-neighbor-map-btn-clip-e2e.js +++ b/tests/e2e/test-neighbor-map-btn-clip-e2e.js @@ -35,15 +35,16 @@ * make the geometry assertions below fail. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('node:fs'); const path = require('node:path'); const assert = require('node:assert'); const { chromium } = require('playwright'); -const CSS = fs.readFileSync(path.join(__dirname, 'public', 'style.css'), 'utf8'); -const NODES_JS = fs.readFileSync(path.join(__dirname, 'public', 'nodes.js'), 'utf8'); -const ANALYTICS_JS = fs.readFileSync(path.join(__dirname, 'public', 'analytics.js'), 'utf8'); +const CSS = fs.readFileSync(path.join(REPO_ROOT, 'public', 'style.css'), 'utf8'); +const NODES_JS = fs.readFileSync(path.join(REPO_ROOT, 'public', 'nodes.js'), 'utf8'); +const ANALYTICS_JS = fs.readFileSync(path.join(REPO_ROOT, 'public', 'analytics.js'), 'utf8'); const HEIGHT = 900; // The bug's window: below ~640px the table finally overflows and the diff --git a/test-node-reach-coverage-e2e.js b/tests/e2e/test-node-reach-coverage-e2e.js similarity index 100% rename from test-node-reach-coverage-e2e.js rename to tests/e2e/test-node-reach-coverage-e2e.js diff --git a/test-node-reach-e2e.js b/tests/e2e/test-node-reach-e2e.js similarity index 100% rename from test-node-reach-e2e.js rename to tests/e2e/test-node-reach-e2e.js diff --git a/test-nodes-export-e2e.js b/tests/e2e/test-nodes-export-e2e.js similarity index 100% rename from test-nodes-export-e2e.js rename to tests/e2e/test-nodes-export-e2e.js diff --git a/test-observer-iata-1188-e2e.js b/tests/e2e/test-observer-iata-1188-e2e.js similarity index 100% rename from test-observer-iata-1188-e2e.js rename to tests/e2e/test-observer-iata-1188-e2e.js diff --git a/test-packets-scope-column.js b/tests/e2e/test-packets-scope-column.js similarity index 100% rename from test-packets-scope-column.js rename to tests/e2e/test-packets-scope-column.js diff --git a/test-path-inspector-coverage-e2e.js b/tests/e2e/test-path-inspector-coverage-e2e.js similarity index 100% rename from test-path-inspector-coverage-e2e.js rename to tests/e2e/test-path-inspector-coverage-e2e.js diff --git a/test-path-inspector-e2e.js b/tests/e2e/test-path-inspector-e2e.js similarity index 100% rename from test-path-inspector-e2e.js rename to tests/e2e/test-path-inspector-e2e.js diff --git a/test-pr-1490-live-map-gpu-animations-e2e.js b/tests/e2e/test-pr-1490-live-map-gpu-animations-e2e.js similarity index 100% rename from test-pr-1490-live-map-gpu-animations-e2e.js rename to tests/e2e/test-pr-1490-live-map-gpu-animations-e2e.js diff --git a/test-rx-coverage-mobile-nav-e2e.js b/tests/e2e/test-rx-coverage-mobile-nav-e2e.js similarity index 100% rename from test-rx-coverage-mobile-nav-e2e.js rename to tests/e2e/test-rx-coverage-mobile-nav-e2e.js diff --git a/test-show-neighbors.js b/tests/e2e/test-show-neighbors.js similarity index 100% rename from test-show-neighbors.js rename to tests/e2e/test-show-neighbors.js diff --git a/test-slideover-1056-e2e.js b/tests/e2e/test-slideover-1056-e2e.js similarity index 100% rename from test-slideover-1056-e2e.js rename to tests/e2e/test-slideover-1056-e2e.js diff --git a/test-slideover-1168-munger-e2e.js b/tests/e2e/test-slideover-1168-munger-e2e.js similarity index 100% rename from test-slideover-1168-munger-e2e.js rename to tests/e2e/test-slideover-1168-munger-e2e.js diff --git a/test-table-fluid-e2e.js b/tests/e2e/test-table-fluid-e2e.js similarity index 100% rename from test-table-fluid-e2e.js rename to tests/e2e/test-table-fluid-e2e.js diff --git a/test-table-sort.js b/tests/e2e/test-table-sort.js similarity index 98% rename from test-table-sort.js rename to tests/e2e/test-table-sort.js index a6255b911..af20c07e1 100644 --- a/test-table-sort.js +++ b/tests/e2e/test-table-sort.js @@ -1,5 +1,6 @@ /* test-table-sort.js — Unit tests for TableSort utility */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const { JSDOM } = require('jsdom'); const fs = require('fs'); @@ -27,7 +28,7 @@ function createDOM(html) { runScripts: 'dangerously' }); // Load TableSort into this DOM - const script = fs.readFileSync(path.join(__dirname, 'public', 'table-sort.js'), 'utf8'); + const script = fs.readFileSync(path.join(REPO_ROOT, 'public', 'table-sort.js'), 'utf8'); const el = dom.window.document.createElement('script'); el.textContent = script; dom.window.document.head.appendChild(el); diff --git a/test-touch-gestures-coverage-e2e.js b/tests/e2e/test-touch-gestures-coverage-e2e.js similarity index 100% rename from test-touch-gestures-coverage-e2e.js rename to tests/e2e/test-touch-gestures-coverage-e2e.js diff --git a/test-touch-targets.js b/tests/e2e/test-touch-targets.js similarity index 99% rename from test-touch-targets.js rename to tests/e2e/test-touch-targets.js index c509c1f37..86575f4d5 100644 --- a/test-touch-targets.js +++ b/tests/e2e/test-touch-targets.js @@ -17,13 +17,14 @@ * in CSS (:focus / :focus-within rule in the Touch Targets section). */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); const assert = require('assert'); const { chromium, devices } = require('playwright'); -const REPO = __dirname; +const REPO = REPO_ROOT; const CSS = fs.readFileSync(path.join(REPO, 'public/style.css'), 'utf8'); // Selectors we claim to make 48x48. Each entry: [selector, tag, classes, diff --git a/test-1108-region-hide-nodes.js b/tests/unit/test-1108-region-hide-nodes.js similarity index 96% rename from test-1108-region-hide-nodes.js rename to tests/unit/test-1108-region-hide-nodes.js index c693bfee9..2a4b8d52c 100644 --- a/test-1108-region-hide-nodes.js +++ b/tests/unit/test-1108-region-hide-nodes.js @@ -10,6 +10,7 @@ * (no DOM, no fetch). Mirrors the pattern from test-area-filter.js. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); const assert = require('assert'); @@ -37,7 +38,7 @@ function buildCtx(initialStorage) { }; ctx.window = ctx; vm.createContext(ctx); - const src = fs.readFileSync(__dirname + '/public/region-filter.js', 'utf8'); + const src = fs.readFileSync(REPO_ROOT + '/public/region-filter.js', 'utf8'); vm.runInContext(src, ctx); return ctx; } diff --git a/test-1659-analytics-warmup.js b/tests/unit/test-1659-analytics-warmup.js similarity index 100% rename from test-1659-analytics-warmup.js rename to tests/unit/test-1659-analytics-warmup.js diff --git a/test-a11y-1705-subpath-hop-prefix-e2e.js b/tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js similarity index 98% rename from test-a11y-1705-subpath-hop-prefix-e2e.js rename to tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js index 98dfbefae..959ef6c45 100644 --- a/test-a11y-1705-subpath-hop-prefix-e2e.js +++ b/tests/unit/test-a11y-1705-subpath-hop-prefix-e2e.js @@ -45,11 +45,12 @@ * serves from). */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); -const STYLE_CSS_PATH = process.env.STYLE_CSS_PATH || path.join(__dirname, 'public', 'style.css'); +const STYLE_CSS_PATH = process.env.STYLE_CSS_PATH || path.join(REPO_ROOT, 'public', 'style.css'); const THEMES = ['dark', 'light']; // -------------------- Pure helpers (unit-testable) -------------------- diff --git a/test-a11y-1715-dark-role-swatches.js b/tests/unit/test-a11y-1715-dark-role-swatches.js similarity index 98% rename from test-a11y-1715-dark-role-swatches.js rename to tests/unit/test-a11y-1715-dark-role-swatches.js index e60b32fac..0b9ed1820 100644 --- a/test-a11y-1715-dark-role-swatches.js +++ b/tests/unit/test-a11y-1715-dark-role-swatches.js @@ -32,12 +32,13 @@ * Usage: node test-a11y-1715-dark-role-swatches.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); const assert = require('assert'); -const ROOT = __dirname; +const ROOT = REPO_ROOT; const STYLE_CSS = process.env.STYLE_CSS_PATH || path.join(ROOT, 'public', 'style.css'); const ROLES = ['repeater', 'companion', 'room', 'sensor', 'observer']; diff --git a/test-a11y-1716-rf-range-btn-active.js b/tests/unit/test-a11y-1716-rf-range-btn-active.js similarity index 98% rename from test-a11y-1716-rf-range-btn-active.js rename to tests/unit/test-a11y-1716-rf-range-btn-active.js index db4b9afd2..9cbbe6413 100644 --- a/test-a11y-1716-rf-range-btn-active.js +++ b/tests/unit/test-a11y-1716-rf-range-btn-active.js @@ -31,11 +31,12 @@ * Usage: node test-a11y-1716-rf-range-btn-active.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); -const ROOT = __dirname; +const ROOT = REPO_ROOT; const STYLE_CSS = process.env.STYLE_CSS_PATH || path.join(ROOT, 'public', 'style.css'); // ---- WCAG helpers (copied small, no shared import) ---- diff --git a/test-a11y-1719-contrast-root-causes-e2e.js b/tests/unit/test-a11y-1719-contrast-root-causes-e2e.js similarity index 99% rename from test-a11y-1719-contrast-root-causes-e2e.js rename to tests/unit/test-a11y-1719-contrast-root-causes-e2e.js index d112555ee..8517fb9db 100644 --- a/test-a11y-1719-contrast-root-causes-e2e.js +++ b/tests/unit/test-a11y-1719-contrast-root-causes-e2e.js @@ -37,11 +37,12 @@ * Usage: node test-a11y-1719-contrast-root-causes-e2e.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); -const ROOT = __dirname; +const ROOT = REPO_ROOT; const STYLE_CSS = process.env.STYLE_CSS_PATH || path.join(ROOT, 'public', 'style.css'); const CUSTOMIZE_JS = path.join(ROOT, 'public', 'customize.js'); diff --git a/test-a11y-axe-1668-selftest.js b/tests/unit/test-a11y-axe-1668-selftest.js similarity index 98% rename from test-a11y-axe-1668-selftest.js rename to tests/unit/test-a11y-axe-1668-selftest.js index 86bb8361f..77f40f528 100644 --- a/test-a11y-axe-1668-selftest.js +++ b/tests/unit/test-a11y-axe-1668-selftest.js @@ -11,13 +11,14 @@ */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const assert = require('assert'); const fs = require('fs'); const os = require('os'); const path = require('path'); -const mod = require('./test-a11y-axe-1668.js'); +const mod = require('../e2e/test-a11y-axe-1668.js'); // ---- routes / themes -------------------------------------------------------- assert.ok(Array.isArray(mod.ROUTES), 'ROUTES must be an array'); @@ -103,7 +104,7 @@ for (const r of mod.ROUTES) { // Cross-check REGISTERED_* against actual source code so the constant cannot // drift silently from `registerPage()` calls or analytics tab `case` arms. -const repoRoot = __dirname; +const repoRoot = REPO_ROOT; const publicDir = path.join(repoRoot, 'public'); if (fs.existsSync(publicDir)) { const files = fs.readdirSync(publicDir).filter(f => f.endsWith('.js')); @@ -184,7 +185,7 @@ assert.throws( // And loadAllowlist must propagate the throw end-to-end via a tmpfile. const tmpMalformed = path.join(os.tmpdir(), `a11y-malformed-${process.pid}.yaml`); fs.writeFileSync(tmpMalformed, 'this :: is :: garbage\n!!! no\n'); -const ALLOWLIST_PATH_ORIG = require.resolve('./test-a11y-axe-1668.js'); +const ALLOWLIST_PATH_ORIG = require.resolve('../e2e/test-a11y-axe-1668.js'); // We can't easily redirect ALLOWLIST_PATH without re-requiring; instead call // parseAllowlistYaml directly on the tmpfile content for end-to-end coverage. assert.throws( @@ -382,7 +383,7 @@ assert.throws( ); // ---- repo allowlist file: shape sanity -------------------------------------- -const allowPath = path.join(__dirname, 'tests', 'a11y-allowlist.yaml'); +const allowPath = path.join(REPO_ROOT, 'tests', 'a11y-allowlist.yaml'); assert.ok(fs.existsSync(allowPath), `tests/a11y-allowlist.yaml missing at ${allowPath}`); const entries = mod.loadAllowlist(); for (const e of entries) { diff --git a/test-a11y-axe-routes-coverage.js b/tests/unit/test-a11y-axe-routes-coverage.js similarity index 93% rename from test-a11y-axe-routes-coverage.js rename to tests/unit/test-a11y-axe-routes-coverage.js index b2846281e..66b25dc64 100644 --- a/test-a11y-axe-routes-coverage.js +++ b/tests/unit/test-a11y-axe-routes-coverage.js @@ -15,15 +15,16 @@ */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const assert = require('assert'); const fs = require('fs'); const path = require('path'); -const mod = require('./test-a11y-axe-1668.js'); +const mod = require('../e2e/test-a11y-axe-1668.js'); const analyticsSrc = fs.readFileSync( - path.join(__dirname, 'public', 'analytics.js'), + path.join(REPO_ROOT, 'public', 'analytics.js'), 'utf8' ); diff --git a/test-aging.js b/tests/unit/test-aging.js similarity index 100% rename from test-aging.js rename to tests/unit/test-aging.js diff --git a/test-analytics-channels-integration.js b/tests/unit/test-analytics-channels-integration.js similarity index 98% rename from test-analytics-channels-integration.js rename to tests/unit/test-analytics-channels-integration.js index 430590944..8d0296edf 100644 --- a/test-analytics-channels-integration.js +++ b/tests/unit/test-analytics-channels-integration.js @@ -16,6 +16,7 @@ * jump to channel activity stats. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); @@ -55,7 +56,7 @@ global.histogram = () => ({ svg: '' }); // Load analytics.js — it self-registers global helpers we test. const analyticsSrc = fs.readFileSync( - path.join(__dirname, 'public/analytics.js'), + path.join(REPO_ROOT, 'public/analytics.js'), 'utf8' ); // Strip top-level `await` / module syntax — analytics.js is plain IIFE so it's diff --git a/test-anl1-tooltip-render.js b/tests/unit/test-anl1-tooltip-render.js similarity index 100% rename from test-anl1-tooltip-render.js rename to tests/unit/test-anl1-tooltip-render.js diff --git a/test-area-filter.js b/tests/unit/test-area-filter.js similarity index 100% rename from test-area-filter.js rename to tests/unit/test-area-filter.js diff --git a/test-channel-color-picker.js b/tests/unit/test-channel-color-picker.js similarity index 96% rename from test-channel-color-picker.js rename to tests/unit/test-channel-color-picker.js index 54de759bd..cbdc7f779 100644 --- a/test-channel-color-picker.js +++ b/tests/unit/test-channel-color-picker.js @@ -9,6 +9,7 @@ */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); const path = require('path'); @@ -113,7 +114,7 @@ assert( // --- Test 3: channel-colors.js getRowStyle uses border-left only --- console.log('\n=== channel-colors.js getRowStyle ==='); -const ccSource = fs.readFileSync(path.join(__dirname, 'public', 'channel-colors.js'), 'utf8'); +const ccSource = fs.readFileSync(path.join(REPO_ROOT, 'public', 'channel-colors.js'), 'utf8'); const ccCtx = { window: {}, localStorage: { @@ -140,7 +141,7 @@ assert(advertStyle === '', 'getRowStyle returns empty for non-GRP_TXT type'); // --- Test 4: channel-color-picker.js palette --- console.log('\n=== channel-color-picker.js palette ==='); -const pickerSource = fs.readFileSync(path.join(__dirname, 'public', 'channel-color-picker.js'), 'utf8'); +const pickerSource = fs.readFileSync(path.join(REPO_ROOT, 'public', 'channel-color-picker.js'), 'utf8'); const pickerCtx = { window: { ChannelColors: ccCtx.window.ChannelColors, matchMedia: () => ({ matches: false }) }, document: { diff --git a/test-channel-colors.js b/tests/unit/test-channel-colors.js similarity index 97% rename from test-channel-colors.js rename to tests/unit/test-channel-colors.js index a3e2426b8..19b537d1c 100644 --- a/test-channel-colors.js +++ b/tests/unit/test-channel-colors.js @@ -1,5 +1,6 @@ /* Unit tests for channel color highlighting (M1) — #271 */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); const assert = require('assert'); @@ -33,7 +34,7 @@ function makeSandbox() { }; ctx.window.ChannelColors = undefined; vm.createContext(ctx); - const src = fs.readFileSync(__dirname + '/public/channel-colors.js', 'utf8'); + const src = fs.readFileSync(REPO_ROOT + '/public/channel-colors.js', 'utf8'); vm.runInContext(src, ctx); return ctx; } @@ -217,7 +218,7 @@ test('channel-color-picker.js loads without error in sandbox', function() { ctx.setTimeout = function(fn) { fn(); }; ctx.window.innerWidth = 1024; ctx.window.innerHeight = 768; - const pickerSrc = fs.readFileSync(__dirname + '/public/channel-color-picker.js', 'utf8'); + const pickerSrc = fs.readFileSync(REPO_ROOT + '/public/channel-color-picker.js', 'utf8'); vm.runInContext(pickerSrc, ctx); assert.ok(ctx.window.ChannelColorPicker, 'ChannelColorPicker should be exported'); assert.strictEqual(typeof ctx.window.ChannelColorPicker.install, 'function'); @@ -247,7 +248,7 @@ test('ChannelColorPicker.install does not throw when elements missing', function ctx.setTimeout = function(fn) { fn(); }; ctx.window.innerWidth = 1024; ctx.window.innerHeight = 768; - const pickerSrc = fs.readFileSync(__dirname + '/public/channel-color-picker.js', 'utf8'); + const pickerSrc = fs.readFileSync(REPO_ROOT + '/public/channel-color-picker.js', 'utf8'); vm.runInContext(pickerSrc, ctx); // Should not throw when feed/table elements don't exist ctx.window.ChannelColorPicker.install(); diff --git a/test-channel-decrypt-ecb.js b/tests/unit/test-channel-decrypt-ecb.js similarity index 95% rename from test-channel-decrypt-ecb.js rename to tests/unit/test-channel-decrypt-ecb.js index 918a6ea17..bf98e2a09 100644 --- a/test-channel-decrypt-ecb.js +++ b/tests/unit/test-channel-decrypt-ecb.js @@ -17,6 +17,7 @@ * ciphertext = 69c4e0d86a7b0430d8cdb78070b4c55a */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); @@ -47,12 +48,12 @@ function loadChannelDecrypt() { vm.createContext(sandbox); // Load vendored AES (if present) before channel-decrypt.js. - const vendorPath = path.join(__dirname, 'public/vendor/aes-ecb.js'); + const vendorPath = path.join(REPO_ROOT, 'public/vendor/aes-ecb.js'); if (fs.existsSync(vendorPath)) { vm.runInContext(fs.readFileSync(vendorPath, 'utf8'), sandbox); } vm.runInContext( - fs.readFileSync(path.join(__dirname, 'public/channel-decrypt.js'), 'utf8'), + fs.readFileSync(path.join(REPO_ROOT, 'public/channel-decrypt.js'), 'utf8'), sandbox ); return sandbox.window.ChannelDecrypt; diff --git a/test-channel-decrypt-insecure-context.js b/tests/unit/test-channel-decrypt-insecure-context.js similarity index 96% rename from test-channel-decrypt-insecure-context.js rename to tests/unit/test-channel-decrypt-insecure-context.js index 7f69179a1..f39a856dd 100644 --- a/test-channel-decrypt-insecure-context.js +++ b/tests/unit/test-channel-decrypt-insecure-context.js @@ -32,6 +32,7 @@ * mac = b0344c61d8db38535ca8afceaf0bf12b881dc200c9833da726e9376c2e32cff7 */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); @@ -63,20 +64,20 @@ function loadChannelDecryptInsecureContext() { vm.createContext(sandbox); // Vendored AES (must load before channel-decrypt.js — same as index.html). - const vendorAesPath = path.join(__dirname, 'public/vendor/aes-ecb.js'); + const vendorAesPath = path.join(REPO_ROOT, 'public/vendor/aes-ecb.js'); if (fs.existsSync(vendorAesPath)) { vm.runInContext(fs.readFileSync(vendorAesPath, 'utf8'), sandbox); } // Optional vendored SHA-256 / HMAC (the fix). Load if present so the test // works whether the fix vendors it as a separate file OR inlines it into // channel-decrypt.js. - const vendorShaPath = path.join(__dirname, 'public/vendor/sha256-hmac.js'); + const vendorShaPath = path.join(REPO_ROOT, 'public/vendor/sha256-hmac.js'); if (fs.existsSync(vendorShaPath)) { vm.runInContext(fs.readFileSync(vendorShaPath, 'utf8'), sandbox); } vm.runInContext( - fs.readFileSync(path.join(__dirname, 'public/channel-decrypt.js'), 'utf8'), + fs.readFileSync(path.join(REPO_ROOT, 'public/channel-decrypt.js'), 'utf8'), sandbox ); return sandbox.window.ChannelDecrypt; diff --git a/test-channel-decrypt-m345.js b/tests/unit/test-channel-decrypt-m345.js similarity index 97% rename from test-channel-decrypt-m345.js rename to tests/unit/test-channel-decrypt-m345.js index 0dccbbdf8..0c77eaa1f 100644 --- a/test-channel-decrypt-m345.js +++ b/tests/unit/test-channel-decrypt-m345.js @@ -3,6 +3,7 @@ * Runs in Node.js via vm.createContext to simulate browser environment. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); @@ -58,7 +59,7 @@ async function runTests() { console.log('\n=== M3: PSK hex key detection ==='); // Load channel-decrypt.js in sandbox - const cdSrc = fs.readFileSync(__dirname + '/public/channel-decrypt.js', 'utf8'); + const cdSrc = fs.readFileSync(REPO_ROOT + '/public/channel-decrypt.js', 'utf8'); const sandbox = createSandbox(); const context = vm.createContext(sandbox); vm.runInContext(cdSrc, context); diff --git a/test-channel-fluid-layout.js b/tests/unit/test-channel-fluid-layout.js similarity index 96% rename from test-channel-fluid-layout.js rename to tests/unit/test-channel-fluid-layout.js index 3d06a8998..07bdb1a12 100644 --- a/test-channel-fluid-layout.js +++ b/tests/unit/test-channel-fluid-layout.js @@ -9,12 +9,13 @@ * Companion E2E (real viewport assertions): test-channel-fluid-e2e.js. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); const assert = require('assert'); -const css = fs.readFileSync(path.join(__dirname, 'public/style.css'), 'utf8'); +const css = fs.readFileSync(path.join(REPO_ROOT, 'public/style.css'), 'utf8'); let passed = 0, failed = 0; function test(name, fn) { diff --git a/test-channel-issue-1087.js b/tests/unit/test-channel-issue-1087.js similarity index 95% rename from test-channel-issue-1087.js rename to tests/unit/test-channel-issue-1087.js index ca3ace349..9bd52abc0 100644 --- a/test-channel-issue-1087.js +++ b/tests/unit/test-channel-issue-1087.js @@ -20,6 +20,7 @@ * Companion E2E coverage: test-channel-issue-1087-e2e.js */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); @@ -31,10 +32,10 @@ function assert(cond, msg) { else { failed++; console.error(' ✗ ' + msg); } } -const chSrc = fs.readFileSync(path.join(__dirname, 'public/channels.js'), 'utf8'); -const qrSrc = fs.readFileSync(path.join(__dirname, 'public/channel-qr.js'), 'utf8'); -const decSrc = fs.readFileSync(path.join(__dirname, 'public/channel-decrypt.js'), 'utf8'); -const idxSrc = fs.readFileSync(path.join(__dirname, 'public/index.html'), 'utf8'); +const chSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channels.js'), 'utf8'); +const qrSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channel-qr.js'), 'utf8'); +const decSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channel-decrypt.js'), 'utf8'); +const idxSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/index.html'), 'utf8'); console.log('\n=== #1087 Bug 1: QR vendor library is wired correctly ==='); diff --git a/test-channel-issue-1101.js b/tests/unit/test-channel-issue-1101.js similarity index 95% rename from test-channel-issue-1101.js rename to tests/unit/test-channel-issue-1101.js index d6c178ea9..2a4877b43 100644 --- a/test-channel-issue-1101.js +++ b/tests/unit/test-channel-issue-1101.js @@ -11,6 +11,7 @@ * text, no Copy Key button overlapping it. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); @@ -21,8 +22,8 @@ function assert(cond, msg) { else { failed++; console.error(' ✗ ' + msg); } } -const channelsSrc = fs.readFileSync(path.join(__dirname, 'public', 'channels.js'), 'utf8'); -const qrSrc = fs.readFileSync(path.join(__dirname, 'public', 'channel-qr.js'), 'utf8'); +const channelsSrc = fs.readFileSync(path.join(REPO_ROOT, 'public', 'channels.js'), 'utf8'); +const qrSrc = fs.readFileSync(path.join(REPO_ROOT, 'public', 'channel-qr.js'), 'utf8'); console.log('\n=== #1101: Share modal markup ==='); diff --git a/test-channel-live-decrypt-userprefix.js b/tests/unit/test-channel-live-decrypt-userprefix.js similarity index 98% rename from test-channel-live-decrypt-userprefix.js rename to tests/unit/test-channel-live-decrypt-userprefix.js index 091bdcf27..eaffdc412 100644 --- a/test-channel-live-decrypt-userprefix.js +++ b/tests/unit/test-channel-live-decrypt-userprefix.js @@ -38,6 +38,7 @@ * 4. unread is NOT bumped on the channel currently being viewed */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); @@ -150,7 +151,7 @@ function makeBrowserLikeSandbox() { } function loadInCtx(ctx, file) { - const src = fs.readFileSync(path.join(__dirname, file), 'utf8'); + const src = fs.readFileSync(path.join(REPO_ROOT, file), 'utf8'); vm.runInContext(src, ctx, { filename: file }); for (const k of Object.keys(ctx.window)) ctx[k] = ctx.window[k]; } diff --git a/test-channel-live-decrypt.js b/tests/unit/test-channel-live-decrypt.js similarity index 96% rename from test-channel-live-decrypt.js rename to tests/unit/test-channel-live-decrypt.js index e7db96a7c..18bd8f5ff 100644 --- a/test-channel-live-decrypt.js +++ b/tests/unit/test-channel-live-decrypt.js @@ -18,6 +18,7 @@ * channels the user is not currently viewing. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); @@ -87,8 +88,8 @@ function buildEncryptedGrpTxt(channelName, sender, message) { async function run() { console.log('\n=== Live PSK decrypt: ChannelDecrypt helpers ==='); - const cdSrc = fs.readFileSync(path.join(__dirname, 'public/channel-decrypt.js'), 'utf8'); - const aesSrc = fs.readFileSync(path.join(__dirname, 'public/vendor/aes-ecb.js'), 'utf8'); + const cdSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channel-decrypt.js'), 'utf8'); + const aesSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/vendor/aes-ecb.js'), 'utf8'); const sandbox = createSandbox(); const ctx = vm.createContext(sandbox); vm.runInContext(aesSrc, ctx); @@ -143,7 +144,7 @@ async function run() { 'tryDecryptLive returns null when encryptedData/mac missing'); console.log('\n=== Live PSK decrypt: channels.js integration contract ==='); - const chSrc = fs.readFileSync(path.join(__dirname, 'public/channels.js'), 'utf8'); + const chSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channels.js'), 'utf8'); assert(/tryDecryptLive\s*\(/.test(chSrc), 'channels.js calls ChannelDecrypt.tryDecryptLive() in the WS path'); assert(/buildKeyMap\s*\(/.test(chSrc), diff --git a/test-channel-modal-ux.js b/tests/unit/test-channel-modal-ux.js similarity index 96% rename from test-channel-modal-ux.js rename to tests/unit/test-channel-modal-ux.js index 5c4e3b179..795e87cfd 100644 --- a/test-channel-modal-ux.js +++ b/tests/unit/test-channel-modal-ux.js @@ -16,6 +16,7 @@ * Runs in Node.js — no browser. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); @@ -27,8 +28,8 @@ function assert(cond, msg) { else { failed++; console.error(' ✗ ' + msg); } } -const chSrc = fs.readFileSync(path.join(__dirname, 'public/channels.js'), 'utf8'); -const cssSrc = fs.readFileSync(path.join(__dirname, 'public/style.css'), 'utf8'); +const chSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channels.js'), 'utf8'); +const cssSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/style.css'), 'utf8'); console.log('\n=== #1034 PR1: [+ Add Channel] sidebar button ==='); assert(/id="chAddChannelBtn"/.test(chSrc), diff --git a/test-channel-psk-ux.js b/tests/unit/test-channel-psk-ux.js similarity index 97% rename from test-channel-psk-ux.js rename to tests/unit/test-channel-psk-ux.js index 9123b7f43..a75ea5f7b 100644 --- a/test-channel-psk-ux.js +++ b/tests/unit/test-channel-psk-ux.js @@ -9,6 +9,7 @@ * Runs in Node.js via vm.createContext to simulate the browser. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); @@ -47,7 +48,7 @@ function createSandbox() { async function run() { console.log('\n=== #1020 PSK UX: ChannelDecrypt label storage ==='); - const cdSrc = fs.readFileSync(path.join(__dirname, 'public/channel-decrypt.js'), 'utf8'); + const cdSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channel-decrypt.js'), 'utf8'); const sandbox = createSandbox(); vm.runInContext(cdSrc, vm.createContext(sandbox)); const CD = sandbox.window.ChannelDecrypt; @@ -78,7 +79,7 @@ async function run() { assert(!CD.getLabel('#LongFast'), 'no label means getLabel returns falsy'); console.log('\n=== #1020 PSK UX: channels.js DOM/contract ==='); - const chSrc = fs.readFileSync(path.join(__dirname, 'public/channels.js'), 'utf8'); + const chSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channels.js'), 'utf8'); // E2E DOM: optional label input in add form (now in #1034 modal as #chPskName) assert(chSrc.includes('id="chPskName"') || chSrc.includes('id="chKeyLabelInput"'), diff --git a/test-channel-qr-wiring.js b/tests/unit/test-channel-qr-wiring.js similarity index 96% rename from test-channel-qr-wiring.js rename to tests/unit/test-channel-qr-wiring.js index 4d92a9663..5284cce5b 100644 --- a/test-channel-qr-wiring.js +++ b/tests/unit/test-channel-qr-wiring.js @@ -8,6 +8,7 @@ * E2E behavior is covered by test-channel-modal-e2e.js extensions. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); @@ -20,7 +21,7 @@ function assert(cond, msg) { } const src = fs.readFileSync( - path.join(__dirname, 'public/channels.js'), + path.join(REPO_ROOT, 'public/channels.js'), 'utf8' ); diff --git a/test-channel-qr.js b/tests/unit/test-channel-qr.js similarity index 96% rename from test-channel-qr.js rename to tests/unit/test-channel-qr.js index 310f6603f..3bbd6f45f 100644 --- a/test-channel-qr.js +++ b/tests/unit/test-channel-qr.js @@ -7,6 +7,7 @@ * elsewhere in the redesign series. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const vm = require('vm'); const fs = require('fs'); @@ -30,7 +31,7 @@ function loadChannelQR() { sandbox.self = sandbox; vm.createContext(sandbox); - const src = fs.readFileSync(path.join(__dirname, 'public/channel-qr.js'), 'utf8'); + const src = fs.readFileSync(path.join(REPO_ROOT, 'public/channel-qr.js'), 'utf8'); vm.runInContext(src, sandbox); return sandbox.window.ChannelQR; } diff --git a/test-channel-sidebar-layout.js b/tests/unit/test-channel-sidebar-layout.js similarity index 95% rename from test-channel-sidebar-layout.js rename to tests/unit/test-channel-sidebar-layout.js index 6d57f4624..ec8e93e34 100644 --- a/test-channel-sidebar-layout.js +++ b/tests/unit/test-channel-sidebar-layout.js @@ -19,6 +19,7 @@ * clearer copy and stays grouped with the channel controls. */ 'use strict'; +const REPO_ROOT = require('path').resolve(__dirname, '..', '..'); const fs = require('fs'); const path = require('path'); @@ -29,8 +30,8 @@ function assert(cond, msg) { else { failed++; console.error(' ✗ ' + msg); } } -const chSrc = fs.readFileSync(path.join(__dirname, 'public/channels.js'), 'utf8'); -const cssSrc = fs.readFileSync(path.join(__dirname, 'public/style.css'), 'utf8'); +const chSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/channels.js'), 'utf8'); +const cssSrc = fs.readFileSync(path.join(REPO_ROOT, 'public/style.css'), 'utf8'); console.log('\n=== Sidebar layout: no nested '; } function msgBox(id) { return ''; } - function say(text, ok, id) { - var el = document.getElementById(id || 'accountMsg'); - if (!el) return; - el.textContent = text; - el.classList.toggle('ok', !!ok); - el.classList.toggle('err', !ok); - } - function errText(r) { return (r.data && r.data.error) || ('Request failed (HTTP ' + r.status + ')'); } function onSubmit(formId, fn, msgId) { var f = document.getElementById(formId); if (!f) return; @@ -53,7 +45,7 @@ e.preventDefault(); var btn = f.querySelector('button[type="submit"]'); if (btn) btn.disabled = true; - Promise.resolve(fn(f)).catch(function () { say('Network error, try again.', false, msgId); }) + Promise.resolve(fn(f)).catch(function () { CSAuth.say(msgId || 'accountMsg', 'Network error, try again.', false); }) .then(function () { if (btn) btn.disabled = false; }); }); } @@ -118,6 +110,7 @@ field('emailPw', 'Current password', 'password', 'current-password') + submitBtn('Change address') + msgBox('emailMsg') + '' + '

Devices

' + msgBox('sessMsg') + + (window.CSSettingsSync ? '

Settings sync

' : '') + '

Delete account

' + ''); onSubmit('resetForm', function () { - if (val('resetPassword') !== val('resetPassword2')) { say('The passwords do not match.', false); return; } + if (val('resetPassword') !== val('resetPassword2')) { CSAuth.say('accountMsg', 'The passwords do not match.', false); return; } return CSAuth.request('POST', '/api/auth/reset', { token: token, password: val('resetPassword') }).then(function (r) { - say(r.ok ? r.data.message : errText(r), r.ok); + CSAuth.say('accountMsg', r.ok ? r.data.message : CSAuth.errText(r), r.ok); // The reset ended every session, this browser's included. if (r.ok) CSAuth.setUser(null); if (!r.ok) showGone(r, '#/account/forgot', 'Send a new link'); @@ -267,36 +260,37 @@ shown.userId = u.id; app.innerHTML = profileHtml(u); document.getElementById('profName').value = u.displayName; + if (window.CSSettingsSync) window.CSSettingsSync.mountSection(document.getElementById('syncSection')); document.getElementById('accountPageLogout').addEventListener('click', function () { return CSAuth.logout('#/account/login').then(function (r) { - if (!r.ok) say(errText(r), false, 'logoutMsg'); - }, function () { say('Network error, try again.', false, 'logoutMsg'); }); + if (!r.ok && !r.cancelled) CSAuth.say('logoutMsg', CSAuth.errText(r), false); + }, function () { CSAuth.say('logoutMsg', 'Network error, try again.', false); }); }); onSubmit('profileForm', function () { return CSAuth.request('PATCH', '/api/account', { displayName: val('profName') }).then(function (r) { if (r.ok) CSAuth.setUser(r.data); - say(r.ok ? 'Saved.' : errText(r), r.ok, 'profMsg'); + CSAuth.say('profMsg', r.ok ? 'Saved.' : CSAuth.errText(r), r.ok); }); }, 'profMsg'); onSubmit('pwForm', function (form) { return CSAuth.request('POST', '/api/account/password', { currentPassword: val('pwCurrent'), newPassword: val('pwNew') }).then(function (r) { if (r.ok) form.reset(); - say(r.ok ? r.data.message : errText(r), r.ok, 'pwMsg'); + CSAuth.say('pwMsg', r.ok ? r.data.message : CSAuth.errText(r), r.ok); if (r.ok) loadSessions(); }); }, 'pwMsg'); onSubmit('emailForm', function (form) { return CSAuth.request('POST', '/api/account/email', { newEmail: val('emailNew'), currentPassword: val('emailPw') }).then(function (r) { if (r.ok) form.reset(); - say(r.ok ? r.data.message : errText(r), r.ok, 'emailMsg'); + CSAuth.say('emailMsg', r.ok ? r.data.message : CSAuth.errText(r), r.ok); }); }, 'emailMsg'); onSubmit('delForm', function () { if (!confirm('Delete your account permanently?')) return; return CSAuth.request('DELETE', '/api/account', { currentPassword: val('delPw') }).then(function (r) { - if (!r.ok) { say(errText(r), false, 'delMsg'); return; } + if (!r.ok) { CSAuth.say('delMsg', CSAuth.errText(r), false); return; } CSAuth.setUser(null); CSAuth.notify('Your account was deleted.'); location.hash = '#/home'; @@ -307,17 +301,17 @@ return CSAuth.request('GET', '/api/account/sessions').then(function (r) { var list = document.getElementById('sessList'); if (!list) return; - if (!r.ok) { say(errText(r), false, 'sessMsg'); return; } + if (!r.ok) { CSAuth.say('sessMsg', CSAuth.errText(r), false); return; } list.innerHTML = sessionsHtml(r.data); - }).catch(function () { say('Network error, try again.', false, 'sessMsg'); }); + }).catch(function () { CSAuth.say('sessMsg', 'Network error, try again.', false); }); } document.getElementById('sessList').addEventListener('click', function (e) { var id = e.target && e.target.getAttribute && e.target.getAttribute('data-sess'); if (!id) return; CSAuth.request('DELETE', '/api/account/sessions/' + encodeURIComponent(id)).then(function (r) { - say(r.ok ? 'Device logged out.' : errText(r), r.ok, 'sessMsg'); + CSAuth.say('sessMsg', r.ok ? 'Device logged out.' : CSAuth.errText(r), r.ok); loadSessions(); - }).catch(function () { say('Network error, try again.', false, 'sessMsg'); }); + }).catch(function () { CSAuth.say('sessMsg', 'Network error, try again.', false); }); }); loadSessions(); } diff --git a/public/admin-users.js b/public/admin-users.js index eb4a69ef4..53e95414c 100644 --- a/public/admin-users.js +++ b/public/admin-users.js @@ -14,15 +14,8 @@ var STATUSES = ['pending', 'active', 'disabled']; var ROLES = ['user', 'admin']; - function errText(r) { return (r.data && r.data.error) || ('Request failed (HTTP ' + r.status + ')'); } function fmt(iso) { if (!iso) return '—'; try { return new Date(iso).toLocaleString(); } catch (_) { return iso; } } - function say(text, ok) { - var el = document.getElementById('umMsg'); - if (!el) return; - el.textContent = text; - el.classList.toggle('ok', !!ok); - el.classList.toggle('err', !ok); - } + function say(text, ok) { CSAuth.say('umMsg', text, ok); } function netErr() { say('Network error, try again.', false); } function chip(event, reason) { var e = String(event || 'sent'); @@ -116,7 +109,7 @@ return CSAuth.request('GET', '/api/admin/users?' + q.toString()).then(function (r) { var body = document.getElementById('umBody'); if (!body || seq !== loadSeq) return; - if (!r.ok) { say(errText(r), false); return; } + if (!r.ok) { say(CSAuth.errText(r), false); return; } var me = CSAuth.user(); var html = ''; // The server caps the list at 1000 rows, so a full tbody rebuild is bounded. @@ -141,7 +134,7 @@ return CSAuth.request('GET', '/api/admin/users/' + encodeURIComponent(id)).then(function (r) { var el = document.getElementById('umDetail'); if (!el || openId !== id) return; - if (!r.ok) { say(errText(r), false); closeDetail(); return; } + if (!r.ok) { say(CSAuth.errText(r), false); closeDetail(); return; } el.innerHTML = detailHtml(r.data); el.hidden = false; if (focus) { @@ -179,13 +172,13 @@ } if (act === 'refresh') { CSAuth.request('POST', '/api/admin/users/' + encodeURIComponent(id) + '/mail/' + encodeURIComponent(btn.getAttribute('data-mail')) + '/refresh') - .then(function (r) { say(r.ok ? 'Mail status refreshed.' : errText(r), r.ok); load(); }).catch(netErr); + .then(function (r) { say(r.ok ? 'Mail status refreshed.' : CSAuth.errText(r), r.ok); load(); }).catch(netErr); return; } if (confirmText[act] && !confirm(confirmText[act])) return; var ep = endpoints[act]; CSAuth.request(ep[0], '/api/admin/users/' + encodeURIComponent(id) + ep[1], ep[2]).then(function (r) { - say(r.ok ? 'Done.' : errText(r), r.ok); + say(r.ok ? 'Done.' : CSAuth.errText(r), r.ok); if (act === 'delete' && r.ok && openId === id) closeDetail(); var me = CSAuth.user(); if (r.ok && (act === 'promote' || act === 'demote') && me && String(me.id) === id) { diff --git a/public/auth.js b/public/auth.js index d38638b42..7409be030 100644 --- a/public/auth.js +++ b/public/auth.js @@ -49,17 +49,49 @@ window.dispatchEvent(new CustomEvent('cs-auth-changed', { detail: state.user })); } + // say shows text in the message box id (account pages, settings sync). + function say(id, text, ok) { + var el = document.getElementById(id); + if (!el) return; + el.textContent = text; + el.classList.toggle('ok', !!ok); + el.classList.toggle('err', !ok); + } + function errText(r) { return (r.data && r.data.error) || ('Request failed (HTTP ' + r.status + ')'); } + + // An optional handler (settings-sync.js) runs first: it may cancel the + // logout or hand back afterLogout, which runs once the server ended the + // session and before the user is cleared. + var logoutHandler = null; + function setLogoutHandler(fn) { logoutHandler = fn; } + // logout ends the session on the server. Only when that succeeded does it // move to next and then clear the user, so pages listening for // 'cs-auth-changed' already see the new view. A refusal keeps the user - // and is returned for the caller to show. + // and is returned for the caller to show; a cancel returns cancelled. One + // logout runs at a time: another call while the dialog is open or the + // POST is out returns cancelled. + var loggingOut = null; function logout(next) { - return request('POST', '/api/auth/logout').then(function (r) { - if (r.ok) { - location.hash = next; - setUser(null); - } - return r; + if (loggingOut) return Promise.resolve({ ok: false, cancelled: true, status: 0, data: {} }); + var done = function () { loggingOut = null; }; + loggingOut = runLogout(next); + loggingOut.then(done, done); + return loggingOut; + } + + function runLogout(next) { + return Promise.resolve(logoutHandler ? logoutHandler() : null).then(function (h) { + h = h || {}; + if (h.cancel) return { ok: false, cancelled: true, status: 0, data: {} }; + return request('POST', '/api/auth/logout').then(function (r) { + if (r.ok) { + if (h.afterLogout) h.afterLogout(); + location.hash = next; + setUser(null); + } + return r; + }); }); } @@ -127,7 +159,7 @@ menu.addEventListener('click', closeMenu); document.getElementById('accountLogout').addEventListener('click', function () { logout('#/home').then(function (r) { - if (!r.ok) notify((r.data && r.data.error) || ('Logout failed (HTTP ' + r.status + ')')); + if (!r.ok && !r.cancelled) notify((r.data && r.data.error) || ('Logout failed (HTTP ' + r.status + ')')); }, function () { notify('Network error, try again.'); }); }); } @@ -150,6 +182,9 @@ request: request, refreshMe: refreshMe, logout: logout, + setLogoutHandler: setLogoutHandler, + say: say, + errText: errText, setUser: setUser, notify: notify, ready: function () { return state.ready; }, diff --git a/public/customize-v2.js b/public/customize-v2.js index e615ef75b..0657c53e4 100644 --- a/public/customize-v2.js +++ b/public/customize-v2.js @@ -2971,6 +2971,8 @@ readOverrides: readOverrides, writeOverrides: writeOverrides, computeEffective: computeEffective, + // Settings sync (settings-sync.js) re-applies overrides that arrived from the account. + runPipeline: _runPipeline, setOverride: setOverride, clearOverride: clearOverride, migrateOldKeys: migrateOldKeys, diff --git a/public/index.html b/public/index.html index b70cdae7c..00a18b6ec 100644 --- a/public/index.html +++ b/public/index.html @@ -188,6 +188,7 @@ + diff --git a/public/settings-sync.js b/public/settings-sync.js new file mode 100644 index 000000000..72583dc42 --- /dev/null +++ b/public/settings-sync.js @@ -0,0 +1,756 @@ +/* Settings sync for optional user management + * (docs/specs/2026-10-06-user-settings-sync-design.md). Inert unless + * window.MC_USER_MGMT is on and CSAuth reports a logged-in user. While + * active it wraps Storage.prototype.setItem/removeItem for the keys the + * server allowlists, merges this device with the account's copy and + * pushes changes. Exposes window.CSSettingsSync. */ +(function () { + 'use strict'; + + // ── Three-way merge (pure) ── + // Values are raw localStorage strings; undefined means "not set". + + function own(o, k) { return o && Object.prototype.hasOwnProperty.call(o, k) ? o[k] : undefined; } + + // parseList returns the array behind raw ([] when unset), or null when + // raw is not a JSON array. + function parseList(raw) { + if (raw === undefined) return []; + try { + var v = JSON.parse(raw); + return Array.isArray(v) ? v : null; + } catch (e) { return null; } + } + + // identity of a set item: its idField when it is an object that has one, + // else its JSON (a string item is compared as itself). + function identity(item, idField) { + if (idField && item && typeof item === 'object' && item[idField] != null) return 'f:' + String(item[idField]); + return 'j:' + JSON.stringify(item); + } + + function indexList(list, idField) { + var map = Object.create(null), order = []; + list.forEach(function (item) { + var id = identity(item, idField); + if (id in map) return; + map[id] = item; + order.push(id); + }); + return { map: map, order: order }; + } + + function sameItem(a, b) { return JSON.stringify(a) === JSON.stringify(b); } + + // mergeSet computes P + (L - B) - (B - L) by identity. Returns the merged + // raw string, undefined when neither side has the key, or null when this + // device or the account holds something that is not a JSON list (the + // caller then merges it as a scalar). An unparseable baseline counts as + // none, so nothing added on either side is dropped. + function mergeSet(l, p, b, idField) { + var al = parseList(l), ap = parseList(p), ab = parseList(b) || []; + if (!al || !ap) return null; + var L = indexList(al, idField), P = indexList(ap, idField), B = indexList(ab, idField); + var out = []; + P.order.forEach(function (id) { + var inL = id in L.map, inB = id in B.map; + if (inB && !inL) return; // removed on this device since the last sync + var onlyLocalChanged = inL && inB && !sameItem(L.map[id], B.map[id]) && sameItem(P.map[id], B.map[id]); + out.push(onlyLocalChanged ? L.map[id] : P.map[id]); + }); + L.order.forEach(function (id) { + if (id in P.map || id in B.map) return; // in the profile, or removed on another device + out.push(L.map[id]); // added on this device since the last sync + }); + if (!out.length && l === undefined && p === undefined) return undefined; + var s = JSON.stringify(out); + // Keep an existing spelling of the same list: formatting alone is no + // change. The account's spelling goes first, so a list equal to both + // sides does not count as differing from the account (two devices + // would otherwise push their spelling back and forth). + if (p !== undefined && JSON.stringify(ap) === s) return p; + if (l !== undefined && JSON.stringify(al) === s) return l; + return s; + } + + function mergeScalar(l, p, b) { return (l !== b && p === b) ? l : p; } + + // mergeDocs merges local, profile and baseline ({key: raw}) for every + // allowlisted key. localChanges lists the keys this device must write; + // differsFromProfile says the result must be pushed. + function mergeDocs(local, profile, base, allowlist) { + var keys = {}, localChanges = [], differs = false; + allowlist.forEach(function (entry) { + var k = entry.key, l = own(local, k), p = own(profile, k), b = own(base, k), v; + if (entry.kind === 'set') { + v = mergeSet(l, p, b, entry.id || ''); + if (v === null) { + console.warn('[settings-sync] ' + k + ' is not a JSON list on this device or in the account; merged as one value'); + v = mergeScalar(l, p, b); + } + } else { + v = mergeScalar(l, p, b); + } + if (v !== undefined) keys[k] = v; + if (v !== l) localChanges.push(k); + if (v !== p) differs = true; + }); + return { keys: keys, localChanges: localChanges, differsFromProfile: differs }; + } + + // ── Sync engine ── + + var BASE_KEY = 'cs-settings-sync-base'; + var REV_KEY = 'cs-settings-sync-rev'; + var PUSH_DELAY_MS = 2000; + var PULL_EVERY_MS = 60000; + var BACKOFF_MIN_MS = 2000; + var BACKOFF_MAX_MS = 300000; + var MAX_CONFLICT_RETRIES = 3; + var FLUSH_TIMEOUT_MS = 5000; + var FIRST_LOGIN_TEXT = 'Your settings are now saved to your account.'; + // Keys whose change an existing storage listener applies (app.js, + // cb-presets.js, map-tile-providers.js). + var LISTENER_KEYS = { + 'meshcore-theme': true, 'meshcore-cb-preset': true, + 'mc-dark-tile-provider': true, 'mc-light-tile-provider': true + }; + // The map-tile-providers.js setter and getter of each tile key. + var TILE_API = { + 'mc-dark-tile-provider': { set: 'MC_setDarkTileProvider', get: 'MC_getDarkTileProvider' }, + 'mc-light-tile-provider': { set: 'MC_setLightTileProvider', get: 'MC_getLightTileProvider' } + }; + + // Assigning localStorage.setItem would store a key named "setItem" + // (Storage has a named-property setter), so the wrap goes on the prototype. + var proto = window.Storage.prototype; + var origGet = proto.getItem, origSet = proto.setItem, origRemove = proto.removeItem; + + var state = { + active: false, userId: null, policy: null, + base: {}, rev: 0, gen: '', hold: false, firstUpload: false, + dirty: false, seq: 0, pushing: null, pushTimer: null, retryTimer: null, pullTimer: null, + backoff: BACKOFF_MIN_MS, blocked: null, status: 'idle', lastSyncedAt: null, tooLarge: [], + // epoch changes on every activate/deactivate: an answer to a request + // from an earlier session is dropped. putsDone counts PUT answers, so a + // pull can tell a push finished while its GET was out. + epoch: 0, putsDone: 0 + }; + + function rawGet(k) { var v = origGet.call(window.localStorage, k); return v === null ? undefined : v; } + function rawSet(k, v) { origSet.call(window.localStorage, k, v); } + function rawRemove(k) { origRemove.call(window.localStorage, k); } + + function setPolicy(list) { + var byKey = Object.create(null); + list.forEach(function (e) { byKey[e.key] = e; }); + state.policy = { list: list, byKey: byKey }; + } + + // pick keeps the allowlisted keys of o. + function pick(o) { + var out = {}; + state.policy.list.forEach(function (e) { var v = own(o, e.key); if (v !== undefined) out[e.key] = v; }); + return out; + } + + function snapshot() { + var out = {}; + state.policy.list.forEach(function (e) { var v = rawGet(e.key); if (v !== undefined) out[e.key] = v; }); + return out; + } + + function sameKeys(a, b) { + var ka = Object.keys(a); + return ka.length === Object.keys(b).length && ka.every(function (k) { return own(b, k) === a[k]; }); + } + + // The baseline is the last document this device and the account agreed + // on. It belongs to one user: another user's baseline counts as none. gen + // is the account document's generation: revisions restart at 1 after a + // delete, so a revision alone does not say which document it belongs to. + function loadBase(userId) { + try { + var b = JSON.parse(rawGet(BASE_KEY) || 'null'); + if (b && b.user === userId && b.keys && typeof b.keys === 'object') { + return { keys: b.keys, rev: Number(rawGet(REV_KEY)) || 0, gen: typeof b.gen === 'string' ? b.gen : '', hold: !!b.hold }; + } + } catch (e) { /* a damaged baseline counts as none */ } + return { keys: {}, rev: 0, gen: '', hold: false }; + } + + // refreshBase re-reads the stored baseline. localStorage is shared by + // every tab of this browser and another tab may have synced since this + // one last looked, so the stored copy is the truth, never the one in + // memory: merging against an older baseline brings back removals. + function refreshBase() { + var b = loadBase(state.userId); + state.base = b.keys; + state.rev = b.rev; + state.gen = b.gen; + state.hold = b.hold; + } + + function saveBase(keys, rev, gen, hold) { + state.base = keys; + state.rev = rev; + state.gen = gen; + state.hold = hold; + rawSet(BASE_KEY, JSON.stringify({ user: state.userId, keys: keys, gen: gen, hold: hold })); + rawSet(REV_KEY, String(rev)); + } + + function setStatus(s) { state.status = s; renderStatus(); } + + // quiet: writes made while applying a default are not the user's changes. + var quiet = false; + + function watched(storage, k) { + return !quiet && state.active && !!state.policy && storage === window.localStorage && !!state.policy.byKey[k]; + } + + function install() { + proto.setItem = function (k, v) { + var watch = watched(this, k), before = watch ? origGet.call(this, k) : null; + origSet.call(this, k, v); + if (watch && before !== String(v)) markDirty(); + }; + proto.removeItem = function (k) { + var watch = watched(this, k), before = watch ? origGet.call(this, k) : null; + origRemove.call(this, k); + if (watch && before !== null) markDirty(); + }; + } + + function uninstall() { + proto.setItem = origSet; + proto.removeItem = origRemove; + } + + // markDirty runs on every allowlisted write (a range input writes many + // times per second). It needs only the hold flag, and a held baseline + // always has revision 0 and no keys: a stored revision above 0 rules the + // hold out without parsing the baseline, which can be 256 KiB. Another + // tab may have entered the hold, so the stored revision is read each time. + function markDirty() { + state.dirty = true; + state.seq++; + if (!(Number(rawGet(REV_KEY)) > 0)) { + refreshBase(); + if (state.hold) saveBase(state.base, state.rev, state.gen, false); // the next change starts a new document + } + schedulePush(PUSH_DELAY_MS); + } + + function schedulePush(ms) { + clearTimeout(state.pushTimer); + state.pushTimer = setTimeout(function () { state.pushTimer = null; push(); }, ms); + } + + function retryLater() { + setStatus('retrying'); + clearTimeout(state.retryTimer); + var wait = state.backoff; + state.backoff = Math.min(state.backoff * 2, BACKOFF_MAX_MS); + state.retryTimer = setTimeout(function () { state.retryTimer = null; push(); }, wait); + } + + function synced(seq) { + state.backoff = BACKOFF_MIN_MS; + state.lastSyncedAt = new Date(); + clearTimeout(state.retryTimer); + state.retryTimer = null; + if (state.seq === seq) state.dirty = false; + else schedulePush(PUSH_DELAY_MS); // written again while the request was out + setStatus('ok'); + } + + function largestKeys(keys) { + return Object.keys(keys).sort(function (a, b) { return keys[b].length - keys[a].length; }).slice(0, 3); + } + + // midEdit: re-rendering now would throw away unsaved input. Account + // pages hold forms; the geofilter editor (customize-v2.js) is a modal + // over the page. The page the user opens next reads the new values. + function midEdit() { + return /^#\/account(\/|\?|$)/.test(location.hash) || !!document.getElementById('cv2-gf-modal-overlay'); + } + + // applyDefaultTile re-applies the effective provider of a removed tile + // key, as customize-v2.js resetAll does: the setter fires the change + // event the maps listen to, then the key it persisted is removed again. + function applyDefaultTile(k) { + var api = TILE_API[k]; + quiet = true; + try { window[api.set](window[api.get]()); } finally { quiet = false; } + rawRemove(k); + } + + // applyListenerKeys hands changed theme, colour-blind preset and map tile + // keys to their storage listeners. Those listeners ignore a removal, so a + // removed key gets the default the app starts with: the OS colour scheme + // (app.js), no preset (cb-presets.js clearPreset) and the effective tile + // provider (applyDefaultTile). + function applyListenerKeys(changed) { + changed.forEach(function (k) { + if (!LISTENER_KEYS[k]) return; + var v = rawGet(k); + if (v === undefined && k === 'meshcore-cb-preset') { window.MeshCorePresets.clearPreset(); return; } + if (v === undefined && TILE_API[k]) { applyDefaultTile(k); return; } + if (v === undefined && k === 'meshcore-theme') v = window.matchMedia('(prefers-color-scheme: dark)').matches ? 'dark' : 'light'; + window.dispatchEvent(new StorageEvent('storage', { key: k, newValue: v === undefined ? null : v })); + }); + } + + // afterRemoteChange makes the running page show values that arrived from + // the account: the storage listeners, the customizer pipeline for its + // overrides, and a router re-render for everything a page reads at init. + // Before the customizer finished its init the pipeline is skipped: it + // would render without the server defaults, and the init reads the new + // overrides itself. + function afterRemoteChange(changed, firstLogin) { + applyListenerKeys(changed); + var cz = window._customizerV2; + if (changed.indexOf('cs-theme-overrides') !== -1 && cz && cz.initDone) cz.runPipeline(); + if (!firstLogin) window.CSAuth.notify('Settings updated from another device'); + if (!midEdit()) window.navigate(); + } + + function writeLocal(keys, changed) { + changed.forEach(function (k) { + var v = own(keys, k); + try { + if (v === undefined) rawRemove(k); else rawSet(k, v); + } catch (e) { console.warn('[settings-sync] could not store ' + k + ': ' + e.message); } + }); + } + + // applyProfile brings the account's document into this device: merge, + // write what changed, make it the new baseline. Returns true when this + // device holds values the account lacks. A baseline of another generation + // (the copy was deleted and started again) counts as none, so nothing on + // this device is taken for a removal made elsewhere. On the first login + // (no baseline at all) the first-login toast replaces "updated from + // another device": after the upload of this device's values, or at once + // when there is nothing to upload. + function applyProfile(rev, gen, doc) { + refreshBase(); + var firstLogin = !state.rev && !state.gen && !state.hold; + var local = snapshot(); + if (!rev) { + if (state.rev > 0 || state.hold) { + // The account's copy was deleted, here or on another device. Keep + // this device as it is; its next change starts a new document. + saveBase({}, 0, '', true); + return false; + } + state.firstUpload = true; // first login: this device's values form the first document + return Object.keys(local).length > 0; + } + var profile = pick((doc && doc.keys) || {}); + var m = mergeDocs(local, profile, state.gen === gen ? state.base : {}, state.policy.list); + writeLocal(m.keys, m.localChanges); + // The baseline can be 256 KiB: an unchanged pull does not rewrite it. + if (state.rev !== rev || state.gen !== gen || state.hold || !sameKeys(profile, state.base)) saveBase(profile, rev, gen, false); + if (m.localChanges.length) afterRemoteChange(m.localChanges, firstLogin); + if (firstLogin && m.differsFromProfile) state.firstUpload = true; + else if (firstLogin) window.CSAuth.notify(FIRST_LOGIN_TEXT); + return m.differsFromProfile; + } + + // push sends this device's allowlisted values. A 409 merges the returned + // document and retries (at most MAX_CONFLICT_RETRIES); a 403 is retried + // once after onForbidden (retried403 marks that retry); network errors, + // 5xx and 429 retry with backoff. Resolves true when the account holds + // this device's values. + function push(attempt, retried403) { + if (!state.active || state.blocked || !state.policy) return Promise.resolve(false); + if (state.pushing) return state.pushing.then(function () { return state.dirty ? push() : true; }); + attempt = attempt || 0; + refreshBase(); + var keys = snapshot(), seq = state.seq, epoch = state.epoch; + if (state.rev > 0 && sameKeys(keys, state.base)) { synced(seq); return Promise.resolve(true); } + setStatus('syncing'); + var p = window.CSAuth.request('PUT', '/api/account/settings', { baseRevision: state.rev, baseGeneration: state.gen, doc: { v: 1, keys: keys } }) + .then(function (r) { + if (state.epoch !== epoch) return false; // logged out (and maybe in again) meanwhile + state.pushing = null; + state.putsDone++; + if (r.ok) { + saveBase(keys, r.data.revision, r.data.generation, false); + synced(seq); + if (state.firstUpload) { + state.firstUpload = false; + window.CSAuth.notify(FIRST_LOGIN_TEXT); + } + return true; + } + if (r.status === 409 && attempt < MAX_CONFLICT_RETRIES) { + applyProfile(r.data.revision, r.data.generation, r.data.doc); + return push(attempt + 1, retried403); + } + if (r.status === 413) { + state.blocked = 'too-large'; + state.tooLarge = largestKeys(keys); + setStatus('too-large'); + return false; + } + if (r.status === 400) { + console.error('[settings-sync] the server refused the settings: ' + (r.data && r.data.error)); + state.blocked = 'rejected'; + setStatus('rejected'); + return false; + } + if (r.status === 403) { + // Held as the push in flight until the retry starts, so no other + // push runs while the session is checked. + var next = onForbidden(epoch, !retried403).then(function (retry) { + if (state.pushing === next) state.pushing = null; + return retry ? push(attempt, true) : false; + }); + state.pushing = next; + return next; + } + if (r.status !== 401) retryLater(); // 401: auth.js logged out, which deactivates this module + return false; + }, function () { + if (state.epoch !== epoch) return false; + state.pushing = null; + retryLater(); + return false; + }); + state.pushing = p; + return p; + } + + // onForbidden handles a 403: this tab's CSRF token belongs to an older + // session (a logout and login in another tab). It asks who is logged in + // now, which also stores the session's current token. Another user, or + // nobody, arrives as cs-auth-changed and re-activates or deactivates this + // module (the epoch moves). For the same user it resolves true when + // mayRetry (the request goes again once with the new token); otherwise + // this tab stops syncing until a reload, because the next attempt would + // get the same answer (an origin misconfiguration, for one). + function onForbidden(epoch, mayRetry) { + clearTimeout(state.retryTimer); + state.retryTimer = null; + var stop = function () { + if (state.epoch !== epoch) return false; + state.blocked = 'forbidden'; + setStatus('forbidden'); + return false; + }; + return window.CSAuth.refreshMe().then(function () { + if (state.epoch !== epoch) return false; + return mayRetry ? true : stop(); + }, function (e) { + console.error('[settings-sync] could not check the session after a 403: ' + (e && e.message)); + return stop(); + }); + } + + // pull fetches the account's document. An answer is dropped when the + // session changed or a push finished (or the stored baseline moved, also + // by another tab) while the GET was out: it describes the account before + // that push, and applying it would revert that change or mistake the + // first upload's revision-0 answer for a deleted copy. The next pull + // catches up. + function pull() { + if (!state.active) return Promise.resolve(); + if (state.pushing) return state.pushing.then(pull); + refreshBase(); + var epoch = state.epoch, puts = state.putsDone, rev = state.rev, gen = state.gen; + var stale = function () { + if (state.epoch !== epoch || state.putsDone !== puts) return true; + refreshBase(); + return state.rev !== rev || state.gen !== gen; + }; + return window.CSAuth.request('GET', '/api/account/settings').then(function (r) { + if (stale()) return; + if (!r.ok) { + if (r.status === 403) onForbidden(epoch, false); + else if (r.status !== 401) setStatus('retrying'); + return; + } + setPolicy(r.data.allowlist || []); + if (applyProfile(r.data.revision, r.data.generation, r.data.doc)) { + state.dirty = true; + state.seq++; + return push(); + } + if (state.dirty) return push(); + if (r.data.revision) { state.lastSyncedAt = new Date(); setStatus('ok'); } + else setStatus(state.hold ? 'held' : 'idle'); + }, function () { if (!stale()) setStatus('retrying'); }); + } + + // syncNow: the account page button. Retries after a 413 (the user may + // have trimmed), never after a 400 (that needs a reload). + function syncNow() { + if (!state.active) return Promise.resolve(); + if (state.blocked === 'too-large') state.blocked = null; + state.backoff = BACKOFF_MIN_MS; + clearTimeout(state.retryTimer); + state.retryTimer = null; + return pull(); + } + + function onVisible() { + if (document.visibilityState !== 'visible') return; + clearTimeout(state.retryTimer); + state.retryTimer = null; + pull(); + } + + function activate(user) { + if (state.active && state.userId === user.id) return Promise.resolve(); + if (state.active) deactivate(); + state.epoch++; + state.active = true; + state.userId = user.id; + state.policy = null; + refreshBase(); + state.firstUpload = false; + state.dirty = false; + state.blocked = null; + state.backoff = BACKOFF_MIN_MS; + install(); + window.CSAuth.setLogoutHandler(onLogout); + document.addEventListener('visibilitychange', onVisible); + state.pullTimer = setInterval(function () { if (document.visibilityState === 'visible') pull(); }, PULL_EVERY_MS); + return pull(); + } + + function deactivate() { + state.active = false; + state.epoch++; + clearTimeout(state.pushTimer); + clearTimeout(state.retryTimer); + clearInterval(state.pullTimer); + state.pushTimer = state.retryTimer = state.pullTimer = null; + document.removeEventListener('visibilitychange', onVisible); + uninstall(); + window.CSAuth.setLogoutHandler(null); + state.policy = null; + state.userId = null; + state.pushing = null; + setStatus('idle'); + } + + // ── Dialog, logout and the account-page section ── + + // showDialog uses the app's modal pattern (.modal-overlay + .modal, as + // the BYOP dialog in packets.js): role=dialog, focus on the choice + // opts.focus names (default the first), Tab trapped, Escape or a + // backdrop click dismiss. Resolves with the chosen id, or null when + // dismissed. + function showDialog(opts) { + return new Promise(function (resolve) { + var prev = document.activeElement; + var overlay = document.createElement('div'); + overlay.className = 'modal-overlay cs-dialog-overlay'; + overlay.innerHTML = ''; + document.body.appendChild(overlay); + var buttons = overlay.querySelectorAll('button'); + function close(choice) { + overlay.remove(); + if (prev && prev.focus) prev.focus(); + resolve(choice); + } + overlay.addEventListener('keydown', function (e) { + if (e.key === 'Escape') { e.preventDefault(); e.stopPropagation(); close(null); return; } + if (e.key !== 'Tab') return; + var first = buttons[0], last = buttons[buttons.length - 1]; + if (e.shiftKey && document.activeElement === first) { e.preventDefault(); last.focus(); } + else if (!e.shiftKey && document.activeElement === last) { e.preventDefault(); first.focus(); } + }); + overlay.addEventListener('click', function (e) { + var btn = e.target.closest && e.target.closest('[data-choice]'); + if (btn) close(btn.getAttribute('data-choice')); + else if (e.target === overlay) close(null); + }); + var start = buttons[0]; + for (var i = 0; i < opts.choices.length; i++) if (opts.choices[i].id === opts.focus) start = buttons[i]; + start.focus(); + }); + } + var dialog = showDialog; + + // unpushed says whether storage holds allowlisted values the stored + // baseline lacks. Storage is shared by every tab, so this sees a change + // another tab made and has not pushed yet. A hold means the account's + // copy was deleted and nothing changed since (a change ends the hold). + // Without a policy no write was watched yet, so nothing is pending. + function unpushed() { + if (!state.policy) return false; + refreshBase(); + return !state.hold && !sameKeys(snapshot(), state.base); + } + + // flush pushes pending changes now. Resolves true when the account holds + // everything this device has, false when the push failed or took longer + // than FLUSH_TIMEOUT_MS (a hanging connection must not hold the logout). + function flush() { + clearTimeout(state.pushTimer); + state.pushTimer = null; + if (!unpushed()) return Promise.resolve(true); + return new Promise(function (resolve) { + var timer = setTimeout(function () { + console.warn('[settings-sync] pending changes were not pushed within ' + FLUSH_TIMEOUT_MS / 1000 + ' s'); + resolve(false); + }, FLUSH_TIMEOUT_MS); + push().then(function (ok) { + clearTimeout(timer); + resolve(ok && !state.dirty); + }, function (e) { + clearTimeout(timer); + console.error('[settings-sync] push failed: ' + (e && e.message)); + resolve(false); + }); + }); + } + + // removeLocal deletes the synced keys in list and the baseline, then + // shows the defaults at once: the customizer's own Reset All teardown + // (it touches only keys removed here) and the theme and preset defaults. + // It runs right before the logout clears the user, so the wrap comes off + // first: the teardown's writes are not changes to push. Channel keys and + // the API key are never on the allowlist, so they stay. + function removeLocal(list) { + uninstall(); + var removed = list.map(function (e) { return e.key; }).filter(function (k) { return rawGet(k) !== undefined; }); + removed.forEach(rawRemove); + rawRemove(BASE_KEY); + rawRemove(REV_KEY); + var cz = window._customizerV2; + if (cz && cz.initDone) { cz.resetAll(); cz.runPipeline(); } + applyListenerKeys(removed); + } + + var LOGOUT_TEXT = [ + 'Your settings stay saved in your account.', + 'Channel keys are never synced and stay on this device. They are not removed, because no copy exists anywhere else.' + ]; + + // onLogout is CSAuth's logout handler while active (see auth.js logout). + function onLogout() { + return dialog({ + title: 'Log out', + text: LOGOUT_TEXT, + choices: [ + { id: 'keep', label: 'Keep my settings on this device', primary: true }, + { id: 'remove', label: 'Remove my settings from this device' }, + { id: 'cancel', label: 'Cancel' } + ] + }).then(function (choice) { + if (choice !== 'keep' && choice !== 'remove') return { cancel: true }; + return flush().then(function (saved) { + if (choice === 'keep') return {}; + if (!saved || !state.policy) { + // Removing now would lose changes that exist nowhere else. + return { afterLogout: function () { window.CSAuth.notify('Your latest settings could not be saved to your account, so they stay on this device.'); } }; + } + var list = state.policy.list; + return { afterLogout: function () { removeLocal(list); } }; + }); + }); + } + + // deleteRemote removes the account's copy; this device keeps its values + // and its next change starts a new document. A push in flight is waited + // for and pending pushes are cancelled first, so no PUT (or its late + // answer) recreates the copy or clears the hold. When the DELETE fails, + // unsynced changes go back to the retry backoff. + function stopPushing() { + clearTimeout(state.pushTimer); + clearTimeout(state.retryTimer); + state.pushTimer = state.retryTimer = null; + } + + function deleteRemote() { + var retryIfDirty = function () { if (state.active && state.dirty) retryLater(); }; + return Promise.resolve(state.pushing).then(function () { + stopPushing(); + return window.CSAuth.request('DELETE', '/api/account/settings'); + }).then(function (r) { + if (r.ok) { + stopPushing(); + state.dirty = false; + saveBase({}, 0, '', true); + setStatus('held'); + } else retryIfDirty(); + return r; + }, function (e) { retryIfDirty(); throw e; }); + } + + function statusText() { + switch (state.status) { + case 'ok': return 'Last synced ' + state.lastSyncedAt.toLocaleString(); + case 'syncing': return 'Syncing…'; + case 'retrying': return 'Not synced: retrying'; + case 'too-large': return 'Not synced: your settings are larger than your account can hold. Largest: ' + state.tooLarge.join(', '); + case 'rejected': return 'Not synced: the server refused your settings. Reload the page to try again.'; + case 'forbidden': return 'Not synced: this tab is out of date. Reload the page to sync again.'; + case 'held': return 'No settings saved in your account. Your next change starts a new copy.'; + default: return 'Not synced yet'; + } + } + + function renderStatus() { + var el = document.getElementById('syncStatus'); + if (!el) return; + el.textContent = statusText(); + el.classList.toggle('ok', state.status === 'ok'); + el.classList.toggle('err', state.status === 'retrying' || state.status === 'too-large' || state.status === 'rejected' || state.status === 'forbidden'); + } + + function mountSection(el) { + el.innerHTML = + '' + + '' + + '' + + '' + + ''; + renderStatus(); + document.getElementById('syncNow').addEventListener('click', function () { return syncNow(); }); + document.getElementById('syncDelete').addEventListener('click', function () { + return dialog({ + title: 'Delete synced settings', + text: ['This deletes the copy of your settings stored in your account. The settings on this device stay. Your next change starts a new copy.'], + choices: [{ id: 'delete', label: 'Delete synced settings', primary: true }, { id: 'cancel', label: 'Cancel' }], + focus: 'cancel' // the destructive choice is never the default + }).then(function (choice) { + if (choice !== 'delete') return; + return deleteRemote().then(function (r) { + window.CSAuth.say('syncMsg', r.ok ? 'Synced settings deleted from your account.' : window.CSAuth.errText(r), r.ok); + }, function () { window.CSAuth.say('syncMsg', 'Network error, try again.', false); }); + }); + }); + } + + window.addEventListener('cs-auth-changed', function (e) { + if (!window.CSAuth.isEnabled()) return; + if (e.detail) activate(e.detail); + else if (state.active) deactivate(); + }); + window.CSAuth.ready().then(function (u) { if (u && window.CSAuth.isEnabled()) activate(u); }); + + window.CSSettingsSync = { + mountSection: mountSection, + syncNow: syncNow, + _test: { + mergeDocs: mergeDocs, state: state, activate: activate, deactivate: deactivate, pull: pull, push: push, + midEdit: midEdit, flush: flush, onLogout: onLogout, deleteRemote: deleteRemote, statusText: statusText, + showDialog: showDialog, useDialog: function (fn) { dialog = fn; } + } + }; +})(); diff --git a/test-all.sh b/test-all.sh index 79a946a8c..198787942 100755 --- a/test-all.sh +++ b/test-all.sh @@ -185,6 +185,7 @@ node tests/unit/test-rx-coverage-escape.js node tests/unit/test-rx-coverage-noise.js node tests/unit/test-rx-coverage-viewport.js node tests/unit/test-scope-audit-styles-linked.js +node tests/unit/test-settings-sync.js node tests/unit/test-slideover-1056-rowsel-strict.js node tests/unit/test-top-routes-overlay.js node tests/unit/test-traces.js diff --git a/tests/e2e/test-user-management-e2e.js b/tests/e2e/test-user-management-e2e.js index 2b9249b10..ff0becf1f 100644 --- a/tests/e2e/test-user-management-e2e.js +++ b/tests/e2e/test-user-management-e2e.js @@ -61,6 +61,27 @@ async function registerAndActivate(page, email, name) { await page.waitForSelector('#accountToggle .nav-account-label:has-text("' + name + '")'); } +// axeClean fails on serious or critical WCAG 2 A/AA violations inside sel. +async function axeClean(pg, sel) { + const res = await new AxeBuilder({ page: pg }).include(sel).withTags(['wcag2a', 'wcag2aa']).analyze(); + const bad = res.violations.filter((v) => v.impact === 'serious' || v.impact === 'critical'); + assert(bad.length === 0, sel + ': ' + bad.map((v) => v.id + ' ' + v.nodes.map((n) => n.target.join(' ') + ' ' + ((n.any[0] || {}).message || '')).join(' | ')).join(', ')); +} + +// The account's synced keys, read through the page's own session. +async function accountKeys(pg) { + return pg.evaluate(() => window.CSAuth.request('GET', '/api/account/settings').then((r) => (r.data.doc && r.data.doc.keys) || {})); +} + +async function until(fn, label) { + const end = Date.now() + 8000; + for (;;) { + if (await fn()) return; + if (Date.now() > end) throw new Error('timed out: ' + label); + await new Promise((r) => setTimeout(r, 200)); + } +} + (async () => { const browser = await chromium.launch({ headless: true, @@ -109,6 +130,8 @@ async function registerAndActivate(page, email, name) { await user.goto(BASE + '/#/account'); await user.waitForSelector('#profileForm'); await user.click('#accountPageLogout'); + // Settings sync is active for a logged-in user: logout asks keep or remove. + await user.click('.cs-dialog [data-choice="keep"]'); await user.waitForSelector('#loginForm'); assert(await user.evaluate(() => location.hash) === '#/account/login', 'not on the login view after logout'); assert(await user.evaluate(() => window.CS_USER === null), 'client still holds the user'); @@ -121,6 +144,69 @@ async function registerAndActivate(page, email, name) { await user.waitForSelector('#accountToggle .nav-account-label:has-text("E2E User")'); }); + // Settings sync: two browser contexts are two devices on one account. + const SYNC_FAV = 'e2e5e7c0000000000000000000000000000000000000000000000000000000a1'; + const d1 = await (await browser.newContext()).newPage(); + const d2 = await (await browser.newContext()).newPage(); + for (const [pg, tag] of [[d1, 'd1'], [d2, 'd2']]) { + pg.setDefaultTimeout(8000); + pg.on('pageerror', (e) => console.error('[pageerror ' + tag + ']', e.message)); + } + + await step('settings sync: device 1 saves a packet time window and a favorite to the account', async () => { + await registerAndActivate(d1, 'sync@e2e.test', 'E2E Sync'); + await d1.goto(BASE + '/#/packets'); + await d1.waitForSelector('#fTimeWindow'); + await d1.selectOption('#fTimeWindow', '180'); + // No favorite star without node rows in view: write the key as nodes.js does. + await d1.evaluate((pk) => localStorage.setItem('meshcore-favorites', JSON.stringify([pk])), SYNC_FAV); + await until(async () => { + const k = await accountKeys(d1); + return k['meshcore-time-window'] === '180' && (k['meshcore-favorites'] || '').includes(SYNC_FAV); + }, 'account holds the time window and the favorite'); + }); + + await step('settings sync: device 2 logs in and gets both; its channel key stays local', async () => { + await d2.goto(BASE + '/#/account/login', { waitUntil: 'domcontentloaded' }); + await d2.waitForSelector('#loginForm'); + await d2.evaluate(() => localStorage.setItem('corescope_channel_keys', JSON.stringify({ '#e2e': '00112233445566778899aabbccddeeff' }))); + await d2.fill('#loginEmail', 'sync@e2e.test'); + await d2.fill('#loginPassword', PW); + await d2.click('#loginForm button[type="submit"]'); + await d2.waitForSelector('#profileForm'); + await d2.waitForFunction((pk) => (localStorage.getItem('meshcore-favorites') || '').includes(pk) && + localStorage.getItem('meshcore-time-window') === '180', SYNC_FAV); + // Check after a full round trip from device 2 (pull, merge, push), not + // just after its first pull. + await d2.evaluate(() => window.CSSettingsSync.syncNow()); + const k = await accountKeys(d2); + assert(!('corescope_channel_keys' in k), 'channel key reached the account'); + }); + + await step('settings sync: a favorite removed on device 1 is gone on device 2', async () => { + await d1.evaluate(() => localStorage.setItem('meshcore-favorites', '[]')); + await until(async () => !((await accountKeys(d1))['meshcore-favorites'] || '').includes(SYNC_FAV), 'removal reached the account'); + await d2.evaluate(() => window.CSSettingsSync.syncNow()); + await d2.waitForFunction((pk) => !(localStorage.getItem('meshcore-favorites') || '').includes(pk), SYNC_FAV); + }); + + await step('settings sync: axe on the section and the logout dialog; Remove keeps the channel key', async () => { + await d2.waitForSelector('#syncStatus'); + await axeClean(d2, '#syncSection'); + await d2.click('#accountPageLogout'); + await d2.waitForSelector('.cs-dialog'); + assert(await d2.evaluate(() => document.activeElement && document.activeElement.getAttribute('data-choice') === 'keep'), 'Keep is not focused'); + await axeClean(d2, '.cs-dialog'); + await d2.click('.cs-dialog [data-choice="remove"]'); + await d2.waitForSelector('#loginForm'); + const left = await d2.evaluate(() => ({ + fav: localStorage.getItem('meshcore-favorites'), tw: localStorage.getItem('meshcore-time-window'), + base: localStorage.getItem('cs-settings-sync-base'), ch: localStorage.getItem('corescope_channel_keys'), + })); + assert(left.fav === null && left.tw === null && left.base === null, 'synced keys left: ' + JSON.stringify(left)); + assert(left.ch && left.ch.includes('#e2e'), 'channel key removed'); + }); + await step('admin disables the user; the live session is logged out without a reload', async () => { await admin.goto(BASE + '/#/admin/users'); const row = admin.locator('tr[data-email="user@e2e.test"]'); @@ -130,8 +216,9 @@ async function registerAndActivate(page, email, name) { // No reload: leave and re-enter the account page; its sessions call answers 401. await user.goto(BASE + '/#/home'); await user.goto(BASE + '/#/account'); - await user.waitForSelector('.cs-auth-toast.visible:has-text("You were logged out.")'); + // The 60 s settings pull may log the user out first, so assert the end state only. await user.waitForSelector('#accountToggle .nav-account-label:has-text("Log in")'); + assert(await user.evaluate(() => window.CS_USER === null), 'CS_USER is not null'); }); await step('axe: no serious or critical violations on the new views', async () => { @@ -140,9 +227,7 @@ async function registerAndActivate(page, email, name) { await pg.waitForSelector(sel); await authReady(pg); await pg.waitForTimeout(1500); - const res = await new AxeBuilder({ page: pg }).include('#app').withTags(['wcag2a', 'wcag2aa']).analyze(); - const bad = res.violations.filter((v) => v.impact === 'serious' || v.impact === 'critical'); - assert(bad.length === 0, route + ': ' + bad.map((v) => v.id + ' ' + v.nodes.map((n) => n.target.join(' ') + ' ' + ((n.any[0] || {}).message || '')).join(' | ')).join(', ')); + await axeClean(pg, '#app'); } }); diff --git a/tests/unit/test-settings-sync.js b/tests/unit/test-settings-sync.js new file mode 100644 index 000000000..433c02dc2 --- /dev/null +++ b/tests/unit/test-settings-sync.js @@ -0,0 +1,1194 @@ +/* Unit tests for public/settings-sync.js (settings sync, sub-project B). + * The real file runs in a vm sandbox with a fake Storage (a real + * prototype, so the module's Storage.prototype wrap is exercised), a fake + * CSAuth backed by a fake server, and fake timers. Tests run one by one. */ +/* global setImmediate */ +'use strict'; +const vm = require('vm'); +const fs = require('fs'); +const path = require('path'); +const assert = require('assert'); + +const ROOT = path.resolve(__dirname, '..', '..'); +const SRC = fs.readFileSync(path.join(ROOT, 'public/settings-sync.js'), 'utf8'); +const J = JSON.stringify; +// Objects made inside the vm have another Object.prototype; compare plain copies. +const plain = (x) => JSON.parse(JSON.stringify(x)); +const settle = () => new Promise((r) => setImmediate(r)); + +const tests = []; +function test(name, fn) { tests.push({ name, fn }); } + +// escapeHtml comes from the real app.js, not a copy. +function loadEscapeHtml() { + const src = fs.readFileSync(path.join(ROOT, 'public/app.js'), 'utf8'); + const m = src.match(/function escapeHtml\(s\) \{[\s\S]*?\n\}/); + assert(m, 'escapeHtml not found in app.js'); + return vm.runInNewContext('(' + m[0].replace(/^function escapeHtml/, 'function') + ')'); +} + +// CSAuth.say and CSAuth.errText come from the real auth.js, bound to ctx +// (say reads ctx.document). +function loadAuthHelpers(ctx) { + const src = fs.readFileSync(path.join(ROOT, 'public/auth.js'), 'utf8'); + const say = src.match(/function say\(id, text, ok\) \{[\s\S]*?\n \}/); + const err = src.match(/function errText\(r\) \{.*\}/); + assert(say && err, 'say/errText not found in auth.js'); + return { say: vm.runInContext('(' + say[0] + ')', ctx), errText: vm.runInContext('(' + err[0] + ')', ctx) }; +} + +function fakeTimers() { + let now = 0, seq = 0; + const list = []; + const api = { + setTimeout(fn, ms) { list.push({ id: ++seq, at: now + (ms || 0), fn }); return seq; }, + clearTimeout(id) { const i = list.findIndex((t) => t.id === id); if (i >= 0) list.splice(i, 1); }, + setInterval(fn, ms) { list.push({ id: ++seq, at: now + ms, fn, every: ms }); return seq; }, + clearInterval(id) { api.clearTimeout(id); }, + async advance(ms) { + const end = now + ms; + for (;;) { + await settle(); + list.sort((a, b) => a.at - b.at); + const t = list[0]; + if (!t || t.at > end) break; + now = t.at; + if (t.every) t.at += t.every; else list.shift(); + t.fn(); + } + now = end; + await settle(); + }, + // Pending one-shot timers, as delays from now. + delays() { return list.filter((t) => !t.every).map((t) => t.at - now).sort((a, b) => a - b); }, + }; + return api; +} + +const ALLOW = [ + { key: 'meshcore-favorites', kind: 'set' }, + { key: 'meshcore-my-nodes', kind: 'set', id: 'pubkey' }, + { key: 'meshcore-time-window', kind: 'scalar' }, + { key: 'meshcore-theme', kind: 'scalar' }, + { key: 'meshcore-cb-preset', kind: 'scalar' }, + { key: 'cs-theme-overrides', kind: 'scalar' }, + { key: 'mc-dark-tile-provider', kind: 'scalar' }, + { key: 'mc-light-tile-provider', kind: 'scalar' }, +]; + +// fakeServer keeps one account document with the semantics of +// cmd/server/settings_handlers.go and internal/users/settings.go: the first +// write starts a new generation, later writes must name it. fail[METHOD] +// queues canned answers ('network' rejects the request). +function fakeServer() { + const s = { rev: 0, gen: '', gens: 0, doc: null, puts: [], gets: 0, deletes: 0, fail: { GET: [], PUT: [], DELETE: [] } }; + s.handle = (method, p, body) => { + if (method === 'PUT') s.puts.push(body); + if (method === 'GET') s.gets++; + if (s.fail[method].length) return s.fail[method].shift(); + if (method === 'GET') return { status: 200, data: { revision: s.rev, generation: s.gen, doc: s.doc, allowlist: ALLOW } }; + if (method === 'PUT') { + if (body.baseRevision !== s.rev || (s.rev && body.baseGeneration !== s.gen)) { + return { status: 409, data: { revision: s.rev, generation: s.gen, doc: s.doc } }; + } + if (!s.rev) s.gen = 'gen' + (++s.gens); + s.rev++; + s.doc = body.doc; + return { status: 200, data: { revision: s.rev, generation: s.gen } }; + } + s.deletes++; + s.rev = 0; + s.gen = ''; + s.doc = null; + return { status: 200, data: { ok: true } }; + }; + return s; +} + +const AUTO_IDS = ['syncStatus', 'syncNow', 'syncDelete', 'syncMsg']; + +// fakeOverlay is the element showDialog creates. Its buttons are read back +// from the rendered HTML; focus() moves doc.activeElement. +function fakeOverlay(doc) { + const node = { className: '', innerHTML: '', handlers: {}, removed: false, buttons: null }; + node.addEventListener = (t, f) => { node.handlers[t] = f; }; + node.remove = () => { node.removed = true; }; + node.closest = () => null; + node.querySelectorAll = () => { + if (!node.buttons) { + node.buttons = [...node.innerHTML.matchAll(/data-choice="([^"]*)"/g)].map((m) => { + const b = { getAttribute: () => m[1], focus() { doc.activeElement = b; } }; + b.closest = () => b; + return b; + }); + } + return node.buttons; + }; + return node; +} + +// makeEnv loads the real module. opts: local {key: raw}, server, +// user (default {id: 7}; null = logged out), enabled (default true), hash, +// customizerReady (default true: the customizer finished its init), +// shared (a Map: localStorage contents shared with another env, as two tabs +// of one browser share them; each env keeps its own Storage prototype), +// prefersDark (the OS colour scheme matchMedia reports). +function makeEnv(opts) { + opts = opts || {}; + function Storage() { this.m = opts.shared || new Map(); } + Storage.prototype.getItem = function (k) { return this.m.has(k) ? this.m.get(k) : null; }; + Storage.prototype.setItem = function (k, v) { this.m.set(k, String(v)); }; + Storage.prototype.removeItem = function (k) { this.m.delete(k); }; + const originalSet = Storage.prototype.setItem; + const ls = new Storage(); + Object.keys(opts.local || {}).forEach((k) => ls.m.set(k, opts.local[k])); + const server = opts.server || fakeServer(); + const timers = fakeTimers(); + const enabled = opts.enabled !== false; + let user = opts.user === undefined ? { id: 7 } : opts.user; + let logoutHandler = null; + const winListeners = {}, docListeners = {}, els = {}; + const toasts = [], events = [], warnings = [], errors = []; + const mkEl = (id) => { + const el = { id, innerHTML: '', textContent: '', handlers: {}, cls: {} }; + el.classList = { toggle(c, on) { el.cls[c] = !!on; } }; + el.addEventListener = (t, fn) => { el.handlers[t] = fn; }; + return el; + }; + const auth = { + isEnabled: () => enabled, + user: () => user, + ready: () => Promise.resolve(enabled ? user : null), + request(method, p, body) { + const r = server.handle(method, p, body === undefined ? undefined : JSON.parse(JSON.stringify(body))); + if (r === 'network') return Promise.reject(new Error('offline')); + return Promise.resolve({ ok: r.status < 400, status: r.status, data: r.data || {} }); + }, + notify(m) { toasts.push(m); }, + setLogoutHandler(fn) { logoutHandler = fn; }, + // env.nextMe: the user /api/auth/me reports now (undefined: unchanged). + refreshMe() { + env.refreshes++; + if (env.nextMe !== undefined) { user = env.nextMe; (winListeners['cs-auth-changed'] || []).forEach((f) => f({ detail: user })); } + return Promise.resolve(user); + }, + }; + const env = { navigations: 0, pipelines: 0, resets: 0, presetClears: 0, refreshes: 0, nextMe: undefined, tileSets: [] }; + // map-tile-providers.js: the getter falls back to the default when the key + // is unset, the setter persists through localStorage (the wrapped setItem). + const tileApi = (type, key, def) => ({ + get: () => ls.getItem(key) || def, + set: (id) => { env.tileSets.push([type, id]); win.localStorage.setItem(key, id); return true; }, + }); + const darkTiles = tileApi('dark', 'mc-dark-tile-provider', 'carto-dark'); + const lightTiles = tileApi('light', 'mc-light-tile-provider', 'carto-light'); + const win = { + localStorage: ls, Storage, CSAuth: auth, MC_USER_MGMT: enabled ? { enabled: true } : null, + addEventListener(t, f) { (winListeners[t] = winListeners[t] || []).push(f); }, + dispatchEvent(e) { events.push(e); }, + navigate() { env.navigations++; }, + _customizerV2: { initDone: opts.customizerReady !== false, runPipeline() { env.pipelines++; }, resetAll() { env.resets++; } }, + MeshCorePresets: { clearPreset() { env.presetClears++; } }, + MC_getDarkTileProvider: darkTiles.get, MC_setDarkTileProvider: darkTiles.set, + MC_getLightTileProvider: lightTiles.get, MC_setLightTileProvider: lightTiles.set, + matchMedia: (q) => ({ matches: q === '(prefers-color-scheme: dark)' && !!opts.prefersDark }), + }; + const doc = { + visibilityState: 'visible', + activeElement: null, + body: { children: [], appendChild(n) { this.children.push(n); } }, + createElement() { return fakeOverlay(doc); }, + addEventListener(t, f) { docListeners[t] = f; }, + removeEventListener(t, f) { if (docListeners[t] === f) delete docListeners[t]; }, + getElementById(id) { return els[id] || (AUTO_IDS.indexOf(id) !== -1 ? (els[id] = mkEl(id)) : null); }, + }; + const ctx = { + window: win, document: doc, location: { hash: opts.hash || '#/packets' }, + console: { warn: (m) => warnings.push(String(m)), error: (m) => errors.push(String(m)), log() {} }, + Promise, + StorageEvent: function (type, init) { this.type = type; this.key = init.key; this.newValue = init.newValue; }, + escapeHtml: loadEscapeHtml(), + setTimeout: timers.setTimeout, clearTimeout: timers.clearTimeout, + setInterval: timers.setInterval, clearInterval: timers.clearInterval, + }; + vm.createContext(ctx); + Object.assign(auth, loadAuthHelpers(ctx)); + vm.runInContext(SRC, ctx); + Object.defineProperty(env, 'logoutHandler', { get: () => logoutHandler }); + Object.assign(env, { + ls, server, timers, toasts, events, warnings, errors, els, doc, ctx, win, originalSet, + api: win.CSSettingsSync, t: win.CSSettingsSync._test, + login(u) { user = u; (winListeners['cs-auth-changed'] || []).forEach((f) => f({ detail: u })); return settle(); }, + logout() { user = null; (winListeners['cs-auth-changed'] || []).forEach((f) => f({ detail: null })); return settle(); }, + fireDoc(t) { if (docListeners[t]) docListeners[t](); return settle(); }, + el(id) { return els[id] || (els[id] = mkEl(id)); }, + }); + return env; +} + +// ── mergeDocs ── +const M_ALLOW = [{ key: 'fav', kind: 'set' }, { key: 'nodes', kind: 'set', id: 'pubkey' }, { key: 'tw', kind: 'scalar' }]; +const n = (pubkey, name) => ({ pubkey, name }); +const MERGE_CASES = [ + // name, local, profile, base, keys, localChanges, differsFromProfile + ['set: an item added here since the last sync is kept', + { fav: J(['a', 'b']) }, { fav: J(['a']) }, { fav: J(['a']) }, { fav: J(['a', 'b']) }, [], true], + ['set: an item removed here since the last sync is removed', + { fav: J(['a']) }, { fav: J(['a', 'b']) }, { fav: J(['a', 'b']) }, { fav: J(['a']) }, [], true], + ['set: an item added on another device arrives', + { fav: J(['a']) }, { fav: J(['a', 'c']) }, { fav: J(['a']) }, { fav: J(['a', 'c']) }, ['fav'], false], + ['set: an item removed on another device is not brought back', + { fav: J(['a', 'b']) }, { fav: J(['a']) }, { fav: J(['a', 'b']) }, { fav: J(['a']) }, ['fav'], false], + ['set: additions on both sides are both kept', + { fav: J(['a', 'x']) }, { fav: J(['a', 'y']) }, { fav: J(['a']) }, { fav: J(['a', 'y', 'x']) }, ['fav'], true], + ['set: without a baseline the lists merge by union', + { fav: J(['x']) }, { fav: J(['y']) }, {}, { fav: J(['y', 'x']) }, ['fav'], true], + ['set: same identity edited only here keeps the local version', + { nodes: J([n('k1', 'new')]) }, { nodes: J([n('k1', 'old')]) }, { nodes: J([n('k1', 'old')]) }, { nodes: J([n('k1', 'new')]) }, [], true], + ['set: same identity edited on both sides keeps the profile version', + { nodes: J([n('k1', 'new')]) }, { nodes: J([n('k1', 'other')]) }, { nodes: J([n('k1', 'old')]) }, { nodes: J([n('k1', 'other')]) }, ['nodes'], false], + ['set: same identity edited only in the profile arrives', + { nodes: J([n('k1', 'old')]) }, { nodes: J([n('k1', 'other')]) }, { nodes: J([n('k1', 'old')]) }, { nodes: J([n('k1', 'other')]) }, ['nodes'], false], + ['scalar: changed only here wins', + { tw: '60' }, { tw: '15' }, { tw: '15' }, { tw: '60' }, [], true], + ['scalar: changed in the profile wins', + { tw: '15' }, { tw: '180' }, { tw: '15' }, { tw: '180' }, ['tw'], false], + ['scalar: changed on both sides, the profile wins', + { tw: '60' }, { tw: '180' }, { tw: '15' }, { tw: '180' }, ['tw'], false], + ['scalar: removed here wins', + {}, { tw: '15' }, { tw: '15' }, {}, [], true], + ['scalar: removed in the profile is removed here', + { tw: '15' }, {}, { tw: '15' }, {}, ['tw'], false], + ['set: the same list spelled differently takes the profile spelling and does not differ from it', + { fav: '["a", "b"]' }, { fav: '["a","b"]' }, { fav: '["a","b"]' }, { fav: '["a","b"]' }, ['fav'], false], + ['set: an unparseable baseline counts as none, so lists merge by union', + { fav: J(['a', 'x']) }, { fav: J(['a', 'y']) }, { fav: 'garbage' }, { fav: J(['a', 'y', 'x']) }, ['fav'], true], + ['set: an item edited here but deleted on another device stays deleted', + { nodes: J([n('k1', 'new')]) }, { nodes: J([]) }, { nodes: J([n('k1', 'old')]) }, { nodes: J([]) }, ['nodes'], false], + ['set: duplicate items on this device dedupe by identity', + { fav: J(['a', 'b', 'a']) }, { fav: J(['a']) }, { fav: J(['a']) }, { fav: J(['a', 'b']) }, ['fav'], true], + ['set: objects without the id field fall back to JSON identity', + { nodes: J([{ name: 'x' }]) }, { nodes: J([{ name: 'y' }]) }, { nodes: J([]) }, { nodes: J([{ name: 'y' }, { name: 'x' }]) }, ['nodes'], true], + ['keys outside the allowlist are ignored', + { other: '1', tw: '15' }, { other: '2', tw: '15' }, {}, { tw: '15' }, [], false], +]; + +MERGE_CASES.forEach(([name, l, p, b, keys, changes, differs]) => { + test('merge ' + name, () => { + const env = makeEnv({ enabled: false }); + const r = plain(env.t.mergeDocs(l, p, b, M_ALLOW)); + assert.deepStrictEqual(r.keys, keys); + assert.deepStrictEqual(r.localChanges, changes); + assert.strictEqual(r.differsFromProfile, differs); + }); +}); + +test('merge: a set value that is not a JSON list merges as a scalar, with a warning', () => { + const env = makeEnv({ enabled: false }); + const r = plain(env.t.mergeDocs({ fav: 'not json' }, { fav: J(['a']) }, { fav: J(['a']) }, M_ALLOW)); + assert.deepStrictEqual(r.keys, { fav: 'not json' }); + assert.strictEqual(env.warnings.length, 1); + assert(env.warnings[0].indexOf('fav') !== -1, env.warnings[0]); +}); + +// ── engine ── +// Tests that do not name a generation use 'g0' on both sides: the device +// synced the account's current document. +const BASE = (user, keys, rev, hold, gen) => ({ 'cs-settings-sync-base': J({ user, keys, hold: !!hold, gen: gen === undefined ? (rev ? 'g0' : '') : gen }), 'cs-settings-sync-rev': String(rev) }); +const synced = (keys, rev, gen) => Object.assign({}, keys, BASE(7, keys, rev, false, gen)); +const serverWith = (rev, keys, gen) => { const s = fakeServer(); s.rev = rev; s.gen = rev ? (gen || 'g0') : ''; s.doc = rev ? { v: 1, keys } : null; return s; }; + +test('feature off: no request and no interception', async () => { + const env = makeEnv({ enabled: false }); + await env.timers.advance(120000); + assert.strictEqual(env.server.gets + env.server.puts.length, 0); + assert.strictEqual(env.win.Storage.prototype.setItem, env.originalSet); +}); + +test('logged out: no request and no interception', async () => { + const env = makeEnv({ user: null }); + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(120000); + assert.strictEqual(env.server.gets + env.server.puts.length, 0); + assert.strictEqual(env.win.Storage.prototype.setItem, env.originalSet); +}); + +test('first login with an empty profile uploads this device (never channel keys)', async () => { + const env = makeEnv({ user: null, local: { 'meshcore-favorites': J(['a']), 'meshcore-time-window': '60', corescope_channel_keys: '{"#x":"00"}', 'meshcore-api-key': 'k' } }); + await env.login({ id: 7 }); + assert.strictEqual(env.server.puts.length, 1); + assert.deepStrictEqual(env.server.puts[0], { baseRevision: 0, baseGeneration: '', doc: { v: 1, keys: { 'meshcore-favorites': J(['a']), 'meshcore-time-window': '60' } } }); + assert.strictEqual(JSON.parse(env.ls.getItem('cs-settings-sync-base')).gen, env.server.gen); + assert.deepStrictEqual(env.toasts, ['Your settings are now saved to your account.']); + assert.strictEqual(env.ls.getItem('cs-settings-sync-rev'), '1'); + assert.strictEqual(JSON.parse(env.ls.getItem('cs-settings-sync-base')).user, 7); +}); + +test('first login with nothing to upload sends nothing', async () => { + const env = makeEnv({ user: null }); + await env.login({ id: 7 }); + assert.strictEqual(env.server.puts.length, 0); + assert.strictEqual(env.t.state.status, 'idle'); +}); + +test('login on another device merges the profile, applies theme and customizer, re-renders', async () => { + const server = serverWith(4, { 'meshcore-favorites': J(['a']), 'meshcore-theme': 'dark', 'cs-theme-overrides': '{"x":1}' }); + const env = makeEnv({ user: null, server, local: { 'meshcore-favorites': J(['b']) } }); + await env.login({ id: 7 }); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['a', 'b'])); + assert.strictEqual(env.ls.getItem('meshcore-theme'), 'dark'); + assert(env.events.some((e) => e.type === 'storage' && e.key === 'meshcore-theme' && e.newValue === 'dark')); + assert.strictEqual(env.pipelines, 1); + assert.strictEqual(env.navigations, 1); + // M3: the first login names the upload, not "updated from another device". + assert.deepStrictEqual(env.toasts, ['Your settings are now saved to your account.']); + // b was only here: pushed on top of revision 4. + assert.strictEqual(server.puts.length, 1); + assert.strictEqual(server.puts[0].baseRevision, 4); + assert.strictEqual(server.doc.keys['meshcore-favorites'], J(['a', 'b'])); +}); + +test('first login with nothing new here: the first-login toast, nothing pushed', async () => { + const server = serverWith(4, { 'meshcore-favorites': J(['a']), 'meshcore-time-window': '180' }); + const env = makeEnv({ user: null, server, local: { 'meshcore-favorites': J(['a']) } }); + await env.login({ id: 7 }); + assert.strictEqual(env.ls.getItem('meshcore-time-window'), '180'); + assert.strictEqual(env.navigations, 1); + assert.strictEqual(server.puts.length, 0); + assert.deepStrictEqual(env.toasts, ['Your settings are now saved to your account.']); +}); + +test('remote tile providers reach the map through a storage event', async () => { + const server = serverWith(2, { 'mc-dark-tile-provider': 'carto-dark', 'mc-light-tile-provider': 'osm' }); + const env = makeEnv({ server, local: synced({}, 1) }); + await env.timers.advance(0); + assert(env.events.some((e) => e.type === 'storage' && e.key === 'mc-dark-tile-provider' && e.newValue === 'carto-dark')); + assert(env.events.some((e) => e.type === 'storage' && e.key === 'mc-light-tile-provider' && e.newValue === 'osm')); +}); + +// M1 (final review): app.js ignores a storage event without a value, so a +// remote removal applies the default the app starts with: the OS colour +// scheme for the theme, no preset (clearPreset) for the colour-blind preset. +test('a theme removed on another device falls back to the OS colour scheme', async () => { + for (const prefersDark of [false, true]) { + const env = makeEnv({ prefersDark, server: serverWith(2, {}), local: synced({ 'meshcore-theme': 'dark' }, 1) }); + await env.timers.advance(0); + assert.strictEqual(env.ls.getItem('meshcore-theme'), null); + const ev = env.events.filter((e) => e.type === 'storage' && e.key === 'meshcore-theme'); + assert.deepStrictEqual(ev.map((e) => e.newValue), [prefersDark ? 'dark' : 'light']); + } +}); + +test('a colour-blind preset removed on another device is cleared through cb-presets', async () => { + const env = makeEnv({ server: serverWith(2, {}), local: synced({ 'meshcore-cb-preset': 'wong' }, 1) }); + await env.timers.advance(0); + assert.strictEqual(env.ls.getItem('meshcore-cb-preset'), null); + assert.strictEqual(env.presetClears, 1); + assert(!env.events.some((e) => e.type === 'storage' && e.key === 'meshcore-cb-preset')); + assert.strictEqual(env.server.puts.length, 0); +}); + +// Polish 2: the tile listener ignores a removal, so a removed provider is +// re-applied as the effective one (the way customize-v2.js resetAll does) +// and the key stays unset. +const TILES = { 'mc-dark-tile-provider': 'osm-dark', 'mc-light-tile-provider': 'osm' }; +test('tile providers removed on another device fall back to the effective provider', async () => { + const env = makeEnv({ server: serverWith(2, {}), local: synced(TILES, 1) }); + await env.timers.advance(0); + assert.deepStrictEqual(plain(env.tileSets), [['dark', 'carto-dark'], ['light', 'carto-light']]); + assert.strictEqual(env.ls.getItem('mc-dark-tile-provider'), null); + assert.strictEqual(env.ls.getItem('mc-light-tile-provider'), null); + assert.strictEqual(env.t.state.seq, 0, 're-applying the default counted as a change'); + await env.timers.advance(10000); + assert.strictEqual(env.server.puts.length, 0); +}); + +test('logout dialog: remove re-applies the effective tile providers', async () => { + const env = makeEnv({ server: serverWith(1, TILES), local: synced(TILES, 1) }); + await env.timers.advance(0); + env.t.useDialog(() => Promise.resolve('remove')); + (await env.logoutHandler()).afterLogout(); + assert.deepStrictEqual(plain(env.tileSets), [['dark', 'carto-dark'], ['light', 'carto-light']]); + assert.strictEqual(env.ls.getItem('mc-dark-tile-provider'), null); + assert.strictEqual(env.ls.getItem('mc-light-tile-provider'), null); + await env.timers.advance(10000); + assert.strictEqual(env.server.puts.length, 0); +}); + +test('customizer not initialised yet: its pipeline is not run (its init reads the new values)', async () => { + const server = serverWith(2, { 'cs-theme-overrides': '{"x":1}' }); + const env = makeEnv({ server, local: synced({}, 1), customizerReady: false }); + await env.timers.advance(0); + assert.strictEqual(env.ls.getItem('cs-theme-overrides'), '{"x":1}'); + assert.strictEqual(env.pipelines, 0); + assert.strictEqual(env.navigations, 1); +}); + +test('interception: allowlisted writes push once, 2 s after the last; other keys never', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const env = makeEnv({ server: serverWith(1, keys), local: synced(keys, 1) }); + await env.timers.advance(0); + env.ls.setItem('meshcore-time-window', '60'); + env.ls.setItem('meshcore-time-window', '180'); + env.ls.setItem('corescope_channel_keys', '{"#x":"00"}'); + env.ls.setItem('meshcore-api-key', 'k'); + env.ls.setItem('panel-drag-packets', '1'); + await env.timers.advance(1999); + assert.strictEqual(env.server.puts.length, 0); + await env.timers.advance(1); + assert.strictEqual(env.server.puts.length, 1); + assert.deepStrictEqual(env.server.puts[0].doc.keys, { 'meshcore-favorites': J(['a']), 'meshcore-time-window': '180' }); + env.ls.removeItem('meshcore-time-window'); + await env.timers.advance(2000); + assert.strictEqual(env.server.puts.length, 2); + env.ls.setItem('corescope_channel_keys', '{}'); + await env.timers.advance(10000); + assert.strictEqual(env.server.puts.length, 2); +}); + +test('writing the same value again does not push', async () => { + const keys = { 'meshcore-theme': 'dark' }; + const env = makeEnv({ server: serverWith(1, keys), local: synced(keys, 1) }); + await env.timers.advance(0); + env.ls.setItem('meshcore-theme', 'dark'); + env.ls.removeItem('meshcore-time-window'); + await env.timers.advance(10000); + assert.strictEqual(env.server.puts.length, 0); +}); + +test('values applied from the account do not trigger a push', async () => { + const env = makeEnv({ server: serverWith(2, { 'meshcore-time-window': '180' }), local: synced({ 'meshcore-time-window': '60' }, 1) }); + await env.timers.advance(10000); + assert.strictEqual(env.ls.getItem('meshcore-time-window'), '180'); + assert.strictEqual(env.server.puts.length, 0); +}); + +test('409: merges the returned document and retries on top of it', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const server = serverWith(1, keys); + const env = makeEnv({ server, local: synced(keys, 1) }); + await env.timers.advance(0); + server.rev = 2; + server.doc = { v: 1, keys: { 'meshcore-favorites': J(['a', 'c']) } }; // another device + env.ls.setItem('meshcore-favorites', J(['a', 'b'])); + await env.timers.advance(2000); + assert.strictEqual(server.puts.length, 2); + assert.strictEqual(server.puts[1].baseRevision, 2); + assert.strictEqual(server.doc.keys['meshcore-favorites'], J(['a', 'c', 'b'])); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['a', 'c', 'b'])); +}); + +test('409 more than three times falls back to the retry backoff', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const server = serverWith(1, keys); + const env = makeEnv({ server, local: synced(keys, 1) }); + await env.timers.advance(0); + const conflict = { status: 409, data: { revision: 1, generation: 'g0', doc: { v: 1, keys } } }; + server.fail.PUT = [conflict, conflict, conflict, conflict]; + env.ls.setItem('meshcore-favorites', J(['a', 'b'])); + await env.timers.advance(2000); + assert.strictEqual(server.puts.length, 4); + assert.strictEqual(env.t.state.status, 'retrying'); + assert.deepStrictEqual(env.timers.delays(), [2000]); +}); + +test('network errors, 5xx and 429 back off from 2 s doubling to 5 min; success resets', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const server = serverWith(1, keys); + const env = makeEnv({ server, local: synced(keys, 1) }); + env.doc.visibilityState = 'hidden'; // no minute pulls in between + await env.timers.advance(0); + const expected = [2000, 4000, 8000, 16000, 32000, 64000, 128000, 256000, 300000]; + server.fail.PUT = expected.map((_, i) => ['network', { status: 503, data: {} }, { status: 429, data: {} }][i % 3]); + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + for (const d of expected) { + assert.deepStrictEqual(env.timers.delays(), [d]); + assert.strictEqual(env.t.state.status, 'retrying'); + await env.timers.advance(d); + } + assert.strictEqual(env.t.state.status, 'ok'); + assert.strictEqual(env.t.state.backoff, 2000); + assert.strictEqual(server.doc.keys['meshcore-time-window'], '60'); +}); + +// M7 (final review): the baseline is up to 256 KiB; an unchanged pull +// must not rewrite it every minute. +test('a pull with nothing new does not rewrite the baseline', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const shared = new Map(); + const set = shared.set.bind(shared); + let baseWrites = 0; + shared.set = (k, v) => { if (k === 'cs-settings-sync-base') baseWrites++; return set(k, v); }; + const server = serverWith(1, keys); + const env = makeEnv({ server, shared, local: synced(keys, 1) }); + baseWrites = 0; + await env.timers.advance(180000); + assert(server.gets >= 4, 'gets ' + server.gets); + assert.strictEqual(baseWrites, 0); + server.rev = 2; + server.doc = { v: 1, keys: { 'meshcore-favorites': J(['a', 'b']) } }; + await env.timers.advance(60000); + assert.strictEqual(baseWrites, 1); + assert.strictEqual(shared.get('cs-settings-sync-rev'), '2'); +}); + +test('tab focus pulls; the minute pull runs only while visible', async () => { + const env = makeEnv({ server: serverWith(1, {}), local: synced({}, 1) }); + await env.timers.advance(0); + const g0 = env.server.gets; + await env.fireDoc('visibilitychange'); + assert.strictEqual(env.server.gets, g0 + 1); + await env.timers.advance(60000); + assert.strictEqual(env.server.gets, g0 + 2); + env.doc.visibilityState = 'hidden'; + await env.timers.advance(180000); + assert.strictEqual(env.server.gets, g0 + 2); +}); + +test('413 stops pushing and names the largest keys; Sync now tries again', async () => { + const server = serverWith(1, {}); + const env = makeEnv({ server, local: synced({}, 1) }); + await env.timers.advance(0); + server.fail.PUT = [{ status: 413, data: { error: 'too large' } }]; + env.ls.setItem('cs-theme-overrides', 'x'.repeat(100)); + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + assert.strictEqual(env.t.state.status, 'too-large'); + assert.strictEqual(env.t.state.tooLarge[0], 'cs-theme-overrides'); + env.ls.setItem('meshcore-time-window', '15'); + await env.timers.advance(10000); + assert.strictEqual(server.puts.length, 1); + await env.api.syncNow(); + await settle(); + assert.strictEqual(server.puts.length, 2); + assert.strictEqual(env.t.state.status, 'ok'); +}); + +test('400 stops pushing until reload and logs the reason', async () => { + const server = serverWith(1, {}); + const env = makeEnv({ server, local: synced({}, 1) }); + await env.timers.advance(0); + server.fail.PUT = [{ status: 400, data: { error: 'key "x" is not a synced setting' } }]; + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + assert.strictEqual(env.t.state.status, 'rejected'); + assert.strictEqual(env.errors.length, 1); + env.ls.setItem('meshcore-time-window', '15'); + await env.api.syncNow(); + await env.timers.advance(10000); + assert.strictEqual(server.puts.length, 1); +}); + +// M6 (final review): after a logout and login in another tab this tab's +// CSRF token is stale and every PUT answers 403; retrying never ends. +// Polish 1: a login in another tab rotates the CSRF token; refreshMe stores +// the new one, so the PUT is retried once with it. +test('403 on a push: same user, retried once with the refreshed token', async () => { + const server = serverWith(1, {}); + const env = makeEnv({ server, local: synced({}, 1) }); + await env.timers.advance(0); + server.fail.PUT = [{ status: 403, data: { error: 'missing CSRF token' } }]; + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + assert.strictEqual(env.refreshes, 1); + assert.strictEqual(server.puts.length, 2); + assert.strictEqual(server.doc.keys['meshcore-time-window'], '60'); + assert.strictEqual(env.t.state.blocked, null); + assert.strictEqual(env.t.state.status, 'ok'); +}); + +test('403 on a push: same user, a second 403 stops syncing until a reload', async () => { + const server = serverWith(1, {}); + const env = makeEnv({ server, local: synced({}, 1) }); + await env.timers.advance(0); + server.fail.PUT = [{ status: 403, data: { error: 'missing CSRF token' } }, { status: 403, data: { error: 'bad origin' } }]; + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + assert.strictEqual(env.refreshes, 2); + assert.strictEqual(server.puts.length, 2); + assert.strictEqual(env.t.state.blocked, 'forbidden'); + assert.strictEqual(env.t.state.status, 'forbidden'); + assert.strictEqual(env.t.statusText(), 'Not synced: this tab is out of date. Reload the page to sync again.'); + assert.deepStrictEqual(env.timers.delays(), [], 'a retry is still scheduled'); + env.ls.setItem('meshcore-time-window', '15'); + await env.timers.advance(600000); + assert.strictEqual(server.puts.length, 2); +}); + +test('403 on a push: another user logged in since re-activates for that user', async () => { + const server = serverWith(1, {}); + const env = makeEnv({ server, local: synced({}, 1) }); + await env.timers.advance(0); + server.fail.PUT = [{ status: 403, data: { error: 'missing CSRF token' } }]; + env.nextMe = { id: 8 }; + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + assert.strictEqual(env.refreshes, 1); + assert.strictEqual(env.t.state.userId, 8); + assert.notStrictEqual(env.t.state.status, 'forbidden'); + assert.strictEqual(env.t.state.blocked, null); +}); + +test('403 on a pull: logged out since deactivates', async () => { + const server = serverWith(1, {}); + const env = makeEnv({ server, local: synced({}, 1) }); + await env.timers.advance(0); + server.fail.GET = [{ status: 403, data: { error: 'forbidden' } }]; + env.nextMe = null; + await env.fireDoc('visibilitychange'); + assert.strictEqual(env.refreshes, 1); + assert.strictEqual(env.t.state.active, false); +}); + +test('account copy deleted elsewhere: values stay, no upload until the next change', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const env = makeEnv({ server: serverWith(0, null), local: synced(keys, 3) }); + await env.timers.advance(60000); + assert.strictEqual(env.server.puts.length, 0); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['a'])); + assert.strictEqual(JSON.parse(env.ls.getItem('cs-settings-sync-base')).hold, true); + assert.strictEqual(env.ls.getItem('cs-settings-sync-rev'), '0'); + env.ls.setItem('meshcore-favorites', J(['a', 'b'])); + await env.timers.advance(2000); + assert.strictEqual(env.server.puts.length, 1); + assert.deepStrictEqual(env.server.puts[0], { baseRevision: 0, baseGeneration: '', doc: { v: 1, keys: { 'meshcore-favorites': J(['a', 'b']) } } }); +}); + +// C2 (final review): revisions restart at 1 after a delete, so the baseline +// carries the document's generation; a baseline of another generation +// counts as none (union, nothing dropped). +test('a copy deleted and started again elsewhere: the old baseline drops nothing here', async () => { + // Synced revision 5 of generation g0, then closed. Meanwhile the copy was + // deleted and a new phone uploaded [p] as revision 1 of generation g9. + const server = serverWith(1, { 'meshcore-favorites': J(['p']) }, 'g9'); + const env = makeEnv({ server, local: synced({ 'meshcore-favorites': J(['a', 'b']) }, 5) }); + await env.timers.advance(0); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['p', 'a', 'b'])); + assert.strictEqual(server.puts.length, 1); + assert.strictEqual(server.puts[0].baseRevision, 1); + assert.strictEqual(server.puts[0].baseGeneration, 'g9'); + assert.strictEqual(server.doc.keys['meshcore-favorites'], J(['p', 'a', 'b'])); + assert.strictEqual(JSON.parse(env.ls.getItem('cs-settings-sync-base')).gen, 'g9'); +}); + +test('an offline retry from an old generation at the same revision gets 409 and merges', async () => { + const server = serverWith(2, { 'meshcore-favorites': J(['a']) }); + const env = makeEnv({ server, local: synced({ 'meshcore-favorites': J(['a']) }, 2) }); + env.doc.visibilityState = 'hidden'; // no pulls: only the retry talks to the server + await env.timers.advance(0); + server.fail.PUT = ['network']; + env.ls.setItem('meshcore-favorites', J(['a', 'b'])); + await env.timers.advance(2000); + assert.strictEqual(env.t.state.status, 'retrying'); + // Meanwhile: deleted, and a new copy reached revision 2 again. + server.gen = 'g9'; + server.doc = { v: 1, keys: { 'meshcore-favorites': J(['c', 'd']) } }; + await env.timers.advance(2000); + const tried = server.puts[1]; + assert.deepStrictEqual([tried.baseRevision, tried.baseGeneration], [2, 'g0']); + assert.strictEqual(server.rev, 3); + assert.strictEqual(server.doc.keys['meshcore-favorites'], J(['c', 'd', 'a', 'b'])); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['c', 'd', 'a', 'b'])); +}); + +test("another user's baseline counts as none: nothing is removed", async () => { + const local = Object.assign({ 'meshcore-favorites': J(['a', 'b']) }, BASE(99, { 'meshcore-favorites': J(['a', 'b', 'c']) }, 5)); + const env = makeEnv({ server: serverWith(2, { 'meshcore-favorites': J(['c']) }), local }); + await env.timers.advance(0); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['c', 'a', 'b'])); +}); + +test('mid-edit: on an account page or with the geofilter editor open, no re-render', async () => { + const env = makeEnv({ hash: '#/account', server: serverWith(2, { 'meshcore-time-window': '180' }), local: synced({ 'meshcore-time-window': '60' }, 1) }); + await env.timers.advance(0); + assert.strictEqual(env.ls.getItem('meshcore-time-window'), '180'); + assert.strictEqual(env.navigations, 0); + assert(env.toasts.indexOf('Settings updated from another device') !== -1); + env.ctx.location.hash = '#/packets'; + assert.strictEqual(env.t.midEdit(), false); + env.el('cv2-gf-modal-overlay'); + assert.strictEqual(env.t.midEdit(), true); + env.ctx.location.hash = '#/accounts-other'; + delete env.els['cv2-gf-modal-overlay']; + assert.strictEqual(env.t.midEdit(), false); +}); + +test('logout makes the module inert: wrap removed, no timers, no requests', async () => { + const env = makeEnv({ server: serverWith(1, {}), local: synced({}, 1) }); + await env.timers.advance(0); + assert.notStrictEqual(env.win.Storage.prototype.setItem, env.originalSet); + await env.logout(); + assert.strictEqual(env.win.Storage.prototype.setItem, env.originalSet); + const before = env.server.gets + env.server.puts.length; + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(180000); + assert.strictEqual(env.server.gets + env.server.puts.length, before); + assert.deepStrictEqual(env.timers.delays(), []); +}); + +// holdNext delays the answer to the next request of method until the +// returned release() is called. The fake server still handles it at send +// time, so the held answer reflects the account as it was then. +function holdNext(env, method) { + const orig = env.win.CSAuth.request; + let release, used = false; + const gate = new Promise((r) => { release = r; }); + env.win.CSAuth.request = function (m, p, b) { + const res = orig.call(this, m, p, b); + if (m !== method || used) return res; + used = true; + return gate.then(() => res); + }; + return () => { release(); return settle(); }; +} + +test('a pull answered after a successful push does not revert the local change', async () => { + const env = makeEnv({ server: serverWith(1, { 'meshcore-time-window': '60' }), local: synced({ 'meshcore-time-window': '60' }, 1) }); + await env.timers.advance(0); + const release = holdNext(env, 'GET'); + await env.fireDoc('visibilitychange'); // GET sent, answer (revision 1) held + env.ls.setItem('meshcore-time-window', '180'); + await env.timers.advance(2000); + assert.strictEqual(env.server.rev, 2); + await release(); + assert.strictEqual(env.ls.getItem('meshcore-time-window'), '180'); + assert.strictEqual(env.ls.getItem('cs-settings-sync-rev'), '2'); + assert.strictEqual(env.navigations, 0); + assert.strictEqual(env.toasts.indexOf('Settings updated from another device'), -1); +}); + +test('a revision-0 pull answered after the first upload does not enter hold', async () => { + const env = makeEnv({ user: null, local: { 'meshcore-favorites': J(['a']) } }); + await settle(); + const release = holdNext(env, 'GET'); + const loggedIn = env.login({ id: 7 }); // first GET held + env.fireDoc('visibilitychange'); // second GET answered at once: revision 0, first upload + await loggedIn; + assert.strictEqual(env.server.rev, 1); + await release(); + assert.strictEqual(JSON.parse(env.ls.getItem('cs-settings-sync-base')).hold, false); + assert.strictEqual(env.ls.getItem('cs-settings-sync-rev'), '1'); + assert.strictEqual(env.t.state.status, 'ok'); +}); + +test('a push answered after logout and login again does not touch the new session', async () => { + const env = makeEnv({ server: serverWith(1, {}), local: synced({}, 1) }); + await env.timers.advance(0); + const releaseOld = holdNext(env, 'PUT'); + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); // old session PUT held + await env.logout(); + await env.login({ id: 7 }); + const releaseNew = holdNext(env, 'PUT'); + env.ls.setItem('meshcore-time-window', '15'); + await env.timers.advance(2000); // new session PUT held + const inFlight = env.t.state.pushing; + assert(inFlight); + await releaseOld(); + assert.strictEqual(env.t.state.pushing, inFlight); + assert.strictEqual(env.t.state.status, 'syncing'); + await releaseNew(); + assert.strictEqual(env.t.state.pushing, null); + assert.strictEqual(env.ls.getItem('cs-settings-sync-rev'), '3'); + assert.strictEqual(JSON.parse(env.ls.getItem('cs-settings-sync-base')).keys['meshcore-time-window'], '15'); +}); + +// I1 (final review): two tabs share localStorage, so the stored baseline is +// the truth for both; a tab never merges against its own stale copy. +test('two tabs: a removal from another device is not undone by a tab with an old baseline', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const server = serverWith(2, keys); + const shared = new Map(); + const tab2 = makeEnv({ server, shared, local: synced(keys, 2) }); + await tab2.timers.advance(0); + const tab1 = makeEnv({ server, shared }); + await tab1.timers.advance(0); + tab1.ls.setItem('meshcore-favorites', J(['a', 'X'])); + await tab1.timers.advance(2000); + assert.strictEqual(server.rev, 3); + // A phone removes X. + server.rev = 4; + server.doc = { v: 1, keys: { 'meshcore-favorites': J(['a']) } }; + await tab2.fireDoc('visibilitychange'); + assert.strictEqual(shared.get('meshcore-favorites'), J(['a'])); + assert.strictEqual(server.doc.keys['meshcore-favorites'], J(['a'])); + assert.strictEqual(server.rev, 4); +}); + +test('two tabs: a pull answered after the other tab pushed does not drop that change', async () => { + const keys = { 'meshcore-favorites': J(['a']) }; + const server = serverWith(2, keys); + const shared = new Map(); + const tab2 = makeEnv({ server, shared, local: synced(keys, 2) }); + await tab2.timers.advance(0); + const tab1 = makeEnv({ server, shared }); + await tab1.timers.advance(0); + const release = holdNext(tab2, 'GET'); + await tab2.fireDoc('visibilitychange'); // GET sent: revision 2, answer held + tab1.ls.setItem('meshcore-favorites', J(['a', 'X'])); + await tab1.timers.advance(2000); + assert.strictEqual(server.rev, 3); + await release(); + assert.strictEqual(shared.get('meshcore-favorites'), J(['a', 'X'])); + assert.strictEqual(server.doc.keys['meshcore-favorites'], J(['a', 'X'])); +}); + +// Polish 3: range inputs write many times per second; a write must not +// parse the stored baseline (up to 256 KiB) to learn the hold flag. +test('allowlisted writes do not read the stored baseline', async () => { + const keys = { 'meshcore-favorites': J(['a']), 'cs-theme-overrides': J({ big: 'x'.repeat(10000) }) }; + const env = makeEnv({ server: serverWith(1, keys), local: synced(keys, 1) }); + await env.timers.advance(0); + let reads = 0; + const get = env.ls.m.get.bind(env.ls.m); + env.ls.m.get = (k) => { if (k === 'cs-settings-sync-base') reads++; return get(k); }; + for (let i = 0; i < 50; i++) env.ls.setItem('meshcore-time-window', String(i)); + assert.strictEqual(reads, 0); + env.ls.m.get = get; + await env.timers.advance(2000); + assert.strictEqual(env.server.doc.keys['meshcore-time-window'], '49'); +}); + +test('two tabs: a change in one tab ends the hold the other tab entered', async () => { + const server = serverWith(1, KEYS); + const shared = new Map(); + const tab2 = makeEnv({ server, shared, local: synced(KEYS, 1) }); + await tab2.timers.advance(0); + const tab1 = makeEnv({ server, shared }); + await tab1.timers.advance(0); + assert.strictEqual((await tab2.t.deleteRemote()).ok, true); + assert.strictEqual(JSON.parse(shared.get('cs-settings-sync-base')).hold, true); + tab1.ls.setItem('meshcore-time-window', '60'); + assert.strictEqual(JSON.parse(shared.get('cs-settings-sync-base')).hold, false); + await tab1.timers.advance(2000); + assert.strictEqual(server.rev, 1); + assert.strictEqual(server.puts[server.puts.length - 1].baseRevision, 0); + assert.strictEqual(server.doc.keys['meshcore-time-window'], '60'); +}); + +// ── logout dialog and account section ── +const KEYS = { 'meshcore-favorites': J(['a']) }; + +test('the logout handler is registered only while active', async () => { + const off = makeEnv({ enabled: false }); + await settle(); + assert.strictEqual(off.logoutHandler, null); + const env = makeEnv({ server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + assert.strictEqual(typeof env.logoutHandler, 'function'); + await env.logout(); + assert.strictEqual(env.logoutHandler, null); +}); + +test('logout dialog: keep first and focused, channel-key text, three choices', async () => { + const env = makeEnv({ server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + let seen = null; + env.t.useDialog((opts) => { seen = plain(opts); return Promise.resolve('keep'); }); + await env.logoutHandler(); + assert.deepStrictEqual(seen.choices.map((c) => c.id), ['keep', 'remove', 'cancel']); + assert.strictEqual(seen.choices[0].label, 'Keep my settings on this device'); + assert.strictEqual(seen.choices[0].primary, true); + assert.strictEqual(seen.choices[1].label, 'Remove my settings from this device'); + assert(seen.text.join(' ').indexOf('Channel keys are never synced') !== -1); +}); + +test('logout dialog: keep pushes pending changes first and leaves local data', async () => { + const env = makeEnv({ server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + env.ls.setItem('meshcore-time-window', '60'); // still in the 2 s debounce + env.t.useDialog(() => Promise.resolve('keep')); + const h = plain(await env.logoutHandler()); + assert.deepStrictEqual(h, {}); + assert.strictEqual(env.server.puts.length, 1); + assert.strictEqual(env.server.doc.keys['meshcore-time-window'], '60'); + assert.strictEqual(env.ls.getItem('meshcore-time-window'), '60'); +}); + +test('logout dialog: remove deletes synced keys and the baseline, never channel keys', async () => { + const local = Object.assign({ 'meshcore-time-window': '60', corescope_channel_keys: '{"#x":"00"}', 'meshcore-api-key': 'k' }, synced(KEYS, 1)); + const env = makeEnv({ server: serverWith(1, KEYS), local }); + await env.timers.advance(0); + env.t.useDialog(() => Promise.resolve('remove')); + const h = await env.logoutHandler(); + assert.strictEqual(typeof h.afterLogout, 'function'); + h.afterLogout(); + for (const k of ['meshcore-favorites', 'meshcore-time-window', 'cs-settings-sync-base', 'cs-settings-sync-rev']) { + assert.strictEqual(env.ls.getItem(k), null, k + ' left behind'); + } + assert.strictEqual(env.ls.getItem('corescope_channel_keys'), '{"#x":"00"}'); + assert.strictEqual(env.ls.getItem('meshcore-api-key'), 'k'); +}); + +// M2 (final review): removing the keys alone left the theme and the +// customizer CSS applied until a reload. +test('logout dialog: remove shows the defaults at once', async () => { + const keys = { 'meshcore-theme': 'dark', 'meshcore-cb-preset': 'wong', 'cs-theme-overrides': '{"x":1}' }; + const env = makeEnv({ server: serverWith(1, keys), local: synced(keys, 1) }); + await env.timers.advance(0); + env.t.useDialog(() => Promise.resolve('remove')); + const h = await env.logoutHandler(); + const p0 = env.pipelines; + h.afterLogout(); + assert.strictEqual(env.resets, 1, 'customizer Reset All teardown not run'); + assert.strictEqual(env.pipelines, p0 + 1); + assert.strictEqual(env.presetClears, 1); + assert(env.events.some((e) => e.type === 'storage' && e.key === 'meshcore-theme' && e.newValue === 'light')); + assert.strictEqual(env.ls.getItem('cs-theme-overrides'), null); + await env.timers.advance(10000); + assert.strictEqual(env.server.puts.length, 0, 'the teardown pushed'); +}); + +test('logout dialog: remove before the customizer finished its init runs no pipeline', async () => { + const env = makeEnv({ customizerReady: false, server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + env.t.useDialog(() => Promise.resolve('remove')); + (await env.logoutHandler()).afterLogout(); + assert.strictEqual(env.resets + env.pipelines, 0); +}); + +test('logout dialog: remove after a failed push keeps the data and says so', async () => { + const server = serverWith(1, KEYS); + const env = makeEnv({ server, local: synced(KEYS, 1) }); + await env.timers.advance(0); + env.ls.setItem('meshcore-time-window', '60'); + server.fail.PUT = ['network']; + env.t.useDialog(() => Promise.resolve('remove')); + const h = await env.logoutHandler(); + h.afterLogout(); + assert.strictEqual(env.ls.getItem('meshcore-time-window'), '60'); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['a'])); + assert(env.toasts.indexOf('Your latest settings could not be saved to your account, so they stay on this device.') !== -1); +}); + +// Polish 4: unpushed changes are judged from storage, which every tab +// shares, not from this tab's dirty flag. +test('two tabs: logout remove pushes a change the other tab made seconds earlier', async () => { + const server = serverWith(1, KEYS); + const shared = new Map(); + const tab1 = makeEnv({ server, shared, local: synced(KEYS, 1) }); + await tab1.timers.advance(0); + const tab2 = makeEnv({ server, shared }); + await tab2.timers.advance(0); + tab1.ls.setItem('meshcore-time-window', '60'); // tab1 still in its 2 s debounce + tab2.t.useDialog(() => Promise.resolve('remove')); + const h = await tab2.logoutHandler(); + assert.strictEqual(server.doc.keys['meshcore-time-window'], '60'); + h.afterLogout(); + assert.strictEqual(shared.get('meshcore-time-window'), undefined); +}); + +test('two tabs: logout remove keeps the other tab change when its push fails', async () => { + const server = serverWith(1, KEYS); + const shared = new Map(); + const tab1 = makeEnv({ server, shared, local: synced(KEYS, 1) }); + await tab1.timers.advance(0); + const tab2 = makeEnv({ server, shared }); + await tab2.timers.advance(0); + tab1.ls.setItem('meshcore-time-window', '60'); + server.fail.PUT = ['network']; + tab2.t.useDialog(() => Promise.resolve('remove')); + (await tab2.logoutHandler()).afterLogout(); + assert.strictEqual(shared.get('meshcore-time-window'), '60'); + assert.strictEqual(shared.get('meshcore-favorites'), J(['a'])); + assert(tab2.toasts.indexOf('Your latest settings could not be saved to your account, so they stay on this device.') !== -1); +}); + +test('logout remove while held: nothing is pushed, the account copy stays deleted', async () => { + const server = serverWith(1, KEYS); + const env = makeEnv({ server, local: synced(KEYS, 1) }); + await env.timers.advance(0); + assert.strictEqual((await env.t.deleteRemote()).ok, true); + const puts = server.puts.length; + env.t.useDialog(() => Promise.resolve('remove')); + (await env.logoutHandler()).afterLogout(); + assert.strictEqual(server.puts.length, puts); + assert.strictEqual(server.doc, null); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), null); +}); + +// M5 (final review): a hanging connection must not hold the logout dialog. +test('logout dialog: a push that hangs counts as failed after 5 s', async () => { + for (const choice of ['remove', 'keep']) { + const env = makeEnv({ server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + holdNext(env, 'PUT'); // never released + env.ls.setItem('meshcore-time-window', '60'); + env.t.useDialog(() => Promise.resolve(choice)); + let h = null; + env.logoutHandler().then((r) => { h = r; }); + await env.timers.advance(4999); + assert.strictEqual(h, null, choice + ': resolved before the timeout'); + await env.timers.advance(1); + assert(h, choice + ': still waiting after 5 s'); + if (choice === 'keep') { assert.deepStrictEqual(plain(h), {}); continue; } + h.afterLogout(); + assert.strictEqual(env.ls.getItem('meshcore-time-window'), '60'); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['a'])); + assert(env.toasts.indexOf('Your latest settings could not be saved to your account, so they stay on this device.') !== -1); + assert(env.warnings.some((w) => w.indexOf('5 s') !== -1), 'timeout not logged'); + } +}); + +test('logout dialog: Cancel, Escape or the backdrop cancel the logout', async () => { + const env = makeEnv({ server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + for (const choice of ['cancel', null]) { + env.t.useDialog(() => Promise.resolve(choice)); + assert.deepStrictEqual(plain(await env.logoutHandler()), { cancel: true }); + } +}); + +test('account section: status, Sync now, delete with confirmation', async () => { + const env = makeEnv({ server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + const el = env.el('syncSection'); + env.api.mountSection(el); + assert(el.innerHTML.indexOf('id="syncNow"') !== -1 && el.innerHTML.indexOf('id="syncDelete"') !== -1); + assert(el.innerHTML.indexOf('Delete synced settings from my account') !== -1); + assert(env.els.syncStatus.textContent.indexOf('Last synced ') === 0, env.els.syncStatus.textContent); + const g0 = env.server.gets; + await env.els.syncNow.handlers.click(); + await settle(); + assert.strictEqual(env.server.gets, g0 + 1); + env.t.useDialog(() => Promise.resolve('cancel')); + await env.els.syncDelete.handlers.click(); + assert.strictEqual(env.server.deletes, 0); + env.t.useDialog(() => Promise.resolve('delete')); + await env.els.syncDelete.handlers.click(); + await settle(); + assert.strictEqual(env.server.deletes, 1); + assert.strictEqual(env.t.state.hold, true); + assert.strictEqual(env.els.syncMsg.textContent, 'Synced settings deleted from your account.'); + assert.strictEqual(env.els.syncStatus.textContent, 'No settings saved in your account. Your next change starts a new copy.'); + assert.strictEqual(env.ls.getItem('meshcore-favorites'), J(['a'])); +}); + +test('dialog: escaped, first choice focused, Tab trapped, Escape and backdrop dismiss, focus returns', async () => { + const env = makeEnv({ enabled: false }); + const prev = { focused: 0, focus() { this.focused++; } }; + const open = (title) => { + env.doc.activeElement = prev; + const p = env.t.showDialog({ title, text: ['t'], choices: [{ id: 'a', label: 'A', primary: true }, { id: 'b', label: 'B' }] }); + return { p, ov: env.doc.body.children[env.doc.body.children.length - 1] }; + }; + let { p, ov } = open(''); + assert(ov.innerHTML.indexOf('') === -1 && ov.innerHTML.indexOf('<img>') !== -1); + assert(ov.innerHTML.indexOf('') === -1); + assert(ov.innerHTML.indexOf('role="dialog"') !== -1 && ov.innerHTML.indexOf('aria-modal="true"') !== -1); + const [a, b] = ov.querySelectorAll(); + assert.strictEqual(env.doc.activeElement, a); + const key = (k, shift) => { + let prevented = false; + ov.handlers.keydown({ key: k, shiftKey: !!shift, preventDefault() { prevented = true; }, stopPropagation() {} }); + return prevented; + }; + assert(key('Tab', true)); + assert.strictEqual(env.doc.activeElement, b); + assert(key('Tab')); + assert.strictEqual(env.doc.activeElement, a); + key('Escape'); + assert.strictEqual(await p, null); + assert(ov.removed); + assert.strictEqual(prev.focused, 1); + ({ p, ov } = open('x')); + ov.handlers.click({ target: ov }); + assert.strictEqual(await p, null); + ({ p, ov } = open('x')); + ov.handlers.click({ target: ov.querySelectorAll()[1] }); + assert.strictEqual(await p, 'b'); +}); + +test('delete while retrying: the pending retry does not recreate the account copy', async () => { + const server = serverWith(1, KEYS); + const env = makeEnv({ server, local: synced(KEYS, 1) }); + await env.timers.advance(0); + server.fail.PUT = ['network']; + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + assert.strictEqual(env.t.state.status, 'retrying'); + const puts = server.puts.length; + const r = await env.t.deleteRemote(); + assert.strictEqual(r.ok, true); + await env.timers.advance(600000); + assert.strictEqual(server.puts.length, puts); + assert.strictEqual(env.t.state.hold, true); + assert.strictEqual(server.doc, null); +}); + +test('delete with a PUT in flight: the late answer does not undo the hold', async () => { + const server = serverWith(1, KEYS); + const env = makeEnv({ server, local: synced(KEYS, 1) }); + await env.timers.advance(0); + const release = holdNext(env, 'PUT'); + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); // PUT sent, answer held + const d = env.t.deleteRemote(); + await release(); + assert.strictEqual((await d).ok, true); + assert.strictEqual(env.t.state.hold, true, 'hold lost when the PUT answer landed'); + assert.strictEqual(env.t.state.rev, 0); + await env.timers.advance(600000); + assert.strictEqual(env.t.state.hold, true); + assert.strictEqual(server.doc, null); + assert.strictEqual(server.deletes, 1); + assert.strictEqual(JSON.parse(env.ls.getItem('cs-settings-sync-base')).hold, true); +}); + +test('a failed delete puts unsynced changes back on the retry backoff', async () => { + const server = serverWith(1, KEYS); + const env = makeEnv({ server, local: synced(KEYS, 1) }); + await env.timers.advance(0); + server.fail.PUT = ['network']; + env.ls.setItem('meshcore-time-window', '60'); + await env.timers.advance(2000); + server.fail.DELETE = [{ status: 500, data: { error: 'boom' } }]; + const r = await env.t.deleteRemote(); + assert.strictEqual(r.status, 500); + assert.strictEqual(env.t.state.hold, false); + assert.strictEqual(env.t.state.status, 'retrying'); + assert.strictEqual(env.timers.delays().length, 1, 'no retry scheduled'); + await env.timers.advance(600000); + assert.strictEqual(server.doc.keys['meshcore-time-window'], '60'); + assert.strictEqual(env.t.state.status, 'ok'); +}); + +test('delete confirmation focuses Cancel; the dialog focuses opts.focus', async () => { + const env = makeEnv({ server: serverWith(1, KEYS), local: synced(KEYS, 1) }); + await env.timers.advance(0); + env.api.mountSection(env.el('syncSection')); + let seen = null; + env.t.useDialog((opts) => { seen = plain(opts); return Promise.resolve('cancel'); }); + await env.els.syncDelete.handlers.click(); + assert.strictEqual(seen.focus, 'cancel'); + const p = env.t.showDialog(Object.assign({}, seen, { title: 'x' })); + const ov = env.doc.body.children[env.doc.body.children.length - 1]; + const btns = ov.querySelectorAll(); + assert.strictEqual(btns[0].getAttribute(), 'delete'); + assert.strictEqual(env.doc.activeElement, btns[1]); + ov.handlers.click({ target: ov }); + assert.strictEqual(await p, null); +}); + +test('status texts', async () => { + const env = makeEnv({ server: serverWith(1, {}), local: synced({}, 1) }); + await env.timers.advance(0); + const st = env.t.state; + st.status = 'retrying'; + assert.strictEqual(env.t.statusText(), 'Not synced: retrying'); + st.status = 'too-large'; st.tooLarge = ['cs-theme-overrides', 'meshcore-my-nodes']; + assert.strictEqual(env.t.statusText(), 'Not synced: your settings are larger than your account can hold. Largest: cs-theme-overrides, meshcore-my-nodes'); + st.status = 'idle'; + assert.strictEqual(env.t.statusText(), 'Not synced yet'); +}); + +(async () => { + let passed = 0, failed = 0; + for (const t of tests) { + try { await t.fn(); passed++; console.log(' ok ' + t.name); } + catch (e) { failed++; console.log(' FAIL ' + t.name + ': ' + (e && e.stack || e)); } + } + console.log('\n' + passed + ' passed, ' + failed + ' failed'); + process.exit(failed ? 1 : 0); +})(); diff --git a/tests/unit/test-user-management-ui.js b/tests/unit/test-user-management-ui.js index 2725165d5..0e1dcd686 100644 --- a/tests/unit/test-user-management-ui.js +++ b/tests/unit/test-user-management-ui.js @@ -25,6 +25,16 @@ function loadEscapeHtml() { return vm.runInNewContext('(' + m[0].replace(/^function escapeHtml/, 'function') + ')'); } +// CSAuth.say and CSAuth.errText come from the real auth.js, bound to ctx +// (say reads ctx.document). +function loadAuthHelpers(ctx) { + const src = fs.readFileSync(path.join(ROOT, 'public/auth.js'), 'utf8'); + const say = src.match(/function say\(id, text, ok\) \{[\s\S]*?\n \}/); + const err = src.match(/function errText\(r\) \{.*\}/); + assert(say && err, 'say/errText not found in auth.js'); + return { say: vm.runInContext('(' + say[0] + ')', ctx), errText: vm.runInContext('(' + err[0] + ')', ctx) }; +} + function makeEnv(routes) { const els = {}; const events = []; @@ -209,6 +219,64 @@ test('a refused logout keeps the user and the view', async () => { assert.strictEqual(env.loc.hash, '#/account'); }); +test('logout handler: cancel keeps the session and posts nothing', async () => { + const env = makeEnv((u) => u === '/api/auth/me' ? { status: 200, body: ME } : { status: 200, body: { ok: true } }); + await env.win.CSAuth.ready(); + env.calls.length = 0; + env.win.CSAuth.setLogoutHandler(() => Promise.resolve({ cancel: true })); + const r = await env.win.CSAuth.logout('#/account/login'); + assert.strictEqual(r.cancelled, true); + assert.strictEqual(r.ok, false); + assert.strictEqual(env.calls.length, 0); + assert.strictEqual(env.win.CS_USER.displayName, 'Ann'); + assert.strictEqual(env.loc.hash, '#/account'); +}); + +test('logout handler: afterLogout runs after the POST, before the user is cleared', async () => { + const env = makeEnv((u) => u === '/api/auth/me' ? { status: 200, body: ME } : { status: 200, body: { ok: true } }); + await env.win.CSAuth.ready(); + env.calls.length = 0; + let userAtAfter = 'not called', postedBefore = false; + env.win.CSAuth.setLogoutHandler(() => Promise.resolve({ afterLogout() { userAtAfter = env.win.CS_USER; postedBefore = env.calls.length === 1; } })); + const r = await env.win.CSAuth.logout('#/account/login'); + assert.strictEqual(r.ok, true); + assert.strictEqual(postedBefore, true); + assert.strictEqual(userAtAfter.displayName, 'Ann'); + assert.strictEqual(env.win.CS_USER, null); +}); + +// M4 (final review): a second Log out click while the dialog is open must +// not stack a second dialog or send a second POST. +test('logout: a second call while one is running opens no second dialog', async () => { + const env = makeEnv((u) => u === '/api/auth/me' ? { status: 200, body: ME } : { status: 200, body: { ok: true } }); + await env.win.CSAuth.ready(); + env.calls.length = 0; + let dialogs = 0, choose; + env.win.CSAuth.setLogoutHandler(() => { dialogs++; return new Promise((r) => { choose = r; }); }); + const first = env.win.CSAuth.logout('#/account/login'); + const second = env.win.CSAuth.logout('#/account/login'); + await new Promise((r) => setTimeout(r, 0)); + assert.strictEqual(dialogs, 1); + assert.strictEqual((await second).cancelled, true); + choose({}); + assert.strictEqual((await first).ok, true); + assert.strictEqual(env.calls.filter((c) => c.url === '/api/auth/logout').length, 1); + // Finished: the next logout runs again. + const third = env.win.CSAuth.logout('#/account/login'); + assert.strictEqual(dialogs, 2); + choose({ cancel: true }); + assert.strictEqual((await third).cancelled, true); +}); + +test('logout handler: a refused POST does not run afterLogout', async () => { + const env = makeEnv((u) => u === '/api/auth/me' ? { status: 200, body: ME } : { status: 403, body: { error: 'no' } }); + await env.win.CSAuth.ready(); + let ran = false; + env.win.CSAuth.setLogoutHandler(() => Promise.resolve({ afterLogout() { ran = true; } })); + await env.win.CSAuth.logout('#/account/login'); + assert.strictEqual(ran, false); +}); + console.log('mobile nav account entry'); // Slice the real route tables and builders out of the two files; markers @@ -257,7 +325,7 @@ NAVS.forEach((n) => { console.log('account.js'); -function loadAccount(hash, routes) { +function loadAccount(hash, routes, extraWin) { const els = {}; const mk = () => { const el = { value: '', textContent: '', handlers: {}, cls: {}, html: '' }; @@ -285,11 +353,12 @@ function loadAccount(hash, routes) { ready() { return Promise.resolve(); }, isEnabled() { return true; }, notify() {}, refreshMe() { return Promise.resolve(); }, }; - const win = { CSAuth, addEventListener(t, fn) { listeners[t] = fn; } }; + const win = Object.assign({ CSAuth, addEventListener(t, fn) { listeners[t] = fn; } }, extraWin || {}); const ctx = { window: win, document: doc, CSAuth, location: loc, URLSearchParams, Promise, String, Object, history: { replaceState(a, b, h) { replaced.push(h); loc.hash = h; } }, escapeHtml: loadEscapeHtml(), registerPage(n, m) { pages[n] = m; }, console }; vm.createContext(ctx); + Object.assign(CSAuth, loadAuthHelpers(ctx)); vm.runInContext(fs.readFileSync(path.join(ROOT, 'public/account.js'), 'utf8'), ctx); return { setRequest(f) { override = f; }, doc, t: ctx.window.CSAccount._test, els, calls, loc, user, pages, logouts, replaced, setLogout(r) { logoutResult = r; }, @@ -483,6 +552,28 @@ test('profile view: a refused logout shows the error next to the button', async assert.strictEqual(env.els.logoutMsg.textContent, 'Network error, try again.'); }); +test('profile view: a cancelled logout shows no message', async () => { + const env = loadAccount('#/account', () => ({ ok: true, status: 200, data: [] })); + env.user.current = { id: 1, email: 'a@b.c', displayName: 'Ann', role: 'user' }; + env.setLogout({ ok: false, cancelled: true, status: 0, data: {} }); + env.t.views.profile({ set innerHTML(v) {} }); + await env.els.accountPageLogout.handlers.click(); + assert.strictEqual(env.doc.getElementById('logoutMsg').textContent, ''); +}); + +test('profile view mounts the settings sync section only when the module is loaded', () => { + const mounted = []; + const env = loadAccount('#/account', () => ({ ok: true, status: 200, data: [] }), { CSSettingsSync: { mountSection(el) { mounted.push(el); } } }); + env.user.current = { id: 1, email: 'a@b.c', displayName: 'Ann', role: 'user' }; + const app = { innerHTML: '' }; + env.t.views.profile(app); + assert(app.innerHTML.indexOf('

Settings sync

') !== -1); + assert.strictEqual(mounted.length, 1); + assert.strictEqual(mounted[0], env.els.syncSection); + const without = loadAccount('#/account', () => ({})); + assert.strictEqual(without.t.profileHtml({ email: 'a', role: 'user', displayName: 'A' }).indexOf('syncSection'), -1); +}); + test('profile view: Log out button for everyone, Manage users link for admins only', () => { const env = loadAccount('#/account', () => ({})); const user = env.t.profileHtml({ email: 'a', role: 'user', displayName: 'A' }); @@ -592,6 +683,7 @@ function loadAdmin(hash, routes) { confirm() { return true; }, debounce(fn) { return fn; }, escapeHtml: loadEscapeHtml(), registerPage(n, m) { pages[n] = m; }, console }; vm.createContext(ctx); + Object.assign(CSAuth, loadAuthHelpers(ctx)); vm.runInContext(fs.readFileSync(path.join(ROOT, 'public/admin-users.js'), 'utf8'), ctx); const app = { innerHTML: '', querySelector() { return els.umPage || (els.umPage = mk('umPage')); } }; return { t: ctx.window.CSAdminUsers._test, pages, els, calls, replaced, loc, app, refreshed: () => refreshed, From 7309bdb5a910eefb025cc63d4b7f4acd04ae4c7d Mon Sep 17 00:00:00 2001 From: dborup <3627142+dborup@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:34:31 +0200 Subject: [PATCH 099/151] fix(store): index a transmission once per relay key in byPathHop (#2117) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relates to #2108 ## Problem `traffic_share_score` grows with server uptime until it is far above reality, and many relays end up clamped at 1.0. The score is the number of non-advert entries in `byPathHop[pubkey]` divided by the number of non-advert transmissions. `indexResolvedPathHops` runs once per **observation**, on the live-ingest path (`IngestNewFromDB`) and on the late-observation path (`IngestNewObservations`). `addResolvedPubkeysToPathHopIndex` only de-duplicates within one call, so every further observation through the same relay appends the transmission to that relay's bucket again. The denominator counts each transmission once. After a restart the values look right, because `buildPathHopIndex` → `retainResolvedPathHops` de-duplicates by `*StoreTx`. They then drift upwards as live observations arrive. None of the affected functions has changed on `master` since the diagnosis against `415362c`. This branch is based on `000d9ab`. ## Fix 1. **Idempotent insert, per (transmission, relay key).** `addResolvedPubkeysToPathHopIndex` keeps a side map `pathHopResolved map[*StoreTx][]string`. It holds the resolved keys each transmission is already indexed under and skips those. - Keys are compared **exactly**, so no collision can ever drop an entry. - A later observation through a **new** relay still adds that relay, once. - `StoreTx` is unchanged. 2. **Interned keys.** `pathHopKeys map[string]string` keeps one shared copy of each resolved key. The record then holds a 16-byte string header per entry, and not the string each observation's resolve allocated: a fresh `json.Unmarshal` string per persisted observation on `Load`, and a `strings.ToLower` result on the live paths. The `byPathHop` key is the same shared copy. 3. **Eviction and rebuild.** - `evictStaleInternal` drops the record of evicted transmissions, plus any interned key whose bucket it deletes. - `retainResolvedPathHops` keeps the records of live transmissions only and drops interned keys whose bucket was not carried over. When a rebuild starts from an empty index, it clears both. 4. **Defence in depth.** The single (`GetRepeaterUsefulnessScore`), batch (`GetRepeaterNodeStatsBatch`) and bulk (`computeRepeaterUsefulnessScoreMap`) scores count **distinct** non-advert transmissions per bucket, via `countDistinctNonAdvert`. - IDs are collected in a reused slice. - A bucket in ascending ID order is already distinct; only out-of-order buckets are sorted. - The bulk pass does this for full-pubkey keys only. Raw-hop buckets get one entry per transmission from `addTxToPathHopIndex`. 5. **Cache side effect.** A repeated observation through known relays no longer mutates `byPathHop`, so it no longer drops the batch relay-stats cache, as the cache contract from `1164` intends. An observation through a new relay still drops it. The other `byPathHop` consumers already de-duplicate by `tx.ID` or read only raw prefix keys: `GetNodeHopAnalytics`, `computeMultiByteCapability`, `handleNodePaths`, `computeRepeaterRelayInfoMap` and the relay info in `GetRepeaterNodeStatsBatch`. Two tests pin that they ignore duplicate entries. ## Exact keys vs. a 64-bit fingerprint The first version of this fix stored a 64-bit FNV-1a hash per key. As requested, I measured both, plus exact keys without interning, on the real `addResolvedPubkeysToPathHopIndex`. **Keys per transmission.** In the e2e fixture, the union of resolved relay keys over all observations of one transmission has a mean of 5.4 and a maximum of 17. The protocol limit is 64 path bytes (`MAX_PATH_SIZE`), i.e. 64 / hash size hops. **Memory** retained by the record, per transmission. `BenchmarkPathHopRecordMemory_2108`: 100K transmissions, keys from 4,000 relays, each transmission heard twice with freshly allocated keys. The record map and the interned keys are included. | keys per tx | exact, interned (this PR) | 64-bit fingerprint | exact, not interned | |---|---|---|---| | 2 | 88 B | 68 B | 207 B | | 5 | 136 B | 101 B | 447 B | | 17 | 344 B | 197 B | 1,423 B | - At realistic key counts, exact keys cost 20–35 B per transmission more than the fingerprint. That is about 3.5 MB per 100K transmissions, and small next to what the store already charges per transmission (`storeTxBaseBytes` alone is 384 B). - Without interning, exact keys would cost 3–4× more, and most of that would be held during every `Load`. Interning is what makes exact keys affordable. **Time.** I ran the variants interleaved, 3 rounds each, median ns/op on an Apple M4. The record step is not measurable inside the full call. The isolated record step was faster with exact keys in a separate micro-benchmark, because comparing a handful of 64-char strings is cheaper than hashing each one. | benchmark | exact, interned | fingerprint | exact, not interned | |---|---|---|---| | late observation, 20K txs | 312 | 358 | 331 | | late observation, 100K txs | 760 | 836 | 758 | | `IngestNewObservations` (SQL + resolve + index) | 531 µs | 497 µs | 527 µs | **Collision probability of the fingerprint.** For k distinct keys in one transmission it is about k(k−1)/2 / 2^64: | k | per transmission | per 10^9 transmissions | |---|---|---| | 2 | 5.4e-20 | 5.4e-11 | | 5 | 5.4e-19 | 5.4e-10 | | 17 | 7.4e-18 | 7.4e-9 | | 64 | 1.1e-16 | 1.1e-7 | For random keys this is negligible. FNV-1a is unkeyed, though, so two relay keys that collide could be found deliberately (a birthday search over about 2^32 key pairs). A collision would leave a transmission out of one relay's bucket. **Decision.** Exact keys are cheap enough once interned: the same speed within noise, and a few tens of bytes per transmission. They remove the question of collision-driven omissions entirely, so this PR uses exact keys and drops the fingerprint. The two new maps (`pathHopResolved`, `pathHopKeys`) are not added to `trackedBytes`, so the `maxMemoryMB` trigger undercounts by roughly the per-transmission figures above. Both are bounded by live state (eviction and rebuild prune them), so this is a steady proportional undercount rather than a leak. ## Tests **New: `cmd/server/pathhop_dedupe_2108_test.go`.** Behaviour tests that use only existing API. Each one runs against the real SQLite schema through `Load`, `IngestNewFromDB` and `IngestNewObservations` where noted. | test | covers | on `master` | |---|---|---| | `TestPathHopIndexOncePerTx_LateObservations_2108` | 1 + 10 late observations through the same relays: one entry per relay | **fails** (11 entries) | | `TestPathHopIndexOncePerTx_LiveIngestBatch_2108` | 11 observations in one `IngestNewFromDB` batch | **fails** (11) | | `TestPathHopIndexOncePerTx_AfterLoad_2108` | `Load` + rebuild, then one live observation | **fails** (2) | | `TestPathHopIndexAddsNewRelayFromLaterObservation_2108` | a later observation through a **new** relay adds it exactly once, and its share becomes correct | **fails** (2) | | `TestTrafficShareStableAcrossLateObservations_2108` | single, batch and bulk scores agree, equal the definition, stay put over rounds of late observations, and equal a fresh `Load` of the same data | **fails** (all four relays at 1.0, want 0.4–0.5) | | `TestPathHopIndexSizeBoundedByTransmissions_2108` | the index size does not grow with observations | **fails** | | `TestRelayStatsCacheAcrossRepeatedObservations_2108` | a repeated observation keeps the relay-stats cache, and the kept cache equals a fresh compute; a new relay drops it, and the next read sees the relay | **fails** | | `TestAddResolvedPubkeysToPathHopIndex_PerRelayIdempotent_2108` | the helper's return value and cache invalidation per (tx, relay), in any key order | **fails** | | `TestTrafficShareCountsDistinctTransmissions_2108` | all three scores count distinct transmissions in a duplicated bucket | **fails** | | `TestPathHopConsumersIgnoreDuplicateEntries_2108` | pin of the consumer audit | passes (by design) | | `TestMultiByteCapabilityIgnoresDuplicateEntries_2108` | pin of the consumer audit | passes (by design) | **New: `cmd/server/pathhop_record_2108_test.go`.** These tests use the new symbols, so they do not compile against `master`. - `TestCountDistinctNonAdvert_2108`: ascending, descending, interleaved duplicates, adverts, nils, untyped. - `TestPathHopResolvedRecordBoundedAndEvicted_2108`: the record stays at 2 keys after 11 observations. Eviction removes the record and every entry. A survivor's next observation adds nothing. Evicting everything leaves no record, no interned key and no bucket. - `TestPathHopResolvedRecordAcrossRebuild_2108`: a rebuild keeps the records of live transmissions, so the next observation adds nothing. It drops the record of a removed transmission and the interned key only that transmission used. - `TestPathHopResolvedRecordClearedWithEmptyIndex_2108`: a rebuild from an empty index clears the record and the interned keys, and the next observation puts the transmission back. - `TestPathHopResolvedRecordInternsKeys_2108`: record entries and the `byPathHop` key share one copy, even though every observation passes freshly allocated keys. **Changed: `pathhop_eviction_1908_test.go`.** It built its duplicate entries by repeating `indexResolvedPathHops`, which no longer duplicates. It now seeds the duplicates directly, so the `1908` sweep is still tested against buckets that hold one transmission several times. The expected buckets are unchanged. **Changed: `db_test.go`.** `setupTestDB` takes `testing.TB`, so the SQL benchmark can use it. **Mutants.** I applied each mutant on its own to this branch. All 14 are killed: | mutant | killed by | |---|---| | record never consulted | the OncePerTx, stable-share, size, cache and record tests | | dedupe per transmission instead of per relay | `AddsNewRelayFromLaterObservation`, `RelayStatsCacheAcrossRepeatedObservations`, `PerRelayIdempotent` | | eviction keeps the record | `RecordBoundedAndEvicted` | | rebuild keeps records of removed transmissions | `RecordAcrossRebuild` | | rebuild from an empty index keeps the record | `RecordClearedWithEmptyIndex` | | cache dropped on every call | `RelayStatsCacheAcrossRepeatedObservations`, `PerRelayIdempotent`, existing `NoMutation_PreservesCache` | | cache kept although `byPathHop` changed | `RelayStatsCacheAcrossRepeatedObservations`, `PerRelayIdempotent`, existing `InvalidatesRelayStatsCache` | | single score counts entries | `TrafficShareCountsDistinctTransmissions` | | batch score counts entries | `TrafficShareCountsDistinctTransmissions` | | bulk score counts entries | `TrafficShareCountsDistinctTransmissions` | | distinct count trusts any bucket order | `TrafficShareCountsDistinctTransmissions`, `CountDistinctNonAdvert` | | keys not interned | `RecordInternsKeys` | | eviction keeps interned keys | `RecordBoundedAndEvicted` | | rebuild keeps interned keys of dropped buckets | `RecordAcrossRebuild` | **Commands run:** - `gofmt -l` on all tracked Go files: clean. - `go vet ./...` in all 14 modules: clean. - `cd cmd/server && go test -race ./...`: pass. - `cd cmd/ingestor && go test ./...`: pass. - `sh test-all.sh`: 184 of 186 suites pass locally. The other two, `test-issue-1956-release-routing.js` and `test-preflight-xss-gate.js`, shell out to scripts that need bash ≥ 4 (`mapfile`). They fail under macOS's bash 3.2 regardless of this change. This PR touches no frontend or script files. ## Benchmark: `master` vs. this branch I ran `master`'s sources (`000d9ab`) and this branch interleaved, 5 rounds, with the same benchmark files. Medians on an Apple M4. | benchmark | `master` | this PR | change | |---|---|---|---| | late observation through known relays, 20K txs | 446 ns, 38 B/op | 441 ns, 0 B/op | within noise | | late observation through known relays, 100K txs | 710 ns, 69 B/op | 755 ns, 0 B/op | within noise (runs overlap) | | … index size afterwards, entries/tx (20K / 100K) | 24.99 / 8.99, still growing | 4.99 / 4.99 | bounded | | `IngestNewObservations`, one observation for each of 20 txs | 532 µs | 513 µs | within noise | | bulk score pass, clean index, 20K, ingest order | 246 µs | 316 µs | +28 % | | bulk score pass, clean index, 100K, ingest order | 3.03 ms | 4.30 ms | +42 % | | bulk score pass, clean index, 20K, reversed buckets | 257 µs | 467 µs | +82 % | | bulk score pass, clean index, 100K, reversed buckets | 3.38 ms | 4.79 ms | +42 % | | bulk score pass after 11 observations per tx, 20K | 663 µs | 284 µs | −57 % | | bulk score pass after 11 observations per tx, 100K | 4.72 ms | 3.19 ms | −32 % | - On an index that is clean on both builds, the distinct count makes the bulk pass slower. That pass runs on the cache-miss path, which the background recomputer refreshes every 5 minutes by default. - On the index a live server actually holds without this fix, `master` walks every accumulated duplicate, so the real-world pass is faster after the fix. The gap grows with uptime. - The late-observation step no longer allocates. On `master` its index grows with every observation. Benchmarks: `BenchmarkLateObservationIndex_2108`, `BenchmarkIngestNewObservations_2108`, `BenchmarkTrafficShareScoreMap_2108` and `BenchmarkPathHopRecordMemory_2108`. ## Production motivation We have run this fix on two production instances. Before the fix, the summed `traffic_share_score` grew past 180 and many relays sat at the 1.0 clamp; with the fix no node reaches `≥ 0.999`. A controlled 12 h A/B run makes the drift explicit. Two servers read one identical database — one on this `master` base, one with the fix — alongside a reference server that freshly loads the same database (a fresh load is correct, because the rebuild dedupes). The unfixed server drifted to 9 relays at the 1.0 clamp and up to 0.95 absolute error per node against the reference; the fixed server stayed within 0.05 of the reference for every node, with no relay at the clamp. A smaller residual rise remains and is a separate cause (startup-vs-live hop resolution); it is deliberately left for a follow-up so its effect stays measurable. ## Out of scope That remaining slow rise comes from a separate cause: live ingest resolves some hops that the startup load does not. This PR deliberately leaves that drift alone, so its effect stays measurable. The fix will follow in a separate PR. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- cmd/server/pathhop_dedupe_2108_bench_test.go | 162 +++++ cmd/server/pathhop_dedupe_2108_test.go | 599 +++++++++++++++++++ cmd/server/pathhop_eviction_1908_test.go | 8 +- cmd/server/pathhop_record_2108_test.go | 294 +++++++++ cmd/server/repeater_enrich_bulk.go | 21 +- cmd/server/repeater_usefulness.go | 85 ++- cmd/server/store.go | 105 +++- 7 files changed, 1220 insertions(+), 54 deletions(-) create mode 100644 cmd/server/pathhop_dedupe_2108_bench_test.go create mode 100644 cmd/server/pathhop_dedupe_2108_test.go create mode 100644 cmd/server/pathhop_record_2108_test.go diff --git a/cmd/server/pathhop_dedupe_2108_bench_test.go b/cmd/server/pathhop_dedupe_2108_bench_test.go new file mode 100644 index 000000000..bd53037aa --- /dev/null +++ b/cmd/server/pathhop_dedupe_2108_bench_test.go @@ -0,0 +1,162 @@ +package main + +import ( + "fmt" + "io" + "log" + "slices" + "testing" + "time" +) + +// phdBenchStore: n non-advert transmissions one second apart, each with 3 +// raw one-byte hops of which 2 resolve to one of 4000 relays (the shape of +// a relay-heavy store), indexed once as by their first observation. +// resolve returns the keys a further observation of transmission j resolves +// to, freshly allocated as every ingest path allocates them. +func phdBenchStore(n int) (store *PacketStore, resolve func(j int) []string) { + start := time.Now().UTC().Add(-time.Duration(n+10) * time.Second) + store = makeTestStore(0, start, 0) + store.byPathHop = make(map[string][]*StoreTx) + hopsSeen := map[string]bool{} + relays := func(i int) (int, int) { return i % 2000, 2000 + (5000+i*31)%2000 } + resolve = func(j int) []string { + r1, r2 := relays(j) + return []string{fmt.Sprintf("%02x%062x", r1%256, r1), fmt.Sprintf("%02x%062x", r2%256, r2)} + } + for i := 0; i < n; i++ { + r1, r2 := relays(i) + pt := 2 + tx := &StoreTx{ + ID: i + 1, + Hash: fmt.Sprintf("late%07d", i), + FirstSeen: start.Add(time.Duration(i) * time.Second).Format(time.RFC3339), + PathJSON: fmt.Sprintf(`["%02x","%02x","%02x"]`, r1%256, r2%256, (i*13+5)%256), + PayloadType: &pt, + } + store.packets = append(store.packets, tx) + store.byHash[tx.Hash] = tx + store.byTxID[tx.ID] = tx + store.byPayloadType[pt] = append(store.byPayloadType[pt], tx) + addTxToPathHopIndex(store.byPathHop, tx) + store.indexResolvedPathHops(tx, resolve(i), hopsSeen) + } + return store, resolve +} + +func phdEntriesPerTx(store *PacketStore) float64 { + entries := 0 + for _, list := range store.byPathHop { + entries += len(list) + } + return float64(entries) / float64(len(store.packets)) +} + +// BenchmarkLateObservationIndex_2108 measures the store side of +// IngestNewObservations for one late observation through the same relays +// (indexResolvedPathHops under the write lock, as the ingest loop calls it; +// the keys are resolved, so allocated, outside the timer as the resolver +// allocates them before this call) and reports how many byPathHop entries +// a transmission ends up with. Without the dedupe that number grows with +// every observation; with it, it stays at 3 raw + 2 resolved. +func BenchmarkLateObservationIndex_2108(b *testing.B) { + for _, n := range []int{20000, 100000} { + b.Run(fmt.Sprintf("txs=%d", n), func(b *testing.B) { + store, resolve := phdBenchStore(n) + hopsSeen := map[string]bool{} + const batch = 1024 + pks := make([][]string, batch) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + j := (i * 7919) % n // spread late observations over the store + if i%batch == 0 { + b.StopTimer() + for k := range pks { + pks[k] = resolve(((i + k) * 7919) % n) + } + b.StartTimer() + } + store.mu.Lock() + store.indexResolvedPathHops(store.packets[j], pks[i%batch], hopsSeen) + store.mu.Unlock() + } + b.StopTimer() + b.ReportMetric(phdEntriesPerTx(store), "pathhop-entries/tx") + }) + } +} + +// BenchmarkIngestNewObservations_2108 drives the real late-observation +// ingest (SQL scan + resolve + index) for a batch of one new observation per +// transmission through the same relays. +func BenchmarkIngestNewObservations_2108(b *testing.B) { + prev := log.Writer() + log.SetOutput(io.Discard) + defer log.SetOutput(prev) + db := setupTestDB(b) + defer db.conn.Close() + const observers = 512 + phdSeed(b, db, observers) + base := time.Now().UTC().Add(-30 * time.Minute) + for i := range phdPaths { + phdInsertTx(b, db, i, base.Add(time.Duration(i)*time.Second)) + phdInsertObs(b, db, i, 1, base.Add(time.Duration(i*1000+1)*time.Second), false) + } + store := phdLoadedStore(b, db) + store.IngestNewFromDB(0, 1000) + b.ReportAllocs() + b.ResetTimer() + for it := 0; it < b.N; it++ { + b.StopTimer() + o := 2 + it%(observers-1) + since := phdMaxObsID(b, db) + for i := range phdPaths { + phdInsertObs(b, db, i, o, base.Add(time.Duration(i*1000+o)*time.Second), false) + } + b.StartTimer() + store.IngestNewObservations(since, 10000) + } +} + +// BenchmarkTrafficShareScoreMap_2108 measures the bulk traffic-share pass +// (cache-miss path of GetRepeaterUsefulnessScoreMap) on realistic sizes. +// obs is how many observations of each transmission were indexed through +// the real indexResolvedPathHops: with obs=1 the index is clean on any +// build; with obs=11 it is what a store holds after 11 observers heard each +// transmission (clean with the dedupe, 10 extra entries per resolved key +// without it). order=reversed reverses every bucket, as when background +// chunks load older transmissions after newer ones: no bucket is in +// ascending ID order, so every full-key bucket takes the sort path. +func BenchmarkTrafficShareScoreMap_2108(b *testing.B) { + for _, c := range []struct { + n, obs int + reversed bool + }{{20000, 1, false}, {100000, 1, false}, {20000, 1, true}, {100000, 1, true}, {20000, 11, false}, {100000, 11, false}} { + order := "ingest" + if c.reversed { + order = "reversed" + } + b.Run(fmt.Sprintf("txs=%d/obs=%d/order=%s", c.n, c.obs, order), func(b *testing.B) { + store, resolve := phdBenchStore(c.n) + hopsSeen := map[string]bool{} + for o := 1; o < c.obs; o++ { + for j, tx := range store.packets { + store.indexResolvedPathHops(tx, resolve(j), hopsSeen) + } + } + if c.reversed { + for _, list := range store.byPathHop { + slices.Reverse(list) + } + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + store.computeRepeaterUsefulnessScoreMap() + } + b.StopTimer() + b.ReportMetric(phdEntriesPerTx(store), "pathhop-entries/tx") + }) + } +} diff --git a/cmd/server/pathhop_dedupe_2108_test.go b/cmd/server/pathhop_dedupe_2108_test.go new file mode 100644 index 000000000..b77b2fe5c --- /dev/null +++ b/cmd/server/pathhop_dedupe_2108_test.go @@ -0,0 +1,599 @@ +package main + +import ( + "fmt" + "math" + "reflect" + "strings" + "testing" + "time" +) + +// Issue #2108: traffic_share_score grew with server uptime. The score is +// |non-advert txs in byPathHop[pubkey]| / |non-advert txs in byPayloadType|, +// but every observation of a transmission that resolved to the same relays +// appended the transmission to the relay's byPathHop bucket again, while the +// denominator counts it once. A rebuild of the index (retainResolvedPathHops +// dedups by *StoreTx) made values look sane right after a restart; they then +// drifted upwards with live observations. + +// Four relays with distinct first bytes, so a raw 1-byte hop resolves to +// exactly one of them (unique prefix) on every ingest path. +var phdRelays = []string{ + "a1" + strings.Repeat("11", 31), + "b2" + strings.Repeat("22", 31), + "c3" + strings.Repeat("33", 31), + "d4" + strings.Repeat("44", 31), +} + +// phdPaths is the fixed transmission set: indexes into phdRelays. The +// longest path has 3 hops, so the sum of all relays' share is at most 3. +var phdPaths = [][]int{ + {0, 1, 2}, {1, 2}, {3}, {0, 3}, {2}, + {0, 1}, {1}, {2, 3}, {0}, {1, 3}, + {0, 1, 2}, {3}, {2, 1}, {0, 2}, {1}, + {3, 2, 0}, {2}, {0, 1}, {1, 2, 3}, {3}, +} + +const ( + phdMaxPathLen = 3 + phdLateObs = 10 // extra observations per transmission + phdObserverCount = 1 + phdLateObs +) + +func phdRawHopsJSON(path []int) string { + hops := make([]string, len(path)) + for i, r := range path { + hops[i] = `"` + strings.ToUpper(phdRelays[r][:2]) + `"` + } + return "[" + strings.Join(hops, ",") + "]" +} + +func phdResolvedJSON(path []int) string { + pks := make([]string, len(path)) + for i, r := range path { + pks[i] = `"` + phdRelays[r] + `"` + } + return "[" + strings.Join(pks, ",") + "]" +} + +// phdSeed creates the relays (as repeater nodes) and the observers. +func phdSeed(t testing.TB, db *DB, observers int) { + t.Helper() + ts := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339) + for i, pk := range phdRelays { + phdExec(t, db, `INSERT INTO nodes (public_key, name, role, last_seen, first_seen, advert_count) VALUES (?, ?, 'repeater', ?, '2026-01-01', 1)`, + pk, fmt.Sprintf("phdRelay-%d", i), ts) + } + for i := 1; i <= observers; i++ { + phdExec(t, db, `INSERT INTO observers (rowid, id, name, iata) VALUES (?, ?, ?, 'TST')`, + i, fmt.Sprintf("phdObs-%02d", i), fmt.Sprintf("Observer %d", i)) + } +} + +func phdExec(t testing.TB, db *DB, q string, args ...any) { + t.Helper() + if _, err := db.conn.Exec(q, args...); err != nil { + t.Fatalf("%s: %v", q, err) + } +} + +// phdInsertTx inserts transmission i of phdPaths (non-advert TXT_MSG, flood). +func phdInsertTx(t testing.TB, db *DB, i int, firstSeen time.Time) { + t.Helper() + phdExec(t, db, `INSERT INTO transmissions (id, raw_hex, hash, first_seen, route_type, payload_type, decoded_json) + VALUES (?, 'CAFE', ?, ?, 1, 2, '{"type":"TXT_MSG"}')`, i+1, fmt.Sprintf("phdTx-%03d", i), firstSeen.Format(time.RFC3339)) +} + +// phdInsertObs inserts one observation of transmission i through its path, +// heard by observer obsIdx. persisted controls resolved_path (the ingestor +// path) vs NULL (resolved fresh by the server on ingest). +func phdInsertObs(t testing.TB, db *DB, i, obsIdx int, ts time.Time, persisted bool) { + t.Helper() + phdInsertObsVia(t, db, i, phdPaths[i], obsIdx, ts, persisted) +} + +// phdInsertObsVia inserts one observation of transmission i heard through +// path (indexes into phdRelays), which need not be the path of phdPaths[i]. +func phdInsertObsVia(t testing.TB, db *DB, i int, path []int, obsIdx int, ts time.Time, persisted bool) { + t.Helper() + var rp any + if persisted { + rp = phdResolvedJSON(path) + } + phdExec(t, db, `INSERT INTO observations (transmission_id, observer_idx, snr, rssi, path_json, timestamp, resolved_path) + VALUES (?, ?, 5, -90, ?, ?, ?)`, i+1, obsIdx, phdRawHopsJSON(path), ts.Unix(), rp) +} + +func phdMaxObsID(t testing.TB, db *DB) int { + t.Helper() + var id int + if err := db.conn.QueryRow(`SELECT COALESCE(MAX(id), 0) FROM observations`).Scan(&id); err != nil { + t.Fatal(err) + } + return id +} + +func phdLoadedStore(t testing.TB, db *DB) *PacketStore { + t.Helper() + store := NewPacketStore(db, nil) + if err := store.Load(); err != nil { + t.Fatal(err) + } + if !store.WaitIndexesReady(5 * time.Second) { + t.Fatal("indexes not ready") + } + return store +} + +func phdCountTxIn(store *PacketStore, key string, tx *StoreTx) int { + store.mu.RLock() + defer store.mu.RUnlock() + n := 0 + for _, t := range store.byPathHop[key] { + if t == tx { + n++ + } + } + return n +} + +func phdAssertOncePerRelay(t *testing.T, store *PacketStore, i int) { + t.Helper() + tx := store.byTxID[i+1] + if tx == nil { + t.Fatalf("tx %d not in store", i+1) + } + if got := len(tx.Observations); got != phdObserverCount { + t.Fatalf("fixture: tx %d has %d observations, want %d", tx.ID, got, phdObserverCount) + } + for _, r := range phdPaths[i] { + if n := phdCountTxIn(store, phdRelays[r], tx); n != 1 { + t.Errorf("tx %d is in byPathHop[%s…] %d times, want exactly 1", tx.ID, phdRelays[r][:8], n) + } + } +} + +// Late-observation path: the transmission is ingested with its first +// observation, then 10 more observers hear it through the same relays. +func TestPathHopIndexOncePerTx_LateObservations_2108(t *testing.T) { + db := setupTestDB(t) + defer db.conn.Close() + phdSeed(t, db, phdObserverCount) + store := phdLoadedStore(t, db) + + now := time.Now().UTC().Add(-10 * time.Minute) + phdInsertTx(t, db, 0, now) + phdInsertObs(t, db, 0, 1, now, false) + store.IngestNewFromDB(0, 100) + for _, r := range phdPaths[0] { + if n := phdCountTxIn(store, phdRelays[r], store.byTxID[1]); n != 1 { + t.Fatalf("fixture: first observation must index tx once under %s…, got %d", phdRelays[r][:8], n) + } + } + + for k := 1; k <= phdLateObs; k++ { + since := phdMaxObsID(t, db) + phdInsertObs(t, db, 0, 1+k, now.Add(time.Duration(k)*time.Second), false) + store.IngestNewObservations(since, 100) + } + phdAssertOncePerRelay(t, store, 0) +} + +// Live-ingest path: the transmission arrives together with all of its +// observations in one IngestNewFromDB batch. +func TestPathHopIndexOncePerTx_LiveIngestBatch_2108(t *testing.T) { + db := setupTestDB(t) + defer db.conn.Close() + phdSeed(t, db, phdObserverCount) + store := phdLoadedStore(t, db) + + now := time.Now().UTC().Add(-10 * time.Minute) + phdInsertTx(t, db, 0, now) + for o := 1; o <= phdObserverCount; o++ { + phdInsertObs(t, db, 0, o, now.Add(time.Duration(o)*time.Second), false) + } + store.IngestNewFromDB(0, 100) + phdAssertOncePerRelay(t, store, 0) +} + +// Restart: Load indexes every persisted observation and rebuilds the index; +// a live observation after that must not add the transmission again. +func TestPathHopIndexOncePerTx_AfterLoad_2108(t *testing.T) { + db := setupTestDB(t) + defer db.conn.Close() + phdSeed(t, db, phdObserverCount) + base := time.Now().UTC().Add(-30 * time.Minute) + phdInsertTx(t, db, 0, base) + for o := 1; o < phdObserverCount; o++ { + phdInsertObs(t, db, 0, o, base.Add(time.Duration(o)*time.Second), true) + } + store := phdLoadedStore(t, db) + since := phdMaxObsID(t, db) + phdInsertObs(t, db, 0, phdObserverCount, base.Add(time.Minute), false) + store.IngestNewObservations(since, 100) + phdAssertOncePerRelay(t, store, 0) +} + +// The dedupe is per (transmission, relay), not per transmission: a later +// observation that was heard through a relay none of the earlier ones went +// through must still add the transmission under that relay, exactly once. +func TestPathHopIndexAddsNewRelayFromLaterObservation_2108(t *testing.T) { + db := setupTestDB(t) + defer db.conn.Close() + phdSeed(t, db, 4) + store := phdLoadedStore(t, db) + + now := time.Now().UTC().Add(-10 * time.Minute) + phdInsertTx(t, db, 0, now) // phdPaths[0] = relays 0, 1, 2 + phdInsertObs(t, db, 0, 1, now, false) + store.IngestNewFromDB(0, 100) + tx := store.byTxID[1] + newRelay := phdRelays[3] + if n := phdCountTxIn(store, newRelay, tx); n != 0 { + t.Fatalf("fixture: tx already under the new relay (%d entries)", n) + } + before := store.GetRepeaterUsefulnessScore(newRelay) + + // Two further observers hear it through relays 0, 1 and the new relay 3. + for o := 2; o <= 3; o++ { + since := phdMaxObsID(t, db) + phdInsertObsVia(t, db, 0, []int{0, 1, 3}, o, now.Add(time.Duration(o)*time.Second), false) + store.IngestNewObservations(since, 100) + if n := phdCountTxIn(store, newRelay, tx); n != 1 { + t.Fatalf("after observation %d through the new relay: tx is in its bucket %d times, want 1", o, n) + } + } + for _, r := range []int{0, 1, 2} { + if n := phdCountTxIn(store, phdRelays[r], tx); n != 1 { + t.Errorf("tx is in byPathHop[%s…] %d times, want 1", phdRelays[r][:8], n) + } + } + if got, want := store.GetRepeaterUsefulnessScore(newRelay), 1.0; before != 0 || got != want { + t.Errorf("new relay's traffic share = %v (was %v), want %v (the only transmission)", got, before, want) + } +} + +// phdShares returns every relay's traffic share as seen by the three score +// functions, failing if they disagree. +func phdShares(t *testing.T, store *PacketStore) map[string]float64 { + t.Helper() + bulk := store.computeRepeaterUsefulnessScoreMap() + batch := store.GetRepeaterNodeStatsBatch(phdRelays, 24) + out := make(map[string]float64, len(phdRelays)) + for _, pk := range phdRelays { + single := store.GetRepeaterUsefulnessScore(pk) + if bulk[pk] != single || batch[pk].Score != single { + t.Fatalf("score functions disagree for %s…: bulk=%v single=%v batch=%v", pk[:8], bulk[pk], single, batch[pk].Score) + } + out[pk] = single + } + return out +} + +// phdExpectedShares is the definition applied to the fixture: the fraction +// of the (all non-advert) transmissions whose path contains the relay. +func phdExpectedShares() map[string]float64 { + out := make(map[string]float64, len(phdRelays)) + for _, path := range phdPaths { + for _, r := range path { + out[phdRelays[r]] += 1 / float64(len(phdPaths)) + } + } + return out +} + +func phdAssertShares(t *testing.T, label string, got map[string]float64) { + t.Helper() + want := phdExpectedShares() + sum := 0.0 + for _, pk := range phdRelays { + sum += got[pk] + if math.Abs(got[pk]-want[pk]) > 1e-9 { + t.Errorf("%s: traffic share of %s… = %.4f, want %.4f", label, pk[:8], got[pk], want[pk]) + } + if got[pk] >= 1 { + t.Errorf("%s: traffic share of %s… reached the 1.0 clamp", label, pk[:8]) + } + } + if sum > phdMaxPathLen+1e-9 { + t.Errorf("%s: sum of all relays' traffic share = %.3f, exceeds the maximum path length %d", label, sum, phdMaxPathLen) + } +} + +// The traffic share of a fixed set of transmissions is the same right after +// a load and after any number of further observations of them. +func TestTrafficShareStableAcrossLateObservations_2108(t *testing.T) { + base := time.Now().UTC().Add(-30 * time.Minute) + + // Restart: every observation is already persisted (resolved_path set by + // the ingestor) and comes in through Load. + loadDB := setupTestDB(t) + defer loadDB.conn.Close() + phdSeed(t, loadDB, phdObserverCount) + for i := range phdPaths { + phdInsertTx(t, loadDB, i, base.Add(time.Duration(i)*time.Second)) + for o := 1; o <= phdObserverCount; o++ { + phdInsertObs(t, loadDB, i, o, base.Add(time.Duration(i*100+o)*time.Second), true) + } + } + loaded := phdLoadedStore(t, loadDB) + afterLoad := phdShares(t, loaded) + phdAssertShares(t, "after load", afterLoad) + + // Live: the same transmissions arrive with one observation each, then + // the late observations arrive in rounds. + liveDB := setupTestDB(t) + defer liveDB.conn.Close() + phdSeed(t, liveDB, phdObserverCount) + live := phdLoadedStore(t, liveDB) + for i := range phdPaths { + phdInsertTx(t, liveDB, i, base.Add(time.Duration(i)*time.Second)) + phdInsertObs(t, liveDB, i, 1, base.Add(time.Duration(i*100+1)*time.Second), false) + } + live.IngestNewFromDB(0, 1000) + afterIngest := phdShares(t, live) + phdAssertShares(t, "after live ingest", afterIngest) + + for round := 0; round < 2; round++ { + since := phdMaxObsID(t, liveDB) + for i := range phdPaths { + for k := 0; k < phdLateObs/2; k++ { + o := 2 + round*(phdLateObs/2) + k + phdInsertObs(t, liveDB, i, o, base.Add(time.Duration(i*100+o)*time.Second), false) + } + } + live.IngestNewObservations(since, 10000) + got := phdShares(t, live) + label := fmt.Sprintf("after late-observation round %d", round+1) + phdAssertShares(t, label, got) + if !reflect.DeepEqual(got, afterIngest) { + t.Errorf("%s: shares changed without new transmissions:\n got %v\nwant %v", label, got, afterIngest) + } + } + for i := range phdPaths { + phdAssertOncePerRelay(t, live, i) + } + if !reflect.DeepEqual(phdShares(t, live), afterLoad) { + t.Errorf("live shares differ from a fresh load of the same data:\nlive %v\nload %v", phdShares(t, live), afterLoad) + } +} + +// Index size is a function of the transmissions, not of how many times they +// were heard: further observations through the same relays add nothing. +func TestPathHopIndexSizeBoundedByTransmissions_2108(t *testing.T) { + db := setupTestDB(t) + defer db.conn.Close() + phdSeed(t, db, phdObserverCount) + store := phdLoadedStore(t, db) + base := time.Now().UTC().Add(-30 * time.Minute) + for i := range phdPaths { + phdInsertTx(t, db, i, base.Add(time.Duration(i)*time.Second)) + phdInsertObs(t, db, i, 1, base.Add(time.Duration(i*100+1)*time.Second), false) + } + store.IngestNewFromDB(0, 1000) + entries := func() int { + store.mu.RLock() + defer store.mu.RUnlock() + n := 0 + for _, list := range store.byPathHop { + n += len(list) + } + return n + } + want := entries() + for o := 2; o <= phdObserverCount; o++ { + since := phdMaxObsID(t, db) + for i := range phdPaths { + phdInsertObs(t, db, i, o, base.Add(time.Duration(i*100+o)*time.Second), false) + } + store.IngestNewObservations(since, 10000) + if got := entries(); got != want { + t.Fatalf("after observation %d of each tx: byPathHop holds %d entries, want %d (one per relay key per tx)", o, got, want) + } + } +} + +// A repeated observation through relays the transmission is already indexed +// under changes nothing in byPathHop, so it must leave the cached batch +// relay stats in place (the cache is dropped only when byPathHop changes, +// #1164). An observation through a new relay does change byPathHop and +// must drop the cache, so the next read sees the new relay. +func TestRelayStatsCacheAcrossRepeatedObservations_2108(t *testing.T) { + db := setupTestDB(t) + defer db.conn.Close() + phdSeed(t, db, 4) + store := phdLoadedStore(t, db) + + now := time.Now().UTC().Add(-10 * time.Minute) + phdInsertTx(t, db, 0, now) // relays 0, 1, 2 + phdInsertObs(t, db, 0, 1, now, false) + store.IngestNewFromDB(0, 100) + cached := func() map[string]RepeaterNodeStats { + store.relayStatsCacheMu.Lock() + defer store.relayStatsCacheMu.Unlock() + return store.relayStatsCache + } + first := store.GetRepeaterNodeStatsBatchCached(phdRelays, 24) + if cached() == nil || first[phdRelays[0]].Score != 1 || first[phdRelays[3]].Score != 0 { + t.Fatalf("fixture: cache not populated as expected: %+v", first) + } + + for o := 2; o <= 3; o++ { + since := phdMaxObsID(t, db) + phdInsertObs(t, db, 0, o, now.Add(time.Duration(o)*time.Second), false) + store.IngestNewObservations(since, 100) + if cached() == nil { + t.Fatalf("repeated observation %d through known relays dropped the relay-stats cache", o) + } + } + if fresh := store.GetRepeaterNodeStatsBatch(phdRelays, 24); !reflect.DeepEqual(fresh, store.GetRepeaterNodeStatsBatchCached(phdRelays, 24)) { + t.Fatalf("cache kept across repeated observations is stale:\nfresh %+v\ncached %+v", fresh, cached()) + } + + since := phdMaxObsID(t, db) + phdInsertObsVia(t, db, 0, []int{0, 1, 3}, 4, now.Add(4*time.Second), false) + store.IngestNewObservations(since, 100) + if cached() != nil { + t.Fatal("observation through a new relay must drop the relay-stats cache") + } + if got := store.GetRepeaterNodeStatsBatchCached(phdRelays, 24)[phdRelays[3]].Score; got != 1 { + t.Errorf("new relay's cached score = %v, want 1", got) + } +} + +// The helper itself: the return value and the cache follow whether byPathHop +// changed, per (transmission, relay). +func TestAddResolvedPubkeysToPathHopIndex_PerRelayIdempotent_2108(t *testing.T) { + keyA, keyB, keyC := strings.Repeat("a", 64), strings.Repeat("b", 64), strings.Repeat("c", 64) + s := &PacketStore{byPathHop: make(map[string][]*StoreTx)} + tx := &StoreTx{ID: 1, parsedPath: []string{"aa", "bb"}, pathParsed: true} + hopsSeen := map[string]bool{} + seed := func() { s.relayStatsCache = map[string]RepeaterNodeStats{"sentinel": {}} } + step := func(label string, pks []string, wantMutated bool) { + t.Helper() + seed() + if got := s.addResolvedPubkeysToPathHopIndex(tx, pks, hopsSeen); got != wantMutated { + t.Fatalf("%s: mutated = %v, want %v", label, got, wantMutated) + } + if dropped := s.relayStatsCache == nil; dropped != wantMutated { + t.Fatalf("%s: relay-stats cache dropped = %v, want %v", label, dropped, wantMutated) + } + } + step("first observation", []string{keyA, keyB}, true) + step("same relays again", []string{keyA, keyB}, false) + step("same relays, other order", []string{keyB, keyA}, false) + step("one new relay", []string{keyA, keyC}, true) + step("all known", []string{keyC, keyB, keyA}, false) + for _, k := range []string{keyA, keyB, keyC} { + if n := len(s.byPathHop[k]); n != 1 { + t.Errorf("byPathHop[%s…] holds %d entries, want 1", k[:4], n) + } + } +} + +// Defence in depth: whatever the index holds, the score counts distinct +// transmissions, not bucket entries. +func TestTrafficShareCountsDistinctTransmissions_2108(t *testing.T) { + store := makeTestStore(0, time.Now().UTC(), 0) + store.byPathHop = make(map[string][]*StoreTx) + pt, advert := 2, payloadTypeAdvert + relay := phdRelays[0] + var txs []*StoreTx + for i := 0; i < 10; i++ { + tx := &StoreTx{ID: i + 1, Hash: fmt.Sprintf("phd-%d", i), PayloadType: &pt} + txs = append(txs, tx) + store.packets = append(store.packets, tx) + store.byTxID[tx.ID] = tx + store.byPayloadType[pt] = append(store.byPayloadType[pt], tx) + } + ad := &StoreTx{ID: 99, Hash: "phd-advert", PayloadType: &advert} + store.byPayloadType[advert] = append(store.byPayloadType[advert], ad) + // Two of the ten transmissions, each present 6 times (the index shape the + // bug produced after 6 observations), plus a duplicated advert. + for k := 0; k < 6; k++ { + store.byPathHop[relay] = append(store.byPathHop[relay], txs[0], txs[1], ad) + } + + const want = 2.0 / 10 + if got := store.GetRepeaterUsefulnessScore(relay); got != want { + t.Errorf("GetRepeaterUsefulnessScore = %v, want %v", got, want) + } + if got := store.computeRepeaterUsefulnessScoreMap()[relay]; got != want { + t.Errorf("computeRepeaterUsefulnessScoreMap = %v, want %v", got, want) + } + if got := store.GetRepeaterNodeStatsBatch([]string{relay}, 24)[relay].Score; got != want { + t.Errorf("GetRepeaterNodeStatsBatch score = %v, want %v", got, want) + } +} + +// phdDuplicateFullKeyEntries reproduces the index shape the bug produced: every +// transmission in a full-pubkey bucket is present `extra` more times. +func phdDuplicateFullKeyEntries(store *PacketStore, extra int) { + store.mu.Lock() + defer store.mu.Unlock() + for key, list := range store.byPathHop { + if len(key) != 64 { + continue + } + orig := append([]*StoreTx(nil), list...) + for k := 0; k < extra; k++ { + store.byPathHop[key] = append(store.byPathHop[key], orig...) + } + } + store.invalidateRelayStatsCache() +} + +// The other byPathHop consumers already deduplicate by transmission (or only +// read raw prefix keys and take a maximum): duplicate entries do not change +// their results. This pins that, so a refactor cannot reintroduce the +// inflation there. +func TestPathHopConsumersIgnoreDuplicateEntries_2108(t *testing.T) { + base := time.Now().UTC().Add(-30 * time.Minute) + db := setupTestDB(t) + defer db.conn.Close() + phdSeed(t, db, phdObserverCount) + for i := range phdPaths { + phdInsertTx(t, db, i, base.Add(time.Duration(i)*time.Second)) + for o := 1; o <= 2; o++ { + phdInsertObs(t, db, i, o, base.Add(time.Duration(i*100+o)*time.Second), true) + } + } + store := phdLoadedStore(t, db) + + type snapshot struct { + info map[string]RepeaterRelayInfo + bulk map[string]RepeaterRelayInfo + multi []MultiByteCapEntry + } + take := func() snapshot { + s := snapshot{info: map[string]RepeaterRelayInfo{}} + batch := store.GetRepeaterNodeStatsBatch(phdRelays, 24) + for _, pk := range phdRelays { + s.info[pk] = batch[pk].Info + } + s.bulk = store.computeRepeaterRelayInfoMap(24) + s.multi = store.computeMultiByteCapability(nil) + return s + } + before := take() + if before.info[phdRelays[0]].RelayCount24h == 0 || before.bulk[phdRelays[0]].RelayCount24h == 0 { + t.Fatalf("fixture: relay 0 must have relay counts: %+v / %+v", before.info[phdRelays[0]], before.bulk[phdRelays[0]]) + } + phdDuplicateFullKeyEntries(store, phdLateObs) + after := take() + for _, pk := range phdRelays { + if !reflect.DeepEqual(before.info[pk], after.info[pk]) { + t.Errorf("relay info of %s… changed with duplicate entries:\nbefore %+v\n after %+v", pk[:8], before.info[pk], after.info[pk]) + } + if !reflect.DeepEqual(before.bulk[pk], after.bulk[pk]) { + t.Errorf("bulk relay info of %s… changed with duplicate entries:\nbefore %+v\n after %+v", pk[:8], before.bulk[pk], after.bulk[pk]) + } + } + if !reflect.DeepEqual(before.multi, after.multi) { + t.Errorf("computeMultiByteCapability changed with duplicate entries:\nbefore %+v\n after %+v", before.multi, after.multi) + } +} + +// computeMultiByteCapability reads raw prefix buckets only and keeps a +// maximum hash size per node, so duplicates there cannot inflate it either. +func TestMultiByteCapabilityIgnoresDuplicateEntries_2108(t *testing.T) { + db := setupCapabilityTestDB(t) + defer db.conn.Close() + db.conn.Exec("INSERT INTO nodes (public_key, name, role, last_seen) VALUES (?, ?, ?, ?)", + "aabbccdd11223344", "RepB", "repeater", recentTS(48)) + store := NewPacketStore(db, nil) + pt := 1 + pkt := &StoreTx{RawHex: "01" + buildPathByte(2, 1) + "aabb", PayloadType: &pt, PathJSON: `["aabb"]`, FirstSeen: recentTS(48)} + addTestPacket(store, pkt) + before := store.computeMultiByteCapability(nil) + store.mu.Lock() + for k := 0; k < phdLateObs; k++ { + store.byPathHop["aabb"] = append(store.byPathHop["aabb"], pkt) + } + store.mu.Unlock() + after := store.computeMultiByteCapability(nil) + if len(before) != 1 || !reflect.DeepEqual(before, after) { + t.Fatalf("duplicates changed multi-byte capability:\nbefore %+v\n after %+v", before, after) + } +} diff --git a/cmd/server/pathhop_eviction_1908_test.go b/cmd/server/pathhop_eviction_1908_test.go index f4b81264b..33b6d3c79 100644 --- a/cmd/server/pathhop_eviction_1908_test.go +++ b/cmd/server/pathhop_eviction_1908_test.go @@ -44,9 +44,11 @@ func TestPathHopEviction1908(t *testing.T) { addTxToPathHopIndex(store.byPathHop, tx) store.indexResolvedPathHops(tx, []string{shared, pk}, hopsSeen) } - // Repeated observations can append the same resolved association again. - store.indexResolvedPathHops(store.packets[0], []string{shared, expiredOnly}, hopsSeen) - store.indexResolvedPathHops(store.packets[7], []string{shared}, hopsSeen) + // Repeated observations no longer append the same resolved + // association again (#2108), but eviction must still drop every + // occurrence from a bucket that holds one, so seed duplicates directly. + store.byPathHop[shared] = append(store.byPathHop[shared], store.packets[0], store.packets[7]) + store.byPathHop[expiredOnly] = append(store.byPathHop[expiredOnly], store.packets[0]) addTxToPathHopIndex(store.byPathHop, store.packets[0]) // Keep the original backing arrays observable: a shortened slice must diff --git a/cmd/server/pathhop_record_2108_test.go b/cmd/server/pathhop_record_2108_test.go new file mode 100644 index 000000000..b10e549f7 --- /dev/null +++ b/cmd/server/pathhop_record_2108_test.go @@ -0,0 +1,294 @@ +package main + +import ( + "fmt" + "runtime" + "strings" + "testing" + "time" + "unsafe" +) + +func TestCountDistinctNonAdvert_2108(t *testing.T) { + pt, advert := 2, payloadTypeAdvert + tx := func(id int) *StoreTx { return &StoreTx{ID: id, PayloadType: &pt} } + t1, t2, t3 := tx(1), tx(2), tx(3) + ad := &StoreTx{ID: 4, PayloadType: &advert} + untyped := &StoreTx{ID: 5} + cases := []struct { + name string + list []*StoreTx + want int + }{ + {"empty", nil, 0}, + {"ascending", []*StoreTx{t1, t2, t3}, 3}, + {"descending (chunk load order)", []*StoreTx{t3, t2, t1}, 3}, + {"adjacent duplicate", []*StoreTx{t1, t1}, 1}, + {"ascending then duplicate", []*StoreTx{t1, t2, t3, t3}, 3}, + {"interleaved duplicates", []*StoreTx{t1, t2, t1, t3, t2}, 3}, + {"adverts and nils skipped", []*StoreTx{nil, ad, t2, ad, nil, t2}, 1}, + {"untyped counted", []*StoreTx{untyped, t1, untyped}, 2}, + } + var ids []int + for _, c := range cases { + var got int + got, ids = countDistinctNonAdvert(c.list, ids) + if got != c.want { + t.Errorf("%s: got %d, want %d", c.name, got, c.want) + } + } +} + +// --- The per-transmission record behind the dedupe (pathHopResolved) --- +// +// It holds the resolved relay keys a live transmission is indexed under. +// These tests pin its lifecycle: bounded by relays (not observations), +// removed on eviction, kept for live transmissions only across an index +// rebuild, and emptied when a rebuild starts from an empty index. + +var ( + phdRecordKey1 = strings.Repeat("a", 64) + phdRecordKey2 = strings.Repeat("b", 64) +) + +// Raw hops of every transmission built by phdRecordStore (makeTestStore's +// PathJSON), plus the two resolved keys above. +const phdRecordRefsPerTx = 3 + 2 + +// phdRecordStore builds count non-advert transmissions, the older half past +// the 1 h retention, each indexed under its raw hops and two resolved keys +// through the real byPathHop helpers. +func phdRecordStore(count int) (store *PacketStore, old, young []*StoreTx) { + now := time.Now().UTC() + store = makeTestStore(count, now.Add(-30*time.Minute), 0) + store.byPayloadType[5] = store.byPayloadType[4] + delete(store.byPayloadType, 4) + store.byPathHop = make(map[string][]*StoreTx) + store.useResolvedPathIndex = true + store.initResolvedPathIndex() + store.retentionHours = 1 + hopsSeen := make(map[string]bool) + for i, tx := range store.packets { + *tx.PayloadType = 5 + if i < count/2 { + tx.FirstSeen = now.Add(-2 * time.Hour).Format(time.RFC3339) + old = append(old, tx) + } else { + young = append(young, tx) + } + addTxToPathHopIndex(store.byPathHop, tx) + store.indexResolvedPathHops(tx, phdFreshKeys(), hopsSeen) + } + return store, old, young +} + +// phdFreshKeys returns the two resolved keys as newly allocated strings, as +// every ingest path produces them (json.Unmarshal / strings.ToLower). +func phdFreshKeys() []string { + return []string{strings.Clone(phdRecordKey1), strings.Clone(phdRecordKey2)} +} + +func phdEntriesOf(store *PacketStore, tx *StoreTx) int { + n := 0 + for _, list := range store.byPathHop { + for _, t := range list { + if t == tx { + n++ + } + } + } + return n +} + +// The record does not grow with observations, and eviction removes it +// together with the transmission's byPathHop entries. +func TestPathHopResolvedRecordBoundedAndEvicted_2108(t *testing.T) { + store, old, young := phdRecordStore(40) + hopsSeen := map[string]bool{} + for _, tx := range store.packets { + for k := 0; k < 10; k++ { + store.indexResolvedPathHops(tx, phdFreshKeys(), hopsSeen) + } + } + if got := len(store.pathHopResolved); got != len(old)+len(young) { + t.Fatalf("record holds %d transmissions, want %d", got, len(old)+len(young)) + } + for _, tx := range young { + if got := len(store.pathHopResolved[tx]); got != 2 { + t.Fatalf("tx %d record holds %d keys after 11 observations, want 2", tx.ID, got) + } + if n := phdEntriesOf(store, tx); n != phdRecordRefsPerTx { + t.Fatalf("tx %d has %d byPathHop entries after 11 observations, want %d", tx.ID, n, phdRecordRefsPerTx) + } + } + + store.mu.Lock() + evicted := store.EvictStale() + store.mu.Unlock() + if evicted != len(old) { + t.Fatalf("evicted %d, want %d", evicted, len(old)) + } + for _, tx := range old { + if _, ok := store.pathHopResolved[tx]; ok { + t.Fatalf("evicted tx %d is still in the record", tx.ID) + } + if n := phdEntriesOf(store, tx); n != 0 { + t.Fatalf("evicted tx %d is still in %d byPathHop buckets", tx.ID, n) + } + } + if got := len(store.pathHopResolved); got != len(young) { + t.Fatalf("record holds %d transmissions after eviction, want %d", got, len(young)) + } + // A survivor's next observation still adds nothing. + for _, tx := range young { + store.indexResolvedPathHops(tx, phdFreshKeys(), hopsSeen) + if n := phdEntriesOf(store, tx); n != phdRecordRefsPerTx { + t.Fatalf("tx %d has %d entries after eviction and another observation, want %d", tx.ID, n, phdRecordRefsPerTx) + } + } + + // Evicting the rest leaves no record and no interned key behind. + for _, tx := range young { + tx.FirstSeen = time.Now().UTC().Add(-2 * time.Hour).Format(time.RFC3339) + } + store.mu.Lock() + store.EvictStale() + store.mu.Unlock() + if len(store.pathHopResolved) != 0 || len(store.pathHopKeys) != 0 || len(store.byPathHop) != 0 { + t.Fatalf("after evicting every transmission: record %d, interned keys %d, byPathHop keys %d; want 0/0/0", + len(store.pathHopResolved), len(store.pathHopKeys), len(store.byPathHop)) + } +} + +// A rebuild keeps the record of live transmissions (their resolved entries +// are carried over), so observations after it still add nothing; it drops +// the record of transmissions that are no longer in the store. +func TestPathHopResolvedRecordAcrossRebuild_2108(t *testing.T) { + store, _, young := phdRecordStore(40) + gone := store.packets[0] + goneOnly := strings.Repeat("e", 64) // a relay only the removed transmission went through + store.indexResolvedPathHops(gone, []string{goneOnly}, map[string]bool{}) + store.packets = store.packets[1:] + delete(store.byTxID, gone.ID) + delete(store.byHash, gone.Hash) + + store.mu.Lock() + store.buildPathHopIndex() + store.mu.Unlock() + + if _, ok := store.pathHopResolved[gone]; ok { + t.Fatal("rebuild kept the record of a transmission no longer in the store") + } + if _, ok := store.byPathHop[goneOnly]; ok { + t.Fatal("fixture: rebuild carried over a bucket of a removed transmission") + } + if _, ok := store.pathHopKeys[goneOnly]; ok || len(store.pathHopKeys) != 2 { + t.Fatalf("rebuild kept interned keys without a bucket: %d interned, want 2", len(store.pathHopKeys)) + } + hopsSeen := map[string]bool{} + for _, tx := range young { + if _, ok := store.pathHopResolved[tx]; !ok { + t.Fatalf("rebuild dropped the record of live tx %d", tx.ID) + } + store.indexResolvedPathHops(tx, phdFreshKeys(), hopsSeen) + if n := phdEntriesOf(store, tx); n != phdRecordRefsPerTx { + t.Fatalf("tx %d has %d entries after a rebuild and another observation, want %d", tx.ID, n, phdRecordRefsPerTx) + } + } +} + +// When the index being rebuilt holds no resolved entries at all, nothing is +// indexed under any resolved key any more, so the record must not claim +// otherwise: the next observation has to put the transmission back. +func TestPathHopResolvedRecordClearedWithEmptyIndex_2108(t *testing.T) { + store, _, young := phdRecordStore(40) + store.byPathHop = make(map[string][]*StoreTx) // nothing carried over + + store.mu.Lock() + store.buildPathHopIndex() + store.mu.Unlock() + + if got := len(store.pathHopResolved); got != 0 { + t.Fatalf("record still holds %d transmissions after a rebuild from an empty index", got) + } + if got := len(store.pathHopKeys); got != 0 { + t.Fatalf("%d interned keys survive a rebuild from an empty index", got) + } + hopsSeen := map[string]bool{} + for _, tx := range young { + store.indexResolvedPathHops(tx, phdFreshKeys(), hopsSeen) + if n := phdEntriesOf(store, tx); n != phdRecordRefsPerTx { + t.Fatalf("tx %d has %d entries after the rebuild and another observation, want %d", tx.ID, n, phdRecordRefsPerTx) + } + } +} + +// The record compares keys exactly, and keeps one shared copy of each key +// rather than the per-observation string it was handed: every record entry +// and the byPathHop key point at the same bytes. +func TestPathHopResolvedRecordInternsKeys_2108(t *testing.T) { + store, _, young := phdRecordStore(10) + canon, ok := store.pathHopKeys[phdRecordKey1] + if !ok || len(store.pathHopKeys) != 2 { + t.Fatalf("interned keys = %d (key1 present %v), want 2", len(store.pathHopKeys), ok) + } + for _, tx := range young { + rec := store.pathHopResolved[tx] + if len(rec) != 2 || unsafe.StringData(rec[0]) != unsafe.StringData(canon) { + t.Fatalf("tx %d record %v does not share the interned key", tx.ID, rec) + } + } + for k := range store.byPathHop { + if k == phdRecordKey1 && unsafe.StringData(k) != unsafe.StringData(canon) { + t.Fatal("byPathHop key is not the interned copy") + } + } +} + +// BenchmarkPathHopRecordMemory_2108 reports what the record costs: heap +// retained by pathHopResolved and pathHopKeys (including any key string +// only they keep alive) per transmission, for n transmissions with k +// distinct resolved relay keys each from 4000 relays, every transmission +// heard twice with freshly allocated keys as the ingest paths produce them. +// The fixture's union of resolved keys per transmission over all of its +// observations has a mean of 5 and a maximum of 17. +func BenchmarkPathHopRecordMemory_2108(b *testing.B) { + heap := func() uint64 { + var m runtime.MemStats + runtime.GC() + runtime.GC() + runtime.ReadMemStats(&m) + return m.HeapAlloc + } + const relays = 4000 + for _, n := range []int{100000} { + for _, k := range []int{2, 5, 17} { + b.Run(fmt.Sprintf("txs=%d/keys=%d", n, k), func(b *testing.B) { + for it := 0; it < b.N; it++ { + b.StopTimer() + store := makeTestStore(0, time.Now().UTC(), 0) + store.byPathHop = make(map[string][]*StoreTx) + hopsSeen := map[string]bool{} + pks := make([]string, k) + for i := 0; i < n; i++ { + tx := &StoreTx{ID: i + 1, parsedPath: []string{"zz"}, pathParsed: true} + store.packets = append(store.packets, tx) + for obs := 0; obs < 2; obs++ { + for j := range pks { + r := (i*31 + j*997) % relays + pks[j] = fmt.Sprintf("%02x%062x", r%256, r) + } + store.indexResolvedPathHops(tx, pks, hopsSeen) + } + } + with := heap() + store.pathHopResolved, store.pathHopKeys = nil, nil + without := heap() + runtime.KeepAlive(store) + b.ReportMetric((float64(with)-float64(without))/float64(n), "record-B/tx") + b.StartTimer() + } + }) + } + } +} diff --git a/cmd/server/repeater_enrich_bulk.go b/cmd/server/repeater_enrich_bulk.go index b26c56699..4fb6cb69e 100644 --- a/cmd/server/repeater_enrich_bulk.go +++ b/cmd/server/repeater_enrich_bulk.go @@ -248,16 +248,19 @@ func (s *PacketStore) computeRepeaterUsefulnessScoreMap() map[string]float64 { return out } denom := float64(totalNonAdvert) + var ids []int for key, list := range s.byPathHop { - relayed := 0 - for _, tx := range list { - if tx == nil { - continue - } - if tx.PayloadType != nil && *tx.PayloadType == payloadTypeAdvert { - continue - } - relayed++ + // Distinct transmissions, matching the denominator (#2108). Only + // full-pubkey keys (the resolved hops, and the only keys a + // repeater's score is read under) can hold a transmission more + // than once; raw hop buckets get one entry per transmission from + // addTxToPathHopIndex and are counted directly, which keeps this + // pass near its previous cost (BenchmarkTrafficShareScoreMap_2108). + var relayed int + if len(key) == 64 { + relayed, ids = countDistinctNonAdvert(list, ids) + } else { + relayed = countNonAdvert(list) } if relayed == 0 { continue diff --git a/cmd/server/repeater_usefulness.go b/cmd/server/repeater_usefulness.go index 49b8f86ce..2e96ccc69 100644 --- a/cmd/server/repeater_usefulness.go +++ b/cmd/server/repeater_usefulness.go @@ -1,6 +1,7 @@ package main import ( + "slices" "sort" "strings" "time" @@ -11,13 +12,10 @@ import ( // repeater as a relay hop. Issue #672 (Traffic axis only — bridge, // coverage, and redundancy axes are deferred to follow-up work). // -// Numerator: count of non-advert StoreTx entries indexed under -// -// pubkey in byPathHop. -// -// Denominator: total non-advert StoreTx entries in the store -// -// (sum of byPayloadType for all keys != payloadTypeAdvert). +// Numerator: the number of distinct non-advert transmissions indexed under +// pubkey in byPathHop (countDistinctNonAdvert, #2108). Denominator: total +// non-advert transmissions in the store (sum of byPayloadType for all keys +// != payloadTypeAdvert). // // Returns 0 when there is no non-advert traffic, the pubkey is empty, // or the repeater never appears as a relay hop. Scores are clamped to @@ -48,17 +46,8 @@ func (s *PacketStore) GetRepeaterUsefulnessScore(pubkey string) float64 { return 0 } - // Numerator: this repeater's non-advert hop appearances. - relayed := 0 - for _, tx := range s.byPathHop[key] { - if tx == nil { - continue - } - if tx.PayloadType != nil && *tx.PayloadType == payloadTypeAdvert { - continue - } - relayed++ - } + // Numerator: the distinct non-advert transmissions this repeater relayed. + relayed, _ := countDistinctNonAdvert(s.byPathHop[key], nil) score := float64(relayed) / float64(totalNonAdvert) if score < 0 { @@ -70,6 +59,55 @@ func (s *PacketStore) GetRepeaterUsefulnessScore(pubkey string) float64 { return score } +// countDistinctNonAdvert returns the number of distinct non-advert +// transmissions in list, a byPathHop bucket. The index holds a transmission +// once per key (#2108); counting distinct transmissions keeps traffic share +// a fraction of transmissions even if a bucket ever held duplicates, since +// the denominator counts each transmission once. ids is caller-owned +// scratch (may be nil), returned for reuse. +// +// Cost: O(len(list)) with no allocation once ids has grown. Buckets are +// appended in ingest order, so their IDs are usually strictly increasing, +// which already proves them distinct; only otherwise (background chunks +// load older transmissions after newer ones) are the IDs sorted, +// O(n log n) on a flat slice. A map per bucket was measured ~10x slower +// on the bulk pass (BenchmarkTrafficShareScoreMap_2108). +func countDistinctNonAdvert(list []*StoreTx, ids []int) (int, []int) { + ids = ids[:0] + ascending := true + for _, tx := range list { + if tx == nil || (tx.PayloadType != nil && *tx.PayloadType == payloadTypeAdvert) { + continue + } + if n := len(ids); n > 0 && tx.ID <= ids[n-1] { + ascending = false + } + ids = append(ids, tx.ID) + } + if ascending || len(ids) < 2 { + return len(ids), ids + } + slices.Sort(ids) + n := 1 + for i := 1; i < len(ids); i++ { + if ids[i] != ids[i-1] { + n++ + } + } + return n, ids +} + +// countNonAdvert returns the number of non-advert entries in list. +func countNonAdvert(list []*StoreTx) int { + n := 0 + for _, tx := range list { + if tx != nil && (tx.PayloadType == nil || *tx.PayloadType != payloadTypeAdvert) { + n++ + } + } + return n +} + // RepeaterNodeStats bundles relay-activity and usefulness data for a single node. type RepeaterNodeStats struct { Info RepeaterRelayInfo @@ -90,7 +128,7 @@ func (s *PacketStore) GetRepeaterNodeStatsBatch(pubkeys []string, windowHours fl type nodeSnap struct { entries []relayEntry - relayed int // non-advert count in full-key list only (for usefulness score) + relayed int // distinct non-advert txs in full-key list only (for usefulness score) } s.mu.RLock() @@ -103,15 +141,12 @@ func (s *PacketStore) GetRepeaterNodeStatsBatch(pubkeys []string, windowHours fl } snaps := make(map[string]nodeSnap, len(pubkeys)) + var ids []int for _, pk := range pubkeys { key := strings.ToLower(pk) entries := s.collectRelayEntriesLocked(key) - relayed := 0 - for _, tx := range s.byPathHop[key] { - if tx != nil && (tx.PayloadType == nil || *tx.PayloadType != payloadTypeAdvert) { - relayed++ - } - } + var relayed int + relayed, ids = countDistinctNonAdvert(s.byPathHop[key], ids) snaps[pk] = nodeSnap{entries: entries, relayed: relayed} } diff --git a/cmd/server/store.go b/cmd/server/store.go index a26f72fa2..e59194188 100644 --- a/cmd/server/store.go +++ b/cmd/server/store.go @@ -394,6 +394,21 @@ type PacketStore struct { usefulnessAxesRecompMu sync.Mutex usefulnessAxesRecompStarted bool + // pathHopResolved records, per transmission, the resolved pubkey keys + // addResolvedPubkeysToPathHopIndex has put it under in byPathHop, so + // further observations through the same relays do not append it again + // (#2108). Keys are compared exactly. At most one entry per relay key + // per live transmission: evictStaleInternal deletes evicted + // transmissions and retainResolvedPathHops drops any no longer in + // s.packets. + pathHopResolved map[*StoreTx][]string + // pathHopKeys interns those keys: one shared copy per resolved byPathHop + // key, so the record holds a 16-byte string header per entry instead of + // keeping alive the string each observation's resolve allocated. It + // holds only keys that have a byPathHop bucket; eviction and rebuild + // drop the others. + pathHopKeys map[string]string + // Precomputed distinct advert pubkey count (refcounted for eviction correctness). // Updated incrementally during Load/Ingest/Evict — avoids JSON parsing in GetPerfStoreStats. advertPubkeys map[string]int // pubkey → number of advert packets referencing it @@ -715,6 +730,8 @@ func NewPacketStore(db *DB, cfg *PacketStoreConfig, cacheTTLs ...map[string]inte spIndex: make(map[string]int, 4096), spTxIndex: make(map[string][]*StoreTx, 4096), advertPubkeys: make(map[string]int), + pathHopResolved: make(map[*StoreTx][]string), + pathHopKeys: make(map[string]string), clockSkew: NewClockSkewEngine(), useResolvedPathIndex: true, areaNodeCache: make(map[string]map[string]bool), @@ -1799,6 +1816,10 @@ func pathLen(pathJSON string) int { // empties); the helper is a no-op when pks is empty. // - hopsSeen is a reusable scratch map; addResolvedPubkeysToPathHopIndex // clear()s it on entry. +// - Safe to call once per observation: every index it feeds is +// idempotent per (transmission, pubkey) — byNode via nodeHashes, the +// resolved index via its forward-list check, byPathHop via +// pathHopResolved (#2108). func (s *PacketStore) indexResolvedPathHops(tx *StoreTx, pks []string, hopsSeen map[string]bool) { if len(pks) == 0 { return @@ -4225,18 +4246,27 @@ func (s *PacketStore) buildPathHopIndex() { // after s.byPathHop has been rebuilt from raw hops. func (s *PacketStore) retainResolvedPathHops(prev map[string][]*StoreTx) int { if len(prev) == 0 { + // No resolved entries anywhere, so nothing is recorded as indexed. + clear(s.pathHopResolved) + clear(s.pathHopKeys) return 0 } live := make(map[*StoreTx]struct{}, len(s.packets)) for _, tx := range s.packets { live[tx] = struct{}{} } + // Keep the per-transmission record in step with what is carried over + // below: transmissions that are gone are not (#2108). + for tx := range s.pathHopResolved { + if _, ok := live[tx]; !ok { + delete(s.pathHopResolved, tx) + } + } // Reused across keys (cleared per key) so a large index does not churn // one map allocation per key. Guards against both a key that the raw - // pass already produced and repeated appends of the same tx in prev — - // indexResolvedPathHops dedups within a call, not across the several - // observations of one transmission. + // pass already produced and repeated appends of the same tx in prev + // (an index built before #2108 made the resolved append idempotent). seen := make(map[*StoreTx]struct{}, 16) retained := 0 for key, list := range prev { @@ -4262,6 +4292,12 @@ func (s *PacketStore) retainResolvedPathHops(prev map[string][]*StoreTx) int { retained++ } } + // Interned keys whose bucket was not carried over have no entries left. + for key := range s.pathHopKeys { + if _, ok := s.byPathHop[key]; !ok { + delete(s.pathHopKeys, key) + } + } return retained } @@ -4378,11 +4414,21 @@ func removeTxFromPathHopIndex(idx map[string][]*StoreTx, tx *StoreTx) { } // addResolvedPubkeysToPathHopIndex appends tx into byPathHop under each -// resolved pubkey key that isn't already present as a raw hop. Mutating -// byPathHop here MUST be paired with invalidateRelayStatsCache so the -// cached batch relay stats don't go stale for up to relayStatsCacheTTL. -// hopsSeen is a scratch map the caller can reuse across calls (it will -// be cleared on entry). +// resolved pubkey key that isn't already present as a raw hop and that tx +// has not been appended under before. It is called once per observation, +// and the observations of one transmission usually resolve to the same +// relays: the per-transmission record in s.pathHopResolved keeps the index +// at one entry per (key, transmission), so per-key counts are distinct +// transmissions (#2108). Mutating byPathHop here MUST be paired with +// invalidateRelayStatsCache so the cached batch relay stats don't go stale +// for up to relayStatsCacheTTL. hopsSeen is a scratch map the caller can +// reuse across calls (it will be cleared on entry). +// +// Cost per call: an exact string compare of each pubkey against the +// transmission's own record (one entry per distinct relay key, a handful). +// A repeat observation through known relays allocates nothing and leaves +// byPathHop, and so the relay-stats cache, untouched. A new key costs one +// pathHopKeys lookup to share the interned copy. // // Must be called with s.mu held. func (s *PacketStore) addResolvedPubkeysToPathHopIndex(tx *StoreTx, pubkeys []string, hopsSeen map[string]bool) bool { @@ -4393,19 +4439,40 @@ func (s *PacketStore) addResolvedPubkeysToPathHopIndex(tx *StoreTx, pubkeys []st for _, hop := range txGetParsedPath(tx) { hopsSeen[strings.ToLower(hop)] = true } - mutated := false + if s.pathHopResolved == nil { + s.pathHopResolved = make(map[*StoreTx][]string) + } + if s.pathHopKeys == nil { + s.pathHopKeys = make(map[string]string) + } + indexed := s.pathHopResolved[tx] + before := len(indexed) for _, pk := range pubkeys { - if !hopsSeen[pk] { - hopsSeen[pk] = true - s.byPathHop[pk] = append(s.byPathHop[pk], tx) - mutated = true + if hopsSeen[pk] { + continue + } + hopsSeen[pk] = true + if slices.Contains(indexed, pk) { + continue + } + key, ok := s.pathHopKeys[pk] + if !ok { + key = pk + s.pathHopKeys[key] = key } + if indexed == nil { + indexed = make([]string, 0, len(pubkeys)) + } + indexed = append(indexed, key) + s.byPathHop[key] = append(s.byPathHop[key], tx) } - // Mutating byPathHop invalidates the batch relay-stats cache (#1164). - if mutated { - s.invalidateRelayStatsCache() + if len(indexed) == before { + return false } - return mutated + s.pathHopResolved[tx] = indexed + // Mutating byPathHop invalidates the batch relay-stats cache (#1164). + s.invalidateRelayStatsCache() + return true } // invalidateRelayStatsCache drops the cached batch relay-stats result so @@ -4917,6 +4984,9 @@ func (s *PacketStore) evictStaleInternal(rpBatch map[int][]string) int { for _, tx := range evicting { delete(s.byHash, tx.Hash) delete(s.byTxID, tx.ID) + // Its record of indexed resolved keys goes with it (#2108), so the + // record stays bounded by the live transmissions. + delete(s.pathHopResolved, tx) evictedTxIDs[tx.ID] = struct{}{} evictedBytes += tx.accountedBytes @@ -4992,6 +5062,7 @@ func (s *PacketStore) evictStaleInternal(rpBatch map[int][]string) int { }) if len(filtered) == 0 { delete(s.byPathHop, key) + delete(s.pathHopKeys, key) } else { s.byPathHop[key] = filtered } From 523a5edfa10b650c6a45498404b95b30f0e2f89a Mon Sep 17 00:00:00 2001 From: nullrouten0 Date: Thu, 8 Oct 2026 06:56:30 -0700 Subject: [PATCH 100/151] fix(ui): escape the route string on the unknown-route page (#2124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `app.js` wrote the URL fragment into `innerHTML` as the heading of the "Page not yet implemented" view for routes it does not know. Browsers percent-encode `<`, `>` and `"` in fragments, so this is unlikely to be exploitable today, but it is a plain `innerHTML` sink on a URL-controlled string and costs one `escapeHtml()` to close. **Tests:** one test added to `tests/unit/test-xss-escape-sinks.js` in the file's existing style. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Mythos 5.1 --- public/app.js | 2 +- tests/unit/test-xss-escape-sinks.js | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/public/app.js b/public/app.js index 6ca80cd07..e5677a3f0 100644 --- a/public/app.js +++ b/public/app.js @@ -1281,7 +1281,7 @@ function navigate() { else { app.setAttribute('tabindex', '-1'); app.focus({ preventScroll: true }); } }); } else { - app.innerHTML = `

${route}

Page not yet implemented.

`; + app.innerHTML = `

${escapeHtml(route)}

Page not yet implemented.

`; } } diff --git a/tests/unit/test-xss-escape-sinks.js b/tests/unit/test-xss-escape-sinks.js index 350927cc8..5867f9ee4 100644 --- a/tests/unit/test-xss-escape-sinks.js +++ b/tests/unit/test-xss-escape-sinks.js @@ -588,6 +588,18 @@ test('loadDetail catch block does NOT interpolate e.message into innerHTML', () 'loadDetail missing textContent assignment for e.message: ' + body); }); +// ========================================================================= +// E. URL-derived strings +// ========================================================================= +console.log('\n=== E. URL-derived strings ==='); + +test('app.js unknown-route page escapes the route taken from location.hash', () => { + const html = evalTemplate('public/app.js', + /(

\$\{[^}]*route[^}]*\}<\/h2>

Page not yet implemented\.<\/p><\/div>)/, + { route: TAG_PAYLOAD + ATTR_PAYLOAD }); + assertNoXss(html, 'app.js unknown route'); +}); + // ========================================================================= // SUMMARY // ========================================================================= From 8b644396354dbeb928ab506ac133ffe71ce29fcc Mon Sep 17 00:00:00 2001 From: nullrouten0 Date: Thu, 8 Oct 2026 06:56:42 -0700 Subject: [PATCH 101/151] fix(server): keep config.json's file mode when saving the geo-filter (#2126) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `SaveGeoFilter` rewrote `config.json` through a temp file created with mode 0644, so a config an operator had made 0600 (it holds the API key and broker passwords) became world-readable after the first geo-filter save. **Fix:** stat the original and reuse its mode for the temp file. Falls back to 0644 when the stat fails. **Tests:** `config_mode_test.go` — a 0600 config stays 0600 after a save; a 0644 config stays 0644. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Mythos 5.1 --- cmd/server/config.go | 10 +++++++-- cmd/server/config_mode_test.go | 40 ++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 cmd/server/config_mode_test.go diff --git a/cmd/server/config.go b/cmd/server/config.go index bb5c43531..038a29cc6 100644 --- a/cmd/server/config.go +++ b/cmd/server/config.go @@ -996,9 +996,15 @@ func SaveGeoFilter(configDir string, gf *GeoFilterConfig) error { } out = append(out, '\n') - // Atomic write: temp file + rename. + // Atomic write: temp file + rename. Keep the original file's mode: + // config.json holds the API key and broker passwords, and an operator + // who made it 0600 should not find it 0644 after a geo-filter save. + mode := os.FileMode(0644) + if fi, err := os.Stat(configPath); err == nil { + mode = fi.Mode().Perm() + } tmp := configPath + ".tmp" - if err := os.WriteFile(tmp, out, 0644); err != nil { + if err := os.WriteFile(tmp, out, mode); err != nil { return fmt.Errorf("write config: %w", err) } if err := os.Rename(tmp, configPath); err != nil { diff --git a/cmd/server/config_mode_test.go b/cmd/server/config_mode_test.go new file mode 100644 index 000000000..601ef7656 --- /dev/null +++ b/cmd/server/config_mode_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// SaveGeoFilter rewrites config.json through a temp file. The file holds the +// API key and broker passwords, so the rewrite must keep whatever mode the +// operator set rather than resetting it to 0644. +func TestSaveGeoFilterPreservesFileMode(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "config.json") + if err := os.WriteFile(path, []byte(`{"apiKey":"secret-key-that-is-long-enough"}`+"\n"), 0600); err != nil { + t.Fatal(err) + } + gf := &GeoFilterConfig{Polygon: [][2]float64{{0, 0}, {0, 1}, {1, 1}, {1, 0}}} + if err := SaveGeoFilter(dir, gf); err != nil { + t.Fatalf("SaveGeoFilter: %v", err) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if got := fi.Mode().Perm(); got != 0600 { + t.Fatalf("config.json mode after save = %o, want 0600", got) + } + // Default stays 0644 for a file that was 0644. + if err := os.Chmod(path, 0644); err != nil { + t.Fatal(err) + } + if err := SaveGeoFilter(dir, nil); err != nil { + t.Fatalf("SaveGeoFilter(nil): %v", err) + } + fi, _ = os.Stat(path) + if got := fi.Mode().Perm(); got != 0644 { + t.Fatalf("config.json mode after save = %o, want 0644", got) + } +} From 4ce6d9f6d40400fc00b5e33976a9a756be716b5a Mon Sep 17 00:00:00 2001 From: nullrouten0 Date: Thu, 8 Oct 2026 06:56:48 -0700 Subject: [PATCH 102/151] fix(ui): pin CDN script versions and add integrity hashes (#2121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `leaflet.heat` and `chart.js` in `index.html`, and swagger-ui on `/api/docs`, were loaded from unpkg with no `integrity` attribute. `chart.js@4` and `swagger-ui-dist@5` also floated on a major version, so a new release would load unreviewed. A compromised CDN or package would run as our own code on every page. Leaflet itself already had a hash. **Fix:** pin `chart.js@4.5.1`, `leaflet.heat@0.2.0`, `swagger-ui-dist@5.33.1`, each with a sha384 hash and `crossorigin="anonymous"`. **How the hashes were made:** download each pinned file, `openssl dgst -sha384 -binary | base64`, then download again and check the hash matches. **Note:** bumping chart.js or swagger-ui now means updating the hash too. Vendoring them into `public/vendor/` (as markercluster already is) would remove the CDN dependency entirely; happy to do that instead if preferred. Running in production on our instance since 2026-10-07. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Mythos 5.1 --- cmd/server/openapi.go | 8 ++++++-- public/index.html | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index ea923eaf7..0a532639c 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -551,7 +551,9 @@ const swaggerUIHTML = ` CoreScope API — Swagger UI - +