diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 8e154d046..af1f51f1e 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -53,6 +53,8 @@ func routeDescriptions() map[string]routeMeta { "GET /api/regions/observed": {Summary: "Observed scoped relay positions", Description: "Bulk Regions map snapshot with selectable 1h, 24h or 7d window. Includes unpositioned scoped relays, current identity visibility filters, conservative retained-history coverage and explicit node/scope truncation flags. Reported positions are self-reported coordinates; neighbor estimates are excluded. Window bounds end at generatedAt, the shared relay snapshot time (normally refreshed every five minutes).", Tag: "analytics", Response: schemaRef("RegionsObservedResponse"), QueryParams: []paramMeta{{Name: "window", Description: "1h, 24h (default), or 7d", Type: "string"}}, Errors: []routeError{{Status: "400", Description: "Invalid window."}, {Status: "500", Description: "Current identity or node metadata could not be read."}, {Status: "503", Description: "Relay snapshot is unavailable or warming up."}}}, + "GET /api/regions/history": {Summary: "Hourly scoped relay history", Description: "One bounded bulk snapshot for Regions timeline and equal-window comparison. Includes 504 UTC clock-hour buckets over 21 days, excluding the unfinished current hour. Counts distinct transmissions per confirmed node and scope, assigned by transmission FirstSeen; summed node counts are relay appearances. Metadata is current, or archived only when no current row exists. Current catalog identity resolution and live visibility policies apply. Coverage and truncation must be checked before interpreting changes.", Tag: "analytics", Response: schemaRef("RegionsHistoryResponse"), + Errors: []routeError{{Status: "500", Description: "Identity or node metadata could not be read."}, {Status: "503", Description: "Relay history snapshot is unavailable or warming up."}}}, "GET /api/config/theme": {Summary: "Get theme configuration", Description: "Returns color maps, CSS variables, and theme defaults.", Tag: "config"}, "GET /api/config/map": {Summary: "Get map configuration", Tag: "config"}, "GET /api/config/geo-filter": {Summary: "Get geo-filter configuration", Tag: "config"}, @@ -1013,6 +1015,9 @@ func componentSchemas() map[string]interface{} { for name, schema := range regionsObservedSchemas() { schemas[name] = schema } + for name, schema := range regionsHistorySchemas() { + schemas[name] = schema + } for name, schema := range heardBySchemas() { schemas[name] = schema } diff --git a/cmd/server/regions_history.go b/cmd/server/regions_history.go new file mode 100644 index 000000000..625466b9b --- /dev/null +++ b/cmd/server/regions_history.go @@ -0,0 +1,290 @@ +package main + +import ( + "log" + "net/http" + "sort" + "time" + + "github.com/meshcore-analyzer/regions" +) + +const ( + regionsHistoryBucketCount = 21 * 24 + maxRegionsHistoryCells = 250000 +) + +// Published with the existing relay snapshot. These slices are immutable and +// contain only confirmed scoped relay evidence, never raw packet contents. +type regionsRelayHistory struct { + start, end time.Time + nodes map[string][]RegionsHistoryScope + truncated bool + scopesTruncated bool + bucketsTruncated bool +} + +type RegionsHistoryCoverage struct { + Start *string `json:"start"` + Loading bool `json:"loading"` +} + +type RegionsHistoryBucket struct { + Index int `json:"index"` + Count int `json:"count"` + LastRelayed string `json:"lastRelayed"` +} + +type RegionsHistoryScope struct { + Name string `json:"name"` + Buckets []RegionsHistoryBucket `json:"buckets"` +} + +type RegionsHistoryNode struct { + PublicKey string `json:"publicKey"` + Name string `json:"name"` + Role string `json:"role"` + Lat *float64 `json:"lat"` + Lon *float64 `json:"lon"` + PositionSource string `json:"positionSource"` + MetadataSource string `json:"metadataSource"` + Scopes []RegionsHistoryScope `json:"scopes"` +} + +type RegionsHistoryResponse struct { + GeneratedAt string `json:"generatedAt"` + HistoryStart string `json:"historyStart"` + HistoryEnd string `json:"historyEnd"` + BucketSeconds int `json:"bucketSeconds"` + BucketCount int `json:"bucketCount"` + Coverage RegionsHistoryCoverage `json:"coverage"` + Nodes []RegionsHistoryNode `json:"nodes"` + Truncated bool `json:"truncated"` + ScopesTruncated bool `json:"scopesTruncated"` + BucketsTruncated bool `json:"bucketsTruncated"` +} + +type regionsHistoryBuilder struct { + history *regionsRelayHistory + remainingCells int +} + +func newRegionsHistoryBuilder(now time.Time) *regionsHistoryBuilder { + end := now.UTC().Truncate(time.Hour) + return ®ionsHistoryBuilder{history: ®ionsRelayHistory{ + start: end.Add(-regionsHistoryBucketCount * time.Hour), end: end, + nodes: make(map[string][]RegionsHistoryScope), + }, remainingCells: maxRegionsHistoryCells} +} + +func historyRelayTime(e relayEntry, start, end time.Time) (time.Time, bool) { + if e.fromPrefix || e.pt == payloadTypeAdvert || regions.ValidateCandidate(e.scope) != nil { + return time.Time{}, false + } + t, valid := e.t, e.valid + if !e.parsed { + t, valid = parseRelayTS(e.ts) + } + return t, valid && !t.Before(start) && t.Before(end) +} + +// add receives the already deduplicated, confirmed entries from the existing +// bulk relay computation, after the packet lock has been released. Callers +// visit keys in sorted order so the shared cell budget is deterministic. +func (b *regionsHistoryBuilder) add(key string, entries []relayEntry) { + if len(key) != 64 { + return + } + h := b.history + // Select the 32 newest valid scopes with bounded scratch space, even if a + // sender introduces arbitrarily many names in the retained packet store. + latest := make(map[string]time.Time, maxTransportedScopes) + for _, e := range entries { + t, valid := historyRelayTime(e, h.start, h.end) + if !valid { + continue + } + if have, exists := latest[e.scope]; exists { + if t.After(have) { + latest[e.scope] = t + } + continue + } + if len(latest) < maxTransportedScopes { + latest[e.scope] = t + continue + } + h.scopesTruncated = true + var oldestName string + var oldest time.Time + for name, when := range latest { + if oldestName == "" || when.Before(oldest) || (when.Equal(oldest) && name > oldestName) { + oldestName, oldest = name, when + } + } + if t.After(oldest) || (t.Equal(oldest) && e.scope < oldestName) { + delete(latest, oldestName) + latest[e.scope] = t + } + } + if len(latest) == 0 { + return + } + if len(h.nodes) >= maxObservedRegionNodes { + h.truncated = true + return + } + if b.remainingCells == 0 { + h.bucketsTruncated = true + return + } + // Scratch aggregation is bounded by 32 scopes x 504 hours. Only after + // sorting scopes and indexes do we spend the global publication budget, + // so its selection never depends on packet or map iteration order. + type historyCell struct { + count int + latest time.Time + } + cells := make(map[string]map[int]historyCell, len(latest)) + for _, e := range entries { + t, valid := historyRelayTime(e, h.start, h.end) + if !valid { + continue + } + if _, selected := latest[e.scope]; !selected { + continue + } + index := int(t.Sub(h.start) / time.Hour) + byIndex := cells[e.scope] + bucket, exists := byIndex[index] + if !exists { + if byIndex == nil { + byIndex = make(map[int]historyCell) + cells[e.scope] = byIndex + } + } + bucket.count++ + if t.After(bucket.latest) { + bucket.latest = t + } + byIndex[index] = bucket + } + scopes := make([]RegionsHistoryScope, 0, len(cells)) + names := make([]string, 0, len(cells)) + for name := range cells { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + byIndex := cells[name] + indexes := make([]int, 0, len(byIndex)) + for index := range byIndex { + indexes = append(indexes, index) + } + sort.Ints(indexes) + retain := len(indexes) + if retain > b.remainingCells { + retain = b.remainingCells + h.bucketsTruncated = true + } + if retain == 0 { + continue + } + buckets := make([]RegionsHistoryBucket, 0, retain) + for _, index := range indexes[:retain] { + cell := byIndex[index] + buckets = append(buckets, RegionsHistoryBucket{Index: index, Count: cell.count, LastRelayed: cell.latest.UTC().Format(time.RFC3339Nano)}) + } + b.remainingCells -= retain + scopes = append(scopes, RegionsHistoryScope{Name: name, Buckets: buckets}) + } + if len(scopes) > 0 { + h.nodes[key] = scopes + } +} + +func (s *Server) handleRegionsHistory(w http.ResponseWriter, r *http.Request) { + if s.store == nil || s.db == nil || s.db.conn == nil { + writeError(w, http.StatusServiceUnavailable, "observed relay data unavailable") + return + } + s.store.repeaterEnrichMu.Lock() + snapshot := s.store.repeaterRelaySnapshot + s.store.repeaterEnrichMu.Unlock() + if snapshot == nil || snapshot.history == nil { + writeError(w, http.StatusServiceUnavailable, "observed relay history is warming up") + return + } + h := snapshot.history + response := RegionsHistoryResponse{GeneratedAt: snapshot.generatedAt.Format(time.RFC3339Nano), + HistoryStart: h.start.Format(time.RFC3339Nano), HistoryEnd: h.end.Format(time.RFC3339Nano), + BucketSeconds: 3600, BucketCount: regionsHistoryBucketCount, + Coverage: RegionsHistoryCoverage{Loading: snapshot.loading}, Nodes: []RegionsHistoryNode{}, + Truncated: h.truncated, ScopesTruncated: h.scopesTruncated, BucketsTruncated: h.bucketsTruncated} + if !snapshot.coverageStart.IsZero() && !snapshot.coverageStart.After(snapshot.generatedAt) { + start := snapshot.coverageStart.Format(time.RFC3339Nano) + response.Coverage.Start = &start + } + keys := make([]string, 0, len(h.nodes)) + for key := range h.nodes { + keys = append(keys, key) + } + err := s.forEachVisibleRegionNode(r.Context(), keys, true, func(n observedRegionNodeMetadata) bool { + scopes := h.nodes[n.publicKey] + if len(scopes) == 0 { + return true + } + if len(response.Nodes) >= maxObservedRegionNodes { + response.Truncated = true + return false + } + node := RegionsHistoryNode{PublicKey: n.publicKey, Name: n.name, Role: n.role, + PositionSource: "none", MetadataSource: n.source, Scopes: scopes} + if node.Name == "" { + node.Name = n.publicKey + } + if reportedRegionPosition(n.lat, n.lon) { + lat, lon := n.lat.Float64, n.lon.Float64 + node.Lat, node.Lon, node.PositionSource = &lat, &lon, "reported" + } + response.Nodes = append(response.Nodes, node) + return true + }) + if err != nil { + log.Printf("[regions-history] node metadata: %v", err) + writeError(w, http.StatusInternalServerError, "could not read region node metadata") + return + } + w.Header().Set("Cache-Control", "no-store") + writeJSON(w, response) +} + +func regionsHistorySchemas() map[string]*openAPISchema { + return map[string]*openAPISchema{ + "RegionsHistoryCoverage": {Type: "object", Properties: map[string]*openAPISchema{ + "start": {Type: "string", Nullable: true, Description: "Conservative retained-history floor, paired with this snapshot; null when unknown."}, + "loading": {Type: "boolean"}, + }}, + "RegionsHistoryBucket": {Type: "object", Properties: map[string]*openAPISchema{ + "index": {Type: "integer", Description: "Zero-based hourly bucket index, 0 through 503."}, + "count": {Type: "integer", Description: "Distinct confirmed transmissions for this node and scope. Summing nodes counts relay appearances, not globally unique transmissions."}, + "lastRelayed": {Type: "string", Description: "Latest transmission FirstSeen in this bucket (RFC3339)."}, + }}, + "RegionsHistoryScope": {Type: "object", Properties: map[string]*openAPISchema{ + "name": {Type: "string"}, "buckets": {Type: "array", Items: openAPIRef("RegionsHistoryBucket"), Description: "Only populated buckets, ascending by index."}, + }}, + "RegionsHistoryNode": {Type: "object", Properties: map[string]*openAPISchema{ + "publicKey": {Type: "string"}, "name": {Type: "string"}, "role": {Type: "string", Enum: []string{"repeater", "room"}}, + "lat": {Type: "number", Nullable: true}, "lon": {Type: "number", Nullable: true}, + "positionSource": {Type: "string", Enum: []string{"reported", "none"}}, + "metadataSource": {Type: "string", Enum: []string{"current", "archived"}, Description: "Current metadata when a nodes row exists; otherwise inactive_nodes metadata. Neither represents a historical position sample."}, + "scopes": {Type: "array", Items: openAPIRef("RegionsHistoryScope")}, + }}, + "RegionsHistoryResponse": {Type: "object", Properties: map[string]*openAPISchema{ + "generatedAt": {Type: "string"}, "historyStart": {Type: "string"}, "historyEnd": {Type: "string", Description: "UTC completed-hour boundary at or before generatedAt. Current unfinished hour is excluded."}, + "bucketSeconds": {Type: "integer", Description: "3600."}, "bucketCount": {Type: "integer", Description: "504 (21 days)."}, + "coverage": openAPIRef("RegionsHistoryCoverage"), "nodes": {Type: "array", Items: openAPIRef("RegionsHistoryNode")}, + "truncated": {Type: "boolean"}, "scopesTruncated": {Type: "boolean"}, "bucketsTruncated": {Type: "boolean"}, + }}, + } +} diff --git a/cmd/server/regions_history_test.go b/cmd/server/regions_history_test.go new file mode 100644 index 000000000..c8a1abb2f --- /dev/null +++ b/cmd/server/regions_history_test.go @@ -0,0 +1,358 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +func historyEntry(at time.Time, scope string) relayEntry { + return relayEntry{t: at, parsed: true, valid: true, pt: 5, scope: scope} +} + +func TestRegionsHistoryBucketsAndScopeProvenance(t *testing.T) { + now := time.Date(2026, 10, 16, 12, 35, 0, 0, time.UTC) + end := now.Truncate(time.Hour) + start := end.Add(-21 * 24 * time.Hour) + entries := []relayEntry{historyEntry(start, "#sample"), historyEntry(end.Add(-time.Nanosecond), "#sample"), + historyEntry(end.Add(-time.Hour), "#sample"), historyEntry(start.Add(-time.Nanosecond), "#old"), + historyEntry(end, "#unfinished"), historyEntry(now.Add(time.Hour), "#future")} + entries = append(entries, relayEntry{ts: "invalid", scope: "#bad"}, historyEntry(end.Add(-time.Minute), "invalid")) + prefix := historyEntry(end.Add(-time.Minute), "#prefix") + prefix.fromPrefix = true + advert := historyEntry(end.Add(-time.Minute), "#advert") + advert.pt = payloadTypeAdvert + entries = append(entries, prefix, advert) + b := newRegionsHistoryBuilder(now) + b.add(strings.Repeat("ab", 32), entries) + h := b.history + if !h.start.Equal(start) || !h.end.Equal(end) || len(h.nodes) != 1 { + t.Fatalf("history bounds/nodes: %+v", h) + } + scopes := h.nodes[strings.Repeat("ab", 32)] + if len(scopes) != 1 || scopes[0].Name != "#sample" || len(scopes[0].Buckets) != 2 { + t.Fatalf("scope provenance/bounds: %+v", scopes) + } + if scopes[0].Buckets[0].Index != 0 || scopes[0].Buckets[0].Count != 1 || scopes[0].Buckets[1].Index != 503 || scopes[0].Buckets[1].Count != 2 { + t.Fatalf("bucket boundary counts: %+v", scopes[0].Buckets) + } + if scopes[0].Buckets[1].LastRelayed != end.Add(-time.Nanosecond).Format(time.RFC3339Nano) { + t.Fatal("latest evidence was lost") + } +} + +func TestRegionsHistoryAdjacentSevenDayWindows(t *testing.T) { + now := time.Date(2026, 10, 16, 12, 35, 0, 0, time.UTC) + end := now.Truncate(time.Hour) + entries := []relayEntry{historyEntry(end.Add(-14*24*time.Hour), "#sample"), historyEntry(end.Add(-7*24*time.Hour), "#sample"), historyEntry(end.Add(-time.Second), "#sample")} + b := newRegionsHistoryBuilder(now) + key := strings.Repeat("ab", 32) + b.add(key, entries) + previous, current := 0, 0 + for _, bucket := range b.history.nodes[key][0].Buckets { + if bucket.Index >= 168 && bucket.Index < 336 { + previous += bucket.Count + } + if bucket.Index >= 336 && bucket.Index < 504 { + current += bucket.Count + } + } + if previous != 1 || current != 2 { + t.Fatalf("adjacent windows overlap or lose boundary: %d %d", previous, current) + } +} + +func TestRegionsHistoryBoundedScopesAndCells(t *testing.T) { + now := time.Date(2026, 10, 16, 12, 35, 0, 0, time.UTC) + b := newRegionsHistoryBuilder(now) + b.remainingCells = 2 + var entries []relayEntry + for i := 0; i < 40; i++ { + entries = append(entries, historyEntry(b.history.end.Add(-time.Duration(i+1)*time.Hour), fmt.Sprintf("#s%02d", i))) + } + key := strings.Repeat("ab", 32) + b.add(key, entries) + if !b.history.scopesTruncated || !b.history.bucketsTruncated || len(b.history.nodes[key]) > 32 || b.remainingCells != 0 { + t.Fatalf("bounds/flags: %+v", b.history) + } + for _, scope := range b.history.nodes[key] { + if scope.Name == "#s39" { + t.Fatal("old scope displaced recent scope") + } + } + reversed := append([]relayEntry(nil), entries...) + for i, j := 0, len(reversed)-1; i < j; i, j = i+1, j-1 { + reversed[i], reversed[j] = reversed[j], reversed[i] + } + other := newRegionsHistoryBuilder(now) + other.remainingCells = 2 + other.add(key, reversed) + if !reflect.DeepEqual(b.history.nodes, other.history.nodes) { + t.Fatal("cell budget depends on entry order") + } +} + +func getRegionsHistory(t testing.TB, srv *Server) RegionsHistoryResponse { + t.Helper() + w := httptest.NewRecorder() + srv.handleRegionsHistory(w, httptest.NewRequest("GET", "/api/regions/history", nil)) + var got RegionsHistoryResponse + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &got) != nil { + t.Fatalf("history response %d: %s", w.Code, w.Body.String()) + } + return got +} + +func TestRegionsHistoryDedupCoverageAndWarmLock(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + pt, rt := 5, routeTypeFlood + tx := &StoreTx{ID: 5, FirstSeen: now.Add(-2 * time.Hour).Format(time.RFC3339Nano), PayloadType: &pt, RouteType: &rt, ScopeName: "#sample", PathJSON: `["` + key + `","` + key + `"]`} + srv.store.byPathHop[key] = []*StoreTx{tx, tx} + srv.store.byNode[key] = []*StoreTx{tx} + recomputeRepeaterEnrichmentSafe(srv.store, 24) + got := getRegionsHistory(t, srv) + if len(got.Nodes) != 1 || len(got.Nodes[0].Scopes) != 1 || got.Nodes[0].Scopes[0].Buckets[0].Count != 1 { + t.Fatalf("duplicate transmission counted: %+v", got.Nodes) + } + if got.Coverage.Start == nil || got.Coverage.Loading || got.HistoryEnd == got.GeneratedAt || got.BucketCount != 504 { + t.Fatalf("paired metadata: %+v", got) + } + srv.store.mu.Lock() + srv.store.oldestLoaded = now.Format(time.RFC3339) + srv.store.backgroundLoadDone.Store(false) + done := make(chan *httptest.ResponseRecorder, 1) + go func() { + w := httptest.NewRecorder() + srv.handleRegionsHistory(w, httptest.NewRequest("GET", "/api/regions/history", nil)) + done <- w + }() + select { + case w := <-done: + if w.Code != 200 { + t.Errorf("warm response: %d", w.Code) + } + case <-time.After(2 * time.Second): + srv.store.mu.Unlock() + t.Fatal("warm history request waits for packet lock") + } + srv.store.mu.Unlock() + warm := getRegionsHistory(t, srv) + if warm.Coverage.Start == nil || *warm.Coverage.Start != *got.Coverage.Start || warm.Coverage.Loading { + t.Fatal("coverage changed independently of cached evidence") + } +} + +func TestRegionsHistoryArchivedMetadataAndVisibility(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + seedRegionRelaySnapshot(t, srv, now, key) + if _, err := srv.db.conn.Exec(`CREATE TABLE inactive_nodes(public_key TEXT PRIMARY KEY, name TEXT, role TEXT, lat REAL, lon REAL, foreign_advert INTEGER); + INSERT INTO inactive_nodes SELECT public_key, 'Archived relay', role, lat, lon, foreign_advert FROM nodes`); err != nil { + t.Fatal(err) + } + got := getRegionsHistory(t, srv) + if len(got.Nodes) != 1 || got.Nodes[0].MetadataSource != "current" || got.Nodes[0].Name != "Synthetic relay" { + t.Fatalf("current precedence: %+v", got.Nodes) + } + if _, err := srv.db.conn.Exec("UPDATE nodes SET role='companion'"); err != nil { + t.Fatal(err) + } + if got := getRegionsHistory(t, srv); len(got.Nodes) != 0 { + t.Fatal("archived repeater resurrected current companion") + } + if _, err := srv.db.conn.Exec("DELETE FROM nodes"); err != nil { + t.Fatal(err) + } + got = getRegionsHistory(t, srv) + if len(got.Nodes) != 1 || got.Nodes[0].MetadataSource != "archived" || got.Nodes[0].Name != "Archived relay" { + t.Fatalf("archive fallback: %+v", got.Nodes) + } + srv.cfg.SetHiddenNamePrefixes([]string{"Hidden"}) + if _, err := srv.db.conn.Exec("INSERT INTO observers(id,name) VALUES(?, 'Hidden observer alias')", strings.ToUpper(key)); err != nil { + t.Fatal(err) + } + if got := getRegionsHistory(t, srv); len(got.Nodes) != 0 { + t.Fatal("archived identity observer alias leaked") + } + srv.cfg.SetHiddenNamePrefixes(nil) + srv.cfg.SetNodeBlacklist([]string{key}) + if got := getRegionsHistory(t, srv); len(got.Nodes) != 0 { + t.Fatal("warm snapshot blacklist leaked") + } + srv.cfg.SetNodeBlacklist(nil) + srv.cfg.ObserverBlacklist = []string{key} + if got := getRegionsHistory(t, srv); len(got.Nodes) != 0 { + t.Fatal("observer blacklist leaked") + } + srv.cfg.ObserverBlacklist = nil + srv.setGeoFilter(&GeoFilterConfig{Polygon: [][2]float64{{51, 4}, {51, 5}, {50, 5}, {50, 4}}}) + if got := getRegionsHistory(t, srv); len(got.Nodes) != 0 { + t.Fatal("archive bypassed geographic filter") + } + if _, err := srv.db.conn.Exec("UPDATE inactive_nodes SET foreign_advert=1"); err != nil { + t.Fatal(err) + } + if got := getRegionsHistory(t, srv); len(got.Nodes) != 1 { + t.Fatal("archive lost foreign-node exception") + } + srv.setGeoFilter(nil) + if _, err := srv.db.conn.Exec("UPDATE inactive_nodes SET lat=NULL,lon=NULL"); err != nil { + t.Fatal(err) + } + got = getRegionsHistory(t, srv) + if len(got.Nodes) != 1 || got.Nodes[0].Lat != nil || got.Nodes[0].PositionSource != "none" { + t.Fatal("unpositioned archived relay lost") + } +} + +func TestRegionsHistoryNodeCapAndAvailability(t *testing.T) { + now := time.Date(2026, 10, 16, 12, 35, 0, 0, time.UTC) + b := newRegionsHistoryBuilder(now) + entries := []relayEntry{historyEntry(b.history.end.Add(-time.Minute), "#sample")} + for i := 0; i <= maxObservedRegionNodes; i++ { + b.add(fmt.Sprintf("%064x", i+1), entries) + } + if len(b.history.nodes) != maxObservedRegionNodes || !b.history.truncated { + t.Fatal("snapshot node bound not enforced") + } + srv, _, _ := regionsObservedFixture(t) + w := httptest.NewRecorder() + srv.handleRegionsHistory(w, httptest.NewRequest("GET", "/api/regions/history", nil)) + if w.Code != 503 || srv.store.repeaterRelayCache != nil { + t.Fatal("warming history computed inline") + } + seedRegionRelaySnapshot(t, srv, now, strings.Repeat("ab", 32)) + if _, err := srv.db.conn.Exec("DROP TABLE nodes"); err != nil { + t.Fatal(err) + } + w = httptest.NewRecorder() + srv.handleRegionsHistory(w, httptest.NewRequest("GET", "/api/regions/history", nil)) + if w.Code != 500 { + t.Fatalf("metadata error became empty success: %d", w.Code) + } +} + +func TestRegionsHistoryActualGlobalCellBudget(t *testing.T) { + now := time.Date(2026, 10, 16, 12, 35, 0, 0, time.UTC) + b := newRegionsHistoryBuilder(now) + entries := make([]relayEntry, 501) + for i := range entries { + entries[i] = historyEntry(b.history.start.Add(time.Duration(i)*time.Hour), "#sample") + } + for i := 0; i < 500; i++ { + b.add(fmt.Sprintf("%064x", i+1), entries) + } + cells := 0 + for _, scopes := range b.history.nodes { + for _, scope := range scopes { + cells += len(scope.Buckets) + } + } + if cells != maxRegionsHistoryCells || b.remainingCells != 0 || !b.history.bucketsTruncated { + t.Fatalf("global budget: cells=%d remaining=%d truncated=%v", cells, b.remainingCells, b.history.bucketsTruncated) + } +} + +func TestOpenAPIRegionsHistorySchema(t *testing.T) { + meta := routeDescriptions()["GET /api/regions/history"] + if meta.Response["$ref"] != "#/components/schemas/RegionsHistoryResponse" { + t.Fatal("history response schema missing") + } + schemas := componentSchemas() + response := schemas["RegionsHistoryResponse"].(*openAPISchema) + if response.Properties["nodes"].Items.Ref != "#/components/schemas/RegionsHistoryNode" || response.Properties["bucketCount"].Type != "integer" { + t.Fatal("typed history contract missing") + } + node := schemas["RegionsHistoryNode"].(*openAPISchema) + if !node.Properties["lat"].Nullable || len(node.Properties["metadataSource"].Enum) != 2 { + t.Fatal("metadata uncertainty missing") + } + if !schemas["RegionsHistoryCoverage"].(*openAPISchema).Properties["start"].Nullable { + t.Fatal("unknown coverage missing") + } +} + +func regionsHistoryBenchmarkFixture(b *testing.B) (*Server, time.Time) { + srv, now, _ := regionsObservedFixture(b) + pt, rt := 5, routeTypeFlood + end := now.Truncate(time.Hour) + transaction, err := srv.db.conn.Begin() + if err != nil { + b.Fatal(err) + } + stmt, err := transaction.Prepare("INSERT INTO nodes(public_key,name,role,lat,lon) VALUES(?,'Synthetic history relay','repeater',55.2,10.3)") + if err != nil { + b.Fatal(err) + } + for i := 0; i < 500; i++ { + key := fmt.Sprintf("%064x", i+1) + if _, err := stmt.Exec(key); err != nil { + b.Fatal(err) + } + for j := 0; j < 60; j++ { + tx := &StoreTx{ID: i*60 + j + 1, FirstSeen: end.Add(-time.Duration(j*8+1) * time.Hour).Format(time.RFC3339Nano), PayloadType: &pt, RouteType: &rt, ScopeName: "#sample", PathJSON: `["` + key + `"]`} + srv.store.byPathHop[key] = append(srv.store.byPathHop[key], tx) + } + } + stmt.Close() + if err := transaction.Commit(); err != nil { + b.Fatal(err) + } + return srv, now +} + +func BenchmarkRegionsHistoryCompute30K(b *testing.B) { + srv, _ := regionsHistoryBenchmarkFixture(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = srv.store.computeRepeaterRelaySnapshot(24) + } +} + +func BenchmarkRegionsHistoryWarm30K(b *testing.B) { + srv, _ := regionsHistoryBenchmarkFixture(b) + recomputeRepeaterEnrichmentSafe(srv.store, 24) + srv.store.byPathHop = nil + response := getRegionsHistory(b, srv) + encoded, err := json.Marshal(response) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + w := httptest.NewRecorder() + srv.handleRegionsHistory(w, httptest.NewRequest("GET", "/api/regions/history", nil)) + if w.Code != 200 { + b.Fatal(w.Code) + } + } + b.ReportMetric(float64(len(encoded)), "payload-bytes") +} + +func BenchmarkRegionsHistoryCapPayload(b *testing.B) { + response := RegionsHistoryResponse{Nodes: []RegionsHistoryNode{}} + for i := 0; i < 500; i++ { + buckets := make([]RegionsHistoryBucket, 500) + for j := range buckets { + buckets[j] = RegionsHistoryBucket{Index: j, Count: 12345, LastRelayed: "2026-10-16T11:59:59.123456789Z"} + } + response.Nodes = append(response.Nodes, RegionsHistoryNode{PublicKey: fmt.Sprintf("%064x", i), Name: "Synthetic benchmark", Role: "repeater", PositionSource: "reported", MetadataSource: "current", Scopes: []RegionsHistoryScope{{Name: "#sample", Buckets: buckets}}}) + } + encoded, err := json.Marshal(response) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := json.Marshal(response); err != nil { + b.Fatal(err) + } + } + b.ReportMetric(float64(len(encoded)), "payload-bytes") +} diff --git a/cmd/server/regions_observed.go b/cmd/server/regions_observed.go index ed95e5998..38a4c397a 100644 --- a/cmd/server/regions_observed.go +++ b/cmd/server/regions_observed.go @@ -21,6 +21,7 @@ type regionScopeRelay struct { type repeaterRelaySnapshot struct { info map[string]RepeaterRelayInfo + history *regionsRelayHistory generatedAt time.Time coverageStart time.Time loading bool @@ -96,17 +97,21 @@ func observedRegionWindow(value string) (string, time.Duration, bool) { } type observedRegionNodeMetadata struct { - publicKey, name, role string - lat, lon sql.NullFloat64 - foreign bool + publicKey, name, role, source string + lat, lon sql.NullFloat64 + foreign bool } // forEachObservedRegionNode streams current metadata with one bulk read. It // runs outside s.mu and checks Scan/Rows errors rather than reporting an empty // successful result when node metadata is unavailable. func (db *DB) forEachObservedRegionNode(ctx context.Context, visit func(observedRegionNodeMetadata) bool) error { - rows, err := db.conn.QueryContext(ctx, `SELECT public_key, name, role, lat, lon, foreign_advert - FROM nodes WHERE role IN ('repeater','room') ORDER BY public_key`) + return db.forEachRegionNodeMetadata(ctx, `SELECT public_key, name, role, lat, lon, foreign_advert, 'current' + FROM nodes WHERE role IN ('repeater','room') ORDER BY public_key`, visit) +} + +func (db *DB) forEachRegionNodeMetadata(ctx context.Context, query string, visit func(observedRegionNodeMetadata) bool) error { + rows, err := db.conn.QueryContext(ctx, query) if err != nil { return err } @@ -115,7 +120,7 @@ func (db *DB) forEachObservedRegionNode(ctx context.Context, visit func(observed var n observedRegionNodeMetadata var name, role sql.NullString var foreign sql.NullInt64 - if err := rows.Scan(&n.publicKey, &name, &role, &n.lat, &n.lon, &foreign); err != nil { + if err := rows.Scan(&n.publicKey, &name, &role, &n.lat, &n.lon, &foreign, &n.source); err != nil { return err } n.publicKey, n.name, n.role, n.foreign = strings.ToLower(n.publicKey), name.String, role.String, foreign.Valid && foreign.Int64 != 0 @@ -126,6 +131,47 @@ func (db *DB) forEachObservedRegionNode(ctx context.Context, visit func(observed return rows.Err() } +// Both Regions endpoints apply live visibility through the same bulk metadata +// cursor. Any current row supersedes archived metadata, even a non-relay role. +func (s *Server) forEachVisibleRegionNode(ctx context.Context, keys []string, includeArchived bool, visit func(observedRegionNodeMetadata) bool) error { + var hiddenNames map[string][]string + if s.cfg != nil && len(s.cfg.EnforcedHiddenNamePrefixes()) > 0 { + var err error + hiddenNames, err = s.hiddenIdentityNames(ctx, keys) + if err != nil { + return err + } + } + geo := s.getGeoFilter() + applyGeo := s.cfg != nil && !s.cfg.GeoFilterExemptNodeList + visible := func(n observedRegionNodeMetadata) bool { + if identityHidden(s.cfg, n.publicKey, n.name) || identityHidden(s.cfg, n.publicKey, hiddenNames[n.publicKey]...) { + return true + } + if geo != nil && applyGeo && !n.foreign && !NodePassesGeoFilter(nullFloat(n.lat), nullFloat(n.lon), geo) { + return true + } + return visit(n) + } + if !includeArchived { + return s.db.forEachObservedRegionNode(ctx, visible) + } + hasArchived, err := s.hasInactiveNodesTable(ctx) + if err != nil { + return err + } + if !hasArchived { + return s.db.forEachObservedRegionNode(ctx, visible) + } + return s.db.forEachRegionNodeMetadata(ctx, `SELECT public_key, name, role, lat, lon, foreign_advert, 'current' AS source + FROM nodes WHERE role IN ('repeater','room') + UNION ALL + SELECT i.public_key, i.name, i.role, i.lat, i.lon, i.foreign_advert, 'archived' AS source + FROM inactive_nodes i WHERE i.role IN ('repeater','room') + AND NOT EXISTS (SELECT 1 FROM nodes n WHERE n.public_key=i.public_key) + ORDER BY public_key`, visible) +} + func reportedRegionPosition(lat, lon sql.NullFloat64) bool { return lat.Valid && lon.Valid && isValidNodePosition(lat.Float64, lon.Float64) } @@ -156,34 +202,16 @@ func (s *Server) handleRegionsObserved(w http.ResponseWriter, r *http.Request) { response.Coverage.Start = &floor response.Coverage.Complete = !snapshot.coverageStart.After(start) } - geo := s.getGeoFilter() - applyGeo := s.cfg != nil && !s.cfg.GeoFilterExemptNodeList // A relay can also be known as an observer under another display name. // Resolve hiding aliases in bulk before opening the metadata cursor: both // queries must also work with SQLite's single-connection test databases. - var hiddenNames map[string][]string - if s.cfg != nil && len(s.cfg.EnforcedHiddenNamePrefixes()) > 0 { - keys := make([]string, 0, len(snapshot.info)) - for key, info := range snapshot.info { - if len(key) == 64 && len(info.regionScopes) > 0 { - keys = append(keys, key) - } - } - var err error - hiddenNames, err = s.hiddenIdentityNames(r.Context(), keys) - if err != nil { - log.Printf("[regions-observed] identity metadata: %v", err) - writeError(w, http.StatusInternalServerError, "could not read region identity metadata") - return + keys := make([]string, 0, len(snapshot.info)) + for key, info := range snapshot.info { + if len(key) == 64 && len(info.regionScopes) > 0 { + keys = append(keys, key) } } - err := s.db.forEachObservedRegionNode(r.Context(), func(n observedRegionNodeMetadata) bool { - if identityHidden(s.cfg, n.publicKey, n.name) || identityHidden(s.cfg, n.publicKey, hiddenNames[n.publicKey]...) { - return true - } - if geo != nil && applyGeo && !n.foreign && !NodePassesGeoFilter(nullFloat(n.lat), nullFloat(n.lon), geo) { - return true - } + err := s.forEachVisibleRegionNode(r.Context(), keys, false, func(n observedRegionNodeMetadata) bool { info, exists := snapshot.info[n.publicKey] if !exists { return true diff --git a/cmd/server/repeater_enrich_bulk.go b/cmd/server/repeater_enrich_bulk.go index 1ee8357d2..ee543e2d5 100644 --- a/cmd/server/repeater_enrich_bulk.go +++ b/cmd/server/repeater_enrich_bulk.go @@ -1,6 +1,7 @@ package main import ( + "sort" "strings" "time" ) @@ -196,6 +197,9 @@ func (s *PacketStore) computeRepeaterRelaySnapshot(windowHours float64) *repeate s.mu.RUnlock() out := make(map[string]RepeaterRelayInfo, len(perKey)) + history := newRegionsHistoryBuilder(now) + // Stable order also makes history's global safety-budget selection stable. + sort.Slice(perKey, func(i, j int) bool { return perKey[i].key < perKey[j].key }) var entries []relayEntry for _, k := range perKey { entries = entries[:0] @@ -205,8 +209,10 @@ func (s *PacketStore) computeRepeaterRelaySnapshot(windowHours float64) *repeate entries = append(entries, e) } out[k.key] = computeRelayInfoFromEntriesAt(entries, windowHours, now) + history.add(k.key, entries) } snapshot.info = out + snapshot.history = history.history return snapshot } diff --git a/cmd/server/routes.go b/cmd/server/routes.go index d013c5942..d802650cb 100644 --- a/cmd/server/routes.go +++ b/cmd/server/routes.go @@ -313,6 +313,7 @@ func (s *Server) RegisterRoutes(r *mux.Router) { r.HandleFunc("/api/config/client", s.handleConfigClient).Methods("GET") r.HandleFunc("/api/config/regions", s.handleConfigRegions).Methods("GET") r.HandleFunc("/api/regions/observed", s.handleRegionsObserved).Methods("GET") + r.HandleFunc("/api/regions/history", s.handleRegionsHistory).Methods("GET") r.HandleFunc("/api/config/theme", s.handleConfigTheme).Methods("GET") r.HandleFunc("/api/config/map", s.handleConfigMap).Methods("GET") r.HandleFunc("/api/config/geo-filter", s.handleConfigGeoFilter).Methods("GET") diff --git a/docs/api-spec.md b/docs/api-spec.md index 5dfe6d66e..f0d7516cd 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -52,6 +52,7 @@ - [GET /api/analytics/subpath-detail](#get-apianalyticssubpath-detail) - [GET /api/scope-stats](#get-apiscope-stats) - [GET /api/regions/observed](#get-apiregionsobserved) +- [GET /api/regions/history](#get-apiregionshistory) - [GET /api/resolve-hops](#get-apiresolve-hops) - [GET /api/traces/:hash](#get-apitraceshash) - [GET /api/config/theme](#get-apiconfigtheme) @@ -2505,6 +2506,78 @@ valid scope timestamps; `scopesTruncated` signals omission from that retained scope evidence. Clients must describe incomplete views when either flag is true. Empty lists serialize as arrays. +## GET /api/regions/history + +One bounded bulk snapshot supplies the Regions timeline and comparisons. The +response contains 504 hourly buckets spanning 21 days and takes no query +parameters. `historyEnd` is `generatedAt` rounded down to a completed UTC hour; +the unfinished current hour is excluded. Bucket `i` covers the half-open +interval `[historyStart + i hours, historyStart + (i + 1) hours)`. Empty buckets +are omitted and each scope's populated buckets are sorted by index. These +absolute clock-hour boundaries support stable bookmarked selections. + +```json +{ + "generatedAt": "2026-10-16T12:35:00Z", + "historyStart": "2026-09-25T12:00:00Z", + "historyEnd": "2026-10-16T12:00:00Z", + "bucketSeconds": 3600, + "bucketCount": 504, + "coverage": {"start": "2026-10-02T12:35:00Z", "loading": false}, + "nodes": [{ + "publicKey": "", "name": "Example relay", "role": "repeater", + "lat": 55.2, "lon": 10.3, "positionSource": "reported", + "metadataSource": "current", + "scopes": [{"name": "#example", "buckets": [ + {"index": 503, "count": 12, "lastRelayed": "2026-10-16T11:55:00Z"} + ]}] + }], + "truncated": false, + "scopesTruncated": false, + "bucketsTruncated": false +} +``` + +`count` is distinct confirmed transmissions **per node and scope**, using the +transmission's `FirstSeen` timestamp. It is not the time each observation was +received. Summing counts over nodes produces relay appearances, not globally +unique packets: one transmission can be confirmed through several relays. +Non-advert confirmed scope attribution follows the same identity-safe rules +as `/api/regions/observed`; heuristic-only and raw-prefix attribution does not +broaden scope claims. Scope spelling and case are preserved. + +`coverage.start` and `loading` are captured together with the relay evidence. +The 21-day horizon does not extend configured retention or restore evicted +packets. A null floor, loading state, or floor after a selected interval's +start means that interval is incomplete. A client comparing equal adjacent +1h, 24h or 7d intervals must check **both** intervals and withhold new/no-longer +observed classifications and percentage changes if either is incomplete. +Even complete retained history does not establish continuous observer uptime. + +Names, roles and positions are the latest known metadata, not historical +samples. A `nodes` row always wins over `inactive_nodes`, even when the current +role is no longer a repeater or room (which excludes the node). An archived +row is used only when no current row exists, and is labelled +`metadataSource: "archived"`. Positions remain self-reported and unverified; +missing positions have null coordinates and `positionSource: "none"`. +Relay identity resolution uses the current catalog; the timeline does not +reconstruct the catalog or node positions as they were in each hour. + +The snapshot is rebuilt during the existing background relay computation, +outside the packet lock. Requests read the published snapshot and current +metadata in bulk, with no packet scan or per-node query. Current identity +visibility and geographic policies (including the foreign-node exception) +are applied on every request; responses use `Cache-Control: no-store`. + +Safety limits bound published evidence to 5,000 keys, the 32 newest valid +scopes per key, and 250,000 populated hourly cells globally. The scratch +aggregation for one key is bounded to 32 x 504 cells. Keys, scope names and +bucket indexes are processed in deterministic order. Flags expose omitted +keys, scopes and cells respectively. Any true truncation flag makes change +classifications incomplete. Current visibility filtering can reduce the +returned node count below the evidence cap. HTTP 503 means the snapshot is +warming up or unavailable; metadata errors return HTTP 500. + ## GET /api/scope-stats Scope-based packet statistics over a time window. Requires ingestor `scope_name_v1` migration to have run. diff --git a/docs/regions-observed.md b/docs/regions-observed.md index afa1fa624..23f71725d 100644 --- a/docs/regions-observed.md +++ b/docs/regions-observed.md @@ -32,20 +32,66 @@ The endpoint reuses the background relay snapshot instead of scanning packet history for each period change. Limits on nodes and scope evidence must be reported as truncation, rather than silently suggesting a complete map. +## Timeline and period comparison + +Latest retains the rolling-window view described above. Timeline reconstructs +activity in complete UTC hours and offers hourly steps through the last seven +days, plus local playback. The selected mode, period, scope, timestamp and +comparison state are shareable. Playback never starts automatically from a +link, stops at the newest frame, and stops when the page is hidden or left. + +The history endpoint returns one bounded snapshot of sparse hourly counts. +Its 21-day horizon allows a seven-day interval ending seven days ago to be +compared with the preceding seven days, when that much history is retained. +This is a maximum display horizon; it does not extend database or memory +retention. Refresh can advance the snapshot. Scrubbing and playback do not +request data for each frame, and the map keeps its viewport while playing. + +Intervals are half-open: `[start, end)`. A transmission exactly at the boundary +between two adjacent periods belongs only to the later period. The unfinished +current UTC hour is excluded from Timeline and remains visible in Latest. + +Compare uses the immediately preceding interval of exactly the same duration. +It distinguishes relays observed only in the selected interval, only in the +previous interval, and in both. "Newly observed" describes this pair of periods; +it does not establish a node's creation date. "Not observed" does not establish +that a node is offline. Comparison always identifies both intervals. + +Counts are distinct transmissions per relay and scope. Adding counts across +relays gives **relay appearances**: a packet carried by several relays can +contribute once for each relay. These totals must not be labelled globally +unique mesh packets. Traffic changes and classifications are withheld when +either interval has unknown, loading, partial or truncated evidence. Raw +observed counts may still be shown separately. A zero reference count has no +defined percentage increase. + +The reconstruction uses transmission `first_seen`, the current identity +catalogue and the latest known node metadata. Archived metadata can be used +when no current node row exists; current metadata takes precedence, including +role changes. Positions are the latest known self-reported coordinates in +both periods, not measurements of historical movement. A refresh can revise +historical attribution as the identity catalogue changes. + +History uses the same identity-safe scoped relay attribution as Latest. +Ambiguous prefixes and heuristic-only candidates do not become confirmed +relays merely because the view is historical. Current visibility rules apply +to both periods, including archived identities and observer aliases. + +The history snapshot bounds populated bucket cells globally as well as nodes +and scopes. Every truncation is exposed. Its retained-history floor and loading +state are captured with the relay evidence, rather than taken from a newer +store state when a request arrives. + ## Following milestones -1. **Timeline and comparison:** introduce historical snapshot selection and - playback, then compare equal, explicitly bounded periods. Show newly observed - and no-longer-observed relays and changed routes. Distinguish missing history - from an observed absence. A comparison must identify its two periods and the - position basis used for each. -2. **Region health:** add active relay counts, last relay and traffic trends +1. **Region health:** add active relay counts, last relay and traffic trends over comparable periods. Quiet relays should be labelled as not observed in the period; lack of reception alone does not establish that a node is offline. -3. **Connections between regions:** add observed paths with direction and +2. **Connections between regions:** add observed paths with direction and distinct-transmission counts. Preserve scope transitions, distinguish ambiguous hop identities and avoid interpreting shared scope membership as - proof of a direct radio link. + proof of a direct radio link. Route changes belong to this path-evidence + milestone; timeline relay membership alone does not establish a changed route. The default period and any future health thresholds or playback settings should be exposed in the customizer in a later milestone. The first milestone uses a @@ -59,3 +105,11 @@ visibility rules, missing coordinates, truncation and reuse of the cached snapshot. Browser tests cover period/scope URLs, browser navigation, pending requests, failures, provenance labels, incomplete history, theme switching and page teardown. Visual checks include narrow mobile and desktop layouts. + +Timeline tests additionally cover hour boundaries, distinct-transmission +counts, adjacent equal-duration intervals, archived/current metadata precedence, +scope and global bucket limits, missing reference history, zero denominators, +stable map bounds, keyboard operation, no per-frame requests, and playback +cleanup on manual changes, hidden documents and navigation. Performance checks +cover background snapshot construction, bounded retained history and warm bulk +responses without acquiring the packet-store lock. diff --git a/public/index.html b/public/index.html index 91a3b5d91..581d350ee 100644 --- a/public/index.html +++ b/public/index.html @@ -214,6 +214,7 @@ + diff --git a/public/regions-history.js b/public/regions-history.js new file mode 100644 index 000000000..ab5509102 --- /dev/null +++ b/public/regions-history.js @@ -0,0 +1,140 @@ +/* Bounded UTC-hour relay history. Counts are relay appearances, not packets. */ +'use strict'; + +(function (root) { + const HOUR = 3600000; + const WINDOW_HOURS = { '1h': 1, '24h': 24, '7d': 168 }; + const MAX_NODES = 5000, MAX_SCOPES_PER_NODE = 32, MAX_BUCKETS = 504, MAX_CELLS = 250000; + + function millis(value) { + return typeof value === 'string' && value ? Date.parse(value) : NaN; + } + + function lowerBound(values, target) { + let low = 0, high = values.length; + while (low < high) { + const mid = (low + high) >>> 1; + if (values[mid] < target) low = mid + 1; else high = mid; + } + return low; + } + + // Sparse prefix sums are constructed once. A frame reads two prefix values + // per scope; it never rescans the bucket payload while scrubbing or playing. + function indexHistory(data, nodes) { + const start = millis(data.historyStart), end = millis(data.historyEnd), generated = millis(data.generatedAt); + if (!Number.isFinite(start) || !Number.isFinite(end) || !Number.isFinite(generated) || generated < end || + start % HOUR || end % HOUR || data.bucketSeconds !== 3600 || data.bucketCount !== MAX_BUCKETS || + end - start !== MAX_BUCKETS * HOUR || !Array.isArray(nodes)) throw new Error('Invalid regions history'); + let limited = !!(data.truncated || data.scopesTruncated || data.bucketsTruncated || nodes.length > MAX_NODES); + let cells = 0, appearances = 0; + const indexed = []; + for (const node of nodes.slice(0, MAX_NODES)) { + const scopes = []; + const sourceScopes = Array.isArray(node.scopes) ? node.scopes : []; + if (sourceScopes.length > MAX_SCOPES_PER_NODE) limited = true; + for (const scope of sourceScopes.slice(0, MAX_SCOPES_PER_NODE)) { + const source = Array.isArray(scope.buckets) ? scope.buckets : []; + if (source.length > MAX_BUCKETS) limited = true; + const buckets = []; + for (const bucket of source.slice(0, MAX_BUCKETS)) { + if (++cells > MAX_CELLS) { limited = true; break; } + if (!bucket || !Number.isInteger(bucket.index) || bucket.index < 0 || bucket.index >= MAX_BUCKETS || + !Number.isSafeInteger(bucket.count) || bucket.count <= 0) { limited = true; continue; } + const last = millis(bucket.lastRelayed); + const bucketStart = start + bucket.index * HOUR; + if (!Number.isFinite(last) || last < bucketStart || last >= bucketStart + HOUR) { limited = true; continue; } + if (!Number.isSafeInteger(appearances + bucket.count)) { limited = true; continue; } + appearances += bucket.count; + buckets.push({ index: bucket.index, count: bucket.count, lastRelayed: new Date(last).toISOString() }); + } + buckets.sort((a, b) => a.index - b.index); + const indices = [], prefix = [0], times = []; + for (const bucket of buckets) { + if (indices.length && indices[indices.length - 1] === bucket.index) { limited = true; continue; } + const total = prefix[prefix.length - 1] + bucket.count; + if (!Number.isSafeInteger(total)) { limited = true; continue; } + indices.push(bucket.index); prefix.push(total); times.push(bucket.lastRelayed); + } + if (indices.length) scopes.push({ name: scope.name, indices, prefix, times }); + } + indexed.push({ ...node, scopes }); + } + const coverage = data.coverage || {}; + return { start, end, generatedAt: new Date(generated).toISOString(), nodes: indexed, + coverageStart: millis(coverage.start), loading: coverage.loading === true, limited }; + } + + function frameIndex(history, at) { + const requested = millis(at); + const end = Number.isFinite(requested) ? Math.floor(requested / HOUR) * HOUR : history.end; + return Math.max(0, Math.min(168, Math.round((end - (history.end - 168 * HOUR)) / HOUR))); + } + + function frameTime(history, index) { + const bounded = Math.max(0, Math.min(168, Number.isFinite(Number(index)) ? Math.round(Number(index)) : 168)); + return new Date(history.end - (168 - bounded) * HOUR).toISOString(); + } + + function interval(history, window, at) { + const end = millis(frameTime(history, frameIndex(history, at))); + const duration = (WINDOW_HOURS[window] || WINDOW_HOURS['24h']) * HOUR; + return { start: new Date(end - duration).toISOString(), end: new Date(end).toISOString(), + previousStart: new Date(end - duration * 2).toISOString(), previousEnd: new Date(end - duration).toISOString() }; + } + + function coverageState(history, start, end) { + if (history.loading) return 'loading'; + if (!Number.isFinite(history.coverageStart)) return 'unknown'; + if (history.coverageStart >= end) return 'unavailable'; + return history.coverageStart <= start ? 'complete' : 'partial'; + } + + function scopeCount(scope, startIndex, endIndex) { + const low = lowerBound(scope.indices, startIndex), high = lowerBound(scope.indices, endIndex); + return { count: scope.prefix[high] - scope.prefix[low], lastRelayed: high > low ? scope.times[high - 1] : '' }; + } + + function selectFrame(history, selection) { + const bounds = interval(history, selection.window, selection.at); + const start = millis(bounds.start), end = millis(bounds.end), previousStart = millis(bounds.previousStart); + const currentCoverage = coverageState(history, start, end); + const previousCoverage = coverageState(history, previousStart, start); + const comparable = !history.limited && currentCoverage === 'complete' && previousCoverage === 'complete'; + const current = [], previous = [], rows = []; + const added = comparable ? [] : null, retained = comparable ? [] : null, notObserved = comparable ? [] : null; + let currentAppearances = 0, previousAppearances = 0; + const startIndex = (start - history.start) / HOUR, endIndex = (end - history.start) / HOUR; + const previousIndex = (previousStart - history.start) / HOUR; + for (const node of history.nodes) { + let count = 0, before = 0, lastRelayed = '', previousLastRelayed = ''; + for (const scope of node.scopes) { + if (selection.scope && scope.name !== selection.scope) continue; + const now = scopeCount(scope, startIndex, endIndex), old = scopeCount(scope, previousIndex, startIndex); + count += now.count; before += old.count; + if (now.lastRelayed > lastRelayed) lastRelayed = now.lastRelayed; + if (old.lastRelayed > previousLastRelayed) previousLastRelayed = old.lastRelayed; + } + currentAppearances += count; previousAppearances += before; + if (count) current.push(node); + if (before) previous.push(node); + if (comparable) { + if (count && before) retained.push(node); + else if (count) added.push(node); + else if (before) notObserved.push(node); + } + if (count || (selection.compare && comparable && before)) rows.push({ node, count, previousCount: before, + lastRelayed, previousLastRelayed, state: selection.compare && comparable ? + (count && before ? 'retained' : count ? 'added' : 'not-observed') : 'observed' }); + } + return { interval: bounds, rows, currentCount: current.length, previousCount: previous.length, + currentAppearances, previousAppearances, currentCoverage, previousCoverage, comparable, + limited: history.limited, added, retained, notObserved, + appearanceChange: comparable ? currentAppearances - previousAppearances : null, + percentChange: comparable && previousAppearances > 0 ? (currentAppearances - previousAppearances) / previousAppearances * 100 : null }; + } + + const api = { indexHistory, frameIndex, frameTime, interval, selectFrame }; + if (typeof module !== 'undefined' && module.exports) module.exports = api; + root.RegionsHistory = api; +})(typeof window === 'undefined' ? globalThis : window); diff --git a/public/regions-map.js b/public/regions-map.js index 4255a0b8a..d1cf3c3a9 100644 --- a/public/regions-map.js +++ b/public/regions-map.js @@ -8,6 +8,8 @@ let leafletMap = null; let themeObserver = null; let generation = 0; + let stopPlayback = null; + let visibilityListener = null; function validPosition(node) { if (!node || node.lat == null || node.lon == null || node.lat === '' || node.lon === '') return false; @@ -30,7 +32,7 @@ return true; } - function collectScopeNodes(nodes) { + function collectScopeNodes(nodes, historical = false) { if (!Array.isArray(nodes)) return []; const seen = new Set(); const out = []; @@ -41,14 +43,16 @@ const key = String(node.publicKey || ''); if (!key || seen.has(key)) continue; const names = new Set(); - const scopes = node.scopes.filter(scope => scope && validScope(scope.name) && !names.has(scope.name) && names.add(scope.name)) - .map(scope => ({ name: scope.name, lastRelayed: Number.isFinite(Date.parse(scope.lastRelayed)) ? scope.lastRelayed : '' })); + const source = historical ? node.scopes.slice(0, 32) : node.scopes; + const scopes = source.filter(scope => scope && validScope(scope.name) && !names.has(scope.name) && names.add(scope.name)) + .map(scope => ({ name: scope.name, lastRelayed: Number.isFinite(Date.parse(scope.lastRelayed)) ? scope.lastRelayed : '', + ...(historical ? { buckets: Array.isArray(scope.buckets) ? scope.buckets : [] } : {}) })); if (!scopes.length) continue; seen.add(key); const positioned = node.positionSource === 'reported' && validPosition(node); out.push({ publicKey: key, name: String(node.name || key.slice(0, 8)), role: node.role, lat: positioned ? Number(node.lat) : null, lon: positioned ? Number(node.lon) : null, - positionSource: positioned ? 'reported' : 'none', scopes }); + positionSource: positioned ? 'reported' : 'none', metadataSource: node.metadataSource === 'archived' ? 'archived' : 'current', scopes }); } return out; } @@ -113,16 +117,28 @@ const params = new URLSearchParams(query); const scope = params.get('scope') || ''; const window = params.get('window'); - return { window: Object.prototype.hasOwnProperty.call(WINDOWS, window) ? window : '24h', scope: validScope(scope) ? scope : '' }; + const selection = { window: Object.prototype.hasOwnProperty.call(WINDOWS, window) ? window : '24h', scope: validScope(scope) ? scope : '' }; + if (params.get('view') === 'timeline') { + selection.view = 'timeline'; + selection.at = isoTime(params.get('at')); + selection.compare = params.get('compare') === '1'; + } + return selection; } function parseSelectedScope(hash) { return parseSelection(hash).scope; } - function scopeHash(scope, window = '24h') { + function scopeHash(scope, window = '24h', selection = {}) { const params = new URLSearchParams({ window: Object.prototype.hasOwnProperty.call(WINDOWS, window) ? window : '24h' }); if (validScope(scope)) params.set('scope', scope); + if (selection.view === 'timeline') { + params.set('view', 'timeline'); + const at = isoTime(selection.at); + if (at) params.set('at', at); + if (selection.compare) params.set('compare', '1'); + } return '#/regions?' + params.toString(); } @@ -208,6 +224,13 @@ shell.appendChild(element('p', 'regions-note', 'Observed scoped relay traffic, not a radio coverage measurement.')); const controls = element('div', 'regions-controls'); const selection = parseSelection(location.hash); + const viewLabel = element('label', '', 'View '); + const view = element('select'); + view.setAttribute('aria-label', 'Region view'); + view.append(new Option('Latest', 'latest'), new Option('Timeline', 'timeline')); + view.value = selection.view || 'latest'; + viewLabel.appendChild(view); + controls.appendChild(viewLabel); const periodLabel = element('label', '', 'Period '); const period = element('select'); period.setAttribute('aria-label', 'Region period'); @@ -223,7 +246,45 @@ const refresh = element('button', '', 'Refresh'); refresh.type = 'button'; controls.appendChild(refresh); + const fitButton = element('button', '', 'Fit map'); + fitButton.type = 'button'; + controls.appendChild(fitButton); shell.appendChild(controls); + const timelineControls = element('div', 'regions-timeline'); + timelineControls.hidden = selection.view !== 'timeline'; + const transport = element('div', 'regions-transport'); + function transportButton(text, label) { + const button = element('button', '', text); + button.type = 'button'; + button.setAttribute('aria-label', label); + transport.appendChild(button); + return button; + } + const previous = transportButton('Previous', 'Previous timeline hour'); + const play = transportButton('Play', 'Play timeline'); + const next = transportButton('Next', 'Next timeline hour'); + play.setAttribute('aria-pressed', 'false'); + timelineControls.appendChild(transport); + const sliderLabel = element('label', 'regions-slider-label', 'Interval ending (completed UTC hours)'); + const slider = element('input'); + slider.type = 'range'; + slider.min = '0'; slider.max = '168'; slider.step = '1'; slider.value = '168'; + slider.setAttribute('aria-label', 'Timeline interval ending'); + sliderLabel.appendChild(slider); + timelineControls.appendChild(sliderLabel); + const compareLabel = element('label', 'regions-compare-label'); + const compare = element('input'); + compare.type = 'checkbox'; compare.checked = selection.compare === true; + compare.setAttribute('aria-label', 'Compare previous period'); + compareLabel.append(compare, 'Compare previous period'); + timelineControls.appendChild(compareLabel); + const intervalLabel = element('p', 'regions-interval'); + timelineControls.appendChild(intervalLabel); + shell.appendChild(timelineControls); + const comparison = element('section', 'regions-comparison'); + comparison.setAttribute('aria-label', 'Timeline comparison'); + comparison.hidden = true; + shell.appendChild(comparison); const status = element('p', 'regions-status', 'Loading observed relay data…'); status.setAttribute('role', 'status'); shell.appendChild(status); @@ -235,6 +296,7 @@ details.appendChild(element('summary', '', 'Data & position details')); details.appendChild(element('p', '', 'Repeaters and rooms are shown where scoped relay traffic was observed. Positions are self-reported; their accuracy is unverified. Nodes without usable positions are counted but not mapped. Dashed outlines join the outermost node positions and do not measure RF coverage or administrative boundaries.')); details.appendChild(element('p', '', 'Retained history is not observer uptime or proof of continuous reception.')); + details.appendChild(element('p', '', 'Timeline counts are relay appearances: one transmission observed via two relay nodes contributes one appearance for each relay. Hours use UTC and intervals include their start and exclude their end. Positions use current or last-known self-reported coordinates; playback shows activity, not historical movement. Current catalog metadata can revise historical attribution after a refresh.')); const exactBounds = element('div', 'regions-exact-bounds'); details.appendChild(exactBounds); shell.appendChild(details); @@ -252,6 +314,7 @@ // period/hash navigation safe when a user just pressed a zoom control. leafletMap = L.map(mapEl, { zoomAnimation: false }).setView([55.7, 10.5], 6); const map = leafletMap; + let redrawMap = null; const applyTiles = function () { if (token !== generation || leafletMap !== map) return; if (tileLayer) map.removeLayer(tileLayer); @@ -262,6 +325,7 @@ attribution: tile && tile.attribution ? tile.attribution : '© OpenStreetMap contributors', }).addTo(map); map.getPane('tilePane').style.filter = tile && tile.invertFilter ? tile.invertFilter : ''; + if (redrawMap) redrawMap(); }; let tileLayer = null; applyTiles(); @@ -269,6 +333,178 @@ themeObserver.observe(document.documentElement, { attributes: true, attributeFilter: ['data-theme'] }); const layer = L.layerGroup().addTo(map); let requestGeneration = 0, snapshot = null, nodes = [], catalog = { scopes: [], truncated: false }; + let indexedHistory = null, timer = null, lastFrame = null, scrubFrame = null; + const markers = new Map(); + let hullLayer = null; + const historyTools = root.RegionsHistory; + + function pause() { + if (timer !== null) clearTimeout(timer); + if (scrubFrame !== null) cancelAnimationFrame(scrubFrame); + timer = null; + scrubFrame = null; + play.textContent = 'Play'; + play.setAttribute('aria-label', 'Play timeline'); + play.setAttribute('aria-pressed', 'false'); + } + stopPlayback = pause; + visibilityListener = () => { if (document.hidden) pause(); }; + document.addEventListener('visibilitychange', visibilityListener); + + function clearPoints() { + layer.clearLayers(); markers.clear(); hullLayer = null; lastFrame = null; + } + + function timelineEnabled(ready) { + const enabled = ready && selection.view === 'timeline'; + for (const control of [slider, compare, previous, next, play]) control.disabled = !enabled; + if (enabled) { + previous.disabled = Number(slider.value) <= 0; + next.disabled = Number(slider.value) >= 168; + play.disabled = Number(slider.value) >= 168; + } + } + + function intervalText(start, end) { + return start.replace('T', ' ').replace(':00.000Z', '') + ' — ' + end.replace('T', ' ').replace(':00.000Z', '') + ' UTC'; + } + + function appendNodeList(parent, labelText, values) { + const details = element('details', 'regions-change-list'); + details.appendChild(element('summary', '', labelText + ' (' + values.length + ')')); + const list = element('ul'); + for (const node of values.slice(0, 20)) { + const item = element('li'); + const link = element('a', '', node.name); + link.href = '#/nodes/' + encodeURIComponent(node.publicKey); + item.appendChild(link); + if (node.positionSource !== 'reported') item.append(' — no usable position'); + list.appendChild(item); + } + details.appendChild(list); + if (values.length > 20) details.appendChild(element('p', '', 'Showing the first 20 of ' + values.length + ' relay nodes.')); + parent.appendChild(details); + } + + function renderComparison(frame) { + comparison.hidden = !selection.compare; + comparison.replaceChildren(); + if (!selection.compare) return; + comparison.appendChild(element('p', 'regions-reference', 'Previous interval: ' + intervalText(frame.interval.previousStart, frame.interval.previousEnd))); + comparison.appendChild(element('p', '', 'Observed relay nodes: ' + frame.currentCount + ' selected / ' + frame.previousCount + + ' previous. Relay appearances: ' + frame.currentAppearances.toLocaleString() + ' selected / ' + frame.previousAppearances.toLocaleString() + ' previous.')); + if (!frame.comparable) { + const reasons = frame.limited ? 'A data safety limit was reached.' : 'Selected history: ' + frame.currentCoverage + '; previous history: ' + frame.previousCoverage + '.'; + comparison.appendChild(element('p', 'regions-comparison-limited', 'Comparison limited by available history. ' + reasons + + ' Changes and new/not-observed classifications are unavailable; counts show only retained observations.')); + comparison.setAttribute('data-comparable', 'false'); + return; + } + comparison.setAttribute('data-comparable', 'true'); + const legend = element('p', 'regions-comparison-legend'); + legend.append(element('span', 'regions-state-added', frame.added.length + ' newly observed'), + element('span', 'regions-state-retained', frame.retained.length + ' observed in both'), + element('span', 'regions-state-not-observed', frame.notObserved.length + ' not observed in selected interval')); + comparison.appendChild(legend); + const change = frame.appearanceChange; + comparison.appendChild(element('p', 'regions-appearance-change', 'Relay appearance change: ' + (change > 0 ? '+' : '') + + change.toLocaleString() + (frame.percentChange == null ? ' (percentage unavailable: previous count is zero).' : + ' (' + (frame.percentChange > 0 ? '+' : '') + frame.percentChange.toFixed(1) + '%).'))); + const lists = element('div', 'regions-change-lists'); + appendNodeList(lists, 'Newly observed in selected interval', frame.added); + appendNodeList(lists, 'Not observed in selected interval', frame.notObserved); + comparison.appendChild(lists); + } + + function timelinePopup(row) { + const point = row.node; + const popup = element('div'); + popup.appendChild(element('strong', '', point.name)); + popup.appendChild(element('div', '', selection.scope || point.scopes.map(s => s.name).join(', '))); + popup.appendChild(element('div', '', 'Selected interval: ' + row.count.toLocaleString() + ' relay appearances.')); + if (selection.compare) popup.appendChild(element('div', '', 'Previous interval: ' + row.previousCount.toLocaleString() + ' relay appearances.')); + if (row.state === 'added') popup.appendChild(element('div', '', 'Newly observed relative to the previous interval.')); + if (row.state === 'retained') popup.appendChild(element('div', '', 'Observed in both intervals.')); + if (row.state === 'not-observed') popup.appendChild(element('div', '', 'Not observed in selected interval; observed in previous interval. This does not establish an outage.')); + if (row.lastRelayed) popup.appendChild(element('div', '', 'Last observed relay in selected interval: ' + row.lastRelayed)); + popup.appendChild(element('div', '', 'Position: ' + (point.metadataSource === 'archived' ? 'last-known' : 'current') + ' self-reported; accuracy unverified. Not a historical position.')); + const link = element('a', '', 'Node page'); + link.href = '#/nodes/' + encodeURIComponent(point.publicKey); + popup.appendChild(link); + return popup; + } + + function renderTimelinePoints(frame) { + const styles = getComputedStyle(document.documentElement); + const colors = { observed: styles.getPropertyValue('--accent').trim(), retained: styles.getPropertyValue('--accent').trim(), + added: styles.getPropertyValue('--status-green').trim(), 'not-observed': styles.getPropertyValue('--status-yellow').trim() }; + const visible = new Set(); + for (const row of frame.rows) { + const point = row.node; + if (point.positionSource !== 'reported') continue; + visible.add(point.publicKey); + const style = { radius: row.state === 'added' ? 7 : 6, weight: 2, color: colors[row.state], + fillOpacity: row.state === 'not-observed' ? 0.15 : 0.75, dashArray: row.state === 'not-observed' ? '3 3' : null }; + let record = markers.get(point.publicKey); + if (!record) { + record = { marker: L.circleMarker([point.lat, point.lon], style), signature: '', style: '' }; + const current = record; + record.marker.bindPopup(() => timelinePopup(current.row)); + markers.set(point.publicKey, record); + } + const styleKey = [style.radius, style.color, style.fillOpacity, style.dashArray].join('|'); + if (record.style !== styleKey) { record.marker.setStyle(style); record.style = styleKey; } + if (!layer.hasLayer(record.marker)) layer.addLayer(record.marker); + record.row = row; + const signature = [row.count, row.previousCount, row.state, row.lastRelayed, row.previousLastRelayed, selection.scope, selection.compare].join('|'); + if (record.signature !== signature) { + if (record.marker.isPopupOpen()) record.marker.getPopup().setContent(timelinePopup(row)); + record.signature = signature; + } + } + for (const [key, record] of markers) if (!visible.has(key) && layer.hasLayer(record.marker)) layer.removeLayer(record.marker); + if (hullLayer) { layer.removeLayer(hullLayer); hullLayer = null; } + const currentPoints = frame.rows.filter(row => row.count > 0 && row.node.positionSource === 'reported').map(row => row.node); + if (selection.scope && currentPoints.length >= 3) { + const hull = convexHull(currentPoints.map(point => [point.lon, point.lat])); + if (hull.length >= 3) { + hullLayer = L.polygon(hull.map(point => [point[1], point[0]]), { color: colors.observed, + fillOpacity: 0.10, weight: 2, dashArray: '6 5', interactive: false }); + layer.addLayer(hullLayer); + } + } + status.textContent = currentPoints.length + ' positioned relay nodes; ' + (frame.currentCount - currentPoints.length) + + ' observed relay nodes without usable positions' + (selection.scope ? ' for ' + selection.scope : '') + + ' in selected interval. ' + frame.currentAppearances.toLocaleString() + ' relay appearances.'; + if (!frame.currentCount) status.textContent += frame.currentCoverage === 'unavailable' ? ' History is unavailable for this interval.' : + ' No scoped relay traffic was observed in the available history; this does not imply there is no coverage.'; + if (frame.limited) status.textContent += ' Data safety limits were reached; counts and map may be incomplete.'; + } + + function fitTimeline() { + const points = lastFrame ? lastFrame.rows.filter(row => row.node.positionSource === 'reported').map(row => row.node) : []; + if (points.length) map.fitBounds(L.latLngBounds(points.map(point => [point.lat, point.lon])).pad(0.15), { maxZoom: 10 }); + } + + function drawTimeline() { + const frame = historyTools.selectFrame(indexedHistory, selection); + lastFrame = frame; + selection.at = frame.interval.end; + slider.value = String(historyTools.frameIndex(indexedHistory, selection.at)); + slider.setAttribute('aria-valuetext', intervalText(frame.interval.start, frame.interval.end)); + intervalLabel.textContent = 'Selected interval: ' + intervalText(frame.interval.start, frame.interval.end); + timelineEnabled(true); + snapshotMeta.replaceChildren('Snapshot: ', timeElement(indexedHistory.generatedAt)); + const messages = { complete: 'Retained history covers the selected interval.', partial: 'Partial retained history for the selected interval.', + unavailable: 'No retained history is available in the selected interval.', unknown: 'Retained history bounds are unknown; this view may be incomplete.', + loading: 'History is still loading; this view may be incomplete.' }; + coverage.textContent = messages[frame.currentCoverage]; + coverage.setAttribute('data-coverage-state', frame.currentCoverage); + exactBounds.replaceChildren(element('p', '', 'Selected interval (UTC): ' + frame.interval.start + ' — ' + frame.interval.end), + element('p', '', 'History dataset (UTC): ' + new Date(indexedHistory.start).toISOString() + ' — ' + new Date(indexedHistory.end).toISOString())); + renderTimelinePoints(frame); + renderComparison(frame); + } function populateScopes(loading) { select.replaceChildren(); @@ -283,17 +519,32 @@ } function draw(fit) { + if (selection.view === 'timeline') { + if (!indexedHistory) return; + drawTimeline(); + if (fit) fitTimeline(); + return; + } if (!snapshot) return; renderPoints(layer, map, nodes, selection.scope, status, snapshot.truncated || snapshot.nodes.length > MAX_POINTS, snapshot.scopesTruncated || catalog.truncated, fit); } + redrawMap = () => { if (selection.view === 'timeline' && indexedHistory) renderTimelinePoints(lastFrame); }; async function load() { + pause(); const request = ++requestGeneration, window = selection.window; + const historical = selection.view === 'timeline'; snapshot = null; + indexedHistory = null; nodes = []; catalog = { scopes: [], truncated: false }; - layer.clearLayers(); + clearPoints(); + timelineEnabled(false); + comparison.hidden = true; + comparison.replaceChildren(); + comparison.removeAttribute('data-comparable'); + intervalLabel.textContent = ''; populateScopes(true); shell.setAttribute('aria-busy', 'true'); status.textContent = 'Loading observed relay data — ' + WINDOWS[window].toLowerCase() + '…'; @@ -302,8 +553,22 @@ exactBounds.replaceChildren(); coverage.removeAttribute('data-coverage-state'); try { - const data = await api('/regions/observed?window=' + encodeURIComponent(window), { ttl: 0, bust: true, retry503: false }); + const data = await api(historical ? '/regions/history' : '/regions/observed?window=' + encodeURIComponent(window), { ttl: 0, bust: true, retry503: false }); if (token !== generation || request !== requestGeneration || leafletMap !== map) return; + if (historical) { + if (!historyTools || !data || !Array.isArray(data.nodes)) throw new Error('Invalid regions history'); + nodes = collectScopeNodes(data.nodes, true); + indexedHistory = historyTools.indexHistory({ ...data, + truncated: data.truncated || data.nodes.length > MAX_POINTS, + scopesTruncated: data.scopesTruncated || data.nodes.some(node => node && Array.isArray(node.scopes) && node.scopes.length > 32), + }, nodes); + catalog = scopeCatalog(nodes); + if (catalog.truncated) indexedHistory.limited = true; + populateScopes(false); + draw(true); + saveSelection(true); + return; + } if (!data || data.window !== window || !Array.isArray(data.nodes) || !isoTime(data.windowStart) || !isoTime(data.windowEnd) || !isoTime(data.generatedAt)) throw new Error('Invalid regions snapshot'); snapshot = data; @@ -332,19 +597,70 @@ } } - function saveSelection() { - const hash = scopeHash(selection.scope, selection.window); - if (location.hash !== hash) history.pushState(null, '', hash); + function saveSelection(replace = false) { + const hash = scopeHash(selection.scope, selection.window, selection); + if (location.hash !== hash) history[replace ? 'replaceState' : 'pushState'](null, '', hash); } period.addEventListener('change', () => { + pause(); selection.window = period.value; saveSelection(); - load(); + if (selection.view === 'timeline') draw(false); else load(); }); select.addEventListener('change', () => { + pause(); selection.scope = select.value; saveSelection(); - draw(true); + draw(selection.view !== 'timeline'); + }); + view.addEventListener('change', () => { + pause(); + selection.view = view.value; + if (selection.view !== 'timeline') { selection.at = ''; selection.compare = false; compare.checked = false; } + timelineControls.hidden = selection.view !== 'timeline'; + saveSelection(); + load(); + }); + compare.addEventListener('change', () => { + pause(); + selection.compare = compare.checked; + saveSelection(); + draw(false); + }); + slider.addEventListener('input', () => { + pause(); + if (!indexedHistory) return; + selection.at = historyTools.frameTime(indexedHistory, slider.value); + scrubFrame = requestAnimationFrame(() => { + scrubFrame = null; + if (token !== generation || leafletMap !== map) return; + draw(false); saveSelection(true); + }); + }); + function step(direction) { + if (!indexedHistory) return; + selection.at = historyTools.frameTime(indexedHistory, Number(slider.value) + direction); + draw(false); saveSelection(true); + } + previous.addEventListener('click', () => { pause(); step(-1); }); + next.addEventListener('click', () => { pause(); step(1); }); + function tick() { + if (token !== generation || !indexedHistory || selection.view !== 'timeline' || document.hidden || Number(slider.value) >= 168) { pause(); return; } + step(1); + if (Number(slider.value) >= 168) { pause(); return; } + timer = setTimeout(tick, 1000); + } + play.addEventListener('click', () => { + if (timer !== null) { pause(); return; } + if (!indexedHistory || document.hidden || Number(slider.value) >= 168) return; + play.textContent = 'Pause'; + play.setAttribute('aria-label', 'Pause timeline'); + play.setAttribute('aria-pressed', 'true'); + timer = setTimeout(tick, 1000); + }); + fitButton.addEventListener('click', () => { + if (selection.view === 'timeline') { if (indexedHistory) fitTimeline(); } + else draw(true); }); refresh.addEventListener('click', load); load(); @@ -353,6 +669,10 @@ function destroy() { generation++; + if (stopPlayback) stopPlayback(); + stopPlayback = null; + if (visibilityListener) document.removeEventListener('visibilitychange', visibilityListener); + visibilityListener = null; if (themeObserver) themeObserver.disconnect(); themeObserver = null; if (leafletMap) leafletMap.remove(); diff --git a/public/style.css b/public/style.css index 472bbaf7f..837ab91e6 100644 --- a/public/style.css +++ b/public/style.css @@ -5942,6 +5942,7 @@ body.embed #app.app-fixed { /* Regions map: an observed-relay footprint, deliberately distinct from RF coverage. */ .regions-page { display: flex; flex-direction: column; height: 100%; min-height: 420px; padding: 16px; gap: 8px; } +.regions-page > :not(.regions-leaflet) { flex-shrink: 0; } .regions-page h1, .regions-page p { margin: 0; } .regions-note { color: var(--text-muted); max-width: 85ch; } .regions-controls { display: flex; flex-wrap: wrap; gap: 8px 12px; align-items: center; } @@ -5949,11 +5950,35 @@ body.embed #app.app-fixed { .regions-controls select, .regions-controls button { background: var(--surface); color: var(--text-primary); border: 1px solid var(--border); padding: 6px 8px; border-radius: 6px; } .regions-controls select { max-width: 100%; } .regions-controls button { cursor: pointer; } +.regions-controls button:disabled, .regions-timeline button:disabled { cursor: default; opacity: 0.55; } +.regions-timeline { display: grid; grid-template-columns: auto minmax(140px, 1fr) auto; gap: 8px 12px; align-items: center; padding: 10px; border: 1px solid var(--border); border-radius: 8px; background: var(--surface); } +.regions-timeline[hidden], .regions-comparison[hidden] { display: none; } +.regions-transport { display: flex; gap: 6px; } +.regions-transport button { background: var(--surface); color: var(--text-primary); border: 1px solid var(--border); padding: 6px 8px; border-radius: 6px; cursor: pointer; } +.regions-slider-label { display: flex; flex-direction: column; gap: 4px; min-width: 0; font-size: 0.85em; color: var(--text-muted); } +.regions-slider-label input { width: 100%; min-width: 0; accent-color: var(--accent); } +.regions-compare-label { display: flex; gap: 6px; align-items: center; font-size: 0.9em; } +.regions-compare-label input { accent-color: var(--accent); } +.regions-interval { grid-column: 1 / -1; font-size: 0.85em; color: var(--text-muted); overflow-wrap: anywhere; } +.regions-comparison { display: flex; flex-direction: column; gap: 6px; font-size: 0.85em; padding: 10px; border: 1px solid var(--border); border-radius: 8px; } +.regions-reference { color: var(--text-muted); overflow-wrap: anywhere; } +.regions-comparison-limited { color: var(--text-muted); border-left: 3px solid var(--warning); padding-left: 8px; } +.regions-comparison-legend { display: flex; flex-wrap: wrap; gap: 6px 16px; } +.regions-comparison-legend span::before { content: ''; display: inline-block; width: 0.7em; height: 0.7em; margin-right: 5px; border: 2px solid currentColor; border-radius: 50%; } +.regions-state-added { color: var(--status-green-text); } +.regions-state-retained { color: var(--accent); } +.regions-state-not-observed { color: var(--warning); } +.regions-state-not-observed::before { border-style: dashed !important; } +.regions-change-lists { display: flex; flex-wrap: wrap; gap: 8px 24px; } +.regions-change-list { min-width: 0; overflow-wrap: anywhere; } +.regions-change-list summary { cursor: pointer; } +.regions-change-list ul { margin: 6px 0; padding-left: 20px; max-height: 180px; overflow-y: auto; } +.regions-controls :focus-visible, .regions-timeline :focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } .regions-status { color: var(--text-muted); min-height: 2.4em; } .regions-snapshot, .regions-coverage, .regions-details { color: var(--text-muted); font-size: 0.85em; overflow-wrap: anywhere; } .regions-details summary { cursor: pointer; } .regions-details p { margin-top: 6px; } -.regions-coverage[data-coverage-state="partial"], .regions-coverage[data-coverage-state="unknown"], .regions-coverage[data-coverage-state="loading"] { border-left: 3px solid var(--warning); padding-left: 8px; } +.regions-coverage[data-coverage-state="partial"], .regions-coverage[data-coverage-state="unknown"], .regions-coverage[data-coverage-state="loading"], .regions-coverage[data-coverage-state="unavailable"] { border-left: 3px solid var(--warning); padding-left: 8px; } .regions-snapshot:empty, .regions-coverage:empty { display: none; } .regions-leaflet { flex: 1; min-height: 280px; border: 1px solid var(--border); border-radius: 8px; } @media (max-width: 600px) { @@ -5961,4 +5986,8 @@ body.embed #app.app-fixed { .regions-controls { align-items: stretch; } .regions-controls label { flex: 1 1 140px; } .regions-controls select { min-width: 0; flex: 1; } + .regions-timeline { grid-template-columns: 1fr; } + .regions-transport button { flex: 1; min-height: 36px; } + .regions-compare-label { min-height: 32px; } + .regions-leaflet { flex-shrink: 0; } } diff --git a/scripts/report-e2e-shards.js b/scripts/report-e2e-shards.js index 5c6f16983..3e4553ca5 100644 --- a/scripts/report-e2e-shards.js +++ b/scripts/report-e2e-shards.js @@ -54,19 +54,31 @@ function aggregateShards({ root, manifestFile, artifactsDir, outputDir = root, e receipts.push(receipt); } const coreShard = manifest.suites.find(suite => suite.file === CORE_SUITE).shard; - const coverageSource = path.join(artifactsDir, `e2e-shard-${coreShard}`, 'coverage'); - fs.mkdirSync(outputDir, { recursive: true }); - fs.writeFileSync(path.join(outputDir, 'e2e-output.txt'), logs.join('\n')); - if (fs.existsSync(coverageSource)) { - const coverageTarget = path.join(outputDir, '.nyc_output'); - fs.mkdirSync(coverageTarget, { recursive: true }); + const coverageFiles = []; + const coverageNames = new Set(); + // Coverage is optional, but every verified shard can contribute it. Retain + // the core names used by the badge workflow and namespace other shards so + // common collector filenames cannot silently overwrite one another. + for (const shard of SHARDS) { + const coverageSource = path.join(artifactsDir, `e2e-shard-${shard}`, 'coverage'); + if (!fs.existsSync(coverageSource)) continue; for (const file of fs.readdirSync(coverageSource)) { assert.ok(/^[a-zA-Z0-9_-]+\.json$/.test(file), `unexpected coverage artifact: ${file}`); const raw = requireFile(path.join(coverageSource, file)); JSON.parse(raw); - fs.writeFileSync(path.join(coverageTarget, file), raw); + const name = shard === coreShard ? file : `shard-${shard}-${file}`; + assert.ok(!coverageNames.has(name), `duplicate coverage artifact target: ${name}`); + coverageNames.add(name); + coverageFiles.push({ name, raw }); } } + fs.mkdirSync(outputDir, { recursive: true }); + fs.writeFileSync(path.join(outputDir, 'e2e-output.txt'), logs.join('\n')); + if (coverageFiles.length) { + const coverageTarget = path.join(outputDir, '.nyc_output'); + fs.mkdirSync(coverageTarget, { recursive: true }); + for (const { name, raw } of coverageFiles) fs.writeFileSync(path.join(coverageTarget, name), raw); + } const report = { version: 1, manifestSha256, status: 'success', coreShard, runContext: runContext(env), diff --git a/test-ci-shards.js b/test-ci-shards.js index 5c149564f..2fa47f426 100644 --- a/test-ci-shards.js +++ b/test-ci-shards.js @@ -168,6 +168,53 @@ test('real child non-zero exit and signal fail the shard without running later s } }); +test('report imports coverage from every successful shard without overwriting shared filenames', t => { + const f = fixture(t); + for (const shard of [1, 2, 3]) assert.equal(f.run(shard).status, 0); + const artifactsDir = f.artifacts(); + for (const shard of [1, 2, 3]) { + const coverage = path.join(artifactsDir, `e2e-shard-${shard}`, 'coverage'); + fs.mkdirSync(coverage); + fs.writeFileSync(path.join(coverage, 'e2e-coverage.json'), JSON.stringify({ shard })); + } + fs.writeFileSync(path.join(artifactsDir, 'e2e-shard-2/coverage/frontend-coverage.json'), '{"core":true}'); + fs.writeFileSync(path.join(artifactsDir, 'e2e-shard-3/coverage/regions-map-coverage.json'), '{"regions":true}'); + const outputDir = path.join(f.root, 'report'); + const options = { root: f.root, manifestFile: f.manifestFile, artifactsDir, outputDir, env: successfulResults() }; + aggregateShards(options); + const target = path.join(outputDir, '.nyc_output'); + assert.deepEqual(fs.readdirSync(target).sort(), ['e2e-coverage.json', 'frontend-coverage.json', 'shard-1-e2e-coverage.json', 'shard-3-e2e-coverage.json', 'shard-3-regions-map-coverage.json']); + for (const [file, shard] of [['e2e-coverage.json', 2], ['shard-1-e2e-coverage.json', 1], ['shard-3-e2e-coverage.json', 3]]) { + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(target, file), 'utf8')), { shard }); + } + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(target, 'shard-3-regions-map-coverage.json'), 'utf8')), { regions: true }); + // Even an unusual core filename must not overwrite a namespaced shard file. + fs.writeFileSync(path.join(artifactsDir, 'e2e-shard-2/coverage/shard-1-e2e-coverage.json'), '{"collision":true}'); + assert.throws(() => aggregateShards(options), /duplicate coverage artifact target/); +}); + +test('report validates optional non-core coverage filenames, regular files and JSON', t => { + const f = fixture(t); + for (const shard of [1, 2, 3]) assert.equal(f.run(shard).status, 0); + const artifactsDir = f.artifacts(); + const folder = path.join(artifactsDir, 'e2e-shard-3/coverage'); + fs.mkdirSync(folder); + const options = { root: f.root, manifestFile: f.manifestFile, artifactsDir, outputDir: path.join(f.root, 'report'), env: successfulResults() }; + const unexpected = path.join(folder, 'coverage.txt'); + fs.writeFileSync(unexpected, '{}'); + assert.throws(() => aggregateShards(options), /unexpected coverage artifact/); + fs.unlinkSync(unexpected); + const file = path.join(folder, 'regions-coverage.json'); + fs.writeFileSync(file, '{invalid JSON'); + assert.throws(() => aggregateShards(options), SyntaxError); + fs.unlinkSync(file); + fs.mkdirSync(file); + assert.throws(() => aggregateShards(options), /non-regular shard artifact/); + fs.rmdirSync(file); + fs.symlinkSync(path.join(artifactsDir, 'e2e-shard-3/result.json'), file); + assert.throws(() => aggregateShards(options), /non-regular shard artifact/); +}); + test('report is fail-closed for every shard status, missing artifacts, missing execution, stale manifest and forged success', t => { const f = fixture(t); for (const shard of [1, 2, 3]) assert.equal(f.run(shard).status, 0); diff --git a/test-e2e-playwright.js b/test-e2e-playwright.js index fa1b6daee..9a5dece64 100644 --- a/test-e2e-playwright.js +++ b/test-e2e-playwright.js @@ -303,6 +303,9 @@ async function run() { await test(`#382 node stats table rows follow the rounded corners in ${scheme} theme`, async () => { const ctx = await browser.newContext({ viewport: { width: 1280, height: 900 }, colorScheme: scheme }); const p = await ctx.newPage(); + await p.addInitScript(() => { + window.addEventListener('theme-refresh', () => { window.__packetCountThemeReady = true; }, { once: true }); + }); const pubkey = 'c'.repeat(64); const node = { public_key: pubkey, name: 'Corner fixture', role: 'repeater', last_seen: new Date().toISOString(), advert_count: 3 }; await p.route('**/api/nodes**', async route => { @@ -315,6 +318,10 @@ async function run() { await route.fulfill({ json: body }); }); try { + // The initial theme refresh replaces this table; settle it before + // evaluating row styles, just as the packet-count fixtures above do. + await p.goto(`${BASE}/#/home`, { waitUntil: 'domcontentloaded' }); + await p.waitForFunction(() => window.__packetCountThemeReady); await p.goto(`${BASE}/#/nodes/${pubkey}`, { waitUntil: 'domcontentloaded' }); const table = p.locator('#nodeFullBody .node-stats-table').first(); await table.waitFor(); diff --git a/test-regions-map-e2e.js b/test-regions-map-e2e.js index 6f54f2920..95bf38763 100644 --- a/test-regions-map-e2e.js +++ b/test-regions-map-e2e.js @@ -33,6 +33,21 @@ function snapshot(window, coverage) { nodes: window === '1h' ? [values[0], values[3]] : values, truncated: false, scopesTruncated: false }; } +function historySnapshot(override = {}) { + const historyStart = '2026-09-25T12:00:00Z', historyEnd = '2026-10-16T12:00:00Z'; + const bucket = (index, count) => ({ index, count, lastRelayed: new Date(Date.parse(historyStart) + index * 3600000 + 1000).toISOString() }); + const mk = (key, name, lat, lon, buckets) => ({ publicKey: key, name, role: 'repeater', lat, lon, + positionSource: lat == null ? 'none' : 'reported', metadataSource: 'current', scopes: [{ name: '#example', buckets }] }); + const values = [mk('aa11', EVIL, 55, 10, [bucket(502, 2), bucket(503, 3)]), + mk('bb22', 'New relay', 55, 11, [bucket(503, 4)]), mk('cc33', 'Previous relay', 56, 10, [bucket(502, 5)]), + mk('dd44', 'No position', null, null, [bucket(503, 1)]), mk('ff66', 'Old distant relay', -40, -120, [bucket(0, 1)])]; + values[0].scopes.push({ name: '#other', buckets: [bucket(503, 10)] }); + values[2].metadataSource = 'archived'; + return { generatedAt: '2026-10-16T12:34:00Z', historyStart, historyEnd, bucketSeconds: 3600, bucketCount: 504, + coverage: { start: historyStart, loading: false }, nodes: values, + truncated: false, scopesTruncated: false, bucketsTruncated: false, ...override }; +} + async function main() { let browser; try { @@ -48,6 +63,8 @@ async function main() { const errors = []; page.on('pageerror', e => errors.push(e.stack || String(e))); let holdHour = false, heldHour = null, failWeek = false, coverageOverride = null; + let historyOverride = {}, failHistory = false; + let historyRequests = 0; const requests = []; const waitForHeldHour = async () => { const deadline = Date.now() + 5000; @@ -64,9 +81,34 @@ async function main() { if (window === '7d' && failWeek) return r.fulfill({ status: 500, body: '{}' }); await fulfill(r, window); }); + await page.route('**/api/regions/history*', async r => { + historyRequests++; + if (failHistory) return r.fulfill({ status: 500, body: '{}' }); + await r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(historySnapshot(historyOverride)) }); + }); let failures = 0; + let coverageCapture = 0; + async function captureCoverage() { + const coverage = await page.evaluate(() => { + if (!window.__coverage__ || !Object.keys(window.__coverage__).length) return null; + const snapshot = JSON.parse(JSON.stringify(window.__coverage__)); + for (const file of Object.values(window.__coverage__)) { + for (const key of Object.keys(file.s || {})) file.s[key] = 0; + for (const key of Object.keys(file.f || {})) file.f[key] = 0; + for (const key of Object.keys(file.b || {})) file.b[key].fill(0); + } + return snapshot; + }); + if (!coverage) return; + const directory = path.join(__dirname, '.nyc_output'); + fs.mkdirSync(directory, { recursive: true }); + fs.writeFileSync(path.join(directory, 'regions-map-e2e-' + process.pid + '-' + (++coverageCapture) + '.json'), JSON.stringify(coverage)); + } const test = async (name, fn) => { try { await fn(); console.log(' ✓ ' + name); } catch (e) { failures++; console.log(' ✗ ' + name + ': ' + e.message); } + finally { + try { await captureCoverage(); } catch (e) { failures++; console.log(' ✗ coverage capture: ' + e.message); } + } }; // Track MutationObservers watching data-theme so teardown can be asserted. await page.addInitScript(() => { @@ -143,7 +185,9 @@ async function main() { await page.evaluate(() => { const original = L.Map.prototype.fitBounds; window.__regionsFitCount = 0; - L.Map.prototype.fitBounds = function (...args) { window.__regionsFitCount++; return original.apply(this, args); }; + L.Map.prototype.fitBounds = function (...args) { window.__regionsFitCount++; window.__regionsFitBounds = { + south: args[0].getSouth(), west: args[0].getWest(), north: args[0].getNorth(), east: args[0].getEast(), + }; return original.apply(this, args); }; }); const dark = await page.evaluate(() => window.MC_getTileSpec('dark')); const light = await page.evaluate(() => window.MC_getTileSpec('light')); @@ -226,16 +270,147 @@ async function main() { coverageOverride = null; }); + await test('timeline deep links clamp to completed hours and compare equal adjacent periods', async () => { + await page.evaluate(() => { location.hash = '#/regions?view=timeline&window=1h&scope=%23example&at=2026-10-16T12%3A47%3A00Z&compare=1'; }); + await page.waitForFunction(() => document.querySelector('.regions-comparison')?.getAttribute('data-comparable') === 'true'); + assert.strictEqual(await page.getByLabel('Region view').inputValue(), 'timeline'); + assert.strictEqual(await page.getByLabel('Timeline interval ending').inputValue(), '168'); + assert.strictEqual(await page.getByLabel('Compare previous period').isChecked(), true); + assert.strictEqual(await page.getByRole('button', { name: 'Play timeline', exact: true }).getAttribute('aria-pressed'), 'false'); + assert.match(await page.locator('.regions-interval').textContent(), /2026-10-16 11:00 — 2026-10-16 12:00 UTC/); + assert.match(await page.locator('.regions-reference').textContent(), /2026-10-16 10:00 — 2026-10-16 11:00 UTC/); + assert.match(await page.locator('.regions-status').textContent(), /2 positioned relay nodes; 1 observed relay nodes without usable positions/); + assert.match(await page.locator('.regions-comparison').textContent(), /8 selected \/ 7 previous/); + assert.match(await page.locator('.regions-state-added').textContent(), /2 newly observed/); + assert.match(await page.locator('.regions-state-retained').textContent(), /1 observed in both/); + assert.match(await page.locator('.regions-state-not-observed').textContent(), /1 not observed/); + assert.match(await page.locator('.regions-appearance-change').textContent(), /\+1 \(\+14\.3%\)/); + assert.strictEqual(await page.locator('.regions-leaflet path.leaflet-interactive').count(), 3, 'previous-only positioned relays remain visible in comparison'); + const params = new URLSearchParams((await page.evaluate(() => location.hash)).split('?')[1]); + assert.strictEqual(params.get('at'), '2026-10-16T12:00:00.000Z'); + assert.match(await page.locator('.regions-details').textContent(), /playback shows activity, not historical movement/); + assert.match(await page.locator('.regions-details').textContent(), /Current catalog metadata can revise historical attribution/); + }); + + if (process.env.REGIONS_SCREENSHOT_DIR) { + await page.setViewportSize({ width: 1440, height: 1000 }); + await page.locator('.regions-change-list summary').first().click(); + await page.evaluate(() => Promise.all(document.querySelector('#app').getAnimations().map(animation => animation.finished.catch(() => {})))); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + await page.screenshot({ path: path.join(process.env.REGIONS_SCREENSHOT_DIR, 'regions-timeline-desktop.png'), fullPage: true }); + await page.setViewportSize({ width: 390, height: 844 }); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + await page.screenshot({ path: path.join(process.env.REGIONS_SCREENSHOT_DIR, 'regions-timeline-mobile.png'), fullPage: true }); + await test('timeline controls and comparison stay within the mobile viewport', async () => { + assert(await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth + 1)); + assert(await page.evaluate(() => { + const status = document.querySelector('.regions-status').getBoundingClientRect(); + const meta = document.querySelector('.regions-snapshot').getBoundingClientRect(); + return status.bottom <= meta.top; + }), 'wrapped status overlaps snapshot metadata'); + }); + await page.locator('.regions-leaflet').scrollIntoViewIfNeeded(); + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))); + await page.screenshot({ path: path.join(process.env.REGIONS_SCREENSHOT_DIR, 'regions-timeline-mobile-map.png'), fullPage: true }); + await page.setViewportSize({ width: 1440, height: 1000 }); + } + + await test('timeline popup uses literal names and interval-specific appearance counts', async () => { + await page.locator('.regions-leaflet path.leaflet-interactive').first().dispatchEvent('click'); + await page.waitForSelector('.leaflet-popup-content'); + assert.strictEqual(await page.locator('.leaflet-popup-content img').count(), 0); + const text = await page.locator('.leaflet-popup-content').textContent(); + assert(text.includes(EVIL)); + assert.match(text, /Selected interval: 3 relay appearances/); + assert.match(text, /Previous interval: 2 relay appearances/); + assert.match(text, /Not a historical position/); + assert.strictEqual(await page.evaluate(() => window.__xss), undefined); + }); + + await test('timeline keyboard, scope, period and comparison changes need no extra requests or map refits', async () => { + const before = historyRequests; + await page.evaluate(() => { window.__regionsFitCount = 0; }); + await page.getByLabel('Timeline interval ending').focus(); + await page.getByLabel('Timeline interval ending').press('ArrowLeft'); + await page.waitForFunction(() => document.querySelector('.regions-interval').textContent.includes('2026-10-16 10:00 — 2026-10-16 11:00')); + assert.strictEqual(await page.getByLabel('Timeline interval ending').inputValue(), '167'); + assert.match(await page.locator('.regions-appearance-change').textContent(), /percentage unavailable/); + await page.getByLabel('Region period').selectOption('7d'); + assert.match(await page.locator('.regions-interval').textContent(), /2026-10-09 11:00/); + await page.getByLabel('Region scope').selectOption('#other'); + await page.getByLabel('Compare previous period').uncheck(); + assert.strictEqual(await page.locator('.regions-comparison').isVisible(), false); + await page.getByLabel('Region period').selectOption('1h'); + await page.getByLabel('Region scope').selectOption('#example'); + await page.getByLabel('Compare previous period').check(); + assert.strictEqual(historyRequests, before); + assert.strictEqual(await page.evaluate(() => window.__regionsFitCount), 0); + await page.getByRole('button', { name: 'Fit map', exact: true }).click(); + assert.strictEqual(await page.evaluate(() => window.__regionsFitCount), 1); + const bounds = await page.evaluate(() => window.__regionsFitBounds); + assert(bounds.south > 50 && bounds.west > 9, 'an old distant relay outside both displayed intervals distorted the fit'); + }); + + await test('playback advances locally, stops at newest hour and pauses on hidden documents', async () => { + const before = historyRequests; + await page.getByLabel('Timeline interval ending').focus(); + await page.getByLabel('Timeline interval ending').press('End'); + await page.getByLabel('Timeline interval ending').press('ArrowLeft'); + await page.getByRole('button', { name: 'Play timeline', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('[aria-label="Timeline interval ending"]').value === '168' && document.querySelector('[aria-label="Play timeline"]')?.getAttribute('aria-pressed') === 'false'); + assert.strictEqual(historyRequests, before); + await page.getByLabel('Timeline interval ending').press('ArrowLeft'); + await page.getByRole('button', { name: 'Play timeline', exact: true }).click(); + await page.evaluate(() => { Object.defineProperty(document, 'hidden', { configurable: true, value: true }); document.dispatchEvent(new Event('visibilitychange')); }); + assert.strictEqual(await page.getByRole('button', { name: 'Play timeline', exact: true }).getAttribute('aria-pressed'), 'false'); + await page.evaluate(() => { delete document.hidden; }); + await page.getByLabel('Timeline interval ending').press('End'); + }); + + await test('partial, unknown, loading and capped history withhold all change claims', async () => { + const overrides = [{ coverage: { start: '2026-10-16T10:30:00Z', loading: false } }, + { coverage: { start: null, loading: false } }, { coverage: { start: null, loading: true } }, { bucketsTruncated: true }]; + for (const value of overrides) { + historyOverride = value; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('.regions-page')?.getAttribute('aria-busy') === 'false' && document.querySelector('.regions-comparison')?.getAttribute('data-comparable') === 'false'); + assert.match(await page.locator('.regions-comparison-limited').textContent(), /Comparison limited by available history/); + assert.strictEqual(await page.locator('.regions-state-added, .regions-state-not-observed, .regions-appearance-change').count(), 0); + assert.match(await page.locator('.regions-status').textContent(), /8 relay appearances/); + assert.strictEqual(await page.locator('.regions-leaflet path.leaflet-interactive').count(), 2, 'limited comparison maps only observed selected relays'); + } + historyOverride = {}; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('.regions-comparison')?.getAttribute('data-comparable') === 'true'); + }); + + await test('failed history refresh clears stale markers and comparison before retry', async () => { + failHistory = true; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('.regions-status').textContent.includes('Could not load')); + assert.strictEqual(await page.locator('.regions-leaflet path.leaflet-interactive').count(), 0); + assert.strictEqual(await page.locator('.regions-comparison').isVisible(), false); + assert.strictEqual(await page.getByLabel('Timeline interval ending').isDisabled(), true); + failHistory = false; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('.regions-comparison')?.getAttribute('data-comparable') === 'true'); + }); + await test('destroy() tears down the map and the theme observer', async () => { // Navigation during Leaflet's 250ms zoom transition used to leave a // callback touching a removed map pane. Exercise the real zoom control. + await page.getByLabel('Timeline interval ending').focus(); + await page.getByLabel('Timeline interval ending').press('ArrowLeft'); + await page.getByRole('button', { name: 'Play timeline', exact: true }).click(); await page.locator('.regions-leaflet .leaflet-control-zoom-in').click(); await page.waitForTimeout(40); await page.evaluate(() => { location.hash = '#/nodes'; }); await page.waitForFunction(() => !document.querySelector('.regions-leaflet')); assert.strictEqual(await page.locator('.leaflet-container').count(), 0, 'leaflet container left behind'); await page.evaluate(() => { document.documentElement.setAttribute('data-theme', 'dark'); }); - await page.waitForTimeout(350); + const hashAfterDestroy = await page.evaluate(() => location.hash); + await page.waitForTimeout(1150); + assert.strictEqual(await page.evaluate(() => location.hash), hashAfterDestroy, 'playback timer changed URL after navigation'); assert.deepStrictEqual(errors, [], 'theme change after destroy raised errors'); const live = await page.evaluate(() => window.__themeObservers.filter(o => o.live && o.page === 'regions').length); assert.strictEqual(live, 0, 'regions theme observer still connected'); diff --git a/test-regions-map.js b/test-regions-map.js index 8184d7178..5cc2e75f1 100644 --- a/test-regions-map.js +++ b/test-regions-map.js @@ -50,6 +50,14 @@ assert.deepEqual(regions.parseSelection('#/regions'), { window: '24h', scope: '' assert.equal(regions.scopeHash('#dk', '1h'), '#/regions?window=1h&scope=%23dk'); assert.equal(regions.scopeHash('', '7d'), '#/regions?window=7d'); assert.equal(regions.parseSelectedScope('#/regions?scope=%23dk'), '#dk'); +assert.deepEqual(regions.parseSelection('#/regions?view=timeline&window=1h&scope=%23dk&at=2026-10-16T12%3A37%3A00Z&compare=1'), + { window: '1h', scope: '#dk', view: 'timeline', at: '2026-10-16T12:37:00.000Z', compare: true }); +assert.deepEqual(regions.parseSelection('#/regions?view=timeline&at=bad&compare=true'), + { window: '24h', scope: '', view: 'timeline', at: '', compare: false }); +assert.equal(regions.scopeHash('#dk', '1h', { view: 'timeline', at: '2026-10-16T12:00:00Z', compare: true }), + '#/regions?window=1h&scope=%23dk&view=timeline&at=2026-10-16T12%3A00%3A00.000Z&compare=1'); +assert.equal(regions.scopeHash('#dk', '1h', { view: 'latest', at: '2026-10-16T12:00:00Z', compare: true }), + '#/regions?window=1h&scope=%23dk', 'latest URLs omit timeline state'); const snapshot = { window: '24h', windowStart: '2026-10-15T12:00:00Z', windowEnd: '2026-10-16T12:00:00Z', generatedAt: '2026-10-16T12:00:00Z', coverage: { start: '2026-10-16T09:00:00Z', complete: false, loading: false } }; @@ -65,4 +73,71 @@ assert.equal(regions.coverageInfo({ ...snapshot, coverage: { start: null, comple assert.match(regions.coverageInfo({ ...snapshot, coverage: { start: '2026-10-17T00:00:00Z', complete: false } }).message, /No retained history/); assert.equal(regions.coverageInfo({ ...snapshot, coverage: { start: '2026-10-16T09:00:00Z', complete: true } }).kind, 'partial', 'inconsistent complete flag cannot overstate available history'); -console.log('regions map unit tests passed'); +const history = require('./public/regions-history.js'); +const historyEnd = '2026-10-16T12:00:00Z'; +const historyStart = '2026-09-25T12:00:00Z'; +const historical = (key, buckets, extra = {}) => node(key, { scopes: [{ name: '#dk', buckets }], ...extra }); +const bucket = (index, count = 1) => ({ index, count, lastRelayed: new Date(Date.parse(historyStart) + index * 3600000 + 1000).toISOString() }); +const historyData = (extra = {}) => ({ generatedAt: '2026-10-16T12:34:00Z', historyStart, historyEnd, + bucketSeconds: 3600, bucketCount: 504, coverage: { start: historyStart, loading: false }, + nodes: [historical('both', [bucket(502, 2), bucket(503, 3)]), historical('added', [bucket(503, 4)]), + historical('before', [bucket(502, 5)], { lat: null, lon: null, positionSource: 'none' })], ...extra }); +let indexed = history.indexHistory(historyData(), historyData().nodes); +let frame = history.selectFrame(indexed, { window: '1h', scope: '#dk', compare: true }); +assert.equal(frame.interval.start, '2026-10-16T11:00:00.000Z'); +assert.equal(frame.interval.previousStart, '2026-10-16T10:00:00.000Z'); +assert.equal(frame.currentCount, 2); +assert.equal(frame.previousCount, 2, 'unpositioned relays belong in comparison counts'); +assert.equal(frame.currentAppearances, 7); +assert.equal(frame.previousAppearances, 7); +assert.deepEqual(frame.added.map(n => n.publicKey), ['added']); +assert.deepEqual(frame.notObserved.map(n => n.publicKey), ['before']); +assert.deepEqual(frame.retained.map(n => n.publicKey), ['both']); +assert.equal(frame.percentChange, 0); +assert.equal(frame.rows[0].node.lat, 55, 'historical frames retain supplied latest coordinates'); +frame = history.selectFrame(indexed, { window: '24h', at: '2026-10-16T11:53:00Z', scope: '#dk' }); +assert.equal(frame.interval.end, '2026-10-16T11:00:00.000Z', 'frame boundaries are completed UTC hours'); +assert.equal(frame.interval.start, '2026-10-15T11:00:00.000Z'); +assert.equal(frame.currentAppearances, 7, 'the selected end is exclusive'); +assert.equal(history.frameTime(indexed, 0), '2026-10-09T12:00:00.000Z'); +assert.equal(history.frameTime(indexed, 168), '2026-10-16T12:00:00.000Z'); +assert.equal(history.frameIndex(indexed, '2099-01-01T00:00:00Z'), 168); +assert.equal(history.frameIndex(indexed, '2000-01-01T00:00:00Z'), 0); +for (const coverage of [{ start: null, loading: false }, { start: historyStart, loading: true }, + { start: '2026-10-16T10:30:00Z', loading: false }]) { + const partial = history.indexHistory(historyData({ coverage }), historyData().nodes); + const limited = history.selectFrame(partial, { window: '1h', compare: true }); + assert.equal(limited.comparable, false); + assert.equal(limited.added, null, 'unavailable history must withhold new/missing classifications'); + assert.equal(limited.percentChange, null); + assert.equal(limited.currentAppearances, 7, 'limited history still shows observed counts'); +} +frame = history.selectFrame(history.indexHistory(historyData({ truncated: true }), historyData().nodes), { window: '1h', compare: true }); +assert.equal(frame.comparable, false, 'caps withhold changes'); +assert.equal(frame.limited, true); +frame = history.selectFrame(indexed, { window: '1h', scope: '#absent', compare: true }); +assert.equal(frame.currentCount, 0); +assert.equal(frame.currentCoverage, 'complete', 'no observations is separate from unavailable history'); +assert.equal(frame.percentChange, null, 'zero baseline has no percentage change'); +const tooMany = Array.from({ length: 5001 }, (_, i) => historical('p' + i, [bucket(503)])); +indexed = history.indexHistory(historyData(), tooMany); +assert.equal(indexed.nodes.length, 5000); +assert.equal(indexed.limited, true); +const badBuckets = [bucket(-1), bucket(504), bucket(503, -1), bucket(503, Number.MAX_VALUE), bucket(503, 2)]; +indexed = history.indexHistory(historyData(), [historical('bad', badBuckets)]); +assert.equal(history.selectFrame(indexed, { window: '1h' }).currentAppearances, 2); +assert.equal(indexed.limited, true, 'invalid cells cannot support a complete comparison'); +indexed = history.indexHistory(historyData(), [historical('many-scopes', [], { + scopes: Array.from({ length: 33 }, (_, i) => ({ name: '#s' + i, buckets: [bucket(503)] })), +})]); +assert.equal(history.selectFrame(indexed, { window: '1h' }).currentAppearances, 32); +assert.equal(indexed.limited, true, 'the 32-scope per-node cap withholds comparisons'); +const fullCells = Array.from({ length: 501 }, (_, i) => bucket(i + 3)); +indexed = history.indexHistory(historyData(), Array.from({ length: 500 }, (_, i) => historical('bounded' + i, fullCells))); +assert.equal(indexed.limited, true, 'the global 250,000-cell bound is enforced'); +assert.equal(history.selectFrame(indexed, { window: '1h', compare: true }).added, null); +assert(indexed.nodes.reduce((total, n) => total + n.scopes.reduce((sum, s) => sum + s.indices.length, 0), 0) <= 250000); +assert.throws(() => history.indexHistory(historyData({ bucketCount: 505 }), []), /history/i); +assert.throws(() => history.indexHistory(historyData({ historyEnd: '2026-10-16T12:30:00Z' }), []), /history/i); + +console.log('regions map and timeline unit tests passed');