diff --git a/cmd/server/openapi.go b/cmd/server/openapi.go index 36f3ca80c..0287430b4 100644 --- a/cmd/server/openapi.go +++ b/cmd/server/openapi.go @@ -1022,6 +1022,9 @@ func componentSchemas() map[string]interface{} { for name, schema := range regionsRoutesSchemas() { schemas[name] = schema } + for name, schema := range regionsEvidenceSchemas() { + schemas[name] = schema + } for name, schema := range heardBySchemas() { schemas[name] = schema } diff --git a/cmd/server/regions_evidence.go b/cmd/server/regions_evidence.go new file mode 100644 index 000000000..77fd19c8d --- /dev/null +++ b/cmd/server/regions_evidence.go @@ -0,0 +1,299 @@ +package main + +import ( + "sort" + "time" +) + +const ( + maxRegionsMembershipEntries = 1000000 + maxRegionsMembershipNodes = 10000 +) + +// regionsObservationClass is the single provenance gate for Regions +// memberships and directed steps. Firmware builds flood paths, but consumes +// direct paths as intended remaining hops (Mesh.cpp routeRecvPacket and +// removeSelfFromPath). Forwarders never rewrite transport codes (they are set +// only by sendFlood/sendZeroHop), so own codes that differ from the stored +// frame identify a frame for another scope code. Canonical transport codes +// only associate the own frame with its stored scope; they do not +// independently authenticate the scope name. +type regionsObservationClass uint8 + +const ( + regionsObservationEligible regionsObservationClass = iota + // Identified as not eligible: a known DIRECT/unscoped header or own + // transport codes that differ from the stored canonical codes. Excluded + // observations never make counts incomplete. + regionsObservationExcluded + // The own header, own codes or canonical scope code are unknown, so + // eligibility cannot be decided. Counts for the bucket/scope are qualified. + regionsObservationUnprovable +) + +func classifyRegionsObservation(obs *StoreObs, canonical uint32, known bool) regionsObservationClass { + if obs == nil || obs.wireRoute == 0 { + return regionsObservationUnprovable + } + if obs.wireRoute != 1 { // Explicit ROUTE_TYPE_TRANSPORT_FLOOD, encoded +1. + return regionsObservationExcluded + } + if !known || !obs.transportKnown { + return regionsObservationUnprovable + } + if obs.wireTransport != canonical { + return regionsObservationExcluded + } + return regionsObservationEligible +} + +// RegionsEvidenceCounts classifies observations of scoped transmissions that +// did not contribute memberships. Excluded observations were identified as not +// eligible. Unprovable observations lacked the evidence to decide. Capped +// counts observations, or per-transmission relay memberships, omitted by a +// safety limit. Only unprovable and capped make a bucket/scope incomplete. +type RegionsEvidenceCounts struct { + Excluded int `json:"excluded"` + Unprovable int `json:"unprovable"` + Capped int `json:"capped"` +} + +func (c *RegionsEvidenceCounts) add(d RegionsEvidenceCounts) { + c.Excluded += d.Excluded + c.Unprovable += d.Unprovable + c.Capped += d.Capped +} + +type RegionsEvidenceBucket struct { + Index int `json:"index"` + RegionsEvidenceCounts +} + +type RegionsEvidenceScope struct { + Name string `json:"name"` + RegionsEvidenceCounts + Buckets []RegionsEvidenceBucket `json:"buckets,omitempty"` +} + +// RegionsEvidence is attributed to each transmission's stored scope and its +// FirstSeen hour. Truncated means the per-bucket budget was exhausted: the +// affected buckets are unknown and every rate or comparison must be withheld. +type RegionsEvidence struct { + RegionsEvidenceCounts + Truncated bool `json:"truncated"` + Scopes []RegionsEvidenceScope `json:"scopes"` +} + +// 128 scopes (the Regions scope selector limit) x 505 hourly buckets: 21 days +// plus Latest's current hour. +const maxRegionsEvidenceCells = 128 * (regionsHistoryBucketCount + 1) + +type regionsEvidenceKey struct { + scope string + index int +} + +type regionsEvidenceBuilder struct { + cells map[regionsEvidenceKey]RegionsEvidenceCounts + truncated bool +} + +func newRegionsEvidenceBuilder() *regionsEvidenceBuilder { + return ®ionsEvidenceBuilder{cells: make(map[regionsEvidenceKey]RegionsEvidenceCounts)} +} + +// add is O(1) and allocation-free for an existing cell, so it may run in +// capture under the packet read lock. +func (b *regionsEvidenceBuilder) add(scope string, index int, d RegionsEvidenceCounts) { + if d == (RegionsEvidenceCounts{}) { + return + } + key := regionsEvidenceKey{scope, index} + cell, exists := b.cells[key] + if !exists && len(b.cells) >= maxRegionsEvidenceCells { + b.truncated = true + return + } + cell.add(d) + b.cells[key] = cell +} + +// regionsEvidenceSnapshot is immutable once published. Scopes are sorted by +// name and buckets by index; index regionsHistoryBucketCount is the current, +// unfinished hour, which only Latest summarises. +type regionsEvidenceSnapshot struct { + start time.Time + scopes []RegionsEvidenceScope + truncated bool + history RegionsEvidence +} + +func (b *regionsEvidenceBuilder) build(start time.Time) *regionsEvidenceSnapshot { + byScope := make(map[string][]RegionsEvidenceBucket) + for key, counts := range b.cells { + byScope[key.scope] = append(byScope[key.scope], RegionsEvidenceBucket{Index: key.index, RegionsEvidenceCounts: counts}) + } + out := ®ionsEvidenceSnapshot{start: start, truncated: b.truncated, scopes: make([]RegionsEvidenceScope, 0, len(byScope))} + for name, buckets := range byScope { + sort.Slice(buckets, func(i, j int) bool { return buckets[i].Index < buckets[j].Index }) + out.scopes = append(out.scopes, RegionsEvidenceScope{Name: name, Buckets: buckets}) + } + sort.Slice(out.scopes, func(i, j int) bool { return out.scopes[i].Name < out.scopes[j].Name }) + out.history = out.summary(0, regionsHistoryBucketCount-1, true) + return out +} + +// summary restricts evidence to buckets first..last inclusive. It is O(cells), +// bounded by maxRegionsEvidenceCells, and never touches the packet store. +func (s *regionsEvidenceSnapshot) summary(first, last int, buckets bool) RegionsEvidence { + out := RegionsEvidence{Truncated: s == nil || s.truncated, Scopes: []RegionsEvidenceScope{}} + if s == nil { + return out + } + for _, scope := range s.scopes { + item := RegionsEvidenceScope{Name: scope.Name} + for _, bucket := range scope.Buckets { + if bucket.Index < first || bucket.Index > last { + continue + } + item.add(bucket.RegionsEvidenceCounts) + if buckets { + item.Buckets = append(item.Buckets, bucket) + } + } + if item.RegionsEvidenceCounts != (RegionsEvidenceCounts{}) { + out.add(item.RegionsEvidenceCounts) + out.Scopes = append(out.Scopes, item) + } + } + return out +} + +// window summarises every bucket that overlaps [start, end], including the +// current hour, for Latest's rolling windows. +func (s *regionsEvidenceSnapshot) window(start, end time.Time) RegionsEvidence { + if s == nil { + return s.summary(0, 0, false) + } + first := int(start.Sub(s.start) / time.Hour) + if start.Before(s.start) { + first = 0 + } + return s.summary(first, int(end.Sub(s.start)/time.Hour), false) +} + +func regionsEvidenceSchemas() map[string]*openAPISchema { + counts := func(extra map[string]*openAPISchema) map[string]*openAPISchema { + properties := map[string]*openAPISchema{ + "excluded": {Type: "integer", Description: "Observations identified as not eligible: known DIRECT/unscoped header or own transport codes that differ from the stored frame. Never makes counts incomplete."}, + "unprovable": {Type: "integer", Description: "Observations whose own header, own transport codes or canonical scope code are unknown or malformed. Makes the bucket/scope incomplete."}, + "capped": {Type: "integer", Description: "Observations or per-transmission relay memberships omitted by a safety limit. Makes the bucket/scope incomplete. Directed-step caps are reported in stepEvidence instead."}, + } + for key, value := range extra { + properties[key] = value + } + return properties + } + return map[string]*openAPISchema{ + "RegionsEvidence": {Type: "object", Description: "Attribution evidence by the transmission's stored scope and FirstSeen hour. Rates and comparisons are withheld only for buckets/scopes with unprovable or capped evidence.", Properties: counts(map[string]*openAPISchema{ + "truncated": {Type: "boolean", Description: "The per-bucket evidence budget was exhausted; affected buckets are unknown, so every rate and comparison is withheld."}, + "scopes": {Type: "array", Items: openAPIRef("RegionsEvidenceScope"), Description: "Scopes with any evidence count, sorted by name."}, + })}, + "RegionsEvidenceScope": {Type: "object", Properties: counts(map[string]*openAPISchema{ + "name": {Type: "string"}, + "buckets": {Type: "array", Items: openAPIRef("RegionsEvidenceBucket"), Description: "History only: buckets with any count, ascending by index."}, + })}, + "RegionsEvidenceBucket": {Type: "object", Properties: counts(map[string]*openAPISchema{ + "index": {Type: "integer", Description: "Zero-based completed UTC hour, 0 through 503."}, + })}, + } +} + +type regionsNodeMembership struct { + scopes []regionScopeRelay + scopesTruncated bool +} + +// Kept separate from generic relay/liveness info: Regions must neither inherit +// its canonical-route heuristics nor broaden its existing public semantics. +type regionsMembershipSnapshot struct { + nodes map[string]regionsNodeMembership + evidence *regionsEvidenceSnapshot +} + +type regionsMembershipBuilder struct { + entries map[string][]relayEntry + remaining int +} + +func newRegionsMembershipBuilder() *regionsMembershipBuilder { + return ®ionsMembershipBuilder{entries: make(map[string][]relayEntry), remaining: maxRegionsMembershipEntries} +} + +// retainLatestRegionScope keeps the newest 32 names with bounded scratch +// storage. Both Latest and hourly history use this exact tie-breaking rule. +func retainLatestRegionScope(latest map[string]time.Time, name string, when time.Time) (truncated bool) { + if have, exists := latest[name]; exists { + if when.After(have) { + latest[name] = when + } + return false + } + if len(latest) < maxTransportedScopes { + latest[name] = when + return false + } + var oldestName string + var oldest time.Time + for scope, stamp := range latest { + if oldestName == "" || stamp.Before(oldest) || (stamp.Equal(oldest) && scope > oldestName) { + oldestName, oldest = scope, stamp + } + } + if when.After(oldest) || (when.Equal(oldest) && name < oldestName) { + delete(latest, oldestName) + latest[name] = when + } + return true +} + +// Called once per resolved node per distinct transmission, outside the packet +// lock. This independent budget is never spent by directed edges or edge cells. +// It reports false when a safety limit omitted the membership. +func (b *regionsMembershipBuilder) add(key string, tx regionsRouteTx) bool { + if b.remaining == 0 { + return false + } + if _, exists := b.entries[key]; !exists && len(b.entries) >= maxRegionsMembershipNodes { + return false + } + // Only time/scope are consumed below; payload exclusion was already made + // before capture. No canonical/direct relay-entry fields are reused. + b.entries[key] = append(b.entries[key], relayEntry{t: tx.when, parsed: true, valid: true, scope: tx.scope}) + b.remaining-- + return true +} + +func (b *regionsMembershipBuilder) build(now time.Time, evidence *regionsEvidenceSnapshot) (*regionsMembershipSnapshot, *regionsRelayHistory) { + out := ®ionsMembershipSnapshot{nodes: make(map[string]regionsNodeMembership, len(b.entries)), evidence: evidence} + history := newRegionsHistoryBuilder(now) + keys := make([]string, 0, len(b.entries)) + for key := range b.entries { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + entries := b.entries[key] + // At most 32 newest scopes per key, with bounded scratch space even + // if an adversary creates a new scope for every retained transmission. + latest := make(map[string]time.Time, maxTransportedScopes) + truncated := false + for _, e := range entries { + truncated = retainLatestRegionScope(latest, e.scope, e.t) || truncated + } + scopes, _ := newestRegionScopeRelays(latest) + out.nodes[key] = regionsNodeMembership{scopes: scopes, scopesTruncated: truncated} + history.add(key, entries) + } + return out, history.history +} diff --git a/cmd/server/regions_evidence_test.go b/cmd/server/regions_evidence_test.go new file mode 100644 index 000000000..86f12857c --- /dev/null +++ b/cmd/server/regions_evidence_test.go @@ -0,0 +1,382 @@ +package main + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" +) + +func latestRegions(t testing.TB, srv *Server) RegionsObservedResponse { + t.Helper() + w := httptest.NewRecorder() + srv.handleRegionsObserved(w, httptest.NewRequest("GET", "/api/regions/observed?window=7d", nil)) + var out RegionsObservedResponse + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &out) != nil { + t.Fatalf("latest response %d: %s", w.Code, w.Body.String()) + } + return out +} + +func TestRegionsEvidenceRejectsPlannedPathsWithoutChangingGenericRelayInfo(t *testing.T) { + srv, now, observed := regionsObservedFixture(t) + planned := strings.Repeat("cd", 32) + if _, err := srv.db.conn.Exec("INSERT INTO nodes(public_key,name,role) VALUES(?,'Planned only','repeater')", planned); err != nil { + t.Fatal(err) + } + tx := routeTransmission(1, now.Truncate(time.Hour).Add(-time.Minute), `["`+observed+`"]`) + tx.PathJSON = `["` + planned + `"]` + tx.Observations = append(tx.Observations, routeObservation(tx.PathJSON, "17aabbccdd00")) + srv.store.byPathHop[planned] = []*StoreTx{tx} + srv.store.byPathHop[observed] = []*StoreTx{tx} + wantGeneric := srv.store.GetRepeaterRelayInfo(planned, 24) + recomputeRepeaterEnrichmentSafe(srv.store, 24) + gotGeneric := srv.store.repeaterRelaySnapshot.info[planned] + wantJSON, _ := json.Marshal(wantGeneric) + gotJSON, _ := json.Marshal(gotGeneric) + if string(wantJSON) != string(gotJSON) || gotGeneric.RelayCount24h != 1 { + t.Fatalf("generic relay semantics changed: %s != %s", wantJSON, gotJSON) + } + history, latest := getRegionsHistory(t, srv), latestRegions(t, srv) + if len(history.Nodes) != 1 || history.Nodes[0].PublicKey != observed || history.Nodes[0].Scopes[0].Buckets[0].Count != 1 { + t.Fatalf("planned DIRECT path entered history: %+v", history.Nodes) + } + if len(latest.Nodes) != 1 || latest.Nodes[0].PublicKey != observed { + t.Fatalf("planned DIRECT path entered Latest: %+v", latest.Nodes) + } +} + +func TestRegionsEvidenceOwnFloodSurvivesCanonicalDirectAndByNodeOnly(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + tx := routeTransmission(1, now.Truncate(time.Hour).Add(-time.Minute), `["`+key+`"]`, `["`+key+`","`+key+`"]`) + direct := 3 + tx.RouteType, tx.RawHex, tx.PathJSON = &direct, "17aabbccdd00", `["unknown-planned"]` + srv.store.byNode[key] = []*StoreTx{tx, tx} + before := srv.store.computeRepeaterRelayInfoMap(24) + recomputeRepeaterEnrichmentSafe(srv.store, 24) + if !reflect.DeepEqual(before, srv.store.repeaterRelaySnapshot.info) { + t.Fatal("canonical direct changed generic relay map") + } + history, latest := getRegionsHistory(t, srv), latestRegions(t, srv) + if len(history.Nodes) != 1 || len(latest.Nodes) != 1 || history.Nodes[0].Scopes[0].Buckets[0].Count != 1 { + t.Fatalf("own singleton flood lost or loop/observation duplicated: history=%+v latest=%+v", history.Nodes, latest.Nodes) + } + if routes := getRegionsRoutes(t, srv); len(routes.Steps) != 0 { + t.Fatal("singleton/loop membership invented a directed step") + } +} + +func TestRegionsEvidenceLatestIncludesCurrentHourButHistoryDoesNot(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + when := now.Truncate(time.Hour) + tx := routeTransmission(1, when, `["`+key+`"]`) + srv.store.byPathHop[key] = []*StoreTx{tx} + recomputeRepeaterEnrichmentSafe(srv.store, 24) + if latest := latestRegions(t, srv); len(latest.Nodes) != 1 || latest.Nodes[0].Scopes[0].LastRelayed != when.Format(time.RFC3339Nano) { + t.Fatalf("current hour missing from Latest: %+v", latest.Nodes) + } + if history := getRegionsHistory(t, srv); len(history.Nodes) != 0 { + t.Fatal("unfinished current hour entered history") + } +} + +func TestRegionsEvidenceFailsClosedOnOwnWireAndCanonicalScopeMismatch(t *testing.T) { + for _, tc := range []struct { + name, canonical, own string + unprovable bool + }{ + {"missing-own-header", "14aabbccdd00", "", true}, + {"missing-own-codes", "14aabbccdd00", "14aa", true}, + {"mismatch-first-code", "14aabbccdd00", "14aabcccdd00", false}, + {"mismatch-second-code", "14aabbccdd00", "14aabbccde00", false}, + {"missing-canonical-codes", "14aa", "14aabbccdd00", true}, + {"zero-canonical-code", "140000ccdd00", "140000ccdd00", true}, + {"unscoped-own", "14aabbccdd00", "1500", false}, + {"direct-own", "14aabbccdd00", "17aabbccdd00", false}, + } { + t.Run(tc.name, func(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + tx := routeTransmission(1, now.Truncate(time.Hour).Add(-time.Minute)) + tx.RawHex, tx.PathJSON = tc.canonical, `["`+key+`"]` + tx.Observations = []*StoreObs{routeObservation(tx.PathJSON, tc.own)} + srv.store.byPathHop[key] = []*StoreTx{tx} + recomputeRepeaterEnrichmentSafe(srv.store, 24) + if history, latest := getRegionsHistory(t, srv), latestRegions(t, srv); len(history.Nodes) != 0 || len(latest.Nodes) != 0 { + t.Fatalf("unverified observation entered Regions: history=%+v latest=%+v", history.Nodes, latest.Nodes) + } + want := RegionsEvidenceCounts{Excluded: 1} + if tc.unprovable { + want = RegionsEvidenceCounts{Unprovable: 1} + } + if history, latest := getRegionsHistory(t, srv), latestRegions(t, srv); history.Coverage.Evidence.RegionsEvidenceCounts != want || latest.Coverage.Evidence.RegionsEvidenceCounts != want { + t.Fatalf("attribution completeness confused with exclusion: history=%+v latest=%+v", history.Coverage, latest.Coverage) + } + }) + } +} + +func TestRegionsEvidencePrefixAmbiguityAndScopeDedup(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + other := "ab" + strings.Repeat("34", 31) + if _, err := srv.db.conn.Exec("INSERT INTO nodes(public_key,name,role) VALUES(?,'Colliding relay','repeater')", other); err != nil { + t.Fatal(err) + } + for i, scope := range []string{"#one", "#two"} { + tx := routeTransmission(i+1, now.Truncate(time.Hour).Add(-time.Minute), `["ab","`+key+`","`+key+`"]`, `["`+key+`"]`) + tx.ScopeName = scope + srv.store.byNode[key] = append(srv.store.byNode[key], tx, tx) + } + recomputeRepeaterEnrichmentSafe(srv.store, 24) + history := getRegionsHistory(t, srv) + // An ambiguous hop qualifies directed steps (stepEvidence.unresolvedHops), + // not the provenance of resolved relays' memberships. + if len(history.Nodes) != 1 || history.Nodes[0].PublicKey != key || len(history.Nodes[0].Scopes) != 2 || history.Coverage.Evidence.Unprovable != 0 { + t.Fatalf("ambiguous prefix broadened membership: %+v", history) + } + for _, scope := range history.Nodes[0].Scopes { + if len(scope.Buckets) != 1 || scope.Buckets[0].Count != 1 { + t.Fatalf("TX/node/scope dedup failed: %+v", scope) + } + } +} + +func TestRegionsEvidenceMembershipBudgetIndependentOfSteps(t *testing.T) { + now := time.Date(2026, 10, 17, 12, 35, 0, 0, time.UTC) + collector := newRegionsRoutesCollector(now) + a := strings.Repeat("ab", 32) + for i := 0; i < maxRegionsRouteSteps+1; i++ { + key := fmt.Sprintf("%064x", i+1) + collector.capture(routeTransmission(i+1, now.Add(-time.Hour), `["`+a+`","`+key+`"]`)) + } + got := collector.build(nil) + last := fmt.Sprintf("%064x", maxRegionsRouteSteps+1) + if len(got.steps) != maxRegionsRouteSteps || !got.stepsTruncated || len(got.memberships.nodes[last].scopes) != 1 || + got.memberships.evidence.history.Capped != 0 || got.memberships.evidence.truncated || got.evidence.CappedStepAppearances != 1 { + t.Fatal("directed-step cap suppressed independent valid membership or polluted its evidence") + } + b := newRegionsMembershipBuilder() + b.remaining = 1 + if !b.add(a, regionsRouteTx{when: now, scope: "#one"}) || b.add(a, regionsRouteTx{when: now, scope: "#two"}) { + t.Fatal("independent membership entry cap was silent") + } + if bounded, _ := b.build(now, nil); len(b.entries[a]) != 1 || len(bounded.nodes[a].scopes) != 1 { + t.Fatal("independent membership entry cap was not applied") + } + // The node cap is counted as capped evidence in the omitted membership's + // own hour and scope only. + collector = newRegionsRoutesCollector(now) + for i := 0; i < maxRegionsMembershipNodes; i++ { + collector.capture(routeTransmission(i+1, now.Add(-time.Hour), fmt.Sprintf(`["%064x"]`, i+1))) + } + collector.capture(routeTransmission(0, now.Add(-2*time.Hour), `["`+a+`"]`)) // Admitted first: lowest ID. + got = collector.build(nil) + evidence := got.memberships.evidence.history + if len(got.memberships.nodes) != maxRegionsMembershipNodes || evidence.Capped != 1 || len(evidence.Scopes) != 1 || + !reflect.DeepEqual(evidence.Scopes[0].Buckets, []RegionsEvidenceBucket{{Index: regionsHistoryBucketCount - 1, RegionsEvidenceCounts: RegionsEvidenceCounts{Capped: 1}}}) || + got.evidence.CappedObservations != 0 { + t.Fatalf("membership node cap must be capped evidence in its own bucket: %+v / %+v", evidence, got.evidence) + } +} + +// The JSON contract is decoded into test-local types so these assertions +// describe the public API independently of the server's Go structs. +type regionsEvidenceBucketJSON struct { + Index int + Excluded, Unprovable, Capped int +} + +type regionsEvidenceScopeJSON struct { + Name string + Excluded, Unprovable, Capped int + Buckets []regionsEvidenceBucketJSON +} + +type regionsEvidenceJSON struct { + Excluded, Unprovable, Capped int + Truncated bool + Scopes []regionsEvidenceScopeJSON +} + +type regionsStepEvidenceJSON struct { + ExcludedObservations, UnprovableObservations, CappedObservations int + UnresolvedHops, CappedStepAppearances int + SourceTruncated, Incomplete bool +} + +type regionsEvidenceBody struct { + Coverage struct { + Evidence *regionsEvidenceJSON + } + StepsTruncated bool + StepEvidence *regionsStepEvidenceJSON +} + +func regionsEvidenceResponse(t testing.TB, srv *Server, target string) regionsEvidenceBody { + t.Helper() + w := httptest.NewRecorder() + r := httptest.NewRequest("GET", target, nil) + switch { + case strings.HasPrefix(target, "/api/regions/observed"): + srv.handleRegionsObserved(w, r) + case strings.HasPrefix(target, "/api/regions/history"): + srv.handleRegionsHistory(w, r) + default: + srv.handleRegionsRoutes(w, r) + } + var out regionsEvidenceBody + if w.Code != 200 || json.Unmarshal(w.Body.Bytes(), &out) != nil { + t.Fatalf("%s: %d %s", target, w.Code, w.Body.String()) + } + if out.Coverage.Evidence == nil { + t.Fatalf("%s: coverage.evidence missing: %s", target, w.Body.String()) + } + return out +} + +// F1 (PR #507 review): one unprovable observation anywhere must not make every +// hour and scope incomplete. A realistic mix of valid, excluded (known direct, +// unscoped, mismatched code), unprovable (unknown header/codes) and capped +// observations is attributed to the transmission's own hour and stored scope. +func TestRegionsEvidenceIncompletenessIsPerBucketAndScope(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + hour := now.Truncate(time.Hour) + at := func(hoursAgo int) time.Time { return hour.Add(-time.Duration(hoursAgo)*time.Hour + time.Minute) } + index := func(hoursAgo int) int { return regionsHistoryBucketCount - hoursAgo } + valid := `["` + key + `"]` + mixed := routeTransmission(1, at(3), valid, valid) + mixed.Observations = append(mixed.Observations, + routeObservation(valid, "17aabbccdd00"), // known TRANSPORT_DIRECT: planned path + routeObservation(valid, "1500"), // known unscoped FLOOD + routeObservation(valid, "14aabcccdd00")) // own scope code differs from the stored frame + unknown := routeTransmission(2, at(2), valid) + unknown.Observations = append(unknown.Observations, routeObservation(valid, "")) + other := routeTransmission(3, at(3), valid) + other.ScopeName = "#other" + other.Observations = append(other.Observations, routeObservation(valid, "14aa")) + oversize := routeTransmission(4, at(1), valid) + oversize.Observations = append(oversize.Observations, routeObservation(`["`+strings.Repeat("a", maxRegionsRoutePathBytes)+`"]`, "14aabbccdd00")) + srv.store.byPathHop[key] = []*StoreTx{mixed, unknown, other, oversize} + recomputeRepeaterEnrichmentSafe(srv.store, 24) + + history := getRegionsHistory(t, srv) + if len(history.Nodes) != 1 || len(history.Nodes[0].Scopes) != 2 { + t.Fatalf("valid observations must still count beside excluded/unprovable ones: %+v", history.Nodes) + } + want := regionsEvidenceJSON{Excluded: 3, Unprovable: 2, Capped: 1, Scopes: []regionsEvidenceScopeJSON{ + {Name: "#other", Unprovable: 1, Buckets: []regionsEvidenceBucketJSON{{Index: index(3), Unprovable: 1}}}, + {Name: "#sample", Excluded: 3, Unprovable: 1, Capped: 1, Buckets: []regionsEvidenceBucketJSON{ + {Index: index(3), Excluded: 3}, {Index: index(2), Unprovable: 1}, {Index: index(1), Capped: 1}}}, + }} + for _, target := range []string{"/api/regions/history", "/api/regions/routes"} { + if got := *regionsEvidenceResponse(t, srv, target).Coverage.Evidence; !reflect.DeepEqual(got, want) { + t.Fatalf("%s evidence is not per bucket/scope:\n got %+v\nwant %+v", target, got, want) + } + } + // Latest summarises the requested rolling window, without bucket detail. + latest := *regionsEvidenceResponse(t, srv, "/api/regions/observed?window=7d").Coverage.Evidence + wantLatest := regionsEvidenceJSON{Excluded: 3, Unprovable: 2, Capped: 1, Scopes: []regionsEvidenceScopeJSON{ + {Name: "#other", Unprovable: 1}, {Name: "#sample", Excluded: 3, Unprovable: 1, Capped: 1}}} + if !reflect.DeepEqual(latest, wantLatest) { + t.Fatalf("7d Latest evidence:\n got %+v\nwant %+v", latest, wantLatest) + } + lastHour := *regionsEvidenceResponse(t, srv, "/api/regions/observed?window=1h").Coverage.Evidence + if want := (regionsEvidenceJSON{Capped: 1, Scopes: []regionsEvidenceScopeJSON{{Name: "#sample", Capped: 1}}}); !reflect.DeepEqual(lastHour, want) { + t.Fatalf("1h Latest evidence must exclude older hours:\n got %+v\nwant %+v", lastHour, want) + } + steps := regionsEvidenceResponse(t, srv, "/api/regions/routes").StepEvidence + if steps == nil || steps.ExcludedObservations != 3 || steps.UnprovableObservations != 2 || steps.CappedObservations != 1 { + t.Fatalf("step evidence must split excluded/unprovable/capped: %+v", steps) + } +} + +// Only known non-eligible observations: nothing is incomplete. +func TestRegionsEvidenceExcludedOnlyIsComplete(t *testing.T) { + srv, now, key := regionsObservedFixture(t) + valid := `["` + key + `"]` + tx := routeTransmission(1, now.Truncate(time.Hour).Add(-time.Minute), valid) + tx.Observations = append(tx.Observations, routeObservation(valid, "17aabbccdd00"), routeObservation(valid, "14aabbccde00")) + srv.store.byPathHop[key] = []*StoreTx{tx} + recomputeRepeaterEnrichmentSafe(srv.store, 24) + got := *regionsEvidenceResponse(t, srv, "/api/regions/history").Coverage.Evidence + if got.Excluded != 2 || got.Unprovable != 0 || got.Capped != 0 || got.Truncated { + t.Fatalf("identified exclusions were reported as incomplete: %+v", got) + } +} + +// F3 (PR #507 review): the #508 directed-step cap is counted as capped step +// appearances in stepEvidence, separately from observation-level evidence, and +// never makes membership/history coverage incomplete. +func TestRegionsEvidenceStepCapCountedSeparately(t *testing.T) { + srv, now, _ := regionsObservedFixture(t) + a := strings.Repeat("ab", 32) + when := now.Truncate(time.Hour).Add(-time.Hour) + for i := 0; i < maxRegionsRouteSteps+1; i++ { + srv.store.byPathHop[a] = append(srv.store.byPathHop[a], routeTransmission(i+1, when, fmt.Sprintf(`["%s","%064x"]`, a, i+1))) + } + recomputeRepeaterEnrichmentSafe(srv.store, 24) + body := regionsEvidenceResponse(t, srv, "/api/regions/routes") + if got := *body.Coverage.Evidence; got.Excluded != 0 || got.Unprovable != 0 || got.Capped != 0 || got.Truncated || len(got.Scopes) != 0 { + t.Fatalf("directed-step cap leaked into membership coverage evidence: %+v", got) + } + steps := body.StepEvidence + if !body.StepsTruncated || steps == nil || steps.CappedStepAppearances != 1 || steps.CappedObservations != 0 || steps.UnprovableObservations != 0 || !steps.Incomplete { + t.Fatalf("step cap must be counted as capped step appearances only: truncated=%v %+v", body.StepsTruncated, steps) + } +} + +// F2 (PR #507 review): a relay appearance is one transmission per distinct +// relay on ANY observer's own attributable flood path (the union across +// observers), not only the canonical display path. Repeated observers, +// identical paths and loops count once. +func TestRegionsEvidenceAppearancesUnionAcrossObservers(t *testing.T) { + srv, now, a := regionsObservedFixture(t) + b, c, d := "b1"+strings.Repeat("12", 31), "c1"+strings.Repeat("12", 31), "d1"+strings.Repeat("12", 31) + for _, key := range []string{b, c, d} { + if _, err := srv.db.conn.Exec("INSERT INTO nodes(public_key,name,role) VALUES(?,'Relay','repeater')", key); err != nil { + t.Fatal(err) + } + } + display := `["` + a + `","` + b + `"]` + tx := routeTransmission(1, now.Truncate(time.Hour).Add(-time.Minute), display, `["`+c+`","`+d+`"]`, display, `["`+b+`","`+a+`","`+b+`"]`) + tx.PathJSON = display + srv.store.byPathHop[a] = []*StoreTx{tx} + srv.store.byPathHop[b] = []*StoreTx{tx} + recomputeRepeaterEnrichmentSafe(srv.store, 24) + history := getRegionsHistory(t, srv) + got := map[string]int{} + total := 0 + for _, node := range history.Nodes { + for _, scope := range node.Scopes { + for _, bucket := range scope.Buckets { + got[node.PublicKey] += bucket.Count + total += bucket.Count + } + } + } + if want := map[string]int{a: 1, b: 1, c: 1, d: 1}; !reflect.DeepEqual(got, want) || total != 4 { + t.Fatalf("relay appearances must be the union across observers, once per relay: got %v (total %d)", got, total) + } +} + +// Worst-case evidence publication: every scope x hour cell of the budget. +func BenchmarkRegionsEvidenceCapPayload(b *testing.B) { + builder := newRegionsEvidenceBuilder() + for i := 0; !builder.truncated; i++ { + builder.add(fmt.Sprintf("#scope%03d", i/(regionsHistoryBucketCount+1)), i%(regionsHistoryBucketCount+1), RegionsEvidenceCounts{Excluded: 12345, Unprovable: 1, Capped: 1}) + } + snapshot := builder.build(time.Now().UTC().Truncate(time.Hour).Add(-regionsHistoryBucketCount * time.Hour)) + encoded, err := json.Marshal(snapshot.history) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = snapshot.window(snapshot.start, snapshot.start.Add(regionsHistoryBucketCount*time.Hour)) + } + b.ReportMetric(float64(len(encoded)), "payload-bytes") + b.ReportMetric(float64(len(builder.cells)), "cells") +} diff --git a/cmd/server/regions_history.go b/cmd/server/regions_history.go index c9b120f92..8505ee1f1 100644 --- a/cmd/server/regions_history.go +++ b/cmd/server/regions_history.go @@ -26,8 +26,9 @@ type regionsRelayHistory struct { } type RegionsHistoryCoverage struct { - Start *string `json:"start"` - Loading bool `json:"loading"` + Start *string `json:"start"` + Loading bool `json:"loading"` + Evidence RegionsEvidence `json:"evidence"` } type RegionsHistoryBucket struct { @@ -89,8 +90,8 @@ func historyRelayTime(e relayEntry, start, end time.Time) (time.Time, bool) { return t, valid && !t.Before(start) && t.Before(end) } -// add receives the already deduplicated, confirmed entries from the existing -// bulk relay computation, after the packet lock has been released. Callers +// add receives deduplicated own-observation memberships from the strict +// Regions collector, after the packet lock has been released. Callers // visit keys in sorted order so the shared cell budget is deterministic. func (b *regionsHistoryBuilder) add(key string, entries []relayEntry) { if len(key) != 64 { @@ -105,28 +106,7 @@ func (b *regionsHistoryBuilder) add(key string, entries []relayEntry) { if !valid { continue } - if have, exists := latest[e.scope]; exists { - if t.After(have) { - latest[e.scope] = t - } - continue - } - if len(latest) < maxTransportedScopes { - latest[e.scope] = t - continue - } - h.scopesTruncated = true - var oldestName string - var oldest time.Time - for name, when := range latest { - if oldestName == "" || when.Before(oldest) || (when.Equal(oldest) && name > oldestName) { - oldestName, oldest = name, when - } - } - if t.After(oldest) || (t.Equal(oldest) && e.scope < oldestName) { - delete(latest, oldestName) - latest[e.scope] = t - } + h.scopesTruncated = retainLatestRegionScope(latest, e.scope, t) || h.scopesTruncated } if len(latest) == 0 { return @@ -236,7 +216,7 @@ func (s *Server) regionsHistoryAndStepNodes(ctx context.Context, snapshot *repea response := RegionsHistoryResponse{GeneratedAt: snapshot.generatedAt.Format(time.RFC3339Nano), HistoryStart: h.start.Format(time.RFC3339Nano), HistoryEnd: h.end.Format(time.RFC3339Nano), BucketSeconds: 3600, BucketCount: regionsHistoryBucketCount, - Coverage: RegionsHistoryCoverage{Loading: snapshot.loading}, Nodes: []RegionsHistoryNode{}, + Coverage: RegionsHistoryCoverage{Loading: snapshot.loading, Evidence: regionsHistoryEvidence(snapshot)}, Nodes: []RegionsHistoryNode{}, Truncated: h.truncated, ScopesTruncated: h.scopesTruncated, BucketsTruncated: h.bucketsTruncated} if !snapshot.coverageStart.IsZero() && !snapshot.coverageStart.After(snapshot.generatedAt) { start := snapshot.coverageStart.Format(time.RFC3339Nano) @@ -279,11 +259,20 @@ func (s *Server) regionsHistoryAndStepNodes(ctx context.Context, snapshot *repea return response, stepNodes, err } +// regionsHistoryEvidence fails closed when no membership evidence exists. +func regionsHistoryEvidence(snapshot *repeaterRelaySnapshot) RegionsEvidence { + if snapshot.regionMemberships == nil || snapshot.regionMemberships.evidence == nil { + return (*regionsEvidenceSnapshot)(nil).summary(0, 0, true) + } + return snapshot.regionMemberships.evidence.history +} + func regionsHistorySchemas() map[string]*openAPISchema { return map[string]*openAPISchema{ "RegionsHistoryCoverage": {Type: "object", Properties: map[string]*openAPISchema{ - "start": {Type: "string", Nullable: true, Description: "Conservative retained-history floor, paired with this snapshot; null when unknown."}, - "loading": {Type: "boolean"}, + "start": {Type: "string", Nullable: true, Description: "Conservative retained-history floor, paired with this snapshot; null when unknown."}, + "loading": {Type: "boolean"}, + "evidence": openAPIRef("RegionsEvidence"), }}, "RegionsHistoryBucket": {Type: "object", Properties: map[string]*openAPISchema{ "index": {Type: "integer", Description: "Zero-based hourly bucket index, 0 through 503."}, diff --git a/cmd/server/regions_history_test.go b/cmd/server/regions_history_test.go index c8a1abb2f..d22e18fab 100644 --- a/cmd/server/regions_history_test.go +++ b/cmd/server/regions_history_test.go @@ -109,8 +109,8 @@ func getRegionsHistory(t testing.TB, srv *Server) RegionsHistoryResponse { func TestRegionsHistoryDedupCoverageAndWarmLock(t *testing.T) { srv, now, key := regionsObservedFixture(t) - pt, rt := 5, routeTypeFlood - tx := &StoreTx{ID: 5, FirstSeen: now.Add(-2 * time.Hour).Format(time.RFC3339Nano), PayloadType: &pt, RouteType: &rt, ScopeName: "#sample", PathJSON: `["` + key + `","` + key + `"]`} + tx := routeTransmission(5, now.Add(-2*time.Hour), `["`+key+`","`+key+`"]`) + tx.PathJSON = `["` + key + `","` + key + `"]` srv.store.byPathHop[key] = []*StoreTx{tx, tx} srv.store.byNode[key] = []*StoreTx{tx} recomputeRepeaterEnrichmentSafe(srv.store, 24) @@ -277,7 +277,6 @@ func TestOpenAPIRegionsHistorySchema(t *testing.T) { func regionsHistoryBenchmarkFixture(b *testing.B) (*Server, time.Time) { srv, now, _ := regionsObservedFixture(b) - pt, rt := 5, routeTypeFlood end := now.Truncate(time.Hour) transaction, err := srv.db.conn.Begin() if err != nil { @@ -293,7 +292,8 @@ func regionsHistoryBenchmarkFixture(b *testing.B) (*Server, time.Time) { b.Fatal(err) } for j := 0; j < 60; j++ { - tx := &StoreTx{ID: i*60 + j + 1, FirstSeen: end.Add(-time.Duration(j*8+1) * time.Hour).Format(time.RFC3339Nano), PayloadType: &pt, RouteType: &rt, ScopeName: "#sample", PathJSON: `["` + key + `"]`} + tx := routeTransmission(i*60+j+1, end.Add(-time.Duration(j*8+1)*time.Hour), `["`+key+`"]`) + tx.PathJSON = `["` + key + `"]` srv.store.byPathHop[key] = append(srv.store.byPathHop[key], tx) } } diff --git a/cmd/server/regions_observed.go b/cmd/server/regions_observed.go index f5657b24a..4173ff875 100644 --- a/cmd/server/regions_observed.go +++ b/cmd/server/regions_observed.go @@ -20,18 +20,20 @@ type regionScopeRelay struct { } type repeaterRelaySnapshot struct { - info map[string]RepeaterRelayInfo - history *regionsRelayHistory - routes *regionsRouteSnapshot - generatedAt time.Time - coverageStart time.Time - loading bool + info map[string]RepeaterRelayInfo + regionMemberships *regionsMembershipSnapshot + history *regionsRelayHistory + routes *regionsRouteSnapshot + generatedAt time.Time + coverageStart time.Time + loading bool } type RegionsObservedCoverage struct { - Start *string `json:"start"` - Complete bool `json:"complete"` - Loading bool `json:"loading"` + Start *string `json:"start"` + Complete bool `json:"complete"` + Loading bool `json:"loading"` + Evidence RegionsEvidence `json:"evidence"` } type RegionsObservedScope struct { @@ -190,14 +192,14 @@ func (s *Server) handleRegionsObserved(w http.ResponseWriter, r *http.Request) { s.store.repeaterEnrichMu.Lock() snapshot := s.store.repeaterRelaySnapshot s.store.repeaterEnrichMu.Unlock() - if snapshot == nil { + if snapshot == nil || snapshot.regionMemberships == nil { writeError(w, http.StatusServiceUnavailable, "observed relay snapshot is warming up") return } start := snapshot.generatedAt.Add(-duration) stamp := snapshot.generatedAt.Format(time.RFC3339Nano) response := RegionsObservedResponse{Window: window, GeneratedAt: stamp, WindowStart: start.Format(time.RFC3339Nano), WindowEnd: stamp, - Coverage: RegionsObservedCoverage{Loading: snapshot.loading}, Nodes: []RegionsObservedNode{}} + Coverage: RegionsObservedCoverage{Loading: snapshot.loading, Evidence: snapshot.regionMemberships.evidence.window(start, snapshot.generatedAt)}, Nodes: []RegionsObservedNode{}} if !snapshot.coverageStart.IsZero() && !snapshot.coverageStart.After(snapshot.generatedAt) { floor := snapshot.coverageStart.Format(time.RFC3339Nano) response.Coverage.Start = &floor @@ -206,19 +208,19 @@ func (s *Server) handleRegionsObserved(w http.ResponseWriter, r *http.Request) { // A relay can also be known as an observer under another display name. // Resolve hiding aliases in bulk before opening the metadata cursor: both // queries must also work with SQLite's single-connection test databases. - keys := make([]string, 0, len(snapshot.info)) - for key, info := range snapshot.info { - if len(key) == 64 && len(info.regionScopes) > 0 { + keys := make([]string, 0, len(snapshot.regionMemberships.nodes)) + for key, info := range snapshot.regionMemberships.nodes { + if len(key) == 64 && len(info.scopes) > 0 { keys = append(keys, key) } } err := s.forEachVisibleRegionNode(r.Context(), keys, false, func(n observedRegionNodeMetadata) bool { - info, exists := snapshot.info[n.publicKey] + info, exists := snapshot.regionMemberships.nodes[n.publicKey] if !exists { return true } - scopes := make([]RegionsObservedScope, 0, len(info.regionScopes)) - for _, scope := range info.regionScopes { + scopes := make([]RegionsObservedScope, 0, len(info.scopes)) + for _, scope := range info.scopes { if !scope.when.Before(start) && !scope.when.After(snapshot.generatedAt) { scopes = append(scopes, RegionsObservedScope{Name: scope.name, LastRelayed: scope.when.Format(time.RFC3339Nano)}) } @@ -240,7 +242,7 @@ func (s *Server) handleRegionsObserved(w http.ResponseWriter, r *http.Request) { node.Lat, node.Lon, node.PositionSource = &lat, &lon, "reported" } response.Nodes = append(response.Nodes, node) - response.ScopesTruncated = response.ScopesTruncated || info.regionScopesTruncated + response.ScopesTruncated = response.ScopesTruncated || info.scopesTruncated return true }) if err != nil { @@ -260,6 +262,7 @@ func regionsObservedSchemas() map[string]*openAPISchema { "start": {Type: "string", Nullable: true, Description: "Known retained-history floor (RFC3339), or null when unknown."}, "complete": {Type: "boolean", Description: "Retained in-memory history covers the requested interval; not an observer uptime guarantee."}, "loading": {Type: "boolean", Description: "Startup loading state when this snapshot was captured."}, + "evidence": openAPIRef("RegionsEvidence"), }}, "RegionsObservedScope": {Type: "object", Properties: map[string]*openAPISchema{ "name": {Type: "string", Description: "Exact observed scope spelling and case."}, diff --git a/cmd/server/regions_observed_test.go b/cmd/server/regions_observed_test.go index caaecb30d..f52382750 100644 --- a/cmd/server/regions_observed_test.go +++ b/cmd/server/regions_observed_test.go @@ -34,10 +34,9 @@ func regionsObservedFixture(t testing.TB) (*Server, time.Time, string) { func seedRegionRelaySnapshot(t testing.TB, srv *Server, now time.Time, key string) { t.Helper() - pt, rt := 5, routeTypeFlood for i, d := range []time.Duration{30 * time.Minute, 12 * time.Hour, 72 * time.Hour} { - tx := &StoreTx{ID: i + 1, FirstSeen: now.Add(-d).Format(time.RFC3339Nano), PayloadType: &pt, RouteType: &rt, - ScopeName: fmt.Sprintf("#scope%d", i), PathJSON: `["` + key + `"]`} + tx := routeTransmission(i+1, now.Add(-d), `["`+key+`"]`) + tx.ScopeName, tx.PathJSON = fmt.Sprintf("#scope%d", i), `["`+key+`"]` srv.store.byPathHop[key] = append(srv.store.byPathHop[key], tx) } recomputeRepeaterEnrichmentSafe(srv.store, 24) @@ -286,9 +285,9 @@ func TestRegionsObservedExactBoundsAndPairedSnapshot(t *testing.T) { seedRegionRelaySnapshot(t, srv, now, key) snapshot := srv.store.repeaterRelaySnapshot end, start := snapshot.generatedAt, snapshot.generatedAt.Add(-time.Hour) - info := snapshot.info[key] - info.regionScopes = []regionScopeRelay{{"#start", start}, {"#end", end}, {"#before", start.Add(-time.Nanosecond)}, {"#future", end.Add(time.Nanosecond)}} - snapshot.info[key] = info + info := snapshot.regionMemberships.nodes[key] + info.scopes = []regionScopeRelay{{"#start", start}, {"#end", end}, {"#before", start.Add(-time.Nanosecond)}, {"#future", end.Add(time.Nanosecond)}} + snapshot.regionMemberships.nodes[key] = info // A later change to live history must not get paired with older cached // relay evidence. The endpoint also must not wait for the packet lock. srv.store.mu.Lock() @@ -372,7 +371,7 @@ func TestRegionsObservedDeterministicLimitAfterVisibility(t *testing.T) { t.Fatal(err) } defer stmt.Close() - info := srv.store.repeaterRelaySnapshot.info[key] + info := srv.store.repeaterRelaySnapshot.regionMemberships.nodes[key] for i := 0; i < maxObservedRegionNodes+2; i++ { pk, name := fmt.Sprintf("%064x", i+1), "Visible" if i < 2 { @@ -381,7 +380,7 @@ func TestRegionsObservedDeterministicLimitAfterVisibility(t *testing.T) { if _, err := stmt.Exec(pk, name); err != nil { t.Fatal(err) } - srv.store.repeaterRelaySnapshot.info[pk] = info + srv.store.repeaterRelaySnapshot.regionMemberships.nodes[pk] = info } if err := transaction.Commit(); err != nil { t.Fatal(err) @@ -419,13 +418,13 @@ func BenchmarkRegionsObservedWarm(b *testing.B) { if err != nil { b.Fatal(err) } - info := srv.store.repeaterRelaySnapshot.info[key] + info := srv.store.repeaterRelaySnapshot.regionMemberships.nodes[key] for i := 0; i < 2000; i++ { pk := fmt.Sprintf("%064x", i+1) if _, err := stmt.Exec(pk); err != nil { b.Fatal(err) } - srv.store.repeaterRelaySnapshot.info[pk] = info + srv.store.repeaterRelaySnapshot.regionMemberships.nodes[pk] = info } stmt.Close() if err := transaction.Commit(); err != nil { diff --git a/cmd/server/regions_routes.go b/cmd/server/regions_routes.go index 325271034..2bbf63837 100644 --- a/cmd/server/regions_routes.go +++ b/cmd/server/regions_routes.go @@ -48,12 +48,19 @@ type RegionsRouteStep struct { Buckets []RegionsHistoryBucket `json:"buckets"` } +// RegionsStepEvidence is snapshot-wide. Observation counts use the same +// classes as RegionsEvidence; CappedStepAppearances counts directed step +// appearances dropped by the step/pair/cell caps, which never affect +// memberships or history coverage. type RegionsStepEvidence struct { - SkippedObservations int `json:"skippedObservations"` - UnresolvedHops int `json:"unresolvedHops"` - SourceTruncated bool `json:"sourceTruncated"` - Incomplete bool `json:"incomplete"` - MissingEndpointSteps int `json:"missingEndpointSteps"` + ExcludedObservations int `json:"excludedObservations"` + UnprovableObservations int `json:"unprovableObservations"` + CappedObservations int `json:"cappedObservations"` + UnresolvedHops int `json:"unresolvedHops"` + SourceTruncated bool `json:"sourceTruncated"` + CappedStepAppearances int `json:"cappedStepAppearances"` + Incomplete bool `json:"incomplete"` + MissingEndpointSteps int `json:"missingEndpointSteps"` } type RegionsRouteNode struct { @@ -91,6 +98,8 @@ type regionsRouteSnapshot struct { steps []RegionsRouteStep stepsTruncated, bucketsTruncated bool evidence RegionsStepEvidence + memberships *regionsMembershipSnapshot + history *regionsRelayHistory } type regionsRoutePath struct { @@ -105,14 +114,30 @@ type regionsRouteTx struct { } type regionsRoutesCollector struct { start, end time.Time + now time.Time txs []regionsRouteTx pathCount int evidence RegionsStepEvidence + cells *regionsEvidenceBuilder } func newRegionsRoutesCollector(now time.Time) *regionsRoutesCollector { end := now.UTC().Truncate(time.Hour) - return ®ionsRoutesCollector{start: end.Add(-regionsHistoryBucketCount * time.Hour), end: end} + return ®ionsRoutesCollector{start: end.Add(-regionsHistoryBucketCount * time.Hour), end: end, now: now.UTC(), cells: newRegionsEvidenceBuilder()} +} + +// addEvidence records one transmission's evidence in its own FirstSeen hour +// and stored scope, and in the snapshot-wide step diagnostics. +func (c *regionsRoutesCollector) addEvidence(tx regionsRouteTx, d RegionsEvidenceCounts) { + c.evidence.ExcludedObservations += d.Excluded + c.evidence.UnprovableObservations += d.Unprovable + c.evidence.CappedObservations += d.Capped + c.cells.add(tx.scope, c.bucket(tx), d) +} + +// bucket is 0 through regionsHistoryBucketCount; the last is the current hour. +func (c *regionsRoutesCollector) bucket(tx regionsRouteTx) int { + return int(tx.when.Sub(c.start) / time.Hour) } // capture copies immutable strings and scalar wire evidence while the packet @@ -127,26 +152,25 @@ func (c *regionsRoutesCollector) captureAt(tx *StoreTx, when time.Time, valid bo if tx.PayloadType == nil || *tx.PayloadType == payloadTypeAdvert || *tx.PayloadType == 9 || regions.ValidateCandidate(tx.ScopeName) != nil { return } - if !valid || when.Before(c.start) || !when.Before(c.end) { + if !valid || when.Before(c.start) || when.After(c.now) { return } canonical, known := wireTransportCodes(tx.RawHex) known = known && canonical>>16 != 0 item := regionsRouteTx{id: tx.ID, when: when, scope: tx.ScopeName} + var counts RegionsEvidenceCounts seen := make(map[string]int) for _, obs := range tx.Observations { - if obs == nil { + switch classifyRegionsObservation(obs, canonical, known) { + case regionsObservationExcluded: + counts.Excluded++ continue - } - if obs.wireRoute == 0 { - c.evidence.SkippedObservations++ + case regionsObservationUnprovable: + counts.Unprovable++ continue } - if obs.wireRoute != 1 { - continue - } // Only explicit TRANSPORT_FLOOD observation headers. - if !known || !obs.transportKnown || obs.wireTransport != canonical || len(obs.PathJSON) > maxRegionsRoutePathBytes { - c.evidence.SkippedObservations++ + if len(obs.PathJSON) > maxRegionsRoutePathBytes { + counts.Capped++ continue } if index, exists := seen[obs.PathJSON]; exists { @@ -155,16 +179,17 @@ func (c *regionsRoutesCollector) captureAt(tx *StoreTx, when time.Time, valid bo } if len(item.paths) >= maxRegionsRoutePathsPerTx || c.pathCount >= maxRegionsRoutePaths { c.evidence.SourceTruncated = true - c.evidence.SkippedObservations++ + counts.Capped++ continue } seen[obs.PathJSON] = len(item.paths) item.paths = append(item.paths, regionsRoutePath{path: obs.PathJSON, observations: 1}) c.pathCount++ } - if len(tx.Observations) == 0 && tx.PathJSON != "" { - c.evidence.SkippedObservations++ + if len(tx.Observations) == 0 { + counts.Unprovable++ } + c.addEvidence(item, counts) if len(item.paths) > 0 { c.txs = append(c.txs, item) } @@ -177,19 +202,32 @@ type regionsRouteCell struct { } func (c *regionsRoutesCollector) build(pm *prefixMap) *regionsRouteSnapshot { - out := ®ionsRouteSnapshot{steps: []RegionsRouteStep{}, evidence: c.evidence} + out := ®ionsRouteSnapshot{steps: []RegionsRouteStep{}} sort.Slice(c.txs, func(i, j int) bool { return c.txs[i].id < c.txs[j].id }) resolve := newRelayTokenResolver(pm) + memberships := newRegionsMembershipBuilder() rows := make(map[regionsRouteKey]map[int]regionsRouteCell) seen := make(map[regionsRouteKey]bool) + seenMembers := make(map[string]bool) cellCount := 0 + // Directed-step caps are counted only as step appearances: they never + // reach membership coverage evidence. + capStep := func(truncated *bool) { + *truncated = true + c.evidence.CappedStepAppearances++ + } for _, tx := range c.txs { clear(seen) - index := int(tx.when.Sub(c.start) / time.Hour) + clear(seenMembers) + index := c.bucket(tx) for _, path := range tx.paths { var tokens []string - if err := json.Unmarshal([]byte(path.path), &tokens); err != nil || len(tokens) > 64 { - out.evidence.SkippedObservations += path.observations + if err := json.Unmarshal([]byte(path.path), &tokens); err != nil { + c.addEvidence(tx, RegionsEvidenceCounts{Unprovable: path.observations}) + continue + } + if len(tokens) > 64 { + c.addEvidence(tx, RegionsEvidenceCounts{Capped: path.observations}) continue } previous := "" @@ -200,14 +238,26 @@ func (c *regionsRoutesCollector) build(pm *prefixMap) *regionsRouteSnapshot { key := resolve.key(token) if key == "" { previous = "" - out.evidence.UnresolvedHops++ + c.evidence.UnresolvedHops++ + continue + } + if !seenMembers[key] { + seenMembers[key] = true + if !memberships.add(key, tx) { + // A dropped membership, not an observation: no step count. + c.cells.add(tx.scope, c.bucket(tx), RegionsEvidenceCounts{Capped: 1}) + } + } + // Latest includes the current hour. Completed hourly history and + // directed steps deliberately do not, even at its exact boundary. + if index >= regionsHistoryBucketCount { continue } if previous != "" && previous != key { step := regionsRouteKey{previous, key, tx.scope} if !seen[step] { if len(seen) >= maxRegionsRoutePairsPerTx { - out.stepsTruncated = true + capStep(&out.stepsTruncated) previous = key continue } @@ -215,7 +265,7 @@ func (c *regionsRoutesCollector) build(pm *prefixMap) *regionsRouteSnapshot { byIndex := rows[step] if byIndex == nil { if len(rows) >= maxRegionsRouteSteps { - out.stepsTruncated = true + capStep(&out.stepsTruncated) previous = key continue } @@ -224,7 +274,7 @@ func (c *regionsRoutesCollector) build(pm *prefixMap) *regionsRouteSnapshot { cell, exists := byIndex[index] if !exists { if cellCount >= maxRegionsRouteCells { - out.bucketsTruncated = true + capStep(&out.bucketsTruncated) previous = key continue } @@ -270,7 +320,13 @@ func (c *regionsRoutesCollector) build(pm *prefixMap) *regionsRouteSnapshot { } out.steps = append(out.steps, step) } - out.evidence.Incomplete = out.evidence.SkippedObservations > 0 || out.evidence.UnresolvedHops > 0 || out.evidence.SourceTruncated || out.stepsTruncated || out.bucketsTruncated + out.memberships, out.history = memberships.build(c.now, c.cells.build(c.start)) + out.evidence = c.evidence + // Directed-step completeness only. Unresolved hops break adjacency, so they + // qualify steps; they do not qualify memberships of resolved relays. + out.evidence.Incomplete = out.evidence.UnprovableObservations > 0 || out.evidence.CappedObservations > 0 || + out.evidence.UnresolvedHops > 0 || out.evidence.SourceTruncated || out.evidence.CappedStepAppearances > 0 || + out.stepsTruncated || out.bucketsTruncated return out } @@ -329,6 +385,14 @@ func regionsRoutesSchemas() map[string]*openAPISchema { "RegionsRouteStep": {Type: "object", Properties: map[string]*openAPISchema{"from": {Type: "string"}, "to": {Type: "string"}, "scope": {Type: "string"}, "buckets": {Type: "array", Items: openAPIRef("RegionsRouteBucket"), Description: "Counts are not additive unique packet totals."}}}, "RegionsRouteBucket": {Type: "object", Properties: map[string]*openAPISchema{"index": {Type: "integer", Description: "Zero-based completed UTC hour, 0 through 503."}, "count": {Type: "integer", Description: "Distinct transmission IDs for this directed adjacent relay step and scope."}, "lastRelayed": {Type: "string", Description: "Latest transmission FirstSeen in this bucket, RFC3339."}}}, "RegionsRouteNode": {Type: "object", Properties: map[string]*openAPISchema{"publicKey": {Type: "string"}, "name": {Type: "string"}, "role": {Type: "string"}, "lat": {Type: "number", Nullable: true}, "lon": {Type: "number", Nullable: true}, "positionSource": {Type: "string", Enum: []string{"reported", "none"}}, "metadataSource": {Type: "string", Enum: []string{"current", "archived"}}}}, - "RegionsStepEvidence": {Type: "object", Description: "Snapshot-wide diagnostics, not selected-window totals or scope-transition evidence.", Properties: map[string]*openAPISchema{"skippedObservations": {Type: "integer"}, "unresolvedHops": {Type: "integer"}, "sourceTruncated": {Type: "boolean"}, "incomplete": {Type: "boolean"}, "missingEndpointSteps": {Type: "integer"}}}, + "RegionsStepEvidence": {Type: "object", Description: "Snapshot-wide directed-step diagnostics, not selected-window totals or scope-transition evidence. Per-bucket membership evidence is coverage.evidence.", Properties: map[string]*openAPISchema{ + "excludedObservations": {Type: "integer", Description: "Known DIRECT/unscoped or mismatched-transport-code observations."}, + "unprovableObservations": {Type: "integer", Description: "Observations with unknown or malformed own/canonical evidence."}, + "cappedObservations": {Type: "integer", Description: "Observations omitted by source path limits."}, + "unresolvedHops": {Type: "integer", Description: "Hop tokens without a unique relay identity; they break adjacency but not resolved memberships."}, + "sourceTruncated": {Type: "boolean"}, + "cappedStepAppearances": {Type: "integer", Description: "Directed step appearances omitted by the step, per-transmission pair or cell caps. Never part of coverage.evidence."}, + "incomplete": {Type: "boolean", Description: "Directed steps may be missing: unprovable/capped observations, unresolved hops, step caps or missing endpoints."}, + "missingEndpointSteps": {Type: "integer"}}}, } } diff --git a/cmd/server/regions_routes_test.go b/cmd/server/regions_routes_test.go index 608947c84..591769734 100644 --- a/cmd/server/regions_routes_test.go +++ b/cmd/server/regions_routes_test.go @@ -99,7 +99,7 @@ func TestRegionRoutesStrictObservationScopeAndBounds(t *testing.T) { if len(got.steps) != 1 || got.steps[0].Buckets[0].Count != 1 || got.steps[0].Buckets[0].Index != 503 { t.Fatalf("unsafe observation or time attribution: %+v", got.steps) } - if !got.evidence.Incomplete || got.evidence.SkippedObservations != 3 { + if e := got.evidence; !e.Incomplete || e.UnprovableObservations != 2 || e.ExcludedObservations != 3 || e.CappedObservations != 0 { t.Fatalf("missing evidence diagnostic: %+v", got.evidence) } } @@ -131,7 +131,7 @@ func TestRegionRoutesZeroScopeMalformedAndDifferentTransportCodes(t *testing.T) } collector.capture(routeTransmission(4, now.Add(-time.Hour), `["`+a+`",null,"`+b+`"]`, `["`+a+`","`+b+`" BROKEN]`)) got := collector.build(nil) - if len(got.steps) != 0 || !got.evidence.Incomplete || got.evidence.SkippedObservations != 4 || got.evidence.UnresolvedHops != 1 { + if e := got.evidence; len(got.steps) != 0 || !e.Incomplete || e.UnprovableObservations != 2 || e.ExcludedObservations != 2 || e.UnresolvedHops != 1 { t.Fatalf("invalid scope/path accepted: %+v", got) } // A collided prefix remains unknown, including when one candidate is hidden @@ -217,8 +217,8 @@ func regionsRoutesFixture(t testing.TB) (*Server, time.Time, string, string) { } path := `["` + a + `","` + b + `"]` tx := routeTransmission(1, now.Add(-2*time.Hour), path) - // Canonical direct with an independently observed flood must work publicly, - // even though the unchanged old relay-history path does not include it. + // Canonical direct with an independently observed flood must work in both + // strict history memberships and directed steps. direct := 3 tx.RouteType = &direct tx.RawHex = "17aabbccdd00" @@ -246,7 +246,7 @@ func getRegionsRoutes(t testing.TB, srv *Server) RegionsRoutesResponse { func TestRegionsRoutesPairedMetadataVisibilityAndWarmLock(t *testing.T) { srv, _, a, b := regionsRoutesFixture(t) got := getRegionsRoutes(t, srv) - if len(got.Nodes) != 0 || len(got.StepNodes) != 2 || len(got.Steps) != 1 || got.Steps[0].From != a || got.Steps[0].To != b || got.Steps[0].Buckets[0].Count != 1 { + if len(got.Nodes) != 2 || len(got.StepNodes) != 2 || len(got.Steps) != 1 || got.Steps[0].From != a || got.Steps[0].To != b || got.Steps[0].Buckets[0].Count != 1 { t.Fatalf("canonical direct scoped flood lost: %+v", got) } srv.store.mu.Lock() @@ -368,6 +368,13 @@ func TestOpenAPIRegionsRoutesSchemas(t *testing.T) { } func regionsRoutesBenchmarkFixture(b *testing.B) *Server { + return regionsRoutesBenchmarkFixtureWith(b, false) +} + +// mixed adds a realistic evidence mix: 8 scopes, a known-direct and a +// mismatched-code observation per transmission and an unknown header on +// every 10th, so per-bucket evidence spans 8 scopes x 168 hours. +func regionsRoutesBenchmarkFixtureWith(b *testing.B, mixed bool) *Server { srv, now, _ := regionsObservedFixture(b) keys := make([]string, 2000) transaction, err := srv.db.conn.Begin() @@ -394,6 +401,13 @@ func regionsRoutesBenchmarkFixture(b *testing.B) *Server { path := `["` + a + `","` + b + `","` + c + `"]` tx := routeTransmission(i+1, end.Add(-time.Duration(i%168+1)*time.Hour), path, path) tx.PathJSON = path + if mixed { + tx.ScopeName = fmt.Sprintf("#s%d", i%8) + tx.Observations = append(tx.Observations, routeObservation(path, "17aabbccdd00"), routeObservation(path, "14aabcccdd00")) + if i%10 == 0 { + tx.Observations = append(tx.Observations, routeObservation(path, "")) + } + } for _, key := range []string{a, b, c} { srv.store.byPathHop[key] = append(srv.store.byPathHop[key], tx) } @@ -410,8 +424,25 @@ func BenchmarkRegionsRoutesCompute30K2K(b *testing.B) { } } +func BenchmarkRegionsRoutesComputeMixedEvidence30K2K(b *testing.B) { + srv := regionsRoutesBenchmarkFixtureWith(b, true) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = srv.store.computeRepeaterRelaySnapshot(24) + } +} + func BenchmarkRegionsRoutesWarm30K2K(b *testing.B) { - srv := regionsRoutesBenchmarkFixture(b) + benchmarkRegionsRoutesWarm(b, false) +} + +func BenchmarkRegionsRoutesWarmMixedEvidence30K2K(b *testing.B) { + benchmarkRegionsRoutesWarm(b, true) +} + +func benchmarkRegionsRoutesWarm(b *testing.B, mixed bool) { + srv := regionsRoutesBenchmarkFixtureWith(b, mixed) recomputeRepeaterEnrichmentSafe(srv.store, 24) response := getRegionsRoutes(b, srv) encoded, err := json.Marshal(response) diff --git a/cmd/server/repeater_enrich_bulk.go b/cmd/server/repeater_enrich_bulk.go index 0f91a46f3..05241277c 100644 --- a/cmd/server/repeater_enrich_bulk.go +++ b/cmd/server/repeater_enrich_bulk.go @@ -202,8 +202,7 @@ func (s *PacketStore) computeRepeaterRelaySnapshot(windowHours float64) *repeate s.mu.RUnlock() out := make(map[string]RepeaterRelayInfo, len(perKey)) - history := newRegionsHistoryBuilder(now) - // Stable order also makes history's global safety-budget selection stable. + // Keep the existing generic relay-map computation independent of Regions. sort.Slice(perKey, func(i, j int) bool { return perKey[i].key < perKey[j].key }) var entries []relayEntry for _, k := range perKey { @@ -214,11 +213,11 @@ func (s *PacketStore) computeRepeaterRelaySnapshot(windowHours float64) *repeate entries = append(entries, e) } out[k.key] = computeRelayInfoFromEntriesAt(entries, windowHours, now) - history.add(k.key, entries) } snapshot.info = out - snapshot.history = history.history snapshot.routes = routes.build(pm) + snapshot.history = snapshot.routes.history + snapshot.regionMemberships = snapshot.routes.memberships return snapshot } diff --git a/docs/api-spec.md b/docs/api-spec.md index 4fe6c13c7..d49e7e10a 100644 --- a/docs/api-spec.md +++ b/docs/api-spec.md @@ -2563,7 +2563,13 @@ absolute clock-hour boundaries support stable bookmarked selections. "historyEnd": "2026-10-16T12:00:00Z", "bucketSeconds": 3600, "bucketCount": 504, - "coverage": {"start": "2026-10-02T12:35:00Z", "loading": false}, + "coverage": {"start": "2026-10-02T12:35:00Z", "loading": false, "evidence": { + "excluded": 7, "unprovable": 1, "capped": 0, "truncated": false, + "scopes": [{"name": "#example", "excluded": 7, "unprovable": 1, "capped": 0, "buckets": [ + {"index": 500, "excluded": 0, "unprovable": 1, "capped": 0}, + {"index": 503, "excluded": 7, "unprovable": 0, "capped": 0} + ]}] + }}, "nodes": [{ "publicKey": "", "name": "Example relay", "role": "repeater", "lat": 55.2, "lon": 10.3, "positionSource": "reported", @@ -2586,6 +2592,18 @@ Non-advert confirmed scope attribution follows the same identity-safe rules as `/api/regions/observed`; heuristic-only and raw-prefix attribution does not broaden scope claims. Scope spelling and case are preserved. +`coverage.evidence` counts observations that supplied no membership, by the +transmission's stored scope and `FirstSeen` hour: `excluded` (known DIRECT or +unscoped header, or own transport codes that differ from the stored frame), +`unprovable` (unknown or malformed own or canonical evidence) and `capped` +(observations or memberships omitted by safety limits). Only unprovable or +capped buckets make a scope's interval incomplete; clients withhold rates, +chart values and comparisons only for intervals that include such a bucket of +the selected scope, or of any scope when no scope is selected. `truncated` +means the per-bucket budget was exhausted and every rate and comparison must +be withheld. `/api/regions/observed` returns the same counts summed over the +requested rolling window, per scope, without bucket detail. + `coverage.start` and `loading` are captured together with the relay evidence. The 21-day horizon does not extend configured retention or restore evicted packets. A null floor, loading state, or floor after a selected interval's diff --git a/docs/regions-observed.md b/docs/regions-observed.md index f381de963..a7ce2c785 100644 --- a/docs/regions-observed.md +++ b/docs/regions-observed.md @@ -17,7 +17,37 @@ available. Unknown or loading history must never be presented as a complete period. Even a complete retained window does not prove uninterrupted radio reception or observer uptime. -Relay identity and scope attribution reuse the existing relay-evidence rules. +Relay identity and scope attribution require each observation's own scoped +transport-flood header, with both known transport codes matching the stored +canonical scoped frame and a nonzero scope code. Direct planned paths, missing +own-frame evidence and mismatched transport codes never acquire membership. +The canonical transmission may itself be direct: a valid flood observation of +that same content still supplies evidence. A single resolved flood hop is +sufficient for membership; a pair of consecutive hops is not required. +Observations of a scoped transmission that do not supply membership are +counted in `coverage.evidence`, per stored scope and `FirstSeen` hour, in +three classes: + +- **excluded**: identified as not eligible, namely a known DIRECT or unscoped + header, or own transport codes that differ from the stored frame. Forwarders + never rewrite transport codes (firmware `Mesh::sendFlood`/`sendZeroHop` set + them only at origin), so a mismatch is a frame for another scope code. + Exclusions never make counts incomplete. +- **unprovable**: the own header, own transport codes or canonical scope code + are unknown or malformed, so eligibility cannot be decided. +- **capped**: observations, or per-transmission relay memberships, omitted by a + safety limit. Directed-step caps are reported in `stepEvidence` instead. + +Unprovable or capped evidence makes only its own hour and scope incomplete: +rates, chart values and comparisons are withheld for intervals that include +an affected hour of the selected scope (or of any scope for "all scopes"), +and remain available elsewhere. If the per-bucket evidence budget (128 scopes +x 505 hours) is exhausted, `coverage.evidence.truncated` withholds every rate +and comparison. Hop tokens without a unique relay identity are counted in +`stepEvidence.unresolvedHops`; they break directed-step adjacency but do not +qualify the memberships of uniquely resolved relays. A fully retained interval +is not necessarily fully attributable; retained observed counts stay +explicitly qualified. The window is based on transmission `first_seen`, as in the existing relay activity fields. It is not an observation-time replay and does not reconstruct the historical position of a moving node. Each scope has its own most recent @@ -58,10 +88,18 @@ it does not establish a node's creation date. "Not observed" does not establish that a node is offline. Comparison always identifies both intervals. Counts are distinct transmissions per relay and scope. Adding counts across -relays gives **relay appearances**: a packet carried by several relays can -contribute once for each relay. These totals must not be labelled globally -unique mesh packets. Traffic changes and classifications are withheld when -either interval has unknown, loading, partial or truncated evidence. Raw +relays gives **relay appearances**: one transmission counts once for each +distinct relay identified on **any** observer's own attributable scoped-flood +path, the union across all observers, not only the displayed path. Repeated +observers, identical paths and loops count once. Before #507 history was +attributed through the canonical display path and its candidate index, so +relays seen only on other observers' paths were not counted; appearance totals +therefore rise where several observers hear the same transmission along +different paths. More observers can reveal more relays for the same traffic. +These totals must not be labelled globally unique mesh packets. Traffic +changes and classifications are withheld when +either interval has unknown, loading, partial or truncated history, or +unprovable or capped evidence for the selected scope. Raw observed counts may still be shown separately. A zero reference count has no defined percentage increase. @@ -106,7 +144,8 @@ as the chart. A zero is shown only for a fully retained, uncapped hour. Partial, unknown, loading or capped evidence produces a gap, not a fabricated zero; retained observations can still be disclosed explicitly as incomplete evidence. The mean appearances/hour is available only when the entire -selected interval is complete and uncapped. Partial counts are never divided +selected interval is complete, uncapped and has no unprovable or capped +evidence for the selected scope. Partial counts are never divided by the full requested duration. The optional comparison reuses Timeline's immediately preceding equal @@ -137,9 +176,8 @@ thresholds must be evidence-based and exposed through the customizer. inferred traffic links between region names: - **Shared relay-history memberships** counts distinct visible identities - attributed to both scopes by the existing transmission-level history during - the selected interval. This is an undirected history relationship, not - independently verified forwarding in both scopes. It does not prove that a + with attributable scoped flood observations in both scopes during the + selected interval. This is an undirected history relationship. It does not prove that a packet crossed between those scopes, that the identity is a gateway, or that the scopes are neighbouring geographic areas. - **Observed directed relay steps** counts consecutive, unambiguously resolved @@ -150,16 +188,15 @@ inferred traffic links between region names: transmissions. Independent steps are never concatenated into a claimed full route. -Shared memberships deliberately reuse the history attribution used by Timeline -and Health. That legacy attribution does not independently check every -observation's route and scope: when a content transmission has mixed flood and -direct observations, planned direct-path candidates can enter its canonical -scope's history. Consequently shared-history counts can contain false -memberships as well as omissions; they are not guaranteed lower bounds on -physically shared relays. The UI discloses this provenance. Only the directed -steps below apply the stricter own-observation frame checks introduced here. -Repairing the older history attribution is a separate correctness task rather -than a silent change to previous stages' metrics in this PR. +Shared memberships, Timeline, Health and Latest use the same strict +own-observation attribution. Mixed flood/direct observations cannot promote +planned direct paths into scope history. Repeated paths, observers and loops +contribute at most once per transmission, relay identity and scope. This +corrects the older transmission-level history: counts can decrease when +previously accepted paths cannot be proven, or increase where a valid flood +observation was previously hidden by a canonical direct transmission. No stored +packets are changed or deleted. Generic node liveness, relay statistics and +traffic-share calculations are deliberately unchanged by this Regions-only fix. Flood forwarding appends the forwarding identity to the path. Direct paths instead name remaining intended hops, so direct observations are not evidence @@ -208,20 +245,29 @@ No database writes, schema changes, retention extension, dependencies or new operator configuration are required. The route collector retains at most 250,000 path variants globally and 128 per -transmission, with an 8,192-byte input-path safety bound. Publication retains +transmission, with an 8,192-byte input-path safety bound. The capture includes +the current hour for Latest, but completed-hour history and steps exclude it. +Membership aggregation has independent budgets of 1,000,000 transmission/node +entries and 10,000 identities; step limits cannot suppress a valid singleton +membership. Each identity retains its 32 newest scope names. Source and +membership limits are counted as `capped` in the affected transmission's hour +and scope; the directed-step, per-transmission pair and cell caps are counted +separately as `stepEvidence.cappedStepAppearances` and never affect +memberships or history coverage. Capped capture is a +retained subset, not a ranking of all possible evidence. Publication retains at most 5,000 directed step/scope rows, 100,000 populated hourly cells and 10,000 endpoint identities. Endpoint metadata is independent of relay-history membership, so a valid flood observation of a canonically direct transmission can still be shown. The UI retains at most 1,000 shared scope pairs, paginates both tables in 25-row pages and maps at most 200 positioned step rows. A capped -subset is not a ranking of all possible relations. Snapshot-wide skipped-path, -unresolved-hop and missing-endpoint diagnostics are not selected-interval +subset is not a ranking of all possible relations. Snapshot-wide +`stepEvidence` counts (excluded, unprovable and capped observations, unresolved +hops, capped step appearances and missing endpoints) are not selected-interval traffic totals. Complete retained history does not imply complete radio reception or that all paths can be attributed. Future work can add independently evidenced scope transitions and comparisons -of observed routes, and align the legacy history's mixed-route/scope attribution -with own-observation evidence. A change in timeline relay membership alone does +of observed routes. A change in timeline relay membership alone does not establish a changed route. The default period, display limits and any future health thresholds or playback @@ -259,7 +305,12 @@ Connections tests additionally cover per-observation route/transport evidence, all observation-loading paths, unchanged observation struct size, separate observation boundaries, ambiguous intermediate hops, repeated/looped steps, reverse directions, live visibility, archived metadata and source/publication -limits. Shared-scope relation keys must remain distinct even when region names +limits. Mixed flood/direct tests additionally cover planned-path exclusion, +canonical direct packets with valid non-display flood observations, singleton +memberships, current-hour Latest versus completed history, independent step +and membership budgets, and unchanged generic relay metrics. Missing wire +evidence suppresses comparisons/rates/chart zeros, not the retention floor. +Shared-scope relation keys must remain distinct even when region names contain delimiter characters. Browser validation covers direction labels, missing positions, partial history, local hour/scope selection without extra requests, keyed and bounded rendering, failures, navigation, themes and narrow diff --git a/public/regions-history.js b/public/regions-history.js index aa2d93e68..54f3dc049 100644 --- a/public/regions-history.js +++ b/public/regions-history.js @@ -5,6 +5,8 @@ const HOUR = 3600000; const WINDOW_HOURS = { '1h': 1, '24h': 24, '7d': 168 }; const MAX_NODES = 5000, MAX_SCOPES_PER_NODE = 32, MAX_BUCKETS = 504, MAX_CELLS = 250000; + const MAX_EVIDENCE_CELLS = 128 * 505; + const COUNT_KEYS = ['excluded', 'unprovable', 'capped']; function millis(value) { return typeof value === 'string' && value ? Date.parse(value) : NaN; @@ -19,6 +21,56 @@ return low; } + // Evidence gaps are attributed to the transmission's stored scope and hour. + // Sparse prefix sums per scope plus a dense all-scopes aggregate answer any + // interval in O(log n). Present-but-unreadable evidence fails closed. + function indexEvidence(raw) { + const all = COUNT_KEYS.map(() => new Float64Array(MAX_BUCKETS + 1)); + const evidence = { truncated: false, scopes: new Map(), all }; + if (raw == null) return evidence; + const sourceScopes = raw && Array.isArray(raw.scopes) ? raw.scopes : null; + if (!sourceScopes || raw.truncated !== false && raw.truncated != null) evidence.truncated = true; + let cells = 0; + for (const scope of sourceScopes || []) { + if (!scope || typeof scope.name !== 'string' || !Array.isArray(scope.buckets)) { evidence.truncated = true; continue; } + const rows = []; + for (const bucket of scope.buckets) { + if (++cells > MAX_EVIDENCE_CELLS || !bucket || !Number.isInteger(bucket.index) || bucket.index < 0 || bucket.index >= MAX_BUCKETS || + COUNT_KEYS.some(key => bucket[key] != null && (!Number.isSafeInteger(bucket[key]) || bucket[key] < 0))) { + evidence.truncated = true; continue; + } + rows.push({ index: bucket.index, counts: COUNT_KEYS.map(key => bucket[key] || 0) }); + } + rows.sort((a, b) => a.index - b.index); + const indices = [], prefix = [COUNT_KEYS.map(() => 0)]; + for (const row of rows) { + if (indices[indices.length - 1] === row.index) { evidence.truncated = true; continue; } + indices.push(row.index); + prefix.push(prefix[prefix.length - 1].map((total, i) => total + row.counts[i])); + row.counts.forEach((count, i) => { all[i][row.index + 1] += count; }); + } + if (evidence.scopes.has(scope.name)) evidence.truncated = true; + else evidence.scopes.set(scope.name, { indices, prefix }); + } + for (const totals of all) for (let i = 1; i <= MAX_BUCKETS; i++) totals[i] += totals[i - 1]; + return evidence; + } + + // Counts for [startIndex, endIndex) and the selected scope ('' = all scopes). + // Only unprovable or capped evidence makes the interval incomplete. + function evidenceCounts(history, scope, startIndex, endIndex) { + const evidence = history.evidence; + let values = [0, 0, 0]; + if (!scope) values = evidence.all.map(totals => totals[endIndex] - totals[startIndex]); + else if (evidence.scopes.has(scope)) { + const entry = evidence.scopes.get(scope); + const low = lowerBound(entry.indices, startIndex), high = lowerBound(entry.indices, endIndex); + values = entry.prefix[high].map((total, i) => total - entry.prefix[low][i]); + } + return { excluded: values[0], unprovable: values[1], capped: values[2], + incomplete: evidence.truncated || values[1] > 0 || values[2] > 0 }; + } + // Sparse prefix sums are constructed once. A frame reads two prefix values // per scope; it never rescans the bucket payload while scrubbing or playing. function indexHistory(data, nodes) { @@ -26,6 +78,7 @@ if (!Number.isFinite(start) || !Number.isFinite(end) || !Number.isFinite(generated) || generated < end || start % HOUR || end % HOUR || data.bucketSeconds !== 3600 || data.bucketCount !== MAX_BUCKETS || end - start !== MAX_BUCKETS * HOUR || !Array.isArray(nodes)) throw new Error('Invalid regions history'); + // Publication caps are snapshot-wide; evidence gaps are per bucket/scope. let limited = !!(data.truncated || data.scopesTruncated || data.bucketsTruncated || nodes.length > MAX_NODES); let cells = 0, appearances = 0; const indexed = []; @@ -62,7 +115,8 @@ } const coverage = data.coverage || {}; return { start, end, generatedAt: new Date(generated).toISOString(), nodes: indexed, - coverageStart: millis(coverage.start), loading: coverage.loading === true, limited }; + coverageStart: millis(coverage.start), loading: coverage.loading === true, + evidence: indexEvidence(coverage.evidence), limited }; } function frameIndex(history, at) { @@ -100,12 +154,15 @@ const start = millis(bounds.start), end = millis(bounds.end), previousStart = millis(bounds.previousStart); const currentCoverage = coverageState(history, start, end); const previousCoverage = coverageState(history, previousStart, start); - const comparable = !history.limited && currentCoverage === 'complete' && previousCoverage === 'complete'; + const startIndex = (start - history.start) / HOUR, endIndex = (end - history.start) / HOUR; + const previousIndex = (previousStart - history.start) / HOUR; + const evidence = evidenceCounts(history, selection.scope || '', startIndex, endIndex); + const previousEvidence = evidenceCounts(history, selection.scope || '', previousIndex, startIndex); + const comparable = !history.limited && currentCoverage === 'complete' && previousCoverage === 'complete' && + !evidence.incomplete && !previousEvidence.incomplete; const current = [], previous = [], rows = []; const added = comparable ? [] : null, retained = comparable ? [] : null, notObserved = comparable ? [] : null; let currentAppearances = 0, previousAppearances = 0; - const startIndex = (start - history.start) / HOUR, endIndex = (end - history.start) / HOUR; - const previousIndex = (previousStart - history.start) / HOUR; for (const node of history.nodes) { let count = 0, before = 0, lastRelayed = '', previousLastRelayed = ''; for (const scope of node.scopes) { @@ -129,7 +186,7 @@ } return { interval: bounds, rows, currentCount: current.length, previousCount: previous.length, currentAppearances, previousAppearances, currentCoverage, previousCoverage, comparable, - limited: history.limited, added, retained, notObserved, + limited: history.limited, evidence, previousEvidence, added, retained, notObserved, appearanceChange: comparable ? currentAppearances - previousAppearances : null, percentChange: comparable && previousAppearances > 0 ? (currentAppearances - previousAppearances) / previousAppearances * 100 : null }; } @@ -153,7 +210,10 @@ } } } - return { history, scope: scope || '', appearances, nodes, times }; + // Withheld hours for this scope, precomputed once per payload/scope change. + const evidenceIncomplete = new Uint8Array(MAX_BUCKETS); + for (let index = 0; index < MAX_BUCKETS; index++) evidenceIncomplete[index] = evidenceCounts(history, scope || '', index, index + 1).incomplete ? 1 : 0; + return { history, scope: scope || '', appearances, nodes, times, evidenceIncomplete }; } // Playback reads only bounded hourly arrays. Partial/unknown history has @@ -172,18 +232,19 @@ const index = first + i, binStart = start + i * HOUR; const coverage = coverageState(history, binStart, binStart + HOUR); const observedAppearances = activity.appearances[index], observedNodes = activity.nodes[index]; - const complete = coverage === 'complete' && !history.limited; + const evidenceIncomplete = activity.evidenceIncomplete[index] === 1; + const complete = coverage === 'complete' && !history.limited && !evidenceIncomplete; bins.push({ start: new Date(binStart).toISOString(), end: new Date(binStart + HOUR).toISOString(), coverage, - observedAppearances, observedNodes, appearances: complete ? observedAppearances : null, + evidenceIncomplete, observedAppearances, observedNodes, appearances: complete ? observedAppearances : null, nodes: complete ? observedNodes : null }); total += observedAppearances; if (activity.times[index] > lastRelayed) lastRelayed = activity.times[index]; } return { bins, lastRelayed, - ratePerHour: frame.currentCoverage === 'complete' && !history.limited ? total / hours : null }; + ratePerHour: frame.currentCoverage === 'complete' && !history.limited && !frame.evidence.incomplete ? total / hours : null }; } - const api = { indexHistory, frameIndex, frameTime, interval, selectFrame, scopeCount, indexActivity, selectActivity }; + const api = { indexHistory, frameIndex, frameTime, interval, selectFrame, scopeCount, evidenceCounts, indexActivity, selectActivity }; if (typeof module !== 'undefined' && module.exports) module.exports = api; root.RegionsHistory = api; })(typeof window === 'undefined' ? globalThis : window); diff --git a/public/regions-map.js b/public/regions-map.js index 210019507..481d167b5 100644 --- a/public/regions-map.js +++ b/public/regions-map.js @@ -3,6 +3,11 @@ (function (root) { const MAX_POINTS = 5000; + // What a relay appearance counts since #507 (own-observation provenance). + const APPEARANCES_TEXT = 'A relay appearance is one transmission counted once for each distinct relay identified on any ' + + 'observer\u2019s own attributable scoped-flood path (the union across observers, not only the displayed path); repeated ' + + 'observers, paths and loops count once. One transmission can contribute several appearances, more observers can reveal more ' + + 'relays for the same traffic, and this is not a global packet total.'; const MAX_SCOPES = 128; const WINDOWS = { '1h': 'Last hour', '24h': 'Last 24 hours', '7d': 'Last 7 days' }; let leafletMap = null; @@ -161,6 +166,21 @@ return Number.isFinite(ms) ? new Date(ms).toISOString() : ''; } + // Evidence gaps are disclosed with counts. Only unprovable or capped evidence + // withholds values; identified exclusions never make counts incomplete. + function evidenceText(evidence) { + if (!evidence) return ''; + const count = value => Number.isSafeInteger(value) && value > 0 ? value : 0; + const excluded = count(evidence.excluded), unprovable = count(evidence.unprovable), capped = count(evidence.capped); + let text = ''; + if (evidence.truncated) text += ' The evidence safety limit was reached; affected hours are unknown, so rates and comparisons are withheld.'; + if (unprovable || capped) text += ' Observation evidence is incomplete in this interval: ' + unprovable.toLocaleString() + + ' observations could not be attributed (unknown header or transport code) and ' + capped.toLocaleString() + + ' were omitted by safety limits. Rates, chart values and comparisons are withheld only for the affected hours and scopes.'; + if (excluded) text += ' ' + excluded.toLocaleString() + ' observations were identified as direct, unscoped or for a different transport code and excluded; this does not make counts incomplete.'; + return text; + } + function coverageInfo(data) { const coverage = data.coverage || {}; const requestedStart = isoTime(data.windowStart), requestedEnd = isoTime(data.windowEnd); @@ -177,6 +197,7 @@ message = complete ? 'Retained history covers the requested interval.' : (availableStart ? 'Partial retained history; earlier activity in the requested interval is unavailable.' : 'No retained history is available in the requested interval.'); } + message += evidenceText(coverage.evidence); return { kind, message, requestedStart, requestedEnd, availableStart, generatedAt: isoTime(data.generatedAt) }; } @@ -292,7 +313,7 @@ head.appendChild(header); const body = element('tbody'); table.append(head, body); tableScroll.appendChild(table); details.appendChild(tableScroll); panel.appendChild(details); - panel.appendChild(element('p', 'regions-health-note', 'Traffic increase does not establish better health. Spam, observer reception and routing changes can affect observed counts. Relay appearances can count the same transmission via several relays; this is not a global packet total.')); + panel.appendChild(element('p', 'regions-health-note', 'Traffic increase does not establish better health. Spam, observer reception and routing changes can affect observed counts. ' + APPEARANCES_TEXT)); const rows = new Map(); function update(frame, activity, scope) { @@ -301,7 +322,8 @@ const missing = frame.rows.filter(row => row.count && row.node.positionSource !== 'reported').length; metrics.nodes.detail.textContent = missing + ' without usable positions'; metrics.appearances.value.textContent = frame.currentAppearances.toLocaleString(); - metrics.appearances.detail.textContent = activity.ratePerHour == null ? 'Rate unavailable: incomplete or limited history.' : + metrics.appearances.detail.textContent = activity.ratePerHour == null ? (frame.evidence.incomplete && !frame.limited ? + 'Rate unavailable: observation evidence is incomplete in this interval.' : 'Rate unavailable: incomplete or limited history.') : activity.ratePerHour.toFixed(2) + ' relay appearances per hour'; metrics['last-relayed'].value.replaceChildren(); if (activity.lastRelayed) metrics['last-relayed'].value.appendChild(timeElement(activity.lastRelayed)); @@ -336,7 +358,7 @@ record = { row, cells, rect, title }; rows.set(bin.start, record); } const complete = bin.appearances != null && bin.nodes != null; - const state = frame.limited ? 'limited' : bin.coverage; + const state = frame.limited ? 'limited' : bin.evidenceIncomplete ? 'incomplete evidence' : bin.coverage; record.cells.hour.textContent = bin.start.slice(0, 16).replace('T', ' ') + ' — ' + bin.end.slice(11, 16); record.cells.appearances.textContent = complete ? bin.appearances.toLocaleString() : '—'; record.cells.nodes.textContent = complete ? bin.nodes.toLocaleString() : '—'; @@ -450,7 +472,7 @@ details.appendChild(element('summary', '', 'Data & position details')); details.appendChild(element('p', '', 'Repeaters and rooms are shown where scoped relay traffic was observed. Positions are self-reported; their accuracy is unverified. Nodes without usable positions are counted but not mapped. Dashed outlines join the outermost node positions and do not measure RF coverage or administrative boundaries.')); details.appendChild(element('p', '', 'Retained history is not observer uptime or proof of continuous reception.')); - details.appendChild(element('p', '', 'Timeline counts are relay appearances: one transmission observed via two relay nodes contributes one appearance for each relay. Hours use UTC and intervals include their start and exclude their end. Positions use current or last-known self-reported coordinates; playback shows activity, not historical movement. Current catalog metadata can revise historical attribution after a refresh.')); + details.appendChild(element('p', '', 'Timeline counts are relay appearances. ' + APPEARANCES_TEXT + ' Hours use UTC and intervals include their start and exclude their end. Positions use current or last-known self-reported coordinates; playback shows activity, not historical movement. Current catalog metadata can revise historical attribution after a refresh.')); const exactBounds = element('div', 'regions-exact-bounds'); details.appendChild(exactBounds); shell.appendChild(details); @@ -570,7 +592,10 @@ comparison.appendChild(element('p', '', 'Observed relay nodes: ' + frame.currentCount + ' selected / ' + frame.previousCount + ' previous. Relay appearances: ' + frame.currentAppearances.toLocaleString() + ' selected / ' + frame.previousAppearances.toLocaleString() + ' previous.')); if (!frame.comparable) { - const reasons = frame.limited ? 'A data safety limit was reached.' : 'Selected history: ' + frame.currentCoverage + '; previous history: ' + frame.previousCoverage + '.'; + const reasons = frame.limited ? 'A data safety limit was reached.' : + frame.evidence.incomplete || frame.previousEvidence.incomplete ? 'Observation evidence is incomplete in the ' + + (frame.evidence.incomplete ? frame.previousEvidence.incomplete ? 'selected and previous intervals.' : 'selected interval.' : 'previous interval.') : + 'Selected history: ' + frame.currentCoverage + '; previous history: ' + frame.previousCoverage + '.'; comparison.appendChild(element('p', 'regions-comparison-limited', 'Comparison limited by available history. ' + reasons + ' Changes and new/not-observed classifications are unavailable; counts show only retained observations.')); comparison.setAttribute('data-comparable', 'false'); @@ -654,7 +679,7 @@ ' in selected interval. ' + frame.currentAppearances.toLocaleString() + ' relay appearances.'; if (!frame.currentCount) status.textContent += frame.currentCoverage === 'unavailable' ? ' History is unavailable for this interval.' : ' No scoped relay traffic was observed in the available history; this does not imply there is no coverage.'; - if (frame.limited) status.textContent += ' Data safety limits were reached; counts and map may be incomplete.'; + if (frame.limited || frame.evidence.incomplete) status.textContent += ' Evidence is incomplete or safety limits were reached; counts and map may be incomplete.'; } function fitTimeline() { @@ -675,6 +700,7 @@ unavailable: 'No retained history is available in the selected interval.', unknown: 'Retained history bounds are unknown; this view may be incomplete.', loading: 'History is still loading; this view may be incomplete.' }; coverage.textContent = messages[frame.currentCoverage]; + coverage.textContent += evidenceText({ ...frame.evidence, truncated: indexedHistory.evidence.truncated }); coverage.setAttribute('data-coverage-state', frame.currentCoverage); exactBounds.replaceChildren(element('p', '', 'Selected interval (UTC): ' + frame.interval.start + ' — ' + frame.interval.end), element('p', '', 'History dataset (UTC): ' + new Date(indexedHistory.start).toISOString() + ' — ' + new Date(indexedHistory.end).toISOString())); @@ -698,7 +724,7 @@ } activityPanel.update(frame, historyTools.selectActivity(activity, indexedHistory, frame), selection.scope); status.textContent = frame.currentCount + ' observed relay nodes; ' + frame.currentAppearances.toLocaleString() + - ' observed relay appearances in the selected interval. ' + (frame.currentCoverage === 'complete' && !frame.limited ? + ' observed relay appearances in the selected interval. ' + (frame.currentCoverage === 'complete' && !frame.limited && !frame.evidence.incomplete ? (frame.currentCount ? 'Observed activity; this does not measure RF coverage or observer uptime.' : 'No scoped relay activity observed in this complete interval; this does not establish an outage.') : 'Retained observations only; history is incomplete or limited.'); @@ -909,7 +935,7 @@ } const internals = { validPosition, validScope, collectScopeNodes, collectScopePoints, selectionSummary, - lastRelayedForScope, listScopes, scopeCatalog, mergeScopeCatalog, convexHull, parseSelection, parseSelectedScope, scopeHash, coverageInfo }; + lastRelayedForScope, listScopes, scopeCatalog, mergeScopeCatalog, convexHull, parseSelection, parseSelectedScope, scopeHash, coverageInfo, evidenceText }; if (typeof module !== 'undefined' && module.exports) module.exports = internals; if (typeof root.registerPage === 'function') root.registerPage('regions', { init, destroy }); })(typeof window === 'undefined' ? globalThis : window); diff --git a/public/regions-routes.js b/public/regions-routes.js index 6bcdfe9a4..e1192ac68 100644 --- a/public/regions-routes.js +++ b/public/regions-routes.js @@ -9,7 +9,9 @@ function indexRoutes(data, history, validScope, validPosition) { const nodes = new Map(history.nodes.map(node => [node.publicKey.toLowerCase(), node])); let limited = !!(data.stepsTruncated || data.stepBucketsTruncated || data.stepEvidence?.sourceTruncated || data.stepEvidence?.incomplete); - for (const key of ['skippedObservations', 'unresolvedHops', 'missingEndpointSteps']) { + // Directed steps stay snapshot-wide: identified exclusions are not gaps, but + // unprovable/capped observations, unresolved hops and step caps can remove steps. + for (const key of ['unprovableObservations', 'cappedObservations', 'unresolvedHops', 'cappedStepAppearances', 'missingEndpointSteps']) { const count = data.stepEvidence?.[key]; if (count != null && (!Number.isSafeInteger(count) || count < 0 || count > 0)) limited = true; } @@ -57,8 +59,9 @@ } const evidence = data.stepEvidence || {}; const count = value => Number.isSafeInteger(value) && value >= 0 ? value : 0; - return { history, steps, limited, evidence: { skippedObservations: count(evidence.skippedObservations), unresolvedHops: count(evidence.unresolvedHops), missingEndpointSteps: count(evidence.missingEndpointSteps), - sourceTruncated: evidence.sourceTruncated === true, incomplete: evidence.incomplete === true }, cached: null }; + return { history, steps, limited, evidence: { excludedObservations: count(evidence.excludedObservations), unprovableObservations: count(evidence.unprovableObservations), + cappedObservations: count(evidence.cappedObservations), unresolvedHops: count(evidence.unresolvedHops), cappedStepAppearances: count(evidence.cappedStepAppearances), + missingEndpointSteps: count(evidence.missingEndpointSteps), sourceTruncated: evidence.sourceTruncated === true, incomplete: evidence.incomplete === true }, cached: null }; } function sharedPairs(history, first, last, scope) { @@ -130,8 +133,8 @@ steps.sort((a, b) => b.count - a.count || a.key.localeCompare(b.key)); const pairs = sharedPairs(history, first, last, selection.scope || ''); const value = { ...pairs, steps, stepAppearances, unmappedSteps, - complete: frame.currentCoverage === 'complete' && !history.limited && !index.limited, - sharedComplete: frame.currentCoverage === 'complete' && !history.limited, + complete: frame.currentCoverage === 'complete' && !history.limited && !frame.evidence.incomplete && !index.limited, + sharedComplete: frame.currentCoverage === 'complete' && !history.limited && !frame.evidence.incomplete, evidence: index.evidence, limited: index.limited || history.limited, mapSteps: steps.filter(step => positioned(step.fromNode) && positioned(step.toNode)).slice(0, MAP_LIMIT) }; index.cached = { key, value }; @@ -150,7 +153,7 @@ panel.setAttribute('aria-label', 'Scope connections and observed relay steps'); const summary = el('p', 'regions-connections-summary'); panel.appendChild(summary); panel.appendChild(el('h2', '', 'Shared relay-history memberships')); - panel.appendChild(el('p', 'regions-connections-provenance', 'Distinct visible identities attributed to both scopes by the transmission-level relay history used by Timeline/Health. Observation route/scope variants are not independently verified here; mixed flood/direct transmissions can include planned-path candidates. Only directed steps below use each observation’s own frame evidence.')); + panel.appendChild(el('p', 'regions-connections-provenance', 'Distinct visible identities with attributable scoped flood observations in both scopes during the selected interval. Timeline, Health and shared memberships use each observation’s own frame evidence; direct planned paths and unknown or mismatched transport codes are excluded. Membership in both scopes does not establish a cross-scope forwarding event.')); panel.appendChild(el('p', 'regions-connections-note', 'Shared history membership does not establish cross-scope traffic, a scope transition or a direct radio link.')); const pairSummary = el('p', 'regions-pairs-summary'); panel.appendChild(pairSummary); @@ -222,8 +225,10 @@ return [direction, el('td', '', step.scope), el('td', '', step.count.toLocaleString()), el('td', '', step.lastRelayed)]; }, step => JSON.stringify([step.count, step.lastRelayed, step.fromNode.name, step.toNode.name])); const evidence = value.evidence; - diagnostics.textContent = 'Snapshot-wide evidence diagnostics (not selected-interval totals): ' + evidence.skippedObservations + - ' excluded or unusable observations; ' + evidence.unresolvedHops + ' unresolved hops; ' + evidence.missingEndpointSteps + + diagnostics.textContent = 'Snapshot-wide directed-step diagnostics (not selected-interval totals): ' + evidence.excludedObservations + + ' excluded observations; ' + evidence.unprovableObservations + ' unprovable observations; ' + evidence.cappedObservations + + ' observations omitted by source limits; ' + evidence.unresolvedHops + ' unresolved hops; ' + evidence.cappedStepAppearances + + ' step appearances omitted by step limits; ' + evidence.missingEndpointSteps + ' steps without publishable endpoint metadata.' + (evidence.incomplete || evidence.sourceTruncated ? ' The source snapshot is incomplete or capped.' : '') + ' Direct planned paths, TRACE/adverts and mismatched or unknown scopes are excluded. Unresolved or hidden hops break adjacency.'; mapSummary.textContent = value.mapSteps.length + ' directed step rows mapped of ' + (value.steps.length - value.unmappedSteps) + diff --git a/test-regions-map-e2e.js b/test-regions-map-e2e.js index 5a43c5f9b..5993f861f 100644 --- a/test-regions-map-e2e.js +++ b/test-regions-map-e2e.js @@ -63,7 +63,8 @@ function routesSnapshot(override = {}) { { from: key('cc33'), to: key('bb22'), scope: '#other', buckets: [bucket(503, 4)] }, { from: key('6666'), to: key('7777'), scope: '#routeonly', buckets: [bucket(503, 7)] }, ], stepsTruncated: false, stepBucketsTruncated: false, - stepEvidence: { skippedObservations: 0, unresolvedHops: 0, missingEndpointSteps: 0, sourceTruncated: false, incomplete: false }, ...override }; + stepEvidence: { excludedObservations: 0, unprovableObservations: 0, cappedObservations: 0, unresolvedHops: 0, cappedStepAppearances: 0, + missingEndpointSteps: 0, sourceTruncated: false, incomplete: false }, ...override }; } async function main() { @@ -296,6 +297,17 @@ async function main() { coverageOverride = null; }); + await test('retained history cannot hide missing observation evidence', async () => { + coverageOverride = { start: '2026-09-01T00:00:00Z', complete: true, loading: false, + evidence: { excluded: 4, unprovable: 2, capped: 1, truncated: false, scopes: [{ name: '#example', excluded: 4, unprovable: 2, capped: 1 }] } }; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('.regions-coverage').textContent.includes('Observation evidence is incomplete')); + assert.strictEqual(await page.locator('.regions-coverage').getAttribute('data-coverage-state'), 'complete', 'retention is separate from attribution'); + assert.match(await page.locator('.regions-coverage').textContent(), /2 observations could not be attributed .* and 1 were omitted by safety limits/); + assert.match(await page.locator('.regions-coverage').textContent(), /4 observations were identified as direct, unscoped or for a different transport code and excluded/); + coverageOverride = null; + }); + await test('timeline deep links clamp to completed hours and compare equal adjacent periods', async () => { await page.evaluate(() => { location.hash = '#/regions?view=timeline&window=1h&scope=%23example&at=2026-10-16T12%3A47%3A00Z&compare=1'; }); await page.waitForFunction(() => document.querySelector('.regions-comparison')?.getAttribute('data-comparable') === 'true'); @@ -395,12 +407,18 @@ async function main() { await test('partial, unknown, loading and capped history withhold all change claims', async () => { const overrides = [{ coverage: { start: '2026-10-16T10:30:00Z', loading: false } }, - { coverage: { start: null, loading: false } }, { coverage: { start: null, loading: true } }, { bucketsTruncated: true }]; + { coverage: { start: null, loading: false } }, { coverage: { start: null, loading: true } }, { bucketsTruncated: true }, + { coverage: { start: '2026-09-01T00:00:00Z', loading: false, evidence: { excluded: 0, unprovable: 1, capped: 0, truncated: false, + scopes: [{ name: '#example', excluded: 0, unprovable: 1, capped: 0, buckets: [{ index: 503, excluded: 0, unprovable: 1, capped: 0 }] }] } } }]; for (const value of overrides) { historyOverride = value; await page.getByRole('button', { name: 'Refresh', exact: true }).click(); await page.waitForFunction(() => document.querySelector('.regions-page')?.getAttribute('aria-busy') === 'false' && document.querySelector('.regions-comparison')?.getAttribute('data-comparable') === 'false'); assert.match(await page.locator('.regions-comparison-limited').textContent(), /Comparison limited by available history/); + if (value.coverage?.evidence) { + assert.match(await page.locator('.regions-comparison-limited').textContent(), /Observation evidence is incomplete in the selected interval/); + assert.doesNotMatch(await page.locator('.regions-comparison-limited').textContent(), /A data safety limit was reached/); + } assert.strictEqual(await page.locator('.regions-state-added, .regions-state-not-observed, .regions-appearance-change').count(), 0); assert.match(await page.locator('.regions-status').textContent(), /8 relay appearances/); assert.strictEqual(await page.locator('.regions-leaflet path.leaflet-interactive').count(), 2, 'limited comparison maps only observed selected relays'); @@ -432,6 +450,9 @@ async function main() { assert.strictEqual(await page.locator('[data-metric="nodes"] .regions-health-value').textContent(), '4'); assert.strictEqual(await page.locator('[data-metric="appearances"] .regions-health-value').textContent(), '15'); assert.match(await page.locator('.regions-health-rate').textContent(), /0\.63 relay appearances per hour/); + assert.match(await page.locator('.regions-health-note').textContent(), /union across observers, not only the displayed path\); repeated observers, paths and loops count once/, + 'the UI states what a relay appearance counts'); + assert.match(await page.locator('.regions-details').textContent(), /Timeline counts are relay appearances\. A relay appearance is one transmission counted once for each distinct relay/); assert.strictEqual(await page.locator('[data-metric="last-relayed"] time').getAttribute('datetime'), '2026-10-16T11:00:01.000Z'); assert.strictEqual(await page.locator('.regions-trend-bar').count(), 24); assert.strictEqual(await page.locator('.regions-trend-gap').count(), 0); @@ -513,7 +534,8 @@ async function main() { assert.match(await partial.textContent(), /7 appearances; 2 relay nodes/); assert.strictEqual(await partial.locator('[data-column="appearances"]').textContent(), '—'); for (const override of [{ coverage: { start: null, loading: false } }, { coverage: { start: null, loading: true } }, - { truncated: true }, { scopesTruncated: true }, { bucketsTruncated: true }]) { + { truncated: true }, { scopesTruncated: true }, { bucketsTruncated: true }, + { coverage: { start: '2026-09-01T00:00:00Z', loading: false, evidence: { truncated: true, scopes: [] } } }]) { historyOverride = override; await page.getByRole('button', { name: 'Refresh', exact: true }).click(); await page.waitForFunction(() => document.querySelector('.regions-health')?.getAttribute('data-ready') === 'true'); @@ -528,6 +550,31 @@ async function main() { await page.waitForFunction(() => document.querySelector('.regions-health')?.getAttribute('data-ready') === 'true'); }); + await test('evidence gaps withhold only the affected hours and scopes', async () => { + historyOverride = { coverage: { start: '2026-09-25T12:00:00Z', loading: false, evidence: { excluded: 6, unprovable: 1, capped: 1, truncated: false, scopes: [ + { name: '#example', excluded: 6, unprovable: 1, capped: 0, buckets: [{ index: 500, excluded: 0, unprovable: 1, capped: 0 }, { index: 503, excluded: 6, unprovable: 0, capped: 0 }] }, + { name: '#other', excluded: 0, unprovable: 0, capped: 1, buckets: [{ index: 503, excluded: 0, unprovable: 0, capped: 1 }] }] } } }; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('.regions-health')?.getAttribute('data-ready') === 'true' && document.querySelectorAll('.regions-trend-gap').length === 1); + assert.strictEqual(await page.locator('.regions-trend-bar').count(), 23, 'only the unprovable #example hour is a gap'); + assert.match(await page.locator('.regions-hourly-table tbody tr[data-start="2026-10-16T08:00:00.000Z"]').textContent(), /incomplete evidence/); + assert.match(await page.locator('.regions-health-rate').textContent(), /Rate unavailable: observation evidence is incomplete/); + await page.getByLabel('Region period').selectOption('1h'); + await page.waitForFunction(() => /relay appearances per hour/.test(document.querySelector('.regions-health-rate')?.textContent || '')); + assert.match(await page.locator('.regions-health-rate').textContent(), /8\.00 relay appearances per hour/, 'an excluded-only #example hour keeps its rate'); + assert.strictEqual(await page.locator('.regions-trend-gap').count(), 0); + assert.match(await page.locator('.regions-coverage').textContent(), /6 observations were identified as direct, unscoped or for a different transport code/); + await page.getByLabel('Region scope').selectOption(''); + await page.waitForFunction(() => /Rate unavailable/.test(document.querySelector('.regions-health-rate')?.textContent || '')); + assert.strictEqual(await page.locator('.regions-trend-gap').count(), 1, 'all scopes include the capped #other hour'); + assert.match(await page.locator('.regions-coverage').textContent(), /0 observations could not be attributed .* and 1 were omitted by safety limits/); + await page.getByLabel('Region scope').selectOption('#example'); + await page.getByLabel('Region period').selectOption('24h'); + historyOverride = {}; + await page.getByRole('button', { name: 'Refresh', exact: true }).click(); + await page.waitForFunction(() => document.querySelector('.regions-health')?.getAttribute('data-ready') === 'true' && document.querySelectorAll('.regions-trend-gap').length === 0); + }); + await test('health SVG and exact-value table are accessible, themed and retain keyed rows', async () => { const chart = page.getByRole('img', { name: /Hourly relay appearances/ }); assert.strictEqual(await chart.count(), 1); @@ -614,9 +661,9 @@ async function main() { assert.strictEqual(await page.getByRole('columnheader', { name: 'History identities', exact: true }).isVisible(), true); const provenance = page.locator('.regions-connections-provenance'); assert.strictEqual(await provenance.isVisible(), true); - assert.match(await provenance.textContent(), /transmission-level relay history used by Timeline\/Health/); - assert.match(await provenance.textContent(), /Observation route\/scope variants are not independently verified here; mixed flood\/direct transmissions can include planned-path candidates/); - assert.match(await provenance.textContent(), /Only directed steps below use each observation’s own frame evidence/); + assert.match(await provenance.textContent(), /Timeline, Health and shared memberships use each observation’s own frame evidence/); + assert.match(await provenance.textContent(), /direct planned paths and unknown or mismatched transport codes are excluded/); + assert.match(await provenance.textContent(), /does not establish a cross-scope forwarding event/); assert.match(await page.locator('.regions-pairs-summary').textContent(), /1 distinct identities with shared history membership/); assert.match(await page.locator('.regions-connections-note').first().textContent(), /does not establish cross-scope traffic/); assert.strictEqual(await page.locator('.regions-steps-table tbody tr').count(), 3); @@ -627,7 +674,7 @@ async function main() { assert(directions.some(value => value.includes('New relay → ' + EVIL))); assert.strictEqual(await page.locator('.regions-directed-step').count(), 2); assert(await page.locator('.regions-directed-step').first().getAttribute('marker-end')); - assert.match(await page.locator('.regions-step-diagnostics').textContent(), /Snapshot-wide evidence diagnostics \(not selected-interval totals\)/); + assert.match(await page.locator('.regions-step-diagnostics').textContent(), /Snapshot-wide directed-step diagnostics \(not selected-interval totals\): 0 excluded observations; 0 unprovable observations; 0 observations omitted by source limits; 0 unresolved hops; 0 step appearances omitted by step limits/); assert.strictEqual(await page.locator('.regions-connections img').count(), 0); assert.strictEqual(await page.evaluate(() => window.__xss), undefined); const params = new URLSearchParams((await page.evaluate(() => location.hash)).split('?')[1]); @@ -660,8 +707,11 @@ async function main() { await test('Connections qualifies partial, unknown, loading and capped evidence without absence claims', async () => { for (const override of [{ coverage: { start: '2026-10-16T11:30:00Z', loading: false } }, { coverage: { start: null, loading: false } }, { coverage: { start: null, loading: true } }, + { coverage: { start: '2026-09-01T00:00:00Z', loading: false, evidence: { excluded: 0, unprovable: 0, capped: 2, truncated: false, + scopes: [{ name: '#example', excluded: 0, unprovable: 0, capped: 2, buckets: [{ index: 503, excluded: 0, unprovable: 0, capped: 2 }] }] } } }, { stepsTruncated: true }, { stepBucketsTruncated: true }, { stepEvidence: { sourceTruncated: true, incomplete: true } }, - { stepEvidence: { skippedObservations: 12, unresolvedHops: 3, missingEndpointSteps: 0, sourceTruncated: false, incomplete: false } }]) { + { stepEvidence: { excludedObservations: 0, unprovableObservations: 12, cappedObservations: 0, unresolvedHops: 3, cappedStepAppearances: 0, missingEndpointSteps: 0, sourceTruncated: false, incomplete: false } }, + { stepEvidence: { excludedObservations: 0, unprovableObservations: 0, cappedObservations: 0, unresolvedHops: 0, cappedStepAppearances: 4, missingEndpointSteps: 0, sourceTruncated: false, incomplete: false } }]) { routesOverride = override; await page.getByRole('button', { name: 'Refresh', exact: true }).click(); await page.waitForFunction(() => document.querySelector('.regions-connections')?.getAttribute('data-ready') === 'true'); diff --git a/test-regions-map.js b/test-regions-map.js index dd0d6aa4b..355798c1a 100644 --- a/test-regions-map.js +++ b/test-regions-map.js @@ -88,6 +88,15 @@ assert.equal(regions.coverageInfo({ ...snapshot, coverage: { start: null, comple assert.equal(regions.coverageInfo({ ...snapshot, coverage: { start: null, complete: false, loading: true } }).kind, 'loading'); assert.match(regions.coverageInfo({ ...snapshot, coverage: { start: '2026-10-17T00:00:00Z', complete: false } }).message, /No retained history/); assert.equal(regions.coverageInfo({ ...snapshot, coverage: { start: '2026-10-16T09:00:00Z', complete: true } }).kind, 'partial', 'inconsistent complete flag cannot overstate available history'); +const omittedEvidence = regions.coverageInfo({ ...snapshot, coverage: { start: '2026-10-01T00:00:00Z', complete: true, + evidence: { excluded: 4, unprovable: 2, capped: 1, truncated: false, scopes: [] } } }); +assert.equal(omittedEvidence.kind, 'complete', 'retention and attributable evidence are separate'); +assert.match(omittedEvidence.message, /Observation evidence is incomplete in this interval: 2 observations could not be attributed .* and 1 were omitted by safety limits/); +assert.match(omittedEvidence.message, /4 observations were identified as direct, unscoped or for a different transport code and excluded; this does not make counts incomplete/); +const excludedOnly = regions.coverageInfo({ ...snapshot, coverage: { start: '2026-10-01T00:00:00Z', complete: true, evidence: { excluded: 3, unprovable: 0, capped: 0, truncated: false, scopes: [] } } }); +assert.doesNotMatch(excludedOnly.message, /incomplete in this interval/, 'identified exclusions are not incomplete evidence'); +assert.match(regions.evidenceText({ truncated: true }), /affected hours are unknown/); +assert.equal(regions.evidenceText(undefined), ''); const history = require('./public/regions-history.js'); const historyEnd = '2026-10-16T12:00:00Z'; @@ -131,6 +140,15 @@ for (const coverage of [{ start: null, loading: false }, { start: historyStart, frame = history.selectFrame(history.indexHistory(historyData({ truncated: true }), historyData().nodes), { window: '1h', compare: true }); assert.equal(frame.comparable, false, 'caps withhold changes'); assert.equal(frame.limited, true); +const unverifiedHistory = history.indexHistory(historyData({ coverage: { start: historyStart, loading: false, + evidence: { excluded: 0, unprovable: 1, capped: 0, truncated: false, scopes: [{ name: '#dk', excluded: 0, unprovable: 1, capped: 0, buckets: [{ index: 503, excluded: 0, unprovable: 1, capped: 0 }] }] } } }), historyData().nodes); +frame = history.selectFrame(unverifiedHistory, { window: '1h', compare: true }); +assert.equal(frame.currentCoverage, 'complete', 'evidence omissions do not rewrite the retained-history floor'); +assert.equal(frame.limited, false, 'evidence gaps are not snapshot-wide publication limits'); +assert.equal(frame.evidence.incomplete, true); +assert.equal(frame.comparable, false, 'unattributable observations cannot support change classifications'); +assert.equal(frame.added, null); +assert.equal(history.selectActivity(history.indexActivity(unverifiedHistory, ''), unverifiedHistory, frame).ratePerHour, null); frame = history.selectFrame(indexed, { window: '1h', scope: '#absent', compare: true }); assert.equal(frame.currentCount, 0); assert.equal(frame.currentCoverage, 'complete', 'no observations is separate from unavailable history'); @@ -169,7 +187,7 @@ const frameBeforeHealth = JSON.stringify(healthFrame); let health = history.selectActivity(activity, healthHistory, healthFrame); assert.equal(health.bins.length, 1); assert.deepEqual(health.bins[0], { start: '2026-10-16T11:00:00.000Z', end: '2026-10-16T12:00:00.000Z', - coverage: 'complete', observedAppearances: 12, observedNodes: 2, appearances: 12, nodes: 2 }); + coverage: 'complete', evidenceIncomplete: false, observedAppearances: 12, observedNodes: 2, appearances: 12, nodes: 2 }); assert.equal(health.lastRelayed, '2026-10-16T11:59:00.000Z'); assert.equal(health.ratePerHour, 12); assert.equal(JSON.stringify(healthFrame), frameBeforeHealth, 'health derivation does not change shared comparison results'); @@ -243,6 +261,49 @@ for (const bounds of [ { start: 'invalid', end: 'invalid' }, ]) assert.throws(() => history.selectActivity(activity, healthHistory, { ...healthFrame, interval: bounds }), /activity interval/i); +// F1 (PR #507 review): evidence gaps are per hour bucket and scope. Excluded +// observations never withhold values; unprovable/capped ones withhold only the +// rates, chart values and comparisons that include an affected bucket/scope. +const evidenceCoverage = { start: historyStart, loading: false, evidence: { excluded: 9, unprovable: 1, capped: 1, truncated: false, scopes: [ + { name: '#dk', excluded: 9, unprovable: 1, capped: 0, buckets: [{ index: 500, excluded: 0, unprovable: 1, capped: 0 }, { index: 503, excluded: 9, unprovable: 0, capped: 0 }] }, + { name: '#se', excluded: 0, unprovable: 0, capped: 1, buckets: [{ index: 503, excluded: 0, unprovable: 0, capped: 1 }] }, +] } }; +const gapHistory = history.indexHistory(historyData({ coverage: evidenceCoverage }), healthNodes); +assert.equal(gapHistory.limited, false, 'evidence gaps are not a snapshot-wide limit'); +let gapFrame = history.selectFrame(gapHistory, { window: '1h', scope: '#dk', compare: true }); +assert.equal(gapFrame.comparable, true, 'excluded-only #dk hours stay comparable'); +assert.deepEqual(gapFrame.evidence, { excluded: 9, unprovable: 0, capped: 0, incomplete: false }); +let gapView = history.selectActivity(history.indexActivity(gapHistory, '#dk'), gapHistory, gapFrame); +assert.equal(gapView.ratePerHour, 8, 'an unaffected bucket keeps its rate despite gaps elsewhere'); +assert.equal(gapView.bins[0].appearances, 8); +gapFrame = history.selectFrame(gapHistory, { window: '1h', compare: true }); +assert.equal(gapFrame.comparable, false, 'all scopes include the capped #se hour'); +assert.deepEqual(gapFrame.evidence, { excluded: 9, unprovable: 0, capped: 1, incomplete: true }); +gapView = history.selectActivity(history.indexActivity(gapHistory, ''), gapHistory, gapFrame); +assert.equal(gapView.ratePerHour, null); +assert.equal(gapView.bins[0].appearances, null); +assert.equal(gapView.bins[0].evidenceIncomplete, true); +assert.equal(gapView.bins[0].observedAppearances, 12, 'retained observations stay visible'); +gapFrame = history.selectFrame(gapHistory, { window: '24h', scope: '#dk' }); +gapView = history.selectActivity(history.indexActivity(gapHistory, '#dk'), gapHistory, gapFrame); +assert.equal(gapView.ratePerHour, null, 'the 24h #dk interval contains the unprovable hour'); +assert.equal(gapView.bins[20].appearances, null, 'bucket 500 is withheld'); +assert.equal(gapView.bins[20].evidenceIncomplete, true); +assert.equal(gapView.bins[23].appearances, 8, 'other #dk bins in the same interval stay available'); +assert.equal(gapView.bins[22].appearances, 2); +assert.equal(history.selectFrame(gapHistory, { window: '1h', at: '2026-10-16T11:00:00Z', scope: '#dk', compare: true }).comparable, true, + 'buckets 502 vs 501 are unaffected'); +assert.equal(history.selectFrame(gapHistory, { window: '1h', at: '2026-10-16T10:00:00Z', scope: '#dk', compare: true }).comparable, false, + 'the previous interval contains unprovable bucket 500'); +assert.equal(history.selectFrame(gapHistory, { window: '1h', at: '2026-10-16T10:00:00Z', scope: '#se', compare: true }).comparable, true, + 'another scope is unaffected by the #dk gap'); +for (const evidence of [{ ...evidenceCoverage.evidence, truncated: true }, { scopes: [{ name: '#dk', buckets: [{ index: 504, unprovable: 1 }] }] }, { scopes: 'bad' }]) { + const failClosed = history.indexHistory(historyData({ coverage: { ...evidenceCoverage, evidence } }), healthNodes); + const selected = history.selectFrame(failClosed, { window: '1h', scope: '#dk', compare: true }); + assert.equal(selected.comparable, false, 'unknown evidence placement fails closed'); + assert.equal(history.selectActivity(history.indexActivity(failClosed, '#dk'), failClosed, selected).ratePerHour, null); +} + // Reading the sparse payload after indexing must not be part of chart playback. const playbackFrame = history.selectFrame(healthHistory, { window: '7d' }); Object.defineProperty(healthHistory, 'nodes', { get() { throw new Error('sparse node rescan'); } }); @@ -286,7 +347,7 @@ const routeData = { ...historyData(), nodes: routeNodes, steps: [ { from: fullKey('a'), to: fullKey('b'), scope: '#a', buckets: [bucket(503, 3)] }, { from: fullKey('b'), to: fullKey('a'), scope: '#a', buckets: [bucket(503, 2)] }, { from: fullKey('a'), to: fullKey('c'), scope: '#c', buckets: [bucket(503)] }, -], stepEvidence: { skippedObservations: 0, unresolvedHops: 0, sourceTruncated: false, incomplete: false } }; +], stepEvidence: { excludedObservations: 5, unprovableObservations: 0, cappedObservations: 0, unresolvedHops: 0, cappedStepAppearances: 0, sourceTruncated: false, incomplete: false } }; const routeHistory = history.indexHistory(routeData, routeNodes); let routeIndex = routes.indexRoutes(routeData, routeHistory, regions.validScope); let routeFrame = history.selectFrame(routeHistory, { window: '1h' }); @@ -303,11 +364,20 @@ assert.equal(connections.steps.length, 2); assert.equal(connections.pairs.length, 2); assert(connections.steps.some(step => step.from === fullKey('a') && step.to === fullKey('b'))); assert(connections.steps.some(step => step.from === fullKey('b') && step.to === fullKey('a'))); +for (const [index, expected] of [[503, false], [400, true]]) { + const gapRoutes = { ...routeData, coverage: { ...routeData.coverage, evidence: { excluded: 0, unprovable: 1, capped: 0, truncated: false, + scopes: [{ name: '#a', excluded: 0, unprovable: 1, capped: 0, buckets: [{ index, excluded: 0, unprovable: 1, capped: 0 }] }] } } }; + const gapHistory = history.indexHistory(gapRoutes, routeNodes); + const selected = routes.selectRoutes(routes.indexRoutes(gapRoutes, gapHistory, regions.validScope), gapHistory, history.selectFrame(gapHistory, { window: '1h' }), {}); + assert.equal(selected.sharedComplete, expected, 'shared memberships are qualified only by gaps inside the selected interval'); + assert.equal(selected.complete, expected); +} const limitedData = { ...routeData, stepBucketsTruncated: true }; connections = routes.selectRoutes(routes.indexRoutes(limitedData, routeHistory, regions.validScope), routeHistory, routeFrame, {}); assert.equal(connections.complete, false); assert.equal(connections.stepAppearances, 6, 'limited evidence remains an explicit lower bound'); -for (const key of ['skippedObservations', 'unresolvedHops', 'missingEndpointSteps']) { +assert.equal(connections.evidence.excludedObservations, 5, 'identified exclusions are disclosed without qualifying steps'); +for (const key of ['unprovableObservations', 'cappedObservations', 'unresolvedHops', 'cappedStepAppearances', 'missingEndpointSteps']) { const contradictory = { ...routeData, stepEvidence: { ...routeData.stepEvidence, [key]: 1, incomplete: false } }; const selected = routes.selectRoutes(routes.indexRoutes(contradictory, routeHistory, regions.validScope), routeHistory, routeFrame, {}); assert.equal(selected.complete, false, 'omitted evidence cannot be complete even when the server flag contradicts diagnostics');