From f07ad4e7e75d0bcfad1fde0cfc3241fd5692a590 Mon Sep 17 00:00:00 2001 From: Babissimo Date: Thu, 24 Sep 2026 11:56:28 +0100 Subject: [PATCH 1/2] Rank nodes by 7-day availability, and keep their counts across restarts The leaderboard's uptime was the time since a node last registered with this process, which every reconnect, config change and startup reset, and which nothing persisted: after a deploy every node read the same few minutes. total_detections, the column the leaderboard ranks by, was not persisted either, so the ranking also restarted with every deploy. retina-analytics now measures availability, the share of the minutes this server was up in which a node delivered a frame, over 7 days and 24 hours. The server's side of that is a clock task marking each minute the process is up. It starts five minutes after boot, because nodes take a while to reconnect after a restart and those minutes are the server's, not theirs. The snapshot carries each node's metrics (its week of minutes, when it was first seen, and its running counts) and the server's minutes with them, restored over what startup priming registered. The leaderboard row publishes availability_7d in place of uptime_s, and the e2e spec asserts the new fields on the deployed API. Ticket: https://app.clickup.com/t/123zgec4k7v Co-Authored-By: Claude Opus 5.5 --- backend/config/constants.py | 6 ++++++ backend/main.py | 2 ++ backend/routes/admin.py | 8 ++++--- backend/services/state_snapshot.py | 20 +++++++++++++++-- backend/services/tasks/__init__.py | 2 ++ backend/services/tasks/periodic.py | 12 ++++++++++- backend/tests/test_admin_routes.py | 5 +++-- backend/tests/test_publication.py | 5 +---- backend/tests/test_server_clock.py | 28 ++++++++++++++++++++++++ backend/tests/test_state_snapshot.py | 32 ++++++++++++++++++++++++++++ backend/vulture_whitelist.py | 1 + e2e/specs/nodes.spec.ts | 11 ++++++---- libs/retina-analytics | 2 +- 13 files changed, 117 insertions(+), 17 deletions(-) create mode 100644 backend/tests/test_server_clock.py diff --git a/backend/config/constants.py b/backend/config/constants.py index ba8353cf1..203db259b 100644 --- a/backend/config/constants.py +++ b/backend/config/constants.py @@ -476,6 +476,12 @@ def _assoc_alt_layers_km() -> tuple[float, ...]: # ── Periodic task intervals ────────────────────────────────────────────────── REPUTATION_INTERVAL_S = 60 # Reputation evaluator sleep (s) +# The server clock behind node availability: the minutes this process could +# take frames. Ticked well inside a minute so none is missed. It starts only +# after a grace, as nodes take a while to reconnect after a restart and those +# minutes are the server's, not theirs. +SERVER_CLOCK_TICK_S = 20 +SERVER_CLOCK_GRACE_S = 300 ADSB_TRUTH_INTERVAL_S = 120 # ADS-B truth fetcher sleep (s) ADSB_BACKOFF_S = 300 # Rate-limit backoff (s) # adsb-service poll for claim candidates (services/tasks/adsb_fallback.py): every diff --git a/backend/main.py b/backend/main.py index 9040e5001..da755767e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -79,6 +79,7 @@ prune_synthetic_nodes, reputation_evaluator, retire_disposable_nodes_task, + server_clock_task, start_solver_workers, stop_solver_workers, storage_refresh_task, @@ -251,6 +252,7 @@ async def _snapshot_loop(): for task_fn in ( server.serve_forever, reputation_evaluator, + server_clock_task, prune_synthetic_nodes, retire_disposable_nodes_task, adsb_truth_fetcher, diff --git a/backend/routes/admin.py b/backend/routes/admin.py index 57c1f0748..d44204ccf 100644 --- a/backend/routes/admin.py +++ b/backend/routes/admin.py @@ -607,7 +607,9 @@ class PublicLeaderboardRow(BaseModel): detections: int frames: int tracks: int - uptime_s: float + # Share of the last week's minutes the node delivered in; None until a + # whole minute has been measured. + availability_7d: float | None avg_snr: float trust_score: float reputation: float @@ -687,7 +689,7 @@ async def _cold_start_summaries() -> dict: @router.get("/leaderboard") async def leaderboard(caller=Depends(get_optional_user)): - """Rankings by detections, uptime and trust, open to anyone. + """Rankings by detections, availability and trust, open to anyone. The odd one out under this prefix, which is otherwise the admin API. A caller with no session gets PublicLeaderboardRow, which is what lets the @@ -742,7 +744,7 @@ async def leaderboard(caller=Depends(get_optional_user)): detections=m.get("total_detections", 0), frames=m.get("total_frames", 0), tracks=m.get("total_tracks", 0), - uptime_s=m.get("uptime_s", 0), + availability_7d=m.get("availability_7d"), avg_snr=m.get("avg_snr", 0), trust_score=s.get("trust", {}).get("trust_score", 0), reputation=s.get("reputation", {}).get("reputation", 0), diff --git a/backend/services/state_snapshot.py b/backend/services/state_snapshot.py index cb7e7d9cb..fbdfc00b6 100644 --- a/backend/services/state_snapshot.py +++ b/backend/services/state_snapshot.py @@ -1,8 +1,9 @@ """Lightweight state snapshot: save/restore high-value in-memory state across restarts. Saved every SAVE_INTERVAL_S (60 s) by a background task. Restored once at startup. -Persists: trust_scores, reputations, accuracy_samples, chain_entries, -node_identities, iq_commitments, anomaly_log, simulation_config. +Persists: trust_scores, reputations, node_metrics, server_minutes, +accuracy_samples, chain_entries, node_identities, iq_commitments, anomaly_log, +simulation_config. """ import hashlib @@ -38,6 +39,10 @@ def save_snapshot() -> None: for nid, rep in state.node_analytics.reputations.items(): reps[nid] = asdict(rep) + # Node availability and the leaderboard's counts. The server's own + # minutes go with them, as they are what availability is measured against. + node_metrics = {nid: m.to_state() for nid, m in list(state.node_analytics.metrics.items())} + identities = {} for nid, ident in state.node_identities.items(): identities[nid] = ident.to_dict() @@ -46,6 +51,8 @@ def save_snapshot() -> None: "saved_at": time.time(), "trust_scores": trust, "reputations": reps, + "node_metrics": node_metrics, + "server_minutes": state.node_analytics.server_minutes.to_state(), "accuracy_samples": list(state.accuracy_samples), "chain_entries": dict(state.chain_entries), "node_identities": identities, @@ -170,6 +177,8 @@ def _restore_simulation_config(snap: dict) -> None: def restore_snapshot() -> bool: """Load state from disk snapshot. Returns True if restored, False if no snapshot found.""" + from retina_analytics.availability import MinuteRing + from retina_analytics.metrics import NodeMetrics from retina_analytics.reputation import NodeReputation from retina_analytics.trust import AdsReportEntry, TrustScoreState from retina_custody.models import NodeIdentity @@ -266,6 +275,13 @@ def restore_snapshot() -> bool: for nid, rep_data in snap.get("reputations", {}).items(): state.node_analytics.reputations[nid] = NodeReputation(**rep_data) + # Node metrics and the server clock they are measured against. These + # replace what startup priming registered, which has counted nothing yet. + for nid, metrics_data in snap.get("node_metrics", {}).items(): + state.node_analytics.metrics[nid] = NodeMetrics.from_state(metrics_data) + if "server_minutes" in snap: + state.node_analytics.server_minutes = MinuteRing.from_state(snap["server_minutes"]) + # Accuracy samples samples_list = snap.get("accuracy_samples", []) state.accuracy_samples = deque(samples_list, maxlen=state.ACCURACY_MAX_SAMPLES) diff --git a/backend/services/tasks/__init__.py b/backend/services/tasks/__init__.py index 041bd4c79..be8601949 100644 --- a/backend/services/tasks/__init__.py +++ b/backend/services/tasks/__init__.py @@ -22,6 +22,7 @@ prune_synthetic_nodes, reputation_evaluator, retire_disposable_nodes_task, + server_clock_task, ) from services.tasks.solver import start_solver_workers, stop_solver_workers from services.tasks.storage_refresh import storage_refresh_task @@ -42,6 +43,7 @@ "archive_flush_task", "archive_lifecycle_task", "reputation_evaluator", + "server_clock_task", "prune_synthetic_nodes", "retire_disposable_nodes_task", "adsb_truth_fetcher", diff --git a/backend/services/tasks/periodic.py b/backend/services/tasks/periodic.py index b7a80dba6..80d87096b 100644 --- a/backend/services/tasks/periodic.py +++ b/backend/services/tasks/periodic.py @@ -1,4 +1,4 @@ -"""Periodic background tasks: archive flush, archive lifecycle, reputation, ADS-B truth fetch.""" +"""Periodic background tasks: archive flush, archive lifecycle, reputation, server clock, ADS-B truth fetch.""" import asyncio import logging @@ -18,6 +18,8 @@ FT_TO_M, KNOTS_TO_MS, REPUTATION_INTERVAL_S, + SERVER_CLOCK_GRACE_S, + SERVER_CLOCK_TICK_S, XVAL_MAX_AGE_S, as_num, is_num, @@ -103,6 +105,14 @@ async def reputation_evaluator(): logging.exception("Reputation evaluation failed") +async def server_clock_task(): + """Mark each minute this process is up, the denominator of every node's availability.""" + await asyncio.sleep(SERVER_CLOCK_GRACE_S) + while True: + state.node_analytics.mark_server_up() + await asyncio.sleep(SERVER_CLOCK_TICK_S) + + async def prune_synthetic_nodes(): """Periodically remove old synthetic/test nodes that have been disconnected.""" # Test nodes (e2e-, synth-, test- prefixes) disconnected >7 days old get pruned diff --git a/backend/tests/test_admin_routes.py b/backend/tests/test_admin_routes.py index 0198e564c..2d901b092 100644 --- a/backend/tests/test_admin_routes.py +++ b/backend/tests/test_admin_routes.py @@ -243,6 +243,7 @@ def test_leaderboard_answers_a_caller_with_no_session(self, client): #: What the route publishes to anyone. Widening this set publishes a field, #: so it is meant to take an edit here as well as one to the model. PUBLIC_FIELDS = { + "availability_7d", "avg_snr", "detections", "frames", @@ -253,7 +254,6 @@ def test_leaderboard_answers_a_caller_with_no_session(self, client): "reputation", "tracks", "trust_score", - "uptime_s", } def test_the_published_row_declares_exactly_the_public_fields(self): @@ -496,7 +496,7 @@ def test_leaderboard_with_node(self, client): "total_detections": 42, "total_frames": 10, "total_tracks": 5, - "uptime_s": 300, + "availability_7d": 0.9731, "avg_snr": 12.0, }, "trust": {}, @@ -514,6 +514,7 @@ def test_leaderboard_with_node(self, client): found = [e for e in entries if e["node_ref"] == "test-lb-1"] assert len(found) == 1 assert found[0]["detections"] == 42 + assert found[0]["availability_7d"] == 0.9731 assert found[0]["rank"] >= 1 finally: state.latest_analytics_bytes = orig diff --git a/backend/tests/test_publication.py b/backend/tests/test_publication.py index fb32d5613..d9330f3b5 100644 --- a/backend/tests/test_publication.py +++ b/backend/tests/test_publication.py @@ -1108,11 +1108,8 @@ def test_the_owners_copy_is_the_same_fuzzed_frame_the_public_would_get(self, cli ) ) # The public handle rides on top of the fuzzed frame; everything under - # it must be exactly what the public would get. uptime_s is the one - # field that legitimately differs between two summaries taken a few - # milliseconds apart (rounded to 0.1 s), so it is compared separately. + # it must be exactly what the public would get. got = {k: v for k, v in body["nodes"][_seed_ref(_PRIV)].items() if k != "node_ref"} - assert abs(got["metrics"].pop("uptime_s") - expected["metrics"].pop("uptime_s")) < 5.0 assert got == expected rx = body["nodes"][_seed_ref(_PRIV)]["detection_area"]["rx"] assert rx["lat"] != self.RX["rx_lat"] diff --git a/backend/tests/test_server_clock.py b/backend/tests/test_server_clock.py new file mode 100644 index 000000000..42a27aba4 --- /dev/null +++ b/backend/tests/test_server_clock.py @@ -0,0 +1,28 @@ +"""The server clock behind node availability: what it marks, and from when.""" + +import asyncio +import time + +from retina_analytics.availability import MinuteRing, minute_of + +from core import state +from services.tasks import periodic + + +async def _marked_after_running_briefly(monkeypatch, grace_s): + monkeypatch.setattr(periodic, "SERVER_CLOCK_GRACE_S", grace_s) + monkeypatch.setattr(state.node_analytics, "server_minutes", MinuteRing()) + task = asyncio.create_task(periodic.server_clock_task()) + await asyncio.sleep(0.05) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + now = minute_of(time.time()) + return state.node_analytics.server_minutes.bits(now - 1, now) + + +async def test_the_server_clock_marks_the_minute_it_is_up_in(monkeypatch): + assert await _marked_after_running_briefly(monkeypatch, grace_s=0) + + +async def test_the_server_clock_marks_nothing_while_nodes_reconnect(monkeypatch): + assert not await _marked_after_running_briefly(monkeypatch, grace_s=60) diff --git a/backend/tests/test_state_snapshot.py b/backend/tests/test_state_snapshot.py index 7377d2f85..202938450 100644 --- a/backend/tests/test_state_snapshot.py +++ b/backend/tests/test_state_snapshot.py @@ -93,6 +93,38 @@ def test_reputations_survive_round_trip(self, tmp_path): state.node_analytics.reputations.pop("node-7", None) + def test_node_metrics_and_the_server_clock_survive_round_trip(self, tmp_path): + from retina_analytics.availability import MinuteRing + from retina_analytics.metrics import NodeMetrics + + from core import state + + now = time.time() + metrics = NodeMetrics(node_id="node-9", first_seen=now - 7200) + up = MinuteRing() + for ago in range(1, 61): + up.mark(int((now - ago * 60) // 60)) + if ago <= 45: + metrics.record_frame({"delay": [1.0, 2.0], "snr": [9.0, 11.0]}, now=now - ago * 60) + expected = metrics.summary(up, now=now) + assert (expected["availability_7d"], expected["total_detections"]) == (0.75, 90) + + saved_up = state.node_analytics.server_minutes + state.node_analytics.metrics["node-9"] = metrics + state.node_analytics.server_minutes = up + snap_path = str(tmp_path / "snap.json") + try: + with patch("services.state_snapshot._SNAPSHOT_PATH", snap_path): + save_snapshot() + state.node_analytics.metrics.pop("node-9") + state.node_analytics.server_minutes = MinuteRing() + restore_snapshot() + restored = state.node_analytics.metrics["node-9"] + assert restored.summary(state.node_analytics.server_minutes, now=now) == expected + finally: + state.node_analytics.metrics.pop("node-9", None) + state.node_analytics.server_minutes = saved_up + def test_accuracy_samples_survive_round_trip(self, tmp_path): from core import state diff --git a/backend/vulture_whitelist.py b/backend/vulture_whitelist.py index 42408fc13..9745ccf34 100644 --- a/backend/vulture_whitelist.py +++ b/backend/vulture_whitelist.py @@ -130,6 +130,7 @@ # Leaderboard response fields, set by keyword and read only by Pydantic's # serialisation. Each is a key the route publishes; see PublicLeaderboardRow. +availability_7d avg_snr trust_score online diff --git a/e2e/specs/nodes.spec.ts b/e2e/specs/nodes.spec.ts index 4dab69c3d..c7918d1d9 100644 --- a/e2e/specs/nodes.spec.ts +++ b/e2e/specs/nodes.spec.ts @@ -561,7 +561,8 @@ describeUnlessProd("Node registration — main integration suite", () => { const m = analyticsBody.metrics as Record; expect(m).toBeDefined(); for (const k of [ - "uptime_s", "total_frames", "total_detections", + "availability_7d", "availability_24h", "availability_measured_s", + "total_frames", "total_detections", "avg_detections_per_frame", "avg_snr", "max_snr", "total_tracks", "geolocated_tracks", "track_quality", ]) { @@ -572,10 +573,12 @@ describeUnlessProd("Node registration — main integration suite", () => { expect(m).not.toHaveProperty("node_id"); }); - test("metrics.uptime_s is a non-negative number", () => { + test("metrics.availability_7d is a share, or null until a whole minute is measured", () => { const m = analyticsBody.metrics as Record; - expect(typeof m.uptime_s).toBe("number"); - expect(m.uptime_s as number).toBeGreaterThanOrEqual(0); + const share = m.availability_7d; + const valid = share === null || (typeof share === "number" && share >= 0 && share <= 1); + expect(valid, `availability_7d: ${share}`).toBe(true); + expect(m.availability_measured_s as number).toBeGreaterThanOrEqual(0); }); test("metrics.total_frames is a non-negative integer", () => { diff --git a/libs/retina-analytics b/libs/retina-analytics index 8b79a1a04..8847a01dd 160000 --- a/libs/retina-analytics +++ b/libs/retina-analytics @@ -1 +1 @@ -Subproject commit 8b79a1a043641e2ce491b3e8bfaadcf3800b8410 +Subproject commit 8847a01ddf486eea9b7b2be941151d3d9fa6b158 From 85d9ad1b561b13006e2d0ed9799bf4f946f7f8d1 Mon Sep 17 00:00:00 2001 From: Babissimo Date: Thu, 24 Sep 2026 11:56:49 +0100 Subject: [PATCH 2/2] Show node availability where the console showed time since registration The five pages that showed a node's uptime (leaderboard, overview, node detail, network health, node management) show its 7-day availability instead. The node's own page adds the 24-hour figure and how much of the week the figure rests on, which for a new node, or in the first week after this ships, is less than the whole of it. The leaderboard sorts on it, with a node not yet measured last. The percentage rounds down, so a node that missed one minute in a week reads 99.9% rather than a perfect 100.0%. The knowledge base said reputation was based on uptime. It is not: it is judged on the heartbeat, agreement with ADS-B and trusted neighbours, and a plausible detection rate, which is what it now says. Availability gets its own entry. Ticket: https://app.clickup.com/t/123zgec4k7v Co-Authored-By: Claude Opus 5.5 --- .../src/pages/admin/NetworkHealthPage.tsx | 6 +-- .../src/pages/admin/NodeManagementPage.tsx | 6 +-- .../src/pages/user/KnowledgeBasePage.tsx | 3 +- dashboard/src/pages/user/LeaderboardPage.tsx | 12 +++-- dashboard/src/pages/user/NodeDetailPage.tsx | 6 ++- dashboard/src/pages/user/OverviewPage.tsx | 6 +-- dashboard/src/test/LeaderboardPage.test.tsx | 54 +++++++++++++++++-- dashboard/src/test/NodeContact.test.tsx | 2 +- dashboard/src/test/NodeDetailPage.test.tsx | 23 +++++++- dashboard/src/test/format.test.ts | 29 +++++----- dashboard/src/utils/format.ts | 11 ++-- 11 files changed, 111 insertions(+), 47 deletions(-) diff --git a/dashboard/src/pages/admin/NetworkHealthPage.tsx b/dashboard/src/pages/admin/NetworkHealthPage.tsx index be45f3f0e..2f90aa14a 100644 --- a/dashboard/src/pages/admin/NetworkHealthPage.tsx +++ b/dashboard/src/pages/admin/NetworkHealthPage.tsx @@ -10,7 +10,7 @@ import { DataTable } from "../../components/DataTable"; import { Pager, clampPage } from "../../components/Pager"; import { StatCard } from "../../components/StatCard"; import { usePolling } from "../../hooks/usePolling"; -import { formatMHz, formatRelativeTime, formatUptime } from "../../utils/format"; +import { formatAvailability, formatMHz, formatRelativeTime } from "../../utils/format"; import { useChartTheme } from "../../utils/chartTheme"; import { RetnodeLink } from "../../components/RetnodeLink"; import { StatusBadge } from "../../components/StatusBadge"; @@ -209,7 +209,7 @@ export default function NetworkHealthPage() { return ( <> @@ -238,7 +238,7 @@ export default function NetworkHealthPage() { {(node._analytics?.metrics?.avg_snr || 0).toFixed(1)} dB {((node._analytics?.trust?.trust_score || 0) * 100).toFixed(0)}% {((node._analytics?.reputation?.reputation || 0) * 100).toFixed(0)}% - {formatUptime(node._analytics?.metrics?.uptime_s || 0)} + {formatAvailability(node._analytics?.metrics?.availability_7d)} ); })} diff --git a/dashboard/src/pages/admin/NodeManagementPage.tsx b/dashboard/src/pages/admin/NodeManagementPage.tsx index d82ea0e01..97c03f6a5 100644 --- a/dashboard/src/pages/admin/NodeManagementPage.tsx +++ b/dashboard/src/pages/admin/NodeManagementPage.tsx @@ -5,7 +5,7 @@ import { FetchNotice, nothingLoaded } from "../../components/Notice"; import { Pager } from "../../components/Pager"; import { StatCard } from "../../components/StatCard"; import { useFetch } from "../../hooks/usePolling"; -import { formatMHz, formatUptime } from "../../utils/format"; +import { formatAvailability, formatMHz } from "../../utils/format"; import { PositionStatusBadge } from "../../components/PositionStatusBadge"; import { PolledRadars } from "./PolledRadars"; import { @@ -166,8 +166,8 @@ export default function NodeManagementPage() { {((summary.reputation?.reputation || 0) * 100).toFixed(0)}% Avg SNR {(summary.metrics?.avg_snr || 0).toFixed(1)} dB - Uptime - {formatUptime(summary.metrics?.uptime_s || 0)} + Availability + {formatAvailability(summary.metrics?.availability_7d)} Owner email {owner?.email || "—"} Site contact diff --git a/dashboard/src/pages/user/KnowledgeBasePage.tsx b/dashboard/src/pages/user/KnowledgeBasePage.tsx index eb6fbf22a..2c7dbe80a 100644 --- a/dashboard/src/pages/user/KnowledgeBasePage.tsx +++ b/dashboard/src/pages/user/KnowledgeBasePage.tsx @@ -38,7 +38,8 @@ export default function KnowledgeBasePage() { description: "What the dashboard numbers mean.", items: [ { q: "Trust Score", a: "Measures how well your node's detections match known ADS-B aircraft positions. Higher is better (0–100%)." }, - { q: "Reputation", a: "Long-term reliability score based on uptime, data quality, and consistency. Drops with outages or bad data." }, + { q: "Availability", a: "The share of the last 7 days' minutes in which your node delivered data, counting only minutes the server was up to receive it. The node's own page also gives the last 24 hours." }, + { q: "Reputation", a: "Long-term reliability score, judged on the node's heartbeat, how well its detections agree with ADS-B and with trusted neighbours, and whether its detection rate is plausible." }, { q: "SNR (Signal-to-Noise Ratio)", a: "The strength of the radar signal relative to background noise. 10+ dB is good, 20+ dB is excellent." }, { q: "Coverage Overlap", a: "How much your detection area intersects with nearby nodes. Higher overlap enables better triangulation." }, ], diff --git a/dashboard/src/pages/user/LeaderboardPage.tsx b/dashboard/src/pages/user/LeaderboardPage.tsx index d64e6c0d3..3e31285e0 100644 --- a/dashboard/src/pages/user/LeaderboardPage.tsx +++ b/dashboard/src/pages/user/LeaderboardPage.tsx @@ -7,7 +7,7 @@ import { StatCard } from "../../components/StatCard"; import { StatusBadge } from "../../components/StatusBadge"; import { usePolling } from "../../hooks/usePolling"; import { useAuth } from "../../context/AuthContext"; -import { formatUptime } from "../../utils/format"; +import { formatAvailability } from "../../utils/format"; import { shortRef } from "../../utils/nodes"; const PAGE_SIZE = 25; @@ -38,7 +38,7 @@ export default function LeaderboardPage() { // Re-sort locally based on selection const sorted = [...entries].sort((a, b) => { if (sortBy === "detections") return b.detections - a.detections; - if (sortBy === "uptime") return b.uptime_s - a.uptime_s; + if (sortBy === "availability") return (b.availability_7d ?? -1) - (a.availability_7d ?? -1); if (sortBy === "trust") return b.trust_score - a.trust_score; if (sortBy === "snr") return b.avg_snr - a.avg_snr; if (sortBy === "miss_rate") return (a.miss_rate || 0) - (b.miss_rate || 0); @@ -111,7 +111,7 @@ export default function LeaderboardPage() { {/* Sort control */}
Sort by: - {["detections", "uptime", "trust", "snr", ...(showsMisses ? ["miss_rate"] : [])].map((key) => ( + {["detections", "availability", "trust", "snr", ...(showsMisses ? ["miss_rate"] : [])].map((key) => (