diff --git a/backend/config/constants.py b/backend/config/constants.py index ba8353cf1..203db259b 100644 --- a/backend/config/constants.py +++ b/backend/config/constants.py @@ -476,6 +476,12 @@ def _assoc_alt_layers_km() -> tuple[float, ...]: # ── Periodic task intervals ────────────────────────────────────────────────── REPUTATION_INTERVAL_S = 60 # Reputation evaluator sleep (s) +# The server clock behind node availability: the minutes this process could +# take frames. Ticked well inside a minute so none is missed. It starts only +# after a grace, as nodes take a while to reconnect after a restart and those +# minutes are the server's, not theirs. +SERVER_CLOCK_TICK_S = 20 +SERVER_CLOCK_GRACE_S = 300 ADSB_TRUTH_INTERVAL_S = 120 # ADS-B truth fetcher sleep (s) ADSB_BACKOFF_S = 300 # Rate-limit backoff (s) # adsb-service poll for claim candidates (services/tasks/adsb_fallback.py): every diff --git a/backend/main.py b/backend/main.py index 9040e5001..da755767e 100644 --- a/backend/main.py +++ b/backend/main.py @@ -79,6 +79,7 @@ prune_synthetic_nodes, reputation_evaluator, retire_disposable_nodes_task, + server_clock_task, start_solver_workers, stop_solver_workers, storage_refresh_task, @@ -251,6 +252,7 @@ async def _snapshot_loop(): for task_fn in ( server.serve_forever, reputation_evaluator, + server_clock_task, prune_synthetic_nodes, retire_disposable_nodes_task, adsb_truth_fetcher, diff --git a/backend/routes/admin.py b/backend/routes/admin.py index 57c1f0748..d44204ccf 100644 --- a/backend/routes/admin.py +++ b/backend/routes/admin.py @@ -607,7 +607,9 @@ class PublicLeaderboardRow(BaseModel): detections: int frames: int tracks: int - uptime_s: float + # Share of the last week's minutes the node delivered in; None until a + # whole minute has been measured. + availability_7d: float | None avg_snr: float trust_score: float reputation: float @@ -687,7 +689,7 @@ async def _cold_start_summaries() -> dict: @router.get("/leaderboard") async def leaderboard(caller=Depends(get_optional_user)): - """Rankings by detections, uptime and trust, open to anyone. + """Rankings by detections, availability and trust, open to anyone. The odd one out under this prefix, which is otherwise the admin API. A caller with no session gets PublicLeaderboardRow, which is what lets the @@ -742,7 +744,7 @@ async def leaderboard(caller=Depends(get_optional_user)): detections=m.get("total_detections", 0), frames=m.get("total_frames", 0), tracks=m.get("total_tracks", 0), - uptime_s=m.get("uptime_s", 0), + availability_7d=m.get("availability_7d"), avg_snr=m.get("avg_snr", 0), trust_score=s.get("trust", {}).get("trust_score", 0), reputation=s.get("reputation", {}).get("reputation", 0), diff --git a/backend/services/state_snapshot.py b/backend/services/state_snapshot.py index cb7e7d9cb..fbdfc00b6 100644 --- a/backend/services/state_snapshot.py +++ b/backend/services/state_snapshot.py @@ -1,8 +1,9 @@ """Lightweight state snapshot: save/restore high-value in-memory state across restarts. Saved every SAVE_INTERVAL_S (60 s) by a background task. Restored once at startup. -Persists: trust_scores, reputations, accuracy_samples, chain_entries, -node_identities, iq_commitments, anomaly_log, simulation_config. +Persists: trust_scores, reputations, node_metrics, server_minutes, +accuracy_samples, chain_entries, node_identities, iq_commitments, anomaly_log, +simulation_config. """ import hashlib @@ -38,6 +39,10 @@ def save_snapshot() -> None: for nid, rep in state.node_analytics.reputations.items(): reps[nid] = asdict(rep) + # Node availability and the leaderboard's counts. The server's own + # minutes go with them, as they are what availability is measured against. + node_metrics = {nid: m.to_state() for nid, m in list(state.node_analytics.metrics.items())} + identities = {} for nid, ident in state.node_identities.items(): identities[nid] = ident.to_dict() @@ -46,6 +51,8 @@ def save_snapshot() -> None: "saved_at": time.time(), "trust_scores": trust, "reputations": reps, + "node_metrics": node_metrics, + "server_minutes": state.node_analytics.server_minutes.to_state(), "accuracy_samples": list(state.accuracy_samples), "chain_entries": dict(state.chain_entries), "node_identities": identities, @@ -170,6 +177,8 @@ def _restore_simulation_config(snap: dict) -> None: def restore_snapshot() -> bool: """Load state from disk snapshot. Returns True if restored, False if no snapshot found.""" + from retina_analytics.availability import MinuteRing + from retina_analytics.metrics import NodeMetrics from retina_analytics.reputation import NodeReputation from retina_analytics.trust import AdsReportEntry, TrustScoreState from retina_custody.models import NodeIdentity @@ -266,6 +275,13 @@ def restore_snapshot() -> bool: for nid, rep_data in snap.get("reputations", {}).items(): state.node_analytics.reputations[nid] = NodeReputation(**rep_data) + # Node metrics and the server clock they are measured against. These + # replace what startup priming registered, which has counted nothing yet. + for nid, metrics_data in snap.get("node_metrics", {}).items(): + state.node_analytics.metrics[nid] = NodeMetrics.from_state(metrics_data) + if "server_minutes" in snap: + state.node_analytics.server_minutes = MinuteRing.from_state(snap["server_minutes"]) + # Accuracy samples samples_list = snap.get("accuracy_samples", []) state.accuracy_samples = deque(samples_list, maxlen=state.ACCURACY_MAX_SAMPLES) diff --git a/backend/services/tasks/__init__.py b/backend/services/tasks/__init__.py index 041bd4c79..be8601949 100644 --- a/backend/services/tasks/__init__.py +++ b/backend/services/tasks/__init__.py @@ -22,6 +22,7 @@ prune_synthetic_nodes, reputation_evaluator, retire_disposable_nodes_task, + server_clock_task, ) from services.tasks.solver import start_solver_workers, stop_solver_workers from services.tasks.storage_refresh import storage_refresh_task @@ -42,6 +43,7 @@ "archive_flush_task", "archive_lifecycle_task", "reputation_evaluator", + "server_clock_task", "prune_synthetic_nodes", "retire_disposable_nodes_task", "adsb_truth_fetcher", diff --git a/backend/services/tasks/periodic.py b/backend/services/tasks/periodic.py index b7a80dba6..80d87096b 100644 --- a/backend/services/tasks/periodic.py +++ b/backend/services/tasks/periodic.py @@ -1,4 +1,4 @@ -"""Periodic background tasks: archive flush, archive lifecycle, reputation, ADS-B truth fetch.""" +"""Periodic background tasks: archive flush, archive lifecycle, reputation, server clock, ADS-B truth fetch.""" import asyncio import logging @@ -18,6 +18,8 @@ FT_TO_M, KNOTS_TO_MS, REPUTATION_INTERVAL_S, + SERVER_CLOCK_GRACE_S, + SERVER_CLOCK_TICK_S, XVAL_MAX_AGE_S, as_num, is_num, @@ -103,6 +105,14 @@ async def reputation_evaluator(): logging.exception("Reputation evaluation failed") +async def server_clock_task(): + """Mark each minute this process is up, the denominator of every node's availability.""" + await asyncio.sleep(SERVER_CLOCK_GRACE_S) + while True: + state.node_analytics.mark_server_up() + await asyncio.sleep(SERVER_CLOCK_TICK_S) + + async def prune_synthetic_nodes(): """Periodically remove old synthetic/test nodes that have been disconnected.""" # Test nodes (e2e-, synth-, test- prefixes) disconnected >7 days old get pruned diff --git a/backend/tests/test_admin_routes.py b/backend/tests/test_admin_routes.py index 0198e564c..2d901b092 100644 --- a/backend/tests/test_admin_routes.py +++ b/backend/tests/test_admin_routes.py @@ -243,6 +243,7 @@ def test_leaderboard_answers_a_caller_with_no_session(self, client): #: What the route publishes to anyone. Widening this set publishes a field, #: so it is meant to take an edit here as well as one to the model. PUBLIC_FIELDS = { + "availability_7d", "avg_snr", "detections", "frames", @@ -253,7 +254,6 @@ def test_leaderboard_answers_a_caller_with_no_session(self, client): "reputation", "tracks", "trust_score", - "uptime_s", } def test_the_published_row_declares_exactly_the_public_fields(self): @@ -496,7 +496,7 @@ def test_leaderboard_with_node(self, client): "total_detections": 42, "total_frames": 10, "total_tracks": 5, - "uptime_s": 300, + "availability_7d": 0.9731, "avg_snr": 12.0, }, "trust": {}, @@ -514,6 +514,7 @@ def test_leaderboard_with_node(self, client): found = [e for e in entries if e["node_ref"] == "test-lb-1"] assert len(found) == 1 assert found[0]["detections"] == 42 + assert found[0]["availability_7d"] == 0.9731 assert found[0]["rank"] >= 1 finally: state.latest_analytics_bytes = orig diff --git a/backend/tests/test_publication.py b/backend/tests/test_publication.py index fb32d5613..d9330f3b5 100644 --- a/backend/tests/test_publication.py +++ b/backend/tests/test_publication.py @@ -1108,11 +1108,8 @@ def test_the_owners_copy_is_the_same_fuzzed_frame_the_public_would_get(self, cli ) ) # The public handle rides on top of the fuzzed frame; everything under - # it must be exactly what the public would get. uptime_s is the one - # field that legitimately differs between two summaries taken a few - # milliseconds apart (rounded to 0.1 s), so it is compared separately. + # it must be exactly what the public would get. got = {k: v for k, v in body["nodes"][_seed_ref(_PRIV)].items() if k != "node_ref"} - assert abs(got["metrics"].pop("uptime_s") - expected["metrics"].pop("uptime_s")) < 5.0 assert got == expected rx = body["nodes"][_seed_ref(_PRIV)]["detection_area"]["rx"] assert rx["lat"] != self.RX["rx_lat"] diff --git a/backend/tests/test_server_clock.py b/backend/tests/test_server_clock.py new file mode 100644 index 000000000..42a27aba4 --- /dev/null +++ b/backend/tests/test_server_clock.py @@ -0,0 +1,28 @@ +"""The server clock behind node availability: what it marks, and from when.""" + +import asyncio +import time + +from retina_analytics.availability import MinuteRing, minute_of + +from core import state +from services.tasks import periodic + + +async def _marked_after_running_briefly(monkeypatch, grace_s): + monkeypatch.setattr(periodic, "SERVER_CLOCK_GRACE_S", grace_s) + monkeypatch.setattr(state.node_analytics, "server_minutes", MinuteRing()) + task = asyncio.create_task(periodic.server_clock_task()) + await asyncio.sleep(0.05) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + now = minute_of(time.time()) + return state.node_analytics.server_minutes.bits(now - 1, now) + + +async def test_the_server_clock_marks_the_minute_it_is_up_in(monkeypatch): + assert await _marked_after_running_briefly(monkeypatch, grace_s=0) + + +async def test_the_server_clock_marks_nothing_while_nodes_reconnect(monkeypatch): + assert not await _marked_after_running_briefly(monkeypatch, grace_s=60) diff --git a/backend/tests/test_state_snapshot.py b/backend/tests/test_state_snapshot.py index 7377d2f85..202938450 100644 --- a/backend/tests/test_state_snapshot.py +++ b/backend/tests/test_state_snapshot.py @@ -93,6 +93,38 @@ def test_reputations_survive_round_trip(self, tmp_path): state.node_analytics.reputations.pop("node-7", None) + def test_node_metrics_and_the_server_clock_survive_round_trip(self, tmp_path): + from retina_analytics.availability import MinuteRing + from retina_analytics.metrics import NodeMetrics + + from core import state + + now = time.time() + metrics = NodeMetrics(node_id="node-9", first_seen=now - 7200) + up = MinuteRing() + for ago in range(1, 61): + up.mark(int((now - ago * 60) // 60)) + if ago <= 45: + metrics.record_frame({"delay": [1.0, 2.0], "snr": [9.0, 11.0]}, now=now - ago * 60) + expected = metrics.summary(up, now=now) + assert (expected["availability_7d"], expected["total_detections"]) == (0.75, 90) + + saved_up = state.node_analytics.server_minutes + state.node_analytics.metrics["node-9"] = metrics + state.node_analytics.server_minutes = up + snap_path = str(tmp_path / "snap.json") + try: + with patch("services.state_snapshot._SNAPSHOT_PATH", snap_path): + save_snapshot() + state.node_analytics.metrics.pop("node-9") + state.node_analytics.server_minutes = MinuteRing() + restore_snapshot() + restored = state.node_analytics.metrics["node-9"] + assert restored.summary(state.node_analytics.server_minutes, now=now) == expected + finally: + state.node_analytics.metrics.pop("node-9", None) + state.node_analytics.server_minutes = saved_up + def test_accuracy_samples_survive_round_trip(self, tmp_path): from core import state diff --git a/backend/vulture_whitelist.py b/backend/vulture_whitelist.py index 42408fc13..9745ccf34 100644 --- a/backend/vulture_whitelist.py +++ b/backend/vulture_whitelist.py @@ -130,6 +130,7 @@ # Leaderboard response fields, set by keyword and read only by Pydantic's # serialisation. Each is a key the route publishes; see PublicLeaderboardRow. +availability_7d avg_snr trust_score online diff --git a/dashboard/src/pages/admin/NetworkHealthPage.tsx b/dashboard/src/pages/admin/NetworkHealthPage.tsx index be45f3f0e..2f90aa14a 100644 --- a/dashboard/src/pages/admin/NetworkHealthPage.tsx +++ b/dashboard/src/pages/admin/NetworkHealthPage.tsx @@ -10,7 +10,7 @@ import { DataTable } from "../../components/DataTable"; import { Pager, clampPage } from "../../components/Pager"; import { StatCard } from "../../components/StatCard"; import { usePolling } from "../../hooks/usePolling"; -import { formatMHz, formatRelativeTime, formatUptime } from "../../utils/format"; +import { formatAvailability, formatMHz, formatRelativeTime } from "../../utils/format"; import { useChartTheme } from "../../utils/chartTheme"; import { RetnodeLink } from "../../components/RetnodeLink"; import { StatusBadge } from "../../components/StatusBadge"; @@ -209,7 +209,7 @@ export default function NetworkHealthPage() { return ( <> @@ -238,7 +238,7 @@ export default function NetworkHealthPage() { {(node._analytics?.metrics?.avg_snr || 0).toFixed(1)} dB {((node._analytics?.trust?.trust_score || 0) * 100).toFixed(0)}% {((node._analytics?.reputation?.reputation || 0) * 100).toFixed(0)}% - {formatUptime(node._analytics?.metrics?.uptime_s || 0)} + {formatAvailability(node._analytics?.metrics?.availability_7d)} ); })} diff --git a/dashboard/src/pages/admin/NodeManagementPage.tsx b/dashboard/src/pages/admin/NodeManagementPage.tsx index d82ea0e01..97c03f6a5 100644 --- a/dashboard/src/pages/admin/NodeManagementPage.tsx +++ b/dashboard/src/pages/admin/NodeManagementPage.tsx @@ -5,7 +5,7 @@ import { FetchNotice, nothingLoaded } from "../../components/Notice"; import { Pager } from "../../components/Pager"; import { StatCard } from "../../components/StatCard"; import { useFetch } from "../../hooks/usePolling"; -import { formatMHz, formatUptime } from "../../utils/format"; +import { formatAvailability, formatMHz } from "../../utils/format"; import { PositionStatusBadge } from "../../components/PositionStatusBadge"; import { PolledRadars } from "./PolledRadars"; import { @@ -166,8 +166,8 @@ export default function NodeManagementPage() { {((summary.reputation?.reputation || 0) * 100).toFixed(0)}% Avg SNR {(summary.metrics?.avg_snr || 0).toFixed(1)} dB - Uptime - {formatUptime(summary.metrics?.uptime_s || 0)} + Availability + {formatAvailability(summary.metrics?.availability_7d)} Owner email {owner?.email || "—"} Site contact diff --git a/dashboard/src/pages/user/KnowledgeBasePage.tsx b/dashboard/src/pages/user/KnowledgeBasePage.tsx index eb6fbf22a..2c7dbe80a 100644 --- a/dashboard/src/pages/user/KnowledgeBasePage.tsx +++ b/dashboard/src/pages/user/KnowledgeBasePage.tsx @@ -38,7 +38,8 @@ export default function KnowledgeBasePage() { description: "What the dashboard numbers mean.", items: [ { q: "Trust Score", a: "Measures how well your node's detections match known ADS-B aircraft positions. Higher is better (0–100%)." }, - { q: "Reputation", a: "Long-term reliability score based on uptime, data quality, and consistency. Drops with outages or bad data." }, + { q: "Availability", a: "The share of the last 7 days' minutes in which your node delivered data, counting only minutes the server was up to receive it. The node's own page also gives the last 24 hours." }, + { q: "Reputation", a: "Long-term reliability score, judged on the node's heartbeat, how well its detections agree with ADS-B and with trusted neighbours, and whether its detection rate is plausible." }, { q: "SNR (Signal-to-Noise Ratio)", a: "The strength of the radar signal relative to background noise. 10+ dB is good, 20+ dB is excellent." }, { q: "Coverage Overlap", a: "How much your detection area intersects with nearby nodes. Higher overlap enables better triangulation." }, ], diff --git a/dashboard/src/pages/user/LeaderboardPage.tsx b/dashboard/src/pages/user/LeaderboardPage.tsx index d64e6c0d3..3e31285e0 100644 --- a/dashboard/src/pages/user/LeaderboardPage.tsx +++ b/dashboard/src/pages/user/LeaderboardPage.tsx @@ -7,7 +7,7 @@ import { StatCard } from "../../components/StatCard"; import { StatusBadge } from "../../components/StatusBadge"; import { usePolling } from "../../hooks/usePolling"; import { useAuth } from "../../context/AuthContext"; -import { formatUptime } from "../../utils/format"; +import { formatAvailability } from "../../utils/format"; import { shortRef } from "../../utils/nodes"; const PAGE_SIZE = 25; @@ -38,7 +38,7 @@ export default function LeaderboardPage() { // Re-sort locally based on selection const sorted = [...entries].sort((a, b) => { if (sortBy === "detections") return b.detections - a.detections; - if (sortBy === "uptime") return b.uptime_s - a.uptime_s; + if (sortBy === "availability") return (b.availability_7d ?? -1) - (a.availability_7d ?? -1); if (sortBy === "trust") return b.trust_score - a.trust_score; if (sortBy === "snr") return b.avg_snr - a.avg_snr; if (sortBy === "miss_rate") return (a.miss_rate || 0) - (b.miss_rate || 0); @@ -111,7 +111,7 @@ export default function LeaderboardPage() { {/* Sort control */}
Sort by: - {["detections", "uptime", "trust", "snr", ...(showsMisses ? ["miss_rate"] : [])].map((key) => ( + {["detections", "availability", "trust", "snr", ...(showsMisses ? ["miss_rate"] : [])].map((key) => (