Rank nodes by 7-day availability, and keep their counts across restarts

backend/config/constants.py

@@ -476,6 +476,12 @@ def _assoc_alt_layers_km() -> tuple[float, ...]:
 
 # ── Periodic task intervals ──────────────────────────────────────────────────
 REPUTATION_INTERVAL_S = 60  # Reputation evaluator sleep (s)
+# The server clock behind node availability: the minutes this process could
+# take frames.  Ticked well inside a minute so none is missed.  It starts only
+# after a grace, as nodes take a while to reconnect after a restart and those
+# minutes are the server's, not theirs.
+SERVER_CLOCK_TICK_S = 20
+SERVER_CLOCK_GRACE_S = 300
 ADSB_TRUTH_INTERVAL_S = 120  # ADS-B truth fetcher sleep (s)
 ADSB_BACKOFF_S = 300  # Rate-limit backoff (s)
 # adsb-service poll for claim candidates (services/tasks/adsb_fallback.py): every

backend/main.py

@@ -79,6 +79,7 @@
     prune_synthetic_nodes,
     reputation_evaluator,
     retire_disposable_nodes_task,
+    server_clock_task,
     start_solver_workers,
     stop_solver_workers,
     storage_refresh_task,
@@ -251,6 +252,7 @@ async def _snapshot_loop():
             for task_fn in (
                 server.serve_forever,
                 reputation_evaluator,
+                server_clock_task,
                 prune_synthetic_nodes,
                 retire_disposable_nodes_task,
                 adsb_truth_fetcher,

backend/routes/admin.py

@@ -607,7 +607,9 @@ class PublicLeaderboardRow(BaseModel):
     detections: int
     frames: int
     tracks: int
-    uptime_s: float
+    # Share of the last week's minutes the node delivered in; None until a
+    # whole minute has been measured.
+    availability_7d: float | None
     avg_snr: float
     trust_score: float
     reputation: float
@@ -687,7 +689,7 @@ async def _cold_start_summaries() -> dict:
 
 @router.get("/leaderboard")
 async def leaderboard(caller=Depends(get_optional_user)):
-    """Rankings by detections, uptime and trust, open to anyone.
+    """Rankings by detections, availability and trust, open to anyone.
 
     The odd one out under this prefix, which is otherwise the admin API. A
     caller with no session gets PublicLeaderboardRow, which is what lets the
@@ -742,7 +744,7 @@ async def leaderboard(caller=Depends(get_optional_user)):
             detections=m.get("total_detections", 0),
             frames=m.get("total_frames", 0),
             tracks=m.get("total_tracks", 0),
-            uptime_s=m.get("uptime_s", 0),
+            availability_7d=m.get("availability_7d"),
             avg_snr=m.get("avg_snr", 0),
             trust_score=s.get("trust", {}).get("trust_score", 0),
             reputation=s.get("reputation", {}).get("reputation", 0),

backend/services/state_snapshot.py

@@ -1,8 +1,9 @@
 """Lightweight state snapshot: save/restore high-value in-memory state across restarts.
 
 Saved every SAVE_INTERVAL_S (60 s) by a background task.  Restored once at startup.
-Persists: trust_scores, reputations, accuracy_samples, chain_entries,
-node_identities, iq_commitments, anomaly_log, simulation_config.
+Persists: trust_scores, reputations, node_metrics, server_minutes,
+accuracy_samples, chain_entries, node_identities, iq_commitments, anomaly_log,
+simulation_config.
 """
 
 import hashlib
@@ -38,6 +39,10 @@ def save_snapshot() -> None:
     for nid, rep in state.node_analytics.reputations.items():
         reps[nid] = asdict(rep)
 
+    # Node availability and the leaderboard's counts.  The server's own
+    # minutes go with them, as they are what availability is measured against.
+    node_metrics = {nid: m.to_state() for nid, m in list(state.node_analytics.metrics.items())}
+
     identities = {}
     for nid, ident in state.node_identities.items():
         identities[nid] = ident.to_dict()
@@ -46,6 +51,8 @@ def save_snapshot() -> None:
         "saved_at": time.time(),
         "trust_scores": trust,
         "reputations": reps,
+        "node_metrics": node_metrics,
+        "server_minutes": state.node_analytics.server_minutes.to_state(),
         "accuracy_samples": list(state.accuracy_samples),
         "chain_entries": dict(state.chain_entries),
         "node_identities": identities,
@@ -170,6 +177,8 @@ def _restore_simulation_config(snap: dict) -> None:
 
 def restore_snapshot() -> bool:
     """Load state from disk snapshot. Returns True if restored, False if no snapshot found."""
+    from retina_analytics.availability import MinuteRing
+    from retina_analytics.metrics import NodeMetrics
     from retina_analytics.reputation import NodeReputation
     from retina_analytics.trust import AdsReportEntry, TrustScoreState
     from retina_custody.models import NodeIdentity
@@ -266,6 +275,13 @@ def restore_snapshot() -> bool:
     for nid, rep_data in snap.get("reputations", {}).items():
         state.node_analytics.reputations[nid] = NodeReputation(**rep_data)
 
+    # Node metrics and the server clock they are measured against.  These
+    # replace what startup priming registered, which has counted nothing yet.
+    for nid, metrics_data in snap.get("node_metrics", {}).items():
+        state.node_analytics.metrics[nid] = NodeMetrics.from_state(metrics_data)
+    if "server_minutes" in snap:
+        state.node_analytics.server_minutes = MinuteRing.from_state(snap["server_minutes"])
+
     # Accuracy samples
     samples_list = snap.get("accuracy_samples", [])
     state.accuracy_samples = deque(samples_list, maxlen=state.ACCURACY_MAX_SAMPLES)

backend/services/tasks/__init__.py

@@ -22,6 +22,7 @@
     prune_synthetic_nodes,
     reputation_evaluator,
     retire_disposable_nodes_task,
+    server_clock_task,
 )
 from services.tasks.solver import start_solver_workers, stop_solver_workers
 from services.tasks.storage_refresh import storage_refresh_task
@@ -42,6 +43,7 @@
     "archive_flush_task",
     "archive_lifecycle_task",
     "reputation_evaluator",
+    "server_clock_task",
     "prune_synthetic_nodes",
     "retire_disposable_nodes_task",
     "adsb_truth_fetcher",

backend/services/tasks/periodic.py

@@ -1,4 +1,4 @@
-"""Periodic background tasks: archive flush, archive lifecycle, reputation, ADS-B truth fetch."""
+"""Periodic background tasks: archive flush, archive lifecycle, reputation, server clock, ADS-B truth fetch."""
 
 import asyncio
 import logging
@@ -18,6 +18,8 @@
     FT_TO_M,
     KNOTS_TO_MS,
     REPUTATION_INTERVAL_S,
+    SERVER_CLOCK_GRACE_S,
+    SERVER_CLOCK_TICK_S,
     XVAL_MAX_AGE_S,
     as_num,
     is_num,
@@ -103,6 +105,14 @@ async def reputation_evaluator():
                 logging.exception("Reputation evaluation failed")
 
 
+async def server_clock_task():
+    """Mark each minute this process is up, the denominator of every node's availability."""
+    await asyncio.sleep(SERVER_CLOCK_GRACE_S)
+    while True:
+        state.node_analytics.mark_server_up()
+        await asyncio.sleep(SERVER_CLOCK_TICK_S)
+
+
 async def prune_synthetic_nodes():
     """Periodically remove old synthetic/test nodes that have been disconnected."""
     # Test nodes (e2e-, synth-, test- prefixes) disconnected >7 days old get pruned

backend/tests/test_admin_routes.py

@@ -243,6 +243,7 @@ def test_leaderboard_answers_a_caller_with_no_session(self, client):
     #: What the route publishes to anyone. Widening this set publishes a field,
     #: so it is meant to take an edit here as well as one to the model.
     PUBLIC_FIELDS = {
+        "availability_7d",
         "avg_snr",
         "detections",
         "frames",
@@ -253,7 +254,6 @@ def test_leaderboard_answers_a_caller_with_no_session(self, client):
         "reputation",
         "tracks",
         "trust_score",
-        "uptime_s",
     }
 
     def test_the_published_row_declares_exactly_the_public_fields(self):
@@ -496,7 +496,7 @@ def test_leaderboard_with_node(self, client):
                         "total_detections": 42,
                         "total_frames": 10,
                         "total_tracks": 5,
-                        "uptime_s": 300,
+                        "availability_7d": 0.9731,
                         "avg_snr": 12.0,
                     },
                     "trust": {},
@@ -514,6 +514,7 @@ def test_leaderboard_with_node(self, client):
             found = [e for e in entries if e["node_ref"] == "test-lb-1"]
             assert len(found) == 1
             assert found[0]["detections"] == 42
+            assert found[0]["availability_7d"] == 0.9731
             assert found[0]["rank"] >= 1
         finally:
             state.latest_analytics_bytes = orig

backend/tests/test_publication.py

@@ -1108,11 +1108,8 @@ def test_the_owners_copy_is_the_same_fuzzed_frame_the_public_would_get(self, cli
             )
         )
         # The public handle rides on top of the fuzzed frame; everything under
-        # it must be exactly what the public would get.  uptime_s is the one
-        # field that legitimately differs between two summaries taken a few
-        # milliseconds apart (rounded to 0.1 s), so it is compared separately.
+        # it must be exactly what the public would get.
         got = {k: v for k, v in body["nodes"][_seed_ref(_PRIV)].items() if k != "node_ref"}
-        assert abs(got["metrics"].pop("uptime_s") - expected["metrics"].pop("uptime_s")) < 5.0
         assert got == expected
         rx = body["nodes"][_seed_ref(_PRIV)]["detection_area"]["rx"]
         assert rx["lat"] != self.RX["rx_lat"]

backend/tests/test_server_clock.py

@@ -0,0 +1,28 @@
+"""The server clock behind node availability: what it marks, and from when."""
+
+import asyncio
+import time
+
+from retina_analytics.availability import MinuteRing, minute_of
+
+from core import state
+from services.tasks import periodic
+
+
+async def _marked_after_running_briefly(monkeypatch, grace_s):
+    monkeypatch.setattr(periodic, "SERVER_CLOCK_GRACE_S", grace_s)
+    monkeypatch.setattr(state.node_analytics, "server_minutes", MinuteRing())
+    task = asyncio.create_task(periodic.server_clock_task())
+    await asyncio.sleep(0.05)
+    task.cancel()
+    await asyncio.gather(task, return_exceptions=True)
+    now = minute_of(time.time())
+    return state.node_analytics.server_minutes.bits(now - 1, now)
+
+
+async def test_the_server_clock_marks_the_minute_it_is_up_in(monkeypatch):
+    assert await _marked_after_running_briefly(monkeypatch, grace_s=0)
+
+
+async def test_the_server_clock_marks_nothing_while_nodes_reconnect(monkeypatch):
+    assert not await _marked_after_running_briefly(monkeypatch, grace_s=60)

backend/tests/test_state_snapshot.py

@@ -93,6 +93,38 @@ def test_reputations_survive_round_trip(self, tmp_path):
 
         state.node_analytics.reputations.pop("node-7", None)
 
+    def test_node_metrics_and_the_server_clock_survive_round_trip(self, tmp_path):
+        from retina_analytics.availability import MinuteRing
+        from retina_analytics.metrics import NodeMetrics
+
+        from core import state
+
+        now = time.time()
+        metrics = NodeMetrics(node_id="node-9", first_seen=now - 7200)
+        up = MinuteRing()
+        for ago in range(1, 61):
+            up.mark(int((now - ago * 60) // 60))
+            if ago <= 45:
+                metrics.record_frame({"delay": [1.0, 2.0], "snr": [9.0, 11.0]}, now=now - ago * 60)
+        expected = metrics.summary(up, now=now)
+        assert (expected["availability_7d"], expected["total_detections"]) == (0.75, 90)
+
+        saved_up = state.node_analytics.server_minutes
+        state.node_analytics.metrics["node-9"] = metrics
+        state.node_analytics.server_minutes = up
+        snap_path = str(tmp_path / "snap.json")
+        try:
+            with patch("services.state_snapshot._SNAPSHOT_PATH", snap_path):
+                save_snapshot()
+                state.node_analytics.metrics.pop("node-9")
+                state.node_analytics.server_minutes = MinuteRing()
+                restore_snapshot()
+            restored = state.node_analytics.metrics["node-9"]
+            assert restored.summary(state.node_analytics.server_minutes, now=now) == expected
+        finally:
+            state.node_analytics.metrics.pop("node-9", None)
+            state.node_analytics.server_minutes = saved_up
+
     def test_accuracy_samples_survive_round_trip(self, tmp_path):
         from core import state
 

backend/vulture_whitelist.py

@@ -130,6 +130,7 @@
 
 # Leaderboard response fields, set by keyword and read only by Pydantic's
 # serialisation. Each is a key the route publishes; see PublicLeaderboardRow.
+availability_7d
 avg_snr
 trust_score
 online

dashboard/src/pages/admin/NetworkHealthPage.tsx

@@ -10,7 +10,7 @@ import { DataTable } from "../../components/DataTable";
 import { Pager, clampPage } from "../../components/Pager";
 import { StatCard } from "../../components/StatCard";
 import { usePolling } from "../../hooks/usePolling";
-import { formatMHz, formatRelativeTime, formatUptime } from "../../utils/format";
+import { formatAvailability, formatMHz, formatRelativeTime } from "../../utils/format";
 import { useChartTheme } from "../../utils/chartTheme";
 import { RetnodeLink } from "../../components/RetnodeLink";
 import { StatusBadge } from "../../components/StatusBadge";
@@ -209,7 +209,7 @@ export default function NetworkHealthPage() {
           return (
             <>
               <DataTable
-                headers={["Node ref", "Node ID", "Status", "Last Heartbeat", "Detections", "Avg SNR", "Trust", "Reputation", "Uptime"]}
+                headers={["Node ref", "Node ID", "Status", "Last Heartbeat", "Detections", "Avg SNR", "Trust", "Reputation", "Availability"]}
                 count={paged.length}
                 empty="No nodes found"
               >
@@ -238,7 +238,7 @@ export default function NetworkHealthPage() {
                       <td>{(node._analytics?.metrics?.avg_snr || 0).toFixed(1)} dB</td>
                       <td>{((node._analytics?.trust?.trust_score || 0) * 100).toFixed(0)}%</td>
                       <td>{((node._analytics?.reputation?.reputation || 0) * 100).toFixed(0)}%</td>
-                      <td>{formatUptime(node._analytics?.metrics?.uptime_s || 0)}</td>
+                      <td>{formatAvailability(node._analytics?.metrics?.availability_7d)}</td>
                     </tr>
                   );
                 })}

dashboard/src/pages/admin/NodeManagementPage.tsx

@@ -5,7 +5,7 @@ import { FetchNotice, nothingLoaded } from "../../components/Notice";
 import { Pager } from "../../components/Pager";
 import { StatCard } from "../../components/StatCard";
 import { useFetch } from "../../hooks/usePolling";
-import { formatMHz, formatUptime } from "../../utils/format";
+import { formatAvailability, formatMHz } from "../../utils/format";
 import { PositionStatusBadge } from "../../components/PositionStatusBadge";
 import { PolledRadars } from "./PolledRadars";
 import {
@@ -166,8 +166,8 @@ export default function NodeManagementPage() {
                   <span>{((summary.reputation?.reputation || 0) * 100).toFixed(0)}%</span>
                   <span className="meta-label">Avg SNR</span>
                   <span>{(summary.metrics?.avg_snr || 0).toFixed(1)} dB</span>
-                  <span className="meta-label">Uptime</span>
-                  <span>{formatUptime(summary.metrics?.uptime_s || 0)}</span>
+                  <span className="meta-label">Availability</span>
+                  <span>{formatAvailability(summary.metrics?.availability_7d)}</span>
                   <span className="meta-label">Owner email</span>
                   <span>{owner?.email || "—"}</span>
                   <span className="meta-label">Site contact</span>

dashboard/src/pages/user/KnowledgeBasePage.tsx

@@ -38,7 +38,8 @@ export default function KnowledgeBasePage() {
       description: "What the dashboard numbers mean.",
       items: [
         { q: "Trust Score", a: "Measures how well your node's detections match known ADS-B aircraft positions. Higher is better (0–100%)." },
-        { q: "Reputation", a: "Long-term reliability score based on uptime, data quality, and consistency. Drops with outages or bad data." },
+        { q: "Availability", a: "The share of the last 7 days' minutes in which your node delivered data, counting only minutes the server was up to receive it. The node's own page also gives the last 24 hours." },
+        { q: "Reputation", a: "Long-term reliability score, judged on the node's heartbeat, how well its detections agree with ADS-B and with trusted neighbours, and whether its detection rate is plausible." },
         { q: "SNR (Signal-to-Noise Ratio)", a: "The strength of the radar signal relative to background noise. 10+ dB is good, 20+ dB is excellent." },
         { q: "Coverage Overlap", a: "How much your detection area intersects with nearby nodes. Higher overlap enables better triangulation." },
       ],

dashboard/src/pages/user/LeaderboardPage.tsx

@@ -7,7 +7,7 @@ import { StatCard } from "../../components/StatCard";
 import { StatusBadge } from "../../components/StatusBadge";
 import { usePolling } from "../../hooks/usePolling";
 import { useAuth } from "../../context/AuthContext";
-import { formatUptime } from "../../utils/format";
+import { formatAvailability } from "../../utils/format";
 import { shortRef } from "../../utils/nodes";
 
 const PAGE_SIZE = 25;
@@ -38,7 +38,7 @@ export default function LeaderboardPage() {
   // Re-sort locally based on selection
   const sorted = [...entries].sort((a, b) => {
     if (sortBy === "detections") return b.detections - a.detections;
-    if (sortBy === "uptime") return b.uptime_s - a.uptime_s;
+    if (sortBy === "availability") return (b.availability_7d ?? -1) - (a.availability_7d ?? -1);
     if (sortBy === "trust") return b.trust_score - a.trust_score;
     if (sortBy === "snr") return b.avg_snr - a.avg_snr;
     if (sortBy === "miss_rate") return (a.miss_rate || 0) - (b.miss_rate || 0);
@@ -111,7 +111,7 @@ export default function LeaderboardPage() {
       {/* Sort control */}
       <div className="toolbar">
         <span className="card-note">Sort by:</span>
-        {["detections", "uptime", "trust", "snr", ...(showsMisses ? ["miss_rate"] : [])].map((key) => (
+        {["detections", "availability", "trust", "snr", ...(showsMisses ? ["miss_rate"] : [])].map((key) => (
           <button
             key={key}
             className={`btn ${sortBy === key ? "btn-primary" : "btn-secondary"} btn-sm`}
@@ -150,7 +150,7 @@ export default function LeaderboardPage() {
                   "Detections",
                   "Tracks",
                   ...(showsMisses ? ["In Range", "Missed", "Miss Rate"] : []),
-                  "Uptime",
+                  "Availability",
                   "Avg SNR",
                   "Trust",
                 ]}
@@ -183,7 +183,9 @@ export default function LeaderboardPage() {
                         </td>
                       </>
                     )}
-                    <td>{formatUptime(entry.uptime_s)}</td>
+                    <td title="Share of the last 7 days' minutes it delivered in">
+                      {formatAvailability(entry.availability_7d)}
+                    </td>
                     <td>{entry.avg_snr.toFixed(1)} dB</td>
                     <td>{(entry.trust_score * 100).toFixed(0)}%</td>
                   </tr>