fix(analytics): derive role breakdown from filtered node rows

public/analytics.js

@@ -151,7 +151,7 @@
   function statusYellow() { return cssVar('--status-yellow') || '#eab308'; }
   function statusRed() { return cssVar('--status-red') || '#ef4444'; }
   function accentColor() { return cssVar('--accent') || '#4a9eff'; }
   function snrColor(snr) { return snr > 6 ? statusGreen() : snr > 0 ? statusYellow() : statusRed(); }
 
   // --- SVG helpers ---
   function sparkSvg(data, color, w = 120, h = 32) {
@@ -635,7 +635,7 @@
 
   // ===================== OVERVIEW =====================
   function renderOverview(el, d) {
     const rf = d.rfData, topo = d.topoData, ch = d.chanData, hs = d.hashData;
     el.innerHTML = `
       <div class="stats-grid">
         <div class="stat-card">
@@ -692,7 +692,7 @@
             const totalHours = pph.length;
             // Pick label interval: <=24h show every 6h, <=72h every 12h, else every 24h
             const labelInterval = totalHours <= 24 ? 6 : totalHours <= 72 ? 12 : 24;
             const labels = pph.map((h, i) => {
               const hh = h.hour.slice(11, 13); // "HH"
               const hourNum = parseInt(hh, 10);
               if (hourNum % labelInterval === 0) {
@@ -2587,14 +2587,39 @@
     }
   }
 
+  // #560 — mirrors the role set and pluralized keys of GetAllRoleCounts
+  // (cmd/server/db.go), but tallies the rows the caller already loaded
+  // instead of querying the whole `nodes` table, so the breakdown describes
+  // the same region/area-filtered population as the Network Status counts
+  // rendered above it. Single bounded O(n) pass; no network call. Roles
+  // outside the known set (including "observer" and missing/unrecognized
+  // values) are not tallied, matching GetAllRoleCounts' existing omission of
+  // them from the breakdown today.
+  const NODE_ROLE_BREAKDOWN_KEYS = { repeater: 'repeaters', room: 'rooms', companion: 'companions', sensor: 'sensors' };
+  function nodeRoleBreakdownCounts(nodes) {
+    const counts = { repeaters: 0, rooms: 0, companions: 0, sensors: 0 };
+    (nodes || []).forEach(n => {
+      const key = NODE_ROLE_BREAKDOWN_KEYS[n && n.role];
+      if (key) counts[key]++;
+    });
+    return counts;
+  }
+
+  // Bumped on every call and checked after the await below so a late
+  // response from a superseded filter/tab change can never overwrite a
+  // newer render (mirrors the _channelRenderGen / _distanceGen pattern
+  // already used elsewhere in this file).
+  let _nodesTabGen = 0;
   async function renderNodesTab(el) {
+    const myGen = ++_nodesTabGen;
     el.innerHTML = '<div style="padding:40px;text-align:center;color:var(--text-muted)">Loading node analytics…</div>';
     try {
       const rq = RegionFilter.regionQueryString() + AreaFilter.areaQueryString();
       const [nodesResp, bulkHealth] = await Promise.all([
         fetchAllNodes('&sortBy=lastSeen' + rq, { ttl: CLIENT_TTL.nodeList }),
         api('/nodes/bulk-health?limit=50' + rq, { ttl: CLIENT_TTL.analyticsRF })
       ]);
+      if (myGen !== _nodesTabGen) return; // superseded by a newer filter/tab change
       const nodes = nodesResp.nodes || nodesResp;
       const myNodes = JSON.parse(localStorage.getItem('meshcore-my-nodes') || '[]');
       const myKeys = new Set(myNodes.map(n => n.pubkey));
@@ -2621,7 +2646,17 @@
         else silent++;
       });
       const totalNodes = nodesResp.total || nodes.length;
-      const roleCounts = nodesResp.counts || {};
+      const roleCounts = nodeRoleBreakdownCounts(nodes);
+      // #560 follow-up: fetchAllNodes silently slices to its safetyCap
+      // (10000 nodes by default, public/app.js). Total Nodes and the Role
+      // Breakdown below both come from this same `nodes` array, so they stay
+      // mutually consistent even above the cap — but presenting a truncated
+      // population as complete would still be misleading. Surface it rather
+      // than guess whether any given deployment can reach it.
+      const nodesTruncatedNotice = nodesResp.truncated ? `
+        <div role="alert" style="margin-bottom:10px;padding:10px 14px;border:1px solid var(--status-yellow);border-radius:6px;background:var(--bg-secondary,var(--bg))">
+          <strong style="color:var(--status-yellow)"><svg class="ph-icon" aria-hidden="true"><use href="/icons/phosphor-sprite.svg#ph-warning"/></svg> Node list truncated at ${nodes.length.toLocaleString()} rows</strong> — Total Nodes and Role Breakdown below may not reflect the complete filtered population.
+        </div>` : '';
 
       function nodeLink(n) {
         return `<a href="#/nodes/${encodeURIComponent(n.public_key)}/analytics" class="analytics-link">${esc(n.name || n.public_key.slice(0, 12))}</a>`;
@@ -2633,6 +2668,7 @@
       // ROLE_COLORS from shared roles.js
 
       el.innerHTML = `
+        ${nodesTruncatedNotice}
         <div class="analytics-section">
           <h3><svg class="ph-icon" aria-hidden="true"><use href="/icons/phosphor-sprite.svg#ph-magnifying-glass"/></svg> Network Status</h3>
           <div style="display:flex;gap:16px;flex-wrap:wrap;margin-bottom:20px">
@@ -2742,6 +2778,7 @@
           </table>
         </div>`;
     } catch (e) {
+      if (myGen !== _nodesTabGen) return;
       el.innerHTML = `<div style="padding:40px;text-align:center;color:#ff6b6b">Failed to load node analytics: ${esc(e.message)}</div>`;
     }
   }
@@ -2810,7 +2847,7 @@
       (Array.isArray(clockData) ? clockData : []).forEach(c => { if (c && c.pubkey) clockByPk[c.pubkey] = c; });
 
       const now = Date.now();
       function ageOf(n) {
         const heardAt = getHealthLastHeard(n);
         const ms = heardAt ? new Date(heardAt).getTime() : 0;
         return ms ? now - ms : Infinity;
@@ -3403,6 +3440,8 @@
     window._analyticsHopDepthTimeSeriesChartHtml = hopDepthTimeSeriesChartHtml;
     window._analyticsHopDepthLookupByPubkey = hopDepthLookupByPubkey;
     window._analyticsBridgeRepeaterPubkeySet = bridgeRepeaterPubkeySet;
+    window._analyticsNodeRoleBreakdownCounts = nodeRoleBreakdownCounts;
+    window._analyticsRenderNodesTab = renderNodesTab;
   }
 
   // ─── Neighbor Graph Tab ─────────────────────────────────────────────────────
@@ -3495,7 +3534,7 @@
   }
 
   function createGraphState(data) {
     const nodes = (data.nodes || []).map((n, i) => ({
       ...n,
       x: 450 + (Math.random() - 0.5) * 400,
       y: 300 + (Math.random() - 0.5) * 300,
@@ -4384,7 +4423,7 @@
       btn.addEventListener('click', () => {
         const panel = document.getElementById(btn.dataset.target);
         if (!panel) return;
         const open = panel.style.display === 'none' || panel.style.display === '';
         // Toggle: if currently hidden ('none'), open; else close.
         const currentlyHidden = panel.style.display === 'none';
         panel.style.display = currentlyHidden ? 'block' : 'none';
@@ -4499,7 +4538,7 @@
     await loadRFHealthData(el);
   }
 
   async function loadRFHealthData(el) {
     const grid = document.getElementById('rfHealthGrid');
     const detail = document.getElementById('rfHealthDetail');
 
@@ -4591,7 +4630,7 @@
     }
   }
 
   async function loadRFSparkline(observerId) {
     const { since, until } = rfHealthTimeRangeToParams(_rfHealthState.range, _rfHealthState.customFrom, _rfHealthState.customTo);
     try {
       const data = await api(`/observers/${observerId}/metrics?since=${encodeURIComponent(since)}&until=${encodeURIComponent(until)}`);
@@ -4768,7 +4807,7 @@
     return svg;
   }
 
   function rfXAxisLabels(data, sx, h, pad) {
     let svg = '';
     const xTicks = Math.min(6, data.length);
     for (let i = 0; i < xTicks; i++) {

public/app.js

@@ -8,7 +8,7 @@
 
 function routeTypeName(n) { return ROUTE_TYPES[n] || 'UNKNOWN'; }
 function payloadTypeName(n) { return PAYLOAD_TYPES[n] || 'UNKNOWN'; }
 function payloadTypeColor(n) { return PAYLOAD_COLORS[n] || 'unknown'; }
 function isTransportRoute(rt) { return rt === 0 || rt === 3; }
 /** Byte offset of path_len in raw_hex: 5 for transport routes (4 bytes of next/last hop codes precede it), 1 otherwise. */
 function getPathLenOffset(routeType) { return isTransportRoute(routeType) ? 5 : 1; }
@@ -411,11 +411,19 @@
 //   (e.g. '&lastHeard=30d&area=x', '&sortBy=lastSeen&region=y'); pass '' for none.
 // safetyCap: hard ceiling on BOTH pages fetched and nodes returned — the result
 //   is sliced to it (callers like live.js pass their render ceiling here).
-// Returns { nodes, counts, total }: counts is from the first page; total is the
-//   real deduped/capped node count (NOT the server's per-query `total`).
+// Returns { nodes, counts, total, truncated }: counts is from the first page;
+//   total is the real deduped/capped node count (NOT the server's per-query
+//   `total`). truncated is true when `nodes` may not be the complete
+//   population for the query — either the page loop never saw a short page
+//   before hitting safetyCap (so more rows may exist beyond it), or dedup
+//   left more unique rows than safetyCap and the final slice below dropped
+//   some. A caller that presents `nodes` as a complete population (e.g.
+//   analytics.js's Role Breakdown, #560) must check this rather than
+//   silently showing a partial count as whole.
 async function fetchAllNodes(extraQuery = '', { ttl = 0, pageSize = 500, safetyCap = 10000 } = {}) {
   const accumulated = [];
   let counts = {};
+  let stoppedShort = false;
   for (let offset = 0; offset < safetyCap; offset += pageSize) {
     const data = await api(`/nodes?limit=${pageSize}&offset=${offset}${extraQuery}`, { ttl });
     const page = data && Array.isArray(data.nodes) ? data.nodes
@@ -426,7 +434,7 @@
     // COUNT(*) for the query, but the handler overwrites it with the filtered
     // length under area/geo/blacklist filtering — so we never loop on it, nor
     // surface it; a short page is the reliable end-of-data signal. See #1606.
-    if (page.length < pageSize) break;
+    if (page.length < pageSize) { stoppedShort = true; break; }
   }
   // Dedup by public_key: the sort window (last_seen DESC by default) can shift
   // under concurrent ingest, repeating a row across a page boundary. Rows
@@ -440,7 +448,8 @@
   // it to the next pageSize multiple), so e.g. live.js's LIVE_MAP_MAX_NODES is
   // honored exactly rather than overshooting by up to pageSize-1.
   const nodes = Array.from(seen.values()).slice(0, safetyCap);
-  return { nodes, counts, total: nodes.length };
+  const truncated = !stoppedShort || seen.size > nodes.length;
+  return { nodes, counts, total: nodes.length, truncated };
 }
 
 function invalidateApiCache(prefix) {

scripts/e2e-shards.json

@@ -33,6 +33,14 @@
       "meanSeconds": 0,
       "measurementNote": "New suite; fresh and SPA ?region= links across five views, listener counts, back/forward, the hash ?region= after a control change on six views, and ?region=All. About 2.5 minutes locally; no CI duration measured yet."
     },
+    {
+      "file": "test-issue-560-role-breakdown-e2e.js",
+      "shard": 2,
+      "env": { "BASE_URL": "$BASE_URL", "CORESCOPE_SERVER_BIN": "./corescope-server", "CORESCOPE_MIGRATE_BIN": "./corescope-migrate" },
+      "output": "e2e",
+      "meanSeconds": 0,
+      "measurementNote": "New suite; #560 Analytics -> Nodes Role Breakdown computed from the filtered rows, not the server's global /api/nodes counts -- an MRY region filter's role-breakdown sum must equal the Network Status total, and must differ from the unfiltered repeaters count; also an unfiltered-vs-server counts-contract check, and (round-2 review) an area-only and combined region+area check against a private, disposable server instance with one test-only area configured via -config-dir, since the shared BASE_URL server/fixture has none. About 15s locally; no CI duration measured yet; assigned to shard 2, the lowest measured total."
+    },
     {
       "file": "test-issue-562-battery-badge-e2e.js",
       "shard": 2,

test-all.sh

@@ -92,6 +92,7 @@ run test-analytics-channels-integration.js
 run test-analytics-foreign-traffic-tab.js
 run test-scope-audit.js
 run test-analytics-nodes-without-scope.js
+run test-issue-560-role-breakdown.js
 run test-analytics-table-ids-unique.js
 run test-analytics-relay-airtime-dumbbell.js
 run test-live-vcr-mode-contrast.js

test-fetch-all-nodes-pagination.js

@@ -109,6 +109,7 @@ test('surfaces ALL nodes past the 500 server cap (1200 > 500)', async () => {
   assert.strictEqual(out.nodes.length, 1200, 'expected all 1200 nodes, got ' + out.nodes.length);
   assert.strictEqual(out.total, 1200, 'total must be the real deduped count, not the clamped per-page total');
   assert.strictEqual(m.calls.length, 3, 'expected 3 pages (500+500+200), got ' + m.calls.length);
+  assert.strictEqual(out.truncated, false, 'a short final page means the population is complete, not truncated');
 });
 
 test('stops on a short page rather than the unreliable server total', async () => {
@@ -120,6 +121,7 @@ test('stops on a short page rather than the unreliable server total', async () =
   const out = await ctx.fetchAllNodes('');
   assert.strictEqual(out.nodes.length, 1000);
   assert.strictEqual(m.calls.length, 3, 'expected one extra empty page after two full pages');
+  assert.strictEqual(out.truncated, false, 'a trailing empty page means the population is complete, not truncated');
 });
 
 test('dedups a public_key repeated across a page boundary', async () => {
@@ -165,6 +167,36 @@ test('safetyCap is a hard node-count ceiling (no pageSize overshoot)', async ()
   const out = await ctx.fetchAllNodes('', { safetyCap: 800 });
   assert.strictEqual(out.nodes.length, 800, 'safetyCap must cap returned nodes exactly, got ' + out.nodes.length);
   assert.strictEqual(out.total, 800, 'total must equal the capped count');
+  assert.strictEqual(out.truncated, true, 'the real 5000-node population exceeds the 800 safetyCap, so the result must say so');
+});
+
+// #560 follow-up: a caller presenting `nodes` as the complete filtered
+// population (analytics.js's Role Breakdown) must be able to tell when it
+// is not — see the two `truncated` assertions above for the "complete"
+// cases. This covers the two ways `truncated` must come back true.
+test('truncated is true when the page loop hits safetyCap without ever seeing a short page', async () => {
+  const ctx = makeSandbox();
+  loadInCtx(ctx, 'public/app.js');
+  // 10000 nodes, safetyCap=1000, pageSize=500: every fetched page (0, 500)
+  // is a full 500-row page, so the loop never learns the real population is
+  // far larger than what it returned.
+  const m = makeNodesFetch(10000, 500);
+  ctx.fetch = m.fetch;
+  const out = await ctx.fetchAllNodes('', { safetyCap: 1000 });
+  assert.strictEqual(out.nodes.length, 1000);
+  assert.strictEqual(out.truncated, true, 'every page fetched was full, so more rows may exist beyond safetyCap');
+});
+
+test('truncated is false when the real population exactly fits under safetyCap', async () => {
+  const ctx = makeSandbox();
+  loadInCtx(ctx, 'public/app.js');
+  // 1000 nodes, safetyCap well above it: the loop sees a short (0-length)
+  // page before ever approaching the cap, so the population is complete.
+  const m = makeNodesFetch(1000, 500);
+  ctx.fetch = m.fetch;
+  const out = await ctx.fetchAllNodes('', { safetyCap: 10000 });
+  assert.strictEqual(out.nodes.length, 1000);
+  assert.strictEqual(out.truncated, false, 'a short page was seen well under safetyCap, so the population is complete');
 });
 
 test('rows missing public_key are NOT collapsed into one', async () => {