From 4fb2f0e545e60ce9a0c70266e6f2cde4b6a6d6c2 Mon Sep 17 00:00:00 2001 From: "Austin S. Lin" Date: Mon, 10 Aug 2026 07:50:57 -0400 Subject: [PATCH] feat(tailscale): opt-in tailnet monitoring per Spark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Spark can be entirely healthy — SSH answering, GPU reporting, LLM serving — while tailscaled has lost its session with the coordination server. Short connections retry and survive, so every LAN-based check (including sparkDash's own SSH liveness) keeps reporting "up" and is correct, but the box is unreachable from anywhere off the LAN: phone on cellular, admin console, other tailnet nodes. The dashboard looks right and the Spark is invisible. Adds `tailscaleMonitoring` (opt-in per Spark, default off), a TailscaleProbe that reads `tailscale status --json`, and a Tailnet card beside Network. It reports the node's OWN `Self.Online` rather than a peer's possibly-stale view, and surfaces Tailscale's `Health` messages so the card explains the failure ("hasn't received a network map from the coordination server in 2m7s") instead of just flagging it. Read-only: never runs tailscale up/down/login. Notes: - Default off, so nothing changes for users who don't run Tailscale. - 30s poll (POLL_INTERVAL_TAILSCALE) — each poll is an SSH round-trip and tailnet state moves slowly. - Local Sparks under Docker: tailscaled's socket lives on the host, so the probe enters the host mount namespace via nsenter, mirroring the existing nvidia-smi approach in SystemCollector. - No timestamps in the snapshot payload, preserving the byte-identical broadcast dedupe. - Parsing is an exported pure function (parseTailscaleStatus) so it is unit tested without I/O, matching how summarizeComfyPrompt is tested. Tests: 9 new (parser incl. wedged-netmap/expired-key/malformed cases, probe default-shape on failure, error-clearing on recovery). Suite 97/97, tsc clean. --- CHANGELOG.md | 11 ++ README.md | 61 ++++++ package.json | 2 +- server/collectors/TailscaleProbe.js | 185 ++++++++++++++++++ .../__tests__/TailscaleProbe.test.js | 122 ++++++++++++ server/config.js | 7 + server/sparks/SparkMonitor.js | 64 ++++++ server/sparks/SparkRegistry.js | 6 + .../sparks/__tests__/role-normalize.test.js | 11 ++ src/App.tsx | 4 + src/api/types.ts | 33 ++++ src/components/EditSparkDialog.tsx | 21 ++ src/components/SparkPage/SparkPage.tsx | 3 + src/components/SparkPage/TailscalePanel.tsx | 102 ++++++++++ 14 files changed, 631 insertions(+), 1 deletion(-) create mode 100644 server/collectors/TailscaleProbe.js create mode 100644 server/collectors/__tests__/TailscaleProbe.test.js create mode 100644 src/components/SparkPage/TailscalePanel.tsx diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d12e5b..ff1258e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ Format: version sections are listed newest first. --- +## [1.7.0] — 2026-08-10 + +### Added +- **Tailnet monitoring** — opt-in per Spark (`tailscaleMonitoring`, default **off**); runs `tailscale status --json` on the host and shows a **Tailnet** card beside Network. Closes a blind spot shared by every LAN-based check: when `tailscaled` loses its coordination-server session, SSH/GPU/LLM all keep reporting healthy while the Spark is unreachable from off-LAN. +- **Off-tailnet verdict from the node itself** — reads `Self.Online` on each host rather than a peer's (possibly stale) view of it. +- **Failure reason, not just state** — surfaces Tailscale's own `Health` messages (e.g. *"hasn't received a network map from the coordination server in 2m7s"*), `BackendState`, tailnet IP, DERP relay region, version, and an expired-node-key warning. +- **Edit Spark** — `Tailnet monitoring` checkbox. +- Env: `POLL_INTERVAL_TAILSCALE` (default `30000`), `TAILSCALE_PROBE_TIMEOUT_MS` (default `8000`). + +--- + ## [1.6.0] — 2026-08-07 ### Added diff --git a/README.md b/README.md index 53865ca..5a2be85 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ sparkDash is a real-time web dashboard for one or more **NVIDIA DGX Spark (GB10) - [Latest version changelog](#latest-version-changelog) - [Features](#features) - [ComfyUI monitoring](#comfyui-monitoring) +- [Tailnet monitoring](#tailnet-monitoring) - [Full changelog](./CHANGELOG.md) - [Quick start](#quick-start) - [Architecture](#architecture) @@ -47,6 +48,10 @@ sparkDash is a real-time web dashboard for one or more **NVIDIA DGX Spark (GB10) ## Latest version changelog +### Version 1.7.0 — Tailnet monitoring +- **Tailnet** — opt-in per Spark (`tailscaleMonitoring`): flags a Spark that is healthy on the LAN but has fallen off its tailnet, which no LAN-based check can see +- **Reason shown, not just state** — surfaces Tailscale's own `Health` messages plus backend state, tailnet IP, DERP relay, version, and expired-key warning + ### Version 1.6.0 — ComfyUI monitoring & compact default - **ComfyUI** — opt-in per Spark (port 8188): live jobs, progress, last run, cancel, queue ETA, Open on LAN IP, model inventory - **Overview** — Comfy status chip (`idle` / `run` / `Nq`) @@ -66,6 +71,7 @@ Full history: [CHANGELOG.md](./CHANGELOG.md) | **Local + remote** | Host metrics via sysfs/proc/`nvidia-smi`; remotes over SSH (key or password) | | **LLM probe** | Auto-detects llama.cpp, vLLM, sglang, or ds4-server; live tok/s per server | | **ComfyUI** | Opt-in probe: queue/jobs, progress, cancel, Open link, inventory, overview chip | +| **Tailnet** | Opt-in probe: flags a Spark that is healthy on the LAN but off its tailnet | | **Decode benchmark** | Multi-concurrency streaming decode tok/s (server + per-stream), persisted last run | | **Prompt Showcase** | Full-page multi-terminal LLM streaming demo (up to 32 prompts) with live tok/s and copy-out | | **vLLM health** | KV cache %, run/wait queue, TTFT/E2E/ITL p95, preemptions, prefix cache, MTP accept from Prometheus `/metrics` | @@ -140,6 +146,61 @@ Env (optional): `COMFY_PORT` (default `8188`), `COMFY_PROBE_TIMEOUT_MS`, `POLL_I --- +## Tailnet monitoring + +Opt-in per Spark. Runs `tailscale status --json` on the host and shows a **Tailnet** card +beside Network. + +This closes a blind spot that every LAN-based check shares, including sparkDash's own SSH +liveness. When `tailscaled` loses its session with the coordination server, short connections +keep retrying and succeeding — SSH answers, the GPU reports, the LLM serves, and the dashboard +says **up**, correctly. But the Spark is unreachable from anywhere *off* the LAN: your phone on +cellular, the admin console, another tailnet node. The box looks perfectly healthy and is +invisible. This card is the difference. + +### What is supported + +| Capability | Details | +|------------|---------| +| **Opt-in per Spark** | `tailscaleMonitoring` (default **off**) | +| **Off-tailnet detection** | `Self.Online` — the node's *own* view of the coordination server | +| **Reason, not just state** | Tailscale's `Health` messages, e.g. *"hasn't received a network map from the coordination server in 2m7s"* | +| **Backend state** | `Running` / `Stopped` / `NeedsLogin` / `NoState` | +| **Identity** | Tailnet IP, hostname, DERP relay region, tailscale version | +| **Key expiry** | Flags an expired node key (which a restart will not fix — it needs re-auth) | + +Asked of **each node about itself**, deliberately: one node's view of a *peer* can be stale, so +peer state is never used as the verdict. + +Not claimed: this does not manage Tailscale. It never runs `tailscale up`, `down`, or `login` — +the probe is read-only. + +### How to enable (per Spark) + +1. Open **Edit Spark**. +2. Tick **Tailnet monitoring**. +3. Save. The Tailnet card appears under Resources, next to Network. + +### Host requirements + +- The `tailscale` CLI on the monitored host, and `tailscaled` running. +- Remote Sparks: reached over the existing SSH connection — no extra setup. +- Local Spark in Docker: tailscaled's socket lives on the *host*, so the probe enters the host + mount namespace via `nsenter` (the same approach used for `nvidia-smi`). This needs + `/host/proc` bind-mounted, which the shipped compose file already does. +- A node with key expiry disabled reports no expiry — that is normal, not an error. + +### Config fields (persisted on the Spark) + +| Field | Default | Description | +|-------|---------|-------------| +| `tailscaleMonitoring` | `false` | Run `tailscale status --json` and show the Tailnet card | + +Env (optional): `POLL_INTERVAL_TAILSCALE` (default `30000` — tailnet state changes slowly and +each poll is an SSH round-trip), `TAILSCALE_PROBE_TIMEOUT_MS` (default `8000`). + +--- + ## Quick start ```bash diff --git a/package.json b/package.json index cb29d7b..c132770 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "sparkdash", - "version": "1.6.0", + "version": "1.7.0", "description": "sparkDash — Multi-DGX Spark Monitoring Dashboard", "type": "module", "scripts": { diff --git a/server/collectors/TailscaleProbe.js b/server/collectors/TailscaleProbe.js new file mode 100644 index 0000000..02cc087 --- /dev/null +++ b/server/collectors/TailscaleProbe.js @@ -0,0 +1,185 @@ +/** + * TailscaleProbe — reports whether a Spark is actually present on its tailnet. + * + * Why this exists: a Spark can be perfectly healthy — SSH answering, GPU idle, + * LLM serving — while `tailscaled` has lost its session with the coordination + * server. Short connections retry and survive, so every LAN-based check + * (including sparkDash's own SSH liveness) still reports "up", but the box is + * unreachable from anywhere off the LAN: phone on cellular, the admin console, + * another tailnet node. The dashboard and the tailnet disagree and the dashboard + * looks right. + * + * `tailscale status --json` is the authoritative answer, and it must be asked of + * the node itself — one node's view of a *peer* can be stale, so peer state is + * deliberately not used here. + * + * Command: `tailscale status --json` (read-only; no state is changed) + */ +import { TAILSCALE_PROBE_TIMEOUT_MS, HOST_PATHS } from "../config.js"; +import { sshExec } from "./ssh.js"; +import fs from "fs"; +import path from "path"; + +/** + * @param {unknown} v + * @returns {string | null} + */ +function str(v) { + if (typeof v !== "string") return null; + const s = v.trim(); + return s.length > 0 ? s : null; +} + +/** + * Normalize `tailscale status --json` into the fields the UI needs. + * + * Exported for unit testing — keep it pure (no I/O, no clock). + * + * @param {object} raw - parsed output of `tailscale status --json` + * @returns {object} normalized fields (without `error`) + */ +export function parseTailscaleStatus(raw) { + const self = raw && typeof raw === "object" && raw.Self && typeof raw.Self === "object" + ? raw.Self + : null; + + // Health is the payload that actually explains a failure, e.g. + // "Tailscale hasn't received a network map from the coordination server in 2m7s." + const health = Array.isArray(raw?.Health) + ? raw.Health.filter((m) => typeof m === "string" && m.trim().length > 0) + : []; + + const ips = Array.isArray(self?.TailscaleIPs) + ? self.TailscaleIPs.filter((ip) => typeof ip === "string") + : []; + + return { + available: self != null, + /** + * Self.Online is the node's OWN view of whether it is talking to the + * coordination server. This is the signal a LAN-only check cannot see. + * null when tailscale did not report it. + */ + online: typeof self?.Online === "boolean" ? self.Online : null, + /** "Running" | "Stopped" | "NeedsLogin" | "NoState" — tailscaled's own state. */ + backendState: str(raw?.BackendState), + hostName: str(self?.HostName), + dnsName: str(self?.DNSName), + tailscaleIp: ips[0] ?? null, + /** DERP relay region, or null when the node has a direct path. */ + relay: str(self?.Relay), + /** ISO timestamp; null when key expiry is disabled for this node. */ + keyExpiry: str(self?.KeyExpiry), + keyExpired: self?.Expired === true, + version: str(raw?.Version), + /** Human-readable reasons tailscale itself considers itself unhealthy. */ + health, + }; +} + +export class TailscaleProbe { + /** + * @param {object} spark + */ + constructor(spark) { + this.spark = spark; + this.error = null; + } + + /** @param {object} spark */ + setTarget(spark) { + this.spark = spark ?? this.spark; + this.error = null; + } + + /** Symmetry with the other probes; nothing persistent to release. */ + dispose() {} + + /** + * Host PID 1 mount namespace, present when running as the bind-mounted + * container. Mirrors SystemCollector._hasHostProc. + */ + _hasHostProc() { + return fs.existsSync(path.join(HOST_PATHS.PROC, "1", "ns", "mnt")); + } + + /** + * Run the status command for an `isLocal` Spark. + * + * In the container the `tailscale` CLI and, more importantly, tailscaled's + * unix socket live on the *host*, so a plain `sh -c` finds neither. Enter the + * host mount namespace when it is available (same approach SystemCollector + * uses for nvidia-smi). + * + * @param {string} cmd + * @returns {Promise} + */ + async _execLocal(cmd) { + const { execFile } = await import("child_process"); + const useHostNs = this._hasHostProc(); + const file = useHostNs ? "nsenter" : "sh"; + const args = useHostNs + ? ["--mount=" + path.join(HOST_PATHS.PROC, "1", "ns", "mnt"), "--", "sh", "-c", cmd] + : ["-c", cmd]; + return new Promise((resolve, reject) => { + execFile(file, args, { timeout: TAILSCALE_PROBE_TIMEOUT_MS }, (err, stdout, stderr) => { + if (err) return reject(new Error(stderr?.trim() || err.message)); + resolve(String(stdout).trim()); + }); + }); + } + + /** + * Never throws — on failure returns `_default()` with `error` set, matching + * the other probes. + * + * @returns {Promise} + */ + async probe() { + const cmd = "tailscale status --json"; + let out; + try { + out = this.spark?.isLocal + ? await this._execLocal(cmd) + : await sshExec(this.spark, cmd, { timeoutMs: TAILSCALE_PROBE_TIMEOUT_MS }); + } catch (err) { + // Most common causes: tailscale not installed, or tailscaled not running + // (the CLI exits non-zero). Both are legitimately "not on the tailnet". + this.error = err.message || "tailscale status failed"; + return this._default(); + } + + let raw; + try { + raw = JSON.parse(out); + } catch { + this.error = "Unparseable `tailscale status --json` output"; + return this._default(); + } + + const parsed = parseTailscaleStatus(raw); + if (!parsed.available) { + this.error = "No Self in `tailscale status --json`"; + return this._default(); + } + this.error = null; + return { ...parsed, error: null }; + } + + _default() { + return { + available: false, + online: null, + backendState: null, + hostName: null, + dnsName: null, + tailscaleIp: null, + relay: null, + keyExpiry: null, + keyExpired: false, + version: null, + health: [], + error: this.error, + }; + } +} diff --git a/server/collectors/__tests__/TailscaleProbe.test.js b/server/collectors/__tests__/TailscaleProbe.test.js new file mode 100644 index 0000000..21e7fe0 --- /dev/null +++ b/server/collectors/__tests__/TailscaleProbe.test.js @@ -0,0 +1,122 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { TailscaleProbe, parseTailscaleStatus } from "../TailscaleProbe.js"; + +/** Shape of a healthy `tailscale status --json`, trimmed to the fields we read. */ +const HEALTHY = { + BackendState: "Running", + Version: "1.102.2-t6cac91817", + Self: { + HostName: "spark-1", + DNSName: "spark-1.example.ts.net.", + Online: true, + TailscaleIPs: ["100.64.0.1", "fd7a:115c:a1e0::1"], + Relay: "nyc", + KeyExpiry: "2026-09-06T20:02:31Z", + Expired: false, + }, + Health: [], +}; + +test("parseTailscaleStatus reads a healthy node", () => { + const p = parseTailscaleStatus(HEALTHY); + assert.equal(p.available, true); + assert.equal(p.online, true); + assert.equal(p.backendState, "Running"); + assert.equal(p.hostName, "spark-1"); + // First IP only — the v4 address is what operators recognize. + assert.equal(p.tailscaleIp, "100.64.0.1"); + assert.equal(p.relay, "nyc"); + assert.equal(p.keyExpired, false); + assert.deepEqual(p.health, []); +}); + +test("parseTailscaleStatus surfaces the wedged-netmap case (healthy box, off tailnet)", () => { + // The failure this probe exists for: tailscaled running, SSH fine, but no + // session with the coordination server — invisible from off-LAN. + const p = parseTailscaleStatus({ + ...HEALTHY, + Self: { ...HEALTHY.Self, Online: false }, + Health: ["Tailscale hasn't received a network map from the coordination server in 2m7s."], + }); + assert.equal(p.available, true); + assert.equal(p.online, false); + assert.equal(p.backendState, "Running"); + assert.equal(p.health.length, 1); + assert.match(p.health[0], /coordination server/); +}); + +test("parseTailscaleStatus flags an expired key", () => { + const p = parseTailscaleStatus({ + ...HEALTHY, + Self: { ...HEALTHY.Self, Online: false, Expired: true }, + }); + assert.equal(p.keyExpired, true); + assert.equal(p.online, false); +}); + +test("parseTailscaleStatus tolerates missing and malformed fields", () => { + const p = parseTailscaleStatus({ Self: {} }); + assert.equal(p.available, true); // Self exists, just empty + assert.equal(p.online, null); // absent boolean stays unknown, not false + assert.equal(p.tailscaleIp, null); + assert.equal(p.relay, null); + assert.equal(p.keyExpiry, null); + assert.equal(p.keyExpired, false); + assert.deepEqual(p.health, []); + + const empty = parseTailscaleStatus({}); + assert.equal(empty.available, false); + assert.equal(empty.online, null); + + // Non-string junk in Health must not reach the UI. + const junk = parseTailscaleStatus({ Self: {}, Health: ["ok", "", 42, null] }); + assert.deepEqual(junk.health, ["ok"]); +}); + +test("parseTailscaleStatus blanks whitespace-only strings", () => { + const p = parseTailscaleStatus({ + BackendState: " ", + Self: { HostName: "", Relay: " ", Online: true }, + }); + assert.equal(p.backendState, null); + assert.equal(p.hostName, null); + assert.equal(p.relay, null); +}); + +test("TailscaleProbe returns default shape with error when the command fails", async () => { + // isLocal avoids SSH; `false` exits non-zero, standing in for a missing + // tailscale binary or a stopped tailscaled. + const probe = new TailscaleProbe({ isLocal: true }); + probe._execLocal = async () => { + throw new Error("tailscale: command not found"); + }; + const snap = await probe.probe(); + assert.equal(snap.available, false); + assert.equal(snap.online, null); + assert.equal(snap.keyExpired, false); + assert.deepEqual(snap.health, []); + assert.match(snap.error, /command not found/); + probe.dispose(); +}); + +test("TailscaleProbe reports an error on unparseable output rather than throwing", async () => { + const probe = new TailscaleProbe({ isLocal: true }); + probe._execLocal = async () => "not json at all"; + const snap = await probe.probe(); + assert.equal(snap.available, false); + assert.match(snap.error, /Unparseable/); +}); + +test("TailscaleProbe clears a stale error after recovery", async () => { + const probe = new TailscaleProbe({ isLocal: true }); + probe._execLocal = async () => { + throw new Error("boom"); + }; + assert.ok((await probe.probe()).error); + + probe._execLocal = async () => JSON.stringify(HEALTHY); + const ok = await probe.probe(); + assert.equal(ok.error, null); + assert.equal(ok.online, true); +}); diff --git a/server/config.js b/server/config.js index 123c2b2..bf43849 100644 --- a/server/config.js +++ b/server/config.js @@ -19,6 +19,8 @@ const SECRETS_KEY_PATH = // ─── LLM / Comfy probe timeouts ────────────────────────── const LLM_PROBE_TIMEOUT_MS = 3000; const COMFY_PROBE_TIMEOUT_MS = parseInt(process.env.COMFY_PROBE_TIMEOUT_MS || "3000", 10); +// `tailscale status --json` runs over SSH, so it needs more headroom than an HTTP probe. +const TAILSCALE_PROBE_TIMEOUT_MS = parseInt(process.env.TAILSCALE_PROBE_TIMEOUT_MS || "8000", 10); const SSH_CONNECT_TIMEOUT = 5; // seconds // ─── Poll intervals (milliseconds) ─────────────────────── @@ -28,6 +30,9 @@ const POLL_INTERVAL_NETWORK = parseInt(process.env.POLL_INTERVAL_NETWORK || "200 const POLL_INTERVAL_STORAGE = parseInt(process.env.POLL_INTERVAL_STORAGE || "5000", 10); const POLL_INTERVAL_LLM = parseInt(process.env.POLL_INTERVAL_LLM || "2000", 10); const POLL_INTERVAL_COMFY = parseInt(process.env.POLL_INTERVAL_COMFY || "2000", 10); +// Tailnet membership changes on the order of minutes, and each poll is an SSH +// round-trip — a slow cadence keeps this cheap. +const POLL_INTERVAL_TAILSCALE = parseInt(process.env.POLL_INTERVAL_TAILSCALE || "30000", 10); // dmon -c 1 -d 1 blocks ~1s; default 2s avoids stacking with in-flight guards const POLL_INTERVAL_BANDWIDTH = parseInt(process.env.POLL_INTERVAL_BANDWIDTH || "2000", 10); // Dedicated liveness (sshTest / local ping) cadence — not a metric domain. @@ -82,6 +87,7 @@ export { SECRETS_KEY_PATH, LLM_PROBE_TIMEOUT_MS, COMFY_PROBE_TIMEOUT_MS, + TAILSCALE_PROBE_TIMEOUT_MS, SSH_CONNECT_TIMEOUT, POLL_INTERVAL_GPU, POLL_INTERVAL_CPU, @@ -89,6 +95,7 @@ export { POLL_INTERVAL_STORAGE, POLL_INTERVAL_LLM, POLL_INTERVAL_COMFY, + POLL_INTERVAL_TAILSCALE, POLL_INTERVAL_BANDWIDTH, POLL_INTERVAL_LIVENESS, PORT, diff --git a/server/sparks/SparkMonitor.js b/server/sparks/SparkMonitor.js index f2292b4..48af6de 100644 --- a/server/sparks/SparkMonitor.js +++ b/server/sparks/SparkMonitor.js @@ -3,6 +3,7 @@ import path from "path"; import { SystemCollector } from "../collectors/SystemCollector.js"; import { LlmProbe } from "../collectors/LlmProbe.js"; import { ComfyProbe } from "../collectors/ComfyProbe.js"; +import { TailscaleProbe } from "../collectors/TailscaleProbe.js"; import { sshTest, sshExec } from "../collectors/ssh.js"; import { POLL_INTERVAL_GPU, @@ -11,6 +12,7 @@ import { POLL_INTERVAL_STORAGE, POLL_INTERVAL_LLM, POLL_INTERVAL_COMFY, + POLL_INTERVAL_TAILSCALE, POLL_INTERVAL_BANDWIDTH, POLL_INTERVAL_LIVENESS, LLM_PORT, @@ -47,6 +49,11 @@ export class SparkMonitor { ? new ComfyProbe(spark, this._comfyPort(spark)) : null; + /** @type {TailscaleProbe | null} */ + this.tailscaleProbe = this._tailscaleMonitoringEnabled(spark) + ? new TailscaleProbe(spark) + : null; + // Online status from dedicated liveness checks (not metric poll success) this.online = false; this.lastOnlineOk = 0; @@ -64,6 +71,7 @@ export class SparkMonitor { unifiedMemory: this.collector._defaultUnifiedMemory(), llm: [], comfy: null, + tailscale: null, }; this._lastUpdate = {}; @@ -73,6 +81,8 @@ export class SparkMonitor { this._llmIntervalId = null; /** @type {ReturnType | null} */ this._comfyIntervalId = null; + /** @type {ReturnType | null} */ + this._tailscaleIntervalId = null; this._running = false; /** @type {Record} in-flight domain guards */ this._inflight = {}; @@ -82,6 +92,7 @@ export class SparkMonitor { updateConfig(spark) { const wasLlm = this._llmMonitoringEnabled(this.spark); const wasComfy = this._comfyMonitoringEnabled(this.spark); + const wasTailscale = this._tailscaleMonitoringEnabled(this.spark); const prevComfyPort = this._comfyPort(this.spark); this.spark = spark; this.collector.spark = spark; @@ -123,6 +134,18 @@ export class SparkMonitor { this._metrics.comfy = null; } + // Tailscale probe — create / update / clear + if (this._tailscaleMonitoringEnabled()) { + if (this.tailscaleProbe) { + this.tailscaleProbe.setTarget(spark); + } else { + this.tailscaleProbe = new TailscaleProbe(spark); + } + } else { + this.tailscaleProbe = null; + this._metrics.tailscale = null; + } + // Toggle LLM poll interval when monitoring enablement flips if (this._running && wasLlm !== this._llmMonitoringEnabled()) { this._restartLlmPollInterval(); @@ -132,6 +155,9 @@ export class SparkMonitor { if (this._running && (wasComfy !== comfyOn || comfyPortChanged)) { this._restartComfyPollInterval(); } + if (this._running && wasTailscale !== this._tailscaleMonitoringEnabled()) { + this._restartTailscalePollInterval(); + } } /** @@ -188,6 +214,31 @@ export class SparkMonitor { } } + /** + * Opt-in tailnet monitoring (all roles; default off). + * @param {object} [spark] + */ + _tailscaleMonitoringEnabled(spark = this.spark) { + return Boolean(spark?.tailscaleMonitoring); + } + + /** Start or clear the tailnet poll timer based on monitoring flag. */ + _restartTailscalePollInterval() { + if (this._tailscaleIntervalId != null) { + clearInterval(this._tailscaleIntervalId); + this._intervals = this._intervals.filter((id) => id !== this._tailscaleIntervalId); + this._tailscaleIntervalId = null; + } + if (this._tailscaleMonitoringEnabled() && this._running) { + this._tailscaleIntervalId = setInterval( + () => this._pollDomain("tailscale"), + POLL_INTERVAL_TAILSCALE + ); + this._intervals.push(this._tailscaleIntervalId); + void this._pollDomain("tailscale"); + } + } + /** Returns array of LLM ports from spark config. */ _llmPorts() { const raw = this.spark?.llmPorts; @@ -216,6 +267,7 @@ export class SparkMonitor { this._intervals.push(setInterval(() => this._pollDomain("memory"), POLL_INTERVAL_BANDWIDTH)); this._restartLlmPollInterval(); this._restartComfyPollInterval(); + this._restartTailscalePollInterval(); // Liveness on a slightly slower cadence this._intervals.push(setInterval(() => this._checkOnline(), POLL_INTERVAL_LIVENESS)); console.log(`[SparkMonitor] ${this.spark.id} started`); @@ -228,6 +280,7 @@ export class SparkMonitor { this._intervals = []; this._llmIntervalId = null; this._comfyIntervalId = null; + this._tailscaleIntervalId = null; this._inflight = {}; if (this.comfyProbe) { try { @@ -243,6 +296,7 @@ export class SparkMonitor { snapshot() { const ports = this._llmMonitoringEnabled() ? this._llmPorts() : []; const comfyOn = this._comfyMonitoringEnabled(); + const tailscaleOn = this._tailscaleMonitoringEnabled(); return { id: this.spark.id, name: this.spark.name, @@ -267,6 +321,7 @@ export class SparkMonitor { .filter((n) => Number.isInteger(n)), comfyMonitoring: comfyOn, comfyPort: this._comfyPort(), + tailscaleMonitoring: tailscaleOn, hardware: this._getHardwareSummary(), metrics: { // NOTE: no `timestamp` here on purpose. The broadcast path skips @@ -284,6 +339,7 @@ export class SparkMonitor { unifiedMemory: this._metrics.unifiedMemory, llm: this._metrics.llm, comfy: comfyOn ? this._metrics.comfy : null, + tailscale: tailscaleOn ? this._metrics.tailscale : null, }, }; } @@ -352,6 +408,7 @@ export class SparkMonitor { this._pollDomain("memory"), this._pollDomain("llm"), this._pollDomain("comfy"), + this._pollDomain("tailscale"), ]); } @@ -362,6 +419,7 @@ export class SparkMonitor { // Worker nodes: no local LLM API if (domain === "llm" && !this._llmMonitoringEnabled()) return; if (domain === "comfy" && !this._comfyMonitoringEnabled()) return; + if (domain === "tailscale" && !this._tailscaleMonitoringEnabled()) return; this._inflight[domain] = true; try { let result; @@ -393,6 +451,9 @@ export class SparkMonitor { case "comfy": result = this.comfyProbe ? await this.comfyProbe.probe() : null; break; + case "tailscale": + result = this.tailscaleProbe ? await this.tailscaleProbe.probe() : null; + break; } // Re-check after the await — `stop()`/`updateSpark()` may have torn // this monitor down mid-flight. Writing `_metrics` on a dead monitor @@ -432,6 +493,9 @@ export class SparkMonitor { case "comfy": this._metrics.comfy = result; break; + case "tailscale": + this._metrics.tailscale = result; + break; } this._lastUpdate[domain] = Date.now(); } catch (err) { diff --git a/server/sparks/SparkRegistry.js b/server/sparks/SparkRegistry.js index d05bcff..f36644e 100644 --- a/server/sparks/SparkRegistry.js +++ b/server/sparks/SparkRegistry.js @@ -511,6 +511,12 @@ export class SparkRegistry { comfyMonitoring: Boolean(config.comfyMonitoring), /** ComfyUI HTTP port (default 8188). */ comfyPort: this._normalizeComfyPort(config.comfyPort), + /** + * Report tailnet presence via `tailscale status --json` (default false; + * all roles). Catches a Spark that is healthy on the LAN but has fallen + * off its tailnet, which no LAN-based check can see. + */ + tailscaleMonitoring: Boolean(config.tailscaleMonitoring), disabledDevices: Array.isArray(config.disabledDevices) ? config.disabledDevices : [], disabledInterfaces: Array.isArray(config.disabledInterfaces) ? config.disabledInterfaces : [], storagePollDisabled: Boolean(config.storagePollDisabled), diff --git a/server/sparks/__tests__/role-normalize.test.js b/server/sparks/__tests__/role-normalize.test.js index 0091c3f..136703a 100644 --- a/server/sparks/__tests__/role-normalize.test.js +++ b/server/sparks/__tests__/role-normalize.test.js @@ -27,6 +27,17 @@ test("comfyMonitoring is opt-in for all roles; comfyPort defaults to 8188", () = assert.equal(n({ comfyPort: "8190" }).comfyPort, 8190); }); +test("tailscaleMonitoring is opt-in for all roles", () => { + assert.equal(n({ role: "head" }).tailscaleMonitoring, false); + assert.equal(n({ role: "worker" }).tailscaleMonitoring, false); + assert.equal(n({ role: "standalone" }).tailscaleMonitoring, false); + assert.equal(n({ role: "standalone", tailscaleMonitoring: true }).tailscaleMonitoring, true); + assert.equal(n({ role: "worker", tailscaleMonitoring: true }).tailscaleMonitoring, true); + // Non-boolean input is coerced, never persisted raw. + assert.equal(n({ tailscaleMonitoring: "yes" }).tailscaleMonitoring, true); + assert.equal(n({ tailscaleMonitoring: 0 }).tailscaleMonitoring, false); +}); + test("legacy workerNode-only becomes worker", () => { const out = n({ workerNode: true, workerLabel: " DS ", workerHeadId: "s5" }); assert.equal(out.role, "worker"); diff --git a/src/App.tsx b/src/App.tsx index 065fa7a..8f4b353 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -27,6 +27,7 @@ function placeholderSnapshot( workerHeadId?: string | null; llmMonitoring?: boolean; comfyMonitoring?: boolean; + tailscaleMonitoring?: boolean; comfyPort?: number; } ): SparkSnapshot { @@ -59,6 +60,7 @@ function placeholderSnapshot( ? true : roleFields?.llmMonitoring !== false, comfyMonitoring: Boolean(roleFields?.comfyMonitoring), + tailscaleMonitoring: Boolean(roleFields?.tailscaleMonitoring), comfyPort: roleFields?.comfyPort ?? 8188, hardware: { device: "NVIDIA DGX Spark", @@ -164,6 +166,7 @@ function DashboardApp() { workerHeadId: c.workerHeadId ?? existing.workerHeadId, llmMonitoring: c.llmMonitoring ?? existing.llmMonitoring, comfyMonitoring: c.comfyMonitoring ?? existing.comfyMonitoring, + tailscaleMonitoring: c.tailscaleMonitoring ?? existing.tailscaleMonitoring, comfyPort: c.comfyPort ?? existing.comfyPort, disabledDevices: c.disabledDevices || existing.disabledDevices, disabledInterfaces: c.disabledInterfaces || existing.disabledInterfaces, @@ -184,6 +187,7 @@ function DashboardApp() { workerHeadId: c.workerHeadId, llmMonitoring: c.llmMonitoring, comfyMonitoring: c.comfyMonitoring, + tailscaleMonitoring: c.tailscaleMonitoring, comfyPort: c.comfyPort, } ); diff --git a/src/api/types.ts b/src/api/types.ts index f4c64ff..8593cbe 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -65,6 +65,10 @@ export interface SparkConfig { comfyMonitoring?: boolean; /** ComfyUI HTTP port (default 8188). */ comfyPort?: number; + /** + * Report tailnet presence via `tailscale status --json` (default false; all roles). + */ + tailscaleMonitoring?: boolean; /** When true, storage is only updated on manual refresh, not auto-polled. */ storagePollDisabled?: boolean; } @@ -310,6 +314,31 @@ export interface ComfyMetrics { error: string | null; } +export interface TailscaleMetrics { + /** True when `tailscale status --json` was read and had a Self entry. */ + available: boolean; + /** + * The node's OWN view of whether it is talking to the coordination server. + * A Spark can be healthy on the LAN with this false — that is the whole point + * of this probe. null when tailscale did not report it. + */ + online: boolean | null; + /** tailscaled's own state: Running | Stopped | NeedsLogin | NoState. */ + backendState: string | null; + hostName: string | null; + dnsName: string | null; + tailscaleIp: string | null; + /** DERP relay region, or null when the node has a direct path. */ + relay: string | null; + /** ISO timestamp; null when key expiry is disabled for this node. */ + keyExpiry: string | null; + keyExpired: boolean; + version: string | null; + /** Tailscale's own health warnings — these explain a false `online`. */ + health: string[]; + error: string | null; +} + // ─── Full metrics snapshot ──────────────────────────────── export interface SparkMetrics { gpu: GpuMetrics | null; @@ -322,6 +351,8 @@ export interface SparkMetrics { llm: LlmMetrics[]; /** ComfyUI probe result when monitoring is enabled; null when off or not yet polled. */ comfy?: ComfyMetrics | null; + /** Tailnet probe result when monitoring is enabled; null when off or not yet polled. */ + tailscale?: TailscaleMetrics | null; } // ─── Spark snapshot (server pushes this) ────────────────── @@ -357,6 +388,8 @@ export interface SparkSnapshot { comfyMonitoring?: boolean; /** ComfyUI HTTP port (default 8188) */ comfyPort?: number; + /** Whether tailnet presence is probed (opt-in; all roles) */ + tailscaleMonitoring?: boolean; hardware: HardwareInfo; metrics: SparkMetrics; } diff --git a/src/components/EditSparkDialog.tsx b/src/components/EditSparkDialog.tsx index 87ccdf2..dcabc13 100644 --- a/src/components/EditSparkDialog.tsx +++ b/src/components/EditSparkDialog.tsx @@ -180,6 +180,7 @@ export function EditSparkDialog({ config.ssh?.user !== savedConfig.ssh?.user || config.ssh?.auth !== savedConfig.ssh?.auth || Boolean(config.comfyMonitoring) !== Boolean(savedConfig.comfyMonitoring) || + Boolean(config.tailscaleMonitoring) !== Boolean(savedConfig.tailscaleMonitoring) || (config.comfyPort ?? 8188) !== (savedConfig.comfyPort ?? 8188); const result = formDirty @@ -249,6 +250,7 @@ export function EditSparkDialog({ const n = Number(config.comfyPort); return Number.isInteger(n) && n >= 1 && n <= 65535 ? n : 8188; })(), + tailscaleMonitoring: Boolean(config.tailscaleMonitoring), ssh: { host: config.ssh.host || config.lanIp, user: config.ssh.user, @@ -463,6 +465,25 @@ export function EditSparkDialog({ +
+ +
+ {role === "worker" && (
diff --git a/src/components/SparkPage/SparkPage.tsx b/src/components/SparkPage/SparkPage.tsx index 7d09b8e..09d248d 100644 --- a/src/components/SparkPage/SparkPage.tsx +++ b/src/components/SparkPage/SparkPage.tsx @@ -7,6 +7,7 @@ import { GpuPanel } from "./GpuPanel"; import { CpuPanel } from "./CpuPanel"; import { StoragePanel } from "./StoragePanel"; import { NetworkPanel } from "./NetworkPanel"; +import { TailscalePanel } from "./TailscalePanel"; import { LlmPanel } from "./LlmPanel"; import { ComfyPanel } from "./ComfyPanel"; import { ChevronDownIcon } from "../ui/icons"; @@ -174,6 +175,7 @@ export function SparkPage({ spark, temperatureUnit, onEdit }: SparkPageProps) { const llmOn = isLlmMonitoringEnabled(spark); const comfyOn = Boolean(spark.comfyMonitoring); + const tailscaleOn = Boolean(spark.tailscaleMonitoring); /** First LLM + Comfy share a row when both are on. */ const primarySideBySide = llmOn && comfyOn; const showServices = llmOn || comfyOn; @@ -241,6 +243,7 @@ export function SparkPage({ spark, temperatureUnit, onEdit }: SparkPageProps) { disabledInterfaces={disabledInterfaces} onDisabledChange={setDisabledInterfaces} /> + {tailscaleOn && } )} {/* diff --git a/src/components/SparkPage/TailscalePanel.tsx b/src/components/SparkPage/TailscalePanel.tsx new file mode 100644 index 0000000..c46b346 --- /dev/null +++ b/src/components/SparkPage/TailscalePanel.tsx @@ -0,0 +1,102 @@ +import type { TailscaleMetrics } from "../../api/types"; +import { Panel } from "../ui/Panel"; +import { NetworkIcon } from "../ui/icons"; + +interface TailscalePanelProps { + tailscale: TailscaleMetrics | null; +} + +/** + * Tailnet presence for one Spark. + * + * The state worth shouting about is "reachable on the LAN but NOT on the + * tailnet" — every other panel here is fed over the LAN, so they all look + * healthy while the box is invisible from anywhere else. + */ +export function TailscalePanel({ tailscale }: TailscalePanelProps) { + const online = tailscale?.online ?? null; + const health = tailscale?.health ?? []; + const available = Boolean(tailscale?.available); + + // Unknown (null) is deliberately not treated as a failure — the first poll + // may not have landed, and tailscale may omit the field. + const offTailnet = available && online === false; + + const status = !available + ? { label: "unknown", cls: "text-muted" } + : online === true + ? { label: "online", cls: "text-accent" } + : online === false + ? { label: "OFF TAILNET", cls: "text-danger" } + : { label: "unknown", cls: "text-muted" }; + + return ( + }> +
+ Status + {status.label} + {tailscale?.backendState && ( + {tailscale.backendState} + )} +
+ + {/* Tailscale's own explanation of a bad state — the actionable part. */} + {health.length > 0 && ( +
+ {health.map((msg) => ( +

+ {msg} +

+ ))} +
+ )} + + {tailscale?.error && ( +

+ {tailscale.error} +

+ )} + +
+ {tailscale?.tailscaleIp && ( + + )} + {tailscale?.hostName && } + {tailscale?.relay && } + {tailscale?.keyExpired && } + {tailscale?.version && } + {!available && !tailscale?.error && ( +

Waiting for first poll…

+ )} +
+ + ); +} + +function Row({ + label, + value, + tabular, + danger, +}: { + label: string; + value: string; + tabular?: boolean; + danger?: boolean; +}) { + return ( +
+ {label} + + {value} + +
+ ); +}