From 2ece9f7ea242b152679d293f9f2edfb2e5405f9a Mon Sep 17 00:00:00 2001 From: Viktor Pelle Date: Tue, 21 Jul 2026 16:07:55 +0200 Subject: [PATCH] feat: add local metrics collection toggle --- docs/use-dkg/node-ui-metrics.md | 36 ++++++++++++ packages/cli/README.md | 6 ++ packages/cli/src/config.ts | 2 + packages/cli/src/daemon/lifecycle.ts | 28 ++++++--- .../cli/src/doctor/checks/config-sanity.ts | 25 ++++++++ packages/cli/src/metrics-collector-config.ts | 50 ++++++++++++++++ packages/cli/test/dkg-doctor.test.ts | 28 +++++++++ .../cli/test/metrics-collector-config.test.ts | 58 +++++++++++++++++++ packages/cli/vitest.unit.config.ts | 1 + tools/observability/RUNBOOK.md | 5 ++ tools/observability/node-config.example.json | 5 +- 11 files changed, 234 insertions(+), 10 deletions(-) create mode 100644 docs/use-dkg/node-ui-metrics.md create mode 100644 packages/cli/src/metrics-collector-config.ts create mode 100644 packages/cli/test/metrics-collector-config.test.ts diff --git a/docs/use-dkg/node-ui-metrics.md b/docs/use-dkg/node-ui-metrics.md new file mode 100644 index 0000000000..9695a2b71c --- /dev/null +++ b/docs/use-dkg/node-ui-metrics.md @@ -0,0 +1,36 @@ +# Node UI metrics collection + +The daemon writes local Node UI metric snapshots to `node-ui.db`. This local +collector is separate from OpenTelemetry metric export: + +- `telemetry.metrics.collectionEnabled` controls local SQLite snapshots and + the store queries used to populate them. +- `telemetry.metrics.enabled`, `endpoint`, and `exportIntervalMs` control OTLP + export. Disabling local collection does not disable OTLP export. + +Local collection remains enabled by default for backward compatibility. To +disable it, add: + +```json +{ + "telemetry": { + "metrics": { + "collectionEnabled": false + } + } +} +``` + +The environment override takes precedence over configuration: + +```bash +export DKG_METRICS_COLLECTION_ENABLED=0 +``` + +The environment value accepts `1`, `0`, `true`, or `false`. Invalid config or +environment values fail daemon startup instead of silently enabling the +collector. Restart the daemon after changing the setting. + +Disabling the collector stops new local snapshots and store scans. Existing +history remains in SQLite. When collection is enabled, the existing metrics +presence gate and `DKG_METRICS_ALWAYS_COLLECT=1` behavior are unchanged. diff --git a/packages/cli/README.md b/packages/cli/README.md index 3c45be7d89..5572d9c0e6 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -47,6 +47,12 @@ dkg ka publish-async notes -c my-project dkg query my-project -q "SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 10" ``` +## Node UI metrics collection + +Operators can disable local dashboard snapshots independently from +OpenTelemetry metric export; see the +[Node UI metrics operator guide](../../docs/use-dkg/node-ui-metrics.md). + ## Running a Core Node (relay operator) A Core Node is a publicly-reachable host that runs a libp2p circuit-relay v2 diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index b3a11d5e7f..0335011907 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -726,6 +726,8 @@ export interface DkgConfig { token?: string; /** PeriodicExportingMetricReader interval. Default 30000ms. */ exportIntervalMs?: number; + /** Enable local Node UI SQLite metric snapshots. Default true. */ + collectionEnabled?: boolean; }; }; /** Shared memory (workspace) data TTL in milliseconds. Default: 30 days (2592000000). Set to 0 to disable cleanup. */ diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index bff815a2e1..2935373700 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -145,6 +145,10 @@ import { } from '../config.js'; import { projectRuntimeEvmChainConfig } from '../runtime-chain-config.js'; import { resolveOtelSignals, resolveLogExporterMode, isUnknownLogExporter } from '../telemetry-config.js'; +import { + formatMetricsCollectorStartupLog, + resolveMetricsCollectorConfig, +} from '../metrics-collector-config.js'; import { createDaemonLogSink } from './log-sink.js'; import { startRpcUsageTelemetry } from './rpc-usage-log.js'; import { startDashboardLogVolumePruner } from './dashboard-log-volume-pruner.js'; @@ -1133,6 +1137,9 @@ export async function runDaemonInner( startedAt: number, ): Promise { configureKaPublishLifecycleDebugLogging(config); + // Resolve the local collector toggle before constructing daemon resources. + // This is independent from OTLP metrics export configuration. + const metricsCollectorConfig = resolveMetricsCollectorConfig(config); const logFile = logPath(); // Rotate before installing the in-process stdout/stderr tee so startup does // not race the truncation with fresh log appends. Existing logs survive @@ -2642,14 +2649,17 @@ export async function runDaemonInner( alwaysCollect: process.env.DKG_METRICS_ALWAYS_COLLECT === "1", }); - const metricsCollector = new MetricsCollector( - dashDb, - metricsSource, - dkgDir(), - () => metricsPresence.hasRecentConsumer(), - ); - metricsCollector.start(); - log("Metrics collector started (30s interval)"); + let metricsCollector: MetricsCollector | undefined; + if (metricsCollectorConfig.enabled) { + metricsCollector = new MetricsCollector( + dashDb, + metricsSource, + dkgDir(), + () => metricsPresence.hasRecentConsumer(), + ); + metricsCollector.start(); + } + log(formatMetricsCollectorStartupLog(metricsCollectorConfig)); // --- Telemetry: syslog log streaming (opt-in) --- const networkKey = network?.networkName?.toLowerCase().includes("testnet") @@ -3729,7 +3739,7 @@ export async function runDaemonInner( // log-derived request totals exact across process lifecycles. rpcUsageTelemetry.stop(); rateLimiter.destroy(); - metricsCollector.stop(); + metricsCollector?.stop(); // Stops log exporters AND flushes + shuts down the OTel SDK. await stopTelemetry(); natStatusWatcherStop?.(); diff --git a/packages/cli/src/doctor/checks/config-sanity.ts b/packages/cli/src/doctor/checks/config-sanity.ts index 86c2ba9165..bdce422bd2 100644 --- a/packages/cli/src/doctor/checks/config-sanity.ts +++ b/packages/cli/src/doctor/checks/config-sanity.ts @@ -18,7 +18,9 @@ import { join } from 'node:path'; import { AUTO_UPDATE_GIT_ONLY_FIELDS, parseAutoUpdateVerifyTagSignature, + type DkgConfig, } from '../../config.js'; +import { resolveMetricsCollectorConfig } from '../../metrics-collector-config.js'; import { formatAutoUpdateTagVerificationWarning, resolveAutoUpdateGitRefPlan, @@ -97,6 +99,29 @@ export async function runConfigSanityCheck(deps: DoctorDeps): Promise } } + const telemetry = parsed.telemetry; + if (telemetry && typeof telemetry === 'object' && !Array.isArray(telemetry)) { + const metrics = (telemetry as Record).metrics; + if (metrics && typeof metrics === 'object' && !Array.isArray(metrics)) { + const collectionEnabled = (metrics as Record).collectionEnabled; + if (collectionEnabled !== undefined) { + try { + resolveMetricsCollectorConfig({ + telemetry: { metrics: { collectionEnabled } }, + } as unknown as Pick, {}); + } catch (err) { + findings.push({ + check: 'config-sanity', + severity: 'error', + message: err instanceof Error ? err.message : String(err), + advisory: 'Set telemetry.metrics.collectionEnabled to true or false.', + subject: 'telemetry.metrics.collectionEnabled', + }); + } + } + } + } + // autoUpdate sub-config const autoUpdate = parsed.autoUpdate; if (autoUpdate && typeof autoUpdate === 'object' && !Array.isArray(autoUpdate)) { diff --git a/packages/cli/src/metrics-collector-config.ts b/packages/cli/src/metrics-collector-config.ts new file mode 100644 index 0000000000..75e531f0b9 --- /dev/null +++ b/packages/cli/src/metrics-collector-config.ts @@ -0,0 +1,50 @@ +import type { DkgConfig } from './config.js'; + +export interface ResolvedMetricsCollectorConfig { + enabled: boolean; +} + +function resolveEnabled(configValue: unknown, envValue: string | undefined): boolean { + if (envValue !== undefined) { + const normalized = envValue.trim().toLowerCase(); + if (normalized === '1' || normalized === 'true') return true; + if (normalized === '0' || normalized === 'false') return false; + throw new Error( + 'DKG_METRICS_COLLECTION_ENABLED must be one of 1, 0, true, or false ' + + `(received ${JSON.stringify(envValue)})`, + ); + } + if (configValue === undefined) return true; + if (typeof configValue !== 'boolean') { + throw new Error( + 'telemetry.metrics.collectionEnabled must be a boolean ' + + `(received ${JSON.stringify(configValue)})`, + ); + } + return configValue; +} + +/** + * Resolve local Node UI snapshot collection independently from OTLP export. + * The dedicated environment variable wins over config; invalid values fail + * startup rather than silently enabling collection. + */ +export function resolveMetricsCollectorConfig( + config: Pick | null | undefined, + env: Record = process.env, +): ResolvedMetricsCollectorConfig { + return { + enabled: resolveEnabled( + config?.telemetry?.metrics?.collectionEnabled, + env.DKG_METRICS_COLLECTION_ENABLED, + ), + }; +} + +export function formatMetricsCollectorStartupLog( + resolved: ResolvedMetricsCollectorConfig, +): string { + return resolved.enabled + ? 'Metrics collector started (30s interval)' + : 'Metrics collector disabled'; +} diff --git a/packages/cli/test/dkg-doctor.test.ts b/packages/cli/test/dkg-doctor.test.ts index d2e3908132..9715d10a9c 100644 --- a/packages/cli/test/dkg-doctor.test.ts +++ b/packages/cli/test/dkg-doctor.test.ts @@ -400,6 +400,34 @@ describe('config-sanity check (§4.7.2)', () => { expect(findings.find((f) => f.subject === 'apiPort' && f.severity === 'error')).toBeDefined(); }); + it('rejects invalid local metrics collector toggle types', async () => { + const deps = makeDeps({ + fs: { + '/test/.dkg/config.json': JSON.stringify({ + telemetry: { metrics: { collectionEnabled: 'yes' } }, + }), + }, + }); + const findings = await runConfigSanityCheck(deps); + expect(findings.find((f) => + f.subject === 'telemetry.metrics.collectionEnabled' && f.severity === 'error', + )).toBeDefined(); + }); + + it('accepts a boolean local metrics collector toggle', async () => { + const deps = makeDeps({ + fs: { + '/test/.dkg/config.json': JSON.stringify({ + telemetry: { metrics: { collectionEnabled: false } }, + }), + }, + }); + const findings = await runConfigSanityCheck(deps); + expect(findings.find((f) => + f.subject === 'telemetry.metrics.collectionEnabled', + )).toBeUndefined(); + }); + it('warns on deprecated autoUpdate fields set to non-empty values', async () => { const deps = makeDeps({ fs: { diff --git a/packages/cli/test/metrics-collector-config.test.ts b/packages/cli/test/metrics-collector-config.test.ts new file mode 100644 index 0000000000..8d6fae9988 --- /dev/null +++ b/packages/cli/test/metrics-collector-config.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from 'vitest'; +import { + formatMetricsCollectorStartupLog, + resolveMetricsCollectorConfig, +} from '../src/metrics-collector-config.js'; + +describe('resolveMetricsCollectorConfig', () => { + it('keeps local metric collection enabled by default', () => { + expect(resolveMetricsCollectorConfig(undefined, {})).toEqual({ enabled: true }); + }); + + it('disables local collection without changing OTLP metrics.enabled', () => { + expect(resolveMetricsCollectorConfig({ + telemetry: { metrics: { enabled: true, collectionEnabled: false } }, + }, {})).toEqual({ enabled: false }); + }); + + it.each([ + ['1', true], + ['true', true], + ['0', false], + ['false', false], + ])('accepts environment toggle %j', (value, enabled) => { + expect(resolveMetricsCollectorConfig(undefined, { + DKG_METRICS_COLLECTION_ENABLED: value, + })).toEqual({ enabled }); + }); + + it('gives the environment override precedence over config', () => { + expect(resolveMetricsCollectorConfig({ + telemetry: { metrics: { collectionEnabled: false } }, + }, { DKG_METRICS_COLLECTION_ENABLED: '1' })).toEqual({ enabled: true }); + }); + + it('rejects invalid environment values', () => { + expect(() => resolveMetricsCollectorConfig(undefined, { + DKG_METRICS_COLLECTION_ENABLED: 'yes', + })).toThrow(/DKG_METRICS_COLLECTION_ENABLED/); + }); + + it('rejects non-boolean config values', () => { + expect(() => resolveMetricsCollectorConfig({ + telemetry: { metrics: { collectionEnabled: 'yes' } }, + } as never, {})).toThrow(/telemetry\.metrics\.collectionEnabled/); + }); +}); + +describe('formatMetricsCollectorStartupLog', () => { + it('preserves the enabled startup log', () => { + expect(formatMetricsCollectorStartupLog({ enabled: true })) + .toBe('Metrics collector started (30s interval)'); + }); + + it('reports when local collection is disabled', () => { + expect(formatMetricsCollectorStartupLog({ enabled: false })) + .toBe('Metrics collector disabled'); + }); +}); diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts index 12fae80403..1b11d0d923 100644 --- a/packages/cli/vitest.unit.config.ts +++ b/packages/cli/vitest.unit.config.ts @@ -53,6 +53,7 @@ export default defineConfig({ 'test/resolve-standalone-install.test.ts', 'test/auto-update.test.ts', 'test/dkg-doctor.test.ts', + 'test/metrics-collector-config.test.ts', 'test/init.test.ts', 'test/nat-status.test.ts', 'test/core-prereq-check.test.ts', diff --git a/tools/observability/RUNBOOK.md b/tools/observability/RUNBOOK.md index ee9fc93f1a..cb2b92eed3 100644 --- a/tools/observability/RUNBOOK.md +++ b/tools/observability/RUNBOOK.md @@ -116,6 +116,11 @@ Restart the node. Local logging (SQLite + daemon.log) is unaffected; this only a **Logs vs traces/metrics (different transports, same endpoint host):** logs ship via a hand-rolled **OTLP/HTTP JSON** exporter (the OTel Logs SDK is still "Development"), while **traces and metrics use the stable OTel SDK** OTLP/protobuf exporters. The polaris setup today only has a **logs** backend (Loki via Alloy), so leave `telemetry.traces`/`telemetry.metrics` out (or set `enabled: false`) until a traces backend (Tempo) and metrics backend (Mimir/Prometheus) are provisioned — the `node-config.example.json` shows the full three-signal shape and `config.alloy` has the matching commented routing. +The local Node UI metrics collector is independent of OTLP export. Set +`telemetry.metrics.collectionEnabled` to `false` to disable local SQLite +snapshots and store scans without changing OTLP settings. See the +[Node UI metrics operator guide](../../docs/use-dkg/node-ui-metrics.md). + ## Step 4 — view in Grafana - **Per-node:** `https://polaris.xtrmstrngth.com/d/dkg-node-logs` → pick a **Node** → set the time range (top-right) → logs appear. `Level` and `Filter (regex)` narrow further; the bottom panel is volume-by-level. - **Fleet overview:** `https://polaris.xtrmstrngth.com/d/dkg-fleet-logs` → active-node count, log volume per node, errors per node, recent fleet-wide errors (filter by `Environment`). diff --git a/tools/observability/node-config.example.json b/tools/observability/node-config.example.json index 20d35408fa..8e79514193 100644 --- a/tools/observability/node-config.example.json +++ b/tools/observability/node-config.example.json @@ -3,6 +3,8 @@ "_comment_signals": "Three independent signals. LOGS use a hand-rolled OTLP/HTTP JSON exporter (the OTel Logs SDK is still 'Development'); set logs.exporter to 'otlp' — if you leave it unset on a hosted node it defaults to legacy syslog/Graylog, NOT OTLP. TRACES and METRICS use the stable OTel SDK exporters (OTLP/protobuf). IMPORTANT: the polaris backend today is LOGS-ONLY (Alloy → Loki; no Tempo for traces, no Prometheus/Mimir for metrics). So traces/metrics are shown here with enabled:false — they will NOT export until you (a) stand up a traces/metrics backend, (b) point config.alloy at it (see its FULL-SIGNAL section), and (c) flip enabled:true. Pointing them at the logs-only ingest host while enabled would just send spans/metrics into a void.", + "_comment_local_metrics": "metrics.enabled/exportIntervalMs control OTLP export. collectionEnabled independently controls local Node UI SQLite snapshots and store scans.", + "name": "testnet-core-01", "telemetry": { @@ -23,7 +25,8 @@ "enabled": false, "endpoint": "https://metrics-ingest.example.com/v1/metrics", "token": "", - "exportIntervalMs": 30000 + "exportIntervalMs": 30000, + "collectionEnabled": true } } }