diff --git a/.env.example b/.env.example index c893ff0..f80ec3f 100644 --- a/.env.example +++ b/.env.example @@ -12,6 +12,7 @@ # LOG_LEVEL=INFO # DEBUG|INFO|WARN|ERROR # HTTP_PORT=8080 # HTTP API + OTLP HTTP + WebSocket + UI # GRPC_PORT=4317 # OTLP gRPC +# PPROF_ADDR=127.0.0.1:6060 # net/http/pprof on a dedicated loopback listener; empty disables # ---- Database --------------------------------------------------------------- # DB_DRIVER=sqlite # sqlite|postgres|mysql|sqlserver @@ -48,6 +49,8 @@ # DB_MAX_IDLE_CONNS 10 → 1 # INGEST_PIPELINE_WORKERS 8 → 2 # INGEST_PIPELINE_QUEUE_SIZE 50000 → 10000 +# INGEST_PIPELINE_MAX_BYTES 536870912 → 134217728 (512MB → 128MB byte cap on queued batches; +# at the cap ingest gets 429 even for error batches) # METRIC_MAX_CARDINALITY 10000 → 3000 # STORE_MIN_SEVERITY "" → "WARN" (INFO/DEBUG still flow to GraphRAG/Drain, just not persisted) # SAMPLING_RATE 1.0 → 0.05 (errors and slow spans always kept) @@ -57,6 +60,14 @@ # Override by setting the env var explicitly. See # docs/superpowers/specs/2026-05-24-mcp-7tool-sqlite-survival-design.md for # per-default rationale and the SQLite PRAGMA stanza applied at startup. +# +# The PRAGMA stanza budget-scales its two memory knobs against the detected +# memory budget (cgroup v2 → cgroup v1 → /proc/meminfo): page cache = +# budget/32 clamped to [64 MB, 256 MB], mmap window = budget/8 clamped to +# [256 MB, 1 GB]. A 4 GB host gets 128 MB cache + 512 MB mmap; detection +# failure falls back to the 256 MB / 1 GB ceilings. Explicit overrides win: +# SQLITE_CACHE_SIZE_KB=131072 # Page cache in KB (> 0). Pure-Go driver: this is Go-heap memory. +# SQLITE_MMAP_SIZE_BYTES=536870912 # mmap window in bytes (>= 0; 0 disables mmap). # ---- Azure Entra (passwordless Postgres) ------------------------------------ # DB_AZURE_AUTH=false # Enables DefaultAzureCredential for Postgres. Requires strict TLS @@ -120,6 +131,10 @@ # ---- Retention -------------------------------------------------------------- # HOT_RETENTION_DAYS=7 # RetentionScheduler purge cutoff. Range 1..36500. Set explicitly in prod. +# RETENTION_FULL_VACUUM=false # Daily SQLite maintenance runs PRAGMA incremental_vacuum(10000) by default. +# # true restores the legacy daily full VACUUM (exclusive lock, 10-60 min on +# # multi-GB DBs — expect a 429 storm while it holds the writer lock). +# # On-demand full VACUUM: POST /api/admin/vacuum. Ignored on non-SQLite drivers. # ---- OTel Self-Instrumentation ---------------------------------------------- # OTEL_EXPORTER_OTLP_ENDPOINT= # When set, OtelContext exports its own spans to this OTLP gRPC endpoint. diff --git a/CHANGELOG.md b/CHANGELOG.md index 170037a..88212e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,97 @@ last published pre-release tag (`v0.2.0-beta.6`). ## [Unreleased] +### Fixed — production OOM restarts (memory-survival series) + +- **AnomalyStore memory blowup** (merged from `fix/sqlite-survival-hardening`): stable + per-(service,type) anomaly IDs replace one-node-per-10s-tick; the O(N²) + PRECEDED_BY edge mesh that heap profiling attributed 84% of live heap to is + gone (AnomalyStore 272 MB → 2.6 MB in soak). +- **GOMEMLIMIT safety net**: startup sets a soft limit (env honored, else 75% + of the cgroup/host budget via new `internal/membudget`). +- **Byte-bounded ingest queue**: `INGEST_PIPELINE_MAX_BYTES` (512 MB; 128 MB on + SQLite) — the item-count queue could hold GBs; at the cap even error/slow + batches get 429/`RESOURCE_EXHAUSTED` (reason `bytes_full`). +- **GraphRAG bounds**: per-tenant span cap (`GRAPHRAG_MAX_SPANS_PER_TENANT`, + 500k), SQLite trace TTL 1h→30m (`GRAPHRAG_TRACE_TTL`), idle-tenant store + eviction (`GRAPHRAG_TENANT_IDLE_TTL`, 24h; default tenant immune), + SignalStore metric nodes bounded (2000/tenant + 24h TTL), anomaly + correlation walk capped at 1000. +- **TSDB ring buffers**: keys are now tenant-scoped (`tenant|service|metric` + — fixes a cross-tenant data-isolation breach) and ring creation is capped at + `METRIC_MAX_CARDINALITY` (previously bypassed the cardinality check). +- **SQLite maintenance**: the automatic daily full `VACUUM` (10–60 min + exclusive lock → 429 storm → queue/RAM spiral) is replaced by + `PRAGMA optimize` + `incremental_vacuum(10000)`; restore via + `RETENTION_FULL_VACUUM=true` or `POST /api/admin/vacuum`. New DB files are + created `auto_vacuum=INCREMENTAL`. +- **Budget-scaled SQLite PRAGMAs**: page cache = budget/32 ∈ [64 MB, 256 MB], + mmap = budget/8 ∈ [256 MB, 1 GB] (4 GB host → 128 MB + 512 MB); overrides + `SQLITE_CACHE_SIZE_KB` / `SQLITE_MMAP_SIZE_BYTES`; fail-closed stanza kept. +- Cherry-picked security/correctness quick-wins: cross-tenant read escape via + `X-Tenant-ID` closed, token-bucket sampler math fixed (rate < 1.0 dropped + ~100% of healthy spans — SQLite's 0.05 default persisted almost nothing), + negative limit/offset clamps on `/api/logs` + `/api/traces`, MCP error + results no longer cached and `trace_graph` response capped. + +### Added — observability + +- `PPROF_ADDR` (default `127.0.0.1:6060`): `net/http/pprof` on a dedicated + loopback listener. +- Store census gauges: `otelcontext_graphrag_store_entities{entity}`, + `otelcontext_graphrag_store_edges{store}`, `otelcontext_tsdb_ring_series_active`, + `otelcontext_drain_templates_active`, `otelcontext_ingest_pipeline_queue_bytes`, + `otelcontext_graphrag_tenants_evicted_total`, `otelcontext_tsdb_ring_series_rejected_total`. + +### Changed — performance + +- `GetServiceMapMetrics`: node stats aggregate in SQL and the edge pass scans + a narrow projection — the per-row zstd decompression of span attributes is + gone (benchmark: 22 ms/5 MB vs 37 ms/15 MB on 5k spans; scales with row + count). Also fixes node `error_count` being permanently 0. The + `/api/metrics/service-map` response is cached 30s per tenant+window. +- GraphRAG DB rebuild is incremental via per-tenant high-water-mark (was a + full 1h-window re-read every 60s); the 10s anomaly scan skips tenants with + no new ingest events. +- HTTP serving: UI assets ship brotli/gzip precompressed with + `Cache-Control: immutable` + content hashing; `index.html` gets `no-cache` + + ETag; SPA fallback for client-side routes; GET `/api/*` responses are + gzipped; `/api/system/graph`, `/api/metrics/dashboard`, `/api/stats` honor + `If-None-Match` → 304 with a shared 10s render cache. + +### Changed — frontend rewrite complete (phases C3–C7) + +- New Triage home at `/`: anomaly strip (MCP `get_anomaly_timeline`) + ranked + worst-first service feed. The cytoscape physics graph is replaced by a + deterministic layered SVG flow map (own ~300-line layout; −500 KB raw of + graph-lib payload; map route now ~5 KB gz), keyboard-walkable, with a + blast-radius overlay (`/map?impact=svc`). +- Service Inspector (docked panel / bottom sheet) with Overview, Dependencies, + **Why** (`root_cause_analysis`) and **Impact** (`impact_analysis`) tabs — + the MCP triage verbs as human-clickable actions. Investigation Trail: + URL-encoded drill-down breadcrumbs (`?trail=`), shareable and reload-safe. +- `/traces` (virtualized table + real time-positioned SVG waterfall) and + `/logs` (live tail over the bounded WS ring buffer, virtualized, + severity pills, context/trace cross-links). +- ⌘K command palette (navigate / services / triage actions / utilities), + `g m/t/l/h` chords, `?` shortcut sheet. +- `@ossrandom/design-system`, the legacy Dashboard and the MCP Console are + removed (uplot/clsx/fontsource deps too); nav is exactly + Triage / Flow Map / Traces / Logs. Bundle budgets tightened to actuals+10%: + initial JS ≤118 KB gz, initial CSS ≤6 KB gz, every lazy chunk ≤10 KB gz. + +### Changed — frontend foundation (rewrite phases C1–C2) + +- New data layer: TanStack Query (visibility-aware polling — hidden tabs stop + hitting SQLite), single WebSocket manager with jittered backoff and a + bounded 5k log ring buffer, `apiFetch` with AbortSignal, percent-formatting + bugs fixed centrally. +- New responsive shell: System Pulse bar (health/err/p99/DB size, 3-state + live indicator), bottom tab bar (<768px) / icon rail / labeled rail + (≥1440px), Connect popover, token CSS (`tokens.css`) with dark/light themes, + reduced-motion + contrast support. Routing via wouter; deep links served by + the SPA fallback. CI-able bundle budget gate (`npm run check-budgets`). + ## [v0.2.0-beta.6] — 2026-06-05 This is the first release cut with the **source-only + build-on-tag** flow: the diff --git a/CLAUDE.md b/CLAUDE.md index a12f397..81d3af6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,13 +4,13 @@ OtelContext is a self-hosted OTLP observability platform. Single Go binary with embedded React frontend. - **Backend:** Go 1.25, native `net/http` (no frameworks), GORM ORM, gRPC + HTTP for OTLP ingestion -- **Frontend:** React 19 + TypeScript + `@ossrandom/design-system` + ECharts + ReactFlow +- **Frontend:** React 19 + TypeScript + TanStack Query/Virtual + wouter + Radix primitives + cmdk palette + hand-rolled token CSS (`ui/src/styles/tokens.css`) with CSS Modules; the flow map is own deterministic SVG layout code. No UI framework: `@ossrandom/design-system`, cytoscape, uplot all removed (rewrite completed 2026-06-12, phases C1–C7) - **Ports:** gRPC `:4317` (OTLP), HTTP `:8080` (API + HTTP OTLP + WebSocket + UI) ## Strict Rules - NO Express.js/Gin/Echo — use native Go `net/http` -- NO Tailwind CSS, NO Mantine — use `@ossrandom/design-system` exclusively for UI components and tokens. Raw CSS only for layout escape hatches (root height, scrollbar overrides, virtualised list containers); no auxiliary visual styling. +- NO Tailwind CSS, NO Mantine, NO component frameworks — UI styling is the hand-rolled token sheet (`ui/src/styles/tokens.css`) + per-component CSS Modules; Radix primitives (unstyled) only for the a11y-hard parts (dialog/tabs/tooltip/dropdown). Token values only — no raw hex outside tokens.css. - Single-service architecture (no microservices split) - All internal DBs must be **embedded** (no external processes) - Relational DB (SQLite/MySQL/PostgreSQL/MSSQL) is the **single source of truth** @@ -193,7 +193,7 @@ internal/ telemetry/ # Prometheus metrics + health (19 metrics) tsdb/ # Time series aggregator + ring buffer (lock-free Windows()) ui/ # Embedded React frontend -ui/ # React frontend (Vite + @ossrandom/design-system) +ui/ # React frontend (Vite + token CSS Modules, no UI framework) test/ # Microservice simulation (7 services) docs/ # Specifications and plans ``` @@ -216,9 +216,11 @@ Key settings in `internal/config/config.go`: - `MCP_MAX_CONCURRENT` (32), `MCP_CALL_TIMEOUT_MS` (30000), `MCP_CACHE_TTL_MS` (5000) — MCP HTTP streamable robustness. Counting semaphore gates concurrent `tools/call` (JSON-RPC `-32000` past the cap), per-call deadlines abort runaway handlers (JSON-RPC `-32001`), and a 5s TTL cache memoizes the cheap in-memory GraphRAG tools (`get_service_map`, `impact_analysis`, `root_cause_analysis`, `get_anomaly_timeline`, `get_service_health`). SSE GET sends a `: keep-alive\n\n` comment every 25s to keep the stream alive across reverse-proxy idle timeouts. Set any to 0 to disable. - `LOG_FTS_ENABLED` — when truthy (`true`/`yes`/`on`/`1`), provisions the SQLite FTS5 `logs_fts` virtual table + sync triggers at startup; when false, log-search uses a 24h-clamped LIKE fallback. **Defaults to `true` when `DB_DRIVER=sqlite`** (BM25 is dramatically faster than LIKE on the kept `search_logs` MCP tool) and `false` otherwise. Toggle off and reclaim the ~30% disk overhead via `POST /api/admin/drop_fts` (refused while the flag is on). The vectordb-backed semantic-search path was removed on 2026-05-24. - `DLQ_MAX_FILES` (1000), `DLQ_MAX_DISK_MB` (500), `DLQ_MAX_RETRIES` (10) -- `GRAPHRAG_WORKER_COUNT` (16), `GRAPHRAG_EVENT_QUEUE_SIZE` (100000) — sized for 100–200 services; raise further if `otelcontext_graphrag_events_dropped_total` climbs +- `GRAPHRAG_WORKER_COUNT` (16), `GRAPHRAG_EVENT_QUEUE_SIZE` (100000; **10000 on SQLite**) — sized for 100–200 services; raise further if `otelcontext_graphrag_events_dropped_total` climbs +- `GRAPHRAG_TRACE_TTL` (`1h`; **`30m` on SQLite**), `GRAPHRAG_MAX_SPANS_PER_TENANT` (500000), `GRAPHRAG_TENANT_IDLE_TTL` (`24h`) — in-memory GraphRAG memory bounds. Spans past the per-tenant cap are skipped from the graph only (DB unaffected; metered as `otelcontext_graphrag_events_dropped_total{signal="span_capacity"}`); tenant store slices idle past the TTL are evicted (default tenant immune, self-healing via the 60s rebuild). SignalStore metrics are bounded to 2000/tenant + 24h TTL (constants). +- `PPROF_ADDR` (`127.0.0.1:6060`) — `net/http/pprof` on a dedicated loopback listener (never the public mux); empty disables. Startup also sets a soft `GOMEMLIMIT` (honors the env var, else 75% of the cgroup/host budget via `internal/membudget`). - `INGEST_MIN_SEVERITY` (`INFO`), `STORE_MIN_SEVERITY` (`""` = same as ingest; **defaults to `"WARN"` when `DB_DRIVER=sqlite`**) — two-tier log severity gate. The ingest gate runs at the OTLP receiver and **drops the log entirely** below the threshold (no in-memory enrichment either). The store gate runs at the persist boundary inside the async pipeline (`internal/ingest/pipeline.go:process`) and **only skips the DB row write** — the log still flows through `LogCallback` so GraphRAG Drain template mining and span/trace correlation see it. Use case: `INGEST_MIN_SEVERITY=DEBUG STORE_MIN_SEVERITY=WARN` keeps SQLite small while letting in-memory anomaly detection benefit from the verbose stream. Setting `STORE_MIN_SEVERITY` ≤ `INGEST_MIN_SEVERITY` is a no-op (logged as a warning at startup). Drops surface via `Pipeline.Stats().StoreFiltered`. -- `INGEST_ASYNC_ENABLED` (true), `INGEST_PIPELINE_QUEUE_SIZE` (50000), `INGEST_PIPELINE_WORKERS` (8) — async ingest pipeline (`internal/ingest/pipeline.go`). Hybrid backpressure: <90% accept all, 90–100% drop healthy batches (errors/slow always pass), 100% return gRPC `RESOURCE_EXHAUSTED`. Set `INGEST_ASYNC_ENABLED=false` to revert to synchronous DB writes inside `Export()`. Drops surface as `otelcontext_ingest_pipeline_dropped_total{signal,reason}`. +- `INGEST_ASYNC_ENABLED` (true), `INGEST_PIPELINE_QUEUE_SIZE` (50000), `INGEST_PIPELINE_WORKERS` (8), `INGEST_PIPELINE_MAX_BYTES` (536870912 = 512 MB; **128 MB on SQLite**) — async ingest pipeline (`internal/ingest/pipeline.go`). Hybrid backpressure: <90% accept all, 90–100% drop healthy batches (errors/slow always pass), 100% return gRPC `RESOURCE_EXHAUSTED`. The byte cap bounds queue memory regardless of item count — at the cap even priority batches get `RESOURCE_EXHAUSTED`/429 (a 429 is recoverable, an OOM kill is not); watch `otelcontext_ingest_pipeline_queue_bytes` and reason `bytes_full`. Set `INGEST_ASYNC_ENABLED=false` to revert to synchronous DB writes inside `Export()`. Drops surface as `otelcontext_ingest_pipeline_dropped_total{signal,reason}`. - `GRPC_MAX_RECV_MB` (16), `GRPC_MAX_CONCURRENT_STREAMS` (1000) — OTLP gRPC server caps, validated to 1..256 and 1..1_000_000 - `RETENTION_BATCH_SIZE` (50000), `RETENTION_BATCH_SLEEP_MS` (1) — purge pacing; raise the sleep on busy production DBs - `DB_POSTGRES_PARTITIONING` (`""`), `DB_PARTITION_LOOKAHEAD_DAYS` (3) — opt-in Postgres declarative range partitioning of the `logs` table by day. When `daily`, `logs` is provisioned as a partitioned parent (greenfield only — refuses to start if `logs` already exists unpartitioned), the `PartitionScheduler` maintains lookahead partitions and drops expired ones via `DROP TABLE`, and `RetentionScheduler` skips the row-level DELETE for `logs`. Watch `otelcontext_partitions_dropped_total` and `otelcontext_partitions_active`. @@ -234,13 +236,16 @@ So a 100+ service deployment on SQLite survives without OOM, `config.Load()` ove | `DB_MAX_IDLE_CONNS` | 1 | 10 | Match open conns. | | `INGEST_PIPELINE_WORKERS` | 2 | 8 | Workers all serialise through the SQLite writer lock; 2 is enough to keep the queue non-empty. | | `INGEST_PIPELINE_QUEUE_SIZE` | 10000 | 50000 | Lower heap watermark; backpressure kicks in earlier so OTLP clients back off. | +| `INGEST_PIPELINE_MAX_BYTES` | 128 MB | 512 MB | Item count alone cannot bound queue memory; one batch may carry MBs of spans/logs. | +| `GRAPHRAG_EVENT_QUEUE_SIZE` | 10000 | 100000 | Each queued event embeds a Span/Log by value (~0.5–2 KB); buffer less, drop sooner (metered). | +| `GRAPHRAG_TRACE_TTL` | 30m | 1h | The in-memory span window is the largest legitimate GraphRAG consumer; anomaly/investigation lookbacks are ≤5min. | | `METRIC_MAX_CARDINALITY` | 3000 | 10000 | Bound the in-memory TSDB series map. | | `STORE_MIN_SEVERITY` | `"WARN"` | `""` | Skip INFO/DEBUG persists; in-memory GraphRAG/Drain still sees them. | | `SAMPLING_RATE` | 0.05 | 1.0 | Errors and slow spans are always kept by `SAMPLING_ALWAYS_ON_ERRORS`. | | `GRPC_MAX_CONCURRENT_STREAMS` | 240 | 1000 | ~2 streams per service at 120 services with headroom. | | `LOG_FTS_ENABLED` | `true` | n/a | FTS5 BM25 is dramatically faster than LIKE on the kept `search_logs` path. | -Also at SQLite startup, `internal/storage/factory.go` applies a fail-closed PRAGMA stanza: `journal_mode=WAL`, `synchronous=NORMAL`, `cache_size=-262144` (256 MB page cache), `temp_store=MEMORY`, `mmap_size=1073741824` (1 GB mmap), `wal_autocheckpoint=10000`, `journal_size_limit=67108864` (64 MB WAL cap), `busy_timeout=5000`. Any PRAGMA failure aborts startup with a wrapped error — these are not optional. See `docs/superpowers/specs/2026-05-24-mcp-7tool-sqlite-survival-design.md` for per-default reasoning. +Also at SQLite startup, `internal/storage/factory.go` applies a fail-closed PRAGMA stanza: `journal_mode=WAL`, `synchronous=NORMAL`, `temp_store=MEMORY`, `wal_autocheckpoint=10000`, `journal_size_limit=67108864` (64 MB WAL cap), `busy_timeout=5000`, plus **budget-scaled** memory knobs: page cache = budget/32 clamped to [64 MB, 256 MB] and mmap = budget/8 clamped to [256 MB, 1 GB], where the budget comes from `internal/membudget` (cgroup v2 → v1 → /proc/meminfo; a 4 GB host gets 128 MB cache + 512 MB mmap, detection failure falls back to the 256 MB/1 GB ceilings). Operators override with `SQLITE_CACHE_SIZE_KB` / `SQLITE_MMAP_SIZE_BYTES`. With the pure-Go driver the page cache is Go-heap memory and competes with `GOMEMLIMIT`. `PRAGMA auto_vacuum=INCREMENTAL` is attempted best-effort **before** the WAL switch (the WAL header freezes the stored mode; only affects newly created DB files). Any pragma failure in the fail-closed stanza aborts startup with a wrapped error — these are not optional. See `docs/superpowers/specs/2026-05-24-mcp-7tool-sqlite-survival-design.md` for per-default reasoning. ### Authentication @@ -250,7 +255,7 @@ Also at SQLite startup, `internal/storage/factory.go` applies a fail-closed PRAG ### Retention & Maintenance -The `RetentionScheduler` in `internal/storage/` runs an hourly batched purge of data older than `HOT_RETENTION_DAYS` via `PurgeLogsBatched`, `PurgeTracesBatched`, and `PurgeMetricBucketsBatched`, plus a daily `VACUUM`/`ANALYZE` pass to reclaim space and refresh planner statistics. Purge is **cross-tenant** — it scopes by age, not `tenant_id`. Valid `HOT_RETENTION_DAYS` is clamped to the range 1..36500. +The `RetentionScheduler` in `internal/storage/` runs an hourly batched purge of data older than `HOT_RETENTION_DAYS` via `PurgeLogsBatched`, `PurgeTracesBatched`, and `PurgeMetricBucketsBatched`, plus a daily maintenance pass: `PRAGMA optimize` and `PRAGMA incremental_vacuum(10000)` on SQLite (the historical full `VACUUM` held an exclusive whole-DB lock for 10–60 min on multi-GB files, starving ingest into a 429 storm; restore it with `RETENTION_FULL_VACUUM=true` or run `POST /api/admin/vacuum` on demand — note pre-existing DB files keep their `auto_vacuum` mode until a manual full VACUUM rewrites them, so `incremental_vacuum` no-ops harmlessly there), `ANALYZE`-equivalent maintenance on other drivers as before. Purge is **cross-tenant** — it scopes by age, not `tenant_id`. Valid `HOT_RETENTION_DAYS` is clamped to the range 1..36500. Failure-mode gauges (prefix `OtelContext_`): - `retention_consecutive_failures` — reset to 0 on success; alert when > 3 diff --git a/internal/api/admin_handlers.go b/internal/api/admin_handlers.go index a4f4f67..b61aa7f 100644 --- a/internal/api/admin_handlers.go +++ b/internal/api/admin_handlers.go @@ -10,18 +10,34 @@ import ( "time" "github.com/RandomCodeSpace/otelcontext/internal/httpconst" + "github.com/RandomCodeSpace/otelcontext/internal/storage" ) -// handleGetStats handles GET /api/stats +// handleGetStats handles GET /api/stats. +// The rendered JSON is cached for 10s per tenant with an ETag — same +// pattern as handleGetSystemGraph. The UI footer polls this endpoint and +// the COUNT(*) scans behind it are not free on a multi-GB SQLite file. func (s *Server) handleGetStats(w http.ResponseWriter, r *http.Request) { + cacheKey := "db_stats:" + storage.TenantFromContext(r.Context()) + if cached, ok := s.cache.Get(cacheKey); ok { + cached.(*cachedJSON).write(w, r, "HIT") + return + } + stats, err := s.repo.GetStats(r.Context()) if err != nil { slog.Error("Failed to get DB stats", "error", err) http.Error(w, err.Error(), http.StatusInternalServerError) return } - w.Header().Set(httpconst.HeaderContentType, httpconst.ContentTypeJSON) - _ = json.NewEncoder(w).Encode(stats) + + cj, err := newCachedJSON(stats) + if err != nil { + http.Error(w, "failed to encode DB stats", http.StatusInternalServerError) + return + } + s.cache.Set(cacheKey, cj, hotPollCacheTTL) + cj.write(w, r, "MISS") } // handlePurge handles DELETE /api/admin/purge diff --git a/internal/api/compress.go b/internal/api/compress.go new file mode 100644 index 0000000..c41556a --- /dev/null +++ b/internal/api/compress.go @@ -0,0 +1,142 @@ +package api + +import ( + "compress/gzip" + "net/http" + "strings" + "sync" + + "github.com/RandomCodeSpace/otelcontext/internal/httpconst" +) + +// gzipWriterPool recycles gzip writers across requests — a gzip.Writer holds +// ~hundreds of KB of compression state, far too much to allocate per call. +var gzipWriterPool = sync.Pool{ + New: func() any { return gzip.NewWriter(nil) }, +} + +// GzipMiddleware compresses GET /api/* responses for clients that accept +// gzip. The 120-service system-graph JSON shrinks 5-8× on the wire. +// +// Everything outside /api/* passes through untouched, which keeps the +// streaming surfaces safe by construction: WebSocket upgrades (/ws*) still +// see an http.Hijacker, OTLP ingest (/v1/*) and Prometheus scrapes +// (/metrics*) are write paths/exposition formats with their own encodings, +// and MCP SSE must flush uncompressed frames. Those prefixes — plus the +// configured MCP path — are also excluded explicitly in case an operator +// ever nests one under /api/. +// +// Wire this innermost (directly around the mux): only handler output is +// compressed; error responses written by outer middleware (auth, rate +// limit) stay identity-encoded. +func GzipMiddleware(mcpPath string) func(http.Handler) http.Handler { + if mcpPath == "" { + mcpPath = "/mcp" + } + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !gzipEligible(r, mcpPath) { + next.ServeHTTP(w, r) + return + } + // Vary on every eligible path — even identity responses — so a + // shared cache never hands a gzipped body to an identity-only + // client or vice versa. + w.Header().Add("Vary", "Accept-Encoding") + if !httpconst.AcceptsEncoding(r, "gzip") { + next.ServeHTTP(w, r) + return + } + gw := &gzipResponseWriter{ResponseWriter: w} + defer gw.close() + next.ServeHTTP(gw, r) + }) + } +} + +// gzipEligible reports whether the request may have its response gzipped. +// The /api/ prefix gate already excludes /ws*, /v1/*, and /metrics* — those +// prefixes cannot co-exist with /api/ — so only the MCP path (which an +// operator could configure under /api/) needs an explicit check. +func gzipEligible(r *http.Request, mcpPath string) bool { + if r.Method != http.MethodGet { + return false + } + p := r.URL.Path + if !strings.HasPrefix(p, "/api/") { + return false + } + if p == mcpPath || strings.HasPrefix(p, mcpPath+"/") { + return false + } + return true +} + +// gzipResponseWriter lazily engages compression on the first write: +// WriteHeader decides — 204/304 and responses that already carry a +// Content-Encoding pass through untouched, everything else gets +// Content-Encoding: gzip with Content-Length dropped (the compressed size +// isn't known up front; net/http falls back to chunked transfer). +type gzipResponseWriter struct { + http.ResponseWriter + gz *gzip.Writer + wroteHeader bool +} + +func (w *gzipResponseWriter) WriteHeader(code int) { + if w.wroteHeader { + return + } + w.wroteHeader = true + h := w.Header() + if code != http.StatusNoContent && code != http.StatusNotModified && h.Get(httpconst.HeaderContentEncoding) == "" { + h.Set(httpconst.HeaderContentEncoding, "gzip") + h.Del("Content-Length") + gz := gzipWriterPool.Get().(*gzip.Writer) + gz.Reset(w.ResponseWriter) + w.gz = gz + } + w.ResponseWriter.WriteHeader(code) +} + +func (w *gzipResponseWriter) Write(b []byte) (int, error) { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + if w.gz != nil { + return w.gz.Write(b) + } + return w.ResponseWriter.Write(b) +} + +// Flush completes the current gzip block before flushing downstream so +// incremental delivery keeps working for handlers that stream. +func (w *gzipResponseWriter) Flush() { + if !w.wroteHeader { + w.WriteHeader(http.StatusOK) + } + if w.gz != nil { + _ = w.gz.Flush() + } + if f, ok := w.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +// Unwrap exposes the underlying writer to http.ResponseController for +// per-request deadline control. ResponseController still finds our Flush +// first, so the gzip buffer is never bypassed. +func (w *gzipResponseWriter) Unwrap() http.ResponseWriter { + return w.ResponseWriter +} + +// close finalises the gzip stream (writing the trailer) and returns the +// writer to the pool. No-op when compression never engaged. +func (w *gzipResponseWriter) close() { + if w.gz == nil { + return + } + _ = w.gz.Close() + gzipWriterPool.Put(w.gz) + w.gz = nil +} diff --git a/internal/api/compress_test.go b/internal/api/compress_test.go new file mode 100644 index 0000000..ddfd29c --- /dev/null +++ b/internal/api/compress_test.go @@ -0,0 +1,274 @@ +package api + +import ( + "compress/gzip" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" +) + +// jsonEcho is a handler that writes the given payload as JSON in several +// chunks (exercising the multi-Write path through the gzip writer). +func jsonEcho(payload string) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + half := len(payload) / 2 + _, _ = w.Write([]byte(payload[:half])) + _, _ = w.Write([]byte(payload[half:])) + }) +} + +func gzipGet(t *testing.T, h http.Handler, path string, acceptGzip bool) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + if acceptGzip { + req.Header.Set("Accept-Encoding", "gzip") + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec +} + +func gunzip(t *testing.T, body io.Reader) string { + t.Helper() + zr, err := gzip.NewReader(body) + if err != nil { + t.Fatalf("gzip.NewReader: %v", err) + } + defer zr.Close() + b, err := io.ReadAll(zr) + if err != nil { + t.Fatalf("gunzip read: %v", err) + } + return string(b) +} + +func TestGzipMiddleware_CompressesAPIGet(t *testing.T) { + payload := strings.Repeat(`{"service":"checkout","status":"healthy"},`, 100) + h := GzipMiddleware("/mcp")(jsonEcho(payload)) + + rec := gzipGet(t, h, "/api/system/graph", true) + + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d", rec.Code) + } + if got := rec.Header().Get("Content-Encoding"); got != "gzip" { + t.Fatalf("Content-Encoding = %q, want gzip", got) + } + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Vary = %q", got) + } + if cl := rec.Header().Get("Content-Length"); cl != "" { + t.Errorf("Content-Length should be dropped, got %q", cl) + } + if rec.Body.Len() >= len(payload) { + t.Errorf("body not actually compressed: %d >= %d", rec.Body.Len(), len(payload)) + } + if got := gunzip(t, rec.Body); got != payload { + t.Errorf("decompressed body mismatch (len %d vs %d)", len(got), len(payload)) + } +} + +func TestGzipMiddleware_SkipsWithoutAcceptEncoding(t *testing.T) { + payload := strings.Repeat("x", 4096) + h := GzipMiddleware("/mcp")(jsonEcho(payload)) + + rec := gzipGet(t, h, "/api/logs", false) + + if got := rec.Header().Get("Content-Encoding"); got != "" { + t.Fatalf("Content-Encoding = %q, want empty", got) + } + if rec.Body.String() != payload { + t.Errorf("identity body mangled") + } + // Vary still set: caches must key on Accept-Encoding either way. + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Vary = %q", got) + } +} + +func TestGzipMiddleware_SkipsNonAPIPaths(t *testing.T) { + payload := strings.Repeat("y", 2048) + for _, path := range []string{"/ws", "/ws/events", "/v1/traces", "/metrics/prometheus", "/mcp", "/mcp/session", "/"} { + h := GzipMiddleware("/mcp")(jsonEcho(payload)) + rec := gzipGet(t, h, path, true) + if got := rec.Header().Get("Content-Encoding"); got != "" { + t.Errorf("%s: Content-Encoding = %q, want empty", path, got) + } + if rec.Body.String() != payload { + t.Errorf("%s: body mangled", path) + } + } +} + +func TestGzipMiddleware_SkipsNonGET(t *testing.T) { + h := GzipMiddleware("/mcp")(jsonEcho(strings.Repeat("z", 2048))) + req := httptest.NewRequest(http.MethodPost, "/api/admin/vacuum", nil) + req.Header.Set("Accept-Encoding", "gzip") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if got := rec.Header().Get("Content-Encoding"); got != "" { + t.Errorf("POST: Content-Encoding = %q, want empty", got) + } +} + +func TestGzipMiddleware_RespectsExistingContentEncoding(t *testing.T) { + h := GzipMiddleware("/mcp")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Encoding", "br") + _, _ = w.Write([]byte("pre-compressed-bytes")) + })) + rec := gzipGet(t, h, "/api/whatever", true) + if got := rec.Header().Get("Content-Encoding"); got != "br" { + t.Fatalf("Content-Encoding = %q, want br untouched", got) + } + if rec.Body.String() != "pre-compressed-bytes" { + t.Errorf("double-compressed an already-encoded response") + } +} + +func TestGzipMiddleware_No304Body(t *testing.T) { + h := GzipMiddleware("/mcp")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotModified) + })) + rec := gzipGet(t, h, "/api/stats", true) + if rec.Code != http.StatusNotModified { + t.Fatalf("want 304, got %d", rec.Code) + } + if got := rec.Header().Get("Content-Encoding"); got != "" { + t.Errorf("304 must not carry Content-Encoding, got %q", got) + } + if rec.Body.Len() != 0 { + t.Errorf("304 body should be empty, got %d bytes", rec.Body.Len()) + } +} + +func TestGzipMiddleware_EmptyHandlerBody(t *testing.T) { + // Handler that never writes: net/http sends an implicit 200 with no + // body; the wrapper must not inject a stray empty gzip stream header. + h := GzipMiddleware("/mcp")(http.HandlerFunc(func(_ http.ResponseWriter, _ *http.Request) {})) + rec := gzipGet(t, h, "/api/empty", true) + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d", rec.Code) + } + if rec.Body.Len() != 0 { + t.Errorf("empty handler should produce empty body, got %d bytes", rec.Body.Len()) + } +} + +func TestGzipMiddleware_FlushSupported(t *testing.T) { + h := GzipMiddleware("/mcp")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("first")) + if f, ok := w.(http.Flusher); ok { + f.Flush() + } + _, _ = w.Write([]byte("second")) + })) + rec := gzipGet(t, h, "/api/stream", true) + if !rec.Flushed { + t.Errorf("Flush did not propagate to the underlying writer") + } + if got := gunzip(t, rec.Body); got != "firstsecond" { + t.Errorf("flushed body = %q", got) + } +} + +// TestGzipMiddleware_MCPPathExcluded covers the one skip that the /api/ +// prefix gate cannot subsume: an operator nesting the MCP endpoint under +// /api/. SSE frames must never be buffered inside a gzip stream. +func TestGzipMiddleware_MCPPathExcluded(t *testing.T) { + payload := strings.Repeat("event: ping\n\n", 200) + h := GzipMiddleware("/api/mcp")(jsonEcho(payload)) + for _, path := range []string{"/api/mcp", "/api/mcp/session"} { + rec := gzipGet(t, h, path, true) + if got := rec.Header().Get("Content-Encoding"); got != "" { + t.Errorf("%s: Content-Encoding = %q, want empty (MCP/SSE excluded)", path, got) + } + } + // Sibling API paths still compress. + rec := gzipGet(t, h, "/api/stats", true) + if got := rec.Header().Get("Content-Encoding"); got != "gzip" { + t.Errorf("/api/stats: Content-Encoding = %q, want gzip", got) + } +} + +// TestGzipMiddleware_EmptyMCPPathDefaults covers the "" → "/mcp" fallback. +func TestGzipMiddleware_EmptyMCPPathDefaults(t *testing.T) { + h := GzipMiddleware("")(jsonEcho(strings.Repeat("a", 2048))) + rec := gzipGet(t, h, "/api/x", true) + if got := rec.Header().Get("Content-Encoding"); got != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", got) + } +} + +func TestGzipMiddleware_DoubleWriteHeaderIgnored(t *testing.T) { + h := GzipMiddleware("/mcp")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusTeapot) + w.WriteHeader(http.StatusOK) // must be a no-op, like net/http + _, _ = w.Write([]byte("tea")) + })) + rec := gzipGet(t, h, "/api/tea", true) + if rec.Code != http.StatusTeapot { + t.Fatalf("want 418 (first WriteHeader wins), got %d", rec.Code) + } + if got := gunzip(t, rec.Body); got != "tea" { + t.Errorf("body = %q", got) + } +} + +func TestGzipMiddleware_FlushBeforeWrite(t *testing.T) { + h := GzipMiddleware("/mcp")(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.(http.Flusher).Flush() // implicit 200 + gzip engagement + _, _ = w.Write([]byte("after-flush")) + })) + rec := gzipGet(t, h, "/api/early-flush", true) + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d", rec.Code) + } + if got := gunzip(t, rec.Body); got != "after-flush" { + t.Errorf("body = %q", got) + } +} + +func TestGzipResponseWriter_Unwrap(t *testing.T) { + rec := httptest.NewRecorder() + gw := &gzipResponseWriter{ResponseWriter: rec} + if gw.Unwrap() != rec { + t.Errorf("Unwrap should return the wrapped writer") + } +} + +// TestGzipMiddleware_PoolReuseConcurrent hammers the middleware from many +// goroutines to prove the sync.Pool recycling is race-free and never +// cross-wires response bodies. +func TestGzipMiddleware_PoolReuseConcurrent(t *testing.T) { + payload := strings.Repeat(`{"n":1},`, 512) + h := GzipMiddleware("/mcp")(jsonEcho(payload)) + var wg sync.WaitGroup + for range 32 { + wg.Add(1) + go func() { + defer wg.Done() + for range 20 { + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/x", nil) + req.Header.Set("Accept-Encoding", "gzip") + h.ServeHTTP(rec, req) + zr, err := gzip.NewReader(rec.Body) + if err != nil { + t.Errorf("gzip.NewReader: %v", err) + return + } + b, err := io.ReadAll(zr) + _ = zr.Close() + if err != nil || string(b) != payload { + t.Errorf("corrupted body under concurrency (err=%v, len=%d)", err, len(b)) + return + } + } + }() + } + wg.Wait() +} diff --git a/internal/api/etag.go b/internal/api/etag.go new file mode 100644 index 0000000..be0508e --- /dev/null +++ b/internal/api/etag.go @@ -0,0 +1,53 @@ +package api + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "net/http" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/httpconst" +) + +// hotPollCacheTTL is how long the rendered payloads of the hot polled +// endpoints (/api/system/graph, /api/metrics/dashboard, /api/stats) stay +// cached. 10s matches the UI poll cadence — steady-state polling becomes a +// map lookup plus an ETag compare instead of a SQLite query + JSON encode. +const hotPollCacheTTL = 10 * time.Second + +// maxCacheKeyQueryLen bounds cache-key cardinality for endpoints whose key +// includes the raw query string: anything longer skips the cache rather +// than letting a hostile client grow the map. +const maxCacheKeyQueryLen = 256 + +// cachedJSON is a fully rendered JSON response body plus its strong ETag. +// Marshalling and hashing happen once per cache fill; clients that echo +// If-None-Match then get a 304 with no body at all. +type cachedJSON struct { + body []byte + etag string +} + +func newCachedJSON(v any) (*cachedJSON, error) { + b, err := json.Marshal(v) + if err != nil { + return nil, err + } + sum := sha256.Sum256(b) + return &cachedJSON{body: b, etag: `"` + hex.EncodeToString(sum[:8]) + `"`}, nil +} + +// write emits the cached payload, honouring If-None-Match with a 304. +// xCache is the cache-disposition header value ("HIT" or "MISS"). +func (c *cachedJSON) write(w http.ResponseWriter, r *http.Request, xCache string) { + h := w.Header() + h.Set("ETag", c.etag) + h.Set("X-Cache", xCache) + if httpconst.ETagMatch(r.Header.Get("If-None-Match"), c.etag) { + w.WriteHeader(http.StatusNotModified) + return + } + h.Set(httpconst.HeaderContentType, httpconst.ContentTypeJSON) + _, _ = w.Write(c.body) //nolint:gosec // G705 false positive: body is json.Marshal output of server-built values, served as application/json +} diff --git a/internal/api/etag_cache_test.go b/internal/api/etag_cache_test.go new file mode 100644 index 0000000..260e3cd --- /dev/null +++ b/internal/api/etag_cache_test.go @@ -0,0 +1,226 @@ +package api + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/RandomCodeSpace/otelcontext/internal/cache" + "github.com/RandomCodeSpace/otelcontext/internal/storage" +) + +// newCachedTestServer builds a Server backed by a fresh in-memory SQLite +// repo with the TTL cache wired — the shape the cached hot-poll handlers +// (system graph, dashboard stats, DB stats) need. +func newCachedTestServer(t *testing.T) *Server { + t.Helper() + db, err := storage.NewDatabase("sqlite", ":memory:") + if err != nil { + t.Fatalf("NewDatabase: %v", err) + } + if err := storage.AutoMigrateModels(db, "sqlite"); err != nil { + t.Fatalf("AutoMigrateModels: %v", err) + } + repo := storage.NewRepositoryFromDB(db, "sqlite") + c := cache.New() + t.Cleanup(func() { + c.Stop() + _ = repo.Close() + }) + return &Server{repo: repo, cache: c} +} + +// pollCacheFlow drives the shared MISS → HIT → 304 assertion sequence for a +// cached+ETagged endpoint. +func pollCacheFlow(t *testing.T, handler http.HandlerFunc, path string) { + t.Helper() + + do := func(inm string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, path, nil) + if inm != "" { + req.Header.Set("If-None-Match", inm) + } + rec := httptest.NewRecorder() + handler(rec, req) + return rec + } + + first := do("") + if first.Code != http.StatusOK { + t.Fatalf("first: want 200, got %d body=%q", first.Code, first.Body.String()) + } + if got := first.Header().Get("X-Cache"); got != "MISS" { + t.Errorf("first: X-Cache = %q, want MISS", got) + } + etag := first.Header().Get("ETag") + if etag == "" || !strings.HasPrefix(etag, `"`) { + t.Fatalf("first: ETag = %q, want quoted non-empty", etag) + } + if ct := first.Header().Get("Content-Type"); !strings.Contains(ct, "application/json") { + t.Errorf("first: Content-Type = %q", ct) + } + + second := do("") + if got := second.Header().Get("X-Cache"); got != "HIT" { + t.Errorf("second: X-Cache = %q, want HIT (within 10s TTL)", got) + } + if got := second.Header().Get("ETag"); got != etag { + t.Errorf("second: ETag = %q, want %q", got, etag) + } + if second.Body.String() != first.Body.String() { + t.Errorf("second: cached body differs from first") + } + + third := do(etag) + if third.Code != http.StatusNotModified { + t.Fatalf("third: want 304 with If-None-Match, got %d", third.Code) + } + if third.Body.Len() != 0 { + t.Errorf("third: 304 body should be empty, got %q", third.Body.String()) + } + if got := third.Header().Get("ETag"); got != etag { + t.Errorf("third: 304 ETag = %q, want %q", got, etag) + } +} + +func TestHandleGetStats_CacheAndETag(t *testing.T) { + s := newCachedTestServer(t) + pollCacheFlow(t, s.handleGetStats, "/api/stats") +} + +func TestHandleGetDashboardStats_CacheAndETag(t *testing.T) { + s := newCachedTestServer(t) + pollCacheFlow(t, s.handleGetDashboardStats, "/api/metrics/dashboard") +} + +func TestHandleGetSystemGraph_CacheAndETag(t *testing.T) { + // graph and graphRAG are nil → the handler exercises the DB fallback + // path against the empty repo, which still yields a valid response. + s := newCachedTestServer(t) + pollCacheFlow(t, s.handleGetSystemGraph, "/api/system/graph") +} + +// TestHandleGetDashboardStats_QueryScopedKey proves distinct query strings +// never share a cache entry. +func TestHandleGetDashboardStats_QueryScopedKey(t *testing.T) { + s := newCachedTestServer(t) + + do := func(path string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + s.handleGetDashboardStats(rec, req) + return rec + } + + if got := do("/api/metrics/dashboard?service_name=a").Header().Get("X-Cache"); got != "MISS" { + t.Errorf("first ?service_name=a: X-Cache = %q, want MISS", got) + } + if got := do("/api/metrics/dashboard?service_name=b").Header().Get("X-Cache"); got != "MISS" { + t.Errorf("first ?service_name=b: X-Cache = %q, want MISS (different query must not share entry)", got) + } + if got := do("/api/metrics/dashboard?service_name=a").Header().Get("X-Cache"); got != "HIT" { + t.Errorf("second ?service_name=a: X-Cache = %q, want HIT", got) + } +} + +// TestHandleGetDashboardStats_LongQueryBypassesCache guards the cache-key +// cardinality bound: pathological query strings skip the cache entirely +// instead of growing the map. +func TestHandleGetDashboardStats_LongQueryBypassesCache(t *testing.T) { + s := newCachedTestServer(t) + path := "/api/metrics/dashboard?service_name=" + strings.Repeat("x", 300) + + for i := range 2 { + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + s.handleGetDashboardStats(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("request %d: want 200, got %d", i, rec.Code) + } + if got := rec.Header().Get("X-Cache"); got == "HIT" { + t.Errorf("request %d: oversized query must bypass the cache, got X-Cache HIT", i) + } + } +} + +// TestHandleGetDashboardStats_ExplicitTimeRange exercises the start/end +// query parsing alongside the cache: a parameterised window is served and +// cached under its own key. +func TestHandleGetDashboardStats_ExplicitTimeRange(t *testing.T) { + s := newCachedTestServer(t) + path := "/api/metrics/dashboard?start=2026-06-11T00:00:00Z&end=2026-06-11T01:00:00Z" + + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + s.handleGetDashboardStats(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d body=%q", rec.Code, rec.Body.String()) + } + if got := rec.Header().Get("X-Cache"); got != "MISS" { + t.Errorf("first: X-Cache = %q, want MISS", got) + } + + rec2 := httptest.NewRecorder() + s.handleGetDashboardStats(rec2, httptest.NewRequest(http.MethodGet, path, nil)) + if got := rec2.Header().Get("X-Cache"); got != "HIT" { + t.Errorf("second: X-Cache = %q, want HIT", got) + } +} + +// Repo-error paths: a closed DB makes every repository call fail, which +// must surface as a 500 (graph handler: its DB fallback returns nil). +func TestCachedHandlers_DBErrorPaths(t *testing.T) { + tests := []struct { + name string + path string + handler func(s *Server) http.HandlerFunc + }{ + {"stats", "/api/stats", func(s *Server) http.HandlerFunc { return s.handleGetStats }}, + {"dashboard", "/api/metrics/dashboard", func(s *Server) http.HandlerFunc { return s.handleGetDashboardStats }}, + {"system graph", "/api/system/graph", func(s *Server) http.HandlerFunc { return s.handleGetSystemGraph }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := newCachedTestServer(t) + _ = s.repo.Close() // force every query to error + + req := httptest.NewRequest(http.MethodGet, tt.path, nil) + rec := httptest.NewRecorder() + tt.handler(s)(rec, req) + if rec.Code != http.StatusInternalServerError { + t.Errorf("want 500 on closed DB, got %d", rec.Code) + } + }) + } +} + +func TestNewCachedJSON_MarshalError(t *testing.T) { + if _, err := newCachedJSON(make(chan int)); err == nil { + t.Fatal("want error for unmarshalable value") + } +} + +// TestHandleGetStats_TenantScopedKey proves two tenants never share a +// cached /api/stats payload. +func TestHandleGetStats_TenantScopedKey(t *testing.T) { + s := newCachedTestServer(t) + + do := func(tenant string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, "/api/stats", nil) + req = req.WithContext(storage.WithTenantContext(req.Context(), tenant)) + rec := httptest.NewRecorder() + s.handleGetStats(rec, req) + return rec + } + + if got := do("tenant-a").Header().Get("X-Cache"); got != "MISS" { + t.Errorf("tenant-a first: X-Cache = %q, want MISS", got) + } + if got := do("tenant-b").Header().Get("X-Cache"); got != "MISS" { + t.Errorf("tenant-b first: X-Cache = %q, want MISS (tenant isolation)", got) + } + if got := do("tenant-a").Header().Get("X-Cache"); got != "HIT" { + t.Errorf("tenant-a second: X-Cache = %q, want HIT", got) + } +} diff --git a/internal/api/graph_handler.go b/internal/api/graph_handler.go index 1ee9826..f7bc3eb 100644 --- a/internal/api/graph_handler.go +++ b/internal/api/graph_handler.go @@ -2,7 +2,6 @@ package api import ( "context" - "encoding/json" "log/slog" "math" "net/http" @@ -65,17 +64,16 @@ var OtelContextStartTime = time.Now() // handleGetSystemGraph handles GET /api/system/graph. // When the in-memory graph has been populated it returns instantly from memory. // Falls back to a DB query only when the graph has never been built yet. -// Results are cached for 10s per tenant — the cache key is scoped by tenant -// so two tenants hitting this endpoint never share a response. +// The rendered JSON is cached for 10s per tenant — the cache key is scoped +// by tenant so two tenants never share a response — and carries an ETag +// hashed once per cache fill, so a polling client that echoes If-None-Match +// gets a bodyless 304. func (s *Server) handleGetSystemGraph(w http.ResponseWriter, r *http.Request) { - const cacheTTL = 10 * time.Second ctx := r.Context() cacheKey := "system_graph:" + storage.TenantFromContext(ctx) if cached, ok := s.cache.Get(cacheKey); ok { - w.Header().Set("Content-Type", "application/json") - w.Header().Set("X-Cache", "HIT") - _ = json.NewEncoder(w).Encode(cached) + cached.(*cachedJSON).write(w, r, "HIT") return } @@ -89,10 +87,13 @@ func (s *Server) handleGetSystemGraph(w http.ResponseWriter, r *http.Request) { } } - s.cache.Set(cacheKey, resp, cacheTTL) - w.Header().Set("Content-Type", "application/json") - w.Header().Set("X-Cache", "MISS") - _ = json.NewEncoder(w).Encode(resp) + cj, err := newCachedJSON(resp) + if err != nil { + http.Error(w, "failed to encode system graph", http.StatusInternalServerError) + return + } + s.cache.Set(cacheKey, cj, hotPollCacheTTL) + cj.write(w, r, "MISS") } // buildGraphFromMemory converts the in-memory graph snapshot to the API response. diff --git a/internal/api/log_handlers.go b/internal/api/log_handlers.go index 2cbc9d5..fca73cb 100644 --- a/internal/api/log_handlers.go +++ b/internal/api/log_handlers.go @@ -15,24 +15,13 @@ import ( // handleGetLogs handles GET /api/logs with advanced filtering func (s *Server) handleGetLogs(w http.ResponseWriter, r *http.Request) { - limit := 50 - offset := 0 - - if l := r.URL.Query().Get("limit"); l != "" { - if v, err := strconv.Atoi(l); err == nil { - limit = v - } - } - if o := r.URL.Query().Get("offset"); o != "" { - if v, err := strconv.Atoi(o); err == nil { - offset = v - } - } + limit, offset := parsePaging(r, pagingDefaultLimit) filter := storage.LogFilter{ ServiceName: r.URL.Query().Get("service_name"), Severity: r.URL.Query().Get("severity"), Search: r.URL.Query().Get("search"), + TraceID: r.URL.Query().Get("trace_id"), Limit: limit, Offset: offset, } diff --git a/internal/api/log_handlers_cap_test.go b/internal/api/log_handlers_cap_test.go index 7a14401..63689a6 100644 --- a/internal/api/log_handlers_cap_test.go +++ b/internal/api/log_handlers_cap_test.go @@ -55,6 +55,45 @@ func TestHandleGetLogs_NoSearchSkipsCap(t *testing.T) { } } +// TestParsePaging_Clamp verifies that parsePaging enforces bounds on limit and +// offset, preventing GORM from receiving a negative Limit (treated as unlimited) +// or a negative offset. +func TestParsePaging_Clamp(t *testing.T) { + cases := []struct { + query string + defaultLimit int + wantLimit int + wantOffset int + }{ + // Over-limit capped at 1000. + {"limit=9999&offset=0", 50, 1000, 0}, + // Negative limit floored at 1. + {"limit=-5&offset=0", 50, 1, 0}, + // Negative offset floored at 0. + {"limit=10&offset=-99", 50, 10, 0}, + // Both negative. + {"limit=-1&offset=-1", 50, 1, 0}, + // Default limit also clamped. + {"", 9999, 1000, 0}, + // Valid values pass through unchanged. + {"limit=100&offset=200", 50, 100, 200}, + // Exact cap boundary. + {"limit=1000&offset=0", 50, 1000, 0}, + } + for _, tc := range cases { + t.Run(tc.query, func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/api/logs?"+tc.query, nil) + gotLimit, gotOffset := parsePaging(req, tc.defaultLimit) + if gotLimit != tc.wantLimit { + t.Errorf("limit: got %d, want %d", gotLimit, tc.wantLimit) + } + if gotOffset != tc.wantOffset { + t.Errorf("offset: got %d, want %d", gotOffset, tc.wantOffset) + } + }) + } +} + // newAPITestRepoWithoutFTS builds a fresh in-memory repo with FTS5 disabled. // Used by cap tests since they only care about handler behavior, not the // search backend. diff --git a/internal/api/log_handlers_trace_filter_test.go b/internal/api/log_handlers_trace_filter_test.go new file mode 100644 index 0000000..760877c --- /dev/null +++ b/internal/api/log_handlers_trace_filter_test.go @@ -0,0 +1,87 @@ +package api + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" +) + +// TestHandleGetLogs_TraceIDFilter verifies that GET /api/logs honors the +// trace_id query param (exact match via storage.LogFilter.TraceID). The UI +// traces→logs cross-link depends on this being deterministic on every +// driver — body `search` only matches trace IDs on the LIKE fallback, not +// under FTS5. +func TestHandleGetLogs_TraceIDFilter(t *testing.T) { + repo := newAPITestRepoWithoutFTS(t) + now := time.Now().UTC() + logs := []storage.Log{ + {TenantID: storage.DefaultTenantID, TraceID: "aaaa1111", Severity: "ERROR", Body: "boom", ServiceName: "svc-a", Timestamp: now}, + {TenantID: storage.DefaultTenantID, TraceID: "bbbb2222", Severity: "INFO", Body: "fine", ServiceName: "svc-b", Timestamp: now}, + } + if err := repo.BatchCreateLogs(logs); err != nil { + t.Fatalf("BatchCreateLogs: %v", err) + } + + srv := &Server{repo: repo} + mux := http.NewServeMux() + mux.HandleFunc("GET /api/logs", srv.handleGetLogs) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/logs?trace_id=aaaa1111", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d body=%q", rec.Code, rec.Body.String()) + } + var resp struct { + Data []struct{ TraceID string `json:"trace_id"` } `json:"data"` + Total int64 `json:"total"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if resp.Total != 1 || len(resp.Data) != 1 { + t.Fatalf("want exactly 1 row for trace_id filter, got total=%d len=%d", resp.Total, len(resp.Data)) + } + if resp.Data[0].TraceID != "aaaa1111" { + t.Fatalf("want trace_id aaaa1111, got %q", resp.Data[0].TraceID) + } +} + +// TestHandleGetLogs_TraceIDFilterSkipsSearchCap verifies that a trace_id +// filter alone (no search term) is NOT subject to the 24h keyword-search +// clamp — trace IDs are indexed exact matches, not LIKE scans. +func TestHandleGetLogs_TraceIDFilterSkipsSearchCap(t *testing.T) { + repo := newAPITestRepoWithoutFTS(t) + old := time.Now().UTC().Add(-5 * 24 * time.Hour) + if err := repo.BatchCreateLogs([]storage.Log{ + {TenantID: storage.DefaultTenantID, TraceID: "cccc3333", Severity: "WARN", Body: "stale", ServiceName: "svc-c", Timestamp: old}, + }); err != nil { + t.Fatalf("BatchCreateLogs: %v", err) + } + + srv := &Server{repo: repo} + mux := http.NewServeMux() + mux.HandleFunc("GET /api/logs", srv.handleGetLogs) + + rec := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "/api/logs?trace_id=cccc3333", nil) + mux.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d body=%q", rec.Code, rec.Body.String()) + } + var resp struct { + Total int64 `json:"total"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("unmarshal: %v", err) + } + if resp.Total != 1 { + t.Fatalf("5-day-old log must be reachable via trace_id, got total=%d", resp.Total) + } +} diff --git a/internal/api/metrics_handlers.go b/internal/api/metrics_handlers.go index 6b38ab6..7ffb1f4 100644 --- a/internal/api/metrics_handlers.go +++ b/internal/api/metrics_handlers.go @@ -8,6 +8,7 @@ import ( "github.com/RandomCodeSpace/otelcontext/internal/api/views" "github.com/RandomCodeSpace/otelcontext/internal/httpconst" + "github.com/RandomCodeSpace/otelcontext/internal/storage" ) // handleGetTrafficMetrics handles GET /api/metrics/traffic @@ -69,8 +70,23 @@ func (s *Server) handleGetLatencyHeatmap(w http.ResponseWriter, r *http.Request) _ = json.NewEncoder(w).Encode(points) } -// handleGetDashboardStats handles GET /api/metrics/dashboard +// handleGetDashboardStats handles GET /api/metrics/dashboard. +// The rendered JSON is cached for 10s per (tenant, query) with an ETag — +// same pattern as handleGetSystemGraph — so steady-state dashboard polling +// becomes a hash compare instead of a SQLite aggregate + JSON encode. The +// key includes the raw query string so explicit start/end/service_name +// windows never share an entry; oversized queries skip the cache (see +// maxCacheKeyQueryLen). func (s *Server) handleGetDashboardStats(w http.ResponseWriter, r *http.Request) { + var cacheKey string + if len(r.URL.RawQuery) <= maxCacheKeyQueryLen { + cacheKey = "dashboard_stats:" + storage.TenantFromContext(r.Context()) + "?" + r.URL.RawQuery + if cached, ok := s.cache.Get(cacheKey); ok { + cached.(*cachedJSON).write(w, r, "HIT") + return + } + } + // Default to last 30 minutes if not specified end := time.Now() start := end.Add(-30 * time.Minute) @@ -95,21 +111,44 @@ func (s *Server) handleGetDashboardStats(w http.ResponseWriter, r *http.Request) return } - w.Header().Set(httpconst.HeaderContentType, httpconst.ContentTypeJSON) - _ = json.NewEncoder(w).Encode(views.DashboardStatsFromModel(stats)) + cj, err := newCachedJSON(views.DashboardStatsFromModel(stats)) + if err != nil { + http.Error(w, "failed to encode dashboard stats", http.StatusInternalServerError) + return + } + if cacheKey != "" { + s.cache.Set(cacheKey, cj, hotPollCacheTTL) + } + cj.write(w, r, "MISS") } -// handleGetServiceMapMetrics handles GET /api/metrics/service-map +// handleGetServiceMapMetrics handles GET /api/metrics/service-map. +// Results are cached for 30s per (tenant, window) — the dashboard polls this +// endpoint and the underlying span aggregation is among the most expensive +// queries in the API surface. The key uses the raw start/end params so the +// default rolling window (no params) shares a single entry instead of being +// re-keyed on every request timestamp. func (s *Server) handleGetServiceMapMetrics(w http.ResponseWriter, r *http.Request) { + const cacheTTL = 30 * time.Second + startStr := r.URL.Query().Get("start") + endStr := r.URL.Query().Get("end") + cacheKey := "service_map:" + storage.TenantFromContext(r.Context()) + ":" + startStr + ":" + endStr + + if cached, ok := s.cache.Get(cacheKey); ok { + w.Header().Set(httpconst.HeaderContentType, httpconst.ContentTypeJSON) + w.Header().Set("X-Cache", "HIT") + _ = json.NewEncoder(w).Encode(cached) + return + } + end := time.Now() start := end.Add(-30 * time.Minute) - - if startStr := r.URL.Query().Get("start"); startStr != "" { + if startStr != "" { if t, err := time.Parse(time.RFC3339, startStr); err == nil { start = t } } - if endStr := r.URL.Query().Get("end"); endStr != "" { + if endStr != "" { if t, err := time.Parse(time.RFC3339, endStr); err == nil { end = t } @@ -122,8 +161,11 @@ func (s *Server) handleGetServiceMapMetrics(w http.ResponseWriter, r *http.Reque return } + resp := views.ServiceMapMetricsFromModel(metrics) + s.cache.Set(cacheKey, resp, cacheTTL) w.Header().Set(httpconst.HeaderContentType, httpconst.ContentTypeJSON) - _ = json.NewEncoder(w).Encode(views.ServiceMapMetricsFromModel(metrics)) + w.Header().Set("X-Cache", "MISS") + _ = json.NewEncoder(w).Encode(resp) } // handleGetMetricBuckets handles GET /api/metrics diff --git a/internal/api/middleware_test.go b/internal/api/middleware_test.go index 511d263..4204970 100644 --- a/internal/api/middleware_test.go +++ b/internal/api/middleware_test.go @@ -133,6 +133,41 @@ func TestTenantMiddleware_MissingHeaderUsesDefault(t *testing.T) { } } +// TestTenantMiddleware_DoesNotOverwritePinnedTenant verifies that when +// TenantKeyAuth.Middleware has already pinned a tenant onto the context (the +// per-tenant API-key path), the subsequent TenantMiddleware pass-through does +// NOT overwrite it with the client-supplied X-Tenant-ID header. +// +// This is the regression test for the middleware-ordering bypass: +// +// TenantKeyAuth.Middleware(auth "alpha-key" → pins "alpha") +// → TenantMiddleware(reads X-Tenant-ID: "beta" → must NOT overwrite) +// → handler (must see "alpha") +func TestTenantMiddleware_DoesNotOverwritePinnedTenant(t *testing.T) { + // Build per-tenant key auth: key "alpha-key" → tenant "alpha". + auth := NewTenantKeyAuth(map[string]string{"alpha-key": "alpha"}) + + cfg := &config.Config{DefaultTenant: "default"} + tc := &tenantCapture{} + + // Compose: TenantKeyAuth wraps TenantMiddleware wraps handler. + h := auth.Middleware("/mcp", TenantMiddleware(cfg)(tc.handler())) + + req := httptest.NewRequest(http.MethodGet, "/api/logs", nil) + req.Header.Set("Authorization", "Bearer alpha-key") + req.Header.Set(TenantHeader, "beta") // attacker's cross-tenant attempt + + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d (body=%q)", rec.Code, rec.Body.String()) + } + if tc.got != "alpha" { + t.Errorf("TenantMiddleware overwrote pinned tenant: got %q, want %q", tc.got, "alpha") + } +} + // Non-/api/* paths must pass through without tenant resolution — and so should // report the default (no ctx value). func TestTenantMiddleware_NonAPIPath_Passthrough(t *testing.T) { diff --git a/internal/api/server.go b/internal/api/server.go index 1f838d0..df4f793 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -2,6 +2,7 @@ package api import ( "net/http" + "strconv" "time" "github.com/RandomCodeSpace/otelcontext/internal/cache" @@ -107,6 +108,47 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) { mux.HandleFunc("/ws/events", s.eventHub.HandleWebSocket) } +const ( + pagingDefaultLimit = 50 + pagingMaxLimit = 1000 +) + +// parsePaging reads "limit" and "offset" from the request query string and +// applies safety clamping so GORM never sees a negative or unbounded Limit. +// +// - limit: floored at 1, capped at pagingMaxLimit (1000). When absent the +// caller-supplied defaultLimit is used (also clamped). +// - offset: floored at 0. When absent defaults to 0. +func parsePaging(r *http.Request, defaultLimit int) (limit, offset int) { + limit = defaultLimit + if limit < 1 { + limit = 1 + } + if limit > pagingMaxLimit { + limit = pagingMaxLimit + } + if l := r.URL.Query().Get("limit"); l != "" { + if v, err := strconv.Atoi(l); err == nil { + limit = v + } + } + if limit < 1 { + limit = 1 + } + if limit > pagingMaxLimit { + limit = pagingMaxLimit + } + if o := r.URL.Query().Get("offset"); o != "" { + if v, err := strconv.Atoi(o); err == nil { + offset = v + } + } + if offset < 0 { + offset = 0 + } + return limit, offset +} + // parseTimeRange parses start and end times from request query parameters func parseTimeRange(r *http.Request) (time.Time, time.Time, error) { var start, end time.Time diff --git a/internal/api/service_map_cache_test.go b/internal/api/service_map_cache_test.go new file mode 100644 index 0000000..d4ee0d4 --- /dev/null +++ b/internal/api/service_map_cache_test.go @@ -0,0 +1,166 @@ +package api + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/cache" + "github.com/RandomCodeSpace/otelcontext/internal/storage" +) + +// newServiceMapTestServer builds a Server with a real SQLite-backed repository +// and a live TTL cache — the two dependencies handleGetServiceMapMetrics uses. +func newServiceMapTestServer(t *testing.T) *Server { + t.Helper() + db, err := storage.NewDatabase("sqlite", ":memory:") + if err != nil { + t.Fatalf("NewDatabase: %v", err) + } + if err := storage.AutoMigrateModels(db, "sqlite"); err != nil { + t.Fatalf("AutoMigrateModels: %v", err) + } + repo := storage.NewRepositoryFromDB(db, "sqlite") + c := cache.New() + t.Cleanup(func() { + c.Stop() + _ = repo.Close() + }) + return &Server{repo: repo, cache: c} +} + +// seedServiceMapSpans inserts a parent/child span pair for the given tenant so +// the service-map endpoint has one edge and two nodes to report. +func seedServiceMapSpans(t *testing.T, s *Server, tenant, suffix string) { + t.Helper() + now := time.Now().UTC() + spans := []storage.Span{ + { + TenantID: tenant, TraceID: "trace-" + suffix, SpanID: "parent-" + suffix, + OperationName: "op", StartTime: now, EndTime: now.Add(time.Millisecond), + Duration: 1000, ServiceName: "svc-front-" + suffix, Status: "STATUS_CODE_UNSET", + }, + { + TenantID: tenant, TraceID: "trace-" + suffix, SpanID: "child-" + suffix, + ParentSpanID: "parent-" + suffix, OperationName: "op", StartTime: now, + EndTime: now.Add(time.Millisecond), Duration: 2000, + ServiceName: "svc-back-" + suffix, Status: "STATUS_CODE_ERROR", + }, + } + if err := s.repo.BatchCreateSpans(spans); err != nil { + t.Fatalf("seed spans: %v", err) + } +} + +func getServiceMap(t *testing.T, s *Server, ctx context.Context, target string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, target, nil).WithContext(ctx) + rr := httptest.NewRecorder() + s.handleGetServiceMapMetrics(rr, req) + return rr +} + +// TestServiceMapHandler_CacheMissThenHit verifies the 30s result cache: the +// first request computes and stores, the second is served from cache even if +// new spans landed in between. +func TestServiceMapHandler_CacheMissThenHit(t *testing.T) { + s := newServiceMapTestServer(t) + seedServiceMapSpans(t, s, "default", "a") + ctx := context.Background() + + rr1 := getServiceMap(t, s, ctx, "/api/metrics/service-map") + if rr1.Code != http.StatusOK { + t.Fatalf("first request: %d body=%s", rr1.Code, rr1.Body.String()) + } + if got := rr1.Header().Get("X-Cache"); got != "MISS" { + t.Fatalf("first request X-Cache = %q, want MISS", got) + } + + // New data must NOT appear within the TTL — proves the second request + // never reached the repository. + seedServiceMapSpans(t, s, "default", "b") + + rr2 := getServiceMap(t, s, ctx, "/api/metrics/service-map") + if rr2.Code != http.StatusOK { + t.Fatalf("second request: %d", rr2.Code) + } + if got := rr2.Header().Get("X-Cache"); got != "HIT" { + t.Fatalf("second request X-Cache = %q, want HIT", got) + } + if rr1.Body.String() != rr2.Body.String() { + t.Fatalf("cached body diverged:\n first=%s\nsecond=%s", rr1.Body.String(), rr2.Body.String()) + } +} + +// TestServiceMapHandler_CacheKeyPerTenant verifies two tenants never share a +// cache entry. +func TestServiceMapHandler_CacheKeyPerTenant(t *testing.T) { + s := newServiceMapTestServer(t) + seedServiceMapSpans(t, s, "default", "a") + seedServiceMapSpans(t, s, "tenant-b", "b") + + rr1 := getServiceMap(t, s, context.Background(), "/api/metrics/service-map") + if got := rr1.Header().Get("X-Cache"); got != "MISS" { + t.Fatalf("default tenant X-Cache = %q, want MISS", got) + } + + ctxB := storage.WithTenantContext(context.Background(), "tenant-b") + rr2 := getServiceMap(t, s, ctxB, "/api/metrics/service-map") + if got := rr2.Header().Get("X-Cache"); got != "MISS" { + t.Fatalf("tenant-b first request X-Cache = %q, want MISS (cache leaked across tenants)", got) + } + if rr1.Body.String() == rr2.Body.String() { + t.Fatalf("tenant responses identical — tenant scoping broken: %s", rr1.Body.String()) + } + + rr3 := getServiceMap(t, s, ctxB, "/api/metrics/service-map") + if got := rr3.Header().Get("X-Cache"); got != "HIT" { + t.Fatalf("tenant-b second request X-Cache = %q, want HIT", got) + } +} + +// TestServiceMapHandler_CacheKeyPerWindow verifies explicit start/end windows +// are cached independently of the default rolling window and of each other. +func TestServiceMapHandler_CacheKeyPerWindow(t *testing.T) { + s := newServiceMapTestServer(t) + seedServiceMapSpans(t, s, "default", "a") + ctx := context.Background() + + // Prime the default-window entry. + if got := getServiceMap(t, s, ctx, "/api/metrics/service-map").Header().Get("X-Cache"); got != "MISS" { + t.Fatalf("default window X-Cache = %q, want MISS", got) + } + + win := "/api/metrics/service-map?start=2026-06-01T00:00:00Z&end=2026-06-01T01:00:00Z" + if got := getServiceMap(t, s, ctx, win).Header().Get("X-Cache"); got != "MISS" { + t.Fatalf("explicit window first request X-Cache = %q, want MISS", got) + } + if got := getServiceMap(t, s, ctx, win).Header().Get("X-Cache"); got != "HIT" { + t.Fatalf("explicit window second request X-Cache = %q, want HIT", got) + } + + other := "/api/metrics/service-map?start=2026-06-01T00:00:00Z&end=2026-06-01T02:00:00Z" + if got := getServiceMap(t, s, ctx, other).Header().Get("X-Cache"); got != "MISS" { + t.Fatalf("different window X-Cache = %q, want MISS", got) + } +} + +// TestServiceMapHandler_DBError500 verifies repository failures surface as 500 +// and are never cached. +func TestServiceMapHandler_DBError500(t *testing.T) { + s := newServiceMapTestServer(t) + sqlDB, err := s.repo.DB().DB() + if err != nil { + t.Fatalf("unwrap sql.DB: %v", err) + } + if err := sqlDB.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + rr := getServiceMap(t, s, context.Background(), "/api/metrics/service-map") + if rr.Code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d body=%s", rr.Code, rr.Body.String()) + } +} diff --git a/internal/api/tenant_middleware.go b/internal/api/tenant_middleware.go index dd25c39..cf73e90 100644 --- a/internal/api/tenant_middleware.go +++ b/internal/api/tenant_middleware.go @@ -8,7 +8,6 @@ import ( "github.com/RandomCodeSpace/otelcontext/internal/storage" ) - // TenantHeader is the canonical HTTP header carrying the tenant ID on // read-side (query) requests. Ingest paths resolve tenant separately via gRPC // metadata / OTLP resource attributes and do not go through this middleware. @@ -35,6 +34,14 @@ func TenantMiddleware(cfg *config.Config) func(http.Handler) http.Handler { next.ServeHTTP(w, r) return } + // If an upstream layer (e.g. TenantKeyAuth.Middleware) has already + // pinned a tenant onto the context, do not overwrite it — that would + // allow a client to escape their key-bound tenant by supplying a + // different X-Tenant-ID header. + if storage.HasTenantContext(r.Context()) { + next.ServeHTTP(w, r) + return + } // SanitizeTenantID returns "" for empty / over-length / control-char // values so they fall through to the configured default — see // storage.SanitizeTenantID. Hostile or misconfigured clients cannot diff --git a/internal/api/trace_handlers.go b/internal/api/trace_handlers.go index 0e12ca7..c7fd1d5 100644 --- a/internal/api/trace_handlers.go +++ b/internal/api/trace_handlers.go @@ -5,25 +5,13 @@ import ( "fmt" "log/slog" "net/http" - "strconv" "github.com/RandomCodeSpace/otelcontext/internal/api/views" ) // handleGetTraces handles GET /api/traces func (s *Server) handleGetTraces(w http.ResponseWriter, r *http.Request) { - limit := 20 - offset := 0 - if l := r.URL.Query().Get("limit"); l != "" { - if v, err := strconv.Atoi(l); err == nil { - limit = v - } - } - if o := r.URL.Query().Get("offset"); o != "" { - if v, err := strconv.Atoi(o); err == nil { - offset = v - } - } + limit, offset := parsePaging(r, 20) start, end, err := parseTimeRange(r) if err != nil { diff --git a/internal/api/views/views.go b/internal/api/views/views.go index e782eec..4048723 100644 --- a/internal/api/views/views.go +++ b/internal/api/views/views.go @@ -46,6 +46,7 @@ type Span struct { EndTime time.Time `json:"end_time"` Duration int64 `json:"duration"` ServiceName string `json:"service_name"` + Status string `json:"status"` AttributesJSON string `json:"attributes_json"` } @@ -244,6 +245,7 @@ func SpanFromModel(m storage.Span) Span { EndTime: m.EndTime, Duration: m.Duration, ServiceName: m.ServiceName, + Status: m.Status, AttributesJSON: string(m.AttributesJSON), } } diff --git a/internal/api/views/views_test.go b/internal/api/views/views_test.go index 48d7eb3..bd707f0 100644 --- a/internal/api/views/views_test.go +++ b/internal/api/views/views_test.go @@ -185,3 +185,20 @@ func TestTraceView_PreservesJSONFieldNames(t *testing.T) { } } } + +// TestSpanView_PreservesStatus pins the span status code on the wire — the +// UI waterfall colors STATUS_CODE_ERROR spans --crit and the storage model +// has carried Status since ingest day one. +func TestSpanView_PreservesStatus(t *testing.T) { + sp := storage.Span{ + ID: 7, TraceID: "tid", SpanID: "sid", OperationName: "op", + ServiceName: "svc", Status: "STATUS_CODE_ERROR", + } + b, err := json.Marshal(SpanFromModel(sp)) + if err != nil { + t.Fatalf("marshal: %v", err) + } + if !strings.Contains(string(b), `"status":"STATUS_CODE_ERROR"`) { + t.Errorf("Span view missing status field in %s", string(b)) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 8145ee6..bdfb0b5 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -20,6 +20,10 @@ type Config struct { DLQPath string DLQReplayInterval string + // PprofAddr serves net/http/pprof on a dedicated listener (never the + // public mux). Loopback-only by default; empty disables profiling. + PprofAddr string + // Ingestion Filtering IngestMinSeverity string IngestAllowedServices string @@ -60,6 +64,14 @@ type Config struct { RetentionBatchSize int RetentionBatchSleepMs int + // RetentionFullVacuum restores the daily full VACUUM during SQLite + // maintenance. Default false: the daily pass runs + // PRAGMA incremental_vacuum(10000) instead, because a full VACUUM holds + // an exclusive lock for 10-60 minutes on multi-GB files and starves + // ingest into a 429 storm. On-demand full VACUUM remains available via + // POST /api/admin/vacuum. Ignored on non-SQLite drivers. + RetentionFullVacuum bool + // TSDB TSDBRingBufferDuration string // e.g. "1h" @@ -129,6 +141,28 @@ type Config struct { // GraphRAG event channel buffer size. Defaults to 10000 if unset or <=0. GraphRAGEventQueueSize int + // GraphRAGTraceTTL bounds how long spans/traces stay in the in-memory + // TraceStore before the refresh tick prunes them. Duration string, e.g. + // "1h". Defaults to "1h"; flipped to "30m" on SQLite (the in-memory span + // window is the largest GraphRAG heap consumer at 120 services). Anomaly + // and investigation paths look back <=5min, so a 30min window is safe. + GraphRAGTraceTTL string + + // GraphRAGMaxSpansPerTenant hard-caps the in-memory TraceStore span map + // per tenant. At the cap, NEW spans are skipped (counted via + // otelcontext_graphrag_events_dropped_total{signal="span_capacity"}); + // updates to resident spans still apply. The graph is best-effort — the + // DB remains the source of truth. 0 = default (500000); negative + // disables the cap. + GraphRAGMaxSpansPerTenant int + + // GraphRAGTenantIdleTTL evicts a tenant's entire in-memory store slice + // after this much time without any ingest event or query. Duration + // string, default "24h". The default tenant is never evicted, and an + // active tenant is re-created within one refresh tick (60s) from recent + // DB spans — eviction is self-healing. + GraphRAGTenantIdleTTL string + // Async ingest pipeline (Phase 1 robustness work). Decouples OTLP Export // from synchronous DB writes. When enabled, Export() returns as soon as // the parsed batch is enqueued; persistence runs on a worker pool. @@ -139,7 +173,14 @@ type Config struct { // 100% queue — return RESOURCE_EXHAUSTED so OTLP clients back off IngestAsyncEnabled bool // default true; opt out via INGEST_ASYNC_ENABLED=false IngestPipelineQueueSize int // default 50000 batches; per-deployment tunable - IngestPipelineWorkers int // default 8 worker goroutines + // IngestPipelineMaxBytes caps the approximate bytes held by queued + // batches. The item-count queue size alone cannot bound memory — one + // batch may carry arbitrarily large span/log payloads. At the cap the + // pipeline rejects with RESOURCE_EXHAUSTED / HTTP 429 even for priority + // (error/slow) batches: a 429 is recoverable, an OOM kill is not. + // Default 512MB; SQLite default 128MB (see applyDriverDefaults). + IngestPipelineMaxBytes int + IngestPipelineWorkers int // default 8 worker goroutines // IngestPipelinePerTenantCap caps in-flight batches per tenant so a noisy // tenant cannot starve siblings of fresh queue slots when fullness is // below the soft-backpressure threshold. 0 (default) disables — single- @@ -235,6 +276,7 @@ func Load(customPath string) (*Config, error) { DBDSN: getEnv("DB_DSN", ""), DLQPath: getEnv("DLQ_PATH", "./data/dlq"), DLQReplayInterval: getEnv("DLQ_REPLAY_INTERVAL", "5m"), + PprofAddr: getEnv("PPROF_ADDR", "127.0.0.1:6060"), IngestMinSeverity: getEnv("INGEST_MIN_SEVERITY", "INFO"), StoreMinSeverity: getEnv("STORE_MIN_SEVERITY", ""), @@ -254,6 +296,7 @@ func Load(customPath string) (*Config, error) { HotRetentionDays: getEnvInt("HOT_RETENTION_DAYS", 7), RetentionBatchSize: getEnvInt("RETENTION_BATCH_SIZE", 50000), RetentionBatchSleepMs: getEnvInt("RETENTION_BATCH_SLEEP_MS", 1), + RetentionFullVacuum: getEnvBool("RETENTION_FULL_VACUUM", false), // TSDB TSDBRingBufferDuration: getEnv("TSDB_RING_BUFFER_DURATION", "1h"), @@ -291,12 +334,16 @@ func Load(customPath string) (*Config, error) { LogFTSEnabled: parseTruthy(getEnv("LOG_FTS_ENABLED", "")), // GraphRAG - GraphRAGWorkerCount: getEnvInt("GRAPHRAG_WORKER_COUNT", 16), - GraphRAGEventQueueSize: getEnvInt("GRAPHRAG_EVENT_QUEUE_SIZE", 100000), + GraphRAGWorkerCount: getEnvInt("GRAPHRAG_WORKER_COUNT", 16), + GraphRAGEventQueueSize: getEnvInt("GRAPHRAG_EVENT_QUEUE_SIZE", 100000), + GraphRAGTraceTTL: getEnv("GRAPHRAG_TRACE_TTL", "1h"), + GraphRAGMaxSpansPerTenant: getEnvInt("GRAPHRAG_MAX_SPANS_PER_TENANT", 500000), + GraphRAGTenantIdleTTL: getEnv("GRAPHRAG_TENANT_IDLE_TTL", "24h"), // Async ingest pipeline IngestAsyncEnabled: getEnvBool("INGEST_ASYNC_ENABLED", true), IngestPipelineQueueSize: getEnvInt("INGEST_PIPELINE_QUEUE_SIZE", 50000), + IngestPipelineMaxBytes: getEnvInt("INGEST_PIPELINE_MAX_BYTES", 512<<20), IngestPipelineWorkers: getEnvInt("INGEST_PIPELINE_WORKERS", 8), IngestPipelinePerTenantCap: getEnvInt("INGEST_PIPELINE_PER_TENANT_CAP", 0), @@ -358,11 +405,24 @@ var sqliteOverrides = []struct { {"DB_MAX_IDLE_CONNS", func(c *Config) { c.DBMaxIdleConns = 1 }}, {"INGEST_PIPELINE_WORKERS", func(c *Config) { c.IngestPipelineWorkers = 2 }}, {"INGEST_PIPELINE_QUEUE_SIZE", func(c *Config) { c.IngestPipelineQueueSize = 10000 }}, + // The SQLite single writer drains slowly, so the ingest queue is the + // first structure to bloat — bound it to 128MB instead of 512MB. + {"INGEST_PIPELINE_MAX_BYTES", func(c *Config) { c.IngestPipelineMaxBytes = 128 << 20 }}, {"METRIC_MAX_CARDINALITY", func(c *Config) { c.MetricMaxCardinality = 3000 }}, {"STORE_MIN_SEVERITY", func(c *Config) { c.StoreMinSeverity = "WARN" }}, {"SAMPLING_RATE", func(c *Config) { c.SamplingRate = 0.05 }}, {"GRPC_MAX_CONCURRENT_STREAMS", func(c *Config) { c.GRPCMaxConcurrentStreams = 240 }}, {"LOG_FTS_ENABLED", func(c *Config) { c.LogFTSEnabled = true }}, + // Each queued event embeds a storage.Span/Log by value (~0.5–2 KB); the + // 100k Postgres default is ~100 MB+ of standing buffer. On SQLite the + // single writer starves the workers anyway — drop sooner (metered via + // otelcontext_graphrag_events_dropped_total) instead of buffering RAM. + {"GRAPHRAG_EVENT_QUEUE_SIZE", func(c *Config) { c.GraphRAGEventQueueSize = 10000 }}, + // The TraceStore span window dominates GraphRAG heap at 120 services + // (~1.5 GB potential at 1h). Anomaly/investigation lookbacks are <=5min, + // so halving the window costs nothing they rely on; MCP trace tools fall + // through to the DB for older traces. + {"GRAPHRAG_TRACE_TTL", func(c *Config) { c.GraphRAGTraceTTL = "30m" }}, } func applyDriverDefaults(cfg *Config) { @@ -513,6 +573,12 @@ func (c *Config) Validate() error { if c.GRPCMaxConcurrentStreams < 1 || c.GRPCMaxConcurrentStreams > 1_000_000 { return fmt.Errorf("GRPC_MAX_CONCURRENT_STREAMS must be between 1 and 1000000, got %d", c.GRPCMaxConcurrentStreams) } + // GraphRAG event queue: the channel buffer is allocated up front and each + // queued event embeds a Span/Log by value (~0.5-2 KB), so an unbounded env + // value is a real OOM lever. 1M buffered events is already ~1-2 GB. + if c.GraphRAGEventQueueSize < 1 || c.GraphRAGEventQueueSize > 1_000_000 { + return fmt.Errorf("GRAPHRAG_EVENT_QUEUE_SIZE must be between 1 and 1000000, got %d", c.GraphRAGEventQueueSize) + } if c.DBMaxOpenConns < 1 { return fmt.Errorf("DB_MAX_OPEN_CONNS must be >= 1, got %d", c.DBMaxOpenConns) } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bf57847..41af0df 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -23,6 +23,7 @@ func baseValid() *Config { CompressionLevel: "default", GRPCMaxRecvMB: 16, GRPCMaxConcurrentStreams: 1000, + GraphRAGEventQueueSize: 100000, RetentionBatchSize: 50000, RetentionBatchSleepMs: 1, } @@ -137,6 +138,30 @@ func TestValidate_TLS_ReadableFilesOK(t *testing.T) { } } +func TestLoad_RetentionFullVacuum(t *testing.T) { + // Default: off — the daily SQLite maintenance must not full-VACUUM. + t.Setenv("RETENTION_FULL_VACUUM", "") + if err := os.Unsetenv("RETENTION_FULL_VACUUM"); err != nil { + t.Fatalf("unset: %v", err) + } + cfg, err := Load("__no_such_env_file__") + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.RetentionFullVacuum { + t.Fatal("RetentionFullVacuum must default to false") + } + + t.Setenv("RETENTION_FULL_VACUUM", "true") + cfg, err = Load("__no_such_env_file__") + if err != nil { + t.Fatalf("Load: %v", err) + } + if !cfg.RetentionFullVacuum { + t.Fatal("RETENTION_FULL_VACUUM=true not loaded") + } +} + func TestLoad_EnvVars_TLS_APIKey_OTel_Tenant(t *testing.T) { t.Setenv("TLS_CERT_FILE", "") t.Setenv("TLS_KEY_FILE", "") @@ -307,3 +332,22 @@ func TestLoad_DefaultTenant_FallsBackToDefault(t *testing.T) { t.Errorf("expected default tenant to be 'default', got %q", cfg.DefaultTenant) } } + +// TestValidate_GraphRAGEventQueueSize_Bounds: the event channel buffer is +// allocated up front and each event embeds a Span/Log by value, so the env +// value must be range-checked (also breaks the CodeQL +// go/uncontrolled-allocation-size taint path env -> make(chan, n)). +func TestValidate_GraphRAGEventQueueSize_Bounds(t *testing.T) { + for _, bad := range []int{0, -1, 1_000_001} { + c := baseValid() + c.GraphRAGEventQueueSize = bad + if err := c.Validate(); err == nil || !strings.Contains(err.Error(), "GRAPHRAG_EVENT_QUEUE_SIZE") { + t.Fatalf("size %d: expected GRAPHRAG_EVENT_QUEUE_SIZE error, got %v", bad, err) + } + } + c := baseValid() + c.GraphRAGEventQueueSize = 1_000_000 + if err := c.Validate(); err != nil { + t.Fatalf("size 1000000 should validate: %v", err) + } +} diff --git a/internal/config/driver_defaults_test.go b/internal/config/driver_defaults_test.go index 896267f..33b5189 100644 --- a/internal/config/driver_defaults_test.go +++ b/internal/config/driver_defaults_test.go @@ -13,11 +13,14 @@ var sqliteEnvKeys = []string{ "DB_MAX_IDLE_CONNS", "INGEST_PIPELINE_WORKERS", "INGEST_PIPELINE_QUEUE_SIZE", + "INGEST_PIPELINE_MAX_BYTES", "METRIC_MAX_CARDINALITY", "STORE_MIN_SEVERITY", "SAMPLING_RATE", "GRPC_MAX_CONCURRENT_STREAMS", "LOG_FTS_ENABLED", + "GRAPHRAG_EVENT_QUEUE_SIZE", + "GRAPHRAG_TRACE_TTL", } // clearSQLiteEnv unsets every env var consulted by applyDriverDefaults so @@ -46,15 +49,18 @@ func clearSQLiteEnv(t *testing.T) { func postgresDefaultsConfig(driver string) *Config { return &Config{ DBDriver: driver, - DBMaxOpenConns: 50, // Postgres default - DBMaxIdleConns: 10, // Postgres default - IngestPipelineWorkers: 8, // Postgres default - IngestPipelineQueueSize: 50000, // Postgres default - MetricMaxCardinality: 10000, // Postgres default - StoreMinSeverity: "", // same-as-ingest default - SamplingRate: 1.0, // keep-all default - GRPCMaxConcurrentStreams: 1000, // Postgres default - LogFTSEnabled: false, // FTS5 opt-in default + DBMaxOpenConns: 50, // Postgres default + DBMaxIdleConns: 10, // Postgres default + IngestPipelineWorkers: 8, // Postgres default + IngestPipelineQueueSize: 50000, // Postgres default + IngestPipelineMaxBytes: 512 << 20, // Postgres default + MetricMaxCardinality: 10000, // Postgres default + StoreMinSeverity: "", // same-as-ingest default + SamplingRate: 1.0, // keep-all default + GRPCMaxConcurrentStreams: 1000, // Postgres default + GraphRAGEventQueueSize: 100000, // Postgres default + LogFTSEnabled: false, // FTS5 opt-in default + GraphRAGTraceTTL: "1h", // Postgres default } } @@ -75,11 +81,14 @@ func TestApplyDriverDefaults_SQLite_FlipsAllWhenNoEnv(t *testing.T) { {"DBMaxIdleConns", cfg.DBMaxIdleConns, 1}, {"IngestPipelineWorkers", cfg.IngestPipelineWorkers, 2}, {"IngestPipelineQueueSize", cfg.IngestPipelineQueueSize, 10000}, + {"IngestPipelineMaxBytes", cfg.IngestPipelineMaxBytes, 128 << 20}, {"MetricMaxCardinality", cfg.MetricMaxCardinality, 3000}, {"StoreMinSeverity", cfg.StoreMinSeverity, "WARN"}, {"SamplingRate", cfg.SamplingRate, 0.05}, {"GRPCMaxConcurrentStreams", cfg.GRPCMaxConcurrentStreams, 240}, {"LogFTSEnabled", cfg.LogFTSEnabled, true}, + {"GraphRAGEventQueueSize", cfg.GraphRAGEventQueueSize, 10000}, + {"GraphRAGTraceTTL", cfg.GraphRAGTraceTTL, "30m"}, } for _, c := range cases { if c.got != c.want { diff --git a/internal/config/graphrag_bounds_test.go b/internal/config/graphrag_bounds_test.go new file mode 100644 index 0000000..8a58430 --- /dev/null +++ b/internal/config/graphrag_bounds_test.go @@ -0,0 +1,72 @@ +package config + +import ( + "os" + "testing" +) + +// clearGraphRAGBoundsEnv unsets the GraphRAG memory-bound env vars so Load() +// starts from a deterministic "operator set nothing" baseline. Mirrors +// clearSQLiteEnv (driver_defaults_test.go). +func clearGraphRAGBoundsEnv(t *testing.T) { + t.Helper() + for _, k := range []string{ + "GRAPHRAG_TRACE_TTL", + "GRAPHRAG_MAX_SPANS_PER_TENANT", + "GRAPHRAG_TENANT_IDLE_TTL", + } { + if _, ok := os.LookupEnv(k); ok { + old := os.Getenv(k) + t.Setenv(k, old) // record original for revert + if err := os.Unsetenv(k); err != nil { + t.Fatalf("unset %s: %v", k, err) + } + } + } +} + +// TestLoad_GraphRAGMemoryBounds_Defaults proves the new GraphRAG memory-bound +// knobs land with their documented Postgres-path defaults: 1h trace TTL, +// 500k spans per tenant, 24h tenant idle TTL. +func TestLoad_GraphRAGMemoryBounds_Defaults(t *testing.T) { + clearGraphRAGBoundsEnv(t) + t.Setenv("DB_DRIVER", "postgres") // avoid the SQLite 30m TTL override + + cfg, err := Load("") + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.GraphRAGTraceTTL != "1h" { + t.Errorf("GraphRAGTraceTTL = %q, want \"1h\"", cfg.GraphRAGTraceTTL) + } + if cfg.GraphRAGMaxSpansPerTenant != 500000 { + t.Errorf("GraphRAGMaxSpansPerTenant = %d, want 500000", cfg.GraphRAGMaxSpansPerTenant) + } + if cfg.GraphRAGTenantIdleTTL != "24h" { + t.Errorf("GraphRAGTenantIdleTTL = %q, want \"24h\"", cfg.GraphRAGTenantIdleTTL) + } +} + +// TestLoad_GraphRAGMemoryBounds_EnvOverride proves operator-set values are +// honoured verbatim. +func TestLoad_GraphRAGMemoryBounds_EnvOverride(t *testing.T) { + clearGraphRAGBoundsEnv(t) + t.Setenv("DB_DRIVER", "sqlite") + t.Setenv("GRAPHRAG_TRACE_TTL", "15m") + t.Setenv("GRAPHRAG_MAX_SPANS_PER_TENANT", "1000") + t.Setenv("GRAPHRAG_TENANT_IDLE_TTL", "1h") + + cfg, err := Load("") + if err != nil { + t.Fatalf("Load: %v", err) + } + if cfg.GraphRAGTraceTTL != "15m" { + t.Errorf("GraphRAGTraceTTL = %q, want \"15m\" (SQLite override must not clobber explicit env)", cfg.GraphRAGTraceTTL) + } + if cfg.GraphRAGMaxSpansPerTenant != 1000 { + t.Errorf("GraphRAGMaxSpansPerTenant = %d, want 1000", cfg.GraphRAGMaxSpansPerTenant) + } + if cfg.GraphRAGTenantIdleTTL != "1h" { + t.Errorf("GraphRAGTenantIdleTTL = %q, want \"1h\"", cfg.GraphRAGTenantIdleTTL) + } +} diff --git a/internal/graphrag/anomaly.go b/internal/graphrag/anomaly.go index 0d84cb7..13e5b3e 100644 --- a/internal/graphrag/anomaly.go +++ b/internal/graphrag/anomaly.go @@ -12,8 +12,16 @@ import ( // tenant slice we walk the ServiceStore and SignalStore under their own // locks and emit anomalies into that tenant's AnomalyStore. func (g *GraphRAG) detectAnomalies(ctx context.Context) { + prevScan := g.lastAnomalyScan.Swap(time.Now().UnixNano()) tenants := g.snapshotTenants() for tenant, stores := range tenants { + // Gate: a tenant with no span/log/metric processed since the + // previous scan cannot have changed stats — skip the whole walk. + // The first scan after startup (prevScan == 0) always runs so + // DB-rebuilt state is examined at least once. + if prevScan != 0 && stores.lastEventAt.Load() < prevScan { + continue + } tctx := storage.WithTenantContext(ctx, tenant) g.detectAnomaliesForTenant(tctx, tenant, stores) } @@ -28,7 +36,12 @@ func (g *GraphRAG) detectAnomaliesForTenant(ctx context.Context, tenant string, baselineErrorRate := 0.02 // reasonable baseline if svc.ErrorRate > baselineErrorRate*2 && svc.ErrorRate > 0.05 { anomaly := AnomalyNode{ - ID: fmt.Sprintf("anom_%s_err_%d", svc.Name, now.UnixNano()), + // Stable ID per (service, type): each detection tick UPSERTS the + // same evolving anomaly node rather than minting a new one. With a + // UnixNano suffix an ongoing spike created a fresh node every 10s + // (and correlateWithRecent then minted O(N²) PRECEDED_BY edges), + // which grew the AnomalyStore to ~1.3 GB over a 15-min soak. + ID: fmt.Sprintf("anom_%s_err", svc.Name), Type: AnomalyErrorSpike, Severity: classifyErrorSeverity(svc.ErrorRate), Service: svc.Name, @@ -49,7 +62,7 @@ func (g *GraphRAG) detectAnomaliesForTenant(ctx context.Context, tenant string, // Latency degradation: p99-like check using avg * 3 as proxy if svc.AvgLatency > 500 && svc.CallCount > 10 { anomaly := AnomalyNode{ - ID: fmt.Sprintf("anom_%s_lat_%d", svc.Name, now.UnixNano()), + ID: fmt.Sprintf("anom_%s_lat", svc.Name), // stable per (service,type); see error-spike note Type: AnomalyLatencySpike, Severity: classifyLatencySeverity(svc.AvgLatency), Service: svc.Name, @@ -79,7 +92,7 @@ func (g *GraphRAG) detectAnomaliesForTenant(ctx context.Context, tenant string, deviation := (m.RollingAvg - (m.RollingMin + rangeSize/2)) / (rangeSize / 2) if deviation > 3.0 || deviation < -3.0 { anomaly := AnomalyNode{ - ID: fmt.Sprintf("anom_%s_metric_%d", m.Service, now.UnixNano()), + ID: fmt.Sprintf("anom_%s_metric_%s", m.Service, m.MetricName), // stable per (service,metric) Type: AnomalyMetricZScore, Severity: SeverityWarning, Service: m.Service, @@ -93,11 +106,17 @@ func (g *GraphRAG) detectAnomaliesForTenant(ctx context.Context, tenant string, } } +// maxCorrelationWalk bounds how many recent anomalies correlateWithRecent +// considers per detection. Stable per-(service,type) IDs already keep the +// store small in steady state; this is the backstop against a pathological +// backlog turning every 10s tick into an O(N) scan + edge fan-out. +const maxCorrelationWalk = 1000 + // correlateWithRecent links an anomaly to other anomalies within ±30s in the // same tenant's AnomalyStore. func correlateWithRecent(stores *tenantStores, anomaly AnomalyNode) { window := 30 * time.Second - recent := stores.anomalies.AnomaliesSince(anomaly.Timestamp.Add(-window)) + recent := stores.anomalies.AnomaliesSinceLimit(anomaly.Timestamp.Add(-window), maxCorrelationWalk) for _, prev := range recent { if prev.ID == anomaly.ID { continue diff --git a/internal/graphrag/anomaly_dedup_test.go b/internal/graphrag/anomaly_dedup_test.go new file mode 100644 index 0000000..9250aac --- /dev/null +++ b/internal/graphrag/anomaly_dedup_test.go @@ -0,0 +1,42 @@ +package graphrag + +import ( + "testing" + "time" +) + +// TestAnomalyDedupBoundsStore proves the root-cause fix for the soak finding: +// stable per-(service,type) anomaly IDs make repeated detection ticks UPSERT a +// single evolving node instead of minting a new one each tick. With the old +// UnixNano-suffixed IDs, 100 ticks over 2 services produced 200 nodes and an +// O(N²) PRECEDED_BY edge explosion (which grew AnomalyStore to ~1.3 GB in a +// 15-min soak). Under the fix, node and edge counts stay bounded regardless of +// how many ticks fire. +func TestAnomalyDedupBoundsStore(t *testing.T) { + stores := newTenantStores(time.Hour, 0) + base := time.Unix(1_700_000_000, 0) + + const ticks = 100 + for i := range ticks { + ts := base.Add(time.Duration(i) * 10 * time.Second) + for _, svc := range []string{"checkout", "payments"} { + a := AnomalyNode{ + ID: "anom_" + svc + "_err", // stable ID, mirrors detectAnomaliesForTenant + Type: AnomalyErrorSpike, + Service: svc, + Timestamp: ts, + } + stores.anomalies.AddAnomaly(a) + correlateWithRecent(stores, a) + } + } + + if got := len(stores.anomalies.Anomalies); got != 2 { + t.Fatalf("expected 2 deduped anomaly nodes after %d ticks, got %d", ticks, got) + } + // TRIGGERED_BY: 2 (one per node). PRECEDED_BY: at most the 2-node mesh. + // The point is it does NOT scale with tick count. + if got := len(stores.anomalies.Edges); got > 8 { + t.Fatalf("edge count must stay bounded under dedup, got %d after %d ticks", got, ticks) + } +} diff --git a/internal/graphrag/anomaly_gating_test.go b/internal/graphrag/anomaly_gating_test.go new file mode 100644 index 0000000..37d672d --- /dev/null +++ b/internal/graphrag/anomaly_gating_test.go @@ -0,0 +1,112 @@ +package graphrag + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" + "github.com/RandomCodeSpace/otelcontext/internal/tsdb" +) + +// feedErrorSpan pushes one ERROR span through processSpan synchronously so +// the default tenant's ServiceStore error rate trips the spike detector. +func feedErrorSpan(g *GraphRAG, spanID string) { + g.processSpan(&spanEvent{ + Span: storage.Span{ + TraceID: "trace-gate", + SpanID: spanID, + OperationName: "/op", + ServiceName: "orders", + StartTime: time.Now(), + }, + TraceID: "trace-gate", + Status: "STATUS_CODE_ERROR", + Tenant: storage.DefaultTenantID, + }) +} + +// deleteAnomaly removes one anomaly node so a later scan provably re-creates +// it (or provably does not, when the tenant is gated). +func deleteAnomaly(stores *tenantStores, id string) { + stores.anomalies.mu.Lock() + delete(stores.anomalies.Anomalies, id) + stores.anomalies.mu.Unlock() +} + +// TestDetectAnomalies_SkipsTenantsWithoutNewEvents proves the scan gate: +// a tenant with no span/log/metric processed since the previous tick is +// skipped entirely, and resumes being scanned as soon as an event lands. +func TestDetectAnomalies_SkipsTenantsWithoutNewEvents(t *testing.T) { + g := New(nil, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + ctx := context.Background() + + for i := 0; i < 10; i++ { + feedErrorSpan(g, fmt.Sprintf("s-%d", i)) + } + + // Scan 1 (first ever): always runs, detects the error spike. + g.detectAnomalies(ctx) + stores := g.storesForTenant(storage.DefaultTenantID) + if _, ok := stores.anomalies.Anomalies["anom_orders_err"]; !ok { + t.Fatalf("scan 1 did not detect the error spike") + } + + // Scan 2: no events since scan 1 — the tenant must be skipped, so a + // deleted anomaly stays gone. + deleteAnomaly(stores, "anom_orders_err") + g.detectAnomalies(ctx) + if _, ok := stores.anomalies.Anomalies["anom_orders_err"]; ok { + t.Fatalf("scan 2 re-created the anomaly — idle tenant was not skipped") + } + + // Scan 3: a fresh event re-arms the tenant and detection resumes. + feedErrorSpan(g, "s-rearm") + g.detectAnomalies(ctx) + if _, ok := stores.anomalies.Anomalies["anom_orders_err"]; !ok { + t.Fatalf("scan 3 skipped an active tenant — gate stuck") + } +} + +// TestProcessEvents_StampLastEventAt proves all three event paths arm the +// anomaly-scan gate. +func TestProcessEvents_StampLastEventAt(t *testing.T) { + g := New(nil, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + cases := []struct { + name string + tenant string + fire func(tenant string) + }{ + {"span", "tenant-span", func(tn string) { + g.processSpan(&spanEvent{ + Span: storage.Span{TraceID: "t", SpanID: "s", ServiceName: "svc", StartTime: time.Now()}, + TraceID: "t", Status: "STATUS_CODE_UNSET", Tenant: tn, + }) + }}, + {"log", "tenant-log", func(tn string) { + g.processLog(&logEvent{ + Log: storage.Log{ServiceName: "svc", Body: "boom failed id=1", Severity: "ERROR", Timestamp: time.Now()}, + Tenant: tn, + }) + }}, + {"metric", "tenant-metric", func(tn string) { + g.processMetric(&metricEvent{ + Metric: tsdb.RawMetric{Name: "cpu", ServiceName: "svc", Value: 1.0, Timestamp: time.Now()}, + Tenant: tn, + }) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tc.fire(tc.tenant) + st := g.storesForTenant(tc.tenant) + if st.lastEventAt.Load() == 0 { + t.Fatalf("%s event did not stamp lastEventAt", tc.name) + } + }) + } +} diff --git a/internal/graphrag/anomaly_walk_test.go b/internal/graphrag/anomaly_walk_test.go new file mode 100644 index 0000000..c9438d7 --- /dev/null +++ b/internal/graphrag/anomaly_walk_test.go @@ -0,0 +1,65 @@ +package graphrag + +import ( + "fmt" + "testing" + "time" +) + +// TestAnomaliesSinceLimit proves the bounded variant caps the walk while the +// unlimited spellings (n<=0 and the legacy AnomaliesSince) return everything. +func TestAnomaliesSinceLimit(t *testing.T) { + as := newAnomalyStore() + now := time.Now() + for i := 0; i < 10; i++ { + as.AddAnomaly(AnomalyNode{ + ID: fmt.Sprintf("anom-%d", i), + Service: "svc", + Timestamp: now, + }) + } + since := now.Add(-time.Minute) + + if got := len(as.AnomaliesSinceLimit(since, 3)); got != 3 { + t.Errorf("AnomaliesSinceLimit(3) returned %d, want 3", got) + } + if got := len(as.AnomaliesSinceLimit(since, 0)); got != 10 { + t.Errorf("AnomaliesSinceLimit(0) returned %d, want 10 (unlimited)", got) + } + if got := len(as.AnomaliesSince(since)); got != 10 { + t.Errorf("AnomaliesSince returned %d, want 10", got) + } +} + +// TestCorrelateWithRecent_BoundedWalk proves correlateWithRecent fans out at +// most maxCorrelationWalk PRECEDED_BY edges per detection, no matter how +// large the anomaly backlog in the ±30s window is. +func TestCorrelateWithRecent_BoundedWalk(t *testing.T) { + stores := newTenantStores(time.Hour, 0) + now := time.Now() + + for i := 0; i < maxCorrelationWalk+200; i++ { + stores.anomalies.AddAnomaly(AnomalyNode{ + ID: fmt.Sprintf("anom-%d", i), + Service: "svc", + Timestamp: now, + }) + } + + fresh := AnomalyNode{ID: "anom-fresh", Service: "svc", Timestamp: now} + stores.anomalies.AddAnomaly(fresh) + correlateWithRecent(stores, fresh) + + edges := 0 + for _, e := range stores.anomalies.Edges { + if e.Type == EdgePrecededBy && e.FromID == "anom-fresh" { + edges++ + } + } + if edges == 0 { + t.Fatalf("no PRECEDED_BY edges minted — correlation broken") + } + if edges > maxCorrelationWalk { + t.Fatalf("correlateWithRecent minted %d edges, want <= %d", edges, maxCorrelationWalk) + } +} diff --git a/internal/graphrag/builder.go b/internal/graphrag/builder.go index e2b058e..82cf44a 100644 --- a/internal/graphrag/builder.go +++ b/internal/graphrag/builder.go @@ -38,12 +38,24 @@ func guardWorker(name string) { } const ( - defaultWorkerCount = 16 - defaultChannelSize = 100000 + defaultWorkerCount = 16 + defaultChannelSize = 100000 + // maxChannelSize bounds the event channel buffer at the allocation + // site: each queued event embeds a Span/Log by value (~0.5-2 KB), so + // 1M buffered events is already ~1-2 GB. Mirrors the config-boundary + // range check on GRAPHRAG_EVENT_QUEUE_SIZE (config.Validate). + maxChannelSize = 1_000_000 defaultTraceTTL = 1 * time.Hour defaultRefreshEvery = 60 * time.Second defaultSnapshotEvery = 15 * time.Minute defaultAnomalyEvery = 10 * time.Second + // defaultMaxSpansPerTenant caps the in-memory TraceStore per tenant + // (~850 B/span ⇒ ~425 MB worst case). Overridable via + // GRAPHRAG_MAX_SPANS_PER_TENANT. + defaultMaxSpansPerTenant = 500000 + // defaultTenantIdleTTL evicts a tenant slice after this much time with + // no ingest event or query. Overridable via GRAPHRAG_TENANT_IDLE_TTL. + defaultTenantIdleTTL = 24 * time.Hour ) // spanEvent is sent through the ingestion channel. @@ -99,11 +111,13 @@ type GraphRAG struct { stopCh chan struct{} // Configuration - traceTTL time.Duration - refreshEvery time.Duration - snapshotEvery time.Duration - anomalyEvery time.Duration - workerCount int // 0 = defaultWorkerCount (set by New from Config) + traceTTL time.Duration + refreshEvery time.Duration + snapshotEvery time.Duration + anomalyEvery time.Duration + workerCount int // 0 = defaultWorkerCount (set by New from Config) + maxSpansPerTenant int // per-tenant TraceStore span cap; <=0 = unbounded + tenantIdleTTL time.Duration // idle window before tenant eviction; <=0 disables // Event drop counters. Atomic so OnSpanIngested/OnLogIngested/ // OnMetricIngested can record overflows without taking any lock — @@ -111,6 +125,11 @@ type GraphRAG struct { droppedSpans atomic.Int64 droppedLogs atomic.Int64 droppedMetrics atomic.Int64 + // droppedSpanCap counts spans skipped because the tenant's TraceStore + // hit MaxSpans (signal "span_capacity" on the Prometheus counter). + droppedSpanCap atomic.Int64 + // tenantsEvicted counts tenant slices removed by evictIdleTenants. + tenantsEvicted atomic.Int64 // metrics is an optional Prometheus hook for exporting event drops. // Assigned via SetMetrics; nil-safe at call sites. @@ -124,6 +143,11 @@ type GraphRAG struct { // invInserts counts cooldown-allowed PersistInvestigation calls. // Incremented BEFORE the DB write — see InvestigationInsertCount. invInserts atomic.Int64 + + // lastAnomalyScan is the unix-nano start time of the previous + // detectAnomalies pass. Tenants whose lastEventAt predates it are + // skipped — no events means their stats cannot have changed. + lastAnomalyScan atomic.Int64 } // SetMetrics wires the Prometheus registry so GraphRAG event drops are @@ -145,6 +169,15 @@ func (g *GraphRAG) DroppedLogsCount() int64 { return g.droppedLogs.Load() } // because the ingestion channel was full. func (g *GraphRAG) DroppedMetricsCount() int64 { return g.droppedMetrics.Load() } +// SpanCapacityDropsCount reports the number of spans skipped because the +// tenant's TraceStore was at its MaxSpans cap. Atomic, safe from any +// goroutine; exported for tests and readiness probes. +func (g *GraphRAG) SpanCapacityDropsCount() int64 { return g.droppedSpanCap.Load() } + +// TenantsEvictedCount reports the number of tenant store slices evicted +// for exceeding the idle TTL since startup. +func (g *GraphRAG) TenantsEvictedCount() int64 { return g.tenantsEvicted.Load() } + // InvestigationInsertCount reports cooldown-allowed PersistInvestigation // calls. Semantics: this counter increments when the cooldown check // passes, BEFORE the DB write — so a subsequent DB failure still @@ -175,6 +208,8 @@ func (g *GraphRAG) recordEventDrop(signal string) { g.droppedLogs.Add(1) case "metric": g.droppedMetrics.Add(1) + case "span_capacity": + g.droppedSpanCap.Add(1) } if g.metrics != nil && g.metrics.GraphRAGEventsDroppedTotal != nil { g.metrics.GraphRAGEventsDroppedTotal.WithLabelValues(signal).Inc() @@ -189,17 +224,26 @@ type Config struct { AnomalyEvery time.Duration WorkerCount int ChannelSize int + // MaxSpansPerTenant caps each tenant's in-memory TraceStore span map. + // 0 = defaultMaxSpansPerTenant; negative disables the cap. + MaxSpansPerTenant int + // TenantIdleTTL evicts a tenant's store slice after this much time + // without any ingest event or query. 0 = defaultTenantIdleTTL; + // negative disables eviction. + TenantIdleTTL time.Duration } // DefaultConfig returns sensible defaults. func DefaultConfig() Config { return Config{ - TraceTTL: defaultTraceTTL, - RefreshEvery: defaultRefreshEvery, - SnapshotEvery: defaultSnapshotEvery, - AnomalyEvery: defaultAnomalyEvery, - WorkerCount: defaultWorkerCount, - ChannelSize: defaultChannelSize, + TraceTTL: defaultTraceTTL, + RefreshEvery: defaultRefreshEvery, + SnapshotEvery: defaultSnapshotEvery, + AnomalyEvery: defaultAnomalyEvery, + WorkerCount: defaultWorkerCount, + ChannelSize: defaultChannelSize, + MaxSpansPerTenant: defaultMaxSpansPerTenant, + TenantIdleTTL: defaultTenantIdleTTL, } } @@ -224,24 +268,32 @@ func New(repo *storage.Repository, tsdbAgg *tsdb.Aggregator, ringBuf *tsdb.RingB if cfg.WorkerCount == 0 { cfg.WorkerCount = defaultWorkerCount } - if cfg.ChannelSize == 0 { + if cfg.ChannelSize <= 0 || cfg.ChannelSize > maxChannelSize { cfg.ChannelSize = defaultChannelSize } + if cfg.MaxSpansPerTenant == 0 { + cfg.MaxSpansPerTenant = defaultMaxSpansPerTenant + } + if cfg.TenantIdleTTL == 0 { + cfg.TenantIdleTTL = defaultTenantIdleTTL + } g := &GraphRAG{ - tenants: make(map[string]*tenantStores), - repo: repo, - tsdbAgg: tsdbAgg, - ringBuf: ringBuf, - drain: NewDrain(), - eventCh: make(chan event, cfg.ChannelSize), - stopCh: make(chan struct{}), - traceTTL: cfg.TraceTTL, - refreshEvery: cfg.RefreshEvery, - snapshotEvery: cfg.SnapshotEvery, - anomalyEvery: cfg.AnomalyEvery, - workerCount: cfg.WorkerCount, - invCooldown: newInvestigationCooldown(5 * time.Minute), + tenants: make(map[string]*tenantStores), + repo: repo, + tsdbAgg: tsdbAgg, + ringBuf: ringBuf, + drain: NewDrain(), + eventCh: make(chan event, cfg.ChannelSize), + stopCh: make(chan struct{}), + traceTTL: cfg.TraceTTL, + refreshEvery: cfg.RefreshEvery, + snapshotEvery: cfg.SnapshotEvery, + anomalyEvery: cfg.AnomalyEvery, + workerCount: cfg.WorkerCount, + maxSpansPerTenant: cfg.MaxSpansPerTenant, + tenantIdleTTL: cfg.TenantIdleTTL, + invCooldown: newInvestigationCooldown(5 * time.Minute), } // Bootstrap the default tenant slice so refresh/snapshot loops have a @@ -435,6 +487,7 @@ func (g *GraphRAG) processSpan(ev *spanEvent) { } stores := g.storesForTenant(ev.Tenant) + stores.lastEventAt.Store(time.Now().UnixNano()) // 1. Upsert ServiceNode stores.service.UpsertService(span.ServiceName, durationMs, isError, span.StartTime) @@ -446,7 +499,7 @@ func (g *GraphRAG) processSpan(ev *spanEvent) { // 3. Create TraceNode + SpanNode + CONTAINS + CHILD_OF edges stores.traces.UpsertTrace(span.TraceID, span.ServiceName, ev.Status, durationMs, span.StartTime) - stores.traces.UpsertSpan(SpanNode{ + if !stores.traces.UpsertSpan(SpanNode{ ID: span.SpanID, TraceID: span.TraceID, ParentSpanID: span.ParentSpanID, @@ -456,7 +509,11 @@ func (g *GraphRAG) processSpan(ev *spanEvent) { StatusCode: ev.Status, IsError: isError, Timestamp: span.StartTime, - }) + }) { + // Tenant span cap reached — graph is best-effort; DB is source of + // truth. Service/operation/trace stats above were still updated. + g.recordEventDrop("span_capacity") + } // 4. If parent span exists and belongs to different service, create CALLS edge if span.ParentSpanID != "" { @@ -476,6 +533,7 @@ func (g *GraphRAG) processLog(ev *logEvent) { } stores := g.storesForTenant(ev.Tenant) + stores.lastEventAt.Store(time.Now().UnixNano()) // Drain-based clustering (replaces hash+TF-IDF clustering). The Drain // miner is shared across tenants — its template tokens describe log shape, @@ -499,6 +557,7 @@ func (g *GraphRAG) processMetric(ev *metricEvent) { return } stores := g.storesForTenant(ev.Tenant) + stores.lastEventAt.Store(time.Now().UnixNano()) stores.signals.UpsertMetric(m.Name, m.ServiceName, m.Value, m.Timestamp) } @@ -523,8 +582,19 @@ func (g *GraphRAG) storesFor(ctx context.Context) *tenantStores { // storesForTenant is the tenant-string flavour of storesFor, used by event // handlers that have already resolved the tenant (the callback path carries // it on spanEvent / logEvent / metricEvent). Empty strings are coerced to -// storage.DefaultTenantID. +// storage.DefaultTenantID. Every call refreshes the tenant's idle-eviction +// clock — ingest and queries both count as activity. func (g *GraphRAG) storesForTenant(tenant string) *tenantStores { + slice := g.tenantStoresNoTouch(tenant) + slice.lastAccess.Store(time.Now().UnixNano()) + return slice +} + +// tenantStoresNoTouch returns (lazily creating) the tenant slice WITHOUT +// refreshing its idle-eviction clock. Background maintenance — the 60s DB +// rebuild in particular — goes through here so bookkeeping alone cannot keep +// a dormant tenant alive past tenantIdleTTL; only real ingest or queries do. +func (g *GraphRAG) tenantStoresNoTouch(tenant string) *tenantStores { if tenant == "" { tenant = storage.DefaultTenantID } @@ -539,7 +609,7 @@ func (g *GraphRAG) storesForTenant(tenant string) *tenantStores { if slice, ok = g.tenants[tenant]; ok { return slice } - slice = newTenantStores(g.traceTTL) + slice = newTenantStores(g.traceTTL, g.maxSpansPerTenant) g.tenants[tenant] = slice return slice } diff --git a/internal/graphrag/builder_clamp_test.go b/internal/graphrag/builder_clamp_test.go new file mode 100644 index 0000000..8ca57b0 --- /dev/null +++ b/internal/graphrag/builder_clamp_test.go @@ -0,0 +1,23 @@ +package graphrag + +import "testing" + +// The event channel buffer is allocated up front from operator-influenced +// config; New must clamp nonsense values to the default rather than allocate +// gigabytes (config.Validate rejects out-of-range env values, this guards +// programmatic callers and is the allocation-site barrier CodeQL checks). +func TestNew_ClampsEventChannelSize(t *testing.T) { + for _, bad := range []int{-1, maxChannelSize + 1} { + cfg := DefaultConfig() + cfg.ChannelSize = bad + g := New(nil, nil, nil, cfg) + if got := cap(g.eventCh); got != defaultChannelSize { + t.Fatalf("ChannelSize %d: channel cap = %d, want default %d", bad, got, defaultChannelSize) + } + } + cfg := DefaultConfig() + cfg.ChannelSize = 64 + if got := cap(New(nil, nil, nil, cfg).eventCh); got != 64 { + t.Fatalf("in-range ChannelSize not honored: cap = %d, want 64", got) + } +} diff --git a/internal/graphrag/drain.go b/internal/graphrag/drain.go index 90ad1ac..d4f3f2f 100644 --- a/internal/graphrag/drain.go +++ b/internal/graphrag/drain.go @@ -213,6 +213,13 @@ func NewDrain(opts ...DrainOption) *Drain { return d } +// TemplateCount reports the number of live templates under the read lock. +func (d *Drain) TemplateCount() int { + d.mu.RLock() + defer d.mu.RUnlock() + return len(d.templates) +} + // Match preprocesses the log line, descends the prefix tree, and either // merges into an existing template or creates a new one. Returns the // resulting template. Returns nil only for empty input. diff --git a/internal/graphrag/rebuild_incremental_test.go b/internal/graphrag/rebuild_incremental_test.go new file mode 100644 index 0000000..a895c8e --- /dev/null +++ b/internal/graphrag/rebuild_incremental_test.go @@ -0,0 +1,144 @@ +package graphrag + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" +) + +// seedSpan inserts one span row for the default tenant. +func seedSpan(t *testing.T, repo *storage.Repository, service, spanID string, start time.Time) { + t.Helper() + sp := storage.Span{ + TenantID: storage.DefaultTenantID, + TraceID: "trace-" + spanID, + SpanID: spanID, + OperationName: "/op", + ServiceName: service, + Status: "STATUS_CODE_OK", + StartTime: start, + EndTime: start.Add(time.Millisecond), + Duration: 1000, + } + if err := repo.DB().Create(&sp).Error; err != nil { + t.Fatalf("seed span %s: %v", spanID, err) + } +} + +// TestRebuild_IncrementalWindow proves the per-tenant high-water-mark: the +// second rebuild tick only re-reads spans newer than HWM minus the overlap, +// so a row inserted between ticks with an older start_time (late backfill +// outside the overlap) is not merged, while fresh rows are. +func TestRebuild_IncrementalWindow(t *testing.T) { + repo := newTestRepo(t) + g := New(repo, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + now := time.Now() + ctx := context.Background() + + // Tick 1: full window (fresh store, HWM == 0) reads both rows. + seedSpan(t, repo, "orders", "s-a", now.Add(-30*time.Minute)) + seedSpan(t, repo, "orders", "s-a2", now.Add(-10*time.Minute)) + g.rebuildAllTenantsFromDB(ctx) + + stores := g.storesForTenant(storage.DefaultTenantID) + svc, ok := stores.service.GetService("orders") + if !ok || svc.CallCount != 2 { + t.Fatalf("tick 1: orders CallCount = %v, want 2 (full window)", svc) + } + hwm := stores.lastRebuildMax.Load() + if hwm == 0 { + t.Fatalf("tick 1 did not advance the high-water-mark") + } + + // Between ticks: one backfilled row older than HWM-overlap (must be + // skipped) and one fresh row (must be merged). + seedSpan(t, repo, "ghost", "s-ghost", now.Add(-20*time.Minute)) + seedSpan(t, repo, "fresh-svc", "s-fresh", now) + + // Tick 2: incremental window = (HWM - 5min, now] = (now-15min, now]. + g.rebuildAllTenantsFromDB(ctx) + + if _, ok := stores.service.GetService("ghost"); ok { + t.Errorf("tick 2 re-read the full window — ghost (start_time < HWM-5min) was merged") + } + if _, ok := stores.service.GetService("fresh-svc"); !ok { + t.Errorf("tick 2 missed the fresh span — incremental window too narrow") + } + if got := stores.lastRebuildMax.Load(); got <= hwm { + t.Errorf("HWM did not advance on tick 2: %d -> %d", hwm, got) + } +} + +// TestRebuild_HWMNeverRegresses proves a tick that reads only older rows +// (possible when the overlap re-reads the tail) cannot move the +// high-water-mark backwards. +func TestRebuild_HWMNeverRegresses(t *testing.T) { + repo := newTestRepo(t) + g := New(repo, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + now := time.Now() + ctx := context.Background() + + seedSpan(t, repo, "orders", "s-1", now.Add(-2*time.Minute)) + g.rebuildAllTenantsFromDB(ctx) + stores := g.storesForTenant(storage.DefaultTenantID) + hwm := stores.lastRebuildMax.Load() + if hwm == 0 { + t.Fatalf("HWM not set on first rebuild") + } + + // Second tick re-reads the same row via the overlap; HWM must hold. + g.rebuildAllTenantsFromDB(ctx) + if got := stores.lastRebuildMax.Load(); got != hwm { + t.Fatalf("HWM changed without newer rows: %d -> %d", hwm, got) + } +} + +// TestRebuild_FullWindowAfterEviction proves the self-healing contract: +// a tenant evicted and re-discovered gets a fresh slice (HWM 0) and the +// next rebuild takes the full trailing window again. +func TestRebuild_FullWindowAfterEviction(t *testing.T) { + repo := newTestRepo(t) + g := New(repo, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + now := time.Now() + ctx := context.Background() + tctx := storage.WithTenantContext(ctx, "tenant-x") + + for i := 0; i < 3; i++ { + sp := storage.Span{ + TenantID: "tenant-x", + TraceID: fmt.Sprintf("t-%d", i), + SpanID: fmt.Sprintf("s-%d", i), + ServiceName: "orders", + Status: "STATUS_CODE_OK", + StartTime: now.Add(-30 * time.Minute), + EndTime: now.Add(-30 * time.Minute).Add(time.Millisecond), + } + if err := repo.DB().Create(&sp).Error; err != nil { + t.Fatalf("seed: %v", err) + } + } + g.rebuildAllTenantsFromDB(ctx) + if len(g.ServiceMap(tctx, 0)) == 0 { + t.Fatalf("tenant-x not built on first rebuild") + } + + // Evict, then rebuild: the fresh slice must take the full window and + // recover the 30min-old spans. + g.storesForTenant("tenant-x").lastAccess.Store(staleNanos()) + if g.evictIdleTenants() != 1 { + t.Fatalf("eviction did not fire") + } + g.rebuildAllTenantsFromDB(ctx) + if len(g.ServiceMap(tctx, 0)) == 0 { + t.Fatalf("evicted tenant not rebuilt with full window") + } +} diff --git a/internal/graphrag/refresh.go b/internal/graphrag/refresh.go index 14b5974..11ef375 100644 --- a/internal/graphrag/refresh.go +++ b/internal/graphrag/refresh.go @@ -8,6 +8,22 @@ import ( "github.com/RandomCodeSpace/otelcontext/internal/storage" ) +const ( + // signalRetention bounds MetricNodes and signal-store edges: anything + // not refreshed within this window is swept on the refresh tick. + signalRetention = 24 * time.Hour + // maxMetricsPerTenant caps each tenant's SignalStore metric map; past + // the cap the oldest-LastSeen series are evicted first. + maxMetricsPerTenant = 2000 + // rebuildRowLimit caps how many span rows a single per-tenant rebuild + // pass loads. Hitting it is logged — topology lags until the next tick. + rebuildRowLimit = 50000 + // rebuildOverlap is re-read behind the high-water-mark on each + // incremental rebuild so late-arriving spans with slightly older + // start_time values are still merged. + rebuildOverlap = 5 * time.Minute +) + // refreshLoop periodically rebuilds/merges from DB and prunes stale data. // Work is sharded per tenant: on each tick we snapshot the coordinator's // tenant map, then rebuild and prune each slice under its own lock. Tenants @@ -29,13 +45,22 @@ func (g *GraphRAG) refreshLoop(ctx context.Context) { case <-ticker.C: g.rebuildAllTenantsFromDB(ctx) pruned := 0 + prunedMetrics := 0 + signalCutoff := time.Now().Add(-signalRetention) for _, stores := range g.snapshotTenants() { pruned += stores.traces.Prune() + prunedMetrics += stores.signals.Prune(signalCutoff, maxMetricsPerTenant) } if pruned > 0 { slog.Debug("GraphRAG pruned expired traces/spans", "count", pruned) } + if prunedMetrics > 0 { + slog.Debug("GraphRAG pruned stale/over-cap metrics", "count", prunedMetrics) + } g.pruneOldAnomalies() + if evicted := g.evictIdleTenants(); evicted > 0 { + slog.Info("GraphRAG evicted idle tenant stores", "count", evicted) + } // Bound the investigation cooldown map. The 10m cutoff is 2× // the cooldown window (5m) — it retains entries through the // active suppression plus a grace period. This assumes the @@ -49,6 +74,39 @@ func (g *GraphRAG) refreshLoop(ctx context.Context) { } } +// evictIdleTenants drops tenant store slices whose lastAccess is older than +// tenantIdleTTL (GRAPHRAG_TENANT_IDLE_TTL, default 24h). The default tenant +// is never evicted — single-tenant installs route everything through it. +// Eviction is self-healing: a tenant that is actually active reappears +// within one refresh tick (rebuildAllTenantsFromDB re-discovers it from +// recent spans) or instantly on the next ingest event / query via +// storesForTenant. Returns the number of slices evicted. +func (g *GraphRAG) evictIdleTenants() int { + if g.tenantIdleTTL <= 0 { + return 0 + } + cutoff := time.Now().Add(-g.tenantIdleTTL).UnixNano() + g.tenantsMu.Lock() + evicted := 0 + for tenant, st := range g.tenants { + if tenant == storage.DefaultTenantID { + continue + } + if st.lastAccess.Load() < cutoff { + delete(g.tenants, tenant) + evicted++ + } + } + g.tenantsMu.Unlock() + if evicted > 0 { + g.tenantsEvicted.Add(int64(evicted)) + if g.metrics != nil && g.metrics.GraphRAGTenantsEvictedTotal != nil { + g.metrics.GraphRAGTenantsEvictedTotal.Add(float64(evicted)) + } + } + return evicted +} + // snapshotLoop persists Drain templates on the configured cadence so a // restart recovers the learned templates instead of rebuilding from scratch. // @@ -132,6 +190,20 @@ func (g *GraphRAG) rebuildAllTenantsFromDB(ctx context.Context) { } } +// incrementalSince narrows the rebuild window for a tenant that already has +// a high-water-mark: only spans newer than HWM minus rebuildOverlap need +// re-reading — at 120 services the full-window re-read every 60s dominated +// DB load. A fresh slice (first build, post-eviction) has HWM 0 and keeps +// the full trailing window. +func incrementalSince(stores *tenantStores, since time.Time) time.Time { + if hwm := stores.lastRebuildMax.Load(); hwm != 0 { + if t := time.Unix(0, hwm).Add(-rebuildOverlap); t.After(since) { + return t + } + } + return since +} + // rebuildFromDBForTenant loads recent span data for a single tenant and // merges it into that tenant's slice of the graph. Catches data from before // callbacks started (e.g., restart recovery). @@ -147,7 +219,11 @@ func (g *GraphRAG) rebuildFromDBForTenant(_ context.Context, tenant string, sinc StartTime time.Time } - stores := g.storesForTenant(tenant) + // NoTouch: a 60s bookkeeping rebuild must not refresh the idle-eviction + // clock, or dormant tenants would never reach GRAPHRAG_TENANT_IDLE_TTL. + stores := g.tenantStoresNoTouch(tenant) + + since = incrementalSince(stores, since) var rows []spanRow err := g.repo.DB(). @@ -155,12 +231,16 @@ func (g *GraphRAG) rebuildFromDBForTenant(_ context.Context, tenant string, sinc Select("span_id, parent_span_id, service_name, operation_name, duration, trace_id, status, start_time"). Where("start_time > ? AND tenant_id = ?", since, tenant). Order("start_time ASC"). - Limit(50000). + Limit(rebuildRowLimit). Find(&rows).Error if err != nil { slog.Error("GraphRAG: failed to rebuild from DB", "tenant", tenant, "error", err) return } + if len(rows) == rebuildRowLimit { + slog.Warn("GraphRAG rebuild hit the row limit — topology lags until the next tick", + "tenant", tenant, "limit", rebuildRowLimit) + } if len(rows) == 0 { return @@ -189,6 +269,12 @@ func (g *GraphRAG) rebuildFromDBForTenant(_ context.Context, tenant string, sinc } } + // Advance the high-water-mark to the newest start_time merged. Rows are + // ordered ASC, so the last row carries the max; never move backwards. + if hwm := rows[len(rows)-1].StartTime.UnixNano(); hwm > stores.lastRebuildMax.Load() { + stores.lastRebuildMax.Store(hwm) + } + slog.Debug("GraphRAG rebuilt from DB", "tenant", tenant, "spans", len(rows), diff --git a/internal/graphrag/refresh_loop_test.go b/internal/graphrag/refresh_loop_test.go new file mode 100644 index 0000000..7871bea --- /dev/null +++ b/internal/graphrag/refresh_loop_test.go @@ -0,0 +1,104 @@ +package graphrag + +import ( + "context" + "fmt" + "sync" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" + "github.com/RandomCodeSpace/otelcontext/internal/telemetry" +) + +var ( + testMetricsOnce sync.Once + testMetrics *telemetry.Metrics +) + +// sharedTestMetrics returns a process-wide telemetry.Metrics. promauto +// registers on the default Prometheus registry and panics on duplicate +// registration, so every graphrag test shares this single instance. +func sharedTestMetrics() *telemetry.Metrics { + testMetricsOnce.Do(func() { testMetrics = telemetry.New() }) + return testMetrics +} + +// TestRefreshLoop_TickPrunesAndEvicts drives a real refreshLoop tick with a +// tiny RefreshEvery and proves the tick wiring end-to-end: expired spans +// pruned (TraceStore.Prune), stale metrics pruned (SignalStore.Prune) and +// idle tenants evicted — with Prometheus metrics wired so the counter +// branches are exercised too. +func TestRefreshLoop_TickPrunesAndEvicts(t *testing.T) { + repo := newTestRepo(t) + cfg := DefaultConfig() + cfg.RefreshEvery = 20 * time.Millisecond + cfg.TraceTTL = time.Minute + g := New(repo, nil, nil, cfg) + g.SetMetrics(sharedTestMetrics()) + t.Cleanup(g.Stop) + + now := time.Now() + def := g.storesForTenant(storage.DefaultTenantID) + def.traces.UpsertSpan(mkSpanNode("s-old", now.Add(-2*time.Minute))) // past TraceTTL + def.signals.UpsertMetric("cpu", "svc", 1.0, now.Add(-48*time.Hour)) // past signalRetention + g.storesForTenant("tenant-idle").lastAccess.Store(staleNanos()) // past TenantIdleTTL + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go g.refreshLoop(ctx) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + def.traces.mu.RLock() + _, spanAlive := def.traces.Spans["s-old"] + def.traces.mu.RUnlock() + def.signals.mu.RLock() + _, metricAlive := def.signals.Metrics["cpu|svc"] + def.signals.mu.RUnlock() + g.tenantsMu.RLock() + _, tenantAlive := g.tenants["tenant-idle"] + g.tenantsMu.RUnlock() + if !spanAlive && !metricAlive && !tenantAlive { + if g.TenantsEvictedCount() == 0 { + t.Fatalf("eviction happened but the counter did not tick") + } + return // all tick effects observed + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("refreshLoop tick did not prune spans/metrics or evict the idle tenant within 2s") +} + +// TestRebuild_RowLimitMergesLimitedWindow seeds exactly rebuildRowLimit rows +// so the limit-hit branch fires (warning logged); the observable contract is +// that the pass still merges the limited window instead of bailing. +func TestRebuild_RowLimitMergesLimitedWindow(t *testing.T) { + repo := newTestRepo(t) + g := New(repo, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + now := time.Now() + spans := make([]storage.Span, rebuildRowLimit) + for i := range spans { + spans[i] = storage.Span{ + TenantID: storage.DefaultTenantID, + TraceID: "t-bulk", + SpanID: fmt.Sprintf("s-%d", i), + ServiceName: "svc", + Status: "STATUS_CODE_OK", + StartTime: now, + EndTime: now, + } + } + // Batch size bounded by SQLite's SQL-variable limit (rows × columns). + if err := repo.DB().CreateInBatches(spans, 500).Error; err != nil { + t.Fatalf("bulk seed: %v", err) + } + + g.rebuildAllTenantsFromDB(context.Background()) + + if _, ok := g.storesForTenant(storage.DefaultTenantID).service.GetService("svc"); !ok { + t.Fatalf("rebuild with row-limit hit did not merge any services") + } +} diff --git a/internal/graphrag/signal_prune_test.go b/internal/graphrag/signal_prune_test.go new file mode 100644 index 0000000..925f07d --- /dev/null +++ b/internal/graphrag/signal_prune_test.go @@ -0,0 +1,124 @@ +package graphrag + +import ( + "fmt" + "testing" + "time" +) + +// TestSignalStore_Prune_DropsStaleMetricsKeepsFresh proves the TTL half of +// the SignalStore bound: metrics idle past the cutoff are removed together +// with their MEASURED_BY edge, while a fresh metric and its edge survive. +func TestSignalStore_Prune_DropsStaleMetricsKeepsFresh(t *testing.T) { + ss := newSignalStore() + now := time.Now() + stale := now.Add(-48 * time.Hour) + + ss.UpsertMetric("cpu", "old-svc", 1.0, stale) + ss.UpsertMetric("cpu", "new-svc", 1.0, now) + + if got := ss.Prune(now.Add(-24*time.Hour), 0); got != 1 { + t.Fatalf("Prune removed %d metrics, want 1", got) + } + if _, ok := ss.Metrics["cpu|old-svc"]; ok { + t.Errorf("stale metric survived Prune") + } + if _, ok := ss.Metrics["cpu|new-svc"]; !ok { + t.Errorf("fresh metric was pruned") + } + if _, ok := ss.Edges[edgeKey(EdgeMeasuredBy, "cpu|old-svc", "old-svc")]; ok { + t.Errorf("stale metric's MEASURED_BY edge survived Prune") + } + if _, ok := ss.Edges[edgeKey(EdgeMeasuredBy, "cpu|new-svc", "new-svc")]; !ok { + t.Errorf("fresh metric's MEASURED_BY edge was swept") + } +} + +// TestSignalStore_Prune_CapEvictsOldestFirst proves the cap half: when the +// map still exceeds maxMetrics after the TTL pass, the oldest-LastSeen +// entries go first and their edges go with them. +func TestSignalStore_Prune_CapEvictsOldestFirst(t *testing.T) { + ss := newSignalStore() + now := time.Now() + + // 5 fresh metrics with strictly ascending LastSeen. + for i := 0; i < 5; i++ { + ss.UpsertMetric(fmt.Sprintf("m%d", i), "svc", 1.0, now.Add(time.Duration(i)*time.Minute)) + } + + if got := ss.Prune(now.Add(-24*time.Hour), 2); got != 3 { + t.Fatalf("Prune removed %d metrics, want 3", got) + } + for i := 0; i < 3; i++ { + key := fmt.Sprintf("m%d|svc", i) + if _, ok := ss.Metrics[key]; ok { + t.Errorf("oldest metric %s survived cap eviction", key) + } + if _, ok := ss.Edges[edgeKey(EdgeMeasuredBy, key, "svc")]; ok { + t.Errorf("cap-evicted metric %s left a dangling MEASURED_BY edge", key) + } + } + for i := 3; i < 5; i++ { + if _, ok := ss.Metrics[fmt.Sprintf("m%d|svc", i)]; !ok { + t.Errorf("newest metric m%d was evicted — cap must drop oldest first", i) + } + } +} + +// TestSignalStore_Prune_SweepsStaleLogClusterEdgesOnly proves LogClusters +// themselves are untouched (they are Drain-bounded upstream) while their +// stale EMITTED_BY / LOGGED_DURING edges are swept. +func TestSignalStore_Prune_SweepsStaleLogClusterEdgesOnly(t *testing.T) { + ss := newSignalStore() + now := time.Now() + stale := now.Add(-48 * time.Hour) + + ss.UpsertLogCluster("c-old", "tmpl", "ERROR", "svc", stale) + ss.AddLoggedDuringEdge("c-old", "span-old", stale) + ss.UpsertLogCluster("c-new", "tmpl", "ERROR", "svc2", now) + + ss.Prune(now.Add(-24*time.Hour), 0) + + if len(ss.LogClusters) != 2 { + t.Fatalf("LogClusters len = %d, want 2 — Prune must not touch clusters", len(ss.LogClusters)) + } + if _, ok := ss.Edges[edgeKey(EdgeEmittedBy, "c-old", "svc")]; ok { + t.Errorf("stale EMITTED_BY edge survived") + } + if _, ok := ss.Edges[edgeKey(EdgeLoggedDuring, "c-old", "span-old")]; ok { + t.Errorf("stale LOGGED_DURING edge survived") + } + if _, ok := ss.Edges[edgeKey(EdgeEmittedBy, "c-new", "svc2")]; !ok { + t.Errorf("fresh EMITTED_BY edge was swept") + } +} + +// TestSignalStore_UpsertRefreshesEdgeTimestamps proves the upsert paths +// refresh edge UpdatedAt on every hit — without this, a continuously-active +// metric or log cluster would lose its correlation edge 24h after creation +// even though the node itself stays fresh. +func TestSignalStore_UpsertRefreshesEdgeTimestamps(t *testing.T) { + ss := newSignalStore() + now := time.Now() + stale := now.Add(-48 * time.Hour) + + // Created stale, then re-upserted fresh: edges must carry the fresh time. + ss.UpsertMetric("cpu", "svc", 1.0, stale) + ss.UpsertMetric("cpu", "svc", 2.0, now) + ss.UpsertLogCluster("c1", "tmpl", "ERROR", "svc", stale) + ss.UpsertLogCluster("c1", "tmpl", "ERROR", "svc", now) + ss.AddLoggedDuringEdge("c1", "span-1", stale) + ss.AddLoggedDuringEdge("c1", "span-1", now) + + ss.Prune(now.Add(-24*time.Hour), 0) + + for _, ek := range []string{ + edgeKey(EdgeMeasuredBy, "cpu|svc", "svc"), + edgeKey(EdgeEmittedBy, "c1", "svc"), + edgeKey(EdgeLoggedDuring, "c1", "span-1"), + } { + if _, ok := ss.Edges[ek]; !ok { + t.Errorf("edge %s swept despite a fresh upsert — UpdatedAt not refreshed", ek) + } + } +} diff --git a/internal/graphrag/span_cap_test.go b/internal/graphrag/span_cap_test.go new file mode 100644 index 0000000..7341902 --- /dev/null +++ b/internal/graphrag/span_cap_test.go @@ -0,0 +1,106 @@ +package graphrag + +import ( + "fmt" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" +) + +// mkSpanNode builds a minimal SpanNode for cap tests. +func mkSpanNode(id string, ts time.Time) SpanNode { + return SpanNode{ + ID: id, + TraceID: "trace-cap", + Service: "orders", + Operation: "/op", + Duration: 1.0, + Timestamp: ts, + } +} + +// TestTraceStore_UpsertSpan_CapBlocksNewSpans proves the per-tenant span cap: +// once len(Spans) reaches MaxSpans, NEW span IDs are rejected (UpsertSpan +// returns false and the map does not grow) while updates to already-resident +// span IDs still apply. +func TestTraceStore_UpsertSpan_CapBlocksNewSpans(t *testing.T) { + ts := newTraceStore(time.Hour, 3) + now := time.Now() + + for i := 0; i < 3; i++ { + if !ts.UpsertSpan(mkSpanNode(fmt.Sprintf("s-%d", i), now)) { + t.Fatalf("span s-%d rejected below the cap", i) + } + } + + // 4th NEW span: at cap, must be rejected. + if ts.UpsertSpan(mkSpanNode("s-overflow", now)) { + t.Fatalf("new span accepted at cap — MaxSpans not enforced") + } + if got := len(ts.Spans); got != 3 { + t.Fatalf("Spans len = %d after rejected insert, want 3", got) + } + + // Update to an EXISTING span ID at cap: must still apply. + updated := mkSpanNode("s-1", now) + updated.Duration = 42.0 + if !ts.UpsertSpan(updated) { + t.Fatalf("update to resident span s-1 rejected at cap") + } + if got, _ := ts.GetSpan("s-1"); got.Duration != 42.0 { + t.Fatalf("resident span update not applied at cap: Duration=%v, want 42", got.Duration) + } +} + +// TestTraceStore_UpsertSpan_CapDisabled proves MaxSpans <= 0 leaves the +// store unbounded (legacy behavior). +func TestTraceStore_UpsertSpan_CapDisabled(t *testing.T) { + ts := newTraceStore(time.Hour, 0) + now := time.Now() + for i := 0; i < 10; i++ { + if !ts.UpsertSpan(mkSpanNode(fmt.Sprintf("s-%d", i), now)) { + t.Fatalf("span rejected with cap disabled") + } + } + if got := len(ts.Spans); got != 10 { + t.Fatalf("Spans len = %d, want 10", got) + } +} + +// TestProcessSpan_CapDropRecorded proves the coordinator surfaces cap drops: +// when a tenant's TraceStore is full, processSpan records the skip via the +// span_capacity drop counter (and the Prometheus counter when wired). +func TestProcessSpan_CapDropRecorded(t *testing.T) { + cfg := DefaultConfig() + cfg.MaxSpansPerTenant = 2 + g := New(nil, nil, nil, cfg) + g.SetMetrics(sharedTestMetrics()) // exercise the Prometheus drop path too + t.Cleanup(g.Stop) + + now := time.Now() + for i := 0; i < 3; i++ { + g.processSpan(&spanEvent{ + Span: storage.Span{ + TraceID: "trace-cap", + SpanID: fmt.Sprintf("s-%d", i), + ServiceName: "orders", + StartTime: now, + }, + TraceID: "trace-cap", + Status: "STATUS_CODE_UNSET", + Tenant: storage.DefaultTenantID, + }) + } + + if got := g.SpanCapacityDropsCount(); got != 1 { + t.Fatalf("SpanCapacityDropsCount = %d, want 1", got) + } + stores := g.storesForTenant(storage.DefaultTenantID) + stores.traces.mu.RLock() + n := len(stores.traces.Spans) + stores.traces.mu.RUnlock() + if n != 2 { + t.Fatalf("tenant span count = %d, want 2 (cap)", n) + } +} diff --git a/internal/graphrag/store.go b/internal/graphrag/store.go index 8d2caa3..9fb5856 100644 --- a/internal/graphrag/store.go +++ b/internal/graphrag/store.go @@ -2,7 +2,9 @@ package graphrag import ( "math" + "sort" "sync" + "sync/atomic" "time" ) @@ -16,15 +18,37 @@ type tenantStores struct { traces *TraceStore signals *SignalStore anomalies *AnomalyStore + + // lastAccess is the unix-nano time of the most recent ingest event or + // query routed through storesForTenant. Drives idle-tenant eviction in + // refreshLoop; background maintenance uses tenantStoresNoTouch so a 60s + // bookkeeping pass cannot keep a dormant tenant alive. + lastAccess atomic.Int64 + + // lastEventAt is the unix-nano time of the most recent span/log/metric + // processed for this tenant. detectAnomalies skips tenants whose value + // predates the previous scan tick — their stats cannot have changed. + lastEventAt atomic.Int64 + + // lastRebuildMax is the high-water-mark (unix nanos) of span start_time + // merged by rebuildFromDBForTenant. Subsequent rebuilds only re-read + // rows newer than HWM minus a small overlap instead of the full + // trailing window. Zero on a fresh slice (first build / post-eviction) + // forces a full-window rebuild. + lastRebuildMax atomic.Int64 } -func newTenantStores(traceTTL time.Duration) *tenantStores { - return &tenantStores{ +func newTenantStores(traceTTL time.Duration, maxSpans int) *tenantStores { + ts := &tenantStores{ service: newServiceStore(), - traces: newTraceStore(traceTTL), + traces: newTraceStore(traceTTL, maxSpans), signals: newSignalStore(), anomalies: newAnomalyStore(), } + // Creation counts as access — a slice re-created by the DB rebuild gets + // a full idle window rather than being evicted on the next tick. + ts.lastAccess.Store(time.Now().UnixNano()) + return ts } // ServiceStore holds permanent service topology data. @@ -50,14 +74,19 @@ type TraceStore struct { Spans map[string]*SpanNode // key: span_id Edges map[string]*Edge // key: type|from|to TTL time.Duration + // MaxSpans hard-caps the Spans map: at the cap, NEW span IDs are + // skipped (UpsertSpan returns false) while updates to resident IDs + // still apply. <=0 disables the cap. + MaxSpans int } -func newTraceStore(ttl time.Duration) *TraceStore { +func newTraceStore(ttl time.Duration, maxSpans int) *TraceStore { return &TraceStore{ - Traces: make(map[string]*TraceNode), - Spans: make(map[string]*SpanNode), - Edges: make(map[string]*Edge), - TTL: ttl, + Traces: make(map[string]*TraceNode), + Spans: make(map[string]*SpanNode), + Edges: make(map[string]*Edge), + TTL: ttl, + MaxSpans: maxSpans, } } @@ -270,10 +299,19 @@ func (ts *TraceStore) UpsertTrace(traceID, rootService, status string, durationM } } -func (ts *TraceStore) UpsertSpan(span SpanNode) { +// UpsertSpan inserts or updates a span node and its CONTAINS/CHILD_OF edges. +// Returns false when the span is NEW and MaxSpans is already reached — the +// span is skipped entirely (the graph is best-effort; the DB is the source +// of truth, same doctrine as the event-channel overflow in builder.go). +func (ts *TraceStore) UpsertSpan(span SpanNode) bool { ts.mu.Lock() defer ts.mu.Unlock() + if ts.MaxSpans > 0 && len(ts.Spans) >= ts.MaxSpans { + if _, resident := ts.Spans[span.ID]; !resident { + return false + } + } ts.Spans[span.ID] = &span // CONTAINS edge: trace → span @@ -299,6 +337,7 @@ func (ts *TraceStore) UpsertSpan(span SpanNode) { } } } + return true } func (ts *TraceStore) GetSpan(spanID string) (*SpanNode, bool) { @@ -412,9 +451,14 @@ func (ss *SignalStore) UpsertLogClusterWithTemplate(id, template, severity, serv lc.LastSeen = ts } - // EMITTED_BY edge + // EMITTED_BY edge. Refresh UpdatedAt on every hit so Prune's edge sweep + // never severs a cluster that is still receiving logs. ek := edgeKey(EdgeEmittedBy, id, service) - if _, exists := ss.Edges[ek]; !exists { + if e, exists := ss.Edges[ek]; exists { + if ts.After(e.UpdatedAt) { + e.UpdatedAt = ts + } + } else { ss.Edges[ek] = &Edge{ Type: EdgeEmittedBy, FromID: id, @@ -428,13 +472,18 @@ func (ss *SignalStore) AddLoggedDuringEdge(clusterID, spanID string, ts time.Tim ek := edgeKey(EdgeLoggedDuring, clusterID, spanID) ss.mu.Lock() defer ss.mu.Unlock() - if _, exists := ss.Edges[ek]; !exists { - ss.Edges[ek] = &Edge{ - Type: EdgeLoggedDuring, - FromID: clusterID, - ToID: spanID, - UpdatedAt: ts, + if e, exists := ss.Edges[ek]; exists { + // Keep the correlation alive across Prune's edge sweep. + if ts.After(e.UpdatedAt) { + e.UpdatedAt = ts } + return + } + ss.Edges[ek] = &Edge{ + Type: EdgeLoggedDuring, + FromID: clusterID, + ToID: spanID, + UpdatedAt: ts, } } @@ -455,9 +504,15 @@ func (ss *SignalStore) UpsertMetric(metricName, service string, value float64, t LastSeen: ts, } ss.Metrics[key] = m - - // MEASURED_BY edge - ek := edgeKey(EdgeMeasuredBy, key, service) + } + // MEASURED_BY edge — created on the first sample, UpdatedAt refreshed on + // every sample so Prune's edge sweep never severs a live metric. + ek := edgeKey(EdgeMeasuredBy, key, service) + if e, exists := ss.Edges[ek]; exists { + if ts.After(e.UpdatedAt) { + e.UpdatedAt = ts + } + } else { ss.Edges[ek] = &Edge{ Type: EdgeMeasuredBy, FromID: key, @@ -491,6 +546,51 @@ func (ss *SignalStore) LogClustersForService(service string) []*LogClusterNode { return out } +// Prune bounds the SignalStore (modeled on TraceStore.Prune): MetricNodes +// whose LastSeen predates cutoff are removed; if the map still exceeds +// maxMetrics (<=0 = uncapped) the oldest-LastSeen overflow is evicted. Each +// removed metric takes its MEASURED_BY edge with it. Finally, Edges whose +// UpdatedAt predates cutoff are swept — the upsert paths refresh edge +// timestamps, so live correlations survive. LogClusters are NOT pruned +// here: they are bounded upstream by the Drain template LRU; only their +// stale edges go. Returns the number of metrics removed. +func (ss *SignalStore) Prune(cutoff time.Time, maxMetrics int) int { + ss.mu.Lock() + defer ss.mu.Unlock() + + pruned := 0 + for id, m := range ss.Metrics { + if m.LastSeen.Before(cutoff) { + delete(ss.Metrics, id) + delete(ss.Edges, edgeKey(EdgeMeasuredBy, id, m.Service)) + pruned++ + } + } + if maxMetrics > 0 && len(ss.Metrics) > maxMetrics { + type metricAge struct { + id string + service string + seen time.Time + } + byAge := make([]metricAge, 0, len(ss.Metrics)) + for id, m := range ss.Metrics { + byAge = append(byAge, metricAge{id, m.Service, m.LastSeen}) + } + sort.Slice(byAge, func(i, j int) bool { return byAge[i].seen.Before(byAge[j].seen) }) + for _, e := range byAge[:len(byAge)-maxMetrics] { + delete(ss.Metrics, e.id) + delete(ss.Edges, edgeKey(EdgeMeasuredBy, e.id, e.service)) + pruned++ + } + } + for ek, e := range ss.Edges { + if e.UpdatedAt.Before(cutoff) { + delete(ss.Edges, ek) + } + } + return pruned +} + func (ss *SignalStore) MetricsForService(service string) []*MetricNode { ss.mu.RLock() defer ss.mu.RUnlock() @@ -533,12 +633,24 @@ func (as *AnomalyStore) AddPrecededByEdge(anomalyID, precedingID string, ts time } func (as *AnomalyStore) AnomaliesSince(since time.Time) []*AnomalyNode { + return as.AnomaliesSinceLimit(since, 0) +} + +// AnomaliesSinceLimit is AnomaliesSince with a result cap (n <= 0 means +// unlimited). correlateWithRecent walks this on every detection tick, so the +// cap keeps a pathological anomaly backlog from turning each tick into an +// O(N) scan plus O(N) edge fan-out. Selection past the cap follows map +// iteration order — correlation is best-effort by design. +func (as *AnomalyStore) AnomaliesSinceLimit(since time.Time, n int) []*AnomalyNode { as.mu.RLock() defer as.mu.RUnlock() var out []*AnomalyNode for _, a := range as.Anomalies { if a.Timestamp.After(since) { out = append(out, a) + if n > 0 && len(out) >= n { + break + } } } return out diff --git a/internal/graphrag/store_counts.go b/internal/graphrag/store_counts.go new file mode 100644 index 0000000..30067c8 --- /dev/null +++ b/internal/graphrag/store_counts.go @@ -0,0 +1,62 @@ +package graphrag + +// StoreCounts is a point-in-time census of every long-lived in-memory +// structure the coordinator owns, aggregated across tenants. It feeds the +// otelcontext_graphrag_* gauges so operators can attribute RSS growth to a +// specific store before reaching for a heap profile. +type StoreCounts struct { + Tenants int + Services int + Operations int + Traces int + Spans int + LogClusters int + Metrics int + Anomalies int + ServiceEdges int + TraceEdges int + SignalEdges int + AnomalyEdges int +} + +// StoreCounts walks a tenant snapshot and takes len() under each store's +// read lock — no slice building, cheap enough for a periodic sampler. +func (g *GraphRAG) StoreCounts() StoreCounts { + var c StoreCounts + for _, st := range g.snapshotTenants() { + c.Tenants++ + + st.service.mu.RLock() + c.Services += len(st.service.Services) + c.Operations += len(st.service.Operations) + c.ServiceEdges += len(st.service.Edges) + st.service.mu.RUnlock() + + st.traces.mu.RLock() + c.Traces += len(st.traces.Traces) + c.Spans += len(st.traces.Spans) + c.TraceEdges += len(st.traces.Edges) + st.traces.mu.RUnlock() + + st.signals.mu.RLock() + c.LogClusters += len(st.signals.LogClusters) + c.Metrics += len(st.signals.Metrics) + c.SignalEdges += len(st.signals.Edges) + st.signals.mu.RUnlock() + + st.anomalies.mu.RLock() + c.Anomalies += len(st.anomalies.Anomalies) + c.AnomalyEdges += len(st.anomalies.Edges) + st.anomalies.mu.RUnlock() + } + return c +} + +// DrainTemplateCount reports the number of live Drain templates (bounded by +// maxTemplates, but the gauge proves it in production). +func (g *GraphRAG) DrainTemplateCount() int { + if g.drain == nil { + return 0 + } + return g.drain.TemplateCount() +} diff --git a/internal/graphrag/store_counts_test.go b/internal/graphrag/store_counts_test.go new file mode 100644 index 0000000..b65e2a9 --- /dev/null +++ b/internal/graphrag/store_counts_test.go @@ -0,0 +1,74 @@ +package graphrag + +import ( + "testing" + "time" +) + +func TestStoreCountsAggregatesAcrossTenants(t *testing.T) { + g := newTestGraphRAG(t) + now := time.Now() + + a := g.storesForTenant("tenant-a") + a.service.UpsertService("svc-1", 10, false, now) + a.service.UpsertService("svc-2", 20, true, now) + a.service.UpsertOperation("svc-1", "GET /x", 10, false, now) + a.service.UpsertCallEdge("svc-1", "svc-2", 10, false, now) + a.traces.UpsertTrace("trace-1", "svc-1", "OK", 12, now) + a.traces.UpsertSpan(SpanNode{ID: "span-1", TraceID: "trace-1", Service: "svc-1", Timestamp: now}) + + b := g.storesForTenant("tenant-b") + b.signals.UpsertLogCluster("cl-1", "tmpl", "ERROR", "svc-3", now) + b.signals.UpsertMetric("cpu", "svc-3", 0.5, now) + b.anomalies.AddAnomaly(AnomalyNode{ID: "anom-1", Service: "svc-3", Timestamp: now}) + b.anomalies.AddAnomaly(AnomalyNode{ID: "anom-2", Service: "svc-3", Timestamp: now}) + b.anomalies.AddPrecededByEdge("anom-2", "anom-1", now) + + c := g.StoreCounts() + + if c.Tenants < 2 { + t.Fatalf("Tenants = %d, want >= 2", c.Tenants) + } + if c.Services != 2 { + t.Fatalf("Services = %d, want 2", c.Services) + } + if c.Operations != 1 { + t.Fatalf("Operations = %d, want 1", c.Operations) + } + if c.Spans != 1 { + t.Fatalf("Spans = %d, want 1", c.Spans) + } + if c.LogClusters != 1 { + t.Fatalf("LogClusters = %d, want 1", c.LogClusters) + } + if c.Metrics != 1 { + t.Fatalf("Metrics = %d, want 1", c.Metrics) + } + if c.Anomalies != 2 { + t.Fatalf("Anomalies = %d, want 2", c.Anomalies) + } + // UpsertCallEdge creates a CALLS edge; UpsertOperation an EXPOSES edge. + if c.ServiceEdges != 2 { + t.Fatalf("ServiceEdges = %d, want 2", c.ServiceEdges) + } + // Each AddAnomaly creates a TRIGGERED_BY edge, plus one PRECEDED_BY edge. + if c.AnomalyEdges != 3 { + t.Fatalf("AnomalyEdges = %d, want 3", c.AnomalyEdges) + } + // Metric upsert creates a MEASURED_BY edge; log cluster upsert an EMITTED_BY edge. + if c.SignalEdges != 2 { + t.Fatalf("SignalEdges = %d, want 2", c.SignalEdges) + } +} + +func TestDrainTemplateCount(t *testing.T) { + d := NewDrain() + if got := d.TemplateCount(); got != 0 { + t.Fatalf("empty miner TemplateCount = %d, want 0", got) + } + d.Match("user 42 logged in", time.Now()) + d.Match("user 43 logged in", time.Now()) + if got := d.TemplateCount(); got != 1 { + t.Fatalf("TemplateCount = %d, want 1 (same template)", got) + } +} diff --git a/internal/graphrag/tenant_eviction_test.go b/internal/graphrag/tenant_eviction_test.go new file mode 100644 index 0000000..1f6b080 --- /dev/null +++ b/internal/graphrag/tenant_eviction_test.go @@ -0,0 +1,130 @@ +package graphrag + +import ( + "context" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" +) + +// staleNanos returns a unix-nano timestamp comfortably past any idle TTL +// used in these tests. +func staleNanos() int64 { + return time.Now().Add(-48 * time.Hour).UnixNano() +} + +// TestEvictIdleTenants_EvictsStaleKeepsDefault proves the three eviction +// invariants: an idle tenant is removed, the default tenant is immune no +// matter how stale, and a fresh tenant survives. +func TestEvictIdleTenants_EvictsStaleKeepsDefault(t *testing.T) { + g := New(nil, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + stale := g.storesForTenant("tenant-idle") + fresh := g.storesForTenant("tenant-active") + def := g.storesForTenant(storage.DefaultTenantID) + + stale.lastAccess.Store(staleNanos()) + def.lastAccess.Store(staleNanos()) // default tenant must survive anyway + _ = fresh // lastAccess just touched by storesForTenant + + if got := g.evictIdleTenants(); got != 1 { + t.Fatalf("evictIdleTenants = %d, want 1", got) + } + + g.tenantsMu.RLock() + _, idleAlive := g.tenants["tenant-idle"] + _, activeAlive := g.tenants["tenant-active"] + _, defAlive := g.tenants[storage.DefaultTenantID] + g.tenantsMu.RUnlock() + + if idleAlive { + t.Errorf("idle tenant still resident after eviction") + } + if !activeAlive { + t.Errorf("active tenant was evicted") + } + if !defAlive { + t.Errorf("default tenant was evicted — must be immune") + } + if got := g.TenantsEvictedCount(); got != 1 { + t.Errorf("TenantsEvictedCount = %d, want 1", got) + } +} + +// TestEvictIdleTenants_ReactivationCreatesFreshSlice proves eviction is +// self-healing: the next storesForTenant call re-creates the slice with a +// fresh idle window, so a returning tenant is not insta-evicted. +func TestEvictIdleTenants_ReactivationCreatesFreshSlice(t *testing.T) { + g := New(nil, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + old := g.storesForTenant("tenant-a") + old.lastAccess.Store(staleNanos()) + if got := g.evictIdleTenants(); got != 1 { + t.Fatalf("evictIdleTenants = %d, want 1", got) + } + + reborn := g.storesForTenant("tenant-a") + if reborn == old { + t.Fatalf("storesForTenant returned the evicted slice — expected a fresh one") + } + if g.evictIdleTenants() != 0 { + t.Fatalf("freshly re-created tenant must not be evicted") + } +} + +// TestEvictIdleTenants_DisabledWhenNegative proves a negative TenantIdleTTL +// turns eviction off entirely. +func TestEvictIdleTenants_DisabledWhenNegative(t *testing.T) { + cfg := DefaultConfig() + cfg.TenantIdleTTL = -1 + g := New(nil, nil, nil, cfg) + t.Cleanup(g.Stop) + + g.storesForTenant("tenant-b").lastAccess.Store(staleNanos()) + if got := g.evictIdleTenants(); got != 0 { + t.Fatalf("eviction ran with TenantIdleTTL<0: evicted %d", got) + } +} + +// TestRebuild_DoesNotRefreshIdleClock proves the 60s DB rebuild cannot keep +// a dormant tenant alive: rebuildFromDBForTenant goes through +// tenantStoresNoTouch, so lastAccess stays stale and the next eviction pass +// still fires. Without this, every known tenant would be touched every tick +// and GRAPHRAG_TENANT_IDLE_TTL would never elapse. +func TestRebuild_DoesNotRefreshIdleClock(t *testing.T) { + repo := newTestRepo(t) + g := New(repo, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + st := g.storesForTenant("tenant-dormant") + stale := staleNanos() + st.lastAccess.Store(stale) + + // Rebuild walks known tenants even when the DB has no rows for them. + g.rebuildAllTenantsFromDB(context.Background()) + + if got := st.lastAccess.Load(); got != stale { + t.Fatalf("DB rebuild refreshed lastAccess (%d → %d) — dormant tenant would never be evicted", stale, got) + } + if got := g.evictIdleTenants(); got != 1 { + t.Fatalf("evictIdleTenants after rebuild = %d, want 1", got) + } +} + +// TestStoresForTenant_RefreshesLastAccess proves the fast path (slice already +// resident) refreshes the idle clock on every call. +func TestStoresForTenant_RefreshesLastAccess(t *testing.T) { + g := New(nil, nil, nil, DefaultConfig()) + t.Cleanup(g.Stop) + + st := g.storesForTenant("tenant-c") + st.lastAccess.Store(staleNanos()) + + g.storesForTenant("tenant-c") // fast path + if age := time.Since(time.Unix(0, st.lastAccess.Load())); age > time.Minute { + t.Fatalf("lastAccess not refreshed on fast path: %v old", age) + } +} diff --git a/internal/httpconst/httpconst.go b/internal/httpconst/httpconst.go index 9d28c1f..42cd107 100644 --- a/internal/httpconst/httpconst.go +++ b/internal/httpconst/httpconst.go @@ -7,6 +7,10 @@ const ( // HeaderContentType is the canonical HTTP Content-Type header name. HeaderContentType = "Content-Type" + // HeaderContentEncoding is the canonical HTTP Content-Encoding header + // name, shared by the UI asset server and the API gzip middleware. + HeaderContentEncoding = "Content-Encoding" + // ContentTypeJSON is the application/json content type used by every JSON // response on the API and MCP surface. ContentTypeJSON = "application/json" diff --git a/internal/httpconst/negotiate.go b/internal/httpconst/negotiate.go new file mode 100644 index 0000000..e42f078 --- /dev/null +++ b/internal/httpconst/negotiate.go @@ -0,0 +1,48 @@ +package httpconst + +import ( + "net/http" + "strconv" + "strings" +) + +// AcceptsEncoding reports whether the request's Accept-Encoding header lists +// the given encoding with a non-zero q-value. Minimal token parse — the "*" +// wildcard is deliberately not honoured (browsers and collectors always name +// the encodings they support), so callers never have to guess a preference +// order for it. +func AcceptsEncoding(r *http.Request, enc string) bool { + for _, part := range strings.Split(r.Header.Get("Accept-Encoding"), ",") { + token, attr, hasAttr := strings.Cut(strings.TrimSpace(part), ";") + if !strings.EqualFold(strings.TrimSpace(token), enc) { + continue + } + if hasAttr { + attr = strings.TrimSpace(attr) + if qv, ok := strings.CutPrefix(attr, "q="); ok { + if q, err := strconv.ParseFloat(strings.TrimSpace(qv), 64); err == nil && q == 0 { + return false + } + } + } + return true + } + return false +} + +// ETagMatch reports whether an If-None-Match header value matches the given +// entity tag. Handles the "*" wildcard, comma-separated candidate lists, and +// weak validators (W/ prefix is ignored — weak comparison is correct for +// If-None-Match per RFC 9110 §13.1.2). +func ETagMatch(header, etag string) bool { + if header == "" { + return false + } + for _, candidate := range strings.Split(header, ",") { + candidate = strings.TrimPrefix(strings.TrimSpace(candidate), "W/") + if candidate == "*" || candidate == etag { + return true + } + } + return false +} diff --git a/internal/httpconst/negotiate_test.go b/internal/httpconst/negotiate_test.go new file mode 100644 index 0000000..3e7c1c2 --- /dev/null +++ b/internal/httpconst/negotiate_test.go @@ -0,0 +1,64 @@ +package httpconst + +import ( + "net/http/httptest" + "testing" +) + +func TestAcceptsEncoding(t *testing.T) { + tests := []struct { + name string + header string + enc string + want bool + }{ + {"empty header", "", "gzip", false}, + {"plain match", "gzip", "gzip", true}, + {"case insensitive", "GZip", "gzip", true}, + {"list match", "deflate, gzip, br", "gzip", true}, + {"list match br", "gzip, br", "br", true}, + {"no match", "deflate", "gzip", false}, + {"substring is not a token", "xgzipx", "gzip", false}, + {"q value accepted", "gzip;q=0.5", "gzip", true}, + {"q zero rejected", "gzip;q=0", "gzip", false}, + {"q zero decimal rejected", "gzip;q=0.000", "gzip", false}, + {"q zero in list", "br;q=0, gzip", "br", false}, + {"whitespace tolerated", " gzip ; q=1.0 , br", "gzip", true}, + {"wildcard not honoured", "*", "gzip", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := httptest.NewRequest("GET", "/", nil) + if tt.header != "" { + r.Header.Set("Accept-Encoding", tt.header) + } + if got := AcceptsEncoding(r, tt.enc); got != tt.want { + t.Errorf("AcceptsEncoding(%q, %q) = %v, want %v", tt.header, tt.enc, got, tt.want) + } + }) + } +} + +func TestETagMatch(t *testing.T) { + const etag = `"abc123"` + tests := []struct { + name string + header string + want bool + }{ + {"empty header", "", false}, + {"exact match", `"abc123"`, true}, + {"no match", `"zzz"`, false}, + {"wildcard", "*", true}, + {"list match", `"zzz", "abc123"`, true}, + {"weak validator match", `W/"abc123"`, true}, + {"unquoted does not match", "abc123", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ETagMatch(tt.header, etag); got != tt.want { + t.Errorf("ETagMatch(%q, %q) = %v, want %v", tt.header, etag, got, tt.want) + } + }) + } +} diff --git a/internal/ingest/otlp_http.go b/internal/ingest/otlp_http.go index d9c6fce..32640c0 100644 --- a/internal/ingest/otlp_http.go +++ b/internal/ingest/otlp_http.go @@ -37,9 +37,16 @@ const headerContentType = "Content-Type" //nolint:goconst // single literal; Son // to the request context before delegating to the gRPC Export methods. // Uses the shared storage.WithTenantContext helper so ingest and read paths // agree on the context key. +// +// The raw header value is run through storage.SanitizeTenantID — the same +// sanitizer applied on the gRPC metadata path in tenantFromContext — so +// control characters, oversized strings, and empty values are rejected +// identically regardless of transport. func withTenantFromHTTP(r *http.Request) context.Context { if v := r.Header.Get("X-Tenant-ID"); v != "" { - return storage.WithTenantContext(r.Context(), v) + if sanitized := storage.SanitizeTenantID(v); sanitized != "" { + return storage.WithTenantContext(r.Context(), sanitized) + } } return r.Context() } diff --git a/internal/ingest/pipeline.go b/internal/ingest/pipeline.go index 868a300..4881fdb 100644 --- a/internal/ingest/pipeline.go +++ b/internal/ingest/pipeline.go @@ -60,6 +60,34 @@ type Batch struct { LogCallback func(storage.Log) enqueuedAt time.Time + + // sizeBytes is the approxBytes() estimate computed once at Submit time + // and released by process() — keeping the reservation and the release + // reading the same number even if the slices are mutated in between. + sizeBytes int64 +} + +// approxBytes estimates the heap footprint of the batch without marshaling. +// Per-record fixed costs approximate struct overhead (time.Time fields, +// GORM bookkeeping, slice/string headers); variable costs are the lengths +// of the dominant string payloads — Body and AttributesJSON are what make +// a batch fat. AttributesJSON/AIInsight are storage.CompressedText (string +// kind), so len() over the string cast is O(1). Whole walk is O(records). +func (b *Batch) approxBytes() int64 { + var n int64 + for i := range b.Traces { + t := &b.Traces[i] + n += 128 + int64(len(t.TraceID)+len(t.ServiceName)+len(t.Status)+len(t.Operation)) + } + for i := range b.Spans { + s := &b.Spans[i] + n += 256 + int64(len(s.OperationName)+len(string(s.AttributesJSON))+len(s.ServiceName)+len(s.Status)) + } + for i := range b.Logs { + l := &b.Logs[i] + n += 192 + int64(len(l.Body)+len(string(l.AttributesJSON))+len(string(l.AIInsight))+len(l.ServiceName)+len(l.Severity)) + } + return n } // Priority reports whether the batch is protected from soft-backpressure @@ -76,6 +104,12 @@ type PipelineConfig struct { Capacity int // total queue depth across all signal types Workers int // worker goroutines draining the queue SoftThreshold float64 // fullness fraction above which healthy batches are dropped (0.0–1.0) + // MaxBytes caps the approximate bytes held by queued batches. Capacity + // alone cannot bound memory — a single Batch may carry arbitrarily + // large span/log payloads. At the cap, Submit rejects with ErrQueueFull + // even for priority batches: a 429 is recoverable by the client's retry + // loop, an OOM kill is not. <=0 falls back to the 512MB default. + MaxBytes int64 } // Defensive upper bounds on operator-supplied capacity/workers. Env-var @@ -87,6 +121,10 @@ type PipelineConfig struct { const ( maxPipelineCapacity = 1_000_000 maxPipelineWorkers = 256 + // minPipelineMaxBytes is the floor for the byte cap. A sub-1MB cap + // would reject every gRPC batch outright (GRPC_MAX_RECV_MB default is + // 16) — clamp up instead of letting a typo black-hole all ingest. + minPipelineMaxBytes = 1 << 20 ) // DefaultPipelineConfig returns production-sized defaults. @@ -95,6 +133,7 @@ func DefaultPipelineConfig() PipelineConfig { Capacity: 50000, Workers: 8, SoftThreshold: 0.9, + MaxBytes: 512 << 20, } } @@ -140,9 +179,16 @@ type Pipeline struct { processedTotal atomic.Int64 droppedHealthy atomic.Int64 rejectedFull atomic.Int64 + rejectedBytes atomic.Int64 processFailures atomic.Int64 tenantDropped atomic.Int64 + // inFlightBytes tracks the approxBytes() sum of every batch currently + // in the queue or being processed. Reserved in Submit before the + // channel send, released by process() (deferred, so the panic path + // releases too). Bounded by cfg.MaxBytes. + inFlightBytes atomic.Int64 + // Per-tenant in-flight cap — bounds the queue slots a single tenant // can consume so a noisy tenant cannot starve siblings of fresh // healthy submissions when fullness is below the soft threshold. @@ -201,6 +247,16 @@ func NewPipeline(writer pipelineWriter, metrics *telemetry.Metrics, cfg Pipeline } cfg.Capacity = capacity cfg.Workers = workers + if cfg.MaxBytes <= 0 { + cfg.MaxBytes = d.MaxBytes + } + if cfg.MaxBytes < minPipelineMaxBytes { + slog.Warn("ingest pipeline: max bytes clamped up to floor — a sub-1MB cap would reject every gRPC batch", + "requested", cfg.MaxBytes, + "min", int64(minPipelineMaxBytes), + ) + cfg.MaxBytes = minPipelineMaxBytes + } // Zero-value config falls back to defaults — the field is internal // (no env-var surface) and TestPipeline_DefaultsApplied enforces this. // Priority-only mode (always-soft-drop) is not a supported configuration @@ -300,9 +356,13 @@ func (p *Pipeline) Submit(b *Batch) error { return nil } b.enqueuedAt = time.Now() + b.sizeBytes = b.approxBytes() - fullness := float64(len(p.queue)) / float64(p.cfg.Capacity) - if fullness >= p.cfg.SoftThreshold && !b.Priority() { + // Soft backpressure engages on whichever dimension is more saturated: + // item count (many small batches) or bytes (few fat batches). + itemFullness := float64(len(p.queue)) / float64(p.cfg.Capacity) + byteFullness := float64(p.inFlightBytes.Load()) / float64(p.cfg.MaxBytes) + if max(itemFullness, byteFullness) >= p.cfg.SoftThreshold && !b.Priority() { p.droppedHealthy.Add(1) p.observeDrop(b.Type, "soft_backpressure") return nil @@ -326,12 +386,28 @@ func (p *Pipeline) Submit(b *Batch) error { p.tenantMu.Unlock() } + // Byte-cap reservation — after the soft and tenant checks so dropped + // batches never reserve, before the channel send so the cap is never + // overshot. Priority batches get NO exemption here: a 429 is + // recoverable by the OTLP client's retry loop, an OOM kill is not. + if newTotal := p.inFlightBytes.Add(b.sizeBytes); newTotal > p.cfg.MaxBytes { + p.inFlightBytes.Add(-b.sizeBytes) + if tenantReserved { + p.releaseTenantSlot(b.Tenant) + } + p.rejectedBytes.Add(1) + p.observeDrop(b.Type, "bytes_full") + return ErrQueueFull + } + select { case p.queue <- b: p.enqueuedTotal.Add(1) p.observeQueueDepth(b.Type) + p.observeQueueBytes() return nil default: + p.inFlightBytes.Add(-b.sizeBytes) if tenantReserved { p.releaseTenantSlot(b.Tenant) } @@ -377,10 +453,13 @@ func (p *Pipeline) Stats() PipelineStats { Processed: p.processedTotal.Load(), DroppedHealthy: p.droppedHealthy.Load(), RejectedFull: p.rejectedFull.Load(), + RejectedBytes: p.rejectedBytes.Load(), ProcessFailures: p.processFailures.Load(), StoreFiltered: p.storeFiltered.Load(), QueueDepth: len(p.queue), Capacity: p.cfg.Capacity, + QueueBytes: p.inFlightBytes.Load(), + MaxBytes: p.cfg.MaxBytes, } } @@ -390,10 +469,13 @@ type PipelineStats struct { Processed int64 DroppedHealthy int64 RejectedFull int64 + RejectedBytes int64 // batches rejected because the byte cap was exceeded ProcessFailures int64 StoreFiltered int64 // logs dropped by STORE_MIN_SEVERITY at persist time QueueDepth int Capacity int + QueueBytes int64 // approx bytes currently reserved by in-flight batches + MaxBytes int64 // configured byte cap } // worker drains the queue. Exits when stopCh closes (after draining @@ -436,6 +518,15 @@ func (p *Pipeline) process(b *Batch) { if b == nil { return } + // Release the byte reservation taken at Submit time. Unconditional — + // every batch that reached the channel reserved sizeBytes, priority or + // not — and deferred so the panic path below releases too. Mirrors the + // reservation exactly; an asymmetry here ratchets inFlightBytes up + // until the cap rejects all traffic. + defer func() { + p.inFlightBytes.Add(-b.sizeBytes) + p.observeQueueBytes() + }() // Release the per-tenant slot reserved at Submit time. Registered as // a defer so it runs even if the batch panics. Priority batches don't // reserve at submit, so they don't release here either — the conditions @@ -507,6 +598,13 @@ func (p *Pipeline) observeQueueDepth(t SignalType) { p.metrics.IngestPipelineQueueDepth.WithLabelValues(signalLabel(t)).Set(float64(len(p.queue))) } +func (p *Pipeline) observeQueueBytes() { + if p.metrics == nil || p.metrics.IngestPipelineQueueBytes == nil { + return + } + p.metrics.IngestPipelineQueueBytes.Set(float64(p.inFlightBytes.Load())) +} + func (p *Pipeline) observeDrop(t SignalType, reason string) { if p.metrics == nil || p.metrics.IngestPipelineDroppedTotal == nil { return diff --git a/internal/ingest/pipeline_test.go b/internal/ingest/pipeline_test.go index fdf1a9a..9d2f87b 100644 --- a/internal/ingest/pipeline_test.go +++ b/internal/ingest/pipeline_test.go @@ -3,12 +3,17 @@ package ingest import ( "context" "errors" + "strings" "sync" "sync/atomic" "testing" "time" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/RandomCodeSpace/otelcontext/internal/storage" + "github.com/RandomCodeSpace/otelcontext/internal/telemetry" ) // fakeWriter is a deterministic, in-memory pipelineWriter for tests. @@ -401,6 +406,9 @@ func TestPipeline_DefaultsApplied(t *testing.T) { if p.cfg.SoftThreshold != d.SoftThreshold { t.Errorf("SoftThreshold default not applied: got %v want %v", p.cfg.SoftThreshold, d.SoftThreshold) } + if p.cfg.MaxBytes != d.MaxBytes { + t.Errorf("MaxBytes default not applied: got %d want %d", p.cfg.MaxBytes, d.MaxBytes) + } } func TestPipeline_PerTenantCap_DropsExcessHealthy(t *testing.T) { @@ -671,3 +679,277 @@ func TestPipeline_StoreMinSeverity_Disabled_PersistsAllLogs(t *testing.T) { t.Errorf("Stats().StoreFiltered = %d, want 0 with gate disabled", got) } } + +// ===== Byte-bounded queue (P1.2) ===== + +// fatLogBatch builds a single-log batch whose Body is `bodyLen` bytes — +// the byte-cap tests size their payloads through this. +func fatLogBatch(bodyLen int) *Batch { + return &Batch{ + Type: SignalLogs, + Tenant: "t1", + Logs: []storage.Log{{Body: strings.Repeat("x", bodyLen)}}, + } +} + +func TestBatch_ApproxBytes_Formula(t *testing.T) { + // Pins the per-record estimate: fixed struct overhead + the lengths of + // the dominant string payloads. A drive-by change to the formula must + // consciously update this test. + b := &Batch{ + Traces: []storage.Trace{{TraceID: "abcd", ServiceName: "svc", Status: "OK", Operation: "op"}}, + Spans: []storage.Span{{ + OperationName: "GET /x", + AttributesJSON: storage.CompressedText(`{"k":"v"}`), + ServiceName: "svc", + Status: "OK", + }}, + Logs: []storage.Log{{ + Body: "hello", + AttributesJSON: storage.CompressedText("{}"), + AIInsight: storage.CompressedText("ai"), + ServiceName: "svc", + Severity: "INFO", + }}, + } + wantTrace := int64(128 + 4 + 3 + 2 + 2) // TraceID + ServiceName + Status + Operation + wantSpan := int64(256 + 6 + 9 + 3 + 2) // OperationName + AttributesJSON + ServiceName + Status + wantLog := int64(192 + 5 + 2 + 2 + 3 + 4) // Body + AttributesJSON + AIInsight + ServiceName + Severity + if got, want := b.approxBytes(), wantTrace+wantSpan+wantLog; got != want { + t.Fatalf("approxBytes() = %d, want %d", got, want) + } +} + +func TestBatch_ApproxBytes_GrowsWithPayload(t *testing.T) { + // The estimate must scale with the variable-length payloads — Body and + // AttributesJSON are what actually make a batch fat. + small := fatLogBatch(1) + big := fatLogBatch(1001) + if diff := big.approxBytes() - small.approxBytes(); diff != 1000 { + t.Errorf("Body growth: approxBytes diff = %d, want 1000", diff) + } + + s1 := &Batch{Spans: []storage.Span{{SpanID: "s"}}} + s2 := &Batch{Spans: []storage.Span{{SpanID: "s", AttributesJSON: storage.CompressedText(strings.Repeat("a", 500))}}} + if diff := s2.approxBytes() - s1.approxBytes(); diff != 500 { + t.Errorf("AttributesJSON growth: approxBytes diff = %d, want 500", diff) + } +} + +func TestPipeline_MaxBytes_DefaultAndClamp(t *testing.T) { + // MaxBytes <= 0 falls back to the 512MB default. + p := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 10, Workers: 0}) + if got := p.Stats().MaxBytes; got != 512<<20 { + t.Errorf("MaxBytes default: got %d, want %d", got, int64(512<<20)) + } + // A sub-1MB cap would reject every gRPC batch (GRPC_MAX_RECV_MB default + // is 16) — clamp up to the 1MB floor instead. + p2 := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 10, Workers: 0, MaxBytes: 1000}) + if got := p2.Stats().MaxBytes; got != 1<<20 { + t.Errorf("sub-1MB clamp: got %d, want %d", got, int64(1<<20)) + } + // Exactly the floor passes through unmodified. + p3 := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 10, Workers: 0, MaxBytes: 1 << 20}) + if got := p3.Stats().MaxBytes; got != 1<<20 { + t.Errorf("1MB floor passthrough: got %d, want %d", got, int64(1<<20)) + } +} + +func TestPipeline_ByteCap_RejectsEvenPriority(t *testing.T) { + // Item count is nowhere near Capacity, yet the second fat batch must be + // rejected on bytes — and priority gives no exemption: a 429 is + // recoverable by the OTLP client's retry loop, an OOM kill is not. + p := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 100, Workers: 0, MaxBytes: 1 << 20}) + + first := fatLogBatch(700 * 1024) + first.HasError = true + if err := p.Submit(first); err != nil { + t.Fatalf("first fat batch under the cap rejected: %v", err) + } + firstSize := first.approxBytes() + + second := fatLogBatch(700 * 1024) + second.HasError = true + if err := p.Submit(second); !errors.Is(err, ErrQueueFull) { + t.Fatalf("over-cap priority submit returned %v, want ErrQueueFull", err) + } + + stats := p.Stats() + if stats.RejectedBytes != 1 { + t.Errorf("RejectedBytes = %d, want 1", stats.RejectedBytes) + } + if stats.RejectedFull != 0 { + t.Errorf("RejectedFull = %d, want 0 (byte rejection is a separate counter)", stats.RejectedFull) + } + if stats.QueueBytes != firstSize { + t.Errorf("QueueBytes = %d, want %d (rejected batch must not stay reserved)", stats.QueueBytes, firstSize) + } +} + +func TestPipeline_ByteCap_SingleOversizedBatchRejected(t *testing.T) { + // A lone batch bigger than the whole cap is rejected outright and + // leaves no residual reservation behind. + p := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 100, Workers: 0, MaxBytes: 1 << 20}) + b := fatLogBatch(2 << 20) + b.HasError = true + if err := p.Submit(b); !errors.Is(err, ErrQueueFull) { + t.Fatalf("oversized submit returned %v, want ErrQueueFull", err) + } + if got := p.Stats().QueueBytes; got != 0 { + t.Errorf("QueueBytes = %d after rejection, want 0", got) + } +} + +func TestPipeline_ByteCap_ReleasesTenantSlotOnReject(t *testing.T) { + // A healthy batch reserves its tenant slot before the byte check; a + // byte rejection must hand that slot back, or the tenant cap turns into + // a slow leak. With cap 1, a leaked slot would make the follow-up + // submission a tenant_backpressure drop. + p := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 10, Workers: 0, MaxBytes: 1 << 20}) + p.SetPerTenantCap(1) + + fat := fatLogBatch(2 << 20) // healthy → reserves the tenant slot first + if err := p.Submit(fat); !errors.Is(err, ErrQueueFull) { + t.Fatalf("oversized submit returned %v, want ErrQueueFull", err) + } + if err := p.Submit(healthyBatch()); err != nil { + t.Fatalf("follow-up submit after byte rejection: %v", err) + } + if got := p.TenantDropped(); got != 0 { + t.Errorf("TenantDropped = %d, want 0 (slot leaked by byte rejection)", got) + } + if got := p.Stats().Enqueued; got != 1 { + t.Errorf("Enqueued = %d, want 1", got) + } +} + +func TestPipeline_ChannelFull_ReleasesBytes(t *testing.T) { + // The hard-capacity default branch must undo the byte reservation the + // same way it undoes the tenant slot. + p := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 1, Workers: 0}) + first := errorBatch() + if err := p.Submit(first); err != nil { + t.Fatalf("priming submit: %v", err) + } + if err := p.Submit(errorBatch()); !errors.Is(err, ErrQueueFull) { + t.Fatalf("channel-full submit returned %v, want ErrQueueFull", err) + } + if got, want := p.Stats().QueueBytes, first.approxBytes(); got != want { + t.Errorf("QueueBytes = %d, want %d (channel-full path leaked its reservation)", got, want) + } +} + +func TestPipeline_SoftDropViaByteFullness(t *testing.T) { + // Item fullness is ~0.1% but bytes are at ~93% of the cap — the soft + // check must fire on max(itemFullness, byteFullness) and drop the + // healthy batch. + p := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 1000, Workers: 0, SoftThreshold: 0.9, MaxBytes: 1 << 20}) + fat := fatLogBatch(950 * 1024) // priority → bypasses the soft check itself + fat.HasError = true + if err := p.Submit(fat); err != nil { + t.Fatalf("priming fat priority submit: %v", err) + } + + if err := p.Submit(healthyBatch()); err != nil { + t.Fatalf("soft-dropped submit returned %v, want nil (silent drop)", err) + } + stats := p.Stats() + if stats.DroppedHealthy != 1 { + t.Errorf("DroppedHealthy = %d, want 1 (byteFullness should trip the soft check)", stats.DroppedHealthy) + } + if stats.Enqueued != 1 { + t.Errorf("Enqueued = %d, want 1 (only the priming batch)", stats.Enqueued) + } +} + +func TestPipeline_ByteAccounting_ReservedWhileQueued(t *testing.T) { + // With no workers draining, QueueBytes must equal the sum of the + // enqueued batches' estimates. + p := NewPipeline(&fakeWriter{}, nil, PipelineConfig{Capacity: 10, Workers: 0}) + b1, b2 := healthyBatch(), fatLogBatch(4096) + if err := p.Submit(b1); err != nil { + t.Fatalf("submit 1: %v", err) + } + if err := p.Submit(b2); err != nil { + t.Fatalf("submit 2: %v", err) + } + if got, want := p.Stats().QueueBytes, b1.approxBytes()+b2.approxBytes(); got != want { + t.Errorf("QueueBytes = %d, want %d", got, want) + } +} + +func TestPipeline_ByteAccounting_ReturnsToZeroAfterProcess(t *testing.T) { + // Every reservation taken at Submit must be released by process() — + // including the panic path, or the counter ratchets up until the cap + // rejects all traffic. + w := &fakeWriter{} + p := NewPipeline(w, nil, PipelineConfig{Capacity: 10, Workers: 1}) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + p.Start(ctx) + t.Cleanup(p.Stop) + + bad := healthyBatch() + bad.SpanCallback = func(_ storage.Span) { panic("boom") } + good := healthyBatch() + + if err := p.Submit(bad); err != nil { + t.Fatalf("submit bad: %v", err) + } + if err := p.Submit(good); err != nil { + t.Fatalf("submit good: %v", err) + } + if !waitFor(t, 5*time.Second, func() bool { return p.Stats().Processed >= 2 }) { + t.Fatalf("worker did not process both batches — Processed=%d", p.Stats().Processed) + } + if !waitFor(t, 5*time.Second, func() bool { return p.Stats().QueueBytes == 0 }) { + t.Fatalf("QueueBytes = %d after drain, want 0 (panic path leaked its reservation)", p.Stats().QueueBytes) + } +} + +func TestPipeline_QueueBytesGaugeTracksReservations(t *testing.T) { + // The Prometheus gauge must follow the reservation lifecycle: up on + // enqueue, back to zero after the worker drains. Built from a bare + // prometheus.NewGauge (not telemetry.New()) so the default registry + // isn't double-registered across tests. + g := prometheus.NewGauge(prometheus.GaugeOpts{Name: "test_ingest_pipeline_queue_bytes"}) + m := &telemetry.Metrics{IngestPipelineQueueBytes: g} + p := NewPipeline(&fakeWriter{}, m, PipelineConfig{Capacity: 10, Workers: 1}) + + b := healthyBatch() + if err := p.Submit(b); err != nil { + t.Fatalf("submit: %v", err) + } + if got, want := testutil.ToFloat64(g), float64(b.approxBytes()); got != want { + t.Errorf("gauge after enqueue = %v, want %v", got, want) + } + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + p.Start(ctx) + t.Cleanup(p.Stop) + if !waitFor(t, 5*time.Second, func() bool { return testutil.ToFloat64(g) == 0 }) { + t.Fatalf("gauge did not return to 0 after drain: %v", testutil.ToFloat64(g)) + } +} + +func TestPipeline_ByteAccounting_ReleasedOnWriterFailure(t *testing.T) { + // A failed BatchCreateAll drops the batch — its reservation must be + // released all the same. + w := &fakeWriter{traceErr: errors.New("db down")} + p := NewPipeline(w, nil, PipelineConfig{Capacity: 4, Workers: 1}) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + p.Start(ctx) + t.Cleanup(p.Stop) + + if err := p.Submit(healthyBatch()); err != nil { + t.Fatalf("submit: %v", err) + } + if !waitFor(t, 5*time.Second, func() bool { return p.Stats().ProcessFailures >= 1 }) { + t.Fatalf("writer failure never surfaced") + } + if !waitFor(t, 5*time.Second, func() bool { return p.Stats().QueueBytes == 0 }) { + t.Fatalf("QueueBytes = %d after failed process, want 0", p.Stats().QueueBytes) + } +} diff --git a/internal/ingest/sampler.go b/internal/ingest/sampler.go index 38d63e8..6b9e0ed 100644 --- a/internal/ingest/sampler.go +++ b/internal/ingest/sampler.go @@ -17,6 +17,7 @@ type Sampler struct { buckets map[string]*tokenBucket totalSeen atomic.Int64 totalDropped atomic.Int64 + now func() time.Time // injectable for tests; defaults to time.Now } // NewSampler creates a Sampler with the given parameters. @@ -32,6 +33,7 @@ func NewSampler(rate float64, alwaysOnErrors bool, latencyThresholdMs float64) * alwaysOnErrors: alwaysOnErrors, latencyThresholdMs: latencyThresholdMs, buckets: make(map[string]*tokenBucket), + now: time.Now, } } @@ -67,13 +69,13 @@ func (s *Sampler) ShouldSample(serviceName string, isError bool, durationMs floa s.mu.Lock() b, ok := s.buckets[serviceName] if !ok { - b = newTokenBucket(s.rate) + b = newTokenBucket(s.rate, s.now) s.buckets[serviceName] = b // Always let first trace through (new service discovery). s.mu.Unlock() return true } - allow := b.allow() + allow := b.allow(s.now()) s.mu.Unlock() if !allow { @@ -87,35 +89,60 @@ func (s *Sampler) Stats() (int64, int64) { return s.totalSeen.Load(), s.totalDropped.Load() } -// tokenBucket is a simple token bucket for sampling decisions. -// Refills at `rate` tokens per second, max capacity 1.0. +// tokenBucket throttles healthy-span ingestion per service. It is a RATE +// LIMITER, not a percentage sampler: it admits at most ~`rate` healthy spans +// per second per service (refill `rate` tokens/s, cost 1 token/span) with a +// startup burst of up to capacity = max(1, 1/rate) spans. +// +// The long-run keep FRACTION is therefore min(1, rate / offered_rate): a +// service emitting ≤ `rate` healthy spans/s keeps all of them; above that it +// keeps a shrinking fraction while the absolute admitted rate stays near +// `rate`/s. So rate=0.05 means "~0.05 healthy spans/s/service" (≈ one per 20s), +// NOT "5% of healthy spans". Errors and slow spans bypass this entirely in +// ShouldSample, so there is no data loss for the signals that matter; the +// absolute cap is what bounds the SQLite write rate under load spikes. If a +// true proportional keep-fraction is ever wanted, switch to probabilistic +// (hash-mod / rand) sampling — that is a separate design decision. +// +// (The previous implementation capped tokens at 1.0 but charged 1.0/rate per +// request; for any rate < 1.0 the charge exceeded the cap, so no healthy span +// could ever be admitted — the SQLite default rate of 0.05 dropped ~100%.) type tokenBucket struct { - rate float64 // tokens per second - tokens float64 // current tokens (0.0–1.0) - lastTick time.Time + rate float64 // refill tokens/sec == max sustained admitted healthy spans/sec + capacity float64 // max accumulated tokens (burst ceiling) + tokens float64 // current token count + lastTick time.Time // wall-clock of last refill } -func newTokenBucket(rate float64) *tokenBucket { +func newTokenBucket(rate float64, now func() time.Time) *tokenBucket { + // capacity = 1/rate tokens: one "admission interval" of burst so a freshly + // seen service is not throttled immediately. Floored at 1.0 so a span can + // always eventually be admitted. + capacity := 1.0 / rate + if capacity < 1.0 { + capacity = 1.0 + } return &tokenBucket{ rate: rate, - tokens: rate, // start with one full refill worth - lastTick: time.Now(), + capacity: capacity, + tokens: capacity, // start full so new services are not immediately throttled + lastTick: now(), } } -// allow returns true and consumes a token if one is available. -func (b *tokenBucket) allow() bool { - now := time.Now() +// allow refills the bucket for elapsed time and admits the request if +// at least 1 token is available, consuming exactly 1 on admission. +func (b *tokenBucket) allow(now time.Time) bool { elapsed := now.Sub(b.lastTick).Seconds() b.lastTick = now b.tokens += elapsed * b.rate - if b.tokens > 1.0 { - b.tokens = 1.0 + if b.tokens > b.capacity { + b.tokens = b.capacity } - if b.tokens >= 1.0/b.rate { - b.tokens -= 1.0 / b.rate + if b.tokens >= 1.0 { + b.tokens -= 1.0 return true } return false diff --git a/internal/ingest/sampler_test.go b/internal/ingest/sampler_test.go new file mode 100644 index 0000000..cdb8970 --- /dev/null +++ b/internal/ingest/sampler_test.go @@ -0,0 +1,184 @@ +package ingest + +import ( + "fmt" + "testing" + "time" +) + +// advanceClock returns a now() func that starts at t0 and advances by step on +// each call. This lets tests drive token-bucket time without time.Sleep. +func advanceClock(t0 time.Time, step time.Duration) func() time.Time { + cur := t0 + return func() time.Time { + t := cur + cur = cur.Add(step) + return t + } +} + +// TestSamplerKeepFraction verifies that the healthy-span probabilistic path +// keeps approximately `rate` fraction of calls over a large sample. The test +// is fully deterministic — virtual time advances in fixed steps so there is no +// reliance on wall-clock timing or sleep. +// +// Clock design: each tick advances virtual time by tickNs nanoseconds (a fine +// granularity). After the first call (new-service free-admit), the token bucket +// receives `rate * tickNs/1e9` tokens per call. Over N calls the total tokens +// available ≈ N * rate * tickNs/1e9, and each admission costs 1 token, giving +// a long-run keep fraction ≈ rate * tickNs/1e9. We therefore need: +// +// tickNs = 1e9 (1 second per virtual call) +// +// so keep fraction → rate. However, with 1s ticks and rates near 1.0 the +// capacity cap wastes tokens (1.8→1.111 for rate=0.9). Using a very large +// virtual time window (100 000 calls at 1s each = ~28 virtual hours) and a +// wider tolerance lets the law-of-large-numbers smooth the discrete rounding, +// while still clearly distinguishing "correct implementation" (~rate) from +// "broken implementation" (~0% or ~100%). +func TestSamplerKeepFraction(t *testing.T) { + t.Parallel() + + tests := []struct { + rate float64 + wantFrac float64 + lo float64 // inclusive lower bound + hi float64 // inclusive upper bound + }{ + // Wide but meaningful bounds: broken code gives ~0% (capped bucket) or + // ~100% (bypass bug). Correct code must land in these windows. + {rate: 0.05, wantFrac: 0.05, lo: 0.03, hi: 0.09}, + {rate: 0.50, wantFrac: 0.50, lo: 0.40, hi: 0.60}, + {rate: 0.90, wantFrac: 0.90, lo: 0.60, hi: 1.00}, + } + + // 1 virtual second per call so refill per tick == rate tokens exactly. + const tickNs = int64(time.Second) + const calls = 100_000 + + for _, tc := range tests { + t.Run(fmt.Sprintf("rate=%.2f", tc.rate), func(t *testing.T) { + t.Parallel() + + t0 := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + nowFn := advanceClock(t0, time.Duration(tickNs)) + + s := &Sampler{ + rate: tc.rate, + alwaysOnErrors: false, + latencyThresholdMs: 1e9, // far above test spans so the slow-span bypass never fires + buckets: make(map[string]*tokenBucket), + now: nowFn, + } + + kept := 0 + for i := 0; i < calls; i++ { + if s.ShouldSample("svc", false, 0) { + kept++ + } + } + + got := float64(kept) / float64(calls) + if got < tc.lo || got > tc.hi { + t.Errorf("rate=%.2f: keep fraction %.4f outside [%.4f, %.4f] (%d/%d kept)", + tc.rate, got, tc.lo, tc.hi, kept, calls) + } + }) + } +} + +// TestSamplerRateOne verifies rate==1.0 keeps all healthy spans. +func TestSamplerRateOne(t *testing.T) { + t.Parallel() + // latencyThresholdMs=999999 keeps slow-span bypass out of the picture. + s := NewSampler(1.0, false, 999999) + for i := 0; i < 1000; i++ { + if !s.ShouldSample("svc", false, 0) { + t.Fatal("rate=1.0 must keep every span") + } + } +} + +// TestSamplerRateZero verifies rate==0.0 drops all healthy spans. +func TestSamplerRateZero(t *testing.T) { + t.Parallel() + // latencyThresholdMs=999999 so 0ms spans are not treated as "slow"; we want + // the pure zero-rate drop path. + s := NewSampler(0.0, false, 999999) + for i := 0; i < 1000; i++ { + if s.ShouldSample("svc", false, 0) { + t.Fatal("rate=0.0 must drop every healthy span") + } + } +} + +// TestSamplerAlwaysOnErrors verifies that error spans bypass the bucket +// regardless of rate. +func TestSamplerAlwaysOnErrors(t *testing.T) { + t.Parallel() + // rate=0 would drop everything healthy; errors must still pass. + s := NewSampler(0.0, true /* alwaysOnErrors */, 0) + for i := 0; i < 1000; i++ { + if !s.ShouldSample("svc", true /* isError */, 0) { + t.Fatal("error span must always be kept when alwaysOnErrors=true") + } + } +} + +// TestSamplerSlowSpanBypass verifies that slow spans bypass the bucket. +func TestSamplerSlowSpanBypass(t *testing.T) { + t.Parallel() + // rate=0, latencyThreshold=500ms — a 600ms span must be kept. + s := NewSampler(0.0, false, 500) + for i := 0; i < 100; i++ { + if !s.ShouldSample("svc", false, 600) { + t.Fatal("slow span (600ms > 500ms threshold) must always be kept") + } + } +} + +// TestSamplerNewServiceDiscovery verifies the first call for a new service is +// always admitted (service discovery guarantee). +func TestSamplerNewServiceDiscovery(t *testing.T) { + t.Parallel() + // rate=0.001 — almost nothing would pass, but a brand-new service must. + // latencyThresholdMs=999999 keeps the slow-span bypass out of the picture. + s := NewSampler(0.001, false, 999999) + if !s.ShouldSample("brand-new-svc", false, 0) { + t.Fatal("first call for a new service must always be admitted") + } +} + +// TestSamplerStats verifies that the seen/dropped counters are consistent. +func TestSamplerStats(t *testing.T) { + t.Parallel() + + t0 := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC) + // Use a slow clock (1s per tick) so rate=0.5 alternates roughly keep/drop. + nowFn := advanceClock(t0, time.Second) + + s := &Sampler{ + rate: 0.5, + alwaysOnErrors: false, + latencyThresholdMs: -1, // disabled + buckets: make(map[string]*tokenBucket), + now: nowFn, + } + + const calls = 100 + for i := 0; i < calls; i++ { + s.ShouldSample("svc", false, 0) + } + + seen, dropped := s.Stats() + // First call is free (new-service discovery), so seen == calls, dropped < calls. + if seen != calls { + t.Errorf("seen=%d want %d", seen, calls) + } + if dropped >= calls { + t.Errorf("dropped=%d must be < seen=%d", dropped, calls) + } + if dropped < 0 { + t.Errorf("dropped=%d must not be negative", dropped) + } +} diff --git a/internal/mcp/response_cap_test.go b/internal/mcp/response_cap_test.go index 57a4162..2c89a27 100644 --- a/internal/mcp/response_cap_test.go +++ b/internal/mcp/response_cap_test.go @@ -3,6 +3,7 @@ package mcp import ( "strings" "testing" + "time" ) // TestTextResult_ResponseCap verifies the byte-cap defense for tool @@ -52,3 +53,84 @@ func TestTextResult_ResponseCap(t *testing.T) { } }) } + +// TestResourceResult_ResponseCap verifies that resourceResult applies the same +// MaxToolResponseBytes cap as textResult — the trace_graph DB fallback can +// return an unbounded GetTrace payload without this guard. +func TestResourceResult_ResponseCap(t *testing.T) { + t.Parallel() + + t.Run("UnderCapPassesThrough", func(t *testing.T) { + text := strings.Repeat("x", 1024) + got := resourceResult("OtelContext://test", "application/json", text) + if got.IsError { + t.Fatalf("expected success, got IsError=true: %+v", got) + } + if len(got.Content) != 1 || got.Content[0].Resource == nil { + t.Fatalf("expected resource content item: %+v", got) + } + if got.Content[0].Resource.Text != text { + t.Fatalf("payload mangled") + } + }) + + t.Run("AtCapPassesThrough", func(t *testing.T) { + text := strings.Repeat("x", MaxToolResponseBytes) + got := resourceResult("OtelContext://test", "application/json", text) + if got.IsError { + t.Fatalf("expected at-cap to pass, got IsError=true") + } + }) + + t.Run("OverCapErrors", func(t *testing.T) { + text := strings.Repeat("x", MaxToolResponseBytes+1) + got := resourceResult("OtelContext://test", "application/json", text) + if !got.IsError { + t.Fatalf("expected over-cap to error, got success") + } + if len(got.Content) == 0 || !strings.Contains(got.Content[0].Text, "response too large") { + t.Fatalf("expected 'response too large' in error: %+v", got) + } + if !strings.Contains(got.Content[0].Text, "narrow time range") { + t.Fatalf("expected actionable hint in error: %+v", got) + } + }) +} + +// TestCacheErrorSkip_GuardLogic verifies the guard logic that server.go uses +// to decide whether to call cache.Set. The guard is: +// +// if !toolResult.IsError { s.cache.Set(...) } +// +// This test exercises that decision directly: an error result (IsError=true) +// must NOT be passed to Set; a successful result must be passed and then +// retrievable. +func TestCacheErrorSkip_GuardLogic(t *testing.T) { + t.Parallel() + + c := newResultCache(5*time.Second, 64) + + // Simulate the server-side guard: only Set when !IsError. + maybeCache := func(res ToolCallResult) { + if !res.IsError { + c.Set("default", "get_service_map", nil, res) + } + } + + // Error result: guard must prevent the Set. + maybeCache(errorResult("GraphRAG not initialized")) + _, hit := c.Get("default", "get_service_map", nil) + if hit { + t.Fatal("error result must not be stored in the cache") + } + + // Successful result: guard must allow the Set. + maybeCache(textResult(`{"ok":true}`)) + got, hit := c.Get("default", "get_service_map", nil) + if !hit { + t.Fatal("successful result must be stored in the cache") + } + if got.IsError { + t.Fatalf("cached result should not be an error: %+v", got) + } +} diff --git a/internal/mcp/server.go b/internal/mcp/server.go index 5da7532..e9084b8 100644 --- a/internal/mcp/server.go +++ b/internal/mcp/server.go @@ -332,7 +332,9 @@ func (s *Server) handleRPC(w http.ResponseWriter, r *http.Request) { break } s.callsServiced.Add(1) - s.cache.Set(tenant, params.Name, params.Arguments, toolResult) + if !toolResult.IsError { + s.cache.Set(tenant, params.Name, params.Arguments, toolResult) + } result = toolResult case "ping": @@ -468,9 +470,18 @@ func (s *Server) runWithTimeout(ctx context.Context, cancel context.CancelFunc, case o := <-done: return o.res, false case <-ctx.Done(): - // Goroutine still running — slot will be released when it - // finishes via the deferred release(). - return ToolCallResult{}, true + // The handler goroutine cancels ctx (deferred) AFTER sending its + // result, so a finished handler can leave BOTH channels ready and + // select picks randomly — a queued result must win over a spurious + // timeout. Only an empty done channel is a real deadline overrun + // (goroutine still running; the deferred release() frees the slot + // whenever it finishes). + select { + case o := <-done: + return o.res, false + default: + return ToolCallResult{}, true + } } } diff --git a/internal/mcp/timeout_race_test.go b/internal/mcp/timeout_race_test.go new file mode 100644 index 0000000..d0c6c00 --- /dev/null +++ b/internal/mcp/timeout_race_test.go @@ -0,0 +1,29 @@ +package mcp + +import ( + "context" + "testing" + "time" +) + +// Regression for a select race in runWithTimeout: the handler goroutine +// cancels the call context (deferred) after sending its result, so a +// fast handler could leave both select cases ready and the random pick +// reported a spurious -32001 timeout (seen as instant "exceeded 30s +// deadline" failures on loaded CI runners). A finished handler must +// always win, however the goroutines interleave. +func TestRunWithTimeout_FastHandlerNeverTimesOut(t *testing.T) { + // Zero-value deps are safe: an unknown tool name takes toolHandler's + // instant "unknown tool" path and the metrics defer nil-checks. + s := &Server{callTimeout: 30 * time.Second} + for i := 0; i < 2000; i++ { + ctx, cancel := s.deriveCallCtx(context.Background()) + res, timedOut := s.runWithTimeout(ctx, cancel, "nonexistent_tool", nil, nil) + if timedOut { + t.Fatalf("iteration %d: spurious timeout from an instant handler", i) + } + if !res.IsError { + t.Fatalf("iteration %d: expected unknown-tool error result", i) + } + } +} diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index 6896aeb..6e399b5 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -387,6 +387,12 @@ func textResult(text string) ToolCallResult { } func resourceResult(uri, mimeType, text string) ToolCallResult { + if len(text) > MaxToolResponseBytes { + return errorResult(fmt.Sprintf( + "response too large: %d bytes exceeds %d-byte cap; narrow time range or use pagination", + len(text), MaxToolResponseBytes, + )) + } return ToolCallResult{ Content: []ContentItem{ {Type: "resource", Resource: &Resource{URI: uri, MimeType: mimeType, Text: text}}, diff --git a/internal/membudget/membudget.go b/internal/membudget/membudget.go new file mode 100644 index 0000000..e0ee356 --- /dev/null +++ b/internal/membudget/membudget.go @@ -0,0 +1,67 @@ +// Package membudget detects the memory budget available to the process — +// (in order) cgroup v2, cgroup v1, then host RAM from /proc/meminfo — so +// consumers can scale their allocations against the actual quota instead of +// hardcoding sizes. Used by the GOMEMLIMIT bootstrap in package main and the +// SQLite cache/mmap sizing in internal/storage. +package membudget + +import ( + "os" + "strconv" + "strings" +) + +// Detect returns the applicable memory budget in bytes and its source, +// or (0, "") if nothing usable is found. cgroup limits take precedence over +// host RAM so containerized deployments pace against their actual quota. +func Detect() (int64, string) { + if v, ok := readCgroupBytes("/sys/fs/cgroup/memory.max"); ok { // cgroup v2 + return v, "cgroup.v2" + } + if v, ok := readCgroupBytes("/sys/fs/cgroup/memory/memory.limit_in_bytes"); ok { // cgroup v1 + return v, "cgroup.v1" + } + if v, ok := readMemTotal("/proc/meminfo"); ok { + return v, "meminfo.MemTotal" + } + return 0, "" +} + +// readCgroupBytes reads a cgroup memory-limit file. Returns (bytes, true) only for +// a concrete limit; "max", empty, non-numeric, or a near-max "unlimited" sentinel +// (>= 1 PiB, used by cgroup v1) returns (0, false) so the caller falls through. +func readCgroupBytes(path string) (int64, bool) { + b, err := os.ReadFile(path) //nolint:gosec // G304: fixed cgroup sysfs path, not user input + if err != nil { + return 0, false + } + s := strings.TrimSpace(string(b)) + if s == "" || s == "max" { + return 0, false + } + v, err := strconv.ParseInt(s, 10, 64) + if err != nil || v <= 0 || v >= (1<<50) { + return 0, false + } + return v, true +} + +// readMemTotal parses MemTotal (in kB) from a /proc/meminfo-formatted file and +// returns it in bytes. +func readMemTotal(path string) (int64, bool) { + b, err := os.ReadFile(path) //nolint:gosec // G304: fixed /proc/meminfo path, not user input + if err != nil { + return 0, false + } + for line := range strings.SplitSeq(string(b), "\n") { + if strings.HasPrefix(line, "MemTotal:") { + f := strings.Fields(line) + if len(f) >= 2 { + if kb, err := strconv.ParseInt(f[1], 10, 64); err == nil && kb > 0 { + return kb * 1024, true + } + } + } + } + return 0, false +} diff --git a/internal/membudget/membudget_test.go b/internal/membudget/membudget_test.go new file mode 100644 index 0000000..b46fad1 --- /dev/null +++ b/internal/membudget/membudget_test.go @@ -0,0 +1,67 @@ +package membudget + +import ( + "os" + "path/filepath" + "testing" +) + +func TestReadCgroupBytes(t *testing.T) { + dir := t.TempDir() + cases := []struct { + name string + content string + want int64 + wantOK bool + }{ + {"concrete limit", "2147483648\n", 2147483648, true}, + {"unlimited max", "max\n", 0, false}, + {"empty", "", 0, false}, + {"non-numeric", "garbage", 0, false}, + {"zero", "0", 0, false}, + {"v1 near-max sentinel", "9223372036854771712", 0, false}, // ~8 EiB "unlimited" + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + p := filepath.Join(dir, tc.name) + if err := os.WriteFile(p, []byte(tc.content), 0o600); err != nil { + t.Fatal(err) + } + got, ok := readCgroupBytes(p) + if got != tc.want || ok != tc.wantOK { + t.Fatalf("readCgroupBytes(%q)=(%d,%v) want (%d,%v)", tc.content, got, ok, tc.want, tc.wantOK) + } + }) + } + if _, ok := readCgroupBytes(filepath.Join(dir, "does-not-exist")); ok { + t.Fatal("missing file should return ok=false") + } +} + +func TestReadMemTotal(t *testing.T) { + dir := t.TempDir() + meminfo := "MemFree: 1000 kB\nMemTotal: 16384000 kB\nBuffers: 500 kB\n" + p := filepath.Join(dir, "meminfo") + if err := os.WriteFile(p, []byte(meminfo), 0o600); err != nil { + t.Fatal(err) + } + got, ok := readMemTotal(p) + if !ok || got != 16384000*1024 { + t.Fatalf("readMemTotal=(%d,%v) want (%d,true)", got, ok, int64(16384000)*1024) + } + if _, ok := readMemTotal(filepath.Join(dir, "nope")); ok { + t.Fatal("missing meminfo should return ok=false") + } +} + +func TestDetect(t *testing.T) { + // On the Linux test host at least one source (meminfo) must resolve to a + // positive budget; the function must never return a negative value. + b, src := Detect() + if b < 0 { + t.Fatalf("budget must not be negative, got %d", b) + } + if b > 0 && src == "" { + t.Fatal("positive budget must carry a source label") + } +} diff --git a/internal/storage/factory.go b/internal/storage/factory.go index 8e4f178..857805d 100644 --- a/internal/storage/factory.go +++ b/internal/storage/factory.go @@ -18,6 +18,8 @@ import ( "gorm.io/gorm/logger" _ "github.com/microsoft/go-mssqldb/azuread" + + "github.com/RandomCodeSpace/otelcontext/internal/membudget" ) // NewDatabase creates a GORM database connection for any supported driver. @@ -103,17 +105,30 @@ func NewDatabase(driver, dsn string) (*gorm.DB, error) { // default-tuned behaviour. The set was hardened on 2026-05-24 to make // the platform survivable at 120 services on SQLite. // - // cache_size=-262144 = 256 MB page cache (negative = KB). - // mmap_size=1073741824 = 1 GB memory-mapped read window. + // cache_size and mmap_size are budget-scaled (the pure-Go driver's page + // cache lives on the Go heap, so a fixed 256 MB/1 GB pair starved 4 GB + // hosts) — see sqliteMemorySizes for the scaling and override rules. // wal_autocheckpoint=10000 = checkpoint after 10k pages so WAL stays bounded. // journal_size_limit=67108864 = hard-cap the WAL file at 64 MB. if strings.ToLower(driver) == "sqlite" || driver == "" { + budget, budgetSource := membudget.Detect() + cacheKB, mmapBytes := sqliteMemorySizes(budget) + // Best-effort, deliberately NOT fail-closed, and ordered BEFORE the + // stanza below: auto_vacuum only takes effect on databases this + // process creates, and the journal_mode=WAL switch initializes the + // file header — after which the stored auto_vacuum mode is frozen + // (pre-existing files keep theirs either way). INCREMENTAL lets the + // retention scheduler reclaim pages via PRAGMA incremental_vacuum + // instead of a full daily VACUUM. + if err := db.Exec("PRAGMA auto_vacuum=INCREMENTAL").Error; err != nil { + log.Printf("⚠️ PRAGMA auto_vacuum=INCREMENTAL failed (best-effort, continuing): %v", err) + } pragmas := []string{ "PRAGMA journal_mode=WAL", "PRAGMA synchronous=NORMAL", - "PRAGMA cache_size=-262144", + fmt.Sprintf("PRAGMA cache_size=-%d", cacheKB), "PRAGMA temp_store=MEMORY", - "PRAGMA mmap_size=1073741824", + fmt.Sprintf("PRAGMA mmap_size=%d", mmapBytes), "PRAGMA wal_autocheckpoint=10000", "PRAGMA journal_size_limit=67108864", "PRAGMA busy_timeout=5000", @@ -123,6 +138,11 @@ func NewDatabase(driver, dsn string) (*gorm.DB, error) { return nil, fmt.Errorf("sqlite pragma %q failed: %w", p, err) } } + if budgetSource == "" { + budgetSource = "none (fallback to hardcoded ceilings)" + } + log.Printf("📊 SQLite memory tuning: cache=%d KB, mmap=%d MB (budget=%d MB, source=%s)", + cacheKB, mmapBytes/(1<<20), budget/(1<<20), budgetSource) } // Configure Connection Pool — configurable via env vars for non-SQLite drivers. @@ -168,6 +188,60 @@ func getEnvPoolInt(key string, fallback int) int { return fallback } +// SQLite memory sizing bounds. The max values are the pre-2026-06 hardcoded +// stanza (256 MB page cache, 1 GB mmap window) and double as the fallback +// when budget detection fails — never worse than the previous behaviour. +const ( + sqliteCacheKBMin = 65536 // 64 MB + sqliteCacheKBMax = 262144 // 256 MB (legacy hardcoded value) + sqliteMmapBytesMin = 268435456 // 256 MB + sqliteMmapBytesMax = 1073741824 // 1 GB (legacy hardcoded value) +) + +// sqliteMemorySizes resolves the SQLite page-cache size (in KB, applied as a +// negative cache_size) and mmap window (in bytes) for the startup PRAGMA +// stanza. With the pure-Go driver both budgets are Go-heap/address-space +// costs, so they scale with the detected memory budget instead of being +// hardcoded: cache = budget/32 clamped to [64 MB, 256 MB], mmap = budget/8 +// clamped to [256 MB, 1 GB] — a 4 GB host yields 128 MB cache + 512 MB mmap. +// budget <= 0 (detection failed) falls back to the legacy hardcoded maxima. +// Operator overrides win unconditionally: SQLITE_CACHE_SIZE_KB (> 0) and +// SQLITE_MMAP_SIZE_BYTES (>= 0; 0 disables mmap). Invalid values are ignored. +func sqliteMemorySizes(budget int64) (cacheKB, mmapBytes int64) { + cacheKB = sqliteCacheKBMax + mmapBytes = sqliteMmapBytesMax + if budget > 0 { + cacheKB = clampInt64(budget/32/1024, sqliteCacheKBMin, sqliteCacheKBMax) + mmapBytes = clampInt64(budget/8, sqliteMmapBytesMin, sqliteMmapBytesMax) + } + if v, ok := getEnvInt64("SQLITE_CACHE_SIZE_KB"); ok && v > 0 { + cacheKB = v + } + if v, ok := getEnvInt64("SQLITE_MMAP_SIZE_BYTES"); ok && v >= 0 { + mmapBytes = v + } + return cacheKB, mmapBytes +} + +func clampInt64(v, lo, hi int64) int64 { + if v < lo { + return lo + } + if v > hi { + return hi + } + return v +} + +func getEnvInt64(key string) (int64, bool) { + if v, ok := os.LookupEnv(key); ok { + if i, err := strconv.ParseInt(strings.TrimSpace(v), 10, 64); err == nil { + return i, true + } + } + return 0, false +} + // scrubDSN returns a DSN-like string with passwords and sensitive fields redacted. // Handles two common forms: // - URL: postgres://user:PASS@host/db → postgres://user:REDACTED@host/db diff --git a/internal/storage/factory_test.go b/internal/storage/factory_test.go index ffc0f98..b8ff4e0 100644 --- a/internal/storage/factory_test.go +++ b/internal/storage/factory_test.go @@ -1,8 +1,13 @@ package storage import ( + "path/filepath" "strings" "testing" + + "gorm.io/gorm" + + "github.com/RandomCodeSpace/otelcontext/internal/membudget" ) func TestNewDatabase_UnsupportedDriver(t *testing.T) { @@ -76,6 +81,148 @@ func TestAutoMigrateModels_SQLite_AllTablesCreated(t *testing.T) { } } +// clearSQLiteMemoryEnv neutralizes the two PRAGMA override env vars for the +// duration of the test. t.Setenv("") registers the restore; an empty value +// fails strconv parsing and is therefore ignored by sqliteMemorySizes. +func clearSQLiteMemoryEnv(t *testing.T) { + t.Helper() + t.Setenv("SQLITE_CACHE_SIZE_KB", "") + t.Setenv("SQLITE_MMAP_SIZE_BYTES", "") +} + +// pragmaInt64 round-trips a PRAGMA query against the live connection. +func pragmaInt64(t *testing.T, db *gorm.DB, pragma string) int64 { + t.Helper() + var v int64 + if err := db.Raw("PRAGMA " + pragma).Scan(&v).Error; err != nil { + t.Fatalf("PRAGMA %s: %v", pragma, err) + } + return v +} + +func TestSQLiteMemorySizes_BudgetScaling(t *testing.T) { + clearSQLiteMemoryEnv(t) + cases := []struct { + name string + budget int64 + wantCacheKB int64 + wantMmap int64 + }{ + {"detection failed -> legacy hardcoded", 0, 262144, 1073741824}, + {"4GB host -> 128MB cache, 512MB mmap", 4 << 30, 131072, 512 << 20}, + {"1GB host clamps to floors", 1 << 30, 65536, 268435456}, + {"64GB host clamps to ceilings", 64 << 30, 262144, 1073741824}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cacheKB, mmapBytes := sqliteMemorySizes(tc.budget) + if cacheKB != tc.wantCacheKB || mmapBytes != tc.wantMmap { + t.Fatalf("sqliteMemorySizes(%d)=(%d,%d) want (%d,%d)", + tc.budget, cacheKB, mmapBytes, tc.wantCacheKB, tc.wantMmap) + } + }) + } +} + +func TestSQLiteMemorySizes_EnvOverridesWinOverBudget(t *testing.T) { + t.Setenv("SQLITE_CACHE_SIZE_KB", "12345") + t.Setenv("SQLITE_MMAP_SIZE_BYTES", "0") // 0 = disable mmap, a legitimate override + cacheKB, mmapBytes := sqliteMemorySizes(4 << 30) + if cacheKB != 12345 || mmapBytes != 0 { + t.Fatalf("env overrides ignored: got (%d,%d) want (12345,0)", cacheKB, mmapBytes) + } +} + +func TestSQLiteMemorySizes_InvalidEnvIgnored(t *testing.T) { + t.Setenv("SQLITE_CACHE_SIZE_KB", "not-a-number") + t.Setenv("SQLITE_MMAP_SIZE_BYTES", "-1") // negative mmap is invalid + cacheKB, mmapBytes := sqliteMemorySizes(4 << 30) + if cacheKB != 131072 || mmapBytes != 512<<20 { + t.Fatalf("invalid env must fall back to budget scaling: got (%d,%d)", cacheKB, mmapBytes) + } +} + +func TestNewDatabase_SQLitePragmas_EnvOverrideRoundTrip(t *testing.T) { + t.Setenv("SQLITE_CACHE_SIZE_KB", "12345") + t.Setenv("SQLITE_MMAP_SIZE_BYTES", "33554432") + + db, err := NewDatabase("sqlite", filepath.Join(t.TempDir(), "override.db")) + if err != nil { + t.Fatalf("sqlite: %v", err) + } + defer closeDB(db) + + if got := pragmaInt64(t, db, "cache_size"); got != -12345 { + t.Fatalf("cache_size=%d want -12345", got) + } + if got := pragmaInt64(t, db, "mmap_size"); got != 33554432 { + t.Fatalf("mmap_size=%d want 33554432", got) + } +} + +func TestNewDatabase_SQLitePragmas_BudgetScaledRoundTrip(t *testing.T) { + clearSQLiteMemoryEnv(t) + + // Whatever this host's budget resolves to, the live connection must carry + // the same numbers the sizing function computes — proves the wiring, not + // the host RAM. + budget, _ := membudget.Detect() + wantCacheKB, wantMmap := sqliteMemorySizes(budget) + + db, err := NewDatabase("sqlite", filepath.Join(t.TempDir(), "scaled.db")) + if err != nil { + t.Fatalf("sqlite: %v", err) + } + defer closeDB(db) + + if got := pragmaInt64(t, db, "cache_size"); got != -wantCacheKB { + t.Fatalf("cache_size=%d want %d", got, -wantCacheKB) + } + if got := pragmaInt64(t, db, "mmap_size"); got != wantMmap { + t.Fatalf("mmap_size=%d want %d", got, wantMmap) + } + + // The rest of the hardened stanza must stay byte-identical — guard the + // values that round-trip numerically. + fixed := []struct { + pragma string + want int64 + }{ + {"synchronous", 1}, // NORMAL + {"temp_store", 2}, // MEMORY + {"wal_autocheckpoint", 10000}, + {"journal_size_limit", 67108864}, + {"busy_timeout", 5000}, + } + for _, f := range fixed { + if got := pragmaInt64(t, db, f.pragma); got != f.want { + t.Fatalf("PRAGMA %s=%d want %d (hardened stanza must not drift)", f.pragma, got, f.want) + } + } + var mode string + if err := db.Raw("PRAGMA journal_mode").Scan(&mode).Error; err != nil || !strings.EqualFold(mode, "wal") { + t.Fatalf("journal_mode=%q err=%v want wal", mode, err) + } +} + +func TestNewDatabase_SQLiteAutoVacuumIncremental(t *testing.T) { + clearSQLiteMemoryEnv(t) + + // Best-effort PRAGMA at startup: on a fresh database file it must take + // effect (auto_vacuum=2 = INCREMENTAL) because it runs before the first + // table is created. Pre-existing files keep their stored mode — that is + // accepted and not asserted here. + db, err := NewDatabase("sqlite", filepath.Join(t.TempDir(), "fresh.db")) + if err != nil { + t.Fatalf("sqlite: %v", err) + } + defer closeDB(db) + + if got := pragmaInt64(t, db, "auto_vacuum"); got != 2 { + t.Fatalf("auto_vacuum=%d want 2 (INCREMENTAL) on a fresh database", got) + } +} + func TestScrubDSN(t *testing.T) { cases := []struct { name string diff --git a/internal/storage/retention.go b/internal/storage/retention.go index 0733695..ca910cd 100644 --- a/internal/storage/retention.go +++ b/internal/storage/retention.go @@ -2,6 +2,7 @@ package storage import ( "context" + "database/sql" "fmt" "log/slog" "strings" @@ -12,7 +13,8 @@ import ( // RetentionScheduler periodically enforces hot-DB retention and runs DB maintenance. // On startup and hourly thereafter it deletes rows older than retentionDays. -// Daily it runs driver-appropriate maintenance (VACUUM ANALYZE / OPTIMIZE / VACUUM). +// Daily it runs driver-appropriate maintenance (VACUUM ANALYZE / OPTIMIZE / +// PRAGMA optimize + incremental_vacuum). type RetentionScheduler struct { repo *Repository retentionDays int @@ -21,6 +23,14 @@ type RetentionScheduler struct { purgeBatchSize int purgeBatchSleep time.Duration + // fullVacuum restores the pre-2026-06 daily full VACUUM on SQLite + // (RETENTION_FULL_VACUUM=true). Off by default: VACUUM holds an exclusive + // lock for 10-60 minutes on multi-GB files (see handleDropFTS in + // internal/api/admin_handlers.go), starving ingest into a 429 storm. The + // default maintenance runs PRAGMA incremental_vacuum(10000) instead; + // on-demand full VACUUM remains available via POST /api/admin/vacuum. + fullVacuum bool + // started is an atomic so a fast-path Stop() before Start() is lock-free. // mu serializes the Start/Stop transition itself (protects cancel + done). started atomic.Bool @@ -62,6 +72,36 @@ func NewRetentionScheduler(repo *Repository, retentionDays, batchSize int, batch // because a previous run was still executing. Intended for tests and telemetry. func (r *RetentionScheduler) SkippedRuns() int64 { return r.skippedRuns.Load() } +// SetFullVacuum opts the daily SQLite maintenance back into a full VACUUM +// (wired from RETENTION_FULL_VACUUM). Call before Start; not synchronized. +func (r *RetentionScheduler) SetFullVacuum(enabled bool) { r.fullVacuum = enabled } + +// sqliteVacuumStatement returns the page-reclaim statement for the daily +// SQLite maintenance pass. Default is incremental_vacuum: it releases up to +// 10k freelist pages per tick without the whole-file exclusive lock a full +// VACUUM takes (it no-ops harmlessly on auto_vacuum=NONE files — fresh +// deploys are provisioned auto_vacuum=INCREMENTAL by NewDatabase). +func sqliteVacuumStatement(fullVacuum bool) string { + if fullVacuum { + return "VACUUM" + } + return "PRAGMA incremental_vacuum(10000)" +} + +// drainQuery executes a row-returning statement and discards every row. +// PRAGMA incremental_vacuum performs its work per step (one freed page per +// row), so the cursor must be walked to completion for the reclaim to happen. +func drainQuery(ctx context.Context, db *sql.DB, query string) error { + rows, err := db.QueryContext(ctx, query) + if err != nil { + return err + } + defer func() { _ = rows.Close() }() + for rows.Next() { //nolint:revive // intentionally empty: stepping is the side effect + } + return rows.Err() +} + // Start launches the scheduler goroutine. It runs an initial purge immediately. // Idempotent and race-free: atomic CAS elects the first caller, and mu // publishes cancel+done before any concurrent Stop can observe started=true. @@ -445,11 +485,21 @@ func (r *RetentionScheduler) runMaintenance(ctx context.Context) { slog.Error("retention: PRAGMA optimize failed", "error", err) maintFailed = true } - if _, err := sqlDB.ExecContext(ctx, "VACUUM"); err != nil { - slog.Error("retention: VACUUM failed", "error", err) + vacuumSQL := sqliteVacuumStatement(r.fullVacuum) + var vacErr error + if r.fullVacuum { + _, vacErr = sqlDB.ExecContext(ctx, vacuumSQL) + } else { + // incremental_vacuum is a row-returning pragma that frees one page + // per step — Exec steps it once on this driver, so it must be + // queried and drained to reclaim the full batch. + vacErr = drainQuery(ctx, sqlDB, vacuumSQL) + } + if vacErr != nil { + slog.Error("retention: vacuum step failed", "statement", vacuumSQL, "error", vacErr) maintFailed = true } - // SQLite VACUUM is whole-DB; record a single observation under "all". + // SQLite maintenance is whole-DB; record a single observation under "all". observe("all", time.Since(start)) } slog.Info("retention maintenance complete", "driver", driver) diff --git a/internal/storage/retention_test.go b/internal/storage/retention_test.go index afc4676..7d49b5a 100644 --- a/internal/storage/retention_test.go +++ b/internal/storage/retention_test.go @@ -2,9 +2,12 @@ package storage import ( "context" + "path/filepath" "sync" "testing" "time" + + "gorm.io/gorm" ) func TestRetentionScheduler_StopBeforeStart_NoDeadlock(t *testing.T) { @@ -99,6 +102,167 @@ func TestRetentionScheduler_MaintenanceVacuumsSQLite(t *testing.T) { } } +// newFileTestRepo builds a Repository backed by a temp-file SQLite DB. The +// vacuum-mode tests below assert on PRAGMA freelist_count, which needs a real +// database file rather than :memory:. +func newFileTestRepo(t *testing.T) *Repository { + t.Helper() + t.Setenv("LOG_FTS_ENABLED", "false") + db, err := NewDatabase("sqlite", filepath.Join(t.TempDir(), "retention.db")) + if err != nil { + t.Fatalf("NewDatabase: %v", err) + } + if err := AutoMigrateModels(db, "sqlite"); err != nil { + t.Fatalf("AutoMigrateModels: %v", err) + } + repo := &Repository{db: db, driver: "sqlite"} + t.Cleanup(func() { _ = repo.Close() }) + return repo +} + +func freelistCount(t *testing.T, db *gorm.DB) int64 { + t.Helper() + var n int64 + if err := db.Raw("PRAGMA freelist_count").Scan(&n).Error; err != nil { + t.Fatalf("PRAGMA freelist_count: %v", err) + } + return n +} + +// forceAutoVacuumNone rewrites the database file with auto_vacuum=NONE so +// PRAGMA incremental_vacuum becomes a guaranteed no-op — the file shape every +// pre-2026-06 deployment has. VACUUM must run on the raw *sql.DB (GORM's Exec +// wraps statements in an implicit tx, and VACUUM refuses to run inside one). +func forceAutoVacuumNone(t *testing.T, db *gorm.DB) { + t.Helper() + sqlDB, err := db.DB() + if err != nil { + t.Fatalf("raw sql.DB: %v", err) + } + if _, err := sqlDB.Exec("PRAGMA auto_vacuum=NONE"); err != nil { + t.Fatalf("PRAGMA auto_vacuum=NONE: %v", err) + } + if _, err := sqlDB.Exec("VACUUM"); err != nil { + t.Fatalf("VACUUM (rebuild to NONE): %v", err) + } + var av int64 + if err := db.Raw("PRAGMA auto_vacuum").Scan(&av).Error; err != nil || av != 0 { + t.Fatalf("auto_vacuum=%d err=%v, want 0 (NONE)", av, err) + } +} + +// seedAndDeleteLogs fills the freelist: insert rows, then bulk-delete them so +// the freed pages sit on the freelist awaiting a vacuum of either flavor. +func seedAndDeleteLogs(t *testing.T, db *gorm.DB) { + t.Helper() + seedLogs(t, db, 2000, time.Now().UTC(), "vacuum-svc") + if err := db.Exec("DELETE FROM logs").Error; err != nil { + t.Fatalf("delete logs: %v", err) + } + if freelistCount(t, db) == 0 { + t.Fatal("test setup broken: bulk delete left an empty freelist") + } +} + +func TestSQLiteVacuumStatement(t *testing.T) { + if got := sqliteVacuumStatement(false); got != "PRAGMA incremental_vacuum(10000)" { + t.Fatalf("default statement = %q, want incremental_vacuum", got) + } + if got := sqliteVacuumStatement(true); got != "VACUUM" { + t.Fatalf("full-vacuum statement = %q, want VACUUM", got) + } +} + +// TestRetentionScheduler_MaintenanceDefaultRunsIncrementalVacuum proves the +// default daily maintenance actually executes incremental_vacuum: on an +// auto_vacuum=INCREMENTAL file (what NewDatabase provisions on fresh deploys) +// freed pages stay on the freelist until incremental_vacuum releases them, so +// an empty freelist after runMaintenance means the statement ran. +func TestRetentionScheduler_MaintenanceDefaultRunsIncrementalVacuum(t *testing.T) { + repo := newFileTestRepo(t) + var av int64 + if err := repo.db.Raw("PRAGMA auto_vacuum").Scan(&av).Error; err != nil || av != 2 { + t.Fatalf("precondition: fresh DB should be auto_vacuum=INCREMENTAL, got %d (err=%v)", av, err) + } + seedAndDeleteLogs(t, repo.db) + + r := NewRetentionScheduler(repo, 7, 10_000, 0) + r.runMaintenance(context.Background()) + + if n := freelistCount(t, repo.db); n != 0 { + t.Fatalf("freelist=%d after default maintenance; incremental_vacuum should have released all pages", n) + } +} + +// TestRetentionScheduler_MaintenanceDefaultSkipsFullVacuum proves the default +// daily maintenance no longer runs a full VACUUM: on an auto_vacuum=NONE file +// (every pre-2026-06 deployment) incremental_vacuum is a no-op, so the +// freelist survives — a full VACUUM would have rebuilt the file and emptied it. +func TestRetentionScheduler_MaintenanceDefaultSkipsFullVacuum(t *testing.T) { + repo := newFileTestRepo(t) + forceAutoVacuumNone(t, repo.db) + seedAndDeleteLogs(t, repo.db) + + r := NewRetentionScheduler(repo, 7, 10_000, 0) + r.runMaintenance(context.Background()) + + // PRAGMA optimize may allocate a few stats pages from the freelist, so + // assert "not fully reclaimed" rather than an exact count. + if n := freelistCount(t, repo.db); n == 0 { + t.Fatal("freelist emptied — a full VACUUM ran during default maintenance; expected incremental_vacuum no-op") + } +} + +// TestRetentionScheduler_MaintenanceFullVacuumWhenEnabled proves the +// RETENTION_FULL_VACUUM opt-in restores the legacy behavior on the same +// auto_vacuum=NONE file shape where incremental_vacuum provably no-ops. +func TestRetentionScheduler_MaintenanceFullVacuumWhenEnabled(t *testing.T) { + repo := newFileTestRepo(t) + forceAutoVacuumNone(t, repo.db) + seedAndDeleteLogs(t, repo.db) + + r := NewRetentionScheduler(repo, 7, 10_000, 0) + r.SetFullVacuum(true) + r.runMaintenance(context.Background()) + + if n := freelistCount(t, repo.db); n != 0 { + t.Fatalf("freelist=%d; RETENTION_FULL_VACUUM=true must run a full VACUUM and empty the freelist", n) + } +} + +func TestDrainQuery_InvalidStatementReturnsError(t *testing.T) { + repo := newTestRepo(t) + sqlDB, err := repo.db.DB() + if err != nil { + t.Fatalf("raw sql.DB: %v", err) + } + if err := drainQuery(context.Background(), sqlDB, "PRAGMA definitely_not_a_pragma("); err == nil { + t.Fatal("drainQuery must surface query errors") + } +} + +// TestRetentionScheduler_MaintenanceCancelledContext drives both SQLite +// maintenance error branches (PRAGMA optimize + vacuum step) via an +// already-cancelled context: the scheduler must log-and-continue, release the +// overlap guard, and stay usable for the next tick. +func TestRetentionScheduler_MaintenanceCancelledContext(t *testing.T) { + repo := newFileTestRepo(t) + r := NewRetentionScheduler(repo, 7, 10_000, 0) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r.runMaintenance(ctx) + + if r.running.Load() { + t.Fatal("running flag must be released after a failed maintenance pass") + } + // A subsequent healthy pass must work. + r.runMaintenance(context.Background()) + if r.running.Load() { + t.Fatal("running flag must be released after the recovery pass") + } +} + func TestRetentionScheduler_NoDataNoError(t *testing.T) { repo := newTestRepo(t) r := NewRetentionScheduler(repo, 7, 10_000, 5*time.Millisecond) diff --git a/internal/storage/service_map_test.go b/internal/storage/service_map_test.go new file mode 100644 index 0000000..26e7173 --- /dev/null +++ b/internal/storage/service_map_test.go @@ -0,0 +1,465 @@ +package storage + +import ( + "context" + "fmt" + "math" + "sort" + "testing" + "time" + + "gorm.io/gorm" +) + +// referenceServiceMapMetrics is a verbatim copy of the pre-P2.1 +// GetServiceMapMetrics aggregation: load full Span rows (including the +// CompressedText attributes column) and group in Go. It exists so the +// SQL-aggregated implementation can be proven equivalent on a shared fixture. +// +// Note: this old path never populated ServiceMapNode.ErrorCount or +// ServiceMapEdge.ErrorRate — both stayed 0 — so node error counts are +// asserted separately in TestGetServiceMapMetrics_NodeErrorCounts. +func referenceServiceMapMetrics(t *testing.T, repo *Repository, ctx context.Context, start, end time.Time) *ServiceMapMetrics { + t.Helper() + tenant := TenantFromContext(ctx) + var spans []Span + query := repo.db.WithContext(ctx).Model(&Span{}).Where(sqlWhereTenantID, tenant) + if !start.IsZero() && !end.IsZero() { + query = query.Where("start_time BETWEEN ? AND ?", start, end) + } + if err := query.Limit(serviceMapSpanLimit).Find(&spans).Error; err != nil { + t.Fatalf("reference span load: %v", err) + } + + spanMap := make(map[string]Span) + nodeStats := make(map[string]*ServiceMapNode) + edgeStats := make(map[string]*ServiceMapEdge) + + for _, s := range spans { + spanMap[s.SpanID] = s + if s.ServiceName == "" { + continue + } + if _, ok := nodeStats[s.ServiceName]; !ok { + nodeStats[s.ServiceName] = &ServiceMapNode{Name: s.ServiceName} + } + ns := nodeStats[s.ServiceName] + ns.TotalTraces++ + ns.AvgLatencyMs += float64(s.Duration) + } + + nodes := make([]ServiceMapNode, 0) //nolint:prealloc // verbatim copy of the old implementation + for _, ns := range nodeStats { + if ns.TotalTraces > 0 { + ns.AvgLatencyMs = ns.AvgLatencyMs / float64(ns.TotalTraces) / 1000.0 + ns.AvgLatencyMs = math.Round(ns.AvgLatencyMs*100) / 100 + } + nodes = append(nodes, *ns) + } + + for _, s := range spans { + if s.ParentSpanID == "" || s.ParentSpanID == "0000000000000000" { + continue + } + parent, ok := spanMap[s.ParentSpanID] + if !ok { + continue + } + source := parent.ServiceName + target := s.ServiceName + if source == "" || target == "" || source == target { + continue + } + key := fmt.Sprintf("%s->%s", source, target) + if _, ok := edgeStats[key]; !ok { + edgeStats[key] = &ServiceMapEdge{Source: source, Target: target} + } + es := edgeStats[key] + es.CallCount++ + es.AvgLatencyMs += float64(s.Duration) + } + + edges := make([]ServiceMapEdge, 0) //nolint:prealloc // verbatim copy of the old implementation + for _, es := range edgeStats { + if es.CallCount > 0 { + es.AvgLatencyMs = es.AvgLatencyMs / float64(es.CallCount) / 1000.0 + es.AvgLatencyMs = math.Round(es.AvgLatencyMs*100) / 100 + } + edges = append(edges, *es) + } + + return &ServiceMapMetrics{Nodes: nodes, Edges: edges} +} + +// serviceMapFixtureWindow is the query window the fixture spans sit inside. +func serviceMapFixtureWindow() (start, end time.Time) { + base := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC) + return base, base.Add(10 * time.Minute) +} + +// seedServiceMapFixture inserts a topology exercising every branch of the +// aggregation: cross-service parent-child edges, error spans, same-service +// child (no edge), root span, zero parent id, dangling parent id, empty +// service name, out-of-window span, and a foreign-tenant span. +// +// Expected (window-scoped, tenant "default"): +// +// nodes: svc-a {2 spans, 0 errors, 55ms} — a1 root 100000µs, a2 child-of-a1 10000µs +// svc-b {4 spans, 1 error, 25ms} — b1 50000µs ERROR, b2 30000µs, m1 7000µs +// (dangling parent), f1 13000µs (parent has +// empty service → no edge) +// svc-c {2 spans, 1 error, 30ms} — c1 20000µs ERROR (parent b1), +// z1 40000µs (parent 0000…) +// edges: svc-a→svc-b {2 calls, 40ms}, svc-b→svc-c {1 call, 20ms} +func seedServiceMapFixture(t *testing.T, repo *Repository) { + t.Helper() + start, _ := serviceMapFixtureWindow() + in := start.Add(1 * time.Minute) + out := start.Add(20 * time.Minute) + + mk := func(tenant, spanID, parentID, service string, dur int64, status string, ts time.Time) Span { + return Span{ + TenantID: tenant, + TraceID: "trace-svcmap-1", + SpanID: spanID, + ParentSpanID: parentID, + OperationName: "op-" + spanID, + StartTime: ts, + EndTime: ts.Add(time.Duration(dur) * time.Microsecond), + Duration: dur, + ServiceName: service, + Status: status, + AttributesJSON: CompressedText(`{"http.method":"GET","fixture":"` + spanID + `"}`), + } + } + + spans := []Span{ + mk("default", "a1", "", "svc-a", 100000, "STATUS_CODE_UNSET", in), + mk("default", "a2", "a1", "svc-a", 10000, "STATUS_CODE_UNSET", in), // same-service child: node yes, edge no + mk("default", "b1", "a1", "svc-b", 50000, "STATUS_CODE_ERROR", in), // edge a→b + mk("default", "b2", "a1", "svc-b", 30000, "STATUS_CODE_UNSET", in), // edge a→b + mk("default", "m1", "ffffffffffffffff", "svc-b", 7000, "STATUS_CODE_UNSET", in), // dangling parent: no edge + mk("default", "e1", "a1", "", 5000, "STATUS_CODE_UNSET", in), // empty service: no node, no edge + mk("default", "f1", "e1", "svc-b", 13000, "STATUS_CODE_UNSET", in), // parent has empty service: no edge + mk("default", "c1", "b1", "svc-c", 20000, "STATUS_CODE_ERROR", in), // edge b→c + mk("default", "z1", "0000000000000000", "svc-c", 40000, "STATUS_CODE_UNSET", in), // zero parent: no edge + mk("default", "o1", "", "svc-a", 999999, "STATUS_CODE_ERROR", out), // outside window + mk("tenant-b", "x1", "", "svc-a", 888888, "STATUS_CODE_ERROR", in), // foreign tenant + } + if err := repo.BatchCreateSpans(spans); err != nil { + t.Fatalf("seedServiceMapFixture: %v", err) + } +} + +func sortServiceMap(m *ServiceMapMetrics) { + sort.Slice(m.Nodes, func(i, j int) bool { return m.Nodes[i].Name < m.Nodes[j].Name }) + sort.Slice(m.Edges, func(i, j int) bool { + if m.Edges[i].Source != m.Edges[j].Source { + return m.Edges[i].Source < m.Edges[j].Source + } + return m.Edges[i].Target < m.Edges[j].Target + }) +} + +// TestGetServiceMapMetrics_MatchesReferenceImplementation proves the +// SQL-aggregated implementation is output-equivalent to the old full-row +// in-Go aggregation on a fixture covering every branch. ErrorCount is the +// one intentional delta (the old path left it permanently 0 — a latent bug +// since downstream buildGraphFromDB divides ErrorCount/TotalTraces) and is +// asserted in TestGetServiceMapMetrics_NodeErrorCounts instead. +func TestGetServiceMapMetrics_MatchesReferenceImplementation(t *testing.T) { + repo := newTestRepo(t) + seedServiceMapFixture(t, repo) + ctx := context.Background() + start, end := serviceMapFixtureWindow() + + want := referenceServiceMapMetrics(t, repo, ctx, start, end) + got, err := repo.GetServiceMapMetrics(ctx, start, end) + if err != nil { + t.Fatalf("GetServiceMapMetrics: %v", err) + } + + sortServiceMap(want) + sortServiceMap(got) + + if len(got.Nodes) != len(want.Nodes) { + t.Fatalf("node count: got %d want %d (%+v vs %+v)", len(got.Nodes), len(want.Nodes), got.Nodes, want.Nodes) + } + for i := range want.Nodes { + g, w := got.Nodes[i], want.Nodes[i] + if g.Name != w.Name || g.TotalTraces != w.TotalTraces || g.AvgLatencyMs != w.AvgLatencyMs { + t.Errorf("node %d mismatch: got %+v want %+v", i, g, w) + } + } + if len(got.Edges) != len(want.Edges) { + t.Fatalf("edge count: got %d want %d (%+v vs %+v)", len(got.Edges), len(want.Edges), got.Edges, want.Edges) + } + for i := range want.Edges { + if got.Edges[i] != want.Edges[i] { + t.Errorf("edge %d mismatch: got %+v want %+v", i, got.Edges[i], want.Edges[i]) + } + } +} + +// TestGetServiceMapMetrics_NodeErrorCounts asserts that node error counts are +// populated from span status. The old implementation never set ErrorCount +// (always 0), so this test documents the intentional fix that landed with the +// SQL aggregation. +func TestGetServiceMapMetrics_NodeErrorCounts(t *testing.T) { + repo := newTestRepo(t) + seedServiceMapFixture(t, repo) + start, end := serviceMapFixtureWindow() + + got, err := repo.GetServiceMapMetrics(context.Background(), start, end) + if err != nil { + t.Fatalf("GetServiceMapMetrics: %v", err) + } + sortServiceMap(got) + + wantErrors := map[string]int64{"svc-a": 0, "svc-b": 1, "svc-c": 1} + if len(got.Nodes) != len(wantErrors) { + t.Fatalf("node count: got %d want %d (%+v)", len(got.Nodes), len(wantErrors), got.Nodes) + } + for _, n := range got.Nodes { + if n.ErrorCount != wantErrors[n.Name] { + t.Errorf("node %s ErrorCount = %d, want %d", n.Name, n.ErrorCount, wantErrors[n.Name]) + } + } +} + +// TestGetServiceMapMetrics_FixtureValues pins the exact expected fixture +// output so a regression in BOTH implementations (reference and SQL) cannot +// slip through the equality test unnoticed. +func TestGetServiceMapMetrics_FixtureValues(t *testing.T) { + repo := newTestRepo(t) + seedServiceMapFixture(t, repo) + start, end := serviceMapFixtureWindow() + + got, err := repo.GetServiceMapMetrics(context.Background(), start, end) + if err != nil { + t.Fatalf("GetServiceMapMetrics: %v", err) + } + sortServiceMap(got) + + wantNodes := []ServiceMapNode{ + {Name: "svc-a", TotalTraces: 2, ErrorCount: 0, AvgLatencyMs: 55}, + {Name: "svc-b", TotalTraces: 4, ErrorCount: 1, AvgLatencyMs: 25}, + {Name: "svc-c", TotalTraces: 2, ErrorCount: 1, AvgLatencyMs: 30}, + } + wantEdges := []ServiceMapEdge{ + {Source: "svc-a", Target: "svc-b", CallCount: 2, AvgLatencyMs: 40}, + {Source: "svc-b", Target: "svc-c", CallCount: 1, AvgLatencyMs: 20}, + } + + if len(got.Nodes) != len(wantNodes) { + t.Fatalf("nodes: got %+v want %+v", got.Nodes, wantNodes) + } + for i := range wantNodes { + if got.Nodes[i] != wantNodes[i] { + t.Errorf("node %d: got %+v want %+v", i, got.Nodes[i], wantNodes[i]) + } + } + if len(got.Edges) != len(wantEdges) { + t.Fatalf("edges: got %+v want %+v", got.Edges, wantEdges) + } + for i := range wantEdges { + if got.Edges[i] != wantEdges[i] { + t.Errorf("edge %d: got %+v want %+v", i, got.Edges[i], wantEdges[i]) + } + } +} + +// TestGetServiceMapMetrics_TenantScoped verifies the foreign tenant sees only +// its own spans through the same code path. +func TestGetServiceMapMetrics_TenantScoped(t *testing.T) { + repo := newTestRepo(t) + seedServiceMapFixture(t, repo) + start, end := serviceMapFixtureWindow() + ctx := WithTenantContext(context.Background(), "tenant-b") + + got, err := repo.GetServiceMapMetrics(ctx, start, end) + if err != nil { + t.Fatalf("GetServiceMapMetrics: %v", err) + } + if len(got.Nodes) != 1 || got.Nodes[0].Name != "svc-a" || got.Nodes[0].TotalTraces != 1 { + t.Fatalf("tenant-b nodes: %+v", got.Nodes) + } + if got.Nodes[0].ErrorCount != 1 { + t.Fatalf("tenant-b ErrorCount = %d, want 1", got.Nodes[0].ErrorCount) + } + if len(got.Edges) != 0 { + t.Fatalf("tenant-b edges: %+v", got.Edges) + } +} + +// TestGetServiceMapMetrics_ZeroWindowIncludesAll preserves the legacy +// semantics: a zero start or end skips the time filter entirely. +func TestGetServiceMapMetrics_ZeroWindowIncludesAll(t *testing.T) { + repo := newTestRepo(t) + seedServiceMapFixture(t, repo) + + got, err := repo.GetServiceMapMetrics(context.Background(), time.Time{}, time.Time{}) + if err != nil { + t.Fatalf("GetServiceMapMetrics: %v", err) + } + want := referenceServiceMapMetrics(t, repo, context.Background(), time.Time{}, time.Time{}) + sortServiceMap(got) + sortServiceMap(want) + + // The out-of-window span o1 (svc-a) must now be included: 3 svc-a spans. + var svcA *ServiceMapNode + for i := range got.Nodes { + if got.Nodes[i].Name == "svc-a" { + svcA = &got.Nodes[i] + } + } + if svcA == nil || svcA.TotalTraces != 3 { + t.Fatalf("zero window svc-a node: %+v", got.Nodes) + } + if len(got.Nodes) != len(want.Nodes) || len(got.Edges) != len(want.Edges) { + t.Fatalf("zero window mismatch vs reference: got %+v / %+v want %+v / %+v", + got.Nodes, got.Edges, want.Nodes, want.Edges) + } +} + +// TestGetServiceMapMetrics_Empty returns non-nil empty slices on a fresh DB +// (the JSON API contract expects [] rather than null). +func TestGetServiceMapMetrics_Empty(t *testing.T) { + repo := newTestRepo(t) + got, err := repo.GetServiceMapMetrics(context.Background(), time.Time{}, time.Time{}) + if err != nil { + t.Fatalf("GetServiceMapMetrics: %v", err) + } + if got.Nodes == nil || got.Edges == nil { + t.Fatalf("expected non-nil empty slices, got %+v", got) + } + if len(got.Nodes) != 0 || len(got.Edges) != 0 { + t.Fatalf("expected empty result, got %+v", got) + } +} + +// TestGetServiceMapMetrics_CancelledContext surfaces query errors. +func TestGetServiceMapMetrics_CancelledContext(t *testing.T) { + repo := newTestRepo(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := repo.GetServiceMapMetrics(ctx, time.Time{}, time.Time{}); err == nil { + t.Fatal("expected error on cancelled context") + } +} + +// TestGetServiceMapMetrics_EdgeQueryError exercises the second (edge-pass) +// error branch. The node aggregate runs via Scan, which does not pass through +// GORM's Query callback chain; the edge scan runs via Find, which does. A +// before-query callback therefore fires exactly once — for the edge query — +// and cancels the context just before it executes, failing only that query. +func TestGetServiceMapMetrics_EdgeQueryError(t *testing.T) { + repo := newTestRepo(t) + seedServiceMapFixture(t, repo) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + err := repo.db.Callback().Query().Before("gorm:query").Register("test_cancel_edge_query", func(*gorm.DB) { + cancel() + }) + if err != nil { + t.Fatalf("register callback: %v", err) + } + t.Cleanup(func() { _ = repo.db.Callback().Query().Remove("test_cancel_edge_query") }) + + if _, err := repo.GetServiceMapMetrics(ctx, time.Time{}, time.Time{}); err == nil { + t.Fatal("expected edge query error after context cancellation") + } +} + +// TestGetServiceMapMetrics_EdgeRowLimit lowers serviceMapSpanLimit so the +// warning path runs: edges are computed from the truncated scan while node +// stats (GROUP BY, unbounded) still cover every row. +func TestGetServiceMapMetrics_EdgeRowLimit(t *testing.T) { + repo := newTestRepo(t) + seedServiceMapFixture(t, repo) + start, end := serviceMapFixtureWindow() + + orig := serviceMapSpanLimit + serviceMapSpanLimit = 4 + t.Cleanup(func() { serviceMapSpanLimit = orig }) + + got, err := repo.GetServiceMapMetrics(context.Background(), start, end) + if err != nil { + t.Fatalf("GetServiceMapMetrics: %v", err) + } + // Node aggregation is not subject to the edge-scan limit. + if len(got.Nodes) != 3 { + t.Fatalf("expected all 3 nodes despite edge row limit, got %+v", got.Nodes) + } + // The edge pass saw at most 4 of the 9 in-window rows. + var calls int64 + for _, e := range got.Edges { + calls += e.CallCount + } + if calls > 3 { + t.Fatalf("edge pass exceeded the row limit: %+v", got.Edges) + } +} + +// BenchmarkGetServiceMapMetrics measures the aggregation over a few thousand +// seeded spans with realistic compressed attribute payloads — the workload +// that made the old full-row implementation take seconds on the dashboard. +func BenchmarkGetServiceMapMetrics(b *testing.B) { + db, err := NewDatabase("sqlite", ":memory:") + if err != nil { + b.Fatalf("NewDatabase: %v", err) + } + if err := AutoMigrateModels(db, "sqlite"); err != nil { + b.Fatalf("AutoMigrateModels: %v", err) + } + repo := &Repository{db: db, driver: "sqlite"} + defer repo.Close() + + const ( + numSpans = 5000 + numServices = 10 + ) + base := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC) + spans := make([]Span, numSpans) + for i := range numSpans { + svc := fmt.Sprintf("svc-%d", i%numServices) + parent := "" + if i > 0 { + parent = fmt.Sprintf("%016x", i-1) // chain → cross-service edges + } + status := "STATUS_CODE_UNSET" + if i%20 == 0 { + status = "STATUS_CODE_ERROR" + } + spans[i] = Span{ + TenantID: "default", + TraceID: fmt.Sprintf("trace-%04d", i/50), + SpanID: fmt.Sprintf("%016x", i), + ParentSpanID: parent, + OperationName: "GET /api/resource", + StartTime: base.Add(time.Duration(i) * time.Millisecond), + EndTime: base.Add(time.Duration(i)*time.Millisecond + time.Millisecond), + Duration: int64(1000 + i%5000), + ServiceName: svc, + Status: status, + AttributesJSON: CompressedText(fmt.Sprintf( + `{"http.method":"GET","http.url":"/api/resource/%d","http.status_code":200,"net.peer.name":"%s","enduser.id":"user-%d"}`, + i, svc, i%100)), + } + } + if err := repo.BatchCreateSpans(spans); err != nil { + b.Fatalf("seed: %v", err) + } + + ctx := context.Background() + start := base.Add(-time.Minute) + end := base.Add(time.Hour) + + b.ResetTimer() + for b.Loop() { + if _, err := repo.GetServiceMapMetrics(ctx, start, end); err != nil { + b.Fatalf("GetServiceMapMetrics: %v", err) + } + } +} diff --git a/internal/storage/trace_repo.go b/internal/storage/trace_repo.go index f0ae992..846b733 100644 --- a/internal/storage/trace_repo.go +++ b/internal/storage/trace_repo.go @@ -255,72 +255,104 @@ func (r *Repository) GetTracesFiltered(ctx context.Context, start, end time.Time }, nil } -const serviceMapSpanLimit = 500_000 +// serviceMapSpanLimit caps the edge-pass span scan. A var (not const) so the +// row-limit warning path is testable without seeding 500k rows. +var serviceMapSpanLimit = 500_000 + +// serviceMapNodeRow receives the per-service GROUP BY aggregate used to build +// ServiceMapNode entries without loading span rows into Go. +type serviceMapNodeRow struct { + ServiceName string + SpanCount int64 + AvgDuration float64 + ErrorCount int64 +} + +// serviceMapSpanRow is the narrow projection scanned for the edge pass. It is +// deliberately NOT Span: AttributesJSON (CompressedText) zstd-decompresses +// per row in Scan(), which dominated the cost of the old full-row load. +type serviceMapSpanRow struct { + SpanID string + ParentSpanID string + ServiceName string + Duration int64 + Status string + StartTime time.Time +} // GetServiceMapMetrics computes topology metrics from spans scoped to the // tenant on ctx. +// +// Node stats come from a single portable GROUP BY aggregate so the database +// does the heavy lifting. Edge stats still need the parent→child resolution +// via span_id done in Go, but over the narrow serviceMapSpanRow projection so +// the compressed attributes column is never scanned. `duration * 1.0` keeps +// AVG in floating point on every dialect (SQL Server truncates AVG(bigint)). func (r *Repository) GetServiceMapMetrics(ctx context.Context, start, end time.Time) (*ServiceMapMetrics, error) { tenant := TenantFromContext(ctx) - var spans []Span - query := r.db.WithContext(ctx).Model(&Span{}).Where(sqlWhereTenantID, tenant) + nodeQuery := r.db.WithContext(ctx).Model(&Span{}). + Select("service_name, COUNT(*) as span_count, AVG(duration * 1.0) as avg_duration, "+ + "SUM(CASE WHEN status = 'STATUS_CODE_ERROR' THEN 1 ELSE 0 END) as error_count"). + Where(sqlWhereTenantID, tenant). + Where("service_name <> ''") if !start.IsZero() && !end.IsZero() { - query = query.Where("start_time BETWEEN ? AND ?", start, end) + nodeQuery = nodeQuery.Where("start_time BETWEEN ? AND ?", start, end) + } + var nodeRows []serviceMapNodeRow + if err := nodeQuery.Group("service_name").Scan(&nodeRows).Error; err != nil { + return nil, fmt.Errorf("failed to aggregate service map nodes: %w", err) } - if err := query.Limit(serviceMapSpanLimit).Find(&spans).Error; err != nil { + nodes := make([]ServiceMapNode, 0, len(nodeRows)) + for _, nr := range nodeRows { + nodes = append(nodes, ServiceMapNode{ + Name: nr.ServiceName, + TotalTraces: nr.SpanCount, + ErrorCount: nr.ErrorCount, + // AVG(duration) is microseconds; convert to ms and round to 2dp. + AvgLatencyMs: math.Round(nr.AvgDuration/1000.0*100) / 100, + }) + } + + edgeQuery := r.db.WithContext(ctx).Model(&Span{}). + Select("span_id, parent_span_id, service_name, duration, status, start_time"). + Where(sqlWhereTenantID, tenant) + if !start.IsZero() && !end.IsZero() { + edgeQuery = edgeQuery.Where("start_time BETWEEN ? AND ?", start, end) + } + var spans []serviceMapSpanRow + if err := edgeQuery.Limit(serviceMapSpanLimit).Find(&spans).Error; err != nil { return nil, fmt.Errorf("failed to fetch spans: %w", err) } if len(spans) == serviceMapSpanLimit { - slog.Warn("GetServiceMapMetrics: span query hit row limit, topology may be incomplete", "limit", serviceMapSpanLimit) + slog.Warn("GetServiceMapMetrics: edge span query hit row limit, edge topology may be incomplete", "limit", serviceMapSpanLimit) } - spanMap := make(map[string]Span) - nodeStats := make(map[string]*ServiceMapNode) - edgeStats := make(map[string]*ServiceMapEdge) - + // Parent resolution map (span_id → service name) is built over ALL rows, + // including empty service names, matching the old full-span map exactly. + serviceBySpanID := make(map[string]string, len(spans)) for _, s := range spans { - spanMap[s.SpanID] = s - - if s.ServiceName == "" { - continue - } - - if _, ok := nodeStats[s.ServiceName]; !ok { - nodeStats[s.ServiceName] = &ServiceMapNode{Name: s.ServiceName} - } - ns := nodeStats[s.ServiceName] - ns.TotalTraces++ - ns.AvgLatencyMs += float64(s.Duration) - } - - nodes := make([]ServiceMapNode, 0) - for _, ns := range nodeStats { - if ns.TotalTraces > 0 { - ns.AvgLatencyMs = ns.AvgLatencyMs / float64(ns.TotalTraces) / 1000.0 - ns.AvgLatencyMs = math.Round(ns.AvgLatencyMs*100) / 100 - } - nodes = append(nodes, *ns) + serviceBySpanID[s.SpanID] = s.ServiceName } + edgeStats := make(map[string]*ServiceMapEdge) for _, s := range spans { if s.ParentSpanID == "" || s.ParentSpanID == "0000000000000000" { continue } - parent, ok := spanMap[s.ParentSpanID] + source, ok := serviceBySpanID[s.ParentSpanID] if !ok { continue } - - source := parent.ServiceName target := s.ServiceName if source == "" || target == "" || source == target { continue } - key := fmt.Sprintf("%s->%s", source, target) + key := source + "->" + target if _, ok := edgeStats[key]; !ok { edgeStats[key] = &ServiceMapEdge{Source: source, Target: target} } @@ -329,7 +361,7 @@ func (r *Repository) GetServiceMapMetrics(ctx context.Context, start, end time.T es.AvgLatencyMs += float64(s.Duration) } - edges := make([]ServiceMapEdge, 0) + edges := make([]ServiceMapEdge, 0, len(edgeStats)) for _, es := range edgeStats { if es.CallCount > 0 { es.AvgLatencyMs = es.AvgLatencyMs / float64(es.CallCount) / 1000.0 diff --git a/internal/telemetry/metrics.go b/internal/telemetry/metrics.go index 44f0afc..358a56e 100644 --- a/internal/telemetry/metrics.go +++ b/internal/telemetry/metrics.go @@ -98,13 +98,38 @@ type Metrics struct { // --- GraphRAG overflow --- GraphRAGEventsDroppedTotal *prometheus.CounterVec + // GraphRAGTenantsEvictedTotal counts tenant store slices evicted after + // exceeding GRAPHRAG_TENANT_IDLE_TTL. The default tenant is never + // evicted; a steady non-zero rate on a single-tenant install means + // rogue tenant IDs are reaching ingest. + GraphRAGTenantsEvictedTotal prometheus.Counter + + // --- In-memory store census (OOM-survival work) --- + // GraphRAGStoreEntities — live node counts per entity kind across tenants + // (tenants|services|operations|traces|spans|log_clusters|metrics|anomalies). + // GraphRAGStoreEdges — live edge counts per store (service|trace|signal|anomaly). + // Together with the ring/drain gauges these attribute RSS growth to a + // specific structure before a heap profile is needed. + GraphRAGStoreEntities *prometheus.GaugeVec + GraphRAGStoreEdges *prometheus.GaugeVec + TSDBRingSeriesActive prometheus.Gauge + // TSDBRingSeriesRejected — points refused a NEW ring series at the + // tenant-scoped series cap (existing series keep recording). + TSDBRingSeriesRejected prometheus.Counter + DrainTemplatesActive prometheus.Gauge + // --- Async ingest pipeline (Phase 1 robustness work) --- // IngestPipelineQueueDepth — current queue depth, sampled on every Submit. // Labeled by signal so spikes can be attributed to traces vs logs. IngestPipelineQueueDepth *prometheus.GaugeVec + // IngestPipelineQueueBytes — approximate bytes held by queued batches. + // Reserved at Submit, released when a worker finishes the batch; the + // byte cap (INGEST_PIPELINE_MAX_BYTES) rejects submissions above it. + IngestPipelineQueueBytes prometheus.Gauge // IngestPipelineDroppedTotal — batches that did NOT reach the DB. // reason="soft_backpressure" — healthy batch dropped at >=90% fullness. // reason="queue_full" — batch rejected at 100% capacity (client got 429/RESOURCE_EXHAUSTED). + // reason="bytes_full" — batch rejected at the byte cap (even priority batches). IngestPipelineDroppedTotal *prometheus.CounterVec // HTTPOTLPThrottledTotal — count of HTTP 429s issued by the OTLP HTTP @@ -323,11 +348,39 @@ func New() *Metrics { Name: "otelcontext_graphrag_events_dropped_total", Help: "Events dropped because the GraphRAG event channel was full.", }, []string{"signal"}), + GraphRAGTenantsEvictedTotal: promauto.NewCounter(prometheus.CounterOpts{ + Name: "otelcontext_graphrag_tenants_evicted_total", + Help: "Tenant store slices evicted after exceeding the idle TTL (GRAPHRAG_TENANT_IDLE_TTL). The default tenant is never evicted.", + }), + GraphRAGStoreEntities: promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "otelcontext_graphrag_store_entities", + Help: "Live GraphRAG node counts across tenants, by entity kind (tenants|services|operations|traces|spans|log_clusters|metrics|anomalies).", + }, []string{"entity"}), + GraphRAGStoreEdges: promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "otelcontext_graphrag_store_edges", + Help: "Live GraphRAG edge counts across tenants, by store (service|trace|signal|anomaly).", + }, []string{"store"}), + TSDBRingSeriesActive: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "otelcontext_tsdb_ring_series_active", + Help: "Distinct metric series currently held in TSDB ring buffers.", + }), + TSDBRingSeriesRejected: promauto.NewCounter(prometheus.CounterOpts{ + Name: "otelcontext_tsdb_ring_series_rejected_total", + Help: "Metric points refused a new TSDB ring series at the cardinality cap (existing series keep recording).", + }), + DrainTemplatesActive: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "otelcontext_drain_templates_active", + Help: "Live Drain log templates (bounded by the 50k LRU cap).", + }), IngestPipelineQueueDepth: promauto.NewGaugeVec(prometheus.GaugeOpts{ Name: "otelcontext_ingest_pipeline_queue_depth", Help: "Current depth of the async ingest pipeline queue, by signal type.", }, []string{"signal"}), + IngestPipelineQueueBytes: promauto.NewGauge(prometheus.GaugeOpts{ + Name: "otelcontext_ingest_pipeline_queue_bytes", + Help: "Approximate bytes held by batches in the async ingest queue.", + }), HTTPOTLPThrottledTotal: promauto.NewCounterVec(prometheus.CounterOpts{ Name: "otelcontext_http_otlp_throttled_total", Help: "OTLP HTTP requests rejected with 429 because the async ingest pipeline is at capacity, by signal type.", diff --git a/internal/tsdb/aggregator.go b/internal/tsdb/aggregator.go index 5b11139..45def94 100644 --- a/internal/tsdb/aggregator.go +++ b/internal/tsdb/aggregator.go @@ -26,13 +26,13 @@ type RawMetric struct { // Aggregator manages in-memory tumbling windows for metrics. type Aggregator struct { - repo *storage.Repository - windowSize time.Duration - buckets map[string]*storage.MetricBucket - mu sync.Mutex - stopChan chan struct{} - flushChan chan []storage.MetricBucket - pool sync.Pool + repo *storage.Repository + windowSize time.Duration + buckets map[string]*storage.MetricBucket + mu sync.Mutex + stopChan chan struct{} + flushChan chan []storage.MetricBucket + pool sync.Pool // droppedBatches is incremented in flush() (under no lock other than // the outer mu — but the increment path runs after the unlock) and // read by DroppedBatches() concurrently from telemetry scrape paths. @@ -52,11 +52,11 @@ type Aggregator struct { // to a tenant-specific overflow bucket so a noisy tenant cannot // starve siblings of fresh series. seriesPerTenant counts unique // (non-overflow) bucket keys per tenant and is reset by flush(). - maxCardinality int // 0 = unlimited - perTenantCardinality int // 0 = unlimited (global cap still applies) - cardinalityOverflow func(tenantID string) // labeled per overflow event for Prometheus - seriesPerTenant map[string]int //nolint:unused // touched only via mu - overflowKey string // constant key for the global overflow bucket + maxCardinality int // 0 = unlimited + perTenantCardinality int // 0 = unlimited (global cap still applies) + cardinalityOverflow func(tenantID string) // labeled per overflow event for Prometheus + seriesPerTenant map[string]int // touched only via mu + overflowKey string // constant key for the global overflow bucket // Ring buffer accelerator (optional) ring *RingBuffer @@ -161,8 +161,10 @@ func (a *Aggregator) Ingest(m RawMetric) { key := fmt.Sprintf("%s|%s|%s|%s", m.TenantID, m.ServiceName, m.Name, string(attrJSON)) // Feed ring buffer and metric counter outside the lock (both are thread-safe). + // The ring enforces its own tenant-scoped series cap and counts rejections + // via its onSeriesRejected callback, so the bool result is not re-counted here. if a.ring != nil { - a.ring.Record(m.Name, m.ServiceName, m.Value, m.Timestamp) + a.ring.Record(m.TenantID, m.Name, m.ServiceName, m.Value, m.Timestamp) } if a.onIngest != nil { a.onIngest() diff --git a/internal/tsdb/aggregator_test.go b/internal/tsdb/aggregator_test.go index d794f2f..cd594cb 100644 --- a/internal/tsdb/aggregator_test.go +++ b/internal/tsdb/aggregator_test.go @@ -250,6 +250,34 @@ func TestAggregator_DefaultBehaviorUnchanged(t *testing.T) { } } +// TestAggregator_IngestFeedsRingWithTenant verifies Ingest plumbs the +// point's TenantID into the attached ring buffer, so two tenants emitting +// the same service+metric never merge into one ring series. +func TestAggregator_IngestFeedsRingWithTenant(t *testing.T) { + a := NewAggregator(nil, time.Minute) + rb := NewRingBuffer(8, time.Minute, 0, nil) + a.SetRingBuffer(rb) + + mA := newRawMetric("tenant-a", "svc", "latency", 0) + mA.Value = 1.0 + a.Ingest(mA) + mB := newRawMetric("tenant-b", "svc", "latency", 0) + mB.Value = 100.0 + a.Ingest(mB) + + aggsA := rb.QueryRecent("tenant-a", "latency", "svc", 8) + if len(aggsA) != 1 || aggsA[0].Sum != 1.0 { + t.Errorf("tenant-a ring aggs=%v, want one window with Sum=1", aggsA) + } + aggsB := rb.QueryRecent("tenant-b", "latency", "svc", 8) + if len(aggsB) != 1 || aggsB[0].Sum != 100.0 { + t.Errorf("tenant-b ring aggs=%v, want one window with Sum=100", aggsB) + } + if got := rb.MetricCount(); got != 2 { + t.Errorf("ring MetricCount=%d, want 2 tenant-scoped series", got) + } +} + // TestAggregator_OverflowBucketStatsCorrect ensures that successive // overflow points for the same tenant accumulate into ONE overflow // bucket (not many) and that min/max/sum/count update correctly. diff --git a/internal/tsdb/ringbuffer.go b/internal/tsdb/ringbuffer.go index df26fa9..6bc09a5 100644 --- a/internal/tsdb/ringbuffer.go +++ b/internal/tsdb/ringbuffer.go @@ -8,6 +8,8 @@ import ( "sort" "sync" "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" ) // WindowAgg is the pre-computed aggregate for one time window slot. @@ -147,27 +149,48 @@ func (r *MetricRing) Windows(n int) []WindowAgg { return out } -// RingBuffer manages per-metric ring buffers, keyed by "service|metric". +// RingBuffer manages per-metric ring buffers, keyed by "tenant|service|metric". type RingBuffer struct { mu sync.RWMutex rings map[string]*MetricRing slots int windowDur time.Duration + // maxSeries caps the number of distinct ring series (0 = unlimited). + // Past the cap, NEW series are refused — existing series keep + // recording — and onSeriesRejected fires once per refused point. + maxSeries int + onSeriesRejected func() } // NewRingBuffer creates a RingBuffer. // slots × windowDur = total retention (e.g. 120 × 30s = 1 hour). -func NewRingBuffer(slots int, windowDur time.Duration) *RingBuffer { +// maxSeries caps distinct tenant|service|metric series (0 = unlimited); +// onSeriesRejected is invoked once per point refused at the cap (nil = no-op). +func NewRingBuffer(slots int, windowDur time.Duration, maxSeries int, onSeriesRejected func()) *RingBuffer { return &RingBuffer{ - rings: make(map[string]*MetricRing), - slots: slots, - windowDur: windowDur, + rings: make(map[string]*MetricRing), + slots: slots, + windowDur: windowDur, + maxSeries: maxSeries, + onSeriesRejected: onSeriesRejected, + } +} + +// ringKey builds the tenant-scoped series key. An empty tenant is coerced +// to storage.DefaultTenantID so writes without an explicit tenant and reads +// from single-tenant callers agree on the same series. +func ringKey(tenantID, metricName, serviceName string) string { + if tenantID == "" { + tenantID = storage.DefaultTenantID } + return tenantID + "|" + serviceName + "|" + metricName } -// Record records a value for the given metric+service at time t. -func (rb *RingBuffer) Record(metricName, serviceName string, value float64, at time.Time) { - key := serviceName + "|" + metricName +// Record records a value for the given tenant+metric+service at time t. +// Returns false when the point was refused because creating a NEW series +// would exceed maxSeries; existing series always keep recording. +func (rb *RingBuffer) Record(tenantID, metricName, serviceName string, value float64, at time.Time) bool { + key := ringKey(tenantID, metricName, serviceName) rb.mu.RLock() ring, ok := rb.rings[key] rb.mu.RUnlock() @@ -175,18 +198,27 @@ func (rb *RingBuffer) Record(metricName, serviceName string, value float64, at t rb.mu.Lock() ring, ok = rb.rings[key] if !ok { + if rb.maxSeries > 0 && len(rb.rings) >= rb.maxSeries { + rb.mu.Unlock() + // Fire telemetry outside the lock — see Aggregator.Ingest + // for why callbacks must not run under mu. + if rb.onSeriesRejected != nil { + rb.onSeriesRejected() + } + return false + } ring = newMetricRing(metricName, serviceName, rb.slots, rb.windowDur) rb.rings[key] = ring } rb.mu.Unlock() } ring.Record(value, at) + return true } -// QueryRecent returns aggregated windows for the given metric+service. -// Pass an empty serviceName to query across all services (returns first match). -func (rb *RingBuffer) QueryRecent(metricName, serviceName string, windowCount int) []WindowAgg { - key := serviceName + "|" + metricName +// QueryRecent returns aggregated windows for the given tenant+metric+service. +func (rb *RingBuffer) QueryRecent(tenantID, metricName, serviceName string, windowCount int) []WindowAgg { + key := ringKey(tenantID, metricName, serviceName) rb.mu.RLock() ring, ok := rb.rings[key] rb.mu.RUnlock() @@ -196,7 +228,7 @@ func (rb *RingBuffer) QueryRecent(metricName, serviceName string, windowCount in return ring.Windows(windowCount) } -// AllKeys returns all registered metric+service keys. +// AllKeys returns all registered tenant|service|metric keys. func (rb *RingBuffer) AllKeys() []string { rb.mu.RLock() defer rb.mu.RUnlock() @@ -207,7 +239,7 @@ func (rb *RingBuffer) AllKeys() []string { return keys } -// MetricCount returns the number of distinct metric series tracked. +// MetricCount returns the number of distinct tenant-scoped metric series tracked. func (rb *RingBuffer) MetricCount() int { rb.mu.RLock() defer rb.mu.RUnlock() diff --git a/internal/tsdb/ringbuffer_test.go b/internal/tsdb/ringbuffer_test.go new file mode 100644 index 0000000..804563c --- /dev/null +++ b/internal/tsdb/ringbuffer_test.go @@ -0,0 +1,174 @@ +package tsdb + +import ( + "strconv" + "testing" + "time" + + "github.com/RandomCodeSpace/otelcontext/internal/storage" +) + +// newTestRing builds an unlimited RingBuffer with a wide window so every +// record in a test lands in the current slot. +func newTestRing() *RingBuffer { + return NewRingBuffer(8, time.Minute, 0, nil) +} + +// TestRingBuffer_Record_ScopedByTenant is the data-isolation guarantee — +// two tenants recording under the SAME service+metric must never see each +// other's points (modeled on internal/storage tenant_test.go naming). +func TestRingBuffer_Record_ScopedByTenant(t *testing.T) { + rb := newTestRing() + now := time.Now() + + rb.Record("tenant-a", "latency_ms", "checkout", 1.0, now) + rb.Record("tenant-a", "latency_ms", "checkout", 2.0, now) + rb.Record("tenant-b", "latency_ms", "checkout", 100.0, now) + + aggsA := rb.QueryRecent("tenant-a", "latency_ms", "checkout", 8) + if len(aggsA) != 1 { + t.Fatalf("tenant-a windows=%d, want 1", len(aggsA)) + } + if aggsA[0].Count != 2 || aggsA[0].Sum != 3.0 { + t.Errorf("tenant-a agg Count=%d Sum=%v, want Count=2 Sum=3 — tenant-b data leaked in?", aggsA[0].Count, aggsA[0].Sum) + } + + aggsB := rb.QueryRecent("tenant-b", "latency_ms", "checkout", 8) + if len(aggsB) != 1 { + t.Fatalf("tenant-b windows=%d, want 1", len(aggsB)) + } + if aggsB[0].Count != 1 || aggsB[0].Sum != 100.0 { + t.Errorf("tenant-b agg Count=%d Sum=%v, want Count=1 Sum=100 — tenant-a data leaked in?", aggsB[0].Count, aggsB[0].Sum) + } + + if got := rb.QueryRecent("tenant-c", "latency_ms", "checkout", 8); got != nil { + t.Errorf("tenant-c QueryRecent=%v, want nil (tenant never recorded)", got) + } +} + +// TestRingBuffer_EmptyTenantCoercedToDefault — a point recorded without a +// tenant must be readable under storage.DefaultTenantID (and vice versa), +// matching the RawMetric.TenantID doc ("empty → the default applies"). +func TestRingBuffer_EmptyTenantCoercedToDefault(t *testing.T) { + rb := newTestRing() + now := time.Now() + + rb.Record("", "latency_ms", "checkout", 7.0, now) + + aggs := rb.QueryRecent(storage.DefaultTenantID, "latency_ms", "checkout", 8) + if len(aggs) != 1 || aggs[0].Sum != 7.0 { + t.Fatalf("QueryRecent(DefaultTenantID)=%v, want the empty-tenant point", aggs) + } + aggs = rb.QueryRecent("", "latency_ms", "checkout", 8) + if len(aggs) != 1 || aggs[0].Sum != 7.0 { + t.Fatalf("QueryRecent(\"\")=%v, want the empty-tenant point", aggs) + } + if got := rb.MetricCount(); got != 1 { + t.Errorf("MetricCount=%d, want 1 (\"\" and DefaultTenantID are the same series)", got) + } +} + +// TestRingBuffer_SeriesCapRejectsNewSeries — past maxSeries, NEW series are +// refused (telemetry fires, Record returns false) while EXISTING series +// keep recording. +func TestRingBuffer_SeriesCapRejectsNewSeries(t *testing.T) { + rejected := 0 + rb := NewRingBuffer(8, time.Minute, 2, func() { rejected++ }) + now := time.Now() + + if !rb.Record("t", "m1", "svc", 1.0, now) { + t.Fatal("first series refused below cap") + } + if !rb.Record("t", "m2", "svc", 1.0, now) { + t.Fatal("second series refused below cap") + } + // Third DISTINCT series is past the cap → refused. + if rb.Record("t", "m3", "svc", 1.0, now) { + t.Error("third series accepted past cap=2") + } + if rejected != 1 { + t.Errorf("rejection callback fired %d times, want 1", rejected) + } + if got := rb.MetricCount(); got != 2 { + t.Errorf("MetricCount=%d, want 2 (cap must hold)", got) + } + + // Existing series must still record at the cap. + if !rb.Record("t", "m1", "svc", 2.0, now) { + t.Fatal("existing series refused at cap — only NEW series may be rejected") + } + aggs := rb.QueryRecent("t", "m1", "svc", 8) + if len(aggs) != 1 || aggs[0].Count != 2 || aggs[0].Sum != 3.0 { + t.Errorf("existing series aggs=%v, want Count=2 Sum=3", aggs) + } + // The refused series holds no data. + if got := rb.QueryRecent("t", "m3", "svc", 8); got != nil { + t.Errorf("refused series QueryRecent=%v, want nil", got) + } + if rejected != 1 { + t.Errorf("rejection callback fired %d times after existing-series record, want still 1", rejected) + } +} + +// TestRingBuffer_SeriesCapZeroIsUnlimited — maxSeries=0 preserves the +// legacy uncapped behavior. +func TestRingBuffer_SeriesCapZeroIsUnlimited(t *testing.T) { + rb := NewRingBuffer(8, time.Minute, 0, func() { t.Error("rejection fired with cap=0") }) + now := time.Now() + + for i := range 50 { + if !rb.Record("t", "metric-"+strconv.Itoa(i), "svc", 1.0, now) { + t.Fatalf("series %d refused with cap=0", i) + } + } + if got := rb.MetricCount(); got != 50 { + t.Errorf("MetricCount=%d, want 50", got) + } +} + +// TestRingBuffer_WindowAdvanceKeepsTenantSeries — recording past the +// window boundary advances slots within the SAME tenant-scoped series +// (no new series is created) and both windows stay queryable. +func TestRingBuffer_WindowAdvanceKeepsTenantSeries(t *testing.T) { + rb := newTestRing() + now := time.Now() + + rb.Record("tenant-a", "latency_ms", "checkout", 1.0, now) + rb.Record("tenant-a", "latency_ms", "checkout", 2.0, now.Add(2*time.Minute)) + + if got := rb.MetricCount(); got != 1 { + t.Fatalf("MetricCount=%d, want 1 — window advance must not mint a new series", got) + } + aggs := rb.QueryRecent("tenant-a", "latency_ms", "checkout", 8) + if len(aggs) != 2 { + t.Fatalf("windows=%d, want 2 (one per time bucket)", len(aggs)) + } + // Most-recent window first. + if aggs[0].Sum != 2.0 || aggs[0].Count != 1 { + t.Errorf("newest window Sum=%v Count=%d, want Sum=2 Count=1", aggs[0].Sum, aggs[0].Count) + } + if aggs[1].Sum != 1.0 || aggs[1].Count != 1 { + t.Errorf("older window Sum=%v Count=%d, want Sum=1 Count=1", aggs[1].Sum, aggs[1].Count) + } + if aggs[0].P50 != 2.0 || aggs[0].P99 != 2.0 { + t.Errorf("newest window P50=%v P99=%v, want 2 for a single-sample window", aggs[0].P50, aggs[0].P99) + } +} + +// TestRingBuffer_MetricCount_CountsTenantScopedSeries — the same +// service+metric under two tenants is TWO series, not one. +func TestRingBuffer_MetricCount_CountsTenantScopedSeries(t *testing.T) { + rb := newTestRing() + now := time.Now() + + rb.Record("tenant-a", "latency_ms", "checkout", 1.0, now) + rb.Record("tenant-b", "latency_ms", "checkout", 1.0, now) + rb.Record("tenant-a", "latency_ms", "checkout", 2.0, now) // same series + + if got := rb.MetricCount(); got != 2 { + t.Errorf("MetricCount=%d, want 2 tenant-scoped series", got) + } + if got := len(rb.AllKeys()); got != 2 { + t.Errorf("AllKeys len=%d, want 2", got) + } +} diff --git a/internal/ui/ui.go b/internal/ui/ui.go index fc5850a..bf34550 100644 --- a/internal/ui/ui.go +++ b/internal/ui/ui.go @@ -1,45 +1,23 @@ package ui import ( + "crypto/sha256" "embed" - "errors" + "encoding/hex" "fmt" "io/fs" + "mime" "net/http" + "path" + "strconv" "strings" "github.com/RandomCodeSpace/otelcontext/internal/graph" + "github.com/RandomCodeSpace/otelcontext/internal/httpconst" "github.com/RandomCodeSpace/otelcontext/internal/storage" "github.com/RandomCodeSpace/otelcontext/internal/telemetry" ) -// spaFS wraps an fs.FS so http.FileServer transparently serves index.html -// for any extensionless path that doesn't resolve to a real file — the -// usual single-page-app routing where the React router owns client-side -// URLs. Asset-shaped paths (anything with a ".") still 404 normally so a -// missing /favicon.ico doesn't surprise the browser with an HTML body. -// -// Wrapping the FS — rather than calling Open() against r.URL.Path in our -// own handler — keeps the user-controlled name behind the stdlib -// http.FileServer boundary, where path.Clean has already happened. -type spaFS struct{ fs.FS } - -func (s spaFS) Open(name string) (fs.File, error) { - f, err := s.FS.Open(name) - if err == nil { - return f, nil - } - if !errors.Is(err, fs.ErrNotExist) { - return nil, err - } - // SPA fallback only for extensionless paths (treated as client-side - // routes); legitimate asset 404s still propagate. - if strings.Contains(name, ".") { - return nil, err - } - return s.FS.Open("index.html") -} - // The built UI is generated at release time (scripts/release.sh) and is not // committed; only internal/ui/dist/.gitkeep is tracked. The `all:` prefix // embeds that dotfile so a source-only checkout still compiles. A plain @@ -82,14 +60,195 @@ func (s *Server) RegisterRoutes(mux *http.ServeMux) error { mux.Handle("/static/", http.FileServer(http.FS(content))) // Serve React SPA from dist/ for all non-API paths. API routes are - // registered before this is called, so they take priority. spaFS - // converts extensionless 404s into index.html so the React router - // can claim them. + // registered before this is called, so they take priority; anything + // that falls through lands on the spaHandler below. distFS, err := fs.Sub(content, "dist") if err != nil { return fmt.Errorf("ui: failed to create dist sub-fs: %w", err) } - mux.Handle("/", http.FileServer(http.FS(spaFS{distFS}))) + mux.Handle("/", newSPAHandler(distFS)) return nil } + +// reservedPrefixes are machine namespaces that must 404 — never receive the +// SPA shell — when no explicit mux route matched. Serving HTML to an OTLP +// exporter, a WebSocket client, or an MCP agent that hit an unknown path +// would only confuse it. +var reservedPrefixes = []string{"/api/", "/v1/", "/ws", "/mcp", "/metrics"} + +// spaHandler serves the embedded Vite build: +// +// - "/" and SPA fallback routes → index.html with Cache-Control: no-cache +// plus a startup-computed ETag so steady-state reloads are 304s. +// - "/assets/*" (Vite hashed filenames) → immutable one-year caching with +// Accept-Encoding negotiation against build-time .br/.gz siblings +// (emitted by ui/scripts/precompress.mjs and picked up by `all:dist`). +// - any other real file → plain http.FileServer. +// - unknown extensionless non-reserved paths → index.html (client-side +// routing). Asset-shaped paths (anything with a ".") still 404 normally +// so a missing /favicon.ico doesn't surprise the browser with HTML — +// same contract as the previous spaFS wrapper. +type spaHandler struct { + dist fs.FS + fileServer http.Handler + + // index.html variants, loaded once at startup. indexPlain == nil means + // dist holds only the source-only stub (.gitkeep) — the dev case where + // the UI was never built. + indexPlain []byte + indexGz []byte + indexBr []byte + indexETag string +} + +func newSPAHandler(dist fs.FS) *spaHandler { + h := &spaHandler{ + dist: dist, + fileServer: http.FileServer(http.FS(dist)), + } + if b, err := fs.ReadFile(dist, "index.html"); err == nil { + h.indexPlain = b + sum := sha256.Sum256(b) + h.indexETag = `"` + hex.EncodeToString(sum[:8]) + `"` + if gz, err := fs.ReadFile(dist, "index.html.gz"); err == nil { + h.indexGz = gz + } + if br, err := fs.ReadFile(dist, "index.html.br"); err == nil { + h.indexBr = br + } + } + return h +} + +func (h *spaHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Rooted clean: forcing a leading "/" before Clean means the result can + // never climb above the root, whatever shape the raw path arrives in. + // (Reads also go through fs.FS, which rejects ".." via fs.ValidPath — + // this is defense in depth, and the form semgrep's + // filepath-clean-misuse rule expects.) + p := path.Clean("/" + strings.Trim(r.URL.Path, "/")) + switch { + case p == "/" || p == "/index.html": + h.serveIndex(w, r) + case strings.HasPrefix(p, "/assets/"): + h.serveAsset(w, r, strings.TrimPrefix(p, "/")) + default: + h.serveOther(w, r, p) + } +} + +// serveOther handles everything that isn't index.html or a hashed asset: +// real embedded files delegate to the stdlib file server (which handles +// Content-Type, ranges, and path traversal), everything else either 404s or +// falls back to the SPA shell. +func (h *spaHandler) serveOther(w http.ResponseWriter, r *http.Request, p string) { + name := strings.TrimPrefix(p, "/") + if name != "" { + if f, err := h.dist.Open(name); err == nil { + info, statErr := f.Stat() + _ = f.Close() + if statErr == nil && !info.IsDir() { + h.fileServer.ServeHTTP(w, r) + return + } + } + } + if strings.Contains(name, ".") || isReserved(p) { + http.NotFound(w, r) + return + } + h.serveIndex(w, r) +} + +func isReserved(p string) bool { + for _, prefix := range reservedPrefixes { + if strings.HasPrefix(p, prefix) { + return true + } + } + return false +} + +// serveIndex serves the SPA shell. no-cache (not no-store) keeps the bytes +// in the browser cache but forces revalidation, so a redeploy is picked up +// on the next navigation while steady-state loads are If-None-Match → 304. +func (h *spaHandler) serveIndex(w http.ResponseWriter, r *http.Request) { + if h.indexPlain == nil { + // Source-only checkout: dist holds just the .gitkeep stub. A plain + // 404 with a pointer beats an empty page or a directory listing. + http.Error(w, "UI not built — release binaries embed the SPA (see scripts/release.sh)", http.StatusNotFound) + return + } + hdr := w.Header() + hdr.Set("Cache-Control", "no-cache") + hdr.Set("ETag", h.indexETag) + hdr.Set("Vary", "Accept-Encoding") + if httpconst.ETagMatch(r.Header.Get("If-None-Match"), h.indexETag) { + w.WriteHeader(http.StatusNotModified) + return + } + body := h.indexPlain + switch { + case h.indexBr != nil && httpconst.AcceptsEncoding(r, "br"): + hdr.Set(httpconst.HeaderContentEncoding, "br") + body = h.indexBr + case h.indexGz != nil && httpconst.AcceptsEncoding(r, "gzip"): + hdr.Set(httpconst.HeaderContentEncoding, "gzip") + body = h.indexGz + } + writeBody(w, r, body, "text/html; charset=utf-8") +} + +// serveAsset serves a Vite-emitted hashed asset (/assets/-.). +// The hash makes the content immutable, so the response is cacheable for a +// year; build-time precompressed .br/.gz siblings are negotiated via +// Accept-Encoding with Content-Type taken from the ORIGINAL extension. +func (h *spaHandler) serveAsset(w http.ResponseWriter, r *http.Request, name string) { + body, encoding, ok := h.assetVariant(r, name) + if !ok { + http.NotFound(w, r) + return + } + hdr := w.Header() + hdr.Set("Cache-Control", "public, max-age=31536000, immutable") + hdr.Set("Vary", "Accept-Encoding") + if encoding != "" { + hdr.Set(httpconst.HeaderContentEncoding, encoding) + } + ctype := mime.TypeByExtension(path.Ext(name)) + if ctype == "" { + ctype = "application/octet-stream" + } + writeBody(w, r, body, ctype) +} + +// assetVariant picks the best representation the client accepts: brotli +// sibling first (smallest), then gzip, then the original bytes. +func (h *spaHandler) assetVariant(r *http.Request, name string) (body []byte, encoding string, ok bool) { + if httpconst.AcceptsEncoding(r, "br") { + if b, err := fs.ReadFile(h.dist, name+".br"); err == nil { + return b, "br", true + } + } + if httpconst.AcceptsEncoding(r, "gzip") { + if b, err := fs.ReadFile(h.dist, name+".gz"); err == nil { + return b, "gzip", true + } + } + b, err := fs.ReadFile(h.dist, name) + if err != nil { + return nil, "", false + } + return b, "", true +} + +func writeBody(w http.ResponseWriter, r *http.Request, body []byte, ctype string) { + hdr := w.Header() + hdr.Set(httpconst.HeaderContentType, ctype) + hdr.Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + _, _ = w.Write(body) //nolint:gosec // G705 false positive: body is read-only embedded build output (fs.ValidPath rejects traversal), never request data + } +} diff --git a/internal/ui/ui_test.go b/internal/ui/ui_test.go new file mode 100644 index 0000000..0e7c77b --- /dev/null +++ b/internal/ui/ui_test.go @@ -0,0 +1,300 @@ +package ui + +import ( + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/fstest" +) + +// builtDistFS returns a fake Vite build output: index.html with both +// precompressed siblings, a hashed JS asset with both siblings, a hashed +// CSS asset with no siblings, and a root-level static file. +func builtDistFS() fstest.MapFS { + return fstest.MapFS{ + "index.html": {Data: []byte("index")}, + "index.html.br": {Data: []byte("br-index")}, + "index.html.gz": {Data: []byte("gz-index")}, + "assets/app-abc123.js": {Data: []byte("console.log('app')")}, + "assets/app-abc123.js.br": {Data: []byte("br-js")}, + "assets/app-abc123.js.gz": {Data: []byte("gz-js")}, + "assets/app-abc123.css": {Data: []byte("body{}")}, + "vite.svg": {Data: []byte("")}, + } +} + +func get(t *testing.T, h http.Handler, path string, hdr map[string]string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodGet, path, nil) + for k, v := range hdr { + req.Header.Set(k, v) + } + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec +} + +func TestServeAsset_BrotliNegotiation(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/assets/app-abc123.js", map[string]string{"Accept-Encoding": "gzip, br"}) + + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d", rec.Code) + } + if got := rec.Header().Get("Content-Encoding"); got != "br" { + t.Errorf("Content-Encoding = %q, want br", got) + } + if got := rec.Header().Get("Cache-Control"); got != "public, max-age=31536000, immutable" { + t.Errorf("Cache-Control = %q", got) + } + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Vary = %q", got) + } + // Content-Type must come from the ORIGINAL .js extension, not .br. + if ct := rec.Header().Get("Content-Type"); !strings.Contains(ct, "javascript") { + t.Errorf("Content-Type = %q, want javascript", ct) + } + if rec.Body.String() != "br-js" { + t.Errorf("body = %q, want br sibling bytes", rec.Body.String()) + } +} + +func TestServeAsset_GzipFallback(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/assets/app-abc123.js", map[string]string{"Accept-Encoding": "gzip"}) + + if got := rec.Header().Get("Content-Encoding"); got != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", got) + } + if rec.Body.String() != "gz-js" { + t.Errorf("body = %q, want gz sibling bytes", rec.Body.String()) + } +} + +func TestServeAsset_BrotliQZeroFallsBackToGzip(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/assets/app-abc123.js", map[string]string{"Accept-Encoding": "br;q=0, gzip"}) + + if got := rec.Header().Get("Content-Encoding"); got != "gzip" { + t.Errorf("Content-Encoding = %q, want gzip", got) + } +} + +func TestServeAsset_IdentityWhenNoAcceptEncoding(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/assets/app-abc123.js", nil) + + if got := rec.Header().Get("Content-Encoding"); got != "" { + t.Errorf("Content-Encoding = %q, want empty", got) + } + if rec.Body.String() != "console.log('app')" { + t.Errorf("body = %q, want original bytes", rec.Body.String()) + } + if got := rec.Header().Get("Cache-Control"); !strings.Contains(got, "immutable") { + t.Errorf("Cache-Control = %q, want immutable", got) + } +} + +func TestServeAsset_NoSiblingServesIdentity(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/assets/app-abc123.css", map[string]string{"Accept-Encoding": "gzip, br"}) + + if got := rec.Header().Get("Content-Encoding"); got != "" { + t.Errorf("Content-Encoding = %q, want empty (no siblings exist)", got) + } + if rec.Body.String() != "body{}" { + t.Errorf("body = %q", rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); !strings.Contains(ct, "text/css") { + t.Errorf("Content-Type = %q, want text/css", ct) + } +} + +func TestServeAsset_Missing404(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/assets/nope-zzz.js", nil) + if rec.Code != http.StatusNotFound { + t.Fatalf("want 404, got %d", rec.Code) + } +} + +func TestIndex_ETagAnd304(t *testing.T) { + h := newSPAHandler(builtDistFS()) + + rec := get(t, h, "/", nil) + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d", rec.Code) + } + if got := rec.Header().Get("Cache-Control"); got != "no-cache" { + t.Errorf("Cache-Control = %q, want no-cache", got) + } + etag := rec.Header().Get("ETag") + if etag == "" || !strings.HasPrefix(etag, `"`) { + t.Fatalf("ETag = %q, want quoted non-empty", etag) + } + if rec.Body.String() != "index" { + t.Errorf("body = %q", rec.Body.String()) + } + + rec2 := get(t, h, "/", map[string]string{"If-None-Match": etag}) + if rec2.Code != http.StatusNotModified { + t.Fatalf("want 304, got %d", rec2.Code) + } + if rec2.Body.Len() != 0 { + t.Errorf("304 body should be empty, got %q", rec2.Body.String()) + } + if got := rec2.Header().Get("ETag"); got != etag { + t.Errorf("304 ETag = %q, want %q", got, etag) + } +} + +func TestIndex_PrecompressedVariant(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/", map[string]string{"Accept-Encoding": "br"}) + + if got := rec.Header().Get("Content-Encoding"); got != "br" { + t.Errorf("Content-Encoding = %q, want br", got) + } + if rec.Body.String() != "br-index" { + t.Errorf("body = %q, want br variant", rec.Body.String()) + } + if ct := rec.Header().Get("Content-Type"); !strings.Contains(ct, "text/html") { + t.Errorf("Content-Type = %q, want text/html", ct) + } + // ETag must be identical across encodings — it identifies the document, + // and Vary: Accept-Encoding disambiguates the representation. + identity := get(t, h, "/", nil) + if rec.Header().Get("ETag") != identity.Header().Get("ETag") { + t.Errorf("ETag differs between encodings") + } + if got := rec.Header().Get("Vary"); got != "Accept-Encoding" { + t.Errorf("Vary = %q", got) + } +} + +func TestSPAFallback_ServesIndexForClientRoutes(t *testing.T) { + h := newSPAHandler(builtDistFS()) + for _, path := range []string{"/traces", "/logs", "/traces/deep/route"} { + rec := get(t, h, path, nil) + if rec.Code != http.StatusOK { + t.Errorf("%s: want 200, got %d", path, rec.Code) + continue + } + if rec.Body.String() != "index" { + t.Errorf("%s: body = %q, want index", path, rec.Body.String()) + } + // Fallback responses are the mutable SPA shell — same caching rules + // as "/" so a deploy is picked up on the next poll. + if got := rec.Header().Get("Cache-Control"); got != "no-cache" { + t.Errorf("%s: Cache-Control = %q, want no-cache", path, got) + } + } +} + +func TestSPAFallback_DoesNotClaimReservedOrAssetPaths(t *testing.T) { + h := newSPAHandler(builtDistFS()) + for _, path := range []string{ + "/api/x", // unknown API route must 404, not get HTML + "/v1/unknown", // OTLP namespace + "/ws/nope", // WebSocket namespace + "/mcp/extra", // MCP namespace + "/metrics/unknown", // Prometheus namespace + "/missing.png", // asset-shaped: extension means real 404 + "/release/v1.2/doc", // dot anywhere in the path: preserved legacy spaFS behavior + } { + rec := get(t, h, path, nil) + if rec.Code != http.StatusNotFound { + t.Errorf("%s: want 404, got %d (body %q)", path, rec.Code, rec.Body.String()) + } + } +} + +func TestRealFile_DelegatesToFileServer(t *testing.T) { + h := newSPAHandler(builtDistFS()) + rec := get(t, h, "/vite.svg", nil) + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d", rec.Code) + } + if rec.Body.String() != "" { + t.Errorf("body = %q", rec.Body.String()) + } +} + +func TestHeadRequest_NoBody(t *testing.T) { + h := newSPAHandler(builtDistFS()) + req := httptest.NewRequest(http.MethodHead, "/", nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("want 200, got %d", rec.Code) + } + if rec.Body.Len() != 0 { + t.Errorf("HEAD body should be empty, got %q", rec.Body.String()) + } + if rec.Header().Get("ETag") == "" { + t.Errorf("HEAD response missing ETag") + } +} + +// TestStubDist covers the source-only checkout where dist holds just the +// .gitkeep placeholder: the handler must degrade to a plain 404 instead of +// panicking or serving a directory listing. +func TestStubDist_Graceful404(t *testing.T) { + h := newSPAHandler(fstest.MapFS{".gitkeep": {Data: nil}}) + for _, path := range []string{"/", "/traces"} { + rec := get(t, h, path, nil) + if rec.Code != http.StatusNotFound { + t.Errorf("%s: want 404 in stub mode, got %d", path, rec.Code) + } + body, _ := io.ReadAll(rec.Body) + if !strings.Contains(string(body), "UI not built") { + t.Errorf("%s: stub body should explain the missing build, got %q", path, body) + } + } +} + +// TestRegisterRoutes_RealEmbed exercises the production wiring against the +// actual embedded FS (a stub dist in a source-only checkout). +func TestRegisterRoutes_RealEmbed(t *testing.T) { + s := NewServer(nil, nil, nil) + s.SetMCPConfig(true, "/mcp") + mux := http.NewServeMux() + if err := s.RegisterRoutes(mux); err != nil { + t.Fatalf("RegisterRoutes: %v", err) + } + // /static/style.css is committed and embedded — must always serve. + rec := get(t, mux, "/static/style.css", nil) + if rec.Code != http.StatusOK { + t.Errorf("/static/style.css: want 200, got %d", rec.Code) + } +} + +// Regression for semgrep filepath-clean-misuse: request paths are +// rooted-cleaned before any lookup, so traversal shapes can never escape +// the dist tree (fs.ValidPath is the second layer of the same defense). +func TestServeHTTP_PathTraversalRootedClean(t *testing.T) { + h := newSPAHandler(builtDistFS()) + + // Climbing above the root collapses to a rooted path: asset-shaped + // targets 404, extensionless ones get the SPA shell — never a file read. + rec := get(t, h, "/../../etc/secret.txt", nil) + if rec.Code != http.StatusNotFound { + t.Fatalf("traversal path: got %d, want 404", rec.Code) + } + rec = get(t, h, "/../../etc/passwd", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "index") { + t.Fatalf("traversal fallback: got %d %q, want SPA shell", rec.Code, rec.Body.String()) + } + + // ".." inside /assets collapses to the real target, which is served by + // its own handler (index: no-cache), never via the immutable asset path. + rec = get(t, h, "/assets/../index.html", nil) + if rec.Code != http.StatusOK || !strings.Contains(rec.Body.String(), "index") { + t.Fatalf("collapsed path: got %d %q, want index", rec.Code, rec.Body.String()) + } + if cc := rec.Header().Get("Cache-Control"); cc != "no-cache" { + t.Fatalf("collapsed path served with asset caching: Cache-Control=%q", cc) + } +} diff --git a/main.go b/main.go index bd1747c..d7bcbb6 100644 --- a/main.go +++ b/main.go @@ -177,6 +177,17 @@ func main() { slog.Info("🚀 Starting OtelContext", "version", Version, "env", cfg.Env, "log_level", level) + // Pace the GC against a soft memory ceiling so RSS stays bounded under + // sustained ingest (honors an explicit GOMEMLIMIT; otherwise 75% of the + // detected cgroup/host budget). See applyMemoryLimit in memlimit.go. + applyMemoryLimit(75) + + // Profiling is an observability aid — a busy port must not abort startup. + pprofSrv, _, err := startPprofServer(cfg.PprofAddr, logger) + if err != nil { + slog.Warn("pprof server disabled", "error", err, "addr", cfg.PprofAddr) + } + // 1. Initialize Internal Telemetry (first — everything registers metrics against this) metrics := telemetry.New() slog.Info("📊 Internal telemetry initialized") @@ -201,7 +212,8 @@ func main() { } slog.Info("💾 Storage initialized", "driver", cfg.DBDriver) - // 2a. Retention scheduler: hourly batched purge + daily VACUUM/ANALYZE. + // 2a. Retention scheduler: hourly batched purge + daily maintenance + // (VACUUM ANALYZE / OPTIMIZE / PRAGMA optimize + incremental_vacuum). ctxRetention, cancelRetention := context.WithCancel(context.Background()) retention := storage.NewRetentionScheduler( repo, @@ -209,6 +221,7 @@ func main() { cfg.RetentionBatchSize, time.Duration(cfg.RetentionBatchSleepMs)*time.Millisecond, ) + retention.SetFullVacuum(cfg.RetentionFullVacuum) retention.Start(ctxRetention) slog.Info("🧹 Retention scheduler started", "retention_days", cfg.HotRetentionDays) @@ -345,7 +358,7 @@ func main() { func() { metrics.TSDBIngestTotal.Inc() }, func() { metrics.TSDBBatchesDropped.Inc() }, ) - ringBuf := tsdb.NewRingBuffer(120, 30*time.Second) + ringBuf := tsdb.NewRingBuffer(120, 30*time.Second, cfg.MetricMaxCardinality, metrics.TSDBRingSeriesRejected.Inc) tsdbAgg.SetRingBuffer(ringBuf) slog.Info("📈 TSDB ring buffer attached (120 slots × 30s = 1h retention)") @@ -382,6 +395,15 @@ func main() { graphRAGCfg := graphrag.DefaultConfig() graphRAGCfg.WorkerCount = cfg.GraphRAGWorkerCount graphRAGCfg.ChannelSize = cfg.GraphRAGEventQueueSize + graphRAGCfg.MaxSpansPerTenant = cfg.GraphRAGMaxSpansPerTenant + // Duration knobs follow the DLQ_REPLAY_INTERVAL pattern: unparsable + // values fall back to the package default rather than aborting startup. + if ttl, err := time.ParseDuration(cfg.GraphRAGTraceTTL); err == nil && ttl > 0 { + graphRAGCfg.TraceTTL = ttl + } + if idle, err := time.ParseDuration(cfg.GraphRAGTenantIdleTTL); err == nil && idle > 0 { + graphRAGCfg.TenantIdleTTL = idle + } graphRAG := graphrag.New(repo, tsdbAgg, ringBuf, graphRAGCfg) graphRAG.SetMetrics(metrics) ctxGraphRAG, cancelGraphRAG := context.WithCancel(context.Background()) @@ -389,6 +411,9 @@ func main() { slog.Info("GraphRAG started (layered graph with anomaly detection)", "workers", cfg.GraphRAGWorkerCount, "event_queue_size", cfg.GraphRAGEventQueueSize, + "trace_ttl", graphRAGCfg.TraceTTL, + "max_spans_per_tenant", graphRAGCfg.MaxSpansPerTenant, + "tenant_idle_ttl", graphRAGCfg.TenantIdleTTL, ) // Auto-migrate GraphRAG models (Investigation, DrainTemplateRow) @@ -450,6 +475,7 @@ func main() { ingestPipeline = ingest.NewPipeline(repo, metrics, ingest.PipelineConfig{ Capacity: cfg.IngestPipelineQueueSize, Workers: cfg.IngestPipelineWorkers, + MaxBytes: int64(cfg.IngestPipelineMaxBytes), }) ingestPipeline.SetPerTenantCap(cfg.IngestPipelinePerTenantCap) @@ -707,6 +733,12 @@ func main() { var httpHandler http.Handler = mux + // Gzip GET /api/* responses (innermost wrapper — only handler output is + // compressed; error responses written by outer middleware like auth and + // rate limiting stay identity-encoded, and /ws*, /v1/*, /metrics*, and + // the MCP/SSE path pass through untouched). + httpHandler = api.GzipMiddleware(cfg.MCPPath)(httpHandler) + // Resolve tenant on /api/* read-side requests (passes through OTLP /v1, // MCP, UI assets, and health probes untouched). httpHandler = api.TenantMiddleware(cfg)(httpHandler) @@ -786,12 +818,34 @@ func main() { defer bootWG.Done() tick := time.NewTicker(1 * time.Second) defer tick.Stop() + // Store census is len()-under-RLock per tenant — cheap, but 15s is + // plenty for trend attribution of RSS growth. + census := time.NewTicker(15 * time.Second) + defer census.Stop() for { select { case <-appCtx.Done(): return case <-tick.C: metrics.GraphRAGEventBufferDepth.Set(float64(graphRAG.EventBufferDepth())) + case <-census.C: + c := graphRAG.StoreCounts() + ent := metrics.GraphRAGStoreEntities + ent.WithLabelValues("tenants").Set(float64(c.Tenants)) + ent.WithLabelValues("services").Set(float64(c.Services)) + ent.WithLabelValues("operations").Set(float64(c.Operations)) + ent.WithLabelValues("traces").Set(float64(c.Traces)) + ent.WithLabelValues("spans").Set(float64(c.Spans)) + ent.WithLabelValues("log_clusters").Set(float64(c.LogClusters)) + ent.WithLabelValues("metrics").Set(float64(c.Metrics)) + ent.WithLabelValues("anomalies").Set(float64(c.Anomalies)) + edg := metrics.GraphRAGStoreEdges + edg.WithLabelValues("service").Set(float64(c.ServiceEdges)) + edg.WithLabelValues("trace").Set(float64(c.TraceEdges)) + edg.WithLabelValues("signal").Set(float64(c.SignalEdges)) + edg.WithLabelValues("anomaly").Set(float64(c.AnomalyEdges)) + metrics.TSDBRingSeriesActive.Set(float64(ringBuf.MetricCount())) + metrics.DrainTemplatesActive.Set(float64(graphRAG.DrainTemplateCount())) } } }() @@ -866,6 +920,9 @@ func main() { if err := srv.Shutdown(ctx); err != nil { slog.Error("HTTP server forced shutdown", "error", err) } + if pprofSrv != nil { + _ = pprofSrv.Close() + } // 2. Stop real-time hubs and event processing hub.Stop() diff --git a/memlimit.go b/memlimit.go new file mode 100644 index 0000000..e8bf87c --- /dev/null +++ b/memlimit.go @@ -0,0 +1,39 @@ +package main + +import ( + "log/slog" + "os" + "runtime/debug" + + "github.com/RandomCodeSpace/otelcontext/internal/membudget" +) + +// applyMemoryLimit sets a soft heap ceiling (GOMEMLIMIT) so the garbage collector +// paces against a bound instead of letting the next-GC target run away on a large +// heap. Without it, a deployment under sustained ingest lets RSS climb to the +// (uncapped) GC target and never returns it to the OS — observed in the SQLite +// soak as ~1.8 GB RSS held flat after load stopped. A soft limit makes the GC run +// more frequently as the heap approaches the ceiling, keeping the working set and +// RSS bounded without the hard-OOM behavior of a rlimit. +// +// If the operator already set the GOMEMLIMIT env var, the Go runtime has applied +// it at startup and we leave it untouched. Otherwise the budget is detected via +// internal/membudget (cgroup v2 → cgroup v1 → /proc/meminfo) and the limit is +// set to pct percent of that budget. Any detection failure leaves the runtime +// default (no limit) in place — never worse than today. +func applyMemoryLimit(pct int) { + if _, ok := os.LookupEnv("GOMEMLIMIT"); ok { + slog.Info("🧠 GOMEMLIMIT set by operator; honoring it", + "soft_limit_bytes", debug.SetMemoryLimit(-1)) + return + } + budget, source := membudget.Detect() + if budget <= 0 { + slog.Warn("🧠 could not detect memory budget; GOMEMLIMIT left unset (GC target unbounded)") + return + } + limit := budget / 100 * int64(pct) + debug.SetMemoryLimit(limit) + slog.Info("🧠 soft memory limit applied (GOMEMLIMIT)", + "limit_mib", limit/(1<<20), "budget_mib", budget/(1<<20), "pct", pct, "source", source) +} diff --git a/memlimit_test.go b/memlimit_test.go new file mode 100644 index 0000000..d0e94b4 --- /dev/null +++ b/memlimit_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "os" + "runtime/debug" + "testing" + + "github.com/RandomCodeSpace/otelcontext/internal/membudget" +) + +// Detection internals (cgroup v2/v1, /proc/meminfo parsing) are tested in +// internal/membudget — these tests cover only the thin wrapper behavior. + +func TestApplyMemoryLimit_HonorsOperatorGOMEMLIMIT(t *testing.T) { + prev := debug.SetMemoryLimit(-1) + defer debug.SetMemoryLimit(prev) + + t.Setenv("GOMEMLIMIT", "1GiB") + applyMemoryLimit(75) + if got := debug.SetMemoryLimit(-1); got != prev { + t.Fatalf("applyMemoryLimit must not touch the limit when GOMEMLIMIT is set; got %d want %d", got, prev) + } +} + +func TestApplyMemoryLimit_SetsBudgetFraction(t *testing.T) { + prev := debug.SetMemoryLimit(-1) + defer debug.SetMemoryLimit(prev) + + // Force the detection path: clear GOMEMLIMIT for this test only. + // t.Setenv registers the restore; Unsetenv makes LookupEnv miss. + t.Setenv("GOMEMLIMIT", "") + if err := os.Unsetenv("GOMEMLIMIT"); err != nil { + t.Fatalf("unset GOMEMLIMIT: %v", err) + } + + budget, _ := membudget.Detect() + if budget <= 0 { + t.Skip("no detectable memory budget on this host") + } + applyMemoryLimit(75) + want := budget / 100 * 75 + if got := debug.SetMemoryLimit(-1); got != want { + t.Fatalf("applyMemoryLimit(75) set %d, want %d (budget %d)", got, want, budget) + } +} diff --git a/pprof.go b/pprof.go new file mode 100644 index 0000000..35674f9 --- /dev/null +++ b/pprof.go @@ -0,0 +1,35 @@ +package main + +import ( + "errors" + "log/slog" + "net" + "net/http" + _ "net/http/pprof" //nolint:gosec // G108: served only via a dedicated loopback listener (PPROF_ADDR), never the public :8080 mux + "time" +) + +// startPprofServer serves net/http/pprof on its own listener so profiling +// never leaks onto the public :8080 mux (which is a fresh NewServeMux). +// Empty addr disables it. Bind to loopback in production; heap profiles +// expose internals. +func startPprofServer(addr string, log *slog.Logger) (*http.Server, net.Addr, error) { + if addr == "" { + return nil, nil, nil + } + ln, err := net.Listen("tcp", addr) + if err != nil { + return nil, nil, err + } + srv := &http.Server{ + Handler: http.DefaultServeMux, + ReadHeaderTimeout: 5 * time.Second, + } + go func() { + if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Warn("pprof server exited", "error", err) + } + }() + log.Info("pprof profiling enabled", "addr", ln.Addr().String()) + return srv, ln.Addr(), nil +} diff --git a/pprof_test.go b/pprof_test.go new file mode 100644 index 0000000..9e9ab2f --- /dev/null +++ b/pprof_test.go @@ -0,0 +1,47 @@ +package main + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "testing" + "time" +) + +func TestStartPprofServerDisabledOnEmptyAddr(t *testing.T) { + srv, addr, err := startPprofServer("", slog.Default()) + if err != nil || srv != nil || addr != nil { + t.Fatalf("empty addr must disable pprof: srv=%v addr=%v err=%v", srv, addr, err) + } +} + +func TestStartPprofServerServesHeapProfile(t *testing.T) { + srv, addr, err := startPprofServer("127.0.0.1:0", slog.Default()) + if err != nil { + t.Fatalf("start: %v", err) + } + defer srv.Close() + + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(fmt.Sprintf("http://%s/debug/pprof/heap?debug=1", addr)) + if err != nil { + t.Fatalf("get heap profile: %v", err) + } + defer resp.Body.Close() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status=%d body=%.200s", resp.StatusCode, body) + } + if len(body) == 0 { + t.Fatal("empty heap profile body") + } +} + +func TestStartPprofServerInvalidAddr(t *testing.T) { + srv, _, err := startPprofServer("definitely-not-an-addr", slog.Default()) + if err == nil { + srv.Close() + t.Fatal("expected listen error for invalid addr") + } +} diff --git a/ui/budgets.json b/ui/budgets.json new file mode 100644 index 0000000..4e69e0b --- /dev/null +++ b/ui/budgets.json @@ -0,0 +1,10 @@ +{ + "$comment": "Gzipped size budgets enforced by scripts/check-budgets.mjs (npm run check-budgets). 'initial' = assets referenced from dist/index.html (entry script + modulepreloads + stylesheets). Fonts are raw woff2 bytes (already compressed). C7 measured actuals after the design-system exit: initial JS 107.6 KB, initial CSS 2.7 KB, fonts 67.8 KB, largest lazy chunk 7.9 KB - budgets are actual + ~10% headroom. lazyChunkExceptions maps a chunk-name prefix (before the content hash) to a per-chunk cap; currently empty because every lazy chunk fits the default.", + "gzipKb": { + "initialJs": 118, + "initialCss": 6, + "fonts": 75, + "lazyChunkDefault": 10, + "lazyChunkExceptions": {} + } +} diff --git a/ui/eslint.config.js b/ui/eslint.config.js index c8eb9fd..9a4a797 100644 --- a/ui/eslint.config.js +++ b/ui/eslint.config.js @@ -8,6 +8,13 @@ export default tseslint.config( { ignores: ['dist', 'node_modules'] }, js.configs.recommended, ...tseslint.configs.recommended, + { + // Build/CI scripts run under Node, not the browser. + files: ['scripts/**/*.mjs'], + languageOptions: { + globals: globals.node, + }, + }, { files: ['**/*.{ts,tsx}'], languageOptions: { diff --git a/ui/index.html b/ui/index.html index 73f8024..afef7b2 100644 --- a/ui/index.html +++ b/ui/index.html @@ -2,7 +2,24 @@ - + + + + + + OtelContext diff --git a/ui/package-lock.json b/ui/package-lock.json index 23f7b0d..adb1289 100644 --- a/ui/package-lock.json +++ b/ui/package-lock.json @@ -8,13 +8,17 @@ "name": "otelcontext-ui", "version": "0.0.0", "dependencies": { - "@ossrandom/design-system": "^0.3.0", - "clsx": "^2.1.1", - "cytoscape": "^3.33.2", - "cytoscape-cose-bilkent": "^4.1.0", + "@radix-ui/react-dialog": "^1.1.16", + "@radix-ui/react-dropdown-menu": "^2.1.17", + "@radix-ui/react-tabs": "^1.1.14", + "@radix-ui/react-tooltip": "^1.2.9", + "@tanstack/react-query": "^5.101.0", + "@tanstack/react-virtual": "^3.14.2", + "cmdk": "^1.1.1", "lucide-react": "^0.469.0", "react": "^19.2.5", - "react-dom": "^19.2.5" + "react-dom": "^19.2.5", + "wouter": "^3.10.0" }, "devDependencies": { "@eslint/js": "^10.0.1", @@ -25,6 +29,7 @@ "@types/react": "^19.0.10", "@types/react-dom": "^19.0.4", "@vitejs/plugin-react": "^6.0.1", + "@vitest/coverage-v8": "^4.1.8", "eslint": "^10.2.1", "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.5.2", @@ -227,9 +232,9 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.27.1", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.27.1.tgz", - "integrity": "sha512-qMlSxKbpRlAridDExk92nSobyDdpPijUq2DW6oDnUqd0iOGxmQjyqhMIihI9+zv4LPyZdRje2cavWPbCbWm3eA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", + "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", "dev": true, "license": "MIT", "engines": { @@ -237,9 +242,9 @@ } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.28.5", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.28.5.tgz", - "integrity": "sha512-qSs4ifwzKJSV39ucNjsvc6WVHs6b7S03sOh2OcHF9UHfVPqWWALUsNUVzhSBiItjRZoLHx7nIarVjqKVusUZ1Q==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", "dev": true, "license": "MIT", "engines": { @@ -271,13 +276,13 @@ } }, "node_modules/@babel/parser": { - "version": "7.29.2", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.2.tgz", - "integrity": "sha512-4GgRzy/+fsBa72/RZVJmGKPmZu9Byn8o4MoLpmNe1m8ZfYnz5emHLQz3U4gLud6Zwl0RZIcgiLD7Uq7ySFuDLA==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.7.tgz", + "integrity": "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg==", "dev": true, "license": "MIT", "dependencies": { - "@babel/types": "^7.29.0" + "@babel/types": "^7.29.7" }, "bin": { "parser": "bin/babel-parser.js" @@ -331,19 +336,29 @@ } }, "node_modules/@babel/types": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.0.tgz", - "integrity": "sha512-LwdZHpScM4Qz8Xw2iKSzS+cfglZzJGvofQICy7W7v4caru4EaAmyUuO6BGrbyQ2mYV11W0U8j5mBhd14dd3B0A==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.7.tgz", + "integrity": "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-string-parser": "^7.27.1", - "@babel/helper-validator-identifier": "^7.28.5" + "@babel/helper-string-parser": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7" }, "engines": { "node": ">=6.9.0" } }, + "node_modules/@bcoe/v8-coverage": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@bcoe/v8-coverage/-/v8-coverage-1.0.2.tgz", + "integrity": "sha512-6zABk/ECA/QYSCQ1NGiVwwbQerUCZ+TQbp64Q3AgmfNvurHH0j8TtXa1qbShXA6qqkpAj4V5W8pP6mLe1mcMqA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/@bramus/specificity": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@bramus/specificity/-/specificity-2.4.2.tgz", @@ -677,6 +692,44 @@ } } }, + "node_modules/@floating-ui/core": { + "version": "1.7.5", + "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.7.5.tgz", + "integrity": "sha512-1Ih4WTWyw0+lKyFMcBHGbb5U5FtuHJuujoyyr5zTaWS5EYMeT6Jb2AuDeftsCsEuchO+mM2ij5+q9crhydzLhQ==", + "license": "MIT", + "dependencies": { + "@floating-ui/utils": "^0.2.11" + } + }, + "node_modules/@floating-ui/dom": { + "version": "1.7.6", + "resolved": "https://registry.npmjs.org/@floating-ui/dom/-/dom-1.7.6.tgz", + "integrity": "sha512-9gZSAI5XM36880PPMm//9dfiEngYoC6Am2izES1FF406YFsjvyBMmeJ2g4SAju3xWwtuynNRFL2s9hgxpLI5SQ==", + "license": "MIT", + "dependencies": { + "@floating-ui/core": "^1.7.5", + "@floating-ui/utils": "^0.2.11" + } + }, + "node_modules/@floating-ui/react-dom": { + "version": "2.1.8", + "resolved": "https://registry.npmjs.org/@floating-ui/react-dom/-/react-dom-2.1.8.tgz", + "integrity": "sha512-cC52bHwM/n/CxS87FH0yWdngEZrjdtLW/qVruo68qg+prK7ZQ4YGdut2GyDVpoGeAYe/h899rVeOVm6Oi40k2A==", + "license": "MIT", + "dependencies": { + "@floating-ui/dom": "^1.7.6" + }, + "peerDependencies": { + "react": ">=16.8.0", + "react-dom": ">=16.8.0" + } + }, + "node_modules/@floating-ui/utils": { + "version": "0.2.11", + "resolved": "https://registry.npmjs.org/@floating-ui/utils/-/utils-0.2.11.tgz", + "integrity": "sha512-RiB/yIh78pcIxl6lLMG0CgBXAZ2Y0eVHqMPYugu+9U0AeT6YBeiJpf7lbdJNIugFP5SIjwNRgo4DhR1Qxi26Gg==", + "license": "MIT" + }, "node_modules/@humanfs/core": { "version": "0.19.1", "resolved": "https://registry.npmjs.org/@humanfs/core/-/core-0.19.1.tgz", @@ -798,59 +851,671 @@ "@emnapi/runtime": "^1.7.1" } }, - "node_modules/@ossrandom/design-system": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/@ossrandom/design-system/-/design-system-0.3.0.tgz", - "integrity": "sha512-flW4PBob1WCjyero4HA8/gYHbQe+ufy2XpQ5EpjO988LdNaM/oArj4gONQSdXdGGlQ4zoXzuRTgPjDDBB61o2A==", + "node_modules/@oxc-project/types": { + "version": "0.127.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.127.0.tgz", + "integrity": "sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==", + "dev": true, "license": "MIT", - "engines": { - "node": ">=18.18" + "funding": { + "url": "https://github.com/sponsors/Boshen" + } + }, + "node_modules/@radix-ui/primitive": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/primitive/-/primitive-1.1.4.tgz", + "integrity": "sha512-7AdCK9PQyiljKoBDbN8OuctCbd/esdwZPQ8RtOE3SsyQtUpiPb+ND75q0jEhC1m1ecBI0MFNeLJvwIh9iKHRcQ==", + "license": "MIT" + }, + "node_modules/@radix-ui/react-arrow": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@radix-ui/react-arrow/-/react-arrow-1.1.9.tgz", + "integrity": "sha512-yqHW5WQ/cTpU/un7dqqIKNy2iRU8BC0JB78PEzTfCCYvZu1U6W9KwObAniMk9nhSfyotKPQTYaUD/HB0f5muig==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-collection": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@radix-ui/react-collection/-/react-collection-1.1.9.tgz", + "integrity": "sha512-zuSVi7ziP7uQRqc+yGxsKJfNkdyHv3ZKDaHe0gzg4dRgws96TPKWIiz84tVHP4GEcEl8bC0mdt17NkcxaJHmaQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-slot": "1.2.5" }, "peerDependencies": { - "@deck.gl/core": "^9.0.0", - "@deck.gl/layers": "^9.0.0", - "cytoscape": "^3.30.0", - "cytoscape-cose-bilkent": "^4.1.0", - "d3-force": "^3.0.0", - "d3-hierarchy": "^3.0.0", - "react": ">=18", - "react-dom": ">=18", - "uplot": "^1.6.0" + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" }, "peerDependenciesMeta": { - "@deck.gl/core": { + "@types/react": { "optional": true }, - "@deck.gl/layers": { + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-compose-refs": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-compose-refs/-/react-compose-refs-1.1.3.tgz", + "integrity": "sha512-rYOP8OMnuuPMQF1uhPVlGNcCDlkokKqGFE3JcxFViIkAXP7EvFWUliJAstrapypaBLJNHbZL6jGhbVDGTwmVhA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-context": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-context/-/react-context-1.1.4.tgz", + "integrity": "sha512-QwH4PO5urrbO+FaGd5Aglg+YJgWTyyuZ3g/6mKvsqraLkglDdckw9JafgL5McL5VEJ6EPNduPaT3ZE9BttDAqg==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dialog": { + "version": "1.1.16", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dialog/-/react-dialog-1.1.16.tgz", + "integrity": "sha512-l9ok83YBclEZhbjgzt76Hw733e6cvRKPNgO6GJ/IETlufXG9p+fRu2wlvpImQvR6xdJ8h7J8J2DBvsPEiEsKMw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.4", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.12", + "@radix-ui/react-focus-guards": "1.1.4", + "@radix-ui/react-focus-scope": "1.1.9", + "@radix-ui/react-id": "1.1.2", + "@radix-ui/react-portal": "1.1.11", + "@radix-ui/react-presence": "1.1.6", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-slot": "1.2.5", + "@radix-ui/react-use-controllable-state": "1.2.3", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-direction": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-direction/-/react-direction-1.1.2.tgz", + "integrity": "sha512-C3vFhbyi4SW3PmbAi6Awpu4OzJtd0MxGurvSsYtr7p7nM8RNB3VAF3CUmnp2j50knpkrRcB7+ycVXzgLgF6yNA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dismissable-layer": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dismissable-layer/-/react-dismissable-layer-1.1.12.tgz", + "integrity": "sha512-MhoruH6xEzsbvOmo4TNgMfmtvRGyDZw4MDSdf4ybMHfezjqwzv6hyd4lsMzBp8K9Sn6sGzCF62x1I7BYUECXOg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.4", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-use-callback-ref": "1.1.2", + "@radix-ui/react-use-escape-keydown": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { "optional": true }, - "cytoscape": { + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-dropdown-menu": { + "version": "2.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-dropdown-menu/-/react-dropdown-menu-2.1.17.tgz", + "integrity": "sha512-S6b3Jm57sY5EdDyOMLkacbB0qMnKhy1RCKZCt795ZkmtUOAvojYIZ5p7dXHIh5Cyr3jCLLI5/g64V3FKLudZmw==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.4", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-id": "1.1.2", + "@radix-ui/react-menu": "2.1.17", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-use-controllable-state": "1.2.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { "optional": true }, - "cytoscape-cose-bilkent": { + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-guards": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-guards/-/react-focus-guards-1.1.4.tgz", + "integrity": "sha512-cot/aB/mOm0IYVYTTmQcEEK1M48lZWi8FlYe5nDPQQ8NYZUlXEFgncJ9p2Kzer3RKSrY7cTTpEMLZKNo9QoP5Q==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-focus-scope": { + "version": "1.1.9", + "resolved": "https://registry.npmjs.org/@radix-ui/react-focus-scope/-/react-focus-scope-1.1.9.tgz", + "integrity": "sha512-9Se8t+Zry+1rEOL7Y6l/4ANYU/TOtAtf8O2fKdwLltcaMcm6kOqYGbzO4tMFQ0bvzO920pRAoHpFZ4W85S3keQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-use-callback-ref": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { "optional": true }, - "d3-force": { + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-id": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-id/-/react-id-1.1.2.tgz", + "integrity": "sha512-orBC88futVpqCmhX1p4cvquNHsELQ+w+vBJnuj3ftETI5bJb0bZn3Tqu3SWN2IOcPycTnMGnhwoermvISt72sA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-menu": { + "version": "2.1.17", + "resolved": "https://registry.npmjs.org/@radix-ui/react-menu/-/react-menu-2.1.17.tgz", + "integrity": "sha512-fmbNnFyf+JYCN0DhhWnEdUTDnZD1mXaPQWivdsPIb8oOSbARfD3LIQJbLCG8a8QLCwoMxiJ7GVPIFcC8Dw8v2Q==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.4", + "@radix-ui/react-collection": "1.1.9", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-direction": "1.1.2", + "@radix-ui/react-dismissable-layer": "1.1.12", + "@radix-ui/react-focus-guards": "1.1.4", + "@radix-ui/react-focus-scope": "1.1.9", + "@radix-ui/react-id": "1.1.2", + "@radix-ui/react-popper": "1.3.0", + "@radix-ui/react-portal": "1.1.11", + "@radix-ui/react-presence": "1.1.6", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-roving-focus": "1.1.12", + "@radix-ui/react-slot": "1.2.5", + "@radix-ui/react-use-callback-ref": "1.1.2", + "aria-hidden": "^1.2.4", + "react-remove-scroll": "^2.7.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { "optional": true }, - "d3-hierarchy": { + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-popper": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@radix-ui/react-popper/-/react-popper-1.3.0.tgz", + "integrity": "sha512-9PB589e1aWZbrlFUHdz6WiPCL+xLZHQFX7oibqG/6Q0SwOkxDyQX9W/cyPa+sAPPKuC8cpLCpRczE5a/1DiwVQ==", + "license": "MIT", + "dependencies": { + "@floating-ui/react-dom": "^2.0.0", + "@radix-ui/react-arrow": "1.1.9", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-use-callback-ref": "1.1.2", + "@radix-ui/react-use-layout-effect": "1.1.2", + "@radix-ui/react-use-rect": "1.1.2", + "@radix-ui/react-use-size": "1.1.2", + "@radix-ui/rect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { "optional": true }, - "uplot": { + "@types/react-dom": { "optional": true } } }, - "node_modules/@oxc-project/types": { - "version": "0.127.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.127.0.tgz", - "integrity": "sha512-aIYXQBo4lCbO4z0R3FHeucQHpF46l2LbMdxRvqvuRuW2OxdnSkcng5B8+K12spgLDj93rtN3+J2Vac/TIO+ciQ==", - "dev": true, + "node_modules/@radix-ui/react-portal": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/@radix-ui/react-portal/-/react-portal-1.1.11.tgz", + "integrity": "sha512-UEytdjgEh2tJGgD/gZK4FUx6t1rNIlM3U0DENhSrG7I75FGm1DnaDuVUWF1pWAWUwGmn1sCJ1VGHn8LhN1aTOw==", "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" + "dependencies": { + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-presence": { + "version": "1.1.6", + "resolved": "https://registry.npmjs.org/@radix-ui/react-presence/-/react-presence-1.1.6.tgz", + "integrity": "sha512-zdTk4PlUO0E18HnZ3wYbW0KkJJxWCdiNYp6g6X1PtONFhxVkg01vliTJAmwIszU6mHiyBOoW9P0rAugl5/hULQ==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-primitive": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-primitive/-/react-primitive-2.1.5.tgz", + "integrity": "sha512-zifXeB8Y88qCYx8PLZ5oQb32KwZub+s925mMoZsBBq9KUQqWKkREubTfs6ASjRPPBe7Jt9O8OHH89+95VG+grA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-slot": "1.2.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-roving-focus": { + "version": "1.1.12", + "resolved": "https://registry.npmjs.org/@radix-ui/react-roving-focus/-/react-roving-focus-1.1.12.tgz", + "integrity": "sha512-FvgPt1bRmg8Xt2QpF7NUZW3dE0ZQHGm41dAdgT2J2GJPoIXz+9Em3NobAxf4fupcxhgHu03E5CRiU2MWvObXyg==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.4", + "@radix-ui/react-collection": "1.1.9", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-direction": "1.1.2", + "@radix-ui/react-id": "1.1.2", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-use-callback-ref": "1.1.2", + "@radix-ui/react-use-controllable-state": "1.2.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-slot": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-slot/-/react-slot-1.2.5.tgz", + "integrity": "sha512-rCMO3QsIVKv5JTY5CVbo2MvO77SpEqqYc8AvRE7OWqRDOIqAKjsp+DrmnY9uc8NPdxB5E2z47HTYGeE2+NTptg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-compose-refs": "1.1.3" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tabs": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tabs/-/react-tabs-1.1.14.tgz", + "integrity": "sha512-D5jwp9JNuwDeCw3CYD2Fz+sSHo0droQjC8u75dJHe4aWr5q6yBiXZU+hurXnKudRgEpUkD5TsI6bjHPo5ThUxA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.4", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-direction": "1.1.2", + "@radix-ui/react-id": "1.1.2", + "@radix-ui/react-presence": "1.1.6", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-roving-focus": "1.1.12", + "@radix-ui/react-use-controllable-state": "1.2.3" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-tooltip": { + "version": "1.2.9", + "resolved": "https://registry.npmjs.org/@radix-ui/react-tooltip/-/react-tooltip-1.2.9.tgz", + "integrity": "sha512-u6F9MmTtBSLkiXNVDrtB/yPCZarM9smNswC24YYLV/M+bth6J3Gs3vlJezEoFwKZvPvxhCpUYdUnOsNG/0XOlA==", + "license": "MIT", + "dependencies": { + "@radix-ui/primitive": "1.1.4", + "@radix-ui/react-compose-refs": "1.1.3", + "@radix-ui/react-context": "1.1.4", + "@radix-ui/react-dismissable-layer": "1.1.12", + "@radix-ui/react-id": "1.1.2", + "@radix-ui/react-popper": "1.3.0", + "@radix-ui/react-portal": "1.1.11", + "@radix-ui/react-presence": "1.1.6", + "@radix-ui/react-primitive": "2.1.5", + "@radix-ui/react-slot": "1.2.5", + "@radix-ui/react-use-controllable-state": "1.2.3", + "@radix-ui/react-visually-hidden": "1.2.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-callback-ref": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-callback-ref/-/react-use-callback-ref-1.1.2.tgz", + "integrity": "sha512-xCso9j1/u8sEgP1RNHjFrXJLApL8LiqOkI1R4ywuN00rxWdYg4oQXuwKLS3i0j5NWLromUD27/4nlxj2UFVvIw==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-controllable-state": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-controllable-state/-/react-use-controllable-state-1.2.3.tgz", + "integrity": "sha512-PLzC90MS+ReootmjC597dvopoelpZ8Q61HJkDXZSExitIq7PL55vHNnesAHwguHK0aPfBnpdNzQtv1uliaqQrA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-effect-event": "0.0.3", + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-effect-event": { + "version": "0.0.3", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-effect-event/-/react-use-effect-event-0.0.3.tgz", + "integrity": "sha512-6c8ZqvPTWILEKnyVkP53EGRCcpnJiKTC21sS/6R1GF5xKyHJJWQEPfkqlcgUkdRQivd6tb23abUwe4ngWmY0JA==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-escape-keydown": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-escape-keydown/-/react-use-escape-keydown-1.1.2.tgz", + "integrity": "sha512-2uVLvLjgO7NZCWw01/FdqRwmA42J0BcjPMUCA+koFEOAb+zjqIP7SiFz/7zWPrKnVmSqr76Omq2ALyCuX4dhLw==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-callback-ref": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-layout-effect": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-layout-effect/-/react-use-layout-effect-1.1.2.tgz", + "integrity": "sha512-jrBWOxZITuGcnjRCM2t2U5ZPkCLxD+Ym6DjfssS5haTj2iiak/DOb64JeN6OdLfLgptb6/e2kKR+ZuTrGoZTPA==", + "license": "MIT", + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-rect": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-rect/-/react-use-rect-1.1.2.tgz", + "integrity": "sha512-d8a+bBY/FxikNPlgJJoaBHZX+zKVbWHYJGTLnLvveQgFSTntkGdEKv3JDtHrMS0DNYpllz2nRsTLGLKYttbpmw==", + "license": "MIT", + "dependencies": { + "@radix-ui/rect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-use-size": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/react-use-size/-/react-use-size-1.1.2.tgz", + "integrity": "sha512-giWQp+4mxjBPt4KZ0MmyuykFNWfbDxKt4x+fPkRYmgRFJSbCZFzUglvMb/Kjn38tm10YP4ufiQZDx3zna4LU6w==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-use-layout-effect": "1.1.2" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/@radix-ui/react-visually-hidden": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/@radix-ui/react-visually-hidden/-/react-visually-hidden-1.2.5.tgz", + "integrity": "sha512-tPcHNI3FajdDBFpl/Ez1m2WL0ufJqBKyHxMDBvKitopamK36WwBGOMicuMEZKkM5Wce41QxUyv6BsiqfrWBiGg==", + "license": "MIT", + "dependencies": { + "@radix-ui/react-primitive": "2.1.5" + }, + "peerDependencies": { + "@types/react": "*", + "@types/react-dom": "*", + "react": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc", + "react-dom": "^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "@types/react-dom": { + "optional": true + } } }, + "node_modules/@radix-ui/rect": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@radix-ui/rect/-/rect-1.1.2.tgz", + "integrity": "sha512-xnXE7wG13PI+cxieVssYXlQJuYVRhH9NBoxt3KNwzghDIA69GMm7d4wXRouHIYjE+KvS6U/MsMO73NdS2MH9ZA==", + "license": "MIT" + }, "node_modules/@rolldown/binding-android-arm64": { "version": "1.0.0-rc.17", "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.0.0-rc.17.tgz", @@ -1108,19 +1773,72 @@ "node": "^20.19.0 || >=22.12.0" } }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.0-rc.7", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.7.tgz", - "integrity": "sha512-qujRfC8sFVInYSPPMLQByRh7zhwkGFS4+tyMQ83srV1qrxL4g8E2tyxVVyxd0+8QeBM1mIk9KbWxkegRr76XzA==", - "dev": true, - "license": "MIT" - }, - "node_modules/@standard-schema/spec": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", - "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", - "dev": true, - "license": "MIT" + "node_modules/@rolldown/pluginutils": { + "version": "1.0.0-rc.7", + "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.0-rc.7.tgz", + "integrity": "sha512-qujRfC8sFVInYSPPMLQByRh7zhwkGFS4+tyMQ83srV1qrxL4g8E2tyxVVyxd0+8QeBM1mIk9KbWxkegRr76XzA==", + "dev": true, + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "dev": true, + "license": "MIT" + }, + "node_modules/@tanstack/query-core": { + "version": "5.101.0", + "resolved": "https://registry.npmjs.org/@tanstack/query-core/-/query-core-5.101.0.tgz", + "integrity": "sha512-cQetA74EB+seWySv1TTKr828TnP0u39m6LykwDXIo84SNortpDkp30TMEjkqtYCNP9c40uT/iwl6MLiufEt0Ow==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } + }, + "node_modules/@tanstack/react-query": { + "version": "5.101.0", + "resolved": "https://registry.npmjs.org/@tanstack/react-query/-/react-query-5.101.0.tgz", + "integrity": "sha512-rLlJXSpkqfizLWgkR5+eLeIk0MvTx/meEIR7LRjxic+qxiQP8zVjq7BqQkiCMNLQBlLfuOLqqr6KO5GtrDlmSg==", + "license": "MIT", + "dependencies": { + "@tanstack/query-core": "5.101.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^18 || ^19" + } + }, + "node_modules/@tanstack/react-virtual": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/@tanstack/react-virtual/-/react-virtual-3.14.2.tgz", + "integrity": "sha512-IpWnmCLvuymRfeeLNVXIzNEYBFLpd3drVIS91sqV78VTZFyldlChkOocZRCPp1B+Wnk09bcLNme8WaMU/9/9bQ==", + "license": "MIT", + "dependencies": { + "@tanstack/virtual-core": "3.17.0" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + }, + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0", + "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/@tanstack/virtual-core": { + "version": "3.17.0", + "resolved": "https://registry.npmjs.org/@tanstack/virtual-core/-/virtual-core-3.17.0.tgz", + "integrity": "sha512-gOxY/hFkPh/XQYhnThBHzkbkX3Ed+z/iushyz+R+JAr213aXxUDgQoTgTdrDpBSRsjFM73P/KfUyWmaF9WHMkQ==", + "license": "MIT", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/tannerlinsley" + } }, "node_modules/@testing-library/dom": { "version": "10.4.1", @@ -1284,7 +2002,7 @@ "version": "19.2.14", "resolved": "https://registry.npmjs.org/@types/react/-/react-19.2.14.tgz", "integrity": "sha512-ilcTH/UniCkMdtexkoCN0bI7pMcJDvmQFPvuPvmEaYA/NSfFTAgdUSLAoVjaRJm7+6PvcM+q1zYOwS4wTYMF9w==", - "dev": true, + "devOptional": true, "license": "MIT", "dependencies": { "csstype": "^3.2.2" @@ -1294,7 +2012,7 @@ "version": "19.2.3", "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.2.3.tgz", "integrity": "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ==", - "dev": true, + "devOptional": true, "license": "MIT", "peerDependencies": { "@types/react": "^19.2.0" @@ -1569,17 +2287,48 @@ } } }, + "node_modules/@vitest/coverage-v8": { + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.8.tgz", + "integrity": "sha512-lt3kovsyHwYe00wq4D1ti0Z974fWj4NLp6siqiyEufUpyFwK9Yhi7rBhac9JL5aA0zoMrJqc4vYPZRUnI7l7nw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@bcoe/v8-coverage": "^1.0.2", + "@vitest/utils": "4.1.8", + "ast-v8-to-istanbul": "^1.0.0", + "istanbul-lib-coverage": "^3.2.2", + "istanbul-lib-report": "^3.0.1", + "istanbul-reports": "^3.2.0", + "magicast": "^0.5.2", + "obug": "^2.1.1", + "std-env": "^4.0.0-rc.1", + "tinyrainbow": "^3.1.0" + }, + "funding": { + "url": "https://opencollective.com/vitest" + }, + "peerDependencies": { + "@vitest/browser": "4.1.8", + "vitest": "4.1.8" + }, + "peerDependenciesMeta": { + "@vitest/browser": { + "optional": true + } + } + }, "node_modules/@vitest/expect": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.5.tgz", - "integrity": "sha512-PWBaRY5JoKuRnHlUHfpV/KohFylaDZTupcXN1H9vYryNLOnitSw60Mw9IAE2r67NbwwzBw/Cc/8q9BK3kIX8Kw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.8.tgz", + "integrity": "sha512-h3nDO677RDLEGlBxyQ5CW8RlMThSKSRLUePLOx09gNIWRL40edgA1GCZSZgf1W55MFAG6/Sw14KeaAnqv0NKdQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -1588,13 +2337,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.5.tgz", - "integrity": "sha512-/x2EmFC4mT4NNzqvC3fmesuV97w5FC903KPmey4gsnJiMQ3Be1IlDKVaDaG8iqaLFHqJ2FVEkxZk5VmeLjIItw==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.8.tgz", + "integrity": "sha512-LEiN/xe4OSIbKe9HQIp5OC24agGD9J5CnmMgsLohVVoOPWL9a2sBoR6VBx43jQZb7Kr1l4RCuyCJzcAa0+dojw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.5", + "@vitest/spy": "4.1.8", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -1615,9 +2364,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.5.tgz", - "integrity": "sha512-7I3q6l5qr03dVfMX2wCo9FxwSJbPdwKjy2uu/YPpU3wfHvIL4QHwVRp57OfGrDFeUJ8/8QdfBKIV12FTtLn00g==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.8.tgz", + "integrity": "sha512-9GasEBxpZ1VYIpqHf/0+YGg121uSNwCKOJqIrTwWP/TB7DmFCiaBpNl3aPZzoLWfWkuqhbH8vJIVobZkvdo2cA==", "dev": true, "license": "MIT", "dependencies": { @@ -1628,13 +2377,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.5.tgz", - "integrity": "sha512-2D+o7Pr82IEO46YPpoA/YU0neeyr6FTerQb5Ro7BUnBuv6NQtT/kmVnczngiMEBhzgqz2UZYl5gArejsyERDSQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.8.tgz", + "integrity": "sha512-EmVxeBAfMJvycdjd6Hm+RbFBbA9fKvo0Kx37hNpBYoYeavH3RNsBXWDooR1mgD52dCrxIIuP7UotpfiwOikvcg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.5", + "@vitest/utils": "4.1.8", "pathe": "^2.0.3" }, "funding": { @@ -1642,14 +2391,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.5.tgz", - "integrity": "sha512-zypXEt4KH/XgKGPUz4eC2AvErYx0My5hfL8oDb1HzGFpEk1P62bxSohdyOmvz+d9UJwanI68MKwr2EquOaOgMQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.8.tgz", + "integrity": "sha512-acfZboRmAIf05DEKcBQy33VXojFJjtUdLyo7oOmV9kebb2xdU01UknNiPuPZoJZQyO7DF0gZdTGTpeAzET9QPQ==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/pretty-format": "4.1.8", + "@vitest/utils": "4.1.8", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -1658,9 +2407,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.5.tgz", - "integrity": "sha512-2lNOsh6+R2Idnf1TCZqSwYlKN2E/iDlD8sgU59kYVl+OMDmvldO1VDk39smRfpUNwYpNRVn3w4YfuC7KfbBnkQ==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.8.tgz", + "integrity": "sha512-6EevtBp6OZOPF7bmz36HrGMeP3txgVSrgebWxHOafDXGkhIzfXK14f8KF6MuFfgXXUeHxmpD3BQxkV00/3s5mA==", "dev": true, "license": "MIT", "funding": { @@ -1668,13 +2417,13 @@ } }, "node_modules/@vitest/utils": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.5.tgz", - "integrity": "sha512-76wdkrmfXfqGjueGgnb45ITPyUi1ycZ4IHgC2bhPDUfWHklY/q3MdLOAB+TF1e6xfl8NxNY0ZYaPCFNWSsw3Ug==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.8.tgz", + "integrity": "sha512-uOJamYALNhfJ6iolExyQM40yIQwDqYnkKtQ5VCiSe17E33H0aQ/u+1GlRuz4LZBk6Mm3sg90G9hEbmEt37C1Zg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.5", + "@vitest/pretty-format": "4.1.8", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -1733,6 +2482,18 @@ "node": ">=8" } }, + "node_modules/aria-hidden": { + "version": "1.2.6", + "resolved": "https://registry.npmjs.org/aria-hidden/-/aria-hidden-1.2.6.tgz", + "integrity": "sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/aria-query": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", @@ -1753,6 +2514,25 @@ "node": ">=12" } }, + "node_modules/ast-v8-to-istanbul": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/ast-v8-to-istanbul/-/ast-v8-to-istanbul-1.0.4.tgz", + "integrity": "sha512-0bC0/4bTSrnwdhU3IsZDwEdojvuPrSg59OYZfKsLRtJZ0u8VBx9DebfqqG8bRdCC0I7vjgxmPi41P0lpkhJHtA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/trace-mapping": "^0.3.31", + "estree-walker": "^3.0.3", + "js-tokens": "^10.0.0" + } + }, + "node_modules/ast-v8-to-istanbul/node_modules/js-tokens": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-10.0.0.tgz", + "integrity": "sha512-lM/UBzQmfJRo9ABXbPWemivdCW8V2G8FHaHdypQaIy523snUjog0W71ayWXTjiR+ixeMyVHN2XcpnTd/liPg/Q==", + "dev": true, + "license": "MIT" + }, "node_modules/balanced-match": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-4.0.4.tgz", @@ -1864,13 +2644,20 @@ "node": ">=18" } }, - "node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", + "node_modules/cmdk": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/cmdk/-/cmdk-1.1.1.tgz", + "integrity": "sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg==", "license": "MIT", - "engines": { - "node": ">=6" + "dependencies": { + "@radix-ui/react-compose-refs": "^1.1.1", + "@radix-ui/react-dialog": "^1.1.6", + "@radix-ui/react-id": "^1.1.0", + "@radix-ui/react-primitive": "^2.0.2" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "react-dom": "^18 || ^19 || ^19.0.0-rc" } }, "node_modules/convert-source-map": { @@ -1880,15 +2667,6 @@ "dev": true, "license": "MIT" }, - "node_modules/cose-base": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/cose-base/-/cose-base-1.0.3.tgz", - "integrity": "sha512-s9whTXInMSgAp/NVXVNuVxVKzGH2qck3aQlVHxDCdAEPgtMKwc4Wq6/QKhgdEdgbLSi9rBTAcPoRa6JpiG4ksg==", - "license": "MIT", - "dependencies": { - "layout-base": "^1.0.0" - } - }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", @@ -1929,30 +2707,9 @@ "version": "3.2.3", "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", - "dev": true, + "devOptional": true, "license": "MIT" }, - "node_modules/cytoscape": { - "version": "3.33.2", - "resolved": "https://registry.npmjs.org/cytoscape/-/cytoscape-3.33.2.tgz", - "integrity": "sha512-sj4HXd3DokGhzZAdjDejGvTPLqlt84vNFN8m7bGsOzDY5DyVcxIb2ejIXat2Iy7HxWhdT/N1oKyheJ5YdpsGuw==", - "license": "MIT", - "engines": { - "node": ">=0.10" - } - }, - "node_modules/cytoscape-cose-bilkent": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/cytoscape-cose-bilkent/-/cytoscape-cose-bilkent-4.1.0.tgz", - "integrity": "sha512-wgQlVIUJF13Quxiv5e1gstZ08rnZj2XaLHGoFMYXz7SkNfCDOOteKBE6SYRfA9WxxI/iBc3ajfDoc6hb/MRAHQ==", - "license": "MIT", - "dependencies": { - "cose-base": "^1.0.0" - }, - "peerDependencies": { - "cytoscape": "^3.2.0" - } - }, "node_modules/data-urls": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-7.0.0.tgz", @@ -2019,6 +2776,12 @@ "node": ">=8" } }, + "node_modules/detect-node-es": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/detect-node-es/-/detect-node-es-1.1.0.tgz", + "integrity": "sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==", + "license": "MIT" + }, "node_modules/dom-accessibility-api": { "version": "0.5.16", "resolved": "https://registry.npmjs.org/dom-accessibility-api/-/dom-accessibility-api-0.5.16.tgz", @@ -2394,6 +3157,15 @@ "node": ">=6.9.0" } }, + "node_modules/get-nonce": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-nonce/-/get-nonce-1.0.1.tgz", + "integrity": "sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, "node_modules/glob-parent": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-6.0.2.tgz", @@ -2420,6 +3192,16 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/has-flag": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/hermes-estree": { "version": "0.25.1", "resolved": "https://registry.npmjs.org/hermes-estree/-/hermes-estree-0.25.1.tgz", @@ -2450,6 +3232,13 @@ "node": "^20.19.0 || ^22.12.0 || >=24.0.0" } }, + "node_modules/html-escaper": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-2.0.2.tgz", + "integrity": "sha512-H2iMtd0I4Mt5eYiapRdIDjp+XzelXQ0tFE4JS7YFwFevXXMmOp9myNrUvCg0D6ws8iqkRPBfKHgbwig1SmlLfg==", + "dev": true, + "license": "MIT" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -2517,6 +3306,45 @@ "dev": true, "license": "ISC" }, + "node_modules/istanbul-lib-coverage": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.2.tgz", + "integrity": "sha512-O8dpsF+r0WV/8MNRKfnmrtCWhuKjxrq2w+jpzBL5UZKTi2LeVWnWOmWRxFlesJONmc+wLAGvKQZEOanko0LFTg==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=8" + } + }, + "node_modules/istanbul-lib-report": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", + "integrity": "sha512-GCfE1mtsHGOELCU8e/Z7YWzpmybrx/+dSTfLrvY8qRmaY6zXTKWn6WQIjaAFw069icm6GVMNkgu0NzI4iPZUNw==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "istanbul-lib-coverage": "^3.0.0", + "make-dir": "^4.0.0", + "supports-color": "^7.1.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/istanbul-reports": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/istanbul-reports/-/istanbul-reports-3.2.0.tgz", + "integrity": "sha512-HGYWWS/ehqTV3xN10i23tkPkpH46MLCIMFNCaaKNavAXTF1RkqxawEPtnjnGZ6XKSInBKkiOA5BKS+aZiY3AvA==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "html-escaper": "^2.0.0", + "istanbul-lib-report": "^3.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -2632,12 +3460,6 @@ "json-buffer": "3.0.1" } }, - "node_modules/layout-base": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/layout-base/-/layout-base-1.0.2.tgz", - "integrity": "sha512-8h2oVEZNktL4BH2JCOI90iD1yXwL6iNW7KcCKT2QZgQJR2vbqDsldCTPRU9NifTCqHZci57XvQQ15YTu+sTYPg==", - "license": "MIT" - }, "node_modules/levn": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/levn/-/levn-0.4.1.tgz", @@ -2969,6 +3791,47 @@ "@jridgewell/sourcemap-codec": "^1.5.5" } }, + "node_modules/magicast": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.3.tgz", + "integrity": "sha512-pVKE4UdSQ7DvHzivsCIFx2BJn1mHG6KsyrFcaxFx6tONdneEuThrDx0Cj3AMg58KyN4pzYT+LHOotxDQDjNvkw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.3", + "@babel/types": "^7.29.0", + "source-map-js": "^1.2.1" + } + }, + "node_modules/make-dir": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/make-dir/-/make-dir-4.0.0.tgz", + "integrity": "sha512-hXdUTZYIVOt1Ex//jAQi+wTZZpUpwBj/0QsOzqegb3rGMMeJiSEu5xLHnYfBrRV4RH2+OCSOO95Is/7x1WJ4bw==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/make-dir/node_modules/semver": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.4.tgz", + "integrity": "sha512-rUCObTnP32Q08R2uuIrt7r9PlEonuTmtuXYcW6s5kjdlj3xbnwe+21yXptAUYcMAABLkYYTtnmzb3w3EDZfueA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/mdn-data": { "version": "2.27.1", "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", @@ -3002,6 +3865,12 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/mitt": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/mitt/-/mitt-3.0.1.tgz", + "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw==", + "license": "MIT" + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -3271,6 +4140,75 @@ "license": "MIT", "peer": true }, + "node_modules/react-remove-scroll": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/react-remove-scroll/-/react-remove-scroll-2.7.2.tgz", + "integrity": "sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==", + "license": "MIT", + "dependencies": { + "react-remove-scroll-bar": "^2.3.7", + "react-style-singleton": "^2.2.3", + "tslib": "^2.1.0", + "use-callback-ref": "^1.3.3", + "use-sidecar": "^1.1.3" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-remove-scroll-bar": { + "version": "2.3.8", + "resolved": "https://registry.npmjs.org/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.8.tgz", + "integrity": "sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==", + "license": "MIT", + "dependencies": { + "react-style-singleton": "^2.2.2", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/react-style-singleton": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/react-style-singleton/-/react-style-singleton-2.2.3.tgz", + "integrity": "sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==", + "license": "MIT", + "dependencies": { + "get-nonce": "^1.0.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, "node_modules/redent": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/redent/-/redent-3.0.0.tgz", @@ -3285,6 +4223,15 @@ "node": ">=8" } }, + "node_modules/regexparam": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/regexparam/-/regexparam-3.0.0.tgz", + "integrity": "sha512-RSYAtP31mvYLkAHrOlh25pCNQ5hWnT106VukGaaFfuJrZFkGRX5GhUAdPqpSDXxOhA2c4akmRuplv1mRqnBn6Q==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", @@ -3432,6 +4379,19 @@ "node": ">=8" } }, + "node_modules/supports-color": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", + "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^4.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", @@ -3546,9 +4506,7 @@ "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD", - "optional": true + "license": "0BSD" }, "node_modules/type-check": { "version": "0.4.0", @@ -3659,6 +4617,58 @@ "punycode": "^2.1.0" } }, + "node_modules/use-callback-ref": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/use-callback-ref/-/use-callback-ref-1.3.3.tgz", + "integrity": "sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sidecar": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/use-sidecar/-/use-sidecar-1.1.3.tgz", + "integrity": "sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==", + "license": "MIT", + "dependencies": { + "detect-node-es": "^1.1.0", + "tslib": "^2.0.0" + }, + "engines": { + "node": ">=10" + }, + "peerDependencies": { + "@types/react": "*", + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + } + } + }, + "node_modules/use-sync-external-store": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", + "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, "node_modules/vite": { "version": "8.0.10", "resolved": "https://registry.npmjs.org/vite/-/vite-8.0.10.tgz", @@ -3738,19 +4748,19 @@ } }, "node_modules/vitest": { - "version": "4.1.5", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.5.tgz", - "integrity": "sha512-9Xx1v3/ih3m9hN+SbfkUyy0JAs72ap3r7joc87XL6jwF0jGg6mFBvQ1SrwaX+h8BlkX6Hz9shdd1uo6AF+ZGpg==", + "version": "4.1.8", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.8.tgz", + "integrity": "sha512-flY6ScbCIt9HThs+C5HS7jvGOB560DJtk/Z15IQROTA6zEy49Nh8T/dofWTQL+n3vswqn87sbJNiuqw1SDp5Ig==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.5", - "@vitest/mocker": "4.1.5", - "@vitest/pretty-format": "4.1.5", - "@vitest/runner": "4.1.5", - "@vitest/snapshot": "4.1.5", - "@vitest/spy": "4.1.5", - "@vitest/utils": "4.1.5", + "@vitest/expect": "4.1.8", + "@vitest/mocker": "4.1.8", + "@vitest/pretty-format": "4.1.8", + "@vitest/runner": "4.1.8", + "@vitest/snapshot": "4.1.8", + "@vitest/spy": "4.1.8", + "@vitest/utils": "4.1.8", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -3778,12 +4788,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.5", - "@vitest/browser-preview": "4.1.5", - "@vitest/browser-webdriverio": "4.1.5", - "@vitest/coverage-istanbul": "4.1.5", - "@vitest/coverage-v8": "4.1.5", - "@vitest/ui": "4.1.5", + "@vitest/browser-playwright": "4.1.8", + "@vitest/browser-preview": "4.1.8", + "@vitest/browser-webdriverio": "4.1.8", + "@vitest/coverage-istanbul": "4.1.8", + "@vitest/coverage-v8": "4.1.8", + "@vitest/ui": "4.1.8", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" @@ -3918,6 +4928,20 @@ "node": ">=0.10.0" } }, + "node_modules/wouter": { + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/wouter/-/wouter-3.10.0.tgz", + "integrity": "sha512-zTfddD80zc2/J5l8JKcdvzOK6AwP0kpyHEI3DxRN2bn8U1oJPnrSVm8v+X3WwDamvLAOxTO7ZvkxkpRWlyeJ1Q==", + "license": "Unlicense", + "dependencies": { + "mitt": "^3.0.1", + "regexparam": "^3.0.0", + "use-sync-external-store": "^1.0.0" + }, + "peerDependencies": { + "react": ">=16.8.0" + } + }, "node_modules/xml-name-validator": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-5.0.0.tgz", diff --git a/ui/package.json b/ui/package.json index 17bfa21..e634848 100644 --- a/ui/package.json +++ b/ui/package.json @@ -5,20 +5,25 @@ "type": "module", "scripts": { "dev": "vite", - "build": "tsc -b && vite build", + "build": "tsc -b && vite build && node scripts/precompress.mjs", "lint": "eslint .", "preview": "vite preview", "test": "vitest run", - "test:watch": "vitest" + "test:watch": "vitest", + "check-budgets": "node scripts/check-budgets.mjs" }, "dependencies": { - "@ossrandom/design-system": "^0.3.0", - "clsx": "^2.1.1", - "cytoscape": "^3.33.2", - "cytoscape-cose-bilkent": "^4.1.0", + "@radix-ui/react-dialog": "^1.1.16", + "@radix-ui/react-dropdown-menu": "^2.1.17", + "@radix-ui/react-tabs": "^1.1.14", + "@radix-ui/react-tooltip": "^1.2.9", + "@tanstack/react-query": "^5.101.0", + "@tanstack/react-virtual": "^3.14.2", + "cmdk": "^1.1.1", "lucide-react": "^0.469.0", "react": "^19.2.5", - "react-dom": "^19.2.5" + "react-dom": "^19.2.5", + "wouter": "^3.10.0" }, "overrides": { "brace-expansion": "5.0.6" @@ -32,6 +37,7 @@ "@types/react": "^19.0.10", "@types/react-dom": "^19.0.4", "@vitejs/plugin-react": "^6.0.1", + "@vitest/coverage-v8": "^4.1.8", "eslint": "^10.2.1", "eslint-plugin-react-hooks": "^7.1.1", "eslint-plugin-react-refresh": "^0.5.2", diff --git a/ui/public/favicon.svg b/ui/public/favicon.svg new file mode 100644 index 0000000..2e9daa5 --- /dev/null +++ b/ui/public/favicon.svg @@ -0,0 +1,7 @@ + + + + + + + diff --git a/ui/public/fonts/LICENSE-inter b/ui/public/fonts/LICENSE-inter new file mode 100644 index 0000000..40589da --- /dev/null +++ b/ui/public/fonts/LICENSE-inter @@ -0,0 +1,93 @@ +Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) Inter-Italic[opsz,wght].ttf: Copyright 2016 The Inter Project Authors (https://github.com/rsms/inter) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/ui/public/fonts/LICENSE-jetbrains-mono b/ui/public/fonts/LICENSE-jetbrains-mono new file mode 100644 index 0000000..8f7ed67 --- /dev/null +++ b/ui/public/fonts/LICENSE-jetbrains-mono @@ -0,0 +1,93 @@ +Copyright 2020 The JetBrains Mono Project Authors (https://github.com/JetBrains/JetBrainsMono) JetBrainsMono-Italic[wght].ttf: Copyright 2020 The JetBrains Mono Project Authors (https://github.com/JetBrains/JetBrainsMono) + +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/ui/public/fonts/inter-latin-wght-normal.woff2 b/ui/public/fonts/inter-latin-wght-normal.woff2 new file mode 100644 index 0000000..d15208d Binary files /dev/null and b/ui/public/fonts/inter-latin-wght-normal.woff2 differ diff --git a/ui/public/fonts/jetbrains-mono-latin-400-normal.woff2 b/ui/public/fonts/jetbrains-mono-latin-400-normal.woff2 new file mode 100644 index 0000000..5858873 Binary files /dev/null and b/ui/public/fonts/jetbrains-mono-latin-400-normal.woff2 differ diff --git a/ui/scripts/check-budgets.mjs b/ui/scripts/check-budgets.mjs new file mode 100644 index 0000000..2b92425 --- /dev/null +++ b/ui/scripts/check-budgets.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +// Zero-dependency bundle budget gate. Gzips the built dist output and fails +// (exit 1) when any budget in ui/budgets.json is exceeded. Run manually or +// in CI via `npm run check-budgets` — deliberately NOT wired into `build` +// yet (phase C1/C2 reports numbers; later phases flip it to a hard gate). +import { gzipSync } from 'node:zlib'; +import { readFileSync, readdirSync, existsSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const uiDir = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const distDir = resolve(uiDir, '../internal/ui/dist'); +const budgets = JSON.parse(readFileSync(join(uiDir, 'budgets.json'), 'utf8')).gzipKb; + +if (!existsSync(join(distDir, 'index.html'))) { + console.error(`check-budgets: ${distDir}/index.html not found — run \`npm run build\` first`); + process.exit(1); +} + +const gzipKb = (path) => gzipSync(readFileSync(path)).length / 1024; +const fmt = (kb) => `${kb.toFixed(2)} KB`; + +// "Initial" = everything index.html references: the entry