From 948967b9664307c3f728518eb69f2d0bd855ac19 Mon Sep 17 00:00:00 2001 From: Amit Kumar Date: Thu, 18 Jun 2026 10:43:00 +0000 Subject: [PATCH] feat(config): default STORE_MIN_SEVERITY=WARN for all drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make WARN the global default for the store-severity gate (was "" globally, WARN only on SQLite). INFO/DEBUG logs now reach in-memory consumers (GraphRAG, Drain clustering, span/trace correlation) but are NOT persisted to the relational DB on any driver — only WARN/ERROR/FATAL grow the store. Analytics are preserved by construction: pipeline.process skips the DB row write for sub-WARN logs but still fires LogCallback over the FULL batch (TestPipeline_StoreMinSeverity_DropsBelowThresholdFromPersist), and the 60s GraphRAG refresh rebuilds topology from the spans table, not logs — so in-memory state/analytics don't depend on what's persisted. - config.go: StoreMinSeverity default "" -> "WARN"; remove the now-redundant SQLite-only override from applyDriverDefaults (WARN is global; SQLite no longer differs). - driver_defaults_test.go + CLAUDE.md updated to match. INGEST_MIN_SEVERITY stays INFO, so DEBUG is still dropped at the receiver (not analysed either). Set INGEST_MIN_SEVERITY=DEBUG to also feed DEBUG into in-memory analytics (raises in-memory event volume). Stats impact: error-rate/latency/p99/health/top-failing-services are trace/span-derived and unaffected; only the DB-derived TotalLogs count and log search reflect the WARN+ stored set (by design). Store-skips are metered via Pipeline.Stats().StoreFiltered. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01LDQVJrixs2nJoea67a8pEG --- CLAUDE.md | 5 ++--- internal/config/config.go | 15 ++++++++------- internal/config/driver_defaults_test.go | 4 +--- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index e6e6903..be0a305 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -221,7 +221,7 @@ Key settings in `internal/config/config.go`: - `GRAPHRAG_WORKER_COUNT` (16), `GRAPHRAG_EVENT_QUEUE_SIZE` (100000; **10000 on SQLite**) — sized for 100–200 services; raise further if `otelcontext_graphrag_events_dropped_total` climbs - `GRAPHRAG_TRACE_TTL` (`1h`; **`30m` on SQLite**), `GRAPHRAG_MAX_SPANS_PER_TENANT` (500000), `GRAPHRAG_TENANT_IDLE_TTL` (`24h`) — in-memory GraphRAG memory bounds. Spans past the per-tenant cap are skipped from the graph only (DB unaffected; metered as `otelcontext_graphrag_events_dropped_total{signal="span_capacity"}`); tenant store slices idle past the TTL are evicted (default tenant immune, self-healing via the 60s rebuild). SignalStore metrics are bounded to 2000/tenant + 24h TTL (constants). - `PPROF_ADDR` (`127.0.0.1:6060`) — `net/http/pprof` on a dedicated loopback listener (never the public mux); empty disables. Startup also sets a soft `GOMEMLIMIT` (honors the env var, else 75% of the cgroup/host budget via `internal/membudget`). -- `INGEST_MIN_SEVERITY` (`INFO`), `STORE_MIN_SEVERITY` (`""` = same as ingest; **defaults to `"WARN"` when `DB_DRIVER=sqlite`**) — two-tier log severity gate. The ingest gate runs at the OTLP receiver and **drops the log entirely** below the threshold (no in-memory enrichment either). The store gate runs at the persist boundary inside the async pipeline (`internal/ingest/pipeline.go:process`) and **only skips the DB row write** — the log still flows through `LogCallback` so GraphRAG Drain template mining and span/trace correlation see it. Use case: `INGEST_MIN_SEVERITY=DEBUG STORE_MIN_SEVERITY=WARN` keeps SQLite small while letting in-memory anomaly detection benefit from the verbose stream. Setting `STORE_MIN_SEVERITY` ≤ `INGEST_MIN_SEVERITY` is a no-op (logged as a warning at startup). Drops surface via `Pipeline.Stats().StoreFiltered`. +- `INGEST_MIN_SEVERITY` (`INFO`), `STORE_MIN_SEVERITY` (**defaults to `"WARN"` for all drivers**; `""` falls back to same-as-ingest) — two-tier log severity gate. The ingest gate runs at the OTLP receiver and **drops the log entirely** below the threshold (no in-memory enrichment either). The store gate runs at the persist boundary inside the async pipeline (`internal/ingest/pipeline.go:process`) and **only skips the DB row write** — the log still flows through `LogCallback` so GraphRAG Drain template mining and span/trace correlation see it. By default (`STORE_MIN_SEVERITY=WARN`, `INGEST_MIN_SEVERITY=INFO`) INFO logs reach GraphRAG/Drain in-memory but are **not** persisted — the DB only grows with WARN+. To also analyse DEBUG in-memory (not just INFO), set `INGEST_MIN_SEVERITY=DEBUG` (raises in-memory event volume). Setting `STORE_MIN_SEVERITY` ≤ `INGEST_MIN_SEVERITY` is a no-op (logged as a warning at startup). Drops surface via `Pipeline.Stats().StoreFiltered`. - `INGEST_ASYNC_ENABLED` (true), `INGEST_PIPELINE_QUEUE_SIZE` (50000), `INGEST_PIPELINE_WORKERS` (8), `INGEST_PIPELINE_MAX_BYTES` (536870912 = 512 MB; **128 MB on SQLite**) — async ingest pipeline (`internal/ingest/pipeline.go`). Hybrid backpressure: <90% accept all, 90–100% drop healthy batches (errors/slow always pass), 100% return gRPC `RESOURCE_EXHAUSTED`. The byte cap bounds queue memory regardless of item count — at the cap even priority batches get `RESOURCE_EXHAUSTED`/429 (a 429 is recoverable, an OOM kill is not); watch `otelcontext_ingest_pipeline_queue_bytes` and reason `bytes_full`. Set `INGEST_ASYNC_ENABLED=false` to revert to synchronous DB writes inside `Export()`. Drops surface as `otelcontext_ingest_pipeline_dropped_total{signal,reason}`. - `GRPC_MAX_RECV_MB` (16), `GRPC_MAX_CONCURRENT_STREAMS` (1000) — OTLP gRPC server caps, validated to 1..256 and 1..1_000_000 - `RETENTION_BATCH_SIZE` (50000), `RETENTION_BATCH_SLEEP_MS` (1) — purge pacing; raise the sleep on busy production DBs @@ -230,7 +230,7 @@ Key settings in `internal/config/config.go`: ### SQLite per-driver defaults (auto-flipped when DB_DRIVER=sqlite) -So a 100+ service deployment on SQLite survives without OOM, `config.Load()` overrides nine defaults at the end of the Load() pass — but **only when the operator did not explicitly set the env var** (detected via `os.LookupEnv` presence, not value comparison). Postgres/MSSQL/MySQL paths are untouched. +So a 100+ service deployment on SQLite survives without OOM, `config.Load()` overrides the defaults listed below at the end of the Load() pass — but **only when the operator did not explicitly set the env var** (detected via `os.LookupEnv` presence, not value comparison). Postgres/MSSQL/MySQL paths are untouched. | Env var | SQLite default | Postgres default | Rationale | |---|---|---|---| @@ -242,7 +242,6 @@ So a 100+ service deployment on SQLite survives without OOM, `config.Load()` ove | `GRAPHRAG_EVENT_QUEUE_SIZE` | 10000 | 100000 | Each queued event embeds a Span/Log by value (~0.5–2 KB); buffer less, drop sooner (metered). | | `GRAPHRAG_TRACE_TTL` | 30m | 1h | The in-memory span window is the largest legitimate GraphRAG consumer; anomaly/investigation lookbacks are ≤5min. | | `METRIC_MAX_CARDINALITY` | 3000 | 10000 | Bound the in-memory TSDB series map. | -| `STORE_MIN_SEVERITY` | `"WARN"` | `""` | Skip INFO/DEBUG persists; in-memory GraphRAG/Drain still sees them. | | `SAMPLING_RATE` | 0.05 | 1.0 | Errors and slow spans are always kept by `SAMPLING_ALWAYS_ON_ERRORS`. | | `GRPC_MAX_CONCURRENT_STREAMS` | 240 | 1000 | ~2 streams per service at 120 services with headroom. | | `LOG_FTS_ENABLED` | `true` | n/a | FTS5 BM25 is dramatically faster than LIKE on the kept `search_logs` path. | diff --git a/internal/config/config.go b/internal/config/config.go index bdfb0b5..4d47b4f 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -30,11 +30,13 @@ type Config struct { IngestExcludedServices string // Storage Filtering. Logs that pass IngestMinSeverity (so they reach the - // receiver and feed in-memory consumers like vectordb / GraphRAG) but - // fall below StoreMinSeverity are skipped during the DB persist pass — - // only the row-write is dropped, not the in-memory enrichment. Empty - // (default) means StoreMinSeverity == IngestMinSeverity, i.e. no - // behavior change vs. the single-threshold semantics. + // receiver and feed in-memory consumers like GraphRAG / Drain) but fall + // below StoreMinSeverity are skipped during the DB persist pass — only the + // row-write is dropped, not the in-memory enrichment. Defaults to "WARN" + // (all drivers): INFO/DEBUG still inform anomaly detection + clustering but + // don't grow the DB. Empty falls back to IngestMinSeverity (no second-tier + // gate); a value <= IngestMinSeverity is a no-op since the receiver already + // drops below that. StoreMinSeverity string // DB Connection Pool @@ -279,7 +281,7 @@ func Load(customPath string) (*Config, error) { PprofAddr: getEnv("PPROF_ADDR", "127.0.0.1:6060"), IngestMinSeverity: getEnv("INGEST_MIN_SEVERITY", "INFO"), - StoreMinSeverity: getEnv("STORE_MIN_SEVERITY", ""), + StoreMinSeverity: getEnv("STORE_MIN_SEVERITY", "WARN"), IngestAllowedServices: getEnv("INGEST_ALLOWED_SERVICES", ""), IngestExcludedServices: getEnv("INGEST_EXCLUDED_SERVICES", ""), @@ -409,7 +411,6 @@ var sqliteOverrides = []struct { // first structure to bloat — bound it to 128MB instead of 512MB. {"INGEST_PIPELINE_MAX_BYTES", func(c *Config) { c.IngestPipelineMaxBytes = 128 << 20 }}, {"METRIC_MAX_CARDINALITY", func(c *Config) { c.MetricMaxCardinality = 3000 }}, - {"STORE_MIN_SEVERITY", func(c *Config) { c.StoreMinSeverity = "WARN" }}, {"SAMPLING_RATE", func(c *Config) { c.SamplingRate = 0.05 }}, {"GRPC_MAX_CONCURRENT_STREAMS", func(c *Config) { c.GRPCMaxConcurrentStreams = 240 }}, {"LOG_FTS_ENABLED", func(c *Config) { c.LogFTSEnabled = true }}, diff --git a/internal/config/driver_defaults_test.go b/internal/config/driver_defaults_test.go index 33b5189..4f21a58 100644 --- a/internal/config/driver_defaults_test.go +++ b/internal/config/driver_defaults_test.go @@ -15,7 +15,6 @@ var sqliteEnvKeys = []string{ "INGEST_PIPELINE_QUEUE_SIZE", "INGEST_PIPELINE_MAX_BYTES", "METRIC_MAX_CARDINALITY", - "STORE_MIN_SEVERITY", "SAMPLING_RATE", "GRPC_MAX_CONCURRENT_STREAMS", "LOG_FTS_ENABLED", @@ -55,7 +54,7 @@ func postgresDefaultsConfig(driver string) *Config { IngestPipelineQueueSize: 50000, // Postgres default IngestPipelineMaxBytes: 512 << 20, // Postgres default MetricMaxCardinality: 10000, // Postgres default - StoreMinSeverity: "", // same-as-ingest default + StoreMinSeverity: "WARN", // global default (all drivers), not driver-flipped SamplingRate: 1.0, // keep-all default GRPCMaxConcurrentStreams: 1000, // Postgres default GraphRAGEventQueueSize: 100000, // Postgres default @@ -83,7 +82,6 @@ func TestApplyDriverDefaults_SQLite_FlipsAllWhenNoEnv(t *testing.T) { {"IngestPipelineQueueSize", cfg.IngestPipelineQueueSize, 10000}, {"IngestPipelineMaxBytes", cfg.IngestPipelineMaxBytes, 128 << 20}, {"MetricMaxCardinality", cfg.MetricMaxCardinality, 3000}, - {"StoreMinSeverity", cfg.StoreMinSeverity, "WARN"}, {"SamplingRate", cfg.SamplingRate, 0.05}, {"GRPCMaxConcurrentStreams", cfg.GRPCMaxConcurrentStreams, 240}, {"LogFTSEnabled", cfg.LogFTSEnabled, true},