From a64c990b2cd6a59dc0d786663f44d50607b70fbb Mon Sep 17 00:00:00 2001 From: CodexManvik Date: Wed, 23 Sep 2026 01:31:36 +0530 Subject: [PATCH] feat(observability): Grafana + Loki - every service's logs, the metrics and the audit trail at one URL Grafana at http://localhost:3000 (compose) / http://localhost:30300 (kind), opening on a provisioned "IntelliOps - Logs, metrics and audit" dashboard: - Service logs from every container (filter by service, regex search), log volume and errors/warnings per minute by service - via Loki. - CPU, p99 latency, error rate and service_up - via the existing Prometheus. - The latest 500 audit_records rows - via the Postgres store. Grafana Alloy ships the logs: through the Docker socket in compose, through the Kubernetes API (loki.source.kubernetes, namespace-scoped read-only Role, no hostPath/DaemonSet) in the chart. It drops the /health, /ready and /metrics access lines, which were most of the volume, and lifts `level` out of JSON lines. Enabling the stack in the chart switches the services to JSON logs. One copy of the config under deploy/k8s/platform/files/observability/ - compose bind-mounts it, Helm reads it with .Files.Get. The dashboard is generated by scripts/make-grafana-dashboard.py. Anonymous visitors are read-only Viewers (switchable off in both targets). Images pinned: grafana 11.3.0, loki 3.2.1, alloy v1.4.3. CI's compose smoke now checks Loki/Grafana readiness, all three datasources healthy, and service logs actually reaching Loki. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 20 + README.md | 5 + deploy/docker-compose.yml | 57 +++ deploy/k8s/README.md | 4 + deploy/k8s/kind-config-full.yaml | 1 + .../observability/alloy-kubernetes.alloy | 70 +++ .../observability/grafana-dashboards.yaml | 10 + .../observability/grafana-datasources.yaml | 38 ++ .../observability/intelliops-dashboard.json | 455 ++++++++++++++++++ .../platform/files/observability/loki.yaml | 45 ++ deploy/k8s/platform/templates/configmap.yaml | 3 + .../k8s/platform/templates/observability.yaml | 273 +++++++++++ deploy/k8s/platform/values.yaml | 19 + deploy/observability/alloy-docker.alloy | 58 +++ docs/DEMO.md | 1 + docs/OBSERVABILITY.md | 50 ++ scripts/make-grafana-dashboard.py | 234 +++++++++ tests/test_observability_config.py | 55 +++ 18 files changed, 1398 insertions(+) create mode 100644 deploy/k8s/platform/files/observability/alloy-kubernetes.alloy create mode 100644 deploy/k8s/platform/files/observability/grafana-dashboards.yaml create mode 100644 deploy/k8s/platform/files/observability/grafana-datasources.yaml create mode 100644 deploy/k8s/platform/files/observability/intelliops-dashboard.json create mode 100644 deploy/k8s/platform/files/observability/loki.yaml create mode 100644 deploy/k8s/platform/templates/observability.yaml create mode 100644 deploy/observability/alloy-docker.alloy create mode 100644 scripts/make-grafana-dashboard.py create mode 100644 tests/test_observability_config.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2889313..09b1e11 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -118,6 +118,26 @@ jobs: sleep 3 done done + # Logs + metrics UI: Loki ready, Grafana up with all three datasources + # healthy, and Alloy actually shipping container logs into Loki. + for url in http://localhost:3100/ready http://localhost:3000/api/health; do + for attempt in $(seq 1 30); do + curl -sf "$url" >/dev/null && { echo " ✓ $url"; break; } + [ "$attempt" -eq 30 ] && { echo " ✗ $url never 200"; exit 1; } + sleep 3 + done + done + for uid in prometheus loki intelliops-postgres; do + curl -sf -u admin:admin "http://localhost:3000/api/datasources/uid/$uid/health" | grep -q '"status":"OK"' \ + && echo " ✓ grafana datasource $uid" \ + || { echo " ✗ grafana datasource $uid unhealthy"; exit 1; } + done + for attempt in $(seq 1 30); do + curl -sf "http://localhost:3100/loki/api/v1/label/service/values" | grep -q '"governance"' \ + && { echo " ✓ loki is receiving service logs"; break; } + [ "$attempt" -eq 30 ] && { echo " ✗ no service logs reached loki"; exit 1; } + sleep 3 + done - if: failure() run: docker compose -f deploy/docker-compose.yml logs - if: always() diff --git a/README.md b/README.md index d7a944c..69c21e3 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,11 @@ npm install && npm run dev # open http://localhost:5173 ``` +**Logs, metrics and the audit trail in one place:** Grafana at + (on kind: ). Every container's +logs (via Loki), the Prometheus metrics, and the Postgres audit trail on one +dashboard, viewable without logging in. See [docs/OBSERVABILITY.md](docs/OBSERVABILITY.md#4-logs-and-metrics-ui-grafana). + **Drive a real incident end to end:** ```bash diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index a1ab3f9..16c0762 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -261,6 +261,62 @@ services: demo-app: condition: service_healthy + # ---- Logs & metrics UI: Grafana at http://localhost:3000 ---------------- + # Loki stores every container's logs (shipped by Alloy through the Docker + # socket); Grafana puts them next to the Prometheus metrics and the Postgres + # audit trail on one provisioned dashboard. Config lives in + # deploy/k8s/platform/files/observability/ so the Helm chart ships the same. + loki: + image: grafana/loki:3.2.1 + command: ["-config.file=/etc/loki/loki.yaml"] + volumes: + - ./k8s/platform/files/observability/loki.yaml:/etc/loki/loki.yaml:ro + - loki-data:/loki + ports: + - "3100:3100" + + alloy: + image: grafana/alloy:v1.4.3 + command: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + volumes: + - ./observability/alloy-docker.alloy:/etc/alloy/config.alloy:ro + # Read-only access to the Docker API, to discover and tail containers. + - /var/run/docker.sock:/var/run/docker.sock:ro + ports: + - "12345:12345" # Alloy's own UI: pipeline graph + per-target status + depends_on: + - loki + + grafana: + image: grafana/grafana:11.3.0 + environment: + # Anyone who can reach localhost:3000 can VIEW (dashboards, Explore, logs). + # Editing needs the admin login. Turn anonymous off for anything shared. + GF_AUTH_ANONYMOUS_ENABLED: "${GRAFANA_ANONYMOUS:-true}" + GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer + GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD:-admin}" + GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/intelliops-dashboard.json + GF_ANALYTICS_REPORTING_ENABLED: "false" + GF_ANALYTICS_CHECK_FOR_UPDATES: "false" + # Read by the provisioned Postgres datasource (audit trail panel). + INTELLIOPS_PG_USER: intelliops + INTELLIOPS_PG_PASSWORD: intelliops + INTELLIOPS_PG_DB: intelliops + volumes: + - ./k8s/platform/files/observability/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/intelliops.yaml:ro + - ./k8s/platform/files/observability/grafana-dashboards.yaml:/etc/grafana/provisioning/dashboards/intelliops.yaml:ro + - ./k8s/platform/files/observability/intelliops-dashboard.json:/var/lib/grafana/dashboards/intelliops-dashboard.json:ro + ports: + - "3000:3000" + depends_on: + - loki + - prometheus + - postgres + read: <<: *service environment: @@ -392,5 +448,6 @@ services: volumes: postgres-data: + loki-data: # Shared between rca and (Task 3) meridian-gateway at rca_context_path. rca-context: diff --git a/deploy/k8s/README.md b/deploy/k8s/README.md index a9cd2cd..c650463 100644 --- a/deploy/k8s/README.md +++ b/deploy/k8s/README.md @@ -40,6 +40,10 @@ the `console`), `helm install`s with `values-live.yaml` (+ your key via dashboard shows **real** situations, embedding-fit confidences, and per-metric verification. - **Read service:** . +- **Grafana (logs + metrics + audit trail):** , requiring + the 30300 port mapping in `kind-config-full.yaml` (recreate an existing cluster to + pick it up, or `kubectl port-forward svc/grafana 3000:3000`). Off with + `--set observability.enabled=false`. **Safe by default.** `helm install` *without* the live overlay (or `SAFE=1 ./scripts/kind-up-full.sh`) keeps the safe posture — `dry_run` / selector diff --git a/deploy/k8s/kind-config-full.yaml b/deploy/k8s/kind-config-full.yaml index b864b9a..0bc8db3 100644 --- a/deploy/k8s/kind-config-full.yaml +++ b/deploy/k8s/kind-config-full.yaml @@ -12,3 +12,4 @@ nodes: - { containerPort: 30007, hostPort: 30007, protocol: TCP } # read-service (direct) - { containerPort: 30090, hostPort: 30090, protocol: TCP } # prometheus (if a NodePort is added) - { containerPort: 30808, hostPort: 30808, protocol: TCP } # meridian-gateway (direct) + - { containerPort: 30300, hostPort: 30300, protocol: TCP } # grafana (logs + metrics UI) diff --git a/deploy/k8s/platform/files/observability/alloy-kubernetes.alloy b/deploy/k8s/platform/files/observability/alloy-kubernetes.alloy new file mode 100644 index 0000000..a5db276 --- /dev/null +++ b/deploy/k8s/platform/files/observability/alloy-kubernetes.alloy @@ -0,0 +1,70 @@ +// Grafana Alloy for the Helm release: tail the release's pods through the +// Kubernetes API (loki.source.kubernetes - no hostPath mounts, no DaemonSet) +// and ship the lines to Loki. +// +// Scoped to the release namespace (NAMESPACE is injected from the pod's own +// metadata). Labels: `service` is the pod's app.kubernetes.io/name (ingestion, +// correlation, ..., meridian-gateway), plus `pod` and `container`. `level` is +// lifted out of JSON log lines when present. + +discovery.kubernetes "pods" { + role = "pod" + namespaces { + names = [sys.env("NAMESPACE")] + } +} + +discovery.relabel "pods" { + targets = discovery.kubernetes.pods.targets + + rule { + source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"] + target_label = "service" + } + // Workloads not labelled by this chart (e.g. the demo app) fall back to + // their `app` label so they are still findable by service. + rule { + source_labels = ["service", "__meta_kubernetes_pod_label_app"] + separator = ";" + regex = ";(.+)" + target_label = "service" + } + rule { + source_labels = ["__meta_kubernetes_namespace"] + target_label = "namespace" + } + rule { + source_labels = ["__meta_kubernetes_pod_name"] + target_label = "pod" + } + rule { + source_labels = ["__meta_kubernetes_pod_container_name"] + target_label = "container" + } +} + +loki.source.kubernetes "pods" { + targets = discovery.relabel.pods.output + forward_to = [loki.process.levels.receiver] +} + +loki.process "levels" { + // Probe and scrape access lines (/health, /ready, /metrics every few seconds + // from every container) were most of the volume and none of the signal. + stage.drop { + expression = "\"GET /(health|ready|metrics)[^\"]*HTTP/1[.]1\" 200" + } + stage.json { + expressions = { level = "level" } + } + stage.labels { + values = { level = "" } + } + forward_to = [loki.write.local.receiver] +} + +loki.write "local" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/deploy/k8s/platform/files/observability/grafana-dashboards.yaml b/deploy/k8s/platform/files/observability/grafana-dashboards.yaml new file mode 100644 index 0000000..ce05c08 --- /dev/null +++ b/deploy/k8s/platform/files/observability/grafana-dashboards.yaml @@ -0,0 +1,10 @@ +apiVersion: 1 + +providers: + - name: intelliops + folder: IntelliOps + type: file + disableDeletion: true + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards diff --git a/deploy/k8s/platform/files/observability/grafana-datasources.yaml b/deploy/k8s/platform/files/observability/grafana-datasources.yaml new file mode 100644 index 0000000..d47904b --- /dev/null +++ b/deploy/k8s/platform/files/observability/grafana-datasources.yaml @@ -0,0 +1,38 @@ +# Grafana datasources, provisioned at startup. The hostnames are the same in +# docker-compose and in the Helm release (service DNS: prometheus, loki, +# postgres), so one file serves both. The Postgres password comes from the +# container environment, never from this file. +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + + - name: Loki + uid: loki + type: loki + access: proxy + url: http://loki:3100 + editable: false + jsonData: + maxLines: 5000 + + # The durable stores: audit trail, approvals, training records, agent runs. + # Read-only use is the intent; point it at a read-only role in anything real. + - name: IntelliOps Postgres + uid: intelliops-postgres + type: grafana-postgresql-datasource + url: postgres:5432 + user: ${INTELLIOPS_PG_USER} + editable: false + jsonData: + database: ${INTELLIOPS_PG_DB} + sslmode: disable + postgresVersion: 1600 + secureJsonData: + password: ${INTELLIOPS_PG_PASSWORD} diff --git a/deploy/k8s/platform/files/observability/intelliops-dashboard.json b/deploy/k8s/platform/files/observability/intelliops-dashboard.json new file mode 100644 index 0000000..f4c2992 --- /dev/null +++ b/deploy/k8s/platform/files/observability/intelliops-dashboard.json @@ -0,0 +1,455 @@ +{ + "uid": "intelliops-overview", + "title": "IntelliOps — Logs, metrics and audit", + "tags": [ + "intelliops" + ], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "10s", + "time": { + "from": "now-30m", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": { + "type": "loki", + "uid": "loki" + }, + "query": { + "label": "service", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1 + }, + "refresh": 2, + "includeAll": true, + "multi": true, + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "sort": 1 + }, + { + "name": "search", + "label": "Search logs", + "type": "textbox", + "query": "", + "current": { + "text": "", + "value": "" + } + } + ] + }, + "annotations": { + "list": [] + }, + "panels": [ + { + "type": "row", + "title": "Logs", + "id": 100, + "gridPos": { + "x": 0, + "y": 0, + "w": 24, + "h": 1 + }, + "collapsed": false + }, + { + "type": "timeseries", + "title": "Log lines per minute, by service", + "description": "", + "id": 1, + "gridPos": { + "x": 0, + "y": 1, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "expr": "sum by (service) (count_over_time({service=~\"$service\"} [1m]))", + "legendFormat": "{{service}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "fillOpacity": 18, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Errors and warnings per minute, by service", + "description": "Counted from the JSON `level` field. Services that log plain text (redis, postgres, uvicorn access lines) carry no level and do not appear here.", + "id": 2, + "gridPos": { + "x": 12, + "y": 1, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "expr": "sum by (service, level) (count_over_time({service=~\"$service\", level=~\"ERROR|WARNING|CRITICAL\"} [1m]))", + "legendFormat": "{{service}} {{level}}", + "queryType": "range" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "fillOpacity": 18, + "stacking": { + "mode": "normal" + } + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "logs", + "title": "Service logs", + "description": "Every container's stdout/stderr. Filter by service above; the search box is a case-insensitive regex over the line.", + "id": 3, + "gridPos": { + "x": 0, + "y": 8, + "w": 24, + "h": 14 + }, + "datasource": { + "type": "loki", + "uid": "loki" + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "loki" + }, + "refId": "A", + "expr": "{service=~\"$service\"} |~ \"(?i)$search\"", + "queryType": "range" + } + ], + "options": { + "showTime": true, + "showLabels": false, + "showCommonLabels": false, + "wrapLogMessage": true, + "prettifyLogMessage": false, + "enableLogDetails": true, + "dedupStrategy": "none", + "sortOrder": "Descending" + } + }, + { + "type": "row", + "title": "Metrics (Prometheus)", + "id": 101, + "gridPos": { + "x": 0, + "y": 22, + "w": 24, + "h": 1 + }, + "collapsed": false + }, + { + "type": "timeseries", + "title": "CPU usage", + "description": "", + "id": 4, + "gridPos": { + "x": 0, + "y": 23, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "cpu_usage", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "fillOpacity": 6 + }, + "unit": "percent" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Latency p99", + "description": "", + "id": 5, + "gridPos": { + "x": 12, + "y": 23, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "latency_p99_ms", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "fillOpacity": 6 + }, + "unit": "ms" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Error rate", + "description": "", + "id": 6, + "gridPos": { + "x": 0, + "y": 30, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "meridian_error_rate", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "fillOpacity": 6 + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "timeseries", + "title": "Service up", + "description": "1 = serving, 0 = down. Scraped from each Meridian service and the demo app.", + "id": 7, + "gridPos": { + "x": 12, + "y": 30, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "refId": "A", + "expr": "service_up", + "legendFormat": "{{service}}" + } + ], + "fieldConfig": { + "defaults": { + "custom": { + "lineWidth": 1, + "fillOpacity": 6 + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + } + }, + { + "type": "row", + "title": "Governance (Postgres)", + "id": 102, + "gridPos": { + "x": 0, + "y": 37, + "w": 24, + "h": 1 + }, + "collapsed": false + }, + { + "type": "table", + "title": "Audit trail", + "description": "The durable audit log (audit_records): every diagnosis, approval, remediation and graduation - the latest 500, whatever the time range. Empty when STORE_BACKEND=file.", + "id": 8, + "gridPos": { + "x": 0, + "y": 38, + "w": 24, + "h": 12 + }, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "intelliops-postgres" + }, + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "intelliops-postgres" + }, + "refId": "A", + "format": "table", + "rawQuery": true, + "editorMode": "code", + "rawSql": "SELECT ts AS \"time\", actor, action, resource, decision, correlation_id FROM audit_records ORDER BY ts DESC LIMIT 500" + } + ], + "options": { + "showHeader": true, + "cellHeight": "sm" + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + } + } + ] +} diff --git a/deploy/k8s/platform/files/observability/loki.yaml b/deploy/k8s/platform/files/observability/loki.yaml new file mode 100644 index 0000000..c49e7bd --- /dev/null +++ b/deploy/k8s/platform/files/observability/loki.yaml @@ -0,0 +1,45 @@ +# Single-binary Loki for the local stack (docker-compose and kind). Filesystem +# storage, no auth, short retention: this is a place to read the services' logs, +# not a production log platform. Shared by both deploy targets - compose +# bind-mounts this file and the Helm chart reads it with .Files.Get. +auth_enabled: false + +server: + http_listen_port: 3100 + log_level: warn + +common: + path_prefix: /loki + replication_factor: 1 + ring: + kvstore: + store: inmemory + storage: + filesystem: + chunks_directory: /loki/chunks + rules_directory: /loki/rules + +schema_config: + configs: + - from: 2024-01-01 + store: tsdb + object_store: filesystem + schema: v13 + index: + prefix: index_ + period: 24h + +limits_config: + retention_period: 72h + allow_structured_metadata: true + # The services poll every few seconds; a burst after a restart is normal. + ingestion_rate_mb: 8 + ingestion_burst_size_mb: 16 + +compactor: + working_directory: /loki/compactor + retention_enabled: true + delete_request_store: filesystem + +analytics: + reporting_enabled: false diff --git a/deploy/k8s/platform/templates/configmap.yaml b/deploy/k8s/platform/templates/configmap.yaml index 62b807d..b84eabe 100644 --- a/deploy/k8s/platform/templates/configmap.yaml +++ b/deploy/k8s/platform/templates/configmap.yaml @@ -12,6 +12,9 @@ metadata: {{- include "intelliops.labels" . | nindent 4 }} data: INTELLIOPS_AUTH_MODE: {{ .Values.env.AUTH_MODE | quote }} + # JSON lines when the Grafana/Loki stack is on, so each record's level is a + # searchable label there; human-readable text otherwise. + INTELLIOPS_LOG_FORMAT: {{ .Values.env.LOG_FORMAT | default (ternary "json" "text" .Values.observability.enabled) | quote }} INTELLIOPS_STORE_BACKEND: {{ .Values.env.STORE_BACKEND | quote }} INTELLIOPS_BUS_BACKEND: {{ .Values.env.BUS_BACKEND | quote }} # Delivery durability (issue #53) + read-model cold start (#58). All default to diff --git a/deploy/k8s/platform/templates/observability.yaml b/deploy/k8s/platform/templates/observability.yaml new file mode 100644 index 0000000..dd8739a --- /dev/null +++ b/deploy/k8s/platform/templates/observability.yaml @@ -0,0 +1,273 @@ +{{/* +observability.yaml +Logs + metrics UI for the release, gated on observability.enabled: + + - Loki stores logs (single binary, emptyDir - logs are for looking at now, + not an archive; they are lost when the pod is rescheduled). + - Alloy tails every pod in the release namespace through the Kubernetes API + (loki.source.kubernetes) and ships the lines to Loki. Namespace-scoped + Role, no hostPath, no DaemonSet. + - Grafana serves one provisioned dashboard: service logs, the Prometheus + metrics, and the Postgres audit trail. NodePort observability.grafana.nodePort. + +All config files live in files/observability/ and are shared with docker-compose. +*/}} +{{- if .Values.observability.enabled }} +{{- $o := .Values.observability }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: observability-config + labels: + {{- include "intelliops.labels" . | nindent 4 }} +data: + loki.yaml: | +{{ .Files.Get "files/observability/loki.yaml" | indent 4 }} + config.alloy: | +{{ .Files.Get "files/observability/alloy-kubernetes.alloy" | indent 4 }} + datasources.yaml: | +{{ .Files.Get "files/observability/grafana-datasources.yaml" | indent 4 }} + dashboards.yaml: | +{{ .Files.Get "files/observability/grafana-dashboards.yaml" | indent 4 }} + intelliops-dashboard.json: | +{{ .Files.Get "files/observability/intelliops-dashboard.json" | indent 4 }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: loki + labels: + {{- include "intelliops.labels" . | nindent 4 }} + {{- include "intelliops.selectorLabels" (dict "root" . "name" "loki") | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "loki") | nindent 6 }} + template: + metadata: + labels: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "loki") | nindent 8 }} + annotations: + checksum/config: {{ .Files.Get "files/observability/loki.yaml" | sha256sum }} + spec: + securityContext: + runAsUser: 10001 + runAsGroup: 10001 + fsGroup: 10001 + containers: + - name: loki + image: {{ $o.loki.image | quote }} + args: ["-config.file=/etc/loki/loki.yaml"] + ports: + - containerPort: 3100 + readinessProbe: + httpGet: + path: /ready + port: 3100 + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/loki/loki.yaml + subPath: loki.yaml + - name: data + mountPath: /loki + volumes: + - name: config + configMap: + name: observability-config + - name: data + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: loki + labels: + {{- include "intelliops.labels" . | nindent 4 }} +spec: + selector: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "loki") | nindent 4 }} + ports: + - port: 3100 + targetPort: 3100 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: intelliops-alloy + labels: + {{- include "intelliops.labels" . | nindent 4 }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: intelliops-alloy + labels: + {{- include "intelliops.labels" . | nindent 4 }} +rules: + # Discover the release's pods and stream their logs. Read-only, this namespace only. + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: intelliops-alloy + labels: + {{- include "intelliops.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: intelliops-alloy +subjects: + - kind: ServiceAccount + name: intelliops-alloy + namespace: {{ .Release.Namespace }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: alloy + labels: + {{- include "intelliops.labels" . | nindent 4 }} + {{- include "intelliops.selectorLabels" (dict "root" . "name" "alloy") | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "alloy") | nindent 6 }} + template: + metadata: + labels: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "alloy") | nindent 8 }} + annotations: + checksum/config: {{ .Files.Get "files/observability/alloy-kubernetes.alloy" | sha256sum }} + spec: + serviceAccountName: intelliops-alloy + containers: + - name: alloy + image: {{ $o.alloy.image | quote }} + args: + - run + - --server.http.listen-addr=0.0.0.0:12345 + - --storage.path=/var/lib/alloy/data + - /etc/alloy/config.alloy + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + ports: + - containerPort: 12345 + volumeMounts: + - name: config + mountPath: /etc/alloy/config.alloy + subPath: config.alloy + - name: data + mountPath: /var/lib/alloy/data + volumes: + - name: config + configMap: + name: observability-config + - name: data + emptyDir: {} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + labels: + {{- include "intelliops.labels" . | nindent 4 }} + {{- include "intelliops.selectorLabels" (dict "root" . "name" "grafana") | nindent 4 }} +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "grafana") | nindent 6 }} + template: + metadata: + labels: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "grafana") | nindent 8 }} + annotations: + checksum/dashboard: {{ .Files.Get "files/observability/intelliops-dashboard.json" | sha256sum }} + checksum/datasources: {{ .Files.Get "files/observability/grafana-datasources.yaml" | sha256sum }} + spec: + containers: + - name: grafana + image: {{ $o.grafana.image | quote }} + env: + # Anyone who can reach the NodePort can VIEW; editing needs the admin + # login. Set observability.grafana.anonymous=false for anything shared. + - name: GF_AUTH_ANONYMOUS_ENABLED + value: {{ $o.grafana.anonymous | quote }} + - name: GF_AUTH_ANONYMOUS_ORG_ROLE + value: Viewer + - name: GF_SECURITY_ADMIN_PASSWORD + value: {{ $o.grafana.adminPassword | quote }} + - name: GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH + value: /var/lib/grafana/dashboards/intelliops-dashboard.json + - name: GF_ANALYTICS_REPORTING_ENABLED + value: "false" + - name: GF_ANALYTICS_CHECK_FOR_UPDATES + value: "false" + # The provisioned Postgres datasource reads these (audit trail panel). + - name: INTELLIOPS_PG_USER + valueFrom: + secretKeyRef: { name: postgres-credentials, key: POSTGRES_USER } + - name: INTELLIOPS_PG_PASSWORD + valueFrom: + secretKeyRef: { name: postgres-credentials, key: POSTGRES_PASSWORD } + - name: INTELLIOPS_PG_DB + valueFrom: + secretKeyRef: { name: postgres-credentials, key: POSTGRES_DB } + ports: + - containerPort: 3000 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: config + mountPath: /etc/grafana/provisioning/datasources/intelliops.yaml + subPath: datasources.yaml + - name: config + mountPath: /etc/grafana/provisioning/dashboards/intelliops.yaml + subPath: dashboards.yaml + - name: config + mountPath: /var/lib/grafana/dashboards/intelliops-dashboard.json + subPath: intelliops-dashboard.json + volumes: + - name: config + configMap: + name: observability-config +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana + labels: + {{- include "intelliops.labels" . | nindent 4 }} +spec: + {{- if $o.grafana.nodePort }} + type: NodePort + {{- end }} + selector: + {{- include "intelliops.selectorLabels" (dict "root" . "name" "grafana") | nindent 4 }} + ports: + - port: 3000 + targetPort: 3000 + {{- if $o.grafana.nodePort }} + nodePort: {{ $o.grafana.nodePort }} + {{- end }} +{{- end }} diff --git a/deploy/k8s/platform/values.yaml b/deploy/k8s/platform/values.yaml index 698aa1a..205c48e 100644 --- a/deploy/k8s/platform/values.yaml +++ b/deploy/k8s/platform/values.yaml @@ -151,3 +151,22 @@ postgres: # Durable storage size for the StatefulSet's PVC (issue #55). The DB survives # a pod reschedule now; size to taste (audit/approval/training/trace rows). storage: 1Gi + +# Logs + metrics UI (Grafana + Loki + Alloy). Grafana is served on a NodePort: +# kind: http://localhost:30300 (deploy/k8s/kind-config-full.yaml maps it) +# other: kubectl port-forward svc/grafana 3000:3000 -> http://localhost:3000 +# One dashboard: every pod's logs (via Loki), the Prometheus metrics, and the +# Postgres audit trail. Enabling it also switches the services to JSON logs so +# levels are searchable. +observability: + enabled: true + grafana: + image: grafana/grafana:11.3.0 + nodePort: 30300 + # Anonymous visitors get read-only Viewer access. Set false for anything shared. + anonymous: true + adminPassword: admin # change with --set-string observability.grafana.adminPassword=... + loki: + image: grafana/loki:3.2.1 + alloy: + image: grafana/alloy:v1.4.3 diff --git a/deploy/observability/alloy-docker.alloy b/deploy/observability/alloy-docker.alloy new file mode 100644 index 0000000..1833319 --- /dev/null +++ b/deploy/observability/alloy-docker.alloy @@ -0,0 +1,58 @@ +// Grafana Alloy for docker-compose: tail every container of this compose +// project through the Docker socket and ship the lines to Loki. +// +// Labels: `service` is the compose service name (ingestion, correlation, ..., +// meridian-gateway), `container` the container name. The IntelliOps services +// log JSON in compose (INTELLIOPS_LOG_FORMAT=json), so `level` is lifted out of +// each line into a label; plain-text lines (redis, postgres, uvicorn access +// logs) pass through untouched. + +discovery.docker "containers" { + host = "unix:///var/run/docker.sock" + filter { + name = "label" + values = ["com.docker.compose.project=intelliops"] + } +} + +discovery.relabel "containers" { + targets = [] + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "service" + } + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "container" + } +} + +loki.source.docker "containers" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.containers.targets + relabel_rules = discovery.relabel.containers.rules + forward_to = [loki.process.levels.receiver] +} + +loki.process "levels" { + // Probe and scrape access lines (/health, /ready, /metrics every few seconds + // from every container) were most of the volume and none of the signal. + stage.drop { + expression = "\"GET /(health|ready|metrics)[^\"]*HTTP/1[.]1\" 200" + } + stage.json { + expressions = { level = "level" } + } + stage.labels { + values = { level = "" } + } + forward_to = [loki.write.local.receiver] +} + +loki.write "local" { + endpoint { + url = "http://loki:3100/loki/api/v1/push" + } +} diff --git a/docs/DEMO.md b/docs/DEMO.md index bc92ea6..88159cd 100644 --- a/docs/DEMO.md +++ b/docs/DEMO.md @@ -190,6 +190,7 @@ When it finishes: | Read service | http://localhost:30007 | | Meridian gateway (inject faults here) | http://localhost:30808 | | Prometheus | http://localhost:30090 | +| Grafana (logs + metrics + audit) | http://localhost:30300 | Confirm it is genuinely up — the banner is not proof on its own: diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md index 74a3ebb..9a79b08 100644 --- a/docs/OBSERVABILITY.md +++ b/docs/OBSERVABILITY.md @@ -166,3 +166,53 @@ For a real cluster deploy, map the two probes to their matching Kubernetes probe outage; `readinessProbe -> /ready` pulls a pod out of the Service's endpoints while a dependency is unreachable, without killing it, and lets it back in when the dependency recovers. See [deploy/k8s/README.md](../deploy/k8s/README.md). + +--- + +## 4. Logs and metrics UI (Grafana) + +Every service's logs, the Prometheus metrics and the Postgres audit trail are +viewable in one place, at their own URL: + +| Deploy | Grafana | Loki API | Alloy pipeline UI | +|---|---|---|---| +| docker-compose | | | | +| Helm on kind | | in-cluster `loki:3100` | in-cluster `alloy:12345` | + +It opens on the provisioned **IntelliOps — Logs, metrics and audit** dashboard: + +- **Service logs.** Every container's stdout/stderr, newest first. Filter by + service, and use the search box, a case-insensitive regex over the line. For + anything more, the *Explore* view accepts raw LogQL, for example + `{service="action"} |= "remediation"` or + `{service=~"rca|governance"} | json | level="WARNING"`. +- **Log volume** and **errors/warnings per minute**, by service. Levels come from + the JSON `level` field, so plain-text lines (redis, postgres, uvicorn access + logs) carry no level. +- **Metrics.** CPU, p99 latency, error rate and `service_up` straight from + Prometheus. Explore can query any metric the stack scrapes. +- **Audit trail.** The latest 500 `audit_records` rows from Postgres (every + diagnosis, approval, remediation and graduation). Empty when `STORE_BACKEND=file`. + +**How it works.** Grafana Alloy collects the logs: through the Docker socket in +compose, and through the Kubernetes API (`loki.source.kubernetes`, a +namespace-scoped read-only Role, no hostPath) in the chart. It drops the +`/health`, `/ready` and `/metrics` access lines, which were most of the volume, +lifts `level` out of JSON lines, and pushes everything to a single-binary Loki. Loki keeps +72h on local disk (an emptyDir in the chart, so a rescheduled pod starts empty). +Enabling the stack in the chart also switches the services to JSON logs +(`INTELLIOPS_LOG_FORMAT`). + +**One copy of the config.** Everything lives in +`deploy/k8s/platform/files/observability/`, where compose bind-mounts it and the +chart reads it with `.Files.Get`. Only the Alloy discovery differs +(`deploy/observability/alloy-docker.alloy` for compose). The dashboard is +generated by `scripts/make-grafana-dashboard.py`; edit that and re-run it rather +than hand-editing the JSON. + +**Access.** Anonymous visitors get read-only *Viewer* access, a local-demo +convenience. Editing needs the admin login (`admin`/`admin` unless you set +`GRAFANA_ADMIN_PASSWORD` in compose or `observability.grafana.adminPassword` in +the chart). For anything shared, turn anonymous access off (`GRAFANA_ANONYMOUS=false`, +or `observability.grafana.anonymous=false`) and point the Postgres datasource at a +read-only role. diff --git a/scripts/make-grafana-dashboard.py b/scripts/make-grafana-dashboard.py new file mode 100644 index 0000000..8e2c372 --- /dev/null +++ b/scripts/make-grafana-dashboard.py @@ -0,0 +1,234 @@ +"""Generate the provisioned Grafana dashboard (logs + metrics + audit trail). + +Run from the repo root: + + uv run python scripts/make-grafana-dashboard.py + +Writes deploy/k8s/platform/files/observability/intelliops-dashboard.json, which +both docker-compose (bind mount) and the Helm chart (.Files.Get) provision. +Generated rather than hand-edited so the JSON stays valid and reviewable here. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +OUT = Path("deploy/k8s/platform/files/observability/intelliops-dashboard.json") + +LOKI = {"type": "loki", "uid": "loki"} +PROM = {"type": "prometheus", "uid": "prometheus"} +PG = {"type": "grafana-postgresql-datasource", "uid": "intelliops-postgres"} + +SVC = '{service=~"$service"}' + + +def _grid(x, y, w, h): + return {"x": x, "y": y, "w": w, "h": h} + + +def row(title, y, pid): + return { + "type": "row", + "title": title, + "id": pid, + "gridPos": _grid(0, y, 24, 1), + "collapsed": False, + } + + +def timeseries(title, pid, grid, ds, targets, unit=None, stack=False, desc=""): + defaults = {"custom": {"lineWidth": 1, "fillOpacity": 18 if stack else 6}} + if stack: + defaults["custom"]["stacking"] = {"mode": "normal"} + if unit: + defaults["unit"] = unit + return { + "type": "timeseries", + "title": title, + "description": desc, + "id": pid, + "gridPos": grid, + "datasource": ds, + "targets": targets, + "fieldConfig": {"defaults": defaults, "overrides": []}, + "options": { + "legend": {"displayMode": "list", "placement": "bottom"}, + "tooltip": {"mode": "multi"}, + }, + } + + +def loki_expr(expr, legend="{{service}}", ref="A"): + return { + "datasource": LOKI, + "refId": ref, + "expr": expr, + "legendFormat": legend, + "queryType": "range", + } + + +def prom_expr(expr, legend="{{service}}", ref="A"): + return {"datasource": PROM, "refId": ref, "expr": expr, "legendFormat": legend} + + +panels = [ + row("Logs", 0, 100), + timeseries( + "Log lines per minute, by service", + 1, + _grid(0, 1, 12, 7), + LOKI, + [loki_expr(f"sum by (service) (count_over_time({SVC} [1m]))")], + stack=True, + ), + timeseries( + "Errors and warnings per minute, by service", + 2, + _grid(12, 1, 12, 7), + LOKI, + [ + loki_expr( + 'sum by (service, level) (count_over_time({service=~"$service", ' + 'level=~"ERROR|WARNING|CRITICAL"} [1m]))', + legend="{{service}} {{level}}", + ) + ], + stack=True, + desc="Counted from the JSON `level` field. Services that log plain text (redis, " + "postgres, uvicorn access lines) carry no level and do not appear here.", + ), + { + "type": "logs", + "title": "Service logs", + "description": "Every container's stdout/stderr. Filter by service above; the search box " + "is a case-insensitive regex over the line.", + "id": 3, + "gridPos": _grid(0, 8, 24, 14), + "datasource": LOKI, + "targets": [ + { + "datasource": LOKI, + "refId": "A", + "expr": SVC + ' |~ "(?i)$search"', + "queryType": "range", + } + ], + "options": { + "showTime": True, + "showLabels": False, + "showCommonLabels": False, + "wrapLogMessage": True, + "prettifyLogMessage": False, + "enableLogDetails": True, + "dedupStrategy": "none", + "sortOrder": "Descending", + }, + }, + row("Metrics (Prometheus)", 22, 101), + timeseries( + "CPU usage", + 4, + _grid(0, 23, 12, 7), + PROM, + [prom_expr("cpu_usage")], + unit="percent", + ), + timeseries( + "Latency p99", + 5, + _grid(12, 23, 12, 7), + PROM, + [prom_expr("latency_p99_ms")], + unit="ms", + ), + timeseries( + "Error rate", + 6, + _grid(0, 30, 12, 7), + PROM, + [prom_expr("meridian_error_rate")], + unit="percentunit", + ), + timeseries( + "Service up", + 7, + _grid(12, 30, 12, 7), + PROM, + [prom_expr("service_up")], + desc="1 = serving, 0 = down. Scraped from each Meridian service and the demo app.", + ), + row("Governance (Postgres)", 37, 102), + { + "type": "table", + "title": "Audit trail", + "description": "The durable audit log (audit_records): every diagnosis, approval, " + "remediation and graduation - the latest 500, whatever the time range. " + "Empty when STORE_BACKEND=file.", + "id": 8, + "gridPos": _grid(0, 38, 24, 12), + "datasource": PG, + "targets": [ + { + "datasource": PG, + "refId": "A", + "format": "table", + "rawQuery": True, + "editorMode": "code", + # Deliberately not bound to the time picker: an audit log is read + # as "what happened most recently", however long ago that was. + "rawSql": 'SELECT ts AS "time", actor, action, resource, decision, ' + "correlation_id FROM audit_records ORDER BY ts DESC LIMIT 500", + } + ], + "options": {"showHeader": True, "cellHeight": "sm"}, + "fieldConfig": {"defaults": {}, "overrides": []}, + }, +] + +dashboard = { + "uid": "intelliops-overview", + "title": "IntelliOps — Logs, metrics and audit", + "tags": ["intelliops"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": True, + "refresh": "10s", + "time": {"from": "now-30m", "to": "now"}, + "templating": { + "list": [ + { + "name": "service", + "label": "Service", + "type": "query", + "datasource": LOKI, + "query": { + "label": "service", + "refId": "LokiVariableQueryEditor-VariableQuery", + "stream": "", + "type": 1, + }, + "refresh": 2, + "includeAll": True, + "multi": True, + "allValue": ".+", + "current": {"selected": True, "text": ["All"], "value": ["$__all"]}, + "sort": 1, + }, + { + "name": "search", + "label": "Search logs", + "type": "textbox", + "query": "", + "current": {"text": "", "value": ""}, + }, + ] + }, + "annotations": {"list": []}, + "panels": panels, +} + +OUT.write_text(json.dumps(dashboard, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") +print(f"wrote {OUT}") diff --git a/tests/test_observability_config.py b/tests/test_observability_config.py new file mode 100644 index 0000000..54f028f --- /dev/null +++ b/tests/test_observability_config.py @@ -0,0 +1,55 @@ +"""The Grafana provisioning files must agree with each other. + +A dashboard panel that names a datasource uid nobody provisions renders as +"datasource not found", and nothing else would catch it before someone opens +the page. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +OBS = Path("deploy/k8s/platform/files/observability") + + +def _uids(node) -> set[str]: + found: set[str] = set() + if isinstance(node, dict): + ds = node.get("datasource") + if isinstance(ds, dict) and "uid" in ds: + found.add(ds["uid"]) + for v in node.values(): + found |= _uids(v) + elif isinstance(node, list): + for v in node: + found |= _uids(v) + return found + + +def test_dashboard_only_uses_provisioned_datasources(): + dashboard = json.loads((OBS / "intelliops-dashboard.json").read_text(encoding="utf-8")) + provisioned = { + d["uid"] + for d in yaml.safe_load((OBS / "grafana-datasources.yaml").read_text(encoding="utf-8"))[ + "datasources" + ] + } + used = _uids(dashboard) + assert used, "dashboard references no datasources" + assert used <= provisioned, used - provisioned + + +def test_every_file_the_chart_reads_exists(): + template = Path("deploy/k8s/platform/templates/observability.yaml").read_text(encoding="utf-8") + for name in ( + "loki.yaml", + "alloy-kubernetes.alloy", + "grafana-datasources.yaml", + "grafana-dashboards.yaml", + "intelliops-dashboard.json", + ): + assert f"files/observability/{name}" in template + assert (OBS / name).is_file(), name