Grafana + Loki: every service's logs, the metrics and the audit trail at one URL

.github/workflows/ci.yml

@@ -118,6 +118,26 @@ jobs:
               sleep 3
             done
           done
+          # Logs + metrics UI: Loki ready, Grafana up with all three datasources
+          # healthy, and Alloy actually shipping container logs into Loki.
+          for url in http://localhost:3100/ready http://localhost:3000/api/health; do
+            for attempt in $(seq 1 30); do
+              curl -sf "$url" >/dev/null && { echo " ✓ $url"; break; }
+              [ "$attempt" -eq 30 ] && { echo " ✗ $url never 200"; exit 1; }
+              sleep 3
+            done
+          done
+          for uid in prometheus loki intelliops-postgres; do
+            curl -sf -u admin:admin "http://localhost:3000/api/datasources/uid/$uid/health" | grep -q '"status":"OK"' \
+              && echo " ✓ grafana datasource $uid" \
+              || { echo " ✗ grafana datasource $uid unhealthy"; exit 1; }
+          done
+          for attempt in $(seq 1 30); do
+            curl -sf "http://localhost:3100/loki/api/v1/label/service/values" | grep -q '"governance"' \
+              && { echo " ✓ loki is receiving service logs"; break; }
+            [ "$attempt" -eq 30 ] && { echo " ✗ no service logs reached loki"; exit 1; }
+            sleep 3
+          done
       - if: failure()
         run: docker compose -f deploy/docker-compose.yml logs
       - if: always()

README.md

@@ -114,6 +114,11 @@ npm install && npm run dev
 # open http://localhost:5173
 ```
 
+**Logs, metrics and the audit trail in one place:** Grafana at
+<http://localhost:3000> (on kind: <http://localhost:30300>). Every container's
+logs (via Loki), the Prometheus metrics, and the Postgres audit trail on one
+dashboard, viewable without logging in. See [docs/OBSERVABILITY.md](docs/OBSERVABILITY.md#4-logs-and-metrics-ui-grafana).
+
 **Drive a real incident end to end:**
 
 ```bash

deploy/docker-compose.yml

@@ -261,6 +261,62 @@ services:
       demo-app:
         condition: service_healthy
 
+  # ---- Logs & metrics UI: Grafana at http://localhost:3000 ----------------
+  # Loki stores every container's logs (shipped by Alloy through the Docker
+  # socket); Grafana puts them next to the Prometheus metrics and the Postgres
+  # audit trail on one provisioned dashboard. Config lives in
+  # deploy/k8s/platform/files/observability/ so the Helm chart ships the same.
+  loki:
+    image: grafana/loki:3.2.1
+    command: ["-config.file=/etc/loki/loki.yaml"]
+    volumes:
+      - ./k8s/platform/files/observability/loki.yaml:/etc/loki/loki.yaml:ro
+      - loki-data:/loki
+    ports:
+      - "3100:3100"
+
+  alloy:
+    image: grafana/alloy:v1.4.3
+    command:
+      - run
+      - --server.http.listen-addr=0.0.0.0:12345
+      - --storage.path=/var/lib/alloy/data
+      - /etc/alloy/config.alloy
+    volumes:
+      - ./observability/alloy-docker.alloy:/etc/alloy/config.alloy:ro
+      # Read-only access to the Docker API, to discover and tail containers.
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    ports:
+      - "12345:12345"   # Alloy's own UI: pipeline graph + per-target status
+    depends_on:
+      - loki
+
+  grafana:
+    image: grafana/grafana:11.3.0
+    environment:
+      # Anyone who can reach localhost:3000 can VIEW (dashboards, Explore, logs).
+      # Editing needs the admin login. Turn anonymous off for anything shared.
+      GF_AUTH_ANONYMOUS_ENABLED: "${GRAFANA_ANONYMOUS:-true}"
+      GF_AUTH_ANONYMOUS_ORG_ROLE: Viewer
+      GF_SECURITY_ADMIN_PASSWORD: "${GRAFANA_ADMIN_PASSWORD:-admin}"
+      GF_DASHBOARDS_DEFAULT_HOME_DASHBOARD_PATH: /var/lib/grafana/dashboards/intelliops-dashboard.json
+      GF_ANALYTICS_REPORTING_ENABLED: "false"
+      GF_ANALYTICS_CHECK_FOR_UPDATES: "false"
+      # Read by the provisioned Postgres datasource (audit trail panel).
+      INTELLIOPS_PG_USER: intelliops
+      INTELLIOPS_PG_PASSWORD: intelliops
+      INTELLIOPS_PG_DB: intelliops
+    volumes:
+      - ./k8s/platform/files/observability/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/intelliops.yaml:ro
+      - ./k8s/platform/files/observability/grafana-dashboards.yaml:/etc/grafana/provisioning/dashboards/intelliops.yaml:ro
+      - ./k8s/platform/files/observability/intelliops-dashboard.json:/var/lib/grafana/dashboards/intelliops-dashboard.json:ro
+    ports:
+      - "3000:3000"
+    depends_on:
+      - loki
+      - prometheus
+      - postgres
+
   read:
     <<: *service
     environment:
@@ -392,5 +448,6 @@ services:
 
 volumes:
   postgres-data:
+  loki-data:
   # Shared between rca and (Task 3) meridian-gateway at rca_context_path.
   rca-context:

deploy/k8s/README.md

@@ -40,6 +40,10 @@ the `console`), `helm install`s with `values-live.yaml` (+ your key via
   dashboard shows **real** situations, embedding-fit confidences, and per-metric
   verification.
 - **Read service:** <http://localhost:30007>.
+- **Grafana (logs + metrics + audit trail):** <http://localhost:30300>, requiring
+  the 30300 port mapping in `kind-config-full.yaml` (recreate an existing cluster to
+  pick it up, or `kubectl port-forward svc/grafana 3000:3000`). Off with
+  `--set observability.enabled=false`.
 
 **Safe by default.** `helm install` *without* the live overlay (or
 `SAFE=1 ./scripts/kind-up-full.sh`) keeps the safe posture — `dry_run` / selector

deploy/k8s/kind-config-full.yaml

@@ -12,3 +12,4 @@ nodes:
       - { containerPort: 30007, hostPort: 30007, protocol: TCP }   # read-service (direct)
       - { containerPort: 30090, hostPort: 30090, protocol: TCP }   # prometheus (if a NodePort is added)
       - { containerPort: 30808, hostPort: 30808, protocol: TCP }   # meridian-gateway (direct)
+      - { containerPort: 30300, hostPort: 30300, protocol: TCP }   # grafana (logs + metrics UI)

deploy/k8s/platform/files/observability/alloy-kubernetes.alloy

@@ -0,0 +1,70 @@
+// Grafana Alloy for the Helm release: tail the release's pods through the
+// Kubernetes API (loki.source.kubernetes - no hostPath mounts, no DaemonSet)
+// and ship the lines to Loki.
+//
+// Scoped to the release namespace (NAMESPACE is injected from the pod's own
+// metadata). Labels: `service` is the pod's app.kubernetes.io/name (ingestion,
+// correlation, ..., meridian-gateway), plus `pod` and `container`. `level` is
+// lifted out of JSON log lines when present.
+
+discovery.kubernetes "pods" {
+  role = "pod"
+  namespaces {
+    names = [sys.env("NAMESPACE")]
+  }
+}
+
+discovery.relabel "pods" {
+  targets = discovery.kubernetes.pods.targets
+
+  rule {
+    source_labels = ["__meta_kubernetes_pod_label_app_kubernetes_io_name"]
+    target_label  = "service"
+  }
+  // Workloads not labelled by this chart (e.g. the demo app) fall back to
+  // their `app` label so they are still findable by service.
+  rule {
+    source_labels = ["service", "__meta_kubernetes_pod_label_app"]
+    separator     = ";"
+    regex         = ";(.+)"
+    target_label  = "service"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_namespace"]
+    target_label  = "namespace"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_name"]
+    target_label  = "pod"
+  }
+  rule {
+    source_labels = ["__meta_kubernetes_pod_container_name"]
+    target_label  = "container"
+  }
+}
+
+loki.source.kubernetes "pods" {
+  targets    = discovery.relabel.pods.output
+  forward_to = [loki.process.levels.receiver]
+}
+
+loki.process "levels" {
+  // Probe and scrape access lines (/health, /ready, /metrics every few seconds
+  // from every container) were most of the volume and none of the signal.
+  stage.drop {
+    expression = "\"GET /(health|ready|metrics)[^\"]*HTTP/1[.]1\" 200"
+  }
+  stage.json {
+    expressions = { level = "level" }
+  }
+  stage.labels {
+    values = { level = "" }
+  }
+  forward_to = [loki.write.local.receiver]
+}
+
+loki.write "local" {
+  endpoint {
+    url = "http://loki:3100/loki/api/v1/push"
+  }
+}

deploy/k8s/platform/files/observability/grafana-dashboards.yaml

@@ -0,0 +1,10 @@
+apiVersion: 1
+
+providers:
+  - name: intelliops
+    folder: IntelliOps
+    type: file
+    disableDeletion: true
+    allowUiUpdates: true
+    options:
+      path: /var/lib/grafana/dashboards

deploy/k8s/platform/files/observability/grafana-datasources.yaml

@@ -0,0 +1,38 @@
+# Grafana datasources, provisioned at startup. The hostnames are the same in
+# docker-compose and in the Helm release (service DNS: prometheus, loki,
+# postgres), so one file serves both. The Postgres password comes from the
+# container environment, never from this file.
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: false
+
+  - name: Loki
+    uid: loki
+    type: loki
+    access: proxy
+    url: http://loki:3100
+    editable: false
+    jsonData:
+      maxLines: 5000
+
+  # The durable stores: audit trail, approvals, training records, agent runs.
+  # Read-only use is the intent; point it at a read-only role in anything real.
+  - name: IntelliOps Postgres
+    uid: intelliops-postgres
+    type: grafana-postgresql-datasource
+    url: postgres:5432
+    user: ${INTELLIOPS_PG_USER}
+    editable: false
+    jsonData:
+      database: ${INTELLIOPS_PG_DB}
+      sslmode: disable
+      postgresVersion: 1600
+    secureJsonData:
+      password: ${INTELLIOPS_PG_PASSWORD}