diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b8cd917..dc12d5e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,3 +28,5 @@ jobs: run: make tf-validate - name: Shell syntax run: find scripts -name '*.sh' -print0 | xargs -0 -n1 bash -n + - name: M10 monitoring configuration + run: make observability-static diff --git a/Makefile b/Makefile index 33830ad..4d04e7f 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,12 @@ CHAOS_SCENARIO ?= primary-dcs-isolation FILE ?= testdata/migrations/safe/staged.sql PGSAFE ?= ./bin/pgsafe RESTORE_MODE ?= latest +ALERT_TEST ?= replica-loss TERRAFORM ?= terraform TF_DIR := terraform/environments/$(PROFILE) VALID_PROFILES := full colocated -.PHONY: tf-fmt tf-validate cluster-up cluster-down check-profile pg-configure pg-verify pg-failover pg-clean etcd-configure etcd-verify etcd-quorum-test etcd-clean patroni-configure patroni-verify haproxy-configure haproxy-verify patroni-failover-test failure-baseline failure-scenario failure-matrix failure-report failure-clean durability-set durability-verify durability-latency durability-failure-test durability-standby-loss durability-strict-test durability-matrix durability-report durability-clean chaos-baseline chaos-scenario chaos-matrix chaos-report chaos-clean pgsafe-build pgsafe-test pgsafe-check pgsafe-fixtures migration-baseline migration-runtime-test migration-clean backup-configure backup-check backup-full backup-info backup-archive-verify restore-latest restore-verify restore-clean pitr-test backup-acceptance backup-clean +.PHONY: tf-fmt tf-validate cluster-up cluster-down check-profile pg-configure pg-verify pg-failover pg-clean etcd-configure etcd-verify etcd-quorum-test etcd-clean patroni-configure patroni-verify haproxy-configure haproxy-verify patroni-failover-test failure-baseline failure-scenario failure-matrix failure-report failure-clean durability-set durability-verify durability-latency durability-failure-test durability-standby-loss durability-strict-test durability-matrix durability-report durability-clean chaos-baseline chaos-scenario chaos-matrix chaos-report chaos-clean pgsafe-build pgsafe-test pgsafe-check pgsafe-fixtures migration-baseline migration-runtime-test migration-clean backup-configure backup-check backup-full backup-info backup-archive-verify restore-latest restore-verify restore-clean pitr-test backup-acceptance backup-clean observability-configure observability-check observability-verify observability-alert-test observability-clean observability-static tf-fmt: $(TERRAFORM) fmt -recursive terraform @@ -192,3 +193,22 @@ backup-acceptance: check-profile backup-clean: check-profile ./scripts/backup/clean.sh $(PROFILE) all + +observability-configure: check-profile + ./scripts/observability/configure.sh $(PROFILE) + +observability-check: check-profile + ./scripts/observability/check.sh $(PROFILE) + +observability-verify: check-profile + ./scripts/observability/verify.sh $(PROFILE) + +# Explicitly destructive and self-restoring; ALERT_TEST=replica-loss|etcd-member-loss|haproxy-loss|replication-lag. +observability-alert-test: check-profile + ./scripts/observability/alert-test.sh $(PROFILE) $(ALERT_TEST) + +observability-clean: check-profile + ./scripts/observability/clean.sh $(PROFILE) + +observability-static: + ./scripts/observability/static-check.sh diff --git a/README.md b/README.md index 3421686..efa384c 100644 --- a/README.md +++ b/README.md @@ -1,168 +1,136 @@ -# pgsentry +# PgSentry -pgsentry is a PostgreSQL reliability engineering lab that measures and verifies database behavior under failover, partitions, unsafe migrations, and recovery — rather than merely demonstrating that an HA stack can be deployed. +PgSentry is a PostgreSQL reliability engineering lab that verifies high availability, failure behavior, migration safety, durability, historical recovery, and operational alerting with reproducible evidence. -It is an evidence-producing reliability lab, not a generic PostgreSQL deployment, Kubernetes demo, Patroni showcase, Terraform showcase, unrelated DevOps toolkit, or consensus implementation. +## Why PgSentry -## Status and roadmap +Deploying an HA stack does not show what clients experience during failure, whether a partition creates two writers, whether acknowledged data survives, whether a migration blocks traffic, or whether a backup can restore history. PgSentry turns those questions into bounded experiments with explicit safety assertions and machine-readable evidence. -M1 provides VM infrastructure, M2 demonstrates manual PostgreSQL replication, M3 proves independent etcd quorum, M4 integrates Patroni-managed PostgreSQL with HAProxy, and M5 adds repeatable client-visible failure experiments. M6 compares asynchronous and synchronous durability policies. +## Architecture -| Milestone | Capability | Status | -| --- | --- | --- | -| M1 | Terraform infrastructure | Implemented and verified | -| M2 | Manual PostgreSQL streaming replication | Implemented; canonical runtime verified | -| M3 | etcd quorum | Implemented and verified | -| M4 | Patroni HA and HAProxy routing | Implemented and verified | -| M5 | Automated failure and durability harness | Implemented and verified | -| M6 | Synchronous durability experiments | Implemented and verified | -| M7 | Network-partition and DCS chaos | Complete | -| M8 | pgsafe migration safety CLI | Complete | -| M9 | Backup, WAL, and PITR verification | Complete | -| M10 | Observability, runbooks, and portfolio polish | Planned | +```text +clients -> HAProxy :5000 -> Patroni PostgreSQL pg-01/02/03 + | physical WAL replication + v + etcd-01/02/03 -## Topologies +all seven VMs + PostgreSQL + Patroni + etcd + HAProxy + pgBackRest + | + v + Prometheus -> Alertmanager -> runbooks + | + v + Grafana -The canonical `full` profile creates seven VMs: three PostgreSQL-role nodes, three etcd-role nodes, and a dedicated control node. Later M5–M9 measurements published as project evidence must use this topology. +PostgreSQL -> full backup + archived WAL -> control-01 repository + | + v + isolated restore / named PITR +``` -The `colocated` profile creates three development VMs. Each can later host PostgreSQL, Patroni, and etcd, but results from this resource-constrained topology are not canonical evidence. +Prometheus observes the system; it never participates in leader election or routing. `control-01` consolidates routing, backup storage, and monitoring to fit a laptop lab. That is not a production colocation recommendation. See [the complete architecture](docs/architecture.md). -See [the architecture](docs/architecture.md) for the separation and network rationale. +## What it demonstrates -## Prerequisites +| Capability | Verified behavior | Evidence | +| --- | --- | --- | +| Infrastructure | Seven deterministic libvirt VMs provision and destroy | M1 | +| Replication | One primary and two read-only streaming replicas | M2 | +| Consensus | Three-member mutual-TLS etcd quorum and bounded failures | M3 | +| Database HA | Patroni promotion and HAProxy role-aware routing | M4 | +| Failure measurement | Client interruption, ambiguity, retained acknowledgements | M5 | +| Durability | Async, sync, and sync-strict policy tradeoffs | M6 | +| Partition safety | Bounded DCS/WAL partitions with writer-count assertions | M7 | +| Migration safety | PostgreSQL 16 AST analysis and real lock demonstrations | M8 | +| Historical recovery | Full backup, post-backup WAL replay, named-target PITR | M9 | +| Operations | Semantic metrics, alerts, Git dashboards, and runbooks | M10 | -- Terraform 1.8 or newer -- Linux with KVM, libvirt, and a running system libvirt daemon -- The `default` libvirt storage pool -- An OpenSSH public key for cloud-init access -- Capacity for either three development VMs or seven canonical VMs +The [evidence matrix](docs/evidence.md) separates measured observations from broader claims. -The provider downloads the configured Ubuntu cloud image during the first apply. No credentials or private keys belong in Terraform variables or state. +## Reliability findings -## Infrastructure quick start +- HAProxy provides a stable write endpoint, but existing sessions still require retry/reconnect behavior after failover. +- No acknowledged-row loss was observed in finite M5/M6 trials; this is not a universal RPO=0 guarantee. +- Sync-strict mode blocked bounded writes when no eligible synchronous standby existed. +- M7 directly sampled every database member and observed no more than one writable primary in its bounded matrix. +- Replication copied an intentional DELETE to both replicas. M9 PITR reconstructed the deleted row from backup plus archived WAL. +- M10 requires real replica, etcd-member, HAProxy, and replay-lag alerts to fire in Prometheus/Alertmanager and resolve after recovery. -```bash -cp terraform/environments/colocated/terraform.tfvars.example \ - terraform/environments/colocated/terraform.tfvars -make tf-validate -make cluster-up PROFILE=colocated -``` +## Migration safety -Inspect the `ssh_targets` output, then connect with the configured SSH user. Remove the profile with: +`pgsafe` is a read-only Go CLI using the PostgreSQL 16 parser AST. It flags dangerous locks, rewrites, destructive statements, transaction-incompatible operations, and missing timeout policy without connecting to a database. ```bash -make cluster-down PROFILE=colocated -``` - -Use `PROFILE=full` only on a host sized for all seven canonical VMs. Terraform state and local variable files are ignored by Git. - -## M2 quick start - -M2 uses Ubuntu 24.04's PostgreSQL 16 packages. Configuration is performed over SSH after Terraform finishes, keeping guest service configuration out of the infrastructure modules. For canonical evidence: - -```bash -make cluster-up PROFILE=full -make pg-configure PROFILE=full -make pg-verify PROFILE=full -make pg-failover PROFILE=full # destructive: stops pg-01 and promotes pg-02 -make cluster-down PROFILE=full +make pgsafe-build +./bin/pgsafe check migration.sql --format=json --fail-on=high +./bin/pgsafe explain PGSAFE001 ``` -Set `PGSENTRY_SSH_KEY` if the private key is not `~/.ssh/id_ed25519`. The replication password is generated at runtime under ignored `.pgsentry/`, is never committed, and is removed by `cluster-down` or `make pg-clean`. +See [the guide](docs/m8-pgsafe.md) and [rule catalog](docs/pgsafe-rules.md). -`pg-verify` is non-destructive to cluster roles: it verifies roles, both streaming connections, replicated data, standby write protection, and restart/reconnect behavior. `pg-failover` is deliberately separate and clearly destructive. See [the M2 runbook](docs/m2-postgresql.md) for inspection commands and split-brain precautions. +## Backup and recovery -## M3 quick start +pgBackRest stores full physical backups and archived WAL on `control-01`. M9 proved that a latest restore included a post-backup marker and named-target PITR restored a row that replication had deleted everywhere. A successful or young backup is not equivalent to a proven restore. See [M9](docs/m9-backup-pitr.md). -M3 pins etcd and etcdctl v3.7.1 and generates runtime-only TLS credentials. On a provisioned profile: +## Observability -```bash -make etcd-configure PROFILE=full -make etcd-verify PROFILE=full -make etcd-quorum-test PROFILE=full # destructive but self-restoring -``` - -The quorum test dynamically stops the elected leader, proves two-member availability, demonstrates one-member quorum loss with bounded operations, restores the members, and verifies persistence across a controlled full service restart. See [the M3 runbook](docs/m3-etcd.md). +Prometheus scrapes node_exporter on all seven VMs plus native Patroni, etcd mTLS, and HAProxy endpoints. A small collector publishes stable primary-count, replica-count, byte-lag, DCS, routing, backup-age, and archive-failure metrics. Alertmanager records local alert lifecycle, Grafana is provisioned from Git, and every alert points to a [runbook](docs/runbooks/). -## M4 quick start +## Quick start -On a fresh environment, configure etcd first, then Patroni and HAProxy: +The canonical path requires Linux/KVM, libvirt, Terraform 1.8+, the `default` libvirt pool, an SSH key, and capacity for seven Ubuntu 24.04 VMs. ```bash +cp terraform/environments/full/terraform.tfvars.example terraform/environments/full/terraform.tfvars +make tf-validate +make cluster-up PROFILE=full make etcd-configure PROFILE=full make patroni-configure PROFILE=full -make patroni-verify PROFILE=full make haproxy-configure PROFILE=full -make haproxy-verify PROFILE=full -make patroni-failover-test PROFILE=full # destructive, self-restoring -``` - -Patroni owns PostgreSQL once M4 is configured; do not run the M2 `pg-*` configuration targets on the same live data directories. See [the M4 runbook](docs/m4-patroni-haproxy.md). - -## M5 quick start - -After the full M4 stack passes verification: - -```bash -make failure-baseline PROFILE=full -make failure-scenario PROFILE=full SCENARIO=primary-service-loss -make failure-matrix PROFILE=full -make failure-report -``` - -The destructive scenarios are individually named, restore their failure state, and store machine-readable evidence under ignored `.pgsentry/results/m5/`. See [the M5 runbook](docs/m5-failure-testing.md). - -## M6 quick start - -M6 reuses the M5 client and failure analysis while changing Patroni's cluster-wide durability policy: - -```bash -make durability-set PROFILE=full MODE=sync -make durability-verify PROFILE=full MODE=sync -make durability-latency PROFILE=full MODE=sync -make durability-failure-test PROFILE=full MODE=sync TRIAL=1 # destructive, self-restoring -make durability-report +make backup-configure PROFILE=full +make backup-full PROFILE=full +make observability-configure PROFILE=full +make observability-verify PROFILE=full ``` -The default after experiments is restored to `async`. Strict mode deliberately blocks bounded client writes when no synchronous standby exists. See [the M6 runbook](docs/m6-synchronous-durability.md). - -## M7 quick start +Use [operations](docs/operations.md) for inspection and teardown or [the demo](docs/demo.md) for a concise walkthrough. -M7 extends the M5 safety and availability harness with bounded, tagged network partitions and DCS quorum experiments: +## Repository structure -```bash -make chaos-baseline PROFILE=full -make chaos-scenario PROFILE=full CHAOS_SCENARIO=replica-dcs-isolation -make chaos-matrix PROFILE=full -make chaos-report -``` +| Path | Purpose | +| --- | --- | +| `terraform/` | Canonical and development libvirt topology | +| `scripts/` | Service lifecycle and bounded reliability experiments | +| `monitoring/` | Prometheus rules/configuration and Grafana assets | +| `cmd/pgsafe`, `internal/pgsafe` | Migration-safety CLI | +| `docs/runbooks/` | Alert response procedures | +| `docs/` | Architecture, evidence, operations, demo, and deep dives | -Every scenario is destructive but self-restoring. Normal writes remain on the same HAProxy endpoint, and direct SQL sampling fails immediately if more than one writable primary is observed. See [the M7 runbook](docs/m7-network-dcs-chaos.md). +## Security model -## M8 quick start +Services are limited to private libvirt subnets. etcd retains mutual TLS. PostgreSQL monitoring uses a dedicated `pg_monitor` login. Grafana requires a generated administrator password. Secrets exist only under ignored `.pgsentry/`; runtime databases, TSDBs, backup data, WAL, images, and Terraform state are not committed. -`pgsafe` statically analyzes PostgreSQL 16 migration SQL and never connects to or executes against a database: +## Limitations -```bash -make pgsafe-build -./bin/pgsafe check migration.sql -./bin/pgsafe check migration.sql --format=json --fail-on=high -./bin/pgsafe check migration.sql --transaction-mode=wrapped -./bin/pgsafe rules -./bin/pgsafe explain PGSAFE001 -``` +PgSentry is a finite single-host lab, not production-ready infrastructure. It does not prove a production SLA, universal RTO/RPO, perfect alerts, long-duration capacity, off-site/immutable backup, geographic DR, HA monitoring/routing, zero false positives/negatives, or correctness under arbitrary failures. -Static fixtures run in hosted CI with `make pgsafe-fixtures`. The explicitly named `make migration-runtime-test PROFILE=full` separately manipulates only disposable objects on the canonical local cluster. See [the M8 runbook](docs/m8-pgsafe.md) and [rule catalog](docs/pgsafe-rules.md). +## Design decisions and deep dives -## M9 quick start +[Architecture](docs/architecture.md) · [Evidence](docs/evidence.md) · [Operations](docs/operations.md) · [Demo](docs/demo.md) · [M5 failures](docs/m5-failure-testing.md) · [M6 durability](docs/m6-synchronous-durability.md) · [M7 partitions](docs/m7-network-dcs-chaos.md) · [M8 migrations](docs/m8-pgsafe.md) · [M9 recovery](docs/m9-backup-pitr.md) · [M10 observability](docs/m10-observability.md) -M9 stores pgBackRest physical backups and archived WAL on `control-01`, then starts loopback-only recovery instances outside Patroni: +## Roadmap -```bash -make backup-configure PROFILE=full -make backup-check PROFILE=full -make backup-acceptance PROFILE=full # intentionally DELETEs disposable M9 data -make backup-clean PROFILE=full -``` +| Milestone | Capability | Status | +| --- | --- | --- | +| M1 | Terraform/libvirt infrastructure | Complete | +| M2 | PostgreSQL physical replication | Complete | +| M3 | Independent etcd quorum | Complete | +| M4 | Patroni HA and HAProxy routing | Complete | +| M5 | Failure and durability evidence harness | Complete | +| M6 | Synchronous durability experiments | Complete | +| M7 | Network-partition and DCS chaos | Complete | +| M8 | pgsafe migration-safety CLI | Complete | +| M9 | Backup, WAL, and PITR verification | Complete | +| M10 | Observability, runbooks, and portfolio polish | Complete | -The acceptance flow proves both post-backup WAL replay and PITR to a named restore point after replication has copied a DELETE to every live replica. See [the M9 runbook](docs/m9-backup-pitr.md). +Planned project roadmap complete. Optional future work—off-site storage, HA routing/monitoring, geographic DR, self-hosted reliability CI, and longer soak tests—is outside the completed roadmap. diff --git a/docs/architecture.md b/docs/architecture.md index 3687e5b..2a6b040 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -6,7 +6,7 @@ The canonical topology uses seven VMs: three PostgreSQL-role nodes, three etcd-r The control VM is also separate. Workload generation, client-visible measurements, HAProxy, and fault injection will eventually run there so the observer and injector remain alive when a database or etcd target is deliberately stopped. M1 provisions only the host and its network attachment. -The three-node `colocated` profile reduces laptop cost by allowing PostgreSQL, Patroni, and etcd to share each node in later milestones. It is useful for development, but only the `full` profile may produce canonical M5–M9 evidence. +The three-node `colocated` profile reduces laptop cost by allowing PostgreSQL, Patroni, and etcd to share each node. It is useful for development, but only the `full` profile may produce canonical M5–M10 evidence. ## Network model @@ -23,17 +23,39 @@ Each profile receives its own libvirt NAT network with deterministic DHCP reserv M1 does not open these ports to `0.0.0.0/0` or install services. Guest-level, role-specific firewall rules belong to the milestone that owns each service, when its bind addresses and operational requirements exist. -## Later CI architecture +## Final system architecture + +```text +Client -> control-01 HAProxy -> Patroni PostgreSQL pg-01/02/03 + | physical WAL + v + etcd-01/02/03 (mTLS) + +all seven hosts + native service endpoints + | + v + control-01 Prometheus (observer) + / \ + Alertmanager Grafana + | + runbooks + +PostgreSQL -> backup + WAL -> control-01 pgBackRest -> isolated restore/PITR +``` + +Prometheus and Alertmanager observe state but never participate in DCS leadership, PostgreSQL promotion, or HAProxy routing. `control-01` consolidates routing, backup, and monitoring to bound lab cost; this failure-domain colocation is not a production recommendation. + +## CI architecture ```text GitHub-hosted PR CI -> Go tests, lint, Terraform validation, static pgsafe checks -Self-hosted reliability CI with real VM access - -> chaos tests, runtime migration tests, backup restore, PITR +Local or future self-hosted reliability CI with real VM access + -> chaos, runtime migration, backup/PITR, monitoring alert lifecycle ``` -This is future architecture, not M1 functionality. No CI workflows or later milestone tools are implemented here. +Hosted CI validates source, Terraform, monitoring rules, Alertmanager configuration, dashboards, and runbook links. It does not claim seven-VM runtime acceptance. No permanently pending self-hosted workflow is configured. ## M2 PostgreSQL boundary @@ -82,3 +104,9 @@ Selected lock demonstrations are separate shell automation for the canonical ful M9 mounts a restricted NFSv4 pgBackRest repository physically hosted at `control-01:/var/lib/pgbackrest` on each Patroni member. Patroni's dynamic PostgreSQL configuration owns `archive_mode`, `archive_command`, and restart semantics; systemd still owns Patroni, and Patroni still owns every live PostgreSQL process. Backups dynamically use the current leader. Latest-state and PITR restores run outside Patroni on control-01 in fresh `/var/lib/pgbackrest-restore/*` directories. They bind only `127.0.0.1:55432`, use explicit `pg_ctl` lifecycle, never join the DCS, and never enter HAProxy. This off-PGDATA repository is operational separation inside one lab network, not geographic disaster recovery. + +## M10 observability and operations boundary + +Prometheus and Alertmanager run loopback-only on `control-01`; authenticated Grafana is reachable only through the lab network/host path. node_exporter runs on all seven VMs. Prometheus scrapes Patroni, HAProxy, and etcd native metrics without weakening mutual TLS. A dedicated `pg_monitor` login and etcd client certificate are generated at runtime. + +A minimal collector publishes cross-component semantics: writable-primary count, streaming replicas, byte replay lag, healthy etcd members, writable HAProxy backends, backup age, and archive failures. These feed actionable rules, Git-provisioned dashboards, and runbooks. Monitoring remains outside the HA control plane. diff --git a/docs/demo.md b/docs/demo.md new file mode 100644 index 0000000..e71192f --- /dev/null +++ b/docs/demo.md @@ -0,0 +1,13 @@ +# PgSentry 10–20 minute demo + +This path assumes the canonical stack is configured and avoids the long destructive matrices. + +1. Run `make patroni-verify PROFILE=full` and `make observability-verify PROFILE=full`. +2. Open the provisioned **PgSentry Overview** dashboard using the tunnel in [operations](operations.md). Show one primary, two replicas, three etcd members, one writable backend, backup age, and alerts. +3. Run `make haproxy-verify PROFILE=full` to prove the client route. +4. Run `make observability-alert-test PROFILE=full ALERT_TEST=replica-loss`. Show the firing `PgSentryReplicaCountLow`, its runbook annotation, and resolution. +5. Run `make pgsafe-build` then `./bin/pgsafe check testdata/migrations/unsafe/risky.sql`. +6. Show `make backup-info PROFILE=full` and [M9 recovery evidence](m9-backup-pitr.md); do not rerun full PITR for a short demo. +7. Finish with `make observability-verify PROFILE=full` and `make patroni-verify PROFILE=full`. + +For deeper demonstrations use [M5](m5-failure-testing.md), [M7](m7-network-dcs-chaos.md), [M8](m8-pgsafe.md), or [M9](m9-backup-pitr.md). diff --git a/docs/evidence.md b/docs/evidence.md new file mode 100644 index 0000000..d3f38d1 --- /dev/null +++ b/docs/evidence.md @@ -0,0 +1,18 @@ +# Consolidated evidence + +Evidence is bounded to canonical lab runs. A successful finite trial is not a universal production guarantee. + +| Area | Measured or asserted evidence | Milestone | +| --- | --- | --- | +| Infrastructure | Seven deterministic VMs booted, accepted SSH, and destroyed with zero Terraform resources | M1 | +| Replication | Exactly one primary, two streaming read-only replicas, replicated marker, restart/reconnect | M2 | +| DCS | Three mutual-TLS etcd voters; 2/3 retained quorum and 1/3 could not commit | M3 | +| HA/routing | Patroni promotion and HAProxy rerouting to exactly one `/primary` backend | M4 | +| Failure behavior | Sequenced client outcomes, interruptions, ambiguity, direct writer count, retained acknowledgements | M5 | +| Durability | Async/sync/sync-strict trials; strict writes blocked without an eligible synchronous standby | M6 | +| Partitions | DCS/WAL partitions with maximum writable-primary count asserted at one | M7 | +| Migrations | PostgreSQL 16 AST fixtures plus bounded real lock and replication demonstrations | M8 | +| Recovery | 23,436,968-byte full backup in 6,499 ms; latest and PITR restores around 2.5 s; deleted row recovered | M9 | +| Operations | 15 targets, semantic metrics, three Git dashboards, and four live alert lifecycles | M10 | + +M9 used backup `20260907-021333F`: latest recovery included post-backup data, and named PITR retained a row before its exact DELETE while excluding a post-target marker. No acknowledged-row loss and no multiple-writer state were observed in accepted finite M5–M7 trials; those observations do not establish universal RPO=0 or arbitrary-partition safety. M10 observes their signals but does not replace failure testing. diff --git a/docs/m10-observability.md b/docs/m10-observability.md new file mode 100644 index 0000000..c76a54a --- /dev/null +++ b/docs/m10-observability.md @@ -0,0 +1,18 @@ +# M10 observability, alerting, and operations + +M10 turns M1–M9 reliability semantics into metrics, alerts, dashboards, and runbooks. Prometheus 3.13.1 and Alertmanager 0.33.1 run on loopback on `control-01` with seven-day retention; authenticated Grafana OSS 13.1.3 is subnet-restricted. node_exporter 1.12.1 runs on all seven VMs. Patroni 4.1.5 `/metrics`, etcd 3.7.1 native metrics over mutual TLS, and HAProxy 2.8.16’s native exporter are scraped directly. + +A small 15-second textfile collector publishes writable-primary count, streaming replicas, maximum replay lag in bytes, healthy etcd members, writable HAProxy backends, latest full-backup timestamp, last archive timestamp, and cumulative archive failures. PostgreSQL uses a generated `pgsentry_monitor` login granted only `pg_monitor`; etcd uses a dedicated generated client certificate. Credentials remain ignored under `.pgsentry/`. + +```bash +make observability-configure PROFILE=full +make observability-check PROFILE=full +make observability-verify PROFILE=full +make observability-alert-test PROFILE=full ALERT_TEST=replica-loss +``` + +Verification is non-destructive. Alert tests are explicitly destructive and self-restoring: they select a replica, stop one etcd member, stop only HAProxy, or pause one replica’s replay while generating bounded WAL. Evidence is ignored under `.pgsentry/results/m10/`. + +The 24-hour backup-age and 15% filesystem-free thresholds are lab policies. Byte lag is an LSN difference, not seconds. Archive alerting detects a counter increase rather than alerting forever on historical failures. A young backup does not prove recovery; M9 restore/PITR evidence does. + +Monitoring is not highly available. `control-01` remains a lab consolidation point for routing, backup, and monitoring. Prometheus does not participate in HA. M10 does not prove a monitoring SLA, 24/7 paging, perfect thresholds, production capacity, HA routing/monitoring, arbitrary failure detection, off-site recovery, or long soak reliability. diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..8560127 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,28 @@ +# Operations index + +Use this entry point for an already-provisioned canonical `full` environment. + +| Question | Command | +| --- | --- | +| Patroni topology | `make patroni-verify PROFILE=full` | +| Current primary | `curl -s http://192.168.130.11:8008/cluster \| jq '.members[]|select(.role=="leader" or .role=="primary")'` | +| Replication | `SELECT * FROM pg_stat_replication;` on the primary | +| etcd quorum | `make etcd-verify PROFILE=full` | +| HAProxy route | `make haproxy-verify PROFILE=full` | +| Backup inventory | `make backup-info PROFILE=full` | +| WAL archive | `make backup-archive-verify PROFILE=full` | +| Monitoring | `make observability-check PROFILE=full && make observability-verify PROFILE=full` | +| Active alerts | `ssh pgsentry@192.168.130.31 'curl -s http://127.0.0.1:9090/api/v1/alerts' \| jq .` | +| Grafana | `ssh -L 3000:192.168.130.31:3000 pgsentry@192.168.130.31`, then `http://127.0.0.1:3000` | + +The generated Grafana password is stored with mode 0600 under `.pgsentry/ha/`. Alert procedures are under [runbooks](runbooks/); start with [no primary](runbooks/no-primary.md), [multiple primaries](runbooks/multiple-primaries.md), or [etcd quorum](runbooks/etcd-quorum.md). + +Safe final teardown order: + +```bash +make observability-clean PROFILE=full +make backup-clean PROFILE=full +make cluster-down PROFILE=full +``` + +The last two commands permanently remove backup/WAL data and VMs. Preserve accepted evidence first. Never delete PGDATA, etcd data, DCS keys, or archived WAL as an ad-hoc response. diff --git a/docs/runbooks/backup-stale.md b/docs/runbooks/backup-stale.md new file mode 100644 index 0000000..0e638b2 --- /dev/null +++ b/docs/runbooks/backup-stale.md @@ -0,0 +1,13 @@ +# Backup stale + +## Meaning and impact + +`PgSentryBackupStale` means the latest successful full backup is older than the lab’s 24-hour threshold. This policy is demonstrative; a young backup does not prove recoverability. Confirm archive continuity and repository integrity. + +```bash +make backup-info PROFILE=full +make backup-check PROFILE=full +make backup-full PROFILE=full +``` + +Inspect pgBackRest JSON inventory, job logs, repository mounts, capacity, and the current Patroni leader. Address scheduling, repository, or database connectivity before taking a reviewed full backup. Do not expire the only valid backup, delete archived WAL, or interpret job success as restore proof. Recovery requires an OK stanza, a current full backup, healthy archival, and separately retained M9 restore/PITR evidence. Escalate if inventory is corrupt or WAL continuity is uncertain. See [M9](../m9-backup-pitr.md) and [archive failure](wal-archive-failure.md). diff --git a/docs/runbooks/disk-space.md b/docs/runbooks/disk-space.md new file mode 100644 index 0000000..ffdd5e8 --- /dev/null +++ b/docs/runbooks/disk-space.md @@ -0,0 +1,13 @@ +# Disk space low + +## Meaning and impact + +`PgSentryDiskSpaceLow` means a monitored non-temporary filesystem has less than 15% available for five minutes. The threshold is a lab policy, not universal capacity planning. Determine whether PostgreSQL PGDATA, WAL, or the control-node backup repository is affected. + +```bash +df -hT +sudo du -xhd1 /var/lib/postgresql /var/lib/pgbackrest /var/lib/prometheus 2>/dev/null +sudo journalctl --disk-usage +``` + +Identify the current PostgreSQL leader and avoid changes that could remove required WAL or backups. Check replication slots, pgBackRest retention, logs, Prometheus retention, and unexpected files. Use reviewed retention/cleanup procedures; do not delete PGDATA, `pg_wal`, etcd data, or backup files manually. Verify free capacity stabilizes, PostgreSQL writes succeed, replicas stream, archival succeeds, and backup inventory remains OK. Escalate if safe reclamation cannot stay ahead of growth. See [M9](../m9-backup-pitr.md). diff --git a/docs/runbooks/etcd-quorum.md b/docs/runbooks/etcd-quorum.md new file mode 100644 index 0000000..e473623 --- /dev/null +++ b/docs/runbooks/etcd-quorum.md @@ -0,0 +1,13 @@ +# etcd member or quorum failure + +## Meaning and impact + +`PgSentryEtcdMemberDown` means fewer than 3/3 members respond; 2/3 is degraded but retains quorum. `PgSentryEtcdQuorumLost` means fewer than 2/3 respond; 1/3 cannot commit new consensus state. Patroni may reduce availability intentionally rather than risk unsafe leadership. + +```bash +make etcd-verify PROFILE=full +. scripts/etcd/common.sh; load_profile full; etcdctl_cmd endpoint status --cluster -w table +ssh pgsentry@192.168.130.21 'systemctl status etcd; journalctl -u etcd --since -15m' +``` + +Identify the elected leader and failed member, then restore only the known member’s service, disk, TLS, or connectivity. Confirm the database has at most one writer. Do not casually create a new cluster, use `--force-new-cluster`, delete member data, or remove DCS keys. Recovery requires 3/3 healthy endpoints, one etcd leader, coherent Patroni membership, and one PostgreSQL primary. Escalate if member identity/data disagree or quorum cannot be restored. See [M3](../m3-etcd.md) and [M7](../m7-network-dcs-chaos.md). diff --git a/docs/runbooks/haproxy-unavailable.md b/docs/runbooks/haproxy-unavailable.md new file mode 100644 index 0000000..6a04f9b --- /dev/null +++ b/docs/runbooks/haproxy-unavailable.md @@ -0,0 +1,14 @@ +# HAProxy or writable route unavailable + +## Meaning and impact + +`PgSentryHAProxyDown` means the metrics/service endpoint is unavailable. `PgSentryNoWritableBackend` means HAProxy has no healthy primary backend. These are client-path incidents; PostgreSQL may still have a healthy leader. First confirm there is exactly one primary. + +```bash +make patroni-verify PROFILE=full +ssh pgsentry@192.168.130.31 'systemctl status haproxy; sudo haproxy -c -f /etc/haproxy/haproxy.cfg; journalctl -u haproxy --since -15m' +ssh pgsentry@192.168.130.31 "printf 'show stat\n' | sudo socat - UNIX-CONNECT:/run/haproxy/admin.sock" +make haproxy-verify PROFILE=full +``` + +Check Patroni `/primary` health responses, HAProxy configuration, socket state, and port 5000. Restore or reload HAProxy only after validation. Do not manually promote PostgreSQL to fix a routing failure or bypass HAProxy with uncontrolled writes. Verify one UP writable backend and a routed query. Escalate if HAProxy and direct Patroni role observations disagree. See [M4](../m4-patroni-haproxy.md). diff --git a/docs/runbooks/multiple-primaries.md b/docs/runbooks/multiple-primaries.md new file mode 100644 index 0000000..56d2388 --- /dev/null +++ b/docs/runbooks/multiple-primaries.md @@ -0,0 +1,21 @@ +# Multiple writable primaries + +## Meaning and impact + +`PgSentryMultiplePrimaries` is a critical safety incident: direct SQL observations indicate more than one writable PostgreSQL node. Availability is secondary to preventing divergent writes. + +## Safety-first response + +1. Stop application writes at the client/load-balancer boundary where possible. +2. Record Prometheus alerts, Patroni `/cluster`, direct `pg_is_in_recovery()` results, timelines, LSNs, and etcd endpoint status. +3. Determine which member owns the Patroni leader key and whether etcd has quorum. +4. Do not manually promote any node and do not write to either suspected primary. +5. Isolate or stop the writer that does not own safe leadership, using the narrowest reviewed fencing action. + +```bash +ssh pgsentry@192.168.130.11 'sudo /opt/patroni/bin/patronictl -c /etc/patroni/patroni.yml list -f json' +for ip in 192.168.130.11 192.168.130.12 192.168.130.13; do ssh pgsentry@$ip "sudo -u postgres psql -Atc 'select pg_is_in_recovery(),timeline_id from pg_control_checkpoint()'"; done +make etcd-verify PROFILE=full +``` + +Inspect `journalctl -u patroni` and M7 partition state before mitigation. Never “restart everything,” delete DCS keys, or discard PGDATA as a first response. Resume traffic only after exactly one writer, coherent DCS ownership, two streaming replicas, and HAProxy routing are verified. Preserve divergent data for reconciliation. Escalate immediately if the safe writer cannot be established. See [M7 evidence](../m7-network-dcs-chaos.md). diff --git a/docs/runbooks/no-primary.md b/docs/runbooks/no-primary.md new file mode 100644 index 0000000..c6ca210 --- /dev/null +++ b/docs/runbooks/no-primary.md @@ -0,0 +1,17 @@ +# No writable primary + +## Meaning and impact + +`PgSentryNoPrimary` means no Patroni member reports ownership of the leader lock and writable PostgreSQL. New writes fail. The immediate safety question is whether every node is truly read-only; do not promote until DCS state is understood. + +## First response + +```bash +make patroni-verify PROFILE=full +ssh pgsentry@192.168.130.11 'sudo /opt/patroni/bin/patronictl -c /etc/patroni/patroni.yml list' +curl -s http://192.168.130.11:8008/cluster | jq . +``` + +Check `systemctl status patroni`, `journalctl -u patroni --since -15m`, PostgreSQL recovery state, and etcd health on all members. Likely causes are DCS quorum loss, a failed leader, failed promotion, or PostgreSQL startup failure. + +Restore etcd quorum or the failed service before considering a reviewed Patroni failover. Do not run `pg_ctl promote`, delete DCS keys, remove PGDATA, or promote multiple nodes. Recovery requires one Patroni leader, two streaming replicas, one writable HAProxy backend, and a successful routed write. Escalate if role observations disagree or quorum cannot be restored. See [M4](../m4-patroni-haproxy.md), [etcd quorum](etcd-quorum.md), and [multiple primaries](multiple-primaries.md). diff --git a/docs/runbooks/replication-degraded.md b/docs/runbooks/replication-degraded.md new file mode 100644 index 0000000..05c87c4 --- /dev/null +++ b/docs/runbooks/replication-degraded.md @@ -0,0 +1,13 @@ +# Replication degraded + +## Meaning and impact + +`PgSentryReplicaCountLow` or `PgSentryPatroniMemberDown` means fewer than two streaming replicas or an unavailable Patroni endpoint. The primary may remain writable, but failover choices and durability margin are reduced. Confirm there is still exactly one primary. + +```bash +make patroni-verify PROFILE=full +ssh pgsentry@192.168.130.11 'sudo /opt/patroni/bin/patronictl -c /etc/patroni/patroni.yml list' +ssh pgsentry@192.168.130.11 "sudo -u postgres psql -x -c 'select * from pg_stat_replication'" +``` + +Inspect the affected member’s `systemctl status patroni`, `journalctl -u patroni`, disk space, network reachability, receiver status, slots, and timelines. Safely restore the stopped service or connectivity and allow Patroni to rejoin it. Do not promote a replica merely because another replica failed; do not delete PGDATA or slots casually. Recovery requires one leader and two streaming replicas with shrinking lag. Escalate on timeline divergence, missing WAL, or repeated reinitialization. See [M2](../m2-postgresql.md) and [replication lag](replication-lag.md). diff --git a/docs/runbooks/replication-lag.md b/docs/runbooks/replication-lag.md new file mode 100644 index 0000000..811a299 --- /dev/null +++ b/docs/runbooks/replication-lag.md @@ -0,0 +1,12 @@ +# Replication lag + +## Meaning and impact + +`PgSentryReplicationLagHigh` means the maximum primary-to-replica replay LSN difference exceeded 1 MiB for 30 seconds. Units are bytes, not seconds. A standby may say `streaming` while replay is paused or slow; confirm one writable primary before action. + +```bash +ssh pgsentry@192.168.130.11 "sudo -u postgres psql -x -c \"select application_name,state,sent_lsn,write_lsn,flush_lsn,replay_lsn,pg_wal_lsn_diff(pg_current_wal_lsn(),replay_lsn) lag_bytes from pg_stat_replication\"" +ssh pgsentry@192.168.130.12 "sudo -u postgres psql -Atc 'select pg_is_wal_replay_paused(),pg_last_wal_replay_lsn()'" +``` + +Check receiver/replay state, host CPU/disk/network, replication slots, WAL availability, and `journalctl -u patroni`. Resume replay only if it was intentionally paused: `SELECT pg_wal_replay_resume();`. Do not pause both replicas, delete slots, or rebuild a standby before identifying missing WAL or resource pressure. Verify lag returns below threshold and both replicas stream. Escalate if lag grows, required WAL is gone, or synchronous policy threatens write availability. See [M6](../m6-synchronous-durability.md). diff --git a/docs/runbooks/wal-archive-failure.md b/docs/runbooks/wal-archive-failure.md new file mode 100644 index 0000000..0223310 --- /dev/null +++ b/docs/runbooks/wal-archive-failure.md @@ -0,0 +1,14 @@ +# WAL archive failure + +## Meaning and impact + +`PgSentryWALArchiveFailure` means PostgreSQL’s cumulative archive failure counter increased during the last five minutes. A cumulative nonzero value alone does not mean a current failure. Continued failure can make recent PITR impossible and fill `pg_wal`; verify the live primary first. + +```bash +make backup-check PROFILE=full +make backup-archive-verify PROFILE=full +ssh pgsentry@192.168.130.11 "sudo -u postgres psql -x -c 'select * from pg_stat_archiver'" +ssh pgsentry@192.168.130.11 'sudo -u postgres pgbackrest --stanza=pgsentry check' +``` + +Inspect `last_failed_wal`, `last_failed_time`, pgBackRest logs, NFS mount health, repository permissions/capacity, and Patroni’s current archive command. Restore repository connectivity or permissions; do not disable archiving, delete unarchived WAL, reset statistics to hide the incident, or rebuild the repository casually. Verify a newly generated segment archives and `failed_count` stops increasing. Escalate if required WAL is missing. See [M9](../m9-backup-pitr.md) and [backup stale](backup-stale.md). diff --git a/monitoring/alertmanager/alertmanager.yml b/monitoring/alertmanager/alertmanager.yml new file mode 100644 index 0000000..ab6577c --- /dev/null +++ b/monitoring/alertmanager/alertmanager.yml @@ -0,0 +1,14 @@ +global: + resolve_timeout: 30s +route: + receiver: local + group_by: [alertname] + group_wait: 5s + group_interval: 15s + repeat_interval: 4h +receivers: + - name: local +inhibit_rules: + - source_matchers: ['alertname="PgSentryEtcdQuorumLost"'] + target_matchers: ['alertname="PgSentryEtcdMemberDown"'] + equal: [] diff --git a/monitoring/grafana/dashboards/dcs-routing-recovery.json b/monitoring/grafana/dashboards/dcs-routing-recovery.json new file mode 100644 index 0000000..8b9d6f5 --- /dev/null +++ b/monitoring/grafana/dashboards/dcs-routing-recovery.json @@ -0,0 +1,11 @@ +{ + "annotations":{"list":[]},"editable":false,"id":null,"schemaVersion":41,"tags":["pgsentry","operations"],"timezone":"utc","title":"PgSentry DCS, Routing, and Recovery","uid":"pgsentry-dcs-routing-recovery","version":1,"time":{"from":"now-1h","to":"now"},"panels":[ + {"type":"stat","title":"Healthy etcd members","id":1,"gridPos":{"h":4,"w":6,"x":0,"y":0},"targets":[{"expr":"pgsentry_etcd_healthy_members","refId":"A"}]}, + {"type":"stat","title":"etcd leader count","id":2,"gridPos":{"h":4,"w":6,"x":6,"y":0},"targets":[{"expr":"sum(etcd_server_has_leader)","refId":"A"}]}, + {"type":"stat","title":"Writable HAProxy backends","id":3,"gridPos":{"h":4,"w":6,"x":12,"y":0},"targets":[{"expr":"pgsentry_haproxy_writable_backends","refId":"A"}]}, + {"type":"stat","title":"HAProxy endpoint","id":4,"gridPos":{"h":4,"w":6,"x":18,"y":0},"targets":[{"expr":"up{job=\"haproxy\"}","refId":"A"}]}, + {"type":"stat","title":"Backup age (hours)","id":5,"gridPos":{"h":5,"w":8,"x":0,"y":4},"targets":[{"expr":"(time()-pgsentry_backup_last_success_timestamp_seconds)/3600","refId":"A"}]}, + {"type":"stat","title":"WAL archive failures","id":6,"gridPos":{"h":5,"w":8,"x":8,"y":4},"targets":[{"expr":"pgsentry_wal_archive_failures_total","refId":"A"}]}, + {"type":"gauge","title":"control-01 root available percent","id":7,"gridPos":{"h":5,"w":8,"x":16,"y":4},"targets":[{"expr":"100*node_filesystem_avail_bytes{instance=\"192.168.130.31:9100\",mountpoint=\"/\"}/node_filesystem_size_bytes{instance=\"192.168.130.31:9100\",mountpoint=\"/\"}","refId":"A"}]} + ] +} diff --git a/monitoring/grafana/dashboards/overview.json b/monitoring/grafana/dashboards/overview.json new file mode 100644 index 0000000..1c97de4 --- /dev/null +++ b/monitoring/grafana/dashboards/overview.json @@ -0,0 +1,14 @@ +{ + "annotations":{"list":[]},"editable":false,"id":null,"schemaVersion":41,"tags":["pgsentry"],"timezone":"utc","title":"PgSentry Overview","uid":"pgsentry-overview","version":1, + "time":{"from":"now-1h","to":"now"},"panels":[ + {"type":"stat","title":"Writable primaries","id":1,"gridPos":{"h":4,"w":4,"x":0,"y":0},"targets":[{"expr":"pgsentry_postgresql_primary_count","refId":"A"}]}, + {"type":"stat","title":"Streaming replicas","id":2,"gridPos":{"h":4,"w":4,"x":4,"y":0},"targets":[{"expr":"pgsentry_postgresql_replica_count","refId":"A"}]}, + {"type":"stat","title":"Largest replication lag (bytes)","id":3,"gridPos":{"h":4,"w":4,"x":8,"y":0},"targets":[{"expr":"pgsentry_replication_lag_bytes","refId":"A"}]}, + {"type":"stat","title":"Healthy etcd members","id":4,"gridPos":{"h":4,"w":4,"x":12,"y":0},"targets":[{"expr":"pgsentry_etcd_healthy_members","refId":"A"}]}, + {"type":"stat","title":"Writable HAProxy backends","id":5,"gridPos":{"h":4,"w":4,"x":16,"y":0},"targets":[{"expr":"pgsentry_haproxy_writable_backends","refId":"A"}]}, + {"type":"stat","title":"Backup age (hours)","id":6,"gridPos":{"h":4,"w":4,"x":20,"y":0},"targets":[{"expr":"(time()-pgsentry_backup_last_success_timestamp_seconds)/3600","refId":"A"}]}, + {"type":"stat","title":"Firing alerts","id":7,"gridPos":{"h":4,"w":6,"x":0,"y":4},"targets":[{"expr":"count(ALERTS{alertstate=\"firing\"})","refId":"A"}]}, + {"type":"timeseries","title":"VM available memory","id":8,"gridPos":{"h":8,"w":9,"x":6,"y":4},"targets":[{"expr":"node_memory_MemAvailable_bytes","legendFormat":"{{instance}}","refId":"A"}]}, + {"type":"timeseries","title":"VM CPU busy","id":9,"gridPos":{"h":8,"w":9,"x":15,"y":4},"targets":[{"expr":"100*(1-avg by(instance)(rate(node_cpu_seconds_total{mode=\"idle\"}[5m])))","legendFormat":"{{instance}}","refId":"A"}]} + ] +} diff --git a/monitoring/grafana/dashboards/postgresql.json b/monitoring/grafana/dashboards/postgresql.json new file mode 100644 index 0000000..1d4ce8f --- /dev/null +++ b/monitoring/grafana/dashboards/postgresql.json @@ -0,0 +1,12 @@ +{ + "annotations":{"list":[]},"editable":false,"id":null,"schemaVersion":41,"tags":["pgsentry","postgresql"],"timezone":"utc","title":"PgSentry PostgreSQL","uid":"pgsentry-postgresql","version":1,"time":{"from":"now-1h","to":"now"},"panels":[ + {"type":"stat","title":"Primary count","id":1,"gridPos":{"h":4,"w":6,"x":0,"y":0},"targets":[{"expr":"pgsentry_postgresql_primary_count","refId":"A"}]}, + {"type":"stat","title":"Replica count","id":2,"gridPos":{"h":4,"w":6,"x":6,"y":0},"targets":[{"expr":"pgsentry_postgresql_replica_count","refId":"A"}]}, + {"type":"timeseries","title":"Replay lag (bytes)","id":3,"gridPos":{"h":8,"w":12,"x":12,"y":0},"targets":[{"expr":"pgsentry_replication_lag_bytes","refId":"A"}]}, + {"type":"state-timeline","title":"Patroni role","id":4,"gridPos":{"h":8,"w":12,"x":0,"y":4},"targets":[{"expr":"patroni_primary","legendFormat":"{{name}} primary","refId":"A"},{"expr":"patroni_replica","legendFormat":"{{name}} replica","refId":"B"}]}, + {"type":"timeseries","title":"PostgreSQL host disk available","id":5,"gridPos":{"h":8,"w":12,"x":12,"y":8},"targets":[{"expr":"node_filesystem_avail_bytes{instance=~\"192.168.130.1[1-3]:9100\",mountpoint=\"/\"}","legendFormat":"{{instance}}","refId":"A"}]}, + {"type":"stat","title":"Connections / waiting locks / long transactions","id":6,"gridPos":{"h":5,"w":12,"x":0,"y":12},"targets":[{"expr":"pgsentry_postgresql_connections","refId":"A"},{"expr":"pgsentry_postgresql_waiting_locks","refId":"B"},{"expr":"pgsentry_postgresql_long_transactions","refId":"C"}]}, + {"type":"timeseries","title":"Transactions","id":7,"gridPos":{"h":5,"w":12,"x":12,"y":16},"targets":[{"expr":"rate(pgsentry_postgresql_commits_total[5m])","refId":"A"},{"expr":"rate(pgsentry_postgresql_rollbacks_total[5m])","refId":"B"}]}, + {"type":"stat","title":"Database size","id":8,"gridPos":{"h":5,"w":12,"x":0,"y":17},"targets":[{"expr":"pgsentry_postgresql_database_size_bytes","refId":"A"}]} + ] +} diff --git a/monitoring/grafana/provisioning/dashboards/pgsentry.yml b/monitoring/grafana/provisioning/dashboards/pgsentry.yml new file mode 100644 index 0000000..a8b0b01 --- /dev/null +++ b/monitoring/grafana/provisioning/dashboards/pgsentry.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +providers: + - name: PgSentry + folder: PgSentry + type: file + disableDeletion: true + editable: false + options: + path: /var/lib/grafana/dashboards diff --git a/monitoring/grafana/provisioning/datasources/prometheus.yml b/monitoring/grafana/provisioning/datasources/prometheus.yml new file mode 100644 index 0000000..b09f975 --- /dev/null +++ b/monitoring/grafana/provisioning/datasources/prometheus.yml @@ -0,0 +1,9 @@ +apiVersion: 1 +datasources: + - name: Prometheus + uid: pgsentry-prometheus + type: prometheus + access: proxy + url: http://127.0.0.1:9090 + isDefault: true + editable: false diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml new file mode 100644 index 0000000..8152a4c --- /dev/null +++ b/monitoring/prometheus/prometheus.yml @@ -0,0 +1,49 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + external_labels: + environment: pgsentry-full + +alerting: + alertmanagers: + - static_configs: + - targets: ['127.0.0.1:9093'] + +rule_files: + - /etc/prometheus/rules.yml + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['127.0.0.1:9090'] + - job_name: node + static_configs: + - targets: + - 192.168.130.11:9100 + - 192.168.130.12:9100 + - 192.168.130.13:9100 + - 192.168.130.21:9100 + - 192.168.130.22:9100 + - 192.168.130.23:9100 + - 192.168.130.31:9100 + - job_name: patroni + metrics_path: /metrics + static_configs: + - targets: ['192.168.130.11:8008', '192.168.130.12:8008', '192.168.130.13:8008'] + - job_name: etcd + scheme: https + tls_config: + ca_file: /etc/prometheus/pki/etcd-ca.crt + cert_file: /etc/prometheus/pki/etcd-monitor.crt + key_file: /etc/prometheus/pki/etcd-monitor.key + static_configs: + - targets: ['192.168.130.21:2379'] + labels: {member: etcd-01} + - targets: ['192.168.130.22:2379'] + labels: {member: etcd-02} + - targets: ['192.168.130.23:2379'] + labels: {member: etcd-03} + - job_name: haproxy + metrics_path: /metrics + static_configs: + - targets: ['127.0.0.1:8405'] diff --git a/monitoring/prometheus/rules.yml b/monitoring/prometheus/rules.yml new file mode 100644 index 0000000..b89b0e5 --- /dev/null +++ b/monitoring/prometheus/rules.yml @@ -0,0 +1,68 @@ +groups: + - name: pgsentry-safety + rules: + - alert: PgSentryMultiplePrimaries + expr: pgsentry_postgresql_primary_count > 1 + for: 15s + labels: {severity: critical} + annotations: + summary: Multiple writable PostgreSQL primaries observed + description: "Writable primary count is {{ $value }}; stop application writes and fence the unsafe writer." + runbook: docs/runbooks/multiple-primaries.md + - alert: PgSentryNoPrimary + expr: pgsentry_postgresql_primary_count == 0 + for: 30s + labels: {severity: critical} + annotations: {summary: No writable PostgreSQL primary, description: "No Patroni member reports a writable primary.", runbook: docs/runbooks/no-primary.md} + - alert: PgSentryNoWritableBackend + expr: pgsentry_haproxy_writable_backends == 0 + for: 30s + labels: {severity: critical} + annotations: {summary: HAProxy has no writable backend, description: "New clients cannot reach a primary through port 5000.", runbook: docs/runbooks/haproxy-unavailable.md} + - alert: PgSentryHAProxyDown + expr: up{job="haproxy"} == 0 + for: 30s + labels: {severity: critical} + annotations: {summary: HAProxy metrics endpoint is down, description: "HAProxy on control-01 is unavailable or unreachable.", runbook: docs/runbooks/haproxy-unavailable.md} + - alert: PgSentryEtcdQuorumLost + expr: pgsentry_etcd_healthy_members < 2 + for: 30s + labels: {severity: critical} + annotations: {summary: etcd quorum is unavailable, description: "Fewer than two of three etcd members are healthy.", runbook: docs/runbooks/etcd-quorum.md} + - name: pgsentry-operations + rules: + - alert: PgSentryReplicaCountLow + expr: pgsentry_postgresql_replica_count < 2 + for: 30s + labels: {severity: warning} + annotations: {summary: PostgreSQL replica count is low, description: "Expected two streaming replicas; observed {{ $value }}.", runbook: docs/runbooks/replication-degraded.md} + - alert: PgSentryReplicationLagHigh + expr: pgsentry_replication_lag_bytes > 1048576 + for: 30s + labels: {severity: warning} + annotations: {summary: PostgreSQL replication byte lag is high, description: "Maximum replay lag is {{ $value }} bytes.", runbook: docs/runbooks/replication-lag.md} + - alert: PgSentryEtcdMemberDown + expr: pgsentry_etcd_healthy_members < 3 + for: 30s + labels: {severity: warning} + annotations: {summary: An etcd member is unavailable, description: "Healthy etcd members: {{ $value }}/3.", runbook: docs/runbooks/etcd-quorum.md} + - alert: PgSentryWALArchiveFailure + expr: increase(pgsentry_wal_archive_failures_total[5m]) > 0 + for: 15s + labels: {severity: warning} + annotations: {summary: PostgreSQL WAL archival failed recently, description: "The cumulative archive failure counter increased in five minutes.", runbook: docs/runbooks/wal-archive-failure.md} + - alert: PgSentryBackupStale + expr: time() - pgsentry_backup_last_success_timestamp_seconds > 86400 + for: 5m + labels: {severity: warning} + annotations: {summary: Latest full backup is older than lab policy, description: "Backup age exceeds the 24-hour demonstration threshold.", runbook: docs/runbooks/backup-stale.md} + - alert: PgSentryDiskSpaceLow + expr: 100 * node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"} < 15 + for: 5m + labels: {severity: warning} + annotations: {summary: Host filesystem free space is low, description: "Filesystem {{ $labels.mountpoint }} on {{ $labels.instance }} has under 15% available.", runbook: docs/runbooks/disk-space.md} + - alert: PgSentryPatroniMemberDown + expr: up{job="patroni"} == 0 + for: 1m + labels: {severity: warning} + annotations: {summary: Patroni member endpoint is unavailable, description: "Patroni metrics scrape failed for {{ $labels.instance }}.", runbook: docs/runbooks/replication-degraded.md} diff --git a/scripts/ha/configure-haproxy-node.sh b/scripts/ha/configure-haproxy-node.sh index eaf85ab..5c9906b 100755 --- a/scripts/ha/configure-haproxy-node.sh +++ b/scripts/ha/configure-haproxy-node.sh @@ -15,6 +15,10 @@ defaults timeout connect 5s timeout client 30s timeout server 30s +frontend prometheus + bind 127.0.0.1:8405 + mode http + http-request use-service prometheus-exporter frontend postgres_write bind *:5000 default_backend patroni_primary diff --git a/scripts/observability/alert-test.sh b/scripts/observability/alert-test.sh new file mode 100755 index 0000000..97efd49 --- /dev/null +++ b/scripts/observability/alert-test.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/common.sh"; observability_load "${1:-}"; scenario=${2:-} +wait_cluster; wait_metric pgsentry_postgresql_primary_count 1; wait_metric pgsentry_postgresql_replica_count 2; wait_metric pgsentry_etcd_healthy_members 3; wait_metric pgsentry_haproxy_writable_backends 1 +cleanup() { set +e; [[ -n ${target_ip:-} && -n ${service:-} ]] && ssh_node "$target_ip" "sudo systemctl start $service"; [[ -n ${replay_pid:-} && -n ${paused_ip:-} ]] && ssh_node "$paused_ip" "sudo kill -CONT '$replay_pid'"; [[ -n ${paused_ip:-} ]] && sql "$paused_ip" 'SELECT pg_wal_replay_resume()' >/dev/null; wait_cluster >/dev/null 2>&1; } +trap cleanup EXIT INT TERM +case "$scenario" in + replica-loss) + target=$(cluster_json | jq -r '.[]|select(.Role!="Leader")|.Member' | head -1); target_ip=$(name_ip "$target"); service=patroni; alert=PgSentryReplicaCountLow; metric=pgsentry_postgresql_replica_count + evidence_event "$scenario" injected "$metric" 2 "$alert"; ssh_node "$target_ip" 'sudo systemctl stop patroni'; wait_metric "$metric" 1; wait_alert "$alert" firing; wait_alertmanager "$alert" firing; evidence_event "$scenario" firing "$metric" 1 "$alert" + [[ $(prom_query pgsentry_postgresql_primary_count) == 1 ]]; ssh_node "$target_ip" 'sudo systemctl start patroni'; service=; wait_cluster; wait_metric "$metric" 2; wait_alert "$alert" resolved; wait_alertmanager "$alert" resolved + ;; + etcd-member-loss) + target_ip=${ETCD_IPS[2]}; service=etcd; alert=PgSentryEtcdMemberDown; metric=pgsentry_etcd_healthy_members + evidence_event "$scenario" injected "$metric" 3 "$alert"; ssh_node "$target_ip" 'sudo systemctl stop etcd'; wait_metric "$metric" 2; wait_alert "$alert" firing; wait_alertmanager "$alert" firing; evidence_event "$scenario" firing "$metric" 2 "$alert" + [[ $(prom_query pgsentry_postgresql_primary_count) == 1 ]]; ssh_node "$target_ip" 'sudo systemctl start etcd'; service=; wait_metric "$metric" 3; wait_alert "$alert" resolved; wait_alertmanager "$alert" resolved + ;; + haproxy-loss) + target_ip=$CONTROL_IP; service=haproxy; alert=PgSentryHAProxyDown; metric='up{job="haproxy"}' + evidence_event "$scenario" injected "$metric" 1 "$alert"; ssh_node "$target_ip" 'sudo systemctl stop haproxy'; wait_metric "$metric" 0; wait_alert "$alert" firing; wait_alertmanager "$alert" firing; evidence_event "$scenario" firing "$metric" 0 "$alert" + [[ $(prom_query pgsentry_postgresql_primary_count) == 1 ]]; ssh_node "$target_ip" 'sudo systemctl start haproxy'; service=; wait_metric "$metric" 1; wait_metric pgsentry_haproxy_writable_backends 1; wait_alert "$alert" resolved; wait_alertmanager "$alert" resolved + ;; + replication-lag) + target=$(cluster_json | jq -r '.[]|select(.Role!="Leader")|.Member' | head -1); paused_ip=$(name_ip "$target"); alert=PgSentryReplicationLagHigh; metric=pgsentry_replication_lag_bytes + replay_pid=$(ssh_node "$paused_ip" "pgrep -f '^postgres: pgsentry: startup recovering' | head -1") + [[ -n $replay_pid ]] || { echo 'standby recovery process not found' >&2; exit 1; } + ssh_node "$paused_ip" "sudo kill -STOP '$replay_pid'" + leader_ip=$(current_leader_ip); sql "$leader_ip" "CREATE TABLE IF NOT EXISTS public.m10_lag(payload text); INSERT INTO public.m10_lag SELECT repeat(md5(g::text),64) FROM generate_series(1,10000) g" >/dev/null + for _ in $(seq 1 40); do [[ $(prom_query "$metric" | cut -d. -f1) -gt 1048576 ]] && break; sleep 3; done + wait_alert "$alert" firing; wait_alertmanager "$alert" firing; evidence_event "$scenario" firing "$metric" "$(prom_query "$metric")" "$alert"; ssh_node "$paused_ip" "sudo kill -CONT '$replay_pid'"; replay_pid=; paused_ip=; wait_cluster + for _ in $(seq 1 60); do [[ $(prom_query "$metric" | cut -d. -f1) -lt 1048576 ]] && break; sleep 3; done + wait_alert "$alert" resolved; wait_alertmanager "$alert" resolved; sql "$leader_ip" 'DROP TABLE IF EXISTS public.m10_lag' >/dev/null + ;; + *) echo 'ALERT_TEST must be replica-loss, etcd-member-loss, haproxy-loss, or replication-lag' >&2; exit 2 ;; +esac +evidence_event "$scenario" resolved "$metric" "$(prom_query "$metric" 2>/dev/null || true)" "$alert" +echo "$scenario alert lifecycle passed" diff --git a/scripts/observability/check.sh b/scripts/observability/check.sh new file mode 100755 index 0000000..58e1132 --- /dev/null +++ b/scripts/observability/check.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/common.sh"; observability_load "${1:-}" +ssh_node "$CONTROL_IP" 'sudo /usr/local/bin/promtool check config /etc/prometheus/prometheus.yml; sudo /usr/local/bin/promtool check rules /etc/prometheus/rules.yml; sudo /usr/local/bin/amtool check-config /etc/alertmanager/alertmanager.yml; systemctl is-active --quiet prometheus alertmanager grafana-server node_exporter pgsentry-metrics.timer' +for ip in "${PG_IPS[@]}" "${ETCD_IPS[@]}"; do ssh_node "$ip" 'systemctl is-active --quiet node_exporter'; done +echo 'M10 services and rendered monitoring configuration are valid' diff --git a/scripts/observability/clean.sh b/scripts/observability/clean.sh new file mode 100755 index 0000000..0734bbc --- /dev/null +++ b/scripts/observability/clean.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/common.sh"; observability_load "${1:-}" +for ip in "${PG_IPS[@]}" "${ETCD_IPS[@]}" "$CONTROL_IP"; do + ssh_node "$ip" "sudo systemctl disable --now node_exporter 2>/dev/null || true; sudo rm -f /etc/systemd/system/node_exporter.service /usr/local/bin/node_exporter; sudo rm -rf /var/lib/node_exporter; sudo ufw --force delete allow from '$SUBNET' to any port 9100 proto tcp >/dev/null 2>&1 || true" || true +done +ssh_node "$CONTROL_IP" "sudo systemctl disable --now prometheus alertmanager grafana-server pgsentry-metrics.timer 2>/dev/null || true; sudo rm -f /etc/systemd/system/prometheus.service /etc/systemd/system/alertmanager.service /etc/systemd/system/pgsentry-metrics.service /etc/systemd/system/pgsentry-metrics.timer /etc/pgsentry-monitor.env /usr/local/bin/prometheus /usr/local/bin/promtool /usr/local/bin/alertmanager /usr/local/bin/amtool /usr/local/sbin/pgsentry-metrics; sudo rm -rf /etc/prometheus /etc/alertmanager /etc/grafana /var/lib/prometheus /var/lib/alertmanager /var/lib/grafana /var/log/grafana; sudo env DEBIAN_FRONTEND=noninteractive apt-get purge -y -qq grafana >/dev/null 2>&1 || true; sudo ufw --force delete allow from '$SUBNET' to any port 3000 proto tcp >/dev/null 2>&1 || true; sudo systemctl daemon-reload" || true +leader=$(current_leader_ip 2>/dev/null || true); [[ -n $leader ]] && sql "$leader" 'DROP ROLE IF EXISTS pgsentry_monitor' >/dev/null || true +[[ -n ${leader:-} ]] && sql "$leader" 'DROP TABLE IF EXISTS public.m10_lag' >/dev/null || true +rm -rf "$OBS_SECRETS" "$OBS_RUNTIME" "$HA_RUNTIME/grafana-admin-password" "$HA_RUNTIME/monitor-password" +echo 'M10 monitoring services, state, credentials, and evidence removed' diff --git a/scripts/observability/collector.sh b/scripts/observability/collector.sh new file mode 100755 index 0000000..f7011a7 --- /dev/null +++ b/scripts/observability/collector.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail +source /etc/pgsentry-monitor.env +out=$(mktemp) +trap 'rm -f "$out"' EXIT +primary=0 replicas=0 lag=0 archive_fail=0 archive_last=0 connections=0 commits=0 rollbacks=0 locks=0 db_size=0 long_tx=0 +leader= +for ip in 192.168.130.11 192.168.130.12 192.168.130.13; do + role=$(curl -fsS --max-time 3 "http://$ip:8008/metrics" 2>/dev/null | awk '/^patroni_primary[{ ]/{print $NF; exit}' || true) + [[ $role == 1 ]] && { primary=$((primary + 1)); leader=$ip; } +done +if [[ -n $leader ]]; then + sql=$(PGPASSWORD="$PGPASSWORD" psql -h "$leader" -U pgsentry_monitor -d postgres -AtF '|' -c "SELECT count(*) FILTER (WHERE state='streaming'),coalesce(max(pg_wal_lsn_diff(pg_current_wal_lsn(),replay_lsn)),0)::bigint FROM pg_stat_replication; SELECT failed_count,coalesce(extract(epoch from last_archived_time),0)::bigint FROM pg_stat_archiver; SELECT (SELECT count(*) FROM pg_stat_activity),coalesce(sum(xact_commit),0),coalesce(sum(xact_rollback),0),(SELECT count(*) FROM pg_locks WHERE NOT granted),coalesce(sum(pg_database_size(datname)),0),(SELECT count(*) FROM pg_stat_activity WHERE xact_start < clock_timestamp()-interval '5 minutes') FROM pg_stat_database" 2>/dev/null || true) + replicas=$(sed -n '1s/|.*//p' <<<"$sql"); lag=$(sed -n '1s/.*|//p' <<<"$sql") + archive_fail=$(sed -n '2s/|.*//p' <<<"$sql"); archive_last=$(sed -n '2s/.*|//p' <<<"$sql") + IFS='|' read -r connections commits rollbacks locks db_size long_tx <<<"$(sed -n '3p' <<<"$sql")" +fi +etcd_healthy=0 +for endpoint in 192.168.130.21 192.168.130.22 192.168.130.23; do + curl -fsS --max-time 3 --cacert /etc/prometheus/pki/etcd-ca.crt --cert /etc/prometheus/pki/etcd-monitor.crt --key /etc/prometheus/pki/etcd-monitor.key "https://$endpoint:2379/health" 2>/dev/null | jq -e '.health=="true" or .health==true' >/dev/null && etcd_healthy=$((etcd_healthy + 1)) || true +done +writable=$(printf 'show stat\n' | socat - UNIX-CONNECT:/run/haproxy/admin.sock 2>/dev/null | awk -F, '$1=="patroni_primary" && $2!="BACKEND" && $18=="UP"{n++} END{print n+0}') +backup_ts=0 +if command -v pgbackrest >/dev/null && sudo -u postgres pgbackrest --stanza=pgsentry info --output=json >/tmp/pgsentry-backup.json 2>/dev/null; then + backup_ts=$(jq '[.[0].backup[]?|select(.type=="full")|.timestamp.stop]|max//0' /tmp/pgsentry-backup.json) + rm -f /tmp/pgsentry-backup.json +fi +cat >"$out" <&2; exit 2; } + backup_load "$1" + OBS_RUNTIME="$PGSENTRY_RUNTIME/results/m10" + OBS_SECRETS="$PGSENTRY_RUNTIME/observability" + mkdir -p "$OBS_RUNTIME" "$OBS_SECRETS" + chmod 700 "$PGSENTRY_RUNTIME" "$PGSENTRY_RUNTIME/results" "$OBS_RUNTIME" "$OBS_SECRETS" +} + +prom_query() { + local query=$1 encoded + encoded=$(jq -rn --arg q "$query" '$q|@uri') + ssh_node "$CONTROL_IP" "curl -fsS 'http://127.0.0.1:$PROMETHEUS_PORT/api/v1/query?query=$encoded'" | + jq -r '.data.result[0].value[1] // empty' +} + +wait_metric() { + local query=$1 expected=$2 value + for _ in $(seq 1 40); do + value=$(prom_query "$query" 2>/dev/null || true) + [[ $value == "$expected" ]] && return + sleep 3 + done + echo "metric did not reach $query = $expected (last=${value:-missing})" >&2 + return 1 +} + +alert_state() { + local name=$1 + ssh_node "$CONTROL_IP" "curl -fsS http://127.0.0.1:$PROMETHEUS_PORT/api/v1/alerts" | + jq -r --arg n "$name" '[.data.alerts[] | select(.labels.alertname==$n and .state=="firing")]|length' +} + +wait_alert() { + local name=$1 wanted=$2 count + for _ in $(seq 1 50); do + count=$(alert_state "$name" 2>/dev/null || echo 0) + if [[ $wanted == firing && $count -gt 0 ]] || [[ $wanted == resolved && $count -eq 0 ]]; then return; fi + sleep 3 + done + echo "alert $name did not become $wanted" >&2 + return 1 +} + +alertmanager_count() { + local name=$1 + ssh_node "$CONTROL_IP" "curl -fsS 'http://127.0.0.1:$ALERTMANAGER_PORT/api/v2/alerts?active=true&silenced=false&inhibited=false'" | + jq -r --arg n "$name" '[.[]|select(.labels.alertname==$n)]|length' +} + +wait_alertmanager() { + local name=$1 wanted=$2 count + for _ in $(seq 1 40); do + count=$(alertmanager_count "$name" 2>/dev/null || echo 0) + if [[ $wanted == firing && $count -gt 0 ]] || [[ $wanted == resolved && $count -eq 0 ]]; then return; fi + sleep 3 + done + echo "Alertmanager alert $name did not become $wanted" >&2 + return 1 +} + +evidence_event() { + local scenario=$1 phase=$2 metric=$3 value=$4 alert=$5 + jq -nc --arg scenario "$scenario" --arg phase "$phase" --arg metric "$metric" --arg value "$value" \ + --arg alert "$alert" --arg timestamp "$(date -u +%FT%TZ)" \ + '{scenario:$scenario,phase:$phase,metric:$metric,value:$value,alert:$alert,timestamp:$timestamp}' >>"$OBS_RUNTIME/alerts.jsonl" +} + +prepare_monitor_etcd_cert() { + local pki="$ETCD_RUNTIME/pki" + [[ -s $pki/ca.key ]] || { echo 'etcd CA missing; run etcd-configure first' >&2; exit 1; } + [[ -s $OBS_SECRETS/etcd-monitor.crt ]] && return + umask 077 + openssl genrsa -out "$OBS_SECRETS/etcd-monitor.key" 2048 >/dev/null 2>&1 + openssl req -new -key "$OBS_SECRETS/etcd-monitor.key" -subj '/CN=pgsentry-m10-monitor' -out "$OBS_SECRETS/etcd-monitor.csr" + printf 'extendedKeyUsage=clientAuth\n' >"$OBS_SECRETS/etcd-monitor.ext" + openssl x509 -req -in "$OBS_SECRETS/etcd-monitor.csr" -CA "$pki/ca.crt" -CAkey "$pki/ca.key" -CAcreateserial -days 825 -sha256 -extfile "$OBS_SECRETS/etcd-monitor.ext" -out "$OBS_SECRETS/etcd-monitor.crt" >/dev/null 2>&1 +} diff --git a/scripts/observability/configure-control.sh b/scripts/observability/configure-control.sh new file mode 100755 index 0000000..b961e19 --- /dev/null +++ b/scripts/observability/configure-control.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +set -euo pipefail +prom_version=$1 prom_sha=$2 am_version=$3 am_sha=$4 grafana_version=$5 grafana_sha=$6 subnet=$7 grafana_password=$8 monitor_password=$9 +install_archive() { + local component=$1 version=$2 sha=$3 binary=$4 + local archive="${component}-${version}.linux-amd64.tar.gz" tmp url + url="https://github.com/prometheus/${component}/releases/download/v${version}/${archive}" + tmp=$(mktemp -d); curl -fsSL "$url" -o "$tmp/$archive"; echo "$sha $tmp/$archive" | sha256sum -c - + tar -xzf "$tmp/$archive" -C "$tmp"; sudo install -m 0755 "$tmp/${component}-${version}.linux-amd64/$binary" "/usr/local/bin/$binary"; rm -rf "$tmp" +} +id prometheus >/dev/null 2>&1 || sudo useradd --system --no-create-home --shell /usr/sbin/nologin prometheus +id alertmanager >/dev/null 2>&1 || sudo useradd --system --no-create-home --shell /usr/sbin/nologin alertmanager +install_archive prometheus "$prom_version" "$prom_sha" prometheus +tmp=$(mktemp -d); curl -fsSL "https://github.com/prometheus/prometheus/releases/download/v${prom_version}/prometheus-${prom_version}.linux-amd64.tar.gz" -o "$tmp/prometheus.tar.gz"; echo "$prom_sha $tmp/prometheus.tar.gz" | sha256sum -c -; tar -xzf "$tmp/prometheus.tar.gz" -C "$tmp"; sudo install -m 0755 "$tmp/prometheus-${prom_version}.linux-amd64/promtool" /usr/local/bin/promtool; rm -rf "$tmp" +install_archive alertmanager "$am_version" "$am_sha" alertmanager +tmp=$(mktemp -d); curl -fsSL "https://github.com/prometheus/alertmanager/releases/download/v${am_version}/alertmanager-${am_version}.linux-amd64.tar.gz" -o "$tmp/alertmanager.tar.gz"; echo "$am_sha $tmp/alertmanager.tar.gz" | sha256sum -c -; tar -xzf "$tmp/alertmanager.tar.gz" -C "$tmp"; sudo install -m 0755 "$tmp/alertmanager-${am_version}.linux-amd64/amtool" /usr/local/bin/amtool; rm -rf "$tmp" +sudo apt-get update -qq +sudo apt-get install -y -qq jq postgresql-client curl >/dev/null +if ! dpkg-query -W -f='${Version}' grafana 2>/dev/null | grep -q "^${grafana_version}"; then + tmp=$(mktemp); curl -fsSL "https://dl.grafana.com/oss/release/grafana_${grafana_version}_amd64.deb" -o "$tmp"; echo "$grafana_sha $tmp" | sha256sum -c -; sudo dpkg -i "$tmp" >/dev/null || sudo apt-get -f install -y -qq; rm -f "$tmp" +fi +sudo install -d -o prometheus -g prometheus -m 0750 /etc/prometheus /var/lib/prometheus /etc/prometheus/pki +sudo install -d -o alertmanager -g alertmanager -m 0750 /etc/alertmanager /var/lib/alertmanager +sudo install -d -o grafana -g grafana -m 0750 /var/lib/grafana/dashboards +sudo tee /etc/pgsentry-monitor.env >/dev/null </dev/null <<'UNIT' +[Unit] +Description=PgSentry Prometheus +After=network-online.target alertmanager.service +[Service] +User=prometheus +Group=prometheus +ExecStart=/usr/local/bin/prometheus --config.file=/etc/prometheus/prometheus.yml --storage.tsdb.path=/var/lib/prometheus --storage.tsdb.retention.time=7d --web.listen-address=127.0.0.1:9090 +Restart=on-failure +[Install] +WantedBy=multi-user.target +UNIT +sudo tee /etc/systemd/system/alertmanager.service >/dev/null <<'UNIT' +[Unit] +Description=PgSentry Alertmanager +After=network-online.target +[Service] +User=alertmanager +Group=alertmanager +ExecStart=/usr/local/bin/alertmanager --config.file=/etc/alertmanager/alertmanager.yml --storage.path=/var/lib/alertmanager --web.listen-address=127.0.0.1:9093 +Restart=on-failure +[Install] +WantedBy=multi-user.target +UNIT +sudo tee /etc/systemd/system/pgsentry-metrics.service >/dev/null <<'UNIT' +[Unit] +Description=Collect PgSentry semantic metrics +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/pgsentry-metrics +UNIT +sudo tee /etc/systemd/system/pgsentry-metrics.timer >/dev/null <<'UNIT' +[Unit] +Description=Collect PgSentry semantic metrics every 15 seconds +[Timer] +OnBootSec=10s +OnUnitActiveSec=15s +AccuracySec=1s +[Install] +WantedBy=timers.target +UNIT +sudo ufw allow from "$subnet" to any port 3000 proto tcp comment 'pgsentry-m10-grafana' >/dev/null +sudo systemctl daemon-reload +sudo systemctl enable --now alertmanager prometheus grafana-server pgsentry-metrics.timer +sudo systemctl start pgsentry-metrics.service diff --git a/scripts/observability/configure.sh b/scripts/observability/configure.sh new file mode 100755 index 0000000..8d7c3b9 --- /dev/null +++ b/scripts/observability/configure.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/common.sh"; observability_load "${1:-}"; prepare_monitor_etcd_cert +grafana_password=$(runtime_secret grafana-admin-password) +monitor_password=$(runtime_secret monitor-password) +leader=$(current_leader_ip) +sql "$leader" "DO \$\$ BEGIN IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='pgsentry_monitor') THEN CREATE ROLE pgsentry_monitor LOGIN PASSWORD '$monitor_password'; END IF; END \$\$; ALTER ROLE pgsentry_monitor PASSWORD '$monitor_password'; GRANT pg_monitor TO pgsentry_monitor;" >/dev/null +for ip in "${PG_IPS[@]}"; do + ssh_node "$ip" "sudo grep -q 'pgsentry_monitor' /etc/postgresql/16/main/pg_hba.conf || echo 'host all pgsentry_monitor $CONTROL_IP/32 scram-sha-256' | sudo tee -a /etc/postgresql/16/main/pg_hba.conf >/dev/null; sudo -u postgres psql -Atc 'select pg_reload_conf()' >/dev/null" +done +all_ips=("${PG_IPS[@]}" "${ETCD_IPS[@]}" "$CONTROL_IP") +for ip in "${all_ips[@]}"; do wait_for_ssh "$ip"; run_remote_script "$ip" "$PGSENTRY_ROOT/scripts/observability/install-node.sh" "$NODE_EXPORTER_VERSION" "$NODE_EXPORTER_SHA256" "$SUBNET"; done +ssh_node "$CONTROL_IP" 'sudo install -d -m 0750 /etc/prometheus/pki /etc/alertmanager /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards /var/lib/grafana/dashboards' +copy_remote "$CONTROL_IP" "$PGSENTRY_ROOT/monitoring/prometheus/prometheus.yml" /etc/prometheus/prometheus.yml 0644 +copy_remote "$CONTROL_IP" "$PGSENTRY_ROOT/monitoring/prometheus/rules.yml" /etc/prometheus/rules.yml 0644 +copy_remote "$CONTROL_IP" "$PGSENTRY_ROOT/monitoring/alertmanager/alertmanager.yml" /etc/alertmanager/alertmanager.yml 0644 +copy_remote "$CONTROL_IP" "$ETCD_RUNTIME/pki/ca.crt" /etc/prometheus/pki/etcd-ca.crt 0644 +copy_remote "$CONTROL_IP" "$OBS_SECRETS/etcd-monitor.crt" /etc/prometheus/pki/etcd-monitor.crt 0644 +copy_remote "$CONTROL_IP" "$OBS_SECRETS/etcd-monitor.key" /etc/prometheus/pki/etcd-monitor.key 0600 +copy_remote "$CONTROL_IP" "$PGSENTRY_ROOT/scripts/observability/collector.sh" /usr/local/sbin/pgsentry-metrics 0755 +for f in "$PGSENTRY_ROOT"/monitoring/grafana/dashboards/*.json; do copy_remote "$CONTROL_IP" "$f" "/var/lib/grafana/dashboards/$(basename "$f")" 0644; done +copy_remote "$CONTROL_IP" "$PGSENTRY_ROOT/monitoring/grafana/provisioning/datasources/prometheus.yml" /etc/grafana/provisioning/datasources/pgsentry.yml 0644 +copy_remote "$CONTROL_IP" "$PGSENTRY_ROOT/monitoring/grafana/provisioning/dashboards/pgsentry.yml" /etc/grafana/provisioning/dashboards/pgsentry.yml 0644 +printf '%s\n%s\n' "$grafana_password" "$monitor_password" | run_remote_script_with_input "$CONTROL_IP" "$PGSENTRY_ROOT/scripts/observability/configure-control.sh" "$PROMETHEUS_VERSION" "$PROMETHEUS_SHA256" "$ALERTMANAGER_VERSION" "$ALERTMANAGER_SHA256" "$GRAFANA_VERSION" "$GRAFANA_SHA256" "$SUBNET" "'$grafana_password'" "'$monitor_password'" +ssh_node "$CONTROL_IP" 'sudo chown -R prometheus:prometheus /etc/prometheus /var/lib/prometheus; sudo chown -R alertmanager:alertmanager /etc/alertmanager /var/lib/alertmanager; sudo chown -R grafana:grafana /var/lib/grafana/dashboards; sudo systemctl restart alertmanager prometheus grafana-server pgsentry-metrics.timer; sudo systemctl start pgsentry-metrics.service' +echo "Grafana=http://$CONTROL_IP:$GRAFANA_PORT user=admin password=$HA_RUNTIME/grafana-admin-password" diff --git a/scripts/observability/install-node.sh b/scripts/observability/install-node.sh new file mode 100755 index 0000000..14fbfb6 --- /dev/null +++ b/scripts/observability/install-node.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail +version=$1 sha=$2 subnet=$3 +archive="node_exporter-${version}.linux-amd64.tar.gz" +url="https://github.com/prometheus/node_exporter/releases/download/v${version}/${archive}" +if ! id node_exporter >/dev/null 2>&1; then sudo useradd --system --no-create-home --shell /usr/sbin/nologin node_exporter; fi +if [[ ! -x /usr/local/bin/node_exporter ]] || [[ $(/usr/local/bin/node_exporter --version 2>&1 | awk '/version/{print $3; exit}') != "$version" ]]; then + tmp=$(mktemp -d); trap 'rm -rf "$tmp"' EXIT + curl -fsSL "$url" -o "$tmp/$archive" + echo "$sha $tmp/$archive" | sha256sum -c - + tar -xzf "$tmp/$archive" -C "$tmp" + sudo install -m 0755 "$tmp/node_exporter-${version}.linux-amd64/node_exporter" /usr/local/bin/node_exporter +fi +sudo install -d -o node_exporter -g node_exporter -m 0755 /var/lib/node_exporter/textfile +sudo tee /etc/systemd/system/node_exporter.service >/dev/null <<'UNIT' +[Unit] +Description=PgSentry node exporter +After=network-online.target +[Service] +User=node_exporter +Group=node_exporter +ExecStart=/usr/local/bin/node_exporter --web.listen-address=:9100 --collector.textfile.directory=/var/lib/node_exporter/textfile +NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=/var/lib/node_exporter/textfile +[Install] +WantedBy=multi-user.target +UNIT +sudo ufw allow from "$subnet" to any port 9100 proto tcp comment 'pgsentry-m10-node-exporter' >/dev/null +sudo systemctl daemon-reload +sudo systemctl enable --now node_exporter diff --git a/scripts/observability/static-check.sh b/scripts/observability/static-check.sh new file mode 100755 index 0000000..a676940 --- /dev/null +++ b/scripts/observability/static-check.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +root=$(cd "$(dirname "$0")/../.." && pwd) +tmp= +cleanup() { [[ -n $tmp ]] && rm -rf "$tmp"; } +trap cleanup EXIT +promtool=${PROMTOOL:-promtool}; amtool=${AMTOOL:-amtool} +if ! command -v "$promtool" >/dev/null || ! command -v "$amtool" >/dev/null; then + tmp=$(mktemp -d) + curl -fsSL https://github.com/prometheus/prometheus/releases/download/v3.13.1/prometheus-3.13.1.linux-amd64.tar.gz -o "$tmp/prometheus.tgz" + echo '962b812371aff838d152b6ff2d56fdb7a6396f5542f48ebf73421b9721f0d103 '"$tmp/prometheus.tgz" | sha256sum -c - + tar -xzf "$tmp/prometheus.tgz" -C "$tmp" + curl -fsSL https://github.com/prometheus/alertmanager/releases/download/v0.33.1/alertmanager-0.33.1.linux-amd64.tar.gz -o "$tmp/alertmanager.tgz" + echo '93d802cba6a8d27239d747ce117df7648d326ab67394e32247540b030e9842ba '"$tmp/alertmanager.tgz" | sha256sum -c - + tar -xzf "$tmp/alertmanager.tgz" -C "$tmp" + promtool="$tmp/prometheus-3.13.1.linux-amd64/promtool" + amtool="$tmp/alertmanager-0.33.1.linux-amd64/amtool" +fi +rendered="$tmp/prometheus.yml" +[[ -n $tmp ]] || { tmp=$(mktemp -d); rendered="$tmp/prometheus.yml"; } +openssl req -x509 -newkey rsa:2048 -nodes -subj /CN=static-check -keyout "$tmp/check.key" -out "$tmp/check.crt" -days 1 >/dev/null 2>&1 +sed -e "s|/etc/prometheus/rules.yml|$root/monitoring/prometheus/rules.yml|" \ + -e "s|/etc/prometheus/pki/etcd-ca.crt|$tmp/check.crt|" \ + -e "s|/etc/prometheus/pki/etcd-monitor.crt|$tmp/check.crt|" \ + -e "s|/etc/prometheus/pki/etcd-monitor.key|$tmp/check.key|" \ + "$root/monitoring/prometheus/prometheus.yml" >"$rendered" +"$promtool" check config "$rendered" +"$promtool" check rules "$root/monitoring/prometheus/rules.yml" +"$amtool" check-config "$root/monitoring/alertmanager/alertmanager.yml" +find "$root/monitoring/grafana/dashboards" -name '*.json' -print0 | xargs -0 -n1 jq empty +grep -Rho 'runbook: docs/runbooks/[^ }]*' "$root/monitoring/prometheus/rules.yml" | awk '{print $2}' | while read -r file; do test -s "$root/$file" || { echo "missing runbook: $file" >&2; exit 1; }; done diff --git a/scripts/observability/verify.sh b/scripts/observability/verify.sh new file mode 100755 index 0000000..54b1282 --- /dev/null +++ b/scripts/observability/verify.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "$0")/common.sh"; observability_load "${1:-}" +wait_metric 'pgsentry_postgresql_primary_count' 1 +wait_metric 'pgsentry_postgresql_replica_count' 2 +wait_metric 'pgsentry_etcd_healthy_members' 3 +wait_metric 'pgsentry_haproxy_writable_backends' 1 +targets=$(ssh_node "$CONTROL_IP" "curl -fsS http://127.0.0.1:$PROMETHEUS_PORT/api/v1/targets") +down=$(jq '[.data.activeTargets[]|select(.health!="up")]|length' <<<"$targets") +expected=$(jq '[.data.activeTargets[]]|length' <<<"$targets") +[[ $down -eq 0 && $expected -eq 15 ]] || { jq '.data.activeTargets[]|select(.health!="up")|{scrapeUrl,lastError}' <<<"$targets"; echo "expected 15 UP targets, got $expected total / $down down" >&2; exit 1; } +backup=$(prom_query 'pgsentry_backup_last_success_timestamp_seconds') +[[ ${backup:-0} -gt 0 ]] || { echo 'backup timestamp metric missing; run backup-full' >&2; exit 1; } +critical=$(ssh_node "$CONTROL_IP" "curl -fsS http://127.0.0.1:$PROMETHEUS_PORT/api/v1/alerts" | jq '[.data.alerts[]|select(.state=="firing" and .labels.severity=="critical")]|length') +[[ $critical -eq 0 ]] || { echo "$critical unexplained critical alerts" >&2; exit 1; } +dashboards=$(ssh_node "$CONTROL_IP" "curl -fsS -u admin:'$(cat "$HA_RUNTIME/grafana-admin-password")' 'http://127.0.0.1:$GRAFANA_PORT/api/search?type=dash-db'") +[[ $(jq 'length' <<<"$dashboards") -ge 3 ]] || { echo 'Grafana dashboards not provisioned' >&2; exit 1; } +cat >"$OBS_RUNTIME/healthy.json" <