diff --git a/README.md b/README.md index 25d336b2..4bcb1a67 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,7 @@ Supported deployment shapes include virtual machines, dedicated physical machine | Build a compatible project contract | [Project CI standard](docs/PROJECT-STANDARD.md) | | Verify project compliance | [Compliance checklist](docs/COMPLIANCE-CHECKLIST.md) | | Configure upgrades, cleanup, draining, and rebooting | [Host maintenance](docs/HOST-MAINTENANCE.md) | +| Configure host health and external missed-heartbeat detection | [Fleet health monitoring](docs/HEALTH-MONITORING.md) | | Understand secret storage and injection | [Secrets model](docs/SECRETS.md) | | Use private fleet workers for a public project | [Public projects and private delivery](docs/PUBLIC-PRIVATE-CONFIGURATION.md) | | See planned work | [Roadmap](docs/ROADMAP.md) | diff --git a/controller/Dockerfile b/controller/Dockerfile index 1c5b92c1..39634c75 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -10,7 +10,8 @@ RUN CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=${CI_FLEET_ FROM debian:13.6-slim ARG CI_FLEET_COMMIT=unknown -LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" +LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" \ + io.randomdevelopment.ci-fleet.managed="true" RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=build /out/ci-fleet-controller /usr/local/bin/ci-fleet-controller USER 65532:65532 diff --git a/docs/ADDING-A-HOST.md b/docs/ADDING-A-HOST.md index e39361b2..d1db8bbd 100644 --- a/docs/ADDING-A-HOST.md +++ b/docs/ADDING-A-HOST.md @@ -122,7 +122,7 @@ Follow [Live pilot](LIVE-PILOT.md) for the complete proof and rollback. ## 8. Verify unattended operations -The installer enables health, scoped cleanup, and pinned desired-state drift timers. Run each service once and inspect its journal as described in [Host maintenance](HOST-MAINTENANCE.md). Monitor controller state, disk thresholds, Docker health, last successful job, cleanup failures, drift, and pending reboot state. +The installer enables health, scoped cleanup, and pinned desired-state drift timers. Run each service once and inspect its journal as described in [Host maintenance](HOST-MAINTENANCE.md). Configure redacted local checks and external missed-heartbeat detection as described in [Fleet health monitoring](HEALTH-MONITORING.md). The steady-state host should need no project-specific edits. Adding a project changes its repository, private fleet configuration, and GitHub runner-group policy—not this host. diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index 3f5bd9d8..547be748 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -153,4 +153,6 @@ Before mutation, the installer records the prior rendered environment, installat These controller checkpoints do not replace machine backups. Operators still create and verify VM snapshots, physical-host recovery media, or equivalent infrastructure backups according to their local policy. +Monitoring thresholds, heartbeat endpoints, and backup hooks are host-local operational facts, not fleet desired state. Keep them in the protected file documented by [Fleet health monitoring](HEALTH-MONITORING.md); the installer preserves that file across upgrades and rollback. + This contract implements the engine portions of [issue #32](https://github.com/RandomDevelopment/ci-fleet/issues/32) and integrates the installer, documentation, phone-first bootstrap, and capacity work tracked by [#21](https://github.com/RandomDevelopment/ci-fleet/issues/21), [#24](https://github.com/RandomDevelopment/ci-fleet/issues/24), [#27](https://github.com/RandomDevelopment/ci-fleet/issues/27), and [#30](https://github.com/RandomDevelopment/ci-fleet/issues/30). diff --git a/docs/HEALTH-MONITORING.md b/docs/HEALTH-MONITORING.md new file mode 100644 index 00000000..b9549dd2 --- /dev/null +++ b/docs/HEALTH-MONITORING.md @@ -0,0 +1,76 @@ +# Fleet health monitoring + +Every managed controller runs a redacted local health check every five minutes. An external monitor must also detect a host that cannot report because it is offline. + +## Local check + +```bash +sudo /opt/ci-fleet/manager/current/scripts/healthcheck.sh +sudo /opt/ci-fleet/manager/current/scripts/healthcheck.sh --json +sudo cat /var/lib/ci-fleet/health/latest.json +``` + +Exit codes are `0` for healthy or intentional maintenance, `1` for warning, and `2` for unhealthy. The JSON schema is versioned and the latest result is replaced atomically. It contains controller identity, desired lifecycle state, status, timestamp, and redacted check results only. + +The check covers: + +- root and Docker filesystem space and inodes; +- available memory, swap use, per-CPU load, and OOM evidence from the last 24 hours; +- Docker availability, controller state/restarts, and configured versus effective capacity; +- inactive, unhealthy, restarting, and stale fleet-labelled resources, including week-old build cache; +- cleanup, drift, health, and update services/timers; +- failed package state, pending reboot, and clock synchronization; +- an optional host-local backup check; +- optional outbound heartbeat delivery. + +It reports but never prunes, restarts, or repairs resources. Project source, logs, environment values, tokens, and private keys are never included. + +## Threshold overrides and hooks + +Defaults are intentionally conservative: disk and inode warning/critical at 80/90%, available memory warning/critical at 15/8%, sustained swap use under five-minute memory pressure warning/critical at 25/50%, per-CPU fifteen-minute load warning/critical at 1.0/1.5, and controller restart warning at 3. + +Optional overrides belong in `/etc/ci-fleet/monitoring.env`, owned by root with mode `0600`: + +```text +CI_FLEET_HEALTH_DISK_WARN_PERCENT=80 +CI_FLEET_HEALTH_DISK_CRITICAL_PERCENT=90 +CI_FLEET_HEALTH_INODE_WARN_PERCENT=80 +CI_FLEET_HEALTH_INODE_CRITICAL_PERCENT=90 +CI_FLEET_HEALTH_MEMORY_WARN_AVAILABLE_PERCENT=15 +CI_FLEET_HEALTH_MEMORY_CRITICAL_AVAILABLE_PERCENT=8 +CI_FLEET_HEALTH_SWAP_WARN_PERCENT=25 +CI_FLEET_HEALTH_SWAP_CRITICAL_PERCENT=50 +CI_FLEET_HEALTH_LOAD_WARN_PER_CPU=1.0 +CI_FLEET_HEALTH_LOAD_CRITICAL_PER_CPU=1.5 +CI_FLEET_HEALTH_RESTART_WARN_COUNT=3 +CI_FLEET_HEALTH_BACKUP_CHECK=/usr/local/sbin/ci-fleet-backup-check +CI_FLEET_HEALTH_HEARTBEAT_URL=https://monitor.example.invalid/heartbeat +CI_FLEET_HEALTH_HEARTBEAT_TOKEN_FILE=/etc/ci-fleet/secrets/heartbeat-token +``` + +The backup hook must be an absolute, executable, root-owned file that is not group- or world-writable. Its output is discarded; only its exit status is reported. The heartbeat URL must use HTTPS. An optional token file must be root-owned and inaccessible to group/other users. The installer never creates, prints, commits, or removes this host-local file or its credentials, so rollback preserves them. + +## External missed-heartbeat detection + +A receiver accepts the redacted JSON POST and stores the most recent body as `.json`. Receiver implementation, endpoint, credential, address, and alert destination are provider-local. The external monitor evaluates those files against reviewed desired state: + +```bash +python3 scripts/health.py heartbeats \ + --config /srv/rd-delivery-config/fleet.json \ + --input-dir /var/lib/ci-fleet-heartbeats \ + --grace-seconds 900 \ + --json +``` + +An active host with no fresh record is unhealthy. A drained host reports maintenance without a false alarm; a disabled host reports retired. A monitoring outage therefore cannot silently turn missing hosts healthy. + +## Operations + +- **Disk/inodes:** inspect fleet-labelled resources and run `scripts/cleanup.sh` in report mode first. Never use global Docker prune. +- **Docker/controller:** drain if possible, inspect Docker and controller journals, then apply only reviewed desired state. +- **Drift/timer failure:** run the named service manually and `install-worker-controller.sh --check`; repair by applying the reviewed pinned configuration, not by editing rendered files. +- **Memory/OOM/load:** let active jobs drain, inspect kernel evidence, and adjust reviewed infrastructure capacity or runner resources. +- **Updates/reboot:** drain before rebooting; verify all timers and the health result afterward. +- **Missed heartbeat:** verify the receiver first, then use the provider console or out-of-band access. Inbound SSH is not required. +- **Add/replace:** enroll the logical controller through reviewed private desired state, configure its host-local heartbeat credential, and verify a fresh external record before relying on it. +- **Retire:** set lifecycle/state through reviewed desired state first. Delete no host, runner, or production resource without separate authorization. diff --git a/docs/HOST-MAINTENANCE.md b/docs/HOST-MAINTENANCE.md index 72e0a145..d9e26a9d 100644 --- a/docs/HOST-MAINTENANCE.md +++ b/docs/HOST-MAINTENANCE.md @@ -30,7 +30,7 @@ Review `/etc/apt/apt.conf.d/50unattended-upgrades` and confirm only the intended `scripts/install-worker-controller.sh` installs and enables all three timer pairs: -- `ci-fleet-health.timer` checks Docker, disk, and the controller's desired runtime state; +- `ci-fleet-health.timer` runs the complete [fleet health contract](HEALTH-MONITORING.md); - `ci-fleet-cleanup.timer` removes only expired inactive fleet-owned resources; - `ci-fleet-drift.timer` compares the installation with the exact pinned configuration commit without applying changes. diff --git a/docs/README.md b/docs/README.md index 4d6eb0e3..3bddefe2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,6 +17,7 @@ Use this index to find ci-fleet concepts, requirements, examples, and step-by-st | Make a project compliant | [Project CI standard](PROJECT-STANDARD.md) and [compliance checklist](COMPLIANCE-CHECKLIST.md) | | Split tests across parallel workers | [Project CI standard](PROJECT-STANDARD.md) and the [parallel workflow example](../examples/workflows/parallel-ci.yml.example) | | Configure automatic updates and cleanup | [Host maintenance](HOST-MAINTENANCE.md) | +| Monitor hosts and detect missed heartbeats | [Fleet health monitoring](HEALTH-MONITORING.md) | | Handle GitHub App, workflow, or deployment secrets | [Secrets model](SECRETS.md) and [security policy](../SECURITY.md) | | Review accepted implementation scope | [Design decisions](DESIGN-DECISIONS.md) | | Run private CI or deployment for a public project | [Public projects, private delivery, and private configuration](PUBLIC-PRIVATE-CONFIGURATION.md) | @@ -55,6 +56,7 @@ These pages are normative for compatible projects and hosts: - [Migration procedure](MIGRATING-EXISTING-CI.md) - [Compliance checklist](COMPLIANCE-CHECKLIST.md) - [Host maintenance standard](HOST-MAINTENANCE.md) +- [Fleet health monitoring](HEALTH-MONITORING.md) - [Git-authored controller desired state](DESIRED-STATE.md) - [Secrets model](SECRETS.md) - [Security policy](../SECURITY.md) diff --git a/host/systemd/ci-fleet-health.service b/host/systemd/ci-fleet-health.service index b33bc8fc..51a2727d 100644 --- a/host/systemd/ci-fleet-health.service +++ b/host/systemd/ci-fleet-health.service @@ -9,3 +9,4 @@ User=root WorkingDirectory=/opt/ci-fleet/manager/current EnvironmentFile=/etc/ci-fleet/ci-fleet.env ExecStart=/opt/ci-fleet/manager/current/scripts/healthcheck.sh +SuccessExitStatus=1 diff --git a/runner/Dockerfile b/runner/Dockerfile index 6b0b6030..d428597d 100644 --- a/runner/Dockerfile +++ b/runner/Dockerfile @@ -6,7 +6,8 @@ ARG CI_FLEET_COMMIT=unknown ARG RUNNER_VERSION=2.335.1 ARG RUNNER_SHA256_AMD64=4ef2f25285f0ae4477f1fe1e346db76d2f3ebf03824e2ddd1973a2819bf6c8cf ARG RUNNER_SHA256_ARM64=6d1e85bfd1a506a8b17c1f1b9b57dba458ffed90898799aaa9f599520b0d9207 -LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" +LABEL org.opencontainers.image.revision="${CI_FLEET_COMMIT}" \ + io.randomdevelopment.ci-fleet.managed="true" SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN apt-get update \ diff --git a/scripts/health.py b/scripts/health.py new file mode 100644 index 00000000..ab054aa6 --- /dev/null +++ b/scripts/health.py @@ -0,0 +1,460 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import os +import re +import stat +import subprocess +import sys +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable + + +@dataclass(frozen=True) +class Thresholds: + disk_warn_percent: int = 80 + disk_critical_percent: int = 90 + inode_warn_percent: int = 80 + inode_critical_percent: int = 90 + memory_warn_available_percent: int = 15 + memory_critical_available_percent: int = 8 + swap_warn_percent: int = 25 + swap_critical_percent: int = 50 + restart_warn_count: int = 3 + load_warn_per_cpu: float = 1.0 + load_critical_per_cpu: float = 1.5 + + +def thresholds_from(values: dict[str, str]) -> Thresholds: + mapping = { + "disk_warn_percent": "CI_FLEET_HEALTH_DISK_WARN_PERCENT", + "disk_critical_percent": "CI_FLEET_HEALTH_DISK_CRITICAL_PERCENT", + "inode_warn_percent": "CI_FLEET_HEALTH_INODE_WARN_PERCENT", + "inode_critical_percent": "CI_FLEET_HEALTH_INODE_CRITICAL_PERCENT", + "memory_warn_available_percent": "CI_FLEET_HEALTH_MEMORY_WARN_AVAILABLE_PERCENT", + "memory_critical_available_percent": "CI_FLEET_HEALTH_MEMORY_CRITICAL_AVAILABLE_PERCENT", + "swap_warn_percent": "CI_FLEET_HEALTH_SWAP_WARN_PERCENT", + "swap_critical_percent": "CI_FLEET_HEALTH_SWAP_CRITICAL_PERCENT", + "restart_warn_count": "CI_FLEET_HEALTH_RESTART_WARN_COUNT", + } + defaults = Thresholds() + kwargs = {field: int(values.get(env, getattr(defaults, field))) for field, env in mapping.items()} + kwargs["load_warn_per_cpu"] = float(values.get("CI_FLEET_HEALTH_LOAD_WARN_PER_CPU", defaults.load_warn_per_cpu)) + kwargs["load_critical_per_cpu"] = float(values.get("CI_FLEET_HEALTH_LOAD_CRITICAL_PER_CPU", defaults.load_critical_per_cpu)) + for field, value in kwargs.items(): + if value < 0 or (field.endswith("percent") and value > 100): + raise ValueError(f"invalid health threshold: {field}") + if kwargs["disk_warn_percent"] >= kwargs["disk_critical_percent"]: + raise ValueError("disk thresholds must increase from warning to critical") + if kwargs["inode_warn_percent"] >= kwargs["inode_critical_percent"]: + raise ValueError("inode thresholds must increase from warning to critical") + if kwargs["memory_critical_available_percent"] >= kwargs["memory_warn_available_percent"]: + raise ValueError("memory available thresholds must decrease from warning to critical") + if kwargs["swap_warn_percent"] >= kwargs["swap_critical_percent"]: + raise ValueError("swap thresholds must increase from warning to critical") + if kwargs["load_warn_per_cpu"] >= kwargs["load_critical_per_cpu"]: + raise ValueError("load thresholds must increase from warning to critical") + return Thresholds(**kwargs) + + +def render_human(report: dict[str, Any]) -> str: + lines = [f'{report["status"].upper()} controller={report.get("controller", "fleet")}'] + lines.extend(f'{check["status"].upper()} {check["id"]}' for check in report.get("checks", []) if check["status"] != "ok") + return "\n".join(lines) + + +def evaluate_heartbeats( + controllers: dict[str, dict[str, Any]], + records: dict[str, dict[str, Any]], + *, + now: int, + grace_seconds: int, +) -> dict[str, Any]: + hosts = [] + rank = 0 + for controller, desired in sorted(controllers.items()): + state = desired["state"] + if state == "disabled": + status = "retired" + elif state == "drained": + status = "maintenance" + else: + record = records.get(controller) + try: + timestamp = int(record["timestamp"]) if record else 0 + reported = record["status"] if record else "" + reported_controller = record["controller"] if record else "" + except (KeyError, TypeError, ValueError): + timestamp, reported, reported_controller = 0, "", "" + if reported_controller != controller or abs(now - timestamp) > grace_seconds or reported not in {"healthy", "warning", "unhealthy"}: + status = "missing" + rank = 2 + else: + status = reported + rank = max(rank, {"healthy": 0, "warning": 1, "unhealthy": 2}[status]) + hosts.append({"controller": controller, "status": status}) + return {"schema_version": 1, "status": ("healthy", "warning", "unhealthy")[rank], "exit_code": rank, "hosts": hosts} + + +def evaluate(snapshot: dict[str, Any], thresholds: Thresholds) -> dict[str, Any]: + checks: list[dict[str, Any]] = [] + + def add(check_id: str, severity: str, **details: Any) -> None: + checks.append({"id": check_id, "status": severity, **details}) + + for name, usage in snapshot["disks"].items(): + used = usage["used_percent"] + severity = "critical" if used >= thresholds.disk_critical_percent else "warning" if used >= thresholds.disk_warn_percent else "ok" + add(f"disk_{name}", severity, used_percent=used) + inodes = usage["inode_used_percent"] + severity = "critical" if inodes >= thresholds.inode_critical_percent else "warning" if inodes >= thresholds.inode_warn_percent else "ok" + add(f"inode_{name}", severity, used_percent=inodes) + + available = snapshot["memory_available_percent"] + add("memory", "critical" if available <= thresholds.memory_critical_available_percent else "warning" if available <= thresholds.memory_warn_available_percent else "ok", available_percent=available) + load = snapshot.get("load_per_cpu", 0) + add("load", "critical" if load >= thresholds.load_critical_per_cpu else "warning" if load >= thresholds.load_warn_per_cpu else "ok", load_per_cpu=load) + swap = snapshot["swap_used_percent"] + add("swap", "critical" if swap >= thresholds.swap_critical_percent else "warning" if swap >= thresholds.swap_warn_percent else "ok", used_percent=swap) + add("oom", "critical" if snapshot["recent_oom"] or snapshot["controller"]["oom_killed"] else "ok") + add("docker", "ok" if snapshot["docker_available"] else "critical") + + desired = snapshot["desired_state"] + controller_state = snapshot["controller"]["state"] + controller_ok = controller_state == "running" if desired == "active" else controller_state in {"missing", "exited", "created"} + add("controller", "ok" if controller_ok else "critical", state=controller_state, desired_state=desired) + restarts = snapshot["controller"]["restart_count"] + add("restarts", "warning" if restarts >= thresholds.restart_warn_count else "ok", count=restarts) + + configured = snapshot["configured_capacity"] + effective = snapshot["effective_capacity"] + expected = configured if desired == "active" else {"min": 0, "max": 0} + add("capacity", "ok" if effective == expected else "critical", configured=configured, effective=effective) + + managed = snapshot["managed"] + add("managed_unhealthy", "critical" if managed["unhealthy"] or managed["restarting"] else "ok", unhealthy=managed["unhealthy"], restarting=managed["restarting"]) + add("managed_inactive", "warning" if managed["inactive"] else "ok", count=managed["inactive"]) + for kind, count in snapshot["stale"].items(): + add(f"stale_{kind[:-1] if kind.endswith('s') else kind}", "warning" if count else "ok", count=count) + + for name, state in snapshot["services"].items(): + add(name, "ok" if state == "ok" else "critical" if state == "failed" else "warning", state=state) + for name, state in snapshot["timers"].items(): + add(f"timer_{name}", "ok" if state == "ok" else "critical" if state == "failed" else "warning", state=state) + add("updates", "critical" if snapshot["failed_packages"] else "ok") + add("reboot", "warning" if snapshot["pending_reboot"] else "ok") + add("clock", "ok" if snapshot["clock_synchronized"] else "warning") + backup = snapshot["backup"] + add("backup", "warning" if backup == "failed" else "ok", state=backup) + + rank = max(({"ok": 0, "warning": 1, "critical": 2}[check["status"]] for check in checks), default=0) + overall = ("healthy", "warning", "unhealthy")[rank] + if rank == 0 and desired in {"drained", "disabled"}: + overall = "maintenance" + return { + "schema_version": 1, + "controller": snapshot["controller_id"], + "desired_state": desired, + "status": overall, + "exit_code": rank, + "checks": checks, + } + + +Runner = Callable[[list[str]], subprocess.CompletedProcess[str]] + + +def _run(args: list[str]) -> subprocess.CompletedProcess[str]: + try: + return subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, check=False, timeout=30) + except (OSError, subprocess.TimeoutExpired) as error: + return subprocess.CompletedProcess(args, 124 if isinstance(error, subprocess.TimeoutExpired) else 127, "", "") + + +def _disk(path: str) -> dict[str, int]: + try: + value = os.statvfs(path) + except OSError: + value = os.statvfs("/") + used = value.f_blocks - value.f_bfree + iused = value.f_files - value.f_ffree + return { + "used_percent": round(100 * used / max(value.f_blocks, 1)), + "inode_used_percent": round(100 * iused / max(value.f_files, 1)), + } + + +def _count(run: Runner, args: list[str]) -> int: + result = run(args) + return len([line for line in result.stdout.splitlines() if line.strip()]) if result.returncode == 0 else 0 + + +def _stale_resources(run: Runner, instance: str) -> dict[str, int]: + result = run([str(Path(__file__).with_name("cleanup.sh")), "--instance", instance]) + stale = {"containers": 0, "networks": 0, "volumes": 0} + if result.returncode == 0: + for line in result.stdout.splitlines(): + match = re.match(r"(?:WOULD_REMOVE|KEEP) (container|network|volume) ", line) + if match: + stale[f"{match.group(1)}s"] += 1 + return stale + + +def _timespan_seconds(value: str) -> float | None: + units = {"y": 365.25 * 86400, "month": 365.25 * 86400 / 12, "w": 7 * 86400, "d": 86400, "h": 3600, "min": 60, "s": 1, "ms": 0.001, "us": 0.000001, "µs": 0.000001, "ns": 0.000000001} + matches = list(re.finditer(r"([0-9]+(?:\.[0-9]+)?)(month|min|ms|us|µs|ns|y|w|d|h|s)", value)) + if not matches or "".join(match.group(0) for match in matches) != re.sub(r"\s+", "", value): + return None + return sum(float(match.group(1)) * units[match.group(2)] for match in matches) + + +def _unit_state(run: Runner, unit: str, timer: bool = False, max_age_seconds: int = 0) -> str: + if timer: + if run(["systemctl", "is-active", unit]).returncode != 0: + return "failed" + if run(["systemctl", "is-enabled", unit]).returncode != 0: + return "stale" + if max_age_seconds: + last = run(["systemctl", "show", unit, "--property=LastTriggerUSecMonotonic", "--value"]) + triggered = _timespan_seconds(last.stdout.strip()) + now = time.clock_gettime(time.CLOCK_BOOTTIME) + if last.returncode != 0 or triggered is None or now - triggered > max_age_seconds: + return "stale" + result = run(["systemctl", "show", unit, "--property=Result", "--value"]) + return "failed" if result.returncode != 0 or result.stdout.strip() not in {"", "success"} else "ok" + + +def _container(run: Runner, name: str) -> tuple[dict[str, Any], dict[str, int]]: + state = run(["docker", "inspect", "--format", "{{.State.Status}}", name]) + if state.returncode != 0: + return {"state": "missing", "restart_count": 0, "oom_killed": False}, {"min": 0, "max": 0} + oom = run(["docker", "inspect", "--format", "{{.State.OOMKilled}}", name]) + restarts = run(["docker", "inspect", "--format", "{{.RestartCount}}", name]) + environment = run(["docker", "inspect", "--format", "{{range .Config.Env}}{{println .}}{{end}}", name]) + env = dict(value.split("=", 1) for value in environment.stdout.splitlines() if "=" in value) + try: + capacity = {"min": int(env.get("CI_FLEET_MIN_RUNNERS", 0)), "max": int(env.get("CI_FLEET_MAX_RUNNERS", 0))} + except ValueError: + return {"state": "invalid", "restart_count": 0, "oom_killed": False}, {"min": 0, "max": 0} + try: + restart_count = int(restarts.stdout.strip() or 0) + except ValueError: + restart_count = 0 + controller_state = state.stdout.strip() or "missing" + if controller_state != "running": + capacity = {"min": 0, "max": 0} + return { + "state": controller_state, + "restart_count": restart_count, + "oom_killed": oom.stdout.strip().lower() == "true", + }, capacity + + +def _memory(root: Path) -> tuple[int, int]: + values: dict[str, int] = {} + try: + for line in (root / "proc/meminfo").read_text().splitlines(): + key, value = line.split(":", 1) + values[key] = int(value.split()[0]) + except (OSError, ValueError, IndexError): + return 0, 0 + available = round(100 * values.get("MemAvailable", 0) / max(values.get("MemTotal", 1), 1)) + swap_total = values.get("SwapTotal", 0) + swap = round(100 * (swap_total - values.get("SwapFree", 0)) / max(swap_total, 1)) if swap_total else 0 + return available, swap + + +def _memory_pressure(root: Path) -> float | None: + try: + for line in (root / "proc/pressure/memory").read_text().splitlines(): + if line.startswith("some "): + match = re.search(r"avg300=([0-9.]+)", line) + return float(match.group(1)) if match else None + except OSError: + pass + return None + + +def _backup_state(values: dict[str, str], run: Runner) -> str: + command = values.get("CI_FLEET_HEALTH_BACKUP_CHECK") + if not command: + return "not_configured" + path = Path(command) + try: + mode = path.stat() + except OSError: + return "failed" + if not path.is_absolute() or mode.st_uid != 0 or mode.st_mode & (stat.S_IWGRP | stat.S_IWOTH) or not os.access(path, os.X_OK): + return "failed" + return "ok" if run([str(path)]).returncode == 0 else "failed" + + +def collect_snapshot(values: dict[str, str], *, root: Path = Path("/"), run: Runner = _run) -> dict[str, Any]: + docker_root = values.get("CI_FLEET_DOCKER_ROOT", "/var/lib/docker") + available, swap = _memory(root) + docker_ok = run(["docker", "info"]).returncode == 0 + controller_name = values.get("CI_FLEET_CONTROLLER_CONTAINER", "ci-fleet-controller-1") + controller, effective = _container(run, controller_name) if docker_ok else ({"state": "missing", "restart_count": 0, "oom_killed": False}, {"min": 0, "max": 0}) + managed = {"running": 0, "inactive": 0, "unhealthy": 0, "restarting": 0} + if docker_ok: + result = run(["docker", "ps", "-a", "--filter", "label=io.randomdevelopment.ci-fleet.managed=true", "--format", "{{json .}}"]) + for line in result.stdout.splitlines() if result.returncode == 0 else []: + try: + item = json.loads(line) + except json.JSONDecodeError: + continue + state, status = item.get("State", ""), item.get("Status", "").lower() + managed["running" if state == "running" else "inactive"] += 1 + managed["unhealthy"] += int("unhealthy" in status) + managed["restarting"] += int(state == "restarting" or "restarting" in status) + oom = run(["journalctl", "--dmesg", "--since=-24h", "--grep=Out of memory|Killed process", "--quiet"]) + configured = {"min": int(values.get("CI_FLEET_MIN_RUNNERS", 0)), "max": int(values.get("CI_FLEET_MAX_RUNNERS", 0))} + timer_ages = {"health": 900, "cleanup": 172800, "drift": 3600} + timers = {name: _unit_state(run, f"ci-fleet-{name}.timer", timer=True, max_age_seconds=age) for name, age in timer_ages.items()} + services = {name: _unit_state(run, unit) for name, unit in { + "cleanup": "ci-fleet-cleanup.service", + "drift": "ci-fleet-drift.service", + }.items()} + debian = (root / "etc/debian_version").exists() + if debian: + timers["updates"] = _unit_state(run, "apt-daily-upgrade.timer", timer=True, max_age_seconds=172800) + services["updates"] = _unit_state(run, "apt-daily-upgrade.service") + if values.get("CI_FLEET_HEALTH_BOOTSTRAP") == "1": + # ponytail: activation validates unit installation separately; scheduled runs verify live timers after enablement. + timers = {name: "ok" for name in timers} + instance = values.get("CI_FLEET_INSTANCE", "unknown") + stale = _stale_resources(run, instance) if docker_ok else {"containers": 0, "networks": 0, "volumes": 0} + stale["images"] = _count(run, ["docker", "images", "-q", "--filter", "dangling=true", "--filter", "label=io.randomdevelopment.ci-fleet.managed=true"]) if docker_ok else 0 + stale["build_cache"] = _count(run, ["docker", "buildx", "du", "--filter", "until=168h", "--format", "json"]) if docker_ok else 0 + return { + "controller_id": instance, + "desired_state": values.get("CI_FLEET_CONTROLLER_STATE", "active"), + "disks": {"root": _disk(str(root)), "docker": _disk(str(root / docker_root.lstrip("/")))}, + "memory_available_percent": available, + "load_per_cpu": os.getloadavg()[2] / max(os.cpu_count() or 1, 1), + "swap_used_percent": swap if (pressure := _memory_pressure(root)) is None or pressure >= 0.1 else 0, + "recent_oom": oom.returncode == 0 and bool(oom.stdout.strip()), + "docker_available": docker_ok, + "controller": controller, + "configured_capacity": configured, + "effective_capacity": effective, + "managed": managed, + "stale": stale, + "services": services, + "timers": timers, + "pending_reboot": (root / "var/run/reboot-required").exists(), + "failed_packages": debian and bool(run(["dpkg", "--audit"]).stdout.strip()), + "clock_synchronized": run(["timedatectl", "show", "--property=NTPSynchronized", "--value"]).stdout.strip() == "yes", + "backup": _backup_state(values, run), + } + + +def load_monitoring_config(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + info = path.stat() + expected_owner = os.getuid() if os.environ.get("CI_FLEET_TESTING") == "1" else 0 + if info.st_uid != expected_owner or stat.S_IMODE(info.st_mode) & 0o077: + raise ValueError(f"monitoring configuration must be root-owned mode 0600: {path}") + values: dict[str, str] = {} + for number, raw in enumerate(path.read_text().splitlines(), 1): + line = raw.strip() + if not line or line.startswith("#"): + continue + if not re.fullmatch(r"CI_FLEET_HEALTH_[A-Z0-9_]+=[^\n]*", line): + raise ValueError(f"invalid monitoring configuration at line {number}") + key, value = line.split("=", 1) + values[key] = value + return values + + +def _write_report(path: Path, report: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True, mode=0o755) + temporary = path.with_suffix(".tmp") + temporary.write_text(json.dumps(report, sort_keys=True) + "\n") + os.chmod(temporary, 0o644) + temporary.replace(path) + + +def _send_heartbeat(values: dict[str, str], report: dict[str, Any]) -> int: + url = values.get("CI_FLEET_HEALTH_HEARTBEAT_URL") + if not url: + return 0 + if not url.startswith("https://"): + return 2 + headers = {"Content-Type": "application/json"} + token_file = values.get("CI_FLEET_HEALTH_HEARTBEAT_TOKEN_FILE") + if token_file: + path = Path(token_file) + try: + info = path.stat() + if info.st_uid != 0 or stat.S_IMODE(info.st_mode) & 0o077: + return 2 + headers["Authorization"] = f"Bearer {path.read_text().strip()}" + except OSError: + return 2 + request = urllib.request.Request(url, data=json.dumps(report).encode(), headers=headers, method="POST") + try: + with urllib.request.urlopen(request, timeout=10) as response: + return 0 if 200 <= response.status < 300 else 1 + except OSError: + return 1 + + +def _local(args: argparse.Namespace) -> int: + values = dict(os.environ) + values.update(load_monitoring_config(args.monitoring_config)) + report = evaluate(collect_snapshot(values), thresholds_from(values)) + report["timestamp"] = int(time.time()) + heartbeat = _send_heartbeat(values, report) + if heartbeat: + severity = "critical" if heartbeat == 2 else "warning" + report["checks"].append({"id": "heartbeat_delivery", "status": severity}) + if heartbeat > report["exit_code"]: + report["status"], report["exit_code"] = ("warning", 1) if heartbeat == 1 else ("unhealthy", 2) + _write_report(args.output, report) + print(json.dumps(report, sort_keys=True) if args.json else render_human(report)) + return int(report["exit_code"]) + + +def _heartbeats(args: argparse.Namespace) -> int: + config = json.loads(args.config.read_text()) + records = {} + for controller in config["controllers"]: + path = args.input_dir / f"{controller}.json" + if path.exists(): + records[controller] = json.loads(path.read_text()) + report = evaluate_heartbeats(config["controllers"], records, now=int(time.time()), grace_seconds=args.grace_seconds) + print(json.dumps(report, sort_keys=True) if args.json else "\n".join(f'{host["status"].upper()} controller={host["controller"]}' for host in report["hosts"])) + return int(report["exit_code"]) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Redacted ci-fleet host health") + commands = parser.add_subparsers(dest="command", required=True) + local = commands.add_parser("local") + local.add_argument("--json", action="store_true") + local.add_argument("--monitoring-config", type=Path, default=Path("/etc/ci-fleet/monitoring.env")) + local.add_argument("--output", type=Path, default=Path("/var/lib/ci-fleet/health/latest.json")) + local.set_defaults(handler=_local) + heartbeats = commands.add_parser("heartbeats") + heartbeats.add_argument("--config", type=Path, required=True) + heartbeats.add_argument("--input-dir", type=Path, required=True) + heartbeats.add_argument("--grace-seconds", type=int, default=900) + heartbeats.add_argument("--json", action="store_true") + heartbeats.set_defaults(handler=_heartbeats) + args = parser.parse_args() + try: + return args.handler(args) + except (OSError, ValueError, KeyError, json.JSONDecodeError) as error: + print(f"CRITICAL health_configuration_invalid: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index 4763d5da..c06bf63f 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -1,37 +1,16 @@ #!/usr/bin/env bash set -Eeuo pipefail - -warn_percent=${CI_FLEET_DISK_WARN_PERCENT:-80} -critical_percent=${CI_FLEET_DISK_CRITICAL_PERCENT:-90} -controller_name=${CI_FLEET_CONTROLLER_CONTAINER:-ci-fleet-controller-1} -desired_state=${CI_FLEET_CONTROLLER_STATE:-active} -status=0 - -emit() { printf '%s\n' "$*"; } -if ! command -v docker >/dev/null || ! docker info >/dev/null 2>&1; then - emit "CRITICAL docker_unavailable" - exit 2 +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +environment=/etc/ci-fleet/ci-fleet.env +args=(local) +if [[ ${CI_FLEET_TESTING:-0} == 1 && -n ${CI_FLEET_ROOT_PREFIX:-} ]]; then + environment="$CI_FLEET_ROOT_PREFIX/etc/ci-fleet/ci-fleet.env" + args+=(--monitoring-config "$CI_FLEET_ROOT_PREFIX/etc/ci-fleet/monitoring.env" --output "$CI_FLEET_ROOT_PREFIX/var/lib/ci-fleet/health/latest.json") fi -emit "OK docker_available" - -docker_root=${CI_FLEET_DOCKER_ROOT:-/var/lib/docker} -used=$(df -P "$docker_root" 2>/dev/null | awk 'NR==2 {gsub(/%/, "", $5); print $5}' || true) -if [[ -z "$used" ]]; then used=$(df -P / | awk 'NR==2 {gsub(/%/, "", $5); print $5}'); fi -if ((used >= critical_percent)); then emit "CRITICAL disk_used_percent=$used"; status=2 -elif ((used >= warn_percent)); then emit "WARN disk_used_percent=$used"; ((status < 1)) && status=1 -else emit "OK disk_used_percent=$used"; fi - -controller_state=$(docker inspect --format '{{.State.Status}}' "$controller_name" 2>/dev/null || true) -if [[ "$desired_state" == active ]]; then - if [[ "$controller_state" == running ]]; then emit "OK controller_state=running" - else emit "CRITICAL controller_state=${controller_state:-missing} desired_state=active"; status=2; fi -elif [[ -z "$controller_state" || "$controller_state" == exited || "$controller_state" == created ]]; then - emit "OK controller_state=${controller_state:-missing} desired_state=$desired_state" -else - emit "CRITICAL controller_state=$controller_state desired_state=$desired_state"; status=2 +if [[ -r $environment ]]; then + set -a + # shellcheck disable=SC1090 + . "$environment" + set +a fi - -stale=$(docker ps -aq --filter label=io.randomdevelopment.ci-fleet.managed=true --filter status=exited | wc -l | tr -d ' ') -if ((stale > 0)); then emit "WARN inactive_managed_containers=$stale"; ((status < 1)) && status=1 -else emit "OK inactive_managed_containers=0"; fi -exit "$status" +exec python3 "$repo_root/scripts/health.py" "${args[@]}" "$@" diff --git a/scripts/install-worker-controller.sh b/scripts/install-worker-controller.sh index 54cb9ac0..6083df48 100755 --- a/scripts/install-worker-controller.sh +++ b/scripts/install-worker-controller.sh @@ -103,6 +103,7 @@ default_host_config=$etc_dir/host.env host_config=${host_config_arg:-$default_host_config} state_root=$(root_path /var/lib/ci-fleet) state_file=$state_root/install-state.json +health_report=$state_root/health/latest.json checkpoints_dir=$state_root/checkpoints systemd_dir=$(root_path /etc/systemd/system) lock_file=$(root_path /run/ci-fleet-installer.lock) @@ -423,6 +424,9 @@ runtime_release_complete() { local path=$1 expected=$2 marker required stored_digest actual_digest [[ -d "$path" && -f "$path/.ci-fleet-engine-ref" && -f "$path/.ci-fleet-tree-sha256" && -f "$path/deploy/compose.yaml" ]] || return 1 [[ -x "$path/scripts/preflight.sh" && -x "$path/scripts/healthcheck.sh" && -x "$path/scripts/cleanup.sh" ]] || return 1 + if grep -Fq 'scripts/health.py' "$path/scripts/healthcheck.sh"; then + [[ -f "$path/scripts/health.py" ]] || return 1 + fi for required in controller/Dockerfile controller/go.mod controller/main.go controller/config.go controller/scaler.go controller/state.go runner/Dockerfile; do [[ -f "$path/$required" ]] || return 1 done @@ -724,6 +728,44 @@ remove_systemd_units() { systemctl daemon-reload } +run_health_check() { + local release=$1 environment=$2 bootstrap=${3:-false} result=0 + ( + local testing_value=${CI_FLEET_TESTING:-} root_value=${CI_FLEET_ROOT_PREFIX:-} variable + while IFS= read -r variable; do unset "$variable"; done < <(compgen -A variable CI_FLEET_) + [[ -z "$testing_value" ]] || export CI_FLEET_TESTING=$testing_value + [[ -z "$root_value" ]] || export CI_FLEET_ROOT_PREFIX=$root_value + set -a + # shellcheck disable=SC1090 + . "$environment" + set +a + [[ "$bootstrap" != true ]] || export CI_FLEET_HEALTH_BOOTSTRAP=1 + "$release/scripts/healthcheck.sh" + ) || result=$? + ((result < 2)) +} + +display_last_health() { + if [[ ! -f "$health_report" ]]; then + note 'HEALTH last=missing' + return + fi + python3 - "$health_report" <<'PY' +import json +import sys +try: + report = json.load(open(sys.argv[1], encoding="utf-8")) + status = report["status"] + timestamp = int(report["timestamp"]) + if status not in {"healthy", "warning", "unhealthy", "maintenance"}: + raise ValueError +except (OSError, ValueError, KeyError, TypeError, json.JSONDecodeError): + print("HEALTH last=invalid") +else: + print(f"HEALTH last={status} timestamp={timestamp}") +PY +} + activate_candidate() { local staged_state install -d -m 0700 "$etc_dir" "$state_root" "$checkpoints_dir" @@ -736,15 +778,6 @@ activate_candidate() { compose "$release_dir" "$rendered_env" up -d --no-deps controller sleep "${CI_FLEET_STARTUP_WAIT_SECONDS:-2}" runtime_matches active || die 'controller did not remain running after activation' - if ! ( - set -a - # shellcheck disable=SC1090 - . "$rendered_env" - set +a - "$release_dir/scripts/healthcheck.sh" - ); then - die 'post-activation health check failed' - fi else compose "$release_dir" "$rendered_env" stop controller >/dev/null 2>&1 || true if ! runtime_matches "$target_state"; then @@ -752,6 +785,9 @@ activate_candidate() { runtime_matches "$target_state" || die 'controller did not reach the requested non-active state' fi fi + if ! run_health_check "$release_dir" "$rendered_env" true; then + die 'post-activation health check failed' + fi staged_state=$(mktemp "$state_root/.install-state.XXXXXX") staging_paths+=("$staged_state") python3 - "$candidate_metadata" "$staged_state" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" <<'PY' @@ -844,15 +880,7 @@ restore_checkpoint() { failed=1 else compose "$release_dir" "$rendered_env" up -d --no-deps controller || failed=1 - if ((failed == 0)); then - ( - set -a - # shellcheck disable=SC1090 - . "$rendered_env" - set +a - "$release_dir/scripts/healthcheck.sh" - ) || failed=1 - fi + if ((failed == 0)); then run_health_check "$release_dir" "$rendered_env" || failed=1; fi fi fi fi @@ -876,6 +904,7 @@ trap on_error ERR perform_check() { local count + display_last_health drift_count count=$DRIFT_COUNT if ((count > 0)); then @@ -953,6 +982,7 @@ perform_uninstall() { fi remove_systemd_units rm -f "$current_link" "$rendered_env" "$state_file" + rm -rf -- "$state_root/health" rm -f "$manager_current" transaction_active=false note "UNINSTALL_OK host_config_preserved=$host_config secrets_preserved=$etc_dir/secrets" diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 4f78b10e..ddd7d8d3 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -209,6 +209,7 @@ export FAKE_CONTROLLER_IMAGE_STATE=$tmp/controller-image-present export FAKE_IMAGE_INSPECT_LOG=$tmp/image-inspects for dockerfile in "$repo_root/controller/Dockerfile" "$repo_root/runner/Dockerfile"; do grep -Fq "LABEL org.opencontainers.image.revision=\"\${CI_FLEET_COMMIT}\"" "$dockerfile" || fail "managed image lacks engine provenance label: $dockerfile" + grep -Fq 'io.randomdevelopment.ci-fleet.managed="true"' "$dockerfile" || fail "managed image lacks fleet ownership label: $dockerfile" done grep -Fq ' user: "0:0"' "$repo_root/deploy/compose.yaml" || fail 'controller cannot read the required root-owned mode-0600 GitHub App PEM' grep -Fq 'export PYTHONDONTWRITEBYTECODE=1' "$repo_root/scripts/install-worker-controller.sh" || fail 'managed validation may write Python bytecode into the immutable manager release' @@ -220,7 +221,8 @@ git -C "$config_repo" config user.email fixture@example.invalid write_config() { local state=$1 maximum=$2 budget=$3 - python3 - "$repo_root/templates/config-repository/fleet.json" "$config_repo/fleet.json" "$engine_ref" "$state" "$maximum" "$budget" <<'PY' + local desired_engine=${4:-$engine_ref} + python3 - "$repo_root/templates/config-repository/fleet.json" "$config_repo/fleet.json" "$desired_engine" "$state" "$maximum" "$budget" <<'PY' import json import sys source, target, engine_ref, state, maximum, budget = sys.argv[1:] @@ -299,6 +301,9 @@ chmod 644 "$rendered_env" expect_failure 'DRIFT rendered_environment' "$installer" --check "${base_args[@]}" --ref "$ref_one" expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one" >/dev/null [[ $(stat -c %a "$rendered_env") == 600 ]] || fail 'convergence did not repair rendered-environment mode' +manual_health_result=0 +"$repo_root/scripts/healthcheck.sh" >/dev/null || manual_health_result=$? +((manual_health_result < 2)) || fail 'manual healthcheck did not source rendered capacity' export FAKE_WRONG_INSTALL_STATE_OWNER=$install_state expect_failure 'install state must be owned by root with mode 0600' env CI_FLEET_INSTALL_STATE_FILE="$install_state" CI_FLEET_INSTALLER="$installer" "$repo_root/scripts/check-installed-state.sh" unset FAKE_WRONG_INSTALL_STATE_OWNER @@ -318,7 +323,8 @@ mv "$host_config.missing" "$host_config" second=$(expect_success "$installer" --install "${base_args[@]}" --ref "$ref_one") grep -Fq 'NO_CHANGE' <<<"$second" || fail 'idempotent rerun changed the host' -expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one" >/dev/null +check=$(expect_success "$installer" --check "${base_args[@]}" --ref "$ref_one") +grep -Fq 'HEALTH last=' <<<"$check" || fail 'check output omitted the last redacted health result' [[ ! -d "$root/opt/ci-fleet/manager/releases/$engine_ref/templates/config-repository/scripts/__pycache__" ]] || fail 'manager validation wrote Python bytecode into the immutable release' python3 - "$install_state" <<'PY' import json @@ -519,6 +525,9 @@ export FAKE_RUNNER_STATE_ONCE=$tmp/orphaned-managed-runner : >"$FAKE_RUNNER_STATE_ONCE" export FAKE_ALL_RUNNER_STATE=$tmp/uninstall-stopped-managed-runner : >"$FAKE_ALL_RUNNER_STATE" +: >"$root/etc/ci-fleet/monitoring.env" +mkdir -p "$root/var/lib/ci-fleet/health" +printf '{"status":"healthy"}\n' >"$root/var/lib/ci-fleet/health/latest.json" : >"$FAKE_DOCKER_PS_LOG" expect_success "$installer" --uninstall >/dev/null [[ ! -f "$FAKE_RUNNER_STATE_ONCE" ]] || fail 'uninstall did not wait for an orphaned managed runner' @@ -528,6 +537,8 @@ if grep -Eq 'label=io.randomdevelopment.ci-fleet.instance=$' "$FAKE_DOCKER_PS_LO unset FAKE_RUNNER_STATE_ONCE FAKE_ALL_RUNNER_STATE [[ ! -e "$root/opt/ci-fleet/current" && ! -e "$root/var/lib/ci-fleet/install-state.json" ]] || fail 'uninstall left active installation state' [[ -f "$host_config" && -f "$pem" ]] || fail 'uninstall removed preserved host credentials' +[[ -f "$root/etc/ci-fleet/monitoring.env" ]] || fail 'uninstall removed host-local monitoring configuration' +[[ ! -e "$root/var/lib/ci-fleet/health" ]] || fail 'uninstall retained fleet-owned health state' adopt_root=$tmp/adopt-host export CI_FLEET_ROOT_PREFIX=$adopt_root @@ -538,7 +549,9 @@ printf 'fixture only\n' >"$adopt_pem" chmod 600 "$adopt_pem" cp "$repo_root/deploy/compose.yaml" "$adopt_root/opt/ci-fleet/deploy/compose.yaml" cp "$repo_root/scripts/healthcheck.sh" "$adopt_root/opt/ci-fleet/scripts/healthcheck.sh" -chmod 0755 "$adopt_root/opt/ci-fleet/scripts/healthcheck.sh" +cp "$repo_root/scripts/health.py" "$adopt_root/opt/ci-fleet/scripts/health.py" +cp "$repo_root/scripts/cleanup.sh" "$adopt_root/opt/ci-fleet/scripts/cleanup.sh" +chmod 0755 "$adopt_root/opt/ci-fleet/scripts/healthcheck.sh" "$adopt_root/opt/ci-fleet/scripts/cleanup.sh" printf '%s\n' \ 'CI_FLEET_GITHUB_APP_CLIENT_ID=Iv1.EXAMPLE' \ 'CI_FLEET_GITHUB_APP_INSTALLATION_ID=123456' \ @@ -547,6 +560,8 @@ printf '%s\n' \ 'CI_FLEET_CONTROLLER_STATE=active' \ 'CI_FLEET_INSTANCE=legacy-ci-01' >"$adopt_root/etc/ci-fleet/ci-fleet.env" chmod 600 "$adopt_root/etc/ci-fleet/ci-fleet.env" +printf 'CI_FLEET_HEALTH_DISK_WARN_PERCENT=75\n' >"$adopt_root/etc/ci-fleet/monitoring.env" +chmod 600 "$adopt_root/etc/ci-fleet/monitoring.env" : >"$FAKE_DOCKER_STATE" chmod 644 "$adopt_root/etc/ci-fleet/ci-fleet.env" expect_failure 'rendered environment must be owned by root with mode 0600' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" @@ -557,6 +572,7 @@ export FAKE_COMPOSE_LOG=$tmp/adopt-compose.log export FAKE_RESTART_AFTER_UP=$tmp/adopt-restart-after-up : >"$FAKE_RESTART_AFTER_UP" expect_failure 'ROLLBACK_RESTORED' "$installer" --adopt "${base_args[@]}" --ref "$ref_one" +grep -Fxq 'CI_FLEET_HEALTH_DISK_WARN_PERCENT=75' "$adopt_root/etc/ci-fleet/monitoring.env" || fail 'rollback changed host-local monitoring configuration' unset FAKE_RESTART_AFTER_UP grep -Fq "stop|$adopt_root/etc/ci-fleet/ci-fleet.env|example-ci-01" "$FAKE_COMPOSE_LOG" || fail 'rollback did not drain the candidate with its rendered environment and identity' grep -Fq 'CI_FLEET_INSTANCE=legacy-ci-01' "$adopt_root/etc/ci-fleet/ci-fleet.env" || fail 'failed adoption did not restore the installed controller identity' @@ -570,6 +586,15 @@ grep -Fq 'CONVERGED mode=adopt' <<<"$adopt" || fail 'adoption did not converge' grep -Fq 'label=io.randomdevelopment.ci-fleet.instance=legacy-ci-01' "$FAKE_DOCKER_PS_LOG" || fail 'adoption did not drain the installed controller instance' unset FAKE_RUNNER_STATE_ONCE FAKE_COMPOSE_LOG +# Public pre-health engine fixture; do not depend on a local remote-tracking ref. +legacy_engine_ref=af9c0c13cd12866ce75dd6c43a4cda01915507e1 +legacy_ref=$(write_config active 1 1 "$legacy_engine_ref") +export FAKE_ENGINE_REF=$legacy_engine_ref +export FAKE_RUNNER_IMAGE=ci-fleet-runner:${legacy_engine_ref:0:12} +export FAKE_CONTROLLER_IMAGE=ci-fleet-controller:${legacy_engine_ref:0:12} +expect_success "$installer" --upgrade "${base_args[@]}" --ref "$legacy_ref" >/dev/null +[[ $(readlink -f "$adopt_root/opt/ci-fleet/current") == "$adopt_root/opt/ci-fleet/releases/$legacy_engine_ref" ]] || fail 'upgrade could not restore a pre-health-contract engine' + grep -Fq 'Issue #7' "$repo_root/docs/DESIGN-DECISIONS.md" || fail 'isolated proof approval is not recorded' if grep -Fq '/etc/ci-fleet/ci-fleet.env.before-max2' "$repo_root/docs/CAPACITY-PROMOTION.md"; then fail 'capacity runbook still edits rendered host state'; fi grep -Fq -- '--upgrade' "$repo_root/docs/CAPACITY-PROMOTION.md" || fail 'capacity runbook does not apply reviewed desired state through the installer' diff --git a/scripts/test_health.py b/scripts/test_health.py new file mode 100644 index 00000000..3dd60ddb --- /dev/null +++ b/scripts/test_health.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +import copy +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location("health", ROOT / "scripts" / "health.py") +health = importlib.util.module_from_spec(SPEC) +assert SPEC.loader +sys.modules[SPEC.name] = health +SPEC.loader.exec_module(health) + + +def healthy_snapshot(): + return { + "controller_id": "example-ci-01", + "desired_state": "active", + "disks": {"root": {"used_percent": 20, "inode_used_percent": 10}, "docker": {"used_percent": 30, "inode_used_percent": 15}}, + "memory_available_percent": 75, + "swap_used_percent": 0, + "recent_oom": False, + "docker_available": True, + "controller": {"state": "running", "restart_count": 0, "oom_killed": False}, + "configured_capacity": {"min": 0, "max": 1}, + "effective_capacity": {"min": 0, "max": 1}, + "managed": {"running": 0, "inactive": 0, "unhealthy": 0, "restarting": 0}, + "stale": {"images": 0, "networks": 0, "volumes": 0, "build_cache_bytes": 0}, + "services": {"cleanup": "ok", "drift": "ok"}, + "timers": {"health": "ok", "cleanup": "ok", "drift": "ok", "updates": "ok"}, + "pending_reboot": False, + "failed_packages": False, + "clock_synchronized": True, + "backup": "not_configured", + } + + +class HealthTests(unittest.TestCase): + def test_healthy_active_host(self) -> None: + report = health.evaluate(healthy_snapshot(), health.Thresholds()) + self.assertEqual(report["status"], "healthy") + self.assertEqual(report["exit_code"], 0) + self.assertEqual(report["controller"], "example-ci-01") + + def test_disk_warning_and_critical_thresholds(self) -> None: + warning = healthy_snapshot() + warning["disks"]["docker"]["used_percent"] = 80 + report = health.evaluate(warning, health.Thresholds()) + self.assertEqual((report["status"], report["exit_code"]), ("warning", 1)) + self.assertIn("disk_docker", {check["id"] for check in report["checks"] if check["status"] == "warning"}) + + critical = copy.deepcopy(warning) + critical["disks"]["docker"]["used_percent"] = 90 + report = health.evaluate(critical, health.Thresholds()) + self.assertEqual((report["status"], report["exit_code"]), ("unhealthy", 2)) + self.assertIn("disk_docker", {check["id"] for check in report["checks"] if check["status"] == "critical"}) + + def test_health_contract_classifies_host_failures(self) -> None: + cases = { + "inode_root": (lambda s: s["disks"]["root"].update(inode_used_percent=90), "unhealthy"), + "memory": (lambda s: s.update(memory_available_percent=8), "unhealthy"), + "load": (lambda s: s.update(load_per_cpu=1.5), "unhealthy"), + "swap": (lambda s: s.update(swap_used_percent=25), "warning"), + "oom": (lambda s: s.update(recent_oom=True), "unhealthy"), + "docker": (lambda s: s.update(docker_available=False), "unhealthy"), + "controller": (lambda s: s["controller"].update(state="exited"), "unhealthy"), + "restarts": (lambda s: s["controller"].update(restart_count=3), "warning"), + "capacity": (lambda s: s.update(effective_capacity={"min": 0, "max": 0}), "unhealthy"), + "managed_unhealthy": (lambda s: s["managed"].update(unhealthy=1), "unhealthy"), + "stale_volume": (lambda s: s["stale"].update(volumes=1), "warning"), + "drift": (lambda s: s["services"].update(drift="failed"), "unhealthy"), + "timer_cleanup": (lambda s: s["timers"].update(cleanup="stale"), "warning"), + "updates": (lambda s: s.update(failed_packages=True), "unhealthy"), + "reboot": (lambda s: s.update(pending_reboot=True), "warning"), + "clock": (lambda s: s.update(clock_synchronized=False), "warning"), + "backup": (lambda s: s.update(backup="failed"), "warning"), + } + for check_id, (mutate, expected) in cases.items(): + with self.subTest(check_id=check_id): + snapshot = healthy_snapshot() + mutate(snapshot) + report = health.evaluate(snapshot, health.Thresholds()) + self.assertEqual(report["status"], expected) + self.assertIn(check_id, {check["id"] for check in report["checks"] if check["status"] != "ok"}) + + def test_drained_host_is_maintenance_not_unhealthy(self) -> None: + snapshot = healthy_snapshot() + snapshot["desired_state"] = "drained" + snapshot["controller"]["state"] = "missing" + snapshot["effective_capacity"] = {"min": 0, "max": 0} + report = health.evaluate(snapshot, health.Thresholds()) + self.assertEqual((report["status"], report["exit_code"]), ("maintenance", 0)) + + def test_external_heartbeats_detect_missing_and_stale_active_hosts(self) -> None: + controllers = { + "fresh": {"state": "active", "lifecycle": "stable"}, + "stale": {"state": "active", "lifecycle": "stable"}, + "missing": {"state": "active", "lifecycle": "stable"}, + "drained": {"state": "drained", "lifecycle": "stable"}, + "retired": {"state": "disabled", "lifecycle": "retiring"}, + } + records = { + "fresh": {"controller": "fresh", "timestamp": 980, "status": "warning"}, + "stale": {"controller": "stale", "timestamp": 800, "status": "healthy"}, + } + report = health.evaluate_heartbeats(controllers, records, now=1000, grace_seconds=60) + states = {host["controller"]: host["status"] for host in report["hosts"]} + self.assertEqual(states, { + "fresh": "warning", + "stale": "missing", + "missing": "missing", + "drained": "maintenance", + "retired": "retired", + }) + self.assertEqual((report["status"], report["exit_code"]), ("unhealthy", 2)) + warning = health.evaluate_heartbeats({"fresh": controllers["fresh"]}, {"fresh": records["fresh"]}, now=1000, grace_seconds=60) + self.assertEqual((warning["status"], warning["exit_code"]), ("warning", 1)) + future = health.evaluate_heartbeats({"fresh": controllers["fresh"]}, {"fresh": {"controller": "fresh", "timestamp": 2000, "status": "healthy"}}, now=1000, grace_seconds=60) + self.assertEqual((future["status"], future["hosts"][0]["status"]), ("unhealthy", "missing")) + wrong = health.evaluate_heartbeats({"fresh": controllers["fresh"]}, {"fresh": {"controller": "other", "timestamp": 980, "status": "healthy"}}, now=1000, grace_seconds=60) + self.assertEqual((wrong["status"], wrong["hosts"][0]["status"]), ("unhealthy", "missing")) + + def test_collector_uses_sustained_metrics_and_all_service_units(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "proc/pressure").mkdir(parents=True) + (root / "proc/meminfo").write_text( + "MemTotal: 100 kB\nMemAvailable: 75 kB\nSwapTotal: 100 kB\nSwapFree: 50 kB\n" + ) + pressure = root / "proc/pressure/memory" + pressure.write_text("some avg10=0.00 avg60=0.05 avg300=0.20 total=1\n") + def run(args): + output = "yes\n" if args[0] == "timedatectl" else "success\n" if args[0] == "systemctl" else "" + return health.subprocess.CompletedProcess(args, 1 if args[:2] == ["docker", "info"] else 0, output, "") + + original_load, original_cpus = health.os.getloadavg, health.os.cpu_count + health.os.getloadavg, health.os.cpu_count = lambda: (1.0, 2.0, 6.0), lambda: 2 + try: + snapshot = health.collect_snapshot( + {"CI_FLEET_INSTANCE": "example", "CI_FLEET_CONTROLLER_STATE": "disabled", "CI_FLEET_HEALTH_BOOTSTRAP": "1"}, + root=root, + run=run, + ) + self.assertEqual((snapshot["load_per_cpu"], snapshot["swap_used_percent"]), (3.0, 50)) + self.assertEqual(set(snapshot["services"]), {"cleanup", "drift"}) + self.assertEqual(set(snapshot["timers"]), {"health", "cleanup", "drift"}) + (root / "etc").mkdir() + (root / "etc/debian_version").write_text("13\n") + debian = health.collect_snapshot({"CI_FLEET_CONTROLLER_STATE": "disabled", "CI_FLEET_HEALTH_BOOTSTRAP": "1"}, root=root, run=run) + self.assertIn("updates", debian["services"]) + self.assertIn("updates", debian["timers"]) + pressure.write_text("some avg10=0.00 avg60=0.00 avg300=0.00 total=1\n") + self.assertEqual(health.collect_snapshot({"CI_FLEET_CONTROLLER_STATE": "disabled", "CI_FLEET_HEALTH_BOOTSTRAP": "1"}, root=root, run=run)["swap_used_percent"], 0) + pressure.unlink() + self.assertEqual(health.collect_snapshot({"CI_FLEET_CONTROLLER_STATE": "disabled", "CI_FLEET_HEALTH_BOOTSTRAP": "1"}, root=root, run=run)["swap_used_percent"], 50) + finally: + health.os.getloadavg, health.os.cpu_count = original_load, original_cpus + + def test_threshold_overrides_validate_ordering(self) -> None: + self.assertAlmostEqual(health._timespan_seconds("3d 1h 41min 40.5s"), 265300.5) + self.assertAlmostEqual(health._timespan_seconds("1y 2month 3w 4d 5h 6min 7.5s"), 365.25 * 86400 + 2 * 365.25 * 86400 / 12 + 3 * 7 * 86400 + 4 * 86400 + 5 * 3600 + 6 * 60 + 7.5) + thresholds = health.thresholds_from({"CI_FLEET_HEALTH_DISK_WARN_PERCENT": "70", "CI_FLEET_HEALTH_DISK_CRITICAL_PERCENT": "85"}) + self.assertEqual((thresholds.disk_warn_percent, thresholds.disk_critical_percent), (70, 85)) + with self.assertRaisesRegex(ValueError, "disk thresholds"): + health.thresholds_from({"CI_FLEET_HEALTH_DISK_WARN_PERCENT": "90", "CI_FLEET_HEALTH_DISK_CRITICAL_PERCENT": "80"}) + + def test_human_output_is_redacted(self) -> None: + report = health.evaluate(healthy_snapshot(), health.Thresholds()) + report["private_token"] = "SHOULD_NOT_PRINT" + output = health.render_human(report) + self.assertIn("HEALTHY controller=example-ci-01", output) + self.assertNotIn("SHOULD_NOT_PRINT", output) + self.assertEqual(health._send_heartbeat({"CI_FLEET_HEALTH_HEARTBEAT_URL": "http://unsafe.invalid"}, report), 2) + + def test_probe_failures_are_results_and_missing_units_fail(self) -> None: + for error in (FileNotFoundError(), health.subprocess.TimeoutExpired(["probe"], 30)): + original = health.subprocess.run + health.subprocess.run = lambda *args, **kwargs: (_ for _ in ()).throw(error) + try: + self.assertNotEqual(health._run(["probe"]).returncode, 0) + finally: + health.subprocess.run = original + + def missing(args): + return health.subprocess.CompletedProcess(args, 1, "", "") + + self.assertEqual(health._unit_state(missing, "missing.service"), "failed") + + def test_expired_active_resources_and_stopped_capacity_are_observable(self) -> None: + cleanup = "KEEP container runner state=running expired=1 (routine cleanup never removes active containers)\nWOULD_REMOVE volume old expired=1\n" + run = lambda args: health.subprocess.CompletedProcess(args, 0, cleanup, "") + self.assertEqual(health._stale_resources(run, "example"), {"containers": 1, "networks": 0, "volumes": 1}) + + def stopped(args): + outputs = { + "{{.State.Status}}": "exited\n", + "{{.State.OOMKilled}}": "false\n", + "{{.RestartCount}}": "0\n", + "{{range .Config.Env}}{{println .}}{{end}}": "CI_FLEET_MIN_RUNNERS=1\nCI_FLEET_MAX_RUNNERS=2\n", + } + return health.subprocess.CompletedProcess(args, 0, outputs.get(args[3], ""), "") + + controller, capacity = health._container(stopped, "controller") + self.assertEqual((controller["state"], capacity), ("exited", {"min": 0, "max": 0})) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate.sh b/scripts/validate.sh index df4d3901..229577d3 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -9,10 +9,13 @@ python3 -m py_compile \ .github/actions/plan/plan.py \ .github/actions/plan/test_plan.py \ scripts/desired_state.py \ + scripts/health.py \ scripts/scan_committed_secrets.py \ - scripts/test_desired_state.py + scripts/test_desired_state.py \ + scripts/test_health.py python3 .github/actions/plan/test_plan.py python3 scripts/test_desired_state.py +python3 scripts/test_health.py python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh