From ed8ad092aaaa39e2e540a1457a08a498ceea0c08 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:16:22 -0500 Subject: [PATCH 01/54] feat: add Docker daemon config renderer for network policy Add render_docker_daemon_config() to desired_state.py, which translates the already-validated CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_* rendered env values into the daemon.json default-address-pools block. Returns an empty dict when no policy is rendered (no-op). Add test_apply_docker_network_policy.py with RED-GREEN coverage for: - valid pool rendering from env - rejection of negative/malformed pool count and CIDRs - no-op when policy absent This is the validation-before-mutation foundation for the Docker network policy rollout engine stage. --- scripts/desired_state.py | 39 +++ scripts/test_apply_docker_network_policy.py | 333 ++++++++++++++++++++ 2 files changed, 372 insertions(+) create mode 100644 scripts/test_apply_docker_network_policy.py diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 8e28b96d..65ce4b0d 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -196,6 +196,45 @@ def validate_docker_network_policy(policy: dict[str, Any], *, path: str, max_run return configured, reserve, networks_per_runner, parsed +def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: + """Build the Docker daemon.json ``default-address-pools`` block from rendered env. + + Reads only the already-validated ``CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_*`` + values produced by ``build_rendered_env``. Returns a dict suitable for + merging into ``daemon.json``. When no policy was rendered, returns an empty + dict (no ``default-address-pools`` key). + """ + count_str = rendered.get("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT", "0") + try: + count = int(count_str) + except ValueError as exc: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must be an integer, got {count_str!r}") from exc + if count == 0 and "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" not in rendered: + return {} + if count < 0: + raise ValueError("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must be non-negative") + pools: list[dict[str, Any]] = [] + for index in range(count): + base = rendered.get(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE") + size_str = rendered.get(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE") + if base is None or size_str is None: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE/SIZE: both required when count > 0") + try: + network = ipaddress.ip_network(base, strict=True) + except ValueError as exc: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE: malformed CIDR {base!r}") from exc + try: + size = int(size_str) + except ValueError as exc: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: must be an integer, got {size_str!r}") from exc + if not isinstance(size, int) or size < 0 or size > 29: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: must be between 0 and 29") + if size < network.prefixlen: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: impossible subnet count for {base!r}") + pools.append({"base": base, "size": size}) + return {"default-address-pools": [{"base": p["base"], "size": p["size"]} for p in pools]} + + def select_controller(config: dict[str, Any], controller_id: str) -> tuple[dict[str, Any], dict[str, Any]]: controllers = config["controllers"] if controller_id not in controllers: diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py new file mode 100644 index 00000000..0c02b2d6 --- /dev/null +++ b/scripts/test_apply_docker_network_policy.py @@ -0,0 +1,333 @@ +#!/usr/bin/env python3 +"""Tests for the Docker daemon network policy apply stage. + +Validates the daemon.json rendering helper and the apply script's +transactional behavior: validation before mutation, preservation of +unrelated daemon JSON keys, rollback on failure, and failure evidence +without secrets. +""" +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +# Scripts directory contains both desired_state.py and the apply script. +SCRIPTS = ROOT / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from desired_state import ( # noqa: E402 (import after sys.path adjustment) + DesiredStateError, + build_rendered_env, + load_and_validate_config, + render_docker_daemon_config, + validate_docker_network_policy, +) + + +CONFIG_COMMIT = "1" * 40 + + +def config() -> dict: + return json.loads((ROOT / "templates" / "config-repository" / "fleet.json").read_text(encoding="utf-8")) + + +def host_values() -> dict[str, str]: + return { + "CI_FLEET_GITHUB_APP_CLIENT_ID": "Iv1.EXAMPLE", + "CI_FLEET_GITHUB_APP_INSTALLATION_ID": "123456", + "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE": "/etc/ci-fleet/secrets/github-app.pem", + "CI_FLEET_RUNNER_TTL": "6h", + } + + +def docker_network_policy() -> dict: + return { + "default_address_pools": [ + {"base": "198.51.100.0/24", "size": 29}, + {"base": "203.0.113.0/24", "size": 29}, + ], + "networks_per_runner": 1, + "reserve_subnets": 1, + } + + +class RenderDaemonConfigTests(unittest.TestCase): + """RED: render_docker_daemon_config does not exist yet.""" + + def test_renders_default_address_pools_from_rendered_env(self) -> None: + value = config() + policy = docker_network_policy() + value["controllers"]["example-ci-01"]["docker_network_policy"] = policy + capabilities = {"status_reporting_config", "required_status_reporting", "docker_network_policy_config"} + rendered, _ = build_rendered_env( + value, + "example-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + engine_capabilities=capabilities, + ) + daemon = render_docker_daemon_config(rendered) + self.assertIn("default-address-pools", daemon) + pools = daemon["default-address-pools"] + self.assertEqual(len(pools), 2) + self.assertEqual(pools[0], {"base": "198.51.100.0/24", "size": 29}) + self.assertEqual(pools[1], {"base": "203.0.113.0/24", "size": 29}) + + def test_rejects_negative_pool_count(self) -> None: + with self.assertRaisesRegex(ValueError, "must be non-negative"): + render_docker_daemon_config({ + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "-1", + }) + + def test_rejects_malformed_pool_index(self) -> None: + env = { + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "1", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE": "not-a-cidr", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE": "29", + } + with self.assertRaisesRegex(ValueError, "malformed"): + render_docker_daemon_config(env) + + def test_rejects_mismatched_pool_size(self) -> None: + env = { + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "1", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE": "198.51.100.0/24", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE": "not-int", + } + with self.assertRaisesRegex(ValueError, "must be an integer"): + render_docker_daemon_config(env) + + def test_not_configured_returns_empty(self) -> None: + # No network policy rendered -> no pools key. + daemon = render_docker_daemon_config({}) + self.assertNotIn("default-address-pools", daemon) + + +class ApplyScriptTests(unittest.TestCase): + """Integration tests for scripts/apply-docker-network-policy.sh. + + Uses real temp files and injected commands (no Docker daemon required). + """ + + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + self.daemon_dir = Path(self.tmp) / "etc" / "docker" + self.daemon_dir.mkdir(parents=True) + + def _write_daemon(self, content: str) -> Path: + path = self.daemon_dir / "daemon.json" + path.write_text(content, encoding="utf-8") + return path + + def _env(self, **extra: str) -> dict[str, str]: + env = dict(os.environ) + env["CI_FLEET_TESTING"] = "1" + env["CI_FLEET_DOCKER_DAEMON_CONFIG"] = str(self.daemon_dir / "daemon.json") + env["CI_FLEET_DOCKER_RESTART_COMMAND"] = str(Path(self.tmp) / "restart.sh") + env["CI_FLEET_DOCKER_NETWORK_PROBE"] = str(Path(self.tmp) / "probe.sh") + env["CI_FLEET_HEALTH_CHECK_COMMAND"] = str(Path(self.tmp) / "health.sh") + env["CI_FLEET_ROOT_PREFIX"] = self.tmp + for key, value in extra.items(): + env[key] = value + return env + + def _write_env_file(self, rendered: dict[str, str]) -> Path: + path = Path(self.tmp) / "ci-fleet.env" + path.write_text("".join(f"{k}={v}\n" for k, v in sorted(rendered.items())), encoding="utf-8") + return path + + def _rendered_with_policy(self) -> dict[str, str]: + value = config() + value["controllers"]["example-ci-01"]["docker_network_policy"] = docker_network_policy() + capabilities = {"status_reporting_config", "required_status_reporting", "docker_network_policy_config"} + rendered, _ = build_rendered_env( + value, + "example-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + engine_capabilities=capabilities, + ) + return rendered + + def _run(self, env_file: str, checkpoint_dir: str = "", expected_rc: int = 0) -> subprocess.CompletedProcess: + script = str(SCRIPTS / "apply-docker-network-policy.sh") + args = [script] + if checkpoint_dir: + args += ["--checkpoint", checkpoint_dir] + args += ["--env", env_file] + return subprocess.run( + args, + capture_output=True, + text=True, + env=self._env(), + timeout=30, + ) + + def test_validates_policy_before_mutation(self) -> None: + """RED: script must reject a malformed daemon config before applying.""" + bad_env = Path(self.tmp) / "bad.env" + bad_env.write_text( + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=1\n" + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE=198.51.100.0/24\n" + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE=29\n", + encoding="utf-8", + ) + # With a valid restart/probe, this should succeed — but we inject a + # probe that reports exhaustion to force rollback. That tests the + # rollback path, not validation-before-mutation. + # For validation-before-mutation, we test render_docker_daemon_config + # raises on malformed input (covered above). + pass # covered by RenderDaemonConfigTests + + def test_applies_daemon_config_preserving_unrelated_keys(self) -> None: + """GREEN: applying a policy preserves unrelated daemon.json keys.""" + self._write_daemon(json.dumps({"bip": "172.17.0.1/16", "icc": False})) + restart = Path(self.tmp) / "restart.sh" + restart.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + probe.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = self._run(str(env_file)) + self.assertEqual(result.returncode, 0, result.stderr) + daemon = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8")) + self.assertIn("default-address-pools", daemon) + self.assertEqual(len(daemon["default-address-pools"]), 2) + # Unrelated keys preserved + self.assertEqual(daemon.get("bip"), "172.17.0.1/16") + self.assertFalse(daemon.get("icc")) + + def test_rolls_back_on_probe_failure(self) -> None: + """RED: if the capacity probe fails, the prior daemon config is restored.""" + prior = {"bip": "172.17.0.1/16", "icc": False} + self._write_daemon(json.dumps(prior)) + restart = Path(self.tmp) / "restart.sh" + restart.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") # fails + probe.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = self._run(str(env_file), expected_rc=1) + self.assertNotEqual(result.returncode, 0) + daemon = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8")) + self.assertNotIn("default-address-pools", daemon) + self.assertEqual(daemon, prior) + + def test_rolls_back_on_restart_failure(self) -> None: + """RED: if the restart command fails, the prior daemon config is restored.""" + prior = {"bip": "172.17.0.1/16"} + self._write_daemon(json.dumps(prior)) + restart = Path(self.tmp) / "restart.sh" + restart.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") # fails + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + probe.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = self._run(str(env_file), expected_rc=1) + self.assertNotEqual(result.returncode, 0) + daemon = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8")) + self.assertNotIn("default-address-pools", daemon) + self.assertEqual(daemon, prior) + + def test_restart_uses_injected_command_boundary(self) -> None: + """GREEN: restart never calls systemctl/dockerd directly.""" + self._write_daemon(json.dumps({})) + restart = Path(self.tmp) / "restart.sh" + restart.write_text("#!/usr/bin/env bash\nprintf '%s\\n' \"restart-invoked\" > \"$1/restart.log\"\nexit 0\n", encoding="utf-8") + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + probe.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = self._run(str(env_file)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((Path(self.tmp) / "restart.log").exists()) + + def test_failure_evidence_excludes_secrets(self) -> None: + """GREEN: failure evidence must not contain secrets or CIDRs.""" + self._write_daemon(json.dumps({})) + restart = Path(self.tmp) / "restart.sh" + restart.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + probe.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + env = self._env(CI_FLEET_HEALTH_STATUS_URL="https://status.example.invalid/v1/status") + env_file = self._write_env_file(self._rendered_with_policy()) + script = str(SCRIPTS / "apply-docker-network-policy.sh") + result = subprocess.run( + [script, "--env", env_file], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + combined = result.stdout + result.stderr + self.assertNotIn("198.51.100", combined) + self.assertNotIn("203.0.113", combined) + + def test_checkpoint_preserves_prior_config(self) -> None: + """GREEN: the checkpoint retains the prior daemon.json for restoration.""" + prior = {"bip": "172.17.0.1/16", "icc": False} + self._write_daemon(json.dumps(prior)) + restart = Path(self.tmp) / "restart.sh" + restart.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + probe.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + checkpoint_dir = Path(self.tmp) / "checkpoint" + env_file = self._write_env_file(self._rendered_with_policy()) + result = self._run(str(env_file), checkpoint_dir=str(checkpoint_dir)) + self.assertEqual(result.returncode, 0, result.stderr) + backup = checkpoint_dir / "daemon.json" + self.assertTrue(backup.exists()) + self.assertEqual(json.loads(backup.read_text(encoding="utf-8")), prior) + + def test_no_policy_is_noop(self) -> None: + """GREEN: when no network policy is rendered, the script does nothing.""" + self._write_daemon(json.dumps({"bip": "172.17.0.1/16"})) + restart = Path(self.tmp) / "restart.sh" + restart.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + restart.chmod(0o755) + env_file = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + result = self._run(str(env_file)) + self.assertEqual(result.returncode, 0, result.stderr) + + +if __name__ == "__main__": + unittest.main() From 48509ecc502a24bfce12022bf256e31f37e41c4e Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 12:28:26 -0500 Subject: [PATCH 02/54] feat: add Docker daemon network policy apply engine stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add scripts/apply-docker-network-policy.sh — the transactional apply half of the Docker network policy rollout engine stage. It: - Is a no-op when no policy is rendered (safe for unrelated controllers) - Renders desired daemon.json default-address-pools via the existing render_docker_daemon_config() validator (detection stays separate) - Preserves all unrelated daemon.json keys during merge - Backs up the exact prior daemon.json for rollback (into --checkpoint dir or a temp work dir) - Restarts Docker ONLY through CI_FLEET_DOCKER_RESTART_COMMAND boundary - Runs bounded capacity probe (CI_FLEET_DOCKER_NETWORK_PROBE) and health verification (CI_FLEET_HEALTH_CHECK_COMMAND) - Rolls back the exact prior config and re-verifies on any restart/probe health failure - Exposes failure evidence without leaking CIDRs or secrets Update test for restart boundary to assert on daemon_dir location. All 13 tests pass; shellcheck clean. --- scripts/apply-docker-network-policy.sh | 199 ++++++++++++++++++++ scripts/test_apply_docker_network_policy.py | 2 +- 2 files changed, 200 insertions(+), 1 deletion(-) create mode 100755 scripts/apply-docker-network-policy.sh diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh new file mode 100755 index 00000000..7e7e0a59 --- /dev/null +++ b/scripts/apply-docker-network-policy.sh @@ -0,0 +1,199 @@ +#!/usr/bin/env bash +# Engine stage: apply the reviewed Docker daemon network policy transactionally. +# +# Validates, drains, applies daemon.json atomically (preserving unrelated keys), +# restarts Docker ONLY via injected command boundary, runs capacity probes + +# health checks, and rolls back the exact prior config on any failure. +# +# Environment variables (all injected, never host-defaulted): +# CI_FLEET_DOCKER_DAEMON_CONFIG absolute path to daemon.json +# CI_FLEET_DOCKER_RESTART_COMMAND path to a Docker restart script +# CI_FLEET_DOCKER_NETWORK_PROBE path to a capacity probe script +# CI_FLEET_HEALTH_CHECK_COMMAND path to a health-check script +# CI_FLEET_TESTING when 1, relaxes root/strict checks +set -Eeuo pipefail + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +testing=${CI_FLEET_TESTING:-0} + +env_file= +checkpoint_dir= + +usage() { + cat >&2 <<'EOF' +usage: apply-docker-network-policy.sh --env PATH [--checkpoint PATH] + +--env PATH path to the rendered ci-fleet env file (required) +--checkpoint PATH directory to back up the prior daemon.json into +EOF +} + +die() { + printf 'ERROR: %s\n' "$*" >&2 + exit 2 +} + +while (($#)); do + case "$1" in + --env) + [[ $# -ge 2 ]] || die '--env requires a value' + env_file=$2 + shift 2 + ;; + --checkpoint) + [[ $# -ge 2 ]] || die '--checkpoint requires a value' + checkpoint_dir=$2 + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + usage + die "unknown argument: $1" + ;; + esac +done + +[[ -n "$env_file" ]] || die '--env is required' +[[ -r "$env_file" ]] || die "rendered env is unreadable: $env_file" + +# --- No-op when no network policy is rendered --- +count=$(awk -F= '$1 == "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" {print substr($0, index($0, "=") + 1)}' "$env_file") +if [[ -z "$count" || "$count" == "0" ]]; then + printf 'NETWORK_POLICY_NOOP\n' + exit 0 +fi + +# --- Resolve required injected commands --- +daemon_config=${CI_FLEET_DOCKER_DAEMON_CONFIG:-} +restart_command=${CI_FLEET_DOCKER_RESTART_COMMAND:-} +probe_command=${CI_FLEET_DOCKER_NETWORK_PROBE:-} +health_command=${CI_FLEET_HEALTH_CHECK_COMMAND:-} + +[[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' +[[ -n "$restart_command" ]] || die 'CI_FLEET_DOCKER_RESTART_COMMAND is required when a network policy is configured' +[[ -n "$probe_command" ]] || die 'CI_FLEET_DOCKER_NETWORK_PROBE is required when a network policy is configured' +[[ -n "$health_command" ]] || die 'CI_FLEET_HEALTH_CHECK_COMMAND is required when a network policy is configured' +[[ -x "$restart_command" ]] || die "restart command is not executable: $restart_command" +[[ -x "$probe_command" ]] || die "network probe is not executable: $probe_command" +[[ -x "$health_command" ]] || die "health-check command is not executable: $health_command" + +# --- Ownership guard (relaxed in testing) --- +if [[ "$testing" != 1 ]]; then + [[ -w "$(dirname "$daemon_config")" ]] || die "daemon config directory is not writable: $(dirname "$daemon_config")" + if [[ -f "$daemon_config" ]]; then + file_owner=$(stat -c %u "$daemon_config") + [[ "$file_owner" == "0" ]] || die "daemon.json must be owned by root: $daemon_config" + fi +else + : # testing mode — skip root checks +fi + +# --- Render desired daemon config block via shared validator --- +desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" <<'PY' +import json, os, sys +env_path, scripts_dir = sys.argv[1], sys.argv[2] +sys.path.insert(0, scripts_dir) +values = {} +with open(env_path, encoding="utf-8") as handle: + for line in handle: + line = line.rstrip("\n") + if "=" in line and line: + key, _, value = line.partition("=") + values[key] = value +from desired_state import render_docker_daemon_config +print(json.dumps(render_docker_daemon_config(values))) +PY +) || die "daemon policy rendering failed" + +# Re-check: if rendering returned empty, treat as no-op. +if [[ "$desired_pools_json" == "{}" ]]; then + printf 'NETWORK_POLICY_NOOP\n' + exit 0 +fi + +# --- Stage merged daemon.json (preserve unrelated keys) --- +work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") +staging_daemon="$work_dir/daemon.json" + +python3 - "$env_file" "$daemon_config" "$staging_daemon" "$desired_pools_json" <<'PY' || { rm -rf "$work_dir"; die "failed to stage merged daemon.json"; } +import json, os, sys +_, _, daemon_path, staging_path, desired_pools_json = sys.argv +prior = {} +if os.path.exists(daemon_path): + try: + text = open(daemon_path, encoding="utf-8").read() + prior = json.loads(text) + if not isinstance(prior, dict): + raise ValueError("daemon.json root must be an object") + except (json.JSONDecodeError, ValueError) as exc: + raise SystemExit(f"ERROR: existing daemon.json is not a valid JSON object: {exc}") +desired_pools = json.loads(desired_pools_json) +merged = dict(prior) +merged["default-address-pools"] = desired_pools.get("default-address-pools", []) +with open(staging_path, "w", encoding="utf-8") as handle: + json.dump(merged, handle, indent=2, sort_keys=True) + handle.write("\n") +os.chmod(staging_path, 0o644) +PY + +# --- Back up exact prior daemon.json for rollback --- +prior_daemon="$work_dir/prior" +mkdir -p "$prior_daemon" +if [[ -n "$checkpoint_dir" ]]; then + mkdir -p "$checkpoint_dir" + backup_dir="$checkpoint_dir" + backup_name="daemon.json" +else + backup_dir="$prior_daemon" + backup_name="daemon.json.before" +fi +if [[ -f "$daemon_config" ]]; then + cp -p "$daemon_config" "$backup_dir/$backup_name" +fi + +restore_daemon() { + local backup_file="$backup_dir/$backup_name" + if [[ -f "$backup_file" ]]; then + cp -p "$backup_file" "$daemon_config" + fi +} + +# --- Transaction: apply → restart → probe → health, with rollback --- +daemon_dir=$(dirname "$daemon_config") + +# Apply daemon.json atomically (rename within same directory) +python3 - "$staging_daemon" "$daemon_dir" "$daemon_config" <<'PY' || { restore_daemon; rm -rf "$work_dir"; die "failed to apply daemon.json"; } +import os, sys +_, _, daemon_dir, target = sys.argv +tmp = os.path.join(daemon_dir, ".daemon.json.tmp") +os.replace(sys.argv[1], tmp) +os.replace(tmp, target) +PY + +# Restart Docker through the injected command boundary (never host-direct) +if ! "$restart_command" "$daemon_dir" 2>&1; then + restore_daemon + rm -rf "$work_dir" + die "Docker restart command failed; prior daemon.json restored" +fi + +# Bounded capacity probe +if ! "$probe_command" 2>&1; then + restore_daemon + rm -rf "$work_dir" + die "capacity probe failed after network-policy restart; prior daemon.json restored" +fi + +# Health verification +if ! "$health_command" 2>&1; then + restore_daemon + rm -rf "$work_dir" + die "health check failed after network-policy restart; prior daemon.json restored" +fi + +# --- Success --- +rm -rf "$work_dir" +printf 'NETWORK_POLICY_APPLIED daemon_config=%s\n' "$daemon_config" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 0c02b2d6..9f1dc346 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -268,7 +268,7 @@ def test_restart_uses_injected_command_boundary(self) -> None: env_file = self._write_env_file(self._rendered_with_policy()) result = self._run(str(env_file)) self.assertEqual(result.returncode, 0, result.stderr) - self.assertTrue((Path(self.tmp) / "restart.log").exists()) + self.assertTrue((self.daemon_dir / "restart.log").exists()) def test_failure_evidence_excludes_secrets(self) -> None: """GREEN: failure evidence must not contain secrets or CIDRs.""" From eb131cea42c6d84a495334a46313398cd57c1e6a Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:26:05 -0500 Subject: [PATCH 03/54] fix: make Docker network policy apply transaction safe --- scripts/apply-docker-network-policy.sh | 62 +++++--- scripts/test_apply_docker_network_policy.py | 152 ++++++++++++++++++-- 2 files changed, 190 insertions(+), 24 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 7e7e0a59..2e32cc59 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -7,9 +7,11 @@ # # Environment variables (all injected, never host-defaulted): # CI_FLEET_DOCKER_DAEMON_CONFIG absolute path to daemon.json +# CI_FLEET_DOCKER_DRAIN_COMMAND path to a host drain script (runs before mutation) # CI_FLEET_DOCKER_RESTART_COMMAND path to a Docker restart script # CI_FLEET_DOCKER_NETWORK_PROBE path to a capacity probe script # CI_FLEET_HEALTH_CHECK_COMMAND path to a health-check script +# CI_FLEET_COMMAND_TIMEOUT_SECONDS command timeout in seconds (default 300) # CI_FLEET_TESTING when 1, relaxes root/strict checks set -Eeuo pipefail @@ -68,15 +70,19 @@ fi # --- Resolve required injected commands --- daemon_config=${CI_FLEET_DOCKER_DAEMON_CONFIG:-} +drain_command=${CI_FLEET_DOCKER_DRAIN_COMMAND:-} restart_command=${CI_FLEET_DOCKER_RESTART_COMMAND:-} probe_command=${CI_FLEET_DOCKER_NETWORK_PROBE:-} health_command=${CI_FLEET_HEALTH_CHECK_COMMAND:-} +command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} +[[ "$command_timeout" =~ ^[1-9][0-9]*$ ]] || die 'CI_FLEET_COMMAND_TIMEOUT_SECONDS must be a positive integer' [[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' [[ -n "$restart_command" ]] || die 'CI_FLEET_DOCKER_RESTART_COMMAND is required when a network policy is configured' [[ -n "$probe_command" ]] || die 'CI_FLEET_DOCKER_NETWORK_PROBE is required when a network policy is configured' [[ -n "$health_command" ]] || die 'CI_FLEET_HEALTH_CHECK_COMMAND is required when a network policy is configured' [[ -x "$restart_command" ]] || die "restart command is not executable: $restart_command" +[[ -z "$drain_command" || -x "$drain_command" ]] || die "drain command is not executable: $drain_command" [[ -x "$probe_command" ]] || die "network probe is not executable: $probe_command" [[ -x "$health_command" ]] || die "health-check command is not executable: $health_command" @@ -114,6 +120,11 @@ if [[ "$desired_pools_json" == "{}" ]]; then exit 0 fi +# --- Drain before any daemon.json mutation or restart --- +if [[ -n "$drain_command" ]] && ! timeout "$command_timeout" "$drain_command" 2>&1; then + die "drain command failed before network-policy apply" +fi + # --- Stage merged daemon.json (preserve unrelated keys) --- work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") staging_daemon="$work_dir/daemon.json" @@ -150,46 +161,65 @@ else backup_dir="$prior_daemon" backup_name="daemon.json.before" fi +had_prior=false if [[ -f "$daemon_config" ]]; then + had_prior=true cp -p "$daemon_config" "$backup_dir/$backup_name" fi +daemon_dir=$(dirname "$daemon_config") + restore_daemon() { - local backup_file="$backup_dir/$backup_name" - if [[ -f "$backup_file" ]]; then - cp -p "$backup_file" "$daemon_config" + if [[ "$had_prior" == true ]]; then + cp -p "$backup_dir/$backup_name" "$daemon_config" + else + rm -f "$daemon_config" fi } -# --- Transaction: apply → restart → probe → health, with rollback --- -daemon_dir=$(dirname "$daemon_config") +# Rollback: restore prior config, restart through the boundary, run health check. +# Failure evidence is surfaced through exit code only — no CIDRs or secrets leaked. +rollback_daemon() { + local failed=0 + restore_daemon || failed=1 + timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1 || failed=1 + timeout "$command_timeout" "$health_command" >/dev/null 2>&1 || failed=1 + return "$failed" +} +# --- Transaction: apply → restart → probe → health, with rollback --- # Apply daemon.json atomically (rename within same directory) python3 - "$staging_daemon" "$daemon_dir" "$daemon_config" <<'PY' || { restore_daemon; rm -rf "$work_dir"; die "failed to apply daemon.json"; } -import os, sys +import os, shutil, sys, tempfile _, _, daemon_dir, target = sys.argv -tmp = os.path.join(daemon_dir, ".daemon.json.tmp") -os.replace(sys.argv[1], tmp) -os.replace(tmp, target) +fd, tmp = tempfile.mkstemp(prefix=".daemon.json.", dir=daemon_dir) +try: + with open(sys.argv[1], "rb") as source, os.fdopen(fd, "wb") as staged: + shutil.copyfileobj(source, staged) + os.chmod(tmp, 0o644) + os.replace(tmp, target) +finally: + if os.path.exists(tmp): + os.unlink(tmp) PY -# Restart Docker through the injected command boundary (never host-direct) -if ! "$restart_command" "$daemon_dir" 2>&1; then - restore_daemon +# Restart Docker through the injected command boundary (never host-direct). +if ! timeout "$command_timeout" "$restart_command" "$daemon_dir" 2>&1; then + rollback_daemon || true rm -rf "$work_dir" die "Docker restart command failed; prior daemon.json restored" fi # Bounded capacity probe -if ! "$probe_command" 2>&1; then - restore_daemon +if ! timeout "$command_timeout" "$probe_command" 2>&1; then + rollback_daemon || true rm -rf "$work_dir" die "capacity probe failed after network-policy restart; prior daemon.json restored" fi # Health verification -if ! "$health_command" 2>&1; then - restore_daemon +if ! timeout "$command_timeout" "$health_command" 2>&1; then + rollback_daemon || true rm -rf "$work_dir" die "health check failed after network-policy restart; prior daemon.json restored" fi diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 9f1dc346..e1160c38 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -14,6 +14,7 @@ import subprocess import sys import tempfile +import time import unittest from pathlib import Path @@ -176,20 +177,155 @@ def _run(self, env_file: str, checkpoint_dir: str = "", expected_rc: int = 0) -> ) def test_validates_policy_before_mutation(self) -> None: - """RED: script must reject a malformed daemon config before applying.""" + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) bad_env = Path(self.tmp) / "bad.env" bad_env.write_text( "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=1\n" - "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE=198.51.100.0/24\n" + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE=not-a-cidr\n" "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE=29\n", encoding="utf-8", ) - # With a valid restart/probe, this should succeed — but we inject a - # probe that reports exhaustion to force rollback. That tests the - # rollback path, not validation-before-mutation. - # For validation-before-mutation, we test render_docker_daemon_config - # raises on malformed input (covered above). - pass # covered by RenderDaemonConfigTests + drain_marker = Path(self.tmp) / "drain.marker" + restart_marker = Path(self.tmp) / "restart.marker" + for name, marker in (("drain.sh", drain_marker), ("restart.sh", restart_marker)): + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + for name in ("probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(bad_env)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_DOCKER_DRAIN_COMMAND=str(Path(self.tmp) / "drain.sh")), + timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + self.assertFalse(restart_marker.exists()) + + def test_drain_failure_prevents_mutation_and_restart(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + drain = Path(self.tmp) / "drain.sh" + drain.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + drain.chmod(0o755) + restart_marker = Path(self.tmp) / "restart.marker" + restart = Path(self.tmp) / "restart.sh" + restart.write_text(f"#!/usr/bin/env bash\ntouch {restart_marker}\n", encoding="utf-8") + restart.chmod(0o755) + for name in ("probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_DOCKER_DRAIN_COMMAND=str(drain)), + timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(restart_marker.exists()) + + def test_probe_failure_restores_absent_config_and_restarts(self) -> None: + restart_log = Path(self.tmp) / "restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text(f"#!/usr/bin/env bash\necho restart >> {restart_log}\n", encoding="utf-8") + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + health_log = Path(self.tmp) / "health.log" + health = Path(self.tmp) / "health.sh" + health.write_text(f"#!/usr/bin/env bash\necho health >> {health_log}\n", encoding="utf-8") + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = self._run(str(env_file), expected_rc=1) + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.daemon_dir / "daemon.json").exists()) + self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) + self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) + + def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: + prior = b'{"bip":"172.17.0.1/16","icc":false}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + restart_log = Path(self.tmp) / "restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {restart_log}\n" + f"[[ $(wc -l < {restart_log}) -eq 1 ]] || {{ echo '198.51.100.0/24 super-secret'; exit 1; }}\n", + encoding="utf-8", + ) + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + health_log = Path(self.tmp) / "health.log" + health = Path(self.tmp) / "health.sh" + health.write_text(f"#!/usr/bin/env bash\necho health >> {health_log}\n", encoding="utf-8") + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = self._run(str(env_file), expected_rc=1) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) + self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) + self.assertNotIn("198.51.100.0/24", result.stdout + result.stderr) + self.assertNotIn("super-secret", result.stdout + result.stderr) + + def test_sleeping_probe_times_out(self) -> None: + self._write_daemon("{}\n") + for name in ("drain.sh", "restart.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nsleep 10\n", encoding="utf-8") + probe.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + started = time.monotonic() + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env( + CI_FLEET_DOCKER_DRAIN_COMMAND=str(Path(self.tmp) / "drain.sh"), + CI_FLEET_COMMAND_TIMEOUT_SECONDS="1", + ), + timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + self.assertLess(time.monotonic() - started, 5) + + @unittest.skipUnless(Path("/dev/shm").is_dir(), "/dev/shm is unavailable") + def test_apply_works_across_temp_filesystems(self) -> None: + self._write_daemon(json.dumps({"bip": "172.17.0.1/16"})) + for name in ("restart.sh", "probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_TEMP_DIR="/dev/shm"), + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + daemon = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8")) + self.assertEqual(daemon["bip"], "172.17.0.1/16") + self.assertIn("default-address-pools", daemon) def test_applies_daemon_config_preserving_unrelated_keys(self) -> None: """GREEN: applying a policy preserves unrelated daemon.json keys.""" From c8848aae8c49092b94532a1f9cf076f0b0a293c2 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:34:25 -0500 Subject: [PATCH 04/54] fix: require safe drain and report rollback failures --- scripts/apply-docker-network-policy.sh | 36 +++++---- scripts/test_apply_docker_network_policy.py | 83 +++++++++++++++++++++ 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 2e32cc59..5c23dbe7 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -78,11 +78,12 @@ command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} [[ "$command_timeout" =~ ^[1-9][0-9]*$ ]] || die 'CI_FLEET_COMMAND_TIMEOUT_SECONDS must be a positive integer' [[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' +[[ -n "$drain_command" ]] || die 'CI_FLEET_DOCKER_DRAIN_COMMAND is required when a network policy is configured' [[ -n "$restart_command" ]] || die 'CI_FLEET_DOCKER_RESTART_COMMAND is required when a network policy is configured' [[ -n "$probe_command" ]] || die 'CI_FLEET_DOCKER_NETWORK_PROBE is required when a network policy is configured' [[ -n "$health_command" ]] || die 'CI_FLEET_HEALTH_CHECK_COMMAND is required when a network policy is configured' [[ -x "$restart_command" ]] || die "restart command is not executable: $restart_command" -[[ -z "$drain_command" || -x "$drain_command" ]] || die "drain command is not executable: $drain_command" +[[ -x "$drain_command" ]] || die "drain command is not executable: $drain_command" [[ -x "$probe_command" ]] || die "network probe is not executable: $probe_command" [[ -x "$health_command" ]] || die "health-check command is not executable: $health_command" @@ -120,11 +121,6 @@ if [[ "$desired_pools_json" == "{}" ]]; then exit 0 fi -# --- Drain before any daemon.json mutation or restart --- -if [[ -n "$drain_command" ]] && ! timeout "$command_timeout" "$drain_command" 2>&1; then - die "drain command failed before network-policy apply" -fi - # --- Stage merged daemon.json (preserve unrelated keys) --- work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") staging_daemon="$work_dir/daemon.json" @@ -169,6 +165,12 @@ fi daemon_dir=$(dirname "$daemon_config") +# --- Drain after local validation/checkpointing, before mutation or restart --- +if ! timeout "$command_timeout" "$drain_command" 2>&1; then + rm -rf "$work_dir" + die "drain command failed before network-policy apply" +fi + restore_daemon() { if [[ "$had_prior" == true ]]; then cp -p "$backup_dir/$backup_name" "$daemon_config" @@ -187,6 +189,16 @@ rollback_daemon() { return "$failed" } +fail_after_apply() { + local failure=$1 + if rollback_daemon; then + rm -rf "$work_dir" + die "$failure; prior daemon.json restored" + fi + rm -rf "$work_dir" + die "$failure; rollback verification failed" +} + # --- Transaction: apply → restart → probe → health, with rollback --- # Apply daemon.json atomically (rename within same directory) python3 - "$staging_daemon" "$daemon_dir" "$daemon_config" <<'PY' || { restore_daemon; rm -rf "$work_dir"; die "failed to apply daemon.json"; } @@ -205,23 +217,17 @@ PY # Restart Docker through the injected command boundary (never host-direct). if ! timeout "$command_timeout" "$restart_command" "$daemon_dir" 2>&1; then - rollback_daemon || true - rm -rf "$work_dir" - die "Docker restart command failed; prior daemon.json restored" + fail_after_apply "Docker restart command failed" fi # Bounded capacity probe if ! timeout "$command_timeout" "$probe_command" 2>&1; then - rollback_daemon || true - rm -rf "$work_dir" - die "capacity probe failed after network-policy restart; prior daemon.json restored" + fail_after_apply "capacity probe failed after network-policy restart" fi # Health verification if ! timeout "$command_timeout" "$health_command" 2>&1; then - rollback_daemon || true - rm -rf "$work_dir" - die "health check failed after network-policy restart; prior daemon.json restored" + fail_after_apply "health check failed after network-policy restart" fi # --- Success --- diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index e1160c38..50c87a8f 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -124,6 +124,9 @@ def setUp(self) -> None: self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) self.daemon_dir = Path(self.tmp) / "etc" / "docker" self.daemon_dir.mkdir(parents=True) + self.drain_command = Path(self.tmp) / "drain.sh" + self.drain_command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + self.drain_command.chmod(0o755) def _write_daemon(self, content: str) -> Path: path = self.daemon_dir / "daemon.json" @@ -134,6 +137,7 @@ def _env(self, **extra: str) -> dict[str, str]: env = dict(os.environ) env["CI_FLEET_TESTING"] = "1" env["CI_FLEET_DOCKER_DAEMON_CONFIG"] = str(self.daemon_dir / "daemon.json") + env["CI_FLEET_DOCKER_DRAIN_COMMAND"] = str(self.drain_command) env["CI_FLEET_DOCKER_RESTART_COMMAND"] = str(Path(self.tmp) / "restart.sh") env["CI_FLEET_DOCKER_NETWORK_PROBE"] = str(Path(self.tmp) / "probe.sh") env["CI_FLEET_HEALTH_CHECK_COMMAND"] = str(Path(self.tmp) / "health.sh") @@ -176,6 +180,58 @@ def _run(self, env_file: str, checkpoint_dir: str = "", expected_rc: int = 0) -> timeout=30, ) + def test_policy_requires_drain_command(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + restart_marker = Path(self.tmp) / "restart.marker" + restart = Path(self.tmp) / "restart.sh" + restart.write_text(f"#!/usr/bin/env bash\ntouch {restart_marker}\n", encoding="utf-8") + restart.chmod(0o755) + for name in ("probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + env = self._env() + env.pop("CI_FLEET_DOCKER_DRAIN_COMMAND") + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(restart_marker.exists()) + + def test_invalid_existing_config_does_not_drain(self) -> None: + daemon = self._write_daemon("{not-json\n") + drain_marker = Path(self.tmp) / "drain.marker" + drain = Path(self.tmp) / "drain-marker.sh" + drain.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + drain.chmod(0o755) + for name in ("restart.sh", "probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_DOCKER_DRAIN_COMMAND=str(drain)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_text(encoding="utf-8"), "{not-json\n") + self.assertFalse(drain_marker.exists()) + def test_validates_policy_before_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -283,6 +339,33 @@ def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: self.assertNotIn("198.51.100.0/24", result.stdout + result.stderr) self.assertNotIn("super-secret", result.stdout + result.stderr) + def test_rollback_failure_is_reported(self) -> None: + self._write_daemon('{"bip":"172.17.0.1/16"}\n') + restart_log = Path(self.tmp) / "restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {restart_log}\n" + f"[[ $(wc -l < {restart_log}) -eq 1 ]] || {{ echo '198.51.100.0/24 super-secret'; exit 1; }}\n", + encoding="utf-8", + ) + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + combined = result.stdout + result.stderr + self.assertIn("rollback verification failed", combined) + self.assertNotIn("prior daemon.json restored", combined) + self.assertNotIn("198.51.100.0/24", combined) + self.assertNotIn("super-secret", combined) + def test_sleeping_probe_times_out(self) -> None: self._write_daemon("{}\n") for name in ("drain.sh", "restart.sh", "health.sh"): From 5ca6e04e047fef686cd5e29a16bf741e3cd540ee Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:48:05 -0500 Subject: [PATCH 05/54] fix: harden Docker network policy file transaction --- scripts/apply-docker-network-policy.sh | 49 ++++-- scripts/test_apply_docker_network_policy.py | 156 ++++++++++++++++++++ 2 files changed, 196 insertions(+), 9 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 5c23dbe7..9a87ef6c 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -87,7 +87,25 @@ command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} [[ -x "$probe_command" ]] || die "network probe is not executable: $probe_command" [[ -x "$health_command" ]] || die "health-check command is not executable: $health_command" +# Serialize with installer mutations using the installer's host-local lock. +lock_file=${CI_FLEET_INSTALLER_LOCK:-${CI_FLEET_ROOT_PREFIX:-}/run/ci-fleet-installer.lock} +if [[ -n ${CI_FLEET_INSTALLER_LOCK_FD:-} ]]; then + [[ "$CI_FLEET_INSTALLER_LOCK_FD" == 9 ]] || die 'inherited installer lock must use file descriptor 9' + [[ $(readlink -f /proc/self/fd/9 2>/dev/null || true) == $(readlink -m "$lock_file") ]] || die 'inherited installer lock does not match the configured lock file' + flock -n 9 || die 'inherited installer lock is unavailable' +else + install -d -m 0755 "$(dirname "$lock_file")" + exec 9>"$lock_file" + flock -n 9 || die 'another ci-fleet installer or drift check is already running' +fi + +[[ ! -L "$daemon_config" ]] || die "daemon.json must not be a symlink: $daemon_config" + # --- Ownership guard (relaxed in testing) --- +daemon_mode=644 +if [[ -f "$daemon_config" ]]; then + daemon_mode=$(stat -c %a "$daemon_config") +fi if [[ "$testing" != 1 ]]; then [[ -w "$(dirname "$daemon_config")" ]] || die "daemon config directory is not writable: $(dirname "$daemon_config")" if [[ -f "$daemon_config" ]]; then @@ -189,26 +207,38 @@ rollback_daemon() { return "$failed" } -fail_after_apply() { - local failure=$1 - if rollback_daemon; then +transaction_failure='network-policy apply interrupted' +rollback_on_exit() { + local status=$? + trap - EXIT INT TERM + ((status != 0)) || status=1 + if rollback_daemon >/dev/null 2>&1; then rm -rf "$work_dir" - die "$failure; prior daemon.json restored" + printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 + else + printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$transaction_failure" "$work_dir" >&2 fi - rm -rf "$work_dir" - die "$failure; rollback verification failed" + exit "$status" +} +trap 'exit 130' INT +trap 'exit 143' TERM +trap rollback_on_exit EXIT + +fail_after_apply() { + transaction_failure=$1 + exit 2 } # --- Transaction: apply → restart → probe → health, with rollback --- # Apply daemon.json atomically (rename within same directory) -python3 - "$staging_daemon" "$daemon_dir" "$daemon_config" <<'PY' || { restore_daemon; rm -rf "$work_dir"; die "failed to apply daemon.json"; } +python3 - "$staging_daemon" "$daemon_dir" "$daemon_config" "$daemon_mode" <<'PY' || { transaction_failure='failed to apply daemon.json'; exit 2; } import os, shutil, sys, tempfile -_, _, daemon_dir, target = sys.argv +_, _, daemon_dir, target, daemon_mode = sys.argv fd, tmp = tempfile.mkstemp(prefix=".daemon.json.", dir=daemon_dir) try: with open(sys.argv[1], "rb") as source, os.fdopen(fd, "wb") as staged: shutil.copyfileobj(source, staged) - os.chmod(tmp, 0o644) + os.chmod(tmp, int(daemon_mode, 8)) os.replace(tmp, target) finally: if os.path.exists(tmp): @@ -231,5 +261,6 @@ if ! timeout "$command_timeout" "$health_command" 2>&1; then fi # --- Success --- +trap - EXIT INT TERM rm -rf "$work_dir" printf 'NETWORK_POLICY_APPLIED daemon_config=%s\n' "$daemon_config" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 50c87a8f..0510c384 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -8,9 +8,11 @@ """ from __future__ import annotations +import fcntl import json import os import shutil +import signal import subprocess import sys import tempfile @@ -208,6 +210,83 @@ def test_policy_requires_drain_command(self) -> None: self.assertEqual(daemon.read_bytes(), prior) self.assertFalse(restart_marker.exists()) + def test_existing_installer_lock_blocks_before_drain_or_checkpoint(self) -> None: + daemon = self._write_daemon("{}\n") + prior = daemon.read_bytes() + drain_marker = Path(self.tmp) / "drain.marker" + drain = Path(self.tmp) / "drain-marker.sh" + drain.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + drain.chmod(0o755) + for name in ("restart.sh", "probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + lock = Path(self.tmp) / "run" / "ci-fleet-installer.lock" + lock.parent.mkdir() + checkpoint = Path(self.tmp) / "checkpoint" + env_file = self._write_env_file(self._rendered_with_policy()) + + with lock.open("w") as lock_handle: + fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env( + CI_FLEET_DOCKER_DRAIN_COMMAND=str(drain), + CI_FLEET_INSTALLER_LOCK=str(lock), + ), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + self.assertFalse(checkpoint.exists()) + + def test_symlinked_daemon_config_is_rejected_before_drain_or_checkpoint(self) -> None: + referent = Path(self.tmp) / "referent.json" + prior = b'{"bip":"172.17.0.1/16"}\n' + referent.write_bytes(prior) + daemon = self.daemon_dir / "daemon.json" + daemon.symlink_to(referent) + drain_marker = Path(self.tmp) / "drain.marker" + drain = Path(self.tmp) / "drain-marker.sh" + drain.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + drain.chmod(0o755) + for name in ("restart.sh", "probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + checkpoint = Path(self.tmp) / "checkpoint" + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_DOCKER_DRAIN_COMMAND=str(drain)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertTrue(daemon.is_symlink()) + self.assertEqual(referent.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + self.assertFalse(checkpoint.exists()) + def test_invalid_existing_config_does_not_drain(self) -> None: daemon = self._write_daemon("{not-json\n") drain_marker = Path(self.tmp) / "drain.marker" @@ -339,6 +418,69 @@ def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: self.assertNotIn("198.51.100.0/24", result.stdout + result.stderr) self.assertNotIn("super-secret", result.stdout + result.stderr) + def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + ready = Path(self.tmp) / "restart.ready" + restart_log = Path(self.tmp) / "restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + "#!/usr/bin/env bash\n" + f"echo restart >> {restart_log}\n" + f"if [[ $(wc -l < {restart_log}) -eq 1 ]]; then\n" + f" touch {ready}\n" + " sleep 30\n" + "else\n" + " echo rollback-restart-output\n" + " exit 1\n" + "fi\n", + encoding="utf-8", + ) + restart.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + probe.chmod(0o755) + health_log = Path(self.tmp) / "health.log" + health = Path(self.tmp) / "health.sh" + health.write_text( + f"#!/usr/bin/env bash\necho health >> {health_log}\necho rollback-health-output\n", + encoding="utf-8", + ) + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + process = subprocess.Popen( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=self._env( + CI_FLEET_COMMAND_TIMEOUT_SECONDS="1", + CI_FLEET_TEMP_DIR=self.tmp, + ), + start_new_session=True, + ) + self.addCleanup(lambda: process.poll() is None and process.kill()) + for _ in range(100): + if ready.exists(): + break + time.sleep(0.02) + self.assertTrue(ready.exists(), "apply did not reach restart after replacement") + self.assertIn("default-address-pools", json.loads(daemon.read_text(encoding="utf-8"))) + + os.killpg(process.pid, signal.SIGTERM) + stdout, stderr = process.communicate(timeout=10) + + self.assertNotEqual(process.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) + self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) + self.assertNotIn("rollback-restart-output", stdout + stderr) + self.assertNotIn("rollback-health-output", stdout + stderr) + recovery_dirs = list(Path(self.tmp).glob(".ci-fleet-apply.*")) + self.assertEqual(len(recovery_dirs), 1) + self.assertEqual((recovery_dirs[0] / "prior" / "daemon.json.before").read_bytes(), prior) + def test_rollback_failure_is_reported(self) -> None: self._write_daemon('{"bip":"172.17.0.1/16"}\n') restart_log = Path(self.tmp) / "restart.log" @@ -410,6 +552,20 @@ def test_apply_works_across_temp_filesystems(self) -> None: self.assertEqual(daemon["bip"], "172.17.0.1/16") self.assertIn("default-address-pools", daemon) + def test_preserves_restrictive_daemon_config_mode(self) -> None: + daemon = self._write_daemon("{}\n") + daemon.chmod(0o600) + for name in ("restart.sh", "probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(daemon.stat().st_mode & 0o777, 0o600) + def test_applies_daemon_config_preserving_unrelated_keys(self) -> None: """GREEN: applying a policy preserves unrelated daemon.json keys.""" self._write_daemon(json.dumps({"bip": "172.17.0.1/16", "icc": False})) From 5a12d4a9bc60e5138c4560711e096d9aa9bef78b Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:16:16 -0500 Subject: [PATCH 06/54] fix: close Docker policy verification gaps --- scripts/apply-docker-network-policy.sh | 21 ++- scripts/healthcheck.sh | 5 + scripts/test_apply_docker_network_policy.py | 176 +++++++++++++++++++- scripts/validate.sh | 2 + 4 files changed, 190 insertions(+), 14 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 9a87ef6c..ff41c054 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -164,6 +164,17 @@ with open(staging_path, "w", encoding="utf-8") as handle: os.chmod(staging_path, 0o644) PY +if [[ -f "$daemon_config" ]] && python3 - "$daemon_config" "$staging_daemon" <<'PY' +import json, sys +with open(sys.argv[1], encoding="utf-8") as current, open(sys.argv[2], encoding="utf-8") as staged: + raise SystemExit(json.load(current) != json.load(staged)) +PY +then + rm -rf "$work_dir" + printf 'NETWORK_POLICY_NO_CHANGE\n' + exit 0 +fi + # --- Back up exact prior daemon.json for rollback --- prior_daemon="$work_dir/prior" mkdir -p "$prior_daemon" @@ -184,7 +195,7 @@ fi daemon_dir=$(dirname "$daemon_config") # --- Drain after local validation/checkpointing, before mutation or restart --- -if ! timeout "$command_timeout" "$drain_command" 2>&1; then +if ! timeout "$command_timeout" "$drain_command" >/dev/null 2>&1; then rm -rf "$work_dir" die "drain command failed before network-policy apply" fi @@ -203,7 +214,7 @@ rollback_daemon() { local failed=0 restore_daemon || failed=1 timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1 || failed=1 - timeout "$command_timeout" "$health_command" >/dev/null 2>&1 || failed=1 + timeout "$command_timeout" "$health_command" --env "$env_file" >/dev/null 2>&1 || failed=1 return "$failed" } @@ -246,17 +257,17 @@ finally: PY # Restart Docker through the injected command boundary (never host-direct). -if ! timeout "$command_timeout" "$restart_command" "$daemon_dir" 2>&1; then +if ! timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1; then fail_after_apply "Docker restart command failed" fi # Bounded capacity probe -if ! timeout "$command_timeout" "$probe_command" 2>&1; then +if ! timeout "$command_timeout" "$probe_command" >/dev/null 2>&1; then fail_after_apply "capacity probe failed after network-policy restart" fi # Health verification -if ! timeout "$command_timeout" "$health_command" 2>&1; then +if ! timeout "$command_timeout" "$health_command" --env "$env_file" >/dev/null 2>&1; then fail_after_apply "health check failed after network-policy restart" fi diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index e02c40cf..9917e6fe 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -9,6 +9,11 @@ if [[ ${CI_FLEET_TESTING:-0} == 1 && -n ${CI_FLEET_ROOT_PREFIX:-} ]]; then environment="$CI_FLEET_ROOT_PREFIX/etc/ci-fleet/ci-fleet.env" args+=(--monitoring-config "$CI_FLEET_ROOT_PREFIX/etc/ci-fleet/monitoring.env" --output "$CI_FLEET_ROOT_PREFIX/var/lib/ci-fleet/health/latest.json") fi +if [[ ${1:-} == --env ]]; then + [[ $# -ge 2 && -r $2 ]] || { printf 'ERROR: --env requires a readable file\n' >&2; exit 2; } + environment=$2 + shift 2 +fi if [[ -r $environment ]]; then set -a # shellcheck disable=SC1090 diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 0510c384..b38682ec 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -115,6 +115,48 @@ def test_not_configured_returns_empty(self) -> None: self.assertNotIn("default-address-pools", daemon) +class ValidationTests(unittest.TestCase): + def test_validate_runs_docker_network_policy_suite(self) -> None: + validate = (SCRIPTS / "validate.sh").read_text(encoding="utf-8") + self.assertIn("python3 scripts/test_apply_docker_network_policy.py", validate) + + +class HealthcheckScriptTests(unittest.TestCase): + def test_env_argument_sources_candidate_rendered_env(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + installed = root / "etc" / "ci-fleet" / "ci-fleet.env" + installed.parent.mkdir(parents=True) + installed.write_text("ENV_MARKER=stale\n", encoding="utf-8") + candidate = root / "candidate.env" + candidate.write_text("ENV_MARKER=candidate\n", encoding="utf-8") + fake_bin = root / "bin" + fake_bin.mkdir() + python = fake_bin / "python3" + python.write_text( + "#!/usr/bin/env bash\n" + "[[ ${ENV_MARKER:-} == candidate ]] || exit 1\n", + encoding="utf-8", + ) + python.chmod(0o755) + env = dict(os.environ) + env.update( + CI_FLEET_TESTING="1", + CI_FLEET_ROOT_PREFIX=tmp, + PATH=f"{fake_bin}:{env['PATH']}", + ) + + result = subprocess.run( + [str(SCRIPTS / "healthcheck.sh"), "--env", str(candidate)], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + class ApplyScriptTests(unittest.TestCase): """Integration tests for scripts/apply-docker-network-policy.sh. @@ -311,6 +353,29 @@ def test_invalid_existing_config_does_not_drain(self) -> None: self.assertEqual(daemon.read_text(encoding="utf-8"), "{not-json\n") self.assertFalse(drain_marker.exists()) + def test_semantically_equal_daemon_config_is_no_change_before_side_effects(self) -> None: + rendered = self._rendered_with_policy() + desired = render_docker_daemon_config(rendered) + daemon = self._write_daemon(json.dumps(desired, separators=(",", ":"))) + prior = daemon.read_bytes() + markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): + marker = Path(self.tmp) / f"{name}.marker" + markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(rendered) + checkpoint = Path(self.tmp) / "checkpoint" + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "NETWORK_POLICY_NO_CHANGE\n") + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(checkpoint.exists()) + self.assertTrue(all(not marker.exists() for marker in markers)) + def test_validates_policy_before_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -349,7 +414,13 @@ def test_drain_failure_prevents_mutation_and_restart(self) -> None: daemon = self.daemon_dir / "daemon.json" daemon.write_bytes(prior) drain = Path(self.tmp) / "drain.sh" - drain.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + drain.write_text( + "#!/usr/bin/env bash\n" + "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" + "echo 'drain-secret-error' >&2\n" + "exit 1\n", + encoding="utf-8", + ) drain.chmod(0o755) restart_marker = Path(self.tmp) / "restart.marker" restart = Path(self.tmp) / "restart.sh" @@ -370,6 +441,12 @@ def test_drain_failure_prevents_mutation_and_restart(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(daemon.read_bytes(), prior) self.assertFalse(restart_marker.exists()) + combined = result.stdout + result.stderr + self.assertEqual(combined, "ERROR: drain command failed before network-policy apply\n") + self.assertNotIn("198.51.100.0/24", combined) + self.assertNotIn("secret.example.invalid", combined) + self.assertNotIn("credential", combined) + self.assertNotIn("drain-secret-error", combined) def test_probe_failure_restores_absent_config_and_restarts(self) -> None: restart_log = Path(self.tmp) / "restart.log" @@ -403,11 +480,22 @@ def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: ) restart.chmod(0o755) probe = Path(self.tmp) / "probe.sh" - probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.write_text( + "#!/usr/bin/env bash\n" + "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" + "echo 'probe-secret-error' >&2\n" + "exit 1\n", + encoding="utf-8", + ) probe.chmod(0o755) health_log = Path(self.tmp) / "health.log" health = Path(self.tmp) / "health.sh" - health.write_text(f"#!/usr/bin/env bash\necho health >> {health_log}\n", encoding="utf-8") + health.write_text( + "#!/usr/bin/env bash\n" + f"[[ $1 == --env && $2 == {self.tmp}/ci-fleet.env ]] || exit 1\n" + f"echo health >> {health_log}\n", + encoding="utf-8", + ) health.chmod(0o755) env_file = self._write_env_file(self._rendered_with_policy()) result = self._run(str(env_file), expected_rc=1) @@ -417,6 +505,9 @@ def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) self.assertNotIn("198.51.100.0/24", result.stdout + result.stderr) self.assertNotIn("super-secret", result.stdout + result.stderr) + self.assertNotIn("secret.example.invalid", result.stdout + result.stderr) + self.assertNotIn("credential", result.stdout + result.stderr) + self.assertNotIn("probe-secret-error", result.stdout + result.stderr) def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' @@ -482,12 +573,13 @@ def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> self.assertEqual((recovery_dirs[0] / "prior" / "daemon.json.before").read_bytes(), prior) def test_rollback_failure_is_reported(self) -> None: - self._write_daemon('{"bip":"172.17.0.1/16"}\n') + prior = b'{"bip":"172.17.0.1/16","registry-mirrors":["https://mirror.example.invalid?token=credential"]}\n' + self._write_daemon(prior.decode()) restart_log = Path(self.tmp) / "restart.log" restart = Path(self.tmp) / "restart.sh" restart.write_text( f"#!/usr/bin/env bash\necho restart >> {restart_log}\n" - f"[[ $(wc -l < {restart_log}) -eq 1 ]] || {{ echo '198.51.100.0/24 super-secret'; exit 1; }}\n", + f"[[ $(wc -l < {restart_log}) -eq 1 ]] || {{ echo '198.51.100.0/24 https://secret.example.invalid token=credential'; echo rollback-secret-error >&2; exit 1; }}\n", encoding="utf-8", ) restart.chmod(0o755) @@ -506,7 +598,64 @@ def test_rollback_failure_is_reported(self) -> None: self.assertIn("rollback verification failed", combined) self.assertNotIn("prior daemon.json restored", combined) self.assertNotIn("198.51.100.0/24", combined) - self.assertNotIn("super-secret", combined) + self.assertNotIn("172.17.0.1/16", combined) + self.assertNotIn("secret.example.invalid", combined) + self.assertNotIn("mirror.example.invalid", combined) + self.assertNotIn("credential", combined) + self.assertNotIn("rollback-secret-error", combined) + recovery = Path(combined.rstrip().rsplit("recovery data retained at ", 1)[1]) + self.addCleanup(shutil.rmtree, recovery, ignore_errors=True) + self.assertTrue(recovery.name.startswith(".ci-fleet-apply.")) + self.assertEqual((recovery / "prior" / "daemon.json.before").read_bytes(), prior) + + def test_post_apply_health_uses_candidate_rendered_env(self) -> None: + self._write_daemon("{}\n") + for name in ("restart.sh", "probe.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + health_log = Path(self.tmp) / "health.log" + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"[[ $1 == --env && $2 == {self.tmp}/ci-fleet.env ]] || exit 1\n" + f"printf '%s\\n' \"$2\" >> {health_log}\n", + encoding="utf-8", + ) + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), [str(env_file)]) + + def test_health_failure_evidence_excludes_command_output(self) -> None: + self._write_daemon("{}\n") + for name in ("restart.sh", "probe.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" + "echo 'health-secret-error' >&2\n" + "exit 1\n", + encoding="utf-8", + ) + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + combined = result.stdout + result.stderr + self.assertIn("ERROR: health check failed after network-policy restart", combined) + self.assertNotIn("198.51.100.0/24", combined) + self.assertNotIn("secret.example.invalid", combined) + self.assertNotIn("credential", combined) + self.assertNotIn("health-secret-error", combined) def test_sleeping_probe_times_out(self) -> None: self._write_daemon("{}\n") @@ -645,11 +794,16 @@ def test_restart_uses_injected_command_boundary(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue((self.daemon_dir / "restart.log").exists()) - def test_failure_evidence_excludes_secrets(self) -> None: - """GREEN: failure evidence must not contain secrets or CIDRs.""" + def test_restart_failure_evidence_excludes_command_output(self) -> None: self._write_daemon(json.dumps({})) restart = Path(self.tmp) / "restart.sh" - restart.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + restart.write_text( + "#!/usr/bin/env bash\n" + "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" + "echo 'restart-secret-error' >&2\n" + "exit 1\n", + encoding="utf-8", + ) restart.chmod(0o755) probe = Path(self.tmp) / "probe.sh" probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") @@ -669,8 +823,12 @@ def test_failure_evidence_excludes_secrets(self) -> None: ) self.assertNotEqual(result.returncode, 0) combined = result.stdout + result.stderr + self.assertIn("ERROR: Docker restart command failed", combined) self.assertNotIn("198.51.100", combined) self.assertNotIn("203.0.113", combined) + self.assertNotIn("secret.example.invalid", combined) + self.assertNotIn("credential", combined) + self.assertNotIn("restart-secret-error", combined) def test_checkpoint_preserves_prior_config(self) -> None: """GREEN: the checkpoint retains the prior daemon.json for restoration.""" diff --git a/scripts/validate.sh b/scripts/validate.sh index 271977ea..c03b8ca2 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -14,11 +14,13 @@ python3 -m py_compile \ scripts/status_receiver.py \ scripts/scan_committed_secrets.py \ scripts/test_desired_state.py \ + scripts/test_apply_docker_network_policy.py \ scripts/test_health.py \ scripts/test_status_receiver.py \ scripts/test_quickstart.py python3 .github/actions/plan/test_plan.py python3 scripts/test_desired_state.py +python3 scripts/test_apply_docker_network_policy.py python3 scripts/test_health.py python3 scripts/test_status_receiver.py python3 scripts/test_quickstart.py From be86098c5ff3bc03b2aa44d6bac6ab0d1e21d8cc Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:45:23 -0500 Subject: [PATCH 07/54] fix: restore prior Docker policy on desired-state removal --- scripts/apply-docker-network-policy.sh | 176 +++++++++++- scripts/test_apply_docker_network_policy.py | 289 +++++++++++++++++++- 2 files changed, 436 insertions(+), 29 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index ff41c054..79186ba7 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -63,9 +63,16 @@ done # --- No-op when no network policy is rendered --- count=$(awk -F= '$1 == "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" {print substr($0, index($0, "=") + 1)}' "$env_file") +removing=false if [[ -z "$count" || "$count" == "0" ]]; then - printf 'NETWORK_POLICY_NOOP\n' - exit 0 + state_file=${checkpoint_dir:+$checkpoint_dir/docker-network-policy.json} + if [[ -z "$state_file" || ! -e "$state_file" ]]; then + printf 'NETWORK_POLICY_NOOP\n' + exit 0 + fi + removing=true +else + [[ -n "$checkpoint_dir" ]] || die '--checkpoint is required when a network policy is configured' fi # --- Resolve required injected commands --- @@ -79,13 +86,15 @@ command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} [[ "$command_timeout" =~ ^[1-9][0-9]*$ ]] || die 'CI_FLEET_COMMAND_TIMEOUT_SECONDS must be a positive integer' [[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' [[ -n "$drain_command" ]] || die 'CI_FLEET_DOCKER_DRAIN_COMMAND is required when a network policy is configured' -[[ -n "$restart_command" ]] || die 'CI_FLEET_DOCKER_RESTART_COMMAND is required when a network policy is configured' -[[ -n "$probe_command" ]] || die 'CI_FLEET_DOCKER_NETWORK_PROBE is required when a network policy is configured' -[[ -n "$health_command" ]] || die 'CI_FLEET_HEALTH_CHECK_COMMAND is required when a network policy is configured' +[[ -n "$restart_command" ]] || die 'CI_FLEET_DOCKER_RESTART_COMMAND is required when network policy is managed' +[[ -n "$health_command" ]] || die 'CI_FLEET_HEALTH_CHECK_COMMAND is required when network policy is managed' [[ -x "$restart_command" ]] || die "restart command is not executable: $restart_command" [[ -x "$drain_command" ]] || die "drain command is not executable: $drain_command" -[[ -x "$probe_command" ]] || die "network probe is not executable: $probe_command" [[ -x "$health_command" ]] || die "health-check command is not executable: $health_command" +if [[ "$removing" == false ]]; then + [[ -n "$probe_command" ]] || die 'CI_FLEET_DOCKER_NETWORK_PROBE is required when a network policy is configured' + [[ -x "$probe_command" ]] || die "network probe is not executable: $probe_command" +fi # Serialize with installer mutations using the installer's host-local lock. lock_file=${CI_FLEET_INSTALLER_LOCK:-${CI_FLEET_ROOT_PREFIX:-}/run/ci-fleet-installer.lock} @@ -116,6 +125,96 @@ else : # testing mode — skip root checks fi +if [[ "$removing" == true ]]; then + [[ -f "$state_file" && ! -L "$state_file" ]] || die 'network-policy checkpoint state is invalid' + mapfile -t managed_state < <(python3 - "$state_file" <<'PY' +import json, sys +state = json.load(open(sys.argv[1], encoding="utf-8")) +if set(state) != {"managed", "prior_mode", "prior_present"} or state["managed"] is not True: + raise SystemExit(1) +if not isinstance(state["prior_present"], bool): + raise SystemExit(1) +mode = state["prior_mode"] +if state["prior_present"]: + if not isinstance(mode, str) or not mode.isdigit(): + raise SystemExit(1) +elif mode is not None: + raise SystemExit(1) +print("true" if state["prior_present"] else "false") +print(mode or "") +PY + ) || die 'network-policy checkpoint state is invalid' + [[ ${#managed_state[@]} == 2 ]] || die 'network-policy checkpoint state is invalid' + prior_present=${managed_state[0]} + prior_mode=${managed_state[1]} + if [[ "$prior_present" == true ]]; then + [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' + [[ -f "$checkpoint_dir/daemon.json" && ! -L "$checkpoint_dir/daemon.json" ]] || die 'network-policy checkpoint backup is invalid' + fi + [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' + + work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-remove.XXXXXX") + managed_daemon=$work_dir/daemon.json.managed + cp -p "$daemon_config" "$managed_daemon" + managed_mode=$(stat -c %a "$daemon_config") + daemon_dir=$(dirname "$daemon_config") + + if ! timeout "$command_timeout" "$drain_command" >/dev/null 2>&1; then + rm -rf "$work_dir" + die 'drain command failed before network-policy removal' + fi + + removal_failure='network-policy removal interrupted' + # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below + rollback_removal() { + local failed=0 + cp -p "$managed_daemon" "$daemon_config" || failed=1 + chmod "$managed_mode" "$daemon_config" || failed=1 + cmp -s "$managed_daemon" "$daemon_config" || failed=1 + timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1 || failed=1 + return "$failed" + } + # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below + removal_on_exit() { + local status=$? + trap - EXIT INT TERM + ((status != 0)) || status=1 + if rollback_removal; then + rm -rf "$work_dir" + printf 'ERROR: %s; managed daemon.json restored\n' "$removal_failure" >&2 + else + printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$removal_failure" "$work_dir" >&2 + fi + exit "$status" + } + trap 'exit 130' INT + trap 'exit 143' TERM + trap removal_on_exit EXIT + + if [[ "$prior_present" == true ]]; then + cp -p "$checkpoint_dir/daemon.json" "$daemon_config" || { removal_failure='failed to restore prior daemon.json'; exit 2; } + chmod "$prior_mode" "$daemon_config" || { removal_failure='failed to restore prior daemon.json mode'; exit 2; } + cmp -s "$checkpoint_dir/daemon.json" "$daemon_config" || { removal_failure='failed to verify prior daemon.json'; exit 2; } + else + rm -f "$daemon_config" || { removal_failure='failed to restore absent daemon.json'; exit 2; } + [[ ! -e "$daemon_config" ]] || { removal_failure='failed to verify absent daemon.json'; exit 2; } + fi + timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1 || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } + timeout "$command_timeout" "$health_command" --env "$env_file" >/dev/null 2>&1 || { removal_failure='health check failed after network-policy removal'; exit 2; } + + # Marker deletion commits removal. Ignore catchable signals across the atomic + # unlink so failure still rolls back and success cannot leave a stale marker. + trap '' INT TERM + if ! rm -f "$state_file"; then + removal_failure='failed to clear network-policy managed marker' + exit 2 + fi + trap - EXIT INT TERM + rm -rf "$work_dir" + printf 'NETWORK_POLICY_REMOVED\n' + exit 0 +fi + # --- Render desired daemon config block via shared validator --- desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" <<'PY' import json, os, sys @@ -164,6 +263,28 @@ with open(staging_path, "w", encoding="utf-8") as handle: os.chmod(staging_path, 0o644) PY +state_file=$checkpoint_dir/docker-network-policy.json +managed_before=false +if [[ -e "$state_file" ]]; then + [[ -f "$state_file" && ! -L "$state_file" ]] || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } + python3 - "$state_file" "$checkpoint_dir/daemon.json" <<'PY' || { rm -rf "$work_dir"; die 'network-policy checkpoint backup is invalid'; } +import json, os, re, sys +state = json.load(open(sys.argv[1], encoding="utf-8")) +if set(state) != {"managed", "prior_mode", "prior_present"} or state["managed"] is not True: + raise SystemExit(1) +if not isinstance(state["prior_present"], bool): + raise SystemExit(1) +if state["prior_present"]: + if not isinstance(state["prior_mode"], str) or not re.fullmatch(r"[0-7]{3,4}", state["prior_mode"]): + raise SystemExit(1) + if not os.path.isfile(sys.argv[2]) or os.path.islink(sys.argv[2]): + raise SystemExit(1) +elif state["prior_mode"] is not None or os.path.exists(sys.argv[2]): + raise SystemExit(1) +PY + managed_before=true +fi + if [[ -f "$daemon_config" ]] && python3 - "$daemon_config" "$staging_daemon" <<'PY' import json, sys with open(sys.argv[1], encoding="utf-8") as current, open(sys.argv[2], encoding="utf-8") as staged: @@ -175,17 +296,11 @@ then exit 0 fi -# --- Back up exact prior daemon.json for rollback --- +# --- Back up exact current daemon.json for transactional rollback --- prior_daemon="$work_dir/prior" mkdir -p "$prior_daemon" -if [[ -n "$checkpoint_dir" ]]; then - mkdir -p "$checkpoint_dir" - backup_dir="$checkpoint_dir" - backup_name="daemon.json" -else - backup_dir="$prior_daemon" - backup_name="daemon.json.before" -fi +backup_dir=$prior_daemon +backup_name=daemon.json.before had_prior=false if [[ -f "$daemon_config" ]]; then had_prior=true @@ -200,6 +315,34 @@ if ! timeout "$command_timeout" "$drain_command" >/dev/null 2>&1; then die "drain command failed before network-policy apply" fi +# Record the original host state before the first managed mutation. Re-applying +# policy keeps this baseline and uses the temp copy above for transaction rollback. +if [[ "$managed_before" == false ]]; then + install -d -m 0700 "$checkpoint_dir" + if [[ "$had_prior" == true ]]; then + cp -p "$backup_dir/$backup_name" "$checkpoint_dir/daemon.json" + fi + python3 - "$state_file" "$had_prior" "$daemon_mode" <<'PY' || { rm -rf "$work_dir"; die 'failed to record network-policy checkpoint state'; } +import json, os, sys, tempfile +path = sys.argv[1] +state = { + "managed": True, + "prior_mode": sys.argv[3] if sys.argv[2] == "true" else None, + "prior_present": sys.argv[2] == "true", +} +fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(state, handle, indent=2, sort_keys=True) + handle.write("\n") + os.chmod(tmp, 0o600) + os.replace(tmp, path) +finally: + if os.path.exists(tmp): + os.unlink(tmp) +PY +fi + restore_daemon() { if [[ "$had_prior" == true ]]; then cp -p "$backup_dir/$backup_name" "$daemon_config" @@ -224,6 +367,9 @@ rollback_on_exit() { trap - EXIT INT TERM ((status != 0)) || status=1 if rollback_daemon >/dev/null 2>&1; then + if [[ "$managed_before" == false ]]; then + rm -f "$state_file" "$checkpoint_dir/daemon.json" + fi rm -rf "$work_dir" printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 else diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index b38682ec..d9d4ae24 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -210,9 +210,11 @@ def _rendered_with_policy(self) -> dict[str, str]: ) return rendered - def _run(self, env_file: str, checkpoint_dir: str = "", expected_rc: int = 0) -> subprocess.CompletedProcess: + def _run(self, env_file: str, checkpoint_dir: str | None = None, expected_rc: int = 0) -> subprocess.CompletedProcess: script = str(SCRIPTS / "apply-docker-network-policy.sh") args = [script] + if checkpoint_dir is None: + checkpoint_dir = str(Path(self.tmp) / "checkpoint-default") if checkpoint_dir: args += ["--checkpoint", checkpoint_dir] args += ["--env", env_file] @@ -224,6 +226,111 @@ def _run(self, env_file: str, checkpoint_dir: str = "", expected_rc: int = 0) -> timeout=30, ) + def _write_success_commands(self) -> None: + for name in ("restart.sh", "probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + + def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + drain_marker = Path(self.tmp) / "drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir="") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("--checkpoint is required", result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + + def test_successful_apply_records_managed_original_presence_and_mode(self) -> None: + for prior_present in (False, True): + with self.subTest(prior_present=prior_present): + daemon = self.daemon_dir / "daemon.json" + daemon.unlink(missing_ok=True) + if prior_present: + daemon.write_bytes(b'{"icc":false}\n') + daemon.chmod(0o600) + checkpoint = Path(self.tmp) / f"checkpoint-{prior_present}" + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertEqual( + state, + { + "managed": True, + "prior_mode": "600" if prior_present else None, + "prior_present": prior_present, + }, + ) + self.assertEqual((checkpoint / "docker-network-policy.json").stat().st_mode & 0o777, 0o600) + self.assertEqual((checkpoint / "daemon.json").exists(), prior_present) + if prior_present: + self.assertEqual((checkpoint / "daemon.json").read_bytes(), b'{"icc":false}\n') + self.assertEqual((checkpoint / "daemon.json").stat().st_mode & 0o777, 0o600) + + def test_policy_reapply_fails_closed_when_recovery_backup_is_missing(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) + prior = daemon.read_bytes() + checkpoint = Path(self.tmp) / "checkpoint-missing-backup" + checkpoint.mkdir() + (checkpoint / "docker-network-policy.json").write_text( + json.dumps({"managed": True, "prior_mode": "600", "prior_present": True}), + encoding="utf-8", + ) + markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): + marker = Path(self.tmp) / f"missing-backup-{name}.marker" + markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(rendered) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkpoint backup is invalid", result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertTrue(all(not marker.exists() for marker in markers)) + + def test_policy_reapply_fails_closed_when_absent_baseline_has_backup(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) + prior = daemon.read_bytes() + checkpoint = Path(self.tmp) / "checkpoint-unexpected-backup" + checkpoint.mkdir() + (checkpoint / "docker-network-policy.json").write_text( + json.dumps({"managed": True, "prior_mode": None, "prior_present": False}), + encoding="utf-8", + ) + (checkpoint / "daemon.json").write_text("{}\n", encoding="utf-8") + markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): + marker = Path(self.tmp) / f"unexpected-backup-{name}.marker" + markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(rendered) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkpoint backup is invalid", result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertTrue(all(not marker.exists() for marker in markers)) + def test_policy_requires_drain_command(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -241,7 +348,7 @@ def test_policy_requires_drain_command(self) -> None: env.pop("CI_FLEET_DOCKER_DRAIN_COMMAND") result = subprocess.run( - [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-direct"), "--env", str(env_file)], capture_output=True, text=True, env=env, @@ -342,7 +449,7 @@ def test_invalid_existing_config_does_not_drain(self) -> None: env_file = self._write_env_file(self._rendered_with_policy()) result = subprocess.run( - [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-direct"), "--env", str(env_file)], capture_output=True, text=True, env=self._env(CI_FLEET_DOCKER_DRAIN_COMMAND=str(drain)), @@ -398,7 +505,7 @@ def test_validates_policy_before_mutation(self) -> None: command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") command.chmod(0o755) result = subprocess.run( - [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(bad_env)], + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-bad-env"), "--env", str(bad_env)], capture_output=True, text=True, env=self._env(CI_FLEET_DOCKER_DRAIN_COMMAND=str(Path(self.tmp) / "drain.sh")), @@ -432,7 +539,7 @@ def test_drain_failure_prevents_mutation_and_restart(self) -> None: command.chmod(0o755) env_file = self._write_env_file(self._rendered_with_policy()) result = subprocess.run( - [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-direct"), "--env", str(env_file)], capture_output=True, text=True, env=self._env(CI_FLEET_DOCKER_DRAIN_COMMAND=str(drain)), @@ -541,7 +648,7 @@ def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> health.chmod(0o755) env_file = self._write_env_file(self._rendered_with_policy()) process = subprocess.Popen( - [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-direct"), "--env", str(env_file)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -669,7 +776,7 @@ def test_sleeping_probe_times_out(self) -> None: env_file = self._write_env_file(self._rendered_with_policy()) started = time.monotonic() result = subprocess.run( - [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-direct"), "--env", str(env_file)], capture_output=True, text=True, env=self._env( @@ -690,7 +797,7 @@ def test_apply_works_across_temp_filesystems(self) -> None: command.chmod(0o755) env_file = self._write_env_file(self._rendered_with_policy()) result = subprocess.run( - [str(SCRIPTS / "apply-docker-network-policy.sh"), "--env", str(env_file)], + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-direct"), "--env", str(env_file)], capture_output=True, text=True, env=self._env(CI_FLEET_TEMP_DIR="/dev/shm"), @@ -815,7 +922,7 @@ def test_restart_failure_evidence_excludes_command_output(self) -> None: env_file = self._write_env_file(self._rendered_with_policy()) script = str(SCRIPTS / "apply-docker-network-policy.sh") result = subprocess.run( - [script, "--env", env_file], + [script, "--checkpoint", str(Path(self.tmp) / "checkpoint-restart-failure"), "--env", env_file], capture_output=True, text=True, env=env, @@ -851,15 +958,169 @@ def test_checkpoint_preserves_prior_config(self) -> None: self.assertTrue(backup.exists()) self.assertEqual(json.loads(backup.read_text(encoding="utf-8")), prior) - def test_no_policy_is_noop(self) -> None: - """GREEN: when no network policy is rendered, the script does nothing.""" - self._write_daemon(json.dumps({"bip": "172.17.0.1/16"})) + def test_managed_policy_removal_restores_original_state_and_clears_only_marker(self) -> None: + for prior_present in (False, True): + with self.subTest(prior_present=prior_present): + daemon = self.daemon_dir / "daemon.json" + daemon.unlink(missing_ok=True) + prior = b'{"icc":false}\n' + if prior_present: + daemon.write_bytes(prior) + daemon.chmod(0o600) + checkpoint = Path(self.tmp) / f"checkpoint-remove-{prior_present}" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.assertTrue((checkpoint / "docker-network-policy.json").exists()) + retained = checkpoint / "retained" + retained.write_text("keep\n", encoding="utf-8") + + command_log = Path(self.tmp) / f"remove-{prior_present}.log" + for name in ("drain.sh", "restart.sh", "probe.sh"): + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + no_policy_env = Path(self.tmp) / f"no-policy-{prior_present}.env" + no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"[[ $1 == --env && $2 == {no_policy_env} ]] || exit 1\n" + f"echo health.sh >> {command_log}\n", + encoding="utf-8", + ) + health.chmod(0o755) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(removed.stdout, "NETWORK_POLICY_REMOVED\n") + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain.sh", "restart.sh", "health.sh"]) + self.assertEqual(daemon.exists(), prior_present) + if prior_present: + self.assertEqual(daemon.read_bytes(), prior) + self.assertEqual(daemon.stat().st_mode & 0o777, 0o600) + self.assertTrue((checkpoint / "daemon.json").exists()) + self.assertFalse((checkpoint / "docker-network-policy.json").exists()) + self.assertTrue(retained.exists()) + + def test_removal_failure_restores_managed_config_and_retains_recovery_state(self) -> None: + prior = b'{"registry-mirrors":["https://mirror.example.invalid?token=credential"]}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + daemon.chmod(0o600) + checkpoint = Path(self.tmp) / "checkpoint-remove-failure" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + managed_mode = daemon.stat().st_mode & 0o777 + state = (checkpoint / "docker-network-policy.json").read_bytes() + baseline = (checkpoint / "daemon.json").read_bytes() + + restart_log = Path(self.tmp) / "remove-failure-restart.log" restart = Path(self.tmp) / "restart.sh" - restart.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + restart.write_text(f"#!/usr/bin/env bash\necho restart >> {restart_log}\n", encoding="utf-8") restart.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" + "echo removal-health-secret >&2\n" + "exit 1\n", + encoding="utf-8", + ) + health.chmod(0o755) + no_policy_env = Path(self.tmp) / "no-policy-failure.env" + no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + self.assertEqual(daemon.stat().st_mode & 0o777, managed_mode) + self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) + self.assertEqual((checkpoint / "docker-network-policy.json").read_bytes(), state) + self.assertEqual((checkpoint / "daemon.json").read_bytes(), baseline) + combined = removed.stdout + removed.stderr + self.assertIn("managed daemon.json restored", combined) + for secret in ("198.51.100.0/24", "secret.example.invalid", "mirror.example.invalid", "credential", "removal-health-secret"): + self.assertNotIn(secret, combined) + + def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) -> None: + daemon = self._write_daemon('{"icc":false}\n') + checkpoint = Path(self.tmp) / "checkpoint-marker-failure" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + state_file = checkpoint / "docker-network-policy.json" + state = state_file.read_bytes() + + restart_log = Path(self.tmp) / "marker-failure-restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text(f"#!/usr/bin/env bash\necho restart >> {restart_log}\n", encoding="utf-8") + restart.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + health.chmod(0o755) + no_policy_env = Path(self.tmp) / "no-policy-marker-failure.env" + no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + fake_bin = Path(self.tmp) / "fake-bin" + fake_bin.mkdir() + fake_rm = fake_bin / "rm" + fake_rm.write_text( + "#!/usr/bin/env bash\n" + f"[[ ${{*: -1}} != {state_file} ]] || exit 1\n" + f"exec {shutil.which('rm')} \"$@\"\n", + encoding="utf-8", + ) + fake_rm.chmod(0o755) + env = self._env(PATH=f"{fake_bin}:{os.environ['PATH']}") + + removed = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(no_policy_env), + ], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + self.assertEqual(state_file.read_bytes(), state) + self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) + + def test_unmanaged_no_policy_is_noop_without_mutation_or_commands(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): + marker = Path(self.tmp) / f"{name}.marker" + markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\nexit 1\n", encoding="utf-8") + command.chmod(0o755) + checkpoint = Path(self.tmp) / "checkpoint" env_file = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) - result = self._run(str(env_file)) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "NETWORK_POLICY_NOOP\n") + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(checkpoint.exists()) + self.assertTrue(all(not marker.exists() for marker in markers)) if __name__ == "__main__": From 17bedc2d338dcfb4a2bcb00f2ede13c1f1b1eba9 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:20:15 -0500 Subject: [PATCH 08/54] fix: harden Docker policy command boundaries --- scripts/apply-docker-network-policy.sh | 67 +++-- scripts/healthcheck.sh | 8 +- scripts/test_apply_docker_network_policy.py | 289 +++++++++++++++++++- 3 files changed, 334 insertions(+), 30 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 79186ba7..807c6e4b 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -59,7 +59,18 @@ while (($#)); do done [[ -n "$env_file" ]] || die '--env is required' -[[ -r "$env_file" ]] || die "rendered env is unreadable: $env_file" +[[ -f "$env_file" && ! -L "$env_file" && -r "$env_file" ]] || die "rendered env must be a readable regular file: $env_file" +env_owner=$(stat -c %u "$env_file") +expected_env_owner=0 +[[ "$testing" != 1 ]] || expected_env_owner=$(id -u) +[[ "$env_owner" == "$expected_env_owner" ]] || die "rendered env has an untrusted owner: $env_file" +env_mode=$(stat -c %a "$env_file") +(( (8#$env_mode & 8#022) == 0 )) || die "rendered env must not be group/world writable: $env_file" +work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") +chmod 0700 "$work_dir" +trap 'rm -rf "$work_dir"' EXIT +install -m 0600 -- "$env_file" "$work_dir/ci-fleet.env" +env_file=$work_dir/ci-fleet.env # --- No-op when no network policy is rendered --- count=$(awk -F= '$1 == "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" {print substr($0, index($0, "=") + 1)}' "$env_file") @@ -83,17 +94,31 @@ probe_command=${CI_FLEET_DOCKER_NETWORK_PROBE:-} health_command=${CI_FLEET_HEALTH_CHECK_COMMAND:-} command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} +validate_command() { + local name=$1 path=$2 canonical + [[ -n "$path" ]] || die "$name is required when network policy is managed" + [[ "$path" == /* && -f "$path" && ! -L "$path" && -x "$path" ]] || die "$name must be an absolute canonical regular executable path" + canonical=$(readlink -f -- "$path") || die "$name must be an absolute canonical regular executable path" + [[ "$path" == "$canonical" ]] || die "$name must be an absolute canonical regular executable path" +} + +run_command() { + timeout --kill-after=5 "$command_timeout" "$@" >/dev/null 2>&1 +} + +run_health() { + local status=0 + CI_FLEET_HEALTH_SUPPRESS_DELIVERY=1 run_command "$health_command" --env "$1" || status=$? + ((status < 2)) +} + [[ "$command_timeout" =~ ^[1-9][0-9]*$ ]] || die 'CI_FLEET_COMMAND_TIMEOUT_SECONDS must be a positive integer' [[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' -[[ -n "$drain_command" ]] || die 'CI_FLEET_DOCKER_DRAIN_COMMAND is required when a network policy is configured' -[[ -n "$restart_command" ]] || die 'CI_FLEET_DOCKER_RESTART_COMMAND is required when network policy is managed' -[[ -n "$health_command" ]] || die 'CI_FLEET_HEALTH_CHECK_COMMAND is required when network policy is managed' -[[ -x "$restart_command" ]] || die "restart command is not executable: $restart_command" -[[ -x "$drain_command" ]] || die "drain command is not executable: $drain_command" -[[ -x "$health_command" ]] || die "health-check command is not executable: $health_command" +validate_command CI_FLEET_DOCKER_DRAIN_COMMAND "$drain_command" +validate_command CI_FLEET_DOCKER_RESTART_COMMAND "$restart_command" +validate_command CI_FLEET_HEALTH_CHECK_COMMAND "$health_command" if [[ "$removing" == false ]]; then - [[ -n "$probe_command" ]] || die 'CI_FLEET_DOCKER_NETWORK_PROBE is required when a network policy is configured' - [[ -x "$probe_command" ]] || die "network probe is not executable: $probe_command" + validate_command CI_FLEET_DOCKER_NETWORK_PROBE "$probe_command" fi # Serialize with installer mutations using the installer's host-local lock. @@ -153,13 +178,12 @@ PY fi [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' - work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-remove.XXXXXX") managed_daemon=$work_dir/daemon.json.managed cp -p "$daemon_config" "$managed_daemon" managed_mode=$(stat -c %a "$daemon_config") daemon_dir=$(dirname "$daemon_config") - if ! timeout "$command_timeout" "$drain_command" >/dev/null 2>&1; then + if ! run_command "$drain_command"; then rm -rf "$work_dir" die 'drain command failed before network-policy removal' fi @@ -171,7 +195,7 @@ PY cp -p "$managed_daemon" "$daemon_config" || failed=1 chmod "$managed_mode" "$daemon_config" || failed=1 cmp -s "$managed_daemon" "$daemon_config" || failed=1 - timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1 || failed=1 + run_command "$restart_command" "$daemon_dir" || failed=1 return "$failed" } # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below @@ -199,8 +223,8 @@ PY rm -f "$daemon_config" || { removal_failure='failed to restore absent daemon.json'; exit 2; } [[ ! -e "$daemon_config" ]] || { removal_failure='failed to verify absent daemon.json'; exit 2; } fi - timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1 || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } - timeout "$command_timeout" "$health_command" --env "$env_file" >/dev/null 2>&1 || { removal_failure='health check failed after network-policy removal'; exit 2; } + run_command "$restart_command" "$daemon_dir" || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } + run_health "$env_file" || { removal_failure='health check failed after network-policy removal'; exit 2; } # Marker deletion commits removal. Ignore catchable signals across the atomic # unlink so failure still rolls back and success cannot leave a stale marker. @@ -216,7 +240,7 @@ PY fi # --- Render desired daemon config block via shared validator --- -desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" <<'PY' +desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" 2>/dev/null <<'PY' import json, os, sys env_path, scripts_dir = sys.argv[1], sys.argv[2] sys.path.insert(0, scripts_dir) @@ -239,7 +263,6 @@ if [[ "$desired_pools_json" == "{}" ]]; then fi # --- Stage merged daemon.json (preserve unrelated keys) --- -work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") staging_daemon="$work_dir/daemon.json" python3 - "$env_file" "$daemon_config" "$staging_daemon" "$desired_pools_json" <<'PY' || { rm -rf "$work_dir"; die "failed to stage merged daemon.json"; } @@ -310,7 +333,7 @@ fi daemon_dir=$(dirname "$daemon_config") # --- Drain after local validation/checkpointing, before mutation or restart --- -if ! timeout "$command_timeout" "$drain_command" >/dev/null 2>&1; then +if ! run_command "$drain_command"; then rm -rf "$work_dir" die "drain command failed before network-policy apply" fi @@ -356,8 +379,8 @@ restore_daemon() { rollback_daemon() { local failed=0 restore_daemon || failed=1 - timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1 || failed=1 - timeout "$command_timeout" "$health_command" --env "$env_file" >/dev/null 2>&1 || failed=1 + run_command "$restart_command" "$daemon_dir" || failed=1 + run_health "$env_file" || failed=1 return "$failed" } @@ -403,17 +426,17 @@ finally: PY # Restart Docker through the injected command boundary (never host-direct). -if ! timeout "$command_timeout" "$restart_command" "$daemon_dir" >/dev/null 2>&1; then +if ! run_command "$restart_command" "$daemon_dir"; then fail_after_apply "Docker restart command failed" fi # Bounded capacity probe -if ! timeout "$command_timeout" "$probe_command" >/dev/null 2>&1; then +if ! run_command "$probe_command"; then fail_after_apply "capacity probe failed after network-policy restart" fi # Health verification -if ! timeout "$command_timeout" "$health_command" --env "$env_file" >/dev/null 2>&1; then +if ! run_health "$env_file"; then fail_after_apply "health check failed after network-policy restart" fi diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index 9917e6fe..7e52ff65 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -14,11 +14,15 @@ if [[ ${1:-} == --env ]]; then environment=$2 shift 2 fi -if [[ -r $environment ]]; then +selected_environment=$environment +health_suppress_delivery=${CI_FLEET_HEALTH_SUPPRESS_DELIVERY-} +while IFS= read -r variable; do unset "$variable"; done < <(compgen -A variable CI_FLEET_) +if [[ -r $selected_environment ]]; then set -a # shellcheck disable=SC1090 - . "$environment" + . "$selected_environment" set +a fi +[[ -z $health_suppress_delivery ]] || export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery use_local_docker exec python3 "$repo_root/scripts/health.py" "${args[@]}" "$@" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index d9d4ae24..8f07f64a 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -156,6 +156,73 @@ def test_env_argument_sources_candidate_rendered_env(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) + def test_selected_env_does_not_inherit_removed_pool_variables(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + candidate = root / "candidate.env" + candidate.write_text( + f"CI_FLEET_TESTING=1\nCI_FLEET_ROOT_PREFIX={tmp}\n", + encoding="utf-8", + ) + fake_bin = root / "bin" + fake_bin.mkdir() + python = fake_bin / "python3" + python.write_text( + "#!/usr/bin/env bash\n" + "[[ -z ${CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT+x} ]] || exit 1\n", + encoding="utf-8", + ) + python.chmod(0o755) + env = dict(os.environ) + env.update( + CI_FLEET_TESTING="1", + CI_FLEET_ROOT_PREFIX=tmp, + CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT="2", + PATH=f"{fake_bin}:{env['PATH']}", + ) + + result = subprocess.run( + [str(SCRIPTS / "healthcheck.sh"), "--env", str(candidate)], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_selected_env_preserves_health_delivery_suppression(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + candidate = root / "candidate.env" + candidate.write_text(f"CI_FLEET_TESTING=1\nCI_FLEET_ROOT_PREFIX={tmp}\n", encoding="utf-8") + fake_bin = root / "bin" + fake_bin.mkdir() + python = fake_bin / "python3" + python.write_text( + "#!/usr/bin/env bash\n" + "[[ ${CI_FLEET_HEALTH_SUPPRESS_DELIVERY:-} == 1 ]]\n", + encoding="utf-8", + ) + python.chmod(0o755) + env = dict(os.environ) + env.update( + CI_FLEET_HEALTH_SUPPRESS_DELIVERY="1", + CI_FLEET_TESTING="1", + CI_FLEET_ROOT_PREFIX=tmp, + PATH=f"{fake_bin}:{env['PATH']}", + ) + + result = subprocess.run( + [str(SCRIPTS / "healthcheck.sh"), "--env", str(candidate)], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + class ApplyScriptTests(unittest.TestCase): """Integration tests for scripts/apply-docker-network-policy.sh. @@ -331,6 +398,213 @@ def test_policy_reapply_fails_closed_when_absent_baseline_has_backup(self) -> No self.assertEqual(daemon.read_bytes(), prior) self.assertTrue(all(not marker.exists() for marker in markers)) + def test_injected_hooks_require_absolute_canonical_regular_executables(self) -> None: + env_file = self._write_env_file(self._rendered_with_policy()) + hook_names = ( + "CI_FLEET_DOCKER_DRAIN_COMMAND", + "CI_FLEET_DOCKER_RESTART_COMMAND", + "CI_FLEET_DOCKER_NETWORK_PROBE", + "CI_FLEET_HEALTH_CHECK_COMMAND", + ) + self._write_success_commands() + target = Path(self.tmp) / "hook-target.sh" + target.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + target.chmod(0o755) + for hook_name in hook_names: + with self.subTest(hook_name=hook_name): + daemon = self._write_daemon("{}\n") + link = Path(self.tmp) / f"{hook_name}.sh" + link.symlink_to(target) + checkpoint = Path(self.tmp) / f"checkpoint-{hook_name}" + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(**{hook_name: str(link)}), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_text(encoding="utf-8"), "{}\n") + self.assertFalse(checkpoint.exists()) + link.unlink() + + def test_all_hooks_use_kill_grace_and_suppress_output(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + timeout_log = Path(self.tmp) / "timeout.log" + hook_log = Path(self.tmp) / "hooks.log" + fake_bin = Path(self.tmp) / "fake-bin" + fake_bin.mkdir() + fake_timeout = fake_bin / "timeout" + fake_timeout.write_text( + "#!/usr/bin/env bash\n" + f"printf '%s\\n' \"$*\" >> {timeout_log}\n" + "[[ $1 != --kill-after=* ]] || shift\n" + "shift\n" + '"$@"\n', + encoding="utf-8", + ) + fake_timeout.chmod(0o755) + hooks = { + "CI_FLEET_DOCKER_DRAIN_COMMAND": self.drain_command, + "CI_FLEET_DOCKER_RESTART_COMMAND": Path(self.tmp) / "restart.sh", + "CI_FLEET_DOCKER_NETWORK_PROBE": Path(self.tmp) / "probe.sh", + "CI_FLEET_HEALTH_CHECK_COMMAND": Path(self.tmp) / "health.sh", + } + for path in hooks.values(): + path.write_text( + "#!/usr/bin/env bash\n" + f"echo {path.name} >> {hook_log}\n" + "echo private-hook-output\n" + "echo private-hook-error >&2\n", + encoding="utf-8", + ) + path.chmod(0o755) + + run_env = self._env(PATH=f"{fake_bin}:{os.environ['PATH']}", **{name: str(path) for name, path in hooks.items()}) + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-timeout"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=run_env, + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotIn("private-hook", result.stdout + result.stderr) + lines = timeout_log.read_text(encoding="utf-8").splitlines() + for path in hooks.values(): + self.assertTrue(any(line.startswith(f"--kill-after=5 300 {path}") for line in lines), path) + self.assertEqual(sorted(hook_log.read_text(encoding="utf-8").splitlines()), sorted(path.name for path in hooks.values())) + + def test_health_accepts_only_success_and_warning_results(self) -> None: + env_file = self._write_env_file(self._rendered_with_policy()) + cases = { + "healthy": ("#!/usr/bin/env bash\nexit 0\n", True), + "warning": ("#!/usr/bin/env bash\nexit 1\n", True), + "critical": ("#!/usr/bin/env bash\nexit 2\n", False), + "signal": ("#!/usr/bin/env bash\nkill -TERM $$\n", False), + "timeout": ("#!/usr/bin/env bash\nsleep 10\n", False), + "execution": ("not an executable format\n", False), + } + for name, (script, accepted) in cases.items(): + with self.subTest(name=name): + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / f"checkpoint-health-{name}" + shutil.rmtree(checkpoint, ignore_errors=True) + self._write_success_commands() + health = Path(self.tmp) / "health.sh" + health.write_text(script, encoding="utf-8") + health.chmod(0o755) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_COMMAND_TIMEOUT_SECONDS="1"), + timeout=15, + ) + + self.assertEqual(result.returncode == 0, accepted, result.stderr) + + def test_transactional_health_suppresses_delivery(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + health_log = Path(self.tmp) / "health-suppression.log" + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"printf '%s\\n' \"${{CI_FLEET_HEALTH_SUPPRESS_DELIVERY:-unset}}\" >> {health_log}\n" + f"(( $(wc -l < {health_log}) > 1 )) || exit 2\n", + encoding="utf-8", + ) + health.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["1", "1"]) + + def test_env_file_is_trusted_and_snapshotted_before_mutation(self) -> None: + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + env_file.chmod(0o666) + drain_marker = Path(self.tmp) / "untrusted-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + rejected = self._run(str(env_file)) + + self.assertNotEqual(rejected.returncode, 0) + self.assertFalse(drain_marker.exists()) + + env_file.chmod(0o600) + health_log = Path(self.tmp) / "snapshot-health.log" + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"printf 'CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=0\\n' > {env_file}\n", + encoding="utf-8", + ) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"[[ $1 == --env && $2 != {env_file} ]] || exit 2\n" + '[[ $(stat -c %a "$2") == 600 && $(stat -c %a "$(dirname "$2")") == 700 ]] || exit 2\n' + 'grep -Fqx "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=2" "$2" || exit 2\n' + f"printf '%s\\n' \"$2\" > {health_log}\n", + encoding="utf-8", + ) + health.chmod(0o755) + + applied = self._run(str(env_file), checkpoint_dir=str(Path(self.tmp) / "checkpoint-snapshot")) + + self.assertEqual(applied.returncode, 0, applied.stderr) + self.assertNotEqual(health_log.read_text(encoding="utf-8").strip(), str(env_file)) + daemon = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8")) + self.assertEqual(len(daemon["default-address-pools"]), 2) + + def test_renderer_failure_suppresses_private_stderr(self) -> None: + daemon = self._write_daemon("{}\n") + prior = daemon.read_bytes() + self._write_success_commands() + private_pool = "10.77.88.0/24" + env_file = self._write_env_file( + { + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "1", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE": private_pool, + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE": "16", + } + ) + drain_marker = Path(self.tmp) / "renderer-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + result = self._run(str(env_file)) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn(private_pool, result.stdout + result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + def test_policy_requires_drain_command(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -599,7 +873,7 @@ def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: health = Path(self.tmp) / "health.sh" health.write_text( "#!/usr/bin/env bash\n" - f"[[ $1 == --env && $2 == {self.tmp}/ci-fleet.env ]] || exit 1\n" + f"[[ $1 == --env && $2 != {self.tmp}/ci-fleet.env ]] || exit 2\n" f"echo health >> {health_log}\n", encoding="utf-8", ) @@ -725,7 +999,7 @@ def test_post_apply_health_uses_candidate_rendered_env(self) -> None: health = Path(self.tmp) / "health.sh" health.write_text( "#!/usr/bin/env bash\n" - f"[[ $1 == --env && $2 == {self.tmp}/ci-fleet.env ]] || exit 1\n" + f"[[ $1 == --env && $2 != {self.tmp}/ci-fleet.env ]] || exit 2\n" f"printf '%s\\n' \"$2\" >> {health_log}\n", encoding="utf-8", ) @@ -735,7 +1009,9 @@ def test_post_apply_health_uses_candidate_rendered_env(self) -> None: result = self._run(str(env_file)) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), [str(env_file)]) + health_paths = health_log.read_text(encoding="utf-8").splitlines() + self.assertEqual(len(health_paths), 1) + self.assertNotEqual(health_paths[0], str(env_file)) def test_health_failure_evidence_excludes_command_output(self) -> None: self._write_daemon("{}\n") @@ -748,7 +1024,7 @@ def test_health_failure_evidence_excludes_command_output(self) -> None: "#!/usr/bin/env bash\n" "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" "echo 'health-secret-error' >&2\n" - "exit 1\n", + "exit 2\n", encoding="utf-8", ) health.chmod(0o755) @@ -986,7 +1262,8 @@ def test_managed_policy_removal_restores_original_state_and_clears_only_marker(s health = Path(self.tmp) / "health.sh" health.write_text( "#!/usr/bin/env bash\n" - f"[[ $1 == --env && $2 == {no_policy_env} ]] || exit 1\n" + f"[[ $1 == --env && $2 != {no_policy_env} ]] || exit 2\n" + 'grep -Fqx "CI_FLEET_INSTANCE=example-ci-01" "$2" || exit 2\n' f"echo health.sh >> {command_log}\n", encoding="utf-8", ) @@ -1029,7 +1306,7 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self "#!/usr/bin/env bash\n" "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" "echo removal-health-secret >&2\n" - "exit 1\n", + "exit 2\n", encoding="utf-8", ) health.chmod(0o755) From ebfbc2796622e329282e36a38eaaea493c4d5f64 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:37:56 -0500 Subject: [PATCH 09/54] fix: preserve Docker policy file durability --- scripts/apply-docker-network-policy.sh | 82 ++++++++---- scripts/test_apply_docker_network_policy.py | 133 +++++++++++++++++++- 2 files changed, 186 insertions(+), 29 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 807c6e4b..721e5803 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -133,12 +133,28 @@ else flock -n 9 || die 'another ci-fleet installer or drift check is already running' fi +checkpoint_owner=0 +[[ "$testing" != 1 ]] || checkpoint_owner=$(id -u) +if [[ -e "$checkpoint_dir" || -L "$checkpoint_dir" ]]; then + [[ -d "$checkpoint_dir" && ! -L "$checkpoint_dir" && $(stat -c %u "$checkpoint_dir") == "$checkpoint_owner" && $(stat -c %a "$checkpoint_dir") == 700 ]] || + die "checkpoint directory must be owned by root with mode 0700: $checkpoint_dir" + for checkpoint_file in "$checkpoint_dir/docker-network-policy.json" "$checkpoint_dir/daemon.json"; do + if [[ -e "$checkpoint_file" || -L "$checkpoint_file" ]]; then + [[ -f "$checkpoint_file" && ! -L "$checkpoint_file" && $(stat -c %u "$checkpoint_file") == "$checkpoint_owner" && $(stat -c %a "$checkpoint_file") == 600 ]] || + die "checkpoint files must be owned by root with mode 0600: $checkpoint_file" + fi + done +fi + [[ ! -L "$daemon_config" ]] || die "daemon.json must not be a symlink: $daemon_config" # --- Ownership guard (relaxed in testing) --- daemon_mode=644 +daemon_gid=0 +[[ "$testing" != 1 ]] || daemon_gid=$(id -g) if [[ -f "$daemon_config" ]]; then daemon_mode=$(stat -c %a "$daemon_config") + daemon_gid=$(stat -c %g "$daemon_config") fi if [[ "$testing" != 1 ]]; then [[ -w "$(dirname "$daemon_config")" ]] || die "daemon config directory is not writable: $(dirname "$daemon_config")" @@ -149,6 +165,31 @@ if [[ "$testing" != 1 ]]; then else : # testing mode — skip root checks fi +daemon_dir=$(dirname "$daemon_config") + +atomic_replace_daemon() { + python3 - "$1" "$daemon_dir" "$daemon_config" "$2" "$3" <<'PY' +import os, shutil, sys, tempfile +_, source, daemon_dir, target, mode, gid = sys.argv +fd, tmp = tempfile.mkstemp(prefix=".daemon.json.", dir=daemon_dir) +try: + with open(source, "rb") as source_handle, os.fdopen(fd, "wb") as staged: + shutil.copyfileobj(source_handle, staged) + os.fchmod(staged.fileno(), int(mode, 8)) + os.fchown(staged.fileno(), -1, int(gid)) + staged.flush() + os.fsync(staged.fileno()) + os.replace(tmp, target) + directory_fd = os.open(daemon_dir, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) +finally: + if os.path.exists(tmp): + os.unlink(tmp) +PY +} if [[ "$removing" == true ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || die 'network-policy checkpoint state is invalid' @@ -175,13 +216,14 @@ PY if [[ "$prior_present" == true ]]; then [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' [[ -f "$checkpoint_dir/daemon.json" && ! -L "$checkpoint_dir/daemon.json" ]] || die 'network-policy checkpoint backup is invalid' + prior_gid=$(stat -c %g "$checkpoint_dir/daemon.json") fi [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' managed_daemon=$work_dir/daemon.json.managed cp -p "$daemon_config" "$managed_daemon" managed_mode=$(stat -c %a "$daemon_config") - daemon_dir=$(dirname "$daemon_config") + managed_gid=$(stat -c %g "$daemon_config") if ! run_command "$drain_command"; then rm -rf "$work_dir" @@ -192,8 +234,7 @@ PY # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below rollback_removal() { local failed=0 - cp -p "$managed_daemon" "$daemon_config" || failed=1 - chmod "$managed_mode" "$daemon_config" || failed=1 + atomic_replace_daemon "$managed_daemon" "$managed_mode" "$managed_gid" || failed=1 cmp -s "$managed_daemon" "$daemon_config" || failed=1 run_command "$restart_command" "$daemon_dir" || failed=1 return "$failed" @@ -216,8 +257,7 @@ PY trap removal_on_exit EXIT if [[ "$prior_present" == true ]]; then - cp -p "$checkpoint_dir/daemon.json" "$daemon_config" || { removal_failure='failed to restore prior daemon.json'; exit 2; } - chmod "$prior_mode" "$daemon_config" || { removal_failure='failed to restore prior daemon.json mode'; exit 2; } + atomic_replace_daemon "$checkpoint_dir/daemon.json" "$prior_mode" "$prior_gid" || { removal_failure='failed to restore prior daemon.json'; exit 2; } cmp -s "$checkpoint_dir/daemon.json" "$daemon_config" || { removal_failure='failed to verify prior daemon.json'; exit 2; } else rm -f "$daemon_config" || { removal_failure='failed to restore absent daemon.json'; exit 2; } @@ -330,8 +370,6 @@ if [[ -f "$daemon_config" ]]; then cp -p "$daemon_config" "$backup_dir/$backup_name" fi -daemon_dir=$(dirname "$daemon_config") - # --- Drain after local validation/checkpointing, before mutation or restart --- if ! run_command "$drain_command"; then rm -rf "$work_dir" @@ -343,7 +381,8 @@ fi if [[ "$managed_before" == false ]]; then install -d -m 0700 "$checkpoint_dir" if [[ "$had_prior" == true ]]; then - cp -p "$backup_dir/$backup_name" "$checkpoint_dir/daemon.json" + install -m 0600 "$backup_dir/$backup_name" "$checkpoint_dir/daemon.json" + chgrp "$daemon_gid" "$checkpoint_dir/daemon.json" fi python3 - "$state_file" "$had_prior" "$daemon_mode" <<'PY' || { rm -rf "$work_dir"; die 'failed to record network-policy checkpoint state'; } import json, os, sys, tempfile @@ -358,8 +397,15 @@ try: with os.fdopen(fd, "w", encoding="utf-8") as handle: json.dump(state, handle, indent=2, sort_keys=True) handle.write("\n") - os.chmod(tmp, 0o600) + os.fchmod(handle.fileno(), 0o600) + handle.flush() + os.fsync(handle.fileno()) os.replace(tmp, path) + directory_fd = os.open(os.path.dirname(path), os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) finally: if os.path.exists(tmp): os.unlink(tmp) @@ -368,7 +414,7 @@ fi restore_daemon() { if [[ "$had_prior" == true ]]; then - cp -p "$backup_dir/$backup_name" "$daemon_config" + atomic_replace_daemon "$backup_dir/$backup_name" "$daemon_mode" "$daemon_gid" else rm -f "$daemon_config" fi @@ -410,20 +456,8 @@ fail_after_apply() { } # --- Transaction: apply → restart → probe → health, with rollback --- -# Apply daemon.json atomically (rename within same directory) -python3 - "$staging_daemon" "$daemon_dir" "$daemon_config" "$daemon_mode" <<'PY' || { transaction_failure='failed to apply daemon.json'; exit 2; } -import os, shutil, sys, tempfile -_, _, daemon_dir, target, daemon_mode = sys.argv -fd, tmp = tempfile.mkstemp(prefix=".daemon.json.", dir=daemon_dir) -try: - with open(sys.argv[1], "rb") as source, os.fdopen(fd, "wb") as staged: - shutil.copyfileobj(source, staged) - os.chmod(tmp, int(daemon_mode, 8)) - os.replace(tmp, target) -finally: - if os.path.exists(tmp): - os.unlink(tmp) -PY +# Apply daemon.json atomically (rename within same directory). +atomic_replace_daemon "$staging_daemon" "$daemon_mode" "$daemon_gid" || { transaction_failure='failed to apply daemon.json'; exit 2; } # Restart Docker through the injected command boundary (never host-direct). if ! run_command "$restart_command" "$daemon_dir"; then diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 8f07f64a..8bbaa5ef 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -315,6 +315,22 @@ def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None self.assertEqual(daemon.read_bytes(), prior) self.assertFalse(drain_marker.exists()) + def test_rejects_unsafe_preexisting_checkpoint_before_drain(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "unsafe-checkpoint" + checkpoint.mkdir(mode=0o755) + drain_marker = Path(self.tmp) / "unsafe-checkpoint-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkpoint directory must be owned by root with mode 0700", result.stderr) + self.assertEqual(daemon.read_text(encoding="utf-8"), "{}\n") + self.assertFalse(drain_marker.exists()) + def test_successful_apply_records_managed_original_presence_and_mode(self) -> None: for prior_present in (False, True): with self.subTest(prior_present=prior_present): @@ -350,11 +366,13 @@ def test_policy_reapply_fails_closed_when_recovery_backup_is_missing(self) -> No daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) prior = daemon.read_bytes() checkpoint = Path(self.tmp) / "checkpoint-missing-backup" - checkpoint.mkdir() - (checkpoint / "docker-network-policy.json").write_text( + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( json.dumps({"managed": True, "prior_mode": "600", "prior_present": True}), encoding="utf-8", ) + state_file.chmod(0o600) markers = [] for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): marker = Path(self.tmp) / f"missing-backup-{name}.marker" @@ -376,12 +394,16 @@ def test_policy_reapply_fails_closed_when_absent_baseline_has_backup(self) -> No daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) prior = daemon.read_bytes() checkpoint = Path(self.tmp) / "checkpoint-unexpected-backup" - checkpoint.mkdir() - (checkpoint / "docker-network-policy.json").write_text( + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( json.dumps({"managed": True, "prior_mode": None, "prior_present": False}), encoding="utf-8", ) - (checkpoint / "daemon.json").write_text("{}\n", encoding="utf-8") + state_file.chmod(0o600) + backup_file = checkpoint / "daemon.json" + backup_file.write_text("{}\n", encoding="utf-8") + backup_file.chmod(0o600) markers = [] for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): marker = Path(self.tmp) / f"unexpected-backup-{name}.marker" @@ -1084,6 +1106,66 @@ def test_apply_works_across_temp_filesystems(self) -> None: self.assertEqual(daemon["bip"], "172.17.0.1/16") self.assertIn("default-address-pools", daemon) + def test_fsyncs_staged_file_and_daemon_directory_around_replace(self) -> None: + daemon = self._write_daemon("{}\n") + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + audit_dir = Path(self.tmp) / "audit-python" + audit_dir.mkdir() + audit_log = Path(self.tmp) / "fsync.log" + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_fsync = os.fsync\n" + "_replace = os.replace\n" + "_log = os.environ['FSYNC_AUDIT_LOG']\n" + "def fsync(fd):\n" + " with open(_log, 'a', encoding='utf-8') as handle:\n" + " handle.write('F ' + os.readlink(f'/proc/self/fd/{fd}') + '\\n')\n" + " return _fsync(fd)\n" + "def replace(source, target):\n" + " with open(_log, 'a', encoding='utf-8') as handle:\n" + " handle.write(f'R {source} {target}\\n')\n" + " return _replace(source, target)\n" + "os.fsync = fsync\n" + "os.replace = replace\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-fsync"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(PYTHONPATH=str(audit_dir), FSYNC_AUDIT_LOG=str(audit_log)), + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + events = audit_log.read_text(encoding="utf-8").splitlines() + replace_indexes = [i for i, event in enumerate(events) if event.startswith("R ")] + for replace_index in replace_indexes: + source = events[replace_index].split(" ", 2)[1] + self.assertIn(f"F {source}", events[:replace_index]) + daemon_replace = next(i for i in replace_indexes if events[i].endswith(f" {daemon}")) + self.assertEqual(events[daemon_replace + 1], f"F {self.daemon_dir}") + + @unittest.skipUnless(os.geteuid() == 0, "changing file GID requires root") + def test_preserves_existing_daemon_config_gid(self) -> None: + daemon = self._write_daemon("{}\n") + daemon.chmod(0o640) + os.chown(daemon, -1, 1) + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(daemon.stat().st_gid, 1) + def test_preserves_restrictive_daemon_config_mode(self) -> None: daemon = self._write_daemon("{}\n") daemon.chmod(0o600) @@ -1234,6 +1316,47 @@ def test_checkpoint_preserves_prior_config(self) -> None: self.assertTrue(backup.exists()) self.assertEqual(json.loads(backup.read_text(encoding="utf-8")), prior) + def test_managed_policy_removal_restores_backup_without_copying_over_daemon(self) -> None: + prior = b'{"icc":false}\n' + daemon = self.daemon_dir / "daemon.json" + daemon.write_bytes(prior) + daemon.chmod(0o640) + checkpoint = Path(self.tmp) / "checkpoint-atomic-removal" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + + no_policy_env = Path(self.tmp) / "no-policy-atomic-removal.env" + no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + fake_bin = Path(self.tmp) / "fake-cp-bin" + fake_bin.mkdir() + fake_cp = fake_bin / "cp" + fake_cp.write_text( + "#!/usr/bin/env bash\n" + f"[[ ${{*: -1}} != {daemon} ]] || exit 99\n" + f"exec {shutil.which('cp')} \"$@\"\n", + encoding="utf-8", + ) + fake_cp.chmod(0o755) + removed = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(no_policy_env), + ], + capture_output=True, + text=True, + env=self._env(PATH=f"{fake_bin}:{os.environ['PATH']}"), + timeout=30, + ) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertEqual(daemon.stat().st_mode & 0o777, 0o640) + def test_managed_policy_removal_restores_original_state_and_clears_only_marker(self) -> None: for prior_present in (False, True): with self.subTest(prior_present=prior_present): From 42775b6e1d5d1ff8d6134cd6cc11f7947d609f9e Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 16:19:47 -0500 Subject: [PATCH 10/54] fix: verify Docker policy runtime transitions --- scripts/apply-docker-network-policy.sh | 179 +++++++++---- scripts/test_apply_docker_network_policy.py | 271 +++++++++++++++++++- 2 files changed, 398 insertions(+), 52 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 721e5803..96d0c4f8 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -9,6 +9,7 @@ # CI_FLEET_DOCKER_DAEMON_CONFIG absolute path to daemon.json # CI_FLEET_DOCKER_DRAIN_COMMAND path to a host drain script (runs before mutation) # CI_FLEET_DOCKER_RESTART_COMMAND path to a Docker restart script +# CI_FLEET_CONTROLLER_RESUME_COMMAND path to a controller resume/start script # CI_FLEET_DOCKER_NETWORK_PROBE path to a capacity probe script # CI_FLEET_HEALTH_CHECK_COMMAND path to a health-check script # CI_FLEET_COMMAND_TIMEOUT_SECONDS command timeout in seconds (default 300) @@ -90,6 +91,7 @@ fi daemon_config=${CI_FLEET_DOCKER_DAEMON_CONFIG:-} drain_command=${CI_FLEET_DOCKER_DRAIN_COMMAND:-} restart_command=${CI_FLEET_DOCKER_RESTART_COMMAND:-} +resume_command=${CI_FLEET_CONTROLLER_RESUME_COMMAND:-} probe_command=${CI_FLEET_DOCKER_NETWORK_PROBE:-} health_command=${CI_FLEET_HEALTH_CHECK_COMMAND:-} command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} @@ -116,6 +118,7 @@ run_health() { [[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' validate_command CI_FLEET_DOCKER_DRAIN_COMMAND "$drain_command" validate_command CI_FLEET_DOCKER_RESTART_COMMAND "$restart_command" +validate_command CI_FLEET_CONTROLLER_RESUME_COMMAND "$resume_command" validate_command CI_FLEET_HEALTH_CHECK_COMMAND "$health_command" if [[ "$removing" == false ]]; then validate_command CI_FLEET_DOCKER_NETWORK_PROBE "$probe_command" @@ -133,6 +136,16 @@ else flock -n 9 || die 'another ci-fleet installer or drift check is already running' fi +# Snapshot the installer's authoritative pre-transaction environment while +# holding its lock. Rollback must not validate against the rejected candidate. +installed_env=${CI_FLEET_ROOT_PREFIX:-}/etc/ci-fleet/ci-fleet.env +[[ -f "$installed_env" && ! -L "$installed_env" && -r "$installed_env" ]] || die "installed rendered env must be a readable regular file: $installed_env" +[[ $(stat -c %u "$installed_env") == "$expected_env_owner" ]] || die "installed rendered env has an untrusted owner: $installed_env" +installed_env_mode=$(stat -c %a "$installed_env") +(( (8#$installed_env_mode & 8#022) == 0 )) || die "installed rendered env must not be group/world writable: $installed_env" +prior_env=$work_dir/prior-ci-fleet.env +install -m 0600 -- "$installed_env" "$prior_env" + checkpoint_owner=0 [[ "$testing" != 1 ]] || checkpoint_owner=$(id -u) if [[ -e "$checkpoint_dir" || -L "$checkpoint_dir" ]]; then @@ -191,12 +204,49 @@ finally: PY } +file_generation() { + python3 - "$1" <<'PY' +import hashlib, sys +with open(sys.argv[1], "rb") as handle: + print(hashlib.file_digest(handle, "sha256").hexdigest()) +PY +} + +set_verified_generation() { + python3 - "$state_file" "${1:-}" <<'PY' +import json, os, sys, tempfile +path, generation = sys.argv[1:] +state = json.load(open(path, encoding="utf-8")) +state["verified_generation"] = generation or None +fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(state, handle, indent=2, sort_keys=True) + handle.write("\n") + os.fchmod(handle.fileno(), 0o600) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + directory_fd = os.open(os.path.dirname(path), os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) +finally: + if os.path.exists(tmp): + os.unlink(tmp) +PY +} + if [[ "$removing" == true ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || die 'network-policy checkpoint state is invalid' mapfile -t managed_state < <(python3 - "$state_file" <<'PY' -import json, sys +import json, re, sys state = json.load(open(sys.argv[1], encoding="utf-8")) -if set(state) != {"managed", "prior_mode", "prior_present"} or state["managed"] is not True: +if set(state) not in ({"managed", "prior_mode", "prior_present"}, {"managed", "prior_mode", "prior_present", "verified_generation"}) or state["managed"] is not True: + raise SystemExit(1) +generation = state.get("verified_generation") +if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) if not isinstance(state["prior_present"], bool): raise SystemExit(1) @@ -208,11 +258,13 @@ elif mode is not None: raise SystemExit(1) print("true" if state["prior_present"] else "false") print(mode or "") +print(generation or "") PY ) || die 'network-policy checkpoint state is invalid' - [[ ${#managed_state[@]} == 2 ]] || die 'network-policy checkpoint state is invalid' + [[ ${#managed_state[@]} == 3 ]] || die 'network-policy checkpoint state is invalid' prior_present=${managed_state[0]} prior_mode=${managed_state[1]} + prior_verified_generation=${managed_state[2]} if [[ "$prior_present" == true ]]; then [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' [[ -f "$checkpoint_dir/daemon.json" && ! -L "$checkpoint_dir/daemon.json" ]] || die 'network-policy checkpoint backup is invalid' @@ -237,6 +289,9 @@ PY atomic_replace_daemon "$managed_daemon" "$managed_mode" "$managed_gid" || failed=1 cmp -s "$managed_daemon" "$daemon_config" || failed=1 run_command "$restart_command" "$daemon_dir" || failed=1 + run_command "$resume_command" --env "$prior_env" || failed=1 + run_health "$prior_env" || failed=1 + set_verified_generation "$prior_verified_generation" || failed=1 return "$failed" } # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below @@ -255,6 +310,7 @@ PY trap 'exit 130' INT trap 'exit 143' TERM trap removal_on_exit EXIT + set_verified_generation "" || { removal_failure='failed to mark network-policy verification pending'; exit 2; } if [[ "$prior_present" == true ]]; then atomic_replace_daemon "$checkpoint_dir/daemon.json" "$prior_mode" "$prior_gid" || { removal_failure='failed to restore prior daemon.json'; exit 2; } @@ -264,6 +320,7 @@ PY [[ ! -e "$daemon_config" ]] || { removal_failure='failed to verify absent daemon.json'; exit 2; } fi run_command "$restart_command" "$daemon_dir" || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } + run_command "$resume_command" --env "$env_file" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } run_health "$env_file" || { removal_failure='health check failed after network-policy removal'; exit 2; } # Marker deletion commits removal. Ignore catchable signals across the atomic @@ -328,12 +385,17 @@ PY state_file=$checkpoint_dir/docker-network-policy.json managed_before=false +prior_verified_generation= if [[ -e "$state_file" ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } - python3 - "$state_file" "$checkpoint_dir/daemon.json" <<'PY' || { rm -rf "$work_dir"; die 'network-policy checkpoint backup is invalid'; } + prior_verified_generation=$( + python3 - "$state_file" "$checkpoint_dir/daemon.json" <<'PY' import json, os, re, sys state = json.load(open(sys.argv[1], encoding="utf-8")) -if set(state) != {"managed", "prior_mode", "prior_present"} or state["managed"] is not True: +if set(state) not in ({"managed", "prior_mode", "prior_present"}, {"managed", "prior_mode", "prior_present", "verified_generation"}) or state["managed"] is not True: + raise SystemExit(1) +generation = state.get("verified_generation") +if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) if not isinstance(state["prior_present"], bool): raise SystemExit(1) @@ -344,19 +406,26 @@ if state["prior_present"]: raise SystemExit(1) elif state["prior_mode"] is not None or os.path.exists(sys.argv[2]): raise SystemExit(1) +print(generation or "") PY + ) || { rm -rf "$work_dir"; die 'network-policy checkpoint backup is invalid'; } managed_before=true fi +daemon_matches=false if [[ -f "$daemon_config" ]] && python3 - "$daemon_config" "$staging_daemon" <<'PY' import json, sys with open(sys.argv[1], encoding="utf-8") as current, open(sys.argv[2], encoding="utf-8") as staged: raise SystemExit(json.load(current) != json.load(staged)) PY then - rm -rf "$work_dir" - printf 'NETWORK_POLICY_NO_CHANGE\n' - exit 0 + daemon_matches=true + current_generation=$(file_generation "$daemon_config") || { rm -rf "$work_dir"; die 'failed to identify daemon.json generation'; } + if [[ -n "$prior_verified_generation" && "$current_generation" == "$prior_verified_generation" ]]; then + rm -rf "$work_dir" + printf 'NETWORK_POLICY_NO_CHANGE\n' + exit 0 + fi fi # --- Back up exact current daemon.json for transactional rollback --- @@ -370,11 +439,49 @@ if [[ -f "$daemon_config" ]]; then cp -p "$daemon_config" "$backup_dir/$backup_name" fi +restore_daemon() { + if [[ "$had_prior" == true ]]; then + atomic_replace_daemon "$backup_dir/$backup_name" "$daemon_mode" "$daemon_gid" + else + rm -f "$daemon_config" + fi +} + +rollback_daemon() { + local failed=0 + restore_daemon || failed=1 + run_command "$restart_command" "$daemon_dir" || failed=1 + run_command "$resume_command" --env "$prior_env" || failed=1 + run_health "$prior_env" || failed=1 + if [[ "$managed_before" == true ]]; then set_verified_generation "$prior_verified_generation" || failed=1; fi + return "$failed" +} + +transaction_failure='network-policy apply interrupted' +rollback_on_exit() { + local status=$? + trap - EXIT INT TERM + ((status != 0)) || status=1 + if rollback_daemon >/dev/null 2>&1; then + if [[ "$managed_before" == false ]]; then + rm -f "$state_file" "$checkpoint_dir/daemon.json" + fi + rm -rf "$work_dir" + printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 + else + printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$transaction_failure" "$work_dir" >&2 + fi + exit "$status" +} + # --- Drain after local validation/checkpointing, before mutation or restart --- if ! run_command "$drain_command"; then rm -rf "$work_dir" die "drain command failed before network-policy apply" fi +trap 'exit 130' INT +trap 'exit 143' TERM +trap rollback_on_exit EXIT # Record the original host state before the first managed mutation. Re-applying # policy keeps this baseline and uses the temp copy above for transaction rollback. @@ -384,13 +491,14 @@ if [[ "$managed_before" == false ]]; then install -m 0600 "$backup_dir/$backup_name" "$checkpoint_dir/daemon.json" chgrp "$daemon_gid" "$checkpoint_dir/daemon.json" fi - python3 - "$state_file" "$had_prior" "$daemon_mode" <<'PY' || { rm -rf "$work_dir"; die 'failed to record network-policy checkpoint state'; } + python3 - "$state_file" "$had_prior" "$daemon_mode" <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } import json, os, sys, tempfile path = sys.argv[1] state = { "managed": True, "prior_mode": sys.argv[3] if sys.argv[2] == "true" else None, "prior_present": sys.argv[2] == "true", + "verified_generation": None, } fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) try: @@ -411,44 +519,9 @@ finally: os.unlink(tmp) PY fi - -restore_daemon() { - if [[ "$had_prior" == true ]]; then - atomic_replace_daemon "$backup_dir/$backup_name" "$daemon_mode" "$daemon_gid" - else - rm -f "$daemon_config" - fi -} - -# Rollback: restore prior config, restart through the boundary, run health check. -# Failure evidence is surfaced through exit code only — no CIDRs or secrets leaked. -rollback_daemon() { - local failed=0 - restore_daemon || failed=1 - run_command "$restart_command" "$daemon_dir" || failed=1 - run_health "$env_file" || failed=1 - return "$failed" -} - -transaction_failure='network-policy apply interrupted' -rollback_on_exit() { - local status=$? - trap - EXIT INT TERM - ((status != 0)) || status=1 - if rollback_daemon >/dev/null 2>&1; then - if [[ "$managed_before" == false ]]; then - rm -f "$state_file" "$checkpoint_dir/daemon.json" - fi - rm -rf "$work_dir" - printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 - else - printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$transaction_failure" "$work_dir" >&2 - fi - exit "$status" -} -trap 'exit 130' INT -trap 'exit 143' TERM -trap rollback_on_exit EXIT +if [[ "$managed_before" == true ]]; then + set_verified_generation "" || { transaction_failure='failed to mark network-policy verification pending'; exit 2; } +fi fail_after_apply() { transaction_failure=$1 @@ -457,7 +530,9 @@ fail_after_apply() { # --- Transaction: apply → restart → probe → health, with rollback --- # Apply daemon.json atomically (rename within same directory). -atomic_replace_daemon "$staging_daemon" "$daemon_mode" "$daemon_gid" || { transaction_failure='failed to apply daemon.json'; exit 2; } +if [[ "$daemon_matches" == false ]]; then + atomic_replace_daemon "$staging_daemon" "$daemon_mode" "$daemon_gid" || { transaction_failure='failed to apply daemon.json'; exit 2; } +fi # Restart Docker through the injected command boundary (never host-direct). if ! run_command "$restart_command" "$daemon_dir"; then @@ -469,11 +544,19 @@ if ! run_command "$probe_command"; then fail_after_apply "capacity probe failed after network-policy restart" fi +# Resume the drained controller before health verification. +if ! run_command "$resume_command" --env "$env_file"; then + fail_after_apply "controller resume command failed after network-policy restart" +fi + # Health verification if ! run_health "$env_file"; then fail_after_apply "health check failed after network-policy restart" fi +verified_generation=$(file_generation "$daemon_config") || fail_after_apply "failed to identify verified daemon.json generation" +set_verified_generation "$verified_generation" || fail_after_apply "failed to record verified daemon.json generation" + # --- Success --- trap - EXIT INT TERM rm -rf "$work_dir" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 8bbaa5ef..9caec797 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -9,6 +9,7 @@ from __future__ import annotations import fcntl +import hashlib import json import os import shutil @@ -235,9 +236,16 @@ def setUp(self) -> None: self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) self.daemon_dir = Path(self.tmp) / "etc" / "docker" self.daemon_dir.mkdir(parents=True) + self.installed_env = Path(self.tmp) / "etc" / "ci-fleet" / "ci-fleet.env" + self.installed_env.parent.mkdir(parents=True) + self.installed_env.write_text("ENV_GENERATION=prior\n", encoding="utf-8") + self.installed_env.chmod(0o600) self.drain_command = Path(self.tmp) / "drain.sh" self.drain_command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") self.drain_command.chmod(0o755) + resume_command = Path(self.tmp) / "resume.sh" + resume_command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + resume_command.chmod(0o755) def _write_daemon(self, content: str) -> Path: path = self.daemon_dir / "daemon.json" @@ -250,6 +258,7 @@ def _env(self, **extra: str) -> dict[str, str]: env["CI_FLEET_DOCKER_DAEMON_CONFIG"] = str(self.daemon_dir / "daemon.json") env["CI_FLEET_DOCKER_DRAIN_COMMAND"] = str(self.drain_command) env["CI_FLEET_DOCKER_RESTART_COMMAND"] = str(Path(self.tmp) / "restart.sh") + env["CI_FLEET_CONTROLLER_RESUME_COMMAND"] = str(Path(self.tmp) / "resume.sh") env["CI_FLEET_DOCKER_NETWORK_PROBE"] = str(Path(self.tmp) / "probe.sh") env["CI_FLEET_HEALTH_CHECK_COMMAND"] = str(Path(self.tmp) / "health.sh") env["CI_FLEET_ROOT_PREFIX"] = self.tmp @@ -294,11 +303,159 @@ def _run(self, env_file: str, checkpoint_dir: str | None = None, expected_rc: in ) def _write_success_commands(self) -> None: + for name in ("restart.sh", "resume.sh", "probe.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + + def test_successful_apply_restarts_probes_resumes_then_checks_health(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "activation.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + result = self._run(str(env_file)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "resume", "health"]) + + def test_failed_apply_restarts_resumes_then_checks_rollback_health(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "rollback-activation.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text(f"#!/usr/bin/env bash\necho probe >> {command_log}\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "restart", "resume", "health"]) + + def test_failed_apply_uses_pretransaction_installed_env_for_rollback(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + self.drain_command.write_text( + f"#!/usr/bin/env bash\nprintf 'ENV_GENERATION=changed-after-drain\\n' > {self.installed_env}\n", + encoding="utf-8", + ) + for name in ("restart.sh",): + command = Path(self.tmp) / name + command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + command.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + rollback_env_log = Path(self.tmp) / "rollback-env.log" + for name in ("resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text( + "#!/usr/bin/env bash\n" + f"printf '%s\\n' \"${{2:-missing}}\" >> {rollback_env_log}\n" + '[[ $1 == --env && -f $2 ]] || exit 2\n' + 'grep -Fqx "ENV_GENERATION=prior" "$2" || exit 2\n', + encoding="utf-8", + ) + command.chmod(0o755) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("prior daemon.json restored", result.stderr) + self.assertNotIn("rollback verification failed", result.stderr) + rollback_envs = rollback_env_log.read_text(encoding="utf-8").splitlines() + self.assertEqual(len(rollback_envs), 2) + self.assertEqual(len(set(rollback_envs)), 1) + self.assertNotEqual(rollback_envs[0], str(self.installed_env)) + + def test_post_drain_state_failure_resumes_prior_controller(self) -> None: + daemon = self._write_daemon("{}\n") + prior = daemon.read_bytes() + checkpoint = Path(self.tmp) / "checkpoint-post-drain-failure" + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "post-drain-failure.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + probe.chmod(0o755) + fake_bin = Path(self.tmp) / "post-drain-bin" + fake_bin.mkdir() + python = fake_bin / "python3" + python.write_text( + "#!/usr/bin/env bash\n" + f"[[ ${{2:-}} != {checkpoint}/docker-network-policy.json ]] || exit 1\n" + f"exec {shutil.which('python3')} \"$@\"\n", + encoding="utf-8", + ) + python.chmod(0o755) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(PATH=f"{fake_bin}:{os.environ['PATH']}"), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "health"]) + + def test_drain_failure_never_resumes_controller(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + self.drain_command.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + resume_marker = Path(self.tmp) / "resume-after-failed-drain.marker" + resume = Path(self.tmp) / "resume.sh" + resume.write_text(f"#!/usr/bin/env bash\ntouch {resume_marker}\n", encoding="utf-8") + resume.chmod(0o755) for name in ("restart.sh", "probe.sh", "health.sh"): command = Path(self.tmp) / name command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") command.chmod(0o755) + result = self._run(str(env_file)) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(resume_marker.exists()) + + def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-reapply-generation" + self._write_success_commands() + prior_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(prior_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + prior_generation = json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"] + self.installed_env.write_bytes(prior_env.read_bytes()) + candidate = self._rendered_with_policy() + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = Path(self.tmp) / "candidate-reapply.env" + candidate_env.write_text("".join(f"{key}={value}\n" for key, value in sorted(candidate.items())), encoding="utf-8") + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + + result = self._run(str(candidate_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"], prior_generation) + self.assertEqual(hashlib.sha256(daemon.read_bytes()).hexdigest(), prior_generation) + def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -353,6 +510,7 @@ def test_successful_apply_records_managed_original_presence_and_mode(self) -> No "managed": True, "prior_mode": "600" if prior_present else None, "prior_present": prior_present, + "verified_generation": hashlib.sha256(daemon.read_bytes()).hexdigest(), }, ) self.assertEqual((checkpoint / "docker-network-policy.json").stat().st_mode & 0o777, 0o600) @@ -425,6 +583,7 @@ def test_injected_hooks_require_absolute_canonical_regular_executables(self) -> hook_names = ( "CI_FLEET_DOCKER_DRAIN_COMMAND", "CI_FLEET_DOCKER_RESTART_COMMAND", + "CI_FLEET_CONTROLLER_RESUME_COMMAND", "CI_FLEET_DOCKER_NETWORK_PROBE", "CI_FLEET_HEALTH_CHECK_COMMAND", ) @@ -478,6 +637,7 @@ def test_all_hooks_use_kill_grace_and_suppress_output(self) -> None: hooks = { "CI_FLEET_DOCKER_DRAIN_COMMAND": self.drain_command, "CI_FLEET_DOCKER_RESTART_COMMAND": Path(self.tmp) / "restart.sh", + "CI_FLEET_CONTROLLER_RESUME_COMMAND": Path(self.tmp) / "resume.sh", "CI_FLEET_DOCKER_NETWORK_PROBE": Path(self.tmp) / "probe.sh", "CI_FLEET_HEALTH_CHECK_COMMAND": Path(self.tmp) / "health.sh", } @@ -756,13 +916,13 @@ def test_invalid_existing_config_does_not_drain(self) -> None: self.assertEqual(daemon.read_text(encoding="utf-8"), "{not-json\n") self.assertFalse(drain_marker.exists()) - def test_semantically_equal_daemon_config_is_no_change_before_side_effects(self) -> None: + def test_semantically_equal_verified_daemon_config_is_no_change_before_side_effects(self) -> None: rendered = self._rendered_with_policy() desired = render_docker_daemon_config(rendered) daemon = self._write_daemon(json.dumps(desired, separators=(",", ":"))) prior = daemon.read_bytes() markers = [] - for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): + for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): marker = Path(self.tmp) / f"{name}.marker" markers.append(marker) command = Path(self.tmp) / name @@ -770,15 +930,55 @@ def test_semantically_equal_daemon_config_is_no_change_before_side_effects(self) command.chmod(0o755) env_file = self._write_env_file(rendered) checkpoint = Path(self.tmp) / "checkpoint" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_mode": None, + "prior_present": False, + "verified_generation": hashlib.sha256(prior).hexdigest(), + }), + encoding="utf-8", + ) + state_file.chmod(0o600) result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "NETWORK_POLICY_NO_CHANGE\n") self.assertEqual(daemon.read_bytes(), prior) - self.assertFalse(checkpoint.exists()) + self.assertTrue(checkpoint.exists()) self.assertTrue(all(not marker.exists() for marker in markers)) + def test_matching_file_without_verified_generation_runs_activation_and_marks_verified(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-pending-generation" + self._write_success_commands() + rendered = self._rendered_with_policy() + env_file = self._write_env_file(rendered) + applied = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state.pop("verified_generation", None) + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + command_log = Path(self.tmp) / "pending-generation.log" + command_log.touch() + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + reconciled = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(reconciled.returncode, 0, reconciled.stderr) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "resume", "health"]) + verified = json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"] + self.assertEqual(verified, hashlib.sha256((self.daemon_dir / "daemon.json").read_bytes()).hexdigest()) + def test_validates_policy_before_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -1010,6 +1210,8 @@ def test_rollback_failure_is_reported(self) -> None: self.addCleanup(shutil.rmtree, recovery, ignore_errors=True) self.assertTrue(recovery.name.startswith(".ci-fleet-apply.")) self.assertEqual((recovery / "prior" / "daemon.json.before").read_bytes(), prior) + recovery_state = json.loads((Path(self.tmp) / "checkpoint-default" / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertIsNone(recovery_state["verified_generation"]) def test_post_apply_health_uses_candidate_rendered_env(self) -> None: self._write_daemon("{}\n") @@ -1357,6 +1559,28 @@ def test_managed_policy_removal_restores_backup_without_copying_over_daemon(self self.assertEqual(daemon.read_bytes(), prior) self.assertEqual(daemon.stat().st_mode & 0o777, 0o640) + def test_successful_removal_restarts_resumes_then_checks_health(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-resume" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(policy_env.read_bytes()) + no_policy_env = Path(self.tmp) / "no-policy-removal-resume.env" + no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + command_log = Path(self.tmp) / "removal-resume.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "health"]) + def test_managed_policy_removal_restores_original_state_and_clears_only_marker(self) -> None: for prior_present in (False, True): with self.subTest(prior_present=prior_present): @@ -1419,6 +1643,7 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self managed_mode = daemon.stat().st_mode & 0o777 state = (checkpoint / "docker-network-policy.json").read_bytes() baseline = (checkpoint / "daemon.json").read_bytes() + self.installed_env.write_bytes(policy_env.read_bytes()) restart_log = Path(self.tmp) / "remove-failure-restart.log" restart = Path(self.tmp) / "restart.sh" @@ -1427,6 +1652,7 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self health = Path(self.tmp) / "health.sh" health.write_text( "#!/usr/bin/env bash\n" + 'grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2" && exit 0\n' "echo '198.51.100.0/24 https://secret.example.invalid token=credential'\n" "echo removal-health-secret >&2\n" "exit 2\n", @@ -1449,6 +1675,43 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self for secret in ("198.51.100.0/24", "secret.example.invalid", "mirror.example.invalid", "credential", "removal-health-secret"): self.assertNotIn(secret, combined) + def test_removal_rollback_resumes_and_checks_health_with_managed_env(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-rollback-env" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + self.installed_env.write_bytes(policy_env.read_bytes()) + no_policy_env = Path(self.tmp) / "no-policy-removal-rollback.env" + no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + command_log = Path(self.tmp) / "removal-rollback-env.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + restart = Path(self.tmp) / "restart.sh" + restart.write_text(f"#!/usr/bin/env bash\necho restart >> {command_log}\n", encoding="utf-8") + restart.chmod(0o755) + for name in ("resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text( + "#!/usr/bin/env bash\n" + 'grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2" && generation=managed || generation=candidate\n' + f"echo {name}-$generation >> {command_log}\n" + f"[[ {name} != health || $generation == managed ]] || exit 2\n", + encoding="utf-8", + ) + command.chmod(0o755) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("managed daemon.json restored", removed.stderr) + self.assertEqual(daemon.read_bytes(), managed) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "resume-candidate", "health-candidate", "restart", "resume-managed", "health-managed"], + ) + def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) -> None: daemon = self._write_daemon('{"icc":false}\n') checkpoint = Path(self.tmp) / "checkpoint-marker-failure" @@ -1505,7 +1768,7 @@ def test_unmanaged_no_policy_is_noop_without_mutation_or_commands(self) -> None: daemon = self.daemon_dir / "daemon.json" daemon.write_bytes(prior) markers = [] - for name in ("drain.sh", "restart.sh", "probe.sh", "health.sh"): + for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): marker = Path(self.tmp) / f"{name}.marker" markers.append(marker) command = Path(self.tmp) / name From eed2bec6331f11397c2fb5bca898c57e0bba0fb4 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 17:05:41 -0500 Subject: [PATCH 11/54] fix: preserve Docker settings on policy removal --- scripts/apply-docker-network-policy.sh | 121 +++++++++---- scripts/test_apply_docker_network_policy.py | 190 ++++++++++++++++---- 2 files changed, 247 insertions(+), 64 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 96d0c4f8..aca885a8 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -151,12 +151,13 @@ checkpoint_owner=0 if [[ -e "$checkpoint_dir" || -L "$checkpoint_dir" ]]; then [[ -d "$checkpoint_dir" && ! -L "$checkpoint_dir" && $(stat -c %u "$checkpoint_dir") == "$checkpoint_owner" && $(stat -c %a "$checkpoint_dir") == 700 ]] || die "checkpoint directory must be owned by root with mode 0700: $checkpoint_dir" - for checkpoint_file in "$checkpoint_dir/docker-network-policy.json" "$checkpoint_dir/daemon.json"; do - if [[ -e "$checkpoint_file" || -L "$checkpoint_file" ]]; then - [[ -f "$checkpoint_file" && ! -L "$checkpoint_file" && $(stat -c %u "$checkpoint_file") == "$checkpoint_owner" && $(stat -c %a "$checkpoint_file") == 600 ]] || - die "checkpoint files must be owned by root with mode 0600: $checkpoint_file" - fi - done + checkpoint_file=$checkpoint_dir/docker-network-policy.json + if [[ -e "$checkpoint_file" || -L "$checkpoint_file" ]]; then + [[ -f "$checkpoint_file" && ! -L "$checkpoint_file" && $(stat -c %u "$checkpoint_file") == "$checkpoint_owner" && $(stat -c %a "$checkpoint_file") == 600 ]] || + die "checkpoint files must be owned by root with mode 0600: $checkpoint_file" + fi + [[ ! -e "$checkpoint_dir/daemon.json" && ! -L "$checkpoint_dir/daemon.json" ]] || + die 'network-policy checkpoint state is invalid' fi [[ ! -L "$daemon_config" ]] || die "daemon.json must not be a symlink: $daemon_config" @@ -243,13 +244,20 @@ if [[ "$removing" == true ]]; then mapfile -t managed_state < <(python3 - "$state_file" <<'PY' import json, re, sys state = json.load(open(sys.argv[1], encoding="utf-8")) -if set(state) not in ({"managed", "prior_mode", "prior_present"}, {"managed", "prior_mode", "prior_present", "verified_generation"}) or state["managed"] is not True: +required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} +if set(state) not in (required, required | {"verified_generation"}) or state["managed"] is not True: raise SystemExit(1) generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) if not isinstance(state["prior_present"], bool): raise SystemExit(1) +if not isinstance(state["prior_default_address_pools_present"], bool): + raise SystemExit(1) +if state["prior_default_address_pools_present"] and not state["prior_present"]: + raise SystemExit(1) +if not state["prior_default_address_pools_present"] and state["prior_default_address_pools"] is not None: + raise SystemExit(1) mode = state["prior_mode"] if state["prior_present"]: if not isinstance(mode, str) or not mode.isdigit(): @@ -267,8 +275,6 @@ PY prior_verified_generation=${managed_state[2]} if [[ "$prior_present" == true ]]; then [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' - [[ -f "$checkpoint_dir/daemon.json" && ! -L "$checkpoint_dir/daemon.json" ]] || die 'network-policy checkpoint backup is invalid' - prior_gid=$(stat -c %g "$checkpoint_dir/daemon.json") fi [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' @@ -285,9 +291,26 @@ PY removal_failure='network-policy removal interrupted' # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below rollback_removal() { - local failed=0 - atomic_replace_daemon "$managed_daemon" "$managed_mode" "$managed_gid" || failed=1 - cmp -s "$managed_daemon" "$daemon_config" || failed=1 + local failed=0 rollback_daemon=$work_dir/daemon.json.rollback + python3 - "$daemon_config" "$managed_daemon" "$rollback_daemon" <<'PY' || failed=1 +import json, os, sys +current_path, managed_path, output_path = sys.argv[1:] +try: + current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} + managed = json.load(open(managed_path, encoding="utf-8")) + if not isinstance(current, dict) or not isinstance(managed, dict) or "default-address-pools" not in managed: + raise ValueError +except (OSError, json.JSONDecodeError, ValueError): + raise SystemExit(1) +current["default-address-pools"] = managed["default-address-pools"] +with open(output_path, "w", encoding="utf-8") as handle: + json.dump(current, handle, indent=2, sort_keys=True) + handle.write("\n") +PY + if ((failed == 0)); then + atomic_replace_daemon "$rollback_daemon" "$managed_mode" "$managed_gid" || failed=1 + cmp -s "$rollback_daemon" "$daemon_config" || failed=1 + fi run_command "$restart_command" "$daemon_dir" || failed=1 run_command "$resume_command" --env "$prior_env" || failed=1 run_health "$prior_env" || failed=1 @@ -312,12 +335,45 @@ PY trap removal_on_exit EXIT set_verified_generation "" || { removal_failure='failed to mark network-policy verification pending'; exit 2; } - if [[ "$prior_present" == true ]]; then - atomic_replace_daemon "$checkpoint_dir/daemon.json" "$prior_mode" "$prior_gid" || { removal_failure='failed to restore prior daemon.json'; exit 2; } - cmp -s "$checkpoint_dir/daemon.json" "$daemon_config" || { removal_failure='failed to verify prior daemon.json'; exit 2; } + removal_daemon=$work_dir/daemon.json.removal + python3 - "$daemon_config" "$state_file" "$removal_daemon" <<'PY' || { removal_failure='failed to stage network-policy removal'; exit 2; } +import json, sys +daemon_path, state_path, output_path = sys.argv[1:] +try: + current = json.load(open(daemon_path, encoding="utf-8")) + state = json.load(open(state_path, encoding="utf-8")) + if not isinstance(current, dict): + raise ValueError +except (OSError, json.JSONDecodeError, ValueError): + raise SystemExit(1) +if state["prior_default_address_pools_present"]: + current["default-address-pools"] = state["prior_default_address_pools"] +else: + current.pop("default-address-pools", None) +with open(output_path, "w", encoding="utf-8") as handle: + json.dump(current, handle, indent=2, sort_keys=True) + handle.write("\n") +PY + if [[ "$prior_present" == false ]] && python3 - "$removal_daemon" <<'PY' +import json, sys +raise SystemExit(bool(json.load(open(sys.argv[1], encoding="utf-8")))) +PY + then + python3 - "$daemon_config" "$daemon_dir" <<'PY' || { removal_failure='failed to remove empty daemon config'; exit 2; } +import os, sys +try: + os.unlink(sys.argv[1]) +except FileNotFoundError: + pass +directory_fd = os.open(sys.argv[2], os.O_RDONLY | os.O_DIRECTORY) +try: + os.fsync(directory_fd) +finally: + os.close(directory_fd) +PY else - rm -f "$daemon_config" || { removal_failure='failed to restore absent daemon.json'; exit 2; } - [[ ! -e "$daemon_config" ]] || { removal_failure='failed to verify absent daemon.json'; exit 2; } + atomic_replace_daemon "$removal_daemon" "$managed_mode" "$managed_gid" || { removal_failure='failed to install prior network-policy key state'; exit 2; } + cmp -s "$removal_daemon" "$daemon_config" || { removal_failure='failed to verify prior network-policy key state'; exit 2; } fi run_command "$restart_command" "$daemon_dir" || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } run_command "$resume_command" --env "$env_file" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } @@ -389,26 +445,31 @@ prior_verified_generation= if [[ -e "$state_file" ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } prior_verified_generation=$( - python3 - "$state_file" "$checkpoint_dir/daemon.json" <<'PY' -import json, os, re, sys + python3 - "$state_file" <<'PY' +import json, re, sys state = json.load(open(sys.argv[1], encoding="utf-8")) -if set(state) not in ({"managed", "prior_mode", "prior_present"}, {"managed", "prior_mode", "prior_present", "verified_generation"}) or state["managed"] is not True: +required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} +if set(state) not in (required, required | {"verified_generation"}) or state["managed"] is not True: raise SystemExit(1) generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) if not isinstance(state["prior_present"], bool): raise SystemExit(1) +if not isinstance(state["prior_default_address_pools_present"], bool): + raise SystemExit(1) +if state["prior_default_address_pools_present"] and not state["prior_present"]: + raise SystemExit(1) +if not state["prior_default_address_pools_present"] and state["prior_default_address_pools"] is not None: + raise SystemExit(1) if state["prior_present"]: if not isinstance(state["prior_mode"], str) or not re.fullmatch(r"[0-7]{3,4}", state["prior_mode"]): raise SystemExit(1) - if not os.path.isfile(sys.argv[2]) or os.path.islink(sys.argv[2]): - raise SystemExit(1) -elif state["prior_mode"] is not None or os.path.exists(sys.argv[2]): +elif state["prior_mode"] is not None: raise SystemExit(1) print(generation or "") PY - ) || { rm -rf "$work_dir"; die 'network-policy checkpoint backup is invalid'; } + ) || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } managed_before=true fi @@ -464,7 +525,7 @@ rollback_on_exit() { ((status != 0)) || status=1 if rollback_daemon >/dev/null 2>&1; then if [[ "$managed_before" == false ]]; then - rm -f "$state_file" "$checkpoint_dir/daemon.json" + rm -f "$state_file" fi rm -rf "$work_dir" printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 @@ -487,15 +548,15 @@ trap rollback_on_exit EXIT # policy keeps this baseline and uses the temp copy above for transaction rollback. if [[ "$managed_before" == false ]]; then install -d -m 0700 "$checkpoint_dir" - if [[ "$had_prior" == true ]]; then - install -m 0600 "$backup_dir/$backup_name" "$checkpoint_dir/daemon.json" - chgrp "$daemon_gid" "$checkpoint_dir/daemon.json" - fi - python3 - "$state_file" "$had_prior" "$daemon_mode" <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } + python3 - "$state_file" "$had_prior" "$daemon_mode" "$backup_dir/$backup_name" <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } import json, os, sys, tempfile path = sys.argv[1] +prior = json.load(open(sys.argv[4], encoding="utf-8")) if sys.argv[2] == "true" else {} +prior_key_present = "default-address-pools" in prior state = { "managed": True, + "prior_default_address_pools": prior.get("default-address-pools") if prior_key_present else None, + "prior_default_address_pools_present": prior_key_present, "prior_mode": sys.argv[3] if sys.argv[2] == "true" else None, "prior_present": sys.argv[2] == "true", "verified_generation": None, diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 9caec797..d7cfab05 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -508,18 +508,17 @@ def test_successful_apply_records_managed_original_presence_and_mode(self) -> No state, { "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, "prior_mode": "600" if prior_present else None, "prior_present": prior_present, "verified_generation": hashlib.sha256(daemon.read_bytes()).hexdigest(), }, ) self.assertEqual((checkpoint / "docker-network-policy.json").stat().st_mode & 0o777, 0o600) - self.assertEqual((checkpoint / "daemon.json").exists(), prior_present) - if prior_present: - self.assertEqual((checkpoint / "daemon.json").read_bytes(), b'{"icc":false}\n') - self.assertEqual((checkpoint / "daemon.json").stat().st_mode & 0o777, 0o600) + self.assertFalse((checkpoint / "daemon.json").exists()) - def test_policy_reapply_fails_closed_when_recovery_backup_is_missing(self) -> None: + def test_policy_reapply_fails_closed_when_managed_key_provenance_is_missing(self) -> None: rendered = self._rendered_with_policy() daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) prior = daemon.read_bytes() @@ -543,11 +542,11 @@ def test_policy_reapply_fails_closed_when_recovery_backup_is_missing(self) -> No result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) self.assertNotEqual(result.returncode, 0) - self.assertIn("checkpoint backup is invalid", result.stderr) + self.assertIn("checkpoint state is invalid", result.stderr) self.assertEqual(daemon.read_bytes(), prior) self.assertTrue(all(not marker.exists() for marker in markers)) - def test_policy_reapply_fails_closed_when_absent_baseline_has_backup(self) -> None: + def test_policy_reapply_rejects_persistent_recovery_backup(self) -> None: rendered = self._rendered_with_policy() daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) prior = daemon.read_bytes() @@ -574,7 +573,7 @@ def test_policy_reapply_fails_closed_when_absent_baseline_has_backup(self) -> No result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) self.assertNotEqual(result.returncode, 0) - self.assertIn("checkpoint backup is invalid", result.stderr) + self.assertIn("checkpoint state is invalid", result.stderr) self.assertEqual(daemon.read_bytes(), prior) self.assertTrue(all(not marker.exists() for marker in markers)) @@ -935,6 +934,8 @@ def test_semantically_equal_verified_daemon_config_is_no_change_before_side_effe state_file.write_text( json.dumps({ "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, "prior_mode": None, "prior_present": False, "verified_generation": hashlib.sha256(prior).hexdigest(), @@ -1497,28 +1498,94 @@ def test_restart_failure_evidence_excludes_command_output(self) -> None: self.assertNotIn("credential", combined) self.assertNotIn("restart-secret-error", combined) - def test_checkpoint_preserves_prior_config(self) -> None: - """GREEN: the checkpoint retains the prior daemon.json for restoration.""" - prior = {"bip": "172.17.0.1/16", "icc": False} - self._write_daemon(json.dumps(prior)) - restart = Path(self.tmp) / "restart.sh" - restart.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") - restart.chmod(0o755) - probe = Path(self.tmp) / "probe.sh" - probe.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") - probe.chmod(0o755) - health = Path(self.tmp) / "health.sh" - health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") - health.chmod(0o755) - checkpoint_dir = Path(self.tmp) / "checkpoint" - env_file = self._write_env_file(self._rendered_with_policy()) - result = self._run(str(env_file), checkpoint_dir=str(checkpoint_dir)) - self.assertEqual(result.returncode, 0, result.stderr) - backup = checkpoint_dir / "daemon.json" - self.assertTrue(backup.exists()) - self.assertEqual(json.loads(backup.read_text(encoding="utf-8")), prior) + def test_removal_restores_prior_pools_and_preserves_current_unrelated_keys(self) -> None: + prior_pools = [{"base": "192.0.2.0/24", "size": 28}] + daemon = self._write_daemon(json.dumps({"default-address-pools": prior_pools, "icc": False})) + checkpoint = Path(self.tmp) / "checkpoint-prior-pools" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(applied.returncode, 0, applied.stderr) + state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertTrue(state["prior_default_address_pools_present"]) + self.assertEqual(state["prior_default_address_pools"], prior_pools) + current = json.loads(daemon.read_text(encoding="utf-8")) + current.pop("icc") + current["live-restore"] = True + daemon.write_text(json.dumps(current), encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual( + json.loads(daemon.read_text(encoding="utf-8")), + {"default-address-pools": prior_pools, "live-restore": True}, + ) + + def test_removal_drops_only_managed_key_when_prior_key_was_absent(self) -> None: + for original_file_present, current_unrelated in ((True, {"live-restore": True}), (False, {})): + with self.subTest(original_file_present=original_file_present): + daemon = self.daemon_dir / "daemon.json" + daemon.unlink(missing_ok=True) + if original_file_present: + daemon.write_text(json.dumps({"icc": False}), encoding="utf-8") + checkpoint = Path(self.tmp) / f"checkpoint-prior-key-absent-{original_file_present}" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(applied.returncode, 0, applied.stderr) + state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertFalse(state["prior_default_address_pools_present"]) + self.assertIsNone(state["prior_default_address_pools"]) + managed = json.loads(daemon.read_text(encoding="utf-8")) + managed.pop("icc", None) + managed.update(current_unrelated) + daemon.write_text(json.dumps(managed), encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(daemon.exists(), bool(current_unrelated) or original_file_present) + if daemon.exists(): + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), current_unrelated) + + def test_persistent_provenance_has_no_recovery_backup_and_rejects_inconsistent_state(self) -> None: + prior_pools = [{"base": "192.0.2.0/24", "size": 28}] + daemon = self._write_daemon(json.dumps({"default-address-pools": prior_pools, "registry-mirrors": ["private-daemon-value"]})) + checkpoint = Path(self.tmp) / "checkpoint-provenance-only" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) - def test_managed_policy_removal_restores_backup_without_copying_over_daemon(self) -> None: + self.assertEqual(applied.returncode, 0, applied.stderr) + self.assertFalse((checkpoint / "daemon.json").exists()) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["prior_default_address_pools_present"] = False + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + managed = daemon.read_bytes() + drain_marker = Path(self.tmp) / "inconsistent-provenance-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + self.assertFalse(drain_marker.exists()) + self.assertIn("network-policy checkpoint state is invalid", removed.stderr) + self.assertNotIn("private-daemon-value", removed.stdout + removed.stderr) + self.assertNotIn("192.0.2.0/24", removed.stdout + removed.stderr) + + def test_managed_policy_removal_replaces_daemon_atomically(self) -> None: prior = b'{"icc":false}\n' daemon = self.daemon_dir / "daemon.json" daemon.write_bytes(prior) @@ -1556,7 +1623,7 @@ def test_managed_policy_removal_restores_backup_without_copying_over_daemon(self ) self.assertEqual(removed.returncode, 0, removed.stderr) - self.assertEqual(daemon.read_bytes(), prior) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), json.loads(prior)) self.assertEqual(daemon.stat().st_mode & 0o777, 0o640) def test_successful_removal_restarts_resumes_then_checks_health(self) -> None: @@ -1623,12 +1690,67 @@ def test_managed_policy_removal_restores_original_state_and_clears_only_marker(s self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain.sh", "restart.sh", "health.sh"]) self.assertEqual(daemon.exists(), prior_present) if prior_present: - self.assertEqual(daemon.read_bytes(), prior) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), json.loads(prior)) self.assertEqual(daemon.stat().st_mode & 0o777, 0o600) - self.assertTrue((checkpoint / "daemon.json").exists()) + self.assertFalse((checkpoint / "daemon.json").exists()) self.assertFalse((checkpoint / "docker-network-policy.json").exists()) self.assertTrue(retained.exists()) + def test_removal_failure_restores_managed_key_into_current_unrelated_json(self) -> None: + daemon = self._write_daemon(json.dumps({"icc": False})) + checkpoint = Path(self.tmp) / "checkpoint-remove-merge-rollback" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + self.installed_env.write_bytes(policy_env.read_bytes()) + current = json.loads(daemon.read_text(encoding="utf-8")) + current.pop("icc") + current["live-restore"] = True + daemon.write_text(json.dumps(current), encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + command_log = Path(self.tmp) / "removal-merge-rollback.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + restart = Path(self.tmp) / "restart.sh" + restart.write_text(f"#!/usr/bin/env bash\necho restart >> {command_log}\n", encoding="utf-8") + restart.chmod(0o755) + mutator = Path(self.tmp) / "mutate-daemon.py" + mutator.write_text( + "import json, sys\n" + "p = sys.argv[1]\n" + "d = json.load(open(p))\n" + "d['debug'] = True\n" + "with open(p, 'w') as handle: json.dump(d, handle)\n", + encoding="utf-8", + ) + resume = Path(self.tmp) / "resume.sh" + resume.write_text( + "#!/usr/bin/env bash\n" + f"echo resume >> {command_log}\n" + 'if ! grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2"; then\n' + f" {shutil.which('python3')} {mutator} {daemon}\n" + " exit 2\n" + "fi\n", + encoding="utf-8", + ) + resume.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text(f"#!/usr/bin/env bash\necho health >> {command_log}\n", encoding="utf-8") + health.chmod(0o755) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual( + json.loads(daemon.read_text(encoding="utf-8")), + {"default-address-pools": managed_pools, "debug": True, "live-restore": True}, + ) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "resume", "restart", "resume", "health"], + ) + def test_removal_failure_restores_managed_config_and_retains_recovery_state(self) -> None: prior = b'{"registry-mirrors":["https://mirror.example.invalid?token=credential"]}\n' daemon = self.daemon_dir / "daemon.json" @@ -1642,7 +1764,7 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self managed = daemon.read_bytes() managed_mode = daemon.stat().st_mode & 0o777 state = (checkpoint / "docker-network-policy.json").read_bytes() - baseline = (checkpoint / "daemon.json").read_bytes() + self.assertFalse((checkpoint / "daemon.json").exists()) self.installed_env.write_bytes(policy_env.read_bytes()) restart_log = Path(self.tmp) / "remove-failure-restart.log" @@ -1669,7 +1791,7 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self self.assertEqual(daemon.stat().st_mode & 0o777, managed_mode) self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) self.assertEqual((checkpoint / "docker-network-policy.json").read_bytes(), state) - self.assertEqual((checkpoint / "daemon.json").read_bytes(), baseline) + self.assertFalse((checkpoint / "daemon.json").exists()) combined = removed.stdout + removed.stderr self.assertIn("managed daemon.json restored", combined) for secret in ("198.51.100.0/24", "secret.example.invalid", "mirror.example.invalid", "credential", "removal-health-secret"): From c21a6209eb39fa9a491922a654b2bdaeb601d06c Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 17:23:30 -0500 Subject: [PATCH 12/54] docs: document Docker policy rollout gates --- docs/DESIRED-STATE.md | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index 759704f7..cc2561f5 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -70,12 +70,23 @@ reconciliation. Once present, the policy requires current evidence naming the selected engine and declaring `docker_network_policy_config: true`; remove the policy before selecting an engine without that evidence. -This phase renders the policy solely for read-only health inspection. It does -not write `daemon.json`, restart Docker, create or remove networks, prune -resources, drain runners, or alter controller scale. Circuit breaking, -frequent orphan reconciliation, daemon configuration rollout, transactional -exhausted-pool recovery, and downstream-consumer label changes remain later -issue #81 slices. +This accepted phase permits the executable policy stage to apply or remove the +managed `default-address-pools` key on an isolated ordinary-CI controller only +from validated desired state. Before mutation, the stage must persist a +root-only checkpoint with prior-key provenance, acquire the shared installer +lock for serialized mutation, and drain the controller and managed runners. It +must atomically and durably write or remove only the managed key, restart +Docker, run the bounded capacity probe, resume the controller to its intended +state, and verify health against the exact candidate rendered environment. + +On interruption or failure, rollback must restore the managed key and prior +rendered environment from the checkpoint, restart Docker, resume the prior +controller state, and verify prior health. A failed rollback must retain its +recovery checkpoint. Rollout requires exact-head CI and proof for the reviewed +engine and desired-state commits before any host mutation. No deployment occurs +in this PR. This scope does not create or remove networks, prune resources, +alter controller scale, change downstream-consumer labels, or authorize +application production deployment. Managed prewarmed runners are not currently supported: `min_runners` is fixed at zero in schema, semantic validation, rendering, and preflight. This keeps idle privileged workers absent and prevents reviewed configuration from passing validation only to fail host adoption. From 11ca1ce37c5786f96b0a125edaf27f31ba3ab25a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 18:20:56 -0500 Subject: [PATCH 13/54] fix: preserve warning health fixtures in CI --- scripts/healthcheck.sh | 10 +++++++++- scripts/test-install-worker-controller.sh | 1 + 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index 7e52ff65..98827cc5 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -15,14 +15,22 @@ if [[ ${1:-} == --env ]]; then shift 2 fi selected_environment=$environment +health_testing=${CI_FLEET_TESTING-} +health_root_prefix=${CI_FLEET_ROOT_PREFIX-} +health_docker_socket=${CI_FLEET_DOCKER_SOCKET-} +health_bootstrap=${CI_FLEET_HEALTH_BOOTSTRAP-} health_suppress_delivery=${CI_FLEET_HEALTH_SUPPRESS_DELIVERY-} while IFS= read -r variable; do unset "$variable"; done < <(compgen -A variable CI_FLEET_) +[[ -z $health_testing ]] || export CI_FLEET_TESTING=$health_testing +[[ -z $health_root_prefix ]] || export CI_FLEET_ROOT_PREFIX=$health_root_prefix +[[ -z $health_docker_socket ]] || export CI_FLEET_DOCKER_SOCKET=$health_docker_socket +[[ -z $health_bootstrap ]] || export CI_FLEET_HEALTH_BOOTSTRAP=$health_bootstrap +[[ -z $health_suppress_delivery ]] || export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery if [[ -r $selected_environment ]]; then set -a # shellcheck disable=SC1090 . "$selected_environment" set +a fi -[[ -z $health_suppress_delivery ]] || export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery use_local_docker exec python3 "$repo_root/scripts/health.py" "${args[@]}" "$@" diff --git a/scripts/test-install-worker-controller.sh b/scripts/test-install-worker-controller.sh index 11d96935..cce7fd89 100755 --- a/scripts/test-install-worker-controller.sh +++ b/scripts/test-install-worker-controller.sh @@ -604,6 +604,7 @@ chmod 600 "$root/etc/ci-fleet/monitoring.env" warning_output=$tmp/warning-upgrade.out "$installer" --upgrade "${base_args[@]}" --ref "$warning_ref" >"$warning_output" 2>&1 warning_upgrade=$(<"$warning_output") +if grep -Fq 'ERROR: alternate Docker endpoints are not supported' <<<"$warning_upgrade"; then fail 'healthcheck lost its test root while clearing candidate environment'; fi grep -Fq 'WARNING disk_root' <<<"$warning_upgrade" || fail 'warning health fixture did not produce a warning result' if grep -Fq 'WARNING status_delivery' <<<"$warning_upgrade"; then fail 'ad-hoc reconciliation health check submitted duplicate status'; fi grep -Fq 'CONVERGED mode=upgrade' <<<"$warning_upgrade" || fail 'warning health result did not report convergence' From 88ab7636aa36b0e6cbfa2a2365d6875a048eb76a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 18:41:32 -0500 Subject: [PATCH 14/54] fix: preserve health operational controls --- scripts/test-healthcheck.sh | 42 +++++++++++++++++++++++++++++++++++++ scripts/validate.sh | 1 + 2 files changed, 43 insertions(+) create mode 100755 scripts/test-healthcheck.sh diff --git a/scripts/test-healthcheck.sh b/scripts/test-healthcheck.sh new file mode 100755 index 00000000..4325b7cd --- /dev/null +++ b/scripts/test-healthcheck.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" +cat >"$tmp/bin/python3" <<'EOF' +#!/usr/bin/env bash +printf 'controller=%s\n' "${CI_FLEET_CONTROLLER-}" +printf 'stale=%s\n' "${CI_FLEET_STALE-unset}" +printf 'testing=%s\n' "${CI_FLEET_TESTING-}" +printf 'root_prefix=%s\n' "${CI_FLEET_ROOT_PREFIX-}" +printf 'docker_socket=%s\n' "${CI_FLEET_DOCKER_SOCKET-}" +printf 'bootstrap=%s\n' "${CI_FLEET_HEALTH_BOOTSTRAP-}" +printf 'suppress_delivery=%s\n' "${CI_FLEET_HEALTH_SUPPRESS_DELIVERY-}" +printf 'docker_host=%s\n' "${DOCKER_HOST-}" +printf 'args=%s\n' "$*" +EOF +chmod +x "$tmp/bin/python3" +printf 'CI_FLEET_CONTROLLER=candidate\n' >"$tmp/candidate.env" + +output=$(env -i \ + "PATH=$tmp/bin:$PATH" \ + CI_FLEET_CONTROLLER=stale \ + CI_FLEET_STALE=stale \ + CI_FLEET_TESTING=1 \ + "CI_FLEET_ROOT_PREFIX=$tmp/root" \ + "CI_FLEET_DOCKER_SOCKET=$tmp/docker.sock" \ + CI_FLEET_HEALTH_BOOTSTRAP=1 \ + CI_FLEET_HEALTH_SUPPRESS_DELIVERY=1 \ + "$repo_root/scripts/healthcheck.sh" --env "$tmp/candidate.env") + +grep -Fqx 'controller=candidate' <<<"$output" +grep -Fqx 'stale=unset' <<<"$output" +grep -Fqx 'testing=1' <<<"$output" +grep -Fqx "root_prefix=$tmp/root" <<<"$output" +grep -Fqx "docker_socket=$tmp/docker.sock" <<<"$output" +grep -Fqx 'bootstrap=1' <<<"$output" +grep -Fqx 'suppress_delivery=1' <<<"$output" +grep -Fqx "docker_host=unix://$tmp/docker.sock" <<<"$output" +grep -Fqx "args=$repo_root/scripts/health.py local --monitoring-config $tmp/root/etc/ci-fleet/monitoring.env --output $tmp/root/var/lib/ci-fleet/health/latest.json" <<<"$output" +printf '%s\n' 'Healthcheck environment tests passed.' diff --git a/scripts/validate.sh b/scripts/validate.sh index c03b8ca2..4b4b185f 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -29,6 +29,7 @@ python3 scripts/desired_state.py validate-engine-capabilities --manifest engine- python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null scripts/test-capacity-preflight.sh +scripts/test-healthcheck.sh scripts/test-install-worker-controller.sh scripts/test-install-status-receiver.sh From e20c8946ed3a6b7ed3d0cd95ba243c6acc164857 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 18:56:32 -0500 Subject: [PATCH 15/54] fix: validate network policy inputs before drain --- scripts/apply-docker-network-policy.sh | 45 +++++++++------- scripts/desired_state.py | 2 + scripts/test_apply_docker_network_policy.py | 60 +++++++++++++++++++++ 3 files changed, 87 insertions(+), 20 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index aca885a8..369e1927 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -279,7 +279,31 @@ PY [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' managed_daemon=$work_dir/daemon.json.managed - cp -p "$daemon_config" "$managed_daemon" + removal_daemon=$work_dir/daemon.json.removal + python3 - "$daemon_config" "$state_file" "$managed_daemon" "$removal_daemon" 2>/dev/null <<'PY' || { +import json, sys +daemon_path, state_path, managed_path, removal_path = sys.argv[1:] +try: + current = json.load(open(daemon_path, encoding="utf-8")) + state = json.load(open(state_path, encoding="utf-8")) + if not isinstance(current, dict): + raise ValueError +except (OSError, json.JSONDecodeError, ValueError): + raise SystemExit(1) +with open(managed_path, "w", encoding="utf-8") as handle: + json.dump(current, handle, indent=2, sort_keys=True) + handle.write("\n") +if state["prior_default_address_pools_present"]: + current["default-address-pools"] = state["prior_default_address_pools"] +else: + current.pop("default-address-pools", None) +with open(removal_path, "w", encoding="utf-8") as handle: + json.dump(current, handle, indent=2, sort_keys=True) + handle.write("\n") +PY + rm -rf "$work_dir" + die 'managed daemon.json is not a valid JSON object' + } managed_mode=$(stat -c %a "$daemon_config") managed_gid=$(stat -c %g "$daemon_config") @@ -335,25 +359,6 @@ PY trap removal_on_exit EXIT set_verified_generation "" || { removal_failure='failed to mark network-policy verification pending'; exit 2; } - removal_daemon=$work_dir/daemon.json.removal - python3 - "$daemon_config" "$state_file" "$removal_daemon" <<'PY' || { removal_failure='failed to stage network-policy removal'; exit 2; } -import json, sys -daemon_path, state_path, output_path = sys.argv[1:] -try: - current = json.load(open(daemon_path, encoding="utf-8")) - state = json.load(open(state_path, encoding="utf-8")) - if not isinstance(current, dict): - raise ValueError -except (OSError, json.JSONDecodeError, ValueError): - raise SystemExit(1) -if state["prior_default_address_pools_present"]: - current["default-address-pools"] = state["prior_default_address_pools"] -else: - current.pop("default-address-pools", None) -with open(output_path, "w", encoding="utf-8") as handle: - json.dump(current, handle, indent=2, sort_keys=True) - handle.write("\n") -PY if [[ "$prior_present" == false ]] && python3 - "$removal_daemon" <<'PY' import json, sys raise SystemExit(bool(json.load(open(sys.argv[1], encoding="utf-8")))) diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 65ce4b0d..7dd9d29c 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -223,6 +223,8 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: network = ipaddress.ip_network(base, strict=True) except ValueError as exc: raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE: malformed CIDR {base!r}") from exc + if network.version != 4: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE: must be IPv4") try: size = int(size_str) except ValueError as exc: diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index d7cfab05..bd9e8784 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -110,6 +110,15 @@ def test_rejects_mismatched_pool_size(self) -> None: with self.assertRaisesRegex(ValueError, "must be an integer"): render_docker_daemon_config(env) + def test_rejects_ipv6_pool(self) -> None: + env = { + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "1", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE": "2001:db8::/29", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE": "29", + } + with self.assertRaisesRegex(ValueError, "must be IPv4"): + render_docker_daemon_config(env) + def test_not_configured_returns_empty(self) -> None: # No network policy rendered -> no pools key. daemon = render_docker_daemon_config({}) @@ -1498,6 +1507,57 @@ def test_restart_failure_evidence_excludes_command_output(self) -> None: self.assertNotIn("credential", combined) self.assertNotIn("restart-secret-error", combined) + def test_removal_rejects_non_object_daemon_before_copy_or_commands(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-non-object-removal" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + prior_state = state_file.read_bytes() + daemon.write_text("[]\n", encoding="utf-8") + copy_marker = Path(self.tmp) / "non-object-copy.marker" + command_markers = [] + for name in ("drain.sh", "restart.sh", "resume.sh", "health.sh"): + marker = Path(self.tmp) / f"non-object-{name}.marker" + command_markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + fake_bin = Path(self.tmp) / "non-object-bin" + fake_bin.mkdir() + fake_cp = fake_bin / "cp" + fake_cp.write_text( + "#!/usr/bin/env bash\n" + f"[[ ${{2:-}} != {daemon} ]] || touch {copy_marker}\n" + f"exec {shutil.which('cp')} \"$@\"\n", + encoding="utf-8", + ) + fake_cp.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(no_policy_env), + ], + capture_output=True, + text=True, + env=self._env(PATH=f"{fake_bin}:{os.environ['PATH']}"), + timeout=30, + ) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("managed daemon.json is not a valid JSON object", removed.stderr) + self.assertEqual(daemon.read_text(encoding="utf-8"), "[]\n") + self.assertEqual(state_file.read_bytes(), prior_state) + self.assertFalse(copy_marker.exists()) + self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_removal_restores_prior_pools_and_preserves_current_unrelated_keys(self) -> None: prior_pools = [{"base": "192.0.2.0/24", "size": 28}] daemon = self._write_daemon(json.dumps({"default-address-pools": prior_pools, "icc": False})) From e789a8a19599fc3818c3969d49bbe414817d4d56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nick=E2=80=99s=20Hermes?= <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:04:11 -0500 Subject: [PATCH 16/54] fix: verify capacity after network policy removal --- scripts/apply-docker-network-policy.sh | 5 ++- scripts/test_apply_docker_network_policy.py | 34 ++++++++++++++++++++- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 369e1927..d5664d38 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -120,9 +120,7 @@ validate_command CI_FLEET_DOCKER_DRAIN_COMMAND "$drain_command" validate_command CI_FLEET_DOCKER_RESTART_COMMAND "$restart_command" validate_command CI_FLEET_CONTROLLER_RESUME_COMMAND "$resume_command" validate_command CI_FLEET_HEALTH_CHECK_COMMAND "$health_command" -if [[ "$removing" == false ]]; then - validate_command CI_FLEET_DOCKER_NETWORK_PROBE "$probe_command" -fi +validate_command CI_FLEET_DOCKER_NETWORK_PROBE "$probe_command" # Serialize with installer mutations using the installer's host-local lock. lock_file=${CI_FLEET_INSTALLER_LOCK:-${CI_FLEET_ROOT_PREFIX:-}/run/ci-fleet-installer.lock} @@ -381,6 +379,7 @@ PY cmp -s "$removal_daemon" "$daemon_config" || { removal_failure='failed to verify prior network-policy key state'; exit 2; } fi run_command "$restart_command" "$daemon_dir" || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } + run_command "$probe_command" || { removal_failure='capacity probe failed after network-policy removal'; exit 2; } run_command "$resume_command" --env "$env_file" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } run_health "$env_file" || { removal_failure='health check failed after network-policy removal'; exit 2; } diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index bd9e8784..20868b33 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -1686,6 +1686,38 @@ def test_managed_policy_removal_replaces_daemon_atomically(self) -> None: self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), json.loads(prior)) self.assertEqual(daemon.stat().st_mode & 0o777, 0o640) + def test_removal_probe_failure_rolls_back_before_resume_health_or_marker_clear(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-probe-failure" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + state = state_file.read_bytes() + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + command_log = Path(self.tmp) / "removal-probe-failure.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + probe = Path(self.tmp) / "probe.sh" + probe.write_text(f"#!/usr/bin/env bash\necho probe >> {command_log}\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + self.assertEqual(state_file.read_bytes(), state) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "restart", "resume", "health"], + ) + def test_successful_removal_restarts_resumes_then_checks_health(self) -> None: self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-removal-resume" @@ -1747,7 +1779,7 @@ def test_managed_policy_removal_restores_original_state_and_clears_only_marker(s self.assertEqual(removed.returncode, 0, removed.stderr) self.assertEqual(removed.stdout, "NETWORK_POLICY_REMOVED\n") - self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain.sh", "restart.sh", "health.sh"]) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain.sh", "restart.sh", "probe.sh", "health.sh"]) self.assertEqual(daemon.exists(), prior_present) if prior_present: self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), json.loads(prior)) From a0fb1a58d12dfbdda9491c5900d2d0327911be26 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:43:00 -0500 Subject: [PATCH 17/54] fix: enforce trusted network policy paths --- scripts/apply-docker-network-policy.sh | 115 ++++++-- scripts/test_apply_docker_network_policy.py | 282 +++++++++++++++++++- 2 files changed, 375 insertions(+), 22 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index d5664d38..5cb3054d 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -77,8 +77,7 @@ env_file=$work_dir/ci-fleet.env count=$(awk -F= '$1 == "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" {print substr($0, index($0, "=") + 1)}' "$env_file") removing=false if [[ -z "$count" || "$count" == "0" ]]; then - state_file=${checkpoint_dir:+$checkpoint_dir/docker-network-policy.json} - if [[ -z "$state_file" || ! -e "$state_file" ]]; then + if [[ -z "$checkpoint_dir" ]]; then printf 'NETWORK_POLICY_NOOP\n' exit 0 fi @@ -96,12 +95,80 @@ probe_command=${CI_FLEET_DOCKER_NETWORK_PROBE:-} health_command=${CI_FLEET_HEALTH_CHECK_COMMAND:-} command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} +validate_trusted_path() { + local name=$1 path=$2 kind=$3 allow_absent=${4:-false} create=${5:-false} + python3 - "$path" "$kind" "$allow_absent" "$create" "$testing" "${CI_FLEET_ROOT_PREFIX:-}" <<'PY' || +import errno, os, stat, sys + +path, kind, allow_absent, create, testing, root_prefix = sys.argv[1:] +expected_owner = os.getuid() if testing == "1" else 0 +anchor = os.path.realpath(root_prefix) if testing == "1" and root_prefix else "/" +try: + if not path.startswith("/") or os.path.normpath(path) != path or os.path.realpath(path) != path: + raise ValueError + if os.path.commonpath((anchor, path)) != anchor: + raise ValueError + metadata = os.lstat(path) + exists = True +except FileNotFoundError: + if allow_absent != "true": + raise SystemExit(1) + exists = False +except (OSError, ValueError): + raise SystemExit(1) + +if exists: + if kind == "executable" and (not stat.S_ISREG(metadata.st_mode) or not os.access(path, os.X_OK)): + raise SystemExit(1) + if kind == "regular" and not stat.S_ISREG(metadata.st_mode): + raise SystemExit(1) + if kind == "checkpoint" and (not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700): + raise SystemExit(1) +current = os.path.dirname(path) + +while True: + try: + metadata = os.lstat(current) + except OSError: + raise SystemExit(1) + if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != expected_owner or metadata.st_mode & 0o022: + raise SystemExit(1) + if current == anchor: + break + current = os.path.dirname(current) + +if exists: + metadata = os.lstat(path) + if metadata.st_uid != expected_owner or metadata.st_mode & 0o022: + raise SystemExit(1) +elif create == "true": + parent, leaf = os.path.split(path) + parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + try: + os.mkdir(leaf, 0o700, dir_fd=parent_fd) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + fd = os.open(leaf, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd) + try: + metadata = os.fstat(fd) + if metadata.st_uid != expected_owner or stat.S_IMODE(metadata.st_mode) != 0o700: + raise OSError + finally: + os.close(fd) + except OSError: + raise SystemExit(1) + finally: + os.close(parent_fd) +PY + die "$name must be a trusted root-owned path" +} + validate_command() { - local name=$1 path=$2 canonical + local name=$1 path=$2 [[ -n "$path" ]] || die "$name is required when network policy is managed" - [[ "$path" == /* && -f "$path" && ! -L "$path" && -x "$path" ]] || die "$name must be an absolute canonical regular executable path" - canonical=$(readlink -f -- "$path") || die "$name must be an absolute canonical regular executable path" - [[ "$path" == "$canonical" ]] || die "$name must be an absolute canonical regular executable path" + validate_trusted_path "$name" "$path" executable } run_command() { @@ -116,6 +183,13 @@ run_health() { [[ "$command_timeout" =~ ^[1-9][0-9]*$ ]] || die 'CI_FLEET_COMMAND_TIMEOUT_SECONDS must be a positive integer' [[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' +validate_trusted_path CI_FLEET_DOCKER_DAEMON_CONFIG "$daemon_config" regular true +validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true +checkpoint_state=$checkpoint_dir/docker-network-policy.json +if [[ "$removing" == true && ( ! -e "$checkpoint_dir" || ! -e "$checkpoint_state" ) ]]; then + printf 'NETWORK_POLICY_NOOP\n' + exit 0 +fi validate_command CI_FLEET_DOCKER_DRAIN_COMMAND "$drain_command" validate_command CI_FLEET_DOCKER_RESTART_COMMAND "$restart_command" validate_command CI_FLEET_CONTROLLER_RESUME_COMMAND "$resume_command" @@ -134,6 +208,14 @@ else flock -n 9 || die 'another ci-fleet installer or drift check is already running' fi +validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true true +exec 8<"$checkpoint_dir" +checkpoint_path_is_pinned() { + [[ ! -L "$checkpoint_dir" && $(readlink -f /proc/self/fd/8) == "$checkpoint_dir" ]] +} +checkpoint_path_is_pinned || die 'checkpoint directory must remain a trusted root-owned path' +state_file=/proc/self/fd/8/docker-network-policy.json + # Snapshot the installer's authoritative pre-transaction environment while # holding its lock. Rollback must not validate against the rejected candidate. installed_env=${CI_FLEET_ROOT_PREFIX:-}/etc/ci-fleet/ci-fleet.env @@ -146,17 +228,12 @@ install -m 0600 -- "$installed_env" "$prior_env" checkpoint_owner=0 [[ "$testing" != 1 ]] || checkpoint_owner=$(id -u) -if [[ -e "$checkpoint_dir" || -L "$checkpoint_dir" ]]; then - [[ -d "$checkpoint_dir" && ! -L "$checkpoint_dir" && $(stat -c %u "$checkpoint_dir") == "$checkpoint_owner" && $(stat -c %a "$checkpoint_dir") == 700 ]] || - die "checkpoint directory must be owned by root with mode 0700: $checkpoint_dir" - checkpoint_file=$checkpoint_dir/docker-network-policy.json - if [[ -e "$checkpoint_file" || -L "$checkpoint_file" ]]; then - [[ -f "$checkpoint_file" && ! -L "$checkpoint_file" && $(stat -c %u "$checkpoint_file") == "$checkpoint_owner" && $(stat -c %a "$checkpoint_file") == 600 ]] || - die "checkpoint files must be owned by root with mode 0600: $checkpoint_file" - fi - [[ ! -e "$checkpoint_dir/daemon.json" && ! -L "$checkpoint_dir/daemon.json" ]] || - die 'network-policy checkpoint state is invalid' +if [[ -e "$state_file" || -L "$state_file" ]]; then + [[ -f "$state_file" && ! -L "$state_file" && $(stat -c %u "$state_file") == "$checkpoint_owner" && $(stat -c %a "$state_file") == 600 ]] || + die "checkpoint files must be owned by root with mode 0600: $checkpoint_dir/docker-network-policy.json" fi +[[ ! -e "/proc/self/fd/8/daemon.json" && ! -L "/proc/self/fd/8/daemon.json" ]] || + die 'network-policy checkpoint state is invalid' [[ ! -L "$daemon_config" ]] || die "daemon.json must not be a symlink: $daemon_config" @@ -212,6 +289,7 @@ PY } set_verified_generation() { + checkpoint_path_is_pinned || return 1 python3 - "$state_file" "${1:-}" <<'PY' import json, os, sys, tempfile path, generation = sys.argv[1:] @@ -309,6 +387,7 @@ PY rm -rf "$work_dir" die 'drain command failed before network-policy removal' fi + checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy removal' removal_failure='network-policy removal interrupted' # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below @@ -386,6 +465,7 @@ PY # Marker deletion commits removal. Ignore catchable signals across the atomic # unlink so failure still rolls back and success cannot leave a stale marker. trap '' INT TERM + checkpoint_path_is_pinned || { removal_failure='checkpoint directory changed during network-policy removal'; exit 2; } if ! rm -f "$state_file"; then removal_failure='failed to clear network-policy managed marker' exit 2 @@ -443,7 +523,6 @@ with open(staging_path, "w", encoding="utf-8") as handle: os.chmod(staging_path, 0o644) PY -state_file=$checkpoint_dir/docker-network-policy.json managed_before=false prior_verified_generation= if [[ -e "$state_file" ]]; then @@ -544,6 +623,7 @@ if ! run_command "$drain_command"; then rm -rf "$work_dir" die "drain command failed before network-policy apply" fi +checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy apply' trap 'exit 130' INT trap 'exit 143' TERM trap rollback_on_exit EXIT @@ -551,7 +631,6 @@ trap rollback_on_exit EXIT # Record the original host state before the first managed mutation. Re-applying # policy keeps this baseline and uses the temp copy above for transaction rollback. if [[ "$managed_before" == false ]]; then - install -d -m 0700 "$checkpoint_dir" python3 - "$state_file" "$had_prior" "$daemon_mode" "$backup_dir/$backup_name" <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } import json, os, sys, tempfile path = sys.argv[1] diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 20868b33..b25c5ad3 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -405,7 +405,7 @@ def test_post_drain_state_failure_resumes_prior_controller(self) -> None: python = fake_bin / "python3" python.write_text( "#!/usr/bin/env bash\n" - f"[[ ${{2:-}} != {checkpoint}/docker-network-policy.json ]] || exit 1\n" + "[[ ${2:-} != /proc/self/fd/*/docker-network-policy.json ]] || exit 1\n" f"exec {shutil.which('python3')} \"$@\"\n", encoding="utf-8", ) @@ -481,6 +481,116 @@ def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None self.assertEqual(daemon.read_bytes(), prior) self.assertFalse(drain_marker.exists()) + def test_creates_and_validates_checkpoint_before_drain(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-before-drain" + drain_marker = Path(self.tmp) / "checkpoint-before-drain.marker" + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"[[ -d {checkpoint} && ! -L {checkpoint} ]] || exit 1\n" + f"touch {drain_marker}\n", + encoding="utf-8", + ) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(drain_marker.exists()) + self.assertEqual(checkpoint.stat().st_mode & 0o777, 0o700) + + def test_checkpoint_swap_between_check_and_state_write_does_not_redirect_state(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-check-use-race" + displaced = Path(self.tmp) / "checkpoint-check-use-original" + attacker = Path(self.tmp) / "checkpoint-check-use-attacker" + attacker.mkdir(mode=0o700) + audit_dir = Path(self.tmp) / "checkpoint-race-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os, tempfile\n" + "_mkstemp = tempfile.mkstemp\n" + "_replace = os.replace\n" + "def mkstemp(*args, **kwargs):\n" + " if kwargs.get('prefix') == '.docker-network-policy.' and not os.path.exists(os.environ['SWAP_DONE']):\n" + " open(os.environ['SWAP_DONE'], 'w').close()\n" + " os.rename(os.environ['CHECKPOINT'], os.environ['DISPLACED'])\n" + " os.symlink(os.environ['ATTACKER'], os.environ['CHECKPOINT'])\n" + " return _mkstemp(*args, **kwargs)\n" + "def replace(source, target):\n" + " if target.endswith('/docker-network-policy.json') and os.path.realpath(os.path.dirname(target)) == os.environ['ATTACKER']:\n" + " open(os.environ['RACE_MARKER'], 'w').close()\n" + " return _replace(source, target)\n" + "tempfile.mkstemp = mkstemp\n" + "os.replace = replace\n", + encoding="utf-8", + ) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env( + PYTHONPATH=str(audit_dir), + CHECKPOINT=str(checkpoint), + DISPLACED=str(displaced), + ATTACKER=str(attacker), + SWAP_DONE=str(Path(self.tmp) / "checkpoint-swap.done"), + RACE_MARKER=str(Path(self.tmp) / "checkpoint-race.marker"), + ), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse((Path(self.tmp) / "checkpoint-race.marker").exists()) + self.assertFalse((attacker / "docker-network-policy.json").exists()) + + def test_checkpoint_symlink_swap_does_not_redirect_state(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-symlink-race" + attacker = Path(self.tmp) / "attacker-checkpoint" + attacker.mkdir(mode=0o700) + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"rm -rf {checkpoint}\n" + f"ln -s {attacker} {checkpoint}\n", + encoding="utf-8", + ) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse((attacker / "docker-network-policy.json").exists()) + + def test_rejects_checkpoint_below_group_writable_parent_before_drain(self) -> None: + daemon = self._write_daemon("{}\n") + self._write_success_commands() + parent = Path(self.tmp) / "writable-checkpoint-parent" + parent.mkdir(mode=0o777) + parent.chmod(0o777) + checkpoint = parent / "checkpoint" + drain_marker = Path(self.tmp) / "writable-checkpoint-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertEqual(daemon.read_text(encoding="utf-8"), "{}\n") + self.assertFalse(drain_marker.exists()) + def test_rejects_unsafe_preexisting_checkpoint_before_drain(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "unsafe-checkpoint" @@ -493,7 +603,7 @@ def test_rejects_unsafe_preexisting_checkpoint_before_drain(self) -> None: result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) self.assertNotEqual(result.returncode, 0) - self.assertIn("checkpoint directory must be owned by root with mode 0700", result.stderr) + self.assertIn("checkpoint directory must be a trusted root-owned path", result.stderr) self.assertEqual(daemon.read_text(encoding="utf-8"), "{}\n") self.assertFalse(drain_marker.exists()) @@ -586,6 +696,170 @@ def test_policy_reapply_rejects_persistent_recovery_backup(self) -> None: self.assertEqual(daemon.read_bytes(), prior) self.assertTrue(all(not marker.exists() for marker in markers)) + def test_rejects_relative_daemon_config_before_lock(self) -> None: + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + lock = Path(self.tmp) / "network-policy.lock" + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-relative-daemon"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env( + CI_FLEET_DOCKER_DAEMON_CONFIG="relative/daemon.json", + CI_FLEET_INSTALLER_LOCK=str(lock), + ), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertFalse(lock.exists()) + + def test_rejects_fifo_daemon_config_before_lock(self) -> None: + daemon = self.daemon_dir / "daemon.json" + os.mkfifo(daemon) + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + lock = Path(self.tmp) / "network-policy.lock" + writer = subprocess.Popen(["bash", "-c", 'printf "{}\\n" >"$1"', "fifo-writer", str(daemon)]) + self.addCleanup(writer.kill) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-fifo-daemon"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + if writer.poll() is None: + writer.terminate() + writer.wait(timeout=5) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertFalse(lock.exists()) + + def test_rejects_group_writable_daemon_file_before_lock(self) -> None: + daemon = self._write_daemon("{}\n") + daemon.chmod(0o664) + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + lock = Path(self.tmp) / "network-policy.lock" + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-writable-daemon"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertFalse(lock.exists()) + + def test_rejects_group_writable_daemon_directory_before_lock(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + self.daemon_dir.chmod(0o775) + env_file = self._write_env_file(self._rendered_with_policy()) + lock = Path(self.tmp) / "network-policy.lock" + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-writable-directory"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertFalse(lock.exists()) + + def test_rejects_hook_below_group_writable_parent_before_lock(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + hook_dir = Path(self.tmp) / "writable-hook-parent" + hook_dir.mkdir(mode=0o777) + hook_dir.chmod(0o777) + hook = hook_dir / "drain.sh" + hook.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + hook.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + lock = Path(self.tmp) / "network-policy.lock" + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-writable-hook-parent"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env( + CI_FLEET_DOCKER_DRAIN_COMMAND=str(hook), + CI_FLEET_INSTALLER_LOCK=str(lock), + ), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertFalse(lock.exists()) + + def test_rejects_group_writable_injected_hook_before_lock(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + self.drain_command.chmod(0o775) + env_file = self._write_env_file(self._rendered_with_policy()) + lock = Path(self.tmp) / "network-policy.lock" + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-writable-hook"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertFalse(lock.exists()) + def test_injected_hooks_require_absolute_canonical_regular_executables(self) -> None: env_file = self._write_env_file(self._rendered_with_policy()) hook_names = ( @@ -1336,7 +1610,7 @@ def test_fsyncs_staged_file_and_daemon_directory_around_replace(self) -> None: " return _fsync(fd)\n" "def replace(source, target):\n" " with open(_log, 'a', encoding='utf-8') as handle:\n" - " handle.write(f'R {source} {target}\\n')\n" + " handle.write(f'R {os.path.realpath(source)} {os.path.realpath(target)}\\n')\n" " return _replace(source, target)\n" "os.fsync = fsync\n" "os.replace = replace\n", @@ -1951,7 +2225,7 @@ def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) fake_rm = fake_bin / "rm" fake_rm.write_text( "#!/usr/bin/env bash\n" - f"[[ ${{*: -1}} != {state_file} ]] || exit 1\n" + "[[ ${*: -1} != /proc/self/fd/*/docker-network-policy.json ]] || exit 1\n" f"exec {shutil.which('rm')} \"$@\"\n", encoding="utf-8", ) From 4b8147864b10aedb9dd2efd9b5eb405580994834 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:53:25 -0500 Subject: [PATCH 18/54] fix: retain durable network policy recovery state --- scripts/apply-docker-network-policy.sh | 50 ++++++++++- scripts/test_apply_docker_network_policy.py | 97 +++++++++++++++++++-- 2 files changed, 139 insertions(+), 8 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 5cb3054d..595ceda5 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -315,6 +315,45 @@ finally: PY } +persist_recovery() { + checkpoint_path_is_pinned || return 1 + python3 - "$state_file" "$had_prior" "$backup_dir/$backup_name" "$prior_env" <<'PY' +import os, shutil, sys, tempfile + +state_path, had_prior, daemon_source, env_source = sys.argv[1:] +parent = os.path.dirname(state_path) +staged = tempfile.mkdtemp(prefix=".recovery.", dir=parent) +try: + os.chmod(staged, 0o700) + sources = [(env_source, "prior-ci-fleet.env")] + if had_prior == "true": + sources.insert(0, (daemon_source, "daemon.json.before")) + for source, name in sources: + target = os.path.join(staged, name) + with open(source, "rb") as source_handle, open(target, "xb") as target_handle: + shutil.copyfileobj(source_handle, target_handle) + os.fchmod(target_handle.fileno(), 0o600) + target_handle.flush() + os.fsync(target_handle.fileno()) + staged_fd = os.open(staged, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(staged_fd) + finally: + os.close(staged_fd) + final = os.path.join(parent, "recovery." + os.path.basename(staged).removeprefix(".recovery.")) + os.replace(staged, final) + parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(parent_fd) + finally: + os.close(parent_fd) + print(os.path.realpath(final)) +except BaseException: + shutil.rmtree(staged, ignore_errors=True) + raise +PY +} + if [[ "$removing" == true ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || die 'network-policy checkpoint state is invalid' mapfile -t managed_state < <(python3 - "$state_file" <<'PY' @@ -597,7 +636,9 @@ rollback_daemon() { run_command "$restart_command" "$daemon_dir" || failed=1 run_command "$resume_command" --env "$prior_env" || failed=1 run_health "$prior_env" || failed=1 - if [[ "$managed_before" == true ]]; then set_verified_generation "$prior_verified_generation" || failed=1; fi + if [[ "$managed_before" == true && "$failed" == 0 ]]; then + set_verified_generation "$prior_verified_generation" || failed=1 + fi return "$failed" } @@ -613,7 +654,12 @@ rollback_on_exit() { rm -rf "$work_dir" printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 else - printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$transaction_failure" "$work_dir" >&2 + recovery_path=$(persist_recovery) || { + printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$transaction_failure" >&2 + exit "$status" + } + rm -rf "$work_dir" + printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$transaction_failure" "$recovery_path" >&2 fi exit "$status" } diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index b25c5ad3..bb5c6de6 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -465,6 +465,29 @@ def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"], prior_generation) self.assertEqual(hashlib.sha256(daemon.read_bytes()).hexdigest(), prior_generation) + def test_failed_managed_reapply_leaves_generation_unverified_when_rollback_health_fails(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-failed-rollback-generation" + self._write_success_commands() + prior_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(prior_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(prior_env.read_bytes()) + candidate = self._rendered_with_policy() + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = Path(self.tmp) / "candidate-failed-rollback.env" + candidate_env.write_text("".join(f"{key}={value}\n" for key, value in sorted(candidate.items())), encoding="utf-8") + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + (Path(self.tmp) / "health.sh").write_text("#!/usr/bin/env bash\nexit 2\n", encoding="utf-8") + (Path(self.tmp) / "health.sh").chmod(0o755) + + result = self._run(str(candidate_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertIsNone(state["verified_generation"]) + def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -1455,9 +1478,9 @@ def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) self.assertNotIn("rollback-restart-output", stdout + stderr) self.assertNotIn("rollback-health-output", stdout + stderr) - recovery_dirs = list(Path(self.tmp).glob(".ci-fleet-apply.*")) - self.assertEqual(len(recovery_dirs), 1) - self.assertEqual((recovery_dirs[0] / "prior" / "daemon.json.before").read_bytes(), prior) + recovery = Path((stdout + stderr).rstrip().rsplit("recovery data retained at ", 1)[1]) + self.assertEqual(recovery.parent, Path(self.tmp) / "checkpoint-direct") + self.assertEqual((recovery / "daemon.json.before").read_bytes(), prior) def test_rollback_failure_is_reported(self) -> None: prior = b'{"bip":"172.17.0.1/16","registry-mirrors":["https://mirror.example.invalid?token=credential"]}\n' @@ -1491,12 +1514,74 @@ def test_rollback_failure_is_reported(self) -> None: self.assertNotIn("credential", combined) self.assertNotIn("rollback-secret-error", combined) recovery = Path(combined.rstrip().rsplit("recovery data retained at ", 1)[1]) - self.addCleanup(shutil.rmtree, recovery, ignore_errors=True) - self.assertTrue(recovery.name.startswith(".ci-fleet-apply.")) - self.assertEqual((recovery / "prior" / "daemon.json.before").read_bytes(), prior) + self.assertEqual(recovery.parent, Path(self.tmp) / "checkpoint-default") + self.assertTrue(recovery.name.startswith("recovery.")) + self.assertEqual((recovery / "daemon.json.before").read_bytes(), prior) recovery_state = json.loads((Path(self.tmp) / "checkpoint-default" / "docker-network-policy.json").read_text(encoding="utf-8")) self.assertIsNone(recovery_state["verified_generation"]) + def test_failed_rollback_persists_exact_recovery_under_checkpoint(self) -> None: + prior_daemon = b'{"bip":"172.17.0.1/16"}\n' + self._write_daemon(prior_daemon.decode()) + prior_env = self.installed_env.read_bytes() + checkpoint = Path(self.tmp) / "checkpoint-durable-recovery" + restart_log = Path(self.tmp) / "durable-recovery-restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {restart_log}\n" + f"[[ $(wc -l < {restart_log}) -eq 1 ]]\n", + encoding="utf-8", + ) + restart.chmod(0o755) + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + (Path(self.tmp) / "health.sh").write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + (Path(self.tmp) / "health.sh").chmod(0o755) + audit_dir = Path(self.tmp) / "recovery-fsync-audit" + audit_dir.mkdir() + audit_log = Path(self.tmp) / "recovery-fsync.log" + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_fsync = os.fsync\n" + "_replace = os.replace\n" + "_log = os.environ['FSYNC_AUDIT_LOG']\n" + "def fsync(fd):\n" + " with open(_log, 'a', encoding='utf-8') as handle:\n" + " handle.write('F ' + os.readlink(f'/proc/self/fd/{fd}') + '\\n')\n" + " return _fsync(fd)\n" + "def replace(source, target):\n" + " with open(_log, 'a', encoding='utf-8') as handle:\n" + " handle.write(f'R {os.path.realpath(source)} {os.path.realpath(target)}\\n')\n" + " return _replace(source, target)\n" + "os.fsync = fsync\n" + "os.replace = replace\n", + encoding="utf-8", + ) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_TEMP_DIR=self.tmp, PYTHONPATH=str(audit_dir), FSYNC_AUDIT_LOG=str(audit_log)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + recovery = Path((result.stdout + result.stderr).rstrip().rsplit("recovery data retained at ", 1)[1]) + self.assertEqual(recovery.parent, checkpoint) + self.assertEqual(recovery.stat().st_mode & 0o777, 0o700) + self.assertEqual((recovery / "daemon.json.before").read_bytes(), prior_daemon) + self.assertEqual((recovery / "prior-ci-fleet.env").read_bytes(), prior_env) + self.assertEqual((recovery / "daemon.json.before").stat().st_mode & 0o777, 0o600) + self.assertEqual((recovery / "prior-ci-fleet.env").stat().st_mode & 0o777, 0o600) + self.assertFalse(list(Path(self.tmp).glob(".ci-fleet-apply.*"))) + audit = audit_log.read_text(encoding="utf-8").splitlines() + self.assertTrue(any(line.startswith("F ") and line.endswith("/daemon.json.before") for line in audit)) + self.assertTrue(any(line.startswith("F ") and line.endswith("/prior-ci-fleet.env") for line in audit)) + self.assertIn(f"F {checkpoint}", audit) + self.assertTrue(any(line.startswith("R ") and line.endswith(f" {recovery}") for line in audit)) + def test_post_apply_health_uses_candidate_rendered_env(self) -> None: self._write_daemon("{}\n") for name in ("restart.sh", "probe.sh"): From e5ffbdca6fb8cfc92e6701b3b552cd18ff9ee9d1 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:01:39 -0500 Subject: [PATCH 19/54] fix: resume controller after failed policy drain --- scripts/apply-docker-network-policy.sh | 39 +++++++++++++++------ scripts/test_apply_docker_network_policy.py | 39 +++++++++++++++++---- 2 files changed, 61 insertions(+), 17 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 595ceda5..a613d56c 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -226,6 +226,31 @@ installed_env_mode=$(stat -c %a "$installed_env") prior_env=$work_dir/prior-ci-fleet.env install -m 0600 -- "$installed_env" "$prior_env" +drain_failure= +# shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below +resume_after_failed_drain() { + local status=$? resume_failed=0 + trap - EXIT INT TERM + run_command "$resume_command" --env "$prior_env" || resume_failed=1 + rm -rf "$work_dir" + if ((resume_failed)); then + printf 'ERROR: %s; controller resume command failed\n' "$drain_failure" >&2 + else + printf 'ERROR: %s\n' "$drain_failure" >&2 + fi + exit "$status" +} + +drain_controller() { + local status=0 + drain_failure=$1 + trap 'exit 130' INT + trap 'exit 143' TERM + trap resume_after_failed_drain EXIT + run_command "$drain_command" || status=$? + ((status == 0)) || exit "$status" +} + checkpoint_owner=0 [[ "$testing" != 1 ]] || checkpoint_owner=$(id -u) if [[ -e "$state_file" || -L "$state_file" ]]; then @@ -422,10 +447,7 @@ PY managed_mode=$(stat -c %a "$daemon_config") managed_gid=$(stat -c %g "$daemon_config") - if ! run_command "$drain_command"; then - rm -rf "$work_dir" - die 'drain command failed before network-policy removal' - fi + drain_controller 'drain command failed before network-policy removal' checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy removal' removal_failure='network-policy removal interrupted' @@ -470,9 +492,9 @@ PY fi exit "$status" } + trap removal_on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM - trap removal_on_exit EXIT set_verified_generation "" || { removal_failure='failed to mark network-policy verification pending'; exit 2; } if [[ "$prior_present" == false ]] && python3 - "$removal_daemon" <<'PY' @@ -665,14 +687,11 @@ rollback_on_exit() { } # --- Drain after local validation/checkpointing, before mutation or restart --- -if ! run_command "$drain_command"; then - rm -rf "$work_dir" - die "drain command failed before network-policy apply" -fi +drain_controller 'drain command failed before network-policy apply' checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy apply' +trap rollback_on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM -trap rollback_on_exit EXIT # Record the original host state before the first managed mutation. Re-applying # policy keeps this baseline and uses the temp copy above for transaction rollback. diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index bb5c6de6..5c6836e1 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -423,23 +423,48 @@ def test_post_drain_state_failure_resumes_prior_controller(self) -> None: self.assertEqual(daemon.read_bytes(), prior) self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "health"]) - def test_drain_failure_never_resumes_controller(self) -> None: + def test_drain_timeout_resumes_prior_controller_and_preserves_failure(self) -> None: self._write_daemon("{}\n") env_file = self._write_env_file(self._rendered_with_policy()) - self.drain_command.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") - resume_marker = Path(self.tmp) / "resume-after-failed-drain.marker" + command_log = Path(self.tmp) / "failed-drain.log" + self.drain_command.write_text( + f"#!/usr/bin/env bash\necho drain >> {command_log}\nsleep 10\n", + encoding="utf-8", + ) resume = Path(self.tmp) / "resume.sh" - resume.write_text(f"#!/usr/bin/env bash\ntouch {resume_marker}\n", encoding="utf-8") + resume.write_text( + "#!/usr/bin/env bash\n" + f"echo resume >> {command_log}\n" + 'grep -Fqx "ENV_GENERATION=prior" "$2" || exit 2\n' + "exit 1\n", + encoding="utf-8", + ) resume.chmod(0o755) for name in ("restart.sh", "probe.sh", "health.sh"): command = Path(self.tmp) / name command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") command.chmod(0o755) - result = self._run(str(env_file)) + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-failed-drain"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_COMMAND_TIMEOUT_SECONDS="1"), + timeout=15, + ) - self.assertNotEqual(result.returncode, 0) - self.assertFalse(resume_marker.exists()) + self.assertEqual(result.returncode, 124) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume"]) + self.assertEqual( + result.stderr, + "ERROR: drain command failed before network-policy apply; controller resume command failed\n", + ) def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None: daemon = self._write_daemon("{}\n") From 12707b118ba2e5e62ec25dc38f77e311156a6217 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:13:23 -0500 Subject: [PATCH 20/54] fix: persist interrupted policy removal state --- scripts/apply-docker-network-policy.sh | 64 ++++++++-- scripts/test_apply_docker_network_policy.py | 131 +++++++++++++++++++- 2 files changed, 183 insertions(+), 12 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index a613d56c..88e61869 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -315,10 +315,13 @@ PY set_verified_generation() { checkpoint_path_is_pinned || return 1 - python3 - "$state_file" "${1:-}" <<'PY' + python3 - "$state_file" "${1:-}" "${2:-}" <<'PY' import json, os, sys, tempfile -path, generation = sys.argv[1:] +path, generation, action = sys.argv[1:] state = json.load(open(path, encoding="utf-8")) +if action == "clear-removal": + state.pop("phase", None) + state.pop("removal_managed_default_address_pools", None) state["verified_generation"] = generation or None fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) try: @@ -340,6 +343,37 @@ finally: PY } +set_removal_pending() { + checkpoint_path_is_pinned || return 1 + python3 - "$state_file" "$1" <<'PY' +import json, os, sys, tempfile +path, managed_path = sys.argv[1:] +state = json.load(open(path, encoding="utf-8")) +if state.get("phase") != "removal-pending": + managed = json.load(open(managed_path, encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = managed["default-address-pools"] +state["verified_generation"] = None +fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(state, handle, indent=2, sort_keys=True) + handle.write("\n") + os.fchmod(handle.fileno(), 0o600) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + directory_fd = os.open(os.path.dirname(path), os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) +finally: + if os.path.exists(tmp): + os.unlink(tmp) +PY +} + persist_recovery() { checkpoint_path_is_pinned || return 1 python3 - "$state_file" "$had_prior" "$backup_dir/$backup_name" "$prior_env" <<'PY' @@ -385,7 +419,12 @@ if [[ "$removing" == true ]]; then import json, re, sys state = json.load(open(sys.argv[1], encoding="utf-8")) required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} -if set(state) not in (required, required | {"verified_generation"}) or state["managed"] is not True: +pending = {"phase", "removal_managed_default_address_pools"} +schemas = (required, required | {"verified_generation"}, required | pending, required | pending | {"verified_generation"}) +if set(state) not in schemas or state["managed"] is not True: + raise SystemExit(1) +is_pending = "phase" in state +if is_pending and (state["phase"] != "removal-pending" or not isinstance(state["removal_managed_default_address_pools"], list)): raise SystemExit(1) generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): @@ -407,9 +446,10 @@ elif mode is not None: print("true" if state["prior_present"] else "false") print(mode or "") print(generation or "") +print("true" if is_pending else "false") PY ) || die 'network-policy checkpoint state is invalid' - [[ ${#managed_state[@]} == 3 ]] || die 'network-policy checkpoint state is invalid' + [[ ${#managed_state[@]} == 4 ]] || die 'network-policy checkpoint state is invalid' prior_present=${managed_state[0]} prior_mode=${managed_state[1]} prior_verified_generation=${managed_state[2]} @@ -428,10 +468,16 @@ try: state = json.load(open(state_path, encoding="utf-8")) if not isinstance(current, dict): raise ValueError -except (OSError, json.JSONDecodeError, ValueError): + if state.get("phase") == "removal-pending": + managed_pools = state["removal_managed_default_address_pools"] + else: + managed_pools = current["default-address-pools"] +except (OSError, json.JSONDecodeError, KeyError, ValueError): raise SystemExit(1) +managed = dict(current) +managed["default-address-pools"] = managed_pools with open(managed_path, "w", encoding="utf-8") as handle: - json.dump(current, handle, indent=2, sort_keys=True) + json.dump(managed, handle, indent=2, sort_keys=True) handle.write("\n") if state["prior_default_address_pools_present"]: current["default-address-pools"] = state["prior_default_address_pools"] @@ -476,7 +522,9 @@ PY run_command "$restart_command" "$daemon_dir" || failed=1 run_command "$resume_command" --env "$prior_env" || failed=1 run_health "$prior_env" || failed=1 - set_verified_generation "$prior_verified_generation" || failed=1 + if ((failed == 0)); then + set_verified_generation "$prior_verified_generation" clear-removal || failed=1 + fi return "$failed" } # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below @@ -495,7 +543,7 @@ PY trap removal_on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM - set_verified_generation "" || { removal_failure='failed to mark network-policy verification pending'; exit 2; } + set_removal_pending "$managed_daemon" || { removal_failure='failed to persist network-policy removal state'; exit 2; } if [[ "$prior_present" == false ]] && python3 - "$removal_daemon" <<'PY' import json, sys diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 5c6836e1..c9505397 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -2070,6 +2070,122 @@ def test_managed_policy_removal_replaces_daemon_atomically(self) -> None: self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), json.loads(prior)) self.assertEqual(daemon.stat().st_mode & 0o777, 0o640) + def test_interrupted_removal_retry_uses_persisted_managed_pools_for_rollback(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-interrupted-removal" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = managed_pools + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + daemon.write_text('{"live-restore":true}\n', encoding="utf-8") + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual( + json.loads(daemon.read_text(encoding="utf-8")), + {"default-address-pools": managed_pools, "live-restore": True}, + ) + rolled_back_state = json.loads(state_file.read_text(encoding="utf-8")) + self.assertNotIn("phase", rolled_back_state) + self.assertNotIn("removal_managed_default_address_pools", rolled_back_state) + + def test_removal_pending_state_is_durable_before_daemon_mutation(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-durable-removal-pending" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + state_file = checkpoint / "docker-network-policy.json" + audit_dir = Path(self.tmp) / "removal-pending-audit" + audit_dir.mkdir() + audit_log = Path(self.tmp) / "removal-pending.log" + (audit_dir / "sitecustomize.py").write_text( + "import json, os\n" + "_fsync = os.fsync\n" + "_replace = os.replace\n" + "_log = os.environ['AUDIT_LOG']\n" + "def record(value):\n" + " with open(_log, 'a', encoding='utf-8') as handle: handle.write(value + '\\n')\n" + "def fsync(fd):\n" + " record('F ' + os.path.realpath(f'/proc/self/fd/{fd}'))\n" + " return _fsync(fd)\n" + "def replace(source, target):\n" + " target = os.path.realpath(target)\n" + " if target == os.environ['DAEMON']:\n" + " state = json.load(open(os.environ['STATE'], encoding='utf-8'))\n" + " assert state['phase'] == 'removal-pending'\n" + " assert state['removal_managed_default_address_pools'] == json.loads(os.environ['POOLS'])\n" + " record('R ' + target)\n" + " return _replace(source, target)\n" + "os.fsync = fsync\n" + "os.replace = replace\n", + encoding="utf-8", + ) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, + text=True, + env=self._env( + PYTHONPATH=str(audit_dir), + AUDIT_LOG=str(audit_log), + DAEMON=str(daemon), + STATE=str(state_file), + POOLS=json.dumps(managed_pools), + ), + timeout=30, + ) + + self.assertEqual(removed.returncode, 0, removed.stderr) + events = audit_log.read_text(encoding="utf-8").splitlines() + state_replace = events.index(f"R {state_file}") + daemon_replace = events.index(f"R {daemon}") + self.assertTrue(any(event.startswith("F ") and ".docker-network-policy." in event for event in events[:state_replace])) + self.assertIn(f"F {checkpoint}", events[state_replace + 1 : daemon_replace]) + + def test_uncertain_removal_rollback_retains_pending_state(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-uncertain-removal" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + self.installed_env.write_bytes(policy_env.read_bytes()) + restart_log = Path(self.tmp) / "uncertain-removal-restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {restart_log}\n" + f"[[ $(wc -l < {restart_log}) -eq 1 ]]\n", + encoding="utf-8", + ) + restart.chmod(0o755) + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertEqual(state["phase"], "removal-pending") + self.assertEqual(state["removal_managed_default_address_pools"], managed_pools) + def test_removal_probe_failure_rolls_back_before_resume_health_or_marker_clear(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-removal-probe-failure" @@ -2102,7 +2218,7 @@ def test_removal_probe_failure_rolls_back_before_resume_health_or_marker_clear(s ["drain", "restart", "probe", "restart", "resume", "health"], ) - def test_successful_removal_restarts_resumes_then_checks_health(self) -> None: + def test_successful_removal_runs_runtime_checks_before_deleting_marker(self) -> None: self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-removal-resume" self._write_success_commands() @@ -2110,19 +2226,26 @@ def test_successful_removal_restarts_resumes_then_checks_health(self) -> None: applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) self.assertEqual(applied.returncode, 0, applied.stderr) self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" no_policy_env = Path(self.tmp) / "no-policy-removal-resume.env" no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") command_log = Path(self.tmp) / "removal-resume.log" self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") - for name in ("restart", "resume", "health"): + for name in ("restart", "probe", "resume", "health"): command = Path(self.tmp) / f"{name}.sh" - command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.write_text( + "#!/usr/bin/env bash\n" + f"grep -Fq '\"phase\": \"removal-pending\"' {state_file} || exit 2\n" + f"echo {name} >> {command_log}\n", + encoding="utf-8", + ) command.chmod(0o755) removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) self.assertEqual(removed.returncode, 0, removed.stderr) - self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "health"]) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "resume", "health"]) + self.assertFalse(state_file.exists()) def test_managed_policy_removal_restores_original_state_and_clears_only_marker(self) -> None: for prior_present in (False, True): From 89cf008ff98a7be7421f2bff06c1d6ba40367dad Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:32:37 -0500 Subject: [PATCH 21/54] fix: separate policy checkpoint and daemon paths --- scripts/apply-docker-network-policy.sh | 14 ++++++++--- scripts/test_apply_docker_network_policy.py | 28 +++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 88e61869..b17a0e19 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -196,6 +196,17 @@ validate_command CI_FLEET_CONTROLLER_RESUME_COMMAND "$resume_command" validate_command CI_FLEET_HEALTH_CHECK_COMMAND "$health_command" validate_command CI_FLEET_DOCKER_NETWORK_PROBE "$probe_command" +checkpoint_path_is_pinned() { + [[ ! -L "$checkpoint_dir" && $(readlink -f /proc/self/fd/8) == "$checkpoint_dir" ]] +} +if [[ -d "$checkpoint_dir" ]]; then + exec 8<"$checkpoint_dir" + checkpoint_path_is_pinned || die 'checkpoint directory must remain a trusted root-owned path' + [[ $(readlink -m /proc/self/fd/8/daemon.json) != "$daemon_config" ]] || + die 'daemon config and checkpoint entry must be separate paths' + exec 8<&- +fi + # Serialize with installer mutations using the installer's host-local lock. lock_file=${CI_FLEET_INSTALLER_LOCK:-${CI_FLEET_ROOT_PREFIX:-}/run/ci-fleet-installer.lock} if [[ -n ${CI_FLEET_INSTALLER_LOCK_FD:-} ]]; then @@ -210,9 +221,6 @@ fi validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true true exec 8<"$checkpoint_dir" -checkpoint_path_is_pinned() { - [[ ! -L "$checkpoint_dir" && $(readlink -f /proc/self/fd/8) == "$checkpoint_dir" ]] -} checkpoint_path_is_pinned || die 'checkpoint directory must remain a trusted root-owned path' state_file=/proc/self/fd/8/docker-network-policy.json diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index c9505397..fae73b85 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -744,6 +744,34 @@ def test_policy_reapply_rejects_persistent_recovery_backup(self) -> None: self.assertEqual(daemon.read_bytes(), prior) self.assertTrue(all(not marker.exists() for marker in markers)) + def test_rejects_absent_daemon_config_in_checkpoint_before_lock_or_drain(self) -> None: + self.daemon_dir.chmod(0o700) + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + lock = Path(self.tmp) / "network-policy.lock" + drain_marker = Path(self.tmp) / "drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(self.daemon_dir), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("daemon config and checkpoint entry must be separate paths", result.stderr) + self.assertFalse((self.daemon_dir / "daemon.json").exists()) + self.assertFalse(lock.exists()) + self.assertFalse(drain_marker.exists()) + def test_rejects_relative_daemon_config_before_lock(self) -> None: self._write_success_commands() env_file = self._write_env_file(self._rendered_with_policy()) From e76fcae85cd7cb502ee673831945a05f28fe883e Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:42:35 -0500 Subject: [PATCH 22/54] fix: validate rendered network policy capacity --- scripts/apply-docker-network-policy.sh | 42 ++++++++--------- scripts/desired_state.py | 12 ++++- scripts/test_apply_docker_network_policy.py | 52 +++++++++++++++++++++ 3 files changed, 82 insertions(+), 24 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index b17a0e19..d02a86b6 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -86,6 +86,25 @@ else [[ -n "$checkpoint_dir" ]] || die '--checkpoint is required when a network policy is configured' fi +# Validate the rendered policy before acquiring the installer lock. +if [[ "$removing" == false ]]; then + desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" 2>/dev/null <<'PY' +import json, sys +env_path, scripts_dir = sys.argv[1], sys.argv[2] +sys.path.insert(0, scripts_dir) +values = {} +with open(env_path, encoding="utf-8") as handle: + for line in handle: + line = line.rstrip("\n") + if "=" in line and line: + key, _, value = line.partition("=") + values[key] = value +from desired_state import render_docker_daemon_config +print(json.dumps(render_docker_daemon_config(values))) +PY + ) || die "daemon policy rendering failed" +fi + # --- Resolve required injected commands --- daemon_config=${CI_FLEET_DOCKER_DAEMON_CONFIG:-} drain_command=${CI_FLEET_DOCKER_DRAIN_COMMAND:-} @@ -593,29 +612,6 @@ PY exit 0 fi -# --- Render desired daemon config block via shared validator --- -desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" 2>/dev/null <<'PY' -import json, os, sys -env_path, scripts_dir = sys.argv[1], sys.argv[2] -sys.path.insert(0, scripts_dir) -values = {} -with open(env_path, encoding="utf-8") as handle: - for line in handle: - line = line.rstrip("\n") - if "=" in line and line: - key, _, value = line.partition("=") - values[key] = value -from desired_state import render_docker_daemon_config -print(json.dumps(render_docker_daemon_config(values))) -PY -) || die "daemon policy rendering failed" - -# Re-check: if rendering returned empty, treat as no-op. -if [[ "$desired_pools_json" == "{}" ]]; then - printf 'NETWORK_POLICY_NOOP\n' - exit 0 -fi - # --- Stage merged daemon.json (preserve unrelated keys) --- staging_daemon="$work_dir/daemon.json" diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 7dd9d29c..246de624 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -234,7 +234,17 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: if size < network.prefixlen: raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: impossible subnet count for {base!r}") pools.append({"base": base, "size": size}) - return {"default-address-pools": [{"base": p["base"], "size": p["size"]} for p in pools]} + try: + policy = { + "default_address_pools": pools, + "networks_per_runner": int(rendered["CI_FLEET_DOCKER_NETWORKS_PER_RUNNER"]), + "reserve_subnets": int(rendered["CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"]), + } + max_runners = 0 if rendered["CI_FLEET_CONTROLLER_STATE"] == "disabled" else int(rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"]) + except (KeyError, ValueError) as exc: + raise ValueError("rendered Docker network policy fields must be present integers") from exc + validate_docker_network_policy(policy, path="rendered Docker network policy", max_runners=max_runners) + return {"default-address-pools": pools} def select_controller(config: dict[str, Any], controller_id: str) -> tuple[dict[str, Any], dict[str, Any]]: diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index fae73b85..8df7d757 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -1339,6 +1339,58 @@ def test_matching_file_without_verified_generation_runs_activation_and_marks_ver verified = json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"] self.assertEqual(verified, hashlib.sha256((self.daemon_dir / "daemon.json").read_bytes()).hexdigest()) + def test_rejects_overlapping_rendered_pools_before_lock_or_drain(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self._write_daemon(prior.decode()) + self._write_success_commands() + rendered = self._rendered_with_policy() + rendered["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_1_BASE"] = "198.51.100.128/25" + env_file = self._write_env_file(rendered) + lock = Path(self.tmp) / "network-policy.lock" + drain_marker = Path(self.tmp) / "drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-overlap"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr, "ERROR: daemon policy rendering failed\n") + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(lock.exists()) + self.assertFalse(drain_marker.exists()) + + def test_rejects_insufficient_rendered_capacity_before_lock_or_drain(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + daemon = self._write_daemon(prior.decode()) + self._write_success_commands() + rendered = self._rendered_with_policy() + rendered["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT"] = "1" + rendered["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "198.51.100.0/28" + rendered["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE"] = "29" + env_file = self._write_env_file(rendered) + lock = Path(self.tmp) / "network-policy.lock" + drain_marker = Path(self.tmp) / "drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-capacity"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr, "ERROR: daemon policy rendering failed\n") + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(lock.exists()) + self.assertFalse(drain_marker.exists()) + def test_validates_policy_before_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" From 82d02a0a79f44def6b22e70c785e194e11420635 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:50:56 -0500 Subject: [PATCH 23/54] fix: abort policy apply on daemon conflicts --- scripts/apply-docker-network-policy.sh | 6 +++++ scripts/test_apply_docker_network_policy.py | 25 +++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index d02a86b6..fd4f71b3 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -740,6 +740,12 @@ rollback_on_exit() { # --- Drain after local validation/checkpointing, before mutation or restart --- drain_controller 'drain command failed before network-policy apply' +if [[ "$had_prior" == true ]]; then + cmp -s "$backup_dir/$backup_name" "$daemon_config" || { drain_failure='daemon.json changed during network-policy apply'; exit 2; } +elif [[ -e "$daemon_config" || -L "$daemon_config" ]]; then + drain_failure='daemon.json changed during network-policy apply' + exit 2 +fi checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy apply' trap rollback_on_exit EXIT trap 'exit 130' INT diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 8df7d757..4b97529f 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -466,6 +466,31 @@ def test_drain_timeout_resumes_prior_controller_and_preserves_failure(self) -> N "ERROR: drain command failed before network-policy apply; controller resume command failed\n", ) + def test_daemon_change_during_drain_aborts_and_preserves_update(self) -> None: + daemon = self._write_daemon('{"bip":"172.17.0.1/16"}\n') + changed = b'{"bip":"172.17.0.1/16","log-level":"debug"}\n' + env_file = self._write_env_file(self._rendered_with_policy()) + checkpoint = Path(self.tmp) / "checkpoint-daemon-conflict" + command_log = Path(self.tmp) / "daemon-conflict.log" + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"echo drain >> {command_log}\n" + f"printf '%s\\n' '{{\"bip\":\"172.17.0.1/16\",\"log-level\":\"debug\"}}' > {daemon}\n", + encoding="utf-8", + ) + for name in ("restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr, "ERROR: daemon.json changed during network-policy apply\n") + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume"]) + self.assertEqual(daemon.read_bytes(), changed) + self.assertFalse((checkpoint / "docker-network-policy.json").exists()) + def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-reapply-generation" From 53b829d74c40ad9576e1dc987da97313093fec42 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:59:55 -0500 Subject: [PATCH 24/54] fix: trust rendered policy environment paths --- scripts/apply-docker-network-policy.sh | 146 ++++++++++---------- scripts/test_apply_docker_network_policy.py | 43 ++++++ 2 files changed, 114 insertions(+), 75 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index fd4f71b3..db94bab5 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -36,6 +36,76 @@ die() { exit 2 } +validate_trusted_path() { + local name=$1 path=$2 kind=$3 allow_absent=${4:-false} create=${5:-false} + python3 - "$path" "$kind" "$allow_absent" "$create" "$testing" "${CI_FLEET_ROOT_PREFIX:-}" <<'PY' || +import errno, os, stat, sys + +path, kind, allow_absent, create, testing, root_prefix = sys.argv[1:] +expected_owner = os.getuid() if testing == "1" else 0 +anchor = os.path.realpath(root_prefix) if testing == "1" and root_prefix else "/" +try: + if not path.startswith("/") or os.path.normpath(path) != path or os.path.realpath(path) != path: + raise ValueError + if os.path.commonpath((anchor, path)) != anchor: + raise ValueError + metadata = os.lstat(path) + exists = True +except FileNotFoundError: + if allow_absent != "true": + raise SystemExit(1) + exists = False +except (OSError, ValueError): + raise SystemExit(1) + +if exists: + if kind == "executable" and (not stat.S_ISREG(metadata.st_mode) or not os.access(path, os.X_OK)): + raise SystemExit(1) + if kind == "regular" and not stat.S_ISREG(metadata.st_mode): + raise SystemExit(1) + if kind == "checkpoint" and (not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700): + raise SystemExit(1) +current = os.path.dirname(path) + +while True: + try: + metadata = os.lstat(current) + except OSError: + raise SystemExit(1) + if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != expected_owner or metadata.st_mode & 0o022: + raise SystemExit(1) + if current == anchor: + break + current = os.path.dirname(current) + +if exists: + metadata = os.lstat(path) + if metadata.st_uid != expected_owner or metadata.st_mode & 0o022: + raise SystemExit(1) +elif create == "true": + parent, leaf = os.path.split(path) + parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) + try: + try: + os.mkdir(leaf, 0o700, dir_fd=parent_fd) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + fd = os.open(leaf, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd) + try: + metadata = os.fstat(fd) + if metadata.st_uid != expected_owner or stat.S_IMODE(metadata.st_mode) != 0o700: + raise OSError + finally: + os.close(fd) + except OSError: + raise SystemExit(1) + finally: + os.close(parent_fd) +PY + die "$name must be a trusted root-owned path" +} + while (($#)); do case "$1" in --env) @@ -60,13 +130,9 @@ while (($#)); do done [[ -n "$env_file" ]] || die '--env is required' -[[ -f "$env_file" && ! -L "$env_file" && -r "$env_file" ]] || die "rendered env must be a readable regular file: $env_file" -env_owner=$(stat -c %u "$env_file") expected_env_owner=0 [[ "$testing" != 1 ]] || expected_env_owner=$(id -u) -[[ "$env_owner" == "$expected_env_owner" ]] || die "rendered env has an untrusted owner: $env_file" -env_mode=$(stat -c %a "$env_file") -(( (8#$env_mode & 8#022) == 0 )) || die "rendered env must not be group/world writable: $env_file" +validate_trusted_path 'rendered env' "$env_file" regular work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") chmod 0700 "$work_dir" trap 'rm -rf "$work_dir"' EXIT @@ -114,76 +180,6 @@ probe_command=${CI_FLEET_DOCKER_NETWORK_PROBE:-} health_command=${CI_FLEET_HEALTH_CHECK_COMMAND:-} command_timeout=${CI_FLEET_COMMAND_TIMEOUT_SECONDS:-300} -validate_trusted_path() { - local name=$1 path=$2 kind=$3 allow_absent=${4:-false} create=${5:-false} - python3 - "$path" "$kind" "$allow_absent" "$create" "$testing" "${CI_FLEET_ROOT_PREFIX:-}" <<'PY' || -import errno, os, stat, sys - -path, kind, allow_absent, create, testing, root_prefix = sys.argv[1:] -expected_owner = os.getuid() if testing == "1" else 0 -anchor = os.path.realpath(root_prefix) if testing == "1" and root_prefix else "/" -try: - if not path.startswith("/") or os.path.normpath(path) != path or os.path.realpath(path) != path: - raise ValueError - if os.path.commonpath((anchor, path)) != anchor: - raise ValueError - metadata = os.lstat(path) - exists = True -except FileNotFoundError: - if allow_absent != "true": - raise SystemExit(1) - exists = False -except (OSError, ValueError): - raise SystemExit(1) - -if exists: - if kind == "executable" and (not stat.S_ISREG(metadata.st_mode) or not os.access(path, os.X_OK)): - raise SystemExit(1) - if kind == "regular" and not stat.S_ISREG(metadata.st_mode): - raise SystemExit(1) - if kind == "checkpoint" and (not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700): - raise SystemExit(1) -current = os.path.dirname(path) - -while True: - try: - metadata = os.lstat(current) - except OSError: - raise SystemExit(1) - if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != expected_owner or metadata.st_mode & 0o022: - raise SystemExit(1) - if current == anchor: - break - current = os.path.dirname(current) - -if exists: - metadata = os.lstat(path) - if metadata.st_uid != expected_owner or metadata.st_mode & 0o022: - raise SystemExit(1) -elif create == "true": - parent, leaf = os.path.split(path) - parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) - try: - try: - os.mkdir(leaf, 0o700, dir_fd=parent_fd) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - fd = os.open(leaf, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=parent_fd) - try: - metadata = os.fstat(fd) - if metadata.st_uid != expected_owner or stat.S_IMODE(metadata.st_mode) != 0o700: - raise OSError - finally: - os.close(fd) - except OSError: - raise SystemExit(1) - finally: - os.close(parent_fd) -PY - die "$name must be a trusted root-owned path" -} - validate_command() { local name=$1 path=$2 [[ -n "$path" ]] || die "$name is required when network policy is managed" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 4b97529f..56020571 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -1148,6 +1148,49 @@ def test_env_file_is_trusted_and_snapshotted_before_mutation(self) -> None: daemon = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8")) self.assertEqual(len(daemon["default-address-pools"]), 2) + def test_env_file_below_writable_parent_is_rejected_before_side_effects(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + untrusted_parent = Path(self.tmp) / "writable-parent" + untrusted_parent.mkdir() + untrusted_parent.chmod(0o777) + env_file = untrusted_parent / "ci-fleet.env" + env_file.write_text( + "".join(f"{key}={value}\n" for key, value in sorted(self._rendered_with_policy().items())), + encoding="utf-8", + ) + env_file.chmod(0o600) + checkpoint = Path(self.tmp) / "checkpoint-untrusted-env-parent" + lock = Path(self.tmp) / "untrusted-env-parent.lock" + drain_marker = Path(self.tmp) / "untrusted-env-parent-drain.marker" + snapshot_marker = Path(self.tmp) / "untrusted-env-parent-snapshot.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + fake_bin = Path(self.tmp) / "untrusted-env-parent-bin" + fake_bin.mkdir() + install = fake_bin / "install" + install.write_text( + "#!/usr/bin/env bash\n" + f"touch {snapshot_marker}\n" + f"exec {shutil.which('install')} \"$@\"\n", + encoding="utf-8", + ) + install.chmod(0o755) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock), PATH=f"{fake_bin}:{os.environ['PATH']}"), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertFalse(snapshot_marker.exists()) + self.assertFalse(checkpoint.exists()) + self.assertFalse(lock.exists()) + self.assertFalse(drain_marker.exists()) + def test_renderer_failure_suppresses_private_stderr(self) -> None: daemon = self._write_daemon("{}\n") prior = daemon.read_bytes() From 2e1e22057363b00208c1a22b7e698303ac5d3c60 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:09:55 -0500 Subject: [PATCH 25/54] fix: persist durable policy rollback recovery --- scripts/apply-docker-network-policy.sh | 28 ++++-- scripts/test_apply_docker_network_policy.py | 96 +++++++++++++++++++++ 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index db94bab5..1d678f19 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -398,17 +398,18 @@ PY } persist_recovery() { + local daemon_source=${1:-} env_source=${2:-$prior_env} checkpoint_path_is_pinned || return 1 - python3 - "$state_file" "$had_prior" "$backup_dir/$backup_name" "$prior_env" <<'PY' + python3 - "$state_file" "$daemon_source" "$env_source" <<'PY' import os, shutil, sys, tempfile -state_path, had_prior, daemon_source, env_source = sys.argv[1:] +state_path, daemon_source, env_source = sys.argv[1:] parent = os.path.dirname(state_path) staged = tempfile.mkdtemp(prefix=".recovery.", dir=parent) try: os.chmod(staged, 0o700) sources = [(env_source, "prior-ci-fleet.env")] - if had_prior == "true": + if daemon_source: sources.insert(0, (daemon_source, "daemon.json.before")) for source, name in sources: target = os.path.join(staged, name) @@ -515,6 +516,8 @@ PY } managed_mode=$(stat -c %a "$daemon_config") managed_gid=$(stat -c %g "$daemon_config") + managed_snapshot=$work_dir/daemon.json.before + cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } drain_controller 'drain command failed before network-policy removal' checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy removal' @@ -559,7 +562,12 @@ PY rm -rf "$work_dir" printf 'ERROR: %s; managed daemon.json restored\n' "$removal_failure" >&2 else - printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$removal_failure" "$work_dir" >&2 + recovery_path=$(persist_recovery "$managed_snapshot" "$prior_env") || { + printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$removal_failure" >&2 + exit "$status" + } + rm -rf "$work_dir" + printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$removal_failure" "$recovery_path" >&2 fi exit "$status" } @@ -697,6 +705,14 @@ restore_daemon() { atomic_replace_daemon "$backup_dir/$backup_name" "$daemon_mode" "$daemon_gid" else rm -f "$daemon_config" + python3 - "$daemon_dir" <<'PY' +import os, sys +fd = os.open(sys.argv[1], os.O_RDONLY | os.O_DIRECTORY) +try: + os.fsync(fd) +finally: + os.close(fd) +PY fi } @@ -724,7 +740,9 @@ rollback_on_exit() { rm -rf "$work_dir" printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 else - recovery_path=$(persist_recovery) || { + recovery_daemon= + [[ "$had_prior" != true ]] || recovery_daemon=$backup_dir/$backup_name + recovery_path=$(persist_recovery "$recovery_daemon" "$prior_env") || { printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$transaction_failure" >&2 exit "$status" } diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 56020571..e870dd3a 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -1550,6 +1550,57 @@ def test_probe_failure_restores_absent_config_and_restarts(self) -> None: self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) + def test_absent_apply_rollback_fsyncs_daemon_dir_after_unlink(self) -> None: + self._write_success_commands() + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + audit_log = Path(self.tmp) / "absent-rollback-fsync.log" + audit_dir = Path(self.tmp) / "absent-rollback-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_fsync = os.fsync\n" + "_log = os.environ['FSYNC_AUDIT_LOG']\n" + "def fsync(fd):\n" + " with open(_log, 'a', encoding='utf-8') as handle:\n" + " handle.write('F ' + os.path.realpath(f'/proc/self/fd/{fd}') + '\\n')\n" + " return _fsync(fd)\n" + "os.fsync = fsync\n", + encoding="utf-8", + ) + fake_bin = Path(self.tmp) / "absent-rollback-bin" + fake_bin.mkdir() + (fake_bin / "rm").write_text( + "#!/usr/bin/env bash\n" + f"if [[ ${{*: -1}} == {self.daemon_dir / 'daemon.json'} ]]; then\n" + f" {shutil.which('rm')} \"$@\"\n" + f" printf 'U %s\\n' {self.daemon_dir / 'daemon.json'} >> {audit_log}\n" + " exit\n" + "fi\n" + f"exec {shutil.which('rm')} \"$@\"\n", + encoding="utf-8", + ) + (fake_bin / "rm").chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-absent-fsync"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env( + PATH=f"{fake_bin}:{os.environ['PATH']}", + PYTHONPATH=str(audit_dir), + FSYNC_AUDIT_LOG=str(audit_log), + ), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("prior daemon.json restored", result.stderr) + events = audit_log.read_text(encoding="utf-8").splitlines() + unlink = events.index(f"U {self.daemon_dir / 'daemon.json'}") + self.assertIn(f"F {self.daemon_dir}", events[unlink + 1 :]) + def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: prior = b'{"bip":"172.17.0.1/16","icc":false}\n' daemon = self.daemon_dir / "daemon.json" @@ -2334,6 +2385,51 @@ def test_uncertain_removal_rollback_retains_pending_state(self) -> None: self.assertEqual(state["phase"], "removal-pending") self.assertEqual(state["removal_managed_default_address_pools"], managed_pools) + def test_failed_removal_from_absent_baseline_persists_durable_recovery(self) -> None: + daemon = self.daemon_dir / "daemon.json" + self.assertFalse(daemon.exists()) + checkpoint = Path(self.tmp) / "checkpoint-durable-removal-recovery" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + managed_pools = json.loads(managed)["default-address-pools"] + self.installed_env.write_bytes(policy_env.read_bytes()) + prior_env = self.installed_env.read_bytes() + restart_log = Path(self.tmp) / "durable-removal-restart.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {restart_log}\n" + f"[[ $(wc -l < {restart_log}) -eq 1 ]]\n", + encoding="utf-8", + ) + restart.chmod(0o755) + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_TEMP_DIR=self.tmp), + timeout=30, + ) + + self.assertNotEqual(removed.returncode, 0) + combined = removed.stdout + removed.stderr + recovery = Path(combined.rstrip().rsplit("recovery data retained at ", 1)[1]) + for volatile in Path(self.tmp).glob(".ci-fleet-apply.*"): + shutil.rmtree(volatile) + self.assertEqual(recovery.parent, checkpoint) + self.assertEqual((recovery / "daemon.json.before").read_bytes(), managed) + self.assertEqual((recovery / "prior-ci-fleet.env").read_bytes(), prior_env) + state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertEqual(state["phase"], "removal-pending") + self.assertEqual(state["removal_managed_default_address_pools"], managed_pools) + self.assertIsNone(state["verified_generation"]) + def test_removal_probe_failure_rolls_back_before_resume_health_or_marker_clear(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-removal-probe-failure" From 035c16aca4bdd7488fbf428a8959c4ccda62ad94 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:20:05 -0500 Subject: [PATCH 26/54] fix: abort policy removal on daemon conflicts --- scripts/apply-docker-network-policy.sh | 20 ++++++++-- scripts/test_apply_docker_network_policy.py | 43 +++++++++++++++++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 1d678f19..2612ad39 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -336,6 +336,18 @@ with open(sys.argv[1], "rb") as handle: PY } +daemon_changed_since_snapshot() { + local snapshot=$1 was_present=$2 + if [[ "$was_present" == true ]]; then + if cmp -s "$snapshot" "$daemon_config"; then + return 1 + fi + return 0 + else + [[ -e "$daemon_config" || -L "$daemon_config" ]] + fi +} + set_verified_generation() { checkpoint_path_is_pinned || return 1 python3 - "$state_file" "${1:-}" "${2:-}" <<'PY' @@ -520,6 +532,10 @@ PY cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } drain_controller 'drain command failed before network-policy removal' + if daemon_changed_since_snapshot "$managed_snapshot" true; then + drain_failure='daemon.json changed during network-policy removal' + exit 2 + fi checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy removal' removal_failure='network-policy removal interrupted' @@ -754,9 +770,7 @@ rollback_on_exit() { # --- Drain after local validation/checkpointing, before mutation or restart --- drain_controller 'drain command failed before network-policy apply' -if [[ "$had_prior" == true ]]; then - cmp -s "$backup_dir/$backup_name" "$daemon_config" || { drain_failure='daemon.json changed during network-policy apply'; exit 2; } -elif [[ -e "$daemon_config" || -L "$daemon_config" ]]; then +if daemon_changed_since_snapshot "$backup_dir/$backup_name" "$had_prior"; then drain_failure='daemon.json changed during network-policy apply' exit 2 fi diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index e870dd3a..71c9ba4e 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -2141,6 +2141,49 @@ def test_removal_rejects_non_object_daemon_before_copy_or_commands(self) -> None self.assertFalse(copy_marker.exists()) self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_removal_daemon_change_during_drain_aborts_and_preserves_update(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-removal-daemon-conflict" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + prior_state = state_file.read_bytes() + changed = json.loads(daemon.read_text(encoding="utf-8")) + changed["log-level"] = "debug" + changed_bytes = json.dumps(changed).encode() + command_log = Path(self.tmp) / "removal-daemon-conflict.log" + mutator = Path(self.tmp) / "change-daemon-during-removal.py" + mutator.write_text( + "import json, sys\n" + "path = sys.argv[1]\n" + "value = json.load(open(path))\n" + "value['log-level'] = 'debug'\n" + "with open(path, 'w') as handle: json.dump(value, handle)\n", + encoding="utf-8", + ) + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"echo drain >> {command_log}\n" + f"{shutil.which('python3')} {mutator} {daemon}\n", + encoding="utf-8", + ) + for name in ("restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual(removed.stderr, "ERROR: daemon.json changed during network-policy removal\n") + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume"]) + self.assertEqual(daemon.read_bytes(), changed_bytes) + self.assertEqual(state_file.read_bytes(), prior_state) + def test_removal_restores_prior_pools_and_preserves_current_unrelated_keys(self) -> None: prior_pools = [{"base": "192.0.2.0/24", "size": 28}] daemon = self._write_daemon(json.dumps({"default-address-pools": prior_pools, "icc": False})) From ebdb964bcfbc864681102a8232242a8c8191b517 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 21:38:54 -0500 Subject: [PATCH 27/54] fix: open installer lock through trusted path --- scripts/apply-docker-network-policy.sh | 43 +++++++++--- scripts/test_apply_docker_network_policy.py | 72 ++++++++++++++++++++- 2 files changed, 105 insertions(+), 10 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 2612ad39..746dae95 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -18,6 +18,7 @@ set -Eeuo pipefail repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) testing=${CI_FLEET_TESTING:-0} +original_args=("$@") env_file= checkpoint_dir= @@ -133,11 +134,6 @@ done expected_env_owner=0 [[ "$testing" != 1 ]] || expected_env_owner=$(id -u) validate_trusted_path 'rendered env' "$env_file" regular -work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") -chmod 0700 "$work_dir" -trap 'rm -rf "$work_dir"' EXIT -install -m 0600 -- "$env_file" "$work_dir/ci-fleet.env" -env_file=$work_dir/ci-fleet.env # --- No-op when no network policy is rendered --- count=$(awk -F= '$1 == "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" {print substr($0, index($0, "=") + 1)}' "$env_file") @@ -229,11 +225,42 @@ if [[ -n ${CI_FLEET_INSTALLER_LOCK_FD:-} ]]; then [[ $(readlink -f /proc/self/fd/9 2>/dev/null || true) == $(readlink -m "$lock_file") ]] || die 'inherited installer lock does not match the configured lock file' flock -n 9 || die 'inherited installer lock is unavailable' else - install -d -m 0755 "$(dirname "$lock_file")" - exec 9>"$lock_file" - flock -n 9 || die 'another ci-fleet installer or drift check is already running' + validate_trusted_path CI_FLEET_INSTALLER_LOCK "$lock_file" regular true + exec python3 - "$lock_file" "$testing" "$repo_root/scripts/apply-docker-network-policy.sh" "${original_args[@]}" <<'PY' +import errno, os, stat, sys + +path, testing, script, *args = sys.argv[1:] +expected_owner = os.getuid() if testing == "1" else 0 +flags = os.O_RDWR | os.O_NOFOLLOW +try: + try: + fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600) + os.fchmod(fd, 0o600) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + fd = os.open(path, flags) + metadata = os.fstat(fd) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != expected_owner or metadata.st_mode & 0o022: + raise OSError + os.dup2(fd, 9) + if fd != 9: + os.close(fd) + os.set_inheritable(9, True) + os.environ["CI_FLEET_INSTALLER_LOCK_FD"] = "9" + os.execv(script, [script, *args]) +except OSError: + print("ERROR: CI_FLEET_INSTALLER_LOCK must remain a trusted root-owned regular file", file=sys.stderr) + raise SystemExit(2) +PY fi +work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") +chmod 0700 "$work_dir" +trap 'rm -rf "$work_dir"' EXIT +install -m 0600 -- "$env_file" "$work_dir/ci-fleet.env" +env_file=$work_dir/ci-fleet.env + validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true true exec 8<"$checkpoint_dir" checkpoint_path_is_pinned || die 'checkpoint directory must remain a trusted root-owned path' diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 71c9ba4e..cde65de3 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -243,6 +243,7 @@ class ApplyScriptTests(unittest.TestCase): def setUp(self) -> None: self.tmp = tempfile.mkdtemp() self.addCleanup(shutil.rmtree, self.tmp, ignore_errors=True) + (Path(self.tmp) / "run").mkdir() self.daemon_dir = Path(self.tmp) / "etc" / "docker" self.daemon_dir.mkdir(parents=True) self.installed_env = Path(self.tmp) / "etc" / "ci-fleet" / "ci-fleet.env" @@ -1253,7 +1254,7 @@ def test_existing_installer_lock_blocks_before_drain_or_checkpoint(self) -> None command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") command.chmod(0o755) lock = Path(self.tmp) / "run" / "ci-fleet-installer.lock" - lock.parent.mkdir() + lock.parent.mkdir(exist_ok=True) checkpoint = Path(self.tmp) / "checkpoint" env_file = self._write_env_file(self._rendered_with_policy()) @@ -1281,6 +1282,73 @@ def test_existing_installer_lock_blocks_before_drain_or_checkpoint(self) -> None self.assertFalse(drain_marker.exists()) self.assertFalse(checkpoint.exists()) + def test_symlinked_installer_lock_below_writable_parent_is_rejected_without_side_effects(self) -> None: + daemon = self._write_daemon("{}\n") + prior = daemon.read_bytes() + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + lock_dir = Path(self.tmp) / "writable-lock-parent" + lock_dir.mkdir(mode=0o777) + lock_dir.chmod(0o777) + victim = Path(self.tmp) / "lock-victim" + victim.write_text("do not truncate\n", encoding="utf-8") + lock = lock_dir / "installer.lock" + lock.symlink_to(victim) + drain_marker = Path(self.tmp) / "symlink-lock-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-symlink-lock"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertEqual(victim.read_text(encoding="utf-8"), "do not truncate\n") + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + + def test_group_writable_installer_lock_parent_fails_before_side_effects(self) -> None: + daemon = self._write_daemon("{}\n") + prior = daemon.read_bytes() + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + lock_dir = Path(self.tmp) / "group-writable-lock-parent" + lock_dir.mkdir(mode=0o770) + lock_dir.chmod(0o770) + lock = lock_dir / "installer.lock" + drain_marker = Path(self.tmp) / "writable-lock-parent-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-writable-lock-parent"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("trusted root-owned path", result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(lock.exists()) + self.assertFalse(drain_marker.exists()) + def test_symlinked_daemon_config_is_rejected_before_drain_or_checkpoint(self) -> None: referent = Path(self.tmp) / "referent.json" prior = b'{"bip":"172.17.0.1/16"}\n' @@ -1686,7 +1754,7 @@ def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> start_new_session=True, ) self.addCleanup(lambda: process.poll() is None and process.kill()) - for _ in range(100): + for _ in range(300): if ready.exists(): break time.sleep(0.02) From 1615e2d1797c50f92536a8be2ebc530ca895f5c9 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:11:06 -0500 Subject: [PATCH 28/54] fix: recover safe interrupted policy removal --- scripts/apply-docker-network-policy.sh | 40 +++++++++++-- scripts/desired_state.py | 52 +++++++++-------- scripts/test_apply_docker_network_policy.py | 65 +++++++++++++++++++++ 3 files changed, 127 insertions(+), 30 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 746dae95..85cd2c3d 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -405,6 +405,20 @@ finally: PY } +clear_managed_marker() { + checkpoint_path_is_pinned || return 1 + python3 - "$state_file" <<'PY' +import os, sys +path = sys.argv[1] +os.unlink(path) +directory_fd = os.open(os.path.dirname(path), os.O_RDONLY | os.O_DIRECTORY) +try: + os.fsync(directory_fd) +finally: + os.close(directory_fd) +PY +} + set_removal_pending() { checkpoint_path_is_pinned || return 1 python3 - "$state_file" "$1" <<'PY' @@ -478,9 +492,11 @@ PY if [[ "$removing" == true ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || die 'network-policy checkpoint state is invalid' - mapfile -t managed_state < <(python3 - "$state_file" <<'PY' + mapfile -t managed_state < <(python3 - "$state_file" "$repo_root/scripts" 2>/dev/null <<'PY' import json, re, sys state = json.load(open(sys.argv[1], encoding="utf-8")) +sys.path.insert(0, sys.argv[2]) +from desired_state import validate_docker_address_pools required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} pending = {"phase", "removal_managed_default_address_pools"} schemas = (required, required | {"verified_generation"}, required | pending, required | pending | {"verified_generation"}) @@ -496,9 +512,14 @@ if not isinstance(state["prior_present"], bool): raise SystemExit(1) if not isinstance(state["prior_default_address_pools_present"], bool): raise SystemExit(1) -if state["prior_default_address_pools_present"] and not state["prior_present"]: - raise SystemExit(1) -if not state["prior_default_address_pools_present"] and state["prior_default_address_pools"] is not None: +if state["prior_default_address_pools_present"]: + if not state["prior_present"]: + raise SystemExit(1) + validate_docker_address_pools( + state["prior_default_address_pools"], + path="checkpoint prior default address pools", + ) +elif state["prior_default_address_pools"] is not None: raise SystemExit(1) mode = state["prior_mode"] if state["prior_present"]: @@ -510,15 +531,24 @@ print("true" if state["prior_present"] else "false") print(mode or "") print(generation or "") print("true" if is_pending else "false") +print("true" if "verified_generation" in state else "false") PY ) || die 'network-policy checkpoint state is invalid' - [[ ${#managed_state[@]} == 4 ]] || die 'network-policy checkpoint state is invalid' + [[ ${#managed_state[@]} == 5 ]] || die 'network-policy checkpoint state is invalid' prior_present=${managed_state[0]} prior_mode=${managed_state[1]} prior_verified_generation=${managed_state[2]} + removal_pending=${managed_state[3]} + has_verified_generation=${managed_state[4]} if [[ "$prior_present" == true ]]; then [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' fi + if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$removal_pending" == false && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" ]]; then + clear_managed_marker || die 'failed to clear interrupted network-policy marker' + rm -rf "$work_dir" + printf 'NETWORK_POLICY_REMOVED\n' + exit 0 + fi [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' managed_daemon=$work_dir/daemon.json.managed diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 246de624..70335504 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -139,31 +139,12 @@ def validate_host_values(values: dict[str, str]) -> dict[str, str]: } -def validate_docker_network_policy(policy: dict[str, Any], *, path: str, max_runners: int) -> tuple[int, int, int, list[dict[str, Any]]]: - if not isinstance(policy, dict): - raise DesiredStateError(f"{path}: must be an object") - required = {"default_address_pools", "networks_per_runner", "reserve_subnets"} - if set(policy) != required: - unknown = sorted(set(policy) - required) - missing = sorted(required - set(policy)) - messages: list[str] = [] - if missing: - messages.append(f"missing keys: {', '.join(missing)}") - if unknown: - messages.append(f"unknown keys: {', '.join(unknown)}") - raise DesiredStateError(f"{path}: " + "; ".join(messages)) - reserve = policy.get("reserve_subnets") - if type(reserve) is not int or reserve < 1: - raise DesiredStateError(f"{path}.reserve_subnets: must be a positive integer") - networks_per_runner = policy.get("networks_per_runner") - if type(networks_per_runner) is not int or networks_per_runner < 1: - raise DesiredStateError(f"{path}.networks_per_runner: must be a positive integer") - pools = policy.get("default_address_pools") +def validate_docker_address_pools(pools: Any, *, path: str) -> list[dict[str, Any]]: if type(pools) is not list or not pools: - raise DesiredStateError(f"{path}.default_address_pools: must be a non-empty list") + raise DesiredStateError(f"{path}: must be a non-empty list") parsed: list[dict[str, Any]] = [] for index, pool in enumerate(pools): - pool_path = f"{path}.default_address_pools[{index}]" + pool_path = f"{path}[{index}]" if not isinstance(pool, dict) or set(pool) != {"base", "size"}: raise DesiredStateError(f"{pool_path}: must contain only base and size") base = pool.get("base") @@ -185,9 +166,30 @@ def validate_docker_network_policy(policy: dict[str, Any], *, path: str, max_run for right in range(left + 1, len(parsed)): other = parsed[right] if item["network"].overlaps(other["network"]): - raise DesiredStateError( - f"{path}.default_address_pools[{left}].base: overlaps configured pool {right}" - ) + raise DesiredStateError(f"{path}[{left}].base: overlaps configured pool {right}") + return parsed + + +def validate_docker_network_policy(policy: dict[str, Any], *, path: str, max_runners: int) -> tuple[int, int, int, list[dict[str, Any]]]: + if not isinstance(policy, dict): + raise DesiredStateError(f"{path}: must be an object") + required = {"default_address_pools", "networks_per_runner", "reserve_subnets"} + if set(policy) != required: + unknown = sorted(set(policy) - required) + missing = sorted(required - set(policy)) + messages: list[str] = [] + if missing: + messages.append(f"missing keys: {', '.join(missing)}") + if unknown: + messages.append(f"unknown keys: {', '.join(unknown)}") + raise DesiredStateError(f"{path}: " + "; ".join(messages)) + reserve = policy.get("reserve_subnets") + if type(reserve) is not int or reserve < 1: + raise DesiredStateError(f"{path}.reserve_subnets: must be a positive integer") + networks_per_runner = policy.get("networks_per_runner") + if type(networks_per_runner) is not int or networks_per_runner < 1: + raise DesiredStateError(f"{path}.networks_per_runner: must be a positive integer") + parsed = validate_docker_address_pools(policy.get("default_address_pools"), path=f"{path}.default_address_pools") configured = sum(1 << (item["size"] - item["network"].prefixlen) for item in parsed) if configured < max_runners * networks_per_runner + reserve + 1: raise DesiredStateError( diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index cde65de3..b4cb7e7b 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -2158,6 +2158,41 @@ def test_restart_failure_evidence_excludes_command_output(self) -> None: self.assertNotIn("credential", combined) self.assertNotIn("restart-secret-error", combined) + def test_removal_clears_interrupted_first_apply_before_rename_without_commands(self) -> None: + daemon = self.daemon_dir / "daemon.json" + self.assertFalse(daemon.exists()) + checkpoint = Path(self.tmp) / "checkpoint-interrupted-before-rename" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, + "prior_mode": None, + "prior_present": False, + "verified_generation": None, + }), + encoding="utf-8", + ) + state_file.chmod(0o600) + command_markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): + marker = Path(self.tmp) / f"interrupted-before-rename-{name}.marker" + command_markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(removed.stdout, "NETWORK_POLICY_REMOVED\n") + self.assertFalse(state_file.exists()) + self.assertFalse(daemon.exists()) + self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_removal_rejects_non_object_daemon_before_copy_or_commands(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-non-object-removal" @@ -2339,6 +2374,36 @@ def test_persistent_provenance_has_no_recovery_backup_and_rejects_inconsistent_s self.assertNotIn("private-daemon-value", removed.stdout + removed.stderr) self.assertNotIn("192.0.2.0/24", removed.stdout + removed.stderr) + def test_removal_rejects_malformed_saved_pools_before_commands_or_mutation(self) -> None: + prior_pools = [{"base": "192.0.2.0/24", "size": 28}] + daemon = self._write_daemon(json.dumps({"default-address-pools": prior_pools})) + checkpoint = Path(self.tmp) / "checkpoint-malformed-saved-pools" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["prior_default_address_pools"] = [{"base": "not-a-cidr", "size": 29}] + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + managed = daemon.read_bytes() + command_markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): + marker = Path(self.tmp) / f"malformed-saved-pools-{name}.marker" + command_markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("network-policy checkpoint state is invalid", removed.stderr) + self.assertEqual(daemon.read_bytes(), managed) + self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_managed_policy_removal_replaces_daemon_atomically(self) -> None: prior = b'{"icc":false}\n' daemon = self.daemon_dir / "daemon.json" From 452aedef0c5202fc641abbc8349a1a98b60a7ac8 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 22:23:37 -0500 Subject: [PATCH 29/54] fix: reject explicit empty network policy --- scripts/apply-docker-network-policy.sh | 3 +-- scripts/test_apply_docker_network_policy.py | 30 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 85cd2c3d..c6589a29 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -136,9 +136,8 @@ expected_env_owner=0 validate_trusted_path 'rendered env' "$env_file" regular # --- No-op when no network policy is rendered --- -count=$(awk -F= '$1 == "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" {print substr($0, index($0, "=") + 1)}' "$env_file") removing=false -if [[ -z "$count" || "$count" == "0" ]]; then +if ! grep -q '^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=' "$env_file"; then if [[ -z "$checkpoint_dir" ]]; then printf 'NETWORK_POLICY_NOOP\n' exit 0 diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index b4cb7e7b..b779ef87 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -1475,6 +1475,36 @@ def test_matching_file_without_verified_generation_runs_activation_and_marks_ver verified = json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"] self.assertEqual(verified, hashlib.sha256((self.daemon_dir / "daemon.json").read_bytes()).hexdigest()) + def test_rejects_explicit_zero_on_managed_host_before_checkpoint_lock_or_drain(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-explicit-zero" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + state_file = checkpoint / "docker-network-policy.json" + state = state_file.read_bytes() + zero_env = self._write_env_file({"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "0"}) + lock = Path(self.tmp) / "network-policy-zero.lock" + drain_marker = Path(self.tmp) / "explicit-zero-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + rejected = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(zero_env)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(rejected.returncode, 0) + self.assertEqual(rejected.stderr, "ERROR: daemon policy rendering failed\n") + self.assertEqual(daemon.read_bytes(), managed) + self.assertEqual(state_file.read_bytes(), state) + self.assertFalse(lock.exists()) + self.assertFalse(drain_marker.exists()) + def test_rejects_overlapping_rendered_pools_before_lock_or_drain(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self._write_daemon(prior.decode()) From 91a25b38f7da93a0994d66137f1ae70559fd438b Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sat, 29 Aug 2026 23:48:26 -0500 Subject: [PATCH 30/54] fix: validate network policy transaction paths --- scripts/apply-docker-network-policy.sh | 30 +++++---- scripts/test_apply_docker_network_policy.py | 70 +++++++++++++++++++++ 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index c6589a29..4d8ac786 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -44,7 +44,7 @@ import errno, os, stat, sys path, kind, allow_absent, create, testing, root_prefix = sys.argv[1:] expected_owner = os.getuid() if testing == "1" else 0 -anchor = os.path.realpath(root_prefix) if testing == "1" and root_prefix else "/" +anchor = os.path.realpath(root_prefix) if testing == "1" and root_prefix and kind != "tempdir" else "/" try: if not path.startswith("/") or os.path.normpath(path) != path or os.path.realpath(path) != path: raise ValueError @@ -66,20 +66,30 @@ if exists: raise SystemExit(1) if kind == "checkpoint" and (not stat.S_ISDIR(metadata.st_mode) or stat.S_IMODE(metadata.st_mode) != 0o700): raise SystemExit(1) -current = os.path.dirname(path) + if kind == "tempdir" and not stat.S_ISDIR(metadata.st_mode): + raise SystemExit(1) +current = path if kind == "tempdir" else os.path.dirname(path) while True: try: metadata = os.lstat(current) except OSError: raise SystemExit(1) - if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != expected_owner or metadata.st_mode & 0o022: + sticky_temp = ( + kind == "tempdir" + and metadata.st_uid in (0, expected_owner) + and metadata.st_mode & stat.S_ISVTX + ) + trusted_owner = metadata.st_uid == expected_owner or (kind == "tempdir" and metadata.st_uid == 0) + if not stat.S_ISDIR(metadata.st_mode) or ( + (not trusted_owner or metadata.st_mode & 0o022) and not sticky_temp + ): raise SystemExit(1) if current == anchor: break current = os.path.dirname(current) -if exists: +if exists and kind != "tempdir": metadata = os.lstat(path) if metadata.st_uid != expected_owner or metadata.st_mode & 0o022: raise SystemExit(1) @@ -131,8 +141,6 @@ while (($#)); do done [[ -n "$env_file" ]] || die '--env is required' -expected_env_owner=0 -[[ "$testing" != 1 ]] || expected_env_owner=$(id -u) validate_trusted_path 'rendered env' "$env_file" regular # --- No-op when no network policy is rendered --- @@ -254,7 +262,9 @@ except OSError: PY fi -work_dir=$(mktemp -d "${CI_FLEET_TEMP_DIR:-/tmp}/.ci-fleet-apply.XXXXXX") +temp_parent=${CI_FLEET_TEMP_DIR:-/tmp} +validate_trusted_path 'transaction temp directory' "$temp_parent" tempdir +work_dir=$(mktemp -d "$temp_parent/.ci-fleet-apply.XXXXXX") chmod 0700 "$work_dir" trap 'rm -rf "$work_dir"' EXIT install -m 0600 -- "$env_file" "$work_dir/ci-fleet.env" @@ -268,10 +278,8 @@ state_file=/proc/self/fd/8/docker-network-policy.json # Snapshot the installer's authoritative pre-transaction environment while # holding its lock. Rollback must not validate against the rejected candidate. installed_env=${CI_FLEET_ROOT_PREFIX:-}/etc/ci-fleet/ci-fleet.env -[[ -f "$installed_env" && ! -L "$installed_env" && -r "$installed_env" ]] || die "installed rendered env must be a readable regular file: $installed_env" -[[ $(stat -c %u "$installed_env") == "$expected_env_owner" ]] || die "installed rendered env has an untrusted owner: $installed_env" -installed_env_mode=$(stat -c %a "$installed_env") -(( (8#$installed_env_mode & 8#022) == 0 )) || die "installed rendered env must not be group/world writable: $installed_env" +validate_trusted_path 'installed rendered env' "$installed_env" regular +[[ -r "$installed_env" ]] || die "installed rendered env must be readable: $installed_env" prior_env=$work_dir/prior-ci-fleet.env install -m 0600 -- "$installed_env" "$prior_env" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index b779ef87..dfb5fb5f 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -12,6 +12,7 @@ import hashlib import json import os +import pwd import shutil import signal import subprocess @@ -318,6 +319,75 @@ def _write_success_commands(self) -> None: command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") command.chmod(0o755) + def test_rejects_untrusted_transaction_temp_parent(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + self._write_success_commands() + temp_parent = Path(self.tmp) / "untrusted-temp" + temp_parent.mkdir(mode=0o777) + temp_parent.chmod(0o777) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-untrusted-temp"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_TEMP_DIR=str(temp_parent)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("transaction temp directory must be a trusted root-owned path", result.stderr) + + @unittest.skipUnless(os.geteuid() == 0 and shutil.which("runuser"), "requires root and runuser") + def test_testing_mode_accepts_default_tmp_as_non_root(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + self._write_success_commands() + test_scripts = Path(self.tmp) / "repo" / "scripts" + test_scripts.mkdir(parents=True) + for name in ("apply-docker-network-policy.sh", "desired_state.py"): + shutil.copy2(SCRIPTS / name, test_scripts / name) + nobody = pwd.getpwnam("nobody") + for path in [Path(self.tmp), *Path(self.tmp).rglob("*")]: + os.chown(path, nobody.pw_uid, nobody.pw_gid) + + result = subprocess.run( + [ + "runuser", + "-u", + "nobody", + "--", + str(test_scripts / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-non-root"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(), + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_rejects_installed_env_under_untrusted_directory(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + self._write_success_commands() + self.installed_env.parent.chmod(0o777) + + result = self._run(str(env_file)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("installed rendered env must be a trusted root-owned path", result.stderr) + def test_successful_apply_restarts_probes_resumes_then_checks_health(self) -> None: self._write_daemon("{}\n") env_file = self._write_env_file(self._rendered_with_policy()) From fe04f5056bff11ba179027e7ef9ee0577eefa67a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 00:36:19 -0500 Subject: [PATCH 31/54] fix: harden network policy recovery --- scripts/apply-docker-network-policy.sh | 195 ++++++++++++++------ scripts/test_apply_docker_network_policy.py | 182 +++++++++++++++++- 2 files changed, 316 insertions(+), 61 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 4d8ac786..35b015c0 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -284,11 +284,18 @@ prior_env=$work_dir/prior-ci-fleet.env install -m 0600 -- "$installed_env" "$prior_env" drain_failure= +new_marker=false +controller_resumed=false +transaction_recovery= # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below resume_after_failed_drain() { local status=$? resume_failed=0 trap - EXIT INT TERM run_command "$resume_command" --env "$prior_env" || resume_failed=1 + if ((resume_failed == 0)) && [[ "$new_marker" == true ]]; then + clear_managed_marker || resume_failed=1 + fi + [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" rm -rf "$work_dir" if ((resume_failed)); then printf 'ERROR: %s; controller resume command failed\n' "$drain_failure" >&2 @@ -435,7 +442,7 @@ state = json.load(open(path, encoding="utf-8")) if state.get("phase") != "removal-pending": managed = json.load(open(managed_path, encoding="utf-8")) state["phase"] = "removal-pending" - state["removal_managed_default_address_pools"] = managed["default-address-pools"] + state["removal_managed_default_address_pools"] = managed.get("default-address-pools") state["verified_generation"] = None fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) try: @@ -510,7 +517,11 @@ schemas = (required, required | {"verified_generation"}, required | pending, req if set(state) not in schemas or state["managed"] is not True: raise SystemExit(1) is_pending = "phase" in state -if is_pending and (state["phase"] != "removal-pending" or not isinstance(state["removal_managed_default_address_pools"], list)): +if is_pending and ( + state["phase"] != "removal-pending" + or state["removal_managed_default_address_pools"] is not None + and not isinstance(state["removal_managed_default_address_pools"], list) +): raise SystemExit(1) generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): @@ -551,6 +562,8 @@ PY [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' fi if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$removal_pending" == false && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" ]]; then + run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' + run_health "$env_file" || die 'health check failed while recovering interrupted network-policy apply' clear_managed_marker || die 'failed to clear interrupted network-policy marker' rm -rf "$work_dir" printf 'NETWORK_POLICY_REMOVED\n' @@ -571,11 +584,14 @@ try: if state.get("phase") == "removal-pending": managed_pools = state["removal_managed_default_address_pools"] else: - managed_pools = current["default-address-pools"] + managed_pools = current.get("default-address-pools") except (OSError, json.JSONDecodeError, KeyError, ValueError): raise SystemExit(1) managed = dict(current) -managed["default-address-pools"] = managed_pools +if managed_pools is None: + managed.pop("default-address-pools", None) +else: + managed["default-address-pools"] = managed_pools with open(managed_path, "w", encoding="utf-8") as handle: json.dump(managed, handle, indent=2, sort_keys=True) handle.write("\n") @@ -606,28 +622,38 @@ PY # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below rollback_removal() { local failed=0 rollback_daemon=$work_dir/daemon.json.rollback - python3 - "$daemon_config" "$managed_daemon" "$rollback_daemon" <<'PY' || failed=1 + if [[ "$controller_resumed" == true ]]; then + run_command "$drain_command" || failed=1 + fi + if ((failed == 0)); then + python3 - "$daemon_config" "$managed_daemon" "$rollback_daemon" <<'PY' || failed=1 import json, os, sys current_path, managed_path, output_path = sys.argv[1:] try: current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} managed = json.load(open(managed_path, encoding="utf-8")) - if not isinstance(current, dict) or not isinstance(managed, dict) or "default-address-pools" not in managed: + if not isinstance(current, dict) or not isinstance(managed, dict): raise ValueError except (OSError, json.JSONDecodeError, ValueError): raise SystemExit(1) -current["default-address-pools"] = managed["default-address-pools"] +if "default-address-pools" in managed: + current["default-address-pools"] = managed["default-address-pools"] +else: + current.pop("default-address-pools", None) with open(output_path, "w", encoding="utf-8") as handle: json.dump(current, handle, indent=2, sort_keys=True) handle.write("\n") PY + fi if ((failed == 0)); then atomic_replace_daemon "$rollback_daemon" "$managed_mode" "$managed_gid" || failed=1 cmp -s "$rollback_daemon" "$daemon_config" || failed=1 fi - run_command "$restart_command" "$daemon_dir" || failed=1 - run_command "$resume_command" --env "$prior_env" || failed=1 - run_health "$prior_env" || failed=1 + if ((failed == 0)); then + run_command "$restart_command" "$daemon_dir" || failed=1 + run_command "$resume_command" --env "$prior_env" || failed=1 + run_health "$prior_env" || failed=1 + fi if ((failed == 0)); then set_verified_generation "$prior_verified_generation" clear-removal || failed=1 fi @@ -680,6 +706,7 @@ PY run_command "$restart_command" "$daemon_dir" || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } run_command "$probe_command" || { removal_failure='capacity probe failed after network-policy removal'; exit 2; } run_command "$resume_command" --env "$env_file" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } + controller_resumed=true run_health "$env_file" || { removal_failure='health check failed after network-policy removal'; exit 2; } # Marker deletion commits removal. Ignore catchable signals across the atomic @@ -781,8 +808,40 @@ if [[ -f "$daemon_config" ]]; then fi restore_daemon() { - if [[ "$had_prior" == true ]]; then + local rollback_daemon=$work_dir/daemon.json.apply-rollback rollback_action + rollback_action=$(python3 - "$daemon_config" "$backup_dir/$backup_name" "$had_prior" "$rollback_daemon" "$staging_daemon" <<'PY' +import json, os, sys +current_path, prior_path, had_prior, output_path, staged_path = sys.argv[1:] +try: + current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} + prior = json.load(open(prior_path, encoding="utf-8")) if had_prior == "true" else {} + staged = json.load(open(staged_path, encoding="utf-8")) + if not isinstance(current, dict) or not isinstance(prior, dict) or not isinstance(staged, dict): + raise ValueError +except (OSError, json.JSONDecodeError, ValueError): + raise SystemExit(1) +current_unrelated = {key: value for key, value in current.items() if key != "default-address-pools"} +staged_unrelated = {key: value for key, value in staged.items() if key != "default-address-pools"} +if current_unrelated == staged_unrelated: + print("exact" if had_prior == "true" else "remove") + raise SystemExit +if "default-address-pools" in prior: + current["default-address-pools"] = prior["default-address-pools"] +else: + current.pop("default-address-pools", None) +if not current and had_prior != "true": + print("remove") +else: + with open(output_path, "w", encoding="utf-8") as handle: + json.dump(current, handle, indent=2, sort_keys=True) + handle.write("\n") + print("replace") +PY + ) || return 1 + if [[ "$rollback_action" == exact ]]; then atomic_replace_daemon "$backup_dir/$backup_name" "$daemon_mode" "$daemon_gid" + elif [[ "$rollback_action" == replace ]]; then + atomic_replace_daemon "$rollback_daemon" "$daemon_mode" "$daemon_gid" else rm -f "$daemon_config" python3 - "$daemon_dir" <<'PY' @@ -796,12 +855,57 @@ PY fi } +# Record the original host state before the first drain. The marker lets a +# later no-policy reconciliation resume a controller interrupted by the drain. +if [[ "$managed_before" == false ]]; then + python3 - "$state_file" "$had_prior" "$daemon_mode" "$backup_dir/$backup_name" <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } +import json, os, sys, tempfile +path = sys.argv[1] +prior = json.load(open(sys.argv[4], encoding="utf-8")) if sys.argv[2] == "true" else {} +prior_key_present = "default-address-pools" in prior +state = { + "managed": True, + "prior_default_address_pools": prior.get("default-address-pools") if prior_key_present else None, + "prior_default_address_pools_present": prior_key_present, + "prior_mode": sys.argv[3] if sys.argv[2] == "true" else None, + "prior_present": sys.argv[2] == "true", + "verified_generation": None, +} +fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) +try: + with os.fdopen(fd, "w", encoding="utf-8") as handle: + json.dump(state, handle, indent=2, sort_keys=True) + handle.write("\n") + os.fchmod(handle.fileno(), 0o600) + handle.flush() + os.fsync(handle.fileno()) + os.replace(tmp, path) + directory_fd = os.open(os.path.dirname(path), os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) +finally: + if os.path.exists(tmp): + os.unlink(tmp) +PY + new_marker=true +fi +if [[ "$managed_before" == true ]]; then + transaction_recovery=$(persist_recovery "$backup_dir/$backup_name" "$prior_env") || die 'failed to persist network-policy transaction recovery' +fi + rollback_daemon() { local failed=0 - restore_daemon || failed=1 - run_command "$restart_command" "$daemon_dir" || failed=1 - run_command "$resume_command" --env "$prior_env" || failed=1 - run_health "$prior_env" || failed=1 + if [[ "$controller_resumed" == true ]]; then + run_command "$drain_command" || failed=1 + fi + if ((failed == 0)); then + restore_daemon || failed=1 + run_command "$restart_command" "$daemon_dir" || failed=1 + run_command "$resume_command" --env "$prior_env" || failed=1 + run_health "$prior_env" || failed=1 + fi if [[ "$managed_before" == true && "$failed" == 0 ]]; then set_verified_generation "$prior_verified_generation" || failed=1 fi @@ -817,15 +921,20 @@ rollback_on_exit() { if [[ "$managed_before" == false ]]; then rm -f "$state_file" fi + [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" rm -rf "$work_dir" printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 else - recovery_daemon= - [[ "$had_prior" != true ]] || recovery_daemon=$backup_dir/$backup_name - recovery_path=$(persist_recovery "$recovery_daemon" "$prior_env") || { - printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$transaction_failure" >&2 - exit "$status" - } + if [[ -n "$transaction_recovery" ]]; then + recovery_path=$transaction_recovery + else + recovery_daemon= + [[ "$had_prior" != true ]] || recovery_daemon=$backup_dir/$backup_name + recovery_path=$(persist_recovery "$recovery_daemon" "$prior_env") || { + printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$transaction_failure" >&2 + exit "$status" + } + fi rm -rf "$work_dir" printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$transaction_failure" "$recovery_path" >&2 fi @@ -843,41 +952,6 @@ trap rollback_on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM -# Record the original host state before the first managed mutation. Re-applying -# policy keeps this baseline and uses the temp copy above for transaction rollback. -if [[ "$managed_before" == false ]]; then - python3 - "$state_file" "$had_prior" "$daemon_mode" "$backup_dir/$backup_name" <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } -import json, os, sys, tempfile -path = sys.argv[1] -prior = json.load(open(sys.argv[4], encoding="utf-8")) if sys.argv[2] == "true" else {} -prior_key_present = "default-address-pools" in prior -state = { - "managed": True, - "prior_default_address_pools": prior.get("default-address-pools") if prior_key_present else None, - "prior_default_address_pools_present": prior_key_present, - "prior_mode": sys.argv[3] if sys.argv[2] == "true" else None, - "prior_present": sys.argv[2] == "true", - "verified_generation": None, -} -fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) -try: - with os.fdopen(fd, "w", encoding="utf-8") as handle: - json.dump(state, handle, indent=2, sort_keys=True) - handle.write("\n") - os.fchmod(handle.fileno(), 0o600) - handle.flush() - os.fsync(handle.fileno()) - os.replace(tmp, path) - directory_fd = os.open(os.path.dirname(path), os.O_RDONLY | os.O_DIRECTORY) - try: - os.fsync(directory_fd) - finally: - os.close(directory_fd) -finally: - if os.path.exists(tmp): - os.unlink(tmp) -PY -fi if [[ "$managed_before" == true ]]; then set_verified_generation "" || { transaction_failure='failed to mark network-policy verification pending'; exit 2; } fi @@ -907,16 +981,25 @@ fi if ! run_command "$resume_command" --env "$env_file"; then fail_after_apply "controller resume command failed after network-policy restart" fi +controller_resumed=true # Health verification if ! run_health "$env_file"; then fail_after_apply "health check failed after network-policy restart" fi +python3 - "$daemon_config" "$staging_daemon" 2>/dev/null <<'PY' || fail_after_apply "daemon.json changed after network-policy verification" +import json, sys +current = json.load(open(sys.argv[1], encoding="utf-8")) +staged = json.load(open(sys.argv[2], encoding="utf-8")) +if not isinstance(current, dict) or current.get("default-address-pools") != staged.get("default-address-pools"): + raise SystemExit(1) +PY verified_generation=$(file_generation "$daemon_config") || fail_after_apply "failed to identify verified daemon.json generation" set_verified_generation "$verified_generation" || fail_after_apply "failed to record verified daemon.json generation" # --- Success --- trap - EXIT INT TERM +[[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" rm -rf "$work_dir" printf 'NETWORK_POLICY_APPLIED daemon_config=%s\n' "$daemon_config" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index dfb5fb5f..b359f4dc 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -421,6 +421,32 @@ def test_failed_apply_restarts_resumes_then_checks_rollback_health(self) -> None self.assertNotEqual(result.returncode, 0) self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "restart", "resume", "health"]) + def test_apply_health_failure_redrains_before_rollback_restart(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "apply-health-rollback.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "resume"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"echo health >> {command_log}\n" + 'grep -Fqx "ENV_GENERATION=prior" "$2" || exit 2\n', + encoding="utf-8", + ) + health.chmod(0o755) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "resume", "health", "drain", "restart", "resume", "health"], + ) + def test_failed_apply_uses_pretransaction_installed_env_for_rollback(self) -> None: self._write_daemon("{}\n") env_file = self._write_env_file(self._rendered_with_policy()) @@ -457,7 +483,7 @@ def test_failed_apply_uses_pretransaction_installed_env_for_rollback(self) -> No self.assertEqual(len(set(rollback_envs)), 1) self.assertNotEqual(rollback_envs[0], str(self.installed_env)) - def test_post_drain_state_failure_resumes_prior_controller(self) -> None: + def test_checkpoint_state_failure_stops_before_drain(self) -> None: daemon = self._write_daemon("{}\n") prior = daemon.read_bytes() checkpoint = Path(self.tmp) / "checkpoint-post-drain-failure" @@ -492,7 +518,7 @@ def test_post_drain_state_failure_resumes_prior_controller(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(daemon.read_bytes(), prior) - self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "health"]) + self.assertFalse(command_log.exists()) def test_drain_timeout_resumes_prior_controller_and_preserves_failure(self) -> None: self._write_daemon("{}\n") @@ -629,10 +655,12 @@ def test_creates_and_validates_checkpoint_before_drain(self) -> None: self._write_daemon("{}\n") self._write_success_commands() checkpoint = Path(self.tmp) / "checkpoint-before-drain" + state_file = checkpoint / "docker-network-policy.json" drain_marker = Path(self.tmp) / "checkpoint-before-drain.marker" self.drain_command.write_text( "#!/usr/bin/env bash\n" f"[[ -d {checkpoint} && ! -L {checkpoint} ]] || exit 1\n" + f"[[ -s {state_file} ]] || exit 1\n" f"touch {drain_marker}\n", encoding="utf-8", ) @@ -1811,6 +1839,45 @@ def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: self.assertNotIn("credential", result.stdout + result.stderr) self.assertNotIn("probe-secret-error", result.stdout + result.stderr) + def test_apply_rollback_preserves_concurrent_unrelated_settings(self) -> None: + daemon = self._write_daemon('{"icc":false}\n') + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "apply-merge-rollback.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "resume"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + mutator = Path(self.tmp) / "mutate-apply-daemon.py" + mutator.write_text( + "import json, sys\n" + "path = sys.argv[1]\n" + "value = json.load(open(path))\n" + "value['debug'] = True\n" + "with open(path, 'w') as handle: json.dump(value, handle)\n", + encoding="utf-8", + ) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"echo health >> {command_log}\n" + 'if grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2"; then\n' + f" {shutil.which('python3')} {mutator} {daemon}\n" + " exit 2\n" + "fi\n", + encoding="utf-8", + ) + health.chmod(0o755) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"debug": True, "icc": False}) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "resume", "health", "drain", "restart", "resume", "health"], + ) + def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -1974,6 +2041,38 @@ def test_failed_rollback_persists_exact_recovery_under_checkpoint(self) -> None: self.assertIn(f"F {checkpoint}", audit) self.assertTrue(any(line.startswith("R ") and line.endswith(f" {recovery}") for line in audit)) + def test_managed_reapply_persists_recovery_before_daemon_restart(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-reapply-recovery" + self._write_success_commands() + restart_count = Path(self.tmp) / "reapply-restart-count" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + "#!/usr/bin/env bash\n" + f"echo restart >> {restart_count}\n" + f"if [[ $(wc -l < {restart_count}) -gt 1 ]]; then\n" + f" recovery=({checkpoint}/recovery.*)\n" + ' [[ -f ${recovery[0]}/daemon.json.before && -f ${recovery[0]}/prior-ci-fleet.env ]] || exit 2\n' + "fi\n", + encoding="utf-8", + ) + restart.chmod(0o755) + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + prior_daemon = daemon.read_bytes() + prior_env = policy_env.read_bytes() + self.installed_env.write_bytes(prior_env) + candidate = self._rendered_with_policy() + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = self._write_env_file(candidate) + + reapplied = self._run(str(candidate_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(reapplied.returncode, 0, reapplied.stderr) + self.assertFalse(list(checkpoint.glob("recovery.*"))) + self.assertNotEqual(daemon.read_bytes(), prior_daemon) + def test_post_apply_health_uses_candidate_rendered_env(self) -> None: self._write_daemon("{}\n") for name in ("restart.sh", "probe.sh"): @@ -1998,6 +2097,45 @@ def test_post_apply_health_uses_candidate_rendered_env(self) -> None: self.assertEqual(len(health_paths), 1) self.assertNotEqual(health_paths[0], str(env_file)) + def test_apply_rejects_managed_pool_change_before_recording_generation(self) -> None: + daemon = self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "final-pool-conflict.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "resume"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + mutator = Path(self.tmp) / "mutate-final-pools.py" + mutator.write_text( + "import json, sys\n" + "path = sys.argv[1]\n" + "value = json.load(open(path))\n" + "value['default-address-pools'] = [{'base': '192.0.2.0/24', 'size': 29}]\n" + "with open(path, 'w') as handle: json.dump(value, handle)\n", + encoding="utf-8", + ) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"echo health >> {command_log}\n" + 'if grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2"; then\n' + f" {shutil.which('python3')} {mutator} {daemon}\n" + "fi\n", + encoding="utf-8", + ) + health.chmod(0o755) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("daemon.json changed after network-policy verification", result.stderr) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {}) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "resume", "health", "drain", "restart", "resume", "health"], + ) + def test_health_failure_evidence_excludes_command_output(self) -> None: self._write_daemon("{}\n") for name in ("restart.sh", "probe.sh"): @@ -2258,7 +2396,7 @@ def test_restart_failure_evidence_excludes_command_output(self) -> None: self.assertNotIn("credential", combined) self.assertNotIn("restart-secret-error", combined) - def test_removal_clears_interrupted_first_apply_before_rename_without_commands(self) -> None: + def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> None: daemon = self.daemon_dir / "daemon.json" self.assertFalse(daemon.exists()) checkpoint = Path(self.tmp) / "checkpoint-interrupted-before-rename" @@ -2291,7 +2429,10 @@ def test_removal_clears_interrupted_first_apply_before_rename_without_commands(s self.assertEqual(removed.stdout, "NETWORK_POLICY_REMOVED\n") self.assertFalse(state_file.exists()) self.assertFalse(daemon.exists()) - self.assertFalse(any(marker.exists() for marker in command_markers)) + self.assertEqual( + [marker.name for marker in command_markers if marker.exists()], + ["interrupted-before-rename-resume.sh.marker", "interrupted-before-rename-health.sh.marker"], + ) def test_removal_rejects_non_object_daemon_before_copy_or_commands(self) -> None: daemon = self._write_daemon("{}\n") @@ -2576,6 +2717,37 @@ def test_interrupted_removal_retry_uses_persisted_managed_pools_for_rollback(sel self.assertNotIn("phase", rolled_back_state) self.assertNotIn("removal_managed_default_address_pools", rolled_back_state) + def test_removal_failure_restores_snapshot_with_absent_managed_key(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-absent-managed-key" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(policy_env.read_bytes()) + daemon.write_text('{"debug":true,"live-restore":true}\n', encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + command_log = Path(self.tmp) / "absent-managed-key.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + restart = Path(self.tmp) / "restart.sh" + restart.write_text(f"#!/usr/bin/env bash\necho restart >> {command_log}\n", encoding="utf-8") + restart.chmod(0o755) + resume = Path(self.tmp) / "resume.sh" + resume.write_text( + "#!/usr/bin/env bash\n" + f"echo resume >> {command_log}\n" + 'grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2"\n', + encoding="utf-8", + ) + resume.chmod(0o755) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("managed daemon.json restored", removed.stderr) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"debug": True, "live-restore": True}) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "restart", "resume"]) + def test_removal_pending_state_is_durable_before_daemon_mutation(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-durable-removal-pending" @@ -2950,7 +3122,7 @@ def test_removal_rollback_resumes_and_checks_health_with_managed_env(self) -> No self.assertEqual(daemon.read_bytes(), managed) self.assertEqual( command_log.read_text(encoding="utf-8").splitlines(), - ["drain", "restart", "resume-candidate", "health-candidate", "restart", "resume-managed", "health-managed"], + ["drain", "restart", "resume-candidate", "health-candidate", "drain", "restart", "resume-managed", "health-managed"], ) def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) -> None: From c1314670a5feeb47463e8fa8d978b48b8e87b71f Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:36:41 -0500 Subject: [PATCH 32/54] fix: serialize network policy recovery --- scripts/apply-docker-network-policy.sh | 129 +++++---- scripts/test_apply_docker_network_policy.py | 278 +++++++++++++++++--- 2 files changed, 325 insertions(+), 82 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 35b015c0..e1de847a 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -143,6 +143,51 @@ done [[ -n "$env_file" ]] || die '--env is required' validate_trusted_path 'rendered env' "$env_file" regular +# Serialize with installer mutations before reading the rendered candidate. +lock_file=${CI_FLEET_INSTALLER_LOCK:-${CI_FLEET_ROOT_PREFIX:-}/run/ci-fleet-installer.lock} +if [[ -n ${CI_FLEET_INSTALLER_LOCK_FD:-} ]]; then + [[ "$CI_FLEET_INSTALLER_LOCK_FD" == 9 ]] || die 'inherited installer lock must use file descriptor 9' + [[ $(readlink -f /proc/self/fd/9 2>/dev/null || true) == $(readlink -m "$lock_file") ]] || die 'inherited installer lock does not match the configured lock file' + flock -n 9 || die 'inherited installer lock is unavailable' +else + validate_trusted_path CI_FLEET_INSTALLER_LOCK "$lock_file" regular true + exec python3 - "$lock_file" "$testing" "$repo_root/scripts/apply-docker-network-policy.sh" "${original_args[@]}" <<'PY' +import errno, os, stat, sys + +path, testing, script, *args = sys.argv[1:] +expected_owner = os.getuid() if testing == "1" else 0 +flags = os.O_RDWR | os.O_NOFOLLOW +try: + try: + fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600) + os.fchmod(fd, 0o600) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + fd = os.open(path, flags) + metadata = os.fstat(fd) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != expected_owner or metadata.st_mode & 0o022: + raise OSError + os.dup2(fd, 9) + if fd != 9: + os.close(fd) + os.set_inheritable(9, True) + os.environ["CI_FLEET_INSTALLER_LOCK_FD"] = "9" + os.execv(script, [script, *args]) +except OSError: + print("ERROR: CI_FLEET_INSTALLER_LOCK must remain a trusted root-owned regular file", file=sys.stderr) + raise SystemExit(2) +PY +fi + +temp_parent=${CI_FLEET_TEMP_DIR:-/tmp} +validate_trusted_path 'transaction temp directory' "$temp_parent" tempdir +work_dir=$(mktemp -d "$temp_parent/.ci-fleet-apply.XXXXXX") +chmod 0700 "$work_dir" +trap 'rm -rf "$work_dir"' EXIT +install -m 0600 -- "$env_file" "$work_dir/ci-fleet.env" +env_file=$work_dir/ci-fleet.env + # --- No-op when no network policy is rendered --- removing=false if ! grep -q '^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=' "$env_file"; then @@ -155,7 +200,7 @@ else [[ -n "$checkpoint_dir" ]] || die '--checkpoint is required when a network policy is configured' fi -# Validate the rendered policy before acquiring the installer lock. +# Validate the locked candidate snapshot before mutation. if [[ "$removing" == false ]]; then desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" 2>/dev/null <<'PY' import json, sys @@ -204,6 +249,7 @@ run_health() { validate_trusted_path CI_FLEET_DOCKER_DAEMON_CONFIG "$daemon_config" regular true validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true checkpoint_state=$checkpoint_dir/docker-network-policy.json +[[ "$removing" != true || ! -L "$checkpoint_state" ]] || die 'network-policy checkpoint state is invalid' if [[ "$removing" == true && ( ! -e "$checkpoint_dir" || ! -e "$checkpoint_state" ) ]]; then printf 'NETWORK_POLICY_NOOP\n' exit 0 @@ -225,51 +271,6 @@ if [[ -d "$checkpoint_dir" ]]; then exec 8<&- fi -# Serialize with installer mutations using the installer's host-local lock. -lock_file=${CI_FLEET_INSTALLER_LOCK:-${CI_FLEET_ROOT_PREFIX:-}/run/ci-fleet-installer.lock} -if [[ -n ${CI_FLEET_INSTALLER_LOCK_FD:-} ]]; then - [[ "$CI_FLEET_INSTALLER_LOCK_FD" == 9 ]] || die 'inherited installer lock must use file descriptor 9' - [[ $(readlink -f /proc/self/fd/9 2>/dev/null || true) == $(readlink -m "$lock_file") ]] || die 'inherited installer lock does not match the configured lock file' - flock -n 9 || die 'inherited installer lock is unavailable' -else - validate_trusted_path CI_FLEET_INSTALLER_LOCK "$lock_file" regular true - exec python3 - "$lock_file" "$testing" "$repo_root/scripts/apply-docker-network-policy.sh" "${original_args[@]}" <<'PY' -import errno, os, stat, sys - -path, testing, script, *args = sys.argv[1:] -expected_owner = os.getuid() if testing == "1" else 0 -flags = os.O_RDWR | os.O_NOFOLLOW -try: - try: - fd = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600) - os.fchmod(fd, 0o600) - except OSError as exc: - if exc.errno != errno.EEXIST: - raise - fd = os.open(path, flags) - metadata = os.fstat(fd) - if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != expected_owner or metadata.st_mode & 0o022: - raise OSError - os.dup2(fd, 9) - if fd != 9: - os.close(fd) - os.set_inheritable(9, True) - os.environ["CI_FLEET_INSTALLER_LOCK_FD"] = "9" - os.execv(script, [script, *args]) -except OSError: - print("ERROR: CI_FLEET_INSTALLER_LOCK must remain a trusted root-owned regular file", file=sys.stderr) - raise SystemExit(2) -PY -fi - -temp_parent=${CI_FLEET_TEMP_DIR:-/tmp} -validate_trusted_path 'transaction temp directory' "$temp_parent" tempdir -work_dir=$(mktemp -d "$temp_parent/.ci-fleet-apply.XXXXXX") -chmod 0700 "$work_dir" -trap 'rm -rf "$work_dir"' EXIT -install -m 0600 -- "$env_file" "$work_dir/ci-fleet.env" -env_file=$work_dir/ci-fleet.env - validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true true exec 8<"$checkpoint_dir" checkpoint_path_is_pinned || die 'checkpoint directory must remain a trusted root-owned path' @@ -289,16 +290,19 @@ controller_resumed=false transaction_recovery= # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below resume_after_failed_drain() { - local status=$? resume_failed=0 + local status=$? resume_failed=0 health_failed=0 trap - EXIT INT TERM run_command "$resume_command" --env "$prior_env" || resume_failed=1 - if ((resume_failed == 0)) && [[ "$new_marker" == true ]]; then + ((resume_failed != 0)) || run_health "$prior_env" || health_failed=1 + if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true ]]; then clear_managed_marker || resume_failed=1 fi [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" rm -rf "$work_dir" if ((resume_failed)); then printf 'ERROR: %s; controller resume command failed\n' "$drain_failure" >&2 + elif ((health_failed)); then + printf 'ERROR: %s; prior controller health check failed\n' "$drain_failure" >&2 else printf 'ERROR: %s\n' "$drain_failure" >&2 fi @@ -523,6 +527,11 @@ if is_pending and ( and not isinstance(state["removal_managed_default_address_pools"], list) ): raise SystemExit(1) +if is_pending and state["removal_managed_default_address_pools"] is not None: + validate_docker_address_pools( + state["removal_managed_default_address_pools"], + path="checkpoint removal managed default address pools", + ) generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) @@ -651,7 +660,11 @@ PY fi if ((failed == 0)); then run_command "$restart_command" "$daemon_dir" || failed=1 + fi + if ((failed == 0)); then run_command "$resume_command" --env "$prior_env" || failed=1 + fi + if ((failed == 0)); then run_health "$prior_env" || failed=1 fi if ((failed == 0)); then @@ -713,7 +726,19 @@ PY # unlink so failure still rolls back and success cannot leave a stale marker. trap '' INT TERM checkpoint_path_is_pinned || { removal_failure='checkpoint directory changed during network-policy removal'; exit 2; } - if ! rm -f "$state_file"; then + python3 - "$daemon_config" "$removal_daemon" 2>/dev/null <<'PY' || { removal_failure='daemon.json changed after network-policy removal verification'; exit 2; } +import json, os, sys +current = json.load(open(sys.argv[1], encoding="utf-8")) if os.path.exists(sys.argv[1]) else {} +expected = json.load(open(sys.argv[2], encoding="utf-8")) +missing = object() +if ( + not isinstance(current, dict) + or not isinstance(expected, dict) + or current.get("default-address-pools", missing) != expected.get("default-address-pools", missing) +): + raise SystemExit(1) +PY + if ! clear_managed_marker; then removal_failure='failed to clear network-policy managed marker' exit 2 fi @@ -902,8 +927,14 @@ rollback_daemon() { fi if ((failed == 0)); then restore_daemon || failed=1 + fi + if ((failed == 0)); then run_command "$restart_command" "$daemon_dir" || failed=1 + fi + if ((failed == 0)); then run_command "$resume_command" --env "$prior_env" || failed=1 + fi + if ((failed == 0)); then run_health "$prior_env" || failed=1 fi if [[ "$managed_before" == true && "$failed" == 0 ]]; then diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index b359f4dc..10790311 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -584,7 +584,7 @@ def test_daemon_change_during_drain_aborts_and_preserves_update(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stderr, "ERROR: daemon.json changed during network-policy apply\n") - self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume"]) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume", "health"]) self.assertEqual(daemon.read_bytes(), changed) self.assertFalse((checkpoint / "docker-network-policy.json").exists()) @@ -868,7 +868,7 @@ def test_policy_reapply_rejects_persistent_recovery_backup(self) -> None: self.assertEqual(daemon.read_bytes(), prior) self.assertTrue(all(not marker.exists() for marker in markers)) - def test_rejects_absent_daemon_config_in_checkpoint_before_lock_or_drain(self) -> None: + def test_rejects_absent_daemon_config_in_checkpoint_after_lock_before_drain(self) -> None: self.daemon_dir.chmod(0o700) self._write_success_commands() env_file = self._write_env_file(self._rendered_with_policy()) @@ -893,10 +893,10 @@ def test_rejects_absent_daemon_config_in_checkpoint_before_lock_or_drain(self) - self.assertNotEqual(result.returncode, 0) self.assertIn("daemon config and checkpoint entry must be separate paths", result.stderr) self.assertFalse((self.daemon_dir / "daemon.json").exists()) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) self.assertFalse(drain_marker.exists()) - def test_rejects_relative_daemon_config_before_lock(self) -> None: + def test_rejects_relative_daemon_config_before_drain(self) -> None: self._write_success_commands() env_file = self._write_env_file(self._rendered_with_policy()) lock = Path(self.tmp) / "network-policy.lock" @@ -920,9 +920,9 @@ def test_rejects_relative_daemon_config_before_lock(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("trusted root-owned path", result.stderr) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) - def test_rejects_fifo_daemon_config_before_lock(self) -> None: + def test_rejects_fifo_daemon_config_before_drain(self) -> None: daemon = self.daemon_dir / "daemon.json" os.mkfifo(daemon) self._write_success_commands() @@ -950,9 +950,9 @@ def test_rejects_fifo_daemon_config_before_lock(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("trusted root-owned path", result.stderr) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) - def test_rejects_group_writable_daemon_file_before_lock(self) -> None: + def test_rejects_group_writable_daemon_file_before_drain(self) -> None: daemon = self._write_daemon("{}\n") daemon.chmod(0o664) self._write_success_commands() @@ -975,9 +975,9 @@ def test_rejects_group_writable_daemon_file_before_lock(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("trusted root-owned path", result.stderr) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) - def test_rejects_group_writable_daemon_directory_before_lock(self) -> None: + def test_rejects_group_writable_daemon_directory_before_drain(self) -> None: self._write_daemon("{}\n") self._write_success_commands() self.daemon_dir.chmod(0o775) @@ -1000,9 +1000,9 @@ def test_rejects_group_writable_daemon_directory_before_lock(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("trusted root-owned path", result.stderr) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) - def test_rejects_hook_below_group_writable_parent_before_lock(self) -> None: + def test_rejects_hook_below_group_writable_parent_before_drain(self) -> None: self._write_daemon("{}\n") self._write_success_commands() hook_dir = Path(self.tmp) / "writable-hook-parent" @@ -1033,9 +1033,9 @@ def test_rejects_hook_below_group_writable_parent_before_lock(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("trusted root-owned path", result.stderr) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) - def test_rejects_group_writable_injected_hook_before_lock(self) -> None: + def test_rejects_group_writable_injected_hook_before_drain(self) -> None: self._write_daemon("{}\n") self._write_success_commands() self.drain_command.chmod(0o775) @@ -1058,7 +1058,7 @@ def test_rejects_group_writable_injected_hook_before_lock(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("trusted root-owned path", result.stderr) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) def test_injected_hooks_require_absolute_canonical_regular_executables(self) -> None: env_file = self._write_env_file(self._rendered_with_policy()) @@ -1247,6 +1247,47 @@ def test_env_file_is_trusted_and_snapshotted_before_mutation(self) -> None: daemon = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8")) self.assertEqual(len(daemon["default-address-pools"]), 2) + def test_candidate_is_snapshotted_before_rendering_under_lock(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + rendered = self._rendered_with_policy() + env_file = self._write_env_file(rendered) + replacement = dict(rendered) + replacement["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + replacement_file = Path(self.tmp) / "replacement.env" + replacement_file.write_text( + "".join(f"{key}={value}\n" for key, value in sorted(replacement.items())), + encoding="utf-8", + ) + fake_bin = Path(self.tmp) / "candidate-snapshot-bin" + fake_bin.mkdir() + install = fake_bin / "install" + install.write_text( + "#!/usr/bin/env bash\n" + f"if [[ ${{*: -1}} == */ci-fleet.env ]]; then {shutil.which('cp')} {replacement_file} {env_file}; fi\n" + f"exec {shutil.which('install')} \"$@\"\n", + encoding="utf-8", + ) + install.chmod(0o755) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-candidate-snapshot"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(PATH=f"{fake_bin}:{os.environ['PATH']}"), + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + pools = json.loads((self.daemon_dir / "daemon.json").read_text(encoding="utf-8"))["default-address-pools"] + self.assertEqual(pools[0]["base"], "192.0.2.0/24") + def test_env_file_below_writable_parent_is_rejected_before_side_effects(self) -> None: self._write_daemon("{}\n") self._write_success_commands() @@ -1380,6 +1421,32 @@ def test_existing_installer_lock_blocks_before_drain_or_checkpoint(self) -> None self.assertFalse(drain_marker.exists()) self.assertFalse(checkpoint.exists()) + def test_removal_noop_requires_installer_lock(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + env_file = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + lock = Path(self.tmp) / "removal-noop.lock" + checkpoint = Path(self.tmp) / "checkpoint-removal-noop-lock" + + with lock.open("w") as lock_handle: + fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("NETWORK_POLICY_NOOP", result.stdout) + def test_symlinked_installer_lock_below_writable_parent_is_rejected_without_side_effects(self) -> None: daemon = self._write_daemon("{}\n") prior = daemon.read_bytes() @@ -1573,7 +1640,7 @@ def test_matching_file_without_verified_generation_runs_activation_and_marks_ver verified = json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"] self.assertEqual(verified, hashlib.sha256((self.daemon_dir / "daemon.json").read_bytes()).hexdigest()) - def test_rejects_explicit_zero_on_managed_host_before_checkpoint_lock_or_drain(self) -> None: + def test_rejects_explicit_zero_on_managed_host_after_lock_before_checkpoint_or_drain(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-explicit-zero" self._write_success_commands() @@ -1600,10 +1667,10 @@ def test_rejects_explicit_zero_on_managed_host_before_checkpoint_lock_or_drain(s self.assertEqual(rejected.stderr, "ERROR: daemon policy rendering failed\n") self.assertEqual(daemon.read_bytes(), managed) self.assertEqual(state_file.read_bytes(), state) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) self.assertFalse(drain_marker.exists()) - def test_rejects_overlapping_rendered_pools_before_lock_or_drain(self) -> None: + def test_rejects_overlapping_rendered_pools_after_lock_before_drain(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self._write_daemon(prior.decode()) self._write_success_commands() @@ -1625,10 +1692,10 @@ def test_rejects_overlapping_rendered_pools_before_lock_or_drain(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stderr, "ERROR: daemon policy rendering failed\n") self.assertEqual(daemon.read_bytes(), prior) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) self.assertFalse(drain_marker.exists()) - def test_rejects_insufficient_rendered_capacity_before_lock_or_drain(self) -> None: + def test_rejects_insufficient_rendered_capacity_after_lock_before_drain(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self._write_daemon(prior.decode()) self._write_success_commands() @@ -1652,7 +1719,7 @@ def test_rejects_insufficient_rendered_capacity_before_lock_or_drain(self) -> No self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stderr, "ERROR: daemon policy rendering failed\n") self.assertEqual(daemon.read_bytes(), prior) - self.assertFalse(lock.exists()) + self.assertTrue(lock.exists()) self.assertFalse(drain_marker.exists()) def test_validates_policy_before_mutation(self) -> None: @@ -1832,7 +1899,7 @@ def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(daemon.read_bytes(), prior) self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) - self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) + self.assertFalse(health_log.exists()) self.assertNotIn("198.51.100.0/24", result.stdout + result.stderr) self.assertNotIn("super-secret", result.stdout + result.stderr) self.assertNotIn("secret.example.invalid", result.stdout + result.stderr) @@ -1934,7 +2001,7 @@ def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> self.assertNotEqual(process.returncode, 0) self.assertEqual(daemon.read_bytes(), prior) self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) - self.assertEqual(health_log.read_text(encoding="utf-8").splitlines(), ["health"]) + self.assertFalse(health_log.exists()) self.assertNotIn("rollback-restart-output", stdout + stderr) self.assertNotIn("rollback-health-output", stdout + stderr) recovery = Path((stdout + stderr).rstrip().rsplit("recovery data retained at ", 1)[1]) @@ -2524,7 +2591,7 @@ def test_removal_daemon_change_during_drain_aborts_and_preserves_update(self) -> self.assertNotEqual(removed.returncode, 0) self.assertEqual(removed.stderr, "ERROR: daemon.json changed during network-policy removal\n") - self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume"]) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume", "health"]) self.assertEqual(daemon.read_bytes(), changed_bytes) self.assertEqual(state_file.read_bytes(), prior_state) @@ -2645,6 +2712,36 @@ def test_removal_rejects_malformed_saved_pools_before_commands_or_mutation(self) self.assertEqual(daemon.read_bytes(), managed) self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_removal_rejects_malformed_pending_pools_before_commands_or_mutation(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-malformed-pending-pools" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = [] + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + managed = daemon.read_bytes() + command_markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): + marker = Path(self.tmp) / f"malformed-pending-pools-{name}.marker" + command_markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("network-policy checkpoint state is invalid", removed.stderr) + self.assertEqual(daemon.read_bytes(), managed) + self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_managed_policy_removal_replaces_daemon_atomically(self) -> None: prior = b'{"icc":false}\n' daemon = self.daemon_dir / "daemon.json" @@ -2824,6 +2921,12 @@ def test_uncertain_removal_rollback_retains_pending_state(self) -> None: restart.chmod(0o755) (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") (Path(self.tmp) / "probe.sh").chmod(0o755) + resume_marker = Path(self.tmp) / "uncertain-removal-resume.marker" + (Path(self.tmp) / "resume.sh").write_text( + f"#!/usr/bin/env bash\ntouch {resume_marker}\n", + encoding="utf-8", + ) + (Path(self.tmp) / "resume.sh").chmod(0o755) no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) @@ -2832,6 +2935,7 @@ def test_uncertain_removal_rollback_retains_pending_state(self) -> None: state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) self.assertEqual(state["phase"], "removal-pending") self.assertEqual(state["removal_managed_default_address_pools"], managed_pools) + self.assertFalse(resume_marker.exists()) def test_failed_removal_from_absent_baseline_persists_durable_recovery(self) -> None: daemon = self.daemon_dir / "daemon.json" @@ -2939,6 +3043,98 @@ def test_successful_removal_runs_runtime_checks_before_deleting_marker(self) -> self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "resume", "health"]) self.assertFalse(state_file.exists()) + def test_successful_removal_fsyncs_checkpoint_after_marker_unlink(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-marker-fsync" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + audit_log = Path(self.tmp) / "marker-fsync.log" + audit_dir = Path(self.tmp) / "marker-fsync-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_unlink = os.unlink\n" + "_fsync = os.fsync\n" + "_log = os.environ['FSYNC_AUDIT_LOG']\n" + "def record(value):\n" + " with open(_log, 'a', encoding='utf-8') as handle: handle.write(value + '\\n')\n" + "def unlink(path, *args, **kwargs):\n" + " if os.path.realpath(path) == os.environ['STATE_FILE']: record('U ' + os.environ['STATE_FILE'])\n" + " return _unlink(path, *args, **kwargs)\n" + "def fsync(fd):\n" + " record('F ' + os.path.realpath(f'/proc/self/fd/{fd}'))\n" + " return _fsync(fd)\n" + "os.unlink = unlink\n" + "os.fsync = fsync\n", + encoding="utf-8", + ) + fake_bin = Path(self.tmp) / "marker-fsync-bin" + fake_bin.mkdir() + (fake_bin / "rm").write_text( + "#!/usr/bin/env bash\n" + f"if [[ ${{*: -1}} == /proc/self/fd/*/docker-network-policy.json ]]; then printf 'U %s\\n' {state_file} >> {audit_log}; fi\n" + f"exec {shutil.which('rm')} \"$@\"\n", + encoding="utf-8", + ) + (fake_bin / "rm").chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, + text=True, + env=self._env( + PATH=f"{fake_bin}:{os.environ['PATH']}", + PYTHONPATH=str(audit_dir), + FSYNC_AUDIT_LOG=str(audit_log), + STATE_FILE=str(state_file), + ), + timeout=30, + ) + + self.assertEqual(removed.returncode, 0, removed.stderr) + events = audit_log.read_text(encoding="utf-8").splitlines() + unlink = events.index(f"U {state_file}") + self.assertIn(f"F {checkpoint}", events[unlink + 1 :]) + + def test_removal_retains_marker_if_managed_key_reappears_during_verification(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-key-conflict" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + mutator = Path(self.tmp) / "reintroduce-pools.py" + mutator.write_text( + "import json, sys\n" + "path = sys.argv[1]\n" + "value = json.load(open(path))\n" + "value['default-address-pools'] = [{'base': '192.0.2.0/24', 'size': 29}]\n" + "with open(path, 'w') as handle: json.dump(value, handle)\n", + encoding="utf-8", + ) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + 'grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2" || ' + f"{shutil.which('python3')} {mutator} {daemon}\n", + encoding="utf-8", + ) + health.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertTrue(state_file.exists()) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"], managed_pools) + def test_managed_policy_removal_restores_original_state_and_clears_only_marker(self) -> None: for prior_present in (False, True): with self.subTest(prior_present=prior_present): @@ -3145,17 +3341,18 @@ def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) health.chmod(0o755) no_policy_env = Path(self.tmp) / "no-policy-marker-failure.env" no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") - fake_bin = Path(self.tmp) / "fake-bin" - fake_bin.mkdir() - fake_rm = fake_bin / "rm" - fake_rm.write_text( - "#!/usr/bin/env bash\n" - "[[ ${*: -1} != /proc/self/fd/*/docker-network-policy.json ]] || exit 1\n" - f"exec {shutil.which('rm')} \"$@\"\n", + audit_dir = Path(self.tmp) / "marker-failure-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_unlink = os.unlink\n" + "def unlink(path, *args, **kwargs):\n" + " if os.path.realpath(path) == os.environ['STATE_FILE']: raise OSError\n" + " return _unlink(path, *args, **kwargs)\n" + "os.unlink = unlink\n", encoding="utf-8", ) - fake_rm.chmod(0o755) - env = self._env(PATH=f"{fake_bin}:{os.environ['PATH']}") + env = self._env(PYTHONPATH=str(audit_dir), STATE_FILE=str(state_file)) removed = subprocess.run( [ @@ -3198,6 +3395,21 @@ def test_unmanaged_no_policy_is_noop_without_mutation_or_commands(self) -> None: self.assertFalse(checkpoint.exists()) self.assertTrue(all(not marker.exists() for marker in markers)) + def test_removal_rejects_broken_checkpoint_state_symlink(self) -> None: + daemon = self._write_daemon('{"default-address-pools":[]}\n') + prior = daemon.read_bytes() + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-broken-state-link" + checkpoint.mkdir(mode=0o700) + (checkpoint / "docker-network-policy.json").symlink_to(checkpoint / "missing-state") + env_file = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkpoint state is invalid", result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + if __name__ == "__main__": unittest.main() From f0eb7e5863abe543677bb6e60712f7561a17069b Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 01:54:51 -0500 Subject: [PATCH 33/54] fix: redact rendered network policy failures --- scripts/desired_state.py | 4 ++-- scripts/test_apply_docker_network_policy.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 70335504..d4753090 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -224,7 +224,7 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: try: network = ipaddress.ip_network(base, strict=True) except ValueError as exc: - raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE: malformed CIDR {base!r}") from exc + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE: malformed CIDR") from exc if network.version != 4: raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE: must be IPv4") try: @@ -234,7 +234,7 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: if not isinstance(size, int) or size < 0 or size > 29: raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: must be between 0 and 29") if size < network.prefixlen: - raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: impossible subnet count for {base!r}") + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: impossible subnet count") pools.append({"base": base, "size": size}) try: policy = { diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 10790311..6c45dff7 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -111,6 +111,19 @@ def test_rejects_mismatched_pool_size(self) -> None: with self.assertRaisesRegex(ValueError, "must be an integer"): render_docker_daemon_config(env) + def test_renderer_failure_does_not_disclose_pool_base(self) -> None: + base = "10.123.45.0/24" + env = { + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "1", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE": base, + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_SIZE": "16", + } + + with self.assertRaises(ValueError) as raised: + render_docker_daemon_config(env) + + self.assertNotIn(base, str(raised.exception)) + def test_rejects_ipv6_pool(self) -> None: env = { "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "1", From f08cbc03b496b2d87cf900e62d93c6443fd615f8 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 02:15:56 -0500 Subject: [PATCH 34/54] fix: close network policy recovery gaps --- scripts/apply-docker-network-policy.sh | 31 ++++++++-- scripts/test_apply_docker_network_policy.py | 63 +++++++++++++++++++++ 2 files changed, 88 insertions(+), 6 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index e1de847a..b0756b9a 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -774,15 +774,27 @@ PY managed_before=false prior_verified_generation= +apply_removal_pending=false if [[ -e "$state_file" ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } - prior_verified_generation=$( - python3 - "$state_file" <<'PY' + apply_checkpoint_state=$( + python3 - "$state_file" "$repo_root/scripts" 2>/dev/null <<'PY' import json, re, sys state = json.load(open(sys.argv[1], encoding="utf-8")) +sys.path.insert(0, sys.argv[2]) +from desired_state import validate_docker_address_pools required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} -if set(state) not in (required, required | {"verified_generation"}) or state["managed"] is not True: +pending = {"phase", "removal_managed_default_address_pools"} +if set(state) not in (required, required | {"verified_generation"}, required | pending, required | pending | {"verified_generation"}) or state["managed"] is not True: raise SystemExit(1) +is_pending = "phase" in state +if is_pending and state["phase"] != "removal-pending": + raise SystemExit(1) +if is_pending and state["removal_managed_default_address_pools"] is not None: + validate_docker_address_pools( + state["removal_managed_default_address_pools"], + path="checkpoint removal managed default address pools", + ) generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) @@ -799,9 +811,10 @@ if state["prior_present"]: raise SystemExit(1) elif state["prior_mode"] is not None: raise SystemExit(1) -print(generation or "") +print(f"{generation or ''}|{'true' if is_pending else 'false'}") PY ) || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } + IFS='|' read -r prior_verified_generation apply_removal_pending <<<"$apply_checkpoint_state" managed_before=true fi @@ -883,11 +896,15 @@ PY # Record the original host state before the first drain. The marker lets a # later no-policy reconciliation resume a controller interrupted by the drain. if [[ "$managed_before" == false ]]; then - python3 - "$state_file" "$had_prior" "$daemon_mode" "$backup_dir/$backup_name" <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } + python3 - "$state_file" "$had_prior" "$daemon_mode" "$backup_dir/$backup_name" "$repo_root/scripts" 2>/dev/null <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } import json, os, sys, tempfile path = sys.argv[1] prior = json.load(open(sys.argv[4], encoding="utf-8")) if sys.argv[2] == "true" else {} prior_key_present = "default-address-pools" in prior +if prior_key_present: + sys.path.insert(0, sys.argv[5]) + from desired_state import validate_docker_address_pools + validate_docker_address_pools(prior["default-address-pools"], path="existing daemon default address pools") state = { "managed": True, "prior_default_address_pools": prior.get("default-address-pools") if prior_key_present else None, @@ -1027,7 +1044,9 @@ if not isinstance(current, dict) or current.get("default-address-pools") != stag raise SystemExit(1) PY verified_generation=$(file_generation "$daemon_config") || fail_after_apply "failed to identify verified daemon.json generation" -set_verified_generation "$verified_generation" || fail_after_apply "failed to record verified daemon.json generation" +verified_action= +[[ "$apply_removal_pending" != true ]] || verified_action=clear-removal +set_verified_generation "$verified_generation" "$verified_action" || fail_after_apply "failed to record verified daemon.json generation" # --- Success --- trap - EXIT INT TERM diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 6c45dff7..d80dbd93 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -822,6 +822,22 @@ def test_successful_apply_records_managed_original_presence_and_mode(self) -> No self.assertEqual((checkpoint / "docker-network-policy.json").stat().st_mode & 0o777, 0o600) self.assertFalse((checkpoint / "daemon.json").exists()) + def test_first_apply_rejects_invalid_baseline_pools_before_checkpoint_or_drain(self) -> None: + daemon = self._write_daemon('{"default-address-pools":[],"live-restore":true}\n') + prior = daemon.read_bytes() + checkpoint = Path(self.tmp) / "checkpoint-invalid-baseline-pools" + self._write_success_commands() + drain_marker = Path(self.tmp) / "invalid-baseline-pools-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse((checkpoint / "docker-network-policy.json").exists()) + self.assertFalse(drain_marker.exists()) + def test_policy_reapply_fails_closed_when_managed_key_provenance_is_missing(self) -> None: rendered = self._rendered_with_policy() daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) @@ -2827,6 +2843,53 @@ def test_interrupted_removal_retry_uses_persisted_managed_pools_for_rollback(sel self.assertNotIn("phase", rolled_back_state) self.assertNotIn("removal_managed_default_address_pools", rolled_back_state) + def test_configured_apply_resumes_removal_pending_checkpoint(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-removal-pending-apply" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + state["verified_generation"] = None + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + daemon.write_text('{"live-restore":true}\n', encoding="utf-8") + + resumed = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(resumed.returncode, 0, resumed.stderr) + self.assertIn("default-address-pools", json.loads(daemon.read_text(encoding="utf-8"))) + resumed_state = json.loads(state_file.read_text(encoding="utf-8")) + self.assertNotIn("phase", resumed_state) + self.assertNotIn("removal_managed_default_address_pools", resumed_state) + + def test_configured_apply_rejects_invalid_removal_pending_pools_before_drain(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-invalid-removal-pending-apply" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = [] + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + managed = daemon.read_bytes() + drain_marker = Path(self.tmp) / "invalid-removal-pending-apply-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + rejected = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(rejected.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + self.assertFalse(drain_marker.exists()) + def test_removal_failure_restores_snapshot_with_absent_managed_key(self) -> None: daemon = self._write_daemon('{"live-restore":true}\n') checkpoint = Path(self.tmp) / "checkpoint-absent-managed-key" From 6aeae2efa44079aff7af9b72647c2269fb8ad516 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 02:35:12 -0500 Subject: [PATCH 35/54] fix: close network policy recovery lock gaps --- scripts/apply-docker-network-policy.sh | 6 ++- scripts/test_apply_docker_network_policy.py | 57 +++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index b0756b9a..18cea971 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -235,7 +235,7 @@ validate_command() { } run_command() { - timeout --kill-after=5 "$command_timeout" "$@" >/dev/null 2>&1 + timeout --kill-after=5 "$command_timeout" "$@" 9>&- >/dev/null 2>&1 } run_health() { @@ -934,7 +934,9 @@ PY new_marker=true fi if [[ "$managed_before" == true ]]; then - transaction_recovery=$(persist_recovery "$backup_dir/$backup_name" "$prior_env") || die 'failed to persist network-policy transaction recovery' + recovery_daemon= + [[ "$had_prior" != true ]] || recovery_daemon=$backup_dir/$backup_name + transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' fi rollback_daemon() { diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index d80dbd93..61af0943 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -625,6 +625,26 @@ def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"], prior_generation) self.assertEqual(hashlib.sha256(daemon.read_bytes()).hexdigest(), prior_generation) + def test_managed_reapply_recovers_when_daemon_snapshot_is_absent(self) -> None: + daemon = self.daemon_dir / "daemon.json" + checkpoint = Path(self.tmp) / "checkpoint-reapply-absent-daemon" + self._write_success_commands() + prior_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(prior_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(prior_env.read_bytes()) + daemon.unlink() + candidate = self._rendered_with_policy() + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = Path(self.tmp) / "candidate-reapply-absent-daemon.env" + candidate_env.write_text("".join(f"{key}={value}\n" for key, value in sorted(candidate.items())), encoding="utf-8") + + result = self._run(str(candidate_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue(daemon.is_file()) + self.assertFalse(list(checkpoint.glob("recovery.*"))) + def test_failed_managed_reapply_leaves_generation_unverified_when_rollback_health_fails(self) -> None: self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-failed-rollback-generation" @@ -1450,6 +1470,43 @@ def test_existing_installer_lock_blocks_before_drain_or_checkpoint(self) -> None self.assertFalse(drain_marker.exists()) self.assertFalse(checkpoint.exists()) + def test_hook_descendants_do_not_retain_installer_lock(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + lock = Path(self.tmp) / "hook-descendant.lock" + child_pid = Path(self.tmp) / "hook-descendant.pid" + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + "sleep 30 &\n" + f"printf '%s\\n' \"$!\" > {child_pid}\n", + encoding="utf-8", + ) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(Path(self.tmp) / "checkpoint-hook-descendant"), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + pid = int(child_pid.read_text(encoding="utf-8")) + try: + with lock.open("w") as lock_handle: + try: + fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) + acquired = True + except BlockingIOError: + acquired = False + self.assertTrue(acquired) + finally: + try: + os.kill(pid, signal.SIGTERM) + except ProcessLookupError: + pass + def test_removal_noop_requires_installer_lock(self) -> None: self._write_daemon("{}\n") self._write_success_commands() From 4ac9ec72c40fc02dca4bb18c80f30ec7458e0593 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:03:38 -0500 Subject: [PATCH 36/54] fix: harden network policy retry rollback --- scripts/apply-docker-network-policy.sh | 87 ++++++++++++++++++--- scripts/test_apply_docker_network_policy.py | 86 ++++++++++++++++++++ 2 files changed, 160 insertions(+), 13 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 18cea971..7d7f1b17 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -333,10 +333,12 @@ fi # --- Ownership guard (relaxed in testing) --- daemon_mode=644 daemon_gid=0 +daemon_metadata= [[ "$testing" != 1 ]] || daemon_gid=$(id -g) if [[ -f "$daemon_config" ]]; then daemon_mode=$(stat -c %a "$daemon_config") daemon_gid=$(stat -c %g "$daemon_config") + daemon_metadata=$(stat -c '%d:%i:%u:%a:%g' "$daemon_config") fi if [[ "$testing" != 1 ]]; then [[ -w "$(dirname "$daemon_config")" ]] || die "daemon config directory is not writable: $(dirname "$daemon_config")" @@ -382,12 +384,41 @@ PY } daemon_changed_since_snapshot() { - local snapshot=$1 was_present=$2 + local snapshot=$1 was_present=$2 expected_metadata=${3:-} if [[ "$was_present" == true ]]; then - if cmp -s "$snapshot" "$daemon_config"; then - return 1 - fi - return 0 + python3 - "$snapshot" "$daemon_config" "$expected_metadata" <<'PY' +import os, stat, sys +snapshot_path, daemon_path, expected = sys.argv[1:] +try: + device, inode, uid, mode, gid = expected.split(":") + expected_metadata = (int(device), int(inode), int(uid), int(mode, 8), int(gid)) + fd = os.open(daemon_path, os.O_RDONLY | os.O_NOFOLLOW) + with open(snapshot_path, "rb") as snapshot, os.fdopen(fd, "rb") as daemon: + before = os.fstat(daemon.fileno()) + metadata = ( + before.st_dev, + before.st_ino, + before.st_uid, + stat.S_IMODE(before.st_mode), + before.st_gid, + ) + same = metadata == expected_metadata and snapshot.read() == daemon.read() + after = os.fstat(daemon.fileno()) + stable = lambda value: ( + value.st_dev, + value.st_ino, + value.st_uid, + stat.S_IMODE(value.st_mode), + value.st_gid, + value.st_size, + value.st_mtime_ns, + value.st_ctime_ns, + ) + same = same and stable(before) == stable(after) +except (OSError, ValueError): + same = False +raise SystemExit(1 if same else 0) +PY else [[ -e "$daemon_config" || -L "$daemon_config" ]] fi @@ -617,11 +648,12 @@ PY } managed_mode=$(stat -c %a "$daemon_config") managed_gid=$(stat -c %g "$daemon_config") + managed_metadata=$(stat -c '%d:%i:%u:%a:%g' "$daemon_config") managed_snapshot=$work_dir/daemon.json.before cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } drain_controller 'drain command failed before network-policy removal' - if daemon_changed_since_snapshot "$managed_snapshot" true; then + if daemon_changed_since_snapshot "$managed_snapshot" true "$managed_metadata"; then drain_failure='daemon.json changed during network-policy removal' exit 2 fi @@ -775,6 +807,8 @@ PY managed_before=false prior_verified_generation= apply_removal_pending=false +checkpoint_prior_present= +checkpoint_prior_mode= if [[ -e "$state_file" ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } apply_checkpoint_state=$( @@ -804,6 +838,11 @@ if not isinstance(state["prior_default_address_pools_present"], bool): raise SystemExit(1) if state["prior_default_address_pools_present"] and not state["prior_present"]: raise SystemExit(1) +if state["prior_default_address_pools_present"]: + validate_docker_address_pools( + state["prior_default_address_pools"], + path="checkpoint prior default address pools", + ) if not state["prior_default_address_pools_present"] and state["prior_default_address_pools"] is not None: raise SystemExit(1) if state["prior_present"]: @@ -811,10 +850,13 @@ if state["prior_present"]: raise SystemExit(1) elif state["prior_mode"] is not None: raise SystemExit(1) -print(f"{generation or ''}|{'true' if is_pending else 'false'}") +print( + f"{generation or ''}|{'true' if is_pending else 'false'}|" + f"{'true' if state['prior_present'] else 'false'}|{state['prior_mode'] or ''}" +) PY ) || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } - IFS='|' read -r prior_verified_generation apply_removal_pending <<<"$apply_checkpoint_state" + IFS='|' read -r prior_verified_generation apply_removal_pending checkpoint_prior_present checkpoint_prior_mode <<<"$apply_checkpoint_state" managed_before=true fi @@ -844,10 +886,29 @@ if [[ -f "$daemon_config" ]]; then had_prior=true cp -p "$daemon_config" "$backup_dir/$backup_name" fi +rollback_source=$backup_dir/$backup_name +if [[ "$managed_before" == true && -z "$prior_verified_generation" ]]; then + rollback_source=$work_dir/daemon.json.durable-baseline + python3 - "$daemon_config" "$state_file" "$rollback_source" <<'PY' || { rm -rf "$work_dir"; die 'failed to construct durable rollback baseline'; } +import json, os, sys +current_path, state_path, output_path = sys.argv[1:] +current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} +state = json.load(open(state_path, encoding="utf-8")) +if state["prior_default_address_pools_present"]: + current["default-address-pools"] = state["prior_default_address_pools"] +else: + current.pop("default-address-pools", None) +with open(output_path, "w", encoding="utf-8") as handle: + json.dump(current, handle, indent=2, sort_keys=True) + handle.write("\n") +PY + had_prior=$checkpoint_prior_present + [[ "$had_prior" != true ]] || daemon_mode=$checkpoint_prior_mode +fi restore_daemon() { local rollback_daemon=$work_dir/daemon.json.apply-rollback rollback_action - rollback_action=$(python3 - "$daemon_config" "$backup_dir/$backup_name" "$had_prior" "$rollback_daemon" "$staging_daemon" <<'PY' + rollback_action=$(python3 - "$daemon_config" "$rollback_source" "$had_prior" "$rollback_daemon" "$staging_daemon" <<'PY' import json, os, sys current_path, prior_path, had_prior, output_path, staged_path = sys.argv[1:] try: @@ -877,7 +938,7 @@ else: PY ) || return 1 if [[ "$rollback_action" == exact ]]; then - atomic_replace_daemon "$backup_dir/$backup_name" "$daemon_mode" "$daemon_gid" + atomic_replace_daemon "$rollback_source" "$daemon_mode" "$daemon_gid" elif [[ "$rollback_action" == replace ]]; then atomic_replace_daemon "$rollback_daemon" "$daemon_mode" "$daemon_gid" else @@ -935,7 +996,7 @@ PY fi if [[ "$managed_before" == true ]]; then recovery_daemon= - [[ "$had_prior" != true ]] || recovery_daemon=$backup_dir/$backup_name + [[ "$had_prior" != true ]] || recovery_daemon=$rollback_source transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' fi @@ -979,7 +1040,7 @@ rollback_on_exit() { recovery_path=$transaction_recovery else recovery_daemon= - [[ "$had_prior" != true ]] || recovery_daemon=$backup_dir/$backup_name + [[ "$had_prior" != true ]] || recovery_daemon=$rollback_source recovery_path=$(persist_recovery "$recovery_daemon" "$prior_env") || { printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$transaction_failure" >&2 exit "$status" @@ -993,7 +1054,7 @@ rollback_on_exit() { # --- Drain after local validation/checkpointing, before mutation or restart --- drain_controller 'drain command failed before network-policy apply' -if daemon_changed_since_snapshot "$backup_dir/$backup_name" "$had_prior"; then +if daemon_changed_since_snapshot "$backup_dir/$backup_name" "$had_prior" "$daemon_metadata"; then drain_failure='daemon.json changed during network-policy apply' exit 2 fi diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 61af0943..a9bf499f 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -601,6 +601,29 @@ def test_daemon_change_during_drain_aborts_and_preserves_update(self) -> None: self.assertEqual(daemon.read_bytes(), changed) self.assertFalse((checkpoint / "docker-network-policy.json").exists()) + def test_daemon_metadata_change_during_drain_aborts(self) -> None: + for field, command in (("mode", "chmod 600"), ("group", "chgrp 1")): + with self.subTest(field=field): + daemon = self._write_daemon('{"bip":"172.17.0.1/16"}\n') + daemon.chmod(0o644) + os.chown(daemon, -1, 0) + checkpoint = Path(self.tmp) / f"checkpoint-daemon-{field}-conflict" + self.drain_command.write_text( + f"#!/usr/bin/env bash\n{command} {daemon}\n", + encoding="utf-8", + ) + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr, "ERROR: daemon.json changed during network-policy apply\n") + if field == "mode": + self.assertEqual(daemon.stat().st_mode & 0o777, 0o600) + else: + self.assertEqual(daemon.stat().st_gid, 1) + def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-reapply-generation" @@ -668,6 +691,40 @@ def test_failed_managed_reapply_leaves_generation_unverified_when_rollback_healt state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) self.assertIsNone(state["verified_generation"]) + def test_failed_unverified_retry_restores_durable_baseline(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps({ + "default-address-pools": render_docker_daemon_config(rendered)["default-address-pools"], + "live-restore": True, + })) + checkpoint = Path(self.tmp) / "checkpoint-unverified-retry" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, + "prior_mode": "600", + "prior_present": True, + "verified_generation": None, + }), + encoding="utf-8", + ) + state_file.chmod(0o600) + candidate = dict(rendered) + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + env_file = self._write_env_file(candidate) + self._write_success_commands() + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"live-restore": True}) + def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -858,6 +915,35 @@ def test_first_apply_rejects_invalid_baseline_pools_before_checkpoint_or_drain(s self.assertFalse((checkpoint / "docker-network-policy.json").exists()) self.assertFalse(drain_marker.exists()) + def test_reapply_rejects_invalid_baseline_pools_before_drain(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) + checkpoint = Path(self.tmp) / "checkpoint-invalid-reapply-baseline" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_default_address_pools": [], + "prior_default_address_pools_present": True, + "prior_mode": "600", + "prior_present": True, + "verified_generation": hashlib.sha256(daemon.read_bytes()).hexdigest(), + }), + encoding="utf-8", + ) + state_file.chmod(0o600) + drain_marker = Path(self.tmp) / "invalid-reapply-baseline-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + self._write_success_commands() + env_file = self._write_env_file(rendered) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkpoint state is invalid", result.stderr) + self.assertFalse(drain_marker.exists()) + def test_policy_reapply_fails_closed_when_managed_key_provenance_is_missing(self) -> None: rendered = self._rendered_with_policy() daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) From ba6cf539bfda9d1e28360991b4e6ab652d0e9d51 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:14:36 -0500 Subject: [PATCH 37/54] test: make daemon metadata regression portable --- scripts/test_apply_docker_network_policy.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index a9bf499f..89466afc 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -602,11 +602,14 @@ def test_daemon_change_during_drain_aborts_and_preserves_update(self) -> None: self.assertFalse((checkpoint / "docker-network-policy.json").exists()) def test_daemon_metadata_change_during_drain_aborts(self) -> None: - for field, command in (("mode", "chmod 600"), ("group", "chgrp 1")): + alternate_gid = 1 if os.geteuid() == 0 else next((gid for gid in os.getgroups() if gid != os.getgid()), None) + cases = [("mode", "chmod 600")] + if alternate_gid is not None: + cases.append(("group", f"chgrp {alternate_gid}")) + for field, command in cases: with self.subTest(field=field): daemon = self._write_daemon('{"bip":"172.17.0.1/16"}\n') daemon.chmod(0o644) - os.chown(daemon, -1, 0) checkpoint = Path(self.tmp) / f"checkpoint-daemon-{field}-conflict" self.drain_command.write_text( f"#!/usr/bin/env bash\n{command} {daemon}\n", @@ -622,7 +625,7 @@ def test_daemon_metadata_change_during_drain_aborts(self) -> None: if field == "mode": self.assertEqual(daemon.stat().st_mode & 0o777, 0o600) else: - self.assertEqual(daemon.stat().st_gid, 1) + self.assertEqual(daemon.stat().st_gid, alternate_gid) def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None: daemon = self._write_daemon("{}\n") From 94268a23bece8b4a549eba3f4178237e7368c82d Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 03:44:36 -0500 Subject: [PATCH 38/54] fix: harden network policy recovery durability --- scripts/apply-docker-network-policy.sh | 32 ++-- scripts/test_apply_docker_network_policy.py | 177 ++++++++++++++++++++ 2 files changed, 193 insertions(+), 16 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 7d7f1b17..69a28183 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -25,10 +25,10 @@ checkpoint_dir= usage() { cat >&2 <<'EOF' -usage: apply-docker-network-policy.sh --env PATH [--checkpoint PATH] +usage: apply-docker-network-policy.sh --env PATH --checkpoint PATH --env PATH path to the rendered ci-fleet env file (required) ---checkpoint PATH directory to back up the prior daemon.json into +--checkpoint PATH directory for durable network-policy state (required) EOF } @@ -97,8 +97,10 @@ elif create == "true": parent, leaf = os.path.split(path) parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW) try: + created = False try: os.mkdir(leaf, 0o700, dir_fd=parent_fd) + created = True except OSError as exc: if exc.errno != errno.EEXIST: raise @@ -109,6 +111,8 @@ elif create == "true": raise OSError finally: os.close(fd) + if created: + os.fsync(parent_fd) except OSError: raise SystemExit(1) finally: @@ -190,14 +194,9 @@ env_file=$work_dir/ci-fleet.env # --- No-op when no network policy is rendered --- removing=false +[[ -n "$checkpoint_dir" ]] || die '--checkpoint is required' if ! grep -q '^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=' "$env_file"; then - if [[ -z "$checkpoint_dir" ]]; then - printf 'NETWORK_POLICY_NOOP\n' - exit 0 - fi removing=true -else - [[ -n "$checkpoint_dir" ]] || die '--checkpoint is required when a network policy is configured' fi # Validate the locked candidate snapshot before mutation. @@ -297,7 +296,9 @@ resume_after_failed_drain() { if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true ]]; then clear_managed_marker || resume_failed=1 fi - [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" + if ((resume_failed == 0 && health_failed == 0)); then + [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" + fi rm -rf "$work_dir" if ((resume_failed)); then printf 'ERROR: %s; controller resume command failed\n' "$drain_failure" >&2 @@ -886,6 +887,7 @@ if [[ -f "$daemon_config" ]]; then had_prior=true cp -p "$daemon_config" "$backup_dir/$backup_name" fi +snapshot_present=$had_prior rollback_source=$backup_dir/$backup_name if [[ "$managed_before" == true && -z "$prior_verified_generation" ]]; then rollback_source=$work_dir/daemon.json.durable-baseline @@ -996,7 +998,7 @@ PY fi if [[ "$managed_before" == true ]]; then recovery_daemon= - [[ "$had_prior" != true ]] || recovery_daemon=$rollback_source + [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' fi @@ -1028,10 +1030,8 @@ rollback_on_exit() { local status=$? trap - EXIT INT TERM ((status != 0)) || status=1 - if rollback_daemon >/dev/null 2>&1; then - if [[ "$managed_before" == false ]]; then - rm -f "$state_file" - fi + if rollback_daemon >/dev/null 2>&1 && + { [[ "$managed_before" == true ]] || clear_managed_marker; }; then [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" rm -rf "$work_dir" printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 @@ -1040,7 +1040,7 @@ rollback_on_exit() { recovery_path=$transaction_recovery else recovery_daemon= - [[ "$had_prior" != true ]] || recovery_daemon=$rollback_source + [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name recovery_path=$(persist_recovery "$recovery_daemon" "$prior_env") || { printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$transaction_failure" >&2 exit "$status" @@ -1054,7 +1054,7 @@ rollback_on_exit() { # --- Drain after local validation/checkpointing, before mutation or restart --- drain_controller 'drain command failed before network-policy apply' -if daemon_changed_since_snapshot "$backup_dir/$backup_name" "$had_prior" "$daemon_metadata"; then +if daemon_changed_since_snapshot "$backup_dir/$backup_name" "$snapshot_present" "$daemon_metadata"; then drain_failure='daemon.json changed during network-policy apply' exit 2 fi diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 89466afc..c105f63a 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -601,6 +601,36 @@ def test_daemon_change_during_drain_aborts_and_preserves_update(self) -> None: self.assertEqual(daemon.read_bytes(), changed) self.assertFalse((checkpoint / "docker-network-policy.json").exists()) + def test_post_drain_conflict_retains_recovery_when_prior_resume_fails(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-post-drain-recovery" + self._write_success_commands() + prior_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(prior_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + prior_env_bytes = prior_env.read_bytes() + self.installed_env.write_bytes(prior_env_bytes) + prior_daemon = daemon.read_bytes() + candidate = self._rendered_with_policy() + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = self._write_env_file(candidate) + self.drain_command.write_text( + f"#!/usr/bin/env bash\nprintf '%s\\n' '{{\"debug\":true}}' > {daemon}\n", + encoding="utf-8", + ) + resume = Path(self.tmp) / "resume.sh" + resume.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + resume.chmod(0o755) + + result = self._run(str(candidate_env), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("controller resume command failed", result.stderr) + recoveries = list(checkpoint.glob("recovery.*")) + self.assertEqual(len(recoveries), 1) + self.assertEqual((recoveries[0] / "daemon.json.before").read_bytes(), prior_daemon) + self.assertEqual((recoveries[0] / "prior-ci-fleet.env").read_bytes(), prior_env_bytes) + def test_daemon_metadata_change_during_drain_aborts(self) -> None: alternate_gid = 1 if os.geteuid() == 0 else next((gid for gid in os.getgroups() if gid != os.getgid()), None) cases = [("mode", "chmod 600")] @@ -728,6 +758,34 @@ def test_failed_unverified_retry_restores_durable_baseline(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"live-restore": True}) + def test_unverified_retry_preserves_current_snapshot_presence(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) + checkpoint = Path(self.tmp) / "checkpoint-unverified-current-present" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, + "prior_mode": None, + "prior_present": False, + "verified_generation": None, + }), + encoding="utf-8", + ) + state_file.chmod(0o600) + self._write_success_commands() + env_file = self._write_env_file(rendered) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, f"NETWORK_POLICY_APPLIED daemon_config={daemon}\n") + state = json.loads(state_file.read_text(encoding="utf-8")) + self.assertEqual(state["verified_generation"], hashlib.sha256(daemon.read_bytes()).hexdigest()) + def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" @@ -744,6 +802,22 @@ def test_policy_apply_requires_checkpoint_before_drain_or_mutation(self) -> None self.assertEqual(daemon.read_bytes(), prior) self.assertFalse(drain_marker.exists()) + def test_policy_removal_requires_checkpoint_before_noop(self) -> None: + prior = b'{"default-address-pools":[{"base":"192.0.2.0/24","size":28}]}\n' + daemon = self._write_daemon(prior.decode()) + drain_marker = Path(self.tmp) / "removal-without-checkpoint-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + self._write_success_commands() + env_file = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + result = self._run(str(env_file), checkpoint_dir="") + + self.assertNotEqual(result.returncode, 0) + self.assertIn("--checkpoint is required", result.stderr) + self.assertNotIn("NETWORK_POLICY_NOOP", result.stdout) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + def test_creates_and_validates_checkpoint_before_drain(self) -> None: self._write_daemon("{}\n") self._write_success_commands() @@ -765,6 +839,52 @@ def test_creates_and_validates_checkpoint_before_drain(self) -> None: self.assertTrue(drain_marker.exists()) self.assertEqual(checkpoint.stat().st_mode & 0o777, 0o700) + def test_checkpoint_creation_fsyncs_parent_directory(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-parent-fsync" + audit_log = Path(self.tmp) / "checkpoint-parent-fsync.log" + audit_dir = Path(self.tmp) / "checkpoint-parent-fsync-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_mkdir = os.mkdir\n" + "_fsync = os.fsync\n" + "_log = os.environ['FSYNC_AUDIT_LOG']\n" + "def record(value):\n" + " with open(_log, 'a', encoding='utf-8') as handle: handle.write(value + '\\n')\n" + "def mkdir(path, mode=0o777, *, dir_fd=None):\n" + " result = _mkdir(path, mode, dir_fd=dir_fd)\n" + " parent = os.path.realpath(f'/proc/self/fd/{dir_fd}') if dir_fd is not None else os.getcwd()\n" + " created = os.path.realpath(os.path.join(parent, path))\n" + " if created == os.environ['CHECKPOINT']: record('M ' + created)\n" + " return result\n" + "def fsync(fd):\n" + " record('F ' + os.path.realpath(f'/proc/self/fd/{fd}'))\n" + " return _fsync(fd)\n" + "os.mkdir = mkdir\n" + "os.fsync = fsync\n", + encoding="utf-8", + ) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env( + PYTHONPATH=str(audit_dir), + FSYNC_AUDIT_LOG=str(audit_log), + CHECKPOINT=str(checkpoint), + ), + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + events = audit_log.read_text(encoding="utf-8").splitlines() + mkdir = events.index(f"M {checkpoint}") + self.assertIn(f"F {checkpoint.parent}", events[mkdir + 1 :]) + def test_checkpoint_swap_between_check_and_state_write_does_not_redirect_state(self) -> None: self._write_daemon("{}\n") self._write_success_commands() @@ -2039,6 +2159,63 @@ def test_absent_apply_rollback_fsyncs_daemon_dir_after_unlink(self) -> None: unlink = events.index(f"U {self.daemon_dir / 'daemon.json'}") self.assertIn(f"F {self.daemon_dir}", events[unlink + 1 :]) + def test_first_apply_rollback_fsyncs_checkpoint_after_marker_unlink(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + checkpoint = Path(self.tmp) / "checkpoint-first-rollback-fsync" + state_file = checkpoint / "docker-network-policy.json" + audit_log = Path(self.tmp) / "first-rollback-fsync.log" + audit_dir = Path(self.tmp) / "first-rollback-fsync-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_unlink = os.unlink\n" + "_fsync = os.fsync\n" + "_log = os.environ['FSYNC_AUDIT_LOG']\n" + "def record(value):\n" + " with open(_log, 'a', encoding='utf-8') as handle: handle.write(value + '\\n')\n" + "def unlink(path, *args, **kwargs):\n" + " if os.path.realpath(path) == os.environ['STATE_FILE']: record('U ' + os.environ['STATE_FILE'])\n" + " return _unlink(path, *args, **kwargs)\n" + "def fsync(fd):\n" + " record('F ' + os.path.realpath(f'/proc/self/fd/{fd}'))\n" + " return _fsync(fd)\n" + "os.unlink = unlink\n" + "os.fsync = fsync\n", + encoding="utf-8", + ) + fake_bin = Path(self.tmp) / "first-rollback-fsync-bin" + fake_bin.mkdir() + (fake_bin / "rm").write_text( + "#!/usr/bin/env bash\n" + f"if [[ ${{*: -1}} == /proc/self/fd/*/docker-network-policy.json ]]; then printf 'U %s\\n' {state_file} >> {audit_log}; fi\n" + f"exec {shutil.which('rm')} \"$@\"\n", + encoding="utf-8", + ) + (fake_bin / "rm").chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env( + PATH=f"{fake_bin}:{os.environ['PATH']}", + PYTHONPATH=str(audit_dir), + FSYNC_AUDIT_LOG=str(audit_log), + STATE_FILE=str(state_file), + ), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("prior daemon.json restored", result.stderr) + events = audit_log.read_text(encoding="utf-8").splitlines() + unlink = events.index(f"U {state_file}") + self.assertIn(f"F {checkpoint}", events[unlink + 1 :]) + def test_probe_failure_rolls_back_restarts_and_health_checks(self) -> None: prior = b'{"bip":"172.17.0.1/16","icc":false}\n' daemon = self.daemon_dir / "daemon.json" From 37212a276f6bf3e35ce948930536d8e687bfa276 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 04:23:18 -0500 Subject: [PATCH 39/54] fix: harden network policy recovery ordering --- scripts/apply-docker-network-policy.sh | 45 +++++++++-- scripts/healthcheck.sh | 10 +-- scripts/test_apply_docker_network_policy.py | 84 ++++++++++++++++++++- 3 files changed, 124 insertions(+), 15 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 69a28183..af92d881 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -246,6 +246,7 @@ run_health() { [[ "$command_timeout" =~ ^[1-9][0-9]*$ ]] || die 'CI_FLEET_COMMAND_TIMEOUT_SECONDS must be a positive integer' [[ -n "$daemon_config" ]] || die 'CI_FLEET_DOCKER_DAEMON_CONFIG is required when a network policy is configured' validate_trusted_path CI_FLEET_DOCKER_DAEMON_CONFIG "$daemon_config" regular true +[[ ! -e "$daemon_config" || ! /proc/self/fd/9 -ef "$daemon_config" ]] || die 'installer lock and daemon paths must be separate' validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true checkpoint_state=$checkpoint_dir/docker-network-policy.json [[ "$removing" != true || ! -L "$checkpoint_state" ]] || die 'network-policy checkpoint state is invalid' @@ -287,6 +288,7 @@ drain_failure= new_marker=false controller_resumed=false transaction_recovery= +removal_checkpoint_started=false # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below resume_after_failed_drain() { local status=$? resume_failed=0 health_failed=0 @@ -296,6 +298,9 @@ resume_after_failed_drain() { if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true ]]; then clear_managed_marker || resume_failed=1 fi + if ((resume_failed == 0 && health_failed == 0)) && [[ "$removal_checkpoint_started" == true ]]; then + set_verified_generation "$prior_verified_generation" clear-removal || resume_failed=1 + fi if ((resume_failed == 0 && health_failed == 0)); then [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" fi @@ -353,7 +358,13 @@ fi daemon_dir=$(dirname "$daemon_config") atomic_replace_daemon() { - python3 - "$1" "$daemon_dir" "$daemon_config" "$2" "$3" <<'PY' + local source=$1 mode=$2 gid=$3 + if [[ -e "$daemon_config" || -L "$daemon_config" ]]; then + validate_trusted_path CI_FLEET_DOCKER_DAEMON_CONFIG "$daemon_config" regular + mode=$(stat -c %a "$daemon_config") + gid=$(stat -c %g "$daemon_config") + fi + python3 - "$source" "$daemon_dir" "$daemon_config" "$mode" "$gid" <<'PY' import os, shutil, sys, tempfile _, source, daemon_dir, target, mode, gid = sys.argv fd, tmp = tempfile.mkstemp(prefix=".daemon.json.", dir=daemon_dir) @@ -458,14 +469,31 @@ PY clear_managed_marker() { checkpoint_path_is_pinned || return 1 python3 - "$state_file" <<'PY' -import os, sys +import os, shutil, sys, tempfile path = sys.argv[1] -os.unlink(path) -directory_fd = os.open(os.path.dirname(path), os.O_RDONLY | os.O_DIRECTORY) +parent = os.path.dirname(path) +fd, backup = tempfile.mkstemp(prefix=".docker-network-policy.clear.", dir=parent) try: - os.fsync(directory_fd) + with open(path, "rb") as source, os.fdopen(fd, "wb") as staged: + shutil.copyfileobj(source, staged) + os.fchmod(staged.fileno(), 0o600) + staged.flush() + os.fsync(staged.fileno()) + directory_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + os.unlink(path) + try: + os.fsync(directory_fd) + except OSError: + os.replace(backup, path) + os.fsync(directory_fd) + raise + finally: + os.close(directory_fd) finally: - os.close(directory_fd) + if os.path.exists(backup): + os.unlink(backup) PY } @@ -603,6 +631,8 @@ PY [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' fi if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$removal_pending" == false && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" ]]; then + run_command "$restart_command" "$daemon_dir" || die 'Docker restart command failed while recovering interrupted network-policy apply' + run_command "$probe_command" || die 'capacity probe failed while recovering interrupted network-policy apply' run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' run_health "$env_file" || die 'health check failed while recovering interrupted network-policy apply' clear_managed_marker || die 'failed to clear interrupted network-policy marker' @@ -653,6 +683,8 @@ PY managed_snapshot=$work_dir/daemon.json.before cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } + set_removal_pending "$managed_daemon" || { rm -rf "$work_dir"; die 'failed to persist network-policy removal state'; } + removal_checkpoint_started=true drain_controller 'drain command failed before network-policy removal' if daemon_changed_since_snapshot "$managed_snapshot" true "$managed_metadata"; then drain_failure='daemon.json changed during network-policy removal' @@ -726,7 +758,6 @@ PY trap removal_on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM - set_removal_pending "$managed_daemon" || { removal_failure='failed to persist network-policy removal state'; exit 2; } if [[ "$prior_present" == false ]] && python3 - "$removal_daemon" <<'PY' import json, sys diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index 98827cc5..99e8c15a 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -21,16 +21,16 @@ health_docker_socket=${CI_FLEET_DOCKER_SOCKET-} health_bootstrap=${CI_FLEET_HEALTH_BOOTSTRAP-} health_suppress_delivery=${CI_FLEET_HEALTH_SUPPRESS_DELIVERY-} while IFS= read -r variable; do unset "$variable"; done < <(compgen -A variable CI_FLEET_) -[[ -z $health_testing ]] || export CI_FLEET_TESTING=$health_testing -[[ -z $health_root_prefix ]] || export CI_FLEET_ROOT_PREFIX=$health_root_prefix -[[ -z $health_docker_socket ]] || export CI_FLEET_DOCKER_SOCKET=$health_docker_socket -[[ -z $health_bootstrap ]] || export CI_FLEET_HEALTH_BOOTSTRAP=$health_bootstrap -[[ -z $health_suppress_delivery ]] || export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery if [[ -r $selected_environment ]]; then set -a # shellcheck disable=SC1090 . "$selected_environment" set +a fi +[[ -z $health_testing ]] || export CI_FLEET_TESTING=$health_testing +[[ -z $health_root_prefix ]] || export CI_FLEET_ROOT_PREFIX=$health_root_prefix +[[ -z $health_docker_socket ]] || export CI_FLEET_DOCKER_SOCKET=$health_docker_socket +[[ -z $health_bootstrap ]] || export CI_FLEET_HEALTH_BOOTSTRAP=$health_bootstrap +[[ -z $health_suppress_delivery ]] || export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery use_local_docker exec python3 "$repo_root/scripts/health.py" "${args[@]}" "$@" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index c105f63a..c1f31984 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -219,7 +219,10 @@ def test_selected_env_preserves_health_delivery_suppression(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) candidate = root / "candidate.env" - candidate.write_text(f"CI_FLEET_TESTING=1\nCI_FLEET_ROOT_PREFIX={tmp}\n", encoding="utf-8") + candidate.write_text( + f"CI_FLEET_TESTING=1\nCI_FLEET_ROOT_PREFIX={tmp}\nCI_FLEET_HEALTH_SUPPRESS_DELIVERY=0\n", + encoding="utf-8", + ) fake_bin = root / "bin" fake_bin.mkdir() python = fake_bin / "python3" @@ -1679,6 +1682,29 @@ def test_existing_installer_lock_blocks_before_drain_or_checkpoint(self) -> None self.assertFalse(drain_marker.exists()) self.assertFalse(checkpoint.exists()) + def test_installer_lock_cannot_alias_daemon_config(self) -> None: + daemon = self._write_daemon("{}\n") + prior = daemon.read_bytes() + self._write_success_commands() + drain_marker = Path(self.tmp) / "aliased-lock-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + checkpoint = Path(self.tmp) / "checkpoint-aliased-lock" + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(daemon)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("lock and daemon paths must be separate", result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + self.assertFalse(checkpoint.exists()) + def test_hook_descendants_do_not_retain_installer_lock(self) -> None: self._write_daemon("{}\n") self._write_success_commands() @@ -2850,7 +2876,12 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> self.assertFalse(daemon.exists()) self.assertEqual( [marker.name for marker in command_markers if marker.exists()], - ["interrupted-before-rename-resume.sh.marker", "interrupted-before-rename-health.sh.marker"], + [ + "interrupted-before-rename-restart.sh.marker", + "interrupted-before-rename-probe.sh.marker", + "interrupted-before-rename-resume.sh.marker", + "interrupted-before-rename-health.sh.marker", + ], ) def test_removal_rejects_non_object_daemon_before_copy_or_commands(self) -> None: @@ -3253,6 +3284,11 @@ def test_removal_pending_state_is_durable_before_daemon_mutation(self) -> None: self.assertEqual(applied.returncode, 0, applied.stderr) managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] state_file = checkpoint / "docker-network-policy.json" + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"grep -Fq '\"phase\": \"removal-pending\"' {state_file}\n", + encoding="utf-8", + ) audit_dir = Path(self.tmp) / "removal-pending-audit" audit_dir.mkdir() audit_log = Path(self.tmp) / "removal-pending.log" @@ -3582,7 +3618,7 @@ def test_managed_policy_removal_restores_original_state_and_clears_only_marker(s self.assertFalse((checkpoint / "docker-network-policy.json").exists()) self.assertTrue(retained.exists()) - def test_removal_failure_restores_managed_key_into_current_unrelated_json(self) -> None: + def test_removal_rollback_preserves_concurrent_content_and_metadata(self) -> None: daemon = self._write_daemon(json.dumps({"icc": False})) checkpoint = Path(self.tmp) / "checkpoint-remove-merge-rollback" self._write_success_commands() @@ -3616,6 +3652,7 @@ def test_removal_failure_restores_managed_key_into_current_unrelated_json(self) f"echo resume >> {command_log}\n" 'if ! grep -q "^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=" "$2"; then\n' f" {shutil.which('python3')} {mutator} {daemon}\n" + f" chmod 600 {daemon}\n" " exit 2\n" "fi\n", encoding="utf-8", @@ -3632,6 +3669,7 @@ def test_removal_failure_restores_managed_key_into_current_unrelated_json(self) json.loads(daemon.read_text(encoding="utf-8")), {"default-address-pools": managed_pools, "debug": True, "live-restore": True}, ) + self.assertEqual(daemon.stat().st_mode & 0o777, 0o600) self.assertEqual( command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "restart", "resume", "health"], @@ -3772,6 +3810,46 @@ def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) self.assertEqual(state_file.read_bytes(), state) self.assertEqual(restart_log.read_text(encoding="utf-8").splitlines(), ["restart", "restart"]) + def test_marker_directory_fsync_failure_restores_state_before_rollback(self) -> None: + daemon = self._write_daemon('{"icc":false}\n') + checkpoint = Path(self.tmp) / "checkpoint-marker-fsync-failure" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + state_file = checkpoint / "docker-network-policy.json" + state = state_file.read_bytes() + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + audit_dir = Path(self.tmp) / "marker-fsync-failure-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os\n" + "_fsync = os.fsync\n" + "_failed = False\n" + "def fsync(fd):\n" + " global _failed\n" + " path = os.path.realpath(f'/proc/self/fd/{fd}')\n" + " if not _failed and path == os.environ['CHECKPOINT'] and not os.path.exists(os.environ['STATE_FILE']):\n" + " _failed = True\n" + " raise OSError('injected checkpoint fsync failure')\n" + " return _fsync(fd)\n" + "os.fsync = fsync\n", + encoding="utf-8", + ) + + removed = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, + text=True, + env=self._env(PYTHONPATH=str(audit_dir), CHECKPOINT=str(checkpoint), STATE_FILE=str(state_file)), + timeout=30, + ) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + self.assertEqual(state_file.read_bytes(), state) + def test_unmanaged_no_policy_is_noop_without_mutation_or_commands(self) -> None: prior = b'{"bip":"172.17.0.1/16"}\n' daemon = self.daemon_dir / "daemon.json" From ed928a36f6360a19a0539cf06580e0f89f0ec4d7 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:04:03 -0500 Subject: [PATCH 40/54] fix: harden network policy reapply consistency --- scripts/apply-docker-network-policy.sh | 62 ++++---- scripts/desired_state.py | 5 +- scripts/test_apply_docker_network_policy.py | 150 ++++++++++++++++++++ 3 files changed, 191 insertions(+), 26 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index af92d881..8c45f5fa 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -289,6 +289,7 @@ new_marker=false controller_resumed=false transaction_recovery= removal_checkpoint_started=false +apply_checkpoint_started=false # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below resume_after_failed_drain() { local status=$? resume_failed=0 health_failed=0 @@ -301,6 +302,9 @@ resume_after_failed_drain() { if ((resume_failed == 0 && health_failed == 0)) && [[ "$removal_checkpoint_started" == true ]]; then set_verified_generation "$prior_verified_generation" clear-removal || resume_failed=1 fi + if ((resume_failed == 0 && health_failed == 0)) && [[ "$apply_checkpoint_started" == true ]]; then + set_verified_generation "$prior_verified_generation" || resume_failed=1 + fi if ((resume_failed == 0 && health_failed == 0)); then [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" fi @@ -644,7 +648,20 @@ PY managed_daemon=$work_dir/daemon.json.managed removal_daemon=$work_dir/daemon.json.removal - python3 - "$daemon_config" "$state_file" "$managed_daemon" "$removal_daemon" 2>/dev/null <<'PY' || { + python3 - "$daemon_config" <<'PY' || { rm -rf "$work_dir"; die 'managed daemon.json is not a valid JSON object'; } +import json, sys +try: + if not isinstance(json.load(open(sys.argv[1], encoding="utf-8")), dict): + raise ValueError +except (OSError, json.JSONDecodeError, ValueError): + raise SystemExit(1) +PY + managed_mode=$(stat -c %a "$daemon_config") + managed_gid=$(stat -c %g "$daemon_config") + managed_metadata=$(stat -c '%d:%i:%u:%a:%g' "$daemon_config") + managed_snapshot=$work_dir/daemon.json.before + cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } + python3 - "$managed_snapshot" "$state_file" "$managed_daemon" "$removal_daemon" 2>/dev/null <<'PY' || { import json, sys daemon_path, state_path, managed_path, removal_path = sys.argv[1:] try: @@ -677,11 +694,6 @@ PY rm -rf "$work_dir" die 'managed daemon.json is not a valid JSON object' } - managed_mode=$(stat -c %a "$daemon_config") - managed_gid=$(stat -c %g "$daemon_config") - managed_metadata=$(stat -c '%d:%i:%u:%a:%g' "$daemon_config") - managed_snapshot=$work_dir/daemon.json.before - cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } set_removal_pending "$managed_daemon" || { rm -rf "$work_dir"; die 'failed to persist network-policy removal state'; } removal_checkpoint_started=true @@ -814,8 +826,19 @@ fi # --- Stage merged daemon.json (preserve unrelated keys) --- staging_daemon="$work_dir/daemon.json" +prior_daemon="$work_dir/prior" +mkdir -p "$prior_daemon" +backup_dir=$prior_daemon +backup_name=daemon.json.before +had_prior=false +if [[ -f "$daemon_config" ]]; then + had_prior=true + cp -p "$daemon_config" "$backup_dir/$backup_name" +fi +snapshot_present=$had_prior +rollback_source=$backup_dir/$backup_name -python3 - "$env_file" "$daemon_config" "$staging_daemon" "$desired_pools_json" <<'PY' || { rm -rf "$work_dir"; die "failed to stage merged daemon.json"; } +python3 - "$env_file" "$rollback_source" "$staging_daemon" "$desired_pools_json" <<'PY' || { rm -rf "$work_dir"; die "failed to stage merged daemon.json"; } import json, os, sys _, _, daemon_path, staging_path, desired_pools_json = sys.argv prior = {} @@ -908,18 +931,7 @@ then fi fi -# --- Back up exact current daemon.json for transactional rollback --- -prior_daemon="$work_dir/prior" -mkdir -p "$prior_daemon" -backup_dir=$prior_daemon -backup_name=daemon.json.before -had_prior=false -if [[ -f "$daemon_config" ]]; then - had_prior=true - cp -p "$daemon_config" "$backup_dir/$backup_name" -fi -snapshot_present=$had_prior -rollback_source=$backup_dir/$backup_name +# --- Use the staged source snapshot for transactional rollback --- if [[ "$managed_before" == true && -z "$prior_verified_generation" ]]; then rollback_source=$work_dir/daemon.json.durable-baseline python3 - "$daemon_config" "$state_file" "$rollback_source" <<'PY' || { rm -rf "$work_dir"; die 'failed to construct durable rollback baseline'; } @@ -946,7 +958,7 @@ import json, os, sys current_path, prior_path, had_prior, output_path, staged_path = sys.argv[1:] try: current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} - prior = json.load(open(prior_path, encoding="utf-8")) if had_prior == "true" else {} + prior = json.load(open(prior_path, encoding="utf-8")) if os.path.exists(prior_path) else {} staged = json.load(open(staged_path, encoding="utf-8")) if not isinstance(current, dict) or not isinstance(prior, dict) or not isinstance(staged, dict): raise ValueError @@ -955,7 +967,7 @@ except (OSError, json.JSONDecodeError, ValueError): current_unrelated = {key: value for key, value in current.items() if key != "default-address-pools"} staged_unrelated = {key: value for key, value in staged.items() if key != "default-address-pools"} if current_unrelated == staged_unrelated: - print("exact" if had_prior == "true" else "remove") + print("exact" if had_prior == "true" or prior else "remove") raise SystemExit if "default-address-pools" in prior: current["default-address-pools"] = prior["default-address-pools"] @@ -1084,6 +1096,10 @@ rollback_on_exit() { } # --- Drain after local validation/checkpointing, before mutation or restart --- +if [[ "$managed_before" == true ]]; then + set_verified_generation "" || die 'failed to mark network-policy verification pending' + apply_checkpoint_started=true +fi drain_controller 'drain command failed before network-policy apply' if daemon_changed_since_snapshot "$backup_dir/$backup_name" "$snapshot_present" "$daemon_metadata"; then drain_failure='daemon.json changed during network-policy apply' @@ -1094,10 +1110,6 @@ trap rollback_on_exit EXIT trap 'exit 130' INT trap 'exit 143' TERM -if [[ "$managed_before" == true ]]; then - set_verified_generation "" || { transaction_failure='failed to mark network-policy verification pending'; exit 2; } -fi - fail_after_apply() { transaction_failure=$1 exit 2 diff --git a/scripts/desired_state.py b/scripts/desired_state.py index d4753090..9f5e41f1 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -242,9 +242,12 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: "networks_per_runner": int(rendered["CI_FLEET_DOCKER_NETWORKS_PER_RUNNER"]), "reserve_subnets": int(rendered["CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"]), } - max_runners = 0 if rendered["CI_FLEET_CONTROLLER_STATE"] == "disabled" else int(rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"]) + state = rendered["CI_FLEET_CONTROLLER_STATE"] + max_runners = 0 if state == "disabled" else int(rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"]) except (KeyError, ValueError) as exc: raise ValueError("rendered Docker network policy fields must be present integers") from exc + if state != "disabled" and max_runners < 1: + raise ValueError("CI_FLEET_CONFIGURED_MAX_RUNNERS: must be a positive integer") validate_docker_network_policy(policy, path="rendered Docker network policy", max_runners=max_runners) return {"default-address-pools": pools} diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index c1f31984..db3248fc 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -93,6 +93,23 @@ def test_rejects_negative_pool_count(self) -> None: "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "-1", }) + def test_rejects_negative_rendered_runner_capacity(self) -> None: + value = config() + value["controllers"]["example-ci-01"]["docker_network_policy"] = docker_network_policy() + rendered, _ = build_rendered_env( + value, + "example-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + engine_capabilities={"status_reporting_config", "required_status_reporting", "docker_network_policy_config"}, + ) + rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"] = "-1" + + with self.assertRaisesRegex(ValueError, "positive integer"): + render_docker_daemon_config(rendered) + def test_rejects_malformed_pool_index(self) -> None: env = { "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "1", @@ -604,6 +621,38 @@ def test_daemon_change_during_drain_aborts_and_preserves_update(self) -> None: self.assertEqual(daemon.read_bytes(), changed) self.assertFalse((checkpoint / "docker-network-policy.json").exists()) + def test_apply_stages_from_the_conflict_detection_snapshot(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + env_file = self._write_env_file(self._rendered_with_policy()) + self._write_success_commands() + fake_bin = Path(self.tmp) / "snapshot-cp-bin" + fake_bin.mkdir() + cp = fake_bin / "cp" + cp.write_text( + "#!/usr/bin/env bash\n" + f"if [[ $2 == {daemon} ]]; then printf '%s\\n' '{{\"live-restore\":true,\"log-level\":\"debug\"}}' > {daemon}; fi\n" + f"exec {shutil.which('cp')} \"$@\"\n", + encoding="utf-8", + ) + cp.chmod(0o755) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-snapshot-stage"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(PATH=f"{fake_bin}:{os.environ['PATH']}"), + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8"))["log-level"], "debug") + def test_post_drain_conflict_retains_recovery_when_prior_resume_fails(self) -> None: daemon = self._write_daemon('{"live-restore":true}\n') checkpoint = Path(self.tmp) / "checkpoint-post-drain-recovery" @@ -684,6 +733,28 @@ def test_failed_managed_reapply_restores_prior_verified_generation(self) -> None self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"], prior_generation) self.assertEqual(hashlib.sha256(daemon.read_bytes()).hexdigest(), prior_generation) + def test_managed_reapply_marks_generation_pending_before_drain(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-reapply-pending-before-drain" + self._write_success_commands() + prior_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(prior_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(prior_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"grep -q '\"verified_generation\": null' {state_file}\n", + encoding="utf-8", + ) + candidate = self._rendered_with_policy() + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = self._write_env_file(candidate) + + result = self._run(str(candidate_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + def test_managed_reapply_recovers_when_daemon_snapshot_is_absent(self) -> None: daemon = self.daemon_dir / "daemon.json" checkpoint = Path(self.tmp) / "checkpoint-reapply-absent-daemon" @@ -761,6 +832,40 @@ def test_failed_unverified_retry_restores_durable_baseline(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"live-restore": True}) + def test_failed_unverified_retry_preserves_new_unrelated_keys_after_absent_baseline(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps({ + **render_docker_daemon_config(rendered), + "live-restore": True, + })) + checkpoint = Path(self.tmp) / "checkpoint-unverified-absent-baseline" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, + "prior_mode": None, + "prior_present": False, + "verified_generation": None, + }), + encoding="utf-8", + ) + state_file.chmod(0o600) + candidate = dict(rendered) + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + env_file = self._write_env_file(candidate) + self._write_success_commands() + probe = Path(self.tmp) / "probe.sh" + probe.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + probe.chmod(0o755) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"live-restore": True}) + def test_unverified_retry_preserves_current_snapshot_presence(self) -> None: rendered = self._rendered_with_policy() daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) @@ -2978,6 +3083,51 @@ def test_removal_daemon_change_during_drain_aborts_and_preserves_update(self) -> self.assertEqual(daemon.read_bytes(), changed_bytes) self.assertEqual(state_file.read_bytes(), prior_state) + def test_removal_stages_from_the_conflict_detection_snapshot(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-removal-snapshot-stage" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(policy_env.read_bytes()) + mutator = Path(self.tmp) / "mutate-before-removal-snapshot.py" + mutator.write_text( + "import json, sys\n" + "value = json.load(open(sys.argv[1]))\n" + "value['log-level'] = 'debug'\n" + "with open(sys.argv[1], 'w') as handle: json.dump(value, handle)\n", + encoding="utf-8", + ) + fake_bin = Path(self.tmp) / "removal-snapshot-cp-bin" + fake_bin.mkdir() + cp = fake_bin / "cp" + cp.write_text( + "#!/usr/bin/env bash\n" + f"if [[ $2 == {daemon} ]]; then {shutil.which('python3')} {mutator} {daemon}; fi\n" + f"exec {shutil.which('cp')} \"$@\"\n", + encoding="utf-8", + ) + cp.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(no_policy_env), + ], + capture_output=True, + text=True, + env=self._env(PATH=f"{fake_bin}:{os.environ['PATH']}"), + timeout=30, + ) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8"))["log-level"], "debug") + def test_removal_restores_prior_pools_and_preserves_current_unrelated_keys(self) -> None: prior_pools = [{"base": "192.0.2.0/24", "size": 28}] daemon = self._write_daemon(json.dumps({"default-address-pools": prior_pools, "icc": False})) From 18070a2d194aa86801164e886719604f56bbc92b Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 05:36:56 -0500 Subject: [PATCH 41/54] fix: harden network policy rollback recovery --- scripts/apply-docker-network-policy.sh | 90 ++++++++++--- scripts/test_apply_docker_network_policy.py | 136 ++++++++++++++++++++ 2 files changed, 209 insertions(+), 17 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 8c45f5fa..5c91986f 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -196,6 +196,9 @@ env_file=$work_dir/ci-fleet.env removing=false [[ -n "$checkpoint_dir" ]] || die '--checkpoint is required' if ! grep -q '^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=' "$env_file"; then + if grep -Eq '^CI_FLEET_DOCKER_(DEFAULT_ADDRESS_POOL_|NETWORKS_PER_RUNNER=|NETWORK_RESERVE_SUBNETS=)' "$env_file"; then + die 'rendered network-policy fields are incomplete' + fi removing=true fi @@ -293,7 +296,8 @@ apply_checkpoint_started=false # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below resume_after_failed_drain() { local status=$? resume_failed=0 health_failed=0 - trap - EXIT INT TERM + trap '' INT TERM + trap - EXIT run_command "$resume_command" --env "$prior_env" || resume_failed=1 ((resume_failed != 0)) || run_health "$prior_env" || health_failed=1 if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true ]]; then @@ -399,6 +403,21 @@ with open(sys.argv[1], "rb") as handle: PY } +daemon_pools_match() { + python3 - "$daemon_config" "$1" 2>/dev/null <<'PY' +import json, os, sys +current = json.load(open(sys.argv[1], encoding="utf-8")) if os.path.exists(sys.argv[1]) else {} +expected = json.load(open(sys.argv[2], encoding="utf-8")) if os.path.exists(sys.argv[2]) else {} +missing = object() +if ( + not isinstance(current, dict) + or not isinstance(expected, dict) + or current.get("default-address-pools", missing) != expected.get("default-address-pools", missing) +): + raise SystemExit(1) +PY +} + daemon_changed_since_snapshot() { local snapshot=$1 was_present=$2 expected_metadata=${3:-} if [[ "$was_present" == true ]]; then @@ -744,6 +763,9 @@ PY if ((failed == 0)); then run_health "$prior_env" || failed=1 fi + if ((failed == 0)); then + daemon_pools_match "$managed_daemon" || failed=1 + fi if ((failed == 0)); then set_verified_generation "$prior_verified_generation" clear-removal || failed=1 fi @@ -752,7 +774,8 @@ PY # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below removal_on_exit() { local status=$? - trap - EXIT INT TERM + trap '' INT TERM + trap - EXIT ((status != 0)) || status=1 if rollback_removal; then rm -rf "$work_dir" @@ -932,9 +955,39 @@ then fi # --- Use the staged source snapshot for transactional rollback --- +interrupted_recovery= if [[ "$managed_before" == true && -z "$prior_verified_generation" ]]; then - rollback_source=$work_dir/daemon.json.durable-baseline - python3 - "$daemon_config" "$state_file" "$rollback_source" <<'PY' || { rm -rf "$work_dir"; die 'failed to construct durable rollback baseline'; } + recovery_info=$(python3 - "$checkpoint_dir" "$checkpoint_owner" <<'PY' +import os, stat, sys +parent, owner = sys.argv[1], int(sys.argv[2]) +recoveries = [entry for entry in os.scandir(parent) if entry.name.startswith("recovery.")] +if not recoveries: + print("|") + raise SystemExit +if len(recoveries) != 1: + raise SystemExit(1) +recovery = recoveries[0] +metadata = recovery.stat(follow_symlinks=False) +if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o700: + raise SystemExit(1) +entries = {entry.name: entry for entry in os.scandir(recovery.path)} +if set(entries) not in ({"prior-ci-fleet.env"}, {"daemon.json.before", "prior-ci-fleet.env"}): + raise SystemExit(1) +for entry in entries.values(): + metadata = entry.stat(follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o600: + raise SystemExit(1) +print(f"{recovery.path}|{'true' if 'daemon.json.before' in entries else 'false'}") +PY + ) || { rm -rf "$work_dir"; die 'network-policy transaction recovery is invalid'; } + IFS='|' read -r interrupted_recovery recovery_present <<<"$recovery_info" + if [[ -n "$interrupted_recovery" ]]; then + had_prior=$recovery_present + prior_env=$interrupted_recovery/prior-ci-fleet.env + rollback_source=$interrupted_recovery/daemon.json.before + else + rollback_source=$work_dir/daemon.json.durable-baseline + python3 - "$daemon_config" "$state_file" "$rollback_source" <<'PY' || { rm -rf "$work_dir"; die 'failed to construct durable rollback baseline'; } import json, os, sys current_path, state_path, output_path = sys.argv[1:] current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} @@ -947,8 +1000,9 @@ with open(output_path, "w", encoding="utf-8") as handle: json.dump(current, handle, indent=2, sort_keys=True) handle.write("\n") PY - had_prior=$checkpoint_prior_present - [[ "$had_prior" != true ]] || daemon_mode=$checkpoint_prior_mode + had_prior=$checkpoint_prior_present + [[ "$had_prior" != true ]] || daemon_mode=$checkpoint_prior_mode + fi fi restore_daemon() { @@ -1040,9 +1094,13 @@ PY new_marker=true fi if [[ "$managed_before" == true ]]; then - recovery_daemon= - [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name - transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' + if [[ -n "$interrupted_recovery" ]]; then + transaction_recovery=$interrupted_recovery + else + recovery_daemon= + [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name + transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' + fi fi rollback_daemon() { @@ -1062,6 +1120,9 @@ rollback_daemon() { if ((failed == 0)); then run_health "$prior_env" || failed=1 fi + if ((failed == 0)); then + daemon_pools_match "$rollback_source" || failed=1 + fi if [[ "$managed_before" == true && "$failed" == 0 ]]; then set_verified_generation "$prior_verified_generation" || failed=1 fi @@ -1071,7 +1132,8 @@ rollback_daemon() { transaction_failure='network-policy apply interrupted' rollback_on_exit() { local status=$? - trap - EXIT INT TERM + trap '' INT TERM + trap - EXIT ((status != 0)) || status=1 if rollback_daemon >/dev/null 2>&1 && { [[ "$managed_before" == true ]] || clear_managed_marker; }; then @@ -1142,13 +1204,7 @@ if ! run_health "$env_file"; then fail_after_apply "health check failed after network-policy restart" fi -python3 - "$daemon_config" "$staging_daemon" 2>/dev/null <<'PY' || fail_after_apply "daemon.json changed after network-policy verification" -import json, sys -current = json.load(open(sys.argv[1], encoding="utf-8")) -staged = json.load(open(sys.argv[2], encoding="utf-8")) -if not isinstance(current, dict) or current.get("default-address-pools") != staged.get("default-address-pools"): - raise SystemExit(1) -PY +daemon_pools_match "$staging_daemon" || fail_after_apply "daemon.json changed after network-policy verification" verified_generation=$(file_generation "$daemon_config") || fail_after_apply "failed to identify verified daemon.json generation" verified_action= [[ "$apply_removal_pending" != true ]] || verified_action=clear-removal diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index db3248fc..ba4ce7aa 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -832,6 +832,41 @@ def test_failed_unverified_retry_restores_durable_baseline(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"live-restore": True}) + def test_failed_interrupted_reapply_retry_restores_immediate_snapshot(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-interrupted-reapply" + self._write_success_commands() + policy_a = self._rendered_with_policy() + env_a = self._write_env_file(policy_a) + applied = self._run(str(env_a), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + daemon_a = daemon.read_bytes() + self.installed_env.write_bytes(env_a.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["verified_generation"] = None + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "daemon.json.before").write_bytes(daemon_a) + (recovery / "prior-ci-fleet.env").write_bytes(env_a.read_bytes()) + for path in recovery.iterdir(): + path.chmod(0o600) + policy_b = dict(policy_a) + policy_b["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + daemon.write_text(json.dumps(render_docker_daemon_config(policy_b)), encoding="utf-8") + policy_c = dict(policy_a) + policy_c["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.3.0/24" + env_c = self._write_env_file(policy_c) + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + + result = self._run(str(env_c), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(daemon.read_bytes(), daemon_a) + def test_failed_unverified_retry_preserves_new_unrelated_keys_after_absent_baseline(self) -> None: rendered = self._rendered_with_policy() daemon = self._write_daemon(json.dumps({ @@ -926,6 +961,21 @@ def test_policy_removal_requires_checkpoint_before_noop(self) -> None: self.assertEqual(daemon.read_bytes(), prior) self.assertFalse(drain_marker.exists()) + def test_partial_no_policy_snapshot_is_rejected_before_removal(self) -> None: + daemon = self._write_daemon('{"default-address-pools":[{"base":"192.0.2.0/24","size":28}]}\n') + prior = daemon.read_bytes() + drain_marker = Path(self.tmp) / "partial-policy-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + self._write_success_commands() + env_file = self._write_env_file({"CI_FLEET_DOCKER_NETWORKS_PER_RUNNER": "1"}) + + result = self._run(str(env_file)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("network-policy fields are incomplete", result.stderr) + self.assertEqual(daemon.read_bytes(), prior) + self.assertFalse(drain_marker.exists()) + def test_creates_and_validates_checkpoint_before_drain(self) -> None: self._write_daemon("{}\n") self._write_success_commands() @@ -2491,6 +2541,52 @@ def test_interrupt_after_replace_rolls_back_and_retains_failed_recovery(self) -> self.assertEqual(recovery.parent, Path(self.tmp) / "checkpoint-direct") self.assertEqual((recovery / "daemon.json.before").read_bytes(), prior) + def test_signal_during_failed_rollback_waits_for_durable_recovery(self) -> None: + prior = b'{"bip":"172.17.0.1/16"}\n' + self._write_daemon(prior.decode()) + checkpoint = Path(self.tmp) / "checkpoint-signal-during-rollback" + rollback_ready = Path(self.tmp) / "rollback.ready" + restart_count = Path(self.tmp) / "rollback-restart-count" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + "#!/usr/bin/env bash\n" + f"echo restart >> {restart_count}\n" + f"if [[ $(wc -l < {restart_count}) -gt 1 ]]; then\n" + f" touch {rollback_ready}\n" + " sleep 1\n" + " exit 1\n" + "fi\n", + encoding="utf-8", + ) + restart.chmod(0o755) + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + (Path(self.tmp) / "health.sh").write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") + (Path(self.tmp) / "health.sh").chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + process = subprocess.Popen( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=self._env(CI_FLEET_COMMAND_TIMEOUT_SECONDS="5"), + ) + self.addCleanup(lambda: process.poll() is None and process.kill()) + for _ in range(300): + if rollback_ready.exists(): + break + time.sleep(0.02) + self.assertTrue(rollback_ready.exists(), "apply did not reach rollback restart") + + os.kill(process.pid, signal.SIGTERM) + _, stderr = process.communicate(timeout=10) + + self.assertNotEqual(process.returncode, 0) + self.assertIn("recovery data retained at", stderr) + recoveries = list(checkpoint.glob("recovery.*")) + self.assertEqual(len(recoveries), 1) + self.assertEqual((recoveries[0] / "daemon.json.before").read_bytes(), prior) + def test_rollback_failure_is_reported(self) -> None: prior = b'{"bip":"172.17.0.1/16","registry-mirrors":["https://mirror.example.invalid?token=credential"]}\n' self._write_daemon(prior.decode()) @@ -2686,6 +2782,46 @@ def test_apply_rejects_managed_pool_change_before_recording_generation(self) -> ["drain", "restart", "probe", "resume", "health", "drain", "restart", "resume", "health"], ) + def test_rollback_rejects_restored_pool_change_before_recording_generation(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-rollback-pool-conflict" + self._write_success_commands() + policy_a = self._rendered_with_policy() + env_a = self._write_env_file(policy_a) + applied = self._run(str(env_a), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(env_a.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + policy_b = dict(policy_a) + policy_b["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + env_b = self._write_env_file(policy_b) + health_count = Path(self.tmp) / "rollback-pool-health-count" + mutator = Path(self.tmp) / "mutate-rollback-pools.py" + mutator.write_text( + "import json, sys\n" + "path = sys.argv[1]\n" + "value = json.load(open(path))\n" + "value['default-address-pools'] = [{'base': '192.0.3.0/24', 'size': 29}]\n" + "with open(path, 'w') as handle: json.dump(value, handle)\n", + encoding="utf-8", + ) + health = Path(self.tmp) / "health.sh" + health.write_text( + "#!/usr/bin/env bash\n" + f"echo health >> {health_count}\n" + f"if [[ $(wc -l < {health_count}) -eq 1 ]]; then exit 2; fi\n" + f"{shutil.which('python3')} {mutator} {daemon}\n", + encoding="utf-8", + ) + health.chmod(0o755) + + result = self._run(str(env_b), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("rollback verification failed", result.stderr) + self.assertIsNone(json.loads(state_file.read_text(encoding="utf-8"))["verified_generation"]) + self.assertEqual(len(list(checkpoint.glob("recovery.*"))), 1) + def test_health_failure_evidence_excludes_command_output(self) -> None: self._write_daemon("{}\n") for name in ("restart.sh", "probe.sh"): From b02085535b6a07a2fa414537c32746d4b2d9ac08 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 06:16:40 -0500 Subject: [PATCH 42/54] fix: preserve network policy removal provenance --- scripts/apply-docker-network-policy.sh | 37 ++++++++--- scripts/test_apply_docker_network_policy.py | 68 +++++++++++++++++++++ 2 files changed, 97 insertions(+), 8 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 5c91986f..3d7a758b 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -490,10 +490,11 @@ PY } clear_managed_marker() { + local expected_daemon=${1:-} checkpoint_path_is_pinned || return 1 - python3 - "$state_file" <<'PY' -import os, shutil, sys, tempfile -path = sys.argv[1] + python3 - "$state_file" "$expected_daemon" "$daemon_config" <<'PY' +import json, os, shutil, sys, tempfile +path, expected_path, daemon_path = sys.argv[1:] parent = os.path.dirname(path) fd, backup = tempfile.mkstemp(prefix=".docker-network-policy.clear.", dir=parent) try: @@ -508,9 +509,25 @@ try: os.unlink(path) try: os.fsync(directory_fd) + if expected_path: + try: + current = json.load(open(daemon_path, encoding="utf-8")) if os.path.exists(daemon_path) else {} + expected = json.load(open(expected_path, encoding="utf-8")) if os.path.exists(expected_path) else {} + missing = object() + if ( + not isinstance(current, dict) + or not isinstance(expected, dict) + or current.get("default-address-pools", missing) != expected.get("default-address-pools", missing) + ): + raise ValueError + except (OSError, json.JSONDecodeError, ValueError): + os.replace(backup, path) + os.fsync(directory_fd) + raise except OSError: - os.replace(backup, path) - os.fsync(directory_fd) + if os.path.exists(backup): + os.replace(backup, path) + os.fsync(directory_fd) raise finally: os.close(directory_fd) @@ -658,7 +675,7 @@ PY run_command "$probe_command" || die 'capacity probe failed while recovering interrupted network-policy apply' run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' run_health "$env_file" || die 'health check failed while recovering interrupted network-policy apply' - clear_managed_marker || die 'failed to clear interrupted network-policy marker' + clear_managed_marker "$work_dir/daemon.json.removal" || die 'failed to clear interrupted network-policy marker' rm -rf "$work_dir" printf 'NETWORK_POLICY_REMOVED\n' exit 0 @@ -837,7 +854,7 @@ if ( ): raise SystemExit(1) PY - if ! clear_managed_marker; then + if ! clear_managed_marker "$removal_daemon"; then removal_failure='failed to clear network-policy managed marker' exit 2 fi @@ -992,7 +1009,11 @@ import json, os, sys current_path, state_path, output_path = sys.argv[1:] current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} state = json.load(open(state_path, encoding="utf-8")) -if state["prior_default_address_pools_present"]: +if state.get("phase") == "removal-pending" and state["removal_managed_default_address_pools"] is not None: + current["default-address-pools"] = state["removal_managed_default_address_pools"] +elif state.get("phase") == "removal-pending": + current.pop("default-address-pools", None) +elif state["prior_default_address_pools_present"]: current["default-address-pools"] = state["prior_default_address_pools"] else: current.pop("default-address-pools", None) diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index ba4ce7aa..c913942d 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -3507,6 +3507,34 @@ def test_configured_apply_resumes_removal_pending_checkpoint(self) -> None: self.assertNotIn("phase", resumed_state) self.assertNotIn("removal_managed_default_address_pools", resumed_state) + def test_failed_configured_apply_from_removal_pending_restores_managed_pools(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-removal-pending-apply-rollback" + self._write_success_commands() + policy_a = self._rendered_with_policy() + env_a = self._write_env_file(policy_a) + applied = self._run(str(env_a), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + managed_pools = json.loads(managed)["default-address-pools"] + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = managed_pools + state["verified_generation"] = None + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + policy_b = dict(policy_a) + policy_b["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + env_b = self._write_env_file(policy_b) + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + + retried = self._run(str(env_b), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(retried.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + def test_configured_apply_rejects_invalid_removal_pending_pools_before_drain(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-invalid-removal-pending-apply" @@ -3856,6 +3884,46 @@ def test_removal_retains_marker_if_managed_key_reappears_during_verification(sel self.assertTrue(state_file.exists()) self.assertEqual(json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"], managed_pools) + def test_removal_restores_marker_if_managed_key_reappears_during_marker_clear(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-marker-clear-conflict" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + audit_dir = Path(self.tmp) / "marker-clear-conflict-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import json, os\n" + "_unlink = os.unlink\n" + "def unlink(path, *args, **kwargs):\n" + " result = _unlink(path, *args, **kwargs)\n" + " if os.path.realpath(path) == os.environ['STATE_FILE']:\n" + " daemon = os.environ['DAEMON']\n" + " value = json.load(open(daemon, encoding='utf-8'))\n" + " value['default-address-pools'] = [{'base': '192.0.2.0/24', 'size': 29}]\n" + " with open(daemon, 'w', encoding='utf-8') as handle: json.dump(value, handle)\n" + " return result\n" + "os.unlink = unlink\n", + encoding="utf-8", + ) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, + text=True, + env=self._env(PYTHONPATH=str(audit_dir), STATE_FILE=str(state_file), DAEMON=str(daemon)), + timeout=30, + ) + + self.assertNotEqual(removed.returncode, 0) + self.assertTrue(state_file.exists()) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"], managed_pools) + def test_managed_policy_removal_restores_original_state_and_clears_only_marker(self) -> None: for prior_present in (False, True): with self.subTest(prior_present=prior_present): From 9d20100ca277662dfa4fdfeec1867594bf4c386a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 10:29:30 -0500 Subject: [PATCH 43/54] fix: harden network policy reconciliation --- scripts/apply-docker-network-policy.sh | 123 ++++++++++++--- scripts/desired_state.py | 13 +- scripts/test_apply_docker_network_policy.py | 164 +++++++++++++++++++- 3 files changed, 275 insertions(+), 25 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 3d7a758b..ccc20061 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -278,6 +278,9 @@ validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true t exec 8<"$checkpoint_dir" checkpoint_path_is_pinned || die 'checkpoint directory must remain a trusted root-owned path' state_file=/proc/self/fd/8/docker-network-policy.json +if [[ $(readlink -m "$state_file") == "$daemon_config" || ( -e "$state_file" && "$state_file" -ef "$daemon_config" ) ]]; then + die 'daemon config and checkpoint state paths must be separate' +fi # Snapshot the installer's authoritative pre-transaction environment while # holding its lock. Rollback must not validate against the rejected candidate. @@ -366,15 +369,15 @@ fi daemon_dir=$(dirname "$daemon_config") atomic_replace_daemon() { - local source=$1 mode=$2 gid=$3 + local source=$1 mode=$2 gid=$3 expected=${4:-} expected_present=${5:-true} if [[ -e "$daemon_config" || -L "$daemon_config" ]]; then validate_trusted_path CI_FLEET_DOCKER_DAEMON_CONFIG "$daemon_config" regular mode=$(stat -c %a "$daemon_config") gid=$(stat -c %g "$daemon_config") fi - python3 - "$source" "$daemon_dir" "$daemon_config" "$mode" "$gid" <<'PY' -import os, shutil, sys, tempfile -_, source, daemon_dir, target, mode, gid = sys.argv + python3 - "$source" "$daemon_dir" "$daemon_config" "$mode" "$gid" "$expected" "$expected_present" <<'PY' +import ctypes, errno, os, shutil, sys, tempfile +_, source, daemon_dir, target, mode, gid, expected, expected_present = sys.argv fd, tmp = tempfile.mkstemp(prefix=".daemon.json.", dir=daemon_dir) try: with open(source, "rb") as source_handle, os.fdopen(fd, "wb") as staged: @@ -383,7 +386,36 @@ try: os.fchown(staged.fileno(), -1, int(gid)) staged.flush() os.fsync(staged.fileno()) - os.replace(tmp, target) + if expected and expected_present == "false": + libc = ctypes.CDLL(None, use_errno=True) + if libc.renameat2(-100, os.fsencode(tmp), -100, os.fsencode(target), 1) != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + elif expected: + staged_identity = (os.stat(tmp).st_dev, os.stat(tmp).st_ino) + libc = ctypes.CDLL(None, use_errno=True) + if libc.renameat2(-100, os.fsencode(tmp), -100, os.fsencode(target), 2) != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + with open(tmp, "rb") as replaced, open(expected, "rb") as snapshot: + matched = replaced.read() == snapshot.read() + if not matched: + current = os.stat(target, follow_symlinks=False) + if (current.st_dev, current.st_ino) == staged_identity: + if libc.renameat2(-100, os.fsencode(tmp), -100, os.fsencode(target), 2) != 0: + error = ctypes.get_errno() + raise OSError(error, os.strerror(error)) + if os.path.exists(tmp): + os.unlink(tmp) + directory_fd = os.open(daemon_dir, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory_fd) + finally: + os.close(directory_fd) + raise OSError(errno.EAGAIN, "daemon.json changed during atomic replacement") + os.unlink(tmp) + else: + os.replace(tmp, target) directory_fd = os.open(daemon_dir, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(directory_fd) @@ -608,6 +640,34 @@ except BaseException: PY } +clear_recovery_artifacts() { + checkpoint_path_is_pinned || return 1 + python3 - "$checkpoint_dir" "$checkpoint_owner" <<'PY' +import os, shutil, stat, sys + +parent, owner = sys.argv[1], int(sys.argv[2]) +for entry in os.scandir(parent): + if not entry.name.startswith("recovery."): + continue + metadata = entry.stat(follow_symlinks=False) + if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o700: + raise SystemExit(1) + files = {child.name: child for child in os.scandir(entry.path)} + if set(files) not in ({"prior-ci-fleet.env"}, {"daemon.json.before", "prior-ci-fleet.env"}): + raise SystemExit(1) + for child in files.values(): + metadata = child.stat(follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o600: + raise SystemExit(1) + shutil.rmtree(entry.path) +parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY) +try: + os.fsync(parent_fd) +finally: + os.close(parent_fd) +PY +} + if [[ "$removing" == true ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || die 'network-policy checkpoint state is invalid' mapfile -t managed_state < <(python3 - "$state_file" "$repo_root/scripts" 2>/dev/null <<'PY' @@ -686,8 +746,10 @@ PY removal_daemon=$work_dir/daemon.json.removal python3 - "$daemon_config" <<'PY' || { rm -rf "$work_dir"; die 'managed daemon.json is not a valid JSON object'; } import json, sys +def reject_constant(_value): + raise ValueError try: - if not isinstance(json.load(open(sys.argv[1], encoding="utf-8")), dict): + if not isinstance(json.load(open(sys.argv[1], encoding="utf-8"), parse_constant=reject_constant), dict): raise ValueError except (OSError, json.JSONDecodeError, ValueError): raise SystemExit(1) @@ -743,21 +805,27 @@ PY removal_failure='network-policy removal interrupted' # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below rollback_removal() { - local failed=0 rollback_daemon=$work_dir/daemon.json.rollback + local failed=0 rollback_daemon=$work_dir/daemon.json.rollback rollback_expected=$work_dir/daemon.json.rollback-expected if [[ "$controller_resumed" == true ]]; then run_command "$drain_command" || failed=1 fi if ((failed == 0)); then - python3 - "$daemon_config" "$managed_daemon" "$rollback_daemon" <<'PY' || failed=1 + python3 - "$daemon_config" "$managed_daemon" "$rollback_daemon" "$rollback_expected" <<'PY' || failed=1 import json, os, sys -current_path, managed_path, output_path = sys.argv[1:] +current_path, managed_path, output_path, expected_path = sys.argv[1:] try: - current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} + current_present = os.path.exists(current_path) + current_text = open(current_path, encoding="utf-8").read() if current_present else "" + current = json.loads(current_text) if current_present else {} managed = json.load(open(managed_path, encoding="utf-8")) if not isinstance(current, dict) or not isinstance(managed, dict): raise ValueError except (OSError, json.JSONDecodeError, ValueError): raise SystemExit(1) +with open(expected_path, "w", encoding="utf-8") as handle: + handle.write(current_text) +with open(expected_path + ".present", "w", encoding="utf-8") as handle: + handle.write("true" if current_present else "false") if "default-address-pools" in managed: current["default-address-pools"] = managed["default-address-pools"] else: @@ -768,7 +836,7 @@ with open(output_path, "w", encoding="utf-8") as handle: PY fi if ((failed == 0)); then - atomic_replace_daemon "$rollback_daemon" "$managed_mode" "$managed_gid" || failed=1 + atomic_replace_daemon "$rollback_daemon" "$managed_mode" "$managed_gid" "$rollback_expected" "$(<"$rollback_expected.present")" || failed=1 cmp -s "$rollback_daemon" "$daemon_config" || failed=1 fi if ((failed == 0)); then @@ -834,8 +902,8 @@ PY fi run_command "$restart_command" "$daemon_dir" || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } run_command "$probe_command" || { removal_failure='capacity probe failed after network-policy removal'; exit 2; } - run_command "$resume_command" --env "$env_file" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } controller_resumed=true + run_command "$resume_command" --env "$env_file" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } run_health "$env_file" || { removal_failure='health check failed after network-policy removal'; exit 2; } # Marker deletion commits removal. Ignore catchable signals across the atomic @@ -881,11 +949,13 @@ rollback_source=$backup_dir/$backup_name python3 - "$env_file" "$rollback_source" "$staging_daemon" "$desired_pools_json" <<'PY' || { rm -rf "$work_dir"; die "failed to stage merged daemon.json"; } import json, os, sys _, _, daemon_path, staging_path, desired_pools_json = sys.argv +def reject_constant(_value): + raise ValueError prior = {} if os.path.exists(daemon_path): try: text = open(daemon_path, encoding="utf-8").read() - prior = json.loads(text) + prior = json.loads(text, parse_constant=reject_constant) if not isinstance(prior, dict): raise ValueError("daemon.json root must be an object") except (json.JSONDecodeError, ValueError) as exc: @@ -965,6 +1035,7 @@ then daemon_matches=true current_generation=$(file_generation "$daemon_config") || { rm -rf "$work_dir"; die 'failed to identify daemon.json generation'; } if [[ -n "$prior_verified_generation" && "$current_generation" == "$prior_verified_generation" ]]; then + clear_recovery_artifacts || { rm -rf "$work_dir"; die 'failed to clear obsolete network-policy recovery data'; } rm -rf "$work_dir" printf 'NETWORK_POLICY_NO_CHANGE\n' exit 0 @@ -1027,18 +1098,24 @@ PY fi restore_daemon() { - local rollback_daemon=$work_dir/daemon.json.apply-rollback rollback_action - rollback_action=$(python3 - "$daemon_config" "$rollback_source" "$had_prior" "$rollback_daemon" "$staging_daemon" <<'PY' + local rollback_daemon=$work_dir/daemon.json.apply-rollback rollback_expected=$work_dir/daemon.json.apply-rollback-expected rollback_action + rollback_action=$(python3 - "$daemon_config" "$rollback_source" "$had_prior" "$rollback_daemon" "$staging_daemon" "$rollback_expected" <<'PY' import json, os, sys -current_path, prior_path, had_prior, output_path, staged_path = sys.argv[1:] +current_path, prior_path, had_prior, output_path, staged_path, expected_path = sys.argv[1:] try: - current = json.load(open(current_path, encoding="utf-8")) if os.path.exists(current_path) else {} + current_present = os.path.exists(current_path) + current_text = open(current_path, encoding="utf-8").read() if current_present else "" + current = json.loads(current_text) if current_present else {} prior = json.load(open(prior_path, encoding="utf-8")) if os.path.exists(prior_path) else {} staged = json.load(open(staged_path, encoding="utf-8")) if not isinstance(current, dict) or not isinstance(prior, dict) or not isinstance(staged, dict): raise ValueError except (OSError, json.JSONDecodeError, ValueError): raise SystemExit(1) +with open(expected_path, "w", encoding="utf-8") as handle: + handle.write(current_text) +with open(expected_path + ".present", "w", encoding="utf-8") as handle: + handle.write("true" if current_present else "false") current_unrelated = {key: value for key, value in current.items() if key != "default-address-pools"} staged_unrelated = {key: value for key, value in staged.items() if key != "default-address-pools"} if current_unrelated == staged_unrelated: @@ -1058,9 +1135,9 @@ else: PY ) || return 1 if [[ "$rollback_action" == exact ]]; then - atomic_replace_daemon "$rollback_source" "$daemon_mode" "$daemon_gid" + atomic_replace_daemon "$rollback_source" "$daemon_mode" "$daemon_gid" "$rollback_expected" "$(<"$rollback_expected.present")" elif [[ "$rollback_action" == replace ]]; then - atomic_replace_daemon "$rollback_daemon" "$daemon_mode" "$daemon_gid" + atomic_replace_daemon "$rollback_daemon" "$daemon_mode" "$daemon_gid" "$rollback_expected" "$(<"$rollback_expected.present")" else rm -f "$daemon_config" python3 - "$daemon_dir" <<'PY' @@ -1122,6 +1199,10 @@ if [[ "$managed_before" == true ]]; then [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' fi +else + recovery_daemon= + [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name + transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' fi rollback_daemon() { @@ -1215,10 +1296,10 @@ if ! run_command "$probe_command"; then fi # Resume the drained controller before health verification. +controller_resumed=true if ! run_command "$resume_command" --env "$env_file"; then fail_after_apply "controller resume command failed after network-policy restart" fi -controller_resumed=true # Health verification if ! run_health "$env_file"; then @@ -1230,9 +1311,9 @@ verified_generation=$(file_generation "$daemon_config") || fail_after_apply "fai verified_action= [[ "$apply_removal_pending" != true ]] || verified_action=clear-removal set_verified_generation "$verified_generation" "$verified_action" || fail_after_apply "failed to record verified daemon.json generation" +clear_recovery_artifacts || fail_after_apply "failed to clear obsolete network-policy recovery data" # --- Success --- trap - EXIT INT TERM -[[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" rm -rf "$work_dir" printf 'NETWORK_POLICY_APPLIED daemon_config=%s\n' "$daemon_config" diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 9f5e41f1..3f8e554b 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -215,6 +215,15 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: return {} if count < 0: raise ValueError("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must be non-negative") + pool_prefix = "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_" + actual_pool_fields = {name for name in rendered if name.startswith(pool_prefix) and name != f"{pool_prefix}COUNT"} + expected_pool_fields = { + f"{pool_prefix}{index}_{field}" + for index in range(count) + for field in ("BASE", "SIZE") + } + if actual_pool_fields != expected_pool_fields: + raise ValueError("rendered Docker address-pool indexed fields must match the declared count") pools: list[dict[str, Any]] = [] for index in range(count): base = rendered.get(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_BASE") @@ -236,13 +245,15 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: if size < network.prefixlen: raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_SIZE: impossible subnet count") pools.append({"base": base, "size": size}) + state = rendered.get("CI_FLEET_CONTROLLER_STATE") + if state not in {"active", "drained", "disabled"}: + raise ValueError("CI_FLEET_CONTROLLER_STATE: must be active, drained, or disabled") try: policy = { "default_address_pools": pools, "networks_per_runner": int(rendered["CI_FLEET_DOCKER_NETWORKS_PER_RUNNER"]), "reserve_subnets": int(rendered["CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"]), } - state = rendered["CI_FLEET_CONTROLLER_STATE"] max_runners = 0 if state == "disabled" else int(rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"]) except (KeyError, ValueError) as exc: raise ValueError("rendered Docker network policy fields must be present integers") from exc diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index c913942d..847078ab 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -93,6 +93,35 @@ def test_rejects_negative_pool_count(self) -> None: "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "-1", }) + def test_rejects_pool_entries_outside_declared_count(self) -> None: + rendered = self._complete_rendered_policy() + rendered["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT"] = "1" + + with self.assertRaisesRegex(ValueError, "indexed fields must match"): + render_docker_daemon_config(rendered) + + def test_rejects_unknown_rendered_controller_state(self) -> None: + rendered = self._complete_rendered_policy() + rendered["CI_FLEET_CONTROLLER_STATE"] = "disable" + + with self.assertRaisesRegex(ValueError, "active, drained, or disabled"): + render_docker_daemon_config(rendered) + + @staticmethod + def _complete_rendered_policy() -> dict[str, str]: + value = config() + value["controllers"]["example-ci-01"]["docker_network_policy"] = docker_network_policy() + rendered, _ = build_rendered_env( + value, + "example-ci-01", + host_values(), + config_repository="example-org/example-fleet-config", + config_ref=CONFIG_COMMIT, + docker_gid=998, + engine_capabilities={"status_reporting_config", "required_status_reporting", "docker_network_policy_config"}, + ) + return rendered + def test_rejects_negative_rendered_runner_capacity(self) -> None: value = config() value["controllers"]["example-ci-01"]["docker_network_policy"] = docker_network_policy() @@ -2871,7 +2900,7 @@ def test_sleeping_probe_times_out(self) -> None: timeout=30, ) self.assertNotEqual(result.returncode, 0) - self.assertLess(time.monotonic() - started, 5) + self.assertLess(time.monotonic() - started, 7) @unittest.skipUnless(Path("/dev/shm").is_dir(), "/dev/shm is unavailable") def test_apply_works_across_temp_filesystems(self) -> None: @@ -3587,7 +3616,7 @@ def test_removal_failure_restores_snapshot_with_absent_managed_key(self) -> None self.assertNotEqual(removed.returncode, 0) self.assertIn("managed daemon.json restored", removed.stderr) self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {"debug": True, "live-restore": True}) - self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "restart", "resume"]) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "resume", "drain", "restart", "resume"]) def test_removal_pending_state_is_durable_before_daemon_mutation(self) -> None: daemon = self._write_daemon("{}\n") @@ -4026,7 +4055,7 @@ def test_removal_rollback_preserves_concurrent_content_and_metadata(self) -> Non self.assertEqual(daemon.stat().st_mode & 0o777, 0o600) self.assertEqual( command_log.read_text(encoding="utf-8").splitlines(), - ["drain", "restart", "resume", "restart", "resume", "health"], + ["drain", "restart", "resume", "drain", "restart", "resume", "health"], ) def test_removal_failure_restores_managed_config_and_retains_recovery_state(self) -> None: @@ -4241,6 +4270,135 @@ def test_removal_rejects_broken_checkpoint_state_symlink(self) -> None: self.assertIn("checkpoint state is invalid", result.stderr) self.assertEqual(daemon.read_bytes(), prior) + def test_first_apply_persists_prior_environment_before_drain(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-first-apply-environment" + self.drain_command.write_text( + "#!/usr/bin/env bash\n" + f"recovery=({checkpoint}/recovery.*)\n" + '[[ ${#recovery[@]} == 1 ]]\n' + f"cmp -s {self.installed_env} \"${{recovery[0]}}/prior-ci-fleet.env\"\n", + encoding="utf-8", + ) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(list(checkpoint.glob("recovery.*"))) + + def test_daemon_config_cannot_alias_checkpoint_state(self) -> None: + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-daemon-state-alias" + checkpoint.mkdir(mode=0o700) + env_file = self._write_env_file(self._rendered_with_policy()) + daemon = checkpoint / "docker-network-policy.json" + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=self._env(CI_FLEET_DOCKER_DAEMON_CONFIG=str(daemon)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkpoint state paths must be separate", result.stderr) + self.assertFalse(daemon.exists()) + + def test_no_change_clears_stale_committed_recovery(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-stale-recovery" + env_file = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + recovery = checkpoint / "recovery.stale" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "NETWORK_POLICY_NO_CHANGE\n") + self.assertFalse(recovery.exists()) + + def test_non_standard_daemon_json_is_rejected_before_drain(self) -> None: + daemon = self._write_daemon('{"log-level":NaN}\n') + prior = daemon.read_bytes() + self._write_success_commands() + drain_marker = Path(self.tmp) / "non-standard-json-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(Path(self.tmp) / "checkpoint-non-standard-json")) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("failed to stage merged daemon.json", result.stderr) + self.assertFalse(drain_marker.exists()) + self.assertEqual(daemon.read_bytes(), prior) + + def test_failed_resume_redrains_before_rollback_restart(self) -> None: + self._write_daemon("{}\n") + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "failed-resume-rollback.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + resume = Path(self.tmp) / "resume.sh" + resume.write_text(f"#!/usr/bin/env bash\necho resume >> {command_log}\nexit 1\n", encoding="utf-8") + resume.chmod(0o755) + + result = self._run(str(env_file), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "resume", "drain", "restart", "resume"], + ) + + def test_rollback_merge_rejects_concurrent_content_change(self) -> None: + daemon = self._write_daemon('{"icc":false}\n') + self._write_success_commands() + trigger = Path(self.tmp) / "mutate-during-rollback" + health = Path(self.tmp) / "health.sh" + health.write_text(f"#!/usr/bin/env bash\ntouch {trigger}\nexit 2\n", encoding="utf-8") + health.chmod(0o755) + fake_bin = Path(self.tmp) / "fake-bin" + fake_bin.mkdir() + stat_command = fake_bin / "stat" + stat_command.write_text( + "#!/usr/bin/env bash\n" + f"{shutil.which('stat')} \"$@\"\n" + "status=$?\n" + f"if [[ -e {trigger} && $1 == -c && $2 == %g && $3 == {daemon} ]]; then\n" + f" {shutil.which('python3')} -c \"import json; p='{daemon}'; v=json.load(open(p)); v['concurrent']=True; json.dump(v,open(p,'w'))\"\n" + f" rm -f {trigger}\n" + "fi\n" + "exit $status\n", + encoding="utf-8", + ) + stat_command.chmod(0o755) + checkpoint = Path(self.tmp) / "checkpoint-rollback-content-conflict" + env_file = self._write_env_file(self._rendered_with_policy()) + env = self._env(PATH=f"{fake_bin}:{os.environ['PATH']}") + + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(env_file)], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("rollback verification failed", result.stderr) + self.assertTrue(json.loads(daemon.read_text(encoding="utf-8"))["concurrent"]) + if __name__ == "__main__": unittest.main() From 95d8bccaf88b736464a1ae6ffd6bf1f71d17b245 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 11:53:11 -0500 Subject: [PATCH 44/54] fix: bound network policy transaction inputs --- scripts/apply-docker-network-policy.sh | 4 ++ scripts/desired_state.py | 5 ++ scripts/test_apply_docker_network_policy.py | 68 +++++++++++++++++++++ 3 files changed, 77 insertions(+) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index ccc20061..05d0cbfb 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -278,6 +278,10 @@ validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true t exec 8<"$checkpoint_dir" checkpoint_path_is_pinned || die 'checkpoint directory must remain a trusted root-owned path' state_file=/proc/self/fd/8/docker-network-policy.json +if [[ $(readlink -m "$checkpoint_state") == $(readlink -f /proc/self/fd/9) || + ( -e "$state_file" && "$state_file" -ef /proc/self/fd/9 ) ]]; then + die 'installer lock and checkpoint state paths must be separate' +fi if [[ $(readlink -m "$state_file") == "$daemon_config" || ( -e "$state_file" && "$state_file" -ef "$daemon_config" ) ]]; then die 'daemon config and checkpoint state paths must be separate' fi diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 3f8e554b..d52edf22 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -32,6 +32,7 @@ REQUIRED_STATUS_CAPABILITY = "required_status_reporting" STATUS_REPORTING_CONFIG_CAPABILITY = "status_reporting_config" DOCKER_NETWORK_POLICY_CONFIG_CAPABILITY = "docker_network_policy_config" +MAX_DOCKER_ADDRESS_POOLS = 64 class DesiredStateError(ValueError): @@ -215,8 +216,12 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: return {} if count < 0: raise ValueError("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must be non-negative") + if count > MAX_DOCKER_ADDRESS_POOLS: + raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must not exceed {MAX_DOCKER_ADDRESS_POOLS}") pool_prefix = "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_" actual_pool_fields = {name for name in rendered if name.startswith(pool_prefix) and name != f"{pool_prefix}COUNT"} + if len(actual_pool_fields) != count * 2: + raise ValueError("rendered Docker address-pool indexed fields must match the declared count") expected_pool_fields = { f"{pool_prefix}{index}_{field}" for index in range(count) diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 847078ab..b2710a87 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -93,6 +93,12 @@ def test_rejects_negative_pool_count(self) -> None: "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "-1", }) + def test_rejects_pool_count_above_repository_limit_before_expansion(self) -> None: + with self.assertRaisesRegex(ValueError, "must not exceed 64"): + render_docker_daemon_config({ + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT": "65", + }) + def test_rejects_pool_entries_outside_declared_count(self) -> None: rendered = self._complete_rendered_policy() rendered["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT"] = "1" @@ -1313,6 +1319,68 @@ def test_policy_reapply_rejects_persistent_recovery_backup(self) -> None: self.assertEqual(daemon.read_bytes(), prior) self.assertTrue(all(not marker.exists() for marker in markers)) + def test_rejects_installer_lock_at_checkpoint_marker_before_drain(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-lock-marker-alias" + checkpoint.mkdir(mode=0o700) + marker = checkpoint / "docker-network-policy.json" + marker.write_text("{}\n", encoding="utf-8") + marker.chmod(0o600) + drain_marker = Path(self.tmp) / "lock-marker-alias-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(marker)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("installer lock and checkpoint state paths must be separate", result.stderr) + self.assertFalse(drain_marker.exists()) + + def test_rejects_installer_lock_hardlink_to_checkpoint_marker_before_drain(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + checkpoint = Path(self.tmp) / "checkpoint-lock-marker-hardlink" + checkpoint.mkdir(mode=0o700) + marker = checkpoint / "docker-network-policy.json" + marker.write_text("{}\n", encoding="utf-8") + marker.chmod(0o600) + lock = Path(self.tmp) / "network-policy-marker-hardlink.lock" + os.link(marker, lock) + drain_marker = Path(self.tmp) / "lock-marker-hardlink-drain.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + env_file = self._write_env_file(self._rendered_with_policy()) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_INSTALLER_LOCK=str(lock)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("installer lock and checkpoint state paths must be separate", result.stderr) + self.assertFalse(drain_marker.exists()) + def test_rejects_absent_daemon_config_in_checkpoint_after_lock_before_drain(self) -> None: self.daemon_dir.chmod(0o700) self._write_success_commands() From abf1a84ad87b337ccdbd365b9ab7cafd5f8f8a78 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 13:23:30 -0500 Subject: [PATCH 45/54] fix: complete network policy recovery validation --- scripts/apply-docker-network-policy.sh | 21 +++- scripts/desired_state.py | 2 + scripts/test_apply_docker_network_policy.py | 109 +++++++++++++++++- scripts/test_desired_state.py | 14 +++ templates/config-repository/fleet.schema.json | 1 + .../config-repository/scripts/test_policy.py | 14 ++- .../config-repository/scripts/validate.py | 6 + 7 files changed, 156 insertions(+), 11 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 05d0cbfb..ab9b5557 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -192,6 +192,18 @@ trap 'rm -rf "$work_dir"' EXIT install -m 0600 -- "$env_file" "$work_dir/ci-fleet.env" env_file=$work_dir/ci-fleet.env +python3 - "$env_file" "$repo_root/scripts" 2>/dev/null <<'PY' || +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[2]) +from desired_state import parse_env + +if parse_env(Path(sys.argv[1]), allow_unknown=True).get("CI_FLEET_DESIRED_STATE_SCHEMA") != "3": + raise SystemExit(1) +PY + die 'rendered env must declare desired-state schema 3' + # --- No-op when no network policy is rendered --- removing=false [[ -n "$checkpoint_dir" ]] || die '--checkpoint is required' @@ -729,12 +741,11 @@ PY prior_present=${managed_state[0]} prior_mode=${managed_state[1]} prior_verified_generation=${managed_state[2]} - removal_pending=${managed_state[3]} has_verified_generation=${managed_state[4]} if [[ "$prior_present" == true ]]; then [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' fi - if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$removal_pending" == false && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" ]]; then + if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" ]]; then run_command "$restart_command" "$daemon_dir" || die 'Docker restart command failed while recovering interrupted network-policy apply' run_command "$probe_command" || die 'capacity probe failed while recovering interrupted network-policy apply' run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' @@ -1200,7 +1211,11 @@ if [[ "$managed_before" == true ]]; then transaction_recovery=$interrupted_recovery else recovery_daemon= - [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name + if [[ "$apply_removal_pending" == true ]]; then + recovery_daemon=$rollback_source + elif [[ "$snapshot_present" == true ]]; then + recovery_daemon=$backup_dir/$backup_name + fi transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' fi else diff --git a/scripts/desired_state.py b/scripts/desired_state.py index d52edf22..8584e9ae 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -143,6 +143,8 @@ def validate_host_values(values: dict[str, str]) -> dict[str, str]: def validate_docker_address_pools(pools: Any, *, path: str) -> list[dict[str, Any]]: if type(pools) is not list or not pools: raise DesiredStateError(f"{path}: must be a non-empty list") + if len(pools) > MAX_DOCKER_ADDRESS_POOLS: + raise DesiredStateError(f"{path}: must not exceed {MAX_DOCKER_ADDRESS_POOLS} pools") parsed: list[dict[str, Any]] = [] for index, pool in enumerate(pools): pool_path = f"{path}[{index}]" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index b2710a87..fadffd1e 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -346,6 +346,7 @@ def _env(self, **extra: str) -> dict[str, str]: return env def _write_env_file(self, rendered: dict[str, str]) -> Path: + rendered = {"CI_FLEET_DESIRED_STATE_SCHEMA": "3", **rendered} path = Path(self.tmp) / "ci-fleet.env" path.write_text("".join(f"{k}={v}\n" for k, v in sorted(rendered.items())), encoding="utf-8") return path @@ -471,6 +472,37 @@ def test_successful_apply_restarts_probes_resumes_then_checks_health(self) -> No self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "resume", "health"]) + def test_rejects_missing_malformed_or_non_v3_schema_before_checkpoint_or_drain(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + for configured in (True, False): + for label, schema in (("missing", None), ("malformed", "three"), ("non-v3", "2")): + with self.subTest(configured=configured, schema=label): + checkpoint = Path(self.tmp) / f"checkpoint-schema-{configured}-{label}" + drain_marker = Path(self.tmp) / f"drain-schema-{configured}-{label}.marker" + self.drain_command.write_text( + f"#!/usr/bin/env bash\ntouch {drain_marker}\n", + encoding="utf-8", + ) + rendered = self._rendered_with_policy() if configured else {"CI_FLEET_INSTANCE": "example-ci-01"} + if schema is None: + rendered.pop("CI_FLEET_DESIRED_STATE_SCHEMA", None) + env_file = Path(self.tmp) / f"schema-{configured}-{label}.env" + env_file.write_text( + "".join(f"{key}={value}\n" for key, value in sorted(rendered.items())), + encoding="utf-8", + ) + else: + rendered["CI_FLEET_DESIRED_STATE_SCHEMA"] = schema + env_file = self._write_env_file(rendered) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("schema 3", result.stderr) + self.assertFalse(checkpoint.exists()) + self.assertFalse(drain_marker.exists()) + def test_failed_apply_restarts_resumes_then_checks_rollback_health(self) -> None: self._write_daemon("{}\n") env_file = self._write_env_file(self._rendered_with_policy()) @@ -3520,7 +3552,7 @@ def test_managed_policy_removal_replaces_daemon_atomically(self) -> None: self.assertEqual(applied.returncode, 0, applied.stderr) no_policy_env = Path(self.tmp) / "no-policy-atomic-removal.env" - no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") fake_bin = Path(self.tmp) / "fake-cp-bin" fake_bin.mkdir() fake_cp = fake_bin / "cp" @@ -3580,6 +3612,71 @@ def test_interrupted_removal_retry_uses_persisted_managed_pools_for_rollback(sel self.assertNotIn("phase", rolled_back_state) self.assertNotIn("removal_managed_default_address_pools", rolled_back_state) + def test_pending_removal_with_absent_daemon_completes_runtime_verification(self) -> None: + daemon = self.daemon_dir / "daemon.json" + checkpoint = Path(self.tmp) / "checkpoint-pending-removal-absent-daemon" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = json.loads( + daemon.read_text(encoding="utf-8") + )["default-address-pools"] + state["verified_generation"] = None + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + daemon.unlink() + command_log = Path(self.tmp) / "pending-removal-absent-daemon.log" + for name in ("restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(removed.stdout, "NETWORK_POLICY_REMOVED\n") + self.assertFalse(state_file.exists()) + self.assertFalse(daemon.exists()) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["restart", "probe", "resume", "health"], + ) + + def test_apply_from_pending_removal_persists_synthesized_rollback_daemon(self) -> None: + daemon = self.daemon_dir / "daemon.json" + checkpoint = Path(self.tmp) / "checkpoint-pending-removal-apply-recovery" + self._write_success_commands() + policy = self._rendered_with_policy() + policy_env = self._write_env_file(policy) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed_pools = json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"] + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = managed_pools + state["verified_generation"] = None + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + daemon.unlink() + changed_policy = dict(policy) + changed_policy["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + changed_env = self._write_env_file(changed_policy) + (Path(self.tmp) / "restart.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "restart.sh").chmod(0o755) + + retried = self._run(str(changed_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(retried.returncode, 0) + recovery = next(checkpoint.glob("recovery.*")) + recovered_daemon = json.loads((recovery / "daemon.json.before").read_text(encoding="utf-8")) + self.assertEqual(recovered_daemon["default-address-pools"], managed_pools) + def test_configured_apply_resumes_removal_pending_checkpoint(self) -> None: daemon = self._write_daemon('{"live-restore":true}\n') checkpoint = Path(self.tmp) / "checkpoint-removal-pending-apply" @@ -3870,7 +3967,7 @@ def test_successful_removal_runs_runtime_checks_before_deleting_marker(self) -> self.installed_env.write_bytes(policy_env.read_bytes()) state_file = checkpoint / "docker-network-policy.json" no_policy_env = Path(self.tmp) / "no-policy-removal-resume.env" - no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") command_log = Path(self.tmp) / "removal-resume.log" self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") for name in ("restart", "probe", "resume", "health"): @@ -4045,7 +4142,7 @@ def test_managed_policy_removal_restores_original_state_and_clears_only_marker(s command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") command.chmod(0o755) no_policy_env = Path(self.tmp) / f"no-policy-{prior_present}.env" - no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") health = Path(self.tmp) / "health.sh" health.write_text( "#!/usr/bin/env bash\n" @@ -4157,7 +4254,7 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self ) health.chmod(0o755) no_policy_env = Path(self.tmp) / "no-policy-failure.env" - no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) @@ -4182,7 +4279,7 @@ def test_removal_rollback_resumes_and_checks_health_with_managed_env(self) -> No managed = daemon.read_bytes() self.installed_env.write_bytes(policy_env.read_bytes()) no_policy_env = Path(self.tmp) / "no-policy-removal-rollback.env" - no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") command_log = Path(self.tmp) / "removal-rollback-env.log" self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") restart = Path(self.tmp) / "restart.sh" @@ -4228,7 +4325,7 @@ def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") health.chmod(0o755) no_policy_env = Path(self.tmp) / "no-policy-marker-failure.env" - no_policy_env.write_text("CI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") audit_dir = Path(self.tmp) / "marker-failure-audit" audit_dir.mkdir() (audit_dir / "sitecustomize.py").write_text( diff --git a/scripts/test_desired_state.py b/scripts/test_desired_state.py index 2b88ee8d..38fce9f7 100755 --- a/scripts/test_desired_state.py +++ b/scripts/test_desired_state.py @@ -232,6 +232,20 @@ def test_docker_network_policy_accounts_for_every_runner_network(self) -> None: ) self.assertEqual((configured, reserve, networks_per_runner), (6, 1, 2)) + def test_docker_network_policy_rejects_more_than_64_pools_before_overlap_checks(self) -> None: + policy = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 29}] * 65, + } + + with self.assertRaisesRegex(DesiredStateError, "must not exceed 64"): + validate_docker_network_policy( + policy, + path="$.controllers.example-ci-01.docker_network_policy", + max_runners=1, + ) + def test_disabled_docker_network_policy_keeps_reserve_and_controller_capacity(self) -> None: configured, _, _, _ = validate_docker_network_policy( { diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index 9407dd73..b9ca3a46 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -91,6 +91,7 @@ "default_address_pools": { "type": "array", "minItems": 1, + "maxItems": 64, "items": { "type": "object", "additionalProperties": false, diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 385af024..20856b07 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -13,7 +13,7 @@ import unittest from pathlib import Path -from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_rollout_evidence, validate_transition +from validate import MAX_DOCKER_ADDRESS_POOLS, Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_rollout_evidence, validate_transition ROOT = Path(__file__).resolve().parents[1] @@ -89,7 +89,9 @@ def test_schema_defines_docker_network_policy_contract(self) -> None: self.assertEqual(set(controller), {"type", "additionalProperties", "required", "properties"}) self.assertEqual(controller["required"], ["default_address_pools", "networks_per_runner", "reserve_subnets"]) self.assertEqual(controller["properties"]["networks_per_runner"], {"type": "integer", "minimum": 1}) - pool = controller["properties"]["default_address_pools"]["items"] + pools = controller["properties"]["default_address_pools"] + self.assertEqual(pools["maxItems"], MAX_DOCKER_ADDRESS_POOLS) + pool = pools["items"] self.assertEqual(set(pool["properties"]), {"base", "size"}) self.assertEqual(pool["properties"]["size"]["maximum"], 29) @@ -147,6 +149,14 @@ def test_docker_network_policy_accounts_for_every_runner_network(self) -> None: ) self.assertEqual(errors_for(config), []) + def test_docker_network_policy_rejects_more_than_64_pools_before_overlap_checks(self) -> None: + config = copy.deepcopy(reference_config()) + first_controller(config)["docker_network_policy"]["default_address_pools"] = [ + {"base": "198.51.100.0/24", "size": 29} + ] * 65 + + self.assert_rejected(config, "must not exceed 64") + def test_disabled_docker_network_policy_keeps_reserve_and_controller_capacity(self) -> None: config = copy.deepcopy(reference_config()) controller = first_controller(config) diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index 7830d4a0..4ef51f13 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -59,6 +59,7 @@ RFC_5737_NETWORKS = tuple( ipaddress.ip_network(value) for value in ("192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24") ) +MAX_DOCKER_ADDRESS_POOLS = 64 class Validation: @@ -179,6 +180,11 @@ def validate_docker_network_policy( if type(pools) is not list or not pools: validation.errors.append(f"{path}.default_address_pools: must be a non-empty list") return 0, 0, [] + if len(pools) > MAX_DOCKER_ADDRESS_POOLS: + validation.errors.append( + f"{path}.default_address_pools: must not exceed {MAX_DOCKER_ADDRESS_POOLS} pools" + ) + return 0, 0, [] parsed: list[dict[str, Any]] = [] for index, pool in enumerate(pools): pool_path = f"{path}.default_address_pools[{index}]" From 148f0d215277815026e994efb3e026a88cc14a4f Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 15:17:30 -0500 Subject: [PATCH 46/54] fix: close network policy validation gaps --- docs/DESIRED-STATE.md | 5 +- scripts/apply-docker-network-policy.sh | 32 +++------- scripts/desired_state.py | 33 +++++++--- scripts/test_apply_docker_network_policy.py | 70 ++++++++++++++++----- templates/config-repository/README.md | 5 +- 5 files changed, 98 insertions(+), 47 deletions(-) diff --git a/docs/DESIRED-STATE.md b/docs/DESIRED-STATE.md index cc2561f5..5d262b7f 100644 --- a/docs/DESIRED-STATE.md +++ b/docs/DESIRED-STATE.md @@ -47,8 +47,9 @@ Active and drained controllers reserve their configured maximum against the pool The Docker network policy uses IPv4 CIDR `base` values and a Docker subnet prefix `size` no longer than `/29`, which leaves enough addresses for an -ordinary Compose network. Validation rejects malformed or overlapping pools, -allocation prefixes broader than their base, and active or drained policies with fewer subnets than +ordinary Compose network. A policy may declare at most 64 pools. Validation +rejects malformed or overlapping pools, allocation prefixes broader than their +base, and active or drained policies with fewer subnets than `max_runners * networks_per_runner + reserve_subnets + 1`. The final subnet is reserved for the persistent controller Compose network. Disabled controllers do not reserve runner subnet capacity, but their retained policy must still cover the reserve diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index ab9b5557..fada5d97 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -204,34 +204,22 @@ if parse_env(Path(sys.argv[1]), allow_unknown=True).get("CI_FLEET_DESIRED_STATE_ PY die 'rendered env must declare desired-state schema 3' -# --- No-op when no network policy is rendered --- -removing=false -[[ -n "$checkpoint_dir" ]] || die '--checkpoint is required' -if ! grep -q '^CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=' "$env_file"; then - if grep -Eq '^CI_FLEET_DOCKER_(DEFAULT_ADDRESS_POOL_|NETWORKS_PER_RUNNER=|NETWORK_RESERVE_SUBNETS=)' "$env_file"; then - die 'rendered network-policy fields are incomplete' - fi - removing=true -fi - # Validate the locked candidate snapshot before mutation. -if [[ "$removing" == false ]]; then - desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" 2>/dev/null <<'PY' +desired_pools_json=$(python3 - "$env_file" "$repo_root/scripts" 2>/dev/null <<'PY' import json, sys env_path, scripts_dir = sys.argv[1], sys.argv[2] sys.path.insert(0, scripts_dir) -values = {} -with open(env_path, encoding="utf-8") as handle: - for line in handle: - line = line.rstrip("\n") - if "=" in line and line: - key, _, value = line.partition("=") - values[key] = value -from desired_state import render_docker_daemon_config +from pathlib import Path +from desired_state import parse_env, render_docker_daemon_config +values = parse_env(Path(env_path), allow_unknown=True) print(json.dumps(render_docker_daemon_config(values))) PY - ) || die "daemon policy rendering failed" -fi +) || die "daemon policy rendering failed" + +# --- No-op when no network policy is rendered --- +removing=false +[[ -n "$checkpoint_dir" ]] || die '--checkpoint is required' +[[ "$desired_pools_json" != '{}' ]] || removing=true # --- Resolve required injected commands --- daemon_config=${CI_FLEET_DOCKER_DAEMON_CONFIG:-} diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 8584e9ae..c2e82204 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -209,18 +209,27 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: merging into ``daemon.json``. When no policy was rendered, returns an empty dict (no ``default-address-pools`` key). """ - count_str = rendered.get("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT", "0") + count_name = "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" + pool_prefix = "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_" + policy_configured = count_name in rendered + if not policy_configured: + if any( + name.startswith(pool_prefix) + or name in {"CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"} + for name in rendered + ): + raise ValueError("rendered Docker network policy fields must include the pool count") + if not rendered: + return {} + count_str = rendered.get(count_name, "0") try: count = int(count_str) except ValueError as exc: raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must be an integer, got {count_str!r}") from exc - if count == 0 and "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT" not in rendered: - return {} if count < 0: raise ValueError("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must be non-negative") if count > MAX_DOCKER_ADDRESS_POOLS: raise ValueError(f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT: must not exceed {MAX_DOCKER_ADDRESS_POOLS}") - pool_prefix = "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_" actual_pool_fields = {name for name in rendered if name.startswith(pool_prefix) and name != f"{pool_prefix}COUNT"} if len(actual_pool_fields) != count * 2: raise ValueError("rendered Docker address-pool indexed fields must match the declared count") @@ -255,17 +264,27 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: state = rendered.get("CI_FLEET_CONTROLLER_STATE") if state not in {"active", "drained", "disabled"}: raise ValueError("CI_FLEET_CONTROLLER_STATE: must be active, drained, or disabled") + try: + configured_max_runners = int(rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"]) + effective_max_runners = int(rendered["CI_FLEET_MAX_RUNNERS"]) + except (KeyError, ValueError) as exc: + raise ValueError("rendered controller capacity fields must be present integers") from exc + if state != "disabled" and configured_max_runners < 1: + raise ValueError("CI_FLEET_CONFIGURED_MAX_RUNNERS: must be a positive integer") + expected_effective_max = configured_max_runners if state == "active" else 0 + if effective_max_runners != expected_effective_max: + raise ValueError("CI_FLEET_MAX_RUNNERS: must match effective controller capacity") + if not policy_configured: + return {} try: policy = { "default_address_pools": pools, "networks_per_runner": int(rendered["CI_FLEET_DOCKER_NETWORKS_PER_RUNNER"]), "reserve_subnets": int(rendered["CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"]), } - max_runners = 0 if state == "disabled" else int(rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"]) except (KeyError, ValueError) as exc: raise ValueError("rendered Docker network policy fields must be present integers") from exc - if state != "disabled" and max_runners < 1: - raise ValueError("CI_FLEET_CONFIGURED_MAX_RUNNERS: must be a positive integer") + max_runners = 0 if state == "disabled" else configured_max_runners validate_docker_network_policy(policy, path="rendered Docker network policy", max_runners=max_runners) return {"default-address-pools": pools} diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index fadffd1e..1d91ffba 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -346,7 +346,13 @@ def _env(self, **extra: str) -> dict[str, str]: return env def _write_env_file(self, rendered: dict[str, str]) -> Path: - rendered = {"CI_FLEET_DESIRED_STATE_SCHEMA": "3", **rendered} + rendered = { + "CI_FLEET_CONFIGURED_MAX_RUNNERS": "1", + "CI_FLEET_CONTROLLER_STATE": "active", + "CI_FLEET_DESIRED_STATE_SCHEMA": "3", + "CI_FLEET_MAX_RUNNERS": "1", + **rendered, + } path = Path(self.tmp) / "ci-fleet.env" path.write_text("".join(f"{k}={v}\n" for k, v in sorted(rendered.items())), encoding="utf-8") return path @@ -472,6 +478,48 @@ def test_successful_apply_restarts_probes_resumes_then_checks_health(self) -> No self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "resume", "health"]) + def test_leading_whitespace_policy_assignments_are_applied(self) -> None: + daemon = self._write_daemon("{}\n") + rendered = self._rendered_with_policy() + env_file = Path(self.tmp) / "leading-whitespace-policy.env" + env_file.write_text( + "".join( + f"{' ' if key.startswith('CI_FLEET_DOCKER_') else ''}{key}={value}\n" + for key, value in sorted(rendered.items()) + ), + encoding="utf-8", + ) + self._write_success_commands() + + result = self._run(str(env_file)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("default-address-pools", json.loads(daemon.read_text(encoding="utf-8"))) + + def test_effective_capacity_mismatch_is_rejected_before_drain(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + for configured in (True, False): + with self.subTest(configured=configured): + rendered = self._rendered_with_policy() + if not configured: + rendered = { + key: value + for key, value in rendered.items() + if not key.startswith("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_") + and key not in {"CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"} + } + rendered["CI_FLEET_MAX_RUNNERS"] = "1000" + env_file = self._write_env_file(rendered) + drain_marker = Path(self.tmp) / f"capacity-mismatch-{configured}.marker" + self.drain_command.write_text(f"#!/usr/bin/env bash\ntouch {drain_marker}\n", encoding="utf-8") + + result = self._run(str(env_file), checkpoint_dir=str(Path(self.tmp) / f"checkpoint-capacity-{configured}")) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr, "ERROR: daemon policy rendering failed\n") + self.assertFalse(drain_marker.exists()) + def test_rejects_missing_malformed_or_non_v3_schema_before_checkpoint_or_drain(self) -> None: self._write_daemon("{}\n") self._write_success_commands() @@ -1039,7 +1087,7 @@ def test_partial_no_policy_snapshot_is_rejected_before_removal(self) -> None: result = self._run(str(env_file)) self.assertNotEqual(result.returncode, 0) - self.assertIn("network-policy fields are incomplete", result.stderr) + self.assertEqual(result.stderr, "ERROR: daemon policy rendering failed\n") self.assertEqual(daemon.read_bytes(), prior) self.assertFalse(drain_marker.exists()) @@ -3551,8 +3599,7 @@ def test_managed_policy_removal_replaces_daemon_atomically(self) -> None: applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) self.assertEqual(applied.returncode, 0, applied.stderr) - no_policy_env = Path(self.tmp) / "no-policy-atomic-removal.env" - no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) fake_bin = Path(self.tmp) / "fake-cp-bin" fake_bin.mkdir() fake_cp = fake_bin / "cp" @@ -3966,8 +4013,7 @@ def test_successful_removal_runs_runtime_checks_before_deleting_marker(self) -> self.assertEqual(applied.returncode, 0, applied.stderr) self.installed_env.write_bytes(policy_env.read_bytes()) state_file = checkpoint / "docker-network-policy.json" - no_policy_env = Path(self.tmp) / "no-policy-removal-resume.env" - no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) command_log = Path(self.tmp) / "removal-resume.log" self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") for name in ("restart", "probe", "resume", "health"): @@ -4141,8 +4187,7 @@ def test_managed_policy_removal_restores_original_state_and_clears_only_marker(s command = Path(self.tmp) / name command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") command.chmod(0o755) - no_policy_env = Path(self.tmp) / f"no-policy-{prior_present}.env" - no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) health = Path(self.tmp) / "health.sh" health.write_text( "#!/usr/bin/env bash\n" @@ -4253,8 +4298,7 @@ def test_removal_failure_restores_managed_config_and_retains_recovery_state(self encoding="utf-8", ) health.chmod(0o755) - no_policy_env = Path(self.tmp) / "no-policy-failure.env" - no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) @@ -4278,8 +4322,7 @@ def test_removal_rollback_resumes_and_checks_health_with_managed_env(self) -> No self.assertEqual(applied.returncode, 0, applied.stderr) managed = daemon.read_bytes() self.installed_env.write_bytes(policy_env.read_bytes()) - no_policy_env = Path(self.tmp) / "no-policy-removal-rollback.env" - no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) command_log = Path(self.tmp) / "removal-rollback-env.log" self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") restart = Path(self.tmp) / "restart.sh" @@ -4324,8 +4367,7 @@ def test_marker_clear_failure_rolls_back_managed_config_and_retains_state(self) health = Path(self.tmp) / "health.sh" health.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") health.chmod(0o755) - no_policy_env = Path(self.tmp) / "no-policy-marker-failure.env" - no_policy_env.write_text("CI_FLEET_DESIRED_STATE_SCHEMA=3\nCI_FLEET_INSTANCE=example-ci-01\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) audit_dir = Path(self.tmp) / "marker-failure-audit" audit_dir.mkdir() (audit_dir / "sitecustomize.py").write_text( diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index b31e6979..dd03986a 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -112,8 +112,9 @@ The validator totals the maximum capacity of every active or drained controller For `docker_network_policy`, each pool has an IPv4 CIDR `base` and Docker subnet prefix `size`. The size must be no longer than `/29` and cannot be -broader than its base. Pools must not overlap, and active or drained controllers -must provide at least `max_runners * networks_per_runner + reserve_subnets + 1` +broader than its base. A policy may declare at most 64 pools. Pools must not +overlap, and active or drained controllers must provide at least +`max_runners * networks_per_runner + reserve_subnets + 1` subnets. The final subnet is reserved for the persistent controller Compose network. Real pool values belong in the private configuration; this template uses RFC 5737 From 777af29697aaed9c755524f0ce142b2e19b4c867 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 17:43:30 -0500 Subject: [PATCH 47/54] fix: complete network policy validation closure --- scripts/apply-docker-network-policy.sh | 13 +- scripts/desired_state.py | 10 +- scripts/test_apply_docker_network_policy.py | 239 ++++++++++++++++++++ templates/config-repository/README.md | 18 +- 4 files changed, 271 insertions(+), 9 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index fada5d97..f32801ed 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -738,6 +738,7 @@ PY run_command "$probe_command" || die 'capacity probe failed while recovering interrupted network-policy apply' run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' run_health "$env_file" || die 'health check failed while recovering interrupted network-policy apply' + clear_recovery_artifacts || die 'failed to clear obsolete network-policy recovery data' clear_managed_marker "$work_dir/daemon.json.removal" || die 'failed to clear interrupted network-policy marker' rm -rf "$work_dir" printf 'NETWORK_POLICY_REMOVED\n' @@ -762,9 +763,11 @@ PY managed_metadata=$(stat -c '%d:%i:%u:%a:%g' "$daemon_config") managed_snapshot=$work_dir/daemon.json.before cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } - python3 - "$managed_snapshot" "$state_file" "$managed_daemon" "$removal_daemon" 2>/dev/null <<'PY' || { + python3 - "$managed_snapshot" "$state_file" "$managed_daemon" "$removal_daemon" "$repo_root/scripts" 2>/dev/null <<'PY' || { import json, sys -daemon_path, state_path, managed_path, removal_path = sys.argv[1:] +daemon_path, state_path, managed_path, removal_path, scripts_path = sys.argv[1:] +sys.path.insert(0, scripts_path) +from desired_state import validate_docker_address_pools try: current = json.load(open(daemon_path, encoding="utf-8")) state = json.load(open(state_path, encoding="utf-8")) @@ -774,6 +777,8 @@ try: managed_pools = state["removal_managed_default_address_pools"] else: managed_pools = current.get("default-address-pools") + if "default-address-pools" in current: + validate_docker_address_pools(managed_pools, path="managed daemon default address pools") except (OSError, json.JSONDecodeError, KeyError, ValueError): raise SystemExit(1) managed = dict(current) @@ -925,6 +930,10 @@ if ( ): raise SystemExit(1) PY + if ! clear_recovery_artifacts; then + removal_failure='failed to clear obsolete network-policy recovery data' + exit 2 + fi if ! clear_managed_marker "$removal_daemon"; then removal_failure='failed to clear network-policy managed marker' exit 2 diff --git a/scripts/desired_state.py b/scripts/desired_state.py index c2e82204..7c66cf69 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -267,10 +267,18 @@ def render_docker_daemon_config(rendered: dict[str, str]) -> dict[str, Any]: try: configured_max_runners = int(rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"]) effective_max_runners = int(rendered["CI_FLEET_MAX_RUNNERS"]) + minimum_runners = int(rendered["CI_FLEET_MIN_RUNNERS"]) + capacity_budget = int(rendered["CI_FLEET_CAPACITY_BUDGET"]) except (KeyError, ValueError) as exc: raise ValueError("rendered controller capacity fields must be present integers") from exc - if state != "disabled" and configured_max_runners < 1: + if minimum_runners != 0: + raise ValueError("CI_FLEET_MIN_RUNNERS: must be zero") + if capacity_budget < 1: + raise ValueError("CI_FLEET_CAPACITY_BUDGET: must be a positive integer") + if configured_max_runners < 1: raise ValueError("CI_FLEET_CONFIGURED_MAX_RUNNERS: must be a positive integer") + if state != "disabled" and configured_max_runners > capacity_budget: + raise ValueError("CI_FLEET_CONFIGURED_MAX_RUNNERS: must not exceed CI_FLEET_CAPACITY_BUDGET") expected_effective_max = configured_max_runners if state == "active" else 0 if effective_max_runners != expected_effective_max: raise ValueError("CI_FLEET_MAX_RUNNERS: must match effective controller capacity") diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 1d91ffba..0688e582 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -347,10 +347,12 @@ def _env(self, **extra: str) -> dict[str, str]: def _write_env_file(self, rendered: dict[str, str]) -> Path: rendered = { + "CI_FLEET_CAPACITY_BUDGET": "1", "CI_FLEET_CONFIGURED_MAX_RUNNERS": "1", "CI_FLEET_CONTROLLER_STATE": "active", "CI_FLEET_DESIRED_STATE_SCHEMA": "3", "CI_FLEET_MAX_RUNNERS": "1", + "CI_FLEET_MIN_RUNNERS": "0", **rendered, } path = Path(self.tmp) / "ci-fleet.env" @@ -394,6 +396,122 @@ def _write_success_commands(self) -> None: command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") command.chmod(0o755) + def _assert_raw_candidate_rejected_before_operational_side_effects( + self, + rendered: dict[str, str], + label: str, + ) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + prior = daemon.read_bytes() + checkpoint = Path(self.tmp) / f"checkpoint-raw-{label}" + command_log = Path(self.tmp) / f"commands-raw-{label}.log" + env_file = Path(self.tmp) / f"candidate-raw-{label}.env" + env_file.write_text( + "".join(f"{key}={value}\n" for key, value in sorted(rendered.items())), + encoding="utf-8", + ) + for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr, "ERROR: daemon policy rendering failed\n") + self.assertFalse(checkpoint.exists()) + self.assertFalse(command_log.exists()) + self.assertEqual(daemon.read_bytes(), prior) + + def test_rejects_raw_minimum_capacity_before_operational_side_effects(self) -> None: + for policy_configured in (True, False): + for state in ("active", "drained", "disabled"): + for label, value in (("missing", None), ("malformed", "none"), ("nonzero", "1")): + with self.subTest(policy_configured=policy_configured, state=state, value=label): + rendered = self._rendered_with_policy() + if not policy_configured: + rendered = { + key: item + for key, item in rendered.items() + if not key.startswith("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_") + and key not in {"CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"} + } + rendered["CI_FLEET_CONTROLLER_STATE"] = state + rendered["CI_FLEET_MAX_RUNNERS"] = "1" if state == "active" else "0" + if value is None: + rendered.pop("CI_FLEET_MIN_RUNNERS") + else: + rendered["CI_FLEET_MIN_RUNNERS"] = value + self._assert_raw_candidate_rejected_before_operational_side_effects( + rendered, + f"minimum-{policy_configured}-{state}-{label}", + ) + + def test_rejects_raw_capacity_budget_before_operational_side_effects(self) -> None: + for policy_configured in (True, False): + for state in ("active", "drained", "disabled"): + for label, value in (("missing", None), ("malformed", "none"), ("zero", "0"), ("negative", "-1")): + with self.subTest(policy_configured=policy_configured, state=state, value=label): + rendered = self._rendered_with_policy() + if not policy_configured: + rendered = { + key: item + for key, item in rendered.items() + if not key.startswith("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_") + and key not in {"CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"} + } + rendered["CI_FLEET_CONTROLLER_STATE"] = state + rendered["CI_FLEET_MAX_RUNNERS"] = "1" if state == "active" else "0" + if value is None: + rendered.pop("CI_FLEET_CAPACITY_BUDGET") + else: + rendered["CI_FLEET_CAPACITY_BUDGET"] = value + self._assert_raw_candidate_rejected_before_operational_side_effects( + rendered, + f"budget-{policy_configured}-{state}-{label}", + ) + for state in ("active", "drained"): + with self.subTest(policy_configured=policy_configured, state=state, value="over-budget"): + rendered = self._rendered_with_policy() + if not policy_configured: + rendered = { + key: item + for key, item in rendered.items() + if not key.startswith("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_") + and key not in {"CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"} + } + rendered["CI_FLEET_CONTROLLER_STATE"] = state + rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"] = "3" + rendered["CI_FLEET_MAX_RUNNERS"] = "3" if state == "active" else "0" + rendered["CI_FLEET_CAPACITY_BUDGET"] = "2" + self._assert_raw_candidate_rejected_before_operational_side_effects( + rendered, + f"budget-{policy_configured}-{state}-over-budget", + ) + + def test_rejects_raw_disabled_nonpositive_configured_maximum_before_operational_side_effects(self) -> None: + for policy_configured in (True, False): + for label, value in (("missing", None), ("malformed", "none"), ("zero", "0"), ("negative", "-1")): + with self.subTest(policy_configured=policy_configured, value=label): + rendered = self._rendered_with_policy() + if not policy_configured: + rendered = { + key: item + for key, item in rendered.items() + if not key.startswith("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_") + and key not in {"CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS"} + } + rendered["CI_FLEET_CONTROLLER_STATE"] = "disabled" + if value is None: + rendered.pop("CI_FLEET_CONFIGURED_MAX_RUNNERS") + else: + rendered["CI_FLEET_CONFIGURED_MAX_RUNNERS"] = value + rendered["CI_FLEET_MAX_RUNNERS"] = "0" + self._assert_raw_candidate_rejected_before_operational_side_effects( + rendered, + f"disabled-configured-maximum-{policy_configured}-{label}", + ) + def test_rejects_untrusted_transaction_temp_parent(self) -> None: self._write_daemon("{}\n") env_file = self._write_env_file(self._rendered_with_policy()) @@ -3277,6 +3395,10 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> encoding="utf-8", ) state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) command_markers = [] for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): marker = Path(self.tmp) / f"interrupted-before-rename-{name}.marker" @@ -3291,6 +3413,7 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> self.assertEqual(removed.returncode, 0, removed.stderr) self.assertEqual(removed.stdout, "NETWORK_POLICY_REMOVED\n") self.assertFalse(state_file.exists()) + self.assertFalse(recovery.exists()) self.assertFalse(daemon.exists()) self.assertEqual( [marker.name for marker in command_markers if marker.exists()], @@ -3558,6 +3681,36 @@ def test_removal_rejects_malformed_saved_pools_before_commands_or_mutation(self) self.assertEqual(daemon.read_bytes(), managed) self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_removal_rejects_malformed_current_managed_pools_before_checkpoint_or_commands(self) -> None: + for label, pools in (("malformed-entry", [{"base": "not-a-cidr", "size": 29}]), ("null", None)): + with self.subTest(pools=label): + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / f"checkpoint-malformed-current-managed-pools-{label}" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + managed_state = state_file.read_bytes() + malformed = json.dumps({"default-address-pools": pools, "live-restore": True}).encode() + daemon.write_bytes(malformed) + command_markers = [] + for name in ("drain.sh", "restart.sh", "probe.sh", "resume.sh", "health.sh"): + marker = Path(self.tmp) / f"malformed-current-managed-pools-{label}-{name}.marker" + command_markers.append(marker) + command = Path(self.tmp) / name + command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") + command.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("managed daemon.json is not a valid JSON object", removed.stderr) + self.assertEqual(state_file.read_bytes(), managed_state) + self.assertEqual(daemon.read_bytes(), malformed) + self.assertFalse(any(marker.exists() for marker in command_markers)) + def test_removal_rejects_malformed_pending_pools_before_commands_or_mutation(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-malformed-pending-pools" @@ -4032,6 +4185,92 @@ def test_successful_removal_runs_runtime_checks_before_deleting_marker(self) -> self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "restart", "probe", "resume", "health"]) self.assertFalse(state_file.exists()) + def test_successful_removal_clears_recovery_before_marker_and_allows_next_reconciliation(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-recovery-cleanup" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + state_file = checkpoint / "docker-network-policy.json" + recovery = checkpoint / "recovery.stale" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) + audit_log = Path(self.tmp) / "removal-recovery-cleanup.log" + audit_dir = Path(self.tmp) / "removal-recovery-cleanup-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os, shutil\n" + "_unlink = os.unlink\n" + "_rmtree = shutil.rmtree\n" + "_log = os.environ['CLEANUP_AUDIT_LOG']\n" + "def record(value):\n" + " with open(_log, 'a', encoding='utf-8') as handle: handle.write(value + '\\n')\n" + "def unlink(path, *args, **kwargs):\n" + " if os.path.realpath(path) == os.environ['STATE_FILE']: record('marker')\n" + " return _unlink(path, *args, **kwargs)\n" + "def rmtree(path, *args, **kwargs):\n" + " if os.path.realpath(path) == os.environ['RECOVERY_DIR']: record('recovery')\n" + " return _rmtree(path, *args, **kwargs)\n" + "os.unlink = unlink\n" + "shutil.rmtree = rmtree\n", + encoding="utf-8", + ) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, + text=True, + env=self._env( + PYTHONPATH=str(audit_dir), + CLEANUP_AUDIT_LOG=str(audit_log), + STATE_FILE=str(state_file), + RECOVERY_DIR=str(recovery), + ), + timeout=30, + ) + + self.assertEqual(removed.returncode, 0, removed.stderr) + self.assertEqual(audit_log.read_text(encoding="utf-8").splitlines(), ["recovery", "marker"]) + self.assertFalse(recovery.exists()) + reconciled = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(reconciled.returncode, 0, reconciled.stderr) + self.assertEqual(reconciled.stdout, "NETWORK_POLICY_NOOP\n") + + def test_removal_recovery_cleanup_failure_rolls_back_and_keeps_marker_usable(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-recovery-cleanup-failure" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + recovery = checkpoint / "recovery.invalid" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) + (recovery / "unexpected").write_text("invalid\n", encoding="utf-8") + (recovery / "unexpected").chmod(0o600) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("failed to clear obsolete network-policy recovery data", removed.stderr) + self.assertEqual(daemon.read_bytes(), managed) + state = json.loads(state_file.read_text(encoding="utf-8")) + self.assertTrue(state["managed"]) + self.assertNotIn("phase", state) + self.assertTrue(recovery.exists()) + shutil.rmtree(recovery) + retried = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(retried.returncode, 0, retried.stderr) + self.assertEqual(retried.stdout, "NETWORK_POLICY_REMOVED\n") + def test_successful_removal_fsyncs_checkpoint_after_marker_unlink(self) -> None: self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-marker-fsync" diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index dd03986a..2994b3e7 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -134,12 +134,18 @@ integrated state when introducing the policy. It also requires matching current evidence whenever a controller retains the policy, including across engine changes or rollbacks. -The public engine renders these values only for read-only health inspection. -This phase detects low water, exhaustion, failed inspection, and legacy -networks; it does not configure or restart Docker, create/delete networks, -clean resources, drain runners, or change controller scale. Circuit breaking, -frequent orphan reconciliation, daemon mutation, transactional recovery, and -consumer-label changes are later issue #81 slices. +After the three-commit engine activation gate above, the pinned public engine +can apply or remove a validated policy with at most 64 pools. The executable +stage holds the installer lock, drains the controller and managed runners, +changes only Docker's managed `default-address-pools` key, restarts Docker, +probes capacity, resumes the intended controller state, and checks health. It +uses prior-key provenance and the prior rendered environment to roll back a +failed or interrupted transaction, and retains recovery data if rollback cannot +be verified. A merged or configured policy does not authorize host mutation; +rollout still requires separate operator approval, exact-head CI, and proof for +the reviewed engine and desired-state commits. This stage does not create or +delete networks, prune resources, change controller scale or consumer labels, +or authorize application deployment. Application repositories do not encode the number of available workers. They submit all independent tasks and shards. Do not use GitHub Actions `strategy.max-parallel` to model fleet size; controllers and the private configuration decide how many jobs run simultaneously. An application may limit concurrency only for a separately documented external-system constraint, not worker availability. From 78de795d8d384cd5fd409e0a5e2533bdc533e362 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 18:39:35 -0500 Subject: [PATCH 48/54] fix: reject daemon paths within checkpoints --- scripts/apply-docker-network-policy.sh | 1 + scripts/test_apply_docker_network_policy.py | 49 ++++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index f32801ed..32a2fb28 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -251,6 +251,7 @@ run_health() { validate_trusted_path CI_FLEET_DOCKER_DAEMON_CONFIG "$daemon_config" regular true [[ ! -e "$daemon_config" || ! /proc/self/fd/9 -ef "$daemon_config" ]] || die 'installer lock and daemon paths must be separate' validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true +[[ "$daemon_config" != "$checkpoint_dir" && "$daemon_config" != "$checkpoint_dir/"* ]] || die 'daemon config must be outside checkpoint directory' checkpoint_state=$checkpoint_dir/docker-network-policy.json [[ "$removing" != true || ! -L "$checkpoint_state" ]] || die 'network-policy checkpoint state is invalid' if [[ "$removing" == true && ( ! -e "$checkpoint_dir" || ! -e "$checkpoint_state" ) ]]; then diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 0688e582..fa02743b 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -581,6 +581,51 @@ def test_rejects_installed_env_under_untrusted_directory(self) -> None: self.assertNotEqual(result.returncode, 0) self.assertIn("installed rendered env must be a trusted root-owned path", result.stderr) + def test_rejects_daemon_config_inside_recovery_before_side_effects(self) -> None: + checkpoint = Path(self.tmp) / "checkpoint-contained-daemon" + checkpoint.mkdir(mode=0o700) + recovery = checkpoint / "recovery.review" + recovery.mkdir(mode=0o700) + daemon = recovery / "daemon.json.before" + daemon.write_text('{"live-restore":true}\n', encoding="utf-8") + daemon.chmod(0o600) + prior = daemon.read_bytes() + prior_env = recovery / "prior-ci-fleet.env" + prior_env.write_bytes(self.installed_env.read_bytes()) + prior_env.chmod(0o600) + env_file = self._write_env_file(self._rendered_with_policy()) + command_log = Path(self.tmp) / "contained-daemon-commands.log" + for command in ( + self.drain_command, + Path(self.tmp) / "restart.sh", + Path(self.tmp) / "probe.sh", + Path(self.tmp) / "resume.sh", + Path(self.tmp) / "health.sh", + ): + command.write_text(f"#!/usr/bin/env bash\necho side-effect >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env(CI_FLEET_DOCKER_DAEMON_CONFIG=str(daemon)), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stderr, "ERROR: daemon config must be outside checkpoint directory\n") + self.assertFalse(command_log.exists()) + self.assertFalse((checkpoint / "docker-network-policy.json").exists()) + self.assertEqual(daemon.read_bytes(), prior) + self.assertTrue(recovery.is_dir()) + def test_successful_apply_restarts_probes_resumes_then_checks_health(self) -> None: self._write_daemon("{}\n") env_file = self._write_env_file(self._rendered_with_policy()) @@ -1602,7 +1647,7 @@ def test_rejects_absent_daemon_config_in_checkpoint_after_lock_before_drain(self ) self.assertNotEqual(result.returncode, 0) - self.assertIn("daemon config and checkpoint entry must be separate paths", result.stderr) + self.assertIn("daemon config must be outside checkpoint directory", result.stderr) self.assertFalse((self.daemon_dir / "daemon.json").exists()) self.assertTrue(lock.exists()) self.assertFalse(drain_marker.exists()) @@ -4750,7 +4795,7 @@ def test_daemon_config_cannot_alias_checkpoint_state(self) -> None: ) self.assertNotEqual(result.returncode, 0) - self.assertIn("checkpoint state paths must be separate", result.stderr) + self.assertIn("daemon config must be outside checkpoint directory", result.stderr) self.assertFalse(daemon.exists()) def test_no_change_clears_stale_committed_recovery(self) -> None: From 3aa7fa7e490d373687dc2d245534dd045a5f7580 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 19:38:43 -0500 Subject: [PATCH 49/54] fix: close network policy recovery gaps --- scripts/apply-docker-network-policy.sh | 76 ++++-- scripts/healthcheck.sh | 15 +- scripts/test-healthcheck.sh | 38 ++- scripts/test_apply_docker_network_policy.py | 241 ++++++++++++++++++++ 4 files changed, 346 insertions(+), 24 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 32a2fb28..f7738660 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -730,26 +730,57 @@ PY prior_present=${managed_state[0]} prior_mode=${managed_state[1]} prior_verified_generation=${managed_state[2]} + removal_pending=${managed_state[3]} has_verified_generation=${managed_state[4]} if [[ "$prior_present" == true ]]; then [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' fi - if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" ]]; then + absent_removal_recovery=false + if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" && "$removal_pending" != true ]]; then + drain_controller 'drain command failed before interrupted network-policy recovery' run_command "$restart_command" "$daemon_dir" || die 'Docker restart command failed while recovering interrupted network-policy apply' run_command "$probe_command" || die 'capacity probe failed while recovering interrupted network-policy apply' run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' run_health "$env_file" || die 'health check failed while recovering interrupted network-policy apply' clear_recovery_artifacts || die 'failed to clear obsolete network-policy recovery data' clear_managed_marker "$work_dir/daemon.json.removal" || die 'failed to clear interrupted network-policy marker' + trap - EXIT INT TERM rm -rf "$work_dir" printf 'NETWORK_POLICY_REMOVED\n' exit 0 fi - [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' + if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" && "$removal_pending" == true ]]; then + absent_removal_recovery=true + else + [[ -f "$daemon_config" ]] || die 'managed daemon.json is missing before policy removal' + fi managed_daemon=$work_dir/daemon.json.managed removal_daemon=$work_dir/daemon.json.removal - python3 - "$daemon_config" <<'PY' || { rm -rf "$work_dir"; die 'managed daemon.json is not a valid JSON object'; } + managed_snapshot=$work_dir/daemon.json.before + if [[ "$absent_removal_recovery" == true ]]; then + python3 - "$state_file" "$managed_daemon" "$removal_daemon" "$managed_snapshot" <<'PY' || { +import json, shutil, sys +state_path, managed_path, removal_path, snapshot_path = sys.argv[1:] +state = json.load(open(state_path, encoding="utf-8")) +managed = {} +if state["removal_managed_default_address_pools"] is not None: + managed["default-address-pools"] = state["removal_managed_default_address_pools"] +for path, value in ((managed_path, managed), (removal_path, {})): + with open(path, "w", encoding="utf-8") as handle: + json.dump(value, handle, indent=2, sort_keys=True) + handle.write("\n") +shutil.copyfile(managed_path, snapshot_path) +PY + rm -rf "$work_dir" + die 'failed to reconstruct interrupted network-policy removal state' + } + managed_mode=$daemon_mode + managed_gid=$daemon_gid + managed_metadata= + prior_verified_generation=$(file_generation "$managed_daemon") || { rm -rf "$work_dir"; die 'failed to identify managed daemon.json generation'; } + else + python3 - "$daemon_config" <<'PY' || { rm -rf "$work_dir"; die 'managed daemon.json is not a valid JSON object'; } import json, sys def reject_constant(_value): raise ValueError @@ -759,12 +790,11 @@ try: except (OSError, json.JSONDecodeError, ValueError): raise SystemExit(1) PY - managed_mode=$(stat -c %a "$daemon_config") - managed_gid=$(stat -c %g "$daemon_config") - managed_metadata=$(stat -c '%d:%i:%u:%a:%g' "$daemon_config") - managed_snapshot=$work_dir/daemon.json.before - cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } - python3 - "$managed_snapshot" "$state_file" "$managed_daemon" "$removal_daemon" "$repo_root/scripts" 2>/dev/null <<'PY' || { + managed_mode=$(stat -c %a "$daemon_config") + managed_gid=$(stat -c %g "$daemon_config") + managed_metadata=$(stat -c '%d:%i:%u:%a:%g' "$daemon_config") + cp -- "$daemon_config" "$managed_snapshot" || { rm -rf "$work_dir"; die 'failed to snapshot managed daemon.json'; } + python3 - "$managed_snapshot" "$state_file" "$managed_daemon" "$removal_daemon" "$repo_root/scripts" 2>/dev/null <<'PY' || { import json, sys daemon_path, state_path, managed_path, removal_path, scripts_path = sys.argv[1:] sys.path.insert(0, scripts_path) @@ -798,16 +828,19 @@ with open(removal_path, "w", encoding="utf-8") as handle: json.dump(current, handle, indent=2, sort_keys=True) handle.write("\n") PY - rm -rf "$work_dir" - die 'managed daemon.json is not a valid JSON object' - } + rm -rf "$work_dir" + die 'managed daemon.json is not a valid JSON object' + } + fi set_removal_pending "$managed_daemon" || { rm -rf "$work_dir"; die 'failed to persist network-policy removal state'; } removal_checkpoint_started=true - drain_controller 'drain command failed before network-policy removal' - if daemon_changed_since_snapshot "$managed_snapshot" true "$managed_metadata"; then - drain_failure='daemon.json changed during network-policy removal' - exit 2 + if [[ "$absent_removal_recovery" != true ]]; then + drain_controller 'drain command failed before network-policy removal' + if daemon_changed_since_snapshot "$managed_snapshot" true "$managed_metadata"; then + drain_failure='daemon.json changed during network-policy removal' + exit 2 + fi fi checkpoint_path_is_pinned || die 'checkpoint directory changed during network-policy removal' @@ -1328,9 +1361,16 @@ verified_generation=$(file_generation "$daemon_config") || fail_after_apply "fai verified_action= [[ "$apply_removal_pending" != true ]] || verified_action=clear-removal set_verified_generation "$verified_generation" "$verified_action" || fail_after_apply "failed to record verified daemon.json generation" -clear_recovery_artifacts || fail_after_apply "failed to clear obsolete network-policy recovery data" + +# The verified generation commits the apply. Disarm rollback before deleting +# the snapshot it consumes; cleanup failure leaves the committed marker usable. +trap '' INT TERM +trap - EXIT +if ! clear_recovery_artifacts; then + rm -rf "$work_dir" + die "failed to clear obsolete network-policy recovery data" +fi # --- Success --- -trap - EXIT INT TERM rm -rf "$work_dir" printf 'NETWORK_POLICY_APPLIED daemon_config=%s\n' "$daemon_config" diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index 99e8c15a..d04f859c 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -15,10 +15,15 @@ if [[ ${1:-} == --env ]]; then shift 2 fi selected_environment=$environment +health_testing_present=${CI_FLEET_TESTING+x} health_testing=${CI_FLEET_TESTING-} +health_root_prefix_present=${CI_FLEET_ROOT_PREFIX+x} health_root_prefix=${CI_FLEET_ROOT_PREFIX-} +health_docker_socket_present=${CI_FLEET_DOCKER_SOCKET+x} health_docker_socket=${CI_FLEET_DOCKER_SOCKET-} +health_bootstrap_present=${CI_FLEET_HEALTH_BOOTSTRAP+x} health_bootstrap=${CI_FLEET_HEALTH_BOOTSTRAP-} +health_suppress_delivery_present=${CI_FLEET_HEALTH_SUPPRESS_DELIVERY+x} health_suppress_delivery=${CI_FLEET_HEALTH_SUPPRESS_DELIVERY-} while IFS= read -r variable; do unset "$variable"; done < <(compgen -A variable CI_FLEET_) if [[ -r $selected_environment ]]; then @@ -27,10 +32,10 @@ if [[ -r $selected_environment ]]; then . "$selected_environment" set +a fi -[[ -z $health_testing ]] || export CI_FLEET_TESTING=$health_testing -[[ -z $health_root_prefix ]] || export CI_FLEET_ROOT_PREFIX=$health_root_prefix -[[ -z $health_docker_socket ]] || export CI_FLEET_DOCKER_SOCKET=$health_docker_socket -[[ -z $health_bootstrap ]] || export CI_FLEET_HEALTH_BOOTSTRAP=$health_bootstrap -[[ -z $health_suppress_delivery ]] || export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery +if [[ $health_testing_present == x ]]; then export CI_FLEET_TESTING=$health_testing; else unset CI_FLEET_TESTING; fi +if [[ $health_root_prefix_present == x ]]; then export CI_FLEET_ROOT_PREFIX=$health_root_prefix; else unset CI_FLEET_ROOT_PREFIX; fi +if [[ $health_docker_socket_present == x ]]; then export CI_FLEET_DOCKER_SOCKET=$health_docker_socket; else unset CI_FLEET_DOCKER_SOCKET; fi +if [[ $health_bootstrap_present == x ]]; then export CI_FLEET_HEALTH_BOOTSTRAP=$health_bootstrap; else unset CI_FLEET_HEALTH_BOOTSTRAP; fi +if [[ $health_suppress_delivery_present == x ]]; then export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery; else unset CI_FLEET_HEALTH_SUPPRESS_DELIVERY; fi use_local_docker exec python3 "$repo_root/scripts/health.py" "${args[@]}" "$@" diff --git a/scripts/test-healthcheck.sh b/scripts/test-healthcheck.sh index 4325b7cd..26136ad2 100755 --- a/scripts/test-healthcheck.sh +++ b/scripts/test-healthcheck.sh @@ -9,15 +9,27 @@ cat >"$tmp/bin/python3" <<'EOF' printf 'controller=%s\n' "${CI_FLEET_CONTROLLER-}" printf 'stale=%s\n' "${CI_FLEET_STALE-unset}" printf 'testing=%s\n' "${CI_FLEET_TESTING-}" +printf 'testing_present=%s\n' "${CI_FLEET_TESTING+set}" printf 'root_prefix=%s\n' "${CI_FLEET_ROOT_PREFIX-}" +printf 'root_prefix_present=%s\n' "${CI_FLEET_ROOT_PREFIX+set}" printf 'docker_socket=%s\n' "${CI_FLEET_DOCKER_SOCKET-}" +printf 'docker_socket_present=%s\n' "${CI_FLEET_DOCKER_SOCKET+set}" printf 'bootstrap=%s\n' "${CI_FLEET_HEALTH_BOOTSTRAP-}" +printf 'bootstrap_present=%s\n' "${CI_FLEET_HEALTH_BOOTSTRAP+set}" printf 'suppress_delivery=%s\n' "${CI_FLEET_HEALTH_SUPPRESS_DELIVERY-}" +printf 'suppress_delivery_present=%s\n' "${CI_FLEET_HEALTH_SUPPRESS_DELIVERY+set}" printf 'docker_host=%s\n' "${DOCKER_HOST-}" printf 'args=%s\n' "$*" EOF chmod +x "$tmp/bin/python3" -printf 'CI_FLEET_CONTROLLER=candidate\n' >"$tmp/candidate.env" +cat >"$tmp/candidate.env" < None: self.assertFalse(list(checkpoint.glob("recovery.*"))) self.assertNotEqual(daemon.read_bytes(), prior_daemon) + def test_signal_after_recovery_cleanup_cannot_run_rollback(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-commit-cleanup-signal" + self._write_success_commands() + policy = self._rendered_with_policy() + policy_env = self._write_env_file(policy) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(policy_env.read_bytes()) + candidate = dict(policy) + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = self._write_env_file(candidate) + command_log = Path(self.tmp) / "commit-cleanup-signal.log" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {command_log}\n" + f"[[ $(wc -l < {command_log}) -lt 7 ]]\n", + encoding="utf-8", + ) + restart.chmod(0o755) + signal_marker = Path(self.tmp) / "cleanup-signal.sent" + audit_dir = Path(self.tmp) / "cleanup-signal-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os, shutil, signal\n" + "_rmtree = shutil.rmtree\n" + "def rmtree(path, *args, **kwargs):\n" + " result = _rmtree(path, *args, **kwargs)\n" + " if os.path.basename(path).startswith('recovery.') and not os.path.exists(os.environ['SIGNAL_MARKER']):\n" + " open(os.environ['SIGNAL_MARKER'], 'w').close()\n" + " os.kill(os.getppid(), signal.SIGTERM)\n" + " return result\n" + "shutil.rmtree = rmtree\n", + encoding="utf-8", + ) + + reapplied = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(candidate_env)], + capture_output=True, + text=True, + env=self._env(PYTHONPATH=str(audit_dir), SIGNAL_MARKER=str(signal_marker)), + timeout=30, + ) + + self.assertEqual(reapplied.returncode, 0, reapplied.stderr) + self.assertTrue(signal_marker.exists()) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "resume", "health"], + ) + self.assertFalse(list(checkpoint.glob("recovery.*"))) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"][0]["base"], "192.0.2.0/24") + + def test_recovery_cleanup_failure_does_not_report_deleted_evidence(self) -> None: + daemon = self._write_daemon('{"live-restore":true}\n') + checkpoint = Path(self.tmp) / "checkpoint-commit-cleanup-failure" + self._write_success_commands() + policy = self._rendered_with_policy() + policy_env = self._write_env_file(policy) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(policy_env.read_bytes()) + candidate = dict(policy) + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + candidate_env = self._write_env_file(candidate) + restart_count = Path(self.tmp) / "commit-cleanup-failure-restarts" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {restart_count}\n" + f"[[ $(wc -l < {restart_count}) -eq 1 ]]\n", + encoding="utf-8", + ) + restart.chmod(0o755) + audit_dir = Path(self.tmp) / "cleanup-failure-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os, shutil\n" + "_rmtree = shutil.rmtree\n" + "def rmtree(path, *args, **kwargs):\n" + " result = _rmtree(path, *args, **kwargs)\n" + " if os.path.basename(path).startswith('recovery.'):\n" + " raise OSError('injected recovery cleanup failure')\n" + " return result\n" + "shutil.rmtree = rmtree\n", + encoding="utf-8", + ) + + reapplied = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(candidate_env)], + capture_output=True, + text=True, + env=self._env(PYTHONPATH=str(audit_dir), CI_FLEET_TEMP_DIR=self.tmp), + timeout=30, + ) + + self.assertNotEqual(reapplied.returncode, 0) + self.assertIn("failed to clear obsolete network-policy recovery data", reapplied.stderr) + self.assertNotIn("recovery data retained at", reapplied.stderr) + self.assertFalse(list(checkpoint.glob("recovery.*"))) + self.assertFalse(list(Path(self.tmp).glob(".ci-fleet-apply.*"))) + state = json.loads((checkpoint / "docker-network-policy.json").read_text(encoding="utf-8")) + self.assertIsNotNone(state["verified_generation"]) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8"))["default-address-pools"][0]["base"], "192.0.2.0/24") + def test_post_apply_health_uses_candidate_rendered_env(self) -> None: self._write_daemon("{}\n") for name in ("restart.sh", "probe.sh"): @@ -3463,6 +3572,7 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> self.assertEqual( [marker.name for marker in command_markers if marker.exists()], [ + "interrupted-before-rename-drain.sh.marker", "interrupted-before-rename-restart.sh.marker", "interrupted-before-rename-probe.sh.marker", "interrupted-before-rename-resume.sh.marker", @@ -3470,6 +3580,45 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> ], ) + def test_interrupted_first_apply_drain_failure_stops_before_restart_or_mutation(self) -> None: + daemon = self.daemon_dir / "daemon.json" + checkpoint = Path(self.tmp) / "checkpoint-interrupted-drain-failure" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, + "prior_mode": None, + "prior_present": False, + "verified_generation": None, + }), + encoding="utf-8", + ) + state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) + command_log = Path(self.tmp) / "interrupted-drain-failure.log" + for name in ("drain", "restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text( + f"#!/usr/bin/env bash\necho {name} >> {command_log}\n" + ("exit 1\n" if name == "drain" else ""), + encoding="utf-8", + ) + command.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertFalse(daemon.exists()) + self.assertTrue(state_file.exists()) + self.assertTrue(recovery.exists()) + self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume", "health"]) + def test_removal_rejects_non_object_daemon_before_copy_or_commands(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-non-object-removal" @@ -3892,6 +4041,98 @@ def test_pending_removal_with_absent_daemon_completes_runtime_verification(self) ["restart", "probe", "resume", "health"], ) + def test_pending_removal_with_absent_daemon_failure_restores_managed_state(self) -> None: + daemon = self.daemon_dir / "daemon.json" + checkpoint = Path(self.tmp) / "checkpoint-pending-removal-absent-failure" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = json.loads(managed)["default-address-pools"] + state["verified_generation"] = None + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + daemon.unlink() + command_log = Path(self.tmp) / "pending-removal-absent-failure.log" + (Path(self.tmp) / "restart.sh").write_text( + f"#!/usr/bin/env bash\necho restart >> {command_log}\n", + encoding="utf-8", + ) + (Path(self.tmp) / "probe.sh").write_text( + f"#!/usr/bin/env bash\necho probe >> {command_log}\nexit 1\n", + encoding="utf-8", + ) + for name in ("resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text( + "#!/usr/bin/env bash\n" + f"echo {name} >> {command_log}\n" + f"grep -Fqx 'CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT=2' \"$2\"\n", + encoding="utf-8", + ) + for name in ("restart", "probe", "resume", "health"): + (Path(self.tmp) / f"{name}.sh").chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertEqual(daemon.read_bytes(), managed) + restored_state = json.loads(state_file.read_text(encoding="utf-8")) + self.assertTrue(restored_state["managed"]) + self.assertNotIn("phase", restored_state) + self.assertIsNotNone(restored_state["verified_generation"]) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["restart", "probe", "restart", "resume", "health"], + ) + + def test_pending_removal_with_absent_daemon_failed_rollback_retains_recovery(self) -> None: + daemon = self.daemon_dir / "daemon.json" + checkpoint = Path(self.tmp) / "checkpoint-pending-removal-absent-recovery" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + managed = daemon.read_bytes() + self.installed_env.write_bytes(policy_env.read_bytes()) + prior_env = self.installed_env.read_bytes() + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state["phase"] = "removal-pending" + state["removal_managed_default_address_pools"] = json.loads(managed)["default-address-pools"] + state["verified_generation"] = None + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + daemon.unlink() + restart_log = Path(self.tmp) / "pending-removal-absent-recovery.log" + restart = Path(self.tmp) / "restart.sh" + restart.write_text( + f"#!/usr/bin/env bash\necho restart >> {restart_log}\n" + f"[[ $(wc -l < {restart_log}) -eq 1 ]]\n", + encoding="utf-8", + ) + restart.chmod(0o755) + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertIn("rollback verification failed; recovery data retained at", removed.stderr) + recovery = next(checkpoint.glob("recovery.*")) + self.assertEqual((recovery / "daemon.json.before").read_bytes(), managed) + self.assertEqual((recovery / "prior-ci-fleet.env").read_bytes(), prior_env) + retained_state = json.loads(state_file.read_text(encoding="utf-8")) + self.assertEqual(retained_state["phase"], "removal-pending") + self.assertIsNone(retained_state["verified_generation"]) + def test_apply_from_pending_removal_persists_synthesized_rollback_daemon(self) -> None: daemon = self.daemon_dir / "daemon.json" checkpoint = Path(self.tmp) / "checkpoint-pending-removal-apply-recovery" From 240f70b1ef9a05e3f48a585a6ae70ed324cdade8 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 21:20:28 -0500 Subject: [PATCH 50/54] fix: redrain before interrupted recovery fallback --- scripts/apply-docker-network-policy.sh | 6 ++- scripts/test_apply_docker_network_policy.py | 48 +++++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index f7738660..303f0cb4 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -306,7 +306,10 @@ resume_after_failed_drain() { local status=$? resume_failed=0 health_failed=0 trap '' INT TERM trap - EXIT - run_command "$resume_command" --env "$prior_env" || resume_failed=1 + if [[ "$controller_resumed" == true ]]; then + run_command "$drain_command" || resume_failed=1 + fi + ((resume_failed != 0)) || run_command "$resume_command" --env "$prior_env" || resume_failed=1 ((resume_failed != 0)) || run_health "$prior_env" || health_failed=1 if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true ]]; then clear_managed_marker || resume_failed=1 @@ -740,6 +743,7 @@ PY drain_controller 'drain command failed before interrupted network-policy recovery' run_command "$restart_command" "$daemon_dir" || die 'Docker restart command failed while recovering interrupted network-policy apply' run_command "$probe_command" || die 'capacity probe failed while recovering interrupted network-policy apply' + controller_resumed=true run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' run_health "$env_file" || die 'health check failed while recovering interrupted network-policy apply' clear_recovery_artifacts || die 'failed to clear obsolete network-policy recovery data' diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 089cea15..2650f6f7 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -3580,6 +3580,54 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> ], ) + def test_interrupted_first_apply_failed_candidate_resume_redrains_before_prior_resume(self) -> None: + checkpoint = Path(self.tmp) / "checkpoint-interrupted-resume-failure" + checkpoint.mkdir(mode=0o700) + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text( + json.dumps({ + "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, + "prior_mode": None, + "prior_present": False, + "verified_generation": None, + }), + encoding="utf-8", + ) + state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) + command_log = Path(self.tmp) / "interrupted-resume-failure.log" + candidate_side_effect = Path(self.tmp) / "candidate-resume-side-effect" + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + resume = Path(self.tmp) / "resume.sh" + resume.write_text( + "#!/usr/bin/env bash\n" + f"if grep -Fqx 'ENV_GENERATION=prior' \"$2\"; then echo resume-prior >> {command_log}; exit 0; fi\n" + f"echo resume-candidate >> {command_log}\n" + f"touch {candidate_side_effect}\n" + "exit 1\n", + encoding="utf-8", + ) + resume.chmod(0o755) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + removed = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(removed.returncode, 0) + self.assertTrue(candidate_side_effect.exists()) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "resume-candidate", "drain", "resume-prior", "health"], + ) + def test_interrupted_first_apply_drain_failure_stops_before_restart_or_mutation(self) -> None: daemon = self.daemon_dir / "daemon.json" checkpoint = Path(self.tmp) / "checkpoint-interrupted-drain-failure" From 5db6b7268214bea1d8c77e81e45e0600e3e71c33 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Sun, 30 Aug 2026 22:05:37 -0500 Subject: [PATCH 51/54] fix: parse rendered health environment safely --- scripts/healthcheck.sh | 28 +++++++++++-- scripts/test_apply_docker_network_policy.py | 45 +++++++++++++++++++-- 2 files changed, 66 insertions(+), 7 deletions(-) diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index d04f859c..14201b40 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -27,10 +27,30 @@ health_suppress_delivery_present=${CI_FLEET_HEALTH_SUPPRESS_DELIVERY+x} health_suppress_delivery=${CI_FLEET_HEALTH_SUPPRESS_DELIVERY-} while IFS= read -r variable; do unset "$variable"; done < <(compgen -A variable CI_FLEET_) if [[ -r $selected_environment ]]; then - set -a - # shellcheck disable=SC1090 - . "$selected_environment" - set +a + parsed_environment=$(mktemp) + trap 'rm -f -- "$parsed_environment"' EXIT + if ! /usr/bin/python3 - "$selected_environment" "$repo_root/scripts" >"$parsed_environment" <<'PY' +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[2]) +from desired_state import parse_env + +values = parse_env(Path(sys.argv[1]), allow_unknown=True) +invalid = next((name for name in values if not name.startswith("CI_FLEET_")), None) +if invalid is not None: + raise SystemExit(f"ERROR: rendered environment variable must start with CI_FLEET_: {invalid}") +for name, value in values.items(): + sys.stdout.buffer.write(name.encode() + b"\0" + value.encode() + b"\0") +PY + then + exit 2 + fi + while IFS= read -r -d '' variable && IFS= read -r -d '' value; do + export "$variable=$value" + done <"$parsed_environment" + rm -f -- "$parsed_environment" + trap - EXIT fi if [[ $health_testing_present == x ]]; then export CI_FLEET_TESTING=$health_testing; else unset CI_FLEET_TESTING; fi if [[ $health_root_prefix_present == x ]]; then export CI_FLEET_ROOT_PREFIX=$health_root_prefix; else unset CI_FLEET_ROOT_PREFIX; fi diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 2650f6f7..458cb160 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -203,15 +203,15 @@ def test_env_argument_sources_candidate_rendered_env(self) -> None: root = Path(tmp) installed = root / "etc" / "ci-fleet" / "ci-fleet.env" installed.parent.mkdir(parents=True) - installed.write_text("ENV_MARKER=stale\n", encoding="utf-8") + installed.write_text("CI_FLEET_ENV_MARKER=stale\n", encoding="utf-8") candidate = root / "candidate.env" - candidate.write_text("ENV_MARKER=candidate\n", encoding="utf-8") + candidate.write_text("CI_FLEET_ENV_MARKER=candidate\n", encoding="utf-8") fake_bin = root / "bin" fake_bin.mkdir() python = fake_bin / "python3" python.write_text( "#!/usr/bin/env bash\n" - "[[ ${ENV_MARKER:-} == candidate ]] || exit 1\n", + "[[ ${CI_FLEET_ENV_MARKER:-} == candidate ]] || exit 1\n", encoding="utf-8", ) python.chmod(0o755) @@ -232,6 +232,45 @@ def test_env_argument_sources_candidate_rendered_env(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) + def test_env_argument_rejects_path_before_command_resolution(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + candidate_bin = root / "candidate-bin" + candidate_bin.mkdir() + candidate_shim_ran = root / "candidate-shim-ran" + (candidate_bin / "python3").write_text( + f"#!/bin/bash\n: > {candidate_shim_ran}\n", + encoding="utf-8", + ) + (candidate_bin / "python3").chmod(0o755) + candidate = root / "candidate.env" + candidate.write_text(f"PATH={candidate_bin}\n", encoding="utf-8") + + trusted_bin = root / "trusted-bin" + trusted_bin.mkdir() + health_substitute_ran = root / "health-substitute-ran" + (trusted_bin / "python3").write_text( + "#!/bin/bash\n" + "if [[ ${1:-} == - ]]; then exec /usr/bin/python3 \"$@\"; fi\n" + f": > {health_substitute_ran}\n", + encoding="utf-8", + ) + (trusted_bin / "python3").chmod(0o755) + env = dict(os.environ) + env["PATH"] = f"{trusted_bin}:{env['PATH']}" + + result = subprocess.run( + [str(SCRIPTS / "healthcheck.sh"), "--env", str(candidate)], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(candidate_shim_ran.exists()) + self.assertFalse(health_substitute_ran.exists()) + def test_selected_env_does_not_inherit_removed_pool_variables(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) From 05ca9015d376df9e62cc2a17398b37c51407c5b7 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 10:51:45 -0500 Subject: [PATCH 52/54] fix: make first-apply recovery crash consistent --- scripts/apply-docker-network-policy.sh | 282 +++++++++++++------- scripts/test_apply_docker_network_policy.py | 252 ++++++++++++++++- 2 files changed, 440 insertions(+), 94 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 303f0cb4..6ee720da 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -254,7 +254,7 @@ validate_trusted_path 'checkpoint directory' "$checkpoint_dir" checkpoint true [[ "$daemon_config" != "$checkpoint_dir" && "$daemon_config" != "$checkpoint_dir/"* ]] || die 'daemon config must be outside checkpoint directory' checkpoint_state=$checkpoint_dir/docker-network-policy.json [[ "$removing" != true || ! -L "$checkpoint_state" ]] || die 'network-policy checkpoint state is invalid' -if [[ "$removing" == true && ( ! -e "$checkpoint_dir" || ! -e "$checkpoint_state" ) ]]; then +if [[ "$removing" == true && ! -e "$checkpoint_dir" ]]; then printf 'NETWORK_POLICY_NOOP\n' exit 0 fi @@ -301,6 +301,8 @@ controller_resumed=false transaction_recovery= removal_checkpoint_started=false apply_checkpoint_started=false +apply_phase= +cancelling_first_apply=false # shellcheck disable=SC2317 # invoked indirectly by the EXIT trap below resume_after_failed_drain() { local status=$? resume_failed=0 health_failed=0 @@ -311,17 +313,18 @@ resume_after_failed_drain() { fi ((resume_failed != 0)) || run_command "$resume_command" --env "$prior_env" || resume_failed=1 ((resume_failed != 0)) || run_health "$prior_env" || health_failed=1 - if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true ]]; then - clear_managed_marker || resume_failed=1 + if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true || "$apply_phase" == first-apply-pending ]]; then + complete_first_apply_rollback || resume_failed=1 fi if ((resume_failed == 0 && health_failed == 0)) && [[ "$removal_checkpoint_started" == true ]]; then - set_verified_generation "$prior_verified_generation" clear-removal || resume_failed=1 + restored_generation=$(file_generation "$daemon_config") || resume_failed=1 + ((resume_failed != 0)) || set_verified_generation "$restored_generation" clear-removal || resume_failed=1 fi if ((resume_failed == 0 && health_failed == 0)) && [[ "$apply_checkpoint_started" == true ]]; then set_verified_generation "$prior_verified_generation" || resume_failed=1 fi - if ((resume_failed == 0 && health_failed == 0)); then - [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" + if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" != true && "$apply_phase" != first-apply-pending && "$cancelling_first_apply" != true ]]; then + clear_recovery_artifacts || resume_failed=1 fi rm -rf "$work_dir" if ((resume_failed)); then @@ -508,6 +511,12 @@ state = json.load(open(path, encoding="utf-8")) if action == "clear-removal": state.pop("phase", None) state.pop("removal_managed_default_address_pools", None) +elif action in ("reapply-pending", "rollback-complete"): + state["phase"] = action + state.pop("removal_managed_default_address_pools", None) +if generation: + state.pop("phase", None) + state.pop("removal_managed_default_address_pools", None) state["verified_generation"] = generation or None fd, tmp = tempfile.mkstemp(prefix=".docker-network-policy.", dir=os.path.dirname(path), text=True) try: @@ -654,20 +663,23 @@ clear_recovery_artifacts() { import os, shutil, stat, sys parent, owner = sys.argv[1], int(sys.argv[2]) +recoveries = [] for entry in os.scandir(parent): - if not entry.name.startswith("recovery."): + if not entry.name.startswith(("recovery.", ".recovery.")): continue + recoveries.append(entry.path) metadata = entry.stat(follow_symlinks=False) if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o700: raise SystemExit(1) files = {child.name: child for child in os.scandir(entry.path)} - if set(files) not in ({"prior-ci-fleet.env"}, {"daemon.json.before", "prior-ci-fleet.env"}): + if not set(files) <= {"daemon.json.before", "prior-ci-fleet.env"}: raise SystemExit(1) for child in files.values(): metadata = child.stat(follow_symlinks=False) if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o600: raise SystemExit(1) - shutil.rmtree(entry.path) +for recovery in recoveries: + shutil.rmtree(recovery) parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY) try: os.fsync(parent_fd) @@ -676,6 +688,80 @@ finally: PY } +complete_first_apply_rollback() { + set_verified_generation "" rollback-complete || return 1 + clear_recovery_artifacts || return 1 + clear_managed_marker || return 1 + transaction_recovery= +} + +authoritative_recovery() { + python3 - "$checkpoint_dir" "$checkpoint_owner" <<'PY' +import os, stat, sys +parent, owner = sys.argv[1], int(sys.argv[2]) +recoveries = [entry for entry in os.scandir(parent) if entry.name.startswith("recovery.")] +if len(recoveries) != 1: + raise SystemExit(1) +recovery = recoveries[0] +metadata = recovery.stat(follow_symlinks=False) +if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o700: + raise SystemExit(1) +entries = {entry.name: entry for entry in os.scandir(recovery.path)} +if set(entries) not in ({"prior-ci-fleet.env"}, {"daemon.json.before", "prior-ci-fleet.env"}): + raise SystemExit(1) +for entry in entries.values(): + metadata = entry.stat(follow_symlinks=False) + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o600: + raise SystemExit(1) +print(f"{recovery.path}|{'true' if 'daemon.json.before' in entries else 'false'}") +PY +} + +# Retire states whose recovery can no longer be consumed before any fast path. +checkpoint_phase=absent +if [[ -e "$state_file" ]]; then + checkpoint_phase=$( + python3 - "$state_file" "$repo_root/scripts" <<'PY' +import json, re, sys +state = json.load(open(sys.argv[1], encoding="utf-8")) +phase = state.get("phase") +generation = state.get("verified_generation") +if phase is not None and generation is not None: + raise SystemExit(1) +required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} +if phase == "rollback-complete": + if set(state) != required | {"phase", "verified_generation"} or state["managed"] is not True: + raise SystemExit(1) + if not isinstance(state["prior_present"], bool) or not isinstance(state["prior_default_address_pools_present"], bool): + raise SystemExit(1) + if state["prior_default_address_pools_present"]: + sys.path.insert(0, sys.argv[2]) + from desired_state import validate_docker_address_pools + validate_docker_address_pools(state["prior_default_address_pools"], path="checkpoint prior default address pools") + elif state["prior_default_address_pools"] is not None: + raise SystemExit(1) + if state["prior_present"] != (state["prior_mode"] is not None): + raise SystemExit(1) +if phase in ("first-apply-pending", "reapply-pending", "removal-pending", "rollback-complete") and generation is None: + print(phase) +elif phase is None and isinstance(generation, str) and re.fullmatch(r"[0-9a-f]{64}", generation): + print("verified") +else: + print("active") +PY + ) || die 'network-policy checkpoint state is invalid' +fi +if [[ "$checkpoint_phase" == absent || "$checkpoint_phase" == rollback-complete ]]; then + clear_recovery_artifacts || die 'failed to clear obsolete network-policy recovery data' +fi +if [[ "$checkpoint_phase" == rollback-complete ]]; then + clear_managed_marker || die 'failed to clear completed network-policy marker' +fi +if [[ "$removing" == true && ! -e "$state_file" ]]; then + printf 'NETWORK_POLICY_NOOP\n' + exit 0 +fi + if [[ "$removing" == true ]]; then [[ -f "$state_file" && ! -L "$state_file" ]] || die 'network-policy checkpoint state is invalid' mapfile -t managed_state < <(python3 - "$state_file" "$repo_root/scripts" 2>/dev/null <<'PY' @@ -685,17 +771,21 @@ sys.path.insert(0, sys.argv[2]) from desired_state import validate_docker_address_pools required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} pending = {"phase", "removal_managed_default_address_pools"} -schemas = (required, required | {"verified_generation"}, required | pending, required | pending | {"verified_generation"}) -if set(state) not in schemas or state["managed"] is not True: +phase = state.get("phase") +if state["managed"] is not True: raise SystemExit(1) -is_pending = "phase" in state -if is_pending and ( - state["phase"] != "removal-pending" - or state["removal_managed_default_address_pools"] is not None +if phase == "removal-pending": + if set(state) != required | pending | {"verified_generation"} or ( + state["removal_managed_default_address_pools"] is not None and not isinstance(state["removal_managed_default_address_pools"], list) -): + ): + raise SystemExit(1) +elif phase == "first-apply-pending": + if set(state) != required | {"phase", "verified_generation"}: + raise SystemExit(1) +elif phase is not None or set(state) not in (required, required | {"verified_generation"}): raise SystemExit(1) -if is_pending and state["removal_managed_default_address_pools"] is not None: +if phase == "removal-pending" and state["removal_managed_default_address_pools"] is not None: validate_docker_address_pools( state["removal_managed_default_address_pools"], path="checkpoint removal managed default address pools", @@ -703,6 +793,8 @@ if is_pending and state["removal_managed_default_address_pools"] is not None: generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) +if "verified_generation" in state and generation is None and phase is None: + raise SystemExit(1) if not isinstance(state["prior_present"], bool): raise SystemExit(1) if not isinstance(state["prior_default_address_pools_present"], bool): @@ -725,7 +817,7 @@ elif mode is not None: print("true" if state["prior_present"] else "false") print(mode or "") print(generation or "") -print("true" if is_pending else "false") +print(phase or "") print("true" if "verified_generation" in state else "false") PY ) || die 'network-policy checkpoint state is invalid' @@ -733,11 +825,22 @@ PY prior_present=${managed_state[0]} prior_mode=${managed_state[1]} prior_verified_generation=${managed_state[2]} - removal_pending=${managed_state[3]} + removal_phase=${managed_state[3]} + removal_pending=false + [[ "$removal_phase" != removal-pending ]] || removal_pending=true has_verified_generation=${managed_state[4]} if [[ "$prior_present" == true ]]; then [[ "$prior_mode" =~ ^[0-7]{3,4}$ ]] || die 'network-policy checkpoint state is invalid' fi + [[ "$checkpoint_phase" != verified ]] || clear_recovery_artifacts || die 'failed to clear obsolete network-policy recovery data' + if [[ "$removal_phase" == first-apply-pending || "$removal_pending" == true ]]; then + recovery_info=$(authoritative_recovery) || die 'network-policy transaction recovery is invalid' + IFS='|' read -r transaction_recovery _ <<<"$recovery_info" + prior_env=$transaction_recovery/prior-ci-fleet.env + fi + if [[ "$removal_phase" == first-apply-pending ]]; then + cancelling_first_apply=true + fi absent_removal_recovery=false if [[ "$prior_present" == false && -z "$prior_verified_generation" && "$has_verified_generation" == true && ! -e "$daemon_config" && ! -L "$daemon_config" && "$removal_pending" != true ]]; then drain_controller 'drain command failed before interrupted network-policy recovery' @@ -837,8 +940,11 @@ PY } fi - set_removal_pending "$managed_daemon" || { rm -rf "$work_dir"; die 'failed to persist network-policy removal state'; } - removal_checkpoint_started=true + if [[ "$cancelling_first_apply" != true ]]; then + [[ -n "$transaction_recovery" ]] || transaction_recovery=$(persist_recovery "$managed_snapshot" "$prior_env") || { rm -rf "$work_dir"; die 'failed to persist network-policy transaction recovery'; } + set_removal_pending "$managed_daemon" || { rm -rf "$work_dir"; die 'failed to persist network-policy removal state'; } + removal_checkpoint_started=true + fi if [[ "$absent_removal_recovery" != true ]]; then drain_controller 'drain command failed before network-policy removal' if daemon_changed_since_snapshot "$managed_snapshot" true "$managed_metadata"; then @@ -897,7 +1003,7 @@ PY if ((failed == 0)); then daemon_pools_match "$managed_daemon" || failed=1 fi - if ((failed == 0)); then + if ((failed == 0)) && [[ "$cancelling_first_apply" != true ]]; then set_verified_generation "$prior_verified_generation" clear-removal || failed=1 fi return "$failed" @@ -949,8 +1055,10 @@ PY run_command "$restart_command" "$daemon_dir" || { removal_failure='Docker restart command failed during network-policy removal'; exit 2; } run_command "$probe_command" || { removal_failure='capacity probe failed after network-policy removal'; exit 2; } controller_resumed=true - run_command "$resume_command" --env "$env_file" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } - run_health "$env_file" || { removal_failure='health check failed after network-policy removal'; exit 2; } + activation_env=$env_file + [[ "$cancelling_first_apply" != true ]] || activation_env=$prior_env + run_command "$resume_command" --env "$activation_env" || { removal_failure='controller resume command failed during network-policy removal'; exit 2; } + run_health "$activation_env" || { removal_failure='health check failed after network-policy removal'; exit 2; } # Marker deletion commits removal. Ignore catchable signals across the atomic # unlink so failure still rolls back and success cannot leave a stale marker. @@ -968,13 +1076,12 @@ if ( ): raise SystemExit(1) PY - if ! clear_recovery_artifacts; then - removal_failure='failed to clear obsolete network-policy recovery data' - exit 2 - fi - if ! clear_managed_marker "$removal_daemon"; then - removal_failure='failed to clear network-policy managed marker' - exit 2 + if [[ "$cancelling_first_apply" == true ]]; then + trap - EXIT + complete_first_apply_rollback || { removal_failure='failed to complete interrupted first-apply rollback'; exit 2; } + else + clear_recovery_artifacts || { removal_failure='failed to clear obsolete network-policy recovery data'; exit 2; } + clear_managed_marker "$removal_daemon" || { removal_failure='failed to clear network-policy managed marker'; exit 2; } fi trap - EXIT INT TERM rm -rf "$work_dir" @@ -1033,13 +1140,19 @@ state = json.load(open(sys.argv[1], encoding="utf-8")) sys.path.insert(0, sys.argv[2]) from desired_state import validate_docker_address_pools required = {"managed", "prior_default_address_pools", "prior_default_address_pools_present", "prior_mode", "prior_present"} -pending = {"phase", "removal_managed_default_address_pools"} -if set(state) not in (required, required | {"verified_generation"}, required | pending, required | pending | {"verified_generation"}) or state["managed"] is not True: +phase = state.get("phase") +extra = set(state) - required - {"verified_generation", "phase", "removal_managed_default_address_pools"} +if extra or not required <= set(state) or state["managed"] is not True: raise SystemExit(1) -is_pending = "phase" in state -if is_pending and state["phase"] != "removal-pending": +if phase == "removal-pending": + if "removal_managed_default_address_pools" not in state: + raise SystemExit(1) +elif phase in ("first-apply-pending", "reapply-pending"): + if "removal_managed_default_address_pools" in state: + raise SystemExit(1) +elif phase is not None or "removal_managed_default_address_pools" in state: raise SystemExit(1) -if is_pending and state["removal_managed_default_address_pools"] is not None: +if phase == "removal-pending" and state["removal_managed_default_address_pools"] is not None: validate_docker_address_pools( state["removal_managed_default_address_pools"], path="checkpoint removal managed default address pools", @@ -1047,6 +1160,8 @@ if is_pending and state["removal_managed_default_address_pools"] is not None: generation = state.get("verified_generation") if generation is not None and (not isinstance(generation, str) or not re.fullmatch(r"[0-9a-f]{64}", generation)): raise SystemExit(1) +if "verified_generation" in state and generation is None and phase is None: + raise SystemExit(1) if not isinstance(state["prior_present"], bool): raise SystemExit(1) if not isinstance(state["prior_default_address_pools_present"], bool): @@ -1066,13 +1181,14 @@ if state["prior_present"]: elif state["prior_mode"] is not None: raise SystemExit(1) print( - f"{generation or ''}|{'true' if is_pending else 'false'}|" - f"{'true' if state['prior_present'] else 'false'}|{state['prior_mode'] or ''}" + f"{generation or ''}|{'true' if phase == 'removal-pending' else 'false'}|" + f"{'true' if state['prior_present'] else 'false'}|{state['prior_mode'] or ''}|{phase or ''}" ) PY ) || { rm -rf "$work_dir"; die 'network-policy checkpoint state is invalid'; } - IFS='|' read -r prior_verified_generation apply_removal_pending checkpoint_prior_present checkpoint_prior_mode <<<"$apply_checkpoint_state" + IFS='|' read -r prior_verified_generation apply_removal_pending checkpoint_prior_present checkpoint_prior_mode apply_phase <<<"$apply_checkpoint_state" managed_before=true + [[ "$checkpoint_phase" != verified ]] || clear_recovery_artifacts || { rm -rf "$work_dir"; die 'failed to clear obsolete network-policy recovery data'; } fi daemon_matches=false @@ -1094,34 +1210,14 @@ fi # --- Use the staged source snapshot for transactional rollback --- interrupted_recovery= -if [[ "$managed_before" == true && -z "$prior_verified_generation" ]]; then - recovery_info=$(python3 - "$checkpoint_dir" "$checkpoint_owner" <<'PY' -import os, stat, sys -parent, owner = sys.argv[1], int(sys.argv[2]) -recoveries = [entry for entry in os.scandir(parent) if entry.name.startswith("recovery.")] -if not recoveries: - print("|") - raise SystemExit -if len(recoveries) != 1: - raise SystemExit(1) -recovery = recoveries[0] -metadata = recovery.stat(follow_symlinks=False) -if not stat.S_ISDIR(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o700: - raise SystemExit(1) -entries = {entry.name: entry for entry in os.scandir(recovery.path)} -if set(entries) not in ({"prior-ci-fleet.env"}, {"daemon.json.before", "prior-ci-fleet.env"}): - raise SystemExit(1) -for entry in entries.values(): - metadata = entry.stat(follow_symlinks=False) - if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != owner or stat.S_IMODE(metadata.st_mode) != 0o600: - raise SystemExit(1) -print(f"{recovery.path}|{'true' if 'daemon.json.before' in entries else 'false'}") -PY - ) || { rm -rf "$work_dir"; die 'network-policy transaction recovery is invalid'; } +if [[ "$managed_before" == true && -n "$apply_phase" && -z "$prior_verified_generation" ]]; then + recovery_info=$(authoritative_recovery) || { rm -rf "$work_dir"; die 'network-policy transaction recovery is invalid'; } IFS='|' read -r interrupted_recovery recovery_present <<<"$recovery_info" if [[ -n "$interrupted_recovery" ]]; then - had_prior=$recovery_present prior_env=$interrupted_recovery/prior-ci-fleet.env + fi + if [[ "$recovery_present" == true ]]; then + had_prior=true rollback_source=$interrupted_recovery/daemon.json.before else rollback_source=$work_dir/daemon.json.durable-baseline @@ -1201,8 +1297,25 @@ PY fi } -# Record the original host state before the first drain. The marker lets a -# later no-policy reconciliation resume a controller interrupted by the drain. +if [[ "$managed_before" == true ]]; then + if [[ -n "$interrupted_recovery" ]]; then + transaction_recovery=$interrupted_recovery + else + recovery_daemon= + if [[ "$apply_removal_pending" == true ]]; then + recovery_daemon=$rollback_source + elif [[ "$snapshot_present" == true ]]; then + recovery_daemon=$backup_dir/$backup_name + fi + transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' + fi +else + recovery_daemon= + [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name + transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' +fi + +# Recovery is authoritative before the pending marker becomes visible. if [[ "$managed_before" == false ]]; then python3 - "$state_file" "$had_prior" "$daemon_mode" "$backup_dir/$backup_name" "$repo_root/scripts" 2>/dev/null <<'PY' || { transaction_failure='failed to record network-policy checkpoint state'; exit 2; } import json, os, sys, tempfile @@ -1215,6 +1328,7 @@ if prior_key_present: validate_docker_address_pools(prior["default-address-pools"], path="existing daemon default address pools") state = { "managed": True, + "phase": "first-apply-pending", "prior_default_address_pools": prior.get("default-address-pools") if prior_key_present else None, "prior_default_address_pools_present": prior_key_present, "prior_mode": sys.argv[3] if sys.argv[2] == "true" else None, @@ -1241,23 +1355,6 @@ finally: PY new_marker=true fi -if [[ "$managed_before" == true ]]; then - if [[ -n "$interrupted_recovery" ]]; then - transaction_recovery=$interrupted_recovery - else - recovery_daemon= - if [[ "$apply_removal_pending" == true ]]; then - recovery_daemon=$rollback_source - elif [[ "$snapshot_present" == true ]]; then - recovery_daemon=$backup_dir/$backup_name - fi - transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' - fi -else - recovery_daemon= - [[ "$snapshot_present" != true ]] || recovery_daemon=$backup_dir/$backup_name - transaction_recovery=$(persist_recovery "$recovery_daemon" "$prior_env") || die 'failed to persist network-policy transaction recovery' -fi rollback_daemon() { local failed=0 @@ -1279,8 +1376,9 @@ rollback_daemon() { if ((failed == 0)); then daemon_pools_match "$rollback_source" || failed=1 fi - if [[ "$managed_before" == true && "$failed" == 0 ]]; then - set_verified_generation "$prior_verified_generation" || failed=1 + if [[ "$managed_before" == true && "$apply_phase" != first-apply-pending && "$failed" == 0 ]]; then + restored_generation=$(file_generation "$daemon_config") || failed=1 + ((failed != 0)) || set_verified_generation "$restored_generation" || failed=1 fi return "$failed" } @@ -1291,9 +1389,13 @@ rollback_on_exit() { trap '' INT TERM trap - EXIT ((status != 0)) || status=1 - if rollback_daemon >/dev/null 2>&1 && - { [[ "$managed_before" == true ]] || clear_managed_marker; }; then - [[ -z "$transaction_recovery" ]] || rm -rf "$transaction_recovery" + if rollback_daemon >/dev/null 2>&1 && { + if [[ "$new_marker" == true || "$apply_phase" == first-apply-pending ]]; then + complete_first_apply_rollback + else + clear_recovery_artifacts + fi + }; then rm -rf "$work_dir" printf 'ERROR: %s; prior daemon.json restored\n' "$transaction_failure" >&2 else @@ -1314,8 +1416,8 @@ rollback_on_exit() { } # --- Drain after local validation/checkpointing, before mutation or restart --- -if [[ "$managed_before" == true ]]; then - set_verified_generation "" || die 'failed to mark network-policy verification pending' +if [[ "$managed_before" == true && -z "$apply_phase" ]]; then + set_verified_generation "" reapply-pending || die 'failed to mark network-policy verification pending' apply_checkpoint_started=true fi drain_controller 'drain command failed before network-policy apply' diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 458cb160..1e45fbe7 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -435,6 +435,216 @@ def _write_success_commands(self) -> None: command.write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8") command.chmod(0o755) + def _seed_checkpoint(self, checkpoint: Path, phase: str | None, *, recovery: bool = False) -> Path: + checkpoint.mkdir(mode=0o700) + state = { + "managed": True, + "prior_default_address_pools": None, + "prior_default_address_pools_present": False, + "prior_mode": "600", + "prior_present": True, + "verified_generation": None, + } + if phase is not None: + state["phase"] = phase + state_file = checkpoint / "docker-network-policy.json" + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + if recovery: + recovery_dir = checkpoint / "recovery.interrupted" + recovery_dir.mkdir(mode=0o700) + (recovery_dir / "daemon.json.before").write_text("{}\n", encoding="utf-8") + (recovery_dir / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + for path in recovery_dir.iterdir(): + path.chmod(0o600) + return state_file + + def _write_recovery(self, checkpoint: Path, daemon: bytes | None = None) -> Path: + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + if daemon is not None: + (recovery / "daemon.json.before").write_bytes(daemon) + for path in recovery.iterdir(): + path.chmod(0o600) + return recovery + + def test_null_generation_without_phase_fails_before_commands(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-ambiguous-null" + state_file = self._seed_checkpoint(checkpoint, None, recovery=True) + command_log = Path(self.tmp) / "ambiguous-null-commands" + for command in (self.drain_command, *(Path(self.tmp) / name for name in ("restart.sh", "probe.sh", "resume.sh", "health.sh"))): + command.write_text(f"#!/usr/bin/env bash\necho command >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + env_file = self._write_env_file(self._rendered_with_policy()) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("checkpoint state is invalid", result.stderr) + self.assertFalse(command_log.exists()) + self.assertTrue(state_file.exists()) + self.assertTrue((checkpoint / "recovery.interrupted").exists()) + + def test_obsolete_checkpoint_states_cleanup_before_noop(self) -> None: + env_file = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + for phase, recovery in (("absent", True), ("rollback-complete", True), ("rollback-complete", False)): + with self.subTest(phase=phase, recovery=recovery): + checkpoint = Path(self.tmp) / f"checkpoint-cleanup-{phase}-{recovery}" + state_file = self._seed_checkpoint(checkpoint, "rollback-complete", recovery=recovery) + if phase == "absent": + state_file.unlink() + command_log = Path(self.tmp) / f"cleanup-{phase}-{recovery}-commands" + for command in (self.drain_command, *(Path(self.tmp) / name for name in ("restart.sh", "probe.sh", "resume.sh", "health.sh"))): + command.write_text(f"#!/usr/bin/env bash\necho command >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "NETWORK_POLICY_NOOP\n") + self.assertFalse(state_file.exists()) + self.assertFalse(list(checkpoint.glob("*recovery.*"))) + self.assertFalse(command_log.exists()) + + def test_obsolete_cleanup_validates_all_nodes_before_deleting_any(self) -> None: + checkpoint = Path(self.tmp) / "checkpoint-validate-obsolete" + checkpoint.mkdir(mode=0o700) + for name in ("recovery.safe", "recovery.unsafe"): + recovery = checkpoint / name + recovery.mkdir(mode=0o700) + payload = recovery / "prior-ci-fleet.env" + payload.write_bytes(self.installed_env.read_bytes()) + payload.chmod(0o600) + (checkpoint / "recovery.unsafe" / "unexpected").write_text("unsafe\n", encoding="utf-8") + env_file = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + self._write_success_commands() + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertTrue((checkpoint / "recovery.safe").exists()) + self.assertTrue((checkpoint / "recovery.unsafe").exists()) + + def test_interrupted_first_apply_retry_rollback_retires_checkpoint(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) + checkpoint = Path(self.tmp) / "checkpoint-first-retry-rollback" + state_file = self._seed_checkpoint(checkpoint, "first-apply-pending", recovery=True) + self._write_success_commands() + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + candidate = dict(rendered) + candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" + env_file = self._write_env_file(candidate) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), {}) + self.assertFalse(state_file.exists()) + self.assertFalse(list(checkpoint.glob("recovery.*"))) + + def test_interrupted_reapply_retry_rollback_preserves_verified_marker(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) + checkpoint = Path(self.tmp) / "checkpoint-reapply-retry-rollback" + state_file = self._seed_checkpoint(checkpoint, "reapply-pending", recovery=True) + self._write_success_commands() + (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + (Path(self.tmp) / "probe.sh").chmod(0o755) + env_file = self._write_env_file(rendered) + + result = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + state = json.loads(state_file.read_text(encoding="utf-8")) + self.assertNotIn("phase", state) + self.assertEqual(state["verified_generation"], hashlib.sha256(daemon.read_bytes()).hexdigest()) + self.assertFalse(list(checkpoint.glob("recovery.*"))) + + def test_retried_removal_failed_drain_restores_verified_marker(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-retry-drain" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + self.assertEqual(self._run(str(policy_env), checkpoint_dir=str(checkpoint)).returncode, 0) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state.update(phase="removal-pending", removal_managed_default_address_pools=json.loads(daemon.read_text())["default-address-pools"], verified_generation=None) + state_file.write_text(json.dumps(state), encoding="utf-8") + state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "daemon.json.before").write_bytes(daemon.read_bytes()) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + for path in recovery.iterdir(): + path.chmod(0o600) + self.drain_command.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + result = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + restored = json.loads(state_file.read_text(encoding="utf-8")) + self.assertNotIn("phase", restored) + self.assertEqual(restored["verified_generation"], hashlib.sha256(daemon.read_bytes()).hexdigest()) + + def test_removal_pending_requires_authoritative_recovery_before_commands(self) -> None: + daemon = self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-without-recovery" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + self.assertEqual(self._run(str(policy_env), checkpoint_dir=str(checkpoint)).returncode, 0) + state_file = checkpoint / "docker-network-policy.json" + state = json.loads(state_file.read_text(encoding="utf-8")) + state.update(phase="removal-pending", removal_managed_default_address_pools=json.loads(daemon.read_text())["default-address-pools"], verified_generation=None) + state_file.write_text(json.dumps(state), encoding="utf-8") + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + result = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + + self.assertNotEqual(result.returncode, 0) + self.assertIn("transaction recovery is invalid", result.stderr) + self.assertTrue(state_file.exists()) + + + def test_first_apply_completion_failures_preserve_authority(self) -> None: + audit = Path(self.tmp) / "completion-faults" + audit.mkdir() + (audit / "sitecustomize.py").write_text( + "import json, os, shutil\n_replace, _rmtree, _unlink = os.replace, shutil.rmtree, os.unlink\ndef replace(src, dst):\n" + " if os.environ['FAULT']=='complete' and os.path.basename(dst)==os.path.basename(os.environ['STATE']) and json.load(open(src)).get('phase')=='rollback-complete': raise OSError('injected')\n" + " return _replace(src, dst)\ndef rmtree(path, *a, **k):\n" + " if os.environ['FAULT']=='recovery' and os.path.basename(path).startswith('recovery.'): raise OSError('injected')\n" + " return _rmtree(path, *a, **k)\ndef unlink(path, *a, **k):\n" + " if os.environ['FAULT']=='marker' and os.path.basename(path)==os.path.basename(os.environ['STATE']): raise OSError('injected')\n" + " return _unlink(path, *a, **k)\nos.replace, shutil.rmtree, os.unlink = replace, rmtree, unlink\n", + encoding="utf-8", + ) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + for fault, phase, has_recovery in (("complete", "first-apply-pending", True), ("recovery", "rollback-complete", True), ("marker", "rollback-complete", False)): + with self.subTest(fault=fault): + checkpoint = Path(self.tmp) / f"checkpoint-completion-{fault}" + self._seed_checkpoint(checkpoint, "first-apply-pending", recovery=True) + self._write_daemon('{"default-address-pools":[{"base":"198.51.100.0/24","size":29}]}') + self._write_success_commands() + command_log = Path(self.tmp) / f"completion-{fault}.log" + for name in ("drain", "restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + state_file = checkpoint / "docker-network-policy.json" + result = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, text=True, env=self._env(PYTHONPATH=str(audit), FAULT=fault, STATE=str(state_file)), timeout=30, + ) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(json.loads(state_file.read_text())["phase"], phase) + self.assertEqual(bool(list(checkpoint.glob("recovery.*"))), has_recovery) + self.assertEqual(command_log.read_text().splitlines(), ["drain", "restart", "probe", "resume", "health"]) + def _assert_raw_candidate_rejected_before_operational_side_effects( self, rendered: dict[str, str], @@ -1061,7 +1271,8 @@ def test_managed_reapply_marks_generation_pending_before_drain(self) -> None: state_file = checkpoint / "docker-network-policy.json" self.drain_command.write_text( "#!/usr/bin/env bash\n" - f"grep -q '\"verified_generation\": null' {state_file}\n", + f"grep -q '\"verified_generation\": null' {state_file}\n" + f"grep -q '\"phase\": \"reapply-pending\"' {state_file}\n", encoding="utf-8", ) candidate = self._rendered_with_policy() @@ -1132,10 +1343,17 @@ def test_failed_unverified_retry_restores_durable_baseline(self) -> None: "prior_mode": "600", "prior_present": True, "verified_generation": None, + "phase": "first-apply-pending", }), encoding="utf-8", ) state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "daemon.json.before").write_text('{"live-restore":true}', encoding="utf-8") + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + for path in recovery.iterdir(): + path.chmod(0o600) candidate = dict(rendered) candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" env_file = self._write_env_file(candidate) @@ -1162,6 +1380,7 @@ def test_failed_interrupted_reapply_retry_restores_immediate_snapshot(self) -> N state_file = checkpoint / "docker-network-policy.json" state = json.loads(state_file.read_text(encoding="utf-8")) state["verified_generation"] = None + state["phase"] = "reapply-pending" state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) recovery = checkpoint / "recovery.interrupted" @@ -1201,10 +1420,15 @@ def test_failed_unverified_retry_preserves_new_unrelated_keys_after_absent_basel "prior_mode": None, "prior_present": False, "verified_generation": None, + "phase": "first-apply-pending", }), encoding="utf-8", ) state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) candidate = dict(rendered) candidate["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" env_file = self._write_env_file(candidate) @@ -1232,10 +1456,15 @@ def test_unverified_retry_preserves_current_snapshot_presence(self) -> None: "prior_mode": None, "prior_present": False, "verified_generation": None, + "phase": "first-apply-pending", }), encoding="utf-8", ) state_file.chmod(0o600) + recovery = checkpoint / "recovery.interrupted" + recovery.mkdir(mode=0o700) + (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) + (recovery / "prior-ci-fleet.env").chmod(0o600) self._write_success_commands() env_file = self._write_env_file(rendered) @@ -3359,7 +3588,7 @@ def test_sleeping_probe_times_out(self) -> None: timeout=30, ) self.assertNotEqual(result.returncode, 0) - self.assertLess(time.monotonic() - started, 7) + self.assertLess(time.monotonic() - started, 10) @unittest.skipUnless(Path("/dev/shm").is_dir(), "/dev/shm is unavailable") def test_apply_works_across_temp_filesystems(self) -> None: @@ -3584,6 +3813,7 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> "prior_mode": None, "prior_present": False, "verified_generation": None, + "phase": "first-apply-pending", }), encoding="utf-8", ) @@ -3631,6 +3861,7 @@ def test_interrupted_first_apply_failed_candidate_resume_redrains_before_prior_r "prior_mode": None, "prior_present": False, "verified_generation": None, + "phase": "first-apply-pending", }), encoding="utf-8", ) @@ -3680,6 +3911,7 @@ def test_interrupted_first_apply_drain_failure_stops_before_restart_or_mutation( "prior_mode": None, "prior_present": False, "verified_generation": None, + "phase": "first-apply-pending", }), encoding="utf-8", ) @@ -3798,7 +4030,10 @@ def test_removal_daemon_change_during_drain_aborts_and_preserves_update(self) -> self.assertEqual(removed.stderr, "ERROR: daemon.json changed during network-policy removal\n") self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume", "health"]) self.assertEqual(daemon.read_bytes(), changed_bytes) - self.assertEqual(state_file.read_bytes(), prior_state) + state = json.loads(state_file.read_text(encoding="utf-8")) + expected = json.loads(prior_state) + expected["verified_generation"] = hashlib.sha256(changed_bytes).hexdigest() + self.assertEqual(state, expected) def test_removal_stages_from_the_conflict_detection_snapshot(self) -> None: daemon = self._write_daemon('{"live-restore":true}\n') @@ -4075,8 +4310,10 @@ def test_interrupted_removal_retry_uses_persisted_managed_pools_for_rollback(sel state = json.loads(state_file.read_text(encoding="utf-8")) state["phase"] = "removal-pending" state["removal_managed_default_address_pools"] = managed_pools + state["verified_generation"] = None state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) + self._write_recovery(checkpoint, daemon.read_bytes()) daemon.write_text('{"live-restore":true}\n', encoding="utf-8") (Path(self.tmp) / "probe.sh").write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") (Path(self.tmp) / "probe.sh").chmod(0o755) @@ -4109,6 +4346,7 @@ def test_pending_removal_with_absent_daemon_completes_runtime_verification(self) state["verified_generation"] = None state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) + self._write_recovery(checkpoint, daemon.read_bytes()) daemon.unlink() command_log = Path(self.tmp) / "pending-removal-absent-daemon.log" for name in ("restart", "probe", "resume", "health"): @@ -4144,6 +4382,7 @@ def test_pending_removal_with_absent_daemon_failure_restores_managed_state(self) state["verified_generation"] = None state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) + self._write_recovery(checkpoint, managed) daemon.unlink() command_log = Path(self.tmp) / "pending-removal-absent-failure.log" (Path(self.tmp) / "restart.sh").write_text( @@ -4196,6 +4435,7 @@ def test_pending_removal_with_absent_daemon_failed_rollback_retains_recovery(sel state["verified_generation"] = None state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) + self._write_recovery(checkpoint, managed) daemon.unlink() restart_log = Path(self.tmp) / "pending-removal-absent-recovery.log" restart = Path(self.tmp) / "restart.sh" @@ -4236,6 +4476,7 @@ def test_apply_from_pending_removal_persists_synthesized_rollback_daemon(self) - state["verified_generation"] = None state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) + self._write_recovery(checkpoint, daemon.read_bytes()) daemon.unlink() changed_policy = dict(policy) changed_policy["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" @@ -4264,6 +4505,7 @@ def test_configured_apply_resumes_removal_pending_checkpoint(self) -> None: state["verified_generation"] = None state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) + self._write_recovery(checkpoint, daemon.read_bytes()) daemon.write_text('{"live-restore":true}\n', encoding="utf-8") resumed = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) @@ -4291,6 +4533,7 @@ def test_failed_configured_apply_from_removal_pending_restores_managed_pools(sel state["verified_generation"] = None state_file.write_text(json.dumps(state), encoding="utf-8") state_file.chmod(0o600) + self._write_recovery(checkpoint, managed) policy_b = dict(policy_a) policy_b["CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_0_BASE"] = "192.0.2.0/24" env_b = self._write_env_file(policy_b) @@ -5097,7 +5340,8 @@ def test_first_apply_persists_prior_environment_before_drain(self) -> None: "#!/usr/bin/env bash\n" f"recovery=({checkpoint}/recovery.*)\n" '[[ ${#recovery[@]} == 1 ]]\n' - f"cmp -s {self.installed_env} \"${{recovery[0]}}/prior-ci-fleet.env\"\n", + f"cmp -s {self.installed_env} \"${{recovery[0]}}/prior-ci-fleet.env\"\n" + f"grep -Fq '\"phase\": \"first-apply-pending\"' {checkpoint / 'docker-network-policy.json'}\n", encoding="utf-8", ) env_file = self._write_env_file(self._rendered_with_policy()) From 4fc3f98ada899174ae72478be08a7762044dd9a3 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 12:37:55 -0500 Subject: [PATCH 53/54] fix: close network policy verification gaps --- scripts/apply-docker-network-policy.sh | 21 +++- scripts/healthcheck.sh | 4 +- scripts/test_apply_docker_network_policy.py | 130 ++++++++++++++++++++ 3 files changed, 153 insertions(+), 2 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 6ee720da..9c4d643e 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -1080,6 +1080,7 @@ PY trap - EXIT complete_first_apply_rollback || { removal_failure='failed to complete interrupted first-apply rollback'; exit 2; } else + set_verified_generation "" rollback-complete || { removal_failure='failed to commit network-policy removal'; exit 2; } clear_recovery_artifacts || { removal_failure='failed to clear obsolete network-policy recovery data'; exit 2; } clear_managed_marker "$removal_daemon" || { removal_failure='failed to clear network-policy managed marker'; exit 2; } fi @@ -1200,7 +1201,25 @@ PY then daemon_matches=true current_generation=$(file_generation "$daemon_config") || { rm -rf "$work_dir"; die 'failed to identify daemon.json generation'; } - if [[ -n "$prior_verified_generation" && "$current_generation" == "$prior_verified_generation" ]]; then + if [[ -n "$prior_verified_generation" && "$current_generation" == "$prior_verified_generation" ]] && + python3 - "$env_file" "$prior_env" "$repo_root/scripts" <<'PY' +import sys +from pathlib import Path + +sys.path.insert(0, sys.argv[3]) +from desired_state import parse_env + +candidate = parse_env(Path(sys.argv[1]), allow_unknown=True) +installed = parse_env(Path(sys.argv[2]), allow_unknown=True) +fields = { + "CI_FLEET_CONFIGURED_MAX_RUNNERS", + "CI_FLEET_CONTROLLER_STATE", + "CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", + "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS", +} +raise SystemExit(any(candidate.get(name) != installed.get(name) for name in fields)) +PY + then clear_recovery_artifacts || { rm -rf "$work_dir"; die 'failed to clear obsolete network-policy recovery data'; } rm -rf "$work_dir" printf 'NETWORK_POLICY_NO_CHANGE\n' diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index 14201b40..52df8769 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -41,6 +41,8 @@ invalid = next((name for name in values if not name.startswith("CI_FLEET_")), No if invalid is not None: raise SystemExit(f"ERROR: rendered environment variable must start with CI_FLEET_: {invalid}") for name, value in values.items(): + if name.startswith("CI_FLEET_HEALTH_"): + continue sys.stdout.buffer.write(name.encode() + b"\0" + value.encode() + b"\0") PY then @@ -57,5 +59,5 @@ if [[ $health_root_prefix_present == x ]]; then export CI_FLEET_ROOT_PREFIX=$hea if [[ $health_docker_socket_present == x ]]; then export CI_FLEET_DOCKER_SOCKET=$health_docker_socket; else unset CI_FLEET_DOCKER_SOCKET; fi if [[ $health_bootstrap_present == x ]]; then export CI_FLEET_HEALTH_BOOTSTRAP=$health_bootstrap; else unset CI_FLEET_HEALTH_BOOTSTRAP; fi if [[ $health_suppress_delivery_present == x ]]; then export CI_FLEET_HEALTH_SUPPRESS_DELIVERY=$health_suppress_delivery; else unset CI_FLEET_HEALTH_SUPPRESS_DELIVERY; fi -use_local_docker +use_local_docker || exit 2 exec python3 "$repo_root/scripts/health.py" "${args[@]}" "$@" diff --git a/scripts/test_apply_docker_network_policy.py b/scripts/test_apply_docker_network_policy.py index 1e45fbe7..86af8a23 100644 --- a/scripts/test_apply_docker_network_policy.py +++ b/scripts/test_apply_docker_network_policy.py @@ -271,6 +271,41 @@ def test_env_argument_rejects_path_before_command_resolution(self) -> None: self.assertFalse(candidate_shim_ran.exists()) self.assertFalse(health_substitute_ran.exists()) + def test_selected_env_does_not_export_health_only_controls(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + candidate = root / "candidate.env" + candidate.write_text( + "CI_FLEET_INSTANCE=candidate\nCI_FLEET_HEALTH_DISK_WARN_PERCENT=99\n", + encoding="utf-8", + ) + fake_bin = root / "bin" + fake_bin.mkdir() + python = fake_bin / "python3" + python.write_text( + "#!/usr/bin/env bash\n" + "[[ ${CI_FLEET_INSTANCE:-} == candidate ]]\n" + "[[ -z ${CI_FLEET_HEALTH_DISK_WARN_PERCENT+x} ]]\n", + encoding="utf-8", + ) + python.chmod(0o755) + env = dict(os.environ) + env.update( + CI_FLEET_TESTING="1", + CI_FLEET_ROOT_PREFIX=tmp, + PATH=f"{fake_bin}:{env['PATH']}", + ) + + result = subprocess.run( + [str(SCRIPTS / "healthcheck.sh"), "--env", str(candidate)], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + def test_selected_env_does_not_inherit_removed_pool_variables(self) -> None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -2215,6 +2250,34 @@ def test_health_accepts_only_success_and_warning_results(self) -> None: self.assertEqual(result.returncode == 0, accepted, result.stderr) + def test_health_wrapper_preflight_failure_is_not_warning(self) -> None: + self._write_daemon("{}\n") + self._write_success_commands() + env_file = self._write_env_file(self._rendered_with_policy()) + release_scripts = Path(self.tmp) / "release" / "scripts" + release_scripts.mkdir(parents=True) + for name in ("healthcheck.sh", "docker-local-env.sh", "desired_state.py"): + shutil.copy2(SCRIPTS / name, release_scripts / name) + + result = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(Path(self.tmp) / "checkpoint-health-preflight"), + "--env", + str(env_file), + ], + capture_output=True, + text=True, + env=self._env( + CI_FLEET_HEALTH_CHECK_COMMAND=str(release_scripts / "healthcheck.sh"), + DOCKER_HOST="tcp://example.invalid:2375", + ), + timeout=30, + ) + + self.assertNotEqual(result.returncode, 0) + def test_transactional_health_suppresses_delivery(self) -> None: self._write_daemon("{}\n") self._write_success_commands() @@ -2672,6 +2735,7 @@ def test_semantically_equal_verified_daemon_config_is_no_change_before_side_effe command.write_text(f"#!/usr/bin/env bash\ntouch {marker}\n", encoding="utf-8") command.chmod(0o755) env_file = self._write_env_file(rendered) + self.installed_env.write_bytes(env_file.read_bytes()) checkpoint = Path(self.tmp) / "checkpoint" checkpoint.mkdir(mode=0o700) state_file = checkpoint / "docker-network-policy.json" @@ -2696,6 +2760,33 @@ def test_semantically_equal_verified_daemon_config_is_no_change_before_side_effe self.assertTrue(checkpoint.exists()) self.assertTrue(all(not marker.exists() for marker in markers)) + def test_capacity_change_with_matching_daemon_runs_verification_gates(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-capacity-change" + self._write_success_commands() + rendered = self._rendered_with_policy() + env_file = self._write_env_file(rendered) + applied = self._run(str(env_file), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(env_file.read_bytes()) + rendered["CI_FLEET_DOCKER_NETWORKS_PER_RUNNER"] = "2" + candidate_env = self._write_env_file(rendered) + command_log = Path(self.tmp) / "capacity-change.log" + command_log.touch() + self.drain_command.write_text(f"#!/usr/bin/env bash\necho drain >> {command_log}\n", encoding="utf-8") + for name in ("restart", "probe", "resume", "health"): + command = Path(self.tmp) / f"{name}.sh" + command.write_text(f"#!/usr/bin/env bash\necho {name} >> {command_log}\n", encoding="utf-8") + command.chmod(0o755) + + reconciled = self._run(str(candidate_env), checkpoint_dir=str(checkpoint)) + + self.assertEqual(reconciled.returncode, 0, reconciled.stderr) + self.assertEqual( + command_log.read_text(encoding="utf-8").splitlines(), + ["drain", "restart", "probe", "resume", "health"], + ) + def test_matching_file_without_verified_generation_runs_activation_and_marks_verified(self) -> None: self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-pending-generation" @@ -4855,6 +4946,44 @@ def test_successful_removal_clears_recovery_before_marker_and_allows_next_reconc self.assertEqual(reconciled.returncode, 0, reconciled.stderr) self.assertEqual(reconciled.stdout, "NETWORK_POLICY_NOOP\n") + def test_removal_completion_survives_crash_after_recovery_cleanup(self) -> None: + self._write_daemon("{}\n") + checkpoint = Path(self.tmp) / "checkpoint-removal-completion-crash" + self._write_success_commands() + policy_env = self._write_env_file(self._rendered_with_policy()) + applied = self._run(str(policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(policy_env.read_bytes()) + state_file = checkpoint / "docker-network-policy.json" + audit_dir = Path(self.tmp) / "removal-completion-crash-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os, shutil, signal\n" + "_rmtree = shutil.rmtree\n" + "def rmtree(path, *args, **kwargs):\n" + " result = _rmtree(path, *args, **kwargs)\n" + " if os.path.basename(path).startswith('recovery.'):\n" + " os.kill(os.getppid(), signal.SIGKILL)\n" + " return result\n" + "shutil.rmtree = rmtree\n", + encoding="utf-8", + ) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + interrupted = subprocess.run( + [str(SCRIPTS / "apply-docker-network-policy.sh"), "--checkpoint", str(checkpoint), "--env", str(no_policy_env)], + capture_output=True, + text=True, + env=self._env(PYTHONPATH=str(audit_dir)), + timeout=30, + ) + + self.assertNotEqual(interrupted.returncode, 0) + self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["phase"], "rollback-complete") + reconciled = self._run(str(no_policy_env), checkpoint_dir=str(checkpoint)) + self.assertEqual(reconciled.returncode, 0, reconciled.stderr) + self.assertEqual(reconciled.stdout, "NETWORK_POLICY_NOOP\n") + def test_removal_recovery_cleanup_failure_rolls_back_and_keeps_marker_usable(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-removal-recovery-cleanup-failure" @@ -5377,6 +5506,7 @@ def test_no_change_clears_stale_committed_recovery(self) -> None: env_file = self._write_env_file(self._rendered_with_policy()) applied = self._run(str(env_file), checkpoint_dir=str(checkpoint)) self.assertEqual(applied.returncode, 0, applied.stderr) + self.installed_env.write_bytes(env_file.read_bytes()) recovery = checkpoint / "recovery.stale" recovery.mkdir(mode=0o700) (recovery / "prior-ci-fleet.env").write_bytes(self.installed_env.read_bytes()) From 032bed9821e98a624179c0b81d2edc20186742f7 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 13:38:03 -0500 Subject: [PATCH 54/54] fix: preserve network policy trust and recovery --- scripts/apply-docker-network-policy.sh | 19 +-- scripts/desired_state.py | 44 +++++++ scripts/healthcheck.sh | 5 +- scripts/test-healthcheck.sh | 12 +- scripts/test_apply_docker_network_policy.py | 130 +++++++++++++++++++- 5 files changed, 190 insertions(+), 20 deletions(-) diff --git a/scripts/apply-docker-network-policy.sh b/scripts/apply-docker-network-policy.sh index 9c4d643e..df9c2d1b 100755 --- a/scripts/apply-docker-network-policy.sh +++ b/scripts/apply-docker-network-policy.sh @@ -313,7 +313,7 @@ resume_after_failed_drain() { fi ((resume_failed != 0)) || run_command "$resume_command" --env "$prior_env" || resume_failed=1 ((resume_failed != 0)) || run_health "$prior_env" || health_failed=1 - if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true || "$apply_phase" == first-apply-pending ]]; then + if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" == true ]]; then complete_first_apply_rollback || resume_failed=1 fi if ((resume_failed == 0 && health_failed == 0)) && [[ "$removal_checkpoint_started" == true ]]; then @@ -323,7 +323,7 @@ resume_after_failed_drain() { if ((resume_failed == 0 && health_failed == 0)) && [[ "$apply_checkpoint_started" == true ]]; then set_verified_generation "$prior_verified_generation" || resume_failed=1 fi - if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" != true && "$apply_phase" != first-apply-pending && "$cancelling_first_apply" != true ]]; then + if ((resume_failed == 0 && health_failed == 0)) && [[ "$new_marker" != true && -z "$apply_phase" && "$cancelling_first_apply" != true ]]; then clear_recovery_artifacts || resume_failed=1 fi rm -rf "$work_dir" @@ -849,8 +849,7 @@ PY controller_resumed=true run_command "$resume_command" --env "$env_file" || die 'controller resume command failed while recovering interrupted network-policy apply' run_health "$env_file" || die 'health check failed while recovering interrupted network-policy apply' - clear_recovery_artifacts || die 'failed to clear obsolete network-policy recovery data' - clear_managed_marker "$work_dir/daemon.json.removal" || die 'failed to clear interrupted network-policy marker' + complete_first_apply_rollback || die 'failed to complete interrupted network-policy rollback' trap - EXIT INT TERM rm -rf "$work_dir" printf 'NETWORK_POLICY_REMOVED\n' @@ -1018,10 +1017,14 @@ PY rm -rf "$work_dir" printf 'ERROR: %s; managed daemon.json restored\n' "$removal_failure" >&2 else - recovery_path=$(persist_recovery "$managed_snapshot" "$prior_env") || { - printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$removal_failure" >&2 - exit "$status" - } + if [[ -n "$transaction_recovery" ]]; then + recovery_path=$transaction_recovery + else + recovery_path=$(persist_recovery "$managed_snapshot" "$prior_env") || { + printf 'ERROR: %s; rollback verification failed; failed to persist recovery data\n' "$removal_failure" >&2 + exit "$status" + } + fi rm -rf "$work_dir" printf 'ERROR: %s; rollback verification failed; recovery data retained at %s\n' "$removal_failure" "$recovery_path" >&2 fi diff --git a/scripts/desired_state.py b/scripts/desired_state.py index 7c66cf69..49c25d76 100755 --- a/scripts/desired_state.py +++ b/scripts/desired_state.py @@ -29,6 +29,33 @@ "CI_FLEET_GITHUB_APP_PRIVATE_KEY_FILE", } HOST_OPTIONAL = {"CI_FLEET_RUNNER_TTL"} +RENDERED_ENV_NAMES = HOST_REQUIRED | HOST_OPTIONAL | { + "CI_FLEET_CAPACITY_BUDGET", + "CI_FLEET_COMMIT", + "CI_FLEET_CONFIGURED_MAX_RUNNERS", + "CI_FLEET_CONFIG_REF", + "CI_FLEET_CONFIG_REPOSITORY", + "CI_FLEET_CONTROLLER_IMAGE", + "CI_FLEET_CONTROLLER_STATE", + "CI_FLEET_DESIRED_STATE_SCHEMA", + "CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT", + "CI_FLEET_DOCKER_GID", + "CI_FLEET_DOCKER_NETWORKS_PER_RUNNER", + "CI_FLEET_DOCKER_NETWORK_RESERVE_SUBNETS", + "CI_FLEET_ENGINE_REF", + "CI_FLEET_GITHUB_URL", + "CI_FLEET_INSTANCE", + "CI_FLEET_LABELS", + "CI_FLEET_MAX_RUNNERS", + "CI_FLEET_MIN_RUNNERS", + "CI_FLEET_RUNNER_CPUS", + "CI_FLEET_RUNNER_GROUP", + "CI_FLEET_RUNNER_IMAGE", + "CI_FLEET_RUNNER_MEMORY_MIB", + "CI_FLEET_SCALE_SET_NAME", + "CI_FLEET_STATUS_REPORTING_REQUIRED", + "CI_FLEET_VERSION", +} REQUIRED_STATUS_CAPABILITY = "required_status_reporting" STATUS_REPORTING_CONFIG_CAPABILITY = "status_reporting_config" DOCKER_NETWORK_POLICY_CONFIG_CAPABILITY = "docker_network_policy_config" @@ -113,6 +140,21 @@ def parse_env(path: Path, *, allow_unknown: bool) -> dict[str, str]: return values +def rendered_env_names(values: dict[str, str]) -> set[str]: + names = set(RENDERED_ENV_NAMES) + try: + count = int(values.get("CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_COUNT", "0")) + except ValueError: + count = 0 + if 0 < count <= MAX_DOCKER_ADDRESS_POOLS: + names.update( + f"CI_FLEET_DOCKER_DEFAULT_ADDRESS_POOL_{index}_{field}" + for index in range(count) + for field in ("BASE", "SIZE") + ) + return names + + def validate_host_values(values: dict[str, str]) -> dict[str, str]: missing = sorted(HOST_REQUIRED - values.keys()) if missing: @@ -380,6 +422,8 @@ def build_rendered_env( raise DesiredStateError("selected engine does not support status reporting configuration") if reporting_required: rendered["CI_FLEET_STATUS_REPORTING_REQUIRED"] = "1" + if set(rendered) - rendered_env_names(rendered): + raise DesiredStateError("renderer produced unsupported environment fields") for name, value in rendered.items(): if not SAFE_ENV_VALUE.fullmatch(value): raise DesiredStateError(f"rendered value for {name} contains unsafe characters") diff --git a/scripts/healthcheck.sh b/scripts/healthcheck.sh index 52df8769..d2230254 100755 --- a/scripts/healthcheck.sh +++ b/scripts/healthcheck.sh @@ -34,14 +34,15 @@ import sys from pathlib import Path sys.path.insert(0, sys.argv[2]) -from desired_state import parse_env +from desired_state import parse_env, rendered_env_names values = parse_env(Path(sys.argv[1]), allow_unknown=True) invalid = next((name for name in values if not name.startswith("CI_FLEET_")), None) if invalid is not None: raise SystemExit(f"ERROR: rendered environment variable must start with CI_FLEET_: {invalid}") +allowed = rendered_env_names(values) for name, value in values.items(): - if name.startswith("CI_FLEET_HEALTH_"): + if name not in allowed: continue sys.stdout.buffer.write(name.encode() + b"\0" + value.encode() + b"\0") PY diff --git a/scripts/test-healthcheck.sh b/scripts/test-healthcheck.sh index 26136ad2..ac6f99f6 100755 --- a/scripts/test-healthcheck.sh +++ b/scripts/test-healthcheck.sh @@ -6,7 +6,7 @@ trap 'rm -rf "$tmp"' EXIT mkdir -p "$tmp/bin" cat >"$tmp/bin/python3" <<'EOF' #!/usr/bin/env bash -printf 'controller=%s\n' "${CI_FLEET_CONTROLLER-}" +printf 'instance=%s\n' "${CI_FLEET_INSTANCE-}" printf 'stale=%s\n' "${CI_FLEET_STALE-unset}" printf 'testing=%s\n' "${CI_FLEET_TESTING-}" printf 'testing_present=%s\n' "${CI_FLEET_TESTING+set}" @@ -23,7 +23,7 @@ printf 'args=%s\n' "$*" EOF chmod +x "$tmp/bin/python3" cat >"$tmp/candidate.env" < None: with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) @@ -3940,6 +3975,46 @@ def test_removal_resumes_interrupted_first_apply_before_clearing_marker(self) -> ], ) + def test_absent_first_apply_cleanup_commits_before_recovery_deletion(self) -> None: + checkpoint = Path(self.tmp) / "checkpoint-absent-first-apply-cleanup-crash" + state_file = self._seed_checkpoint(checkpoint, "first-apply-pending", recovery=True) + state = json.loads(state_file.read_text(encoding="utf-8")) + state.update(prior_mode=None, prior_present=False) + state_file.write_text(json.dumps(state), encoding="utf-8") + (checkpoint / "recovery.interrupted" / "daemon.json.before").unlink() + self._write_success_commands() + audit_dir = Path(self.tmp) / "absent-first-apply-cleanup-crash-audit" + audit_dir.mkdir() + (audit_dir / "sitecustomize.py").write_text( + "import os, shutil, signal\n" + "_rmtree = shutil.rmtree\n" + "def rmtree(path, *args, **kwargs):\n" + " result = _rmtree(path, *args, **kwargs)\n" + " if os.path.basename(path).startswith('recovery.'):\n" + " os.kill(os.getppid(), signal.SIGKILL)\n" + " return result\n" + "shutil.rmtree = rmtree\n", + encoding="utf-8", + ) + no_policy_env = self._write_env_file({"CI_FLEET_INSTANCE": "example-ci-01"}) + + interrupted = subprocess.run( + [ + str(SCRIPTS / "apply-docker-network-policy.sh"), + "--checkpoint", + str(checkpoint), + "--env", + str(no_policy_env), + ], + capture_output=True, + text=True, + env=self._env(PYTHONPATH=str(audit_dir)), + timeout=30, + ) + + self.assertNotEqual(interrupted.returncode, 0) + self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["phase"], "rollback-complete") + def test_interrupted_first_apply_failed_candidate_resume_redrains_before_prior_resume(self) -> None: checkpoint = Path(self.tmp) / "checkpoint-interrupted-resume-failure" checkpoint.mkdir(mode=0o700) @@ -4029,6 +4104,51 @@ def test_interrupted_first_apply_drain_failure_stops_before_restart_or_mutation( self.assertTrue(recovery.exists()) self.assertEqual(command_log.read_text(encoding="utf-8").splitlines(), ["drain", "resume", "health"]) + def test_configured_first_apply_retry_failed_drain_preserves_recovery(self) -> None: + rendered = self._rendered_with_policy() + daemon = self._write_daemon(json.dumps(render_docker_daemon_config(rendered))) + checkpoint = Path(self.tmp) / "checkpoint-configured-first-apply-drain-failure" + state_file = self._seed_checkpoint(checkpoint, "first-apply-pending", recovery=True) + state = json.loads(state_file.read_text(encoding="utf-8")) + state.update(prior_mode=None, prior_present=False) + state_file.write_text(json.dumps(state), encoding="utf-8") + recovery = checkpoint / "recovery.interrupted" + (recovery / "daemon.json.before").unlink() + self._write_success_commands() + self.drain_command.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + env_file = self._write_env_file(rendered) + + retried = self._run(str(env_file), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(retried.returncode, 0) + self.assertTrue(state_file.exists()) + self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["phase"], "first-apply-pending") + self.assertTrue(recovery.exists()) + self.assertEqual(json.loads(daemon.read_text(encoding="utf-8")), render_docker_daemon_config(rendered)) + + def test_configured_pending_retry_failed_drain_preserves_recovery(self) -> None: + rendered = self._rendered_with_policy() + managed = render_docker_daemon_config(rendered) + env_file = self._write_env_file(rendered) + for phase in ("reapply-pending", "removal-pending"): + with self.subTest(phase=phase): + self._write_daemon(json.dumps(managed)) + checkpoint = Path(self.tmp) / f"checkpoint-configured-{phase}-drain-failure" + state_file = self._seed_checkpoint(checkpoint, phase, recovery=True) + state = json.loads(state_file.read_text(encoding="utf-8")) + if phase == "removal-pending": + state["removal_managed_default_address_pools"] = managed["default-address-pools"] + state_file.write_text(json.dumps(state), encoding="utf-8") + recovery = checkpoint / "recovery.interrupted" + self._write_success_commands() + self.drain_command.write_text("#!/usr/bin/env bash\nexit 1\n", encoding="utf-8") + + retried = self._run(str(env_file), checkpoint_dir=str(checkpoint), expected_rc=1) + + self.assertNotEqual(retried.returncode, 0) + self.assertEqual(json.loads(state_file.read_text(encoding="utf-8"))["phase"], phase) + self.assertTrue(recovery.exists()) + def test_removal_rejects_non_object_daemon_before_copy_or_commands(self) -> None: daemon = self._write_daemon("{}\n") checkpoint = Path(self.tmp) / "checkpoint-non-object-removal" @@ -4544,7 +4664,9 @@ def test_pending_removal_with_absent_daemon_failed_rollback_retains_recovery(sel self.assertNotEqual(removed.returncode, 0) self.assertIn("rollback verification failed; recovery data retained at", removed.stderr) - recovery = next(checkpoint.glob("recovery.*")) + recoveries = list(checkpoint.glob("recovery.*")) + self.assertEqual(len(recoveries), 1) + recovery = recoveries[0] self.assertEqual((recovery / "daemon.json.before").read_bytes(), managed) self.assertEqual((recovery / "prior-ci-fleet.env").read_bytes(), prior_env) retained_state = json.loads(state_file.read_text(encoding="utf-8"))