From 4ffa3be81394b13063d0c2ff4d840d01f554795e Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 20:23:15 -0500 Subject: [PATCH 01/15] feat: prepare template release compatibility contract --- .github/workflows/validate.yml | 12 +- README.md | 9 + docs/RELEASE.md | 16 ++ docs/UPDATING.md | 4 + engine-rollout-evidence.json | 4 + examples/multi-host/fleet.json | 20 ++ fleet.json | 10 + fleet.schema.json | 34 +++- scripts/init.py | 15 +- scripts/test_core_compatibility.py | 67 ++++++ scripts/test_policy.py | 225 ++++++++++++++++++++ scripts/test_release_update.py | 103 ++++++++++ scripts/validate.py | 316 ++++++++++++++++++++++++++++- template-compatibility.json | 34 ++++ 14 files changed, 863 insertions(+), 6 deletions(-) create mode 100644 docs/RELEASE.md create mode 100644 engine-rollout-evidence.json create mode 100644 scripts/test_core_compatibility.py create mode 100644 scripts/test_release_update.py create mode 100644 template-compatibility.json diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 033a91a..84942c1 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -27,11 +27,16 @@ jobs: - name: Validate shell and Python syntax run: | bash -n scripts/init.sh scripts/validate.sh - python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py scripts/scan_committed_secrets.py + python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py scripts/test_core_compatibility.py scripts/test_release_update.py scripts/scan_committed_secrets.py - name: Prove forbidden configurations fail closed run: python3 scripts/test_policy.py + - name: Verify exact core compatibility and release update flow + run: | + python3 scripts/test_core_compatibility.py + python3 scripts/test_release_update.py + - name: Scan committed file contents for secrets env: EVENT_NAME: ${{ github.event_name }} @@ -59,6 +64,11 @@ jobs: run: | ./scripts/validate.sh ./scripts/validate.sh --config examples/multi-host/fleet.json + if ./scripts/validate.sh --strict 2>"$RUNNER_TEMP/strict-errors"; then + echo 'documentation configuration unexpectedly passed strict validation' >&2 + exit 1 + fi + grep -F 'reviewed operational Docker pool CIDR' "$RUNNER_TEMP/strict-errors" - name: Prove initialized configurations pass strict policy run: | diff --git a/README.md b/README.md index 2df86ad..dccaa6a 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ flowchart LR The initializer refuses to replace a configured file unless `--force` is explicit. Run `./scripts/init.sh --help` for repository, registry, runner-group, controller, location, capacity, resource, and output options. +## Reviewed compatibility and release state + +`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records per-controller capability evidence only after an engine revision has already been integrated. New status-reporting or Docker network-policy fields require that prior-state evidence. + +The examples use RFC 5737 address pools so they remain public documentation. Ordinary validation accepts them. `./scripts/validate.sh --strict` rejects them until a private adopter supplies reviewed operational Docker pool CIDRs. Initial release preparation and exclusions are in `docs/RELEASE.md`; no tag or GitHub release is published by this repository change. + ## Schema v3: Git-authored controller desired state `fleet.json` is the reviewed authority for logical controller state. Each entry in `controllers` has a unique ID and declares: @@ -143,6 +149,8 @@ Adding workers reduces wall-clock time only while independent shards remain queu |---|---| | `fleet.json` | Fictional, valid schema-v3 configuration with one controller | | `fleet.schema.json` | JSON Schema draft 2020-12 editor contract | +| `template-compatibility.json` | Exact reviewed core/template compatibility and prepared release state | +| `engine-rollout-evidence.json` | Staged optional-capability evidence, empty in the public template | | `scripts/init.sh` | Safe first-project and first-controller initializer | | `scripts/validate.sh` | Structural, relational, capacity, and secret-boundary validation | | `scripts/test_policy.py` | Regression tests proving unsafe configurations fail closed | @@ -150,6 +158,7 @@ Adding workers reduces wall-clock time only while independent shards remain queu | `SECURITY.md` | Secret handling and vulnerability reporting | | `AGENTS.md` | Non-negotiable rules for humans and coding agents | | `docs/UPDATING.md` | Keeping a derived private repository current: schema vs template versions, releases, migrations, Dependabot | +| `docs/RELEASE.md` | Prepared initial release notes and immutable-tag publication checklist | ## Public and private boundary diff --git a/docs/RELEASE.md b/docs/RELEASE.md new file mode 100644 index 0000000..0706452 --- /dev/null +++ b/docs/RELEASE.md @@ -0,0 +1,16 @@ +# Initial template release preparation + +The `v1.0.0` template release is prepared but not published. Creating the tag or GitHub release requires separate authorization after this change merges. + +The candidate synchronizes the standalone schema-v3 contract with `templates/config-repository` at reviewed ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It adds optional staged status-reporting and Docker network-policy support. Existing configurations that omit either field remain valid. + +The fictional examples retain engine commit `8df97cc7575f47696fa82a179bbe39cd2874b1ca`. That older pin remains compatible only while the new optional fields are omitted. The examples use RFC 5737 Docker pools for documentation, so ordinary validation passes and strict validation deliberately fails until an adopter selects reviewed operational pool CIDRs. + +When release is authorized: + +1. Confirm the release commit passes every required check. +2. Create annotated tag `v1.0.0` at that exact commit. +3. Publish release notes that name the tag object ID, peeled template commit, and reviewed core commit above. +4. Never retarget the tag. Correct a bad release with a new higher tag and mark the old release as superseded. + +The compatibility details are machine-readable in `template-compatibility.json`. No tag or GitHub release exists as part of this preparation change. diff --git a/docs/UPDATING.md b/docs/UPDATING.md index 79556a4..24d3001 100644 --- a/docs/UPDATING.md +++ b/docs/UPDATING.md @@ -29,6 +29,8 @@ the same rewrite-detection baseline; a local-only ref cannot do that. Treat template files as vendored code: update them deliberately, not casually. +The initial `v1.0.0` release is currently prepared, not published. Do not use it until the repository has an immutable tag and GitHub release. If a published release is wrong, leave its tag object untouched, publish a new higher tag, and mark the old release as superseded. Never retarget or recreate an existing release tag. + ## GitHub template repositories have no fork ancestry A repository created with GitHub's "Use this template" button is **not** @@ -160,6 +162,8 @@ pinned contract. Releases that introduce a new `schema_version` are the exception: import the new schema/validator in phase 2 of the two-phase rollout below, after every deployed controller runs the new engine. +Migration programs use `scripts/migrate-v-to-v.py`. Release notes name the exact path. If a release changes `schema_version` but does not ship that named program, stop the update. + ## Dependabot update PRs Keep a `.github/dependabot.yml` in the private repository covering diff --git a/engine-rollout-evidence.json b/engine-rollout-evidence.json new file mode 100644 index 0000000..934e58a --- /dev/null +++ b/engine-rollout-evidence.json @@ -0,0 +1,4 @@ +{ + "schema_version": 1, + "status_reporting_engine_capabilities": {} +} diff --git a/examples/multi-host/fleet.json b/examples/multi-host/fleet.json index e9f071f..2abd3fe 100644 --- a/examples/multi-host/fleet.json +++ b/examples/multi-host/fleet.json @@ -30,6 +30,16 @@ "runner_resources": { "cpu_cores": 4, "memory_mib": 8192 + }, + "docker_network_policy": { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [ + { + "base": "198.51.100.0/24", + "size": 28 + } + ] } }, "sample-ci-remote": { @@ -44,6 +54,16 @@ "runner_resources": { "cpu_cores": 2, "memory_mib": 4096 + }, + "docker_network_policy": { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [ + { + "base": "203.0.113.0/24", + "size": 28 + } + ] } } }, diff --git a/fleet.json b/fleet.json index 198456c..47c1553 100644 --- a/fleet.json +++ b/fleet.json @@ -30,6 +30,16 @@ "runner_resources": { "cpu_cores": 2, "memory_mib": 4096 + }, + "docker_network_policy": { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [ + { + "base": "198.51.100.0/24", + "size": 28 + } + ] } } }, diff --git a/fleet.schema.json b/fleet.schema.json index 606536a..b9ca3a4 100644 --- a/fleet.schema.json +++ b/fleet.schema.json @@ -80,7 +80,39 @@ "engine_ref": {"type": "string", "pattern": "^(?!0{40}$)[0-9a-f]{40}$"}, "min_runners": {"const": 0}, "max_runners": {"type": "integer", "minimum": 1}, - "runner_resources": {"$ref": "#/$defs/runner_resources"} + "runner_resources": {"$ref": "#/$defs/runner_resources"}, + "docker_network_policy": { + "type": "object", + "additionalProperties": false, + "required": ["default_address_pools", "networks_per_runner", "reserve_subnets"], + "properties": { + "networks_per_runner": {"type": "integer", "minimum": 1}, + "reserve_subnets": {"type": "integer", "minimum": 1}, + "default_address_pools": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["base", "size"], + "properties": { + "base": {"type": "string"}, + "size": {"type": "integer", "minimum": 0, "maximum": 29} + } + } + } + } + }, + "status_reporting": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "config_file"], + "properties": { + "enabled": {"type": "boolean"}, + "config_file": {"const": "/etc/ci-fleet/monitoring.env"} + } + } } }, "runner_resources": { diff --git a/scripts/init.py b/scripts/init.py index 3d3ac48..17527e8 100755 --- a/scripts/init.py +++ b/scripts/init.py @@ -38,6 +38,7 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--location", default="primary-site", help="logical location slug; never an address") parser.add_argument("--capacity-budget", type=positive_integer, default=1, help="maximum capacity reserved by the pool") parser.add_argument("--max-runners", type=positive_integer, default=1, help="initial controller maximum") + parser.add_argument("--networks-per-runner", type=positive_integer, default=1, help="reviewed maximum Compose networks per runner") parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") @@ -67,6 +68,9 @@ def main() -> int: fail("--engine-ref must be a nonzero full lowercase commit SHA") if args.max_runners > args.capacity_budget: fail("--max-runners must not exceed --capacity-budget") + runner_networks = args.max_runners * args.networks_per_runner + if runner_networks > 30: + fail("--max-runners multiplied by --networks-per-runner must not exceed 30 so generated Docker networks stay at /29 or larger with reserved capacity") if args.runner_memory_mib < 512: fail("--runner-memory-mib must be at least 512") @@ -115,6 +119,13 @@ def main() -> int: "cpu_cores": args.runner_cpu_cores, "memory_mib": args.runner_memory_mib, }, + "docker_network_policy": { + "networks_per_runner": args.networks_per_runner, + "reserve_subnets": 1, + "default_address_pools": [ + {"base": "198.51.100.0/24", "size": 24 + (runner_networks + 1).bit_length()}, + ], + }, } }, "host_groups": { @@ -164,7 +175,7 @@ def main() -> int: handle.flush() os.fsync(handle.fileno()) subprocess.run( - [str(ROOT / "scripts" / "validate.sh"), "--strict", "--skip-path-scan", "--config", str(temporary)], + [str(ROOT / "scripts" / "validate.sh"), "--skip-path-scan", "--config", str(temporary)], check=True, ) os.chmod(temporary, 0o644) @@ -172,7 +183,7 @@ def main() -> int: finally: temporary.unlink(missing_ok=True) print(f"Initialized {output}") - print("Next: review controller capacity, configure GitHub policy, and keep every secret value outside Git.") + print("Next: replace the RFC 5737 Docker pool, run ./scripts/validate.sh --strict, and keep every secret value outside Git.") return 0 diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py new file mode 100644 index 0000000..f31a0b7 --- /dev/null +++ b/scripts/test_core_compatibility.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Fail when the standalone contract drifts from one reviewed embedded template.""" + +from __future__ import annotations + +import argparse +import json +import sys +import urllib.request +from pathlib import Path + +CORE_COMMIT = "0aed0d7e85e10050028b7d11fb12b84b3619e638" +CORE_PREFIX = "templates/config-repository" +ROOT = Path(__file__).resolve().parents[1] +EXACT_FILES = ( + "fleet.schema.json", + "engine-rollout-evidence.json", + "scripts/init.py", + "scripts/validate.py", +) +CONFIG_FILES = ("fleet.json", "examples/multi-host/fleet.json") + + +def core_bytes(relative: str, core_root: Path | None) -> bytes: + if core_root is not None: + return (core_root / relative).read_bytes() + url = ( + "https://raw.githubusercontent.com/RandomDevelopment/ci-fleet/" + f"{CORE_COMMIT}/{CORE_PREFIX}/{relative}" + ) + with urllib.request.urlopen(url, timeout=30) as response: + return response.read() + + +def normalized_config(raw: bytes) -> dict: + value = json.loads(raw) + # Intentional standalone difference: issue #12 preserves the older reviewed + # example engine pin. Every other embedded configuration value must match. + for controller in value["controllers"].values(): + controller.pop("engine_ref", None) + return value + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--core-root", type=Path, help="local exact-commit template tree for offline tests") + parser.add_argument("--standalone-root", type=Path, default=ROOT) + args = parser.parse_args() + errors: list[str] = [] + for relative in EXACT_FILES: + if (args.standalone_root / relative).read_bytes() != core_bytes(relative, args.core_root): + errors.append(f"{relative}: differs from ci-fleet {CORE_COMMIT}") + for relative in CONFIG_FILES: + standalone = normalized_config((args.standalone_root / relative).read_bytes()) + embedded = normalized_config(core_bytes(relative, args.core_root)) + if standalone != embedded: + errors.append(f"{relative}: differs beyond the allowlisted example engine_ref") + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print(f"OK: standalone contract matches embedded template at {CORE_COMMIT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 42493dc..1c3222f 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -71,6 +71,231 @@ def assert_delivery_engine_contract(self, value: str, accepted: bool) -> None: def test_reference_configuration_is_valid(self) -> None: self.assertEqual(errors_for(reference_config()), []) + def test_schema_defines_optional_controller_capabilities(self) -> None: + controller = contract_schema()["$defs"]["controller"] + self.assertNotIn("docker_network_policy", controller["required"]) + self.assertNotIn("status_reporting", controller["required"]) + network = controller["properties"]["docker_network_policy"] + self.assertEqual( + network["required"], + ["default_address_pools", "networks_per_runner", "reserve_subnets"], + ) + self.assertEqual(network["properties"]["default_address_pools"]["maxItems"], 64) + self.assertEqual( + network["properties"]["default_address_pools"]["items"]["properties"]["size"]["maximum"], + 29, + ) + reporting = controller["properties"]["status_reporting"] + self.assertEqual(reporting["required"], ["enabled", "config_file"]) + self.assertEqual( + reporting["properties"]["config_file"]["const"], + "/etc/ci-fleet/monitoring.env", + ) + + def test_optional_docker_network_policy_enforces_core_semantics(self) -> None: + config = copy.deepcopy(reference_config()) + controller = first_controller(config) + policy = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + controller["docker_network_policy"] = policy + self.assertEqual(errors_for(config), []) + + cases = ( + ({**policy, "extra": True}, "unknown keys: extra"), + (None, "must be an object"), + ({**policy, "default_address_pools": [{"base": "2001:db8::/64", "size": 28}]}, "IPv4"), + ({**policy, "default_address_pools": [{"base": "198.51.100.1/24", "size": 28}]}, "malformed"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/24", "size": 23}]}, "impossible subnet count"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/24", "size": 30}]}, "between 0 and 29"), + ({**policy, "default_address_pools": [ + {"base": "198.51.100.0/24", "size": 28}, + {"base": "198.51.100.128/25", "size": 28}, + ]}, "overlaps"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/24", "size": 29}] * 65}, "must not exceed 64"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/29", "size": 29}]}, "controller Compose network"), + ) + for value, expected in cases: + with self.subTest(expected=expected): + controller["docker_network_policy"] = value + self.assert_rejected(config, expected) + + controller["state"] = "disabled" + controller["max_runners"] = 100 + controller["docker_network_policy"] = { + **policy, + "networks_per_runner": 100, + "default_address_pools": [{"base": "198.51.100.0/28", "size": 29}], + } + self.assertEqual(errors_for(config), []) + self.assert_rejected(config, "reviewed operational Docker pool CIDR", strict=True) + + def test_optional_capabilities_require_previous_integrated_engine_evidence(self) -> None: + previous = copy.deepcopy(reference_config()) + current = copy.deepcopy(previous) + controller_name = next(iter(current["controllers"])) + controller = first_controller(current) + controller["engine_ref"] = "2" * 40 + controller["status_reporting"] = { + "enabled": False, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": "2" * 40, + "status_reporting_config": True, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("current.json", current), + ("evidence.json", evidence), + ("previous-evidence.json", {"schema_version": 1, "status_reporting_engine_capabilities": {}}), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "evidence.json"), + "--previous-rollout-evidence", str(root / "previous-evidence.json"), + "--skip-path-scan", + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("later commit", result.stderr) + + previous = copy.deepcopy(current) + first_controller(previous).pop("status_reporting") + first_controller(previous).pop("docker_network_policy") + (root / "previous.json").write_text(json.dumps(previous), encoding="utf-8") + (root / "previous-evidence.json").write_text(json.dumps(evidence), encoding="utf-8") + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + invalid = copy.deepcopy(current) + first_controller(invalid)["status_reporting"] = None + self.assert_rejected(invalid, "must be an object") + + def test_initializer_emits_sized_network_policy_and_omits_status_reporting(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "fleet.json" + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init.py"), + "--organization", "sample-org", + "--project", "sample-app", + "--engine-ref", "1" * 40, + "--capacity-budget", "2", + "--max-runners", "2", + "--networks-per-runner", "2", + "--output", str(output), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + config = json.loads(output.read_text(encoding="utf-8")) + controller = first_controller(config) + self.assertNotIn("status_reporting", controller) + self.assertEqual(controller["docker_network_policy"]["networks_per_runner"], 2) + pool = controller["docker_network_policy"]["default_address_pools"][0] + self.assertEqual(pool["base"], "198.51.100.0/24") + self.assertGreaterEqual(1 << (pool["size"] - 24), 6) + self.assertEqual(errors_for(config), []) + + def test_reference_examples_publish_optional_network_policy_without_status_reporting(self) -> None: + for path in (ROOT / "fleet.json", ROOT / "examples" / "multi-host" / "fleet.json"): + with self.subTest(path=path): + config = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(config["schema_version"], 3) + for controller in config["controllers"].values(): + self.assertEqual( + controller["docker_network_policy"]["networks_per_runner"], + 1, + ) + self.assertNotIn("status_reporting", controller) + self.assertEqual( + controller["engine_ref"], + "8df97cc7575f47696fa82a179bbe39cd2874b1ca", + ) + self.assertEqual(errors_for(config), []) + + def test_compatibility_record_names_exact_core_and_staged_template_contract(self) -> None: + compatibility = json.loads((ROOT / "template-compatibility.json").read_text(encoding="utf-8")) + self.assertEqual(set(compatibility), { + "schema_version", + "reviewed_core", + "template_contract", + "optional_capabilities", + "example_engine", + "template_release", + }) + self.assertEqual( + compatibility["reviewed_core"], + { + "repository": "RandomDevelopment/ci-fleet", + "commit": "0aed0d7e85e10050028b7d11fb12b84b3619e638", + "embedded_template_path": "templates/config-repository", + }, + ) + self.assertEqual(compatibility["template_contract"]["fleet_schema_version"], 3) + self.assertEqual(compatibility["template_contract"]["validator"], "scripts/validate.py") + self.assertEqual(compatibility["template_contract"]["initializer"], "scripts/init.py") + self.assertEqual( + set(compatibility["optional_capabilities"]), + {"status_reporting", "docker_network_policy"}, + ) + self.assertTrue(all( + value["support"] == "optional-staged" + for value in compatibility["optional_capabilities"].values() + )) + self.assertEqual(compatibility["example_engine"]["commit"], "8df97cc7575f47696fa82a179bbe39cd2874b1ca") + self.assertEqual(compatibility["example_engine"]["pin_status"], "unchanged") + self.assertEqual(compatibility["template_release"]["state"], "prepared-not-published") + + def test_exact_core_drift_check_is_pinned_and_runnable(self) -> None: + checker = ROOT / "scripts" / "test_core_compatibility.py" + text = checker.read_text(encoding="utf-8") + self.assertIn("0aed0d7e85e10050028b7d11fb12b84b3619e638", text) + result = subprocess.run( + [sys.executable, str(checker)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_release_checks_and_artifacts_are_documented_in_ci(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + for command in ( + "python3 scripts/test_core_compatibility.py", + "python3 scripts/test_release_update.py", + "./scripts/validate.sh --strict", + ): + self.assertIn(command, workflow) + readme = (ROOT / "README.md").read_text(encoding="utf-8") + updating = (ROOT / "docs" / "UPDATING.md").read_text(encoding="utf-8") + for artifact in ("template-compatibility.json", "engine-rollout-evidence.json", "docs/RELEASE.md"): + self.assertIn(artifact, readme) + self.assertIn("new higher tag", updating) + self.assertIn("scripts/migrate-v-to-v.py", updating) + def test_multi_host_multi_location_configuration_is_valid(self) -> None: config = json.loads((ROOT / "examples" / "multi-host" / "fleet.json").read_text(encoding="utf-8")) self.assertEqual(errors_for(config), []) diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py new file mode 100644 index 0000000..61eb93b --- /dev/null +++ b/scripts/test_release_update.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +"""Exercise the documented immutable release update in a fictional adopter.""" + +from __future__ import annotations + +import json +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def git(repository: Path, *args: str, capture: bool = False) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *args], + check=True, + stdout=subprocess.PIPE if capture else subprocess.DEVNULL, + text=True, + ) + return result.stdout.strip() if capture else "" + + +def configure(repository: Path) -> None: + git(repository, "config", "user.name", "Release Test") + git(repository, "config", "user.email", "release@example.invalid") + + +class ReleaseUpdateTests(unittest.TestCase): + def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite(self) -> None: + release_notes = (ROOT / "docs" / "RELEASE.md").read_text(encoding="utf-8") + for required in ("v1.0.0", "0aed0d7e85e10050028b7d11fb12b84b3619e638", "not published"): + self.assertIn(required, release_notes) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + template = root / "template" + adopter = root / "adopter" + shutil.copytree( + ROOT, + template, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + git(template, "init", "-q") + configure(template) + git(template, "add", ".") + git(template, "commit", "-qm", "template release") + git(template, "tag", "-a", "v1.0.0", "-m", "v1.0.0") + release_oid = git(template, "rev-parse", "v1.0.0", capture=True) + release_commit = git(template, "rev-parse", "v1.0.0^{commit}", capture=True) + + adopter.mkdir() + git(adopter, "init", "-q") + configure(adopter) + subprocess.run( + [ + str(template / "scripts" / "init.sh"), + "--organization", "fictional-org", + "--project", "fictional-app", + "--engine-ref", "1" * 40, + "--output", str(adopter / "fleet.json"), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + fleet = json.loads((adopter / "fleet.json").read_text(encoding="utf-8")) + next(iter(fleet["controllers"].values()))["docker_network_policy"]["default_address_pools"][0]["base"] = "198.18.0.0/24" + (adopter / "fleet.json").write_text(json.dumps(fleet, indent=2) + "\n", encoding="utf-8") + (adopter / "TEMPLATE_RELEASE").write_text(f"v1.0.0 {release_oid}\n", encoding="utf-8") + git(adopter, "add", ".") + git(adopter, "commit", "-qm", "adopter state") + adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) + expected_fleet = (adopter / "fleet.json").read_bytes() + + git(adopter, "remote", "add", "template", str(template)) + git(adopter, "fetch", "--no-tags", "template", release_commit) + merge = subprocess.run( + ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", "FETCH_HEAD"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self.assertIn(merge.returncode, (0, 1)) + git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", "fleet.json") + self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) + subprocess.run( + [str(adopter / "scripts" / "validate.sh"), "--strict"], + cwd=adopter, + check=True, + stdout=subprocess.DEVNULL, + ) + + (template / "rewrite-marker").write_text("rewrite\n", encoding="utf-8") + git(template, "add", "rewrite-marker") + git(template, "commit", "-qm", "rewrite target") + git(template, "tag", "-f", "-a", "v1.0.0", "-m", "rewritten") + rewritten_oid = git(template, "rev-parse", "v1.0.0", capture=True) + self.assertNotEqual(rewritten_oid, release_oid) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate.py b/scripts/validate.py index 77cd706..4ef51f1 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import ipaddress import json import re import sys @@ -55,6 +56,10 @@ } FORBIDDEN_FILENAMES = re.compile(r"(?:^|/)(?:\.env(?:\..+)?|host\.env|ci-fleet\.env)$|\.(?:key|pem|p12|pfx)$", re.IGNORECASE) FORBIDDEN_DIRECTORIES = {"credentials", "private", "secrets"} +RFC_5737_NETWORKS = tuple( + ipaddress.ip_network(value) for value in ("192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24") +) +MAX_DOCKER_ADDRESS_POOLS = 64 class Validation: @@ -89,6 +94,9 @@ def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: value[key] = child return value + if path.is_symlink(): + validation.errors.append(f"{path}: symlinked JSON files are forbidden") + return None try: return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicate_keys) except FileNotFoundError: @@ -137,6 +145,90 @@ def scan_keys(value: Any, path: str = "$") -> None: break +def validate_docker_network_policy( + policy: Any, + path: str, + max_runners: int, + validation: Validation, + *, + strict: bool = False, +) -> tuple[int, int, list[dict[str, Any]]]: + if not isinstance(policy, dict): + validation.errors.append(f"{path}: must be an object") + return 0, 0, [] + required = {"default_address_pools", "networks_per_runner", "reserve_subnets"} + keys = set(policy) + missing = sorted(required - keys) + unknown = sorted(keys - required) + if missing or unknown: + parts = [] + if missing: + parts.append(f"missing keys: {', '.join(missing)}") + if unknown: + parts.append(f"unknown keys: {', '.join(unknown)}") + validation.errors.append(f"{path}: {'; '.join(parts)}") + return 0, 0, [] + reserve = policy["reserve_subnets"] + if type(reserve) is not int or reserve < 1: + validation.errors.append(f"{path}.reserve_subnets: must be a positive integer") + return 0, 0, [] + networks_per_runner = policy["networks_per_runner"] + if type(networks_per_runner) is not int or networks_per_runner < 1: + validation.errors.append(f"{path}.networks_per_runner: must be a positive integer") + return 0, 0, [] + pools = policy["default_address_pools"] + if type(pools) is not list or not pools: + validation.errors.append(f"{path}.default_address_pools: must be a non-empty list") + return 0, 0, [] + if len(pools) > MAX_DOCKER_ADDRESS_POOLS: + validation.errors.append( + f"{path}.default_address_pools: must not exceed {MAX_DOCKER_ADDRESS_POOLS} pools" + ) + return 0, 0, [] + parsed: list[dict[str, Any]] = [] + for index, pool in enumerate(pools): + pool_path = f"{path}.default_address_pools[{index}]" + if not isinstance(pool, dict) or set(pool) != {"base", "size"}: + validation.errors.append(f"{pool_path}: must contain only base and size") + return 0, 0, [] + base = pool["base"] + size = pool["size"] + if not isinstance(base, str): + validation.errors.append(f"{pool_path}.base: must be a CIDR prefix") + return 0, 0, [] + if type(size) is not int or size < 0 or size > 29: + validation.errors.append(f"{pool_path}.size: must be an IPv4 prefix length between 0 and 29") + return 0, 0, [] + try: + network = ipaddress.ip_network(base, strict=True) + except ValueError: + validation.errors.append(f"{pool_path}.base: malformed address pool IPv4 prefix") + return 0, 0, [] + if network.version != 4: + validation.errors.append(f"{pool_path}.base: malformed address pool IPv4 prefix") + return 0, 0, [] + if strict and any(network.overlaps(documentation) for documentation in RFC_5737_NETWORKS): + validation.errors.append( + f"{pool_path}.base: replace the RFC 5737 documentation address pool with a reviewed operational Docker pool CIDR" + ) + return 0, 0, [] + if size < network.prefixlen: + validation.errors.append(f"{pool_path}.size: impossible subnet count for {base}") + return 0, 0, [] + parsed.append({"base": base, "network": network, "size": size}) + for left, item in enumerate(parsed): + for right in range(left + 1, len(parsed)): + if item["network"].overlaps(parsed[right]["network"]): + validation.errors.append(f"{path}.default_address_pools[{left}].base: overlaps configured pool {right}") + return 0, 0, [] + configured = sum(1 << (item["size"] - item["network"].prefixlen) for item in parsed) + if configured < max_runners * networks_per_runner + reserve + 1: + validation.errors.append( + f"{path}: network capacity cannot satisfy max_runners * networks_per_runner + reserve_subnets + one controller Compose network" + ) + return configured, reserve, parsed + + def scan_forbidden_paths(repo_root: Path, validation: Validation) -> None: for path in repo_root.rglob("*"): try: @@ -264,11 +356,12 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: "min_runners", "max_runners", "runner_resources", + "docker_network_policy", } for name, controller in controllers.items(): path = f"$.controllers.{name}" validation.require(isinstance(name, str) and bool(SLUG.fullmatch(name)), path, "controller ID must be a unique lowercase slug") - if not validation.exact_keys(controller, path, controller_keys): + if not validation.exact_keys(controller, path, controller_keys - {"docker_network_policy"}, {"status_reporting", "docker_network_policy"}): continue pool_name = controller.get("pool") location = controller.get("location") @@ -296,12 +389,26 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: if type(minimum) is int and type(maximum) is int: validation.require(minimum <= maximum, f"{path}.min_runners", "must not exceed max_runners") validation.require(minimum == 0, f"{path}.min_runners", "must be zero because managed prewarmed runners are not supported") + status_reporting = controller.get("status_reporting") + if "status_reporting" in controller and validation.exact_keys(status_reporting, f"{path}.status_reporting", {"enabled", "config_file"}): + validation.require(type(status_reporting.get("enabled")) is bool, f"{path}.status_reporting.enabled", "must be a boolean") + validation.require(status_reporting.get("config_file") == "/etc/ci-fleet/monitoring.env", f"{path}.status_reporting.config_file", "must use the fixed host-local monitoring configuration") resources = controller.get("runner_resources") if validation.exact_keys(resources, f"{path}.runner_resources", {"cpu_cores", "memory_mib"}): cpu = resources.get("cpu_cores") memory = resources.get("memory_mib") validation.require(type(cpu) is int and cpu > 0, f"{path}.runner_resources.cpu_cores", "must be a positive integer") validation.require(type(memory) is int and memory >= 512, f"{path}.runner_resources.memory_mib", "must be at least 512 MiB") + network_policy = controller.get("docker_network_policy") + capacity_maximum = maximum if state != "disabled" and type(maximum) is int and maximum > 0 else 0 + if "docker_network_policy" in controller: + validate_docker_network_policy( + network_policy, + f"{path}.docker_network_policy", + capacity_maximum, + validation, + strict=strict, + ) if isinstance(pool_name, str) and pool_name in pools and state != "disabled" and type(maximum) is int and maximum > 0: reserved_capacity[pool_name] += maximum @@ -400,9 +507,156 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(repository != "example-org/example-app", f"{path}.repository", "replace the example repository before use") +def validate_rollout_evidence(value: Any, validation: Validation) -> dict[str, dict[str, Any]]: + if not validation.exact_keys( + value, + "engine-rollout-evidence.json", + {"schema_version", "status_reporting_engine_capabilities"}, + ): + return {} + validation.require(value.get("schema_version") == 1, "engine-rollout-evidence.json.schema_version", "must equal 1") + capabilities = value.get("status_reporting_engine_capabilities") + if not isinstance(capabilities, dict): + validation.errors.append("engine-rollout-evidence.json.status_reporting_engine_capabilities: must be an object mapping controller IDs to capability evidence") + return {} + valid: dict[str, dict[str, Any]] = {} + for controller, evidence in capabilities.items(): + path = f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}" + controller_valid = bool(SLUG.fullmatch(controller)) + validation.require(controller_valid, path, "controller ID must be a lowercase slug") + if not validation.exact_keys( + evidence, + path, + {"engine_ref", "status_reporting_config", "required_status_reporting"}, + {"docker_network_policy_config"}, + ): + continue + ref = evidence.get("engine_ref") + configured = evidence.get("status_reporting_config") + required = evidence.get("required_status_reporting") + network_policy = evidence.get("docker_network_policy_config") + ref_valid = isinstance(ref, str) and bool(COMMIT_SHA.fullmatch(ref)) and ref != "0" * 40 + validation.require(ref_valid, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") + validation.require(type(configured) is bool, f"{path}.status_reporting_config", "must be a boolean") + validation.require(type(required) is bool, f"{path}.required_status_reporting", "must be a boolean") + if "docker_network_policy_config" in evidence: + validation.require(type(network_policy) is bool, f"{path}.docker_network_policy_config", "must be a boolean") + network_policy_valid = "docker_network_policy_config" not in evidence or type(network_policy) is bool + if controller_valid and ref_valid and type(configured) is bool and type(required) is bool and network_policy_valid: + valid[controller] = { + "engine_ref": ref, + "status_reporting_config": configured, + "required_status_reporting": required, + } + if "docker_network_policy_config" in evidence: + valid[controller]["docker_network_policy_config"] = network_policy + return valid + + +def validate_reporting_evidence( + name: str, + controller: dict[str, Any], + evidence: dict[str, Any], + validation: Validation, +) -> None: + reporting = controller.get("status_reporting") + if not isinstance(reporting, dict): + return + validation.require( + evidence.get("engine_ref") == controller.get("engine_ref") + and evidence.get("status_reporting_config") is True, + f"$.controllers.{name}.status_reporting", + "requires status-reporting configuration capability evidence for this controller and engine_ref", + ) + if reporting.get("enabled") is True: + validation.require( + evidence.get("engine_ref") == controller.get("engine_ref") + and evidence.get("required_status_reporting") is True, + f"$.controllers.{name}.status_reporting.enabled", + "requires required status-reporting rollout evidence for this controller and engine_ref", + ) + + +def validate_transition( + previous: Any, + current: Any, + compatible_engine_refs: dict[str, dict[str, Any]], + validation: Validation, + previous_compatible_engine_refs: dict[str, dict[str, Any]] | None = None, +) -> None: + if not isinstance(previous, dict) or not isinstance(current, dict): + return + old_controllers = previous.get("controllers") + new_controllers = current.get("controllers") + if not isinstance(old_controllers, dict) or not isinstance(new_controllers, dict): + return + for name, new in new_controllers.items(): + old = old_controllers.get(name) + if not isinstance(new, dict): + continue + if name not in old_controllers: + if "status_reporting" in new: + validation.errors.append( + f"$.controllers.{name}.status_reporting: must be omitted from a new controller until its engine rollout is proven" + ) + continue + if not isinstance(old, dict): + continue + current_evidence = compatible_engine_refs.get(name, {}) + previous_evidence_source = ( + compatible_engine_refs + if previous_compatible_engine_refs is None + else previous_compatible_engine_refs + ) + previous_evidence = previous_evidence_source.get(name, {}) + old_reporting = old.get("status_reporting") + new_reporting = new.get("status_reporting") + if "docker_network_policy" not in old and "docker_network_policy" in new: + validation.require( + old.get("engine_ref") == new.get("engine_ref"), + f"$.controllers.{name}.docker_network_policy", + "must be introduced in a later commit after the compatible engine_ref is active", + ) + validation.require( + previous_evidence.get("engine_ref") == new.get("engine_ref") + and previous_evidence.get("docker_network_policy_config") is True, + f"$.controllers.{name}.docker_network_policy", + "requires reviewed evidence from the previous integrated state that this controller activated the same engine_ref with Docker network policy configuration capability", + ) + if "docker_network_policy" in new: + validation.require( + current_evidence.get("engine_ref") == new.get("engine_ref") + and current_evidence.get("docker_network_policy_config") is True, + f"$.controllers.{name}.docker_network_policy", + "requires Docker network policy configuration capability evidence for this controller and engine_ref", + ) + staged_capability_required = ( + "status_reporting" not in old + or ( + isinstance(new_reporting, dict) + and new_reporting.get("enabled") is True + and (not isinstance(old_reporting, dict) or old_reporting.get("enabled") is not True) + ) + ) + evidence = previous_evidence if staged_capability_required else current_evidence + validate_reporting_evidence(name, new, evidence, validation) + if "status_reporting" not in old and "status_reporting" in new: + validation.require( + old.get("engine_ref") == new.get("engine_ref"), + f"$.controllers.{name}.status_reporting", + "must be introduced in a later commit after the compatible engine_ref is active", + ) + validation.require( + evidence.get("engine_ref") == old.get("engine_ref"), + f"$.controllers.{name}.status_reporting", + "requires reviewed rollout evidence for this controller and its already-active compatible engine_ref", + ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", type=Path, default=ROOT / "fleet.json", help="configuration file to validate") + parser.add_argument("--previous-config", type=Path, help="previous integrated configuration for rollout validation") + parser.add_argument("--rollout-evidence", type=Path, help="rollout evidence file (defaults to engine-rollout-evidence.json only for the default fleet.json)") + parser.add_argument("--previous-rollout-evidence", type=Path, help="previous integrated rollout evidence") parser.add_argument("--strict", action="store_true", help="reject unchanged example values") parser.add_argument("--skip-path-scan", action="store_true", help="skip repository path checks (for external fixtures)") parser.add_argument("--tree-paths", type=Path, help="NUL-delimited committed paths to scan instead of the local template tree") @@ -412,13 +666,71 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() validation = Validation() - config = load_json(args.config.resolve(), validation) + config_path = args.config.absolute() + config = load_json(config_path, validation) + evidence_path = args.rollout_evidence.absolute() if args.rollout_evidence else None + if evidence_path is None and config_path == ROOT / "fleet.json": + evidence_path = ROOT / "engine-rollout-evidence.json" + evidence = load_json(evidence_path, validation) if evidence_path is not None else None + current_compatible_engine_refs = ( + validate_rollout_evidence(evidence, validation) + if evidence is not None + else {} + ) schema = load_json(ROOT / "fleet.schema.json", validation) if schema is not None: validation.require(schema.get("$schema") == "https://json-schema.org/draft/2020-12/schema", "fleet.schema.json.$schema", "must use JSON Schema draft 2020-12") if config is not None: scan_secret_material(config, validation) validate_config(config, validation, args.strict) + current_controllers = config.get("controllers", {}) if isinstance(config, dict) else {} + for controller, evidence in current_compatible_engine_refs.items(): + current_controller = current_controllers.get(controller) if isinstance(current_controllers, dict) else None + validation.require( + isinstance(current_controller, dict) and current_controller.get("engine_ref") == evidence["engine_ref"], + f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + "must match the current controller engine_ref; remove stale evidence before changing or removing the controller", + ) + if isinstance(current_controllers, dict): + for controller, value in current_controllers.items(): + if isinstance(value, dict): + validate_reporting_evidence( + controller, + value, + current_compatible_engine_refs.get(controller, {}), + validation, + ) + if args.previous_config is not None: + previous = load_json(args.previous_config.absolute(), validation) + previous_evidence = ( + load_json(args.previous_rollout_evidence.absolute(), validation) + if args.previous_rollout_evidence + else None + ) + previous_compatible_engine_refs = ( + validate_rollout_evidence(previous_evidence, validation) + if previous_evidence is not None + else {} + ) + if previous is not None: + previous_controllers = previous.get("controllers", {}) if isinstance(previous, dict) else {} + for controller, evidence in current_compatible_engine_refs.items(): + if previous_compatible_engine_refs.get(controller) == evidence: + continue + ref = evidence["engine_ref"] + previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None + validation.require( + isinstance(previous_controller, dict) and previous_controller.get("engine_ref") == ref, + f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + f"{ref} must already be selected for this controller in the previous integrated fleet configuration", + ) + validate_transition( + previous, + config, + current_compatible_engine_refs, + validation, + previous_compatible_engine_refs, + ) if args.tree_paths is not None: scan_tree_path_list(args.tree_paths, validation) elif not args.skip_path_scan: diff --git a/template-compatibility.json b/template-compatibility.json new file mode 100644 index 0000000..1903e70 --- /dev/null +++ b/template-compatibility.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "reviewed_core": { + "repository": "RandomDevelopment/ci-fleet", + "commit": "0aed0d7e85e10050028b7d11fb12b84b3619e638", + "embedded_template_path": "templates/config-repository" + }, + "template_contract": { + "fleet_schema_version": 3, + "schema": "fleet.schema.json", + "validator": "scripts/validate.py", + "initializer": "scripts/init.py", + "rollout_evidence": "engine-rollout-evidence.json" + }, + "optional_capabilities": { + "status_reporting": { + "support": "optional-staged", + "evidence_key": "status_reporting_engine_capabilities" + }, + "docker_network_policy": { + "support": "optional-staged", + "evidence_key": "docker_network_policy_config" + } + }, + "example_engine": { + "commit": "8df97cc7575f47696fa82a179bbe39cd2874b1ca", + "pin_status": "unchanged", + "compatibility": "optional-fields-omitted" + }, + "template_release": { + "planned_tag": "v1.0.0", + "state": "prepared-not-published" + } +} From 1395f9dbade9d7f16397b9b7d4459a8ae1ff12a4 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 21:32:22 -0500 Subject: [PATCH 02/15] fix: enforce staged network policy rollout --- .github/workflows/validate.yml | 26 ++++-- AGENTS.md | 2 +- README.md | 6 +- docs/RELEASE.md | 2 +- examples/multi-host/fleet.json | 20 ----- fleet.json | 10 --- scripts/init.py | 13 +-- scripts/test_core_compatibility.py | 70 ++++++++++++++- scripts/test_policy.py | 138 ++++++++++++++++++++++++++--- scripts/test_release_update.py | 56 +++++++++--- scripts/validate.py | 4 + template-compatibility.json | 7 ++ 12 files changed, 271 insertions(+), 83 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 84942c1..553e94a 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -61,14 +61,21 @@ jobs: done - name: Validate reference configurations + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} run: | - ./scripts/validate.sh - ./scripts/validate.sh --config examples/multi-host/fleet.json - if ./scripts/validate.sh --strict 2>"$RUNNER_TEMP/strict-errors"; then - echo 'documentation configuration unexpectedly passed strict validation' >&2 - exit 1 + if [[ -n "$BASE_SHA" && ! "$BASE_SHA" =~ ^0+$ ]] && git cat-file -e "$BASE_SHA:fleet.json" 2>/dev/null; then + git show "$BASE_SHA:fleet.json" >"$RUNNER_TEMP/previous-fleet.json" + args=(--previous-config "$RUNNER_TEMP/previous-fleet.json") + if git cat-file -e "$BASE_SHA:engine-rollout-evidence.json" 2>/dev/null; then + git show "$BASE_SHA:engine-rollout-evidence.json" >"$RUNNER_TEMP/previous-rollout-evidence.json" + args+=(--previous-rollout-evidence "$RUNNER_TEMP/previous-rollout-evidence.json") + fi + ./scripts/validate.sh "${args[@]}" + else + ./scripts/validate.sh fi - grep -F 'reviewed operational Docker pool CIDR' "$RUNNER_TEMP/strict-errors" + ./scripts/validate.sh --config examples/multi-host/fleet.json - name: Prove initialized configurations pass strict policy run: | @@ -98,3 +105,10 @@ jobs: --engine-ref "${engine_ref}" \ --output "${temporary_directory}/fleet.json" test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 + ./scripts/validate.sh --strict --skip-path-scan --config "${temporary_directory}/fleet.json" + python3 -c 'import json,sys; p=sys.argv[1]; value=json.load(open(p, encoding="utf-8")); next(iter(value["controllers"].values()))["docker_network_policy"]={"networks_per_runner": 1, "reserve_subnets": 1, "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}]}; json.dump(value, open(sys.argv[2], "w", encoding="utf-8"), indent=2)' "${temporary_directory}/fleet.json" "${temporary_directory}/rfc-5737.json" + if ./scripts/validate.sh --strict --skip-path-scan --config "${temporary_directory}/rfc-5737.json" 2>"${temporary_directory}/strict-errors"; then + echo 'RFC 5737 fixture unexpectedly passed strict validation' >&2 + exit 1 + fi + grep -F 'reviewed operational Docker pool CIDR' "${temporary_directory}/strict-errors" diff --git a/AGENTS.md b/AGENTS.md index 1332cfb..71ff36d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,7 +15,7 @@ Before committing configuration changes, run: ## Hard rules - Never add real `.env` files, credentials, tokens, private keys, cookies, or passwords. -- Never add addresses, VM IDs, storage identifiers, backup identifiers, SSH details, or rendered runtime configuration. +- Never add addresses, VM IDs, storage identifiers, backup identifiers, SSH details, or rendered runtime configuration. A reviewed Docker `default_address_pools[].base` CIDR is the sole address exception after engine support is staged. - Do not weaken `public_repositories: false` for Docker-socket runner pools. - Infrastructure configuration owns capacity. Application workflows submit all independent jobs and do not use `max-parallel` to model fleet size. - Each GitHub runner group belongs to exactly one runner pool; do not create ambiguous cross-pool assignments. diff --git a/README.md b/README.md index dccaa6a..18cdd79 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This is the public, secret-free starting point for an organization's private `ci-fleet` configuration repository. It records which trusted projects may use each CI pool, the reviewed desired state for controller machines, infrastructure capacity budgets, logical deployment environments, and the standardized commands every project must expose. -It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, VM IDs, storage names, backup identifiers, or `.env` files. +It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, VM IDs, storage names, backup identifiers, or `.env` files. The sole address exception is a reviewed Docker `default_address_pools[].base` CIDR after an adopter has staged engine support. It is fleet capacity policy, not a host or service address. Public examples and fixtures use RFC 5737 ranges only. ```mermaid flowchart LR @@ -52,9 +52,9 @@ The initializer refuses to replace a configured file unless `--force` is explici ## Reviewed compatibility and release state -`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records per-controller capability evidence only after an engine revision has already been integrated. New status-reporting or Docker network-policy fields require that prior-state evidence. +`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, the standalone new-controller safety check, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records per-controller capability evidence only after an engine revision has already been integrated. New status-reporting or Docker network-policy fields require that prior-state evidence. -The examples use RFC 5737 address pools so they remain public documentation. Ordinary validation accepts them. `./scripts/validate.sh --strict` rejects them until a private adopter supplies reviewed operational Docker pool CIDRs. Initial release preparation and exclusions are in `docs/RELEASE.md`; no tag or GitHub release is published by this repository change. +The committed examples and initializer omit both optional fields because their engine capability has not been staged. When a private adopter later stages `docker_network_policy`, it must replace the RFC 5737 review fixture with a reviewed operational pool before strict validation passes. Initial release preparation and exclusions are in `docs/RELEASE.md`; no tag or GitHub release is published by this repository change. ## Schema v3: Git-authored controller desired state diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 0706452..52b4773 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -4,7 +4,7 @@ The `v1.0.0` template release is prepared but not published. Creating the tag or The candidate synchronizes the standalone schema-v3 contract with `templates/config-repository` at reviewed ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It adds optional staged status-reporting and Docker network-policy support. Existing configurations that omit either field remain valid. -The fictional examples retain engine commit `8df97cc7575f47696fa82a179bbe39cd2874b1ca`. That older pin remains compatible only while the new optional fields are omitted. The examples use RFC 5737 Docker pools for documentation, so ordinary validation passes and strict validation deliberately fails until an adopter selects reviewed operational pool CIDRs. +The fictional examples retain engine commit `8df97cc7575f47696fa82a179bbe39cd2874b1ca` and omit both optional fields. Network-policy validation fixtures use RFC 5737 ranges only. A private adopter may add a reviewed operational `default_address_pools[].base` CIDR only after the compatible engine and capability evidence have existed in prior integrated states. When release is authorized: diff --git a/examples/multi-host/fleet.json b/examples/multi-host/fleet.json index 2abd3fe..e9f071f 100644 --- a/examples/multi-host/fleet.json +++ b/examples/multi-host/fleet.json @@ -30,16 +30,6 @@ "runner_resources": { "cpu_cores": 4, "memory_mib": 8192 - }, - "docker_network_policy": { - "networks_per_runner": 1, - "reserve_subnets": 1, - "default_address_pools": [ - { - "base": "198.51.100.0/24", - "size": 28 - } - ] } }, "sample-ci-remote": { @@ -54,16 +44,6 @@ "runner_resources": { "cpu_cores": 2, "memory_mib": 4096 - }, - "docker_network_policy": { - "networks_per_runner": 1, - "reserve_subnets": 1, - "default_address_pools": [ - { - "base": "203.0.113.0/24", - "size": 28 - } - ] } } }, diff --git a/fleet.json b/fleet.json index 47c1553..198456c 100644 --- a/fleet.json +++ b/fleet.json @@ -30,16 +30,6 @@ "runner_resources": { "cpu_cores": 2, "memory_mib": 4096 - }, - "docker_network_policy": { - "networks_per_runner": 1, - "reserve_subnets": 1, - "default_address_pools": [ - { - "base": "198.51.100.0/24", - "size": 28 - } - ] } } }, diff --git a/scripts/init.py b/scripts/init.py index 17527e8..1d0024f 100755 --- a/scripts/init.py +++ b/scripts/init.py @@ -38,7 +38,6 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--location", default="primary-site", help="logical location slug; never an address") parser.add_argument("--capacity-budget", type=positive_integer, default=1, help="maximum capacity reserved by the pool") parser.add_argument("--max-runners", type=positive_integer, default=1, help="initial controller maximum") - parser.add_argument("--networks-per-runner", type=positive_integer, default=1, help="reviewed maximum Compose networks per runner") parser.add_argument("--runner-cpu-cores", type=positive_integer, default=2, help="CPU cores available to each runner") parser.add_argument("--runner-memory-mib", type=positive_integer, default=4096, help="memory available to each runner") parser.add_argument("--engine-ref", required=True, help="reviewed full ci-fleet commit SHA") @@ -68,9 +67,6 @@ def main() -> int: fail("--engine-ref must be a nonzero full lowercase commit SHA") if args.max_runners > args.capacity_budget: fail("--max-runners must not exceed --capacity-budget") - runner_networks = args.max_runners * args.networks_per_runner - if runner_networks > 30: - fail("--max-runners multiplied by --networks-per-runner must not exceed 30 so generated Docker networks stay at /29 or larger with reserved capacity") if args.runner_memory_mib < 512: fail("--runner-memory-mib must be at least 512") @@ -119,13 +115,6 @@ def main() -> int: "cpu_cores": args.runner_cpu_cores, "memory_mib": args.runner_memory_mib, }, - "docker_network_policy": { - "networks_per_runner": args.networks_per_runner, - "reserve_subnets": 1, - "default_address_pools": [ - {"base": "198.51.100.0/24", "size": 24 + (runner_networks + 1).bit_length()}, - ], - }, } }, "host_groups": { @@ -183,7 +172,7 @@ def main() -> int: finally: temporary.unlink(missing_ok=True) print(f"Initialized {output}") - print("Next: replace the RFC 5737 Docker pool, run ./scripts/validate.sh --strict, and keep every secret value outside Git.") + print("Next: review controller capacity, run ./scripts/validate.sh --strict, and keep every secret value outside Git.") return 0 diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index f31a0b7..f1feaf2 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import hashlib import json import sys import urllib.request @@ -12,6 +13,31 @@ CORE_COMMIT = "0aed0d7e85e10050028b7d11fb12b84b3619e638" CORE_PREFIX = "templates/config-repository" ROOT = Path(__file__).resolve().parents[1] +STANDALONE_FILES = { + ".github/dependabot.yml", + ".github/workflows/validate.yml", + ".gitignore", + "AGENTS.md", + "LICENSE", + "README.md", + "SECURITY.md", + "THIRD_PARTY_NOTICES.md", + "docs/RELEASE.md", + "docs/UPDATING.md", + "engine-rollout-evidence.json", + "examples/multi-host/fleet.json", + "fleet.json", + "fleet.schema.json", + "scripts/init.py", + "scripts/init.sh", + "scripts/scan_committed_secrets.py", + "scripts/test_core_compatibility.py", + "scripts/test_policy.py", + "scripts/test_release_update.py", + "scripts/validate.py", + "scripts/validate.sh", + "template-compatibility.json", +} EXACT_FILES = ( "fleet.schema.json", "engine-rollout-evidence.json", @@ -19,6 +45,12 @@ "scripts/validate.py", ) CONFIG_FILES = ("fleet.json", "examples/multi-host/fleet.json") +ALLOWED_STANDALONE_HASHES = { + "examples/multi-host/fleet.json": "ec0104a3891795664288c145a16e94be44eac628f8bf7aacc953ae5b3802e036", + "fleet.json": "23a434eee489bc359589f74e9ec57b07382af61f43c00905b38816df0ef5b3db", + "scripts/init.py": "f058369d22eccac3c9e042272460bcf066e3b1d1af00d07027e0e45489e5bfa3", + "scripts/validate.py": "6cef1cbc918e9026d30561c4f95697f354cd79cb102d7a18151a25aa3eb961eb", +} def core_bytes(relative: str, core_root: Path | None) -> bytes: @@ -47,14 +79,44 @@ def main() -> int: parser.add_argument("--standalone-root", type=Path, default=ROOT) args = parser.parse_args() errors: list[str] = [] + used_allowlist: set[str] = set() + actual_files = { + path.relative_to(args.standalone_root).as_posix() + for path in args.standalone_root.rglob("*") + if path.is_file() + and ".git" not in path.relative_to(args.standalone_root).parts + and "__pycache__" not in path.relative_to(args.standalone_root).parts + and path.suffix != ".pyc" + } + if actual_files != STANDALONE_FILES: + errors.append( + "tree membership differs: " + f"missing={sorted(STANDALONE_FILES - actual_files)} " + f"unexpected={sorted(actual_files - STANDALONE_FILES)}" + ) for relative in EXACT_FILES: - if (args.standalone_root / relative).read_bytes() != core_bytes(relative, args.core_root): - errors.append(f"{relative}: differs from ci-fleet {CORE_COMMIT}") + standalone = (args.standalone_root / relative).read_bytes() + if standalone != core_bytes(relative, args.core_root): + digest = hashlib.sha256(standalone).hexdigest() + if ALLOWED_STANDALONE_HASHES.get(relative) == digest: + used_allowlist.add(relative) + else: + errors.append(f"{relative}: differs from ci-fleet {CORE_COMMIT}; standalone sha256={digest}") for relative in CONFIG_FILES: - standalone = normalized_config((args.standalone_root / relative).read_bytes()) + standalone_raw = (args.standalone_root / relative).read_bytes() + standalone = normalized_config(standalone_raw) embedded = normalized_config(core_bytes(relative, args.core_root)) if standalone != embedded: - errors.append(f"{relative}: differs beyond the allowlisted example engine_ref") + digest = hashlib.sha256(standalone_raw).hexdigest() + if ALLOWED_STANDALONE_HASHES.get(relative) == digest: + used_allowlist.add(relative) + else: + errors.append( + f"{relative}: differs beyond the example engine_ref; standalone sha256={digest}" + ) + unused_allowlist = sorted(ALLOWED_STANDALONE_HASHES.keys() - used_allowlist) + if unused_allowlist: + errors.append(f"unused standalone hash allowlist entries: {unused_allowlist}") if errors: for error in errors: print(f"ERROR: {error}", file=sys.stderr) diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 1c3222f..5274022 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -6,13 +6,14 @@ import copy import json import re +import shutil import subprocess import sys import tempfile import unittest from pathlib import Path -from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config +from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_transition ROOT = Path(__file__).resolve().parents[1] @@ -51,6 +52,29 @@ def first_controller(config: dict) -> dict: return next(iter(config["controllers"].values())) +def documented_artifact_errors(root: Path) -> list[str]: + allowed_external = { + "TEMPLATE_RELEASE", + "scripts/ci/plan.json", + "scripts/ci/run.sh", + "scripts/install-worker-controller.sh", + "scripts/migrate-v-to-v.py", + } + token = re.compile( + r"`(?:python3 )?(?:\./)?((?:scripts|docs|examples)/[A-Za-z0-9_./<>-]+|" + r"fleet(?:\.schema)?\.json|template-compatibility\.json|" + r"engine-rollout-evidence\.json|AGENTS\.md|SECURITY\.md|" + r"THIRD_PARTY_NOTICES\.md|LICENSE|TEMPLATE_RELEASE)" + ) + errors = [] + for document in ("README.md", "docs/UPDATING.md"): + text = (root / document).read_text(encoding="utf-8") + for relative in token.findall(text): + if relative not in allowed_external and not (root / relative).is_file(): + errors.append(f"{document}: documented path does not exist: {relative}") + return sorted(set(errors)) + + class PolicyTests(unittest.TestCase): def assert_rejected(self, config: dict, expected: str, *, strict: bool = False) -> None: errors = errors_for(config, strict=strict) @@ -192,7 +216,28 @@ def test_optional_capabilities_require_previous_integrated_engine_evidence(self) first_controller(invalid)["status_reporting"] = None self.assert_rejected(invalid, "must be an object") - def test_initializer_emits_sized_network_policy_and_omits_status_reporting(self) -> None: + def test_new_controller_cannot_introduce_docker_network_policy(self) -> None: + previous = reference_config() + current = copy.deepcopy(previous) + controller = copy.deepcopy(first_controller(current)) + controller["state"] = "disabled" + controller["scale_set_name"] = "new-ci-02" + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + current["controllers"]["new-ci-02"] = controller + validation = Validation() + + validate_transition(previous, current, {}, validation, {}) + + self.assertTrue(any( + "new-ci-02.docker_network_policy" in error and "new controller" in error + for error in validation.errors + ), validation.errors) + + def test_initializer_omits_optional_capabilities_without_engine_evidence(self) -> None: with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "fleet.json" subprocess.run( @@ -202,9 +247,6 @@ def test_initializer_emits_sized_network_policy_and_omits_status_reporting(self) "--organization", "sample-org", "--project", "sample-app", "--engine-ref", "1" * 40, - "--capacity-budget", "2", - "--max-runners", "2", - "--networks-per-runner", "2", "--output", str(output), ], check=True, @@ -213,22 +255,16 @@ def test_initializer_emits_sized_network_policy_and_omits_status_reporting(self) config = json.loads(output.read_text(encoding="utf-8")) controller = first_controller(config) self.assertNotIn("status_reporting", controller) - self.assertEqual(controller["docker_network_policy"]["networks_per_runner"], 2) - pool = controller["docker_network_policy"]["default_address_pools"][0] - self.assertEqual(pool["base"], "198.51.100.0/24") - self.assertGreaterEqual(1 << (pool["size"] - 24), 6) + self.assertNotIn("docker_network_policy", controller) self.assertEqual(errors_for(config), []) - def test_reference_examples_publish_optional_network_policy_without_status_reporting(self) -> None: + def test_old_engine_examples_omit_optional_capabilities(self) -> None: for path in (ROOT / "fleet.json", ROOT / "examples" / "multi-host" / "fleet.json"): with self.subTest(path=path): config = json.loads(path.read_text(encoding="utf-8")) self.assertEqual(config["schema_version"], 3) for controller in config["controllers"].values(): - self.assertEqual( - controller["docker_network_policy"]["networks_per_runner"], - 1, - ) + self.assertNotIn("docker_network_policy", controller) self.assertNotIn("status_reporting", controller) self.assertEqual( controller["engine_ref"], @@ -243,6 +279,7 @@ def test_compatibility_record_names_exact_core_and_staged_template_contract(self "reviewed_core", "template_contract", "optional_capabilities", + "standalone_safety_deltas", "example_engine", "template_release", }) @@ -265,6 +302,11 @@ def test_compatibility_record_names_exact_core_and_staged_template_contract(self value["support"] == "optional-staged" for value in compatibility["optional_capabilities"].values() )) + self.assertEqual(compatibility["standalone_safety_deltas"], [{ + "path": "scripts/validate.py", + "reviewed_core_gap": "new controllers may introduce docker_network_policy without prior staged capability evidence", + "standalone_enforcement": "new controllers must omit docker_network_policy until engine rollout is proven", + }]) self.assertEqual(compatibility["example_engine"]["commit"], "8df97cc7575f47696fa82a179bbe39cd2874b1ca") self.assertEqual(compatibility["example_engine"]["pin_status"], "unchanged") self.assertEqual(compatibility["template_release"]["state"], "prepared-not-published") @@ -281,6 +323,24 @@ def test_exact_core_drift_check_is_pinned_and_runnable(self) -> None: ) self.assertEqual(result.returncode, 0, result.stderr) + def test_exact_core_drift_check_rejects_required_tree_and_allowlisted_hash_drift(self) -> None: + with tempfile.TemporaryDirectory() as directory: + standalone = Path(directory) / "standalone" + shutil.copytree(ROOT, standalone, ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc")) + (standalone / ".github" / "workflows" / "validate.yml").unlink() + with (standalone / "scripts" / "init.py").open("a", encoding="utf-8") as initializer: + initializer.write("\n# drift\n") + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "test_core_compatibility.py"), "--standalone-root", str(standalone)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("tree membership", result.stderr) + self.assertIn("scripts/init.py: differs", result.stderr) + self.assertIn("standalone sha256=", result.stderr) + def test_release_checks_and_artifacts_are_documented_in_ci(self) -> None: workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") for command in ( @@ -296,6 +356,56 @@ def test_release_checks_and_artifacts_are_documented_in_ci(self) -> None: self.assertIn("new higher tag", updating) self.assertIn("scripts/migrate-v-to-v.py", updating) + def test_documented_artifact_check_is_path_aware(self) -> None: + self.assertEqual(documented_artifact_errors(ROOT), []) + with tempfile.TemporaryDirectory() as directory: + copy = Path(directory) / "standalone" + shutil.copytree(ROOT, copy, ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc")) + with (copy / "README.md").open("a", encoding="utf-8") as readme: + readme.write("\nRun `python3 scripts/missing-required.py`.\n") + self.assertEqual( + documented_artifact_errors(copy), + ["README.md: documented path does not exist: scripts/missing-required.py"], + ) + + def test_pr_workflow_validates_against_base_fleet_and_rollout_evidence(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + for required in ( + 'git show "$BASE_SHA:fleet.json"', + '--previous-config "$RUNNER_TEMP/previous-fleet.json"', + 'git show "$BASE_SHA:engine-rollout-evidence.json"', + '--previous-rollout-evidence "$RUNNER_TEMP/previous-rollout-evidence.json"', + ): + self.assertIn(required, workflow) + + def test_workflow_passes_strict_operational_fixture_and_rejects_rfc_5737_fixture(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertIn('./scripts/validate.sh --strict --skip-path-scan --config "${temporary_directory}/fleet.json"', workflow) + self.assertIn('"base": "198.51.100.0/24"', workflow) + self.assertIn('grep -F \'reviewed operational Docker pool CIDR\'', workflow) + self.assertNotIn("documentation configuration unexpectedly passed strict validation", workflow) + + def test_documentation_names_the_narrow_docker_pool_address_exception(self) -> None: + agents = (ROOT / "AGENTS.md").read_text(encoding="utf-8") + readme = (ROOT / "README.md").read_text(encoding="utf-8") + initializer = (ROOT / "scripts" / "init.py").read_text(encoding="utf-8") + for text in (agents, readme): + self.assertIn("default_address_pools[].base", text) + self.assertIn("sole address exception", text) + self.assertIn("review controller capacity", initializer) + self.assertNotIn("replace the RFC 5737 Docker pool", initializer) + + def test_release_update_test_executes_recorded_tag_rewrite_refusal(self) -> None: + release_test = (ROOT / "scripts" / "test_release_update.py").read_text(encoding="utf-8") + for required in ( + "TEMPLATE_RELEASE", + "refs/tmp/template-tag-check", + "was rewritten upstream; refusing to use it", + "v1.1.0", + ): + self.assertIn(required, release_test) + self.assertIn("assertNotEqual(rewrite.returncode, 0)", release_test) + def test_multi_host_multi_location_configuration_is_valid(self) -> None: config = json.loads((ROOT / "examples" / "multi-host" / "fleet.json").read_text(encoding="utf-8")) self.assertEqual(errors_for(config), []) diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py index 61eb93b..f3d34ce 100644 --- a/scripts/test_release_update.py +++ b/scripts/test_release_update.py @@ -28,6 +28,30 @@ def configure(repository: Path) -> None: git(repository, "config", "user.email", "release@example.invalid") +def fetch_recorded_release(repository: Path, tag: str) -> subprocess.CompletedProcess[str]: + records = dict( + line.split() + for line in (repository / "TEMPLATE_RELEASE").read_text(encoding="utf-8").splitlines() + ) + temporary_ref = "refs/tmp/template-tag-check" + subprocess.run(["git", "-C", str(repository), "update-ref", "-d", temporary_ref], check=False) + fetch = subprocess.run( + ["git", "-C", str(repository), "fetch", "--no-tags", "template", f"refs/tags/{tag}:{temporary_ref}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if fetch.returncode: + return fetch + oid = git(repository, "rev-parse", temporary_ref, capture=True) + if tag in records and records[tag] != oid: + git(repository, "update-ref", "-d", temporary_ref) + return subprocess.CompletedProcess(fetch.args, 1, "", f"template tag {tag} was rewritten upstream; refusing to use it") + commit = git(repository, "rev-parse", f"{temporary_ref}^{{commit}}", capture=True) + git(repository, "update-ref", "-d", temporary_ref) + return subprocess.CompletedProcess(fetch.args, 0, f"{oid} {commit}", "") + + class ReleaseUpdateTests(unittest.TestCase): def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite(self) -> None: release_notes = (ROOT / "docs" / "RELEASE.md").read_text(encoding="utf-8") @@ -46,10 +70,9 @@ def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite git(template, "init", "-q") configure(template) git(template, "add", ".") - git(template, "commit", "-qm", "template release") + git(template, "commit", "-qm", "template v1.0.0") git(template, "tag", "-a", "v1.0.0", "-m", "v1.0.0") release_oid = git(template, "rev-parse", "v1.0.0", capture=True) - release_commit = git(template, "rev-parse", "v1.0.0^{commit}", capture=True) adopter.mkdir() git(adopter, "init", "-q") @@ -65,38 +88,47 @@ def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite check=True, stdout=subprocess.DEVNULL, ) - fleet = json.loads((adopter / "fleet.json").read_text(encoding="utf-8")) - next(iter(fleet["controllers"].values()))["docker_network_policy"]["default_address_pools"][0]["base"] = "198.18.0.0/24" - (adopter / "fleet.json").write_text(json.dumps(fleet, indent=2) + "\n", encoding="utf-8") (adopter / "TEMPLATE_RELEASE").write_text(f"v1.0.0 {release_oid}\n", encoding="utf-8") git(adopter, "add", ".") git(adopter, "commit", "-qm", "adopter state") - adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) - expected_fleet = (adopter / "fleet.json").read_bytes() + (template / "release-marker").write_text("v1.1.0\n", encoding="utf-8") + git(template, "add", "release-marker") + git(template, "commit", "-qm", "template v1.1.0") + git(template, "tag", "-a", "v1.1.0", "-m", "v1.1.0") git(adopter, "remote", "add", "template", str(template)) - git(adopter, "fetch", "--no-tags", "template", release_commit) + release = fetch_recorded_release(adopter, "v1.1.0") + self.assertEqual(release.returncode, 0, release.stderr) + new_oid, release_commit = release.stdout.split() + adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) + expected_fleet = (adopter / "fleet.json").read_bytes() merge = subprocess.run( - ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", "FETCH_HEAD"], + ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", release_commit], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) self.assertIn(merge.returncode, (0, 1)) git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", "fleet.json") self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) + git(adopter, "add", ".") subprocess.run( [str(adopter / "scripts" / "validate.sh"), "--strict"], cwd=adopter, check=True, stdout=subprocess.DEVNULL, ) + with (adopter / "TEMPLATE_RELEASE").open("a", encoding="utf-8") as record: + record.write(f"v1.1.0 {new_oid}\n") + git(adopter, "add", "TEMPLATE_RELEASE") + git(adopter, "commit", "-qm", "update to template v1.1.0") (template / "rewrite-marker").write_text("rewrite\n", encoding="utf-8") git(template, "add", "rewrite-marker") git(template, "commit", "-qm", "rewrite target") - git(template, "tag", "-f", "-a", "v1.0.0", "-m", "rewritten") - rewritten_oid = git(template, "rev-parse", "v1.0.0", capture=True) - self.assertNotEqual(rewritten_oid, release_oid) + git(template, "tag", "-f", "-a", "v1.1.0", "-m", "rewritten") + rewrite = fetch_recorded_release(adopter, "v1.1.0") + self.assertNotEqual(rewrite.returncode, 0) + self.assertIn("was rewritten upstream; refusing to use it", rewrite.stderr) if __name__ == "__main__": diff --git a/scripts/validate.py b/scripts/validate.py index 4ef51f1..8b2f5b6 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -599,6 +599,10 @@ def validate_transition( validation.errors.append( f"$.controllers.{name}.status_reporting: must be omitted from a new controller until its engine rollout is proven" ) + if "docker_network_policy" in new: + validation.errors.append( + f"$.controllers.{name}.docker_network_policy: must be omitted from a new controller until its engine rollout is proven" + ) continue if not isinstance(old, dict): continue diff --git a/template-compatibility.json b/template-compatibility.json index 1903e70..d9da5ef 100644 --- a/template-compatibility.json +++ b/template-compatibility.json @@ -22,6 +22,13 @@ "evidence_key": "docker_network_policy_config" } }, + "standalone_safety_deltas": [ + { + "path": "scripts/validate.py", + "reviewed_core_gap": "new controllers may introduce docker_network_policy without prior staged capability evidence", + "standalone_enforcement": "new controllers must omit docker_network_policy until engine rollout is proven" + } + ], "example_engine": { "commit": "8df97cc7575f47696fa82a179bbe39cd2874b1ca", "pin_status": "unchanged", From 6ba28dd111e8c958d4b343903b0a9a673d2e608b Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:13:48 -0500 Subject: [PATCH 03/15] fix: enforce adopter-safe release validation --- scripts/test_core_compatibility.py | 15 ++++++++++- scripts/test_policy.py | 42 ++++++++++++++++++++++++++++++ scripts/test_release_update.py | 30 +++++++++++++++++++++ scripts/validate.py | 14 +++++----- 4 files changed, 93 insertions(+), 8 deletions(-) diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index f1feaf2..2b86ce7 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -6,6 +6,7 @@ import argparse import hashlib import json +import subprocess import sys import urllib.request from pathlib import Path @@ -49,7 +50,7 @@ "examples/multi-host/fleet.json": "ec0104a3891795664288c145a16e94be44eac628f8bf7aacc953ae5b3802e036", "fleet.json": "23a434eee489bc359589f74e9ec57b07382af61f43c00905b38816df0ef5b3db", "scripts/init.py": "f058369d22eccac3c9e042272460bcf066e3b1d1af00d07027e0e45489e5bfa3", - "scripts/validate.py": "6cef1cbc918e9026d30561c4f95697f354cd79cb102d7a18151a25aa3eb961eb", + "scripts/validate.py": "cf48e62469cb1e60d066b43b3ebba8b2fbff22c4e7e7e0758403688561350bda", } @@ -78,6 +79,18 @@ def main() -> int: parser.add_argument("--core-root", type=Path, help="local exact-commit template tree for offline tests") parser.add_argument("--standalone-root", type=Path, default=ROOT) args = parser.parse_args() + if args.standalone_root == ROOT: + origin = subprocess.run( + ["git", "-C", str(ROOT), "remote", "get-url", "origin"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + if origin.returncode or not origin.stdout.strip().removesuffix(".git").endswith( + "RandomDevelopment/ci-fleet-config-template" + ): + print("OK: exact core compatibility is upstream-only; skipped for derived repository") + return 0 errors: list[str] = [] used_allowlist: set[str] = set() actual_files = { diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 5274022..7379c6c 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -156,6 +156,48 @@ def test_optional_docker_network_policy_enforces_core_semantics(self) -> None: self.assertEqual(errors_for(config), []) self.assert_rejected(config, "reviewed operational Docker pool CIDR", strict=True) + def test_standalone_docker_network_policy_requires_current_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + config_path = Path(directory) / "fleet.json" + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init.py"), + "--organization", "sample-org", + "--project", "sample-app", + "--engine-ref", "1" * 40, + "--output", str(config_path), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + config = json.loads(config_path.read_text(encoding="utf-8")) + first_controller(config)["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "10.255.255.0/24", "size": 28}], + } + config_path.write_text(json.dumps(config), encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--strict", + "--skip-path-scan", + "--config", str(config_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "requires Docker network policy configuration capability evidence for this controller and engine_ref", + result.stderr, + ) + def test_optional_capabilities_require_previous_integrated_engine_evidence(self) -> None: previous = copy.deepcopy(reference_config()) current = copy.deepcopy(previous) diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py index f3d34ce..426c7d2 100644 --- a/scripts/test_release_update.py +++ b/scripts/test_release_update.py @@ -6,6 +6,7 @@ import json import shutil import subprocess +import sys import tempfile import unittest from pathlib import Path @@ -53,6 +54,35 @@ def fetch_recorded_release(repository: Path, tag: str) -> subprocess.CompletedPr class ReleaseUpdateTests(unittest.TestCase): + def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> None: + with tempfile.TemporaryDirectory() as directory: + derived = Path(directory) / "derived" + shutil.copytree( + ROOT, + derived, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + config_path = derived / "fleet.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["organization"]["slug"] = "derived-org" + project = next(iter(config["projects"].values())) + project["repository"] = "derived-org/derived-app" + config["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + (derived / "TEMPLATE_RELEASE").write_text("v1.0.0 " + "1" * 40 + "\n", encoding="utf-8") + git(derived, "init", "-q") + git(derived, "remote", "add", "origin", "https://github.com/derived-org/derived-config.git") + + result = subprocess.run( + [sys.executable, str(derived / "scripts" / "test_policy.py")], + cwd=derived, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite(self) -> None: release_notes = (ROOT / "docs" / "RELEASE.md").read_text(encoding="utf-8") for required in ("v1.0.0", "0aed0d7e85e10050028b7d11fb12b84b3619e638", "not published"): diff --git a/scripts/validate.py b/scripts/validate.py index 8b2f5b6..65d6b1a 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -559,6 +559,13 @@ def validate_reporting_evidence( evidence: dict[str, Any], validation: Validation, ) -> None: + if "docker_network_policy" in controller: + validation.require( + evidence.get("engine_ref") == controller.get("engine_ref") + and evidence.get("docker_network_policy_config") is True, + f"$.controllers.{name}.docker_network_policy", + "requires Docker network policy configuration capability evidence for this controller and engine_ref", + ) reporting = controller.get("status_reporting") if not isinstance(reporting, dict): return @@ -627,13 +634,6 @@ def validate_transition( f"$.controllers.{name}.docker_network_policy", "requires reviewed evidence from the previous integrated state that this controller activated the same engine_ref with Docker network policy configuration capability", ) - if "docker_network_policy" in new: - validation.require( - current_evidence.get("engine_ref") == new.get("engine_ref") - and current_evidence.get("docker_network_policy_config") is True, - f"$.controllers.{name}.docker_network_policy", - "requires Docker network policy configuration capability evidence for this controller and engine_ref", - ) staged_capability_required = ( "status_reporting" not in old or ( From fbd7ab56bbb5de3eab3f2d5ebb35792b702363e3 Mon Sep 17 00:00:00 2001 From: Nick <1572453+Nickfost@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:37:02 -0500 Subject: [PATCH 04/15] fix: keep adopter policy tests portable --- scripts/init.py | 2 +- scripts/test_core_compatibility.py | 29 +++++++++++--------- scripts/test_policy.py | 43 +++++++++++++++++++++++++++--- scripts/test_release_update.py | 7 +++++ 4 files changed, 63 insertions(+), 18 deletions(-) diff --git a/scripts/init.py b/scripts/init.py index 1d0024f..1f85ca3 100755 --- a/scripts/init.py +++ b/scripts/init.py @@ -164,7 +164,7 @@ def main() -> int: handle.flush() os.fsync(handle.fileno()) subprocess.run( - [str(ROOT / "scripts" / "validate.sh"), "--skip-path-scan", "--config", str(temporary)], + [str(ROOT / "scripts" / "validate.sh"), "--strict", "--skip-path-scan", "--config", str(temporary)], check=True, ) os.chmod(temporary, 0o644) diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index 2b86ce7..201fc74 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -49,7 +49,7 @@ ALLOWED_STANDALONE_HASHES = { "examples/multi-host/fleet.json": "ec0104a3891795664288c145a16e94be44eac628f8bf7aacc953ae5b3802e036", "fleet.json": "23a434eee489bc359589f74e9ec57b07382af61f43c00905b38816df0ef5b3db", - "scripts/init.py": "f058369d22eccac3c9e042272460bcf066e3b1d1af00d07027e0e45489e5bfa3", + "scripts/init.py": "bb47f464763be1324f13af6bd64b3017e085103f2e0c1637b7d66c33e01b3c46", "scripts/validate.py": "cf48e62469cb1e60d066b43b3ebba8b2fbff22c4e7e7e0758403688561350bda", } @@ -74,23 +74,26 @@ def normalized_config(raw: bytes) -> dict: return value +def is_upstream_repository(root: Path) -> bool: + origin = subprocess.run( + ["git", "-C", str(root), "remote", "get-url", "origin"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + return not origin.returncode and origin.stdout.strip().removesuffix(".git").endswith( + "RandomDevelopment/ci-fleet-config-template" + ) + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--core-root", type=Path, help="local exact-commit template tree for offline tests") parser.add_argument("--standalone-root", type=Path, default=ROOT) args = parser.parse_args() - if args.standalone_root == ROOT: - origin = subprocess.run( - ["git", "-C", str(ROOT), "remote", "get-url", "origin"], - stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL, - text=True, - ) - if origin.returncode or not origin.stdout.strip().removesuffix(".git").endswith( - "RandomDevelopment/ci-fleet-config-template" - ): - print("OK: exact core compatibility is upstream-only; skipped for derived repository") - return 0 + if args.standalone_root == ROOT and not is_upstream_repository(ROOT): + print("OK: exact core compatibility is upstream-only; skipped for derived repository") + return 0 errors: list[str] = [] used_allowlist: set[str] = set() actual_files = { diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 7379c6c..33b344f 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -13,6 +13,7 @@ import unittest from pathlib import Path +from test_core_compatibility import is_upstream_repository from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_transition @@ -200,6 +201,7 @@ def test_standalone_docker_network_policy_requires_current_evidence(self) -> Non def test_optional_capabilities_require_previous_integrated_engine_evidence(self) -> None: previous = copy.deepcopy(reference_config()) + first_controller(previous)["engine_ref"] = "1" * 40 current = copy.deepcopy(previous) controller_name = next(iter(current["controllers"])) controller = first_controller(current) @@ -295,12 +297,44 @@ def test_initializer_omits_optional_capabilities_without_engine_evidence(self) - stdout=subprocess.DEVNULL, ) config = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(output.stat().st_mode & 0o777, 0o644) + subprocess.run( + [ + str(ROOT / "scripts" / "validate.sh"), + "--strict", + "--skip-path-scan", + "--config", str(output), + ], + check=True, + stdout=subprocess.DEVNULL, + ) controller = first_controller(config) self.assertNotIn("status_reporting", controller) self.assertNotIn("docker_network_policy", controller) self.assertEqual(errors_for(config), []) + def test_initializer_rejects_strict_placeholders_without_output(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "fleet.json" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init.py"), + "--organization", "example-org", + "--project", "example-app", + "--engine-ref", "2" * 40, + "--output", str(output), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(output.exists()) + def test_old_engine_examples_omit_optional_capabilities(self) -> None: + upstream = is_upstream_repository(ROOT) for path in (ROOT / "fleet.json", ROOT / "examples" / "multi-host" / "fleet.json"): with self.subTest(path=path): config = json.loads(path.read_text(encoding="utf-8")) @@ -308,10 +342,11 @@ def test_old_engine_examples_omit_optional_capabilities(self) -> None: for controller in config["controllers"].values(): self.assertNotIn("docker_network_policy", controller) self.assertNotIn("status_reporting", controller) - self.assertEqual( - controller["engine_ref"], - "8df97cc7575f47696fa82a179bbe39cd2874b1ca", - ) + if upstream: + self.assertEqual( + controller["engine_ref"], + "8df97cc7575f47696fa82a179bbe39cd2874b1ca", + ) self.assertEqual(errors_for(config), []) def test_compatibility_record_names_exact_core_and_staged_template_contract(self) -> None: diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py index 426c7d2..2fd01be 100644 --- a/scripts/test_release_update.py +++ b/scripts/test_release_update.py @@ -68,7 +68,14 @@ def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> No project = next(iter(config["projects"].values())) project["repository"] = "derived-org/derived-app" config["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + for controller in config["controllers"].values(): + controller["engine_ref"] = "2" * 40 config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + multi_host_path = derived / "examples" / "multi-host" / "fleet.json" + multi_host = json.loads(multi_host_path.read_text(encoding="utf-8")) + for controller in multi_host["controllers"].values(): + controller["engine_ref"] = "2" * 40 + multi_host_path.write_text(json.dumps(multi_host, indent=2) + "\n", encoding="utf-8") (derived / "TEMPLATE_RELEASE").write_text("v1.0.0 " + "1" * 40 + "\n", encoding="utf-8") git(derived, "init", "-q") git(derived, "remote", "add", "origin", "https://github.com/derived-org/derived-config.git") From 8220e7c4b13f0082908483bda4b4bcfcf2a91848 Mon Sep 17 00:00:00 2001 From: Nick <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:18:34 -0500 Subject: [PATCH 05/15] fix: close rollout validation gaps --- .github/workflows/validate.yml | 3 +- docs/UPDATING.md | 18 ++-- scripts/test_core_compatibility.py | 44 +++++++-- scripts/test_policy.py | 150 ++++++++++++++++++++++++++++- scripts/test_release_update.py | 52 +++++++++- scripts/validate.py | 48 ++++++--- 6 files changed, 275 insertions(+), 40 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 553e94a..8221e82 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -73,7 +73,8 @@ jobs: fi ./scripts/validate.sh "${args[@]}" else - ./scripts/validate.sh + printf '{"controllers":{}}\n' >"$RUNNER_TEMP/previous-fleet.json" + ./scripts/validate.sh --previous-config "$RUNNER_TEMP/previous-fleet.json" fi ./scripts/validate.sh --config examples/multi-host/fleet.json diff --git a/docs/UPDATING.md b/docs/UPDATING.md index 24d3001..0ac35c4 100644 --- a/docs/UPDATING.md +++ b/docs/UPDATING.md @@ -39,17 +39,17 @@ a fork: it has no git ancestry link to the template, so Updating is an explicit operation: 1. Start from a clean tree — no uncommitted or unstaged changes, - especially to `fleet.json`; the procedure restores `fleet.json` from - the recorded pre-merge commit and would silently discard an - uncommitted edit. Then add the template as a remote and fetch its - tags into that remote's tracking namespace. `--no-tags` prevents Git + especially to `fleet.json` or `engine-rollout-evidence.json`; the + procedure restores both files from the recorded pre-merge commit and + would silently discard an uncommitted edit. Then add the template as a + remote and fetch its tags into that remote's tracking namespace. `--no-tags` prevents Git from also creating adopter-visible tags. A retargeted upstream tag updates a `refs/remotes/*` ref silently, so verify the reviewed object ID yourself before using any previously fetched tag ref: ```bash # Hard stop on any uncommitted state; a dirty tree would be silently - # overwritten by the fleet.json restore below. + # overwritten by the adopter-owned state restore below. test -z "$(git status --porcelain)" || { echo "clean the tree first" >&2; exit 1; } # one-time setup; on later runs require the existing remote to be the # template, not an unrelated remote that happens to share the name: @@ -106,15 +106,15 @@ Updating is an explicit operation: conflict: ```bash - git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json + git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json engine-rollout-evidence.json git status --short ``` - If the release keeps the same `schema_version`, prove `fleet.json` - still has no staged change: + If the release keeps the same `schema_version`, prove the adopter-owned + configuration and rollout evidence still have no staged changes: ```bash - git diff --cached --exit-code -- fleet.json + git diff --cached --exit-code -- fleet.json engine-rollout-evidence.json ``` 3. Review the complete staged result — including any changes the merge diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index 201fc74..5a7c728 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -47,10 +47,22 @@ ) CONFIG_FILES = ("fleet.json", "examples/multi-host/fleet.json") ALLOWED_STANDALONE_HASHES = { - "examples/multi-host/fleet.json": "ec0104a3891795664288c145a16e94be44eac628f8bf7aacc953ae5b3802e036", - "fleet.json": "23a434eee489bc359589f74e9ec57b07382af61f43c00905b38816df0ef5b3db", - "scripts/init.py": "bb47f464763be1324f13af6bd64b3017e085103f2e0c1637b7d66c33e01b3c46", - "scripts/validate.py": "cf48e62469cb1e60d066b43b3ebba8b2fbff22c4e7e7e0758403688561350bda", + "examples/multi-host/fleet.json": ( + "ec0104a3891795664288c145a16e94be44eac628f8bf7aacc953ae5b3802e036", + "04908089d4d1f5f483a815ef9ef859ae053ba572d1e6d1c898866a677bc226de", + ), + "fleet.json": ( + "23a434eee489bc359589f74e9ec57b07382af61f43c00905b38816df0ef5b3db", + "12ce4b9f7146f80e5eaaa693cbeb0802f9f2f7aeaf1dcece5c0e86009d1b2e1c", + ), + "scripts/init.py": ( + "bb47f464763be1324f13af6bd64b3017e085103f2e0c1637b7d66c33e01b3c46", + "0acf5b340317d3b9f97ae7c0686d7c6e0513e2084f9aeb295bc3d96b90fbe5dd", + ), + "scripts/validate.py": ( + "cbb401924961138e1b575d5143b622de391dd21f5647bbcc07ae4f94ba6f0f35", + "3c202840ce00ae31568d3ac2137cd1acdebf5ff9fa8807b9823e4310c9e39568", + ), } @@ -112,20 +124,32 @@ def main() -> int: ) for relative in EXACT_FILES: standalone = (args.standalone_root / relative).read_bytes() - if standalone != core_bytes(relative, args.core_root): + embedded = core_bytes(relative, args.core_root) + if standalone != embedded: digest = hashlib.sha256(standalone).hexdigest() - if ALLOWED_STANDALONE_HASHES.get(relative) == digest: - used_allowlist.add(relative) + allowed = ALLOWED_STANDALONE_HASHES.get(relative) + if allowed and allowed[0] == digest: + core_digest = hashlib.sha256(embedded).hexdigest() + if allowed[1] == core_digest: + used_allowlist.add(relative) + else: + errors.append(f"{relative}: reviewed core bytes differ; core sha256={core_digest}") else: errors.append(f"{relative}: differs from ci-fleet {CORE_COMMIT}; standalone sha256={digest}") for relative in CONFIG_FILES: standalone_raw = (args.standalone_root / relative).read_bytes() standalone = normalized_config(standalone_raw) - embedded = normalized_config(core_bytes(relative, args.core_root)) + embedded_raw = core_bytes(relative, args.core_root) + embedded = normalized_config(embedded_raw) if standalone != embedded: digest = hashlib.sha256(standalone_raw).hexdigest() - if ALLOWED_STANDALONE_HASHES.get(relative) == digest: - used_allowlist.add(relative) + allowed = ALLOWED_STANDALONE_HASHES.get(relative) + if allowed and allowed[0] == digest: + core_digest = hashlib.sha256(embedded_raw).hexdigest() + if allowed[1] == core_digest: + used_allowlist.add(relative) + else: + errors.append(f"{relative}: reviewed core bytes differ; core sha256={core_digest}") else: errors.append( f"{relative}: differs beyond the example engine_ref; standalone sha256={digest}" diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 33b344f..2c77bbc 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -13,7 +13,7 @@ import unittest from pathlib import Path -from test_core_compatibility import is_upstream_repository +from test_core_compatibility import CONFIG_FILES, EXACT_FILES, core_bytes, is_upstream_repository from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_transition @@ -201,7 +201,10 @@ def test_standalone_docker_network_policy_requires_current_evidence(self) -> Non def test_optional_capabilities_require_previous_integrated_engine_evidence(self) -> None: previous = copy.deepcopy(reference_config()) - first_controller(previous)["engine_ref"] = "1" * 40 + previous_controller = first_controller(previous) + previous_controller["engine_ref"] = "1" * 40 + previous_controller.pop("status_reporting", None) + previous_controller.pop("docker_network_policy", None) current = copy.deepcopy(previous) controller_name = next(iter(current["controllers"])) controller = first_controller(current) @@ -260,6 +263,65 @@ def test_optional_capabilities_require_previous_integrated_engine_evidence(self) first_controller(invalid)["status_reporting"] = None self.assert_rejected(invalid, "must be an object") + def test_engine_upgrade_retains_previously_proven_optional_capabilities(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + previous_controller = first_controller(previous) + previous_controller["engine_ref"] = "1" * 40 + previous_controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + previous_controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + current = copy.deepcopy(previous) + first_controller(current)["engine_ref"] = "2" * 40 + + def evidence(engine_ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + }, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("current.json", current), + ("previous-evidence.json", evidence("1" * 40)), + ("current-evidence.json", evidence("2" * 40)), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "current-evidence.json"), + "--previous-rollout-evidence", str(root / "previous-evidence.json"), + "--skip-path-scan", + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + (root / "previous-evidence.json").write_text( + json.dumps({"schema_version": 1, "status_reporting_engine_capabilities": {}}), + encoding="utf-8", + ) + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("requires Docker network policy configuration capability evidence", result.stderr) + def test_new_controller_cannot_introduce_docker_network_policy(self) -> None: previous = reference_config() current = copy.deepcopy(previous) @@ -340,9 +402,9 @@ def test_old_engine_examples_omit_optional_capabilities(self) -> None: config = json.loads(path.read_text(encoding="utf-8")) self.assertEqual(config["schema_version"], 3) for controller in config["controllers"].values(): - self.assertNotIn("docker_network_policy", controller) - self.assertNotIn("status_reporting", controller) if upstream: + self.assertNotIn("docker_network_policy", controller) + self.assertNotIn("status_reporting", controller) self.assertEqual( controller["engine_ref"], "8df97cc7575f47696fa82a179bbe39cd2874b1ca", @@ -401,6 +463,8 @@ def test_exact_core_drift_check_is_pinned_and_runnable(self) -> None: self.assertEqual(result.returncode, 0, result.stderr) def test_exact_core_drift_check_rejects_required_tree_and_allowlisted_hash_drift(self) -> None: + if not is_upstream_repository(ROOT): + self.skipTest("exact core compatibility is upstream-only") with tempfile.TemporaryDirectory() as directory: standalone = Path(directory) / "standalone" shutil.copytree(ROOT, standalone, ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc")) @@ -418,6 +482,27 @@ def test_exact_core_drift_check_rejects_required_tree_and_allowlisted_hash_drift self.assertIn("scripts/init.py: differs", result.stderr) self.assertIn("standalone sha256=", result.stderr) + def test_exact_core_drift_check_rejects_corrupt_allowlisted_core_bytes(self) -> None: + if not is_upstream_repository(ROOT): + self.skipTest("exact core compatibility is upstream-only") + with tempfile.TemporaryDirectory() as directory: + core = Path(directory) / "core" + for relative in (*EXACT_FILES, *CONFIG_FILES): + path = core / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(core_bytes(relative, None)) + with (core / "scripts" / "validate.py").open("ab") as validator: + validator.write(b"\n# corrupt core bytes\n") + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "test_core_compatibility.py"), "--core-root", str(core)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("scripts/validate.py: reviewed core bytes differ", result.stderr) + self.assertIn("core sha256=", result.stderr) + def test_release_checks_and_artifacts_are_documented_in_ci(self) -> None: workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") for command in ( @@ -455,6 +540,61 @@ def test_pr_workflow_validates_against_base_fleet_and_rollout_evidence(self) -> ): self.assertIn(required, workflow) + def test_pr_workflow_rejects_first_state_optional_capabilities(self) -> None: + current = copy.deepcopy(reference_config()) + controller_name = next(iter(current["controllers"])) + controller = first_controller(current) + controller["engine_ref"] = "2" * 40 + controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": "2" * 40, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("current.json", current), + ("previous.json", {"controllers": {}}), + ("evidence.json", evidence), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "evidence.json"), + "--skip-path-scan", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be omitted from a new controller", result.stderr) + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertRegex( + workflow, + r'else\n\s+printf \'\{\"controllers\":\{\}\}\\n\' >\"\$RUNNER_TEMP/previous-fleet\.json\"\n' + r'\s+\./scripts/validate\.sh --previous-config \"\$RUNNER_TEMP/previous-fleet\.json\"\n\s+fi', + ) + def test_workflow_passes_strict_operational_fixture_and_rejects_rfc_5737_fixture(self) -> None: workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") self.assertIn('./scripts/validate.sh --strict --skip-path-scan --config "${temporary_directory}/fleet.json"', workflow) @@ -828,7 +968,7 @@ def test_updating_guide_preserves_adopter_state_before_commit(self) -> None: "git fetch --no-tags template", 'refs/tags/$NEW_TAG:refs/tmp/template-tag-check', 'ADOPTER_HEAD="$(git rev-parse HEAD)"', - 'git restore --source="$ADOPTER_HEAD"', + 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json engine-rollout-evidence.json', "run the now-reviewed target", "./scripts/validate.sh --strict", "git commit", diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py index 2fd01be..68db109 100644 --- a/scripts/test_release_update.py +++ b/scripts/test_release_update.py @@ -4,6 +4,7 @@ from __future__ import annotations import json +import os import shutil import subprocess import sys @@ -68,9 +69,34 @@ def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> No project = next(iter(config["projects"].values())) project["repository"] = "derived-org/derived-app" config["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + controller_name, controller = next(iter(config["controllers"].items())) for controller in config["controllers"].values(): controller["engine_ref"] = "2" * 40 + controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "10.255.255.0/24", "size": 28}], + } config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": "2" * 40, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + }, + } + (derived / "engine-rollout-evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", + encoding="utf-8", + ) multi_host_path = derived / "examples" / "multi-host" / "fleet.json" multi_host = json.loads(multi_host_path.read_text(encoding="utf-8")) for controller in multi_host["controllers"].values(): @@ -83,6 +109,13 @@ def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> No result = subprocess.run( [sys.executable, str(derived / "scripts" / "test_policy.py")], cwd=derived, + env={ + **os.environ, + "HTTPS_PROXY": "http://127.0.0.1:9", + "https_proxy": "http://127.0.0.1:9", + "NO_PROXY": "", + "no_proxy": "", + }, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, @@ -126,6 +159,21 @@ def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite stdout=subprocess.DEVNULL, ) (adopter / "TEMPLATE_RELEASE").write_text(f"v1.0.0 {release_oid}\n", encoding="utf-8") + staged_evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + "ci-01": { + "engine_ref": "1" * 40, + "status_reporting_config": True, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + }, + } + (adopter / "engine-rollout-evidence.json").write_text( + json.dumps(staged_evidence, indent=2) + "\n", + encoding="utf-8", + ) git(adopter, "add", ".") git(adopter, "commit", "-qm", "adopter state") @@ -139,14 +187,16 @@ def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite new_oid, release_commit = release.stdout.split() adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) expected_fleet = (adopter / "fleet.json").read_bytes() + expected_evidence = (adopter / "engine-rollout-evidence.json").read_bytes() merge = subprocess.run( ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", release_commit], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) self.assertIn(merge.returncode, (0, 1)) - git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", "fleet.json") + git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", "fleet.json", "engine-rollout-evidence.json") self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) + self.assertEqual((adopter / "engine-rollout-evidence.json").read_bytes(), expected_evidence) git(adopter, "add", ".") subprocess.run( [str(adopter / "scripts" / "validate.sh"), "--strict"], diff --git a/scripts/validate.py b/scripts/validate.py index 65d6b1a..eb47a4d 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -558,7 +558,8 @@ def validate_reporting_evidence( controller: dict[str, Any], evidence: dict[str, Any], validation: Validation, -) -> None: +) -> bool: + errors_before = len(validation.errors) if "docker_network_policy" in controller: validation.require( evidence.get("engine_ref") == controller.get("engine_ref") @@ -567,21 +568,21 @@ def validate_reporting_evidence( "requires Docker network policy configuration capability evidence for this controller and engine_ref", ) reporting = controller.get("status_reporting") - if not isinstance(reporting, dict): - return - validation.require( - evidence.get("engine_ref") == controller.get("engine_ref") - and evidence.get("status_reporting_config") is True, - f"$.controllers.{name}.status_reporting", - "requires status-reporting configuration capability evidence for this controller and engine_ref", - ) - if reporting.get("enabled") is True: + if isinstance(reporting, dict): validation.require( evidence.get("engine_ref") == controller.get("engine_ref") - and evidence.get("required_status_reporting") is True, - f"$.controllers.{name}.status_reporting.enabled", - "requires required status-reporting rollout evidence for this controller and engine_ref", + and evidence.get("status_reporting_config") is True, + f"$.controllers.{name}.status_reporting", + "requires status-reporting configuration capability evidence for this controller and engine_ref", ) + if reporting.get("enabled") is True: + validation.require( + evidence.get("engine_ref") == controller.get("engine_ref") + and evidence.get("required_status_reporting") is True, + f"$.controllers.{name}.status_reporting.enabled", + "requires required status-reporting rollout evidence for this controller and engine_ref", + ) + return len(validation.errors) == errors_before def validate_transition( @@ -723,8 +724,27 @@ def main() -> int: continue ref = evidence["engine_ref"] previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None + current_controller = current_controllers.get(controller) if isinstance(current_controllers, dict) else None + retained_capability_is_proven = ( + isinstance(previous_controller, dict) + and isinstance(current_controller, dict) + and any( + capability in previous_controller and capability in current_controller + for capability in ("docker_network_policy", "status_reporting") + ) + and validate_reporting_evidence( + controller, + previous_controller, + previous_compatible_engine_refs.get(controller, {}), + validation, + ) + ) validation.require( - isinstance(previous_controller, dict) and previous_controller.get("engine_ref") == ref, + ( + isinstance(previous_controller, dict) + and previous_controller.get("engine_ref") == ref + ) + or retained_capability_is_proven, f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", f"{ref} must already be selected for this controller in the previous integrated fleet configuration", ) From 0bbf14fcd953538f940531d8e75ebc1c5ee7448c Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:52:18 -0500 Subject: [PATCH 06/15] fix: validate derived release state strictly --- .github/workflows/validate.yml | 9 +++- docs/UPDATING.md | 26 +++++++--- scripts/test_policy.py | 7 ++- scripts/test_release_update.py | 89 +++++++++++++++++++++++++++++++++- 4 files changed, 118 insertions(+), 13 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 8221e82..64f6de4 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -64,9 +64,13 @@ jobs: env: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} run: | + args=() + if [[ "$GITHUB_REPOSITORY" != RandomDevelopment/ci-fleet-config-template ]]; then + args+=(--strict) + fi if [[ -n "$BASE_SHA" && ! "$BASE_SHA" =~ ^0+$ ]] && git cat-file -e "$BASE_SHA:fleet.json" 2>/dev/null; then git show "$BASE_SHA:fleet.json" >"$RUNNER_TEMP/previous-fleet.json" - args=(--previous-config "$RUNNER_TEMP/previous-fleet.json") + args+=(--previous-config "$RUNNER_TEMP/previous-fleet.json") if git cat-file -e "$BASE_SHA:engine-rollout-evidence.json" 2>/dev/null; then git show "$BASE_SHA:engine-rollout-evidence.json" >"$RUNNER_TEMP/previous-rollout-evidence.json" args+=(--previous-rollout-evidence "$RUNNER_TEMP/previous-rollout-evidence.json") @@ -74,7 +78,8 @@ jobs: ./scripts/validate.sh "${args[@]}" else printf '{"controllers":{}}\n' >"$RUNNER_TEMP/previous-fleet.json" - ./scripts/validate.sh --previous-config "$RUNNER_TEMP/previous-fleet.json" + args+=(--previous-config "$RUNNER_TEMP/previous-fleet.json") + ./scripts/validate.sh "${args[@]}" fi ./scripts/validate.sh --config examples/multi-host/fleet.json diff --git a/docs/UPDATING.md b/docs/UPDATING.md index 0ac35c4..89138da 100644 --- a/docs/UPDATING.md +++ b/docs/UPDATING.md @@ -40,10 +40,11 @@ Updating is an explicit operation: 1. Start from a clean tree — no uncommitted or unstaged changes, especially to `fleet.json` or `engine-rollout-evidence.json`; the - procedure restores both files from the recorded pre-merge commit and - would silently discard an uncommitted edit. Then add the template as a - remote and fetch its tags into that remote's tracking namespace. `--no-tags` prevents Git - from also creating adopter-visible tags. A retargeted upstream tag + procedure restores adopter-owned files from the recorded pre-merge + commit and would silently discard an uncommitted edit. Then add the + template as a remote and fetch its tags into that remote's tracking + namespace. `--no-tags` prevents Git from also creating adopter-visible + tags. A retargeted upstream tag updates a `refs/remotes/*` ref silently, so verify the reviewed object ID yourself before using any previously fetched tag ref: @@ -102,11 +103,15 @@ Updating is an explicit operation: If Git reports conflicts, leave the merge in progress and continue. Whether or not it conflicted, restore the adopter-owned configuration - from the recorded pre-merge commit, then resolve and stage every other - conflict: + from the recorded pre-merge commit. Preserve rollout evidence from that + commit when it exists; otherwise keep the evidence introduced by the new + template release. Then resolve and stage every other conflict: ```bash - git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json engine-rollout-evidence.json + git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json + if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then + git restore --source="$ADOPTER_HEAD" --staged --worktree -- engine-rollout-evidence.json + fi git status --short ``` @@ -114,7 +119,12 @@ Updating is an explicit operation: configuration and rollout evidence still have no staged changes: ```bash - git diff --cached --exit-code -- fleet.json engine-rollout-evidence.json + git diff --cached --exit-code "$ADOPTER_HEAD" -- fleet.json + if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then + git diff --cached --exit-code "$ADOPTER_HEAD" -- engine-rollout-evidence.json + else + git diff --cached --exit-code "$MERGE_SOURCE" -- engine-rollout-evidence.json + fi ``` 3. Review the complete staged result — including any changes the merge diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 2c77bbc..31b1313 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -592,7 +592,8 @@ def test_pr_workflow_rejects_first_state_optional_capabilities(self) -> None: self.assertRegex( workflow, r'else\n\s+printf \'\{\"controllers\":\{\}\}\\n\' >\"\$RUNNER_TEMP/previous-fleet\.json\"\n' - r'\s+\./scripts/validate\.sh --previous-config \"\$RUNNER_TEMP/previous-fleet\.json\"\n\s+fi', + r'\s+args\+=\(--previous-config \"\$RUNNER_TEMP/previous-fleet\.json\"\)\n' + r'\s+\./scripts/validate\.sh \"\$\{args\[@\]\}\"\n\s+fi', ) def test_workflow_passes_strict_operational_fixture_and_rejects_rfc_5737_fixture(self) -> None: @@ -968,7 +969,9 @@ def test_updating_guide_preserves_adopter_state_before_commit(self) -> None: "git fetch --no-tags template", 'refs/tags/$NEW_TAG:refs/tmp/template-tag-check', 'ADOPTER_HEAD="$(git rev-parse HEAD)"', - 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json engine-rollout-evidence.json', + 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json', + 'if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then', + 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- engine-rollout-evidence.json', "run the now-reviewed target", "./scripts/validate.sh --strict", "git commit", diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py index 68db109..18a6b6f 100644 --- a/scripts/test_release_update.py +++ b/scripts/test_release_update.py @@ -55,7 +55,7 @@ def fetch_recorded_release(repository: Path, tag: str) -> subprocess.CompletedPr class ReleaseUpdateTests(unittest.TestCase): - def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> None: + def test_derived_repository_workflow_uses_strict_policy_without_exact_core_identity(self) -> None: with tempfile.TemporaryDirectory() as directory: derived = Path(directory) / "derived" shutil.copytree( @@ -64,6 +64,19 @@ def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> No ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), ) config_path = derived / "fleet.json" + unchanged = subprocess.run( + [str(derived / "scripts" / "validate.sh"), "--strict"], + cwd=derived, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(unchanged.returncode, 0) + self.assertTrue( + "replace the example organization" in unchanged.stderr + or "reviewed operational Docker pool CIDR" in unchanged.stderr, + unchanged.stderr, + ) config = json.loads(config_path.read_text(encoding="utf-8")) config["organization"]["slug"] = "derived-org" project = next(iter(config["projects"].values())) @@ -106,6 +119,13 @@ def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> No git(derived, "init", "-q") git(derived, "remote", "add", "origin", "https://github.com/derived-org/derived-config.git") + subprocess.run( + [str(derived / "scripts" / "validate.sh"), "--strict"], + cwd=derived, + check=True, + stdout=subprocess.DEVNULL, + ) + result = subprocess.run( [sys.executable, str(derived / "scripts" / "test_policy.py")], cwd=derived, @@ -122,6 +142,72 @@ def test_derived_repository_policy_suite_ignores_exact_core_identity(self) -> No ) self.assertEqual(result.returncode, 0, result.stderr) + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertIn('if [[ "$GITHUB_REPOSITORY" != RandomDevelopment/ci-fleet-config-template ]]; then', workflow) + self.assertIn("args+=(--strict)", workflow) + + def test_legacy_adopter_keeps_new_template_rollout_evidence(self) -> None: + guide = (ROOT / "docs" / "UPDATING.md").read_text(encoding="utf-8") + conditional_restore = 'if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then' in guide + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + template = root / "template" + adopter = root / "adopter" + shutil.copytree( + ROOT, + template, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + git(template, "init", "-q") + configure(template) + git(template, "add", ".") + git(template, "commit", "-qm", "template release") + release_commit = git(template, "rev-parse", "HEAD", capture=True) + expected_evidence = (template / "engine-rollout-evidence.json").read_bytes() + + adopter.mkdir() + git(adopter, "init", "-q") + configure(adopter) + subprocess.run( + [ + str(template / "scripts" / "init.sh"), + "--organization", "legacy-org", + "--project", "legacy-app", + "--engine-ref", "1" * 40, + "--output", str(adopter / "fleet.json"), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + git(adopter, "add", "fleet.json") + git(adopter, "commit", "-qm", "legacy adopter state") + adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) + expected_fleet = (adopter / "fleet.json").read_bytes() + git(adopter, "remote", "add", "template", str(template)) + git(adopter, "fetch", "template", release_commit) + + merge = subprocess.run( + ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", release_commit], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self.assertIn(merge.returncode, (0, 1)) + restore_paths = ["fleet.json"] + if not conditional_restore: + restore_paths.append("engine-rollout-evidence.json") + git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", *restore_paths) + + self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) + self.assertEqual((adopter / "engine-rollout-evidence.json").read_bytes(), expected_evidence) + git(adopter, "diff", "--cached", "--exit-code", adopter_head, "--", "fleet.json") + git(adopter, "diff", "--cached", "--exit-code", release_commit, "--", "engine-rollout-evidence.json") + git(adopter, "add", ".") + subprocess.run( + [str(adopter / "scripts" / "validate.sh"), "--strict"], + cwd=adopter, + check=True, + stdout=subprocess.DEVNULL, + ) def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite(self) -> None: release_notes = (ROOT / "docs" / "RELEASE.md").read_text(encoding="utf-8") @@ -197,6 +283,7 @@ def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", "fleet.json", "engine-rollout-evidence.json") self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) self.assertEqual((adopter / "engine-rollout-evidence.json").read_bytes(), expected_evidence) + git(adopter, "diff", "--cached", "--exit-code", adopter_head, "--", "fleet.json", "engine-rollout-evidence.json") git(adopter, "add", ".") subprocess.run( [str(adopter / "scripts" / "validate.sh"), "--strict"], From d503adff42d376399b0fa7eaf1dc0149bfd7c286 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 10:48:32 -0500 Subject: [PATCH 07/15] fix: stage evidence before engine upgrades --- AGENTS.md | 1 + README.md | 5 +- scripts/test_core_compatibility.py | 2 +- scripts/test_policy.py | 154 ++++++++++++++++++++++++++++- scripts/validate.py | 106 ++++++++++++-------- 5 files changed, 224 insertions(+), 44 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 71ff36d..93e7b63 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,6 +21,7 @@ Before committing configuration changes, run: - Each GitHub runner group belongs to exactly one runner pool; do not create ambiguous cross-pool assignments. - The sum of active and drained controller maxima must not exceed the pool capacity budget. - Controller engine revisions and reusable workflows must be pinned to full reviewed commit SHAs. +- Upgrade an engine with active optional capabilities in two integrated commits: first keep engine A active and stage reviewed engine B evidence in `next_engine`; then change `engine_ref` to engine B and promote the matching `next_engine` record to active evidence. - Production environments must require approval and must not deploy automatically. - CI runner hosts and application deployment hosts are separate roles. - Image promotion uses immutable digests; do not rebuild separately for production. diff --git a/README.md b/README.md index 18cdd79..1faf92d 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,10 @@ The initializer refuses to replace a configured file unless `--force` is explici ## Reviewed compatibility and release state -`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, the standalone new-controller safety check, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records per-controller capability evidence only after an engine revision has already been integrated. New status-reporting or Docker network-policy fields require that prior-state evidence. +`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, the standalone new-controller safety check, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records per-controller capability evidence. New status-reporting or Docker network-policy fields require prior-state evidence. Upgrade an engine with active optional capabilities in exactly two integrated commits: + +1. Keep `fleet.json` and the active evidence on engine A, and add reviewed engine B capability evidence as `next_engine`. +2. Change `engine_ref` to engine B and promote that matching `next_engine` record to the active evidence. The committed examples and initializer omit both optional fields because their engine capability has not been staged. When a private adopter later stages `docker_network_policy`, it must replace the RFC 5737 review fixture with a reviewed operational pool before strict validation passes. Initial release preparation and exclusions are in `docs/RELEASE.md`; no tag or GitHub release is published by this repository change. diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index 5a7c728..c3cb800 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -60,7 +60,7 @@ "0acf5b340317d3b9f97ae7c0686d7c6e0513e2084f9aeb295bc3d96b90fbe5dd", ), "scripts/validate.py": ( - "cbb401924961138e1b575d5143b622de391dd21f5647bbcc07ae4f94ba6f0f35", + "a86a7fc4d9cc6aaf5098c5ef37808f5d5dc76a0e49b7a7d967f23043f4ecd122", "3c202840ce00ae31568d3ac2137cd1acdebf5ff9fa8807b9823e4310c9e39568", ), } diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 31b1313..3c9e81e 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -263,7 +263,7 @@ def test_optional_capabilities_require_previous_integrated_engine_evidence(self) first_controller(invalid)["status_reporting"] = None self.assert_rejected(invalid, "must be an object") - def test_engine_upgrade_retains_previously_proven_optional_capabilities(self) -> None: + def test_engine_upgrade_requires_previously_staged_next_engine_evidence(self) -> None: previous = copy.deepcopy(reference_config()) controller_name = next(iter(previous["controllers"])) previous_controller = first_controller(previous) @@ -312,7 +312,11 @@ def evidence(engine_ref: str) -> dict: "--skip-path-scan", ] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) - self.assertEqual(result.returncode, 0, result.stderr) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "requires next-engine capability evidence from the previous integrated rollout evidence", + result.stderr, + ) (root / "previous-evidence.json").write_text( json.dumps({"schema_version": 1, "status_reporting_engine_capabilities": {}}), @@ -322,6 +326,152 @@ def evidence(engine_ref: str) -> dict: self.assertNotEqual(result.returncode, 0) self.assertIn("requires Docker network policy configuration capability evidence", result.stderr) + def test_staged_next_engine_evidence_permits_later_promotion(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + controller = first_controller(previous) + controller["engine_ref"] = "1" * 40 + controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + promoted = copy.deepcopy(previous) + first_controller(promoted)["engine_ref"] = "2" * 40 + + active = { + "engine_ref": "1" * 40, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + staged_evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + **active, + "next_engine": {**active, "engine_ref": "2" * 40}, + } + }, + } + promoted_evidence = copy.deepcopy(staged_evidence) + promoted_evidence["status_reporting_engine_capabilities"][controller_name] = { + **active, + "engine_ref": "2" * 40, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("promoted.json", promoted), + ("active-evidence.json", { + "schema_version": 1, + "status_reporting_engine_capabilities": {controller_name: active}, + }), + ("staged-evidence.json", staged_evidence), + ("promoted-evidence.json", promoted_evidence), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + base_command = [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--skip-path-scan", + ] + staging = subprocess.run( + [ + *base_command, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "staged-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(staging.returncode, 0, staging.stderr) + promotion = subprocess.run( + [ + *base_command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "staged-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(promotion.returncode, 0, promotion.stderr) + + invalid = copy.deepcopy(staged_evidence) + invalid["status_reporting_engine_capabilities"][controller_name]["engine_ref"] = "9" * 40 + (root / "staged-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") + nonintegrated = subprocess.run( + [ + *base_command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "staged-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(nonintegrated.returncode, 0) + self.assertIn("must match the previous integrated controller engine_ref", nonintegrated.stderr) + + invalid = copy.deepcopy(staged_evidence) + invalid["status_reporting_engine_capabilities"][controller_name]["next_engine"]["engine_ref"] = "3" * 40 + (root / "staged-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") + mismatch = subprocess.run( + [ + *base_command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "staged-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(mismatch.returncode, 0) + self.assertIn( + "requires next-engine capability evidence from the previous integrated rollout evidence", + mismatch.stderr, + ) + + for capability, expected in ( + ("status_reporting_config", "requires status-reporting configuration capability evidence"), + ("required_status_reporting", "requires required status-reporting rollout evidence"), + ("docker_network_policy_config", "requires Docker network policy configuration capability evidence"), + ): + with self.subTest(capability=capability): + invalid = copy.deepcopy(staged_evidence) + invalid["status_reporting_engine_capabilities"][controller_name]["next_engine"][capability] = False + (root / "staged-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") + insufficient = subprocess.run( + [ + *base_command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "staged-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(insufficient.returncode, 0) + self.assertIn(expected, insufficient.stderr) + def test_new_controller_cannot_introduce_docker_network_policy(self) -> None: previous = reference_config() current = copy.deepcopy(previous) diff --git a/scripts/validate.py b/scripts/validate.py index eb47a4d..21ff8dc 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -508,6 +508,40 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: def validate_rollout_evidence(value: Any, validation: Validation) -> dict[str, dict[str, Any]]: + def validate_engine_evidence( + evidence: Any, + path: str, + optional: set[str], + ) -> dict[str, Any] | None: + if not validation.exact_keys( + evidence, + path, + {"engine_ref", "status_reporting_config", "required_status_reporting"}, + optional, + ): + return None + ref = evidence.get("engine_ref") + configured = evidence.get("status_reporting_config") + required = evidence.get("required_status_reporting") + network_policy = evidence.get("docker_network_policy_config") + ref_valid = isinstance(ref, str) and bool(COMMIT_SHA.fullmatch(ref)) and ref != "0" * 40 + validation.require(ref_valid, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") + validation.require(type(configured) is bool, f"{path}.status_reporting_config", "must be a boolean") + validation.require(type(required) is bool, f"{path}.required_status_reporting", "must be a boolean") + if "docker_network_policy_config" in evidence: + validation.require(type(network_policy) is bool, f"{path}.docker_network_policy_config", "must be a boolean") + network_policy_valid = "docker_network_policy_config" not in evidence or type(network_policy) is bool + if not (ref_valid and type(configured) is bool and type(required) is bool and network_policy_valid): + return None + parsed = { + "engine_ref": ref, + "status_reporting_config": configured, + "required_status_reporting": required, + } + if "docker_network_policy_config" in evidence: + parsed["docker_network_policy_config"] = network_policy + return parsed + if not validation.exact_keys( value, "engine-rollout-evidence.json", @@ -524,32 +558,21 @@ def validate_rollout_evidence(value: Any, validation: Validation) -> dict[str, d path = f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}" controller_valid = bool(SLUG.fullmatch(controller)) validation.require(controller_valid, path, "controller ID must be a lowercase slug") - if not validation.exact_keys( + parsed = validate_engine_evidence( evidence, path, - {"engine_ref", "status_reporting_config", "required_status_reporting"}, - {"docker_network_policy_config"}, - ): - continue - ref = evidence.get("engine_ref") - configured = evidence.get("status_reporting_config") - required = evidence.get("required_status_reporting") - network_policy = evidence.get("docker_network_policy_config") - ref_valid = isinstance(ref, str) and bool(COMMIT_SHA.fullmatch(ref)) and ref != "0" * 40 - validation.require(ref_valid, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") - validation.require(type(configured) is bool, f"{path}.status_reporting_config", "must be a boolean") - validation.require(type(required) is bool, f"{path}.required_status_reporting", "must be a boolean") - if "docker_network_policy_config" in evidence: - validation.require(type(network_policy) is bool, f"{path}.docker_network_policy_config", "must be a boolean") - network_policy_valid = "docker_network_policy_config" not in evidence or type(network_policy) is bool - if controller_valid and ref_valid and type(configured) is bool and type(required) is bool and network_policy_valid: - valid[controller] = { - "engine_ref": ref, - "status_reporting_config": configured, - "required_status_reporting": required, - } - if "docker_network_policy_config" in evidence: - valid[controller]["docker_network_policy_config"] = network_policy + {"docker_network_policy_config", "next_engine"}, + ) + if parsed is not None and "next_engine" in evidence: + next_engine = validate_engine_evidence( + evidence["next_engine"], + f"{path}.next_engine", + {"docker_network_policy_config"}, + ) + if next_engine is not None: + parsed["next_engine"] = next_engine + if controller_valid and parsed is not None: + valid[controller] = parsed return valid @@ -623,6 +646,16 @@ def validate_transition( previous_evidence = previous_evidence_source.get(name, {}) old_reporting = old.get("status_reporting") new_reporting = new.get("status_reporting") + if old.get("engine_ref") != new.get("engine_ref") and ( + "docker_network_policy" in new or "status_reporting" in new + ): + next_engine = previous_evidence.get("next_engine", {}) + validation.require( + next_engine.get("engine_ref") == new.get("engine_ref"), + f"$.controllers.{name}.engine_ref", + "requires next-engine capability evidence from the previous integrated rollout evidence", + ) + validate_reporting_evidence(name, new, next_engine, validation) if "docker_network_policy" not in old and "docker_network_policy" in new: validation.require( old.get("engine_ref") == new.get("engine_ref"), @@ -719,32 +752,25 @@ def main() -> int: ) if previous is not None: previous_controllers = previous.get("controllers", {}) if isinstance(previous, dict) else {} + for controller, evidence in previous_compatible_engine_refs.items(): + previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None + validation.require( + isinstance(previous_controller, dict) + and previous_controller.get("engine_ref") == evidence["engine_ref"], + f"previous engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + "must match the previous integrated controller engine_ref", + ) for controller, evidence in current_compatible_engine_refs.items(): if previous_compatible_engine_refs.get(controller) == evidence: continue ref = evidence["engine_ref"] previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None - current_controller = current_controllers.get(controller) if isinstance(current_controllers, dict) else None - retained_capability_is_proven = ( - isinstance(previous_controller, dict) - and isinstance(current_controller, dict) - and any( - capability in previous_controller and capability in current_controller - for capability in ("docker_network_policy", "status_reporting") - ) - and validate_reporting_evidence( - controller, - previous_controller, - previous_compatible_engine_refs.get(controller, {}), - validation, - ) - ) validation.require( ( isinstance(previous_controller, dict) and previous_controller.get("engine_ref") == ref ) - or retained_capability_is_proven, + or previous_compatible_engine_refs.get(controller, {}).get("next_engine") == evidence, f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", f"{ref} must already be selected for this controller in the previous integrated fleet configuration", ) From a74be487fef885ff26de6935ac73b18919f36ac7 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:36:52 -0500 Subject: [PATCH 08/15] fix: keep staged engine evidence core-compatible --- .github/workflows/validate.yml | 7 + AGENTS.md | 2 +- README.md | 9 +- docs/UPDATING.md | 9 +- scripts/test_core_compatibility.py | 118 +++++++++++- scripts/test_policy.py | 279 ++++++++++++++++++++++------- scripts/test_release_update.py | 35 +++- scripts/validate.py | 98 +++++++--- 8 files changed, 460 insertions(+), 97 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 64f6de4..44de413 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -68,6 +68,9 @@ jobs: if [[ "$GITHUB_REPOSITORY" != RandomDevelopment/ci-fleet-config-template ]]; then args+=(--strict) fi + if [[ -f next-engine-rollout-evidence.json ]]; then + args+=(--next-engine-rollout-evidence next-engine-rollout-evidence.json) + fi if [[ -n "$BASE_SHA" && ! "$BASE_SHA" =~ ^0+$ ]] && git cat-file -e "$BASE_SHA:fleet.json" 2>/dev/null; then git show "$BASE_SHA:fleet.json" >"$RUNNER_TEMP/previous-fleet.json" args+=(--previous-config "$RUNNER_TEMP/previous-fleet.json") @@ -75,6 +78,10 @@ jobs: git show "$BASE_SHA:engine-rollout-evidence.json" >"$RUNNER_TEMP/previous-rollout-evidence.json" args+=(--previous-rollout-evidence "$RUNNER_TEMP/previous-rollout-evidence.json") fi + if git cat-file -e "$BASE_SHA:next-engine-rollout-evidence.json" 2>/dev/null; then + git show "$BASE_SHA:next-engine-rollout-evidence.json" >"$RUNNER_TEMP/previous-next-engine-rollout-evidence.json" + args+=(--previous-next-engine-rollout-evidence "$RUNNER_TEMP/previous-next-engine-rollout-evidence.json") + fi ./scripts/validate.sh "${args[@]}" else printf '{"controllers":{}}\n' >"$RUNNER_TEMP/previous-fleet.json" diff --git a/AGENTS.md b/AGENTS.md index 93e7b63..6188f14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,7 @@ Before committing configuration changes, run: - Each GitHub runner group belongs to exactly one runner pool; do not create ambiguous cross-pool assignments. - The sum of active and drained controller maxima must not exceed the pool capacity budget. - Controller engine revisions and reusable workflows must be pinned to full reviewed commit SHAs. -- Upgrade an engine with active optional capabilities in two integrated commits: first keep engine A active and stage reviewed engine B evidence in `next_engine`; then change `engine_ref` to engine B and promote the matching `next_engine` record to active evidence. +- Upgrade an engine with active optional capabilities in two integrated commits: first keep engine A active and stage reviewed engine B evidence in optional adopter-owned `next-engine-rollout-evidence.json`; then change `engine_ref` to engine B, promote the matching sidecar record to active evidence, and remove the sidecar when empty. - Production environments must require approval and must not deploy automatically. - CI runner hosts and application deployment hosts are separate roles. - Image promotion uses immutable digests; do not rebuild separately for production. diff --git a/README.md b/README.md index 1faf92d..1245e45 100644 --- a/README.md +++ b/README.md @@ -52,10 +52,10 @@ The initializer refuses to replace a configured file unless `--force` is explici ## Reviewed compatibility and release state -`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, the standalone new-controller safety check, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records per-controller capability evidence. New status-reporting or Docker network-policy fields require prior-state evidence. Upgrade an engine with active optional capabilities in exactly two integrated commits: +`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, the standalone new-controller safety check, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records active per-controller capability evidence in the shape consumed by that pinned core. New status-reporting or Docker network-policy fields require prior-state evidence. Upgrade an engine with active optional capabilities in exactly two integrated commits: -1. Keep `fleet.json` and the active evidence on engine A, and add reviewed engine B capability evidence as `next_engine`. -2. Change `engine_ref` to engine B and promote that matching `next_engine` record to the active evidence. +1. Keep `fleet.json` and `engine-rollout-evidence.json` on engine A. Add reviewed engine B capability evidence to optional adopter-owned `next-engine-rollout-evidence.json`, using the same schema-version-1 record shape. +2. Change `engine_ref` to engine B, promote the matching sidecar record to `engine-rollout-evidence.json`, and remove the sidecar when it has no remaining staged records. The committed examples and initializer omit both optional fields because their engine capability has not been staged. When a private adopter later stages `docker_network_policy`, it must replace the RFC 5737 review fixture with a reviewed operational pool before strict validation passes. Initial release preparation and exclusions are in `docs/RELEASE.md`; no tag or GitHub release is published by this repository change. @@ -153,7 +153,8 @@ Adding workers reduces wall-clock time only while independent shards remain queu | `fleet.json` | Fictional, valid schema-v3 configuration with one controller | | `fleet.schema.json` | JSON Schema draft 2020-12 editor contract | | `template-compatibility.json` | Exact reviewed core/template compatibility and prepared release state | -| `engine-rollout-evidence.json` | Staged optional-capability evidence, empty in the public template | +| `engine-rollout-evidence.json` | Active optional-capability evidence, empty in the public template | +| `next-engine-rollout-evidence.json` | Optional adopter-owned next-engine evidence sidecar; not committed by this public template | | `scripts/init.sh` | Safe first-project and first-controller initializer | | `scripts/validate.sh` | Structural, relational, capacity, and secret-boundary validation | | `scripts/test_policy.py` | Regression tests proving unsafe configurations fail closed | diff --git a/docs/UPDATING.md b/docs/UPDATING.md index 89138da..7d02058 100644 --- a/docs/UPDATING.md +++ b/docs/UPDATING.md @@ -39,7 +39,8 @@ a fork: it has no git ancestry link to the template, so Updating is an explicit operation: 1. Start from a clean tree — no uncommitted or unstaged changes, - especially to `fleet.json` or `engine-rollout-evidence.json`; the + especially to `fleet.json`, `engine-rollout-evidence.json`, or the optional + `next-engine-rollout-evidence.json`; the procedure restores adopter-owned files from the recorded pre-merge commit and would silently discard an uncommitted edit. Then add the template as a remote and fetch its tags into that remote's tracking @@ -112,6 +113,9 @@ Updating is an explicit operation: if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then git restore --source="$ADOPTER_HEAD" --staged --worktree -- engine-rollout-evidence.json fi + if git cat-file -e "$ADOPTER_HEAD:next-engine-rollout-evidence.json" 2>/dev/null; then + git restore --source="$ADOPTER_HEAD" --staged --worktree -- next-engine-rollout-evidence.json + fi git status --short ``` @@ -125,6 +129,9 @@ Updating is an explicit operation: else git diff --cached --exit-code "$MERGE_SOURCE" -- engine-rollout-evidence.json fi + if git cat-file -e "$ADOPTER_HEAD:next-engine-rollout-evidence.json" 2>/dev/null; then + git diff --cached --exit-code "$ADOPTER_HEAD" -- next-engine-rollout-evidence.json + fi ``` 3. Review the complete staged result — including any changes the merge diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index c3cb800..97cf6f7 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -8,6 +8,7 @@ import json import subprocess import sys +import tempfile import urllib.request from pathlib import Path @@ -60,7 +61,7 @@ "0acf5b340317d3b9f97ae7c0686d7c6e0513e2084f9aeb295bc3d96b90fbe5dd", ), "scripts/validate.py": ( - "a86a7fc4d9cc6aaf5098c5ef37808f5d5dc76a0e49b7a7d967f23043f4ecd122", + "e15b229b78d4d7649d08510d37302b7df668b52a87bfd7fe2cf4918e4a71c567", "3c202840ce00ae31568d3ac2137cd1acdebf5ff9fa8807b9823e4310c9e39568", ), } @@ -98,6 +99,116 @@ def is_upstream_repository(root: Path) -> bool: ) +def verify_pinned_consumer_staging(core_root: Path | None) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + pinned = root / "pinned" + (pinned / "scripts").mkdir(parents=True) + (pinned / "scripts" / "validate.py").write_bytes(core_bytes("scripts/validate.py", core_root)) + (pinned / "fleet.schema.json").write_bytes(core_bytes("fleet.schema.json", core_root)) + previous = json.loads((ROOT / "fleet.json").read_text(encoding="utf-8")) + previous["organization"]["slug"] = "compatibility-org" + project = next(iter(previous["projects"].values())) + project["repository"] = "compatibility-org/example-app" + previous["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + controller_name, controller = next(iter(previous["controllers"].items())) + controller["engine_ref"] = "1" * 40 + controller.pop("status_reporting", None) + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "10.255.254.0/24", "size": 28}], + } + promoted = json.loads(json.dumps(previous)) + promoted["controllers"][controller_name]["engine_ref"] = "2" * 40 + + def evidence(engine_ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": False, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + }, + } + + paths = { + "previous.json": previous, + "promoted.json": promoted, + "active-evidence.json": evidence("1" * 40), + "promoted-evidence.json": evidence("2" * 40), + "next-engine-rollout-evidence.json": evidence("2" * 40), + } + for name, value in paths.items(): + (root / name).write_text(json.dumps(value), encoding="utf-8") + tree_paths = root / "tree-paths" + tree_paths.write_bytes( + b"fleet.json\0engine-rollout-evidence.json\0next-engine-rollout-evidence.json\0" + ) + pinned_result = subprocess.run( + [ + sys.executable, + str(pinned / "scripts" / "validate.py"), + "--config", str(root / "previous.json"), + "--strict", + "--tree-paths", str(tree_paths), + "--rollout-evidence", str(root / "active-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if pinned_result.returncode: + raise RuntimeError(f"pinned core validator rejected sidecar staging: {pinned_result.stderr}") + + validator = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] + staging = subprocess.run( + [ + *validator, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-engine-rollout-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if staging.returncode: + raise RuntimeError(f"standalone validator rejected sidecar staging: {staging.stderr}") + promotion_command = [ + *validator, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + ] + promotion = subprocess.run( + [ + *promotion_command, + "--previous-next-engine-rollout-evidence", + str(root / "next-engine-rollout-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if promotion.returncode: + raise RuntimeError(f"standalone validator rejected staged promotion: {promotion.stderr}") + missing_sidecar = subprocess.run( + promotion_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if missing_sidecar.returncode == 0 or "previous integrated sidecar" not in missing_sidecar.stderr: + raise RuntimeError("standalone validator accepted promotion without prior sidecar evidence") + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--core-root", type=Path, help="local exact-commit template tree for offline tests") @@ -161,6 +272,11 @@ def main() -> int: for error in errors: print(f"ERROR: {error}", file=sys.stderr) return 1 + try: + verify_pinned_consumer_staging(args.core_root) + except RuntimeError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 print(f"OK: standalone contract matches embedded template at {CORE_COMMIT}") return 0 diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 3c9e81e..fbd2dce 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -14,7 +14,7 @@ from pathlib import Path from test_core_compatibility import CONFIG_FILES, EXACT_FILES, core_bytes, is_upstream_repository -from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_transition +from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_rollout_evidence, validate_transition ROOT = Path(__file__).resolve().parents[1] @@ -56,6 +56,7 @@ def first_controller(config: dict) -> dict: def documented_artifact_errors(root: Path) -> list[str]: allowed_external = { "TEMPLATE_RELEASE", + "next-engine-rollout-evidence.json", "scripts/ci/plan.json", "scripts/ci/run.sh", "scripts/install-worker-controller.sh", @@ -64,7 +65,7 @@ def documented_artifact_errors(root: Path) -> list[str]: token = re.compile( r"`(?:python3 )?(?:\./)?((?:scripts|docs|examples)/[A-Za-z0-9_./<>-]+|" r"fleet(?:\.schema)?\.json|template-compatibility\.json|" - r"engine-rollout-evidence\.json|AGENTS\.md|SECURITY\.md|" + r"(?:next-engine-rollout-evidence|engine-rollout-evidence)\.json|AGENTS\.md|SECURITY\.md|" r"THIRD_PARTY_NOTICES\.md|LICENSE|TEMPLATE_RELEASE)" ) errors = [] @@ -309,12 +310,13 @@ def evidence(engine_ref: str) -> dict: "--previous-config", str(root / "previous.json"), "--rollout-evidence", str(root / "current-evidence.json"), "--previous-rollout-evidence", str(root / "previous-evidence.json"), + "--next-engine-rollout-evidence", str(root / "current-evidence.json"), "--skip-path-scan", ] result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) self.assertNotEqual(result.returncode, 0) self.assertIn( - "requires next-engine capability evidence from the previous integrated rollout evidence", + "requires next-engine capability evidence from the previous integrated sidecar", result.stderr, ) @@ -342,66 +344,59 @@ def test_staged_next_engine_evidence_permits_later_promotion(self) -> None: } promoted = copy.deepcopy(previous) first_controller(promoted)["engine_ref"] = "2" * 40 - active = { "engine_ref": "1" * 40, "status_reporting_config": True, "required_status_reporting": True, "docker_network_policy_config": True, } - staged_evidence = { - "schema_version": 1, - "status_reporting_engine_capabilities": { - controller_name: { - **active, - "next_engine": {**active, "engine_ref": "2" * 40}, - } - }, - } - promoted_evidence = copy.deepcopy(staged_evidence) - promoted_evidence["status_reporting_engine_capabilities"][controller_name] = { - **active, - "engine_ref": "2" * 40, - } + def evidence(record: dict) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": {controller_name: record}, + } + + active_evidence = evidence(active) + next_evidence = evidence({**active, "engine_ref": "2" * 40}) + promoted_evidence = evidence({**active, "engine_ref": "2" * 40}) with tempfile.TemporaryDirectory() as directory: root = Path(directory) for name, value in ( ("previous.json", previous), ("promoted.json", promoted), - ("active-evidence.json", { - "schema_version": 1, - "status_reporting_engine_capabilities": {controller_name: active}, - }), - ("staged-evidence.json", staged_evidence), + ("active-evidence.json", active_evidence), + ("next-evidence.json", next_evidence), ("promoted-evidence.json", promoted_evidence), ): (root / name).write_text(json.dumps(value), encoding="utf-8") - base_command = [ - sys.executable, - str(ROOT / "scripts" / "validate.py"), - "--skip-path-scan", - ] + base_command = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] staging = subprocess.run( [ *base_command, "--config", str(root / "previous.json"), "--previous-config", str(root / "previous.json"), - "--rollout-evidence", str(root / "staged-evidence.json"), + "--rollout-evidence", str(root / "active-evidence.json"), "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) self.assertEqual(staging.returncode, 0, staging.stderr) + + promotion_command = [ + *base_command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + ] promotion = subprocess.run( [ - *base_command, - "--config", str(root / "promoted.json"), - "--previous-config", str(root / "previous.json"), - "--rollout-evidence", str(root / "promoted-evidence.json"), - "--previous-rollout-evidence", str(root / "staged-evidence.json"), + *promotion_command, + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -409,61 +404,50 @@ def test_staged_next_engine_evidence_permits_later_promotion(self) -> None: ) self.assertEqual(promotion.returncode, 0, promotion.stderr) - invalid = copy.deepcopy(staged_evidence) - invalid["status_reporting_engine_capabilities"][controller_name]["engine_ref"] = "9" * 40 - (root / "staged-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") - nonintegrated = subprocess.run( - [ - *base_command, - "--config", str(root / "promoted.json"), - "--previous-config", str(root / "previous.json"), - "--rollout-evidence", str(root / "promoted-evidence.json"), - "--previous-rollout-evidence", str(root / "staged-evidence.json"), - ], + missing = subprocess.run( + promotion_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) - self.assertNotEqual(nonintegrated.returncode, 0) - self.assertIn("must match the previous integrated controller engine_ref", nonintegrated.stderr) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("previous integrated sidecar", missing.stderr) - invalid = copy.deepcopy(staged_evidence) - invalid["status_reporting_engine_capabilities"][controller_name]["next_engine"]["engine_ref"] = "3" * 40 - (root / "staged-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") - mismatch = subprocess.run( + invalid = copy.deepcopy(active_evidence) + invalid["status_reporting_engine_capabilities"][controller_name]["engine_ref"] = "9" * 40 + (root / "invalid-active-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") + nonintegrated = subprocess.run( [ *base_command, "--config", str(root / "promoted.json"), "--previous-config", str(root / "previous.json"), "--rollout-evidence", str(root / "promoted-evidence.json"), - "--previous-rollout-evidence", str(root / "staged-evidence.json"), + "--previous-rollout-evidence", str(root / "invalid-active-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, ) - self.assertNotEqual(mismatch.returncode, 0) - self.assertIn( - "requires next-engine capability evidence from the previous integrated rollout evidence", - mismatch.stderr, - ) + self.assertNotEqual(nonintegrated.returncode, 0) + self.assertIn("must match the previous integrated controller engine_ref", nonintegrated.stderr) for capability, expected in ( + ("engine_ref", "requires next-engine capability evidence from the previous integrated sidecar"), ("status_reporting_config", "requires status-reporting configuration capability evidence"), ("required_status_reporting", "requires required status-reporting rollout evidence"), ("docker_network_policy_config", "requires Docker network policy configuration capability evidence"), ): with self.subTest(capability=capability): - invalid = copy.deepcopy(staged_evidence) - invalid["status_reporting_engine_capabilities"][controller_name]["next_engine"][capability] = False - (root / "staged-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") + invalid = copy.deepcopy(next_evidence) + record = invalid["status_reporting_engine_capabilities"][controller_name] + record[capability] = "3" * 40 if capability == "engine_ref" else False + (root / "invalid-next-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") insufficient = subprocess.run( [ - *base_command, - "--config", str(root / "promoted.json"), - "--previous-config", str(root / "previous.json"), - "--rollout-evidence", str(root / "promoted-evidence.json"), - "--previous-rollout-evidence", str(root / "staged-evidence.json"), + *promotion_command, + "--previous-next-engine-rollout-evidence", + str(root / "invalid-next-evidence.json"), ], stdout=subprocess.PIPE, stderr=subprocess.PIPE, @@ -472,6 +456,159 @@ def test_staged_next_engine_evidence_permits_later_promotion(self) -> None: self.assertNotEqual(insufficient.returncode, 0) self.assertIn(expected, insufficient.stderr) + def test_network_only_staged_next_engine_evidence_permits_later_promotion(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + controller = first_controller(previous) + controller["engine_ref"] = "1" * 40 + controller.pop("status_reporting", None) + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + promoted = copy.deepcopy(previous) + first_controller(promoted)["engine_ref"] = "2" * 40 + active = { + "engine_ref": "1" * 40, + "status_reporting_config": False, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + evidence = lambda record: { + "schema_version": 1, + "status_reporting_engine_capabilities": {controller_name: record}, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("promoted.json", promoted), + ("active-evidence.json", evidence(active)), + ("next-evidence.json", evidence({**active, "engine_ref": "2" * 40})), + ("promoted-evidence.json", evidence({**active, "engine_ref": "2" * 40})), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] + staging = subprocess.run( + [ + *command, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + promotion = subprocess.run( + [ + *command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(staging.returncode, 0, staging.stderr) + self.assertEqual(promotion.returncode, 0, promotion.stderr) + + def test_status_only_disabled_staged_next_engine_evidence_permits_later_promotion(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + controller = first_controller(previous) + controller["engine_ref"] = "1" * 40 + controller.pop("docker_network_policy", None) + controller["status_reporting"] = { + "enabled": False, + "config_file": "/etc/ci-fleet/monitoring.env", + } + promoted = copy.deepcopy(previous) + first_controller(promoted)["engine_ref"] = "2" * 40 + + def evidence(engine_ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": True, + "required_status_reporting": False, + } + }, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("promoted.json", promoted), + ("active-evidence.json", evidence("1" * 40)), + ("next-evidence.json", evidence("2" * 40)), + ("promoted-evidence.json", evidence("2" * 40)), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] + staging = subprocess.run( + [ + *command, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + promotion = subprocess.run( + [ + *command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(staging.returncode, 0, staging.stderr) + self.assertEqual(promotion.returncode, 0, promotion.stderr) + + def test_primary_rollout_evidence_rejects_nested_next_engine(self) -> None: + validation = Validation() + validate_rollout_evidence( + { + "schema_version": 1, + "status_reporting_engine_capabilities": { + "example-ci-01": { + "engine_ref": "1" * 40, + "status_reporting_config": False, + "required_status_reporting": False, + "next_engine": { + "engine_ref": "2" * 40, + "status_reporting_config": False, + "required_status_reporting": False, + }, + } + }, + }, + validation, + ) + self.assertTrue(any("unknown keys: next_engine" in error for error in validation.errors), validation.errors) + def test_new_controller_cannot_introduce_docker_network_policy(self) -> None: previous = reference_config() current = copy.deepcopy(previous) @@ -663,7 +800,12 @@ def test_release_checks_and_artifacts_are_documented_in_ci(self) -> None: self.assertIn(command, workflow) readme = (ROOT / "README.md").read_text(encoding="utf-8") updating = (ROOT / "docs" / "UPDATING.md").read_text(encoding="utf-8") - for artifact in ("template-compatibility.json", "engine-rollout-evidence.json", "docs/RELEASE.md"): + for artifact in ( + "template-compatibility.json", + "engine-rollout-evidence.json", + "next-engine-rollout-evidence.json", + "docs/RELEASE.md", + ): self.assertIn(artifact, readme) self.assertIn("new higher tag", updating) self.assertIn("scripts/migrate-v-to-v.py", updating) @@ -687,6 +829,9 @@ def test_pr_workflow_validates_against_base_fleet_and_rollout_evidence(self) -> '--previous-config "$RUNNER_TEMP/previous-fleet.json"', 'git show "$BASE_SHA:engine-rollout-evidence.json"', '--previous-rollout-evidence "$RUNNER_TEMP/previous-rollout-evidence.json"', + 'args+=(--next-engine-rollout-evidence next-engine-rollout-evidence.json)', + 'git show "$BASE_SHA:next-engine-rollout-evidence.json"', + '--previous-next-engine-rollout-evidence "$RUNNER_TEMP/previous-next-engine-rollout-evidence.json"', ): self.assertIn(required, workflow) @@ -1122,6 +1267,8 @@ def test_updating_guide_preserves_adopter_state_before_commit(self) -> None: 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json', 'if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then', 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- engine-rollout-evidence.json', + 'if git cat-file -e "$ADOPTER_HEAD:next-engine-rollout-evidence.json" 2>/dev/null; then', + 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- next-engine-rollout-evidence.json', "run the now-reviewed target", "./scripts/validate.sh --strict", "git commit", diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py index 18a6b6f..c6e97ca 100644 --- a/scripts/test_release_update.py +++ b/scripts/test_release_update.py @@ -260,6 +260,12 @@ def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite json.dumps(staged_evidence, indent=2) + "\n", encoding="utf-8", ) + next_evidence = json.loads(json.dumps(staged_evidence)) + next_evidence["status_reporting_engine_capabilities"]["ci-01"]["engine_ref"] = "2" * 40 + (adopter / "next-engine-rollout-evidence.json").write_text( + json.dumps(next_evidence, indent=2) + "\n", + encoding="utf-8", + ) git(adopter, "add", ".") git(adopter, "commit", "-qm", "adopter state") @@ -274,16 +280,41 @@ def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) expected_fleet = (adopter / "fleet.json").read_bytes() expected_evidence = (adopter / "engine-rollout-evidence.json").read_bytes() + expected_next_evidence = (adopter / "next-engine-rollout-evidence.json").read_bytes() merge = subprocess.run( ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", release_commit], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) self.assertIn(merge.returncode, (0, 1)) - git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", "fleet.json", "engine-rollout-evidence.json") + git( + adopter, + "restore", + f"--source={adopter_head}", + "--staged", + "--worktree", + "--", + "fleet.json", + "engine-rollout-evidence.json", + "next-engine-rollout-evidence.json", + ) self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) self.assertEqual((adopter / "engine-rollout-evidence.json").read_bytes(), expected_evidence) - git(adopter, "diff", "--cached", "--exit-code", adopter_head, "--", "fleet.json", "engine-rollout-evidence.json") + self.assertEqual( + (adopter / "next-engine-rollout-evidence.json").read_bytes(), + expected_next_evidence, + ) + git( + adopter, + "diff", + "--cached", + "--exit-code", + adopter_head, + "--", + "fleet.json", + "engine-rollout-evidence.json", + "next-engine-rollout-evidence.json", + ) git(adopter, "add", ".") subprocess.run( [str(adopter / "scripts" / "validate.sh"), "--strict"], diff --git a/scripts/validate.py b/scripts/validate.py index 21ff8dc..df0d009 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -507,7 +507,11 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(repository != "example-org/example-app", f"{path}.repository", "replace the example repository before use") -def validate_rollout_evidence(value: Any, validation: Validation) -> dict[str, dict[str, Any]]: +def validate_rollout_evidence( + value: Any, + validation: Validation, + source: str = "engine-rollout-evidence.json", +) -> dict[str, dict[str, Any]]: def validate_engine_evidence( evidence: Any, path: str, @@ -544,33 +548,25 @@ def validate_engine_evidence( if not validation.exact_keys( value, - "engine-rollout-evidence.json", + source, {"schema_version", "status_reporting_engine_capabilities"}, ): return {} - validation.require(value.get("schema_version") == 1, "engine-rollout-evidence.json.schema_version", "must equal 1") + validation.require(value.get("schema_version") == 1, f"{source}.schema_version", "must equal 1") capabilities = value.get("status_reporting_engine_capabilities") if not isinstance(capabilities, dict): - validation.errors.append("engine-rollout-evidence.json.status_reporting_engine_capabilities: must be an object mapping controller IDs to capability evidence") + validation.errors.append(f"{source}.status_reporting_engine_capabilities: must be an object mapping controller IDs to capability evidence") return {} valid: dict[str, dict[str, Any]] = {} for controller, evidence in capabilities.items(): - path = f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}" + path = f"{source}.status_reporting_engine_capabilities.{controller}" controller_valid = bool(SLUG.fullmatch(controller)) validation.require(controller_valid, path, "controller ID must be a lowercase slug") parsed = validate_engine_evidence( evidence, path, - {"docker_network_policy_config", "next_engine"}, + {"docker_network_policy_config"}, ) - if parsed is not None and "next_engine" in evidence: - next_engine = validate_engine_evidence( - evidence["next_engine"], - f"{path}.next_engine", - {"docker_network_policy_config"}, - ) - if next_engine is not None: - parsed["next_engine"] = next_engine if controller_valid and parsed is not None: valid[controller] = parsed return valid @@ -581,12 +577,15 @@ def validate_reporting_evidence( controller: dict[str, Any], evidence: dict[str, Any], validation: Validation, + *, + network_evidence: dict[str, Any] | None = None, ) -> bool: errors_before = len(validation.errors) + network_evidence = evidence if network_evidence is None else network_evidence if "docker_network_policy" in controller: validation.require( - evidence.get("engine_ref") == controller.get("engine_ref") - and evidence.get("docker_network_policy_config") is True, + network_evidence.get("engine_ref") == controller.get("engine_ref") + and network_evidence.get("docker_network_policy_config") is True, f"$.controllers.{name}.docker_network_policy", "requires Docker network policy configuration capability evidence for this controller and engine_ref", ) @@ -614,6 +613,7 @@ def validate_transition( compatible_engine_refs: dict[str, dict[str, Any]], validation: Validation, previous_compatible_engine_refs: dict[str, dict[str, Any]] | None = None, + previous_next_engine_refs: dict[str, dict[str, Any]] | None = None, ) -> None: if not isinstance(previous, dict) or not isinstance(current, dict): return @@ -649,11 +649,11 @@ def validate_transition( if old.get("engine_ref") != new.get("engine_ref") and ( "docker_network_policy" in new or "status_reporting" in new ): - next_engine = previous_evidence.get("next_engine", {}) + next_engine = (previous_next_engine_refs or {}).get(name, {}) validation.require( next_engine.get("engine_ref") == new.get("engine_ref"), f"$.controllers.{name}.engine_ref", - "requires next-engine capability evidence from the previous integrated rollout evidence", + "requires next-engine capability evidence from the previous integrated sidecar", ) validate_reporting_evidence(name, new, next_engine, validation) if "docker_network_policy" not in old and "docker_network_policy" in new: @@ -676,8 +676,19 @@ def validate_transition( and (not isinstance(old_reporting, dict) or old_reporting.get("enabled") is not True) ) ) - evidence = previous_evidence if staged_capability_required else current_evidence - validate_reporting_evidence(name, new, evidence, validation) + reporting_evidence = previous_evidence if staged_capability_required else current_evidence + network_evidence = ( + previous_evidence + if "docker_network_policy" not in old and "docker_network_policy" in new + else current_evidence + ) + validate_reporting_evidence( + name, + new, + reporting_evidence, + validation, + network_evidence=network_evidence, + ) if "status_reporting" not in old and "status_reporting" in new: validation.require( old.get("engine_ref") == new.get("engine_ref"), @@ -685,7 +696,7 @@ def validate_transition( "must be introduced in a later commit after the compatible engine_ref is active", ) validation.require( - evidence.get("engine_ref") == old.get("engine_ref"), + reporting_evidence.get("engine_ref") == old.get("engine_ref"), f"$.controllers.{name}.status_reporting", "requires reviewed rollout evidence for this controller and its already-active compatible engine_ref", ) @@ -695,6 +706,8 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--previous-config", type=Path, help="previous integrated configuration for rollout validation") parser.add_argument("--rollout-evidence", type=Path, help="rollout evidence file (defaults to engine-rollout-evidence.json only for the default fleet.json)") parser.add_argument("--previous-rollout-evidence", type=Path, help="previous integrated rollout evidence") + parser.add_argument("--next-engine-rollout-evidence", type=Path, help="optional next-engine rollout evidence sidecar") + parser.add_argument("--previous-next-engine-rollout-evidence", type=Path, help="previous integrated next-engine rollout evidence sidecar") parser.add_argument("--strict", action="store_true", help="reject unchanged example values") parser.add_argument("--skip-path-scan", action="store_true", help="skip repository path checks (for external fixtures)") parser.add_argument("--tree-paths", type=Path, help="NUL-delimited committed paths to scan instead of the local template tree") @@ -715,6 +728,20 @@ def main() -> int: if evidence is not None else {} ) + next_evidence_path = ( + args.next_engine_rollout_evidence.absolute() + if args.next_engine_rollout_evidence + else None + ) + default_next_evidence = ROOT / "next-engine-rollout-evidence.json" + if next_evidence_path is None and config_path == ROOT / "fleet.json" and default_next_evidence.is_file(): + next_evidence_path = default_next_evidence + next_evidence = load_json(next_evidence_path, validation) if next_evidence_path is not None else None + current_next_engine_refs = ( + validate_rollout_evidence(next_evidence, validation, "next-engine-rollout-evidence.json") + if next_evidence is not None + else {} + ) schema = load_json(ROOT / "fleet.schema.json", validation) if schema is not None: validation.require(schema.get("$schema") == "https://json-schema.org/draft/2020-12/schema", "fleet.schema.json.$schema", "must use JSON Schema draft 2020-12") @@ -729,6 +756,12 @@ def main() -> int: f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", "must match the current controller engine_ref; remove stale evidence before changing or removing the controller", ) + for controller in current_next_engine_refs: + validation.require( + isinstance(current_controllers.get(controller), dict), + f"next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}", + "must reference a current controller", + ) if isinstance(current_controllers, dict): for controller, value in current_controllers.items(): if isinstance(value, dict): @@ -750,6 +783,20 @@ def main() -> int: if previous_evidence is not None else {} ) + previous_next_evidence = ( + load_json(args.previous_next_engine_rollout_evidence.absolute(), validation) + if args.previous_next_engine_rollout_evidence + else None + ) + previous_next_engine_refs = ( + validate_rollout_evidence( + previous_next_evidence, + validation, + "previous next-engine-rollout-evidence.json", + ) + if previous_next_evidence is not None + else {} + ) if previous is not None: previous_controllers = previous.get("controllers", {}) if isinstance(previous, dict) else {} for controller, evidence in previous_compatible_engine_refs.items(): @@ -760,6 +807,12 @@ def main() -> int: f"previous engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", "must match the previous integrated controller engine_ref", ) + for controller in previous_next_engine_refs: + validation.require( + isinstance(previous_controllers.get(controller), dict), + f"previous next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}", + "must reference a previous integrated controller", + ) for controller, evidence in current_compatible_engine_refs.items(): if previous_compatible_engine_refs.get(controller) == evidence: continue @@ -770,7 +823,7 @@ def main() -> int: isinstance(previous_controller, dict) and previous_controller.get("engine_ref") == ref ) - or previous_compatible_engine_refs.get(controller, {}).get("next_engine") == evidence, + or previous_next_engine_refs.get(controller) == evidence, f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", f"{ref} must already be selected for this controller in the previous integrated fleet configuration", ) @@ -780,6 +833,7 @@ def main() -> int: current_compatible_engine_refs, validation, previous_compatible_engine_refs, + previous_next_engine_refs, ) if args.tree_paths is not None: scan_tree_path_list(args.tree_paths, validation) From b93212f2342a4ce3960dd340cfb9df02e9ef0d1a Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:56:03 -0500 Subject: [PATCH 09/15] fix: reject promoted next-engine sidecar records --- scripts/test_core_compatibility.py | 2 +- scripts/test_policy.py | 13 +++++++++++++ scripts/validate.py | 11 +++++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index 97cf6f7..c058870 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -61,7 +61,7 @@ "0acf5b340317d3b9f97ae7c0686d7c6e0513e2084f9aeb295bc3d96b90fbe5dd", ), "scripts/validate.py": ( - "e15b229b78d4d7649d08510d37302b7df668b52a87bfd7fe2cf4918e4a71c567", + "30d9afbfadd0b6aca38aadd3f5b1c362999bf18cd5dd8b4337930122abfc301e", "3c202840ce00ae31568d3ac2137cd1acdebf5ff9fa8807b9823e4310c9e39568", ), } diff --git a/scripts/test_policy.py b/scripts/test_policy.py index fbd2dce..12078e6 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -404,6 +404,19 @@ def evidence(record: dict) -> dict: ) self.assertEqual(promotion.returncode, 0, promotion.stderr) + retained = subprocess.run( + [ + *promotion_command, + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(retained.returncode, 0) + self.assertIn("remove the promoted record from the sidecar", retained.stderr) + missing = subprocess.run( promotion_command, stdout=subprocess.PIPE, diff --git a/scripts/validate.py b/scripts/validate.py index df0d009..ce3bdd2 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -756,12 +756,19 @@ def main() -> int: f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", "must match the current controller engine_ref; remove stale evidence before changing or removing the controller", ) - for controller in current_next_engine_refs: + for controller, evidence in current_next_engine_refs.items(): + current_controller = current_controllers.get(controller) validation.require( - isinstance(current_controllers.get(controller), dict), + isinstance(current_controller, dict), f"next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}", "must reference a current controller", ) + if isinstance(current_controller, dict): + validation.require( + current_controller.get("engine_ref") != evidence["engine_ref"], + f"next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + "must differ from the current controller engine_ref; remove the promoted record from the sidecar", + ) if isinstance(current_controllers, dict): for controller, value in current_controllers.items(): if isinstance(value, dict): From 5fdd290eeb38c4979ab9c4b55546c75b440d4265 Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:13:34 -0500 Subject: [PATCH 10/15] fix: reject empty next-engine sidecars --- scripts/test_core_compatibility.py | 2 +- scripts/test_policy.py | 17 +++++++++++++++++ scripts/validate.py | 2 ++ 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index c058870..b938cbb 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -61,7 +61,7 @@ "0acf5b340317d3b9f97ae7c0686d7c6e0513e2084f9aeb295bc3d96b90fbe5dd", ), "scripts/validate.py": ( - "30d9afbfadd0b6aca38aadd3f5b1c362999bf18cd5dd8b4337930122abfc301e", + "797c7d3dc67275ad6d31d11243fc9e841596341f5e5c11e8ec8824ef81b3652b", "3c202840ce00ae31568d3ac2137cd1acdebf5ff9fa8807b9823e4310c9e39568", ), } diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 12078e6..938aaae 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -367,6 +367,10 @@ def evidence(record: dict) -> dict: ("promoted.json", promoted), ("active-evidence.json", active_evidence), ("next-evidence.json", next_evidence), + ("empty-next-evidence.json", { + "schema_version": 1, + "status_reporting_engine_capabilities": {}, + }), ("promoted-evidence.json", promoted_evidence), ): (root / name).write_text(json.dumps(value), encoding="utf-8") @@ -404,6 +408,19 @@ def evidence(record: dict) -> dict: ) self.assertEqual(promotion.returncode, 0, promotion.stderr) + empty = subprocess.run( + [ + *promotion_command, + "--next-engine-rollout-evidence", str(root / "empty-next-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(empty.returncode, 0) + self.assertIn("must contain at least one staged record", empty.stderr) + retained = subprocess.run( [ *promotion_command, diff --git a/scripts/validate.py b/scripts/validate.py index ce3bdd2..0ec6caf 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -557,6 +557,8 @@ def validate_engine_evidence( if not isinstance(capabilities, dict): validation.errors.append(f"{source}.status_reporting_engine_capabilities: must be an object mapping controller IDs to capability evidence") return {} + if source.endswith("next-engine-rollout-evidence.json"): + validation.require(bool(capabilities), f"{source}.status_reporting_engine_capabilities", "must contain at least one staged record; remove the sidecar when empty") valid: dict[str, dict[str, Any]] = {} for controller, evidence in capabilities.items(): path = f"{source}.status_reporting_engine_capabilities.{controller}" From c6579fb0df6c5fe09cc4e09e56e34f6e832d7f4e Mon Sep 17 00:00:00 2001 From: Nickfost <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 21:33:14 -0500 Subject: [PATCH 11/15] fix: scope release tests to upstream template --- .github/workflows/validate.yml | 4 +++- scripts/test_release_update.py | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 44de413..557b6ce 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -35,7 +35,9 @@ jobs: - name: Verify exact core compatibility and release update flow run: | python3 scripts/test_core_compatibility.py - python3 scripts/test_release_update.py + if [[ "$GITHUB_REPOSITORY" == RandomDevelopment/ci-fleet-config-template ]]; then + python3 scripts/test_release_update.py + fi - name: Scan committed file contents for secrets env: diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py index c6e97ca..585ffef 100644 --- a/scripts/test_release_update.py +++ b/scripts/test_release_update.py @@ -55,6 +55,16 @@ def fetch_recorded_release(repository: Path, tag: str) -> subprocess.CompletedPr class ReleaseUpdateTests(unittest.TestCase): + def test_workflow_runs_release_update_fixtures_only_upstream(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + guarded_invocation = ( + 'if [[ "$GITHUB_REPOSITORY" == RandomDevelopment/ci-fleet-config-template ]]; then\n' + " python3 scripts/test_release_update.py\n" + " fi" + ) + self.assertIn(guarded_invocation, workflow) + self.assertEqual(workflow.count("python3 scripts/test_release_update.py"), 1) + def test_derived_repository_workflow_uses_strict_policy_without_exact_core_identity(self) -> None: with tempfile.TemporaryDirectory() as directory: derived = Path(directory) / "derived" From cfaf9af506c2658b9997c79bdba8f2eee6770374 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:13:08 -0500 Subject: [PATCH 12/15] fix: guard malformed sidecar controller lookups --- scripts/test_core_compatibility.py | 2 +- scripts/test_policy.py | 94 ++++++++++++++++++++++++++++++ scripts/validate.py | 5 +- 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index b938cbb..f403dd0 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -61,7 +61,7 @@ "0acf5b340317d3b9f97ae7c0686d7c6e0513e2084f9aeb295bc3d96b90fbe5dd", ), "scripts/validate.py": ( - "797c7d3dc67275ad6d31d11243fc9e841596341f5e5c11e8ec8824ef81b3652b", + "8659252cb0eab669a978e55284f0f370093402543f9facc90a86b69ade3502f8", "3c202840ce00ae31568d3ac2137cd1acdebf5ff9fa8807b9823e4310c9e39568", ), } diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 938aaae..3377cd6 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -639,6 +639,100 @@ def test_primary_rollout_evidence_rejects_nested_next_engine(self) -> None: ) self.assertTrue(any("unknown keys: next_engine" in error for error in validation.errors), validation.errors) + def test_malformed_current_controllers_with_next_engine_sidecar_fails_cleanly(self) -> None: + config = reference_config() + controller_name = next(iter(config["controllers"])) + engine_ref = first_controller(config)["engine_ref"] + config["controllers"] = [] + + def evidence(ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": ref, + "status_reporting_config": False, + "required_status_reporting": False, + } + }, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("current.json", config), + ("active-evidence.json", evidence(engine_ref)), + ("next-evidence.json", evidence("1" * 40)), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + "--skip-path-scan", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + self.assertNotIn("AttributeError", result.stderr) + self.assertIn("$.controllers: must be a non-empty object", result.stderr) + self.assertIn("FAILED:", result.stderr) + + def test_malformed_previous_controllers_with_next_engine_sidecar_fails_cleanly(self) -> None: + config = reference_config() + controller_name = next(iter(config["controllers"])) + engine_ref = first_controller(config)["engine_ref"] + previous = copy.deepcopy(config) + previous["controllers"] = [] + + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": False, + "required_status_reporting": False, + } + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("current.json", config), + ("previous.json", previous), + ("evidence.json", evidence), + ("previous-next-evidence.json", evidence), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "evidence.json"), + "--previous-rollout-evidence", str(root / "evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "previous-next-evidence.json"), + "--skip-path-scan", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + self.assertNotIn("AttributeError", result.stderr) + self.assertIn("must reference a previous integrated controller", result.stderr) + self.assertIn("FAILED:", result.stderr) + def test_new_controller_cannot_introduce_docker_network_policy(self) -> None: previous = reference_config() current = copy.deepcopy(previous) diff --git a/scripts/validate.py b/scripts/validate.py index 0ec6caf..94e63bb 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -759,7 +759,7 @@ def main() -> int: "must match the current controller engine_ref; remove stale evidence before changing or removing the controller", ) for controller, evidence in current_next_engine_refs.items(): - current_controller = current_controllers.get(controller) + current_controller = current_controllers.get(controller) if isinstance(current_controllers, dict) else None validation.require( isinstance(current_controller, dict), f"next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}", @@ -817,8 +817,9 @@ def main() -> int: "must match the previous integrated controller engine_ref", ) for controller in previous_next_engine_refs: + previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None validation.require( - isinstance(previous_controllers.get(controller), dict), + isinstance(previous_controller, dict), f"previous next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}", "must reference a previous integrated controller", ) From 81cd4e91be20cbf2ea250846e088b381eddbb903 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:36:48 -0500 Subject: [PATCH 13/15] fix: isolate downstream policy fixtures --- scripts/test_core_compatibility.py | 47 ++++++++++++++++- scripts/test_policy.py | 82 +++++++++++++++++++++++++++++- 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index f403dd0..8daad43 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -9,6 +9,8 @@ import subprocess import sys import tempfile +import unittest +import urllib.parse import urllib.request from pathlib import Path @@ -94,11 +96,54 @@ def is_upstream_repository(root: Path) -> bool: stderr=subprocess.DEVNULL, text=True, ) - return not origin.returncode and origin.stdout.strip().removesuffix(".git").endswith( + if origin.returncode: + return False + remote = origin.stdout.strip() + if "://" not in remote: + user_host, separator, path = remote.partition(":") + user, at, host = user_host.partition("@") + if not separator or not at or user != "git": + return False + else: + parsed = urllib.parse.urlparse(remote) + if parsed.scheme not in {"https", "ssh"} or (parsed.scheme == "ssh" and parsed.username != "git"): + return False + host, path = parsed.hostname, parsed.path + return host == "github.com" and path.strip("/").removesuffix(".git") == ( "RandomDevelopment/ci-fleet-config-template" ) +class UpstreamRepositoryTests(unittest.TestCase): + def assert_origin_identity(self, origin: str, expected: bool) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "remote", "add", "origin", origin], check=True) + self.assertEqual(is_upstream_repository(root), expected) + + def test_supported_github_origins_identify_upstream(self) -> None: + for origin in ( + "https://github.com/RandomDevelopment/ci-fleet-config-template.git", + "ssh://git@github.com/RandomDevelopment/ci-fleet-config-template.git", + "git@github.com:RandomDevelopment/ci-fleet-config-template.git", + ): + with self.subTest(origin=origin): + self.assert_origin_identity(origin, True) + + def test_similar_owner_suffix_is_not_upstream(self) -> None: + self.assert_origin_identity( + "https://github.com/AcmeRandomDevelopment/ci-fleet-config-template.git", + False, + ) + + def test_different_repository_is_not_upstream(self) -> None: + self.assert_origin_identity( + "https://github.com/RandomDevelopment/not-ci-fleet-config-template.git", + False, + ) + + def verify_pinned_consumer_staging(core_root: Path | None) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 3377cd6..42c5f76 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -202,12 +202,13 @@ def test_standalone_docker_network_policy_requires_current_evidence(self) -> Non def test_optional_capabilities_require_previous_integrated_engine_evidence(self) -> None: previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} previous_controller = first_controller(previous) previous_controller["engine_ref"] = "1" * 40 previous_controller.pop("status_reporting", None) previous_controller.pop("docker_network_policy", None) current = copy.deepcopy(previous) - controller_name = next(iter(current["controllers"])) controller = first_controller(current) controller["engine_ref"] = "2" * 40 controller["status_reporting"] = { @@ -331,6 +332,7 @@ def evidence(engine_ref: str) -> dict: def test_staged_next_engine_evidence_permits_later_promotion(self) -> None: previous = copy.deepcopy(reference_config()) controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} controller = first_controller(previous) controller["engine_ref"] = "1" * 40 controller["status_reporting"] = { @@ -489,6 +491,7 @@ def evidence(record: dict) -> dict: def test_network_only_staged_next_engine_evidence_permits_later_promotion(self) -> None: previous = copy.deepcopy(reference_config()) controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} controller = first_controller(previous) controller["engine_ref"] = "1" * 40 controller.pop("status_reporting", None) @@ -554,6 +557,7 @@ def test_network_only_staged_next_engine_evidence_permits_later_promotion(self) def test_status_only_disabled_staged_next_engine_evidence_permits_later_promotion(self) -> None: previous = copy.deepcopy(reference_config()) controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} controller = first_controller(previous) controller["engine_ref"] = "1" * 40 controller.pop("docker_network_policy", None) @@ -617,6 +621,82 @@ def evidence(engine_ref: str) -> dict: self.assertEqual(staging.returncode, 0, staging.stderr) self.assertEqual(promotion.returncode, 0, promotion.stderr) + def test_rollout_fixtures_ignore_additional_adopter_controllers(self) -> None: + with tempfile.TemporaryDirectory() as directory: + derived = Path(directory) / "derived" + shutil.copytree( + ROOT, + derived, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + config_path = derived / "fleet.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["organization"]["slug"] = "derived-org" + project = first_project(config) + project["repository"] = "derived-org/derived-app" + config["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + first_controller(config).pop("status_reporting", None) + first_controller(config).pop("docker_network_policy", None) + extra = copy.deepcopy(first_controller(config)) + extra.update({ + "location": "derived-site-b", + "state": "disabled", + "scale_set_name": "derived-ci-02", + "engine_ref": "3" * 40, + "status_reporting": { + "enabled": False, + "config_file": "/etc/ci-fleet/monitoring.env", + }, + }) + config["controllers"]["derived-ci-02"] = extra + config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + "derived-ci-02": { + "engine_ref": "3" * 40, + "status_reporting_config": True, + "required_status_reporting": False, + } + }, + } + (derived / "engine-rollout-evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", + encoding="utf-8", + ) + subprocess.run(["git", "init", "-q", str(derived)], check=True) + subprocess.run( + [ + "git", "-C", str(derived), "remote", "add", "origin", + "https://github.com/derived-org/derived-config.git", + ], + check=True, + ) + strict = subprocess.run( + [str(derived / "scripts" / "validate.sh"), "--strict"], + cwd=derived, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(strict.returncode, 0, strict.stderr) + result = subprocess.run( + [ + sys.executable, + str(derived / "scripts" / "test_policy.py"), + "PolicyTests.test_optional_capabilities_require_previous_integrated_engine_evidence", + "PolicyTests.test_staged_next_engine_evidence_permits_later_promotion", + "PolicyTests.test_network_only_staged_next_engine_evidence_permits_later_promotion", + "PolicyTests.test_status_only_disabled_staged_next_engine_evidence_permits_later_promotion", + ], + cwd=derived, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + def test_primary_rollout_evidence_rejects_nested_next_engine(self) -> None: validation = Validation() validate_rollout_evidence( From 5fd6be60580453e2a06ba78feda37f5330727045 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Tue, 1 Sep 2026 23:49:05 -0500 Subject: [PATCH 14/15] test: run upstream origin regressions --- scripts/test_core_compatibility.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py index 8daad43..e2e0189 100644 --- a/scripts/test_core_compatibility.py +++ b/scripts/test_core_compatibility.py @@ -259,6 +259,10 @@ def main() -> int: parser.add_argument("--core-root", type=Path, help="local exact-commit template tree for offline tests") parser.add_argument("--standalone-root", type=Path, default=ROOT) args = parser.parse_args() + if not unittest.TextTestRunner().run( + unittest.defaultTestLoader.loadTestsFromTestCase(UpstreamRepositoryTests) + ).wasSuccessful(): + return 1 if args.standalone_root == ROOT and not is_upstream_repository(ROOT): print("OK: exact core compatibility is upstream-only; skipped for derived repository") return 0 From 5f5ef5de36e171f28472b004f98bb2576d365288 Mon Sep 17 00:00:00 2001 From: Nick's Hermes <1572453+Nickfost@users.noreply.github.com> Date: Wed, 2 Sep 2026 09:37:06 -0500 Subject: [PATCH 15/15] test: isolate network fixtures from adopter capacity --- scripts/test_policy.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 42c5f76..558f41d 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -121,6 +121,7 @@ def test_schema_defines_optional_controller_capabilities(self) -> None: def test_optional_docker_network_policy_enforces_core_semantics(self) -> None: config = copy.deepcopy(reference_config()) controller = first_controller(config) + controller["max_runners"] = 1 policy = { "networks_per_runner": 1, "reserve_subnets": 1, @@ -211,6 +212,7 @@ def test_optional_capabilities_require_previous_integrated_engine_evidence(self) current = copy.deepcopy(previous) controller = first_controller(current) controller["engine_ref"] = "2" * 40 + controller["max_runners"] = 1 controller["status_reporting"] = { "enabled": False, "config_file": "/etc/ci-fleet/monitoring.env", @@ -335,6 +337,7 @@ def test_staged_next_engine_evidence_permits_later_promotion(self) -> None: previous["controllers"] = {controller_name: previous["controllers"][controller_name]} controller = first_controller(previous) controller["engine_ref"] = "1" * 40 + controller["max_runners"] = 1 controller["status_reporting"] = { "enabled": True, "config_file": "/etc/ci-fleet/monitoring.env", @@ -494,6 +497,7 @@ def test_network_only_staged_next_engine_evidence_permits_later_promotion(self) previous["controllers"] = {controller_name: previous["controllers"][controller_name]} controller = first_controller(previous) controller["engine_ref"] = "1" * 40 + controller["max_runners"] = 1 controller.pop("status_reporting", None) controller["docker_network_policy"] = { "networks_per_runner": 1, @@ -635,9 +639,12 @@ def test_rollout_fixtures_ignore_additional_adopter_controllers(self) -> None: project = first_project(config) project["repository"] = "derived-org/derived-app" config["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] - first_controller(config).pop("status_reporting", None) - first_controller(config).pop("docker_network_policy", None) - extra = copy.deepcopy(first_controller(config)) + controller = first_controller(config) + controller["max_runners"] = 20 + config["runner_pools"][controller["pool"]]["capacity_budget"] = 20 + controller.pop("status_reporting", None) + controller.pop("docker_network_policy", None) + extra = copy.deepcopy(controller) extra.update({ "location": "derived-site-b", "state": "disabled",