diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 033a91a..557b6ce 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -27,11 +27,18 @@ jobs: - name: Validate shell and Python syntax run: | bash -n scripts/init.sh scripts/validate.sh - python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py scripts/scan_committed_secrets.py + python3 -m py_compile scripts/init.py scripts/validate.py scripts/test_policy.py scripts/test_core_compatibility.py scripts/test_release_update.py scripts/scan_committed_secrets.py - name: Prove forbidden configurations fail closed run: python3 scripts/test_policy.py + - name: Verify exact core compatibility and release update flow + run: | + python3 scripts/test_core_compatibility.py + if [[ "$GITHUB_REPOSITORY" == RandomDevelopment/ci-fleet-config-template ]]; then + python3 scripts/test_release_update.py + fi + - name: Scan committed file contents for secrets env: EVENT_NAME: ${{ github.event_name }} @@ -56,8 +63,33 @@ jobs: done - name: Validate reference configurations + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} run: | - ./scripts/validate.sh + args=() + if [[ "$GITHUB_REPOSITORY" != RandomDevelopment/ci-fleet-config-template ]]; then + args+=(--strict) + fi + if [[ -f next-engine-rollout-evidence.json ]]; then + args+=(--next-engine-rollout-evidence next-engine-rollout-evidence.json) + fi + if [[ -n "$BASE_SHA" && ! "$BASE_SHA" =~ ^0+$ ]] && git cat-file -e "$BASE_SHA:fleet.json" 2>/dev/null; then + git show "$BASE_SHA:fleet.json" >"$RUNNER_TEMP/previous-fleet.json" + args+=(--previous-config "$RUNNER_TEMP/previous-fleet.json") + if git cat-file -e "$BASE_SHA:engine-rollout-evidence.json" 2>/dev/null; then + git show "$BASE_SHA:engine-rollout-evidence.json" >"$RUNNER_TEMP/previous-rollout-evidence.json" + args+=(--previous-rollout-evidence "$RUNNER_TEMP/previous-rollout-evidence.json") + fi + if git cat-file -e "$BASE_SHA:next-engine-rollout-evidence.json" 2>/dev/null; then + git show "$BASE_SHA:next-engine-rollout-evidence.json" >"$RUNNER_TEMP/previous-next-engine-rollout-evidence.json" + args+=(--previous-next-engine-rollout-evidence "$RUNNER_TEMP/previous-next-engine-rollout-evidence.json") + fi + ./scripts/validate.sh "${args[@]}" + else + printf '{"controllers":{}}\n' >"$RUNNER_TEMP/previous-fleet.json" + args+=(--previous-config "$RUNNER_TEMP/previous-fleet.json") + ./scripts/validate.sh "${args[@]}" + fi ./scripts/validate.sh --config examples/multi-host/fleet.json - name: Prove initialized configurations pass strict policy @@ -88,3 +120,10 @@ jobs: --engine-ref "${engine_ref}" \ --output "${temporary_directory}/fleet.json" test "$(stat -c '%a' "${temporary_directory}/fleet.json")" = 644 + ./scripts/validate.sh --strict --skip-path-scan --config "${temporary_directory}/fleet.json" + python3 -c 'import json,sys; p=sys.argv[1]; value=json.load(open(p, encoding="utf-8")); next(iter(value["controllers"].values()))["docker_network_policy"]={"networks_per_runner": 1, "reserve_subnets": 1, "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}]}; json.dump(value, open(sys.argv[2], "w", encoding="utf-8"), indent=2)' "${temporary_directory}/fleet.json" "${temporary_directory}/rfc-5737.json" + if ./scripts/validate.sh --strict --skip-path-scan --config "${temporary_directory}/rfc-5737.json" 2>"${temporary_directory}/strict-errors"; then + echo 'RFC 5737 fixture unexpectedly passed strict validation' >&2 + exit 1 + fi + grep -F 'reviewed operational Docker pool CIDR' "${temporary_directory}/strict-errors" diff --git a/AGENTS.md b/AGENTS.md index 1332cfb..6188f14 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,12 +15,13 @@ Before committing configuration changes, run: ## Hard rules - Never add real `.env` files, credentials, tokens, private keys, cookies, or passwords. -- Never add addresses, VM IDs, storage identifiers, backup identifiers, SSH details, or rendered runtime configuration. +- Never add addresses, VM IDs, storage identifiers, backup identifiers, SSH details, or rendered runtime configuration. A reviewed Docker `default_address_pools[].base` CIDR is the sole address exception after engine support is staged. - Do not weaken `public_repositories: false` for Docker-socket runner pools. - Infrastructure configuration owns capacity. Application workflows submit all independent jobs and do not use `max-parallel` to model fleet size. - Each GitHub runner group belongs to exactly one runner pool; do not create ambiguous cross-pool assignments. - The sum of active and drained controller maxima must not exceed the pool capacity budget. - Controller engine revisions and reusable workflows must be pinned to full reviewed commit SHAs. +- Upgrade an engine with active optional capabilities in two integrated commits: first keep engine A active and stage reviewed engine B evidence in optional adopter-owned `next-engine-rollout-evidence.json`; then change `engine_ref` to engine B, promote the matching sidecar record to active evidence, and remove the sidecar when empty. - Production environments must require approval and must not deploy automatically. - CI runner hosts and application deployment hosts are separate roles. - Image promotion uses immutable digests; do not rebuild separately for production. diff --git a/README.md b/README.md index 2df86ad..1245e45 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This is the public, secret-free starting point for an organization's private `ci-fleet` configuration repository. It records which trusted projects may use each CI pool, the reviewed desired state for controller machines, infrastructure capacity budgets, logical deployment environments, and the standardized commands every project must expose. -It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, VM IDs, storage names, backup identifiers, or `.env` files. +It does **not** contain runner registration tokens, deploy credentials, private keys, host addresses, VM IDs, storage names, backup identifiers, or `.env` files. The sole address exception is a reviewed Docker `default_address_pools[].base` CIDR after an adopter has staged engine support. It is fleet capacity policy, not a host or service address. Public examples and fixtures use RFC 5737 ranges only. ```mermaid flowchart LR @@ -50,6 +50,15 @@ flowchart LR The initializer refuses to replace a configured file unless `--force` is explicit. Run `./scripts/init.sh --help` for repository, registry, runner-group, controller, location, capacity, resource, and output options. +## Reviewed compatibility and release state + +`template-compatibility.json` pins the reviewed embedded template to ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It records schema and validator/initializer support, optional staged capabilities, the standalone new-controller safety check, and the unchanged fictional example engine pin. `engine-rollout-evidence.json` records active per-controller capability evidence in the shape consumed by that pinned core. New status-reporting or Docker network-policy fields require prior-state evidence. Upgrade an engine with active optional capabilities in exactly two integrated commits: + +1. Keep `fleet.json` and `engine-rollout-evidence.json` on engine A. Add reviewed engine B capability evidence to optional adopter-owned `next-engine-rollout-evidence.json`, using the same schema-version-1 record shape. +2. Change `engine_ref` to engine B, promote the matching sidecar record to `engine-rollout-evidence.json`, and remove the sidecar when it has no remaining staged records. + +The committed examples and initializer omit both optional fields because their engine capability has not been staged. When a private adopter later stages `docker_network_policy`, it must replace the RFC 5737 review fixture with a reviewed operational pool before strict validation passes. Initial release preparation and exclusions are in `docs/RELEASE.md`; no tag or GitHub release is published by this repository change. + ## Schema v3: Git-authored controller desired state `fleet.json` is the reviewed authority for logical controller state. Each entry in `controllers` has a unique ID and declares: @@ -143,6 +152,9 @@ Adding workers reduces wall-clock time only while independent shards remain queu |---|---| | `fleet.json` | Fictional, valid schema-v3 configuration with one controller | | `fleet.schema.json` | JSON Schema draft 2020-12 editor contract | +| `template-compatibility.json` | Exact reviewed core/template compatibility and prepared release state | +| `engine-rollout-evidence.json` | Active optional-capability evidence, empty in the public template | +| `next-engine-rollout-evidence.json` | Optional adopter-owned next-engine evidence sidecar; not committed by this public template | | `scripts/init.sh` | Safe first-project and first-controller initializer | | `scripts/validate.sh` | Structural, relational, capacity, and secret-boundary validation | | `scripts/test_policy.py` | Regression tests proving unsafe configurations fail closed | @@ -150,6 +162,7 @@ Adding workers reduces wall-clock time only while independent shards remain queu | `SECURITY.md` | Secret handling and vulnerability reporting | | `AGENTS.md` | Non-negotiable rules for humans and coding agents | | `docs/UPDATING.md` | Keeping a derived private repository current: schema vs template versions, releases, migrations, Dependabot | +| `docs/RELEASE.md` | Prepared initial release notes and immutable-tag publication checklist | ## Public and private boundary diff --git a/docs/RELEASE.md b/docs/RELEASE.md new file mode 100644 index 0000000..52b4773 --- /dev/null +++ b/docs/RELEASE.md @@ -0,0 +1,16 @@ +# Initial template release preparation + +The `v1.0.0` template release is prepared but not published. Creating the tag or GitHub release requires separate authorization after this change merges. + +The candidate synchronizes the standalone schema-v3 contract with `templates/config-repository` at reviewed ci-fleet commit `0aed0d7e85e10050028b7d11fb12b84b3619e638`. It adds optional staged status-reporting and Docker network-policy support. Existing configurations that omit either field remain valid. + +The fictional examples retain engine commit `8df97cc7575f47696fa82a179bbe39cd2874b1ca` and omit both optional fields. Network-policy validation fixtures use RFC 5737 ranges only. A private adopter may add a reviewed operational `default_address_pools[].base` CIDR only after the compatible engine and capability evidence have existed in prior integrated states. + +When release is authorized: + +1. Confirm the release commit passes every required check. +2. Create annotated tag `v1.0.0` at that exact commit. +3. Publish release notes that name the tag object ID, peeled template commit, and reviewed core commit above. +4. Never retarget the tag. Correct a bad release with a new higher tag and mark the old release as superseded. + +The compatibility details are machine-readable in `template-compatibility.json`. No tag or GitHub release exists as part of this preparation change. diff --git a/docs/UPDATING.md b/docs/UPDATING.md index 79556a4..7d02058 100644 --- a/docs/UPDATING.md +++ b/docs/UPDATING.md @@ -29,6 +29,8 @@ the same rewrite-detection baseline; a local-only ref cannot do that. Treat template files as vendored code: update them deliberately, not casually. +The initial `v1.0.0` release is currently prepared, not published. Do not use it until the repository has an immutable tag and GitHub release. If a published release is wrong, leave its tag object untouched, publish a new higher tag, and mark the old release as superseded. Never retarget or recreate an existing release tag. + ## GitHub template repositories have no fork ancestry A repository created with GitHub's "Use this template" button is **not** @@ -37,17 +39,19 @@ a fork: it has no git ancestry link to the template, so Updating is an explicit operation: 1. Start from a clean tree — no uncommitted or unstaged changes, - especially to `fleet.json`; the procedure restores `fleet.json` from - the recorded pre-merge commit and would silently discard an - uncommitted edit. Then add the template as a remote and fetch its - tags into that remote's tracking namespace. `--no-tags` prevents Git - from also creating adopter-visible tags. A retargeted upstream tag + especially to `fleet.json`, `engine-rollout-evidence.json`, or the optional + `next-engine-rollout-evidence.json`; the + procedure restores adopter-owned files from the recorded pre-merge + commit and would silently discard an uncommitted edit. Then add the + template as a remote and fetch its tags into that remote's tracking + namespace. `--no-tags` prevents Git from also creating adopter-visible + tags. A retargeted upstream tag updates a `refs/remotes/*` ref silently, so verify the reviewed object ID yourself before using any previously fetched tag ref: ```bash # Hard stop on any uncommitted state; a dirty tree would be silently - # overwritten by the fleet.json restore below. + # overwritten by the adopter-owned state restore below. test -z "$(git status --porcelain)" || { echo "clean the tree first" >&2; exit 1; } # one-time setup; on later runs require the existing remote to be the # template, not an unrelated remote that happens to share the name: @@ -100,19 +104,34 @@ Updating is an explicit operation: If Git reports conflicts, leave the merge in progress and continue. Whether or not it conflicted, restore the adopter-owned configuration - from the recorded pre-merge commit, then resolve and stage every other - conflict: + from the recorded pre-merge commit. Preserve rollout evidence from that + commit when it exists; otherwise keep the evidence introduced by the new + template release. Then resolve and stage every other conflict: ```bash git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json + if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then + git restore --source="$ADOPTER_HEAD" --staged --worktree -- engine-rollout-evidence.json + fi + if git cat-file -e "$ADOPTER_HEAD:next-engine-rollout-evidence.json" 2>/dev/null; then + git restore --source="$ADOPTER_HEAD" --staged --worktree -- next-engine-rollout-evidence.json + fi git status --short ``` - If the release keeps the same `schema_version`, prove `fleet.json` - still has no staged change: + If the release keeps the same `schema_version`, prove the adopter-owned + configuration and rollout evidence still have no staged changes: ```bash - git diff --cached --exit-code -- fleet.json + git diff --cached --exit-code "$ADOPTER_HEAD" -- fleet.json + if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then + git diff --cached --exit-code "$ADOPTER_HEAD" -- engine-rollout-evidence.json + else + git diff --cached --exit-code "$MERGE_SOURCE" -- engine-rollout-evidence.json + fi + if git cat-file -e "$ADOPTER_HEAD:next-engine-rollout-evidence.json" 2>/dev/null; then + git diff --cached --exit-code "$ADOPTER_HEAD" -- next-engine-rollout-evidence.json + fi ``` 3. Review the complete staged result — including any changes the merge @@ -160,6 +179,8 @@ pinned contract. Releases that introduce a new `schema_version` are the exception: import the new schema/validator in phase 2 of the two-phase rollout below, after every deployed controller runs the new engine. +Migration programs use `scripts/migrate-v-to-v.py`. Release notes name the exact path. If a release changes `schema_version` but does not ship that named program, stop the update. + ## Dependabot update PRs Keep a `.github/dependabot.yml` in the private repository covering diff --git a/engine-rollout-evidence.json b/engine-rollout-evidence.json new file mode 100644 index 0000000..934e58a --- /dev/null +++ b/engine-rollout-evidence.json @@ -0,0 +1,4 @@ +{ + "schema_version": 1, + "status_reporting_engine_capabilities": {} +} diff --git a/fleet.schema.json b/fleet.schema.json index 606536a..b9ca3a4 100644 --- a/fleet.schema.json +++ b/fleet.schema.json @@ -80,7 +80,39 @@ "engine_ref": {"type": "string", "pattern": "^(?!0{40}$)[0-9a-f]{40}$"}, "min_runners": {"const": 0}, "max_runners": {"type": "integer", "minimum": 1}, - "runner_resources": {"$ref": "#/$defs/runner_resources"} + "runner_resources": {"$ref": "#/$defs/runner_resources"}, + "docker_network_policy": { + "type": "object", + "additionalProperties": false, + "required": ["default_address_pools", "networks_per_runner", "reserve_subnets"], + "properties": { + "networks_per_runner": {"type": "integer", "minimum": 1}, + "reserve_subnets": {"type": "integer", "minimum": 1}, + "default_address_pools": { + "type": "array", + "minItems": 1, + "maxItems": 64, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["base", "size"], + "properties": { + "base": {"type": "string"}, + "size": {"type": "integer", "minimum": 0, "maximum": 29} + } + } + } + } + }, + "status_reporting": { + "type": "object", + "additionalProperties": false, + "required": ["enabled", "config_file"], + "properties": { + "enabled": {"type": "boolean"}, + "config_file": {"const": "/etc/ci-fleet/monitoring.env"} + } + } } }, "runner_resources": { diff --git a/scripts/init.py b/scripts/init.py index 3d3ac48..1f85ca3 100755 --- a/scripts/init.py +++ b/scripts/init.py @@ -172,7 +172,7 @@ def main() -> int: finally: temporary.unlink(missing_ok=True) print(f"Initialized {output}") - print("Next: review controller capacity, configure GitHub policy, and keep every secret value outside Git.") + print("Next: review controller capacity, run ./scripts/validate.sh --strict, and keep every secret value outside Git.") return 0 diff --git a/scripts/test_core_compatibility.py b/scripts/test_core_compatibility.py new file mode 100644 index 0000000..e2e0189 --- /dev/null +++ b/scripts/test_core_compatibility.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +"""Fail when the standalone contract drifts from one reviewed embedded template.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +import tempfile +import unittest +import urllib.parse +import urllib.request +from pathlib import Path + +CORE_COMMIT = "0aed0d7e85e10050028b7d11fb12b84b3619e638" +CORE_PREFIX = "templates/config-repository" +ROOT = Path(__file__).resolve().parents[1] +STANDALONE_FILES = { + ".github/dependabot.yml", + ".github/workflows/validate.yml", + ".gitignore", + "AGENTS.md", + "LICENSE", + "README.md", + "SECURITY.md", + "THIRD_PARTY_NOTICES.md", + "docs/RELEASE.md", + "docs/UPDATING.md", + "engine-rollout-evidence.json", + "examples/multi-host/fleet.json", + "fleet.json", + "fleet.schema.json", + "scripts/init.py", + "scripts/init.sh", + "scripts/scan_committed_secrets.py", + "scripts/test_core_compatibility.py", + "scripts/test_policy.py", + "scripts/test_release_update.py", + "scripts/validate.py", + "scripts/validate.sh", + "template-compatibility.json", +} +EXACT_FILES = ( + "fleet.schema.json", + "engine-rollout-evidence.json", + "scripts/init.py", + "scripts/validate.py", +) +CONFIG_FILES = ("fleet.json", "examples/multi-host/fleet.json") +ALLOWED_STANDALONE_HASHES = { + "examples/multi-host/fleet.json": ( + "ec0104a3891795664288c145a16e94be44eac628f8bf7aacc953ae5b3802e036", + "04908089d4d1f5f483a815ef9ef859ae053ba572d1e6d1c898866a677bc226de", + ), + "fleet.json": ( + "23a434eee489bc359589f74e9ec57b07382af61f43c00905b38816df0ef5b3db", + "12ce4b9f7146f80e5eaaa693cbeb0802f9f2f7aeaf1dcece5c0e86009d1b2e1c", + ), + "scripts/init.py": ( + "bb47f464763be1324f13af6bd64b3017e085103f2e0c1637b7d66c33e01b3c46", + "0acf5b340317d3b9f97ae7c0686d7c6e0513e2084f9aeb295bc3d96b90fbe5dd", + ), + "scripts/validate.py": ( + "8659252cb0eab669a978e55284f0f370093402543f9facc90a86b69ade3502f8", + "3c202840ce00ae31568d3ac2137cd1acdebf5ff9fa8807b9823e4310c9e39568", + ), +} + + +def core_bytes(relative: str, core_root: Path | None) -> bytes: + if core_root is not None: + return (core_root / relative).read_bytes() + url = ( + "https://raw.githubusercontent.com/RandomDevelopment/ci-fleet/" + f"{CORE_COMMIT}/{CORE_PREFIX}/{relative}" + ) + with urllib.request.urlopen(url, timeout=30) as response: + return response.read() + + +def normalized_config(raw: bytes) -> dict: + value = json.loads(raw) + # Intentional standalone difference: issue #12 preserves the older reviewed + # example engine pin. Every other embedded configuration value must match. + for controller in value["controllers"].values(): + controller.pop("engine_ref", None) + return value + + +def is_upstream_repository(root: Path) -> bool: + origin = subprocess.run( + ["git", "-C", str(root), "remote", "get-url", "origin"], + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + text=True, + ) + if origin.returncode: + return False + remote = origin.stdout.strip() + if "://" not in remote: + user_host, separator, path = remote.partition(":") + user, at, host = user_host.partition("@") + if not separator or not at or user != "git": + return False + else: + parsed = urllib.parse.urlparse(remote) + if parsed.scheme not in {"https", "ssh"} or (parsed.scheme == "ssh" and parsed.username != "git"): + return False + host, path = parsed.hostname, parsed.path + return host == "github.com" and path.strip("/").removesuffix(".git") == ( + "RandomDevelopment/ci-fleet-config-template" + ) + + +class UpstreamRepositoryTests(unittest.TestCase): + def assert_origin_identity(self, origin: str, expected: bool) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + subprocess.run(["git", "init", "-q", str(root)], check=True) + subprocess.run(["git", "-C", str(root), "remote", "add", "origin", origin], check=True) + self.assertEqual(is_upstream_repository(root), expected) + + def test_supported_github_origins_identify_upstream(self) -> None: + for origin in ( + "https://github.com/RandomDevelopment/ci-fleet-config-template.git", + "ssh://git@github.com/RandomDevelopment/ci-fleet-config-template.git", + "git@github.com:RandomDevelopment/ci-fleet-config-template.git", + ): + with self.subTest(origin=origin): + self.assert_origin_identity(origin, True) + + def test_similar_owner_suffix_is_not_upstream(self) -> None: + self.assert_origin_identity( + "https://github.com/AcmeRandomDevelopment/ci-fleet-config-template.git", + False, + ) + + def test_different_repository_is_not_upstream(self) -> None: + self.assert_origin_identity( + "https://github.com/RandomDevelopment/not-ci-fleet-config-template.git", + False, + ) + + +def verify_pinned_consumer_staging(core_root: Path | None) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + pinned = root / "pinned" + (pinned / "scripts").mkdir(parents=True) + (pinned / "scripts" / "validate.py").write_bytes(core_bytes("scripts/validate.py", core_root)) + (pinned / "fleet.schema.json").write_bytes(core_bytes("fleet.schema.json", core_root)) + previous = json.loads((ROOT / "fleet.json").read_text(encoding="utf-8")) + previous["organization"]["slug"] = "compatibility-org" + project = next(iter(previous["projects"].values())) + project["repository"] = "compatibility-org/example-app" + previous["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + controller_name, controller = next(iter(previous["controllers"].items())) + controller["engine_ref"] = "1" * 40 + controller.pop("status_reporting", None) + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "10.255.254.0/24", "size": 28}], + } + promoted = json.loads(json.dumps(previous)) + promoted["controllers"][controller_name]["engine_ref"] = "2" * 40 + + def evidence(engine_ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": False, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + }, + } + + paths = { + "previous.json": previous, + "promoted.json": promoted, + "active-evidence.json": evidence("1" * 40), + "promoted-evidence.json": evidence("2" * 40), + "next-engine-rollout-evidence.json": evidence("2" * 40), + } + for name, value in paths.items(): + (root / name).write_text(json.dumps(value), encoding="utf-8") + tree_paths = root / "tree-paths" + tree_paths.write_bytes( + b"fleet.json\0engine-rollout-evidence.json\0next-engine-rollout-evidence.json\0" + ) + pinned_result = subprocess.run( + [ + sys.executable, + str(pinned / "scripts" / "validate.py"), + "--config", str(root / "previous.json"), + "--strict", + "--tree-paths", str(tree_paths), + "--rollout-evidence", str(root / "active-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if pinned_result.returncode: + raise RuntimeError(f"pinned core validator rejected sidecar staging: {pinned_result.stderr}") + + validator = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] + staging = subprocess.run( + [ + *validator, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-engine-rollout-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if staging.returncode: + raise RuntimeError(f"standalone validator rejected sidecar staging: {staging.stderr}") + promotion_command = [ + *validator, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + ] + promotion = subprocess.run( + [ + *promotion_command, + "--previous-next-engine-rollout-evidence", + str(root / "next-engine-rollout-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if promotion.returncode: + raise RuntimeError(f"standalone validator rejected staged promotion: {promotion.stderr}") + missing_sidecar = subprocess.run( + promotion_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if missing_sidecar.returncode == 0 or "previous integrated sidecar" not in missing_sidecar.stderr: + raise RuntimeError("standalone validator accepted promotion without prior sidecar evidence") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--core-root", type=Path, help="local exact-commit template tree for offline tests") + parser.add_argument("--standalone-root", type=Path, default=ROOT) + args = parser.parse_args() + if not unittest.TextTestRunner().run( + unittest.defaultTestLoader.loadTestsFromTestCase(UpstreamRepositoryTests) + ).wasSuccessful(): + return 1 + if args.standalone_root == ROOT and not is_upstream_repository(ROOT): + print("OK: exact core compatibility is upstream-only; skipped for derived repository") + return 0 + errors: list[str] = [] + used_allowlist: set[str] = set() + actual_files = { + path.relative_to(args.standalone_root).as_posix() + for path in args.standalone_root.rglob("*") + if path.is_file() + and ".git" not in path.relative_to(args.standalone_root).parts + and "__pycache__" not in path.relative_to(args.standalone_root).parts + and path.suffix != ".pyc" + } + if actual_files != STANDALONE_FILES: + errors.append( + "tree membership differs: " + f"missing={sorted(STANDALONE_FILES - actual_files)} " + f"unexpected={sorted(actual_files - STANDALONE_FILES)}" + ) + for relative in EXACT_FILES: + standalone = (args.standalone_root / relative).read_bytes() + embedded = core_bytes(relative, args.core_root) + if standalone != embedded: + digest = hashlib.sha256(standalone).hexdigest() + allowed = ALLOWED_STANDALONE_HASHES.get(relative) + if allowed and allowed[0] == digest: + core_digest = hashlib.sha256(embedded).hexdigest() + if allowed[1] == core_digest: + used_allowlist.add(relative) + else: + errors.append(f"{relative}: reviewed core bytes differ; core sha256={core_digest}") + else: + errors.append(f"{relative}: differs from ci-fleet {CORE_COMMIT}; standalone sha256={digest}") + for relative in CONFIG_FILES: + standalone_raw = (args.standalone_root / relative).read_bytes() + standalone = normalized_config(standalone_raw) + embedded_raw = core_bytes(relative, args.core_root) + embedded = normalized_config(embedded_raw) + if standalone != embedded: + digest = hashlib.sha256(standalone_raw).hexdigest() + allowed = ALLOWED_STANDALONE_HASHES.get(relative) + if allowed and allowed[0] == digest: + core_digest = hashlib.sha256(embedded_raw).hexdigest() + if allowed[1] == core_digest: + used_allowlist.add(relative) + else: + errors.append(f"{relative}: reviewed core bytes differ; core sha256={core_digest}") + else: + errors.append( + f"{relative}: differs beyond the example engine_ref; standalone sha256={digest}" + ) + unused_allowlist = sorted(ALLOWED_STANDALONE_HASHES.keys() - used_allowlist) + if unused_allowlist: + errors.append(f"unused standalone hash allowlist entries: {unused_allowlist}") + if errors: + for error in errors: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + try: + verify_pinned_consumer_staging(args.core_root) + except RuntimeError as error: + print(f"ERROR: {error}", file=sys.stderr) + return 1 + print(f"OK: standalone contract matches embedded template at {CORE_COMMIT}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_policy.py b/scripts/test_policy.py index 42493dc..558f41d 100755 --- a/scripts/test_policy.py +++ b/scripts/test_policy.py @@ -6,13 +6,15 @@ import copy import json import re +import shutil import subprocess import sys import tempfile import unittest from pathlib import Path -from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config +from test_core_compatibility import CONFIG_FILES, EXACT_FILES, core_bytes, is_upstream_repository +from validate import Validation, load_json, scan_secret_material, scan_tree_path_list, validate_config, validate_rollout_evidence, validate_transition ROOT = Path(__file__).resolve().parents[1] @@ -51,6 +53,30 @@ def first_controller(config: dict) -> dict: return next(iter(config["controllers"].values())) +def documented_artifact_errors(root: Path) -> list[str]: + allowed_external = { + "TEMPLATE_RELEASE", + "next-engine-rollout-evidence.json", + "scripts/ci/plan.json", + "scripts/ci/run.sh", + "scripts/install-worker-controller.sh", + "scripts/migrate-v-to-v.py", + } + token = re.compile( + r"`(?:python3 )?(?:\./)?((?:scripts|docs|examples)/[A-Za-z0-9_./<>-]+|" + r"fleet(?:\.schema)?\.json|template-compatibility\.json|" + r"(?:next-engine-rollout-evidence|engine-rollout-evidence)\.json|AGENTS\.md|SECURITY\.md|" + r"THIRD_PARTY_NOTICES\.md|LICENSE|TEMPLATE_RELEASE)" + ) + errors = [] + for document in ("README.md", "docs/UPDATING.md"): + text = (root / document).read_text(encoding="utf-8") + for relative in token.findall(text): + if relative not in allowed_external and not (root / relative).is_file(): + errors.append(f"{document}: documented path does not exist: {relative}") + return sorted(set(errors)) + + class PolicyTests(unittest.TestCase): def assert_rejected(self, config: dict, expected: str, *, strict: bool = False) -> None: errors = errors_for(config, strict=strict) @@ -71,6 +97,1039 @@ def assert_delivery_engine_contract(self, value: str, accepted: bool) -> None: def test_reference_configuration_is_valid(self) -> None: self.assertEqual(errors_for(reference_config()), []) + def test_schema_defines_optional_controller_capabilities(self) -> None: + controller = contract_schema()["$defs"]["controller"] + self.assertNotIn("docker_network_policy", controller["required"]) + self.assertNotIn("status_reporting", controller["required"]) + network = controller["properties"]["docker_network_policy"] + self.assertEqual( + network["required"], + ["default_address_pools", "networks_per_runner", "reserve_subnets"], + ) + self.assertEqual(network["properties"]["default_address_pools"]["maxItems"], 64) + self.assertEqual( + network["properties"]["default_address_pools"]["items"]["properties"]["size"]["maximum"], + 29, + ) + reporting = controller["properties"]["status_reporting"] + self.assertEqual(reporting["required"], ["enabled", "config_file"]) + self.assertEqual( + reporting["properties"]["config_file"]["const"], + "/etc/ci-fleet/monitoring.env", + ) + + def test_optional_docker_network_policy_enforces_core_semantics(self) -> None: + config = copy.deepcopy(reference_config()) + controller = first_controller(config) + controller["max_runners"] = 1 + policy = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + controller["docker_network_policy"] = policy + self.assertEqual(errors_for(config), []) + + cases = ( + ({**policy, "extra": True}, "unknown keys: extra"), + (None, "must be an object"), + ({**policy, "default_address_pools": [{"base": "2001:db8::/64", "size": 28}]}, "IPv4"), + ({**policy, "default_address_pools": [{"base": "198.51.100.1/24", "size": 28}]}, "malformed"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/24", "size": 23}]}, "impossible subnet count"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/24", "size": 30}]}, "between 0 and 29"), + ({**policy, "default_address_pools": [ + {"base": "198.51.100.0/24", "size": 28}, + {"base": "198.51.100.128/25", "size": 28}, + ]}, "overlaps"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/24", "size": 29}] * 65}, "must not exceed 64"), + ({**policy, "default_address_pools": [{"base": "198.51.100.0/29", "size": 29}]}, "controller Compose network"), + ) + for value, expected in cases: + with self.subTest(expected=expected): + controller["docker_network_policy"] = value + self.assert_rejected(config, expected) + + controller["state"] = "disabled" + controller["max_runners"] = 100 + controller["docker_network_policy"] = { + **policy, + "networks_per_runner": 100, + "default_address_pools": [{"base": "198.51.100.0/28", "size": 29}], + } + self.assertEqual(errors_for(config), []) + self.assert_rejected(config, "reviewed operational Docker pool CIDR", strict=True) + + def test_standalone_docker_network_policy_requires_current_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + config_path = Path(directory) / "fleet.json" + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init.py"), + "--organization", "sample-org", + "--project", "sample-app", + "--engine-ref", "1" * 40, + "--output", str(config_path), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + config = json.loads(config_path.read_text(encoding="utf-8")) + first_controller(config)["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "10.255.255.0/24", "size": 28}], + } + config_path.write_text(json.dumps(config), encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--strict", + "--skip-path-scan", + "--config", str(config_path), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "requires Docker network policy configuration capability evidence for this controller and engine_ref", + result.stderr, + ) + + def test_optional_capabilities_require_previous_integrated_engine_evidence(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} + previous_controller = first_controller(previous) + previous_controller["engine_ref"] = "1" * 40 + previous_controller.pop("status_reporting", None) + previous_controller.pop("docker_network_policy", None) + current = copy.deepcopy(previous) + controller = first_controller(current) + controller["engine_ref"] = "2" * 40 + controller["max_runners"] = 1 + controller["status_reporting"] = { + "enabled": False, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": "2" * 40, + "status_reporting_config": True, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("current.json", current), + ("evidence.json", evidence), + ("previous-evidence.json", {"schema_version": 1, "status_reporting_engine_capabilities": {}}), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "evidence.json"), + "--previous-rollout-evidence", str(root / "previous-evidence.json"), + "--skip-path-scan", + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("later commit", result.stderr) + + previous = copy.deepcopy(current) + first_controller(previous).pop("status_reporting") + first_controller(previous).pop("docker_network_policy") + (root / "previous.json").write_text(json.dumps(previous), encoding="utf-8") + (root / "previous-evidence.json").write_text(json.dumps(evidence), encoding="utf-8") + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + + invalid = copy.deepcopy(current) + first_controller(invalid)["status_reporting"] = None + self.assert_rejected(invalid, "must be an object") + + def test_engine_upgrade_requires_previously_staged_next_engine_evidence(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + previous_controller = first_controller(previous) + previous_controller["engine_ref"] = "1" * 40 + previous_controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + previous_controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + current = copy.deepcopy(previous) + first_controller(current)["engine_ref"] = "2" * 40 + + def evidence(engine_ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + }, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("current.json", current), + ("previous-evidence.json", evidence("1" * 40)), + ("current-evidence.json", evidence("2" * 40)), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "current-evidence.json"), + "--previous-rollout-evidence", str(root / "previous-evidence.json"), + "--next-engine-rollout-evidence", str(root / "current-evidence.json"), + "--skip-path-scan", + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "requires next-engine capability evidence from the previous integrated sidecar", + result.stderr, + ) + + (root / "previous-evidence.json").write_text( + json.dumps({"schema_version": 1, "status_reporting_engine_capabilities": {}}), + encoding="utf-8", + ) + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("requires Docker network policy configuration capability evidence", result.stderr) + + def test_staged_next_engine_evidence_permits_later_promotion(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} + controller = first_controller(previous) + controller["engine_ref"] = "1" * 40 + controller["max_runners"] = 1 + controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + promoted = copy.deepcopy(previous) + first_controller(promoted)["engine_ref"] = "2" * 40 + active = { + "engine_ref": "1" * 40, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + + def evidence(record: dict) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": {controller_name: record}, + } + + active_evidence = evidence(active) + next_evidence = evidence({**active, "engine_ref": "2" * 40}) + promoted_evidence = evidence({**active, "engine_ref": "2" * 40}) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("promoted.json", promoted), + ("active-evidence.json", active_evidence), + ("next-evidence.json", next_evidence), + ("empty-next-evidence.json", { + "schema_version": 1, + "status_reporting_engine_capabilities": {}, + }), + ("promoted-evidence.json", promoted_evidence), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + base_command = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] + staging = subprocess.run( + [ + *base_command, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(staging.returncode, 0, staging.stderr) + + promotion_command = [ + *base_command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + ] + promotion = subprocess.run( + [ + *promotion_command, + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(promotion.returncode, 0, promotion.stderr) + + empty = subprocess.run( + [ + *promotion_command, + "--next-engine-rollout-evidence", str(root / "empty-next-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(empty.returncode, 0) + self.assertIn("must contain at least one staged record", empty.stderr) + + retained = subprocess.run( + [ + *promotion_command, + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(retained.returncode, 0) + self.assertIn("remove the promoted record from the sidecar", retained.stderr) + + missing = subprocess.run( + promotion_command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(missing.returncode, 0) + self.assertIn("previous integrated sidecar", missing.stderr) + + invalid = copy.deepcopy(active_evidence) + invalid["status_reporting_engine_capabilities"][controller_name]["engine_ref"] = "9" * 40 + (root / "invalid-active-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") + nonintegrated = subprocess.run( + [ + *base_command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "invalid-active-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(nonintegrated.returncode, 0) + self.assertIn("must match the previous integrated controller engine_ref", nonintegrated.stderr) + + for capability, expected in ( + ("engine_ref", "requires next-engine capability evidence from the previous integrated sidecar"), + ("status_reporting_config", "requires status-reporting configuration capability evidence"), + ("required_status_reporting", "requires required status-reporting rollout evidence"), + ("docker_network_policy_config", "requires Docker network policy configuration capability evidence"), + ): + with self.subTest(capability=capability): + invalid = copy.deepcopy(next_evidence) + record = invalid["status_reporting_engine_capabilities"][controller_name] + record[capability] = "3" * 40 if capability == "engine_ref" else False + (root / "invalid-next-evidence.json").write_text(json.dumps(invalid), encoding="utf-8") + insufficient = subprocess.run( + [ + *promotion_command, + "--previous-next-engine-rollout-evidence", + str(root / "invalid-next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(insufficient.returncode, 0) + self.assertIn(expected, insufficient.stderr) + + def test_network_only_staged_next_engine_evidence_permits_later_promotion(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} + controller = first_controller(previous) + controller["engine_ref"] = "1" * 40 + controller["max_runners"] = 1 + controller.pop("status_reporting", None) + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + promoted = copy.deepcopy(previous) + first_controller(promoted)["engine_ref"] = "2" * 40 + active = { + "engine_ref": "1" * 40, + "status_reporting_config": False, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + evidence = lambda record: { + "schema_version": 1, + "status_reporting_engine_capabilities": {controller_name: record}, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("promoted.json", promoted), + ("active-evidence.json", evidence(active)), + ("next-evidence.json", evidence({**active, "engine_ref": "2" * 40})), + ("promoted-evidence.json", evidence({**active, "engine_ref": "2" * 40})), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] + staging = subprocess.run( + [ + *command, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + promotion = subprocess.run( + [ + *command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(staging.returncode, 0, staging.stderr) + self.assertEqual(promotion.returncode, 0, promotion.stderr) + + def test_status_only_disabled_staged_next_engine_evidence_permits_later_promotion(self) -> None: + previous = copy.deepcopy(reference_config()) + controller_name = next(iter(previous["controllers"])) + previous["controllers"] = {controller_name: previous["controllers"][controller_name]} + controller = first_controller(previous) + controller["engine_ref"] = "1" * 40 + controller.pop("docker_network_policy", None) + controller["status_reporting"] = { + "enabled": False, + "config_file": "/etc/ci-fleet/monitoring.env", + } + promoted = copy.deepcopy(previous) + first_controller(promoted)["engine_ref"] = "2" * 40 + + def evidence(engine_ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": True, + "required_status_reporting": False, + } + }, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("previous.json", previous), + ("promoted.json", promoted), + ("active-evidence.json", evidence("1" * 40)), + ("next-evidence.json", evidence("2" * 40)), + ("promoted-evidence.json", evidence("2" * 40)), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + command = [sys.executable, str(ROOT / "scripts" / "validate.py"), "--skip-path-scan"] + staging = subprocess.run( + [ + *command, + "--config", str(root / "previous.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + promotion = subprocess.run( + [ + *command, + "--config", str(root / "promoted.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "promoted-evidence.json"), + "--previous-rollout-evidence", str(root / "active-evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "next-evidence.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(staging.returncode, 0, staging.stderr) + self.assertEqual(promotion.returncode, 0, promotion.stderr) + + def test_rollout_fixtures_ignore_additional_adopter_controllers(self) -> None: + with tempfile.TemporaryDirectory() as directory: + derived = Path(directory) / "derived" + shutil.copytree( + ROOT, + derived, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + config_path = derived / "fleet.json" + config = json.loads(config_path.read_text(encoding="utf-8")) + config["organization"]["slug"] = "derived-org" + project = first_project(config) + project["repository"] = "derived-org/derived-app" + config["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + controller = first_controller(config) + controller["max_runners"] = 20 + config["runner_pools"][controller["pool"]]["capacity_budget"] = 20 + controller.pop("status_reporting", None) + controller.pop("docker_network_policy", None) + extra = copy.deepcopy(controller) + extra.update({ + "location": "derived-site-b", + "state": "disabled", + "scale_set_name": "derived-ci-02", + "engine_ref": "3" * 40, + "status_reporting": { + "enabled": False, + "config_file": "/etc/ci-fleet/monitoring.env", + }, + }) + config["controllers"]["derived-ci-02"] = extra + config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + "derived-ci-02": { + "engine_ref": "3" * 40, + "status_reporting_config": True, + "required_status_reporting": False, + } + }, + } + (derived / "engine-rollout-evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", + encoding="utf-8", + ) + subprocess.run(["git", "init", "-q", str(derived)], check=True) + subprocess.run( + [ + "git", "-C", str(derived), "remote", "add", "origin", + "https://github.com/derived-org/derived-config.git", + ], + check=True, + ) + strict = subprocess.run( + [str(derived / "scripts" / "validate.sh"), "--strict"], + cwd=derived, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(strict.returncode, 0, strict.stderr) + result = subprocess.run( + [ + sys.executable, + str(derived / "scripts" / "test_policy.py"), + "PolicyTests.test_optional_capabilities_require_previous_integrated_engine_evidence", + "PolicyTests.test_staged_next_engine_evidence_permits_later_promotion", + "PolicyTests.test_network_only_staged_next_engine_evidence_permits_later_promotion", + "PolicyTests.test_status_only_disabled_staged_next_engine_evidence_permits_later_promotion", + ], + cwd=derived, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + + def test_primary_rollout_evidence_rejects_nested_next_engine(self) -> None: + validation = Validation() + validate_rollout_evidence( + { + "schema_version": 1, + "status_reporting_engine_capabilities": { + "example-ci-01": { + "engine_ref": "1" * 40, + "status_reporting_config": False, + "required_status_reporting": False, + "next_engine": { + "engine_ref": "2" * 40, + "status_reporting_config": False, + "required_status_reporting": False, + }, + } + }, + }, + validation, + ) + self.assertTrue(any("unknown keys: next_engine" in error for error in validation.errors), validation.errors) + + def test_malformed_current_controllers_with_next_engine_sidecar_fails_cleanly(self) -> None: + config = reference_config() + controller_name = next(iter(config["controllers"])) + engine_ref = first_controller(config)["engine_ref"] + config["controllers"] = [] + + def evidence(ref: str) -> dict: + return { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": ref, + "status_reporting_config": False, + "required_status_reporting": False, + } + }, + } + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("current.json", config), + ("active-evidence.json", evidence(engine_ref)), + ("next-evidence.json", evidence("1" * 40)), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--rollout-evidence", str(root / "active-evidence.json"), + "--next-engine-rollout-evidence", str(root / "next-evidence.json"), + "--skip-path-scan", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + self.assertNotIn("AttributeError", result.stderr) + self.assertIn("$.controllers: must be a non-empty object", result.stderr) + self.assertIn("FAILED:", result.stderr) + + def test_malformed_previous_controllers_with_next_engine_sidecar_fails_cleanly(self) -> None: + config = reference_config() + controller_name = next(iter(config["controllers"])) + engine_ref = first_controller(config)["engine_ref"] + previous = copy.deepcopy(config) + previous["controllers"] = [] + + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": engine_ref, + "status_reporting_config": False, + "required_status_reporting": False, + } + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("current.json", config), + ("previous.json", previous), + ("evidence.json", evidence), + ("previous-next-evidence.json", evidence), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "evidence.json"), + "--previous-rollout-evidence", str(root / "evidence.json"), + "--previous-next-engine-rollout-evidence", str(root / "previous-next-evidence.json"), + "--skip-path-scan", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("Traceback", result.stderr) + self.assertNotIn("AttributeError", result.stderr) + self.assertIn("must reference a previous integrated controller", result.stderr) + self.assertIn("FAILED:", result.stderr) + + def test_new_controller_cannot_introduce_docker_network_policy(self) -> None: + previous = reference_config() + current = copy.deepcopy(previous) + controller = copy.deepcopy(first_controller(current)) + controller["state"] = "disabled" + controller["scale_set_name"] = "new-ci-02" + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + current["controllers"]["new-ci-02"] = controller + validation = Validation() + + validate_transition(previous, current, {}, validation, {}) + + self.assertTrue(any( + "new-ci-02.docker_network_policy" in error and "new controller" in error + for error in validation.errors + ), validation.errors) + + def test_initializer_omits_optional_capabilities_without_engine_evidence(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "fleet.json" + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init.py"), + "--organization", "sample-org", + "--project", "sample-app", + "--engine-ref", "1" * 40, + "--output", str(output), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + config = json.loads(output.read_text(encoding="utf-8")) + self.assertEqual(output.stat().st_mode & 0o777, 0o644) + subprocess.run( + [ + str(ROOT / "scripts" / "validate.sh"), + "--strict", + "--skip-path-scan", + "--config", str(output), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + controller = first_controller(config) + self.assertNotIn("status_reporting", controller) + self.assertNotIn("docker_network_policy", controller) + self.assertEqual(errors_for(config), []) + + def test_initializer_rejects_strict_placeholders_without_output(self) -> None: + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "fleet.json" + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init.py"), + "--organization", "example-org", + "--project", "example-app", + "--engine-ref", "2" * 40, + "--output", str(output), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertNotEqual(result.returncode, 0) + self.assertFalse(output.exists()) + + def test_old_engine_examples_omit_optional_capabilities(self) -> None: + upstream = is_upstream_repository(ROOT) + for path in (ROOT / "fleet.json", ROOT / "examples" / "multi-host" / "fleet.json"): + with self.subTest(path=path): + config = json.loads(path.read_text(encoding="utf-8")) + self.assertEqual(config["schema_version"], 3) + for controller in config["controllers"].values(): + if upstream: + self.assertNotIn("docker_network_policy", controller) + self.assertNotIn("status_reporting", controller) + self.assertEqual( + controller["engine_ref"], + "8df97cc7575f47696fa82a179bbe39cd2874b1ca", + ) + self.assertEqual(errors_for(config), []) + + def test_compatibility_record_names_exact_core_and_staged_template_contract(self) -> None: + compatibility = json.loads((ROOT / "template-compatibility.json").read_text(encoding="utf-8")) + self.assertEqual(set(compatibility), { + "schema_version", + "reviewed_core", + "template_contract", + "optional_capabilities", + "standalone_safety_deltas", + "example_engine", + "template_release", + }) + self.assertEqual( + compatibility["reviewed_core"], + { + "repository": "RandomDevelopment/ci-fleet", + "commit": "0aed0d7e85e10050028b7d11fb12b84b3619e638", + "embedded_template_path": "templates/config-repository", + }, + ) + self.assertEqual(compatibility["template_contract"]["fleet_schema_version"], 3) + self.assertEqual(compatibility["template_contract"]["validator"], "scripts/validate.py") + self.assertEqual(compatibility["template_contract"]["initializer"], "scripts/init.py") + self.assertEqual( + set(compatibility["optional_capabilities"]), + {"status_reporting", "docker_network_policy"}, + ) + self.assertTrue(all( + value["support"] == "optional-staged" + for value in compatibility["optional_capabilities"].values() + )) + self.assertEqual(compatibility["standalone_safety_deltas"], [{ + "path": "scripts/validate.py", + "reviewed_core_gap": "new controllers may introduce docker_network_policy without prior staged capability evidence", + "standalone_enforcement": "new controllers must omit docker_network_policy until engine rollout is proven", + }]) + self.assertEqual(compatibility["example_engine"]["commit"], "8df97cc7575f47696fa82a179bbe39cd2874b1ca") + self.assertEqual(compatibility["example_engine"]["pin_status"], "unchanged") + self.assertEqual(compatibility["template_release"]["state"], "prepared-not-published") + + def test_exact_core_drift_check_is_pinned_and_runnable(self) -> None: + checker = ROOT / "scripts" / "test_core_compatibility.py" + text = checker.read_text(encoding="utf-8") + self.assertIn("0aed0d7e85e10050028b7d11fb12b84b3619e638", text) + result = subprocess.run( + [sys.executable, str(checker)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_exact_core_drift_check_rejects_required_tree_and_allowlisted_hash_drift(self) -> None: + if not is_upstream_repository(ROOT): + self.skipTest("exact core compatibility is upstream-only") + with tempfile.TemporaryDirectory() as directory: + standalone = Path(directory) / "standalone" + shutil.copytree(ROOT, standalone, ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc")) + (standalone / ".github" / "workflows" / "validate.yml").unlink() + with (standalone / "scripts" / "init.py").open("a", encoding="utf-8") as initializer: + initializer.write("\n# drift\n") + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "test_core_compatibility.py"), "--standalone-root", str(standalone)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("tree membership", result.stderr) + self.assertIn("scripts/init.py: differs", result.stderr) + self.assertIn("standalone sha256=", result.stderr) + + def test_exact_core_drift_check_rejects_corrupt_allowlisted_core_bytes(self) -> None: + if not is_upstream_repository(ROOT): + self.skipTest("exact core compatibility is upstream-only") + with tempfile.TemporaryDirectory() as directory: + core = Path(directory) / "core" + for relative in (*EXACT_FILES, *CONFIG_FILES): + path = core / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(core_bytes(relative, None)) + with (core / "scripts" / "validate.py").open("ab") as validator: + validator.write(b"\n# corrupt core bytes\n") + result = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "test_core_compatibility.py"), "--core-root", str(core)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("scripts/validate.py: reviewed core bytes differ", result.stderr) + self.assertIn("core sha256=", result.stderr) + + def test_release_checks_and_artifacts_are_documented_in_ci(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + for command in ( + "python3 scripts/test_core_compatibility.py", + "python3 scripts/test_release_update.py", + "./scripts/validate.sh --strict", + ): + self.assertIn(command, workflow) + readme = (ROOT / "README.md").read_text(encoding="utf-8") + updating = (ROOT / "docs" / "UPDATING.md").read_text(encoding="utf-8") + for artifact in ( + "template-compatibility.json", + "engine-rollout-evidence.json", + "next-engine-rollout-evidence.json", + "docs/RELEASE.md", + ): + self.assertIn(artifact, readme) + self.assertIn("new higher tag", updating) + self.assertIn("scripts/migrate-v-to-v.py", updating) + + def test_documented_artifact_check_is_path_aware(self) -> None: + self.assertEqual(documented_artifact_errors(ROOT), []) + with tempfile.TemporaryDirectory() as directory: + copy = Path(directory) / "standalone" + shutil.copytree(ROOT, copy, ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc")) + with (copy / "README.md").open("a", encoding="utf-8") as readme: + readme.write("\nRun `python3 scripts/missing-required.py`.\n") + self.assertEqual( + documented_artifact_errors(copy), + ["README.md: documented path does not exist: scripts/missing-required.py"], + ) + + def test_pr_workflow_validates_against_base_fleet_and_rollout_evidence(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + for required in ( + 'git show "$BASE_SHA:fleet.json"', + '--previous-config "$RUNNER_TEMP/previous-fleet.json"', + 'git show "$BASE_SHA:engine-rollout-evidence.json"', + '--previous-rollout-evidence "$RUNNER_TEMP/previous-rollout-evidence.json"', + 'args+=(--next-engine-rollout-evidence next-engine-rollout-evidence.json)', + 'git show "$BASE_SHA:next-engine-rollout-evidence.json"', + '--previous-next-engine-rollout-evidence "$RUNNER_TEMP/previous-next-engine-rollout-evidence.json"', + ): + self.assertIn(required, workflow) + + def test_pr_workflow_rejects_first_state_optional_capabilities(self) -> None: + current = copy.deepcopy(reference_config()) + controller_name = next(iter(current["controllers"])) + controller = first_controller(current) + controller["engine_ref"] = "2" * 40 + controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "198.51.100.0/24", "size": 28}], + } + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": "2" * 40, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + }, + } + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for name, value in ( + ("current.json", current), + ("previous.json", {"controllers": {}}), + ("evidence.json", evidence), + ): + (root / name).write_text(json.dumps(value), encoding="utf-8") + result = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "validate.py"), + "--config", str(root / "current.json"), + "--previous-config", str(root / "previous.json"), + "--rollout-evidence", str(root / "evidence.json"), + "--skip-path-scan", + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("must be omitted from a new controller", result.stderr) + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertRegex( + workflow, + r'else\n\s+printf \'\{\"controllers\":\{\}\}\\n\' >\"\$RUNNER_TEMP/previous-fleet\.json\"\n' + r'\s+args\+=\(--previous-config \"\$RUNNER_TEMP/previous-fleet\.json\"\)\n' + r'\s+\./scripts/validate\.sh \"\$\{args\[@\]\}\"\n\s+fi', + ) + + def test_workflow_passes_strict_operational_fixture_and_rejects_rfc_5737_fixture(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertIn('./scripts/validate.sh --strict --skip-path-scan --config "${temporary_directory}/fleet.json"', workflow) + self.assertIn('"base": "198.51.100.0/24"', workflow) + self.assertIn('grep -F \'reviewed operational Docker pool CIDR\'', workflow) + self.assertNotIn("documentation configuration unexpectedly passed strict validation", workflow) + + def test_documentation_names_the_narrow_docker_pool_address_exception(self) -> None: + agents = (ROOT / "AGENTS.md").read_text(encoding="utf-8") + readme = (ROOT / "README.md").read_text(encoding="utf-8") + initializer = (ROOT / "scripts" / "init.py").read_text(encoding="utf-8") + for text in (agents, readme): + self.assertIn("default_address_pools[].base", text) + self.assertIn("sole address exception", text) + self.assertIn("review controller capacity", initializer) + self.assertNotIn("replace the RFC 5737 Docker pool", initializer) + + def test_release_update_test_executes_recorded_tag_rewrite_refusal(self) -> None: + release_test = (ROOT / "scripts" / "test_release_update.py").read_text(encoding="utf-8") + for required in ( + "TEMPLATE_RELEASE", + "refs/tmp/template-tag-check", + "was rewritten upstream; refusing to use it", + "v1.1.0", + ): + self.assertIn(required, release_test) + self.assertIn("assertNotEqual(rewrite.returncode, 0)", release_test) + def test_multi_host_multi_location_configuration_is_valid(self) -> None: config = json.loads((ROOT / "examples" / "multi-host" / "fleet.json").read_text(encoding="utf-8")) self.assertEqual(errors_for(config), []) @@ -416,7 +1475,11 @@ def test_updating_guide_preserves_adopter_state_before_commit(self) -> None: "git fetch --no-tags template", 'refs/tags/$NEW_TAG:refs/tmp/template-tag-check', 'ADOPTER_HEAD="$(git rev-parse HEAD)"', - 'git restore --source="$ADOPTER_HEAD"', + 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- fleet.json', + 'if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then', + 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- engine-rollout-evidence.json', + 'if git cat-file -e "$ADOPTER_HEAD:next-engine-rollout-evidence.json" 2>/dev/null; then', + 'git restore --source="$ADOPTER_HEAD" --staged --worktree -- next-engine-rollout-evidence.json', "run the now-reviewed target", "./scripts/validate.sh --strict", "git commit", diff --git a/scripts/test_release_update.py b/scripts/test_release_update.py new file mode 100644 index 0000000..585ffef --- /dev/null +++ b/scripts/test_release_update.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Exercise the documented immutable release update in a fictional adopter.""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +def git(repository: Path, *args: str, capture: bool = False) -> str: + result = subprocess.run( + ["git", "-C", str(repository), *args], + check=True, + stdout=subprocess.PIPE if capture else subprocess.DEVNULL, + text=True, + ) + return result.stdout.strip() if capture else "" + + +def configure(repository: Path) -> None: + git(repository, "config", "user.name", "Release Test") + git(repository, "config", "user.email", "release@example.invalid") + + +def fetch_recorded_release(repository: Path, tag: str) -> subprocess.CompletedProcess[str]: + records = dict( + line.split() + for line in (repository / "TEMPLATE_RELEASE").read_text(encoding="utf-8").splitlines() + ) + temporary_ref = "refs/tmp/template-tag-check" + subprocess.run(["git", "-C", str(repository), "update-ref", "-d", temporary_ref], check=False) + fetch = subprocess.run( + ["git", "-C", str(repository), "fetch", "--no-tags", "template", f"refs/tags/{tag}:{temporary_ref}"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if fetch.returncode: + return fetch + oid = git(repository, "rev-parse", temporary_ref, capture=True) + if tag in records and records[tag] != oid: + git(repository, "update-ref", "-d", temporary_ref) + return subprocess.CompletedProcess(fetch.args, 1, "", f"template tag {tag} was rewritten upstream; refusing to use it") + commit = git(repository, "rev-parse", f"{temporary_ref}^{{commit}}", capture=True) + git(repository, "update-ref", "-d", temporary_ref) + return subprocess.CompletedProcess(fetch.args, 0, f"{oid} {commit}", "") + + +class ReleaseUpdateTests(unittest.TestCase): + def test_workflow_runs_release_update_fixtures_only_upstream(self) -> None: + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + guarded_invocation = ( + 'if [[ "$GITHUB_REPOSITORY" == RandomDevelopment/ci-fleet-config-template ]]; then\n' + " python3 scripts/test_release_update.py\n" + " fi" + ) + self.assertIn(guarded_invocation, workflow) + self.assertEqual(workflow.count("python3 scripts/test_release_update.py"), 1) + + def test_derived_repository_workflow_uses_strict_policy_without_exact_core_identity(self) -> None: + with tempfile.TemporaryDirectory() as directory: + derived = Path(directory) / "derived" + shutil.copytree( + ROOT, + derived, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + config_path = derived / "fleet.json" + unchanged = subprocess.run( + [str(derived / "scripts" / "validate.sh"), "--strict"], + cwd=derived, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + self.assertNotEqual(unchanged.returncode, 0) + self.assertTrue( + "replace the example organization" in unchanged.stderr + or "reviewed operational Docker pool CIDR" in unchanged.stderr, + unchanged.stderr, + ) + config = json.loads(config_path.read_text(encoding="utf-8")) + config["organization"]["slug"] = "derived-org" + project = next(iter(config["projects"].values())) + project["repository"] = "derived-org/derived-app" + config["runner_pools"][project["ci_pool"]]["allowed_repositories"] = [project["repository"]] + controller_name, controller = next(iter(config["controllers"].items())) + for controller in config["controllers"].values(): + controller["engine_ref"] = "2" * 40 + controller["status_reporting"] = { + "enabled": True, + "config_file": "/etc/ci-fleet/monitoring.env", + } + controller["docker_network_policy"] = { + "networks_per_runner": 1, + "reserve_subnets": 1, + "default_address_pools": [{"base": "10.255.255.0/24", "size": 28}], + } + config_path.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8") + evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + controller_name: { + "engine_ref": "2" * 40, + "status_reporting_config": True, + "required_status_reporting": True, + "docker_network_policy_config": True, + } + }, + } + (derived / "engine-rollout-evidence.json").write_text( + json.dumps(evidence, indent=2) + "\n", + encoding="utf-8", + ) + multi_host_path = derived / "examples" / "multi-host" / "fleet.json" + multi_host = json.loads(multi_host_path.read_text(encoding="utf-8")) + for controller in multi_host["controllers"].values(): + controller["engine_ref"] = "2" * 40 + multi_host_path.write_text(json.dumps(multi_host, indent=2) + "\n", encoding="utf-8") + (derived / "TEMPLATE_RELEASE").write_text("v1.0.0 " + "1" * 40 + "\n", encoding="utf-8") + git(derived, "init", "-q") + git(derived, "remote", "add", "origin", "https://github.com/derived-org/derived-config.git") + + subprocess.run( + [str(derived / "scripts" / "validate.sh"), "--strict"], + cwd=derived, + check=True, + stdout=subprocess.DEVNULL, + ) + + result = subprocess.run( + [sys.executable, str(derived / "scripts" / "test_policy.py")], + cwd=derived, + env={ + **os.environ, + "HTTPS_PROXY": "http://127.0.0.1:9", + "https_proxy": "http://127.0.0.1:9", + "NO_PROXY": "", + "no_proxy": "", + }, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr) + workflow = (ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") + self.assertIn('if [[ "$GITHUB_REPOSITORY" != RandomDevelopment/ci-fleet-config-template ]]; then', workflow) + self.assertIn("args+=(--strict)", workflow) + + def test_legacy_adopter_keeps_new_template_rollout_evidence(self) -> None: + guide = (ROOT / "docs" / "UPDATING.md").read_text(encoding="utf-8") + conditional_restore = 'if git cat-file -e "$ADOPTER_HEAD:engine-rollout-evidence.json" 2>/dev/null; then' in guide + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + template = root / "template" + adopter = root / "adopter" + shutil.copytree( + ROOT, + template, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + git(template, "init", "-q") + configure(template) + git(template, "add", ".") + git(template, "commit", "-qm", "template release") + release_commit = git(template, "rev-parse", "HEAD", capture=True) + expected_evidence = (template / "engine-rollout-evidence.json").read_bytes() + + adopter.mkdir() + git(adopter, "init", "-q") + configure(adopter) + subprocess.run( + [ + str(template / "scripts" / "init.sh"), + "--organization", "legacy-org", + "--project", "legacy-app", + "--engine-ref", "1" * 40, + "--output", str(adopter / "fleet.json"), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + git(adopter, "add", "fleet.json") + git(adopter, "commit", "-qm", "legacy adopter state") + adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) + expected_fleet = (adopter / "fleet.json").read_bytes() + git(adopter, "remote", "add", "template", str(template)) + git(adopter, "fetch", "template", release_commit) + + merge = subprocess.run( + ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", release_commit], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self.assertIn(merge.returncode, (0, 1)) + restore_paths = ["fleet.json"] + if not conditional_restore: + restore_paths.append("engine-rollout-evidence.json") + git(adopter, "restore", f"--source={adopter_head}", "--staged", "--worktree", "--", *restore_paths) + + self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) + self.assertEqual((adopter / "engine-rollout-evidence.json").read_bytes(), expected_evidence) + git(adopter, "diff", "--cached", "--exit-code", adopter_head, "--", "fleet.json") + git(adopter, "diff", "--cached", "--exit-code", release_commit, "--", "engine-rollout-evidence.json") + git(adopter, "add", ".") + subprocess.run( + [str(adopter / "scripts" / "validate.sh"), "--strict"], + cwd=adopter, + check=True, + stdout=subprocess.DEVNULL, + ) + + def test_fictional_unrelated_adopter_update_preserves_config_and_detects_rewrite(self) -> None: + release_notes = (ROOT / "docs" / "RELEASE.md").read_text(encoding="utf-8") + for required in ("v1.0.0", "0aed0d7e85e10050028b7d11fb12b84b3619e638", "not published"): + self.assertIn(required, release_notes) + + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + template = root / "template" + adopter = root / "adopter" + shutil.copytree( + ROOT, + template, + ignore=shutil.ignore_patterns(".git", "__pycache__", "*.pyc"), + ) + git(template, "init", "-q") + configure(template) + git(template, "add", ".") + git(template, "commit", "-qm", "template v1.0.0") + git(template, "tag", "-a", "v1.0.0", "-m", "v1.0.0") + release_oid = git(template, "rev-parse", "v1.0.0", capture=True) + + adopter.mkdir() + git(adopter, "init", "-q") + configure(adopter) + subprocess.run( + [ + str(template / "scripts" / "init.sh"), + "--organization", "fictional-org", + "--project", "fictional-app", + "--engine-ref", "1" * 40, + "--output", str(adopter / "fleet.json"), + ], + check=True, + stdout=subprocess.DEVNULL, + ) + (adopter / "TEMPLATE_RELEASE").write_text(f"v1.0.0 {release_oid}\n", encoding="utf-8") + staged_evidence = { + "schema_version": 1, + "status_reporting_engine_capabilities": { + "ci-01": { + "engine_ref": "1" * 40, + "status_reporting_config": True, + "required_status_reporting": False, + "docker_network_policy_config": True, + } + }, + } + (adopter / "engine-rollout-evidence.json").write_text( + json.dumps(staged_evidence, indent=2) + "\n", + encoding="utf-8", + ) + next_evidence = json.loads(json.dumps(staged_evidence)) + next_evidence["status_reporting_engine_capabilities"]["ci-01"]["engine_ref"] = "2" * 40 + (adopter / "next-engine-rollout-evidence.json").write_text( + json.dumps(next_evidence, indent=2) + "\n", + encoding="utf-8", + ) + git(adopter, "add", ".") + git(adopter, "commit", "-qm", "adopter state") + + (template / "release-marker").write_text("v1.1.0\n", encoding="utf-8") + git(template, "add", "release-marker") + git(template, "commit", "-qm", "template v1.1.0") + git(template, "tag", "-a", "v1.1.0", "-m", "v1.1.0") + git(adopter, "remote", "add", "template", str(template)) + release = fetch_recorded_release(adopter, "v1.1.0") + self.assertEqual(release.returncode, 0, release.stderr) + new_oid, release_commit = release.stdout.split() + adopter_head = git(adopter, "rev-parse", "HEAD", capture=True) + expected_fleet = (adopter / "fleet.json").read_bytes() + expected_evidence = (adopter / "engine-rollout-evidence.json").read_bytes() + expected_next_evidence = (adopter / "next-engine-rollout-evidence.json").read_bytes() + merge = subprocess.run( + ["git", "-C", str(adopter), "merge", "--no-ff", "--no-commit", "--allow-unrelated-histories", release_commit], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self.assertIn(merge.returncode, (0, 1)) + git( + adopter, + "restore", + f"--source={adopter_head}", + "--staged", + "--worktree", + "--", + "fleet.json", + "engine-rollout-evidence.json", + "next-engine-rollout-evidence.json", + ) + self.assertEqual((adopter / "fleet.json").read_bytes(), expected_fleet) + self.assertEqual((adopter / "engine-rollout-evidence.json").read_bytes(), expected_evidence) + self.assertEqual( + (adopter / "next-engine-rollout-evidence.json").read_bytes(), + expected_next_evidence, + ) + git( + adopter, + "diff", + "--cached", + "--exit-code", + adopter_head, + "--", + "fleet.json", + "engine-rollout-evidence.json", + "next-engine-rollout-evidence.json", + ) + git(adopter, "add", ".") + subprocess.run( + [str(adopter / "scripts" / "validate.sh"), "--strict"], + cwd=adopter, + check=True, + stdout=subprocess.DEVNULL, + ) + with (adopter / "TEMPLATE_RELEASE").open("a", encoding="utf-8") as record: + record.write(f"v1.1.0 {new_oid}\n") + git(adopter, "add", "TEMPLATE_RELEASE") + git(adopter, "commit", "-qm", "update to template v1.1.0") + + (template / "rewrite-marker").write_text("rewrite\n", encoding="utf-8") + git(template, "add", "rewrite-marker") + git(template, "commit", "-qm", "rewrite target") + git(template, "tag", "-f", "-a", "v1.1.0", "-m", "rewritten") + rewrite = fetch_recorded_release(adopter, "v1.1.0") + self.assertNotEqual(rewrite.returncode, 0) + self.assertIn("was rewritten upstream; refusing to use it", rewrite.stderr) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate.py b/scripts/validate.py index 77cd706..94e63bb 100755 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import ipaddress import json import re import sys @@ -55,6 +56,10 @@ } FORBIDDEN_FILENAMES = re.compile(r"(?:^|/)(?:\.env(?:\..+)?|host\.env|ci-fleet\.env)$|\.(?:key|pem|p12|pfx)$", re.IGNORECASE) FORBIDDEN_DIRECTORIES = {"credentials", "private", "secrets"} +RFC_5737_NETWORKS = tuple( + ipaddress.ip_network(value) for value in ("192.0.2.0/24", "198.51.100.0/24", "203.0.113.0/24") +) +MAX_DOCKER_ADDRESS_POOLS = 64 class Validation: @@ -89,6 +94,9 @@ def reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: value[key] = child return value + if path.is_symlink(): + validation.errors.append(f"{path}: symlinked JSON files are forbidden") + return None try: return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=reject_duplicate_keys) except FileNotFoundError: @@ -137,6 +145,90 @@ def scan_keys(value: Any, path: str = "$") -> None: break +def validate_docker_network_policy( + policy: Any, + path: str, + max_runners: int, + validation: Validation, + *, + strict: bool = False, +) -> tuple[int, int, list[dict[str, Any]]]: + if not isinstance(policy, dict): + validation.errors.append(f"{path}: must be an object") + return 0, 0, [] + required = {"default_address_pools", "networks_per_runner", "reserve_subnets"} + keys = set(policy) + missing = sorted(required - keys) + unknown = sorted(keys - required) + if missing or unknown: + parts = [] + if missing: + parts.append(f"missing keys: {', '.join(missing)}") + if unknown: + parts.append(f"unknown keys: {', '.join(unknown)}") + validation.errors.append(f"{path}: {'; '.join(parts)}") + return 0, 0, [] + reserve = policy["reserve_subnets"] + if type(reserve) is not int or reserve < 1: + validation.errors.append(f"{path}.reserve_subnets: must be a positive integer") + return 0, 0, [] + networks_per_runner = policy["networks_per_runner"] + if type(networks_per_runner) is not int or networks_per_runner < 1: + validation.errors.append(f"{path}.networks_per_runner: must be a positive integer") + return 0, 0, [] + pools = policy["default_address_pools"] + if type(pools) is not list or not pools: + validation.errors.append(f"{path}.default_address_pools: must be a non-empty list") + return 0, 0, [] + if len(pools) > MAX_DOCKER_ADDRESS_POOLS: + validation.errors.append( + f"{path}.default_address_pools: must not exceed {MAX_DOCKER_ADDRESS_POOLS} pools" + ) + return 0, 0, [] + parsed: list[dict[str, Any]] = [] + for index, pool in enumerate(pools): + pool_path = f"{path}.default_address_pools[{index}]" + if not isinstance(pool, dict) or set(pool) != {"base", "size"}: + validation.errors.append(f"{pool_path}: must contain only base and size") + return 0, 0, [] + base = pool["base"] + size = pool["size"] + if not isinstance(base, str): + validation.errors.append(f"{pool_path}.base: must be a CIDR prefix") + return 0, 0, [] + if type(size) is not int or size < 0 or size > 29: + validation.errors.append(f"{pool_path}.size: must be an IPv4 prefix length between 0 and 29") + return 0, 0, [] + try: + network = ipaddress.ip_network(base, strict=True) + except ValueError: + validation.errors.append(f"{pool_path}.base: malformed address pool IPv4 prefix") + return 0, 0, [] + if network.version != 4: + validation.errors.append(f"{pool_path}.base: malformed address pool IPv4 prefix") + return 0, 0, [] + if strict and any(network.overlaps(documentation) for documentation in RFC_5737_NETWORKS): + validation.errors.append( + f"{pool_path}.base: replace the RFC 5737 documentation address pool with a reviewed operational Docker pool CIDR" + ) + return 0, 0, [] + if size < network.prefixlen: + validation.errors.append(f"{pool_path}.size: impossible subnet count for {base}") + return 0, 0, [] + parsed.append({"base": base, "network": network, "size": size}) + for left, item in enumerate(parsed): + for right in range(left + 1, len(parsed)): + if item["network"].overlaps(parsed[right]["network"]): + validation.errors.append(f"{path}.default_address_pools[{left}].base: overlaps configured pool {right}") + return 0, 0, [] + configured = sum(1 << (item["size"] - item["network"].prefixlen) for item in parsed) + if configured < max_runners * networks_per_runner + reserve + 1: + validation.errors.append( + f"{path}: network capacity cannot satisfy max_runners * networks_per_runner + reserve_subnets + one controller Compose network" + ) + return configured, reserve, parsed + + def scan_forbidden_paths(repo_root: Path, validation: Validation) -> None: for path in repo_root.rglob("*"): try: @@ -264,11 +356,12 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: "min_runners", "max_runners", "runner_resources", + "docker_network_policy", } for name, controller in controllers.items(): path = f"$.controllers.{name}" validation.require(isinstance(name, str) and bool(SLUG.fullmatch(name)), path, "controller ID must be a unique lowercase slug") - if not validation.exact_keys(controller, path, controller_keys): + if not validation.exact_keys(controller, path, controller_keys - {"docker_network_policy"}, {"status_reporting", "docker_network_policy"}): continue pool_name = controller.get("pool") location = controller.get("location") @@ -296,12 +389,26 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: if type(minimum) is int and type(maximum) is int: validation.require(minimum <= maximum, f"{path}.min_runners", "must not exceed max_runners") validation.require(minimum == 0, f"{path}.min_runners", "must be zero because managed prewarmed runners are not supported") + status_reporting = controller.get("status_reporting") + if "status_reporting" in controller and validation.exact_keys(status_reporting, f"{path}.status_reporting", {"enabled", "config_file"}): + validation.require(type(status_reporting.get("enabled")) is bool, f"{path}.status_reporting.enabled", "must be a boolean") + validation.require(status_reporting.get("config_file") == "/etc/ci-fleet/monitoring.env", f"{path}.status_reporting.config_file", "must use the fixed host-local monitoring configuration") resources = controller.get("runner_resources") if validation.exact_keys(resources, f"{path}.runner_resources", {"cpu_cores", "memory_mib"}): cpu = resources.get("cpu_cores") memory = resources.get("memory_mib") validation.require(type(cpu) is int and cpu > 0, f"{path}.runner_resources.cpu_cores", "must be a positive integer") validation.require(type(memory) is int and memory >= 512, f"{path}.runner_resources.memory_mib", "must be at least 512 MiB") + network_policy = controller.get("docker_network_policy") + capacity_maximum = maximum if state != "disabled" and type(maximum) is int and maximum > 0 else 0 + if "docker_network_policy" in controller: + validate_docker_network_policy( + network_policy, + f"{path}.docker_network_policy", + capacity_maximum, + validation, + strict=strict, + ) if isinstance(pool_name, str) and pool_name in pools and state != "disabled" and type(maximum) is int and maximum > 0: reserved_capacity[pool_name] += maximum @@ -400,9 +507,209 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(repository != "example-org/example-app", f"{path}.repository", "replace the example repository before use") +def validate_rollout_evidence( + value: Any, + validation: Validation, + source: str = "engine-rollout-evidence.json", +) -> dict[str, dict[str, Any]]: + def validate_engine_evidence( + evidence: Any, + path: str, + optional: set[str], + ) -> dict[str, Any] | None: + if not validation.exact_keys( + evidence, + path, + {"engine_ref", "status_reporting_config", "required_status_reporting"}, + optional, + ): + return None + ref = evidence.get("engine_ref") + configured = evidence.get("status_reporting_config") + required = evidence.get("required_status_reporting") + network_policy = evidence.get("docker_network_policy_config") + ref_valid = isinstance(ref, str) and bool(COMMIT_SHA.fullmatch(ref)) and ref != "0" * 40 + validation.require(ref_valid, f"{path}.engine_ref", "must be a nonzero full lowercase commit SHA") + validation.require(type(configured) is bool, f"{path}.status_reporting_config", "must be a boolean") + validation.require(type(required) is bool, f"{path}.required_status_reporting", "must be a boolean") + if "docker_network_policy_config" in evidence: + validation.require(type(network_policy) is bool, f"{path}.docker_network_policy_config", "must be a boolean") + network_policy_valid = "docker_network_policy_config" not in evidence or type(network_policy) is bool + if not (ref_valid and type(configured) is bool and type(required) is bool and network_policy_valid): + return None + parsed = { + "engine_ref": ref, + "status_reporting_config": configured, + "required_status_reporting": required, + } + if "docker_network_policy_config" in evidence: + parsed["docker_network_policy_config"] = network_policy + return parsed + + if not validation.exact_keys( + value, + source, + {"schema_version", "status_reporting_engine_capabilities"}, + ): + return {} + validation.require(value.get("schema_version") == 1, f"{source}.schema_version", "must equal 1") + capabilities = value.get("status_reporting_engine_capabilities") + if not isinstance(capabilities, dict): + validation.errors.append(f"{source}.status_reporting_engine_capabilities: must be an object mapping controller IDs to capability evidence") + return {} + if source.endswith("next-engine-rollout-evidence.json"): + validation.require(bool(capabilities), f"{source}.status_reporting_engine_capabilities", "must contain at least one staged record; remove the sidecar when empty") + valid: dict[str, dict[str, Any]] = {} + for controller, evidence in capabilities.items(): + path = f"{source}.status_reporting_engine_capabilities.{controller}" + controller_valid = bool(SLUG.fullmatch(controller)) + validation.require(controller_valid, path, "controller ID must be a lowercase slug") + parsed = validate_engine_evidence( + evidence, + path, + {"docker_network_policy_config"}, + ) + if controller_valid and parsed is not None: + valid[controller] = parsed + return valid + + +def validate_reporting_evidence( + name: str, + controller: dict[str, Any], + evidence: dict[str, Any], + validation: Validation, + *, + network_evidence: dict[str, Any] | None = None, +) -> bool: + errors_before = len(validation.errors) + network_evidence = evidence if network_evidence is None else network_evidence + if "docker_network_policy" in controller: + validation.require( + network_evidence.get("engine_ref") == controller.get("engine_ref") + and network_evidence.get("docker_network_policy_config") is True, + f"$.controllers.{name}.docker_network_policy", + "requires Docker network policy configuration capability evidence for this controller and engine_ref", + ) + reporting = controller.get("status_reporting") + if isinstance(reporting, dict): + validation.require( + evidence.get("engine_ref") == controller.get("engine_ref") + and evidence.get("status_reporting_config") is True, + f"$.controllers.{name}.status_reporting", + "requires status-reporting configuration capability evidence for this controller and engine_ref", + ) + if reporting.get("enabled") is True: + validation.require( + evidence.get("engine_ref") == controller.get("engine_ref") + and evidence.get("required_status_reporting") is True, + f"$.controllers.{name}.status_reporting.enabled", + "requires required status-reporting rollout evidence for this controller and engine_ref", + ) + return len(validation.errors) == errors_before + + +def validate_transition( + previous: Any, + current: Any, + compatible_engine_refs: dict[str, dict[str, Any]], + validation: Validation, + previous_compatible_engine_refs: dict[str, dict[str, Any]] | None = None, + previous_next_engine_refs: dict[str, dict[str, Any]] | None = None, +) -> None: + if not isinstance(previous, dict) or not isinstance(current, dict): + return + old_controllers = previous.get("controllers") + new_controllers = current.get("controllers") + if not isinstance(old_controllers, dict) or not isinstance(new_controllers, dict): + return + for name, new in new_controllers.items(): + old = old_controllers.get(name) + if not isinstance(new, dict): + continue + if name not in old_controllers: + if "status_reporting" in new: + validation.errors.append( + f"$.controllers.{name}.status_reporting: must be omitted from a new controller until its engine rollout is proven" + ) + if "docker_network_policy" in new: + validation.errors.append( + f"$.controllers.{name}.docker_network_policy: must be omitted from a new controller until its engine rollout is proven" + ) + continue + if not isinstance(old, dict): + continue + current_evidence = compatible_engine_refs.get(name, {}) + previous_evidence_source = ( + compatible_engine_refs + if previous_compatible_engine_refs is None + else previous_compatible_engine_refs + ) + previous_evidence = previous_evidence_source.get(name, {}) + old_reporting = old.get("status_reporting") + new_reporting = new.get("status_reporting") + if old.get("engine_ref") != new.get("engine_ref") and ( + "docker_network_policy" in new or "status_reporting" in new + ): + next_engine = (previous_next_engine_refs or {}).get(name, {}) + validation.require( + next_engine.get("engine_ref") == new.get("engine_ref"), + f"$.controllers.{name}.engine_ref", + "requires next-engine capability evidence from the previous integrated sidecar", + ) + validate_reporting_evidence(name, new, next_engine, validation) + if "docker_network_policy" not in old and "docker_network_policy" in new: + validation.require( + old.get("engine_ref") == new.get("engine_ref"), + f"$.controllers.{name}.docker_network_policy", + "must be introduced in a later commit after the compatible engine_ref is active", + ) + validation.require( + previous_evidence.get("engine_ref") == new.get("engine_ref") + and previous_evidence.get("docker_network_policy_config") is True, + f"$.controllers.{name}.docker_network_policy", + "requires reviewed evidence from the previous integrated state that this controller activated the same engine_ref with Docker network policy configuration capability", + ) + staged_capability_required = ( + "status_reporting" not in old + or ( + isinstance(new_reporting, dict) + and new_reporting.get("enabled") is True + and (not isinstance(old_reporting, dict) or old_reporting.get("enabled") is not True) + ) + ) + reporting_evidence = previous_evidence if staged_capability_required else current_evidence + network_evidence = ( + previous_evidence + if "docker_network_policy" not in old and "docker_network_policy" in new + else current_evidence + ) + validate_reporting_evidence( + name, + new, + reporting_evidence, + validation, + network_evidence=network_evidence, + ) + if "status_reporting" not in old and "status_reporting" in new: + validation.require( + old.get("engine_ref") == new.get("engine_ref"), + f"$.controllers.{name}.status_reporting", + "must be introduced in a later commit after the compatible engine_ref is active", + ) + validation.require( + reporting_evidence.get("engine_ref") == old.get("engine_ref"), + f"$.controllers.{name}.status_reporting", + "requires reviewed rollout evidence for this controller and its already-active compatible engine_ref", + ) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--config", type=Path, default=ROOT / "fleet.json", help="configuration file to validate") + parser.add_argument("--previous-config", type=Path, help="previous integrated configuration for rollout validation") + parser.add_argument("--rollout-evidence", type=Path, help="rollout evidence file (defaults to engine-rollout-evidence.json only for the default fleet.json)") + parser.add_argument("--previous-rollout-evidence", type=Path, help="previous integrated rollout evidence") + parser.add_argument("--next-engine-rollout-evidence", type=Path, help="optional next-engine rollout evidence sidecar") + parser.add_argument("--previous-next-engine-rollout-evidence", type=Path, help="previous integrated next-engine rollout evidence sidecar") parser.add_argument("--strict", action="store_true", help="reject unchanged example values") parser.add_argument("--skip-path-scan", action="store_true", help="skip repository path checks (for external fixtures)") parser.add_argument("--tree-paths", type=Path, help="NUL-delimited committed paths to scan instead of the local template tree") @@ -412,13 +719,132 @@ def parse_args() -> argparse.Namespace: def main() -> int: args = parse_args() validation = Validation() - config = load_json(args.config.resolve(), validation) + config_path = args.config.absolute() + config = load_json(config_path, validation) + evidence_path = args.rollout_evidence.absolute() if args.rollout_evidence else None + if evidence_path is None and config_path == ROOT / "fleet.json": + evidence_path = ROOT / "engine-rollout-evidence.json" + evidence = load_json(evidence_path, validation) if evidence_path is not None else None + current_compatible_engine_refs = ( + validate_rollout_evidence(evidence, validation) + if evidence is not None + else {} + ) + next_evidence_path = ( + args.next_engine_rollout_evidence.absolute() + if args.next_engine_rollout_evidence + else None + ) + default_next_evidence = ROOT / "next-engine-rollout-evidence.json" + if next_evidence_path is None and config_path == ROOT / "fleet.json" and default_next_evidence.is_file(): + next_evidence_path = default_next_evidence + next_evidence = load_json(next_evidence_path, validation) if next_evidence_path is not None else None + current_next_engine_refs = ( + validate_rollout_evidence(next_evidence, validation, "next-engine-rollout-evidence.json") + if next_evidence is not None + else {} + ) schema = load_json(ROOT / "fleet.schema.json", validation) if schema is not None: validation.require(schema.get("$schema") == "https://json-schema.org/draft/2020-12/schema", "fleet.schema.json.$schema", "must use JSON Schema draft 2020-12") if config is not None: scan_secret_material(config, validation) validate_config(config, validation, args.strict) + current_controllers = config.get("controllers", {}) if isinstance(config, dict) else {} + for controller, evidence in current_compatible_engine_refs.items(): + current_controller = current_controllers.get(controller) if isinstance(current_controllers, dict) else None + validation.require( + isinstance(current_controller, dict) and current_controller.get("engine_ref") == evidence["engine_ref"], + f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + "must match the current controller engine_ref; remove stale evidence before changing or removing the controller", + ) + for controller, evidence in current_next_engine_refs.items(): + current_controller = current_controllers.get(controller) if isinstance(current_controllers, dict) else None + validation.require( + isinstance(current_controller, dict), + f"next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}", + "must reference a current controller", + ) + if isinstance(current_controller, dict): + validation.require( + current_controller.get("engine_ref") != evidence["engine_ref"], + f"next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + "must differ from the current controller engine_ref; remove the promoted record from the sidecar", + ) + if isinstance(current_controllers, dict): + for controller, value in current_controllers.items(): + if isinstance(value, dict): + validate_reporting_evidence( + controller, + value, + current_compatible_engine_refs.get(controller, {}), + validation, + ) + if args.previous_config is not None: + previous = load_json(args.previous_config.absolute(), validation) + previous_evidence = ( + load_json(args.previous_rollout_evidence.absolute(), validation) + if args.previous_rollout_evidence + else None + ) + previous_compatible_engine_refs = ( + validate_rollout_evidence(previous_evidence, validation) + if previous_evidence is not None + else {} + ) + previous_next_evidence = ( + load_json(args.previous_next_engine_rollout_evidence.absolute(), validation) + if args.previous_next_engine_rollout_evidence + else None + ) + previous_next_engine_refs = ( + validate_rollout_evidence( + previous_next_evidence, + validation, + "previous next-engine-rollout-evidence.json", + ) + if previous_next_evidence is not None + else {} + ) + if previous is not None: + previous_controllers = previous.get("controllers", {}) if isinstance(previous, dict) else {} + for controller, evidence in previous_compatible_engine_refs.items(): + previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None + validation.require( + isinstance(previous_controller, dict) + and previous_controller.get("engine_ref") == evidence["engine_ref"], + f"previous engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + "must match the previous integrated controller engine_ref", + ) + for controller in previous_next_engine_refs: + previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None + validation.require( + isinstance(previous_controller, dict), + f"previous next-engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}", + "must reference a previous integrated controller", + ) + for controller, evidence in current_compatible_engine_refs.items(): + if previous_compatible_engine_refs.get(controller) == evidence: + continue + ref = evidence["engine_ref"] + previous_controller = previous_controllers.get(controller) if isinstance(previous_controllers, dict) else None + validation.require( + ( + isinstance(previous_controller, dict) + and previous_controller.get("engine_ref") == ref + ) + or previous_next_engine_refs.get(controller) == evidence, + f"engine-rollout-evidence.json.status_reporting_engine_capabilities.{controller}.engine_ref", + f"{ref} must already be selected for this controller in the previous integrated fleet configuration", + ) + validate_transition( + previous, + config, + current_compatible_engine_refs, + validation, + previous_compatible_engine_refs, + previous_next_engine_refs, + ) if args.tree_paths is not None: scan_tree_path_list(args.tree_paths, validation) elif not args.skip_path_scan: diff --git a/template-compatibility.json b/template-compatibility.json new file mode 100644 index 0000000..d9da5ef --- /dev/null +++ b/template-compatibility.json @@ -0,0 +1,41 @@ +{ + "schema_version": 1, + "reviewed_core": { + "repository": "RandomDevelopment/ci-fleet", + "commit": "0aed0d7e85e10050028b7d11fb12b84b3619e638", + "embedded_template_path": "templates/config-repository" + }, + "template_contract": { + "fleet_schema_version": 3, + "schema": "fleet.schema.json", + "validator": "scripts/validate.py", + "initializer": "scripts/init.py", + "rollout_evidence": "engine-rollout-evidence.json" + }, + "optional_capabilities": { + "status_reporting": { + "support": "optional-staged", + "evidence_key": "status_reporting_engine_capabilities" + }, + "docker_network_policy": { + "support": "optional-staged", + "evidence_key": "docker_network_policy_config" + } + }, + "standalone_safety_deltas": [ + { + "path": "scripts/validate.py", + "reviewed_core_gap": "new controllers may introduce docker_network_policy without prior staged capability evidence", + "standalone_enforcement": "new controllers must omit docker_network_policy until engine rollout is proven" + } + ], + "example_engine": { + "commit": "8df97cc7575f47696fa82a179bbe39cd2874b1ca", + "pin_status": "unchanged", + "compatibility": "optional-fields-omitted" + }, + "template_release": { + "planned_tag": "v1.0.0", + "state": "prepared-not-published" + } +}