diff --git a/.github/actions/plan/action.yml b/.github/actions/plan/action.yml new file mode 100644 index 00000000..f87928b4 --- /dev/null +++ b/.github/actions/plan/action.yml @@ -0,0 +1,34 @@ +name: Expand ci-fleet task plan +description: Validate a project CI plan and emit a deterministic GitHub Actions matrix. + +inputs: + plan-path: + description: Path to the project-owned CI task plan. + required: false + default: scripts/ci/plan.json + group: + description: Aggregate group to expand (fast or full). + required: false + default: fast + +outputs: + matrix: + description: GitHub Actions matrix containing one entry per task shard. + value: ${{ steps.expand.outputs.matrix }} + job-count: + description: Number of independently schedulable jobs. + value: ${{ steps.expand.outputs.job-count }} + estimated-test-minutes: + description: Sum of expected test payload across all emitted shards. + value: ${{ steps.expand.outputs.estimated-test-minutes }} + +runs: + using: composite + steps: + - id: expand + shell: bash + run: >- + python3 "${GITHUB_ACTION_PATH}/plan.py" + --plan "${{ inputs.plan-path }}" + --group "${{ inputs.group }}" + --github-output "${GITHUB_OUTPUT}" diff --git a/.github/actions/plan/plan.py b/.github/actions/plan/plan.py new file mode 100755 index 00000000..a486a5d3 --- /dev/null +++ b/.github/actions/plan/plan.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +"""Validate and expand a ci-fleet project task plan.""" + +from __future__ import annotations + +import argparse +import json +import math +import re +import sys +from pathlib import Path +from typing import Any + + +TASK_ID = re.compile(r"^[a-z0-9][a-z0-9-]{0,62}$") +PLAN_KEYS = { + "schema_version", + "target_wall_clock_minutes", + "max_job_minutes", + "shard_target_minutes", + "tasks", +} +TASK_KEYS = {"id", "groups", "shards", "estimated_minutes_per_shard"} + + +def fail(message: str) -> None: + raise ValueError(message) + + +def exact_keys(value: Any, required: set[str], path: str, optional: set[str] | None = None) -> None: + if not isinstance(value, dict): + fail(f"{path} must be an object") + optional = optional or set() + missing = required - set(value) + unknown = set(value) - required - optional + if missing: + fail(f"{path} is missing: {', '.join(sorted(missing))}") + if unknown: + fail(f"{path} contains unknown keys: {', '.join(sorted(unknown))}") + + +def load_plan(path: Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError as exc: + fail(f"{path} was not found") + raise AssertionError from exc + except json.JSONDecodeError as exc: + fail(f"{path}:{exc.lineno}:{exc.colno} is not valid JSON: {exc.msg}") + raise AssertionError from exc + exact_keys(value, PLAN_KEYS, "$", {"$schema"}) + return value + + +def validate_and_expand(plan: dict[str, Any], group: str) -> tuple[dict[str, list[dict[str, Any]]], int]: + if group not in {"fast", "full"}: + fail("group must be fast or full") + if plan["schema_version"] != 1: + fail("$.schema_version must equal 1") + if plan["target_wall_clock_minutes"] != 5: + fail("$.target_wall_clock_minutes must equal 5") + if plan["max_job_minutes"] != 5: + fail("$.max_job_minutes must equal 5") + target = plan["shard_target_minutes"] + if type(target) is not int or not 1 <= target <= 4: + fail("$.shard_target_minutes must be an integer between 1 and 4") + tasks = plan["tasks"] + if not isinstance(tasks, list) or not tasks: + fail("$.tasks must be a non-empty array") + + seen: set[str] = set() + coverage = {"fast": 0, "full": 0} + include: list[dict[str, Any]] = [] + estimated_total = 0 + for index, task in enumerate(tasks): + path = f"$.tasks[{index}]" + exact_keys(task, TASK_KEYS, path) + task_id = task["id"] + if not isinstance(task_id, str) or not TASK_ID.fullmatch(task_id): + fail(f"{path}.id must be a lowercase slug") + if task_id in {"fast", "full"}: + fail(f"{path}.id uses a reserved aggregate name") + if task_id in seen: + fail(f"{path}.id duplicates {task_id}") + seen.add(task_id) + + groups = task["groups"] + if not isinstance(groups, list) or not groups or any(value not in {"fast", "full"} for value in groups): + fail(f"{path}.groups must contain fast and/or full") + if len(groups) != len(set(groups)): + fail(f"{path}.groups must be unique") + if "fast" in groups and "full" not in groups: + fail(f"{path}.groups must include full whenever it includes fast") + for value in groups: + coverage[value] += 1 + + shards = task["shards"] + if type(shards) is not int or shards < 1: + fail(f"{path}.shards must be a positive integer") + estimate = task["estimated_minutes_per_shard"] + if type(estimate) not in {int, float} or isinstance(estimate, bool) or estimate <= 0 or estimate > target: + fail(f"{path}.estimated_minutes_per_shard must be greater than zero and at most {target}") + + if group in groups: + for shard in range(1, shards + 1): + include.append( + { + "task": task_id, + "shard": shard, + "shards": shards, + "estimated_minutes": estimate, + } + ) + estimated_total += estimate + + if coverage["fast"] == 0 or coverage["full"] == 0: + fail("$.tasks must provide both fast and full coverage") + if len(include) > 256: + fail(f"expanded {group} matrix has {len(include)} jobs; GitHub permits at most 256") + if not include: + fail(f"group {group} contains no tasks") + return {"include": include}, math.ceil(estimated_total) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--plan", type=Path, required=True) + parser.add_argument("--group", choices=("fast", "full"), default="fast") + parser.add_argument("--github-output", type=Path) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + plan = load_plan(args.plan) + matrix, estimated_total = validate_and_expand(plan, args.group) + except ValueError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + encoded = json.dumps(matrix, separators=(",", ":")) + if args.github_output: + with args.github_output.open("a", encoding="utf-8") as output: + output.write(f"matrix={encoded}\n") + output.write(f"job-count={len(matrix['include'])}\n") + output.write(f"estimated-test-minutes={estimated_total}\n") + print(encoded) + print( + f"OK: {args.group} expands to {len(matrix['include'])} jobs " + f"covering approximately {estimated_total} test-minutes", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/actions/plan/test_plan.py b/.github/actions/plan/test_plan.py new file mode 100755 index 00000000..64c6b30f --- /dev/null +++ b/.github/actions/plan/test_plan.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Regression tests for ci-fleet task-plan expansion.""" + +from __future__ import annotations + +import copy +import json +import unittest +from pathlib import Path + +from plan import validate_and_expand + + +ROOT = Path(__file__).resolve().parents[3] + + +def sample_plan() -> dict: + return json.loads((ROOT / "examples/project/scripts/ci/plan.json").read_text(encoding="utf-8")) + + +class PlanTests(unittest.TestCase): + def assert_rejected(self, plan: dict, expected: str) -> None: + with self.assertRaisesRegex(ValueError, expected): + validate_and_expand(plan, "full") + + def test_example_expands_to_independent_shards(self) -> None: + matrix, total = validate_and_expand(sample_plan(), "full") + self.assertEqual(len(matrix["include"]), 13) + self.assertEqual(total, 45) + + def test_fast_is_a_smaller_group(self) -> None: + matrix, total = validate_and_expand(sample_plan(), "fast") + self.assertEqual(len(matrix["include"]), 6) + self.assertEqual(total, 20) + + def test_five_minute_job_ceiling_is_mandatory(self) -> None: + plan = sample_plan() + plan["max_job_minutes"] = 6 + self.assert_rejected(plan, "must equal 5") + + def test_shard_estimate_cannot_consume_startup_reserve(self) -> None: + plan = sample_plan() + plan["tasks"][0]["estimated_minutes_per_shard"] = 5 + self.assert_rejected(plan, "at most 4") + + def test_duplicate_task_ids_are_rejected(self) -> None: + plan = sample_plan() + plan["tasks"][1]["id"] = plan["tasks"][0]["id"] + self.assert_rejected(plan, "duplicates") + + def test_fast_tasks_must_also_run_in_full(self) -> None: + plan = sample_plan() + plan["tasks"][0]["groups"] = ["fast"] + self.assert_rejected(plan, "whenever it includes fast") + + def test_matrix_limit_is_enforced(self) -> None: + plan = sample_plan() + plan["tasks"][0]["shards"] = 257 + self.assert_rejected(plan, "at most 256") + + +if __name__ == "__main__": + unittest.main() diff --git a/AGENTS.md b/AGENTS.md index 701b77ea..bbbd282d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -30,7 +30,9 @@ Agents modifying this repository or adapting a project MUST apply these document - Prefer ephemeral, one-job runners. - Treat every group of runners sharing one Docker daemon as one security boundary. - Keep normal CI, release, deployment, repository-writing, and internal-network workloads separable. -- Make cleanup concurrency-aware and scoped to a workflow run. +- Make cleanup concurrency-aware and scoped to a workflow run, task, and shard. +- Keep ordinary task-matrix jobs at a five-minute hard timeout and expected test payload at four minutes or less. +- Preserve `fast` and `full` as aggregates, but schedule granular deterministic task shards on the fleet. - Do not use unrestricted global Docker pruning as per-job cleanup. - Pin production dependencies and container images to reviewed versions or digests. - Do not remove or weaken existing required CI until parallel validation and rollback verification are complete. @@ -49,6 +51,8 @@ Runnable changes must eventually include: - cleanup verification after success, failure, cancellation, and timeout; - proof that long-lived controller credentials are unavailable to jobs; - proof that ordinary CI uses read-only permissions; +- task-plan validation and deterministic matrix expansion; +- measured shard duration and five-minute timeout enforcement; - comparison against the existing CI path during migration. ## Change policy diff --git a/README.md b/README.md index 59631f46..be8f8cfa 100644 --- a/README.md +++ b/README.md @@ -34,14 +34,16 @@ flowchart TD ## Project contract -Every participating project must expose: +Every participating project publishes a task plan and a sharding-aware Docker entrypoint: ```bash -./scripts/ci/run.sh fast -./scripts/ci/run.sh full +./scripts/ci/run.sh unit --shard 1/4 +./scripts/ci/run.sh integration --shard 2/3 ``` -Those commands must execute project validation inside project-owned containers. The fleet runner image does not carry project runtimes. +The fleet expands `scripts/ci/plan.json` into one GitHub job per task shard. Ordinary jobs have a hard five-minute timeout and target no more than four minutes of test payload. With sufficient independent work and available workers, total wall-clock time approaches the slowest shard rather than the sum of all tests. + +`./scripts/ci/run.sh fast` and `full` remain aggregate local commands. All application validation still executes inside project-owned containers; the fleet runner image does not carry project runtimes. The mandatory rules are defined in the [Project CI Standard](docs/PROJECT-STANDARD.md). Existing projects follow [Migrating Existing CI](docs/MIGRATING-EXISTING-CI.md) and must complete the [Compliance Checklist](docs/COMPLIANCE-CHECKLIST.md). @@ -109,6 +111,8 @@ The scaffold is validated in this repository before being published as the stand ### Copyable examples - [Experimental read-only workflow](examples/workflows/experimental-smoke.yml.example) +- [Five-minute parallel matrix workflow](examples/workflows/parallel-ci.yml.example) +- [Project task plan](examples/project/scripts/ci/plan.json) - [Private-repository live pilot workflow](examples/workflows/live-pilot.yml.example) - [Standard project entrypoint](examples/project/scripts/ci/run.sh) - [Isolated Compose project](examples/project/compose.ci.yaml) diff --git a/docs/COMPLIANCE-CHECKLIST.md b/docs/COMPLIANCE-CHECKLIST.md index 193a6ac9..b6f49871 100644 --- a/docs/COMPLIANCE-CHECKLIST.md +++ b/docs/COMPLIANCE-CHECKLIST.md @@ -4,15 +4,20 @@ A project MUST complete this checklist before its existing required CI is moved ## Project contract -- [ ] `scripts/ci/run.sh fast` exists and is executable. -- [ ] `scripts/ci/run.sh full` exists when the project has integration tests. -- [ ] Both commands run project validation inside project-owned containers. +- [ ] `scripts/ci/plan.json` declares every ordinary task and deterministic shard count. +- [ ] `scripts/ci/run.sh --shard INDEX/TOTAL` exists and is executable. +- [ ] `scripts/ci/run.sh fast` and `full` remain working aggregate local commands. +- [ ] Every task/shard runs project validation inside project-owned containers. +- [ ] Every ordinary matrix job has `timeout-minutes: 5`. +- [ ] Expected test payload per shard is four minutes or less. +- [ ] Fast tasks are included in the full group. +- [ ] Task IDs are unique and the expanded matrix stays within 256 jobs. - [ ] The runner host does not provide the project language runtime. -- [ ] Local and CI execution use the same entrypoint. +- [ ] Local aggregate and CI shard execution use the same task implementations. ## Isolation -- [ ] Compose project names are unique per run and attempt. +- [ ] Compose project names are unique per run, attempt, task, and shard. - [ ] No fixed `container_name` exists. - [ ] No fixed host port is published. - [ ] Tests use internal service DNS where possible. @@ -49,7 +54,7 @@ A project MUST complete this checklist before its existing required CI is moved - [ ] Manual experimental run passed. - [ ] Old and new CI ran on the same commits. - [ ] Results and artifacts matched. -- [ ] Resource consumption was recorded. +- [ ] Resource consumption, total test-minutes, shard durations, and wall-clock duration were recorded. - [ ] Forced failure cleanup passed. - [ ] Cancellation cleanup passed. - [ ] Existing required checks remained available during validation. diff --git a/docs/MIGRATING-EXISTING-CI.md b/docs/MIGRATING-EXISTING-CI.md index 2f98415e..fec24d3a 100644 --- a/docs/MIGRATING-EXISTING-CI.md +++ b/docs/MIGRATING-EXISTING-CI.md @@ -51,16 +51,22 @@ Create or adapt the project's test image so the runner host no longer supplies t The runner host should not determine whether a project needs Node 22, PHP 8.4, PostgreSQL 16, or a future runtime. Those versions belong to the project. -## Step 4: Add the standard adapter +## Step 4: Add the task plan and standard adapter Add: ```text +scripts/ci/plan.json +scripts/ci/run.sh --shard INDEX/TOTAL scripts/ci/run.sh fast scripts/ci/run.sh full ``` -The adapter may call existing scripts. This allows a mature Dockerized project to preserve working orchestration while presenting the same interface as a newly containerized project. +Inventory the existing suite by test-minutes, then divide it into independently runnable tasks and deterministic shards. Begin with measured or conservative estimates. Each shard SHOULD target no more than four minutes of test payload and MUST run inside a five-minute job timeout. + +The aggregate `fast` and `full` commands may call existing scripts. Fleet workflows expand `plan.json` and execute individual task shards so additional workers reduce wall-clock time. + +Do not split solely by test count when durations differ materially. Use historical timing data as it becomes available and rebalance shards that repeatedly approach the ceiling. ## Step 5: Eliminate shared-host collisions @@ -69,7 +75,7 @@ Before using a shared host: - remove fixed `container_name` values; - remove fixed host port bindings; - use service DNS on internal Compose networks; -- derive the Compose project name from the workflow run; +- derive the Compose project name from the workflow run, attempt, task, and shard; - make caches explicitly namespaced; - ensure cleanup targets only the current run. @@ -81,7 +87,7 @@ The first workflow MUST: - use the experimental runner label; - declare `permissions: contents: read`; - set a timeout; -- run one non-destructive suite; +- run one non-destructive task shard inside the five-minute ceiling; - leave the existing required workflow unchanged. See [the experimental example](../examples/workflows/experimental-smoke.yml.example). @@ -92,7 +98,8 @@ Run both paths on the same commits and compare: - pass/fail results; - generated artifacts; -- suite duration; +- total test-minutes, matrix job count, shard balance, and wall-clock duration; +- any shard approaching or exceeding the five-minute ceiling; - CPU, memory, disk, and cache growth; - containers, networks, volumes, and workspaces remaining afterward; - behavior after cancellation and forced failure. diff --git a/docs/PROJECT-STANDARD.md b/docs/PROJECT-STANDARD.md index 48330e30..24939b02 100644 --- a/docs/PROJECT-STANDARD.md +++ b/docs/PROJECT-STANDARD.md @@ -6,29 +6,50 @@ The key words **MUST**, **MUST NOT**, **REQUIRED**, **SHOULD**, and **SHOULD NOT ## Required repository interface -Every participating project MUST provide one executable entrypoint: +Every participating project MUST provide: ```text -scripts/ci/run.sh +scripts/ci/plan.json +scripts/ci/run.sh --shard INDEX/TOTAL ``` -Supported suites SHOULD include: +The task plan MUST declare named, independently runnable tasks, their `fast` and/or `full` group membership, deterministic shard counts, and expected minutes per shard. Task IDs MUST NOT use the reserved aggregate names `fast` or `full`. -- `fast`: syntax, formatting, build, unit tests, and short smoke validation; -- `full`: integration tests, migrations, database validation, and complete behavior tests. +Projects MUST also preserve these aggregate developer interfaces: -A project MAY support additional suites, but the shared fleet MUST NOT contain project-specific test logic. +```bash +./scripts/ci/run.sh fast +./scripts/ci/run.sh full +``` + +The fleet MUST schedule the named task shards, not the aggregate command. Aggregate commands exist for local reproduction, migration compatibility, and constrained environments with only one worker. -Existing scripts do not need to be discarded. The standard entrypoint MAY delegate to an established project script. +Existing scripts do not need to be discarded. The standard entrypoint MAY delegate to established project scripts, but the shared fleet MUST NOT contain project-specific test logic. ```mermaid flowchart LR - A["Reusable workflow"] --> B["scripts/ci/run.sh"] - B --> C["Project Dockerfile"] - B --> D["Project services"] - B --> E["Project tests"] + A["plan.json"] --> B["GitHub matrix"] + B --> C["task A"] + B --> D["task B: shard 1/N"] + B --> E["task B: shard 2/N"] + C --> F["Project Docker environment"] + D --> F + E --> F ``` +## Five-minute scheduling objective + +Ordinary CI MUST target a wall-clock duration of five minutes or less when sufficient workers are available. + +- Every task-matrix job MUST set `timeout-minutes: 5`. +- Expected test payload per shard MUST be four minutes or less, reserving time for checkout, container startup, and reporting. +- Sharding MUST be deterministic: the same revision and `INDEX/TOTAL` pair select the same work. +- Tasks and shards SHOULD be balanced using measured historical duration rather than test count. +- Projects MUST split, optimize, or move any indivisible test that cannot fit the ordinary five-minute job ceiling. +- A task plan MUST expand to no more than GitHub's 256-job matrix limit. + +Forty-five test-minutes divided among nine workers is a theoretical five-minute lower bound. Real plans generally need more than nine shards because job setup consumes part of the ceiling and test work is not perfectly balanced. Adding workers reduces wall-clock time only while runnable shards remain queued. + ## Host independence A project CI job: @@ -43,7 +64,7 @@ A project CI job: A project: -- MUST use a unique Compose project name derived from repository, workflow run ID, and run attempt; +- MUST use a unique Compose project name derived from repository, workflow run ID, run attempt, task, and shard; - MUST NOT set fixed `container_name` values; - MUST NOT publish fixed host ports; - SHOULD run tests against service names on an internal Compose network; @@ -56,12 +77,12 @@ A project: Recommended project name construction: ```bash -repo_slug="${GITHUB_REPOSITORY#*/}" -raw_name="ci-${repo_slug}-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" -COMPOSE_PROJECT_NAME="$(printf '%s' "$raw_name" | - tr '[:upper:]' '[:lower:]' | - tr -cs 'a-z0-9_-' '-' | - cut -c1-63)" +repository="${GITHUB_REPOSITORY:-local/project}" +repo_component="$(printf '%s' "${repository#*/}" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9_-' '-' | cut -c1-12)" +task_component="$(printf '%s' "${CI_FLEET_TASK:-aggregate}" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9_-' '-' | cut -c1-12)" +shard_component="${CI_FLEET_SHARD_INDEX:-1}of${CI_FLEET_SHARD_TOTAL:-1}" +raw_name="ci-${repo_component}-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${task_component}-${shard_component}" +COMPOSE_PROJECT_NAME="$(printf '%s' "$raw_name" | tr -cs 'a-z0-9_-' '-' | cut -c1-63)" export COMPOSE_PROJECT_NAME ``` @@ -87,7 +108,7 @@ Ordinary CI: - MUST declare `permissions: contents: read`; - MUST NOT receive deployment, release, production, or internal-network credentials; - MUST NOT push branches, tags, releases, packages, or commits; -- MUST set a job timeout; +- MUST set `timeout-minutes: 5` on every ordinary task-matrix job; - SHOULD use concurrency controls appropriate to the project; - MUST treat pull-request code as untrusted unless repository policy explicitly establishes otherwise. @@ -122,21 +143,25 @@ flowchart TD ## Reproducibility -A project MUST document a local command that exercises the same entrypoint used by GitHub: +A project MUST document both aggregate local commands and a representative direct shard: ```bash ./scripts/ci/run.sh fast ./scripts/ci/run.sh full +./scripts/ci/run.sh unit --shard 1/4 ``` -CI-only behavior SHOULD be limited to run identity, artifact upload, and GitHub status reporting. +Local and CI execution MUST use the same project image and task implementation. CI-only behavior SHOULD be limited to matrix selection, run identity, artifact upload, and GitHub status reporting. ## Required verification before migration A project is not compliant until all of these pass: -- fast suite locally through Docker; -- full suite locally through Docker; +- task plan validation and deterministic matrix expansion; +- every declared task/shard through Docker; +- fast aggregate locally through Docker; +- full aggregate locally through Docker; +- measured evidence that ordinary shards fit the five-minute ceiling; - manual experimental fleet run; - parallel old/new CI comparison; - forced failure cleanup; diff --git a/examples/project/scripts/ci/plan.json b/examples/project/scripts/ci/plan.json new file mode 100644 index 00000000..d97bd31b --- /dev/null +++ b/examples/project/scripts/ci/plan.json @@ -0,0 +1,16 @@ +{ + "$schema": "./plan.schema.json", + "schema_version": 1, + "target_wall_clock_minutes": 5, + "max_job_minutes": 5, + "shard_target_minutes": 4, + "tasks": [ + {"id": "lint", "groups": ["fast", "full"], "shards": 1, "estimated_minutes_per_shard": 2}, + {"id": "syntax", "groups": ["fast", "full"], "shards": 1, "estimated_minutes_per_shard": 2}, + {"id": "unit", "groups": ["fast", "full"], "shards": 4, "estimated_minutes_per_shard": 4}, + {"id": "integration", "groups": ["full"], "shards": 3, "estimated_minutes_per_shard": 4}, + {"id": "database", "groups": ["full"], "shards": 2, "estimated_minutes_per_shard": 4}, + {"id": "browser", "groups": ["full"], "shards": 1, "estimated_minutes_per_shard": 4}, + {"id": "security", "groups": ["full"], "shards": 1, "estimated_minutes_per_shard": 1} + ] +} diff --git a/examples/project/scripts/ci/plan.schema.json b/examples/project/scripts/ci/plan.schema.json new file mode 100644 index 00000000..6f4ea470 --- /dev/null +++ b/examples/project/scripts/ci/plan.schema.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "ci-fleet project task plan", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "target_wall_clock_minutes", "max_job_minutes", "shard_target_minutes", "tasks"], + "properties": { + "$schema": {"type": "string"}, + "schema_version": {"const": 1}, + "target_wall_clock_minutes": {"const": 5}, + "max_job_minutes": {"const": 5}, + "shard_target_minutes": {"type": "integer", "minimum": 1, "maximum": 4}, + "tasks": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "groups", "shards", "estimated_minutes_per_shard"], + "properties": { + "id": {"type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,62}$", "not": {"enum": ["fast", "full"]}}, + "groups": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"enum": ["fast", "full"]}}, + "shards": {"type": "integer", "minimum": 1, "maximum": 256}, + "estimated_minutes_per_shard": {"type": "number", "exclusiveMinimum": 0, "maximum": 4} + } + } + } + } +} diff --git a/examples/project/scripts/ci/run.sh b/examples/project/scripts/ci/run.sh old mode 100644 new mode 100755 index a73519c4..1c5eccea --- a/examples/project/scripts/ci/run.sh +++ b/examples/project/scripts/ci/run.sh @@ -1,17 +1,53 @@ #!/usr/bin/env bash set -Eeuo pipefail -suite="${1:-}" -case "$suite" in - fast|full) ;; +task="${1:-}" +shift || true + +run_aggregate() { + local spec aggregate_task shard + for spec in "$@"; do + aggregate_task=${spec%%:*} + shard=${spec#*:} + "$0" "$aggregate_task" --shard "$shard" + done +} + +case "$task" in + fast) + run_aggregate lint:1/1 syntax:1/1 unit:1/4 unit:2/4 unit:3/4 unit:4/4 + exit 0 + ;; + full) + run_aggregate \ + lint:1/1 syntax:1/1 \ + unit:1/4 unit:2/4 unit:3/4 unit:4/4 \ + integration:1/3 integration:2/3 integration:3/3 \ + database:1/2 database:2/2 browser:1/1 security:1/1 + exit 0 + ;; + lint|syntax|unit|integration|database|browser|security) ;; *) - echo "usage: $0 " >&2 + echo "usage: $0 [--shard INDEX/TOTAL]" >&2 exit 64 ;; esac -repo_slug="${GITHUB_REPOSITORY#*/}" -raw_name="ci-${repo_slug}-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}" +if [[ "${1:-}" != "--shard" || ! "${2:-}" =~ ^([1-9][0-9]*)/([1-9][0-9]*)$ || -n "${3:-}" ]]; then + echo "usage: $0 ${task} --shard INDEX/TOTAL" >&2 + exit 64 +fi +shard_index=${BASH_REMATCH[1]} +shard_total=${BASH_REMATCH[2]} +if (( shard_index > shard_total )); then + echo "shard index cannot exceed shard total" >&2 + exit 64 +fi + +repository="${GITHUB_REPOSITORY:-local/project}" +repo_component="$(printf '%s' "${repository#*/}" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9_-' '-' | cut -c1-12)" +task_component="$(printf '%s' "$task" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9_-' '-' | cut -c1-12)" +raw_name="ci-${repo_component}-${GITHUB_RUN_ID:-local}-${GITHUB_RUN_ATTEMPT:-1}-${task_component}-${shard_index}of${shard_total}" COMPOSE_PROJECT_NAME="$(printf '%s' "$raw_name" | tr '[:upper:]' '[:lower:]' | tr -cs 'a-z0-9_-' '-' | @@ -30,12 +66,16 @@ trap cleanup EXIT INT TERM "${compose[@]}" build --pull test -case "$suite" in - fast) - "${compose[@]}" run --rm --no-deps test ./scripts/test-fast.sh - ;; - full) +case "$task" in + integration|database|browser) "${compose[@]}" up -d --wait database - "${compose[@]}" run --rm test ./scripts/test-full.sh ;; esac + +# Replace these sample scripts with project-owned task implementations. A +# sharding-aware test framework should receive both values deterministically. +"${compose[@]}" run --rm \ + -e "CI_FLEET_TASK=${task}" \ + -e "CI_FLEET_SHARD_INDEX=${shard_index}" \ + -e "CI_FLEET_SHARD_TOTAL=${shard_total}" \ + test "./scripts/test-${task}.sh" "$shard_index" "$shard_total" diff --git a/examples/workflows/parallel-ci.yml.example b/examples/workflows/parallel-ci.yml.example new file mode 100644 index 00000000..a4a14cf8 --- /dev/null +++ b/examples/workflows/parallel-ci.yml.example @@ -0,0 +1,50 @@ +# Copy into a private project as .github/workflows/ci.yml. +# Replace CI_FLEET_COMMIT_SHA with a reviewed immutable ci-fleet commit SHA. + +name: Parallel container CI + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + plan: + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + matrix: ${{ steps.plan.outputs.matrix }} + steps: + - name: Check out project + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - id: plan + name: Validate and expand project task plan + uses: RandomDevelopment/ci-fleet/.github/actions/plan@CI_FLEET_COMMIT_SHA + with: + group: full + + test: + needs: plan + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.matrix) }} + runs-on: [self-hosted, linux, x64, docker-ci] + timeout-minutes: 5 + steps: + - name: Check out project + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Run isolated task shard + run: >- + ./scripts/ci/run.sh + "${{ matrix.task }}" + --shard "${{ matrix.shard }}/${{ matrix.shards }}" diff --git a/scripts/validate.sh b/scripts/validate.sh index d2a622b0..7710279b 100755 --- a/scripts/validate.sh +++ b/scripts/validate.sh @@ -5,6 +5,10 @@ repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) cd "$repo_root" for script in scripts/*.sh examples/project/scripts/ci/*.sh; do bash -n "$script"; done +python3 -m py_compile .github/actions/plan/plan.py .github/actions/plan/test_plan.py +python3 .github/actions/plan/test_plan.py +python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group fast >/dev/null +python3 .github/actions/plan/plan.py --plan examples/project/scripts/ci/plan.json --group full >/dev/null tmp=$(mktemp) trap 'rm -f "$tmp"' EXIT diff --git a/templates/config-repository/AGENTS.md b/templates/config-repository/AGENTS.md index c2408e00..d340d357 100644 --- a/templates/config-repository/AGENTS.md +++ b/templates/config-repository/AGENTS.md @@ -20,4 +20,6 @@ Before committing configuration changes, run: - CI runner hosts and application deployment hosts are separate roles. - Image promotion uses immutable digests; do not rebuild separately for production. - Reusable workflows must be pinned to a full reviewed commit SHA. +- Keep ordinary task jobs at a five-minute hard ceiling and expected shard payload at four minutes or less. +- Preserve deterministic task/shard isolation; Compose identity must include task and shard as well as run identity. - Update schema and validator together. diff --git a/templates/config-repository/README.md b/templates/config-repository/README.md index 161ede75..9c536c13 100644 --- a/templates/config-repository/README.md +++ b/templates/config-repository/README.md @@ -8,7 +8,7 @@ It does **not** contain runner registration tokens, deploy credentials, private flowchart LR E[Public ci-fleet engine] -->|pinned reusable workflow| P[Private project repos] C[Private config created from this template] -->|logical policy| P - P -->|fast / full| R[Trusted Docker CI pool] + P -->|task matrix| R[Trusted Docker CI pool] P -->|approved image digest| D[Development hosts] P -->|manual approval + image digest| X[Production hosts] S[GitHub Environments / host secret store] -. secret values .-> P @@ -42,7 +42,9 @@ The initializer refuses to replace a configured file unless `--force` is explici ## Hard rules - Public repositories never receive access to the trusted self-hosted runner pool. -- Every project implements exactly `./scripts/ci/run.sh fast` and `./scripts/ci/run.sh full`; each command runs the project's own Docker-defined test environment. +- Every project publishes `scripts/ci/plan.json` and implements `./scripts/ci/run.sh --shard INDEX/TOTAL` in its own Docker-defined test environment. +- `./scripts/ci/run.sh fast` and `full` remain aggregate developer commands; fleet scheduling expands their named tasks across available workers. +- Every matrix job has a five-minute hard timeout, while expected test payload targets four minutes or less to reserve startup and reporting time. - CI runner pools and deployment host groups are separate trust roles. - Production deployment is manual and requires GitHub Environment approval. - Reusable workflows and third-party actions are pinned to immutable commits. @@ -51,6 +53,22 @@ The initializer refuses to replace a configured file unless `--force` is explici `fleet.schema.json` provides editor completion and structural documentation. `scripts/validate.py` is the authoritative dependency-free policy check, including relationships JSON Schema cannot express clearly. +## Five-minute parallelism contract + +Projects divide their total test-minutes into independent named tasks and deterministic shards. Forty-five test-minutes require at least nine perfectly balanced five-minute jobs in theory. In practice, projects should create additional shards targeting four minutes of test payload so checkout, image preparation, and reporting remain inside the five-minute job ceiling. + +```mermaid +flowchart LR + P[plan.json] --> M[GitHub matrix] + M --> A[lint] + M --> B[unit 1/4] + M --> C[unit 2/4] + M --> D[integration 1/3] + M --> E[other independent shards] +``` + +Adding workers reduces wall-clock time only while independent shards remain queued. A genuinely indivisible test longer than five minutes must be optimized, split, or moved into an explicitly slower scheduled class outside ordinary CI. + ## Repository map | Path | Purpose | diff --git a/templates/config-repository/examples/multi-host/fleet.json b/templates/config-repository/examples/multi-host/fleet.json index 6690e2cd..603ee21f 100644 --- a/templates/config-repository/examples/multi-host/fleet.json +++ b/templates/config-repository/examples/multi-host/fleet.json @@ -1,6 +1,6 @@ { "$schema": "../../fleet.schema.json", - "schema_version": 1, + "schema_version": 2, "organization": { "slug": "sample-company", "registry": "ghcr.io/sample-company", @@ -54,9 +54,16 @@ "repository": "sample-company/api", "image": "ghcr.io/sample-company/api", "ci_pool": "trusted-ci", - "ci_entrypoints": { - "fast": "./scripts/ci/run.sh fast", - "full": "./scripts/ci/run.sh full" + "ci_contract": { + "runner_entrypoint": "./scripts/ci/run.sh", + "task_plan": "./scripts/ci/plan.json", + "aggregate_entrypoints": { + "fast": "./scripts/ci/run.sh fast", + "full": "./scripts/ci/run.sh full" + }, + "target_wall_clock_minutes": 5, + "max_job_minutes": 5, + "shard_target_minutes": 4 }, "deployments": ["development", "staging", "production"] }, @@ -64,9 +71,16 @@ "repository": "sample-company/web", "image": "ghcr.io/sample-company/web", "ci_pool": "trusted-ci", - "ci_entrypoints": { - "fast": "./scripts/ci/run.sh fast", - "full": "./scripts/ci/run.sh full" + "ci_contract": { + "runner_entrypoint": "./scripts/ci/run.sh", + "task_plan": "./scripts/ci/plan.json", + "aggregate_entrypoints": { + "fast": "./scripts/ci/run.sh fast", + "full": "./scripts/ci/run.sh full" + }, + "target_wall_clock_minutes": 5, + "max_job_minutes": 5, + "shard_target_minutes": 4 }, "deployments": ["development", "production"] } diff --git a/templates/config-repository/fleet.json b/templates/config-repository/fleet.json index 36f33c65..0a3177aa 100644 --- a/templates/config-repository/fleet.json +++ b/templates/config-repository/fleet.json @@ -1,6 +1,6 @@ { "$schema": "./fleet.schema.json", - "schema_version": 1, + "schema_version": 2, "organization": { "slug": "example-org", "registry": "ghcr.io/example-org", @@ -44,9 +44,16 @@ "repository": "example-org/example-app", "image": "ghcr.io/example-org/example-app", "ci_pool": "trusted-ci", - "ci_entrypoints": { - "fast": "./scripts/ci/run.sh fast", - "full": "./scripts/ci/run.sh full" + "ci_contract": { + "runner_entrypoint": "./scripts/ci/run.sh", + "task_plan": "./scripts/ci/plan.json", + "aggregate_entrypoints": { + "fast": "./scripts/ci/run.sh fast", + "full": "./scripts/ci/run.sh full" + }, + "target_wall_clock_minutes": 5, + "max_job_minutes": 5, + "shard_target_minutes": 4 }, "deployments": ["development", "production"] } diff --git a/templates/config-repository/fleet.schema.json b/templates/config-repository/fleet.schema.json index f00e36d3..9dd012e8 100644 --- a/templates/config-repository/fleet.schema.json +++ b/templates/config-repository/fleet.schema.json @@ -7,7 +7,7 @@ "required": ["schema_version", "organization", "runner_pools", "host_groups", "environments", "projects"], "properties": { "$schema": {"type": "string"}, - "schema_version": {"const": 1}, + "schema_version": {"const": 2}, "organization": { "type": "object", "additionalProperties": false, @@ -75,18 +75,30 @@ "project": { "type": "object", "additionalProperties": false, - "required": ["repository", "image", "ci_pool", "ci_entrypoints", "deployments"], + "required": ["repository", "image", "ci_pool", "ci_contract", "deployments"], "properties": { "repository": {"type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$"}, "image": {"type": "string", "pattern": "^[a-z0-9.-]+/[a-z0-9._/-]+$"}, "ci_pool": {"type": "string"}, - "ci_entrypoints": { + "ci_contract": { "type": "object", "additionalProperties": false, - "required": ["fast", "full"], + "required": ["runner_entrypoint", "task_plan", "aggregate_entrypoints", "target_wall_clock_minutes", "max_job_minutes", "shard_target_minutes"], "properties": { - "fast": {"const": "./scripts/ci/run.sh fast"}, - "full": {"const": "./scripts/ci/run.sh full"} + "runner_entrypoint": {"const": "./scripts/ci/run.sh"}, + "task_plan": {"const": "./scripts/ci/plan.json"}, + "aggregate_entrypoints": { + "type": "object", + "additionalProperties": false, + "required": ["fast", "full"], + "properties": { + "fast": {"const": "./scripts/ci/run.sh fast"}, + "full": {"const": "./scripts/ci/run.sh full"} + } + }, + "target_wall_clock_minutes": {"const": 5}, + "max_job_minutes": {"const": 5}, + "shard_target_minutes": {"type": "integer", "minimum": 1, "maximum": 4} } }, "deployments": {"type": "array", "minItems": 1, "uniqueItems": true, "items": {"type": "string"}} diff --git a/templates/config-repository/scripts/init.py b/templates/config-repository/scripts/init.py index 4d52134a..0a761d61 100755 --- a/templates/config-repository/scripts/init.py +++ b/templates/config-repository/scripts/init.py @@ -54,7 +54,7 @@ def main() -> int: config = { "$schema": str((ROOT / "fleet.schema.json").resolve()) if output.parent != ROOT else "./fleet.schema.json", - "schema_version": 1, + "schema_version": 2, "organization": { "slug": args.organization, "registry": registry, @@ -92,9 +92,16 @@ def main() -> int: "repository": repository, "image": f"{registry}/{args.project}", "ci_pool": "trusted-ci", - "ci_entrypoints": { - "fast": "./scripts/ci/run.sh fast", - "full": "./scripts/ci/run.sh full", + "ci_contract": { + "runner_entrypoint": "./scripts/ci/run.sh", + "task_plan": "./scripts/ci/plan.json", + "aggregate_entrypoints": { + "fast": "./scripts/ci/run.sh fast", + "full": "./scripts/ci/run.sh full", + }, + "target_wall_clock_minutes": 5, + "max_job_minutes": 5, + "shard_target_minutes": 4, }, "deployments": ["development", "production"], } diff --git a/templates/config-repository/scripts/test_policy.py b/templates/config-repository/scripts/test_policy.py index 5fbc3ac8..417d1ec1 100755 --- a/templates/config-repository/scripts/test_policy.py +++ b/templates/config-repository/scripts/test_policy.py @@ -68,8 +68,23 @@ def test_strict_mode_rejects_unchanged_example(self) -> None: def test_nonstandard_ci_entrypoint_is_rejected(self) -> None: config = copy.deepcopy(reference_config()) - config["projects"]["example-app"]["ci_entrypoints"]["fast"] = "npm test" - self.assert_rejected(config, "standard fast entrypoint") + config["projects"]["example-app"]["ci_contract"]["aggregate_entrypoints"]["fast"] = "npm test" + self.assert_rejected(config, "standard aggregate fast entrypoint") + + def test_job_ceiling_above_five_minutes_is_rejected(self) -> None: + config = copy.deepcopy(reference_config()) + config["projects"]["example-app"]["ci_contract"]["max_job_minutes"] = 10 + self.assert_rejected(config, "five-minute hard job ceiling") + + def test_shard_target_must_reserve_startup_time(self) -> None: + config = copy.deepcopy(reference_config()) + config["projects"]["example-app"]["ci_contract"]["shard_target_minutes"] = 5 + self.assert_rejected(config, "reserve startup time") + + def test_standard_task_plan_path_is_required(self) -> None: + config = copy.deepcopy(reference_config()) + config["projects"]["example-app"]["ci_contract"]["task_plan"] = "ci/custom.json" + self.assert_rejected(config, "standard task-plan path") if __name__ == "__main__": diff --git a/templates/config-repository/scripts/validate.py b/templates/config-repository/scripts/validate.py index faecfb76..f644f657 100755 --- a/templates/config-repository/scripts/validate.py +++ b/templates/config-repository/scripts/validate.py @@ -125,7 +125,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: if not validation.exact_keys(config, "$", required_top, {"$schema"}): return - validation.require(config.get("schema_version") == 1, "$.schema_version", "must equal 1") + validation.require(config.get("schema_version") == 2, "$.schema_version", "must equal 2") organization = config.get("organization") organization_keys = {"slug", "registry", "delivery_engine", "workflow_ref_policy"} @@ -207,7 +207,7 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: for name, project in projects.items(): path = f"$.projects.{name}" validation.require(bool(SLUG.fullmatch(name)), path, "project name must be a lowercase slug") - if not validation.exact_keys(project, path, {"repository", "image", "ci_pool", "ci_entrypoints", "deployments"}): + if not validation.exact_keys(project, path, {"repository", "image", "ci_pool", "ci_contract", "deployments"}): continue repository = project.get("repository") image = project.get("image") @@ -217,10 +217,28 @@ def validate_config(config: Any, validation: Validation, strict: bool) -> None: validation.require(pool_name in pools, f"{path}.ci_pool", "must reference a declared runner pool") if pool_name in pools and isinstance(pools[pool_name].get("allowed_repositories"), list): validation.require(repository in pools[pool_name]["allowed_repositories"], f"{path}.repository", "must be explicitly allowed by its CI pool") - entrypoints = project.get("ci_entrypoints") - if validation.exact_keys(entrypoints, f"{path}.ci_entrypoints", {"fast", "full"}): - validation.require(entrypoints.get("fast") == "./scripts/ci/run.sh fast", f"{path}.ci_entrypoints.fast", "must use the standard fast entrypoint") - validation.require(entrypoints.get("full") == "./scripts/ci/run.sh full", f"{path}.ci_entrypoints.full", "must use the standard full entrypoint") + contract = project.get("ci_contract") + contract_path = f"{path}.ci_contract" + contract_keys = { + "runner_entrypoint", + "task_plan", + "aggregate_entrypoints", + "target_wall_clock_minutes", + "max_job_minutes", + "shard_target_minutes", + } + if validation.exact_keys(contract, contract_path, contract_keys): + validation.require(contract.get("runner_entrypoint") == "./scripts/ci/run.sh", f"{contract_path}.runner_entrypoint", "must use the standard task runner") + validation.require(contract.get("task_plan") == "./scripts/ci/plan.json", f"{contract_path}.task_plan", "must use the standard task-plan path") + validation.require(contract.get("target_wall_clock_minutes") == 5, f"{contract_path}.target_wall_clock_minutes", "must equal the five-minute fleet goal") + validation.require(contract.get("max_job_minutes") == 5, f"{contract_path}.max_job_minutes", "must enforce a five-minute hard job ceiling") + shard_target = contract.get("shard_target_minutes") + validation.require(type(shard_target) is int and 1 <= shard_target <= 4, f"{contract_path}.shard_target_minutes", "must be between one and four minutes to reserve startup time") + entrypoints = contract.get("aggregate_entrypoints") + aggregate_path = f"{contract_path}.aggregate_entrypoints" + if validation.exact_keys(entrypoints, aggregate_path, {"fast", "full"}): + validation.require(entrypoints.get("fast") == "./scripts/ci/run.sh fast", f"{aggregate_path}.fast", "must use the standard aggregate fast entrypoint") + validation.require(entrypoints.get("full") == "./scripts/ci/run.sh full", f"{aggregate_path}.full", "must use the standard aggregate full entrypoint") deployments = project.get("deployments") validation.require(isinstance(deployments, list) and bool(deployments), f"{path}.deployments", "must be a non-empty list") if isinstance(deployments, list):