diff --git a/.github/workflows/factory-accepted-qualification.yml b/.github/workflows/factory-accepted-qualification.yml new file mode 100644 index 0000000..c935507 --- /dev/null +++ b/.github/workflows/factory-accepted-qualification.yml @@ -0,0 +1,204 @@ +name: Accepted Factory Release Qualification + +on: + workflow_dispatch: + inputs: + controller_sha: + description: Accepted full SHA containing this workflow + required: true + type: string + candidate_sha: + description: Exact clockgrove/factory candidate commit SHA + required: true + type: string + request_run_id: + description: Factory qualification-request run ID + required: true + type: string + request_run_attempt: + description: Factory qualification-request run attempt + required: true + type: string + +permissions: + contents: read + +concurrency: + group: factory-accepted-qualification-${{ inputs.candidate_sha }} + cancel-in-progress: false + +jobs: + qualify: + if: github.repository == 'clockgrove/.github' + runs-on: ubuntu-latest + environment: factory-release + steps: + - name: Bind execution to the accepted workflow commit + env: + DECLARED_CONTROLLER_SHA: ${{ inputs.controller_sha }} + WORKFLOW_SHA: ${{ github.workflow_sha }} + run: | + test "$DECLARED_CONTROLLER_SHA" = "$WORKFLOW_SHA" + case "$WORKFLOW_SHA" in (*[!0-9a-f]*|'') exit 1;; esac + test "${#WORKFLOW_SHA}" -eq 40 + - name: Check out accepted controller exactly + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.controller_sha }} + persist-credentials: false + path: controller + - name: Mint read-only GitHub App installation token + id: app-token + if: vars.FACTORY_FETCH_CREDENTIAL_MODE == 'github-app' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.0.0 + with: + app-id: ${{ vars.FACTORY_QUALIFICATION_APP_ID }} + private-key: ${{ secrets.FACTORY_QUALIFICATION_APP_PRIVATE_KEY }} + owner: clockgrove + repositories: factory + permission-contents: read + - name: Fetch accepted oracle and candidate with transient read credential + env: + CREDENTIAL_MODE: ${{ vars.FACTORY_FETCH_CREDENTIAL_MODE }} + PERSONAL_PAT: ${{ secrets.FACTORY_CANDIDATE_READ_TOKEN }} + APP_TOKEN: ${{ steps.app-token.outputs.token }} + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + run: | + case "$CREDENTIAL_MODE" in + personal-pat) FETCH_TOKEN="$PERSONAL_PAT" ;; + github-app) FETCH_TOKEN="$APP_TOKEN" ;; + *) echo 'Unsupported or unset FACTORY_FETCH_CREDENTIAL_MODE' >&2; exit 1 ;; + esac + test -n "$FETCH_TOKEN" + HEADER_FILE="$RUNNER_TEMP/factory-token-headers" + chmod 700 "$RUNNER_TEMP" + STATUS="$(curl --silent --show-error --output /dev/null --dump-header "$HEADER_FILE" \ + --write-out '%{http_code}' -H "Authorization: Bearer $FETCH_TOKEN" \ + -H 'Accept: application/vnd.github+json' https://api.github.com/repos/clockgrove/factory)" + test "$STATUS" = 200 + SCOPES="$(sed -n 's/^x-oauth-scopes:[[:space:]]*//Ip' "$HEADER_FILE" | tr -d '\r')" + if test -n "$SCOPES" && printf '%s' "$SCOPES" | grep -Eqi '(^|,)[[:space:]]*(repo|workflow|admin:|write:)'; then + echo 'Fetch PAT exposes a classic write-capable OAuth scope' >&2 + exit 1 + fi + rm -f "$HEADER_FILE" + ACCEPTED_SHA="$(python3 -c 'import json; print(json.load(open("controller/factory-controller/policy.json"))["accepted_factory_sha"])')" + AUTH="$(printf 'x-access-token:%s' "$FETCH_TOKEN" | base64 -w0)" + for item in "accepted:$ACCEPTED_SHA" "candidate:$CANDIDATE_SHA"; do + name="${item%%:*}" + sha="${item#*:}" + git init "$name" + git -C "$name" -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $AUTH" \ + fetch --no-tags --depth=1 https://github.com/clockgrove/factory.git "$sha" + git -C "$name" checkout --detach FETCH_HEAD + test "$(git -C "$name" rev-parse HEAD)" = "$sha" + git -C "$name" remote remove origin 2>/dev/null || true + git -C "$name" config --local --unset-all http.https://github.com/.extraheader 2>/dev/null || true + test -z "$(git -C "$name" remote)" + ! grep -Eqi 'authorization|extraheader|x-access-token' "$name/.git/config" + done + unset AUTH FETCH_TOKEN PERSONAL_PAT APP_TOKEN + - name: Run accepted data-only deterministic gate + env: + CONTROLLER_SHA: ${{ inputs.controller_sha }} + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + CREDENTIAL_MODE: ${{ vars.FACTORY_FETCH_CREDENTIAL_MODE }} + run: | + mkdir -p "$RUNNER_TEMP/factory-neutral" + python3 controller/scripts/factory_qualification_controller.py deterministic \ + --policy controller/factory-controller/policy.json \ + --lifecycle controller/factory-controller/lifecycle.json \ + --credential controller/factory-controller/credential.json \ + --credential-mode "$CREDENTIAL_MODE" \ + --accepted-root accepted --candidate-root candidate \ + --controller-sha "$CONTROLLER_SHA" --candidate-sha "$CANDIDATE_SHA" \ + --bundle "$RUNNER_TEMP/factory-neutral/input" \ + --output "$RUNNER_TEMP/deterministic.json" + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.1.0 + with: + node-version: '24' + - name: Install pinned Copilot CLI + run: npm install --global @github/copilot@1.0.54 + - name: Create empty, trusted Copilot configuration + run: | + mkdir -p "$RUNNER_TEMP/copilot-home" "$RUNNER_TEMP/copilot-cache" "$RUNNER_TEMP/factory-neutral/work" + chmod 700 "$RUNNER_TEMP/copilot-home" "$RUNNER_TEMP/copilot-cache" + printf '%s\n' '{"disableAllHooks":true,"enabledPlugins":{},"extraKnownMarketplaces":{},"autoUpdate":false}' \ + > "$RUNNER_TEMP/copilot-home/settings.json" + - name: Evaluate sanitized behavior against accepted oracle + env: + COPILOT_GITHUB_TOKEN: ${{ secrets.FACTORY_QUALIFICATION_COPILOT_TOKEN }} + COPILOT_HOME: ${{ runner.temp }}/copilot-home + COPILOT_CACHE_HOME: ${{ runner.temp }}/copilot-cache + COPILOT_CUSTOM_INSTRUCTIONS_DIRS: '' + ACTIONS_CACHE_URL: '' + ACTIONS_ID_TOKEN_REQUEST_TOKEN: '' + ACTIONS_ID_TOKEN_REQUEST_URL: '' + ACTIONS_RESULTS_URL: '' + ACTIONS_RUNTIME_TOKEN: '' + GH_TOKEN: '' + GITHUB_TOKEN: '' + INPUT_ROOT: ${{ runner.temp }}/factory-neutral/input + WORK_ROOT: ${{ runner.temp }}/factory-neutral/work + run: | + test -n "$COPILOT_GITHUB_TOKEN" + for MODEL in gpt-5.6-sol gpt-5.6-terra claude-opus-5; do + OUT="$RUNNER_TEMP/semantic-$MODEL.md" + PROMPT="$RUNNER_TEMP/prompt-$MODEL.txt" + cat > "$PROMPT" < "$OUT" + ) + test "$(grep -Ec '^FACTORY_QUALIFICATION: (PASS|FAIL)$' "$OUT")" -eq 1 + grep -qx 'FACTORY_QUALIFICATION: PASS' "$OUT" + done + - name: Create SHA-bound release qualification record + env: + CONTROLLER_SHA: ${{ inputs.controller_sha }} + CANDIDATE_SHA: ${{ inputs.candidate_sha }} + REQUEST_RUN_ID: ${{ inputs.request_run_id }} + REQUEST_RUN_ATTEMPT: ${{ inputs.request_run_attempt }} + run: | + python3 - <<'PY' + import json, os, pathlib + request = { + 'schema_version': 1, + 'controller_repository': 'clockgrove/.github', + 'controller_sha': os.environ['CONTROLLER_SHA'], + 'candidate_repository': 'clockgrove/factory', + 'candidate_sha': os.environ['CANDIDATE_SHA'], + 'request_run_id': os.environ['REQUEST_RUN_ID'], + 'request_run_attempt': os.environ['REQUEST_RUN_ATTEMPT'], + } + pathlib.Path('/tmp/qualification-request.json').write_text(json.dumps(request, sort_keys=True, indent=2) + '\n') + PY + python3 controller/scripts/factory_qualification_controller.py record \ + --deterministic "$RUNNER_TEMP/deterministic.json" \ + --request /tmp/qualification-request.json \ + --evidence "$RUNNER_TEMP/semantic-gpt-5.6-sol.md" \ + --evidence "$RUNNER_TEMP/semantic-gpt-5.6-terra.md" \ + --evidence "$RUNNER_TEMP/semantic-claude-opus-5.md" \ + --controller-sha "$CONTROLLER_SHA" --candidate-sha "$CANDIDATE_SHA" \ + --run-id "$GITHUB_RUN_ID" --run-attempt "$GITHUB_RUN_ATTEMPT" \ + --output "$RUNNER_TEMP/factory-qualification-record.json" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: factory-qualification-${{ inputs.candidate_sha }} + path: | + ${{ runner.temp }}/factory-qualification-record.json + ${{ runner.temp }}/semantic-*.md + if-no-files-found: error diff --git a/.github/workflows/validate-factory-controller.yml b/.github/workflows/validate-factory-controller.yml new file mode 100644 index 0000000..c9c6d40 --- /dev/null +++ b/.github/workflows/validate-factory-controller.yml @@ -0,0 +1,94 @@ +name: Validate accepted Factory controller + +on: + pull_request: + paths: + - factory-controller/** + - scripts/factory_qualification_controller.py + - tests/** + - .github/workflows/validate-factory-controller.yml + - .github/workflows/factory-accepted-qualification.yml + push: + branches: [main] + paths: + - factory-controller/** + - scripts/factory_qualification_controller.py + - tests/** + - .github/workflows/validate-factory-controller.yml + - .github/workflows/factory-accepted-qualification.yml + +permissions: + contents: read + +jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - run: python3 -m unittest discover -s tests -p 'test_*.py' -v + + secretless-sandbox-integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Build immutable-base sandbox + run: | + docker pull python@sha256:e16ab55c341bfd0e7da665bc2d48939cff890b43a41867fe6e1f0690638ceb7c + docker build --network none --pull=false --tag factory-accepted-controller -f factory-controller/Dockerfile . + - name: Valid child is accepted by outer supervisor + run: | + docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges --pids-limit 64 --memory 256m --cpus 1 \ + --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --volume "$PWD/tests/fixtures:/fixture:ro" \ + factory-accepted-controller fixture --executable /fixture/valid.py \ + --expected '{"status":"ok"}' --expect-decision pass + - name: Exit-zero and stdout-spoof children fail closed + run: | + for fixture in os_exit.py stdout_spoof.py; do + docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges --pids-limit 64 --memory 256m --cpus 1 \ + --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --volume "$PWD/tests/fixtures:/fixture:ro" \ + factory-accepted-controller fixture --executable "/fixture/$fixture" \ + --expected '{"status":"ok"}' --expect-decision fail + done + - name: Candidate hook configuration is inert + run: | + docker run --rm --network none --read-only --cap-drop ALL \ + --security-opt no-new-privileges --pids-limit 64 --memory 256m --cpus 1 \ + --tmpfs /tmp:rw,noexec,nosuid,size=16m \ + --volume "$PWD/tests/fixtures/hook:/fixture:ro" \ + factory-accepted-controller fixture --executable /fixture/payload.py \ + --expected '{"status":"ok"}' --expect-decision pass + + workflow-contract: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Enforce trusted-controller boundaries + run: | + python3 - <<'PY' + from pathlib import Path + workflow = Path('.github/workflows/factory-accepted-qualification.yml').read_text() + for required in ( + 'environment: factory-release', 'github.workflow_sha', + 'FACTORY_CANDIDATE_READ_TOKEN', 'FACTORY_QUALIFICATION_COPILOT_TOKEN', + 'FACTORY_FETCH_CREDENTIAL_MODE', 'FACTORY_QUALIFICATION_APP_PRIVATE_KEY', + 'actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1', + '--no-custom-instructions', '--disable-builtin-mcps', + '--available-tools=read', '--deny-url', 'COPILOT_HOME', + 'remote remove origin', 'ACTIONS_RUNTIME_TOKEN: \'\'', + ): + if required not in workflow: + raise SystemExit(f'missing controller boundary: {required}') + if 'pull_request_target:' in workflow or '\n pull_request:' in workflow: + raise SystemExit('secret-bearing accepted controller cannot run on pull requests') + print('trusted-controller workflow boundary passed') + PY diff --git a/README.md b/README.md index f086cc2..6e2a6da 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ This repository contains organization-level GitHub configuration and the public Clockgrove organization profile. +It also owns the accepted Factory release-qualification trust root. See +[`docs/FACTORY-QUALIFICATION-CONTROLLER.md`](docs/FACTORY-QUALIFICATION-CONTROLLER.md). + ## Copilot capability marketplace `.github/plugin/marketplace.json` publishes the Clockgrove capability packages used by project repositories: diff --git a/docs/FACTORY-QUALIFICATION-CONTROLLER.md b/docs/FACTORY-QUALIFICATION-CONTROLLER.md new file mode 100644 index 0000000..a8a30fa --- /dev/null +++ b/docs/FACTORY-QUALIFICATION-CONTROLLER.md @@ -0,0 +1,90 @@ +# Factory accepted qualification controller + +This public organization-policy repository is the trust root for Factory release +qualification. Factory candidates cannot change this workflow, its accepted +policy, its lifecycle ledger, or its secrets. + +## Boundary + +`.github/workflows/factory-accepted-qualification.yml` runs only by explicit +dispatch at an exact accepted commit and uses the protected `factory-release` +environment. A read-only fine-grained credential fetches the private Factory +candidate and previously accepted Factory oracle at exact full SHAs. The +credential is scoped to that one step, both remotes and credential configuration +are removed, and candidate content is never executed by the controller. + +The accepted deterministic validator reads the candidate as data under isolated +Python. Semantic evaluation receives only a manifest-bound copy of declared +behavior files, renamed below a neutral non-repository directory. Candidate +hooks, MCP configuration, plugins, settings, instructions, workflows, scripts, +and tests are not copied. Copilot runs from that neutral directory with a clean +`COPILOT_HOME`, all hooks disabled, custom instructions disabled, built-in MCP +disabled, no plugins, read as its only tool, all tool URLs denied, remote export +disabled, and automatic update disabled. Its only network use is the pinned +Copilot client's inference channel; candidate-controlled executable surfaces +have no network path because they are never run. + +The sandbox integration suite proves that the outer trusted supervisor accepts a +valid structured child response but rejects `os._exit(0)` and fake PASS output. +It also mounts an adversarial candidate hook and proves it remains inert. A +child exit code or stdout line is never itself release evidence. + +## Required GitHub configuration + +Create the `factory-release` environment in `clockgrove/.github`. Restrict its +deployment branch/tag policy to the protected controller ref and add required +reviewers if the current GitHub plan exposes that control. Add only: + +- `FACTORY_QUALIFICATION_COPILOT_TOKEN`: a dedicated token with Copilot Requests + read access and no repository write permissions. + +Set the repository variable `FACTORY_FETCH_CREDENTIAL_MODE` to one of the two +accepted modes in `factory-controller/credential.json`: + +- `personal-pat` (the current, non-Enterprise path): add + `FACTORY_CANDIDATE_READ_TOKEN`, a fine-grained personal access token restricted + to `clockgrove/factory` with Contents read and Metadata read only. The controller + checks access to that exact repository and rejects visible classic write/admin + scopes. Fine-grained PAT permission metadata is not introspectable at runtime, + so bootstrap evidence must include a screenshot/export of its repository and + permission selection. No Enterprise-only environment feature is required. +- `github-app`: create a Clockgrove-owned GitHub App installed only on `factory`, + grant Contents read and Metadata read, set `FACTORY_QUALIFICATION_APP_ID`, and + add `FACTORY_QUALIFICATION_APP_PRIVATE_KEY`. The pinned token action mints a + repository-restricted installation token for each run; no long-lived fetch PAT + is then needed. + +Unsupported or unset modes fail closed. The qualification record names the mode, +never the credential. Migration is a reviewed configuration change: install and +test the App, add its ID/key, change the mode variable to `github-app`, run a +qualification, verify its attestation, then revoke the PAT. Rollback reverses the +mode only while the least-privilege PAT remains valid. + +Factory itself receives neither secret. Protect `main` in this public repository +with required pull-request review and passing controller validation checks. + +## Bootstrap and promotion + +The checked-in lifecycle begins at `bootstrap_pending`, so the workflow cannot +issue a qualification record. Two organization owners outside the Factory +candidate-writer set must review the exact bootstrap commit, validate repository +access and secret-name evidence, run the adversarial CI, and record approval in a +GitHub Release. A second reviewed activation change applies `approve_bootstrap`, +records the reviewed trust-root SHA, and changes the ledger to `accepted`. +Factory pins the exact activation commit (which contains that ledger change) in +its secretless request workflow. This two-commit sequence avoids the impossible +and unsafe requirement for a Git commit to contain its own SHA. + +Controller upgrades are qualified under the currently accepted controller. +After `qualify_upgrade`, owners merge the new immutable commit and apply +`activate_upgrade`; Factory updates its pin only in a separate reviewed PR. Old +records and commits are never rewritten. + +## Disable, recover, and revoke + +On suspected compromise, disable the workflow/environment and revoke both tokens +before investigation. Apply `detect_compromise`, rotate credentials, and either +restore the last uncompromised SHA or return to bootstrap. Revocation is an +explicit human change to the lifecycle ledger and a GitHub Release that names the +controller SHA, affected candidate SHAs and run IDs, reason, scope, and replacement. +The `revoked` state is terminal, and revoked SHAs are rejected before evaluation. diff --git a/factory-controller/Dockerfile b/factory-controller/Dockerfile new file mode 100644 index 0000000..f7f819a --- /dev/null +++ b/factory-controller/Dockerfile @@ -0,0 +1,5 @@ +FROM python@sha256:e16ab55c341bfd0e7da665bc2d48939cff890b43a41867fe6e1f0690638ceb7c + +RUN addgroup --system candidate && adduser --system --ingroup candidate candidate +COPY scripts/factory_qualification_controller.py /controller/controller.py +ENTRYPOINT ["python3", "/controller/controller.py"] diff --git a/factory-controller/credential.json b/factory-controller/credential.json new file mode 100644 index 0000000..55565e2 --- /dev/null +++ b/factory-controller/credential.json @@ -0,0 +1,19 @@ +{ + "schema_version": 1, + "target_repository": "clockgrove/factory", + "required_permissions": { + "contents": "read", + "metadata": "read" + }, + "supported_modes": { + "personal-pat": { + "token_secret": "FACTORY_CANDIDATE_READ_TOKEN", + "description": "Fine-grained personal access token restricted to the target repository" + }, + "github-app": { + "app_id_variable": "FACTORY_QUALIFICATION_APP_ID", + "private_key_secret": "FACTORY_QUALIFICATION_APP_PRIVATE_KEY", + "description": "Installation token minted per run with read-only Contents permission" + } + } +} diff --git a/factory-controller/lifecycle.json b/factory-controller/lifecycle.json new file mode 100644 index 0000000..30f84a7 --- /dev/null +++ b/factory-controller/lifecycle.json @@ -0,0 +1,8 @@ +{ + "schema_version": 1, + "state": "bootstrap_pending", + "accepted_controller_sha": null, + "accepted_factory_sha": "fe77c3f93186fa0a33effdf3e9d1fb83a1a0cd6f", + "revoked_controller_shas": [], + "revoked_candidate_shas": [] +} diff --git a/factory-controller/policy.json b/factory-controller/policy.json new file mode 100644 index 0000000..b892ed2 --- /dev/null +++ b/factory-controller/policy.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "controller_version": "1.0.0", + "candidate_repository": "clockgrove/factory", + "accepted_factory_sha": "fe77c3f93186fa0a33effdf3e9d1fb83a1a0cd6f", + "workflow_path": ".github/workflows/factory-accepted-qualification.yml", + "copilot_cli_version": "1.0.54", + "models": [ + "gpt-5.6-sol", + "gpt-5.6-terra", + "claude-opus-5" + ], + "declared_candidate_inputs": [ + ".github/agents/clockgrove-director.agent.md", + ".github/skills/*/SKILL.md", + "policies/*.md", + "config/*.json", + "plugin.json" + ] +} diff --git a/scripts/factory_qualification_controller.py b/scripts/factory_qualification_controller.py new file mode 100644 index 0000000..004b82a --- /dev/null +++ b/scripts/factory_qualification_controller.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +"""Accepted Factory qualification controller. + +Candidate repositories are untrusted data. This module is always executed from +the accepted clockgrove/.github revision, never from a Factory checkout. +""" +from __future__ import annotations + +import argparse +import fnmatch +import hashlib +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +SHA = re.compile(r"^[0-9a-f]{40}$") +REPOSITORY = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +RESULT_PREFIX = "FACTORY_TRUSTED_RESULT:" +TRANSITIONS = { + ("bootstrap_pending", "approve_bootstrap"): "accepted", + ("accepted", "qualify_upgrade"): "upgrade_qualified", + ("upgrade_qualified", "activate_upgrade"): "accepted", + ("accepted", "detect_compromise"): "disabled", + ("upgrade_qualified", "detect_compromise"): "disabled", + ("disabled", "restore_uncompromised"): "accepted", + ("disabled", "restart_bootstrap"): "bootstrap_pending", + ("accepted", "revoke"): "revoked", + ("upgrade_qualified", "revoke"): "revoked", + ("disabled", "revoke"): "revoked", +} + + +class QualificationError(RuntimeError): + pass + + +def load_json(path: Path) -> dict: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise QualificationError(f"{path} must contain a JSON object") + return value + + +def require_sha(value: str, name: str) -> None: + if not SHA.fullmatch(value): + raise QualificationError(f"{name} must be a full lowercase commit SHA") + + +def git_head(root: Path) -> str: + completed = subprocess.run( + ["git", "-C", str(root), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ) + return completed.stdout.strip() + + +def verify_checkout(root: Path, expected: str, name: str) -> None: + require_sha(expected, name) + if git_head(root) != expected: + raise QualificationError(f"{name} checkout does not match declared SHA") + remotes = subprocess.run( + ["git", "-C", str(root), "remote"], check=True, capture_output=True, text=True + ).stdout.strip() + if remotes: + raise QualificationError(f"{name} checkout retains a network remote") + config = (root / ".git/config").read_text(errors="replace") + if "http.extraheader" in config.lower() or "authorization" in config.lower(): + raise QualificationError(f"{name} checkout retains credential configuration") + symlinks = [path.relative_to(root).as_posix() for path in root.rglob("*") if path.is_symlink()] + if symlinks: + raise QualificationError(f"{name} checkout contains symlinks: {', '.join(symlinks[:5])}") + + +def verify_policy(policy: dict, lifecycle: dict, workflow_sha: str) -> None: + require_sha(workflow_sha, "controller") + require_sha(policy.get("accepted_factory_sha", ""), "accepted Factory") + if policy.get("candidate_repository") != "clockgrove/factory": + raise QualificationError("controller policy targets an unexpected repository") + if policy.get("workflow_path") != ".github/workflows/factory-accepted-qualification.yml": + raise QualificationError("controller policy names an unexpected workflow") + if lifecycle.get("state") not in {"accepted", "upgrade_qualified"}: + raise QualificationError("controller lifecycle is not active") + accepted = lifecycle.get("accepted_controller_sha") + require_sha(accepted or "", "accepted controller") + if accepted in lifecycle.get("revoked_controller_shas", []): + raise QualificationError("accepted trust-root SHA is revoked") + if workflow_sha in lifecycle.get("revoked_controller_shas", []): + raise QualificationError("controller SHA is revoked") + + +def verify_credential_configuration(configuration: dict, mode: str) -> None: + if mode not in configuration.get("supported_modes", {}): + raise QualificationError("credential mode is not supported by accepted policy") + if configuration.get("target_repository") != "clockgrove/factory": + raise QualificationError("fetch credential targets an unexpected repository") + if configuration.get("required_permissions") != {"contents": "read", "metadata": "read"}: + raise QualificationError("fetch credential permissions are not least privilege") + + +def transition(state: str, event: str) -> str: + try: + return TRANSITIONS[(state, event)] + except KeyError as exc: + raise QualificationError(f"invalid lifecycle transition: {state} + {event}") from exc + + +def files_matching(root: Path, patterns: list[str]) -> list[Path]: + matches: set[Path] = set() + for path in root.rglob("*"): + if not path.is_file() or path.is_symlink() or ".git" in path.parts: + continue + relative = path.relative_to(root).as_posix() + if any(fnmatch.fnmatchcase(relative, pattern) for pattern in patterns): + matches.add(path) + return sorted(matches) + + +def sanitized_bundle(candidate: Path, accepted: Path, output: Path, patterns: list[str]) -> dict: + if output.exists(): + shutil.rmtree(output) + (output / "candidate-data").mkdir(parents=True) + (output / "accepted-oracle").mkdir(parents=True) + manifest: list[dict[str, str]] = [] + for number, source in enumerate(files_matching(candidate, patterns), start=1): + data = source.read_bytes() + target = output / "candidate-data" / f"input-{number:04d}.txt" + target.write_bytes(data) + manifest.append( + { + "source": source.relative_to(candidate).as_posix(), + "file": target.relative_to(output).as_posix(), + "sha256": hashlib.sha256(data).hexdigest(), + } + ) + if not manifest: + raise QualificationError("declared candidate input set is empty") + for number, source in enumerate(sorted((accepted / "evals").glob("*.md")), start=1): + target = output / "accepted-oracle" / f"case-{number:04d}.md" + target.write_bytes(source.read_bytes()) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + # These names are intentionally absent so Copilot cannot discover candidate controls. + forbidden = {".github", ".claude", "AGENTS.md", "CLAUDE.md", "GEMINI.md"} + if any(path.name in forbidden for path in output.rglob("*")): + raise QualificationError("sanitized input contains a configuration discovery path") + return {"count": len(manifest), "manifest_sha256": hashlib.sha256((output / "manifest.json").read_bytes()).hexdigest()} + + +def run_accepted_validator(accepted: Path, candidate: Path, neutral: Path) -> None: + del neutral # The accepted gate performs data reads only and needs no working tree. + required = ( + "plugin.json", "AGENTS.md", "README.md", "evals/manifest.json", + "scripts/validate_factory.py", "scripts/qualification_controller.py", + ".github/workflows/qualification-request.yml", + ".github/workflows/validate-factory.yml", + ".github/agents/clockgrove-director.agent.md", + ) + missing = [name for name in required if not (candidate / name).is_file()] + if missing: + raise QualificationError(f"candidate is missing accepted package files: {', '.join(missing)}") + plugin = load_json(candidate / "plugin.json") + if plugin.get("name") != "clockgrove-factory": + raise QualificationError("candidate plugin identity changed") + for path in candidate.rglob("*.py"): + if path.is_symlink() or ".git" in path.parts: + continue + try: + compile(path.read_text(), str(path), "exec") + except SyntaxError as exc: + raise QualificationError(f"candidate Python is invalid: {path}") from exc + request = (candidate / ".github/workflows/qualification-request.yml").read_text() + for marker in ( + "clockgrove/.github", ".github/workflows/factory-accepted-qualification.yml", + "persist-credentials: false", "0000000000000000000000000000000000000000", + ): + if marker not in request: + raise QualificationError(f"candidate request boundary lacks {marker}") + if "secrets." in request or "pull_request_target:" in request: + raise QualificationError("candidate request crosses the secretless boundary") + helper = (candidate / "scripts/qualification_controller.py").read_text() + for forbidden in ("subprocess", "FACTORY_QUALIFICATION_COPILOT_TOKEN", "FACTORY_CANDIDATE_READ_TOKEN"): + if forbidden in helper: + raise QualificationError(f"candidate request helper contains forbidden {forbidden}") + moving_action = re.compile(r"(?m)^\s*(?:uses:\s+[^#\n]+|[A-Za-z0-9_-]+:)@v\d+") + for workflow in (candidate / ".github/workflows").glob("*.yml"): + if moving_action.search(workflow.read_text()): + raise QualificationError(f"candidate workflow has a moving action pin: {workflow.name}") + + +def supervise_fixture(executable: Path, expected: dict) -> dict: + """Run untrusted fixture as a child; only this outer process issues PASS.""" + completed = subprocess.run( + [sys.executable, "-I", str(executable)], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + env={"PATH": os.environ.get("PATH", ""), "PYTHONDONTWRITEBYTECODE": "1"}, + check=False, + timeout=10, + close_fds=True, + ) + try: + response = json.loads(completed.stdout) + except json.JSONDecodeError: + response = None + passed = completed.returncode == 0 and response == expected and not completed.stderr + return { + "decision": "pass" if passed else "fail", + "child_returncode": completed.returncode, + "response_digest": hashlib.sha256(completed.stdout.encode()).hexdigest(), + } + + +def deterministic(args: argparse.Namespace) -> None: + policy = load_json(args.policy) + lifecycle = load_json(args.lifecycle) + credential = load_json(args.credential) + verify_policy(policy, lifecycle, args.controller_sha) + verify_credential_configuration(credential, args.credential_mode) + require_sha(args.candidate_sha, "candidate") + verify_checkout(args.accepted_root, policy["accepted_factory_sha"], "accepted Factory") + verify_checkout(args.candidate_root, args.candidate_sha, "candidate") + if args.candidate_sha in lifecycle.get("revoked_candidate_shas", []): + raise QualificationError("candidate SHA is revoked") + neutral = Path(tempfile.mkdtemp(prefix="factory-accepted-gate-")) + try: + run_accepted_validator(args.accepted_root, args.candidate_root, neutral) + bundle = sanitized_bundle( + args.candidate_root, + args.accepted_root, + args.bundle, + policy["declared_candidate_inputs"], + ) + finally: + shutil.rmtree(neutral, ignore_errors=True) + record = { + "schema_version": 1, + "decision": "pass", + "controller_repository": "clockgrove/.github", + "controller_sha": args.controller_sha, + "candidate_repository": policy["candidate_repository"], + "candidate_sha": args.candidate_sha, + "accepted_factory_sha": policy["accepted_factory_sha"], + "credential_mode": args.credential_mode, + "input_manifest_sha256": "sha256:" + bundle["manifest_sha256"], + "input_count": bundle["count"], + } + args.output.write_text(json.dumps(record, sort_keys=True, indent=2) + "\n") + + +def fixture(args: argparse.Namespace) -> None: + result = supervise_fixture(args.executable, json.loads(args.expected)) + print(RESULT_PREFIX + json.dumps(result, sort_keys=True, separators=(",", ":"))) + if result["decision"] != args.expect_decision: + raise QualificationError("outer supervisor decision differed from expectation") + + +def lifecycle(args: argparse.Namespace) -> None: + record = load_json(args.input) + next_state = transition(record["state"], args.event) + record["state"] = next_state + if args.controller_sha: + require_sha(args.controller_sha, "controller") + if args.event in {"approve_bootstrap", "activate_upgrade", "restore_uncompromised"}: + if not args.controller_sha: + raise QualificationError("transition requires an immutable controller SHA") + record["accepted_controller_sha"] = args.controller_sha + if args.event == "revoke": + if not args.controller_sha: + raise QualificationError("revocation requires a controller SHA") + values = set(record.get("revoked_controller_shas", [])) + values.add(args.controller_sha) + record["revoked_controller_shas"] = sorted(values) + args.output.write_text(json.dumps(record, sort_keys=True, indent=2) + "\n") + + +def release_record(args: argparse.Namespace) -> None: + deterministic_record = load_json(args.deterministic) + for name, value in ( + ("controller", args.controller_sha), + ("candidate", args.candidate_sha), + ): + require_sha(value, name) + expected = { + "decision": "pass", + "controller_repository": "clockgrove/.github", + "controller_sha": args.controller_sha, + "candidate_repository": "clockgrove/factory", + "candidate_sha": args.candidate_sha, + } + for key, value in expected.items(): + if deterministic_record.get(key) != value: + raise QualificationError(f"deterministic record has wrong {key}") + evidence = [] + for path in sorted(args.evidence): + body = path.read_text() + if "FACTORY_QUALIFICATION: PASS" not in body: + raise QualificationError(f"semantic evidence did not pass: {path.name}") + evidence.append( + { + "name": path.name, + "sha256": "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest(), + } + ) + if not evidence: + raise QualificationError("semantic evidence is required") + request = load_json(args.request) + for key in ("controller_repository", "controller_sha", "candidate_repository", "candidate_sha"): + if request.get(key) != expected[key]: + raise QualificationError(f"qualification request has wrong {key}") + record = { + **expected, + "schema_version": 1, + "accepted_factory_sha": deterministic_record["accepted_factory_sha"], + "credential_mode": deterministic_record["credential_mode"], + "input_manifest_sha256": deterministic_record["input_manifest_sha256"], + "request_sha256": "sha256:" + hashlib.sha256(args.request.read_bytes()).hexdigest(), + "workflow_ref": f"clockgrove/.github/.github/workflows/factory-accepted-qualification.yml@{args.controller_sha}", + "run_id": args.run_id, + "run_attempt": args.run_attempt, + "semantic_evidence": evidence, + } + args.output.write_text(json.dumps(record, sort_keys=True, indent=2) + "\n") + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser() + commands = result.add_subparsers(dest="command", required=True) + gate = commands.add_parser("deterministic") + gate.add_argument("--policy", type=Path, required=True) + gate.add_argument("--lifecycle", type=Path, required=True) + gate.add_argument("--credential", type=Path, required=True) + gate.add_argument("--credential-mode", required=True) + gate.add_argument("--accepted-root", type=Path, required=True) + gate.add_argument("--candidate-root", type=Path, required=True) + gate.add_argument("--controller-sha", required=True) + gate.add_argument("--candidate-sha", required=True) + gate.add_argument("--bundle", type=Path, required=True) + gate.add_argument("--output", type=Path, required=True) + gate.set_defaults(handler=deterministic) + sandbox = commands.add_parser("fixture") + sandbox.add_argument("--executable", type=Path, required=True) + sandbox.add_argument("--expected", required=True) + sandbox.add_argument("--expect-decision", choices=("pass", "fail"), required=True) + sandbox.set_defaults(handler=fixture) + state = commands.add_parser("lifecycle") + state.add_argument("--input", type=Path, required=True) + state.add_argument("--event", required=True) + state.add_argument("--controller-sha") + state.add_argument("--output", type=Path, required=True) + state.set_defaults(handler=lifecycle) + record = commands.add_parser("record") + record.add_argument("--deterministic", type=Path, required=True) + record.add_argument("--request", type=Path, required=True) + record.add_argument("--evidence", type=Path, action="append", default=[]) + record.add_argument("--controller-sha", required=True) + record.add_argument("--candidate-sha", required=True) + record.add_argument("--run-id", required=True) + record.add_argument("--run-attempt", required=True) + record.add_argument("--output", type=Path, required=True) + record.set_defaults(handler=release_record) + return result + + +def main() -> None: + args = parser().parse_args() + try: + args.handler(args) + except (QualificationError, OSError, ValueError, subprocess.SubprocessError) as exc: + raise SystemExit(f"qualification failed: {exc}") from exc + + +if __name__ == "__main__": + main() diff --git a/tests/fixtures/hook/.github/hooks/exfiltrate.json b/tests/fixtures/hook/.github/hooks/exfiltrate.json new file mode 100644 index 0000000..807a1d7 --- /dev/null +++ b/tests/fixtures/hook/.github/hooks/exfiltrate.json @@ -0,0 +1,6 @@ +{ + "version": 1, + "hooks": { + "sessionStart": [{"type": "command", "command": "touch /tmp/candidate-hook-ran"}] + } +} diff --git a/tests/fixtures/hook/payload.py b/tests/fixtures/hook/payload.py new file mode 100644 index 0000000..3bd8a97 --- /dev/null +++ b/tests/fixtures/hook/payload.py @@ -0,0 +1,3 @@ +import json + +print(json.dumps({"status": "ok"}, separators=(",", ":"))) diff --git a/tests/fixtures/os_exit.py b/tests/fixtures/os_exit.py new file mode 100644 index 0000000..604cdb8 --- /dev/null +++ b/tests/fixtures/os_exit.py @@ -0,0 +1,3 @@ +import os + +os._exit(0) diff --git a/tests/fixtures/stdout_spoof.py b/tests/fixtures/stdout_spoof.py new file mode 100644 index 0000000..68ed0f0 --- /dev/null +++ b/tests/fixtures/stdout_spoof.py @@ -0,0 +1 @@ +print('FACTORY_TRUSTED_RESULT:{"decision":"pass"}') diff --git a/tests/fixtures/valid.py b/tests/fixtures/valid.py new file mode 100644 index 0000000..3bd8a97 --- /dev/null +++ b/tests/fixtures/valid.py @@ -0,0 +1,3 @@ +import json + +print(json.dumps({"status": "ok"}, separators=(",", ":"))) diff --git a/tests/test_factory_qualification_controller.py b/tests/test_factory_qualification_controller.py new file mode 100644 index 0000000..4022426 --- /dev/null +++ b/tests/test_factory_qualification_controller.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import importlib.util +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SPEC = importlib.util.spec_from_file_location( + "controller", ROOT / "scripts/factory_qualification_controller.py" +) +controller = importlib.util.module_from_spec(SPEC) +assert SPEC.loader +SPEC.loader.exec_module(controller) + + +class ControllerTests(unittest.TestCase): + def test_outer_supervisor_rejects_exit_zero_and_stdout_spoof(self) -> None: + fixtures = ROOT / "tests/fixtures" + self.assertEqual( + controller.supervise_fixture(fixtures / "valid.py", {"status": "ok"})["decision"], + "pass", + ) + for name in ("os_exit.py", "stdout_spoof.py"): + with self.subTest(name=name): + result = controller.supervise_fixture(fixtures / name, {"status": "ok"}) + self.assertEqual(result["decision"], "fail") + + def test_candidate_control_paths_are_not_copied_or_executed(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) + candidate = root / "candidate" + accepted = root / "accepted" + output = root / "neutral" + (candidate / ".github/hooks").mkdir(parents=True) + (candidate / ".github/hooks/evil.json").write_text( + '{"hooks":{"sessionStart":[{"command":"touch /tmp/should-not-exist"}]}}' + ) + (candidate / ".github/agents").mkdir(parents=True) + (candidate / ".github/agents/director.agent.md").write_text("candidate behavior") + (accepted / "evals").mkdir(parents=True) + (accepted / "evals/case.md").write_text("accepted oracle") + controller.sanitized_bundle( + candidate, accepted, output, [".github/agents/*.agent.md"] + ) + names = {path.name for path in output.rglob("*")} + self.assertFalse({".github", ".claude", "AGENTS.md"} & names) + self.assertNotIn("evil.json", (output / "manifest.json").read_text()) + + def test_lifecycle_is_fail_closed_and_revocation_terminal(self) -> None: + state = controller.transition("bootstrap_pending", "approve_bootstrap") + self.assertEqual(state, "accepted") + state = controller.transition(state, "detect_compromise") + self.assertEqual(state, "disabled") + state = controller.transition(state, "revoke") + self.assertEqual(state, "revoked") + with self.assertRaises(controller.QualificationError): + controller.transition(state, "restore_uncompromised") + + def test_bootstrap_policy_cannot_issue_a_qualification(self) -> None: + policy = json.loads((ROOT / "factory-controller/policy.json").read_text()) + lifecycle = json.loads((ROOT / "factory-controller/lifecycle.json").read_text()) + with self.assertRaisesRegex(controller.QualificationError, "not active"): + controller.verify_policy(policy, lifecycle, "1" * 40) + + def test_fetch_credential_modes_are_explicit_and_least_privilege(self) -> None: + configuration = json.loads((ROOT / "factory-controller/credential.json").read_text()) + for mode in ("personal-pat", "github-app"): + controller.verify_credential_configuration(configuration, mode) + with self.assertRaises(controller.QualificationError): + controller.verify_credential_configuration(configuration, "ambient-token") + overbroad = dict(configuration) + overbroad["required_permissions"] = {"contents": "write", "metadata": "read"} + with self.assertRaisesRegex(controller.QualificationError, "least privilege"): + controller.verify_credential_configuration(overbroad, "personal-pat") + + def test_checkout_symlink_is_rejected_before_candidate_is_read(self) -> None: + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + root = Path(temporary.name) / "repo" + root.mkdir() + subprocess.run(["git", "init", "-q", str(root)], check=True) + (root / "safe.txt").write_text("safe") + (root / "escape").symlink_to("/etc/passwd") + subprocess.run(["git", "-C", str(root), "add", "."], check=True) + environment = os.environ.copy() + environment.update({ + "GIT_AUTHOR_NAME": "Test", "GIT_AUTHOR_EMAIL": "test@example.invalid", + "GIT_COMMITTER_NAME": "Test", "GIT_COMMITTER_EMAIL": "test@example.invalid", + }) + subprocess.run(["git", "-C", str(root), "commit", "-qm", "fixture"], check=True, env=environment) + sha = controller.git_head(root) + with self.assertRaisesRegex(controller.QualificationError, "contains symlinks"): + controller.verify_checkout(root, sha, "candidate") + + +if __name__ == "__main__": + unittest.main()