diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..4059d0b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,52 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test-package: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + timeout-minutes: 10 + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PYTHONDONTWRITEBYTECODE: "1" + steps: + # actions/checkout v7.0.1 + - name: Check out source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + # actions/setup-python v7.0.0 + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14" + + - name: Run tests + run: PYTHONPATH=src python -m unittest discover -s tests -v + + - name: Build source and wheel distributions + run: | + python -m pip install build==1.6.0 + python -m build + python tools/normalize_sdist.py dist/*.tar.gz + + - name: Install and inspect wheel + run: | + python -m venv "${RUNNER_TEMP}/agent-release-gate-smoke" + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/python" -m pip install --no-deps dist/*.whl + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/python" -c \ + 'from importlib.metadata import metadata, requires; m = metadata("agent-release-gate"); assert m["Version"] == "0.1.0"; assert m["License-Expression"] == "Apache-2.0"; assert not (requires("agent-release-gate") or [])' + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/agent-release-gate" --help + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/python" tests/package_smoke.py \ + --cli "${RUNNER_TEMP}/agent-release-gate-smoke/bin/agent-release-gate" diff --git a/.gitignore b/.gitignore index b80d7bd..b95d978 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Local planning artifacts docs/superpowers/ +.worktrees/ # Python caches and local environments __pycache__/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..bf88107 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,6 @@ +include SECURITY.md +recursive-include docs *.md +recursive-include integrations *.json +recursive-include policies *.toml +recursive-include tests *.py *.json +recursive-include tools *.py diff --git a/README.md b/README.md index 738e4df..6278a68 100644 --- a/README.md +++ b/README.md @@ -4,9 +4,38 @@ Agent Release Gate turns completed agent-benchmark evidence into a deterministic The tool reads existing JSON reports. It does not install, import, execute, fetch, or modify ClawProBench. +## Status + +Version `0.1.0` is an early, source-distributed release. The public repository +is intended for inspection and use, but outside pull requests and feature +requests are not being solicited for v0. + +## Installation + +Agent Release Gate is not published to PyPI. Install it from a trusted source +checkout: + +```bash +python3.14 -m venv .venv +.venv/bin/python -m pip install . +.venv/bin/agent-release-gate --help +``` + +To install a wheel produced from the checkout: + +```bash +uv build +python3.14 -m venv /tmp/agent-release-gate +/tmp/agent-release-gate/bin/python -m pip install \ + dist/agent_release_gate-0.1.0-py3-none-any.whl +/tmp/agent-release-gate/bin/agent-release-gate --help +``` + ## Requirements - Python 3.14 or newer; development is verified with `python3.14`. +- A POSIX-style operating system with descriptor-relative filesystem operations; + v0 is verified on macOS and Linux. - Git, used only for read-only provenance checks. - The audited ClawProBench checkout at `../ClawProBench`. @@ -18,7 +47,9 @@ No package installation is needed for repository development. Prefix commands wi PYTHONPATH=src python3.14 -m agent_release_gate doctor ``` -`doctor` validates the checkout, origin URL, audited commit, clean worktree, and absence of prohibited vendored directories. It never runs upstream code. +`doctor` pins and validates the checkout, origin URL, audited commit, clean +worktree, and absence of prohibited vendored directories. It never runs +upstream code. ## Evaluate a Report @@ -31,7 +62,12 @@ PYTHONPATH=src python3.14 -m agent_release_gate evaluate \ --output decisions/release-decision.json ``` -The output is written atomically. A failed evaluation leaves an existing output file unchanged. +Evaluation inputs and the validated benchmark checkout are pinned by file +descriptor, and output is written atomically through a held directory +descriptor. A failed evaluation leaves an existing output file unchanged; +concurrent path or parent-symlink swaps cannot substitute an input, mix +checkout provenance, redirect the write into the checkout, or overwrite a +protected input through a case-variant or hard-link alias. Exit codes are: @@ -85,6 +121,7 @@ Tests use synthetic JSON and disposable Git repositories. They do not execute th - `integrations/`: audited benchmark manifests. - `policies/`: release threshold policies. - `tests/`: synthetic fixtures and behavior tests. +- `tools/`: release-artifact metadata normalization. - `docs/architecture.md`: system boundaries and data flow. - `docs/adding-an-adapter.md`: extension contract for another agent benchmark. @@ -98,3 +135,38 @@ The v0 manifest pins: - prohibited checked-out paths: `ironclaw` and `nanoclaw`. Generated benchmark reports are inputs to this repository. ClawProBench remains a read-only upstream dependency. + +ClawProBench is licensed under Apache-2.0. Agent Release Gate does not vendor, +modify, or redistribute its source. See [dependency boundaries](docs/dependencies.md) +for the complete build, test, CI, upstream, and audit inventory. + +## Trust Model and Limitations + +- The CLI evaluates supplied reports; it does not prove that a report was + produced honestly or by the pinned benchmark source. +- v0 preserves the report timestamp but does not enforce evidence freshness. +- Custom policies and integration manifests are trusted local configuration. +- The CLI does not fetch or execute benchmark code. +- Filesystem identity and ancestry are checked at acquisition and immediately + before output commit. A hostile concurrent process with write access to both + directory trees is outside the v0 threat model; run evaluations where + untrusted processes cannot rename the output or benchmark directories. +- Decision output must be separate from reports, policies, manifests, and + benchmark checkouts; protected paths are rejected after symlink resolution + while input files, the benchmark checkout, and the output directory remain + pinned by descriptor. +- The source distribution includes the default policy and integration manifest. + A standalone wheel contains only the CLI package, so invoke it from a source + checkout or pass explicit `--policy` and `--integration` paths. +- v0 supports only the documented ClawProBench report shape and default + integration. Unknown additive report fields are tolerated. +- A deterministic `go` means only that the supplied evidence satisfies the + supplied policy. It is not a general security certification. + +Serialized decisions include the pinned repository URL and commit but omit the +absolute local checkout path to avoid leaking machine-specific information. + +## Security and License + +Report vulnerabilities through the private process in [SECURITY.md](SECURITY.md). +Agent Release Gate is licensed under the [Apache License 2.0](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..19d60bc --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,31 @@ +# Security Policy + +## Supported Versions + +Agent Release Gate is an early-stage project. Only the latest `0.1.x` release +is supported with security fixes. + +## Reporting a Vulnerability + +Use GitHub's **Security** tab and select **Report a vulnerability**. This opens +a private report with the repository owner. Do not disclose suspected +vulnerabilities in a public issue, pull request, discussion, or benchmark +artifact, and do not include credentials, tokens, private data, or exploit data +that is not necessary to reproduce the problem. + +If private vulnerability reporting is unavailable, do not publish the details. +Wait until the repository provides a private reporting channel. + +Include the affected version or commit, the relevant command and input shape, +the observed impact, and the smallest safe reproduction. Redact personal data +and secrets. + +Reports are acknowledged and handled on a best-effort basis. No response or +remediation timeline is guaranteed for this early release. + +## Scope + +Security reports may cover the CLI, report and policy parsing, integration +provenance checks, output-path handling, packaging, and CI configuration. +ClawProBench is an independent upstream project; report vulnerabilities in its +code to its maintainers. diff --git a/docs/adding-an-adapter.md b/docs/adding-an-adapter.md index a91fc33..8ff3292 100644 --- a/docs/adding-an-adapter.md +++ b/docs/adding-an-adapter.md @@ -48,6 +48,11 @@ _ADAPTERS = MappingProxyType( Add a strict manifest under `integrations/` for the benchmark source. The manifest identifies the source repository, audited version, expected local checkout, and prohibited paths. Adapter code receives the validated source version; it does not perform Git operations itself. +The manifest's `adapter` field must exactly match the lowercase registry key. +Evaluation rejects a different `--adapter` value before reading a report or +writing a decision. Keep the checkout as a direct sibling, and never use a +decision output path inside that checkout. + ## Tests Start with synthetic, hand-checked reports. Tests must cover: diff --git a/docs/architecture.md b/docs/architecture.md index 4302ec6..d9af269 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -31,19 +31,24 @@ This direction lets another agent benchmark reuse the same policies, evaluator, For `doctor`: 1. Load and strictly validate the integration manifest. -2. Resolve its checkout as a direct sibling of this repository. -3. verify Git worktree state, `HEAD`, `origin`, cleanliness, and prohibited paths. -4. Emit validated provenance as JSON. +2. Bind the manifest's declared adapter to the registry key used for evaluation. +3. Resolve and descriptor-pin its checkout as a direct sibling of this repository. +4. Verify Git worktree state, `HEAD`, `origin`, cleanliness, and prohibited paths + relative to that held descriptor. +5. Emit validated provenance as JSON without the absolute local checkout path. For `evaluate`: -1. Perform the same integration validation. -2. Select the named benchmark adapter. -3. Validate and normalize the report into `BenchmarkEvidence`. -4. Parse and validate the TOML policy, retaining its SHA-256 digest. -5. Apply every gate rule in stable order. -6. Combine decision, observed metrics, report identity, integration provenance, policy identity, and UTC evaluation time. -7. Write sorted, indented JSON to a sibling temporary file, sync it, and replace the target atomically. +1. Open and pin the report, policy, and integration manifest as regular files. +2. Perform integration validation from the pinned manifest. +3. Select the named benchmark adapter and require it to match the manifest. +4. Open and validate the output directory, rejecting benchmark-checkout or + input-file targets while holding the directory descriptor through the write. +5. Validate and normalize the pinned report into `BenchmarkEvidence`. +6. Parse and validate the pinned TOML policy, retaining its SHA-256 digest. +7. Apply every gate rule in stable order. +8. Combine decision, observed metrics, report identity, integration provenance, policy identity, and UTC evaluation time. +9. Write sorted, indented JSON to a sibling temporary file, sync it, and replace the target atomically. Input errors are distinct from release outcomes. A valid `no_go` is exit code `1`; malformed evidence or unproven provenance is exit code `2` and produces no decision. @@ -71,12 +76,54 @@ The evaluation timestamp is the only time-varying output field. Tests inject a c The integration manifest pins the expected origin and full Git commit. Validation uses only: - `git rev-parse --is-inside-work-tree`; +- `git rev-parse --show-prefix`; - `git rev-parse HEAD`; - `git remote get-url origin`; -- `git status --porcelain`. +- `git status --porcelain --untracked-files=all`; +- `git ls-files --others` without exclude rules; +- `git ls-files -v -z` to reject assume-unchanged entries and skip-worktree + entries that remain present (absent sparse-checkout entries are allowed). Git hooks and fsmonitor are disabled for these read-only subprocesses, terminal prompts are disabled, and `GIT_OPTIONAL_LOCKS=0` prevents status checks from refreshing the upstream index. The validator also disables Git's untracked cache, ignores global/system Git configuration, and removes inherited `GIT_*` variables before setting its explicit safe environment. This prevents ambient `GIT_DIR`, `GIT_WORK_TREE`, or index overrides from redirecting a probe away from the pinned checkout. Validation never fetches, checks out, resets, cleans, or writes upstream files. +The validator opens the resolved checkout through a no-follow descriptor walk +and retains that descriptor through integration validation and output-path +validation. Git subprocesses enter the held directory by descriptor, and +prohibited paths are inspected relative to it. Renaming or replacing the +manifest path therefore cannot mix provenance from one checkout with +cleanliness from another or make output protection follow a replacement path. + +The integration manifest declares the adapter that may consume its evidence. +This prevents a CLI invocation from presenting one benchmark's source +provenance alongside a different adapter. Prohibited paths are detected even +when represented by dangling symlinks. The checkout must resolve to a distinct +direct sibling. Held project, checkout, and parent descriptors are compared by +device and inode, so the project itself, case-variant aliases, and sibling +symlinks back to it are rejected. + +Before evaluation, the CLI opens and pins the report, policy, and integration +manifest by file and parent-directory descriptor, and the validator pins the +benchmark checkout by directory descriptor. It then resolves the requested +output path, opens its directory without following the final path component, +and verifies the opened directory by device and inode. It rejects paths inside +the held benchmark checkout and destination leaves with the same device and +inode as a pinned input, including case variants, hard links, and aliases +reached through symlinked parents. It repeats the input-identity check +and benchmark-ancestry check before creating the temporary file and again +immediately before replacement. Pinned descriptors are used for every input +read and for creating and replacing the decision file, so +concurrent path and parent-symlink swaps cannot substitute an input, mix +checkout validation, or redirect the write into the checkout. The resolved +checkout path stays internal and is not serialized. + +These checks validate path identity and ancestry when descriptors are acquired +and immediately before the output commit. v0 does not claim to contain a +hostile local process that already has write access to both the output and +benchmark directory trees and races a directory rename between the final +validation syscall and `os.replace`; such a process can already modify the +read-only checkout directly. Run evaluations in a filesystem namespace not +writable by untrusted concurrent processes. + ## Failure Handling Expected input and filesystem failures produce a concise stderr message without a traceback. Atomic output prevents a failed evaluation from truncating a previous decision. Unexpected failures are contained at the command boundary and return exit code `2`. diff --git a/docs/dependencies.md b/docs/dependencies.md new file mode 100644 index 0000000..c5e1a1f --- /dev/null +++ b/docs/dependencies.md @@ -0,0 +1,31 @@ +# Dependencies and Third-Party Boundary + +Agent Release Gate has no third-party Python package runtime or test +dependencies. The table below inventories external tools involved in running, +building, verifying, or integrating the project; none are shipped in the wheel. + +| Role | Component | Version constraint | License | Distribution boundary | +| --- | --- | --- | --- | --- | +| Runtime | Python standard library | Python 3.14+ | PSF License | Required interpreter; not bundled | +| Runtime and tests | Git CLI | Git 2.x; tests require `git init -b` support | GPL-2.0-only | External provenance tool; not bundled or invoked over a network | +| Tests | `unittest` | Python 3.14+ | PSF License | Standard library; not bundled | +| Build backend | setuptools | `>=80` | MIT | Isolated build dependency; not bundled | +| Build frontend | build | `1.6.0` in CI | MIT | CI/development tool; not bundled | +| CI action | `actions/checkout` | `v7.0.1`, pinned by SHA | MIT | GitHub Actions only | +| CI action | `actions/setup-python` | `v7.0.0`, pinned by SHA | MIT | GitHub Actions only | +| Upstream evidence producer | ClawProBench | commit `c4b8395854fe0752eef435b44f140366efd44d8e` | Apache-2.0 | External read-only checkout; no source vendored | +| Release audit | Gitleaks | Exact release recorded per audit | MIT | Temporary audit tool; not bundled | + +License sources: + +- Python: +- Git: +- setuptools: +- build: +- actions/checkout: +- actions/setup-python: +- ClawProBench: +- Gitleaks: + +The JSON fixtures under `tests/fixtures/` are original, hand-authored synthetic +data. They contain no copied benchmark results or user data. diff --git a/integrations/clawprobench.lock.json b/integrations/clawprobench.lock.json index 6f588d7..19da442 100644 --- a/integrations/clawprobench.lock.json +++ b/integrations/clawprobench.lock.json @@ -1,5 +1,6 @@ { "schema_version": 1, + "adapter": "clawprobench", "name": "ClawProBench", "repository_url": "https://github.com/suyoumo/ClawProBench.git", "checkout_path": "../ClawProBench", diff --git a/pyproject.toml b/pyproject.toml index 53839c0..916b38c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,14 @@ name = "agent-release-gate" version = "0.1.0" description = "Turn agent benchmark evidence into deterministic release decisions." readme = "README.md" +license = "Apache-2.0" requires-python = ">=3.14" dependencies = [] +[project.urls] +Repository = "https://github.com/bsha6/agent-release-gate" +Issues = "https://github.com/bsha6/agent-release-gate/issues" + [project.scripts] agent-release-gate = "agent_release_gate.cli:main" diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index 349f4ae..ed78e52 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -3,10 +3,11 @@ import argparse import json import os +import secrets import sys -import tempfile from collections.abc import Callable, Sequence -from contextlib import redirect_stderr, redirect_stdout +from contextlib import ExitStack, redirect_stderr, redirect_stdout +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import NoReturn, TextIO @@ -15,8 +16,19 @@ from agent_release_gate.adapters.registry import get_adapter from agent_release_gate.domain.policy import PolicyError, load_policy from agent_release_gate.evaluation.evaluator import evaluate +from agent_release_gate.filesystem import ( + FileIdentity, + close_best_effort, + directory_identity, + directory_is_within, + file_identity, + open_directory, + open_regular_file, + same_directory, +) from agent_release_gate.integration.validator import ( IntegrationError, + IntegrationEvidence, load_manifest, validate_integration, ) @@ -26,6 +38,58 @@ class DecisionWriteError(ValueError): """Raised when a completed decision cannot be written atomically.""" +class InputReadError(ValueError): + """Raised when an evaluation input cannot be pinned for a safe read.""" + + +@dataclass(slots=True) +class _InputTarget: + file_fd: int + directory_fd: int + name: str + display_path: Path + + @property + def read_path(self) -> Path: + return Path("/dev/fd") / str(self.file_fd) + + def close(self) -> None: + file_fd = self.file_fd + directory_fd = self.directory_fd + self.file_fd = -1 + self.directory_fd = -1 + for descriptor in (file_fd, directory_fd): + close_best_effort(descriptor) + + def __enter__(self) -> _InputTarget: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + +@dataclass(slots=True) +class _OutputTarget: + directory_fd: int + name: str + display_path: Path + protected_file_identities: tuple[FileIdentity, ...] + protected_directory_identity: tuple[int, int] + + def close(self) -> None: + directory_fd = self.directory_fd + self.directory_fd = -1 + # This read-only guard descriptor cannot affect a committed output, + # and a close error must not mask the body result. + close_best_effort(directory_fd) + + def __enter__(self) -> _OutputTarget: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="agent-release-gate", @@ -66,51 +130,261 @@ def build_parser() -> argparse.ArgumentParser: return parser -def _write_json_atomic(path: Path, document: dict[str, object]) -> None: - temporary_path: Path | None = None +def _prepare_input_target(path: Path) -> _InputTarget: + directory_fd: int | None = None + file_fd: int | None = None + try: + file_fd, directory_fd, resolved_path = open_regular_file(path) + target = _InputTarget( + file_fd=file_fd, + directory_fd=directory_fd, + name=resolved_path.name, + display_path=path, + ) + file_fd = None + directory_fd = None + return target + except InputReadError: + raise + except OSError as exc: + raise InputReadError(f"unable to open evaluation input {path}: {exc}") from exc + finally: + for descriptor in (file_fd, directory_fd): + if descriptor is not None: + close_best_effort(descriptor) + + +def _write_json_atomic( + target: _OutputTarget, + document: dict[str, object], +) -> None: + temporary_name: str | None = None try: + _validate_pinned_output_target(target) serialized = json.dumps( document, indent=2, sort_keys=True, allow_nan=False, ) + "\n" - with tempfile.NamedTemporaryFile( + temporary_name = f".{target.name}.{secrets.token_hex(16)}" + temporary_fd = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=target.directory_fd, + ) + with os.fdopen( + temporary_fd, mode="w", encoding="utf-8", - dir=path.parent, - prefix=f".{path.name}.", - delete=False, ) as temporary: - temporary_path = Path(temporary.name) temporary.write(serialized) temporary.flush() os.fsync(temporary.fileno()) - os.replace(temporary_path, path) - temporary_path = None + _validate_pinned_output_target(target) + os.replace( + temporary_name, + target.name, + src_dir_fd=target.directory_fd, + dst_dir_fd=target.directory_fd, + ) + temporary_name = None + try: + os.fsync(target.directory_fd) + except OSError: + # The rename has already committed the new file. Reporting failure + # here would falsely promise that an existing output was preserved. + pass + except DecisionWriteError: + raise except (OSError, TypeError, ValueError) as exc: - raise DecisionWriteError(f"unable to write decision {path}: {exc}") from exc + raise DecisionWriteError( + f"unable to write decision {target.display_path}: {exc}" + ) from exc finally: - if temporary_path is not None: + if temporary_name is not None: try: - temporary_path.unlink(missing_ok=True) + os.unlink(temporary_name, dir_fd=target.directory_fd) except OSError: pass -def _validated_integration(path: Path): +def _validate_output_path( + output: Path, + *, + protected_files: Sequence[Path], + protected_directory: Path, +) -> Path: + resolved_output = output.resolve(strict=False) + resolved_directory = protected_directory.resolve(strict=False) + if resolved_output.is_relative_to(resolved_directory): + raise DecisionWriteError( + "output path must not be inside the benchmark checkout" + ) + if any( + resolved_output == protected.resolve(strict=False) + for protected in protected_files + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) + return resolved_output + + +def _leaf_matches_protected_input( + directory_fd: int, + name: str, + protected_identities: Sequence[FileIdentity], +) -> bool: + try: + observed = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + return False + observed_identity = observed.st_dev, observed.st_ino + return observed_identity in protected_identities + + +def _validate_pinned_output_target(target: _OutputTarget) -> None: + if directory_is_within( + target.directory_fd, + target.protected_directory_identity, + ): + raise DecisionWriteError( + "output path must not be inside the benchmark checkout" + ) + if _leaf_matches_protected_input( + target.directory_fd, + target.name, + target.protected_file_identities, + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) + + +def _prepare_output_target( + output: Path, + *, + protected_files: Sequence[Path | _InputTarget], + protected_directory: Path | IntegrationEvidence, +) -> _OutputTarget: + if isinstance(protected_directory, IntegrationEvidence): + protected_path = protected_directory.checkout_path + protected_identity = protected_directory.checkout_identity + else: + protected_path = protected_directory + protected_fd: int | None = None + try: + protected_fd, _ = open_directory(protected_path) + protected_identity = directory_identity(protected_fd) + finally: + if protected_fd is not None: + close_best_effort(protected_fd) + protected_paths = tuple( + protected.display_path + if isinstance(protected, _InputTarget) + else protected + for protected in protected_files + ) + _validate_output_path( + output, + protected_files=protected_paths, + protected_directory=protected_path, + ) + output_name = output.name + if not output_name: + raise DecisionWriteError("output path must name a file") + + directory_fd: int | None = None + try: + directory_fd, _ = open_directory(output.parent) + if directory_is_within(directory_fd, protected_identity): + raise DecisionWriteError( + "output path must not be inside the benchmark checkout" + ) + + protected_identities: list[FileIdentity] = [] + for protected in protected_files: + protected_parent_fd: int | None = None + close_protected_parent = False + try: + if isinstance(protected, _InputTarget): + protected_name = protected.name + protected_parent_fd = protected.directory_fd + protected_identities.append(file_identity(protected.file_fd)) + else: + resolved_protected = protected.resolve(strict=False) + protected_name = resolved_protected.name + protected_parent_fd, _ = open_directory( + resolved_protected.parent + ) + close_protected_parent = True + observed = os.stat(resolved_protected, follow_symlinks=False) + protected_identities.append((observed.st_dev, observed.st_ino)) + assert protected_parent_fd is not None + if output_name == protected_name and same_directory( + directory_fd, + protected_parent_fd, + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) + finally: + if close_protected_parent and protected_parent_fd is not None: + close_best_effort(protected_parent_fd) + + if _leaf_matches_protected_input( + directory_fd, + output_name, + protected_identities, + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) + + target = _OutputTarget( + directory_fd=directory_fd, + name=output_name, + display_path=output, + protected_file_identities=tuple(protected_identities), + protected_directory_identity=protected_identity, + ) + directory_fd = None + return target + except DecisionWriteError: + raise + except OSError as exc: + raise DecisionWriteError(f"unable to write decision {output}: {exc}") from exc + finally: + if directory_fd is not None: + close_best_effort(directory_fd) + + +def _validated_integration( + path: Path, + *, + expected_adapter: str | None = None, +) -> IntegrationEvidence: manifest = load_manifest(path, project_root=Path.cwd()) + if expected_adapter is not None and manifest.adapter != expected_adapter: + raise IntegrationError( + f"requested adapter {expected_adapter!r} does not match " + f"integration adapter {manifest.adapter!r}" + ) return validate_integration(manifest) def _doctor(args: argparse.Namespace, stdout: TextIO) -> int: - integration = _validated_integration(args.integration) - document: dict[str, object] = { - "schema_version": 1, - "valid": True, - "integration": integration.to_dict(), - } - stdout.write(json.dumps(document, indent=2, sort_keys=True, allow_nan=False) + "\n") + with _validated_integration(args.integration) as integration: + document: dict[str, object] = { + "schema_version": 1, + "valid": True, + "integration": integration.to_dict(), + } + stdout.write( + json.dumps(document, indent=2, sort_keys=True, allow_nan=False) + "\n" + ) return 0 @@ -118,20 +392,41 @@ def _evaluate( args: argparse.Namespace, clock: Callable[[], datetime], ) -> int: - integration = _validated_integration(args.integration) adapter = get_adapter(args.adapter) - evidence = adapter.load(args.report, source_version=integration.commit) - policy, policy_sha256 = load_policy(args.policy) - decision = evaluate(evidence, policy) - - document = decision.to_dict() - document["schema_version"] = 1 - document["evaluated_at"] = clock().astimezone(timezone.utc).isoformat() - policy_document = dict(document["policy"]) # type: ignore[arg-type] - policy_document["sha256"] = policy_sha256 - document["policy"] = policy_document - document["integration"] = integration.to_dict() - _write_json_atomic(args.output, document) + with ExitStack() as resources: + report_input = resources.enter_context(_prepare_input_target(args.report)) + policy_input = resources.enter_context(_prepare_input_target(args.policy)) + integration_input = resources.enter_context( + _prepare_input_target(args.integration) + ) + integration = resources.enter_context( + _validated_integration( + integration_input.read_path, + expected_adapter=args.adapter, + ) + ) + output_target = resources.enter_context( + _prepare_output_target( + args.output, + protected_files=(report_input, policy_input, integration_input), + protected_directory=integration, + ) + ) + evidence = adapter.load( + report_input.read_path, + source_version=integration.commit, + ) + policy, policy_sha256 = load_policy(policy_input.read_path) + decision = evaluate(evidence, policy) + + document = decision.to_dict() + document["schema_version"] = 1 + document["evaluated_at"] = clock().astimezone(timezone.utc).isoformat() + policy_document = dict(document["policy"]) # type: ignore[arg-type] + policy_document["sha256"] = policy_sha256 + document["policy"] = policy_document + document["integration"] = integration.to_dict() + _write_json_atomic(output_target, document) return 0 if decision.decision == "go" else 1 diff --git a/src/agent_release_gate/filesystem.py b/src/agent_release_gate/filesystem.py new file mode 100644 index 0000000..77462e2 --- /dev/null +++ b/src/agent_release_gate/filesystem.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import errno +import os +import stat +from pathlib import Path + + +DirectoryIdentity = tuple[int, int] +FileIdentity = tuple[int, int] + + +def close_best_effort(descriptor: int) -> None: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + +def directory_flags() -> int: + required = ("O_DIRECTORY", "O_NOFOLLOW") + if any(not hasattr(os, name) for name in required): + raise OSError(errno.ENOTSUP, "secure descriptor walks are not supported") + return ( + os.O_RDONLY + | os.O_DIRECTORY + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0) + ) + + +def directory_identity(descriptor: int) -> DirectoryIdentity: + observed = os.fstat(descriptor) + return observed.st_dev, observed.st_ino + + +def file_identity(descriptor: int) -> FileIdentity: + observed = os.fstat(descriptor) + return observed.st_dev, observed.st_ino + + +def same_directory(left_fd: int, right_fd: int) -> bool: + return directory_identity(left_fd) == directory_identity(right_fd) + + +def _walk_absolute_directory(path: Path) -> int: + if not path.is_absolute(): + raise ValueError("descriptor walk requires an absolute path") + current_fd = os.open(os.sep, directory_flags()) + try: + for component in path.parts[1:]: + next_fd = os.open( + component, + directory_flags(), + dir_fd=current_fd, + ) + os.close(current_fd) + current_fd = next_fd + result = current_fd + current_fd = -1 + return result + finally: + close_best_effort(current_fd) + + +def _open_resolved_directory( + resolved: Path, + expected_identity: DirectoryIdentity | None = None, +) -> int: + expected = os.stat(resolved, follow_symlinks=False) + if not stat.S_ISDIR(expected.st_mode): + raise NotADirectoryError(errno.ENOTDIR, "not a directory", str(resolved)) + expected_identity = expected_identity or (expected.st_dev, expected.st_ino) + descriptor = _walk_absolute_directory(resolved) + if directory_identity(descriptor) != expected_identity: + close_best_effort(descriptor) + raise OSError( + errno.ESTALE, + "directory changed while being opened", + str(resolved), + ) + return descriptor + + +def open_directory(path: Path) -> tuple[int, Path]: + expected = os.stat(path) + if not stat.S_ISDIR(expected.st_mode): + raise NotADirectoryError(errno.ENOTDIR, "not a directory", str(path)) + resolved = path.resolve(strict=True) + descriptor = _open_resolved_directory( + resolved, + (expected.st_dev, expected.st_ino), + ) + return descriptor, resolved + + +def open_regular_file(path: Path) -> tuple[int, int, Path]: + expected = os.stat(path) + if not stat.S_ISREG(expected.st_mode): + raise OSError(errno.EINVAL, "not a regular file", str(path)) + resolved = path.resolve(strict=True) + parent_fd = _open_resolved_directory(resolved.parent) + file_fd: int | None = None + try: + file_fd = os.open( + resolved.name, + os.O_RDONLY + | os.O_NOFOLLOW + | os.O_NONBLOCK + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent_fd, + ) + observed = os.fstat(file_fd) + if not stat.S_ISREG(observed.st_mode): + raise OSError(errno.EINVAL, "not a regular file", str(path)) + if (observed.st_dev, observed.st_ino) != (expected.st_dev, expected.st_ino): + raise OSError(errno.ESTALE, "file changed while being opened", str(path)) + result = file_fd + file_fd = None + return result, parent_fd, resolved + except Exception: + close_best_effort(parent_fd) + raise + finally: + if file_fd is not None: + close_best_effort(file_fd) + + +def directory_is_within( + directory_fd: int, + ancestor: DirectoryIdentity, +) -> bool: + current_fd = os.dup(directory_fd) + try: + while True: + if directory_identity(current_fd) == ancestor: + return True + parent_fd = os.open("..", directory_flags(), dir_fd=current_fd) + if same_directory(current_fd, parent_fd): + os.close(parent_fd) + return False + os.close(current_fd) + current_fd = parent_fd + finally: + close_best_effort(current_fd) diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index 8bce878..9fbcd05 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -8,6 +8,15 @@ from pathlib import Path, PurePosixPath from typing import Any +from agent_release_gate.filesystem import ( + DirectoryIdentity, + close_best_effort, + directory_flags, + directory_identity, + open_directory, + same_directory, +) + class IntegrationError(ValueError): """Raised when benchmark provenance cannot be established.""" @@ -15,24 +24,43 @@ class IntegrationError(ValueError): @dataclass(frozen=True, slots=True) class IntegrationManifest: + adapter: str name: str repository_url: str + project_path: Path checkout_path: Path commit: str prohibited_paths: tuple[str, ...] -@dataclass(frozen=True, slots=True) +@dataclass(slots=True) class IntegrationEvidence: + adapter: str name: str checkout_path: Path repository_url: str commit: str + checkout_fd: int + + @property + def checkout_identity(self) -> DirectoryIdentity: + return directory_identity(self.checkout_fd) + + def close(self) -> None: + checkout_fd = self.checkout_fd + self.checkout_fd = -1 + close_best_effort(checkout_fd) + + def __enter__(self) -> IntegrationEvidence: + return self + + def __exit__(self, *_: object) -> None: + self.close() def to_dict(self) -> dict[str, object]: return { + "adapter": self.adapter, "name": self.name, - "checkout_path": str(self.checkout_path), "repository_url": self.repository_url, "commit": self.commit, } @@ -40,6 +68,7 @@ def to_dict(self) -> dict[str, object]: _MANIFEST_KEYS = { "schema_version", + "adapter", "name", "repository_url", "checkout_path", @@ -47,6 +76,7 @@ def to_dict(self) -> dict[str, object]: "prohibited_paths", } _COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_ADAPTER_RE = re.compile(r"^[a-z][a-z0-9_-]*$") def _nonempty_string(raw: dict[str, Any], key: str) -> str: @@ -75,6 +105,10 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: if not isinstance(schema_version, int) or isinstance(schema_version, bool) or schema_version != 1: raise IntegrationError("schema_version must be integer 1") + adapter = _nonempty_string(raw, "adapter") + if not _ADAPTER_RE.fullmatch(adapter): + raise IntegrationError("adapter must be a lowercase identifier") + name = _nonempty_string(raw, "name") repository_url = _nonempty_string(raw, "repository_url") checkout_raw = _nonempty_string(raw, "checkout_path") @@ -84,7 +118,10 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: resolved_project_root = project_root.resolve() checkout_path = (resolved_project_root / checkout_relative).resolve() - if checkout_path.parent != resolved_project_root.parent: + if ( + checkout_path == resolved_project_root + or checkout_path.parent != resolved_project_root.parent + ): raise IntegrationError("checkout_path must resolve to a direct sibling of the project") commit = _nonempty_string(raw, "commit") @@ -104,15 +141,17 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: prohibited_paths.append(normalized.as_posix()) return IntegrationManifest( + adapter=adapter, name=name, repository_url=repository_url, + project_path=resolved_project_root, checkout_path=checkout_path, commit=commit, prohibited_paths=tuple(prohibited_paths), ) -def _git(checkout: Path, *args: str) -> subprocess.CompletedProcess[str]: +def _git(checkout_fd: int, *args: str) -> subprocess.CompletedProcess[str]: environment = { key: value for key, value in os.environ.items() @@ -127,6 +166,10 @@ def _git(checkout: Path, *args: str) -> subprocess.CompletedProcess[str]: "GIT_TERMINAL_PROMPT": "0", } ) + + def enter_checkout() -> None: + os.fchdir(checkout_fd) + return subprocess.run( [ "git", @@ -136,14 +179,14 @@ def _git(checkout: Path, *args: str) -> subprocess.CompletedProcess[str]: "core.fsmonitor=false", "-c", "core.untrackedCache=false", - "-C", - str(checkout), *args, ], check=False, capture_output=True, text=True, env=environment, + pass_fds=(checkout_fd,), + preexec_fn=enter_checkout, ) @@ -152,41 +195,151 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: if not checkout.is_dir(): raise IntegrationError(f"checkout does not exist or is not a directory: {checkout}") - worktree = _git(checkout, "rev-parse", "--is-inside-work-tree") - if worktree.returncode != 0 or worktree.stdout.strip() != "true": - raise IntegrationError(f"checkout is not a Git worktree: {checkout}") - - failures: list[str] = [] - head = _git(checkout, "rev-parse", "HEAD") - observed_commit = head.stdout.strip() if head.returncode == 0 else "unavailable" - if observed_commit != manifest.commit: - failures.append( - f"expected commit {manifest.commit}, observed {observed_commit}" - ) - - origin = _git(checkout, "remote", "get-url", "origin") - observed_origin = origin.stdout.strip() if origin.returncode == 0 else "unavailable" - if observed_origin != manifest.repository_url: - failures.append( - f"unexpected origin URL: expected {manifest.repository_url}, observed {observed_origin}" + checkout_fd: int | None = None + project_fd: int | None = None + checkout_parent_fd: int | None = None + project_parent_fd: int | None = None + try: + try: + checkout_fd, _ = open_directory(checkout) + except OSError as exc: + raise IntegrationError(f"unable to pin checkout {checkout}: {exc}") from exc + + try: + project_fd, _ = open_directory(manifest.project_path) + checkout_parent_fd = os.open( + "..", + directory_flags(), + dir_fd=checkout_fd, + ) + project_parent_fd = os.open( + "..", + directory_flags(), + dir_fd=project_fd, + ) + except OSError as exc: + raise IntegrationError(f"unable to verify checkout placement: {exc}") from exc + if same_directory(checkout_fd, project_fd) or not same_directory( + checkout_parent_fd, + project_parent_fd, + ): + raise IntegrationError( + "checkout must be a distinct direct sibling of the project" + ) + + worktree = _git(checkout_fd, "rev-parse", "--is-inside-work-tree") + if worktree.returncode != 0 or worktree.stdout.strip() != "true": + raise IntegrationError(f"checkout is not a Git worktree: {checkout}") + + prefix = _git(checkout_fd, "rev-parse", "--show-prefix") + if prefix.returncode != 0 or prefix.stdout.strip(): + raise IntegrationError(f"checkout is not a Git worktree root: {checkout}") + + failures: list[str] = [] + head = _git(checkout_fd, "rev-parse", "HEAD") + observed_commit = head.stdout.strip() if head.returncode == 0 else "unavailable" + if observed_commit != manifest.commit: + failures.append( + f"expected commit {manifest.commit}, observed {observed_commit}" + ) + + origin = _git(checkout_fd, "remote", "get-url", "origin") + observed_origin = origin.stdout.strip() if origin.returncode == 0 else "unavailable" + if observed_origin != manifest.repository_url: + failures.append( + f"unexpected origin URL: expected {manifest.repository_url}, observed {observed_origin}" + ) + + worktree_dirty = False + status = _git( + checkout_fd, + "status", + "--porcelain", + "--untracked-files=all", ) - - status = _git(checkout, "status", "--porcelain") - if status.returncode != 0: - failures.append("unable to determine worktree status") - elif status.stdout: - failures.append("worktree is not clean") - - for prohibited_path in manifest.prohibited_paths: - if (checkout / prohibited_path).exists(): + if status.returncode != 0: + failures.append("unable to determine worktree status") + elif status.stdout: + worktree_dirty = True + + untracked = _git(checkout_fd, "ls-files", "--others") + if untracked.returncode != 0: + failures.append("unable to enumerate untracked worktree files") + elif untracked.stdout: + worktree_dirty = True + + if worktree_dirty: + failures.append("worktree is not clean") + + index_flags = _git(checkout_fd, "ls-files", "-v", "-z") + if index_flags.returncode != 0: + failures.append("unable to inspect index flags") + else: + entries = [ + entry + for entry in index_flags.stdout.split("\0") + if entry + ] + if any(entry[0].islower() for entry in entries): + failures.append("index contains assume-unchanged entries") + present_skip_worktree = False + for entry in entries: + if not entry.startswith("S "): + continue + try: + os.stat( + entry[2:], + dir_fd=checkout_fd, + follow_symlinks=False, + ) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as exc: + failures.append( + f"unable to inspect skip-worktree path {entry[2:]}: {exc}" + ) + continue + present_skip_worktree = True + if present_skip_worktree: + failures.append("index contains present skip-worktree entries") + + for prohibited_path in manifest.prohibited_paths: + try: + os.stat( + prohibited_path, + dir_fd=checkout_fd, + follow_symlinks=False, + ) + except FileNotFoundError: + continue + except OSError as exc: + failures.append( + f"unable to inspect prohibited path {prohibited_path}: {exc}" + ) + continue failures.append(f"prohibited path is present: {prohibited_path}") - if failures: - raise IntegrationError("integration validation failed: " + "; ".join(failures)) - - return IntegrationEvidence( - name=manifest.name, - checkout_path=checkout, - repository_url=manifest.repository_url, - commit=manifest.commit, - ) + if failures: + raise IntegrationError( + "integration validation failed: " + "; ".join(failures) + ) + + evidence = IntegrationEvidence( + adapter=manifest.adapter, + name=manifest.name, + checkout_path=checkout, + repository_url=manifest.repository_url, + commit=manifest.commit, + checkout_fd=checkout_fd, + ) + checkout_fd = None + return evidence + finally: + for descriptor in ( + checkout_fd, + project_fd, + checkout_parent_fd, + project_parent_fd, + ): + if descriptor is not None: + close_best_effort(descriptor) diff --git a/tests/package_smoke.py b/tests/package_smoke.py new file mode 100644 index 0000000..835dfe0 --- /dev/null +++ b/tests/package_smoke.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +ORIGIN = "https://example.com/synthetic-benchmark.git" + + +def run(command: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + environment = { + key: value for key, value in os.environ.items() if not key.startswith("GIT_") + } + return subprocess.run( + command, + cwd=cwd, + check=False, + capture_output=True, + text=True, + env=environment, + ) + + +def git(repo: Path, *args: str) -> str: + result = run( + ["git", "-c", "core.hooksPath=/dev/null", "-C", str(repo), *args], + cwd=repo, + ) + if result.returncode != 0: + raise RuntimeError(f"git {' '.join(args)} failed: {result.stderr.strip()}") + return result.stdout.strip() + + +def seed_project(base: Path) -> tuple[Path, Path, Path, Path]: + project = base / "agent-release-gate" + checkout = base / "SyntheticBench" + project.mkdir() + checkout.mkdir() + + git(checkout, "init", "-b", "main") + git(checkout, "config", "user.name", "Agent Release Gate CI") + git(checkout, "config", "user.email", "ci@example.com") + (checkout / "README.md").write_text("synthetic benchmark\n", encoding="utf-8") + git(checkout, "add", "README.md") + git(checkout, "commit", "-m", "test: seed synthetic benchmark") + git(checkout, "remote", "add", "origin", ORIGIN) + commit = git(checkout, "rev-parse", "HEAD") + + manifest = project / "integration.json" + manifest.write_text( + json.dumps( + { + "schema_version": 1, + "adapter": "clawprobench", + "name": "SyntheticBench", + "repository_url": ORIGIN, + "checkout_path": "../SyntheticBench", + "commit": commit, + "prohibited_paths": ["vendor"], + } + ), + encoding="utf-8", + ) + policy = project / "policy.toml" + shutil.copyfile(ROOT / "policies" / "default.toml", policy) + go_report = project / "go.json" + no_go_report = project / "no-go.json" + shutil.copyfile(ROOT / "tests" / "fixtures" / "clawprobench_go.json", go_report) + shutil.copyfile( + ROOT / "tests" / "fixtures" / "clawprobench_no_go.json", + no_go_report, + ) + return project, manifest, go_report, no_go_report + + +def evaluate( + cli: Path, + project: Path, + manifest: Path, + report: Path, + output: Path, +) -> subprocess.CompletedProcess[str]: + return run( + [ + str(cli), + "evaluate", + "--adapter", + "clawprobench", + "--report", + str(report), + "--policy", + str(project / "policy.toml"), + "--integration", + str(manifest), + "--output", + str(output), + ], + cwd=project, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--cli", required=True, type=Path) + args = parser.parse_args() + cli = args.cli.resolve(strict=True) + + with tempfile.TemporaryDirectory() as directory: + project, manifest, go_report, no_go_report = seed_project(Path(directory)) + cases = ( + (go_report, project / "go-decision.json", 0, "go"), + (no_go_report, project / "no-go-decision.json", 1, "no_go"), + ) + for report, output, expected_exit, expected_decision in cases: + result = evaluate(cli, project, manifest, report, output) + if result.returncode != expected_exit: + raise RuntimeError( + f"{expected_decision} smoke returned {result.returncode}: " + f"{result.stderr.strip()}" + ) + observed = json.loads(output.read_text(encoding="utf-8")) + if observed["decision"] != expected_decision: + raise RuntimeError( + f"expected {expected_decision}, observed {observed['decision']}" + ) + if "checkout_path" in observed["integration"]: + raise RuntimeError("decision leaked the local checkout path") + + print("installed-wheel GO and NO_GO smoke checks passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/support.py b/tests/support.py index 5f3c9cf..b256e3a 100644 --- a/tests/support.py +++ b/tests/support.py @@ -42,12 +42,14 @@ def write_manifest( checkout: Path, commit: str, *, + adapter: str = "clawprobench", repository_url: str = TEST_ORIGIN, prohibited_paths: list[str] | None = None, updates: dict[str, object] | None = None, ) -> Path: data: dict[str, object] = { "schema_version": 1, + "adapter": adapter, "name": "SyntheticBench", "repository_url": repository_url, "checkout_path": os.path.relpath(checkout, project_root), diff --git a/tests/test_cli.py b/tests/test_cli.py index 7dd1f3c..631557a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,15 +1,24 @@ from __future__ import annotations +import errno import hashlib import io import json +import os import tempfile import unittest from contextlib import chdir from datetime import datetime, timezone from pathlib import Path - -from agent_release_gate.cli import run +from unittest.mock import patch + +from agent_release_gate.adapters.registry import get_adapter +from agent_release_gate.cli import ( + _prepare_output_target, + _write_json_atomic, + run, +) +from agent_release_gate.domain.models import BenchmarkEvidence from tests.support import create_git_repo, write_manifest @@ -25,8 +34,8 @@ def setUp(self) -> None: self.base = Path(directory.name) self.project_root = self.base / "agent-release-gate" self.project_root.mkdir() - self.checkout, commit = create_git_repo(self.base) - self.manifest = write_manifest(self.project_root, self.checkout, commit) + self.checkout, self.commit = create_git_repo(self.base) + self.manifest = write_manifest(self.project_root, self.checkout, self.commit) def invoke(self, args: list[str]) -> tuple[int, str, str]: stdout = io.StringIO() @@ -59,7 +68,10 @@ def test_doctor_emits_json_and_zero_for_valid_integration(self) -> None: self.assertEqual(0, code) self.assertEqual("", stderr) self.assertTrue(document["valid"]) + self.assertEqual("clawprobench", document["integration"]["adapter"]) self.assertEqual("SyntheticBench", document["integration"]["name"]) + self.assertNotIn("checkout_path", document["integration"]) + self.assertNotIn(str(self.base), stdout) def test_doctor_reports_invalid_integration_and_returns_two(self) -> None: (self.checkout / "dirty.txt").write_text("dirty\n", encoding="utf-8") @@ -142,6 +154,23 @@ def test_unknown_adapter_returns_two(self) -> None: self.assertIn("unknown adapter", stderr) self.assertFalse(output.exists()) + def test_adapter_mismatch_returns_two_without_output(self) -> None: + manifest = write_manifest( + self.project_root, + self.checkout, + self.commit, + adapter="otherbench", + ) + output = self.project_root / "decision.json" + args = self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + args[args.index("--integration") + 1] = str(manifest) + + code, _, stderr = self.invoke(args) + + self.assertEqual(2, code) + self.assertIn("does not match integration adapter", stderr) + self.assertFalse(output.exists()) + def test_output_parent_failure_returns_two(self) -> None: output = self.project_root / "missing" / "decision.json" @@ -153,6 +182,352 @@ def test_output_parent_failure_returns_two(self) -> None: self.assertIn("unable to write decision", stderr) self.assertFalse(output.exists()) + def test_output_cannot_overwrite_report(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + + code, _, stderr = self.invoke(self.evaluate_args(report, report)) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, report.read_bytes()) + + def test_output_hard_link_cannot_overwrite_report(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + output = self.project_root / "decision.json" + os.link(report, output) + + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, report.read_bytes()) + self.assertEqual(original, output.read_bytes()) + + def test_output_case_variant_cannot_overwrite_report(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + output = self.project_root / "REPORT.JSON" + if not output.exists(): + self.skipTest("requires a case-insensitive filesystem") + + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, report.read_bytes()) + + def test_protected_input_linked_to_output_before_replace_is_preserved(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + output = self.project_root / "decision.json" + + with _prepare_output_target( + output, + protected_files=(report, POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + os.link(report, output) + with self.assertRaisesRegex( + ValueError, + "must not overwrite an evaluation input", + ): + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual(original, report.read_bytes()) + self.assertEqual(original, output.read_bytes()) + + def test_output_cannot_overwrite_policy(self) -> None: + policy = self.project_root / "policy.toml" + original = POLICY.read_bytes() + policy.write_bytes(original) + args = self.evaluate_args(FIXTURES / "clawprobench_go.json", policy) + args[args.index("--policy") + 1] = str(policy) + + code, _, stderr = self.invoke(args) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, policy.read_bytes()) + + def test_output_cannot_overwrite_integration_manifest(self) -> None: + original = self.manifest.read_bytes() + + code, _, stderr = self.invoke( + self.evaluate_args( + FIXTURES / "clawprobench_go.json", + self.manifest, + ) + ) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, self.manifest.read_bytes()) + + def test_output_inside_benchmark_checkout_is_rejected_without_write(self) -> None: + output = self.checkout / "decision.json" + + code, _, stderr = self.invoke( + self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + ) + + self.assertEqual(2, code) + self.assertIn("must not be inside the benchmark checkout", stderr) + self.assertFalse(output.exists()) + + def test_output_symlinked_into_benchmark_checkout_is_rejected(self) -> None: + linked_checkout = self.project_root / "linked-checkout" + linked_checkout.symlink_to(self.checkout, target_is_directory=True) + output = linked_checkout / "decision.json" + + code, _, stderr = self.invoke( + self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + ) + + self.assertEqual(2, code) + self.assertIn("must not be inside the benchmark checkout", stderr) + self.assertFalse((self.checkout / "decision.json").exists()) + + def test_output_parent_symlink_swap_cannot_redirect_atomic_write(self) -> None: + safe_directory = self.project_root / "safe" + safe_directory.mkdir() + linked_directory = self.project_root / "out" + linked_directory.symlink_to(safe_directory, target_is_directory=True) + output = linked_directory / "decision.json" + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + linked_directory.unlink() + linked_directory.symlink_to(self.checkout, target_is_directory=True) + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual( + {"decision": "go"}, + json.loads((safe_directory / "decision.json").read_text()), + ) + self.assertFalse((self.checkout / "decision.json").exists()) + + def test_input_parent_symlink_swap_cannot_change_or_overwrite_pinned_report( + self, + ) -> None: + reports = self.base / "reports" + reports.mkdir() + report = reports / "report.json" + original_report = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original_report) + linked_reports = self.project_root / "linked-reports" + linked_reports.symlink_to(reports, target_is_directory=True) + linked_report = linked_reports / "report.json" + output = self.project_root / "report.json" + output.write_bytes((FIXTURES / "clawprobench_no_go.json").read_bytes()) + delegate = get_adapter("clawprobench") + project_root = self.project_root + + class SwappingAdapter: + def load( + self, + report_path: Path, + *, + source_version: str, + ) -> BenchmarkEvidence: + linked_reports.unlink() + linked_reports.symlink_to(project_root, target_is_directory=True) + return delegate.load(report_path, source_version=source_version) + + with patch( + "agent_release_gate.cli.get_adapter", + return_value=SwappingAdapter(), + ): + code, _, stderr = self.invoke(self.evaluate_args(linked_report, output)) + + document = json.loads(output.read_text()) + self.assertEqual(0, code) + self.assertEqual("", stderr) + self.assertEqual("go", document["decision"]) + self.assertEqual("agent-go", document["benchmark"]["subject"]) + self.assertEqual(original_report, report.read_bytes()) + + def test_input_ancestor_swap_during_pin_is_rejected(self) -> None: + requested_directory = self.base / "requested" + requested_inner = requested_directory / "inner" + requested_inner.mkdir(parents=True) + report = requested_inner / "report.json" + report.write_bytes((FIXTURES / "clawprobench_go.json").read_bytes()) + saved_directory = self.base / "saved" + substitute_directory = self.base / "substitute" + substitute_inner = substitute_directory / "inner" + substitute_inner.mkdir(parents=True) + (substitute_inner / "report.json").write_bytes( + (FIXTURES / "clawprobench_no_go.json").read_bytes() + ) + output = self.project_root / "decision.json" + original_resolve = Path.resolve + swapped = False + + def resolve_then_swap(path: Path, *args: object, **kwargs: object) -> Path: + nonlocal swapped + resolved = original_resolve(path, *args, **kwargs) # type: ignore[arg-type] + if path == report and not swapped: + swapped = True + requested_directory.rename(saved_directory) + requested_directory.symlink_to( + substitute_directory, + target_is_directory=True, + ) + return resolved + + with patch.object(Path, "resolve", resolve_then_swap): + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("unable to open evaluation input", stderr) + self.assertFalse(output.exists()) + + def test_input_ancestor_directory_replacement_during_pin_is_rejected( + self, + ) -> None: + requested_directory = self.base / "requested-directory" + requested_inner = requested_directory / "inner" + requested_inner.mkdir(parents=True) + report = requested_inner / "report.json" + report.write_bytes((FIXTURES / "clawprobench_go.json").read_bytes()) + saved_directory = self.base / "saved-directory" + substitute_directory = self.base / "substitute-directory" + substitute_inner = substitute_directory / "inner" + substitute_inner.mkdir(parents=True) + (substitute_inner / "report.json").write_bytes( + (FIXTURES / "clawprobench_no_go.json").read_bytes() + ) + output = self.project_root / "decision.json" + original_resolve = Path.resolve + swapped = False + + def resolve_then_replace(path: Path, *args: object, **kwargs: object) -> Path: + nonlocal swapped + resolved = original_resolve(path, *args, **kwargs) # type: ignore[arg-type] + if path == report and not swapped: + swapped = True + requested_directory.rename(saved_directory) + substitute_directory.rename(requested_directory) + return resolved + + with patch.object(Path, "resolve", resolve_then_replace): + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("unable to open evaluation input", stderr) + self.assertFalse(output.exists()) + + def test_validated_checkout_rename_cannot_redirect_output_into_it(self) -> None: + safe_directory = self.project_root / "safe" + safe_directory.mkdir() + output_link = self.project_root / "out" + output_link.symlink_to(safe_directory, target_is_directory=True) + output = output_link / "decision.json" + renamed_checkout = self.base / "RenamedBenchmark" + original_prepare = _prepare_output_target + + def rename_before_prepare(*args: object, **kwargs: object): + self.checkout.rename(renamed_checkout) + self.checkout.mkdir() + output_link.unlink() + output_link.symlink_to(renamed_checkout, target_is_directory=True) + return original_prepare(*args, **kwargs) # type: ignore[arg-type] + + with patch( + "agent_release_gate.cli._prepare_output_target", + side_effect=rename_before_prepare, + ): + code, _, stderr = self.invoke( + self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + ) + + self.assertEqual(2, code) + self.assertIn("must not be inside the benchmark checkout", stderr) + self.assertFalse((renamed_checkout / "decision.json").exists()) + + def test_output_directory_moved_into_checkout_before_write_is_rejected( + self, + ) -> None: + safe_directory = self.project_root / "safe-output" + safe_directory.mkdir() + output = safe_directory / "decision.json" + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + moved_directory = self.checkout / "moved-output" + safe_directory.rename(moved_directory) + with self.assertRaisesRegex( + ValueError, + "must not be inside the benchmark checkout", + ): + _write_json_atomic(target, {"decision": "go"}) + + self.assertFalse((moved_directory / "decision.json").exists()) + self.assertEqual([], list(moved_directory.iterdir())) + + def test_output_leaf_symlink_is_replaced_without_overwriting_its_target(self) -> None: + victim = self.project_root / "victim.txt" + victim.write_text("keep me\n", encoding="utf-8") + output = self.project_root / "decision.json" + output.symlink_to(victim) + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual("keep me\n", victim.read_text()) + self.assertFalse(output.is_symlink()) + self.assertEqual({"decision": "go"}, json.loads(output.read_text())) + + def test_directory_sync_failure_after_replace_does_not_report_failure(self) -> None: + output = self.project_root / "decision.json" + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + with patch( + "agent_release_gate.cli.os.fsync", + side_effect=(None, OSError(errno.EIO, "simulated directory sync failure")), + ): + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual({"decision": "go"}, json.loads(output.read_text())) + + def test_directory_close_failure_after_replace_does_not_report_failure(self) -> None: + output = self.project_root / "decision.json" + target = _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) + _write_json_atomic(target, {"decision": "go"}) + + with patch( + "agent_release_gate.cli.os.close", + side_effect=OSError(errno.EIO, "simulated close failure"), + ): + target.close() + + self.assertEqual(-1, target.directory_fd) + self.assertEqual({"decision": "go"}, json.loads(output.read_text())) + def test_help_is_available_without_loading_inputs(self) -> None: code, stdout, stderr = self.invoke(["--help"]) diff --git a/tests/test_distribution.py b/tests/test_distribution.py new file mode 100644 index 0000000..8af63c1 --- /dev/null +++ b/tests/test_distribution.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import gzip +import io +import tarfile +import tempfile +import unittest +from pathlib import Path + +from tools.normalize_sdist import normalize_sdist, verify_sdist + + +class DistributionTests(unittest.TestCase): + def test_normalize_sdist_removes_owner_metadata_and_preserves_content(self) -> None: + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "package.tar.gz" + payload = b"release data\n" + with archive.open("wb") as raw: + with gzip.GzipFile( + filename="personal-build.tar.gz", + mode="wb", + fileobj=raw, + mtime=1_777_777_777, + ) as compressed: + with tarfile.open(fileobj=compressed, mode="w|") as destination: + member = tarfile.TarInfo("package/README.md") + member.size = len(payload) + member.mode = 0o640 + member.uid = 501 + member.gid = 20 + member.uname = "localuser" + member.gname = "localgroup" + destination.addfile(member, io.BytesIO(payload)) + + normalize_sdist(archive) + verify_sdist(archive) + + with tarfile.open(archive, "r:gz") as normalized: + members = normalized.getmembers() + self.assertEqual(1, len(members)) + member = members[0] + self.assertEqual((0, 0, "", ""), ( + member.uid, + member.gid, + member.uname, + member.gname, + )) + self.assertEqual(0o640, member.mode) + extracted = normalized.extractfile(member) + self.assertIsNotNone(extracted) + self.assertEqual(payload, extracted.read()) # type: ignore[union-attr] + + header = archive.read_bytes()[:64] + self.assertNotIn(b"personal-build", header) + + def test_verify_sdist_rejects_non_neutral_owner_metadata(self) -> None: + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "package.tar.gz" + with tarfile.open(archive, "w:gz") as destination: + member = tarfile.TarInfo("package/") + member.type = tarfile.DIRTYPE + member.uid = 501 + member.uname = "localuser" + destination.addfile(member) + + with self.assertRaisesRegex(ValueError, "non-neutral owner metadata"): + verify_sdist(archive) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index 76368ee..96383b0 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -7,6 +7,7 @@ from pathlib import Path from unittest.mock import patch +from agent_release_gate.integration import validator as validator_module from agent_release_gate.integration.validator import ( IntegrationError, IntegrationManifest, @@ -38,7 +39,9 @@ def manifest(self, **kwargs: object) -> IntegrationManifest: def test_valid_checkout_returns_provenance(self) -> None: evidence = validate_integration(self.manifest()) + self.addCleanup(evidence.close) + self.assertEqual("clawprobench", evidence.adapter) self.assertEqual("SyntheticBench", evidence.name) self.assertEqual(self.checkout.resolve(), evidence.checkout_path) self.assertEqual(TEST_ORIGIN, evidence.repository_url) @@ -55,6 +58,32 @@ def test_missing_or_non_git_checkout_is_rejected(self) -> None: with self.assertRaisesRegex(IntegrationError, "not a Git worktree"): validate_integration(self.manifest(checkout=nongit)) + def test_nested_directory_inside_ancestor_repo_is_not_a_checkout(self) -> None: + ancestor = self.base / "ancestor" + ancestor.mkdir() + run_git(ancestor, "init", "-b", "main") + run_git(ancestor, "config", "user.name", "Agent Release Gate Tests") + run_git(ancestor, "config", "user.email", "tests@example.com") + nested_project = ancestor / "project" + nested_checkout = ancestor / "nested" + nested_project.mkdir() + nested_checkout.mkdir() + (ancestor / ".gitignore").write_text("nested/\nproject/\n", encoding="utf-8") + (ancestor / "README.md").write_text("ancestor repository\n", encoding="utf-8") + run_git(ancestor, "add", ".gitignore", "README.md") + run_git(ancestor, "commit", "-m", "test: seed ancestor repository") + run_git(ancestor, "remote", "add", "origin", TEST_ORIGIN) + ancestor_commit = run_git(ancestor, "rev-parse", "HEAD") + manifest_path = write_manifest( + nested_project, + nested_checkout, + ancestor_commit, + ) + manifest = load_manifest(manifest_path, project_root=nested_project) + + with self.assertRaisesRegex(IntegrationError, "not a Git worktree root"): + validate_integration(manifest) + def test_wrong_commit_and_origin_are_rejected(self) -> None: wrong_commit = "0" * 40 with self.assertRaisesRegex(IntegrationError, f"expected commit {wrong_commit}"): @@ -69,12 +98,49 @@ def test_dirty_worktree_is_rejected(self) -> None: with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): validate_integration(self.manifest()) + def test_repository_config_cannot_hide_untracked_files(self) -> None: + run_git(self.checkout, "config", "status.showUntrackedFiles", "no") + (self.checkout / "untracked.txt").write_text("dirty\n", encoding="utf-8") + + with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): + validate_integration(self.manifest()) + + def test_repository_excludes_file_cannot_hide_untracked_files(self) -> None: + excludes = self.base / "local-excludes" + excludes.write_text("hidden.txt\n", encoding="utf-8") + run_git(self.checkout, "config", "core.excludesFile", str(excludes)) + (self.checkout / "hidden.txt").write_text("dirty\n", encoding="utf-8") + + with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): + validate_integration(self.manifest()) + + def test_assume_unchanged_cannot_hide_tracked_modification(self) -> None: + run_git(self.checkout, "update-index", "--assume-unchanged", "README.md") + (self.checkout / "README.md").write_text("modified\n", encoding="utf-8") + + with self.assertRaisesRegex( + IntegrationError, + "assume-unchanged", + ): + validate_integration(self.manifest()) + + def test_present_skip_worktree_file_cannot_hide_modification(self) -> None: + run_git(self.checkout, "update-index", "--skip-worktree", "README.md") + (self.checkout / "README.md").write_text("modified\n", encoding="utf-8") + + with self.assertRaisesRegex( + IntegrationError, + "skip-worktree", + ): + validate_integration(self.manifest()) + def test_validation_does_not_refresh_or_rewrite_git_index(self) -> None: index_path = self.checkout / ".git" / "index" before = index_path.read_bytes() os.utime(self.checkout / "README.md", (1_577_836_800, 1_577_836_800)) - validate_integration(self.manifest()) + evidence = validate_integration(self.manifest()) + evidence.close() self.assertEqual(before, index_path.read_bytes()) @@ -95,6 +161,7 @@ def test_inherited_git_repository_context_cannot_override_checkout(self) -> None }, ): evidence = validate_integration(self.manifest()) + self.addCleanup(evidence.close) self.assertEqual(self.checkout.resolve(), evidence.checkout_path) self.assertEqual(self.commit, evidence.commit) @@ -105,6 +172,12 @@ def test_prohibited_directory_is_rejected(self) -> None: with self.assertRaisesRegex(IntegrationError, "prohibited path is present: vendor"): validate_integration(self.manifest()) + def test_prohibited_dangling_symlink_is_rejected(self) -> None: + (self.checkout / "vendor").symlink_to(self.base / "missing-target") + + with self.assertRaisesRegex(IntegrationError, "prohibited path is present: vendor"): + validate_integration(self.manifest()) + def test_validation_aggregates_independent_mismatches(self) -> None: (self.checkout / "vendor").mkdir() (self.checkout / "dirty.txt").write_text("dirty\n", encoding="utf-8") @@ -118,6 +191,34 @@ def test_validation_aggregates_independent_mismatches(self) -> None: self.assertIn("worktree is not clean", message) self.assertIn("prohibited path is present: vendor", message) + def test_checkout_path_swap_cannot_mix_provenance_and_cleanliness(self) -> None: + (self.checkout / "dirty.txt").write_text("dirty\n", encoding="utf-8") + alternate_parent = self.base / "alternate-repository" + alternate_parent.mkdir() + alternate_checkout, _ = create_git_repo(alternate_parent) + saved_checkout = self.base / "PinnedBenchmark" + original_git = validator_module._git + swapped = False + + def swap_before_status(checkout_fd: int, *args: str): + nonlocal swapped + if args == ( + "status", + "--porcelain", + "--untracked-files=all", + ) and not swapped: + swapped = True + self.checkout.rename(saved_checkout) + alternate_checkout.rename(self.checkout) + return original_git(checkout_fd, *args) + + with patch( + "agent_release_gate.integration.validator._git", + side_effect=swap_before_status, + ): + with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): + validate_integration(self.manifest()) + def test_manifest_rejects_unknown_or_missing_keys(self) -> None: path = write_manifest( self.project_root, @@ -134,6 +235,13 @@ def test_manifest_rejects_unknown_or_missing_keys(self) -> None: with self.assertRaisesRegex(IntegrationError, "missing keys: name"): load_manifest(path, project_root=self.project_root) + path = write_manifest(self.project_root, self.checkout, self.commit) + data = json.loads(path.read_text()) + del data["adapter"] + path.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(IntegrationError, "missing keys: adapter"): + load_manifest(path, project_root=self.project_root) + def test_manifest_rejects_unsafe_paths_and_commit(self) -> None: with self.assertRaisesRegex(IntegrationError, "schema_version must be integer 1"): self.manifest(updates={"schema_version": 1.0}) @@ -141,11 +249,48 @@ def test_manifest_rejects_unsafe_paths_and_commit(self) -> None: self.manifest(updates={"checkout_path": str(self.checkout.resolve())}) with self.assertRaisesRegex(IntegrationError, "checkout_path must resolve to a direct sibling"): self.manifest(updates={"checkout_path": "../../escape"}) + for self_reference in (".", "../agent-release-gate"): + with self.subTest(checkout_path=self_reference): + with self.assertRaisesRegex( + IntegrationError, + "checkout_path must resolve to a direct sibling", + ): + self.manifest(updates={"checkout_path": self_reference}) + project_alias = self.base / "project-alias" + project_alias.symlink_to(self.project_root, target_is_directory=True) + with self.assertRaisesRegex( + IntegrationError, + "checkout_path must resolve to a direct sibling", + ): + self.manifest(updates={"checkout_path": "../project-alias"}) with self.assertRaisesRegex(IntegrationError, "commit must be 40 lowercase hexadecimal"): self.manifest(updates={"commit": "ABC"}) with self.assertRaisesRegex(IntegrationError, "prohibited_paths entries must be safe relative paths"): self.manifest(prohibited_paths=["../escape"]) + def test_case_variant_of_project_is_not_checkout(self) -> None: + checkout_alias = self.project_root.parent / self.project_root.name.upper() + if not checkout_alias.is_dir(): + self.skipTest("requires a case-insensitive filesystem") + manifest = self.manifest( + updates={"checkout_path": f"../{self.project_root.name.upper()}"} + ) + + with self.assertRaisesRegex( + IntegrationError, + "distinct direct sibling", + ): + validate_integration(manifest) + + def test_manifest_rejects_unsafe_adapter_names(self) -> None: + for adapter in ("ClawProBench", "claw pro bench", "-clawprobench"): + with self.subTest(adapter=adapter): + with self.assertRaisesRegex( + IntegrationError, + "adapter must be a lowercase identifier", + ): + self.manifest(updates={"adapter": adapter}) + if __name__ == "__main__": unittest.main() diff --git a/tools/normalize_sdist.py b/tools/normalize_sdist.py new file mode 100644 index 0000000..31f4874 --- /dev/null +++ b/tools/normalize_sdist.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import argparse +import gzip +import os +import stat +import tarfile +import tempfile +from collections.abc import Sequence +from pathlib import Path + + +_OWNER_PAX_KEYS = frozenset({"uid", "gid", "uname", "gname"}) + + +def verify_sdist(path: Path) -> None: + with tarfile.open(path, "r:gz") as archive: + for member in archive: + ownership = (member.uid, member.gid, member.uname, member.gname) + if ownership != (0, 0, "", ""): + raise ValueError( + f"non-neutral owner metadata in {path}: " + f"{member.name} has {ownership!r}" + ) + + +def normalize_sdist(path: Path) -> None: + original_mode = stat.S_IMODE(path.stat().st_mode) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w+b", + dir=path.parent, + prefix=f".{path.name}.", + delete=False, + ) as raw_destination: + temporary_path = Path(raw_destination.name) + with tarfile.open(path, "r:gz") as source: + with gzip.GzipFile( + filename="", + mode="wb", + fileobj=raw_destination, + mtime=0, + ) as compressed: + with tarfile.open( + fileobj=compressed, + mode="w|", + format=tarfile.PAX_FORMAT, + ) as destination: + for member in source: + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + member.pax_headers = { + key: value + for key, value in member.pax_headers.items() + if key not in _OWNER_PAX_KEYS + } + payload = source.extractfile(member) if member.isfile() else None + try: + destination.addfile(member, payload) + finally: + if payload is not None: + payload.close() + raw_destination.flush() + os.fsync(raw_destination.fileno()) + + os.chmod(temporary_path, original_mode) + verify_sdist(temporary_path) + os.replace(temporary_path, path) + temporary_path = None + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Remove user and machine ownership metadata from Python sdists." + ) + parser.add_argument("archives", nargs="+", type=Path) + args = parser.parse_args(argv) + for archive in args.archives: + normalize_sdist(archive) + print(f"normalized {archive}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())