From 0981f503d7dcc2db8bf7bf0a06d70398788c4a73 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:54:33 -0400 Subject: [PATCH 01/14] chore: ignore local worktrees --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index b80d7bd..b95d978 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Local planning artifacts docs/superpowers/ +.worktrees/ # Python caches and local environments __pycache__/ From 6c18c4e611da052e420fad1ffca0c806691cb4b6 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:55:40 -0400 Subject: [PATCH 02/14] fix: bind integrations to benchmark adapters --- integrations/clawprobench.lock.json | 1 + src/agent_release_gate/cli.py | 16 ++++++++++++-- .../integration/validator.py | 11 ++++++++++ tests/support.py | 2 ++ tests/test_cli.py | 22 +++++++++++++++++-- tests/test_integration_validator.py | 17 ++++++++++++++ 6 files changed, 65 insertions(+), 4 deletions(-) diff --git a/integrations/clawprobench.lock.json b/integrations/clawprobench.lock.json index 6f588d7..19da442 100644 --- a/integrations/clawprobench.lock.json +++ b/integrations/clawprobench.lock.json @@ -1,5 +1,6 @@ { "schema_version": 1, + "adapter": "clawprobench", "name": "ClawProBench", "repository_url": "https://github.com/suyoumo/ClawProBench.git", "checkout_path": "../ClawProBench", diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index 349f4ae..f58e65c 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -98,8 +98,17 @@ def _write_json_atomic(path: Path, document: dict[str, object]) -> None: pass -def _validated_integration(path: Path): +def _validated_integration( + path: Path, + *, + expected_adapter: str | None = None, +): manifest = load_manifest(path, project_root=Path.cwd()) + if expected_adapter is not None and manifest.adapter != expected_adapter: + raise IntegrationError( + f"requested adapter {expected_adapter!r} does not match " + f"integration adapter {manifest.adapter!r}" + ) return validate_integration(manifest) @@ -118,8 +127,11 @@ def _evaluate( args: argparse.Namespace, clock: Callable[[], datetime], ) -> int: - integration = _validated_integration(args.integration) adapter = get_adapter(args.adapter) + integration = _validated_integration( + args.integration, + expected_adapter=args.adapter, + ) evidence = adapter.load(args.report, source_version=integration.commit) policy, policy_sha256 = load_policy(args.policy) decision = evaluate(evidence, policy) diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index 8bce878..6632096 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -15,6 +15,7 @@ class IntegrationError(ValueError): @dataclass(frozen=True, slots=True) class IntegrationManifest: + adapter: str name: str repository_url: str checkout_path: Path @@ -24,6 +25,7 @@ class IntegrationManifest: @dataclass(frozen=True, slots=True) class IntegrationEvidence: + adapter: str name: str checkout_path: Path repository_url: str @@ -31,6 +33,7 @@ class IntegrationEvidence: def to_dict(self) -> dict[str, object]: return { + "adapter": self.adapter, "name": self.name, "checkout_path": str(self.checkout_path), "repository_url": self.repository_url, @@ -40,6 +43,7 @@ def to_dict(self) -> dict[str, object]: _MANIFEST_KEYS = { "schema_version", + "adapter", "name", "repository_url", "checkout_path", @@ -47,6 +51,7 @@ def to_dict(self) -> dict[str, object]: "prohibited_paths", } _COMMIT_RE = re.compile(r"^[0-9a-f]{40}$") +_ADAPTER_RE = re.compile(r"^[a-z][a-z0-9_-]*$") def _nonempty_string(raw: dict[str, Any], key: str) -> str: @@ -75,6 +80,10 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: if not isinstance(schema_version, int) or isinstance(schema_version, bool) or schema_version != 1: raise IntegrationError("schema_version must be integer 1") + adapter = _nonempty_string(raw, "adapter") + if not _ADAPTER_RE.fullmatch(adapter): + raise IntegrationError("adapter must be a lowercase identifier") + name = _nonempty_string(raw, "name") repository_url = _nonempty_string(raw, "repository_url") checkout_raw = _nonempty_string(raw, "checkout_path") @@ -104,6 +113,7 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: prohibited_paths.append(normalized.as_posix()) return IntegrationManifest( + adapter=adapter, name=name, repository_url=repository_url, checkout_path=checkout_path, @@ -185,6 +195,7 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: raise IntegrationError("integration validation failed: " + "; ".join(failures)) return IntegrationEvidence( + adapter=manifest.adapter, name=manifest.name, checkout_path=checkout, repository_url=manifest.repository_url, diff --git a/tests/support.py b/tests/support.py index 5f3c9cf..b256e3a 100644 --- a/tests/support.py +++ b/tests/support.py @@ -42,12 +42,14 @@ def write_manifest( checkout: Path, commit: str, *, + adapter: str = "clawprobench", repository_url: str = TEST_ORIGIN, prohibited_paths: list[str] | None = None, updates: dict[str, object] | None = None, ) -> Path: data: dict[str, object] = { "schema_version": 1, + "adapter": adapter, "name": "SyntheticBench", "repository_url": repository_url, "checkout_path": os.path.relpath(checkout, project_root), diff --git a/tests/test_cli.py b/tests/test_cli.py index 7dd1f3c..7a5a4ad 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -25,8 +25,8 @@ def setUp(self) -> None: self.base = Path(directory.name) self.project_root = self.base / "agent-release-gate" self.project_root.mkdir() - self.checkout, commit = create_git_repo(self.base) - self.manifest = write_manifest(self.project_root, self.checkout, commit) + self.checkout, self.commit = create_git_repo(self.base) + self.manifest = write_manifest(self.project_root, self.checkout, self.commit) def invoke(self, args: list[str]) -> tuple[int, str, str]: stdout = io.StringIO() @@ -59,6 +59,7 @@ def test_doctor_emits_json_and_zero_for_valid_integration(self) -> None: self.assertEqual(0, code) self.assertEqual("", stderr) self.assertTrue(document["valid"]) + self.assertEqual("clawprobench", document["integration"]["adapter"]) self.assertEqual("SyntheticBench", document["integration"]["name"]) def test_doctor_reports_invalid_integration_and_returns_two(self) -> None: @@ -142,6 +143,23 @@ def test_unknown_adapter_returns_two(self) -> None: self.assertIn("unknown adapter", stderr) self.assertFalse(output.exists()) + def test_adapter_mismatch_returns_two_without_output(self) -> None: + manifest = write_manifest( + self.project_root, + self.checkout, + self.commit, + adapter="otherbench", + ) + output = self.project_root / "decision.json" + args = self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + args[args.index("--integration") + 1] = str(manifest) + + code, _, stderr = self.invoke(args) + + self.assertEqual(2, code) + self.assertIn("does not match integration adapter", stderr) + self.assertFalse(output.exists()) + def test_output_parent_failure_returns_two(self) -> None: output = self.project_root / "missing" / "decision.json" diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index 76368ee..a341248 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -39,6 +39,7 @@ def manifest(self, **kwargs: object) -> IntegrationManifest: def test_valid_checkout_returns_provenance(self) -> None: evidence = validate_integration(self.manifest()) + self.assertEqual("clawprobench", evidence.adapter) self.assertEqual("SyntheticBench", evidence.name) self.assertEqual(self.checkout.resolve(), evidence.checkout_path) self.assertEqual(TEST_ORIGIN, evidence.repository_url) @@ -134,6 +135,13 @@ def test_manifest_rejects_unknown_or_missing_keys(self) -> None: with self.assertRaisesRegex(IntegrationError, "missing keys: name"): load_manifest(path, project_root=self.project_root) + path = write_manifest(self.project_root, self.checkout, self.commit) + data = json.loads(path.read_text()) + del data["adapter"] + path.write_text(json.dumps(data), encoding="utf-8") + with self.assertRaisesRegex(IntegrationError, "missing keys: adapter"): + load_manifest(path, project_root=self.project_root) + def test_manifest_rejects_unsafe_paths_and_commit(self) -> None: with self.assertRaisesRegex(IntegrationError, "schema_version must be integer 1"): self.manifest(updates={"schema_version": 1.0}) @@ -146,6 +154,15 @@ def test_manifest_rejects_unsafe_paths_and_commit(self) -> None: with self.assertRaisesRegex(IntegrationError, "prohibited_paths entries must be safe relative paths"): self.manifest(prohibited_paths=["../escape"]) + def test_manifest_rejects_unsafe_adapter_names(self) -> None: + for adapter in ("ClawProBench", "claw pro bench", "-clawprobench"): + with self.subTest(adapter=adapter): + with self.assertRaisesRegex( + IntegrationError, + "adapter must be a lowercase identifier", + ): + self.manifest(updates={"adapter": adapter}) + if __name__ == "__main__": unittest.main() From a241fd10c9f418f0007221cf78d67c5e15f02f39 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:56:29 -0400 Subject: [PATCH 03/14] fix: protect benchmark and input paths --- src/agent_release_gate/cli.py | 26 ++++++++ .../integration/validator.py | 3 +- tests/test_cli.py | 62 +++++++++++++++++++ tests/test_integration_validator.py | 6 ++ 4 files changed, 96 insertions(+), 1 deletion(-) diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index f58e65c..130c701 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -98,6 +98,27 @@ def _write_json_atomic(path: Path, document: dict[str, object]) -> None: pass +def _validate_output_path( + output: Path, + *, + protected_files: Sequence[Path], + protected_directory: Path, +) -> None: + resolved_output = output.resolve(strict=False) + resolved_directory = protected_directory.resolve(strict=False) + if resolved_output.is_relative_to(resolved_directory): + raise DecisionWriteError( + "output path must not be inside the benchmark checkout" + ) + if any( + resolved_output == protected.resolve(strict=False) + for protected in protected_files + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) + + def _validated_integration( path: Path, *, @@ -132,6 +153,11 @@ def _evaluate( args.integration, expected_adapter=args.adapter, ) + _validate_output_path( + args.output, + protected_files=(args.report, args.policy, args.integration), + protected_directory=integration.checkout_path, + ) evidence = adapter.load(args.report, source_version=integration.commit) policy, policy_sha256 = load_policy(args.policy) decision = evaluate(evidence, policy) diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index 6632096..b8af7ca 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -188,7 +188,8 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: failures.append("worktree is not clean") for prohibited_path in manifest.prohibited_paths: - if (checkout / prohibited_path).exists(): + candidate = checkout / prohibited_path + if candidate.exists() or candidate.is_symlink(): failures.append(f"prohibited path is present: {prohibited_path}") if failures: diff --git a/tests/test_cli.py b/tests/test_cli.py index 7a5a4ad..552de08 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -171,6 +171,68 @@ def test_output_parent_failure_returns_two(self) -> None: self.assertIn("unable to write decision", stderr) self.assertFalse(output.exists()) + def test_output_cannot_overwrite_report(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + + code, _, stderr = self.invoke(self.evaluate_args(report, report)) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, report.read_bytes()) + + def test_output_cannot_overwrite_policy(self) -> None: + policy = self.project_root / "policy.toml" + original = POLICY.read_bytes() + policy.write_bytes(original) + args = self.evaluate_args(FIXTURES / "clawprobench_go.json", policy) + args[args.index("--policy") + 1] = str(policy) + + code, _, stderr = self.invoke(args) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, policy.read_bytes()) + + def test_output_cannot_overwrite_integration_manifest(self) -> None: + original = self.manifest.read_bytes() + + code, _, stderr = self.invoke( + self.evaluate_args( + FIXTURES / "clawprobench_go.json", + self.manifest, + ) + ) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, self.manifest.read_bytes()) + + def test_output_inside_benchmark_checkout_is_rejected_without_write(self) -> None: + output = self.checkout / "decision.json" + + code, _, stderr = self.invoke( + self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + ) + + self.assertEqual(2, code) + self.assertIn("must not be inside the benchmark checkout", stderr) + self.assertFalse(output.exists()) + + def test_output_symlinked_into_benchmark_checkout_is_rejected(self) -> None: + linked_checkout = self.project_root / "linked-checkout" + linked_checkout.symlink_to(self.checkout, target_is_directory=True) + output = linked_checkout / "decision.json" + + code, _, stderr = self.invoke( + self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + ) + + self.assertEqual(2, code) + self.assertIn("must not be inside the benchmark checkout", stderr) + self.assertFalse((self.checkout / "decision.json").exists()) + def test_help_is_available_without_loading_inputs(self) -> None: code, stdout, stderr = self.invoke(["--help"]) diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index a341248..3ee6eb7 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -106,6 +106,12 @@ def test_prohibited_directory_is_rejected(self) -> None: with self.assertRaisesRegex(IntegrationError, "prohibited path is present: vendor"): validate_integration(self.manifest()) + def test_prohibited_dangling_symlink_is_rejected(self) -> None: + (self.checkout / "vendor").symlink_to(self.base / "missing-target") + + with self.assertRaisesRegex(IntegrationError, "prohibited path is present: vendor"): + validate_integration(self.manifest()) + def test_validation_aggregates_independent_mismatches(self) -> None: (self.checkout / "vendor").mkdir() (self.checkout / "dirty.txt").write_text("dirty\n", encoding="utf-8") From ab04a70865ef64cab8ca67f0c0ff4df1377d59e0 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:57:35 -0400 Subject: [PATCH 04/14] fix: omit local paths from provenance --- src/agent_release_gate/integration/validator.py | 1 - tests/test_cli.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index b8af7ca..4378454 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -35,7 +35,6 @@ def to_dict(self) -> dict[str, object]: return { "adapter": self.adapter, "name": self.name, - "checkout_path": str(self.checkout_path), "repository_url": self.repository_url, "commit": self.commit, } diff --git a/tests/test_cli.py b/tests/test_cli.py index 552de08..9fbe722 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -61,6 +61,8 @@ def test_doctor_emits_json_and_zero_for_valid_integration(self) -> None: self.assertTrue(document["valid"]) self.assertEqual("clawprobench", document["integration"]["adapter"]) self.assertEqual("SyntheticBench", document["integration"]["name"]) + self.assertNotIn("checkout_path", document["integration"]) + self.assertNotIn(str(self.base), stdout) def test_doctor_reports_invalid_integration_and_returns_two(self) -> None: (self.checkout / "dirty.txt").write_text("dirty\n", encoding="utf-8") From 8f66d7188a37ba176ce19df8fc4ffa59997b6e08 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 13:58:43 -0400 Subject: [PATCH 05/14] docs: prepare public release guidance --- LICENSE | 202 ++++++++++++++++++++++++++++++++++++++ README.md | 53 ++++++++++ SECURITY.md | 31 ++++++ docs/adding-an-adapter.md | 5 + docs/architecture.md | 30 ++++-- docs/dependencies.md | 29 ++++++ pyproject.toml | 5 + 7 files changed, 346 insertions(+), 9 deletions(-) create mode 100644 LICENSE create mode 100644 SECURITY.md create mode 100644 docs/dependencies.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 738e4df..70ac177 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,33 @@ Agent Release Gate turns completed agent-benchmark evidence into a deterministic The tool reads existing JSON reports. It does not install, import, execute, fetch, or modify ClawProBench. +## Status + +Version `0.1.0` is an early, source-distributed release. The public repository +is intended for inspection and use, but outside pull requests and feature +requests are not being solicited for v0. + +## Installation + +Agent Release Gate is not published to PyPI. Install it from a trusted source +checkout: + +```bash +python3.14 -m venv .venv +.venv/bin/python -m pip install . +.venv/bin/agent-release-gate --help +``` + +To install a wheel produced from the checkout: + +```bash +uv build +python3.14 -m venv /tmp/agent-release-gate +/tmp/agent-release-gate/bin/python -m pip install \ + dist/agent_release_gate-0.1.0-py3-none-any.whl +/tmp/agent-release-gate/bin/agent-release-gate --help +``` + ## Requirements - Python 3.14 or newer; development is verified with `python3.14`. @@ -98,3 +125,29 @@ The v0 manifest pins: - prohibited checked-out paths: `ironclaw` and `nanoclaw`. Generated benchmark reports are inputs to this repository. ClawProBench remains a read-only upstream dependency. + +ClawProBench is licensed under Apache-2.0. Agent Release Gate does not vendor, +modify, or redistribute its source. See [dependency boundaries](docs/dependencies.md) +for the complete build, test, CI, upstream, and audit inventory. + +## Trust Model and Limitations + +- The CLI evaluates supplied reports; it does not prove that a report was + produced honestly or by the pinned benchmark source. +- v0 preserves the report timestamp but does not enforce evidence freshness. +- Custom policies and integration manifests are trusted local configuration. +- The CLI does not fetch or execute benchmark code. +- Decision output must be separate from reports, policies, manifests, and + benchmark checkouts; protected paths are rejected after symlink resolution. +- v0 supports only the documented ClawProBench report shape and default + integration. Unknown additive report fields are tolerated. +- A deterministic `go` means only that the supplied evidence satisfies the + supplied policy. It is not a general security certification. + +Serialized decisions include the pinned repository URL and commit but omit the +absolute local checkout path to avoid leaking machine-specific information. + +## Security and License + +Report vulnerabilities through the private process in [SECURITY.md](SECURITY.md). +Agent Release Gate is licensed under the [Apache License 2.0](LICENSE). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..19d60bc --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,31 @@ +# Security Policy + +## Supported Versions + +Agent Release Gate is an early-stage project. Only the latest `0.1.x` release +is supported with security fixes. + +## Reporting a Vulnerability + +Use GitHub's **Security** tab and select **Report a vulnerability**. This opens +a private report with the repository owner. Do not disclose suspected +vulnerabilities in a public issue, pull request, discussion, or benchmark +artifact, and do not include credentials, tokens, private data, or exploit data +that is not necessary to reproduce the problem. + +If private vulnerability reporting is unavailable, do not publish the details. +Wait until the repository provides a private reporting channel. + +Include the affected version or commit, the relevant command and input shape, +the observed impact, and the smallest safe reproduction. Redact personal data +and secrets. + +Reports are acknowledged and handled on a best-effort basis. No response or +remediation timeline is guaranteed for this early release. + +## Scope + +Security reports may cover the CLI, report and policy parsing, integration +provenance checks, output-path handling, packaging, and CI configuration. +ClawProBench is an independent upstream project; report vulnerabilities in its +code to its maintainers. diff --git a/docs/adding-an-adapter.md b/docs/adding-an-adapter.md index a91fc33..8ff3292 100644 --- a/docs/adding-an-adapter.md +++ b/docs/adding-an-adapter.md @@ -48,6 +48,11 @@ _ADAPTERS = MappingProxyType( Add a strict manifest under `integrations/` for the benchmark source. The manifest identifies the source repository, audited version, expected local checkout, and prohibited paths. Adapter code receives the validated source version; it does not perform Git operations itself. +The manifest's `adapter` field must exactly match the lowercase registry key. +Evaluation rejects a different `--adapter` value before reading a report or +writing a decision. Keep the checkout as a direct sibling, and never use a +decision output path inside that checkout. + ## Tests Start with synthetic, hand-checked reports. Tests must cover: diff --git a/docs/architecture.md b/docs/architecture.md index 4302ec6..5319e01 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -31,19 +31,21 @@ This direction lets another agent benchmark reuse the same policies, evaluator, For `doctor`: 1. Load and strictly validate the integration manifest. -2. Resolve its checkout as a direct sibling of this repository. -3. verify Git worktree state, `HEAD`, `origin`, cleanliness, and prohibited paths. -4. Emit validated provenance as JSON. +2. Bind the manifest's declared adapter to the registry key used for evaluation. +3. Resolve its checkout as a direct sibling of this repository. +4. Verify Git worktree state, `HEAD`, `origin`, cleanliness, and prohibited paths. +5. Emit validated provenance as JSON without the absolute local checkout path. For `evaluate`: 1. Perform the same integration validation. -2. Select the named benchmark adapter. -3. Validate and normalize the report into `BenchmarkEvidence`. -4. Parse and validate the TOML policy, retaining its SHA-256 digest. -5. Apply every gate rule in stable order. -6. Combine decision, observed metrics, report identity, integration provenance, policy identity, and UTC evaluation time. -7. Write sorted, indented JSON to a sibling temporary file, sync it, and replace the target atomically. +2. Select the named benchmark adapter and require it to match the manifest. +3. Resolve the output path and reject benchmark-checkout or input-file targets. +4. Validate and normalize the report into `BenchmarkEvidence`. +5. Parse and validate the TOML policy, retaining its SHA-256 digest. +6. Apply every gate rule in stable order. +7. Combine decision, observed metrics, report identity, integration provenance, policy identity, and UTC evaluation time. +8. Write sorted, indented JSON to a sibling temporary file, sync it, and replace the target atomically. Input errors are distinct from release outcomes. A valid `no_go` is exit code `1`; malformed evidence or unproven provenance is exit code `2` and produces no decision. @@ -77,6 +79,16 @@ The integration manifest pins the expected origin and full Git commit. Validatio Git hooks and fsmonitor are disabled for these read-only subprocesses, terminal prompts are disabled, and `GIT_OPTIONAL_LOCKS=0` prevents status checks from refreshing the upstream index. The validator also disables Git's untracked cache, ignores global/system Git configuration, and removes inherited `GIT_*` variables before setting its explicit safe environment. This prevents ambient `GIT_DIR`, `GIT_WORK_TREE`, or index overrides from redirecting a probe away from the pinned checkout. Validation never fetches, checks out, resets, cleans, or writes upstream files. +The integration manifest declares the adapter that may consume its evidence. +This prevents a CLI invocation from presenting one benchmark's source +provenance alongside a different adapter. Prohibited paths are detected even +when represented by dangling symlinks. + +Before creating a temporary decision file, the CLI resolves the requested +output path. It rejects paths inside the benchmark checkout and paths equal to +the report, policy, or integration manifest, including aliases reached through +symlinked parents. The resolved checkout stays internal and is not serialized. + ## Failure Handling Expected input and filesystem failures produce a concise stderr message without a traceback. Atomic output prevents a failed evaluation from truncating a previous decision. Unexpected failures are contained at the command boundary and return exit code `2`. diff --git a/docs/dependencies.md b/docs/dependencies.md new file mode 100644 index 0000000..50d1d81 --- /dev/null +++ b/docs/dependencies.md @@ -0,0 +1,29 @@ +# Dependencies and Third-Party Boundary + +Agent Release Gate has no third-party runtime or test dependencies. The table +below inventories tools involved in building, verifying, or integrating the +project; build and audit tools are not shipped in the wheel. + +| Role | Component | Version constraint | License | Distribution boundary | +| --- | --- | --- | --- | --- | +| Runtime | Python standard library | Python 3.14+ | PSF License | Required interpreter; not bundled | +| Tests | `unittest` | Python 3.14+ | PSF License | Standard library; not bundled | +| Build backend | setuptools | `>=80` | MIT | Isolated build dependency; not bundled | +| Build frontend | build | `1.3.0` in CI | MIT | CI/development tool; not bundled | +| CI action | `actions/checkout` | `v7.0.1`, pinned by SHA | MIT | GitHub Actions only | +| CI action | `actions/setup-python` | `v7.0.0`, pinned by SHA | MIT | GitHub Actions only | +| Upstream evidence producer | ClawProBench | commit `c4b8395854fe0752eef435b44f140366efd44d8e` | Apache-2.0 | External read-only checkout; no source vendored | +| Release audit | Gitleaks | Exact release recorded per audit | MIT | Temporary audit tool; not bundled | + +License sources: + +- Python: +- setuptools: +- build: +- actions/checkout: +- actions/setup-python: +- ClawProBench: +- Gitleaks: + +The JSON fixtures under `tests/fixtures/` are original, hand-authored synthetic +data. They contain no copied benchmark results or user data. diff --git a/pyproject.toml b/pyproject.toml index 53839c0..916b38c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,9 +7,14 @@ name = "agent-release-gate" version = "0.1.0" description = "Turn agent benchmark evidence into deterministic release decisions." readme = "README.md" +license = "Apache-2.0" requires-python = ">=3.14" dependencies = [] +[project.urls] +Repository = "https://github.com/bsha6/agent-release-gate" +Issues = "https://github.com/bsha6/agent-release-gate/issues" + [project.scripts] agent-release-gate = "agent_release_gate.cli:main" From d5a86d66c2ea41fb137aa1abdc511a1668c17dcd Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:00:50 -0400 Subject: [PATCH 06/14] ci: verify Python 3.14 package --- .github/workflows/ci.yml | 45 ++++++++++++++++++++++++++++++++++++++++ MANIFEST.in | 5 +++++ README.md | 3 +++ docs/dependencies.md | 2 +- 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/ci.yml create mode 100644 MANIFEST.in diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..1a1e941 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + test-package: + runs-on: ubuntu-latest + timeout-minutes: 10 + env: + PIP_DISABLE_PIP_VERSION_CHECK: "1" + PYTHONDONTWRITEBYTECODE: "1" + steps: + # actions/checkout v7.0.1 + - name: Check out source + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + + # actions/setup-python v7.0.0 + - name: Set up Python 3.14 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14" + + - name: Run tests + run: PYTHONPATH=src python -m unittest discover -s tests -v + + - name: Build source and wheel distributions + run: | + python -m pip install build==1.6.0 + python -m build + + - name: Install and inspect wheel + run: | + python -m venv "${RUNNER_TEMP}/agent-release-gate-smoke" + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/python" -m pip install --no-deps dist/*.whl + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/python" -c \ + 'from importlib.metadata import metadata, requires; m = metadata("agent-release-gate"); assert m["Version"] == "0.1.0"; assert m["License-Expression"] == "Apache-2.0"; assert not (requires("agent-release-gate") or [])' + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/agent-release-gate" --help diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..a416b85 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include SECURITY.md +recursive-include docs *.md +recursive-include integrations *.json +recursive-include policies *.toml +recursive-include tests *.py *.json diff --git a/README.md b/README.md index 70ac177..1091c75 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,9 @@ for the complete build, test, CI, upstream, and audit inventory. - The CLI does not fetch or execute benchmark code. - Decision output must be separate from reports, policies, manifests, and benchmark checkouts; protected paths are rejected after symlink resolution. +- The source distribution includes the default policy and integration manifest. + A standalone wheel contains only the CLI package, so invoke it from a source + checkout or pass explicit `--policy` and `--integration` paths. - v0 supports only the documented ClawProBench report shape and default integration. Unknown additive report fields are tolerated. - A deterministic `go` means only that the supplied evidence satisfies the diff --git a/docs/dependencies.md b/docs/dependencies.md index 50d1d81..2118f5a 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -9,7 +9,7 @@ project; build and audit tools are not shipped in the wheel. | Runtime | Python standard library | Python 3.14+ | PSF License | Required interpreter; not bundled | | Tests | `unittest` | Python 3.14+ | PSF License | Standard library; not bundled | | Build backend | setuptools | `>=80` | MIT | Isolated build dependency; not bundled | -| Build frontend | build | `1.3.0` in CI | MIT | CI/development tool; not bundled | +| Build frontend | build | `1.6.0` in CI | MIT | CI/development tool; not bundled | | CI action | `actions/checkout` | `v7.0.1`, pinned by SHA | MIT | GitHub Actions only | | CI action | `actions/setup-python` | `v7.0.0`, pinned by SHA | MIT | GitHub Actions only | | Upstream evidence producer | ClawProBench | commit `c4b8395854fe0752eef435b44f140366efd44d8e` | Apache-2.0 | External read-only checkout; no source vendored | From d8c719ade949fdd135c31cabc95d167eadcc5538 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:22:13 -0400 Subject: [PATCH 07/14] fix: Secure provenance and decision output Co-Authored-By: Codex GPT-5.6 --- src/agent_release_gate/cli.py | 196 +++++++++++++++--- .../integration/validator.py | 9 + tests/test_cli.py | 81 +++++++- tests/test_integration_validator.py | 26 +++ 4 files changed, 283 insertions(+), 29 deletions(-) diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index 130c701..fd10d47 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -3,10 +3,11 @@ import argparse import json import os +import secrets import sys -import tempfile from collections.abc import Callable, Sequence from contextlib import redirect_stderr, redirect_stdout +from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import NoReturn, TextIO @@ -17,6 +18,7 @@ from agent_release_gate.evaluation.evaluator import evaluate from agent_release_gate.integration.validator import ( IntegrationError, + IntegrationEvidence, load_manifest, validate_integration, ) @@ -26,6 +28,30 @@ class DecisionWriteError(ValueError): """Raised when a completed decision cannot be written atomically.""" +@dataclass(slots=True) +class _OutputTarget: + directory_fd: int + name: str + display_path: Path + + def close(self) -> None: + directory_fd = self.directory_fd + self.directory_fd = -1 + if directory_fd >= 0: + try: + os.close(directory_fd) + except OSError: + # This read-only guard descriptor cannot affect a committed + # output, and a close error must not mask the body result. + pass + + def __enter__(self) -> _OutputTarget: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="agent-release-gate", @@ -66,8 +92,51 @@ def build_parser() -> argparse.ArgumentParser: return parser -def _write_json_atomic(path: Path, document: dict[str, object]) -> None: - temporary_path: Path | None = None +def _directory_flags() -> int: + required = ("O_DIRECTORY", "O_NOFOLLOW") + if any(not hasattr(os, name) for name in required): + raise DecisionWriteError("secure output writes are not supported on this platform") + return ( + os.O_RDONLY + | os.O_DIRECTORY + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0) + ) + + +def _open_directory(path: Path) -> int: + return os.open(path, _directory_flags()) + + +def _same_directory(left_fd: int, right_fd: int) -> bool: + left = os.fstat(left_fd) + right = os.fstat(right_fd) + return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) + + +def _directory_is_within(directory_fd: int, ancestor: Path) -> bool: + ancestor_fd = _open_directory(ancestor.resolve(strict=True)) + current_fd = os.dup(directory_fd) + try: + while True: + if _same_directory(current_fd, ancestor_fd): + return True + parent_fd = os.open("..", _directory_flags(), dir_fd=current_fd) + if _same_directory(current_fd, parent_fd): + os.close(parent_fd) + return False + os.close(current_fd) + current_fd = parent_fd + finally: + os.close(current_fd) + os.close(ancestor_fd) + + +def _write_json_atomic( + target: _OutputTarget, + document: dict[str, object], +) -> None: + temporary_name: str | None = None try: serialized = json.dumps( document, @@ -75,25 +144,42 @@ def _write_json_atomic(path: Path, document: dict[str, object]) -> None: sort_keys=True, allow_nan=False, ) + "\n" - with tempfile.NamedTemporaryFile( + temporary_name = f".{target.name}.{secrets.token_hex(16)}" + temporary_fd = os.open( + temporary_name, + os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, + 0o600, + dir_fd=target.directory_fd, + ) + with os.fdopen( + temporary_fd, mode="w", encoding="utf-8", - dir=path.parent, - prefix=f".{path.name}.", - delete=False, ) as temporary: - temporary_path = Path(temporary.name) temporary.write(serialized) temporary.flush() os.fsync(temporary.fileno()) - os.replace(temporary_path, path) - temporary_path = None + os.replace( + temporary_name, + target.name, + src_dir_fd=target.directory_fd, + dst_dir_fd=target.directory_fd, + ) + temporary_name = None + try: + os.fsync(target.directory_fd) + except OSError: + # The rename has already committed the new file. Reporting failure + # here would falsely promise that an existing output was preserved. + pass except (OSError, TypeError, ValueError) as exc: - raise DecisionWriteError(f"unable to write decision {path}: {exc}") from exc + raise DecisionWriteError( + f"unable to write decision {target.display_path}: {exc}" + ) from exc finally: - if temporary_path is not None: + if temporary_name is not None: try: - temporary_path.unlink(missing_ok=True) + os.unlink(temporary_name, dir_fd=target.directory_fd) except OSError: pass @@ -103,7 +189,7 @@ def _validate_output_path( *, protected_files: Sequence[Path], protected_directory: Path, -) -> None: +) -> Path: resolved_output = output.resolve(strict=False) resolved_directory = protected_directory.resolve(strict=False) if resolved_output.is_relative_to(resolved_directory): @@ -117,13 +203,66 @@ def _validate_output_path( raise DecisionWriteError( "output path must not overwrite an evaluation input" ) + return resolved_output + + +def _prepare_output_target( + output: Path, + *, + protected_files: Sequence[Path], + protected_directory: Path, +) -> _OutputTarget: + resolved_output = _validate_output_path( + output, + protected_files=protected_files, + protected_directory=protected_directory, + ) + output_name = output.name + if not output_name: + raise DecisionWriteError("output path must name a file") + + directory_fd: int | None = None + try: + directory_fd = _open_directory(output.parent.resolve(strict=False)) + if _directory_is_within(directory_fd, protected_directory): + raise DecisionWriteError( + "output path must not be inside the benchmark checkout" + ) + + for protected in protected_files: + resolved_protected = protected.resolve(strict=False) + if output_name != resolved_protected.name: + continue + protected_parent_fd = _open_directory(resolved_protected.parent) + try: + if _same_directory(directory_fd, protected_parent_fd): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) + finally: + os.close(protected_parent_fd) + + target = _OutputTarget( + directory_fd=directory_fd, + name=output_name, + display_path=output, + ) + directory_fd = None + return target + except DecisionWriteError: + raise + except OSError as exc: + raise DecisionWriteError(f"unable to write decision {output}: {exc}") from exc + finally: + if directory_fd is not None: + os.close(directory_fd) def _validated_integration( path: Path, *, expected_adapter: str | None = None, -): +) -> IntegrationEvidence: manifest = load_manifest(path, project_root=Path.cwd()) if expected_adapter is not None and manifest.adapter != expected_adapter: raise IntegrationError( @@ -153,23 +292,24 @@ def _evaluate( args.integration, expected_adapter=args.adapter, ) - _validate_output_path( + output_target = _prepare_output_target( args.output, protected_files=(args.report, args.policy, args.integration), protected_directory=integration.checkout_path, ) - evidence = adapter.load(args.report, source_version=integration.commit) - policy, policy_sha256 = load_policy(args.policy) - decision = evaluate(evidence, policy) - - document = decision.to_dict() - document["schema_version"] = 1 - document["evaluated_at"] = clock().astimezone(timezone.utc).isoformat() - policy_document = dict(document["policy"]) # type: ignore[arg-type] - policy_document["sha256"] = policy_sha256 - document["policy"] = policy_document - document["integration"] = integration.to_dict() - _write_json_atomic(args.output, document) + with output_target: + evidence = adapter.load(args.report, source_version=integration.commit) + policy, policy_sha256 = load_policy(args.policy) + decision = evaluate(evidence, policy) + + document = decision.to_dict() + document["schema_version"] = 1 + document["evaluated_at"] = clock().astimezone(timezone.utc).isoformat() + policy_document = dict(document["policy"]) # type: ignore[arg-type] + policy_document["sha256"] = policy_sha256 + document["policy"] = policy_document + document["integration"] = integration.to_dict() + _write_json_atomic(output_target, document) return 0 if decision.decision == "go" else 1 diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index 4378454..8b10d3c 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -165,6 +165,15 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: if worktree.returncode != 0 or worktree.stdout.strip() != "true": raise IntegrationError(f"checkout is not a Git worktree: {checkout}") + top_level = _git(checkout, "rev-parse", "--show-toplevel") + observed_top_level = ( + Path(top_level.stdout.strip()).resolve() + if top_level.returncode == 0 and top_level.stdout.strip() + else None + ) + if observed_top_level != checkout.resolve(): + raise IntegrationError(f"checkout is not a Git worktree root: {checkout}") + failures: list[str] = [] head = _git(checkout, "rev-parse", "HEAD") observed_commit = head.stdout.strip() if head.returncode == 0 else "unavailable" diff --git a/tests/test_cli.py b/tests/test_cli.py index 9fbe722..d77c11f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import hashlib import io import json @@ -8,8 +9,13 @@ from contextlib import chdir from datetime import datetime, timezone from pathlib import Path +from unittest.mock import patch -from agent_release_gate.cli import run +from agent_release_gate.cli import ( + _prepare_output_target, + _write_json_atomic, + run, +) from tests.support import create_git_repo, write_manifest @@ -235,6 +241,79 @@ def test_output_symlinked_into_benchmark_checkout_is_rejected(self) -> None: self.assertIn("must not be inside the benchmark checkout", stderr) self.assertFalse((self.checkout / "decision.json").exists()) + def test_output_parent_symlink_swap_cannot_redirect_atomic_write(self) -> None: + safe_directory = self.project_root / "safe" + safe_directory.mkdir() + linked_directory = self.project_root / "out" + linked_directory.symlink_to(safe_directory, target_is_directory=True) + output = linked_directory / "decision.json" + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + linked_directory.unlink() + linked_directory.symlink_to(self.checkout, target_is_directory=True) + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual( + {"decision": "go"}, + json.loads((safe_directory / "decision.json").read_text()), + ) + self.assertFalse((self.checkout / "decision.json").exists()) + + def test_output_leaf_symlink_is_replaced_without_overwriting_its_target(self) -> None: + victim = self.project_root / "victim.txt" + victim.write_text("keep me\n", encoding="utf-8") + output = self.project_root / "decision.json" + output.symlink_to(victim) + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual("keep me\n", victim.read_text()) + self.assertFalse(output.is_symlink()) + self.assertEqual({"decision": "go"}, json.loads(output.read_text())) + + def test_directory_sync_failure_after_replace_does_not_report_failure(self) -> None: + output = self.project_root / "decision.json" + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + with patch( + "agent_release_gate.cli.os.fsync", + side_effect=(None, OSError(errno.EIO, "simulated directory sync failure")), + ): + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual({"decision": "go"}, json.loads(output.read_text())) + + def test_directory_close_failure_after_replace_does_not_report_failure(self) -> None: + output = self.project_root / "decision.json" + target = _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) + _write_json_atomic(target, {"decision": "go"}) + + with patch( + "agent_release_gate.cli.os.close", + side_effect=OSError(errno.EIO, "simulated close failure"), + ): + target.close() + + self.assertEqual(-1, target.directory_fd) + self.assertEqual({"decision": "go"}, json.loads(output.read_text())) + def test_help_is_available_without_loading_inputs(self) -> None: code, stdout, stderr = self.invoke(["--help"]) diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index 3ee6eb7..412b265 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -56,6 +56,32 @@ def test_missing_or_non_git_checkout_is_rejected(self) -> None: with self.assertRaisesRegex(IntegrationError, "not a Git worktree"): validate_integration(self.manifest(checkout=nongit)) + def test_nested_directory_inside_ancestor_repo_is_not_a_checkout(self) -> None: + ancestor = self.base / "ancestor" + ancestor.mkdir() + run_git(ancestor, "init", "-b", "main") + run_git(ancestor, "config", "user.name", "Agent Release Gate Tests") + run_git(ancestor, "config", "user.email", "tests@example.com") + nested_project = ancestor / "project" + nested_checkout = ancestor / "nested" + nested_project.mkdir() + nested_checkout.mkdir() + (ancestor / ".gitignore").write_text("nested/\nproject/\n", encoding="utf-8") + (ancestor / "README.md").write_text("ancestor repository\n", encoding="utf-8") + run_git(ancestor, "add", ".gitignore", "README.md") + run_git(ancestor, "commit", "-m", "test: seed ancestor repository") + run_git(ancestor, "remote", "add", "origin", TEST_ORIGIN) + ancestor_commit = run_git(ancestor, "rev-parse", "HEAD") + manifest_path = write_manifest( + nested_project, + nested_checkout, + ancestor_commit, + ) + manifest = load_manifest(manifest_path, project_root=nested_project) + + with self.assertRaisesRegex(IntegrationError, "not a Git worktree root"): + validate_integration(manifest) + def test_wrong_commit_and_origin_are_rejected(self) -> None: wrong_commit = "0" * 40 with self.assertRaisesRegex(IntegrationError, f"expected commit {wrong_commit}"): From 6744ee47598eff1ca4fe1cda494b39f3165f5458 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:22:24 -0400 Subject: [PATCH 08/14] build: Sanitize and smoke-test release artifacts Co-Authored-By: Codex GPT-5.6 --- .github/workflows/ci.yml | 3 + MANIFEST.in | 1 + tests/package_smoke.py | 140 +++++++++++++++++++++++++++++++++++++ tests/test_distribution.py | 71 +++++++++++++++++++ tools/normalize_sdist.py | 91 ++++++++++++++++++++++++ 5 files changed, 306 insertions(+) create mode 100644 tests/package_smoke.py create mode 100644 tests/test_distribution.py create mode 100644 tools/normalize_sdist.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a1e941..bb64efa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: run: | python -m pip install build==1.6.0 python -m build + python tools/normalize_sdist.py dist/*.tar.gz - name: Install and inspect wheel run: | @@ -43,3 +44,5 @@ jobs: "${RUNNER_TEMP}/agent-release-gate-smoke/bin/python" -c \ 'from importlib.metadata import metadata, requires; m = metadata("agent-release-gate"); assert m["Version"] == "0.1.0"; assert m["License-Expression"] == "Apache-2.0"; assert not (requires("agent-release-gate") or [])' "${RUNNER_TEMP}/agent-release-gate-smoke/bin/agent-release-gate" --help + "${RUNNER_TEMP}/agent-release-gate-smoke/bin/python" tests/package_smoke.py \ + --cli "${RUNNER_TEMP}/agent-release-gate-smoke/bin/agent-release-gate" diff --git a/MANIFEST.in b/MANIFEST.in index a416b85..bf88107 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ recursive-include docs *.md recursive-include integrations *.json recursive-include policies *.toml recursive-include tests *.py *.json +recursive-include tools *.py diff --git a/tests/package_smoke.py b/tests/package_smoke.py new file mode 100644 index 0000000..835dfe0 --- /dev/null +++ b/tests/package_smoke.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + + +ROOT = Path(__file__).parents[1] +ORIGIN = "https://example.com/synthetic-benchmark.git" + + +def run(command: list[str], *, cwd: Path) -> subprocess.CompletedProcess[str]: + environment = { + key: value for key, value in os.environ.items() if not key.startswith("GIT_") + } + return subprocess.run( + command, + cwd=cwd, + check=False, + capture_output=True, + text=True, + env=environment, + ) + + +def git(repo: Path, *args: str) -> str: + result = run( + ["git", "-c", "core.hooksPath=/dev/null", "-C", str(repo), *args], + cwd=repo, + ) + if result.returncode != 0: + raise RuntimeError(f"git {' '.join(args)} failed: {result.stderr.strip()}") + return result.stdout.strip() + + +def seed_project(base: Path) -> tuple[Path, Path, Path, Path]: + project = base / "agent-release-gate" + checkout = base / "SyntheticBench" + project.mkdir() + checkout.mkdir() + + git(checkout, "init", "-b", "main") + git(checkout, "config", "user.name", "Agent Release Gate CI") + git(checkout, "config", "user.email", "ci@example.com") + (checkout / "README.md").write_text("synthetic benchmark\n", encoding="utf-8") + git(checkout, "add", "README.md") + git(checkout, "commit", "-m", "test: seed synthetic benchmark") + git(checkout, "remote", "add", "origin", ORIGIN) + commit = git(checkout, "rev-parse", "HEAD") + + manifest = project / "integration.json" + manifest.write_text( + json.dumps( + { + "schema_version": 1, + "adapter": "clawprobench", + "name": "SyntheticBench", + "repository_url": ORIGIN, + "checkout_path": "../SyntheticBench", + "commit": commit, + "prohibited_paths": ["vendor"], + } + ), + encoding="utf-8", + ) + policy = project / "policy.toml" + shutil.copyfile(ROOT / "policies" / "default.toml", policy) + go_report = project / "go.json" + no_go_report = project / "no-go.json" + shutil.copyfile(ROOT / "tests" / "fixtures" / "clawprobench_go.json", go_report) + shutil.copyfile( + ROOT / "tests" / "fixtures" / "clawprobench_no_go.json", + no_go_report, + ) + return project, manifest, go_report, no_go_report + + +def evaluate( + cli: Path, + project: Path, + manifest: Path, + report: Path, + output: Path, +) -> subprocess.CompletedProcess[str]: + return run( + [ + str(cli), + "evaluate", + "--adapter", + "clawprobench", + "--report", + str(report), + "--policy", + str(project / "policy.toml"), + "--integration", + str(manifest), + "--output", + str(output), + ], + cwd=project, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--cli", required=True, type=Path) + args = parser.parse_args() + cli = args.cli.resolve(strict=True) + + with tempfile.TemporaryDirectory() as directory: + project, manifest, go_report, no_go_report = seed_project(Path(directory)) + cases = ( + (go_report, project / "go-decision.json", 0, "go"), + (no_go_report, project / "no-go-decision.json", 1, "no_go"), + ) + for report, output, expected_exit, expected_decision in cases: + result = evaluate(cli, project, manifest, report, output) + if result.returncode != expected_exit: + raise RuntimeError( + f"{expected_decision} smoke returned {result.returncode}: " + f"{result.stderr.strip()}" + ) + observed = json.loads(output.read_text(encoding="utf-8")) + if observed["decision"] != expected_decision: + raise RuntimeError( + f"expected {expected_decision}, observed {observed['decision']}" + ) + if "checkout_path" in observed["integration"]: + raise RuntimeError("decision leaked the local checkout path") + + print("installed-wheel GO and NO_GO smoke checks passed") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_distribution.py b/tests/test_distribution.py new file mode 100644 index 0000000..8af63c1 --- /dev/null +++ b/tests/test_distribution.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import gzip +import io +import tarfile +import tempfile +import unittest +from pathlib import Path + +from tools.normalize_sdist import normalize_sdist, verify_sdist + + +class DistributionTests(unittest.TestCase): + def test_normalize_sdist_removes_owner_metadata_and_preserves_content(self) -> None: + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "package.tar.gz" + payload = b"release data\n" + with archive.open("wb") as raw: + with gzip.GzipFile( + filename="personal-build.tar.gz", + mode="wb", + fileobj=raw, + mtime=1_777_777_777, + ) as compressed: + with tarfile.open(fileobj=compressed, mode="w|") as destination: + member = tarfile.TarInfo("package/README.md") + member.size = len(payload) + member.mode = 0o640 + member.uid = 501 + member.gid = 20 + member.uname = "localuser" + member.gname = "localgroup" + destination.addfile(member, io.BytesIO(payload)) + + normalize_sdist(archive) + verify_sdist(archive) + + with tarfile.open(archive, "r:gz") as normalized: + members = normalized.getmembers() + self.assertEqual(1, len(members)) + member = members[0] + self.assertEqual((0, 0, "", ""), ( + member.uid, + member.gid, + member.uname, + member.gname, + )) + self.assertEqual(0o640, member.mode) + extracted = normalized.extractfile(member) + self.assertIsNotNone(extracted) + self.assertEqual(payload, extracted.read()) # type: ignore[union-attr] + + header = archive.read_bytes()[:64] + self.assertNotIn(b"personal-build", header) + + def test_verify_sdist_rejects_non_neutral_owner_metadata(self) -> None: + with tempfile.TemporaryDirectory() as directory: + archive = Path(directory) / "package.tar.gz" + with tarfile.open(archive, "w:gz") as destination: + member = tarfile.TarInfo("package/") + member.type = tarfile.DIRTYPE + member.uid = 501 + member.uname = "localuser" + destination.addfile(member) + + with self.assertRaisesRegex(ValueError, "non-neutral owner metadata"): + verify_sdist(archive) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/normalize_sdist.py b/tools/normalize_sdist.py new file mode 100644 index 0000000..31f4874 --- /dev/null +++ b/tools/normalize_sdist.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import argparse +import gzip +import os +import stat +import tarfile +import tempfile +from collections.abc import Sequence +from pathlib import Path + + +_OWNER_PAX_KEYS = frozenset({"uid", "gid", "uname", "gname"}) + + +def verify_sdist(path: Path) -> None: + with tarfile.open(path, "r:gz") as archive: + for member in archive: + ownership = (member.uid, member.gid, member.uname, member.gname) + if ownership != (0, 0, "", ""): + raise ValueError( + f"non-neutral owner metadata in {path}: " + f"{member.name} has {ownership!r}" + ) + + +def normalize_sdist(path: Path) -> None: + original_mode = stat.S_IMODE(path.stat().st_mode) + temporary_path: Path | None = None + try: + with tempfile.NamedTemporaryFile( + mode="w+b", + dir=path.parent, + prefix=f".{path.name}.", + delete=False, + ) as raw_destination: + temporary_path = Path(raw_destination.name) + with tarfile.open(path, "r:gz") as source: + with gzip.GzipFile( + filename="", + mode="wb", + fileobj=raw_destination, + mtime=0, + ) as compressed: + with tarfile.open( + fileobj=compressed, + mode="w|", + format=tarfile.PAX_FORMAT, + ) as destination: + for member in source: + member.uid = 0 + member.gid = 0 + member.uname = "" + member.gname = "" + member.pax_headers = { + key: value + for key, value in member.pax_headers.items() + if key not in _OWNER_PAX_KEYS + } + payload = source.extractfile(member) if member.isfile() else None + try: + destination.addfile(member, payload) + finally: + if payload is not None: + payload.close() + raw_destination.flush() + os.fsync(raw_destination.fileno()) + + os.chmod(temporary_path, original_mode) + verify_sdist(temporary_path) + os.replace(temporary_path, path) + temporary_path = None + finally: + if temporary_path is not None: + temporary_path.unlink(missing_ok=True) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Remove user and machine ownership metadata from Python sdists." + ) + parser.add_argument("archives", nargs="+", type=Path) + args = parser.parse_args(argv) + for archive in args.archives: + normalize_sdist(archive) + print(f"normalized {archive}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 3859751ce0ab73bfbb54517d5728fadda65e5674 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:22:34 -0400 Subject: [PATCH 09/14] docs: Document release safety boundaries Co-Authored-By: Codex GPT-5.6 --- README.md | 10 ++++++++-- docs/architecture.md | 16 +++++++++++----- docs/dependencies.md | 14 ++++++++------ 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 1091c75..1a13465 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,8 @@ python3.14 -m venv /tmp/agent-release-gate ## Requirements - Python 3.14 or newer; development is verified with `python3.14`. +- A POSIX-style operating system with descriptor-relative filesystem operations; + v0 is verified on macOS and Linux. - Git, used only for read-only provenance checks. - The audited ClawProBench checkout at `../ClawProBench`. @@ -58,7 +60,9 @@ PYTHONPATH=src python3.14 -m agent_release_gate evaluate \ --output decisions/release-decision.json ``` -The output is written atomically. A failed evaluation leaves an existing output file unchanged. +The output is written atomically through a held directory descriptor. A failed +evaluation leaves an existing output file unchanged, and a concurrent +symlink-parent swap cannot redirect the write. Exit codes are: @@ -112,6 +116,7 @@ Tests use synthetic JSON and disposable Git repositories. They do not execute th - `integrations/`: audited benchmark manifests. - `policies/`: release threshold policies. - `tests/`: synthetic fixtures and behavior tests. +- `tools/`: release-artifact metadata normalization. - `docs/architecture.md`: system boundaries and data flow. - `docs/adding-an-adapter.md`: extension contract for another agent benchmark. @@ -138,7 +143,8 @@ for the complete build, test, CI, upstream, and audit inventory. - Custom policies and integration manifests are trusted local configuration. - The CLI does not fetch or execute benchmark code. - Decision output must be separate from reports, policies, manifests, and - benchmark checkouts; protected paths are rejected after symlink resolution. + benchmark checkouts; protected paths are rejected after symlink resolution + and the output directory remains pinned by descriptor through the write. - The source distribution includes the default policy and integration manifest. A standalone wheel contains only the CLI package, so invoke it from a source checkout or pass explicit `--policy` and `--integration` paths. diff --git a/docs/architecture.md b/docs/architecture.md index 5319e01..57ca4d7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -40,7 +40,8 @@ For `evaluate`: 1. Perform the same integration validation. 2. Select the named benchmark adapter and require it to match the manifest. -3. Resolve the output path and reject benchmark-checkout or input-file targets. +3. Open and validate the output directory, rejecting benchmark-checkout or + input-file targets while holding the directory descriptor through the write. 4. Validate and normalize the report into `BenchmarkEvidence`. 5. Parse and validate the TOML policy, retaining its SHA-256 digest. 6. Apply every gate rule in stable order. @@ -73,6 +74,7 @@ The evaluation timestamp is the only time-varying output field. Tests inject a c The integration manifest pins the expected origin and full Git commit. Validation uses only: - `git rev-parse --is-inside-work-tree`; +- `git rev-parse --show-toplevel`; - `git rev-parse HEAD`; - `git remote get-url origin`; - `git status --porcelain`. @@ -84,10 +86,14 @@ This prevents a CLI invocation from presenting one benchmark's source provenance alongside a different adapter. Prohibited paths are detected even when represented by dangling symlinks. -Before creating a temporary decision file, the CLI resolves the requested -output path. It rejects paths inside the benchmark checkout and paths equal to -the report, policy, or integration manifest, including aliases reached through -symlinked parents. The resolved checkout stays internal and is not serialized. +Before reading evaluation inputs, the CLI resolves the requested output path, +opens its directory without following the final path component, and verifies +the opened directory by device and inode. It rejects paths inside the benchmark +checkout and paths equal to the report, policy, or integration manifest, +including aliases reached through symlinked parents. The same directory +descriptor is used to create and replace the decision file, so a concurrent +symlink-parent swap cannot redirect the write. The resolved checkout stays +internal and is not serialized. ## Failure Handling diff --git a/docs/dependencies.md b/docs/dependencies.md index 2118f5a..c5e1a1f 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -1,12 +1,13 @@ # Dependencies and Third-Party Boundary -Agent Release Gate has no third-party runtime or test dependencies. The table -below inventories tools involved in building, verifying, or integrating the -project; build and audit tools are not shipped in the wheel. +Agent Release Gate has no third-party Python package runtime or test +dependencies. The table below inventories external tools involved in running, +building, verifying, or integrating the project; none are shipped in the wheel. | Role | Component | Version constraint | License | Distribution boundary | | --- | --- | --- | --- | --- | | Runtime | Python standard library | Python 3.14+ | PSF License | Required interpreter; not bundled | +| Runtime and tests | Git CLI | Git 2.x; tests require `git init -b` support | GPL-2.0-only | External provenance tool; not bundled or invoked over a network | | Tests | `unittest` | Python 3.14+ | PSF License | Standard library; not bundled | | Build backend | setuptools | `>=80` | MIT | Isolated build dependency; not bundled | | Build frontend | build | `1.6.0` in CI | MIT | CI/development tool; not bundled | @@ -18,11 +19,12 @@ project; build and audit tools are not shipped in the wheel. License sources: - Python: +- Git: - setuptools: - build: -- actions/checkout: -- actions/setup-python: -- ClawProBench: +- actions/checkout: +- actions/setup-python: +- ClawProBench: - Gitleaks: The JSON fixtures under `tests/fixtures/` are original, hand-authored synthetic From ddae4ca30cb3d9390b9e0c289bfc4db8069cdb23 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:35:06 -0400 Subject: [PATCH 10/14] fix: Pin evaluation inputs and sibling provenance --- README.md | 9 +- docs/architecture.md | 39 ++--- src/agent_release_gate/cli.py | 139 +++++++++++++++--- .../integration/validator.py | 5 +- tests/test_cli.py | 42 ++++++ tests/test_integration_validator.py | 14 ++ 6 files changed, 205 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 1a13465..f74b219 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,10 @@ PYTHONPATH=src python3.14 -m agent_release_gate evaluate \ --output decisions/release-decision.json ``` -The output is written atomically through a held directory descriptor. A failed -evaluation leaves an existing output file unchanged, and a concurrent -symlink-parent swap cannot redirect the write. +Evaluation inputs are pinned by file descriptor, and output is written +atomically through a held directory descriptor. A failed evaluation leaves an +existing output file unchanged; concurrent parent-symlink swaps cannot +substitute an input or redirect the write. Exit codes are: @@ -144,7 +145,7 @@ for the complete build, test, CI, upstream, and audit inventory. - The CLI does not fetch or execute benchmark code. - Decision output must be separate from reports, policies, manifests, and benchmark checkouts; protected paths are rejected after symlink resolution - and the output directory remains pinned by descriptor through the write. + while input files and the output directory remain pinned by descriptor. - The source distribution includes the default policy and integration manifest. A standalone wheel contains only the CLI package, so invoke it from a source checkout or pass explicit `--policy` and `--integration` paths. diff --git a/docs/architecture.md b/docs/architecture.md index 57ca4d7..1cd0336 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -38,15 +38,16 @@ For `doctor`: For `evaluate`: -1. Perform the same integration validation. -2. Select the named benchmark adapter and require it to match the manifest. -3. Open and validate the output directory, rejecting benchmark-checkout or +1. Open and pin the report, policy, and integration manifest as regular files. +2. Perform integration validation from the pinned manifest. +3. Select the named benchmark adapter and require it to match the manifest. +4. Open and validate the output directory, rejecting benchmark-checkout or input-file targets while holding the directory descriptor through the write. -4. Validate and normalize the report into `BenchmarkEvidence`. -5. Parse and validate the TOML policy, retaining its SHA-256 digest. -6. Apply every gate rule in stable order. -7. Combine decision, observed metrics, report identity, integration provenance, policy identity, and UTC evaluation time. -8. Write sorted, indented JSON to a sibling temporary file, sync it, and replace the target atomically. +5. Validate and normalize the pinned report into `BenchmarkEvidence`. +6. Parse and validate the pinned TOML policy, retaining its SHA-256 digest. +7. Apply every gate rule in stable order. +8. Combine decision, observed metrics, report identity, integration provenance, policy identity, and UTC evaluation time. +9. Write sorted, indented JSON to a sibling temporary file, sync it, and replace the target atomically. Input errors are distinct from release outcomes. A valid `no_go` is exit code `1`; malformed evidence or unproven provenance is exit code `2` and produces no decision. @@ -84,16 +85,18 @@ Git hooks and fsmonitor are disabled for these read-only subprocesses, terminal The integration manifest declares the adapter that may consume its evidence. This prevents a CLI invocation from presenting one benchmark's source provenance alongside a different adapter. Prohibited paths are detected even -when represented by dangling symlinks. - -Before reading evaluation inputs, the CLI resolves the requested output path, -opens its directory without following the final path component, and verifies -the opened directory by device and inode. It rejects paths inside the benchmark -checkout and paths equal to the report, policy, or integration manifest, -including aliases reached through symlinked parents. The same directory -descriptor is used to create and replace the decision file, so a concurrent -symlink-parent swap cannot redirect the write. The resolved checkout stays -internal and is not serialized. +when represented by dangling symlinks. The checkout must resolve to a distinct +direct sibling; the project itself and sibling symlinks back to it are rejected. + +Before evaluation, the CLI opens and pins the report, policy, and integration +manifest by file and parent-directory descriptor. It then resolves the requested +output path, opens its directory without following the final path component, +and verifies the opened directory by device and inode. It rejects paths inside +the benchmark checkout and paths equal to a pinned input, including aliases +reached through symlinked parents. Pinned descriptors are used for every input +read and for creating and replacing the decision file, so concurrent parent +symlink swaps cannot substitute an input or redirect the write. The resolved +checkout stays internal and is not serialized. ## Failure Handling diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index fd10d47..3f5ae05 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -4,9 +4,10 @@ import json import os import secrets +import stat import sys from collections.abc import Callable, Sequence -from contextlib import redirect_stderr, redirect_stdout +from contextlib import ExitStack, redirect_stderr, redirect_stdout from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path @@ -28,6 +29,40 @@ class DecisionWriteError(ValueError): """Raised when a completed decision cannot be written atomically.""" +class InputReadError(ValueError): + """Raised when an evaluation input cannot be pinned for a safe read.""" + + +@dataclass(slots=True) +class _InputTarget: + file_fd: int + directory_fd: int + name: str + display_path: Path + + @property + def read_path(self) -> Path: + return Path("/dev/fd") / str(self.file_fd) + + def close(self) -> None: + file_fd = self.file_fd + directory_fd = self.directory_fd + self.file_fd = -1 + self.directory_fd = -1 + for descriptor in (file_fd, directory_fd): + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + def __enter__(self) -> _InputTarget: + return self + + def __exit__(self, *_: object) -> None: + self.close() + + @dataclass(slots=True) class _OutputTarget: directory_fd: int @@ -108,6 +143,44 @@ def _open_directory(path: Path) -> int: return os.open(path, _directory_flags()) +def _prepare_input_target(path: Path) -> _InputTarget: + directory_fd: int | None = None + file_fd: int | None = None + try: + resolved_path = path.resolve(strict=True) + directory_fd = _open_directory(resolved_path.parent) + file_fd = os.open( + resolved_path.name, + os.O_RDONLY + | os.O_NOFOLLOW + | os.O_NONBLOCK + | getattr(os, "O_CLOEXEC", 0), + dir_fd=directory_fd, + ) + if not stat.S_ISREG(os.fstat(file_fd).st_mode): + raise InputReadError(f"evaluation input is not a regular file: {path}") + target = _InputTarget( + file_fd=file_fd, + directory_fd=directory_fd, + name=resolved_path.name, + display_path=path, + ) + file_fd = None + directory_fd = None + return target + except InputReadError: + raise + except OSError as exc: + raise InputReadError(f"unable to open evaluation input {path}: {exc}") from exc + finally: + for descriptor in (file_fd, directory_fd): + if descriptor is not None: + try: + os.close(descriptor) + except OSError: + pass + + def _same_directory(left_fd: int, right_fd: int) -> bool: left = os.fstat(left_fd) right = os.fstat(right_fd) @@ -209,12 +282,18 @@ def _validate_output_path( def _prepare_output_target( output: Path, *, - protected_files: Sequence[Path], + protected_files: Sequence[Path | _InputTarget], protected_directory: Path, ) -> _OutputTarget: - resolved_output = _validate_output_path( + protected_paths = tuple( + protected.display_path + if isinstance(protected, _InputTarget) + else protected + for protected in protected_files + ) + _validate_output_path( output, - protected_files=protected_files, + protected_files=protected_paths, protected_directory=protected_directory, ) output_name = output.name @@ -230,17 +309,27 @@ def _prepare_output_target( ) for protected in protected_files: - resolved_protected = protected.resolve(strict=False) - if output_name != resolved_protected.name: + if isinstance(protected, _InputTarget): + protected_name = protected.name + protected_parent_fd = protected.directory_fd + close_protected_parent = False + else: + resolved_protected = protected.resolve(strict=False) + protected_name = resolved_protected.name + protected_parent_fd = _open_directory(resolved_protected.parent) + close_protected_parent = True + if output_name != protected_name: + if close_protected_parent: + os.close(protected_parent_fd) continue - protected_parent_fd = _open_directory(resolved_protected.parent) try: if _same_directory(directory_fd, protected_parent_fd): raise DecisionWriteError( "output path must not overwrite an evaluation input" ) finally: - os.close(protected_parent_fd) + if close_protected_parent: + os.close(protected_parent_fd) target = _OutputTarget( directory_fd=directory_fd, @@ -288,18 +377,28 @@ def _evaluate( clock: Callable[[], datetime], ) -> int: adapter = get_adapter(args.adapter) - integration = _validated_integration( - args.integration, - expected_adapter=args.adapter, - ) - output_target = _prepare_output_target( - args.output, - protected_files=(args.report, args.policy, args.integration), - protected_directory=integration.checkout_path, - ) - with output_target: - evidence = adapter.load(args.report, source_version=integration.commit) - policy, policy_sha256 = load_policy(args.policy) + with ExitStack() as resources: + report_input = resources.enter_context(_prepare_input_target(args.report)) + policy_input = resources.enter_context(_prepare_input_target(args.policy)) + integration_input = resources.enter_context( + _prepare_input_target(args.integration) + ) + integration = _validated_integration( + integration_input.read_path, + expected_adapter=args.adapter, + ) + output_target = resources.enter_context( + _prepare_output_target( + args.output, + protected_files=(report_input, policy_input, integration_input), + protected_directory=integration.checkout_path, + ) + ) + evidence = adapter.load( + report_input.read_path, + source_version=integration.commit, + ) + policy, policy_sha256 = load_policy(policy_input.read_path) decision = evaluate(evidence, policy) document = decision.to_dict() diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index 8b10d3c..a021844 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -92,7 +92,10 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: resolved_project_root = project_root.resolve() checkout_path = (resolved_project_root / checkout_relative).resolve() - if checkout_path.parent != resolved_project_root.parent: + if ( + checkout_path == resolved_project_root + or checkout_path.parent != resolved_project_root.parent + ): raise IntegrationError("checkout_path must resolve to a direct sibling of the project") commit = _nonempty_string(raw, "commit") diff --git a/tests/test_cli.py b/tests/test_cli.py index d77c11f..2c761c6 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,11 +11,13 @@ from pathlib import Path from unittest.mock import patch +from agent_release_gate.adapters.registry import get_adapter from agent_release_gate.cli import ( _prepare_output_target, _write_json_atomic, run, ) +from agent_release_gate.domain.models import BenchmarkEvidence from tests.support import create_git_repo, write_manifest @@ -263,6 +265,46 @@ def test_output_parent_symlink_swap_cannot_redirect_atomic_write(self) -> None: ) self.assertFalse((self.checkout / "decision.json").exists()) + def test_input_parent_symlink_swap_cannot_change_or_overwrite_pinned_report( + self, + ) -> None: + reports = self.base / "reports" + reports.mkdir() + report = reports / "report.json" + original_report = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original_report) + linked_reports = self.project_root / "linked-reports" + linked_reports.symlink_to(reports, target_is_directory=True) + linked_report = linked_reports / "report.json" + output = self.project_root / "report.json" + output.write_bytes((FIXTURES / "clawprobench_no_go.json").read_bytes()) + delegate = get_adapter("clawprobench") + project_root = self.project_root + + class SwappingAdapter: + def load( + self, + report_path: Path, + *, + source_version: str, + ) -> BenchmarkEvidence: + linked_reports.unlink() + linked_reports.symlink_to(project_root, target_is_directory=True) + return delegate.load(report_path, source_version=source_version) + + with patch( + "agent_release_gate.cli.get_adapter", + return_value=SwappingAdapter(), + ): + code, _, stderr = self.invoke(self.evaluate_args(linked_report, output)) + + document = json.loads(output.read_text()) + self.assertEqual(0, code) + self.assertEqual("", stderr) + self.assertEqual("go", document["decision"]) + self.assertEqual("agent-go", document["benchmark"]["subject"]) + self.assertEqual(original_report, report.read_bytes()) + def test_output_leaf_symlink_is_replaced_without_overwriting_its_target(self) -> None: victim = self.project_root / "victim.txt" victim.write_text("keep me\n", encoding="utf-8") diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index 412b265..bcd6354 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -181,6 +181,20 @@ def test_manifest_rejects_unsafe_paths_and_commit(self) -> None: self.manifest(updates={"checkout_path": str(self.checkout.resolve())}) with self.assertRaisesRegex(IntegrationError, "checkout_path must resolve to a direct sibling"): self.manifest(updates={"checkout_path": "../../escape"}) + for self_reference in (".", "../agent-release-gate"): + with self.subTest(checkout_path=self_reference): + with self.assertRaisesRegex( + IntegrationError, + "checkout_path must resolve to a direct sibling", + ): + self.manifest(updates={"checkout_path": self_reference}) + project_alias = self.base / "project-alias" + project_alias.symlink_to(self.project_root, target_is_directory=True) + with self.assertRaisesRegex( + IntegrationError, + "checkout_path must resolve to a direct sibling", + ): + self.manifest(updates={"checkout_path": "../project-alias"}) with self.assertRaisesRegex(IntegrationError, "commit must be 40 lowercase hexadecimal"): self.manifest(updates={"commit": "ABC"}) with self.assertRaisesRegex(IntegrationError, "prohibited_paths entries must be safe relative paths"): From 1afddff95cd0995bf124c21f7351cd8efefbb6f1 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 14:57:40 -0400 Subject: [PATCH 11/14] fix: pin checkout and secure path acquisition --- README.md | 16 +- docs/architecture.md | 28 ++-- src/agent_release_gate/cli.py | 135 ++++++---------- src/agent_release_gate/filesystem.py | 128 +++++++++++++++ .../integration/validator.py | 150 ++++++++++++------ tests/test_cli.py | 64 ++++++++ tests/test_integration_validator.py | 30 +++- 7 files changed, 397 insertions(+), 154 deletions(-) create mode 100644 src/agent_release_gate/filesystem.py diff --git a/README.md b/README.md index f74b219..1302482 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,9 @@ No package installation is needed for repository development. Prefix commands wi PYTHONPATH=src python3.14 -m agent_release_gate doctor ``` -`doctor` validates the checkout, origin URL, audited commit, clean worktree, and absence of prohibited vendored directories. It never runs upstream code. +`doctor` pins and validates the checkout, origin URL, audited commit, clean +worktree, and absence of prohibited vendored directories. It never runs +upstream code. ## Evaluate a Report @@ -60,10 +62,11 @@ PYTHONPATH=src python3.14 -m agent_release_gate evaluate \ --output decisions/release-decision.json ``` -Evaluation inputs are pinned by file descriptor, and output is written -atomically through a held directory descriptor. A failed evaluation leaves an -existing output file unchanged; concurrent parent-symlink swaps cannot -substitute an input or redirect the write. +Evaluation inputs and the validated benchmark checkout are pinned by file +descriptor, and output is written atomically through a held directory +descriptor. A failed evaluation leaves an existing output file unchanged; +concurrent path or parent-symlink swaps cannot substitute an input, mix +checkout provenance, or redirect the write into the checkout. Exit codes are: @@ -145,7 +148,8 @@ for the complete build, test, CI, upstream, and audit inventory. - The CLI does not fetch or execute benchmark code. - Decision output must be separate from reports, policies, manifests, and benchmark checkouts; protected paths are rejected after symlink resolution - while input files and the output directory remain pinned by descriptor. + while input files, the benchmark checkout, and the output directory remain + pinned by descriptor. - The source distribution includes the default policy and integration manifest. A standalone wheel contains only the CLI package, so invoke it from a source checkout or pass explicit `--policy` and `--integration` paths. diff --git a/docs/architecture.md b/docs/architecture.md index 1cd0336..501684b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -32,8 +32,9 @@ For `doctor`: 1. Load and strictly validate the integration manifest. 2. Bind the manifest's declared adapter to the registry key used for evaluation. -3. Resolve its checkout as a direct sibling of this repository. -4. Verify Git worktree state, `HEAD`, `origin`, cleanliness, and prohibited paths. +3. Resolve and descriptor-pin its checkout as a direct sibling of this repository. +4. Verify Git worktree state, `HEAD`, `origin`, cleanliness, and prohibited paths + relative to that held descriptor. 5. Emit validated provenance as JSON without the absolute local checkout path. For `evaluate`: @@ -75,13 +76,20 @@ The evaluation timestamp is the only time-varying output field. Tests inject a c The integration manifest pins the expected origin and full Git commit. Validation uses only: - `git rev-parse --is-inside-work-tree`; -- `git rev-parse --show-toplevel`; +- `git rev-parse --show-prefix`; - `git rev-parse HEAD`; - `git remote get-url origin`; - `git status --porcelain`. Git hooks and fsmonitor are disabled for these read-only subprocesses, terminal prompts are disabled, and `GIT_OPTIONAL_LOCKS=0` prevents status checks from refreshing the upstream index. The validator also disables Git's untracked cache, ignores global/system Git configuration, and removes inherited `GIT_*` variables before setting its explicit safe environment. This prevents ambient `GIT_DIR`, `GIT_WORK_TREE`, or index overrides from redirecting a probe away from the pinned checkout. Validation never fetches, checks out, resets, cleans, or writes upstream files. +The validator opens the resolved checkout through a no-follow descriptor walk +and retains that descriptor through integration validation and output-path +validation. Git subprocesses enter the held directory by descriptor, and +prohibited paths are inspected relative to it. Renaming or replacing the +manifest path therefore cannot mix provenance from one checkout with +cleanliness from another or make output protection follow a replacement path. + The integration manifest declares the adapter that may consume its evidence. This prevents a CLI invocation from presenting one benchmark's source provenance alongside a different adapter. Prohibited paths are detected even @@ -89,14 +97,16 @@ when represented by dangling symlinks. The checkout must resolve to a distinct direct sibling; the project itself and sibling symlinks back to it are rejected. Before evaluation, the CLI opens and pins the report, policy, and integration -manifest by file and parent-directory descriptor. It then resolves the requested +manifest by file and parent-directory descriptor, and the validator pins the +benchmark checkout by directory descriptor. It then resolves the requested output path, opens its directory without following the final path component, and verifies the opened directory by device and inode. It rejects paths inside -the benchmark checkout and paths equal to a pinned input, including aliases -reached through symlinked parents. Pinned descriptors are used for every input -read and for creating and replacing the decision file, so concurrent parent -symlink swaps cannot substitute an input or redirect the write. The resolved -checkout stays internal and is not serialized. +the held benchmark checkout and paths equal to a pinned input, including +aliases reached through symlinked parents. Pinned descriptors are used for +every input read and for creating and replacing the decision file, so +concurrent path and parent-symlink swaps cannot substitute an input, mix +checkout validation, or redirect the write into the checkout. The resolved +checkout path stays internal and is not serialized. ## Failure Handling diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index 3f5ae05..55b1bc7 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -4,7 +4,6 @@ import json import os import secrets -import stat import sys from collections.abc import Callable, Sequence from contextlib import ExitStack, redirect_stderr, redirect_stdout @@ -17,6 +16,14 @@ from agent_release_gate.adapters.registry import get_adapter from agent_release_gate.domain.policy import PolicyError, load_policy from agent_release_gate.evaluation.evaluator import evaluate +from agent_release_gate.filesystem import ( + close_best_effort, + directory_identity, + directory_is_within, + open_directory, + open_regular_file, + same_directory, +) from agent_release_gate.integration.validator import ( IntegrationError, IntegrationEvidence, @@ -50,11 +57,7 @@ def close(self) -> None: self.file_fd = -1 self.directory_fd = -1 for descriptor in (file_fd, directory_fd): - if descriptor >= 0: - try: - os.close(descriptor) - except OSError: - pass + close_best_effort(descriptor) def __enter__(self) -> _InputTarget: return self @@ -72,13 +75,9 @@ class _OutputTarget: def close(self) -> None: directory_fd = self.directory_fd self.directory_fd = -1 - if directory_fd >= 0: - try: - os.close(directory_fd) - except OSError: - # This read-only guard descriptor cannot affect a committed - # output, and a close error must not mask the body result. - pass + # This read-only guard descriptor cannot affect a committed output, + # and a close error must not mask the body result. + close_best_effort(directory_fd) def __enter__(self) -> _OutputTarget: return self @@ -127,38 +126,11 @@ def build_parser() -> argparse.ArgumentParser: return parser -def _directory_flags() -> int: - required = ("O_DIRECTORY", "O_NOFOLLOW") - if any(not hasattr(os, name) for name in required): - raise DecisionWriteError("secure output writes are not supported on this platform") - return ( - os.O_RDONLY - | os.O_DIRECTORY - | os.O_NOFOLLOW - | getattr(os, "O_CLOEXEC", 0) - ) - - -def _open_directory(path: Path) -> int: - return os.open(path, _directory_flags()) - - def _prepare_input_target(path: Path) -> _InputTarget: directory_fd: int | None = None file_fd: int | None = None try: - resolved_path = path.resolve(strict=True) - directory_fd = _open_directory(resolved_path.parent) - file_fd = os.open( - resolved_path.name, - os.O_RDONLY - | os.O_NOFOLLOW - | os.O_NONBLOCK - | getattr(os, "O_CLOEXEC", 0), - dir_fd=directory_fd, - ) - if not stat.S_ISREG(os.fstat(file_fd).st_mode): - raise InputReadError(f"evaluation input is not a regular file: {path}") + file_fd, directory_fd, resolved_path = open_regular_file(path) target = _InputTarget( file_fd=file_fd, directory_fd=directory_fd, @@ -175,34 +147,7 @@ def _prepare_input_target(path: Path) -> _InputTarget: finally: for descriptor in (file_fd, directory_fd): if descriptor is not None: - try: - os.close(descriptor) - except OSError: - pass - - -def _same_directory(left_fd: int, right_fd: int) -> bool: - left = os.fstat(left_fd) - right = os.fstat(right_fd) - return (left.st_dev, left.st_ino) == (right.st_dev, right.st_ino) - - -def _directory_is_within(directory_fd: int, ancestor: Path) -> bool: - ancestor_fd = _open_directory(ancestor.resolve(strict=True)) - current_fd = os.dup(directory_fd) - try: - while True: - if _same_directory(current_fd, ancestor_fd): - return True - parent_fd = os.open("..", _directory_flags(), dir_fd=current_fd) - if _same_directory(current_fd, parent_fd): - os.close(parent_fd) - return False - os.close(current_fd) - current_fd = parent_fd - finally: - os.close(current_fd) - os.close(ancestor_fd) + close_best_effort(descriptor) def _write_json_atomic( @@ -283,8 +228,20 @@ def _prepare_output_target( output: Path, *, protected_files: Sequence[Path | _InputTarget], - protected_directory: Path, + protected_directory: Path | IntegrationEvidence, ) -> _OutputTarget: + if isinstance(protected_directory, IntegrationEvidence): + protected_path = protected_directory.checkout_path + protected_identity = protected_directory.checkout_identity + else: + protected_path = protected_directory + protected_fd: int | None = None + try: + protected_fd, _ = open_directory(protected_path) + protected_identity = directory_identity(protected_fd) + finally: + if protected_fd is not None: + close_best_effort(protected_fd) protected_paths = tuple( protected.display_path if isinstance(protected, _InputTarget) @@ -294,7 +251,7 @@ def _prepare_output_target( _validate_output_path( output, protected_files=protected_paths, - protected_directory=protected_directory, + protected_directory=protected_path, ) output_name = output.name if not output_name: @@ -302,8 +259,8 @@ def _prepare_output_target( directory_fd: int | None = None try: - directory_fd = _open_directory(output.parent.resolve(strict=False)) - if _directory_is_within(directory_fd, protected_directory): + directory_fd, _ = open_directory(output.parent) + if directory_is_within(directory_fd, protected_identity): raise DecisionWriteError( "output path must not be inside the benchmark checkout" ) @@ -316,14 +273,14 @@ def _prepare_output_target( else: resolved_protected = protected.resolve(strict=False) protected_name = resolved_protected.name - protected_parent_fd = _open_directory(resolved_protected.parent) + protected_parent_fd, _ = open_directory(resolved_protected.parent) close_protected_parent = True if output_name != protected_name: if close_protected_parent: os.close(protected_parent_fd) continue try: - if _same_directory(directory_fd, protected_parent_fd): + if same_directory(directory_fd, protected_parent_fd): raise DecisionWriteError( "output path must not overwrite an evaluation input" ) @@ -344,7 +301,7 @@ def _prepare_output_target( raise DecisionWriteError(f"unable to write decision {output}: {exc}") from exc finally: if directory_fd is not None: - os.close(directory_fd) + close_best_effort(directory_fd) def _validated_integration( @@ -362,13 +319,15 @@ def _validated_integration( def _doctor(args: argparse.Namespace, stdout: TextIO) -> int: - integration = _validated_integration(args.integration) - document: dict[str, object] = { - "schema_version": 1, - "valid": True, - "integration": integration.to_dict(), - } - stdout.write(json.dumps(document, indent=2, sort_keys=True, allow_nan=False) + "\n") + with _validated_integration(args.integration) as integration: + document: dict[str, object] = { + "schema_version": 1, + "valid": True, + "integration": integration.to_dict(), + } + stdout.write( + json.dumps(document, indent=2, sort_keys=True, allow_nan=False) + "\n" + ) return 0 @@ -383,15 +342,17 @@ def _evaluate( integration_input = resources.enter_context( _prepare_input_target(args.integration) ) - integration = _validated_integration( - integration_input.read_path, - expected_adapter=args.adapter, + integration = resources.enter_context( + _validated_integration( + integration_input.read_path, + expected_adapter=args.adapter, + ) ) output_target = resources.enter_context( _prepare_output_target( args.output, protected_files=(report_input, policy_input, integration_input), - protected_directory=integration.checkout_path, + protected_directory=integration, ) ) evidence = adapter.load( diff --git a/src/agent_release_gate/filesystem.py b/src/agent_release_gate/filesystem.py new file mode 100644 index 0000000..ae7a327 --- /dev/null +++ b/src/agent_release_gate/filesystem.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import errno +import os +import stat +from pathlib import Path + + +DirectoryIdentity = tuple[int, int] + + +def close_best_effort(descriptor: int) -> None: + if descriptor >= 0: + try: + os.close(descriptor) + except OSError: + pass + + +def directory_flags() -> int: + required = ("O_DIRECTORY", "O_NOFOLLOW") + if any(not hasattr(os, name) for name in required): + raise OSError(errno.ENOTSUP, "secure descriptor walks are not supported") + return ( + os.O_RDONLY + | os.O_DIRECTORY + | os.O_NOFOLLOW + | getattr(os, "O_CLOEXEC", 0) + ) + + +def directory_identity(descriptor: int) -> DirectoryIdentity: + observed = os.fstat(descriptor) + return observed.st_dev, observed.st_ino + + +def same_directory(left_fd: int, right_fd: int) -> bool: + return directory_identity(left_fd) == directory_identity(right_fd) + + +def _walk_absolute_directory(path: Path) -> int: + if not path.is_absolute(): + raise ValueError("descriptor walk requires an absolute path") + current_fd = os.open(os.sep, directory_flags()) + try: + for component in path.parts[1:]: + next_fd = os.open( + component, + directory_flags(), + dir_fd=current_fd, + ) + os.close(current_fd) + current_fd = next_fd + result = current_fd + current_fd = -1 + return result + finally: + close_best_effort(current_fd) + + +def _open_resolved_directory(resolved: Path) -> int: + expected = os.stat(resolved, follow_symlinks=False) + if not stat.S_ISDIR(expected.st_mode): + raise NotADirectoryError(errno.ENOTDIR, "not a directory", str(resolved)) + descriptor = _walk_absolute_directory(resolved) + if directory_identity(descriptor) != (expected.st_dev, expected.st_ino): + close_best_effort(descriptor) + raise OSError( + errno.ESTALE, + "directory changed while being opened", + str(resolved), + ) + return descriptor + + +def open_directory(path: Path) -> tuple[int, Path]: + resolved = path.resolve(strict=True) + descriptor = _open_resolved_directory(resolved) + return descriptor, resolved + + +def open_regular_file(path: Path) -> tuple[int, int, Path]: + resolved = path.resolve(strict=True) + expected = os.stat(resolved, follow_symlinks=False) + parent_fd = _open_resolved_directory(resolved.parent) + file_fd: int | None = None + try: + file_fd = os.open( + resolved.name, + os.O_RDONLY + | os.O_NOFOLLOW + | os.O_NONBLOCK + | getattr(os, "O_CLOEXEC", 0), + dir_fd=parent_fd, + ) + observed = os.fstat(file_fd) + if not stat.S_ISREG(observed.st_mode): + raise OSError(errno.EINVAL, "not a regular file", str(path)) + if (observed.st_dev, observed.st_ino) != (expected.st_dev, expected.st_ino): + raise OSError(errno.ESTALE, "file changed while being opened", str(path)) + result = file_fd + file_fd = None + return result, parent_fd, resolved + except Exception: + close_best_effort(parent_fd) + raise + finally: + if file_fd is not None: + close_best_effort(file_fd) + + +def directory_is_within( + directory_fd: int, + ancestor: DirectoryIdentity, +) -> bool: + current_fd = os.dup(directory_fd) + try: + while True: + if directory_identity(current_fd) == ancestor: + return True + parent_fd = os.open("..", directory_flags(), dir_fd=current_fd) + if same_directory(current_fd, parent_fd): + os.close(parent_fd) + return False + os.close(current_fd) + current_fd = parent_fd + finally: + close_best_effort(current_fd) diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index a021844..b9c3d8d 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -8,6 +8,13 @@ from pathlib import Path, PurePosixPath from typing import Any +from agent_release_gate.filesystem import ( + DirectoryIdentity, + close_best_effort, + directory_identity, + open_directory, +) + class IntegrationError(ValueError): """Raised when benchmark provenance cannot be established.""" @@ -23,13 +30,29 @@ class IntegrationManifest: prohibited_paths: tuple[str, ...] -@dataclass(frozen=True, slots=True) +@dataclass(slots=True) class IntegrationEvidence: adapter: str name: str checkout_path: Path repository_url: str commit: str + checkout_fd: int + + @property + def checkout_identity(self) -> DirectoryIdentity: + return directory_identity(self.checkout_fd) + + def close(self) -> None: + checkout_fd = self.checkout_fd + self.checkout_fd = -1 + close_best_effort(checkout_fd) + + def __enter__(self) -> IntegrationEvidence: + return self + + def __exit__(self, *_: object) -> None: + self.close() def to_dict(self) -> dict[str, object]: return { @@ -124,7 +147,7 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: ) -def _git(checkout: Path, *args: str) -> subprocess.CompletedProcess[str]: +def _git(checkout_fd: int, *args: str) -> subprocess.CompletedProcess[str]: environment = { key: value for key, value in os.environ.items() @@ -139,6 +162,10 @@ def _git(checkout: Path, *args: str) -> subprocess.CompletedProcess[str]: "GIT_TERMINAL_PROMPT": "0", } ) + + def enter_checkout() -> None: + os.fchdir(checkout_fd) + return subprocess.run( [ "git", @@ -148,14 +175,14 @@ def _git(checkout: Path, *args: str) -> subprocess.CompletedProcess[str]: "core.fsmonitor=false", "-c", "core.untrackedCache=false", - "-C", - str(checkout), *args, ], check=False, capture_output=True, text=True, env=environment, + pass_fds=(checkout_fd,), + preexec_fn=enter_checkout, ) @@ -164,52 +191,73 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: if not checkout.is_dir(): raise IntegrationError(f"checkout does not exist or is not a directory: {checkout}") - worktree = _git(checkout, "rev-parse", "--is-inside-work-tree") - if worktree.returncode != 0 or worktree.stdout.strip() != "true": - raise IntegrationError(f"checkout is not a Git worktree: {checkout}") - - top_level = _git(checkout, "rev-parse", "--show-toplevel") - observed_top_level = ( - Path(top_level.stdout.strip()).resolve() - if top_level.returncode == 0 and top_level.stdout.strip() - else None - ) - if observed_top_level != checkout.resolve(): - raise IntegrationError(f"checkout is not a Git worktree root: {checkout}") - - failures: list[str] = [] - head = _git(checkout, "rev-parse", "HEAD") - observed_commit = head.stdout.strip() if head.returncode == 0 else "unavailable" - if observed_commit != manifest.commit: - failures.append( - f"expected commit {manifest.commit}, observed {observed_commit}" - ) - - origin = _git(checkout, "remote", "get-url", "origin") - observed_origin = origin.stdout.strip() if origin.returncode == 0 else "unavailable" - if observed_origin != manifest.repository_url: - failures.append( - f"unexpected origin URL: expected {manifest.repository_url}, observed {observed_origin}" - ) - - status = _git(checkout, "status", "--porcelain") - if status.returncode != 0: - failures.append("unable to determine worktree status") - elif status.stdout: - failures.append("worktree is not clean") - - for prohibited_path in manifest.prohibited_paths: - candidate = checkout / prohibited_path - if candidate.exists() or candidate.is_symlink(): + checkout_fd: int | None = None + try: + try: + checkout_fd, _ = open_directory(checkout) + except OSError as exc: + raise IntegrationError(f"unable to pin checkout {checkout}: {exc}") from exc + + worktree = _git(checkout_fd, "rev-parse", "--is-inside-work-tree") + if worktree.returncode != 0 or worktree.stdout.strip() != "true": + raise IntegrationError(f"checkout is not a Git worktree: {checkout}") + + prefix = _git(checkout_fd, "rev-parse", "--show-prefix") + if prefix.returncode != 0 or prefix.stdout.strip(): + raise IntegrationError(f"checkout is not a Git worktree root: {checkout}") + + failures: list[str] = [] + head = _git(checkout_fd, "rev-parse", "HEAD") + observed_commit = head.stdout.strip() if head.returncode == 0 else "unavailable" + if observed_commit != manifest.commit: + failures.append( + f"expected commit {manifest.commit}, observed {observed_commit}" + ) + + origin = _git(checkout_fd, "remote", "get-url", "origin") + observed_origin = origin.stdout.strip() if origin.returncode == 0 else "unavailable" + if observed_origin != manifest.repository_url: + failures.append( + f"unexpected origin URL: expected {manifest.repository_url}, observed {observed_origin}" + ) + + status = _git(checkout_fd, "status", "--porcelain") + if status.returncode != 0: + failures.append("unable to determine worktree status") + elif status.stdout: + failures.append("worktree is not clean") + + for prohibited_path in manifest.prohibited_paths: + try: + os.stat( + prohibited_path, + dir_fd=checkout_fd, + follow_symlinks=False, + ) + except FileNotFoundError: + continue + except OSError as exc: + failures.append( + f"unable to inspect prohibited path {prohibited_path}: {exc}" + ) + continue failures.append(f"prohibited path is present: {prohibited_path}") - if failures: - raise IntegrationError("integration validation failed: " + "; ".join(failures)) - - return IntegrationEvidence( - adapter=manifest.adapter, - name=manifest.name, - checkout_path=checkout, - repository_url=manifest.repository_url, - commit=manifest.commit, - ) + if failures: + raise IntegrationError( + "integration validation failed: " + "; ".join(failures) + ) + + evidence = IntegrationEvidence( + adapter=manifest.adapter, + name=manifest.name, + checkout_path=checkout, + repository_url=manifest.repository_url, + commit=manifest.commit, + checkout_fd=checkout_fd, + ) + checkout_fd = None + return evidence + finally: + if checkout_fd is not None: + close_best_effort(checkout_fd) diff --git a/tests/test_cli.py b/tests/test_cli.py index 2c761c6..9b46638 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -305,6 +305,70 @@ def load( self.assertEqual("agent-go", document["benchmark"]["subject"]) self.assertEqual(original_report, report.read_bytes()) + def test_input_ancestor_swap_during_pin_is_rejected(self) -> None: + requested_directory = self.base / "requested" + requested_inner = requested_directory / "inner" + requested_inner.mkdir(parents=True) + report = requested_inner / "report.json" + report.write_bytes((FIXTURES / "clawprobench_go.json").read_bytes()) + saved_directory = self.base / "saved" + substitute_directory = self.base / "substitute" + substitute_inner = substitute_directory / "inner" + substitute_inner.mkdir(parents=True) + (substitute_inner / "report.json").write_bytes( + (FIXTURES / "clawprobench_no_go.json").read_bytes() + ) + output = self.project_root / "decision.json" + original_resolve = Path.resolve + swapped = False + + def resolve_then_swap(path: Path, *args: object, **kwargs: object) -> Path: + nonlocal swapped + resolved = original_resolve(path, *args, **kwargs) # type: ignore[arg-type] + if path == report and not swapped: + swapped = True + requested_directory.rename(saved_directory) + requested_directory.symlink_to( + substitute_directory, + target_is_directory=True, + ) + return resolved + + with patch.object(Path, "resolve", resolve_then_swap): + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("unable to open evaluation input", stderr) + self.assertFalse(output.exists()) + + def test_validated_checkout_rename_cannot_redirect_output_into_it(self) -> None: + safe_directory = self.project_root / "safe" + safe_directory.mkdir() + output_link = self.project_root / "out" + output_link.symlink_to(safe_directory, target_is_directory=True) + output = output_link / "decision.json" + renamed_checkout = self.base / "RenamedBenchmark" + original_prepare = _prepare_output_target + + def rename_before_prepare(*args: object, **kwargs: object): + self.checkout.rename(renamed_checkout) + self.checkout.mkdir() + output_link.unlink() + output_link.symlink_to(renamed_checkout, target_is_directory=True) + return original_prepare(*args, **kwargs) # type: ignore[arg-type] + + with patch( + "agent_release_gate.cli._prepare_output_target", + side_effect=rename_before_prepare, + ): + code, _, stderr = self.invoke( + self.evaluate_args(FIXTURES / "clawprobench_go.json", output) + ) + + self.assertEqual(2, code) + self.assertIn("must not be inside the benchmark checkout", stderr) + self.assertFalse((renamed_checkout / "decision.json").exists()) + def test_output_leaf_symlink_is_replaced_without_overwriting_its_target(self) -> None: victim = self.project_root / "victim.txt" victim.write_text("keep me\n", encoding="utf-8") diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index bcd6354..3b72f15 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -7,6 +7,7 @@ from pathlib import Path from unittest.mock import patch +from agent_release_gate.integration import validator as validator_module from agent_release_gate.integration.validator import ( IntegrationError, IntegrationManifest, @@ -38,6 +39,7 @@ def manifest(self, **kwargs: object) -> IntegrationManifest: def test_valid_checkout_returns_provenance(self) -> None: evidence = validate_integration(self.manifest()) + self.addCleanup(evidence.close) self.assertEqual("clawprobench", evidence.adapter) self.assertEqual("SyntheticBench", evidence.name) @@ -101,7 +103,8 @@ def test_validation_does_not_refresh_or_rewrite_git_index(self) -> None: before = index_path.read_bytes() os.utime(self.checkout / "README.md", (1_577_836_800, 1_577_836_800)) - validate_integration(self.manifest()) + evidence = validate_integration(self.manifest()) + evidence.close() self.assertEqual(before, index_path.read_bytes()) @@ -122,6 +125,7 @@ def test_inherited_git_repository_context_cannot_override_checkout(self) -> None }, ): evidence = validate_integration(self.manifest()) + self.addCleanup(evidence.close) self.assertEqual(self.checkout.resolve(), evidence.checkout_path) self.assertEqual(self.commit, evidence.commit) @@ -151,6 +155,30 @@ def test_validation_aggregates_independent_mismatches(self) -> None: self.assertIn("worktree is not clean", message) self.assertIn("prohibited path is present: vendor", message) + def test_checkout_path_swap_cannot_mix_provenance_and_cleanliness(self) -> None: + (self.checkout / "dirty.txt").write_text("dirty\n", encoding="utf-8") + alternate_parent = self.base / "alternate-repository" + alternate_parent.mkdir() + alternate_checkout, _ = create_git_repo(alternate_parent) + saved_checkout = self.base / "PinnedBenchmark" + original_git = validator_module._git + swapped = False + + def swap_before_status(checkout_fd: int, *args: str): + nonlocal swapped + if args == ("status", "--porcelain") and not swapped: + swapped = True + self.checkout.rename(saved_checkout) + alternate_checkout.rename(self.checkout) + return original_git(checkout_fd, *args) + + with patch( + "agent_release_gate.integration.validator._git", + side_effect=swap_before_status, + ): + with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): + validate_integration(self.manifest()) + def test_manifest_rejects_unknown_or_missing_keys(self) -> None: path = write_manifest( self.project_root, From 656e53980d83f8c23a88d02fb5354c1a5331ab03 Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:07:19 -0400 Subject: [PATCH 12/14] fix: reject filesystem identity aliases --- .github/workflows/ci.yml | 6 +- README.md | 3 +- docs/architecture.md | 12 ++- src/agent_release_gate/cli.py | 75 +++++++++++++++---- src/agent_release_gate/filesystem.py | 6 ++ .../integration/validator.py | 39 +++++++++- tests/test_cli.py | 50 +++++++++++++ tests/test_integration_validator.py | 14 ++++ 8 files changed, 181 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb64efa..4059d0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,11 @@ permissions: jobs: test-package: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} timeout-minutes: 10 env: PIP_DISABLE_PIP_VERSION_CHECK: "1" diff --git a/README.md b/README.md index 1302482..8a91494 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,8 @@ Evaluation inputs and the validated benchmark checkout are pinned by file descriptor, and output is written atomically through a held directory descriptor. A failed evaluation leaves an existing output file unchanged; concurrent path or parent-symlink swaps cannot substitute an input, mix -checkout provenance, or redirect the write into the checkout. +checkout provenance, redirect the write into the checkout, or overwrite a +protected input through a case-variant or hard-link alias. Exit codes are: diff --git a/docs/architecture.md b/docs/architecture.md index 501684b..339b2ef 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -94,16 +94,20 @@ The integration manifest declares the adapter that may consume its evidence. This prevents a CLI invocation from presenting one benchmark's source provenance alongside a different adapter. Prohibited paths are detected even when represented by dangling symlinks. The checkout must resolve to a distinct -direct sibling; the project itself and sibling symlinks back to it are rejected. +direct sibling. Held project, checkout, and parent descriptors are compared by +device and inode, so the project itself, case-variant aliases, and sibling +symlinks back to it are rejected. Before evaluation, the CLI opens and pins the report, policy, and integration manifest by file and parent-directory descriptor, and the validator pins the benchmark checkout by directory descriptor. It then resolves the requested output path, opens its directory without following the final path component, and verifies the opened directory by device and inode. It rejects paths inside -the held benchmark checkout and paths equal to a pinned input, including -aliases reached through symlinked parents. Pinned descriptors are used for -every input read and for creating and replacing the decision file, so +the held benchmark checkout and destination leaves with the same device and +inode as a pinned input, including case variants, hard links, and aliases +reached through symlinked parents. It repeats the input-identity check +immediately before replacement. Pinned descriptors are used for every input +read and for creating and replacing the decision file, so concurrent path and parent-symlink swaps cannot substitute an input, mix checkout validation, or redirect the write into the checkout. The resolved checkout path stays internal and is not serialized. diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index 55b1bc7..5e33498 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -17,9 +17,11 @@ from agent_release_gate.domain.policy import PolicyError, load_policy from agent_release_gate.evaluation.evaluator import evaluate from agent_release_gate.filesystem import ( + FileIdentity, close_best_effort, directory_identity, directory_is_within, + file_identity, open_directory, open_regular_file, same_directory, @@ -71,6 +73,7 @@ class _OutputTarget: directory_fd: int name: str display_path: Path + protected_file_identities: tuple[FileIdentity, ...] def close(self) -> None: directory_fd = self.directory_fd @@ -177,6 +180,14 @@ def _write_json_atomic( temporary.write(serialized) temporary.flush() os.fsync(temporary.fileno()) + if _leaf_matches_protected_input( + target.directory_fd, + target.name, + target.protected_file_identities, + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) os.replace( temporary_name, target.name, @@ -190,6 +201,8 @@ def _write_json_atomic( # The rename has already committed the new file. Reporting failure # here would falsely promise that an existing output was preserved. pass + except DecisionWriteError: + raise except (OSError, TypeError, ValueError) as exc: raise DecisionWriteError( f"unable to write decision {target.display_path}: {exc}" @@ -224,6 +237,19 @@ def _validate_output_path( return resolved_output +def _leaf_matches_protected_input( + directory_fd: int, + name: str, + protected_identities: Sequence[FileIdentity], +) -> bool: + try: + observed = os.stat(name, dir_fd=directory_fd, follow_symlinks=False) + except FileNotFoundError: + return False + observed_identity = observed.st_dev, observed.st_ino + return observed_identity in protected_identities + + def _prepare_output_target( output: Path, *, @@ -265,33 +291,50 @@ def _prepare_output_target( "output path must not be inside the benchmark checkout" ) + protected_identities: list[FileIdentity] = [] for protected in protected_files: - if isinstance(protected, _InputTarget): - protected_name = protected.name - protected_parent_fd = protected.directory_fd - close_protected_parent = False - else: - resolved_protected = protected.resolve(strict=False) - protected_name = resolved_protected.name - protected_parent_fd, _ = open_directory(resolved_protected.parent) - close_protected_parent = True - if output_name != protected_name: - if close_protected_parent: - os.close(protected_parent_fd) - continue + protected_parent_fd: int | None = None + close_protected_parent = False try: - if same_directory(directory_fd, protected_parent_fd): + if isinstance(protected, _InputTarget): + protected_name = protected.name + protected_parent_fd = protected.directory_fd + protected_identities.append(file_identity(protected.file_fd)) + else: + resolved_protected = protected.resolve(strict=False) + protected_name = resolved_protected.name + protected_parent_fd, _ = open_directory( + resolved_protected.parent + ) + close_protected_parent = True + observed = os.stat(resolved_protected, follow_symlinks=False) + protected_identities.append((observed.st_dev, observed.st_ino)) + assert protected_parent_fd is not None + if output_name == protected_name and same_directory( + directory_fd, + protected_parent_fd, + ): raise DecisionWriteError( "output path must not overwrite an evaluation input" ) finally: - if close_protected_parent: - os.close(protected_parent_fd) + if close_protected_parent and protected_parent_fd is not None: + close_best_effort(protected_parent_fd) + + if _leaf_matches_protected_input( + directory_fd, + output_name, + protected_identities, + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) target = _OutputTarget( directory_fd=directory_fd, name=output_name, display_path=output, + protected_file_identities=tuple(protected_identities), ) directory_fd = None return target diff --git a/src/agent_release_gate/filesystem.py b/src/agent_release_gate/filesystem.py index ae7a327..33fae74 100644 --- a/src/agent_release_gate/filesystem.py +++ b/src/agent_release_gate/filesystem.py @@ -7,6 +7,7 @@ DirectoryIdentity = tuple[int, int] +FileIdentity = tuple[int, int] def close_best_effort(descriptor: int) -> None: @@ -34,6 +35,11 @@ def directory_identity(descriptor: int) -> DirectoryIdentity: return observed.st_dev, observed.st_ino +def file_identity(descriptor: int) -> FileIdentity: + observed = os.fstat(descriptor) + return observed.st_dev, observed.st_ino + + def same_directory(left_fd: int, right_fd: int) -> bool: return directory_identity(left_fd) == directory_identity(right_fd) diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index b9c3d8d..5c5cf0a 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -11,8 +11,10 @@ from agent_release_gate.filesystem import ( DirectoryIdentity, close_best_effort, + directory_flags, directory_identity, open_directory, + same_directory, ) @@ -25,6 +27,7 @@ class IntegrationManifest: adapter: str name: str repository_url: str + project_path: Path checkout_path: Path commit: str prohibited_paths: tuple[str, ...] @@ -141,6 +144,7 @@ def load_manifest(path: Path, *, project_root: Path) -> IntegrationManifest: adapter=adapter, name=name, repository_url=repository_url, + project_path=resolved_project_root, checkout_path=checkout_path, commit=commit, prohibited_paths=tuple(prohibited_paths), @@ -192,12 +196,37 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: raise IntegrationError(f"checkout does not exist or is not a directory: {checkout}") checkout_fd: int | None = None + project_fd: int | None = None + checkout_parent_fd: int | None = None + project_parent_fd: int | None = None try: try: checkout_fd, _ = open_directory(checkout) except OSError as exc: raise IntegrationError(f"unable to pin checkout {checkout}: {exc}") from exc + try: + project_fd, _ = open_directory(manifest.project_path) + checkout_parent_fd = os.open( + "..", + directory_flags(), + dir_fd=checkout_fd, + ) + project_parent_fd = os.open( + "..", + directory_flags(), + dir_fd=project_fd, + ) + except OSError as exc: + raise IntegrationError(f"unable to verify checkout placement: {exc}") from exc + if same_directory(checkout_fd, project_fd) or not same_directory( + checkout_parent_fd, + project_parent_fd, + ): + raise IntegrationError( + "checkout must be a distinct direct sibling of the project" + ) + worktree = _git(checkout_fd, "rev-parse", "--is-inside-work-tree") if worktree.returncode != 0 or worktree.stdout.strip() != "true": raise IntegrationError(f"checkout is not a Git worktree: {checkout}") @@ -259,5 +288,11 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: checkout_fd = None return evidence finally: - if checkout_fd is not None: - close_best_effort(checkout_fd) + for descriptor in ( + checkout_fd, + project_fd, + checkout_parent_fd, + project_parent_fd, + ): + if descriptor is not None: + close_best_effort(descriptor) diff --git a/tests/test_cli.py b/tests/test_cli.py index 9b46638..f39c4ce 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -4,6 +4,7 @@ import hashlib import io import json +import os import tempfile import unittest from contextlib import chdir @@ -192,6 +193,55 @@ def test_output_cannot_overwrite_report(self) -> None: self.assertIn("must not overwrite an evaluation input", stderr) self.assertEqual(original, report.read_bytes()) + def test_output_hard_link_cannot_overwrite_report(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + output = self.project_root / "decision.json" + os.link(report, output) + + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, report.read_bytes()) + self.assertEqual(original, output.read_bytes()) + + def test_output_case_variant_cannot_overwrite_report(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + output = self.project_root / "REPORT.JSON" + if not output.exists(): + self.skipTest("requires a case-insensitive filesystem") + + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("must not overwrite an evaluation input", stderr) + self.assertEqual(original, report.read_bytes()) + + def test_protected_input_linked_to_output_before_replace_is_preserved(self) -> None: + report = self.project_root / "report.json" + original = (FIXTURES / "clawprobench_go.json").read_bytes() + report.write_bytes(original) + output = self.project_root / "decision.json" + + with _prepare_output_target( + output, + protected_files=(report, POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + os.link(report, output) + with self.assertRaisesRegex( + ValueError, + "must not overwrite an evaluation input", + ): + _write_json_atomic(target, {"decision": "go"}) + + self.assertEqual(original, report.read_bytes()) + self.assertEqual(original, output.read_bytes()) + def test_output_cannot_overwrite_policy(self) -> None: policy = self.project_root / "policy.toml" original = POLICY.read_bytes() diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index 3b72f15..59819f7 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -228,6 +228,20 @@ def test_manifest_rejects_unsafe_paths_and_commit(self) -> None: with self.assertRaisesRegex(IntegrationError, "prohibited_paths entries must be safe relative paths"): self.manifest(prohibited_paths=["../escape"]) + def test_case_variant_of_project_is_not_checkout(self) -> None: + checkout_alias = self.project_root.parent / self.project_root.name.upper() + if not checkout_alias.is_dir(): + self.skipTest("requires a case-insensitive filesystem") + manifest = self.manifest( + updates={"checkout_path": f"../{self.project_root.name.upper()}"} + ) + + with self.assertRaisesRegex( + IntegrationError, + "distinct direct sibling", + ): + validate_integration(manifest) + def test_manifest_rejects_unsafe_adapter_names(self) -> None: for adapter in ("ClawProBench", "claw pro bench", "-clawprobench"): with self.subTest(adapter=adapter): From 4d46ae760733dc617f6f7b97f440b7e7a69a739d Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:12:46 -0400 Subject: [PATCH 13/14] fix: close remaining filesystem race gaps --- docs/architecture.md | 3 +- src/agent_release_gate/cli.py | 30 +++++++--- src/agent_release_gate/filesystem.py | 20 +++++-- .../integration/validator.py | 7 ++- tests/test_cli.py | 58 +++++++++++++++++++ tests/test_integration_validator.py | 13 ++++- 6 files changed, 116 insertions(+), 15 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 339b2ef..1cbec53 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,7 +79,7 @@ The integration manifest pins the expected origin and full Git commit. Validatio - `git rev-parse --show-prefix`; - `git rev-parse HEAD`; - `git remote get-url origin`; -- `git status --porcelain`. +- `git status --porcelain --untracked-files=all`. Git hooks and fsmonitor are disabled for these read-only subprocesses, terminal prompts are disabled, and `GIT_OPTIONAL_LOCKS=0` prevents status checks from refreshing the upstream index. The validator also disables Git's untracked cache, ignores global/system Git configuration, and removes inherited `GIT_*` variables before setting its explicit safe environment. This prevents ambient `GIT_DIR`, `GIT_WORK_TREE`, or index overrides from redirecting a probe away from the pinned checkout. Validation never fetches, checks out, resets, cleans, or writes upstream files. @@ -106,6 +106,7 @@ and verifies the opened directory by device and inode. It rejects paths inside the held benchmark checkout and destination leaves with the same device and inode as a pinned input, including case variants, hard links, and aliases reached through symlinked parents. It repeats the input-identity check +and benchmark-ancestry check before creating the temporary file and again immediately before replacement. Pinned descriptors are used for every input read and for creating and replacing the decision file, so concurrent path and parent-symlink swaps cannot substitute an input, mix diff --git a/src/agent_release_gate/cli.py b/src/agent_release_gate/cli.py index 5e33498..ed78e52 100644 --- a/src/agent_release_gate/cli.py +++ b/src/agent_release_gate/cli.py @@ -74,6 +74,7 @@ class _OutputTarget: name: str display_path: Path protected_file_identities: tuple[FileIdentity, ...] + protected_directory_identity: tuple[int, int] def close(self) -> None: directory_fd = self.directory_fd @@ -159,6 +160,7 @@ def _write_json_atomic( ) -> None: temporary_name: str | None = None try: + _validate_pinned_output_target(target) serialized = json.dumps( document, indent=2, @@ -180,14 +182,7 @@ def _write_json_atomic( temporary.write(serialized) temporary.flush() os.fsync(temporary.fileno()) - if _leaf_matches_protected_input( - target.directory_fd, - target.name, - target.protected_file_identities, - ): - raise DecisionWriteError( - "output path must not overwrite an evaluation input" - ) + _validate_pinned_output_target(target) os.replace( temporary_name, target.name, @@ -250,6 +245,24 @@ def _leaf_matches_protected_input( return observed_identity in protected_identities +def _validate_pinned_output_target(target: _OutputTarget) -> None: + if directory_is_within( + target.directory_fd, + target.protected_directory_identity, + ): + raise DecisionWriteError( + "output path must not be inside the benchmark checkout" + ) + if _leaf_matches_protected_input( + target.directory_fd, + target.name, + target.protected_file_identities, + ): + raise DecisionWriteError( + "output path must not overwrite an evaluation input" + ) + + def _prepare_output_target( output: Path, *, @@ -335,6 +348,7 @@ def _prepare_output_target( name=output_name, display_path=output, protected_file_identities=tuple(protected_identities), + protected_directory_identity=protected_identity, ) directory_fd = None return target diff --git a/src/agent_release_gate/filesystem.py b/src/agent_release_gate/filesystem.py index 33fae74..77462e2 100644 --- a/src/agent_release_gate/filesystem.py +++ b/src/agent_release_gate/filesystem.py @@ -64,12 +64,16 @@ def _walk_absolute_directory(path: Path) -> int: close_best_effort(current_fd) -def _open_resolved_directory(resolved: Path) -> int: +def _open_resolved_directory( + resolved: Path, + expected_identity: DirectoryIdentity | None = None, +) -> int: expected = os.stat(resolved, follow_symlinks=False) if not stat.S_ISDIR(expected.st_mode): raise NotADirectoryError(errno.ENOTDIR, "not a directory", str(resolved)) + expected_identity = expected_identity or (expected.st_dev, expected.st_ino) descriptor = _walk_absolute_directory(resolved) - if directory_identity(descriptor) != (expected.st_dev, expected.st_ino): + if directory_identity(descriptor) != expected_identity: close_best_effort(descriptor) raise OSError( errno.ESTALE, @@ -80,14 +84,22 @@ def _open_resolved_directory(resolved: Path) -> int: def open_directory(path: Path) -> tuple[int, Path]: + expected = os.stat(path) + if not stat.S_ISDIR(expected.st_mode): + raise NotADirectoryError(errno.ENOTDIR, "not a directory", str(path)) resolved = path.resolve(strict=True) - descriptor = _open_resolved_directory(resolved) + descriptor = _open_resolved_directory( + resolved, + (expected.st_dev, expected.st_ino), + ) return descriptor, resolved def open_regular_file(path: Path) -> tuple[int, int, Path]: + expected = os.stat(path) + if not stat.S_ISREG(expected.st_mode): + raise OSError(errno.EINVAL, "not a regular file", str(path)) resolved = path.resolve(strict=True) - expected = os.stat(resolved, follow_symlinks=False) parent_fd = _open_resolved_directory(resolved.parent) file_fd: int | None = None try: diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index 5c5cf0a..57118c1 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -250,7 +250,12 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: f"unexpected origin URL: expected {manifest.repository_url}, observed {observed_origin}" ) - status = _git(checkout_fd, "status", "--porcelain") + status = _git( + checkout_fd, + "status", + "--porcelain", + "--untracked-files=all", + ) if status.returncode != 0: failures.append("unable to determine worktree status") elif status.stdout: diff --git a/tests/test_cli.py b/tests/test_cli.py index f39c4ce..631557a 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -391,6 +391,41 @@ def resolve_then_swap(path: Path, *args: object, **kwargs: object) -> Path: self.assertIn("unable to open evaluation input", stderr) self.assertFalse(output.exists()) + def test_input_ancestor_directory_replacement_during_pin_is_rejected( + self, + ) -> None: + requested_directory = self.base / "requested-directory" + requested_inner = requested_directory / "inner" + requested_inner.mkdir(parents=True) + report = requested_inner / "report.json" + report.write_bytes((FIXTURES / "clawprobench_go.json").read_bytes()) + saved_directory = self.base / "saved-directory" + substitute_directory = self.base / "substitute-directory" + substitute_inner = substitute_directory / "inner" + substitute_inner.mkdir(parents=True) + (substitute_inner / "report.json").write_bytes( + (FIXTURES / "clawprobench_no_go.json").read_bytes() + ) + output = self.project_root / "decision.json" + original_resolve = Path.resolve + swapped = False + + def resolve_then_replace(path: Path, *args: object, **kwargs: object) -> Path: + nonlocal swapped + resolved = original_resolve(path, *args, **kwargs) # type: ignore[arg-type] + if path == report and not swapped: + swapped = True + requested_directory.rename(saved_directory) + substitute_directory.rename(requested_directory) + return resolved + + with patch.object(Path, "resolve", resolve_then_replace): + code, _, stderr = self.invoke(self.evaluate_args(report, output)) + + self.assertEqual(2, code) + self.assertIn("unable to open evaluation input", stderr) + self.assertFalse(output.exists()) + def test_validated_checkout_rename_cannot_redirect_output_into_it(self) -> None: safe_directory = self.project_root / "safe" safe_directory.mkdir() @@ -419,6 +454,29 @@ def rename_before_prepare(*args: object, **kwargs: object): self.assertIn("must not be inside the benchmark checkout", stderr) self.assertFalse((renamed_checkout / "decision.json").exists()) + def test_output_directory_moved_into_checkout_before_write_is_rejected( + self, + ) -> None: + safe_directory = self.project_root / "safe-output" + safe_directory.mkdir() + output = safe_directory / "decision.json" + + with _prepare_output_target( + output, + protected_files=(FIXTURES / "clawprobench_go.json", POLICY, self.manifest), + protected_directory=self.checkout, + ) as target: + moved_directory = self.checkout / "moved-output" + safe_directory.rename(moved_directory) + with self.assertRaisesRegex( + ValueError, + "must not be inside the benchmark checkout", + ): + _write_json_atomic(target, {"decision": "go"}) + + self.assertFalse((moved_directory / "decision.json").exists()) + self.assertEqual([], list(moved_directory.iterdir())) + def test_output_leaf_symlink_is_replaced_without_overwriting_its_target(self) -> None: victim = self.project_root / "victim.txt" victim.write_text("keep me\n", encoding="utf-8") diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index 59819f7..d9ab117 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -98,6 +98,13 @@ def test_dirty_worktree_is_rejected(self) -> None: with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): validate_integration(self.manifest()) + def test_repository_config_cannot_hide_untracked_files(self) -> None: + run_git(self.checkout, "config", "status.showUntrackedFiles", "no") + (self.checkout / "untracked.txt").write_text("dirty\n", encoding="utf-8") + + with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): + validate_integration(self.manifest()) + def test_validation_does_not_refresh_or_rewrite_git_index(self) -> None: index_path = self.checkout / ".git" / "index" before = index_path.read_bytes() @@ -166,7 +173,11 @@ def test_checkout_path_swap_cannot_mix_provenance_and_cleanliness(self) -> None: def swap_before_status(checkout_fd: int, *args: str): nonlocal swapped - if args == ("status", "--porcelain") and not swapped: + if args == ( + "status", + "--porcelain", + "--untracked-files=all", + ) and not swapped: swapped = True self.checkout.rename(saved_checkout) alternate_checkout.rename(self.checkout) From 648a24c7855ca64b595f46d3fedb5d5ea9d24dff Mon Sep 17 00:00:00 2001 From: bsha6 <54374986+bsha6@users.noreply.github.com> Date: Sat, 29 Aug 2026 15:19:23 -0400 Subject: [PATCH 14/14] fix: make checkout cleanliness configuration-independent --- README.md | 4 ++ docs/architecture.md | 13 +++++- .../integration/validator.py | 42 +++++++++++++++++++ tests/test_integration_validator.py | 29 +++++++++++++ 4 files changed, 87 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8a91494..6278a68 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,10 @@ for the complete build, test, CI, upstream, and audit inventory. - v0 preserves the report timestamp but does not enforce evidence freshness. - Custom policies and integration manifests are trusted local configuration. - The CLI does not fetch or execute benchmark code. +- Filesystem identity and ancestry are checked at acquisition and immediately + before output commit. A hostile concurrent process with write access to both + directory trees is outside the v0 threat model; run evaluations where + untrusted processes cannot rename the output or benchmark directories. - Decision output must be separate from reports, policies, manifests, and benchmark checkouts; protected paths are rejected after symlink resolution while input files, the benchmark checkout, and the output directory remain diff --git a/docs/architecture.md b/docs/architecture.md index 1cbec53..d9af269 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -79,7 +79,10 @@ The integration manifest pins the expected origin and full Git commit. Validatio - `git rev-parse --show-prefix`; - `git rev-parse HEAD`; - `git remote get-url origin`; -- `git status --porcelain --untracked-files=all`. +- `git status --porcelain --untracked-files=all`; +- `git ls-files --others` without exclude rules; +- `git ls-files -v -z` to reject assume-unchanged entries and skip-worktree + entries that remain present (absent sparse-checkout entries are allowed). Git hooks and fsmonitor are disabled for these read-only subprocesses, terminal prompts are disabled, and `GIT_OPTIONAL_LOCKS=0` prevents status checks from refreshing the upstream index. The validator also disables Git's untracked cache, ignores global/system Git configuration, and removes inherited `GIT_*` variables before setting its explicit safe environment. This prevents ambient `GIT_DIR`, `GIT_WORK_TREE`, or index overrides from redirecting a probe away from the pinned checkout. Validation never fetches, checks out, resets, cleans, or writes upstream files. @@ -113,6 +116,14 @@ concurrent path and parent-symlink swaps cannot substitute an input, mix checkout validation, or redirect the write into the checkout. The resolved checkout path stays internal and is not serialized. +These checks validate path identity and ancestry when descriptors are acquired +and immediately before the output commit. v0 does not claim to contain a +hostile local process that already has write access to both the output and +benchmark directory trees and races a directory rename between the final +validation syscall and `os.replace`; such a process can already modify the +read-only checkout directly. Run evaluations in a filesystem namespace not +writable by untrusted concurrent processes. + ## Failure Handling Expected input and filesystem failures produce a concise stderr message without a traceback. Atomic output prevents a failed evaluation from truncating a previous decision. Unexpected failures are contained at the command boundary and return exit code `2`. diff --git a/src/agent_release_gate/integration/validator.py b/src/agent_release_gate/integration/validator.py index 57118c1..9fbcd05 100644 --- a/src/agent_release_gate/integration/validator.py +++ b/src/agent_release_gate/integration/validator.py @@ -250,6 +250,7 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: f"unexpected origin URL: expected {manifest.repository_url}, observed {observed_origin}" ) + worktree_dirty = False status = _git( checkout_fd, "status", @@ -259,8 +260,49 @@ def validate_integration(manifest: IntegrationManifest) -> IntegrationEvidence: if status.returncode != 0: failures.append("unable to determine worktree status") elif status.stdout: + worktree_dirty = True + + untracked = _git(checkout_fd, "ls-files", "--others") + if untracked.returncode != 0: + failures.append("unable to enumerate untracked worktree files") + elif untracked.stdout: + worktree_dirty = True + + if worktree_dirty: failures.append("worktree is not clean") + index_flags = _git(checkout_fd, "ls-files", "-v", "-z") + if index_flags.returncode != 0: + failures.append("unable to inspect index flags") + else: + entries = [ + entry + for entry in index_flags.stdout.split("\0") + if entry + ] + if any(entry[0].islower() for entry in entries): + failures.append("index contains assume-unchanged entries") + present_skip_worktree = False + for entry in entries: + if not entry.startswith("S "): + continue + try: + os.stat( + entry[2:], + dir_fd=checkout_fd, + follow_symlinks=False, + ) + except (FileNotFoundError, NotADirectoryError): + continue + except OSError as exc: + failures.append( + f"unable to inspect skip-worktree path {entry[2:]}: {exc}" + ) + continue + present_skip_worktree = True + if present_skip_worktree: + failures.append("index contains present skip-worktree entries") + for prohibited_path in manifest.prohibited_paths: try: os.stat( diff --git a/tests/test_integration_validator.py b/tests/test_integration_validator.py index d9ab117..96383b0 100644 --- a/tests/test_integration_validator.py +++ b/tests/test_integration_validator.py @@ -105,6 +105,35 @@ def test_repository_config_cannot_hide_untracked_files(self) -> None: with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): validate_integration(self.manifest()) + def test_repository_excludes_file_cannot_hide_untracked_files(self) -> None: + excludes = self.base / "local-excludes" + excludes.write_text("hidden.txt\n", encoding="utf-8") + run_git(self.checkout, "config", "core.excludesFile", str(excludes)) + (self.checkout / "hidden.txt").write_text("dirty\n", encoding="utf-8") + + with self.assertRaisesRegex(IntegrationError, "worktree is not clean"): + validate_integration(self.manifest()) + + def test_assume_unchanged_cannot_hide_tracked_modification(self) -> None: + run_git(self.checkout, "update-index", "--assume-unchanged", "README.md") + (self.checkout / "README.md").write_text("modified\n", encoding="utf-8") + + with self.assertRaisesRegex( + IntegrationError, + "assume-unchanged", + ): + validate_integration(self.manifest()) + + def test_present_skip_worktree_file_cannot_hide_modification(self) -> None: + run_git(self.checkout, "update-index", "--skip-worktree", "README.md") + (self.checkout / "README.md").write_text("modified\n", encoding="utf-8") + + with self.assertRaisesRegex( + IntegrationError, + "skip-worktree", + ): + validate_integration(self.manifest()) + def test_validation_does_not_refresh_or_rewrite_git_index(self) -> None: index_path = self.checkout / ".git" / "index" before = index_path.read_bytes()