diff --git a/.github/workflows/noema-review.yml b/.github/workflows/noema-review.yml index 21ea967201..d99758c6e2 100644 --- a/.github/workflows/noema-review.yml +++ b/.github/workflows/noema-review.yml @@ -607,35 +607,32 @@ jobs: run: | set -euo pipefail if [ -z "${GH_TOKEN:-}" ]; then - echo "::error::Noema target visibility cannot be resolved without the selected repository-scoped reviewer token." - exit 1 + echo "::notice::Noema target visibility audit was unavailable; continuing with the mandatory ZDR-only review pool." fi visibility="" - for target_visibility_attempt in 1 2 3 4 5 6; do - if visibility="$( - gh api "/repos/${TARGET_REPOSITORY}" --jq '.visibility // (if .private then "private" else "public" end)' - )"; then - break - fi - visibility="" - if [ "$target_visibility_attempt" -lt 6 ]; then - echo "Repository visibility lookup failed (attempt ${target_visibility_attempt}/6), possibly a transient GitHub API rate limit; retrying after backoff." >&2 - sleep "$(( target_visibility_attempt * 5 ))" - fi - done + if [ -n "${GH_TOKEN:-}" ]; then + for target_visibility_attempt in 1 2 3 4 5 6; do + if visibility="$( + gh api "/repos/${TARGET_REPOSITORY}" --jq '.visibility // (if .private then "private" else "public" end)' + )"; then + break + fi + visibility="" + if [ "$target_visibility_attempt" -lt 6 ]; then + echo "Repository visibility lookup failed (attempt ${target_visibility_attempt}/6), possibly a transient GitHub API rate limit; retrying after backoff." >&2 + sleep "$(( target_visibility_attempt * 5 ))" + fi + done + fi case "$visibility" in - private|internal) - echo "require_zdr=true" >>"$GITHUB_OUTPUT" - echo "::notice::Private/internal target requires an attested ZDR-only review pool." - ;; - public) - echo "require_zdr=false" >>"$GITHUB_OUTPUT" + private|internal|public) + echo "::notice::Every target requires an attested ZDR-only review pool." ;; *) - echo "::error::Noema target repository visibility is missing or unsupported: ${visibility:-}." - exit 1 + echo "::notice::Noema target visibility was unavailable; the review remains bound to the mandatory ZDR-only pool." ;; esac + echo "require_zdr=true" >>"$GITHUB_OUTPUT" - name: Provision contextual-orchestrator review sidecar if: env.PR_NUMBER != '' diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 26e8555967..a6fa24a624 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -668,10 +668,15 @@ jobs: esac done <"$python_change_files" if [ "$python_coverage_required" -eq 1 ]; then + # --target-python-version must track the coverage image's pinned + # "FROM docker.io/library/python:3.14-slim@sha256:..." tag below + # so an archive marker excluding that version skips its download + # instead of failing the whole job on an unrelated URL outage. python3 -I "$GITHUB_WORKSPACE/scripts/ci/materialize_base_python_requirements.py" \ --repo-root "$COVERAGE_SOURCE_WORKDIR" \ --base-sha "$PR_BASE_SHA" \ - --output-dir "$coverage_build_dir/base-python-requirements" + --output-dir "$coverage_build_dir/base-python-requirements" \ + --target-python-version 3.14 else mkdir -p "$coverage_build_dir/base-python-requirements" printf '[]\n' >"$coverage_build_dir/base-python-requirements/manifest.json" @@ -782,6 +787,7 @@ jobs: -r /tmp/requirements-opencode-review-ci-hashes.txt \ && rm -f /tmp/requirements-opencode-review-ci-hashes.txt COPY base-python-requirements /tmp/base-python-requirements + ENV CARGO_HOME=/opt/base-cargo RUN set -eu; \ mkdir -p /opt/base-vcs-dependencies; \ site_packages="$(python3 -c 'import site; print(site.getsitepackages()[0])')"; \ @@ -859,6 +865,93 @@ jobs: COPY install-base-python-locks.py /usr/local/libexec/install-base-python-locks.py RUN python3 -I /usr/local/libexec/install-base-python-locks.py \ --requirements-root /tmp/base-python-requirements \ + --no-archives + RUN mkdir -p /opt/base-python-archive-sources \ + && python3 - <<'PYTHON' + import json + import pathlib + import re + import stat + import tarfile + import tomllib + import zipfile + from packaging.markers import Marker, default_environment + + requirements_root = pathlib.Path("/tmp/base-python-requirements") + source_root = pathlib.Path("/opt/base-python-archive-sources").resolve() + archive_manifest_path = requirements_root / "archive-manifest.json" + manifest = ( + json.loads(archive_manifest_path.read_text(encoding="utf-8")) + if archive_manifest_path.exists() + else [] + ) + coverage_environment = default_environment() + for index, entry in enumerate(manifest): + marker = entry.get("marker") + if marker is not None and not Marker(marker).evaluate(coverage_environment): + continue + relative_file = entry["file"] + archive = (requirements_root / relative_file).resolve() + destination = (source_root / f"archive-{index:03d}").resolve() + destination.mkdir(parents=True, exist_ok=True) + if not archive.is_file() or archive.is_symlink(): + raise SystemExit(f"archive input is not a regular file: {relative_file}") + if relative_file.endswith(".tar.gz"): + with tarfile.open(archive, "r:gz") as bundle: + members = bundle.getmembers() + for member in members: + target = (destination / member.name).resolve() + if not target.is_relative_to(destination) or member.issym() or member.islnk() or member.isdev(): + raise SystemExit(f"archive contains an unsafe member: {relative_file}") + bundle.extractall(destination) + elif relative_file.endswith(".zip"): + with zipfile.ZipFile(archive) as bundle: + for member in bundle.infolist(): + target = (destination / member.filename).resolve() + mode = (member.external_attr >> 16) & 0o170000 + if not target.is_relative_to(destination) or mode == stat.S_IFLNK: + raise SystemExit(f"archive contains an unsafe member: {relative_file}") + bundle.extractall(destination) + else: + raise SystemExit(f"unsupported archive suffix: {relative_file}") + pyprojects = [ + path for path in destination.rglob("pyproject.toml") if path.is_file() + ] + if len(pyprojects) != 1: + raise SystemExit( + f"archive must expose exactly one pyproject.toml: {relative_file}" + ) + project = tomllib.loads(pyprojects[0].read_text(encoding="utf-8")) + build_system = project.get("build-system") + build_requires = ( + build_system.get("requires") + if isinstance(build_system, dict) + else None + ) + if ( + not isinstance(build_system, dict) + or build_system.get("build-backend") != "maturin" + or not isinstance(build_requires, list) + or not any( + isinstance(requirement, str) + and re.fullmatch( + r"maturin(?:\[.*\])?(?:\s*[<>=!~].*)?", + requirement, + ) + for requirement in build_requires + ) + ): + raise SystemExit( + f"archive build backend is not the installed maturin contract: {relative_file}" + ) + PYTHON + RUN set -eu; \ + find /opt/base-python-archive-sources -name Cargo.toml -print0 \ + | sort -z -u \ + | xargs -0 -r -n1 cargo fetch --locked --manifest-path + RUN --network=none python3 -I /usr/local/libexec/install-base-python-locks.py \ + --requirements-root /tmp/base-python-requirements \ + --archives-only \ && rm -rf /tmp/base-python-requirements \ && rm -f /usr/local/libexec/install-base-python-locks.py DOCKERFILE @@ -2436,7 +2529,7 @@ jobs: NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ needs.validate-pr-metadata.outputs.is_private }} + CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: "true" run: | set -euo pipefail bash "$GITHUB_WORKSPACE/scripts/ci/contextual_orchestrator_review_sidecar.sh" diff --git a/.github/workflows/pr-review-autofix.yml b/.github/workflows/pr-review-autofix.yml index 1b7849a0c5..1d36bd191d 100644 --- a/.github/workflows/pr-review-autofix.yml +++ b/.github/workflows/pr-review-autofix.yml @@ -270,6 +270,7 @@ jobs: NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: "true" run: | set -euo pipefail bash "$GITHUB_WORKSPACE/trusted-autofix-source/scripts/ci/contextual_orchestrator_review_sidecar.sh" diff --git a/.github/workflows/strix.yml b/.github/workflows/strix.yml index 58ed3dab8d..f540e29bd2 100644 --- a/.github/workflows/strix.yml +++ b/.github/workflows/strix.yml @@ -751,7 +751,7 @@ jobs: NVIDIA_NIM_API_KEY_SUB: ${{ secrets.NVIDIA_NIM_API_KEY_SUB }} OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ steps.target_visibility.outputs.is_private }} + CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: "true" CONTEXTUAL_ORCHESTRATOR_POOL: free run: | set -euo pipefail diff --git a/CHANGELOG.md b/CHANGELOG.md index ca81dcea1d..721e83b542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -118,6 +118,20 @@ this file. The format follows Keep a Changelog, and versioned releases follow Semantic Versioning where the repository publishes a release. ## [Unreleased] +- Materialize hash-pinned organization archive dependencies as verified local + inputs and install them in a separate Docker `--network=none` phase with the + pinned `maturin` build backend, so archive build hooks cannot use image-build + network access or alter the regular pip lock closure. +- Preserve the private contextual-orchestrator bearer-file owner and mode gate + across GNU, BusyBox, and BSD/macOS `stat` implementations without relaxing + the required current-user ownership or exact mode `600` contract. +- Accept valid leading-dot repository names such as `ContextualWisdomLab/.github` + in both OpenCode receipt and coverage-identity gates while rejecting dot-path + segments and option-like names before any GitHub CLI call. +- Keep canonical `openrouter/` endpoint-feed ZDR evidence exact for + OpenRouter routes while allowing an unambiguous matching feed model identity + to attest supplied non-OpenRouter provider rows; noncanonical, nonmatching, or + ambiguous rows remain non-ZDR. - **Pin `opencode-review-dispatch.yml` off the starved floating `ubuntu-latest` image.** The 2026-09-01 floating-image fix (see that entry below) pinned `strix.yml`, `opencode-review.yml`, and `noema-review.yml` -- the three required-check @@ -1132,7 +1146,7 @@ Semantic Versioning where the repository publishes a release. - Give stacked pull requests a separately bounded organization-sweep OpenCode dispatch budget, so default-branch review traffic cannot leave a stacked PR at `OpenCode review absent` without changing the protected merge - or exact-head evidence rules. + or exact-head evidence rules. - Add a bounded hourly LineageWeave stacked-PR review-repair caller while preserving the existing review-agent, model-routing, and protected-merge boundaries. Product-gap development remains a separately gated coordinator diff --git a/docs/doctoring/trusted-uv-lock-materialization.md b/docs/doctoring/trusted-uv-lock-materialization.md index 2d83e8bda8..73070fecf3 100644 --- a/docs/doctoring/trusted-uv-lock-materialization.md +++ b/docs/doctoring/trusted-uv-lock-materialization.md @@ -41,10 +41,19 @@ The implementation therefore: 10. keeps project metadata discovery enabled because the reconstructed `pyproject.toml` is an authoritative input; `--no-config` is deliberately not used because uv documents that it disables `pyproject.toml` discovery; -11. rejects every nonempty export unless every logical line is an exact normalized - package `==` pin followed only by complete SHA-256 hashes; and -12. exposes only generated requirements files and a source manifest to the later - networkless coverage environment. +11. accepts a direct archive only when it is an exact HTTPS + `github.com/ContextualWisdomLab//archive/...tar.gz|zip` reference with + a complete SHA-256 hash; its environment marker is preserved in the archive + manifest and later pip direct-reference command so false-marker archives are + skipped by pip rather than built unconditionally; +12. rejects every nonempty export unless every logical line is an exact normalized + package `==` pin, an allowlisted organization archive, or an exact immutable + organization VCS source; and +13. exposes generated requirements files, VCS metadata, and locally verified + archive inputs to the later networkless coverage environment. The coverage + image currently accepts only archives declaring the installed `maturin` build + backend and its `maturin` build requirement; unsupported PEP 517 backends + fail before extraction completes. ## Standards and current-tool rationale @@ -70,6 +79,17 @@ sets `UV_NO_ENV_FILE=1` and `UV_PYTHON_DOWNLOADS=never`, and passes only a fixed `PATH`. This preserves the exact reconstructed project metadata while excluding user-level and runner-level configuration state. +Organization archive entries are not installed from a networked pip build. The +materializer downloads each archive from the fixed GitHub HTTPS origins, +verifies its exported SHA-256 digest, and writes it separately from regular +`--require-hashes` locks. The coverage Dockerfile rejects unsafe tar/zip members, +requires exactly one `pyproject.toml` with the installed `maturin` backend and +requirement, fetches any Rust dependency manifests while networked, and invokes +archive build hooks only in the subsequent `RUN --network=none` layer. The image +preparation evaluates each archive marker for the coverage interpreter before +extraction or Cargo fetching, while the marker remains part of the later +direct-reference requirement so pip evaluates it again before any build hook. + Generic requirements discovery continues to accept a global `--require-hashes` directive because pip performs a later closure preflight. Trusted `uv export` output uses a stricter rule: every logical line must begin diff --git a/requirements-opencode-review-ci-hashes.txt b/requirements-opencode-review-ci-hashes.txt index d8aaca3ad8..e58b02f50f 100644 --- a/requirements-opencode-review-ci-hashes.txt +++ b/requirements-opencode-review-ci-hashes.txt @@ -4,9 +4,9 @@ attrs==26.1.0 \ --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 # via interrogate -click==8.4.2 \ - --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ - --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 +click==8.5.0 \ + --hash=sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360 \ + --hash=sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34 # via interrogate colorama==0.4.6 \ --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \ @@ -137,69 +137,88 @@ coverage==7.15.4 \ # via # -r requirements-opencode-review-ci.txt # pytest-cov -hypothesis==6.163.0 \ - --hash=sha256:002a9709345892279fb0e81b5a05b72d08cfe81f937339827be0d588607ca9b0 \ - --hash=sha256:00d3091b28de83c5116e0ccd9a4bcb28ef61d2aace5df91093bb22434fd2350c \ - --hash=sha256:0a0c396244c13805edcb73ff467c4c8178ccefc41c4ef5ed00a68e612fd773e9 \ - --hash=sha256:0a933aca9ebf9daf951d07cf01200c94c321b6ee0b42cc7b67675c9686d914c2 \ - --hash=sha256:0cba5202f74e7e4cdb676d86f26e8cc1b4fdc88f7f58ba73c8ac45b6b22f3070 \ - --hash=sha256:213527755f0fc2b1f3721e73fd60023e2752a48f914e3e2df8d35111956ae5c8 \ - --hash=sha256:21e72e8d5818e5ef8cd6a2191c386e3fd1a6d9e3739cf97289b4d9b5dbc8e38d \ - --hash=sha256:2849c23b2e0fe2eef4c1ec336b01eac7ad7397c49fca43c264f59ec1e6046eac \ - --hash=sha256:28a6cc1c25a6cc9b6ec079eaabd32ac769994831ecddd57123ce43c9056dcf34 \ - --hash=sha256:31dc46c48aa53c3ec92d03120978ca7f19b9cf96d195ed3fc93503f1433c94a6 \ - --hash=sha256:320b076bf6436f971f1c73ee651e60001226d1b4e341f2c4a1ca87248261ca03 \ - --hash=sha256:331906cb029b6b360b8ebac3ec00c3cfa720037fe2efb294a503a1979c9a9a8f \ - --hash=sha256:34fc895691a2420595506eb17f3a104f2fa9039f013c0770a6cc2743ccaf6fed \ - --hash=sha256:3b6cee2afe6c67b31a4a64b63a876e0b020befdc61daabea80f7a0e14f19203a \ - --hash=sha256:3f3cceb4720a39127622fbf3bcebe1775b894372c53b5edddfdef10bbdeef9ec \ - --hash=sha256:40dfab6fe6a02a80abef81aebf88e53cd529e3f2f6ba3486b674a67b1f4a3512 \ - --hash=sha256:4159a1c2560e10de51b1c14956e277eb1b37526c9abef9e87c1e531760486448 \ - --hash=sha256:487ab8ec2f01a225d6a1e2ceadc5290cde2c691952bd2e7f76199cf82e06fb25 \ - --hash=sha256:4ab0dadc09c537d4ac57e564039dfe7daf09c98375306d54bfc0fd6c218efcca \ - --hash=sha256:50073f8e63c1e7d3403899755657a990d8bba7b5b5bff66b1c56796d4969bb28 \ - --hash=sha256:520480d4bd3a17557616c25923640953e360332c89d012fffcebd69857e674a9 \ - --hash=sha256:52f16840add2eb02c2416f3b83cec4f527b6c19699f2d31eff4859233c715526 \ - --hash=sha256:56ed585baab75cb98462c57ca88bbdc6a9d935a14118dd572fb476c3ecec2a06 \ - --hash=sha256:58be45d1737bf8c2e10cf29505c0f10f8a23d61bc82e4339182a6c8251cbc2d9 \ - --hash=sha256:59f5fdb8addb44c17520a60d50542d9db6ceba577bbf54efefa9c10ee20be140 \ - --hash=sha256:5a3ac6c62d49f7fe518dfe7fa924fa03aac839993702207802b0e45f9e1b0dab \ - --hash=sha256:67d1593941ede41052b4a35ec25b50d0e280358c7674ef7812d520010e7e8bdf \ - --hash=sha256:6ae63dec6d1d467b7f4737455f81a7a82f14a41c14510937fcfbc726a085b5f8 \ - --hash=sha256:7a3db868a943c814cc557104712d43bf609adfe5ea9f708f38377d366b4855f8 \ - --hash=sha256:7ca7b20bf38d51e15f7808b0239791c4792b1709ce0c63093acaff56a09c31e6 \ - --hash=sha256:7cb3d927360fe73f9a06d646e6082237142ee39c24679c7133d22bf06dd03b45 \ - --hash=sha256:7ef8954e37c80e0c46e6161eef1c72c71059b95250e620a77bd646f6c7a52a2d \ - --hash=sha256:8aac96db8a6c7ee43aba2ee0d3c43893da1fb7c38ed54790c1be2b6d8fd87b96 \ - --hash=sha256:8c5d1e6bad47edf6fb1d7406cf6d67314ac08325c63a49550d782a4596ea302b \ - --hash=sha256:9105c66ea8dbc108adc42058bb7b65bd953f53ee178bf63bf9ebb0cded6c8c96 \ - --hash=sha256:9be37b7ddf0af9e3f9112cd133afc34e78a56da1f96db5f2b4fc289fe1c4d1c3 \ - --hash=sha256:9c084749c115ea7918cf7efa144682783da17eec70d1276689182b871126e715 \ - --hash=sha256:9d23f0f3a14bb6e6f99c793d340196dba4af95ba25bfcab624d1794f540f5e27 \ - --hash=sha256:a16ebce774755a7a652bd44c62101dc914372ed1a98935969624848c9627b4a4 \ - --hash=sha256:a2a20e9835d3c4b293a709ee6ef769bcb18c6ed4ef337a9e251c1a9496d5e8be \ - --hash=sha256:a57352efa938889ea9992667a5014c0fc870d03945de71918574d1cf28276378 \ - --hash=sha256:ab34c61d9249f1a8129cb4276062c04e3e47b5be8de6446e7c7fe11362d6fe43 \ - --hash=sha256:b123b4995a7612f1130e2b2362c9a5d0568df887bf7e7bdb45c23af8cd5423c9 \ - --hash=sha256:b268211e625cd550e361fc387bf1db5deb1e9cae0ce4041116f0a0aafeef7c06 \ - --hash=sha256:b2ddcdaf6691101e06dc4a5add7b8c8fdf1e68daba599255a281f3f3550d3331 \ - --hash=sha256:b4ad2134405d5345434c22dea96bbc12c85abcfc3c253a8063dbc9ff01164555 \ - --hash=sha256:b839dfd1342bb50570cb0c66b80322307cdb468abf14faf5df4dab022bc1b9ce \ - --hash=sha256:b8f22fb8218ba6a452bf9000fc656e1ed57625d17cc8a3871a0fcea3b1b69ebf \ - --hash=sha256:bd312b15044b1c1a0920a5827a830559b2d1fa380851cedf509f8b835309c5b9 \ - --hash=sha256:c0ec3b709508ccd835d8ded1db025b7800618f2289a22a6bfd4927da5f4eb33c \ - --hash=sha256:c4f5be1482189c7b0a1dcac269fffe97a7d18cc04ac9a9a4d6613212dd87f38b \ - --hash=sha256:ca1b48bde68c528a79dec2a2859e05035802e5b1c9c3579f388c9de6ed6d0148 \ - --hash=sha256:d0838a28e9943d5b834ebae59b02adda76e2cd1e65caa808104c72102052057d \ - --hash=sha256:e165f6cc2075059b7c95dac1612bfb25494f72d90f56880e84c288b089f8a896 \ - --hash=sha256:e568a3d766b7ba8df00e0c33efc4c6530cde14fbc72daabe4824eed211ed7596 \ - --hash=sha256:ee47c2cb1be03a052ebd3549dad07f636a98b3ccfd7acbe5e17b3b7da0ab9e37 \ - --hash=sha256:f1fe222f50a1898e87a1e7323ab35f9e956278efabe4dd55a1342808206d05ad \ - --hash=sha256:f28ad27193c1fbcfb52ef2ee63d2b721563525089e80962b4268b306dac45507 \ - --hash=sha256:f2f1b67a48da86d3e41c9445367b49a49f7efdb60fc8b5e3593f05e6afb2efbe \ - --hash=sha256:f7f706df6839dcc53f20833f2933cbcd126fd2fdee7c312e053de49df4b64e44 \ - --hash=sha256:fae7305ae20fddeea09df317b920c45d3e20bfedbdb041f4db6ca5267c458189 \ - --hash=sha256:ffdda3006a383a48f71a23b4f2b3fae3fe1b09af67925d885985f7ec34d66bcb +hypothesis==6.167.0 \ + --hash=sha256:004ddebe0f85bc96f63b570659d0d37ff5ad4f4be7fa4b98f3313ce405019610 \ + --hash=sha256:014b237fe33d9abf50a4bbdc1d9de04307a768bf0fee43422f541169b91eae53 \ + --hash=sha256:064c6341919abb5776e6b51f701858670b584d1963923b336e78e6e1edf2152d \ + --hash=sha256:07ea389b2351b1119142b2e736c5eb96aae361f988db3e1c36488c294b41552c \ + --hash=sha256:0b2846cac259c96b81c6539f722c3b9fd2458a85e6273b7986dc08ee05c92318 \ + --hash=sha256:0d0e57b70dbd8b6a1fdd3791614104f7565a48928d6edde0a6c819f9778327fd \ + --hash=sha256:1b156573357689e2056030383d9012408c898dbcf3b245e8c639e559494d6865 \ + --hash=sha256:1db3d15a8f77c347c711bd64b844c4610e97e41d27fb3b2c96d9611f60f5d7f4 \ + --hash=sha256:1f9c13b3680e37f30184ec9a9fc19d99eaf2278313252a6bc1c51a5fce5b2be4 \ + --hash=sha256:21039a571752c8faa69733b94f4fe04250a1dff759a87089bc3b423c1d216b4d \ + --hash=sha256:2149948b652a74795cd5b92b624624ff1228f29f2837b1f354a968178ac82860 \ + --hash=sha256:220e68920f0da423c78aa0e06ee8461ad2eb964ff3fdb7cafb78754f3a992411 \ + --hash=sha256:23110d0cafd4bb146b30903f1ce268dd98784ab3a49e7e420e67d0c94f2cac98 \ + --hash=sha256:23a60707a2bc5e0c59ef95e94264c4e8fd1c54b4ca1b95cbde74fb68b9c2eeef \ + --hash=sha256:2402fd4f170a4023d3ba5a20858a39e9e8fac105eced4eb619d7d11271be6f6b \ + --hash=sha256:25434371146b9d59a552ff0bbaba5ec198cbbab5fe24448b20a1df6b220a660b \ + --hash=sha256:29f6fa02f7e98ca1ea2b67083d3267a6de443a513694d5d30214aa62fc2c13cf \ + --hash=sha256:2dae6eeddabd8f420dea09e203b1c5945639a9b240cf8f38d63e954d5fa3b219 \ + --hash=sha256:31bdca9e569d28a2b17399163a150d7de51fe3d3161203087ee9ee34cba13390 \ + --hash=sha256:342576693423821587babc15064c4f0f13630624a5523db1963be9109686ef09 \ + --hash=sha256:34270e93976cf038852a990836a45f9df55883ee8d661131497ec5a1a067af37 \ + --hash=sha256:3532de0fef4cb383e5959aca1b9d2af66804f661cac855e328bb8b00992e8800 \ + --hash=sha256:3796498cae6894e737d8569d8fe9d3be0650019ca6f378184d18f14cd522c6fd \ + --hash=sha256:3a235e3ef76f73e32922383bd507970cbc01fc08f17043d18a72508957af8f06 \ + --hash=sha256:3d717181cbc0daeb673a64b421276f21d1d619deb00877cc383601ac6dce9571 \ + --hash=sha256:42be1bf942e4279892a9af46745363d053931daf4aef50634789a2301bcbed78 \ + --hash=sha256:42e4e0a97dba87463a7636c370d2b8dff690a74896d20ec016c5b2eba72b50e5 \ + --hash=sha256:4346b4ab05c59613f828452bbf3c8165d31536141dfaece9ad22f9c56b23310d \ + --hash=sha256:44ae0dc7cf9b0a20d39da08ec1fbe2c31c9811781582a4b7aea5bb9dd4509cef \ + --hash=sha256:4a287d1cc06c4591ac2369997e400c99d31ad2debe52cc0f9bff12f216f80fa8 \ + --hash=sha256:5024a40cd2c82f7391499d5348c7cbf0360f6e6d3a116fb9020aa48f5a212483 \ + --hash=sha256:534430c0dd66bee5d4f9263a34ee2877912e796e0ff2827b98d91837e071dd84 \ + --hash=sha256:56e0073e8bac881ae7d7e3f6bed36dd766cc3941b9c2ec76dfdf5ae77cc6acf2 \ + --hash=sha256:578e5cefa5a293158342e3bbabc71e426ec29d5cdb41635789b55f4b48eb5f9e \ + --hash=sha256:5ad2bafeb2193981c5cc44490b4f329dd435b1c76c981844cd3549f031481bb2 \ + --hash=sha256:5dfbfa3b76052270fb5b021c172506b7c72db3cabb1b57f96490f49e14006822 \ + --hash=sha256:601965116dce06c1052ca0c89b69f0f1a49ad5b7d2da2b8352351c23296bd607 \ + --hash=sha256:6d3b8435c73a4502e57e757dd80eac73bbc0c9664b0149a9e31250d0711753c8 \ + --hash=sha256:7356b0c2cd1d0978b66c1e4b604a4a1221695b6396f4fd6f302a817ab0a09e59 \ + --hash=sha256:7852a0469cba37e777c01cf6601146a48f76c8407c33e4ee093c5502aa571685 \ + --hash=sha256:7973858cf8d8546322efca0ecf3728f55495641da48fd70fd9e8d47fb4fa4935 \ + --hash=sha256:7b6369c7dc98ed0889121f5013145d5dc621977d0f594a8db737b8986a7e14a5 \ + --hash=sha256:7bc84771e9f0d93c1bebf2ae8729bd967053ea44985758351a21fe628c9413ae \ + --hash=sha256:8789069253ef2e9091f1b3dbf364a658177de2711eac86626fa07851a37b7d4e \ + --hash=sha256:8b5a5f27ae48453ec2296f7a97600e965d4982bfcce772a66d12204672237848 \ + --hash=sha256:8b9542a2199016998600091ea4f7bf5239f368c7c965fd335abed87b2c4bd1de \ + --hash=sha256:8decf66c7027dfa06b6e6bb9cd0ea0f19ae1fe12e9fa5c7b691e5d237c6622fd \ + --hash=sha256:902f42b1f3c438278281266ba7f0c55202345287e8065e2eb98fabdcda8939eb \ + --hash=sha256:91dbd4d673d696075b4aa9506b0c94421756364edb46e38f63ab5a0fa33dd3e1 \ + --hash=sha256:9a5cad77635f77494b65b4869dcf0b6398dd0eececfe31a6868c8bf4a5b5bf28 \ + --hash=sha256:9a6896bf96592cadb3c268be95e47be8d2d093d3d9c3785fab77c13fc9b89143 \ + --hash=sha256:a4841ec07f0af94633e7f38b875cd472232c315cb58f2dc08f927e00084b159c \ + --hash=sha256:a95359d9dbe8057b4ed6ef51217540357303e2e558c61f1a2eaa641362648b7c \ + --hash=sha256:aa55764b18f139e126d3360e6227d88ca25cfd110573f6051bab1235308098b1 \ + --hash=sha256:aac94c4414be605e6a2e6b95e0d0a976f2c09a6397faac3da6b2f9b1a7207e08 \ + --hash=sha256:b121253ff03e89656cab6e8ceea51c08215f56bc58c5d9eea35d250a885300c5 \ + --hash=sha256:b44839487832addbc248998dcbef103237b03d2c59b5b10597139a784a8b69ca \ + --hash=sha256:b7ecfb0d63107fad49e56b9ccdb64e76c37ca96559509c8114ed68a27affa5bf \ + --hash=sha256:b8730e59b9e0439e4cd40d6ef47c61fb5b9d09185ea56e9a91573b1cb0e1ccf9 \ + --hash=sha256:bc721a2fc6511806b680b0cc618de9a29385c25125544065a0c11c11095629d6 \ + --hash=sha256:bdbef6d0b2e7ee36192a5e457ff0258d221ffd807bc1e10e9a25dae0dd06dcbf \ + --hash=sha256:be0d0543e767ad1341736cc6e279a4e27de7f7ccbd62ca0fd0b4a5bd293f12d2 \ + --hash=sha256:c07b1cdf065c8a62e209951d26d8ce21f14e3b998755f1422c6e529dfd940e3f \ + --hash=sha256:c27f11ee4a13960ea64f58b744ee134a50c160b83b8d3de24e7bb744f302fe0a \ + --hash=sha256:c33f3d7c817e42d6f2c39199fdf94d54593b935df02e177ad26a2b5dbd64fbe3 \ + --hash=sha256:c628536f8a1b29ade49668388afd5978da8fbbd678bfe960e297525d853c3707 \ + --hash=sha256:c7b8658329582e1dddee4f0a563ac53c6ed7262a52e1eebbd72810a443ea6a19 \ + --hash=sha256:cb75ab6af7c6e84717a718d208e8e59054c0eaa06cb290d8ec440a8b99d9fad4 \ + --hash=sha256:cc7da295b1692701ca190c0e99c190edaf10a69260b200a95edf220fe35c0ed2 \ + --hash=sha256:cf4ee1dd5eb44f15c221b7bebab5c9f823e3593f6e09c3adb6ad8cc4af5f1eb0 \ + --hash=sha256:d02f1c7901c81009a8f24f03d717d9ef1d2ba2012e1b912a9a4f142ca428a40a \ + --hash=sha256:d0a82542a44bc192139c4d4e50800adf6e91b6629dc15ddd43ab870fe71a3654 \ + --hash=sha256:d9a01908ee8820800bd20b4ff289654650f4058583d3a80e58bae6c293ce78ca \ + --hash=sha256:dc30b21a0b9c0f1af43c6c2c53c009bbf08580e904f1f108bf4d13650a7c7784 \ + --hash=sha256:dee0107bb4a80dcd040f037c86f7fc96b03a70e8cf3e003761904ecb85db0b40 \ + --hash=sha256:eea70c11aab935580afab3247e3a54c7eeceaf80d8452cf33e2a38015c2c139e \ + --hash=sha256:f0301a5b9c04b6ec7b642e22dd581b64addfc1c07e9188e10394c94816c9b059 \ + --hash=sha256:f060535e56a16029f7b3ef98da5a1a1e0816e76d9b4940c8b6237142d93401ed \ + --hash=sha256:f38a2fcdc318795f22de72155c9991114f3fa041c1cbbadd82297cb40ca81f1b \ + --hash=sha256:f7117613a8b62d32cda04be4ee31ef8482bfa9d2ba31107f878b38f9cd4e440d \ + --hash=sha256:f9315a9adfa59a1c0a9ae0d41e7fc6fdb826cd09a8e1aa3fd3e07a042fd75ffe # via -r requirements-opencode-review-ci.txt iniconfig==2.3.0 \ --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ @@ -209,9 +228,25 @@ interrogate==1.7.0 \ --hash=sha256:a320d6ec644dfd887cc58247a345054fc4d9f981100c45184470068f4b3719b0 \ --hash=sha256:b13ff4dd8403369670e2efe684066de9fcb868ad9d7f2b4095d8112142dc9d12 # via -r requirements-opencode-review-ci.txt -packaging==26.2 \ - --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ - --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 +maturin==1.15.0 \ + --hash=sha256:0ebf9767892725083138e671c34482c660317a2f3d6a29fc0e0f34e9d8c99136 \ + --hash=sha256:126e12e618b4db42f68c779a56d41f82a390145ba36ac3f621d057eb34f5ad9d \ + --hash=sha256:4f9d33e6c3f9615c8caceecbbbd440f8eb25a3ddeb687077682cd5eca2e9ae15 \ + --hash=sha256:552c2be4afd43fe8d5c9f3ec8d4c4756d973b8dcbe94c14084390301f50243e1 \ + --hash=sha256:653020a63525bb224e5ab0adf02e17a2e08bc86dbea7fc1399c9a56d7529b99e \ + --hash=sha256:6bf6dc62e22d4dcfd5a51244ff0d58975fa4979c48209fe84159617648956d82 \ + --hash=sha256:7ab7eebffd7b8debca2265985de4eaeb332141276d24b9560b5ad484d4b3add1 \ + --hash=sha256:7eb066372f541f8eb4909c79c5d9bd0b9e8125980bdf1ec9e8aba23c6c8d6c55 \ + --hash=sha256:94b26cc8e8aba61a5f2099715fe640e18c5f678e9a500408b38761263954228a \ + --hash=sha256:bf29beddd0c6708f112db51d5275fc28b28b9e9c9c5faae387eaef662918b176 \ + --hash=sha256:c40b4eae7bf5ef1f4b1af8d623fe4105016f93578fb15b764e741d08ec3b92dd \ + --hash=sha256:c7dc0c66c78d3debdd9c5aa807e861fbcbf07f3505d34b125df74c03986b0f48 \ + --hash=sha256:cd35772633f489841132bc8e71d6fc7f842df30b9c05cd5cdf1ee1ddcb744cc7 \ + --hash=sha256:da649988be98e87e009e51b1bf0d301b6a301bc0cecbdd60d40d8ba60748d1ca + # via -r requirements-opencode-review-ci.txt +packaging==26.3 \ + --hash=sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79 \ + --hash=sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c # via pytest pluggy==1.6.0 \ --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ @@ -223,9 +258,9 @@ py==1.11.0 \ --hash=sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719 \ --hash=sha256:607c53218732647dff4acdfcd50cb62615cedf612e72d1724fb1a0cc6405b378 # via interrogate -pygments==2.20.0 \ - --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ - --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 +pygments==2.21.0 \ + --hash=sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9 \ + --hash=sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c # via pytest pytest==9.1.1 \ --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ diff --git a/requirements-opencode-review-ci.txt b/requirements-opencode-review-ci.txt index 1e9a42f6a0..7af6909a32 100644 --- a/requirements-opencode-review-ci.txt +++ b/requirements-opencode-review-ci.txt @@ -1,4 +1,5 @@ coverage==7.15.4 +maturin>=1.10,<2.0 # hypothesis (MPL-2.0, permissive test tool) so the coverage-evidence sandbox can # run repos' always-on property tests (tests/fuzz/*) instead of ImportError-ing on # collection. Matches the >=6.100 floor used by consumer repos (e.g. contextual-orchestrator). diff --git a/scripts/ci/contextual_orchestrator_review_sidecar.sh b/scripts/ci/contextual_orchestrator_review_sidecar.sh index a96e854a51..8788a6e617 100755 --- a/scripts/ci/contextual_orchestrator_review_sidecar.sh +++ b/scripts/ci/contextual_orchestrator_review_sidecar.sh @@ -48,17 +48,26 @@ log() { printf '[contextual-orchestrator-sidecar] %s\n' "$*"; } fail() { log "error: $*" >&2; exit 1; } -# Require at least one of the five provider secrets so we never boot an empty -# (or mock) pool. Missing individual secrets are allowed — discovery skips the -# unregistered provider — matching the review gateway contract. +# Require the OpenRouter evidence credential plus at least one serving-provider +# credential for the mandatory-ZDR review pool. Missing other individual +# secrets are allowed — discovery skips that unregistered provider. provider_secret_count=0 for secret_name in BYTEZ_API_KEY NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB OPENROUTER_API_KEY OPENAI_API_KEY; do if [ -n "${!secret_name:-}" ]; then provider_secret_count=$((provider_secret_count + 1)) fi done -if [ "$provider_secret_count" -lt 1 ]; then - fail "at least one of BYTEZ_API_KEY / NVIDIA_NIM_API_KEY / NVIDIA_NIM_API_KEY_SUB / OPENROUTER_API_KEY / OPENAI_API_KEY is required" +serving_provider_secret_count=0 +for secret_name in BYTEZ_API_KEY NVIDIA_NIM_API_KEY NVIDIA_NIM_API_KEY_SUB OPENAI_API_KEY; do + if [ -n "${!secret_name:-}" ]; then + serving_provider_secret_count=$((serving_provider_secret_count + 1)) + fi +done +if [ -z "${OPENROUTER_API_KEY:-}" ]; then + fail "OPENROUTER_API_KEY is required for mandatory-ZDR evidence discovery" +fi +if [ "$serving_provider_secret_count" -lt 1 ]; then + fail "at least one non-OpenRouter serving provider credential is required for mandatory-ZDR review" fi log "provider secrets present: $provider_secret_count of 5" @@ -192,6 +201,31 @@ try: finally: connection.close() + def post_stream_payload(payload): + encoded = json.dumps(payload, ensure_ascii=False).encode("utf-8") + connection = http.client.HTTPConnection( + "127.0.0.1", server.server_address[1], timeout=5 + ) + try: + connection.request( + "POST", + "/v1/chat/completions", + body=encoded, + headers={ + "Authorization": "Bearer contract", + "Content-Type": "application/json", + "Content-Length": str(len(encoded)), + }, + ) + response = connection.getresponse() + return ( + response.status, + response.getheader("Content-Type", ""), + response.read().decode("utf-8"), + ) + finally: + connection.close() + large_status, large_body, encoded_size = post_payload({ "model": "openai/gpt-5", "messages": [{"role": "user", "content": "x" * accepted_size}], @@ -218,6 +252,23 @@ try: assert len(description) == description_length forwarded = client.proxy_payloads[-1]["tools"][0]["function"]["description"] assert forwarded.encode("utf-8") == description.encode("utf-8") + + stream_status, content_type, stream_body = post_stream_payload({ + "model": "openai/gpt-5", + "messages": [{"role": "user", "content": "tool stream probe"}], + "stream": True, + "stream_options": {"include_usage": True}, + "tools": [{ + "type": "function", + "function": { + "name": "scan_target", + "parameters": {"type": "object", "properties": {}}, + }, + }], + }) + assert stream_status == 200, stream_body + assert content_type.startswith("text/event-stream") + assert "usage_source" in stream_body finally: server.shutdown() server.server_close() @@ -256,7 +307,9 @@ publish_sidecar_evidence() { # Optional authoritative ZDR route feed. Failure is non-fatal: the policy falls # back to the dated static attestation table in scripts/ci/zdr_policy.py. -if curl -fsSL "https://openrouter.ai/api/v1/endpoints/zdr" -o "$zdr_feed" 2>/dev/null; then +if [ -n "${OPENROUTER_API_KEY:-}" ] && curl -fsSL \ + -H "Authorization: Bearer ${OPENROUTER_API_KEY}" \ + "https://openrouter.ai/api/v1/endpoints/zdr" -o "$zdr_feed" 2>/dev/null; then log "using live OpenRouter ZDR endpoint feed" zdr_args=(--zdr-endpoints "$zdr_feed") else @@ -264,13 +317,13 @@ else zdr_args=() fi -case "${CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR:-false}" in +case "${CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR:-}" in true) privacy_args=(--require-zdr) - log "private/internal target: requiring attested ZDR routes" + log "requiring attested ZDR routes for every central review target" ;; false|"") - privacy_args=() + fail "central review sidecar requires CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR=true" ;; *) fail "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR must be true or false" diff --git a/scripts/ci/install_base_python_locks.py b/scripts/ci/install_base_python_locks.py index 1b9ab10693..cb4bf12ac9 100644 --- a/scripts/ci/install_base_python_locks.py +++ b/scripts/ci/install_base_python_locks.py @@ -14,6 +14,7 @@ from __future__ import annotations import argparse +import hashlib import json import pathlib import re @@ -26,6 +27,8 @@ GENERATED_LOCK_RE = re.compile(r"^requirements-[0-9]{3}\.txt$") +GENERATED_ARCHIVE_RE = re.compile(r"^archives/archive-[0-9]{3}\.(?:tar\.gz|zip)$") +SHA256_RE = re.compile(r"^[0-9a-f]{64}$") DEFERABLE_PREFLIGHT_FAILURES = ( re.compile( r"In --require-hashes mode, all requirements must have their versions " @@ -57,6 +60,71 @@ def source_directory(self) -> str: return "" if parent == "." else parent +@dataclass(frozen=True) +class ArchiveCandidate: + """One materialized archive source with a verified content digest.""" + + package: str + file: pathlib.Path + hashes: tuple[str, ...] + marker: str | None = None + + +def _archive_entries( + requirements_root: pathlib.Path, +) -> list[ArchiveCandidate]: + """Load and validate the archive files materialized from the base lock.""" + manifest_path = requirements_root.resolve() / "archive-manifest.json" + if not manifest_path.exists(): + return [] + if not manifest_path.is_file() or manifest_path.is_symlink(): + raise ValueError("base Python archive manifest must be a regular non-symlink file") + try: + manifest: Any = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ValueError(f"base Python archive manifest is invalid: {exc}") from exc + if not isinstance(manifest, list): + raise ValueError("base Python archive manifest must be a JSON array") + + entries: list[ArchiveCandidate] = [] + seen_files: set[str] = set() + for entry in manifest: + if not isinstance(entry, dict): + raise ValueError("base Python archive manifest entries must be objects") + package = entry.get("package") + relative_file = entry.get("file") + hashes = entry.get("hashes") + marker = entry.get("marker") + if ( + not isinstance(package, str) + or not package + or not isinstance(relative_file, str) + or GENERATED_ARCHIVE_RE.fullmatch(relative_file) is None + or not isinstance(hashes, list) + or not hashes + or any(not isinstance(value, str) or not SHA256_RE.fullmatch(value) for value in hashes) + or ( + marker is not None + and (not isinstance(marker, str) or not marker.strip()) + ) + ): + raise ValueError("base Python archive manifest contains an invalid entry") + if relative_file in seen_files: + raise ValueError("base Python archive manifest contains duplicate files") + seen_files.add(relative_file) + archive = requirements_root.resolve() / relative_file + if not archive.is_file() or archive.is_symlink(): + raise ValueError(f"materialized base Python archive {relative_file} must be a regular file") + digest = hashlib.sha256() + with archive.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + if digest.hexdigest() not in hashes: + raise ValueError(f"materialized base Python archive {relative_file} failed hash verification") + entries.append(ArchiveCandidate(package, archive, tuple(hashes), marker)) + return entries + + def _manifest_entries( requirements_root: pathlib.Path, ) -> list[LockCandidate]: @@ -129,6 +197,24 @@ def _pip_command(requirements: Sequence[pathlib.Path], *, preflight: bool) -> li return command +def _archive_pip_command(archive: ArchiveCandidate) -> list[str]: + """Build a no-network source-install command for one verified archive.""" + requirement = str(archive.file) + if archive.marker is not None: + requirement = f"{archive.package} @ {archive.file.as_uri()} ; {archive.marker}" + return [ + sys.executable, + "-m", + "pip", + "install", + "--break-system-packages", + "--disable-pip-version-check", + "--no-deps", + "--no-build-isolation", + requirement, + ] + + def _bounded_failure_output(output: str, *, maximum_lines: int = 120) -> str: """Keep the dependency root cause visible without flooding Actions logs.""" lines = output.rstrip().splitlines() @@ -183,17 +269,40 @@ def _report_fatal_preflight_failure( def install_materialized_locks( requirements_root: pathlib.Path, *, + install_archives: bool = True, + archives_only: bool = False, runner: Runner = subprocess.run, stdout: TextIO = sys.stdout, stderr: TextIO = sys.stderr, ) -> int: """Preflight and install independent base lock closures.""" try: - entries = _manifest_entries(requirements_root) + archive_entries = _archive_entries(requirements_root) + entries = [] if archives_only else _manifest_entries(requirements_root) except (OSError, ValueError) as exc: - print(f"::error::Could not validate base Python locks: {exc}", file=stderr) + print(f"::error::Could not validate base Python lock inputs: {exc}", file=stderr) return 2 + if archives_only: + for archive in archive_entries: + print( + f"Installing verified trusted base Python archive {archive.package}.", + file=stdout, + flush=True, + ) + installation = runner(_archive_pip_command(archive), check=False) + if installation.returncode != 0: + print( + f"::error::Verified trusted base Python archive failed to install: {archive.package}.", + file=stderr, + ) + return installation.returncode or 1 + print( + f"Trusted base Python archive installation summary: installed={len(archive_entries)}.", + file=stdout, + ) + return 0 + installed = 0 skipped = 0 preflight_results: dict[str, subprocess.CompletedProcess[str]] = {} @@ -312,6 +421,21 @@ def install_materialized_locks( return installation.returncode or 1 installed += len(plan) + if install_archives: + for archive in archive_entries: + print( + f"Installing verified trusted base Python archive {archive.package}.", + file=stdout, + flush=True, + ) + installation = runner(_archive_pip_command(archive), check=False) + if installation.returncode != 0: + print( + f"::error::Verified trusted base Python archive failed to install: {archive.package}.", + file=stderr, + ) + return installation.returncode or 1 + print( "Trusted base Python lock installation summary: " f"candidates={len(entries)} installed={installed} skipped={skipped}.", @@ -324,8 +448,14 @@ def main(argv: Sequence[str] | None = None) -> int: """Install materialized lock candidates supplied by the trusted workflow.""" parser = argparse.ArgumentParser() parser.add_argument("--requirements-root", required=True, type=pathlib.Path) + parser.add_argument("--no-archives", action="store_true") + parser.add_argument("--archives-only", action="store_true") args = parser.parse_args(argv) - return install_materialized_locks(args.requirements_root) + return install_materialized_locks( + args.requirements_root, + install_archives=not args.no_archives, + archives_only=args.archives_only, + ) if __name__ == "__main__": # pragma: no cover diff --git a/scripts/ci/load_contextual_orchestrator_token.sh b/scripts/ci/load_contextual_orchestrator_token.sh index a30b182c20..558a9de9eb 100755 --- a/scripts/ci/load_contextual_orchestrator_token.sh +++ b/scripts/ci/load_contextual_orchestrator_token.sh @@ -11,15 +11,28 @@ _contextual_orchestrator_token_fail() { _contextual_orchestrator_stat() { local format="$1" target="$2" value + # Probe the exact GNU/BusyBox operation instead of the implementation's + # version flag. BusyBox does not need the BSD fallback when its -c form is + # available, while macOS/BSD stat rejects -c and reaches the BSD form. if value="$(stat -c "$format" -- "$target" 2>/dev/null)"; then printf '%s\n' "$value" return 0 fi if [ "$format" = "%a" ]; then - stat -f '%OMp %OLp' "$target" + if value="$(stat -f '%Mp %Lp' "$target" 2>/dev/null)"; then + printf '%s\n' "$value" + return 0 + fi + if value="$(stat -f '%OMp %OLp' "$target" 2>/dev/null)"; then + printf '%s\n' "$value" + return 0 + fi + elif value="$(stat -f "$format" "$target" 2>/dev/null)"; then + printf '%s\n' "$value" return 0 fi - stat -f "$format" "$target" + _contextual_orchestrator_token_fail "stat cannot report the token file owner or mode." + return 1 } _contextual_orchestrator_load_token() { diff --git a/scripts/ci/materialize_base_python_requirements.py b/scripts/ci/materialize_base_python_requirements.py index a052123547..e16aede453 100755 --- a/scripts/ci/materialize_base_python_requirements.py +++ b/scripts/ci/materialize_base_python_requirements.py @@ -4,6 +4,7 @@ from __future__ import annotations import argparse +import ast import atexit import fnmatch import functools @@ -19,6 +20,7 @@ import sys import tarfile import tempfile +import urllib.error import urllib.parse import urllib.request from typing import Any @@ -43,6 +45,16 @@ r"(?P[A-Za-z0-9_.-]{1,100})\.git@" r"(?P[0-9a-fA-F]{40})" ) +UV_EXACT_ORG_ARCHIVE_RE = re.compile( + r"(?P[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?" + r"(?:\[[A-Za-z0-9._-]+(?:,[A-Za-z0-9._-]+)*\])?)\s+@\s+" + r"(?Phttps://github\.com/ContextualWisdomLab/" + r"[A-Za-z0-9_.-]{1,100}/archive/" + r"(?:refs/(?:tags|heads)/)?" + r"[A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9._-]+)*" + r"\.(?:tar\.gz|zip)" + r")(?:\s*;\s*(?P\S(?:.*\S)?))?" +) UV_EXPORT_TIMEOUT_SECONDS = 120 TRUSTED_UV_VERSION = "0.12.1" TRUSTED_UV_TARGET_TRIPLE = "x86_64-unknown-linux-gnu" @@ -70,6 +82,9 @@ TRUSTED_UV_ORIGIN_ERROR = ( "trusted uv archive redirected outside the fixed GitHub release HTTPS origin" ) +TRUSTED_ORG_ARCHIVE_HOSTS = frozenset({"github.com", "codeload.github.com"}) +TRUSTED_ORG_ARCHIVE_MAX_BYTES = 256 * 1024 * 1024 +TRUSTED_ORG_ARCHIVE_TIMEOUT_SECONDS = 120 def _https_default_port(parsed: urllib.parse.ParseResult) -> bool: @@ -110,6 +125,120 @@ def _is_trusted_uv_final_origin(url: str) -> bool: return _is_trusted_uv_https_host(url, TRUSTED_UV_FINAL_HOSTS) +def _is_trusted_org_archive_url(url: str) -> bool: + """Return whether one archive URL stays on GitHub's HTTPS origins.""" + parsed = urllib.parse.urlparse(url) + return ( + parsed.scheme == "https" + and parsed.hostname in TRUSTED_ORG_ARCHIVE_HOSTS + and parsed.username is None + and parsed.password is None + and _https_default_port(parsed) + ) + + +_GITHUB_COM_ARCHIVE_PATH_RE = re.compile(r"^/(?P[^/]+)/(?P[^/]+)/archive/") +_CODELOAD_ARCHIVE_PATH_RE = re.compile( + r"^/(?P[^/]+)/(?P[^/]+)/(?:tar\.gz|zip|legacy\.tar\.gz|legacy\.zip)(?:/|$)" +) + + +def _org_archive_repository(url: str) -> tuple[str, str] | None: + """Return the ``(owner, repository)`` one organization archive URL names. + + ``github.com`` archive links and their ``codeload.github.com`` redirect + target use different path shapes for the exact same repository + (``/{owner}/{repo}/archive/...`` versus + ``/{owner}/{repo}/{tar.gz|zip}[/...]``). Parsing both here lets a redirect + be proven to stay on the exact same repository rather than merely the + same allowlisted host, so ``codeload.github.com/other-org/other-repo`` + cannot be reached through a host-only allowlist. Returns ``None`` when the + URL is not one of the two known trusted archive path shapes. + """ + parsed = urllib.parse.urlparse(url) + if parsed.hostname == "github.com": + match = _GITHUB_COM_ARCHIVE_PATH_RE.match(parsed.path) + elif parsed.hostname == "codeload.github.com": + match = _CODELOAD_ARCHIVE_PATH_RE.match(parsed.path) + else: + return None + if match is None: + return None + return (match.group("owner").casefold(), match.group("repo").casefold()) + + +def _is_same_org_archive_repository(source_url: str, target_url: str) -> bool: + """Return whether two trusted archive URLs name the identical repository.""" + source_repository = _org_archive_repository(source_url) + target_repository = _org_archive_repository(target_url) + return source_repository is not None and source_repository == target_repository + + +class _TrustedOrgArchiveRedirects(urllib.request.HTTPRedirectHandler): + """Follow GitHub archive redirects only onto GitHub's code-download host.""" + + def redirect_request( + self, + request: urllib.request.Request, + response: Any, + code: int, + message: str, + headers: Any, + new_url: str, + ) -> urllib.request.Request: + """Reject archive redirects that leave the fixed GitHub origins or repository.""" + if ( + not _is_trusted_org_archive_url(request.full_url) + or not _is_trusted_org_archive_url(new_url) + or not _is_same_org_archive_repository(request.full_url, new_url) + ): + raise RuntimeError("trusted organization archive redirected outside GitHub") + followed = super().redirect_request( + request, + response, + code, + message, + headers, + new_url, + ) + if followed is None: + raise RuntimeError("trusted organization archive redirect was rejected") + return followed + + +def _download_trusted_org_archive(url: str, hashes: list[str]) -> bytes: + """Download and verify one exact organization archive before image build.""" + if not _is_trusted_org_archive_url(url): + raise RuntimeError("trusted organization archive URL is not GitHub HTTPS") + opener = urllib.request.build_opener( + urllib.request.ProxyHandler({}), + _TrustedOrgArchiveRedirects(), + ) + try: + with opener.open(url, timeout=TRUSTED_ORG_ARCHIVE_TIMEOUT_SECONDS) as response: + final_url = response.geturl() + if not _is_trusted_org_archive_url(final_url) or not _is_same_org_archive_repository( + url, final_url + ): + raise RuntimeError("trusted organization archive left GitHub origins") + payload = bytearray() + while len(payload) <= TRUSTED_ORG_ARCHIVE_MAX_BYTES: + chunk = response.read(TRUSTED_ORG_ARCHIVE_MAX_BYTES + 1 - len(payload)) + if not chunk: + break + payload.extend(chunk) + except (OSError, urllib.error.URLError) as exc: + raise RuntimeError( + f"trusted organization archive download failed: {type(exc).__name__}" + ) from exc + if len(payload) > TRUSTED_ORG_ARCHIVE_MAX_BYTES: + raise RuntimeError("trusted organization archive exceeded the bounded size") + digest = hashlib.sha256(payload).hexdigest() + if digest not in hashes: + raise RuntimeError("trusted organization archive checksum verification failed") + return bytes(payload) + + class _TrustedUvReleaseAssetRedirects(urllib.request.HTTPRedirectHandler): """Follow one GitHub Releases hop onto the official asset CDN only.""" @@ -250,7 +379,7 @@ def _is_hash_pinned(content: bytes) -> bool: if not requirement_lines: return False return all( - _is_fully_hash_pinned_requirement(line) + _is_registry_hash_pinned_requirement(line) or _is_bounded_requirement_include(line) for line in requirement_lines ) @@ -261,21 +390,36 @@ def _is_flat_materializable_lock(content: bytes) -> bool: Selected sources are renamed to generated flat files. Relative ``-r`` and ``--requirement`` edges therefore lose the source directory that gives them - meaning. Only independent exact package pins cross this publication boundary + meaning. Only independent exact package pins or hash-pinned organization + archive URLs cross this publication boundary until a complete immutable include graph can be reconstructed and rewritten. """ lines = _requirement_lines(content) requirement_lines = [line for line in lines if line != "--require-hashes"] return bool(requirement_lines) and all( - _is_fully_hash_pinned_requirement(line) for line in requirement_lines + _is_registry_hash_pinned_requirement(line) for line in requirement_lines + ) + + +def _is_registry_hash_pinned_requirement(line: str) -> bool: + """Return whether one registry requirement is an exact SHA-256 pin.""" + fields = re.split(r"\s+(?=--hash=)", line) + if len(fields) < 2: + return False + requirement, *hashes = fields + return UV_EXACT_REQUIREMENT_RE.fullmatch(requirement) is not None and all( + UV_SHA256_HASH_RE.fullmatch(hash_value) for hash_value in hashes ) def _is_fully_hash_pinned_requirement(line: str) -> bool: - """Return whether one uv-export line is an exact package pin with SHA-256 hashes.""" + """Return whether one uv-export line is an exact hash-pinned package or archive.""" fields = re.split(r"\s+(?=--hash=)", line) if len(fields) < 2: return False requirement, *hashes = fields - if UV_EXACT_REQUIREMENT_RE.fullmatch(requirement) is None: + if ( + UV_EXACT_REQUIREMENT_RE.fullmatch(requirement) is None + and UV_EXACT_ORG_ARCHIVE_RE.fullmatch(requirement) is None + ): return False return all(UV_SHA256_HASH_RE.fullmatch(hash_value) for hash_value in hashes) @@ -285,19 +429,62 @@ def _is_fully_hash_pinned_export(content: bytes) -> bool: The fixed exporter invocation does not request index, find-links, binary, or global hash directives. Every non-comment logical line must therefore be one - normalized package ``==`` pin with at least one complete SHA-256 hash. Option - lines, local/direct references, other algorithms, and truncated hashes are - rejected even when they contain a ``--hash=`` substring. + normalized package ``==`` pin or a hash-pinned archive URL from the trusted + organization, each with at least one complete SHA-256 hash. Option lines, + local/direct references, other origins, other algorithms, and truncated + hashes are rejected even when they contain a ``--hash=`` substring. """ lines = _requirement_lines(content) return bool(lines) and all(_is_fully_hash_pinned_requirement(line) for line in lines) -def _partition_uv_export(content: bytes) -> tuple[bytes, list[dict[str, str]]]: - """Separate registry hash pins from exact organization VCS source pins.""" +def _archive_from_uv_line(line: str) -> dict[str, object] | None: + """Return one validated organization archive descriptor from an export line.""" + fields = re.split(r"\s+(?=--hash=)", line) + if len(fields) < 2: + return None + requirement, *hash_fields = fields + match = UV_EXACT_ORG_ARCHIVE_RE.fullmatch(requirement) + if match is None: + return None + hashes = [field.removeprefix("--hash=sha256:").lower() for field in hash_fields] + if not hashes or any(not re.fullmatch(r"[0-9a-fA-F]{64}", value) for value in hashes): + raise ValueError("organization archive must carry complete SHA-256 hashes") + descriptor: dict[str, object] = { + "package": match.group("package"), + "url": match.group("url"), + "hashes": hashes, + } + marker = match.group("marker") + if marker is not None: + descriptor["marker"] = marker + return descriptor + + +def _archive_identity(archive: dict[str, Any]) -> tuple[str, str, str | None]: + """Return the dependency identity used to deduplicate archive entries.""" + return str(archive["package"]), str(archive["url"]), archive.get("marker") + + +def _partition_uv_export( + content: bytes, +) -> tuple[bytes, list[dict[str, str]], list[dict[str, object]]]: + """Separate registry pins, VCS sources, and verified archive sources.""" registry_requirements: list[str] = [] vcs_by_repository: dict[str, dict[str, str]] = {} + archive_hashes_by_url: dict[str, object] = {} + archives_by_identity: dict[tuple[str, str, str | None], dict[str, object]] = {} for line in _requirement_lines(content): + archive = _archive_from_uv_line(line) + if archive is not None: + url = str(archive["url"]) + previous_hashes = archive_hashes_by_url.get(url) + if previous_hashes is not None and previous_hashes != archive["hashes"]: + raise ValueError("uv export pins one archive URL to conflicting hashes") + archive_hashes_by_url[url] = archive["hashes"] + identity = _archive_identity(archive) + archives_by_identity[identity] = archive + continue if _is_fully_hash_pinned_requirement(line): registry_requirements.append(line) continue @@ -323,9 +510,19 @@ def _partition_uv_export(content: bytes) -> tuple[bytes, list[dict[str, str]]]: if registry_requirements else b"" ) - return registry_content, sorted( - vcs_by_repository.values(), - key=lambda dependency: dependency["repository"].casefold(), + return ( + registry_content, + sorted( + vcs_by_repository.values(), + key=lambda dependency: dependency["repository"].casefold(), + ), + sorted( + archives_by_identity.values(), + key=lambda dependency: ( + str(dependency["url"]), + str(dependency.get("marker", "")), + ), + ), ) @@ -535,7 +732,7 @@ def _reject_unsupported_uv_workspace( def _export_uv_lock( repo_root: pathlib.Path, base_sha: str, lock_path: str -) -> tuple[bytes, list[dict[str, str]]] | None: +) -> tuple[bytes, list[dict[str, str]], list[dict[str, object]]] | None: """Export one tracked base ``uv.lock`` into a trusted hash-pinned closure. The caller proves that the sibling ``pyproject.toml`` is a regular blob in @@ -617,8 +814,8 @@ def _regular_base_blob_paths(entries: bytes) -> list[tuple[str, pathlib.PurePosi def _base_python_inputs( repo_root: pathlib.Path, base_sha: str -) -> tuple[list[tuple[str, bytes]], list[dict[str, str]]]: - """Return hash locks and exact VCS sources from one validated base commit.""" +) -> tuple[list[tuple[str, bytes]], list[dict[str, str]], list[dict[str, object]]]: + """Return locks, exact VCS sources, and verified archive sources.""" if not SHA_RE.fullmatch(base_sha): raise ValueError("base SHA must be exactly 40 hexadecimal characters") @@ -627,6 +824,8 @@ def _base_python_inputs( regular_paths = {path for path, _candidate in regular_blobs} locks: list[tuple[str, bytes]] = [] vcs_by_repository: dict[str, dict[str, str]] = {} + archive_hashes_by_url: dict[str, object] = {} + archives_by_identity: dict[tuple[str, str, str | None], dict[str, object]] = {} for path, candidate in regular_blobs: if _is_candidate_lock_path(candidate): content = _git(repo_root, "show", f"{base_sha}:{path}") @@ -637,7 +836,7 @@ def _base_python_inputs( continue exported = _export_uv_lock(repo_root, base_sha, path) if exported is not None: - registry_content, vcs_dependencies = exported + registry_content, vcs_dependencies, archive_dependencies = exported if registry_content: locks.append((path, registry_content)) for dependency in vcs_dependencies: @@ -653,12 +852,32 @@ def _base_python_inputs( "to conflicting commits" ) vcs_by_repository[repository_key] = dependency + for archive in archive_dependencies: + url = str(archive["url"]) + previous_hashes = archive_hashes_by_url.get(url) + if previous_hashes is not None and previous_hashes != archive["hashes"]: + raise RuntimeError( + "base uv locks pin one archive URL to conflicting hashes" + ) + archive_hashes_by_url[url] = archive["hashes"] + identity = _archive_identity(archive) + archives_by_identity[identity] = { + **archive, + "source": path, + } return ( sorted(locks, key=lambda item: item[0]), sorted( vcs_by_repository.values(), key=lambda dependency: dependency["repository"].casefold(), ), + sorted( + archives_by_identity.values(), + key=lambda dependency: ( + str(dependency["url"]), + str(dependency.get("marker", "")), + ), + ), ) @@ -721,12 +940,187 @@ def _rewrite_materialized_includes( return "".join(rewritten).encode("utf-8") +_SUPPORTED_MARKER_VARIABLES = frozenset({"python_version", "python_full_version"}) + +# ``target_python_version`` is a plain "major.minor" string (e.g. "3.14") -- +# the coverage workflow genuinely cannot know the exact patch release of its +# pinned interpreter ahead of time. That value is a confident, exact stand-in +# for the ``python_version`` marker variable (which is itself major.minor), +# but it is NOT a confident stand-in for ``python_full_version`` (patch- +# sensitive): padding "3.14" into "3.14.0" would silently assume the patch +# component is zero, which is not something the target string actually says. +# Only variables in this set may have their missing components zero-padded +# for a comparison against an equal-or-shorter target; a ``python_full_version`` +# comparison against a major.minor-only target is trusted only when the +# literal's own major.minor already falls outside the target's series (see +# ``_python_full_version_comparison_is_ambiguous``) -- otherwise it needs the +# target string to carry patch precision (3+ components) itself. +_PATCH_INSENSITIVE_MARKER_VARIABLES = frozenset({"python_version"}) + + +class _UnsupportedMarkerError(Exception): + """Raised when a marker shape is outside the narrow supported subset.""" + + +def _marker_version_tuple(value: str) -> tuple[int, ...]: + """Parse a dotted numeric version literal into a comparable integer tuple.""" + if not re.fullmatch(r"[0-9]+(?:\.[0-9]+)*", value): + raise _UnsupportedMarkerError(f"unsupported marker version literal: {value!r}") + return tuple(int(part) for part in value.split(".")) + + +def _compare_marker_versions(left: str, operator: type, right: str) -> bool: + """Compare two dotted version literals as zero-padded integer tuples.""" + left_tuple = _marker_version_tuple(left) + right_tuple = _marker_version_tuple(right) + length = max(len(left_tuple), len(right_tuple)) + left_padded = left_tuple + (0,) * (length - len(left_tuple)) + right_padded = right_tuple + (0,) * (length - len(right_tuple)) + if operator is ast.Eq: + return left_padded == right_padded + if operator is ast.NotEq: + return left_padded != right_padded + if operator is ast.Lt: + return left_padded < right_padded + if operator is ast.LtE: + return left_padded <= right_padded + if operator is ast.Gt: + return left_padded > right_padded + if operator is ast.GtE: + return left_padded >= right_padded + raise _UnsupportedMarkerError("unsupported marker comparison operator") + + +def _python_full_version_comparison_is_ambiguous( + target_python_version: str, literal: str +) -> bool: + """Return whether a ``python_full_version`` comparison needs the unknown patch. + + ``target_python_version`` is only known to major.minor precision; the real + interpreter's trailing components (patch, and beyond) are genuinely + unknown. ``literal`` is a fully specified PEP 440 version constant, so + treating any of *its* missing trailing components as zero is exact, not a + guess -- that is standard version-comparison normalization, not an + assumption about the target. + + Comparing the two, element by element, up to the target's known length: + if a real difference already shows up within that shared, fully-known + prefix, the comparison is decided right there and no later component -- + real or unknown -- can change it, because ordinary tuple/lexicographic + comparison stops at the first differing position. That covers both kinds + of confidently-decidable case: a different minor (``3.9`` vs. ``3.14``) + and a minor entirely outside the target's series (``3.0`` or ``4.0`` vs. + ``3.14``). + + The comparison is ambiguous only when the literal's prefix, truncated (or + zero-padded, if shorter) to the target's known length, exactly equals the + target's known prefix -- i.e. the literal shares the target's major.minor + series and therefore the outcome hinges on the interpreter's real, + not-yet-known trailing components. That is the only case left to fail + open, regardless of which operator is being evaluated (``==``, ``!=``, + ``<``, ``<=``, ``>``, ``>=`` all depend on the unknown patch digit once + the known prefix already matches). + """ + target_tuple = _marker_version_tuple(target_python_version) + literal_tuple = _marker_version_tuple(literal) + known_length = len(target_tuple) + literal_prefix = literal_tuple[:known_length] + literal_prefix = literal_prefix + (0,) * (known_length - len(literal_prefix)) + return literal_prefix == target_tuple + + +def _evaluate_marker_node(node: ast.AST, target_python_version: str) -> bool: + """Evaluate one parsed marker expression node against a fixed Python version. + + Only boolean combinations (``and``/``or``/``not``/parentheses) of + ``python_version``/``python_full_version`` comparisons against a literal + dotted version string are understood -- the shape ``uv export`` emits for + Python-version-gated organization archive sources. Any other marker shape + (``sys_platform``, ``extra``, ``in``/``not in``, function calls, chained + comparisons, and so on) raises ``_UnsupportedMarkerError`` so the caller + fails open and still downloads the archive. A ``python_full_version`` + comparison is patch-sensitive: given only a major.minor + ``target_python_version``, it resolves confidently when the literal's + major.minor differs from the target's (the outcome cannot depend on the + target's unknown patch digit in that case -- see + ``_python_full_version_comparison_is_ambiguous``), and otherwise raises + ``_UnsupportedMarkerError`` rather than assume the missing patch is + ``.0``. + """ + if isinstance(node, ast.Expression): + return _evaluate_marker_node(node.body, target_python_version) + if isinstance(node, ast.BoolOp): + values = [_evaluate_marker_node(value, target_python_version) for value in node.values] + # ast.BoolOp.op is exhaustively either And or Or in Python's grammar; + # there is no third case to fail open on. + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not _evaluate_marker_node(node.operand, target_python_version) + if isinstance(node, ast.Compare) and len(node.ops) == 1 and len(node.comparators) == 1: + left, right = node.left, node.comparators[0] + variable_node, literal_node = (left, right) if isinstance(left, ast.Name) else (right, left) + if not isinstance(variable_node, ast.Name) or variable_node.id not in _SUPPORTED_MARKER_VARIABLES: + raise _UnsupportedMarkerError("unsupported marker variable") + if not isinstance(literal_node, ast.Constant) or not isinstance(literal_node.value, str): + raise _UnsupportedMarkerError("unsupported marker literal") + if ( + variable_node.id not in _PATCH_INSENSITIVE_MARKER_VARIABLES + and len(_marker_version_tuple(target_python_version)) < 3 + and _python_full_version_comparison_is_ambiguous( + target_python_version, literal_node.value + ) + ): + # ``python_full_version`` is patch-sensitive, and the target is + # only major.minor -- but that only makes a comparison unknown + # when the literal shares the target's major.minor series (see + # ``_python_full_version_comparison_is_ambiguous``). A literal in + # a different minor series is decidable for every possible patch + # and falls through to the ordinary comparison below instead. + # Fail open rather than assume the missing patch is ".0" (see the + # module docstring above). + raise _UnsupportedMarkerError( + "python_full_version comparison requires a patch-precise target version" + ) + operator = type(node.ops[0]) + if variable_node is left: + return _compare_marker_versions(target_python_version, operator, literal_node.value) + return _compare_marker_versions(literal_node.value, operator, target_python_version) + raise _UnsupportedMarkerError("unsupported marker expression shape") + + +def _marker_excludes_target_python(marker: str, target_python_version: str) -> bool: + """Return whether ``marker`` can be proven false for one fixed Python version. + + This only ever removes redundant network work: an archive this evaluator + cannot confidently rule out (an unsupported marker shape, or a parse + failure) is treated as included, exactly like today's unconditional + download, so it can never wrongly drop a dependency the target Python + version actually needs. + """ + try: + tree = ast.parse(marker, mode="eval") + return not _evaluate_marker_node(tree, target_python_version) + except (SyntaxError, _UnsupportedMarkerError, RecursionError, ValueError): + return False + + def materialize( repo_root: pathlib.Path, base_sha: str, output_dir: pathlib.Path, + *, + target_python_version: str | None = None, ) -> list[dict[str, str]]: - """Write base locks and resolvable bounded includes into a safe context.""" + """Write base locks and resolvable bounded includes into a safe context. + + ``target_python_version`` (a plain ``"major.minor"`` string such as + ``"3.14"``, matching the coverage image's pinned interpreter) lets an + archive whose ``marker`` field can be proven false for that version skip + the network download entirely -- the coverage image would never install + it anyway. Leave it ``None`` to download every archive unconditionally, + as before. This is purely an optimization: an archive that this cannot + confidently exclude is still downloaded and verified exactly as today. + """ if output_dir.exists() and output_dir.is_symlink(): raise ValueError("output directory must not be a symlink") output_dir.mkdir(parents=True, exist_ok=True) @@ -736,7 +1130,7 @@ def materialize( regular_paths = { path for path, _candidate in _regular_base_blob_paths(entries) } - locks, vcs_manifest = _base_python_inputs(resolved_repo, base_sha) + locks, vcs_manifest, archive_sources = _base_python_inputs(resolved_repo, base_sha) manifest: list[dict[str, str]] = [] for index, (source_path, content) in enumerate(locks): generated_name = f"requirements-{index:03d}.txt" @@ -770,6 +1164,33 @@ def materialize( json.dumps(vcs_manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8", ) + archive_directory = output_dir / "archives" + archive_manifest: list[dict[str, object]] = [] + for index, archive in enumerate(archive_sources): + marker = archive.get("marker") + if ( + target_python_version is not None + and marker is not None + and _marker_excludes_target_python(str(marker), target_python_version) + ): + # This archive's own marker rules it out for the coverage image's + # pinned interpreter; the download would never be installed, so it + # is dropped entirely rather than recorded as an unmaterialized + # manifest entry (see _archive_entries in install_base_python_locks.py, + # which requires every manifest entry to have a real file on disk). + continue + url = str(archive["url"]) + suffix = ".tar.gz" if url.endswith(".tar.gz") else ".zip" + archive_file = f"archive-{index:03d}{suffix}" + archive_directory.mkdir(parents=True, exist_ok=True) + (archive_directory / archive_file).write_bytes( + _download_trusted_org_archive(url, list(archive["hashes"])) + ) + archive_manifest.append({**archive, "file": f"archives/{archive_file}"}) + (output_dir / "archive-manifest.json").write_text( + json.dumps(archive_manifest, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) return manifest @@ -779,10 +1200,25 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--repo-root", required=True, type=pathlib.Path) parser.add_argument("--base-sha", required=True) parser.add_argument("--output-dir", required=True, type=pathlib.Path) + parser.add_argument( + "--target-python-version", + default=None, + help=( + "major.minor Python version of the coverage image (e.g. 3.14), " + "used only to skip downloading an organization archive whose " + "marker already excludes it. Omit to download every archive " + "unconditionally." + ), + ) args = parser.parse_args(argv) try: - manifest = materialize(args.repo_root, args.base_sha, args.output_dir) + manifest = materialize( + args.repo_root, + args.base_sha, + args.output_dir, + target_python_version=args.target_python_version, + ) except (OSError, RuntimeError, ValueError) as exc: print( f"::error::Could not materialize base Python locks: {exc}", file=sys.stderr diff --git a/scripts/ci/opencode_coverage_identity.py b/scripts/ci/opencode_coverage_identity.py index 0bf439e746..52db3c5bb3 100644 --- a/scripts/ci/opencode_coverage_identity.py +++ b/scripts/ci/opencode_coverage_identity.py @@ -19,7 +19,9 @@ CANONICAL_WORKFLOW_NAMES = frozenset({"Required OpenCode Review"}) DISPATCH_WORKFLOW_NAME = "OpenCode Review Dispatch" SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") -REPO_RE = re.compile(r"^[A-Za-z0-9_][A-Za-z0-9_.-]*/[A-Za-z0-9_.-]+$") +REPO_RE = re.compile( + r"^[A-Za-z0-9_][A-Za-z0-9_.-]*/(?!\.{1,2}$)[A-Za-z0-9_.][A-Za-z0-9_.-]*$" +) TERMINAL_RESULTS = frozenset( {"success", "failure", "cancelled", "skipped", "neutral", "timed_out", "action_required"} ) diff --git a/scripts/ci/pr_auto_rebase.py b/scripts/ci/pr_auto_rebase.py index c0f04c3aa2..c62c70ebf3 100755 --- a/scripts/ci/pr_auto_rebase.py +++ b/scripts/ci/pr_auto_rebase.py @@ -80,7 +80,9 @@ ) -REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +REPO_RE = re.compile( + r"^[A-Za-z0-9_][A-Za-z0-9_.-]*/(?!\.{1,2}$)[A-Za-z0-9_.][A-Za-z0-9_.-]*$" +) OPEN_PRS_PAGE_SIZE = 25 LABELS_PAGE_SIZE = 50 DEFAULT_MAX_PER_RUN = 10 diff --git a/scripts/ci/pr_review_autofix_context.py b/scripts/ci/pr_review_autofix_context.py index cc7b6fb003..a32eb43779 100755 --- a/scripts/ci/pr_review_autofix_context.py +++ b/scripts/ci/pr_review_autofix_context.py @@ -19,7 +19,9 @@ from scripts.ci.pr_review_fix_scheduler import current_head_failed_checks -REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +REPO_RE = re.compile( + r"^[A-Za-z0-9_][A-Za-z0-9_.-]*/(?!\.{1,2}$)[A-Za-z0-9_.][A-Za-z0-9_.-]*$" +) SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$") _AUTOFIX_CONTROL_PREFIXES = (".github/", "scripts/ci/") _REPAIR_MODES = ("review", "rca", "conflict") diff --git a/scripts/ci/pr_review_fix_scheduler.py b/scripts/ci/pr_review_fix_scheduler.py index bc2868c5a4..3bfc0486c8 100755 --- a/scripts/ci/pr_review_fix_scheduler.py +++ b/scripts/ci/pr_review_fix_scheduler.py @@ -53,7 +53,9 @@ r"" ) -REPO_RE = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$") +REPO_RE = re.compile( + r"^[A-Za-z0-9_][A-Za-z0-9_.-]*/(?!\.{1,2}$)[A-Za-z0-9_.][A-Za-z0-9_.-]*$" +) REPAIR_MODES = frozenset({"review", "rca", "conflict"}) AUTOFIX_RUN_NAME_RE = re.compile( r"^PR Review Autofix (?P[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)" diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index b9b1c43de3..52175355a4 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -666,7 +666,7 @@ assert_opencode_review_uses_codegraph_and_contextual_orchestrator() { assert_file_matches "$workflow_file" 'uses:[[:space:]]+actions/checkout@[0-9a-fA-F]{40}([[:space:]]|$)' "opencode review workflow pins checkout to a full commit SHA" assert_file_contains "$workflow_file" "Provision contextual-orchestrator review sidecar" "opencode review provisions the central contextual-orchestrator sidecar" assert_file_contains "$workflow_file" 'NVIDIA_NIM_API_KEY: ${{ secrets.NVIDIA_NIM_API_KEY }}' "opencode review passes the scoped provider credentials only to sidecar bootstrap" - assert_file_contains "$workflow_file" "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" "opencode review passes repository privacy to the gateway ZDR policy" + assert_file_contains "$workflow_file" 'CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: "true"' "opencode review requires ZDR for every gateway review" assert_file_contains "$workflow_file" 'is_private: ${{ steps.validate.outputs.is_private }}' "opencode review carries validated repository privacy into gateway routing" assert_file_contains "$workflow_file" '"model": "contextual-orchestrator/orchestrator/free"' "opencode review uses the gateway free pool" assert_file_contains "$workflow_file" '"small_model": "contextual-orchestrator/orchestrator/free"' "opencode review uses the gateway for the small model" diff --git a/scripts/ci/zdr_policy.py b/scripts/ci/zdr_policy.py index eb327c8cad..ae4cafe40c 100644 --- a/scripts/ci/zdr_policy.py +++ b/scripts/ci/zdr_policy.py @@ -13,8 +13,9 @@ Two authoritative, machine-readable sources feed the policy at runtime: 1. OpenRouter ZDR endpoint feed (``https://openrouter.ai/api/v1/endpoints/zdr``) - — the exact list of model endpoints OpenRouter serves under a zero-data- - retention policy. Used verbatim for the ``openrouter`` provider scope. + — the machine-readable model evidence used to match discovered model + identities across the caller's candidate providers. It is not a routing + target; direct OpenRouter routes still require exact feed membership. 2. OpenRouter provider data-policy catalog (``https://openrouter.ai/api/frontend/v1/all-providers``) — per-provider ``dataPolicy`` (``retainsPrompts`` / ``retentionDays`` / ``training``), @@ -33,6 +34,9 @@ from typing import Mapping +OPENROUTER_ZDR_ENDPOINTS_SOURCE = "https://openrouter.ai/api/v1/endpoints/zdr" + + @dataclasses.dataclass(frozen=True) class ProviderZdrScope: """One provider's ZDR attestation for the CI review sidecar. @@ -45,9 +49,9 @@ class ProviderZdrScope: source: URL or document that grounds the attestation. as_of: ISO date the attestation was last verified. note: One-sentence scope note; never fabricated policy language. - openrouter_endpoints_feed: When True, the authoritative OpenRouter - ``/api/v1/endpoints/zdr`` feed decides per-model ZDR membership for - this provider; the static table is then only the fallback. + openrouter_endpoints_feed: When True, the provider requires exact + membership in the OpenRouter endpoint feed; other providers may + use the feed as model-level evidence for matching candidates. """ provider_name: str @@ -197,24 +201,70 @@ def is_zdr_model( Args: provider_name: Orchestrator provider identifier of the model route. - model: Specific model or route identifier. Required for an exact - OpenRouter feed match; omitted or empty never grants ZDR from - the feed. - zdr_endpoints: Frozen set of exact ``\"provider/model\"`` route keys - from the OpenRouter ``/api/v1/endpoints/zdr`` feed. When the - provider uses the feed, an empty set is not a fallback to - \"all OpenRouter is ZDR\". + model: Specific model or route identifier. OpenRouter rows require + exact route membership; other provider rows may match the same + model identity in the feed. + zdr_endpoints: Frozen set of ``\"provider/model\"`` keys from the + OpenRouter ``/api/v1/endpoints/zdr`` feed. An empty set never + grants feed-based ZDR. Returns: - True only for an attested zero-retention scope or an exact feed - membership match. + True only for an attested zero-retention scope, an exact OpenRouter + feed route, or an unambiguous matching model identity from that feed. """ + return zdr_evidence_source( + provider_name, model=model, zdr_endpoints=zdr_endpoints + ) is not None + + +def zdr_evidence_source( + provider_name: str, + *, + model: str | None = None, + zdr_endpoints: frozenset[str] = frozenset(), +) -> str | None: + """Return the source that attests one model, or ``None`` when unattested.""" scope = provider_zdr_scope(provider_name) + if not isinstance(model, str) or not model.strip(): + return ( + scope.source + if scope.zero_data_retention and not scope.openrouter_endpoints_feed + else None + ) + candidate = model.strip().lstrip("/").casefold() + feed = { + endpoint + for endpoint in (str(value).strip().casefold() for value in zdr_endpoints) + if endpoint.startswith("openrouter/") + and all(segment for segment in endpoint.split("/")) + } if scope.openrouter_endpoints_feed: - if not zdr_endpoints or not model: - return False - return route_key(provider_name, model) in zdr_endpoints - return scope.zero_data_retention + return ( + OPENROUTER_ZDR_ENDPOINTS_SOURCE + if feed and route_key(provider_name, model).casefold() in feed + else None + ) + elif feed: + matched = route_key(provider_name, model).casefold() in feed + feed_models = { + endpoint.split("/", 1)[1] if "/" in endpoint else endpoint + for endpoint in feed + } + if not matched and candidate in feed_models: + matched = True + if not matched: + suffix = candidate.rsplit("/", 1)[-1] + suffix_matches = [ + feed_model + for feed_model in feed_models + if feed_model.rsplit("/", 1)[-1] == suffix + ] + matched = bool(suffix) and len(suffix_matches) == 1 + else: + matched = False + if matched: + return OPENROUTER_ZDR_ENDPOINTS_SOURCE + return scope.source if scope.zero_data_retention else None def is_free_route(is_free: object) -> bool: @@ -229,4 +279,4 @@ def is_free_route(is_free: object) -> bool: """ if isinstance(is_free, str): return is_free.strip().lower() in {"1", "true", "yes"} - return bool(is_free) \ No newline at end of file + return bool(is_free) diff --git a/tests/test_contextual_orchestrator_review_live_discovery_contract.py b/tests/test_contextual_orchestrator_review_live_discovery_contract.py index 627af2d879..9b100a8118 100644 --- a/tests/test_contextual_orchestrator_review_live_discovery_contract.py +++ b/tests/test_contextual_orchestrator_review_live_discovery_contract.py @@ -9,6 +9,10 @@ FREE_MODEL = "qwen/qwen3-coder:free" PRICED_MODEL = "anthropic/claude-sonnet-4.6" +# OpenRouter is a ZDR evidence source, never a routed upstream (see +# EVIDENCE_ONLY_PROVIDERS): the illustrative free/priced rows below carry a +# real routable provider, while the feed still attests them via OpenRouter's +# per-model ZDR evidence, exactly as it does for any other discovered provider. PRICED_ZDR_FEED = frozenset({f"openrouter/{PRICED_MODEL}"}) @@ -17,18 +21,18 @@ def _live_discovery_report() -> dict[str, object]: return { "models": [ { - "provider": "openrouter", + "provider": "nvidia_nim", "model": FREE_MODEL, - "agent_id": "openrouter_qwen3_coder_free", + "agent_id": "nvidia_nim_qwen3_coder_free", "is_free": True, "prompt_price_per_1k": 0.0, "completion_price_per_1k": 0.0, "currency_code": "USD", }, { - "provider": "openrouter", + "provider": "bytez", "model": PRICED_MODEL, - "agent_id": "openrouter_claude_sonnet_46", + "agent_id": "bytez_claude_sonnet_46", "is_free": False, "prompt_price_per_1k": 0.003, "completion_price_per_1k": 0.015, diff --git a/tests/test_contextual_orchestrator_review_sidecar_contract.py b/tests/test_contextual_orchestrator_review_sidecar_contract.py index 79c74a4d43..f54836dac8 100644 --- a/tests/test_contextual_orchestrator_review_sidecar_contract.py +++ b/tests/test_contextual_orchestrator_review_sidecar_contract.py @@ -71,10 +71,11 @@ def test_sidecar_adr_names_the_current_vendored_revision() -> None: assert ORCH_PIN_SHA in _read(SIDECAR_ADR) -def test_sidecar_requires_the_five_provider_secrets() -> None: - """At least one of the five secrets must be present as bootstrap transport.""" +def test_sidecar_requires_zdr_evidence_and_serving_provider() -> None: + """Mandatory ZDR needs the evidence key and a non-evidence provider.""" text = _read(SIDECAR) - assert '"$provider_secret_count" -lt 1 ]; then' in text + assert 'OPENROUTER_API_KEY is required for mandatory-ZDR evidence discovery' in text + assert '"$serving_provider_secret_count" -lt 1 ]; then' in text for secret in FIVE_SECRETS: assert secret in text @@ -89,6 +90,7 @@ def test_sidecar_feeds_discovery_and_policy_artifacts_to_the_launcher() -> None: "--zdr-endpoints \"$zdr_feed\"", ): assert arg in text + assert 'Authorization: Bearer ${OPENROUTER_API_KEY}' in text assert "https://openrouter.ai/api/v1/endpoints/zdr" in text @@ -166,6 +168,7 @@ def test_token_loader_rehydrates_and_masks_bearer_inside_each_consumer_step() -> assert '_contextual_orchestrator_stat()' in text assert 'stat -c "$format" -- "$target"' in text assert '[ "$format" = "%a" ]' in text + assert "stat -f '%Mp %Lp' \"$target\"" in text assert "stat -f '%OMp %OLp' \"$target\"" in text assert 'stat -f "$format" "$target"' in text assert "CONTEXTUAL_ORCHESTRATOR_TOKEN must not contain CR or LF" in text @@ -241,6 +244,49 @@ def run(candidate: Path) -> subprocess.CompletedProcess[str]: assert "must not contain CR or LF" in multiline.stderr +def test_token_loader_probes_busybox_style_stat_without_version_flag(tmp_path: Path) -> None: + """A stat implementation without --version still uses its working -c form.""" + token_file = tmp_path / "bearer.token" + token_file.write_text("synthetic-test-bearer", encoding="utf-8") + token_file.chmod(0o600) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + fake_stat = fake_bin / "stat" + fake_stat.write_text( + "#!/usr/bin/env bash\n" + "case \"${1:-}\" in\n" + " --version) exit 1 ;;\n" + " -c)\n" + " case \"${2:-}\" in\n" + " %u) id -u ;;\n" + " %a) printf '600\\n' ;;\n" + " *) exit 1 ;;\n" + " esac\n" + " ;;\n" + " -f) exit 1 ;;\n" + " *) exit 1 ;;\n" + "esac\n", + encoding="utf-8", + ) + fake_stat.chmod(0o700) + result = subprocess.run( + ["bash", "-c", 'set -euo pipefail; source "$TOKEN_LOADER"; printf "loaded=%s\\n" "$CONTEXTUAL_ORCHESTRATOR_TOKEN"'], + env={ + **os.environ, + "GITHUB_ACTIONS": "false", + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "TOKEN_LOADER": str(TOKEN_LOADER), + "CONTEXTUAL_ORCHESTRATOR_TOKEN_FILE": str(token_file), + }, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + assert "loaded=synthetic-test-bearer" in result.stdout + + def test_token_loader_preserves_caller_locals_and_removes_helpers(tmp_path: Path) -> None: """Sourcing the loader must not clobber common caller names or leak functions.""" token_path = tmp_path / "bearer.token" @@ -418,6 +464,10 @@ def test_sidecar_probes_the_pinned_server_body_limit_at_http_boundary() -> None: assert "assert status == 200" in text assert "proxy_payloads[-1]" in text assert '"utf-8"' in text + assert '"stream_options": {"include_usage": True}' in text + assert '"stream": True' in text + assert "post_stream_payload" in text + assert 'text/event-stream' in text def test_autofix_workflow_provisions_sidecar_with_all_five_secrets() -> None: @@ -426,6 +476,7 @@ def test_autofix_workflow_provisions_sidecar_with_all_five_secrets() -> None: assert "contextual_orchestrator_review_sidecar.sh" in workflow for secret in FIVE_SECRETS: assert f"{secret}: ${{{{ secrets.{secret} }}}}" in workflow + assert 'CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: "true"' in workflow assert GATEWAY_MODEL in workflow assert workflow.count(f"MODEL: {GATEWAY_MODEL}") == 2 assert "https://integrate.api.nvidia.com/v1" not in workflow @@ -536,8 +587,8 @@ def test_noema_review_workflow_provisions_sidecar_with_all_five_secrets() -> Non assert "NOEMA_REVIEW_TOKEN: ${{ secrets.NOEMA_REVIEW_TOKEN }}" in workflow -def test_noema_private_targets_require_zdr_only_sidecar_routing() -> None: - """Repository visibility binds private review content to an attested ZDR-only pool.""" +def test_noema_review_targets_require_zdr_only_sidecar_routing() -> None: + """Every Noema review target uses an attested ZDR-only pool.""" workflow = _read(NOEMA_WORKFLOW) sidecar = _read(SIDECAR) launcher = _read(LAUNCHER) @@ -545,8 +596,11 @@ def test_noema_private_targets_require_zdr_only_sidecar_routing() -> None: assert "Resolve Noema target repository visibility" in workflow assert "target_visibility.outputs.require_zdr" in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow + assert 'private|internal|public)' in workflow + assert 'echo "require_zdr=false"' not in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in sidecar assert "--require-zdr" in sidecar + assert 'fail "central review sidecar requires CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR=true"' in sidecar assert 'parser.add_argument("--require-zdr", action="store_true")' in launcher assert "require_zdr=args.require_zdr" in launcher @@ -568,10 +622,10 @@ def test_required_opencode_dispatch_uses_the_gateway_for_model_pool_and_diagnosi def test_required_strix_uses_the_gateway_and_zdr_visibility_contract() -> None: - """Strix accepts only the gateway route and binds private scans to ZDR.""" + """Strix accepts only the gateway route and requires ZDR for every scan.""" workflow = _read(STRIX_WORKFLOW) assert "Provision contextual-orchestrator Strix sidecar" in workflow - assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow + assert 'CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: "true"' in workflow assert 'STRIX_MODEL: contextual-orchestrator/orchestrator/free' in workflow assert "provider_mode=contextual_orchestrator" in workflow assert "STRIX_LLM_DEFAULT_PROVIDER: contextual_orchestrator" in workflow diff --git a/tests/test_install_base_python_locks.py b/tests/test_install_base_python_locks.py index b6f1782a02..1175e97a83 100644 --- a/tests/test_install_base_python_locks.py +++ b/tests/test_install_base_python_locks.py @@ -3,6 +3,7 @@ from __future__ import annotations import io +import hashlib import json import pathlib import subprocess @@ -91,6 +92,325 @@ def fake_runner(command: list[str], **kwargs): assert "candidates=2 installed=2 skipped=0" in stdout.getvalue() +def test_installs_verified_archives_without_network_or_dependency_resolution(tmp_path) -> None: + """Archive build hooks run only in the caller's network-isolated phase.""" + archive = tmp_path / "archives" / "archive-000.tar.gz" + archive.parent.mkdir() + archive.write_bytes(b"verified archive") + (tmp_path / "archive-manifest.json").write_text( + json.dumps( + [ + { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": [hashlib.sha256(archive.read_bytes()).hexdigest()], + } + ] + ), + encoding="utf-8", + ) + commands: list[list[str]] = [] + + def fake_runner(command: list[str], **kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout="") + + assert installer.install_materialized_locks( + tmp_path, + archives_only=True, + runner=fake_runner, + ) == 0 + assert commands == [ + [ + installer.sys.executable, + "-m", + "pip", + "install", + "--break-system-packages", + "--disable-pip-version-check", + "--no-deps", + "--no-build-isolation", + str(archive), + ] + ] + + +@pytest.mark.parametrize( + "marker", + ["python_version >= '3.12'", "python_version < '3.10'"], +) +def test_archive_marker_is_preserved_for_pip_to_evaluate(tmp_path, marker: str) -> None: + """Archive markers remain attached to the direct local requirement.""" + archive = tmp_path / "archives" / "archive-000.tar.gz" + archive.parent.mkdir() + archive.write_bytes(b"verified archive") + (tmp_path / "archive-manifest.json").write_text( + json.dumps( + [ + { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": [hashlib.sha256(archive.read_bytes()).hexdigest()], + "marker": marker, + } + ] + ), + encoding="utf-8", + ) + commands: list[list[str]] = [] + + def fake_runner(command: list[str], **kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout="") + + assert installer.install_materialized_locks( + tmp_path, + archives_only=True, + runner=fake_runner, + ) == 0 + assert commands[0][-1] == ( + f"demo @ {archive.as_uri()} ; {marker}" + ) + + +def test_archive_only_install_failure_is_fatal(tmp_path) -> None: + """A verified archive that fails to build must fail the isolated phase.""" + archive = tmp_path / "archives" / "archive-000.tar.gz" + archive.parent.mkdir() + archive.write_bytes(b"verified archive") + (tmp_path / "archive-manifest.json").write_text( + json.dumps( + [ + { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": [hashlib.sha256(archive.read_bytes()).hexdigest()], + } + ] + ), + encoding="utf-8", + ) + + def fake_runner(command: list[str], **kwargs): + return subprocess.CompletedProcess(command, 23, stdout="") + + stderr = io.StringIO() + assert installer.install_materialized_locks( + tmp_path, + archives_only=True, + runner=fake_runner, + stderr=stderr, + ) == 23 + assert "failed to install: demo" in stderr.getvalue() + + +def _write_valid_archive_manifest(root: pathlib.Path) -> pathlib.Path: + """Create one valid archive manifest and return its materialized file.""" + archive = root / "archives" / "archive-000.tar.gz" + archive.parent.mkdir() + archive.write_bytes(b"verified archive") + (root / "archive-manifest.json").write_text( + json.dumps( + [ + { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": [hashlib.sha256(archive.read_bytes()).hexdigest()], + } + ] + ), + encoding="utf-8", + ) + return archive + + +def test_installs_verified_archives_after_validated_locks(tmp_path) -> None: + """Normal lock installation includes verified archives by default.""" + write_candidate( + tmp_path, + generated_file="requirements-000.txt", + source="requirements-hashes.txt", + ) + archive = _write_valid_archive_manifest(tmp_path) + commands: list[list[str]] = [] + + def fake_runner(command: list[str], **kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout="") + + assert installer.install_materialized_locks(tmp_path, runner=fake_runner) == 0 + assert commands[-1][-1] == str(archive) + + +def test_can_skip_verified_archives_after_validated_locks(tmp_path) -> None: + """The normal phase can explicitly defer archive installation.""" + write_candidate( + tmp_path, + generated_file="requirements-000.txt", + source="requirements-hashes.txt", + ) + archive = _write_valid_archive_manifest(tmp_path) + commands: list[list[str]] = [] + + def fake_runner(command: list[str], **kwargs): + commands.append(command) + return subprocess.CompletedProcess(command, 0, stdout="") + + assert installer.install_materialized_locks( + tmp_path, + install_archives=False, + runner=fake_runner, + ) == 0 + assert commands[-1][-1] != str(archive) + + +def test_archive_install_failure_after_validated_locks_is_fatal(tmp_path) -> None: + """A normal install cannot hide a verified archive build failure.""" + write_candidate( + tmp_path, + generated_file="requirements-000.txt", + source="requirements-hashes.txt", + ) + archive = _write_valid_archive_manifest(tmp_path) + call_count = 0 + + def fake_runner(command: list[str], **kwargs): + nonlocal call_count + call_count += 1 + return subprocess.CompletedProcess( + command, + 17 if call_count == 3 else 0, + stdout="", + ) + + stderr = io.StringIO() + assert installer.install_materialized_locks( + tmp_path, + runner=fake_runner, + stderr=stderr, + ) == 17 + assert str(archive) in stderr.getvalue() or "demo" in stderr.getvalue() + + +def test_archive_manifest_directory_is_rejected(tmp_path) -> None: + """The archive manifest itself must be a regular file.""" + (tmp_path / "archive-manifest.json").mkdir() + + with pytest.raises(ValueError, match="regular non-symlink file"): + installer._archive_entries(tmp_path) + + +@pytest.mark.parametrize("manifest_text", ["{not-json", "{}", "[1]"]) +def test_archive_manifest_json_shape_is_validated(tmp_path, manifest_text: str) -> None: + """Archive manifest syntax and top-level shape are fail-closed.""" + (tmp_path / "archive-manifest.json").write_text(manifest_text, encoding="utf-8") + + error = "invalid" if manifest_text == "{not-json" else ( + "JSON array" if manifest_text == "{}" else "entries must be objects" + ) + with pytest.raises(ValueError, match=error): + installer._archive_entries(tmp_path) + + +@pytest.mark.parametrize( + "entry", + [ + { + "package": "", + "file": "archives/archive-000.tar.gz", + "hashes": ["a" * 64], + }, + { + "package": "demo", + "file": "archive-000.tar.gz", + "hashes": ["a" * 64], + }, + { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": ["not-a-sha256"], + }, + ], +) +def test_archive_manifest_entry_fields_are_validated(tmp_path, entry) -> None: + """Package, generated path, and digest fields must be exact types and shapes.""" + (tmp_path / "archive-manifest.json").write_text( + json.dumps([entry]), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="invalid entry"): + installer._archive_entries(tmp_path) + + +def test_archive_manifest_rejects_duplicate_files(tmp_path) -> None: + """One generated archive path cannot represent two source entries.""" + archive = _write_valid_archive_manifest(tmp_path) + entry = { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": [hashlib.sha256(archive.read_bytes()).hexdigest()], + } + (tmp_path / "archive-manifest.json").write_text( + json.dumps([entry, entry]), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="duplicate files"): + installer._archive_entries(tmp_path) + + +def test_archive_manifest_rejects_missing_archive_file(tmp_path) -> None: + """Every manifest entry must resolve to a regular materialized file.""" + (tmp_path / "archive-manifest.json").write_text( + json.dumps( + [ + { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": ["a" * 64], + } + ] + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="must be a regular file"): + installer._archive_entries(tmp_path) + + +def test_archive_manifest_rejects_symlink_archive_file(tmp_path) -> None: + """Archive entries cannot escape the materialized root through a symlink.""" + archive = _write_valid_archive_manifest(tmp_path) + target = tmp_path / "real-archive.tar.gz" + target.write_bytes(archive.read_bytes()) + archive.unlink() + archive.symlink_to(target) + + with pytest.raises(ValueError, match="must be a regular file"): + installer._archive_entries(tmp_path) + + +def test_archive_manifest_rejects_hash_mismatch(tmp_path) -> None: + """The local archive bytes must match the digest exported by the base lock.""" + _write_valid_archive_manifest(tmp_path) + (tmp_path / "archive-manifest.json").write_text( + json.dumps( + [ + { + "package": "demo", + "file": "archives/archive-000.tar.gz", + "hashes": ["a" * 64], + } + ] + ), + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="failed hash verification"): + installer._archive_entries(tmp_path) + + def test_skips_partial_candidate_without_completing_sibling(tmp_path) -> None: """An unrecoverable hash-bearing supplement remains visible and non-fatal.""" write_candidate( @@ -443,7 +763,7 @@ def test_main_forwards_requirements_root(monkeypatch, tmp_path) -> None: """The CLI delegates the exact requirements root to the installer.""" seen: list[pathlib.Path] = [] - def fake_install(root: pathlib.Path) -> int: + def fake_install(root: pathlib.Path, **_kwargs) -> int: seen.append(root) return 7 diff --git a/tests/test_materialize_base_python_requirements.py b/tests/test_materialize_base_python_requirements.py index a2da04ae25..3903210017 100644 --- a/tests/test_materialize_base_python_requirements.py +++ b/tests/test_materialize_base_python_requirements.py @@ -129,7 +129,7 @@ def test_materializes_exact_vcs_sources_in_a_separate_manifest( monkeypatch.setattr( materializer, "_base_python_inputs", - lambda *_args: ([("uv.lock", hash_lock)], vcs_sources), + lambda *_args: ([("uv.lock", hash_lock)], vcs_sources, []), ) output = tmp_path / "output" @@ -142,6 +142,525 @@ def test_materializes_exact_vcs_sources_in_a_separate_manifest( ) +def test_materializes_archive_sources_separately_from_pip_locks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Archive sources are verified before the image's no-network install step.""" + repository = tmp_path / "repo" + repository.mkdir() + git(repository, "init") + git(repository, "config", "user.name", "Test") + git(repository, "config", "user.email", "test@example.invalid") + git(repository, "commit", "--allow-empty", "-m", "base") + base_sha = git(repository, "rev-parse", "HEAD") + archive = b"verified archive" + archive_source = { + "package": "demo", + "url": "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz", + "hashes": [hashlib.sha256(archive).hexdigest()], + "source": "uv.lock", + } + monkeypatch.setattr( + materializer, + "_base_python_inputs", + lambda *_args: ([], [], [archive_source]), + ) + monkeypatch.setattr( + materializer, + "_download_trusted_org_archive", + lambda _url, _hashes: archive, + ) + + output = tmp_path / "output" + materializer.materialize(repository, base_sha, output) + + assert not list(output.glob("requirements-*.txt")) + assert json.loads((output / "archive-manifest.json").read_text()) == [ + {**archive_source, "file": "archives/archive-000.tar.gz"} + ] + assert (output / "archives/archive-000.tar.gz").read_bytes() == archive + + +def test_materialize_skips_downloading_an_archive_excluded_for_the_target_python( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A marker-excluded archive's unreachable URL must not fail the whole job. + + The coverage image would never install this archive for its pinned + interpreter anyway, so an unrelated outage on its URL (404, network blip, + a removed tag) must not fail base Python lock materialization. + """ + repository = tmp_path / "repo" + repository.mkdir() + git(repository, "init") + git(repository, "config", "user.name", "Test") + git(repository, "config", "user.email", "test@example.invalid") + git(repository, "commit", "--allow-empty", "-m", "base") + base_sha = git(repository, "rev-parse", "HEAD") + included_archive = b"included archive" + excluded_source = { + "package": "demo-py311-only", + "url": "https://github.com/ContextualWisdomLab/demo/archive/py311.tar.gz", + "hashes": ["a" * 64], + "marker": "python_version == '3.11'", + "source": "uv.lock", + } + included_source = { + "package": "demo", + "url": "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz", + "hashes": [hashlib.sha256(included_archive).hexdigest()], + "marker": "python_version == '3.14'", + "source": "uv.lock", + } + monkeypatch.setattr( + materializer, + "_base_python_inputs", + lambda *_args: ([], [], [excluded_source, included_source]), + ) + + def fail_if_called_for_excluded_archive(url: str, _hashes: list[str]) -> bytes: + if url == excluded_source["url"]: + raise AssertionError( + "materialize() must not download an archive its own marker " + "excludes for the target coverage Python version" + ) + return included_archive + + monkeypatch.setattr( + materializer, + "_download_trusted_org_archive", + fail_if_called_for_excluded_archive, + ) + + output = tmp_path / "output" + manifest = materializer.materialize( + repository, base_sha, output, target_python_version="3.14" + ) + + assert manifest == [] + archive_manifest = json.loads((output / "archive-manifest.json").read_text()) + assert archive_manifest == [ + {**included_source, "file": "archives/archive-001.tar.gz"} + ] + assert (output / "archives/archive-001.tar.gz").read_bytes() == included_archive + assert not (output / "archives/archive-000.tar.gz").exists() + + +def test_materialize_downloads_every_archive_when_no_target_python_is_given( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Omitting the target version keeps today's unconditional download behavior.""" + repository = tmp_path / "repo" + repository.mkdir() + git(repository, "init") + git(repository, "config", "user.name", "Test") + git(repository, "config", "user.email", "test@example.invalid") + git(repository, "commit", "--allow-empty", "-m", "base") + base_sha = git(repository, "rev-parse", "HEAD") + archive_source = { + "package": "demo-py311-only", + "url": "https://github.com/ContextualWisdomLab/demo/archive/py311.tar.gz", + "hashes": ["a" * 64], + "marker": "python_version == '3.11'", + "source": "uv.lock", + } + monkeypatch.setattr( + materializer, + "_base_python_inputs", + lambda *_args: ([], [], [archive_source]), + ) + calls: list[str] = [] + + def record_call(url: str, _hashes: list[str]) -> bytes: + calls.append(url) + return b"payload" + + monkeypatch.setattr(materializer, "_download_trusted_org_archive", record_call) + + output = tmp_path / "output" + materializer.materialize(repository, base_sha, output) + + assert calls == [archive_source["url"]] + + +@pytest.mark.parametrize( + ("marker", "target_python_version", "expected"), + [ + ("python_version == '3.11'", "3.14", True), + ("python_version == '3.14'", "3.14", False), + ("python_version != '3.14'", "3.14", True), + ("python_version < '3.11'", "3.14", True), + ("python_version < '3.11'", "3.9", False), + ("python_version <= '3.14'", "3.14", False), + ("python_version > '3.14'", "3.14", True), + ("python_version >= '3.14'", "3.14", False), + ("python_version >= '3.9'", "3.14", False), + ("python_version == '3.9' or python_version == '3.14'", "3.14", False), + ("python_version == '3.9' or python_version == '3.10'", "3.14", True), + ( + "python_version >= '3.10' and python_version < '3.12'", + "3.14", + True, + ), + ( + "python_version >= '3.10' and python_version < '3.14'", + "3.9", + True, + ), + ("not python_version == '3.14'", "3.14", True), + # A literal on the left of the comparison is handled the same way. + ("'3.10' <= python_version", "3.9", True), + ("'3.10' <= python_version", "3.11", False), + # Unsupported/unparseable marker shapes must fail open (never exclude). + ("sys_platform == 'linux'", "3.14", False), + ("extra == 'test'", "3.14", False), + ("python_version in '3.11'", "3.14", False), + ("python_version == 'not-a-version'", "3.14", False), + ("not a valid marker (((", "3.14", False), + ("python_version == python_full_version", "3.14", False), + ("extra", "3.14", False), + # python_full_version is patch-sensitive: a major.minor-only target + # (as the coverage workflow genuinely only ever has) must NOT be + # zero-padded into a confident patch value *when the literal shares + # the target's own major.minor series* -- the outcome then genuinely + # depends on the interpreter's real, unknown patch digit, so these + # must fail open (never exclude/skip the download) regardless of + # operator or of whether the zero-padded guess would have happened + # to match. + ("python_full_version == '3.14.0'", "3.14", False), + ("python_full_version != '3.14.0'", "3.14", False), + ("python_full_version >= '3.14.1'", "3.14", False), + ("python_full_version > '3.14.0'", "3.14", False), + ("python_full_version < '3.14.5'", "3.14", False), + # But a literal whose major.minor falls OUTSIDE the target's series + # is decidable for every possible patch of the target's minor series + # -- the major.minor mismatch alone settles it, so these must now + # resolve confidently instead of failing open. + # Cross-minor equality: 3.14.x is never version 3.9.0. + ("python_full_version == '3.9.0'", "3.14", True), + # Below the target's entire minor series: 3.14.x is never < 3.0.0. + ("python_full_version < '3.0.0'", "3.14", True), + # Above the target's entire minor series: 3.14.x is never >= 4.0.0. + ("python_full_version >= '4.0.0'", "3.14", True), + # A cross-minor literal with the variable on the left of the compare. + ("'3.9.0' == python_full_version", "3.14", True), + # A literal with fewer components than the target still resolves + # confidently once zero-padded, provided the padded major.minor + # differs from the target's. + ("python_full_version == '3'", "3.14", True), + ( + "python_full_version >= '3.10.0' and python_full_version < '3.15.0'", + "3.14", + False, + ), + # A same-minor-series operand alongside a decidable one: the overall + # marker still must not be resolved confidently, because ``and`` + # cannot decide without the ambiguous operand's true value, and + # ``_evaluate_marker_node`` propagates the ambiguity by raising. + ( + "python_full_version >= '3.14.1' and python_full_version < '3.15.0'", + "3.14", + False, + ), + # Once the target itself carries patch precision, python_full_version + # comparisons become confident again. + ("python_full_version >= '3.14.1'", "3.14.5", False), + ("python_full_version >= '3.14.6'", "3.14.5", True), + ("python_full_version == '3.14.5'", "3.14.5", False), + ], +) +def test_marker_excludes_target_python( + marker: str, target_python_version: str, expected: bool +) -> None: + """Only a confidently-false python_version/python_full_version marker excludes.""" + assert ( + materializer._marker_excludes_target_python(marker, target_python_version) + == expected + ) + + +def test_download_trusted_org_archive_uses_bounded_verified_https_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A trusted archive is read in bounded chunks and checksum-verified.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + payload = b"verified archive" + + class FakeOpener: + def open(self, request_url: str, *, timeout: int) -> FakeHttpResponse: + assert request_url == url + assert timeout == materializer.TRUSTED_ORG_ARCHIVE_TIMEOUT_SECONDS + return FakeHttpResponse(url, payload, maximum_chunk_size=3) + + monkeypatch.setattr( + materializer.urllib.request, + "build_opener", + lambda *_handlers: FakeOpener(), + ) + + assert materializer._download_trusted_org_archive( + url, [hashlib.sha256(payload).hexdigest()] + ) == payload + + +@pytest.mark.parametrize( + ("source_url", "target_url"), + [ + ( + "https://not-github.invalid/demo.tar.gz", + "https://codeload.github.com/ContextualWisdomLab/demo/legacy.tar.gz", + ), + ( + "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz", + "https://not-github.invalid/demo.tar.gz", + ), + ], +) +def test_trusted_org_archive_redirects_reject_non_github_origins( + source_url: str, + target_url: str, +) -> None: + """Archive redirects must remain on the two explicit GitHub origins.""" + with pytest.raises(RuntimeError, match="redirected outside GitHub"): + materializer._TrustedOrgArchiveRedirects().redirect_request( + materializer.urllib.request.Request(source_url), + object(), + 302, + "Found", + {}, + target_url, + ) + + +def test_trusted_org_archive_redirects_follow_codeload() -> None: + """A valid archive redirect is passed through unchanged as a GET request.""" + followed = materializer._TrustedOrgArchiveRedirects().redirect_request( + materializer.urllib.request.Request( + "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + ), + object(), + 302, + "Found", + {}, + "https://codeload.github.com/ContextualWisdomLab/demo/legacy.tar.gz", + ) + + assert followed.full_url == ( + "https://codeload.github.com/ContextualWisdomLab/demo/legacy.tar.gz" + ) + + +def test_trusted_org_archive_redirects_reject_cross_repository_hop() -> None: + """A same-host redirect to a different repository must not be followed. + + Both hosts are allowlisted, so a host-only check would let + ``github.com/ContextualWisdomLab/demo`` redirect onto + ``codeload.github.com/some-other-org/some-other-repo``. The redirect must + stay on the exact repository the original request named. + """ + with pytest.raises(RuntimeError, match="redirected outside GitHub"): + materializer._TrustedOrgArchiveRedirects().redirect_request( + materializer.urllib.request.Request( + "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + ), + object(), + 302, + "Found", + {}, + "https://codeload.github.com/some-other-org/some-other-repo/tar.gz/v1", + ) + + +def test_trusted_org_archive_redirects_accept_case_insensitive_same_repository() -> None: + """GitHub repository names are case-insensitive; the same-repo check must match.""" + followed = materializer._TrustedOrgArchiveRedirects().redirect_request( + materializer.urllib.request.Request( + "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + ), + object(), + 302, + "Found", + {}, + "https://codeload.github.com/contextualwisdomlab/DEMO/tar.gz/v1", + ) + + assert followed.full_url == ( + "https://codeload.github.com/contextualwisdomlab/DEMO/tar.gz/v1" + ) + + +def test_trusted_org_archive_redirects_reject_unrecognized_archive_path() -> None: + """An allowlisted-host URL that is not a recognized archive path shape is rejected.""" + with pytest.raises(RuntimeError, match="redirected outside GitHub"): + materializer._TrustedOrgArchiveRedirects().redirect_request( + materializer.urllib.request.Request( + "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + ), + object(), + 302, + "Found", + {}, + "https://github.com/ContextualWisdomLab", + ) + + +def test_org_archive_repository_rejects_unrecognized_hosts() -> None: + """A host outside the two trusted archive hosts names no repository.""" + assert materializer._org_archive_repository("https://example.invalid/demo/archive/v1.tar.gz") is None + assert not materializer._is_same_org_archive_repository( + "https://example.invalid/demo/archive/v1.tar.gz", + "https://example.invalid/demo/archive/v1.tar.gz", + ) + + +def test_trusted_org_archive_redirects_fail_when_parent_rejects_redirect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A redirect rejected by urllib remains a hard failure.""" + monkeypatch.setattr( + materializer.urllib.request.HTTPRedirectHandler, + "redirect_request", + lambda *_args: None, + ) + + with pytest.raises(RuntimeError, match="redirect was rejected"): + materializer._TrustedOrgArchiveRedirects().redirect_request( + materializer.urllib.request.Request( + "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + ), + object(), + 302, + "Found", + {}, + "https://codeload.github.com/ContextualWisdomLab/demo/legacy.tar.gz", + ) + + +def test_download_trusted_org_archive_rejects_invalid_source_url() -> None: + """The initial archive URL must be an allowlisted GitHub HTTPS URL.""" + with pytest.raises(RuntimeError, match="URL is not GitHub HTTPS"): + materializer._download_trusted_org_archive( + "https://example.invalid/demo.tar.gz", ["a" * 64] + ) + + +def test_download_trusted_org_archive_rejects_untrusted_final_url( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A response that leaves GitHub is rejected before its bytes are trusted.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + + class FakeOpener: + def open(self, _request_url: str, *, timeout: int) -> FakeHttpResponse: + del timeout + return FakeHttpResponse("https://example.invalid/demo.tar.gz") + + monkeypatch.setattr( + materializer.urllib.request, + "build_opener", + lambda *_handlers: FakeOpener(), + ) + + with pytest.raises(RuntimeError, match="left GitHub origins"): + materializer._download_trusted_org_archive(url, ["a" * 64]) + + +def test_download_trusted_org_archive_rejects_final_url_naming_another_repository( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A same-allowlisted-host final URL for a different repository is rejected. + + Defense in depth alongside the ``redirect_request`` check: even if the + observed final URL reached the response through some other path, it must + still name the exact repository that was originally requested. + """ + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + + class FakeOpener: + def open(self, _request_url: str, *, timeout: int) -> FakeHttpResponse: + del timeout + return FakeHttpResponse( + "https://codeload.github.com/some-other-org/some-other-repo/tar.gz/v1" + ) + + monkeypatch.setattr( + materializer.urllib.request, + "build_opener", + lambda *_handlers: FakeOpener(), + ) + + with pytest.raises(RuntimeError, match="left GitHub origins"): + materializer._download_trusted_org_archive(url, ["a" * 64]) + + +def test_download_trusted_org_archive_wraps_network_errors( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Network failures do not escape as ambiguous low-level exceptions.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + + class FakeOpener: + def open(self, _request_url: str, *, timeout: int) -> FakeHttpResponse: + del timeout + raise OSError("offline") + + monkeypatch.setattr( + materializer.urllib.request, + "build_opener", + lambda *_handlers: FakeOpener(), + ) + + with pytest.raises(RuntimeError, match="download failed: OSError"): + materializer._download_trusted_org_archive(url, ["a" * 64]) + + +def test_download_trusted_org_archive_rejects_oversized_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Archive downloads remain bounded before checksum processing.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + monkeypatch.setattr(materializer, "TRUSTED_ORG_ARCHIVE_MAX_BYTES", 3) + + class FakeOpener: + def open(self, _request_url: str, *, timeout: int) -> FakeHttpResponse: + del timeout + return FakeHttpResponse(url, b"1234") + + monkeypatch.setattr( + materializer.urllib.request, + "build_opener", + lambda *_handlers: FakeOpener(), + ) + + with pytest.raises(RuntimeError, match="bounded size"): + materializer._download_trusted_org_archive(url, ["a" * 64]) + + +def test_download_trusted_org_archive_rejects_checksum_mismatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A trusted origin is insufficient without the exact exported hash.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + payload = b"archive" + + class FakeOpener: + def open(self, _request_url: str, *, timeout: int) -> FakeHttpResponse: + del timeout + return FakeHttpResponse(url, payload) + + monkeypatch.setattr( + materializer.urllib.request, + "build_opener", + lambda *_handlers: FakeOpener(), + ) + + with pytest.raises(RuntimeError, match="checksum verification failed"): + materializer._download_trusted_org_archive(url, ["a" * 64]) + + def test_base_inputs_preserve_a_vcs_only_export( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -161,13 +680,16 @@ def test_base_inputs_preserve_a_vcs_only_export( monkeypatch.setattr( materializer, "_export_uv_lock", - lambda *_args: (b"", [dependency]), + lambda *_args: (b"", [dependency], []), ) - locks, vcs_sources = materializer._base_python_inputs(tmp_path, "a" * 40) + locks, vcs_sources, archive_sources = materializer._base_python_inputs( + tmp_path, "a" * 40 + ) assert locks == [] assert vcs_sources == [{**dependency, "source": "uv.lock"}] + assert archive_sources == [] def test_base_inputs_reject_conflicting_vcs_revisions_across_locks( @@ -188,7 +710,7 @@ def test_base_inputs_reject_conflicting_vcs_revisions_across_locks( def export(_repo: Path, _sha: str, lock_path: str): commit = "a" * 40 if lock_path.startswith("first/") else "b" * 40 - return b"", [{"package": "demo", "repository": "demo", "commit": commit}] + return b"", [{"package": "demo", "repository": "demo", "commit": commit}], [] monkeypatch.setattr(materializer, "_export_uv_lock", export) @@ -196,6 +718,111 @@ def export(_repo: Path, _sha: str, lock_path: str): materializer._base_python_inputs(tmp_path, "a" * 40) +def test_base_inputs_reject_conflicting_archive_hashes_across_locks( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Separate uv projects cannot select conflicting hashes for one archive.""" + tree = b"".join( + b"100644 blob " + bytes(character, "ascii") * 40 + b"\t" + path + b"\0" + for character, path in ( + ("a", b"first/pyproject.toml"), + ("b", b"first/uv.lock"), + ("c", b"second/pyproject.toml"), + ("d", b"second/uv.lock"), + ) + ) + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + monkeypatch.setattr(materializer, "_git", lambda *_args: tree) + + def export(_repo: Path, _sha: str, lock_path: str): + digest = "a" * 64 if lock_path.startswith("first/") else "b" * 64 + return b"", [], [{"package": "demo", "url": url, "hashes": [digest]}] + + monkeypatch.setattr(materializer, "_export_uv_lock", export) + + with pytest.raises(RuntimeError, match="conflicting hashes"): + materializer._base_python_inputs(tmp_path, "a" * 40) + + +def test_base_inputs_keeps_same_archive_url_for_distinct_markers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Separate uv projects may retain conditional alternatives for one URL.""" + tree = b"".join( + b"100644 blob " + bytes(character, "ascii") * 40 + b"\t" + path + b"\0" + for character, path in ( + ("a", b"first/pyproject.toml"), + ("b", b"first/uv.lock"), + ("c", b"second/pyproject.toml"), + ("d", b"second/uv.lock"), + ) + ) + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + monkeypatch.setattr(materializer, "_git", lambda *_args: tree) + + def export(_repo: Path, _sha: str, lock_path: str): + marker = ( + "python_version < '3.10'" + if lock_path.startswith("first/") + else "python_version >= '3.10'" + ) + return b"", [], [ + { + "package": "demo", + "url": url, + "hashes": ["a" * 64], + "marker": marker, + } + ] + + monkeypatch.setattr(materializer, "_export_uv_lock", export) + + _locks, _vcs_sources, archive_sources = materializer._base_python_inputs( + tmp_path, "a" * 40 + ) + + assert {archive["marker"] for archive in archive_sources} == { + "python_version < '3.10'", + "python_version >= '3.10'", + } + + +def test_base_inputs_rejects_different_hashes_for_same_url_across_markers( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Conditional alternatives cannot change one immutable archive payload.""" + tree = b"".join( + b"100644 blob " + bytes(character, "ascii") * 40 + b"\t" + path + b"\0" + for character, path in ( + ("a", b"first/pyproject.toml"), + ("b", b"first/uv.lock"), + ("c", b"second/pyproject.toml"), + ("d", b"second/uv.lock"), + ) + ) + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + monkeypatch.setattr(materializer, "_git", lambda *_args: tree) + + def export(_repo: Path, _sha: str, lock_path: str): + marker = ( + "python_version < '3.10'" + if lock_path.startswith("first/") + else "python_version >= '3.10'" + ) + digest = "a" * 64 if lock_path.startswith("first/") else "b" * 64 + return b"", [], [ + {"package": "demo", "url": url, "hashes": [digest], "marker": marker} + ] + + monkeypatch.setattr(materializer, "_export_uv_lock", export) + + with pytest.raises(RuntimeError, match="conflicting hashes"): + materializer._base_python_inputs(tmp_path, "a" * 40) + + def test_materializes_hash_pinned_locks_named_beyond_the_legacy_whitelist( tmp_path: Path, ) -> None: @@ -454,8 +1081,13 @@ def test_main_reports_each_materialized_lock( """The CLI identifies the exact trusted source and generated lock name.""" def fake_materialize( - _repo_root: Path, _base_sha: str, _output_dir: Path + _repo_root: Path, + _base_sha: str, + _output_dir: Path, + *, + target_python_version: str | None = None, ) -> list[dict[str, str]]: + del target_python_version return [ { "file": "requirements-000.txt", @@ -490,7 +1122,7 @@ def test_main_reports_when_no_locks_exist( capsys: pytest.CaptureFixture[str], ) -> None: """The CLI distinguishes an empty trusted base from a failed extraction.""" - monkeypatch.setattr(materializer, "materialize", lambda *_args: []) + monkeypatch.setattr(materializer, "materialize", lambda *_args, **_kwargs: []) assert ( materializer.main( @@ -518,7 +1150,14 @@ def test_main_fails_with_the_materialization_reason( ) -> None: """A materialization exception fails closed and remains diagnosable in CI.""" - def fail_materialize(_repo_root: Path, _base_sha: str, _output_dir: Path) -> None: + def fail_materialize( + _repo_root: Path, + _base_sha: str, + _output_dir: Path, + *, + target_python_version: str | None = None, + ) -> None: + del target_python_version raise OSError("fixture failure") monkeypatch.setattr(materializer, "materialize", fail_materialize) diff --git a/tests/test_noema_orchestrator_workflow_contract.py b/tests/test_noema_orchestrator_workflow_contract.py index 937cf6fe97..6a3363975e 100644 --- a/tests/test_noema_orchestrator_workflow_contract.py +++ b/tests/test_noema_orchestrator_workflow_contract.py @@ -183,6 +183,8 @@ def test_noema_review_credentials_and_llm_use_orchestrator_free() -> None: assert "nvidia/nemotron-3-ultra-550b-a55b" not in workflow assert "Resolve Noema target repository visibility" in workflow assert "target_visibility.outputs.require_zdr" in workflow + assert 'private|internal|public)' in workflow + assert 'echo "require_zdr=false"' not in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow assert ( "NOEMA_LLM_API_KEY: ${{ secrets.NOEMA_LLM_API_KEY || secrets.OPENAI_API_KEY || '' }}" @@ -348,7 +350,7 @@ def test_stale_trigger_step_still_rejects_a_genuinely_different_head( def test_noema_visibility_lookup_retries_transient_api_failures() -> None: - """Bound transient GitHub API failures without weakening visibility validation.""" + """Keep the visibility audit bounded without making it a routing dependency.""" workflow = workflow_text("noema-review.yml") start = workflow.index(" - name: Resolve Noema target repository visibility") end = workflow.index(" - name: Provision contextual-orchestrator review sidecar", start) @@ -359,6 +361,8 @@ def test_noema_visibility_lookup_retries_transient_api_failures() -> None: assert 'sleep "$(( target_visibility_attempt * 5 ))"' in visibility_step assert "possibly a transient GitHub API rate limit; retrying after backoff." in visibility_step assert "case \"$visibility\" in" in visibility_step + assert 'echo "require_zdr=true" >>"$GITHUB_OUTPUT"' in visibility_step + assert "Noema target visibility was unavailable; the review remains bound to the mandatory ZDR-only pool." in visibility_step def test_strix_gateway_default_and_noema_sidecar_fail_closed(tmp_path: Path) -> None: diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 37ec068db9..dc1cf7e481 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -741,12 +741,44 @@ def test_opencode_target_coverage_materializes_only_after_authorized_dispatch(): assert 'install -m 0755 "$trusted_base_python_installer"' in measure_step assert "COPY install-base-python-locks.py" in measure_step assert "python3 -I /usr/local/libexec/install-base-python-locks.py" in measure_step + assert "--no-archives" in measure_step + assert "--archives-only" in measure_step + assert "RUN --network=none python3 -I /usr/local/libexec/install-base-python-locks.py" in measure_step + assert "from packaging.markers import Marker, default_environment" in measure_step + assert ( + 'archive_manifest_path = requirements_root / "archive-manifest.json"' + in measure_step + ) + assert ( + "json.loads(archive_manifest_path.read_text(encoding=\"utf-8\"))\n" + " if archive_manifest_path.exists()\n" + " else []" + ) in measure_step + assert measure_step.index("archive_manifest_path.exists()") < measure_step.index( + "coverage_environment = default_environment()" + ) + assert "coverage_environment = default_environment()" in measure_step + assert "not Marker(marker).evaluate(coverage_environment)" in measure_step + assert measure_step.index("not Marker(marker).evaluate") < measure_step.index( + "archive = (requirements_root / relative_file).resolve()" + ) + assert measure_step.index("not Marker(marker).evaluate") < measure_step.index( + "cargo fetch --locked" + ) + assert "import tomllib" in measure_step + assert 'build_system.get("build-backend") != "maturin"' in measure_step + assert 'r"maturin(?:\\[.*\\])?(?:\\s*[<>=!~].*)?"' in measure_step + assert "archive must expose exactly one pyproject.toml" in measure_step + assert "archive build backend is not the installed maturin contract" in measure_step assert '"https://github.com/ContextualWisdomLab/${repository}.git"' in measure_step assert '--quiet --no-tags --depth=1 origin "$commit"' in measure_step assert 'rev-parse FETCH_HEAD)" = "$commit"' in measure_step assert 'rev-parse HEAD)" = "$commit"' in measure_step assert "opencode-base-vcs-dependencies.pth" in measure_step assert 'vcs-manifest.json >"$dependency_list"' in measure_step + assert 'maturin>=1.10,<2.0' in Path( + "requirements-opencode-review-ci.txt" + ).read_text(encoding="utf-8") assert 'done <"$dependency_list"' in measure_step assert 'candidate_count=$((candidate_count + 1))' in measure_step assert '[ "$candidate_count" -ne 1 ]' in measure_step diff --git a/tests/test_opencode_coverage_identity.py b/tests/test_opencode_coverage_identity.py index c446901008..8ebec8aaa3 100644 --- a/tests/test_opencode_coverage_identity.py +++ b/tests/test_opencode_coverage_identity.py @@ -150,6 +150,14 @@ def unexpected_run(args, **kwargs): identity.fetch_check_runs("ContextualWisdomLab/kaefa", "not-a-sha") +def test_repository_identity_accepts_leading_dot_but_rejects_path_segments() -> None: + """Central dot repositories are valid while dot paths and options fail closed.""" + assert identity.REPO_RE.fullmatch("ContextualWisdomLab/.github") + assert not identity.REPO_RE.fullmatch("owner/.") + assert not identity.REPO_RE.fullmatch("owner/..") + assert not identity.REPO_RE.fullmatch("owner/-repo") + + def test_fetch_check_runs_retries_transient_github_read_failure(monkeypatch) -> None: """A transient 429 is retried before exact-head identity fails closed.""" diff --git a/tests/test_opencode_review_receipt_gate.py b/tests/test_opencode_review_receipt_gate.py index c971e2128a..e625496bc9 100644 --- a/tests/test_opencode_review_receipt_gate.py +++ b/tests/test_opencode_review_receipt_gate.py @@ -298,6 +298,11 @@ def fake_pages(args, **kwargs): == 0 ) + assert receipt.REPO_RE.fullmatch("ContextualWisdomLab/.github") + assert not receipt.REPO_RE.fullmatch("owner/.") + assert not receipt.REPO_RE.fullmatch("owner/..") + assert not receipt.REPO_RE.fullmatch("owner/-repo") + def fake_fail(args, **kwargs): return type("Completed", (), {"returncode": 1, "stdout": "", "stderr": "nope"})() diff --git a/tests/test_opencode_workflow_shell_syntax.py b/tests/test_opencode_workflow_shell_syntax.py index b0a672b1a2..c4f4eb2a1a 100644 --- a/tests/test_opencode_workflow_shell_syntax.py +++ b/tests/test_opencode_workflow_shell_syntax.py @@ -95,6 +95,51 @@ def test_opencode_review_comment_helpers_are_shared_and_valid_bash(): assert result.returncode == 0, result.stderr +def test_cargo_fetch_run_step_is_valid_posix_sh_not_only_bash(): + """The coverage image's base is Debian, whose default ``/bin/sh`` is dash. + + Docker's shell-form ``RUN`` executes under the image's default shell, not + bash, and dash does not implement bash's ``read -d`` extension. A prior + version of this step used ``while IFS= read -r -d "" ...`` to walk + NUL-delimited ``find -print0`` output; under dash that ``read`` fails on + every invocation, the ``while`` loop body never runs, and -- because a + failing loop condition is not itself a ``set -e`` trigger -- the whole + ``RUN`` step still reports success with the Cargo archive cache left + empty. Assert both that the bash-only construct is gone and that the + replacement genuinely parses under a real POSIX ``dash``. + """ + workflow_text = (REPO_ROOT / ".github/workflows/opencode-review-dispatch.yml").read_text( + encoding="utf-8" + ) + start_anchor = "RUN set -eu; \\\n find /opt/base-python-archive-sources" + start = workflow_text.index(start_anchor) + end_anchor = ( + "\n RUN --network=none " + "python3 -I /usr/local/libexec/install-base-python-locks.py" + ) + end = workflow_text.index(end_anchor, start) + script = workflow_text[start:end] + + assert "read -d" not in script + assert "xargs -0" in script + assert "cargo fetch --locked --manifest-path" in script + + if sys.platform == "win32": + return + dash = shutil.which("dash") + if dash is None: + return + result = subprocess.run( + [dash, "-n"], + input=script.removeprefix("RUN "), + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + def test_merge_scheduler_review_followup_run_block_is_valid_bash(): """The App-review follow-up keeps its dynamic wait logic valid Bash.""" if sys.platform == "win32": diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 2d2304aaf1..83f077aadc 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "26e8555967171a5f3974602ac05700c27bddebf1" +REVIEW_DISPATCH_BLOB_SHA = "a6fa24a624a8f1547b2f1a7f277137013feef01b" def _workflow_text(path: Path) -> str: diff --git a/tests/test_repository_branch_coverage_review_schedulers.py b/tests/test_repository_branch_coverage_review_schedulers.py index 04defb2d3f..03405f83cb 100644 --- a/tests/test_repository_branch_coverage_review_schedulers.py +++ b/tests/test_repository_branch_coverage_review_schedulers.py @@ -127,6 +127,16 @@ def test_autofix_context_renders_legacy_status_context() -> None: ) == ["- security: SUCCESS"] +@pytest.mark.parametrize("module", [autofix_context, fix_scheduler, auto_rebase]) +def test_review_schedulers_reject_path_like_repository_names(module: Any) -> None: + """All sibling scheduler entrypoints reject dot path segments consistently.""" + + assert module.REPO_RE.fullmatch("ContextualWisdomLab/.github") + assert not module.REPO_RE.fullmatch("owner/.") + assert not module.REPO_RE.fullmatch("owner/..") + assert not module.REPO_RE.fullmatch("owner/-repo") + + def test_fix_scheduler_queue_includes_eligible_pr_without_fix_need( monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], diff --git a/tests/test_required_workflow_queue_contract.py b/tests/test_required_workflow_queue_contract.py index 803d43ab59..131af02a6f 100644 --- a/tests/test_required_workflow_queue_contract.py +++ b/tests/test_required_workflow_queue_contract.py @@ -765,6 +765,8 @@ def test_noema_review_credentials_and_orchestrator_configuration_fail_closed() - ) assert "Resolve Noema target repository visibility" in workflow assert "target_visibility.outputs.require_zdr" in workflow + assert 'private|internal|public)' in workflow + assert 'echo "require_zdr=false"' not in workflow assert "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR" in workflow assert "https://integrate.api.nvidia.com/v1/chat/completions" not in workflow assert "nvidia/nemotron-3-ultra-550b-a55b" not in workflow diff --git a/tests/test_strix_contextual_orchestrator_contract.py b/tests/test_strix_contextual_orchestrator_contract.py index 52763ecc88..737a394730 100644 --- a/tests/test_strix_contextual_orchestrator_contract.py +++ b/tests/test_strix_contextual_orchestrator_contract.py @@ -54,10 +54,10 @@ def test_model_override_cannot_escape_the_gateway(self) -> None: for direct_route in ("nvidia_nim/*)", "openrouter/free", "openai-direct/gpt-5.4"): self.assertNotIn(direct_route, self.workflow) - def test_private_gateway_scans_require_zdr_only_routing(self) -> None: - """Private source never enters the gateway's non-ZDR fallback tier.""" + def test_gateway_scans_require_zdr_only_routing(self) -> None: + """Every source never enters the gateway's non-ZDR fallback tier.""" self.assertIn( - "CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: ${{ steps.target_visibility.outputs.is_private }}", + 'CONTEXTUAL_ORCHESTRATOR_REQUIRE_ZDR: "true"', self.workflow, ) diff --git a/tests/test_uv_export_isolation_contract.py b/tests/test_uv_export_isolation_contract.py index 76b72fdc7f..95e3afb459 100644 --- a/tests/test_uv_export_isolation_contract.py +++ b/tests/test_uv_export_isolation_contract.py @@ -97,6 +97,80 @@ def test_uv_export_accepts_exact_package_pins_with_markers_and_multiple_hashes() assert materializer._is_fully_hash_pinned_export(content) is True +def test_uv_export_accepts_hash_pinned_organization_archive_as_registry_lock() -> None: + """A trusted HTTPS archive with a complete hash remains a pip lock entry.""" + content = ( + b"fast-mlsirm @ https://github.com/ContextualWisdomLab/fast-mlsirm/" + b"archive/refs/tags/v0.9.1.tar.gz ; python_full_version >= '3.12' \\\n" + b" --hash=sha256:" + b"a" * 64 + b"\n" + ) + + registry, vcs_sources, archive_sources = materializer._partition_uv_export(content) + + assert materializer._is_fully_hash_pinned_export(content) is True + assert registry == b"" + assert vcs_sources == [] + assert archive_sources == [ + { + "package": "fast-mlsirm", + "url": "https://github.com/ContextualWisdomLab/fast-mlsirm/archive/refs/tags/v0.9.1.tar.gz", + "hashes": ["a" * 64], + "marker": "python_full_version >= '3.12'", + } + ] + + +def test_uv_export_rejects_organization_archive_without_complete_sha256_hash() -> None: + """Organization archives must carry a complete SHA-256 hash before partitioning.""" + content = ( + b"demo @ https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz " + b"--hash=sha256:abcd\n" + ) + + with pytest.raises(ValueError, match="complete SHA-256 hashes"): + materializer._partition_uv_export(content) + + +def test_uv_export_rejects_conflicting_hashes_for_one_archive_url() -> None: + """One archive URL cannot be admitted with two different digests.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + content = ( + f"demo @ {url} --hash=sha256:{'a' * 64}\n" + f"demo @ {url} --hash=sha256:{'b' * 64}\n" + ).encode() + + with pytest.raises(ValueError, match="conflicting hashes"): + materializer._partition_uv_export(content) + + +def test_uv_export_keeps_same_archive_url_for_distinct_markers() -> None: + """Conditional alternatives sharing a URL remain separate requirements.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + content = ( + f"demo @ {url} ; python_version < '3.10' --hash=sha256:{'a' * 64}\n" + f"demo @ {url} ; python_version >= '3.10' --hash=sha256:{'a' * 64}\n" + ).encode() + + _registry, _vcs_sources, archive_sources = materializer._partition_uv_export(content) + + assert [archive["marker"] for archive in archive_sources] == [ + "python_version < '3.10'", + "python_version >= '3.10'", + ] + + +def test_uv_export_rejects_different_hashes_for_same_url_across_markers() -> None: + """Conditional alternatives cannot change the immutable archive payload.""" + url = "https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz" + content = ( + f"demo @ {url} ; python_version < '3.10' --hash=sha256:{'a' * 64}\n" + f"demo @ {url} ; python_version >= '3.10' --hash=sha256:{'b' * 64}\n" + ).encode() + + with pytest.raises(ValueError, match="conflicting hashes"): + materializer._partition_uv_export(content) + + def test_uv_export_partitions_hashes_and_exact_organization_vcs_sources() -> None: """An immutable organization source pin is separated from pip hash locks.""" content = ( @@ -105,7 +179,7 @@ def test_uv_export_partitions_hashes_and_exact_organization_vcs_sources() -> Non b"61c49c50d3b4a24fc9bd7c6d3a7f2f4ba19d7be6\n" ) - registry, vcs_sources = materializer._partition_uv_export(content) + registry, vcs_sources, archive_sources = materializer._partition_uv_export(content) assert registry == b"demo==1.2.3 --hash=sha256:" + b"a" * 64 + b"\n" assert vcs_sources == [ @@ -116,6 +190,7 @@ def test_uv_export_partitions_hashes_and_exact_organization_vcs_sources() -> Non "commit": "61c49c50d3b4a24fc9bd7c6d3a7f2f4ba19d7be6", } ] + assert archive_sources == [] @pytest.mark.parametrize( @@ -135,6 +210,24 @@ def test_uv_export_rejects_unbounded_vcs_sources(requirement: str) -> None: materializer._partition_uv_export(f"{requirement}\n".encode()) +@pytest.mark.parametrize( + "requirement", + [ + "demo @ http://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz", + "demo @ https://github.com/other/demo/archive/v1.tar.gz", + "demo @ https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz?download=1", + "demo @ https://github.com/ContextualWisdomLab/demo/archive/v1.tar.gz#fragment", + "demo @ https://github.com/ContextualWisdomLab/demo/archive/../v1.tar.gz", + ], +) +def test_uv_export_rejects_unbounded_archive_sources(requirement: str) -> None: + """Only archive URLs from the exact organization origin are accepted.""" + with pytest.raises(ValueError, match="unsupported dependency"): + materializer._partition_uv_export( + f"{requirement} --hash=sha256:{'a' * 64}\n".encode() + ) + + def test_uv_export_rejects_conflicting_commits_for_one_repository() -> None: """One import path cannot ambiguously combine two repository revisions.""" with pytest.raises(ValueError, match="conflicting commits"): diff --git a/tests/test_zdr_policy.py b/tests/test_zdr_policy.py index 90c7fe4197..7b1cbe66e0 100644 --- a/tests/test_zdr_policy.py +++ b/tests/test_zdr_policy.py @@ -36,6 +36,18 @@ def test_provider_zdr_scope_rejects_unknown_provider() -> None: zdr_policy.provider_zdr_scope("made_up_provider") +def test_is_zdr_model_rejects_non_string_model() -> None: + """Defensive route evaluation must fail closed for malformed model values.""" + assert ( + zdr_policy.is_zdr_model( + "openrouter", + model=object(), # type: ignore[arg-type] + zdr_endpoints=frozenset({"openrouter/provider/model"}), + ) + is False + ) + + @pytest.mark.parametrize( ("provider_name", "expected_zdr"), [ @@ -80,6 +92,14 @@ def test_is_zdr_model_openrouter_feed_is_authoritative_when_present() -> None: ) is False ) + assert ( + zdr_policy.is_zdr_model( + "openrouter", + model="other/deepseek-r1:free", + zdr_endpoints=frozenset({"openrouter/deepseek/deepseek-r1:free"}), + ) + is False + ) def test_route_key_strips_a_leading_slash() -> None: @@ -89,10 +109,78 @@ def test_route_key_strips_a_leading_slash() -> None: ) -def test_is_zdr_model_feed_only_applies_to_the_openrouter_scope() -> None: - """Static non-ZDR providers stay non-ZDR even if a route key is present.""" - feed = frozenset({"nvidia_nim/nvidia/nemotron-3-nano-30b-a3b"}) - assert zdr_policy.is_zdr_model("nvidia_nim", zdr_endpoints=feed) is False +def test_is_zdr_model_feed_evidence_matches_other_provider_model_ids() -> None: + """OpenRouter model evidence selects matching candidates from other providers.""" + feed = frozenset({"openrouter/deepseek/deepseek-r1:free"}) + assert ( + zdr_policy.is_zdr_model( + "nvidia_nim", + model="deepseek/deepseek-r1:free", + zdr_endpoints=feed, + ) + is True + ) + assert ( + zdr_policy.is_zdr_model( + "nvidia_nim", + model="nvidia/nemotron-3-nano-30b-a3b", + zdr_endpoints=feed, + ) + is False + ) + assert ( + zdr_policy.is_zdr_model( + "nvidia_nim", + model="deepseek-r1:free", + zdr_endpoints=feed, + ) + is True + ) + assert ( + zdr_policy.is_zdr_model( + "nvidia_nim", + model="nvidia/deepseek-r1:free", + zdr_endpoints=frozenset( + { + "openrouter/deepseek/deepseek-r1:free", + "openrouter/other/deepseek-r1:free", + } + ), + ) + is False + ) + + +def test_is_zdr_model_rejects_noncanonical_feed_provider_keys() -> None: + """Only canonical OpenRouter feed routes may provide cross-provider evidence.""" + assert ( + zdr_policy.is_zdr_model( + "nvidia_nim", + model="deepseek/deepseek-r1:free", + zdr_endpoints=frozenset({"nvidia_nim/deepseek/deepseek-r1:free"}), + ) + is False + ) + + +@pytest.mark.parametrize( + "feed_key", + [ + "openrouter//deepseek/deepseek-r1:free", + "openrouter/deepseek/deepseek-r1:free/", + "openrouter/", + ], +) +def test_is_zdr_model_rejects_feed_keys_with_empty_segments(feed_key: str) -> None: + """Malformed feed paths cannot grant suffix-based ZDR evidence.""" + assert ( + zdr_policy.is_zdr_model( + "nvidia_nim", + model="deepseek/deepseek-r1:free", + zdr_endpoints=frozenset({feed_key}), + ) + is False + ) @pytest.mark.parametrize( @@ -116,4 +204,4 @@ def test_is_zdr_model_feed_only_applies_to_the_openrouter_scope() -> None: ) def test_is_free_route(value: object, expected: bool) -> None: """Only explicitly truthy free markers count; strings are case-folded.""" - assert zdr_policy.is_free_route(value) is expected \ No newline at end of file + assert zdr_policy.is_free_route(value) is expected