From 401152b45fbb9e55922df9413f9af101753da483 Mon Sep 17 00:00:00 2001 From: Patrick Ryan Date: Sat, 10 Oct 2026 23:10:03 -0500 Subject: [PATCH] policy v2.2: IMG-18 input hashes, IMG-19/20 VEX coverage report; assert-image hardening checks --- .github/scripts/assert-image.sh | 8 +++ .github/scripts/check-vex-coverage.py | 5 +- .github/scripts/verify_builder_policy.py | 79 +++++++++++++++++++++- .github/workflows/release.yml | 18 +++-- .gitignore | 2 + policy/builder-image/POLICY.md | 68 ++++++++++++++++++- policy/builder-image/policy.json | 53 ++++++++++++++- tests/test_assert_image.py | 8 +++ tests/test_policy_tool.py | 2 +- tests/test_verify_builder_policy.py | 86 ++++++++++++++++++++++-- 10 files changed, 310 insertions(+), 19 deletions(-) diff --git a/.github/scripts/assert-image.sh b/.github/scripts/assert-image.sh index 20448ca..8ebe624 100755 --- a/.github/scripts/assert-image.sh +++ b/.github/scripts/assert-image.sh @@ -13,6 +13,10 @@ no_netstack() { for c in opendal reqwest rustls openssl hyper-rustls; do [ "$( pinned() { [ "$(count_purl "pkg:cargo/$1")" -ge 1 ]; } frame_env() { podman image inspect "$IMG" --format '{{range .Config.Env}}{{println .}}{{end}}' | grep -q '^SCCACHE_MAX_FRAME_LENGTH='; } listen_ok() { podman run --rm -v "$HERE/listen-check.sh:/listen-check.sh:ro" --entrypoint sh "$IMG" /listen-check.sh; } +no_readelf_objdump() { podman run --rm --entrypoint sh "$IMG" -c '! find / -xdev ! -type d \( -name "*readelf*" -o -name "*objdump*" \) | grep -q .'; } +no_perl() { podman run --rm --entrypoint sh "$IMG" -c '! command -v perl >/dev/null 2>&1'; } +no_pip() { podman run --rm --entrypoint sh "$IMG" -c '! python3 -m pip --version >/dev/null 2>&1'; } +no_xcoff() { podman run --rm --entrypoint sh "$IMG" -c 'for t in ld nm objcopy strip; do $t --help 2>&1 | grep -i "supported targets" | grep -qiE "xcoff|aix|rs6000" && exit 1; done; exit 0'; } check "control: sccache binary exists in image" have_sccache check "no sccache-dist binary" no_dist_bin @@ -23,4 +27,8 @@ check "pinned bytes@1.10.1 present" pinned bytes@1.10.1 check "pinned rand@0.8.5 present" pinned rand@0.8.5 check "SCCACHE_MAX_FRAME_LENGTH set in image" frame_env check "server seen on loopback; no other listeners" listen_ok +check "no readelf or objdump in image" no_readelf_objdump +check "no perl in image" no_perl +check "no pip in image" no_pip +check "no XCOFF/AIX target in binutils" no_xcoff exit $fail diff --git a/.github/scripts/check-vex-coverage.py b/.github/scripts/check-vex-coverage.py index 4adcd5c..8fd807f 100644 --- a/.github/scripts/check-vex-coverage.py +++ b/.github/scripts/check-vex-coverage.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # usage: check-vex-coverage.py SCAN.json SBOM.json VEX.json OUT.json -import json, sys, datetime as dt +import hashlib, json, sys, datetime as dt MAX_DB_AGE_DAYS, MIN_COMPONENTS = 3, 100 @@ -27,6 +27,7 @@ def ok_status(s, severity): def main(scan_p, sbom_p, vex_p, out_p): scan, sbom, vex = (json.load(open(p)) for p in (scan_p, sbom_p, vex_p)) + sha = {k: hashlib.sha256(open(q, 'rb').read()).hexdigest() for k, q in (('scan', scan_p), ('sbom', sbom_p), ('vex', vex_p))} problems, covered, uncovered, seen = [], [], [], set() n = len(sbom.get("components", [])) if n < MIN_COMPONENTS: @@ -55,7 +56,7 @@ def main(scan_p, sbom_p, vex_p, out_p): json.dump({"verdict": verdict, "scanner": desc.get("name"), "scanner_version": desc.get("version"), "db_built": built, "sbom_components": n, "findings": len(covered) + len(uncovered), "covered": covered, "uncovered": uncovered, "problems": problems, - "vex_statements_without_finding": unused}, open(out_p, "w"), indent=2, sort_keys=True) + "vex_statements_without_finding": unused, "input_sha256": sha}, open(out_p, "w"), indent=2, sort_keys=True) print(f"{verdict}: {len(covered)} covered, {len(uncovered)} uncovered, problems={problems}") return 0 if verdict == "pass" else 1 diff --git a/.github/scripts/verify_builder_policy.py b/.github/scripts/verify_builder_policy.py index 5b0a2bf..1e433e1 100644 --- a/.github/scripts/verify_builder_policy.py +++ b/.github/scripts/verify_builder_policy.py @@ -38,7 +38,7 @@ class Ctx: def __init__(self, *, root, tag, tag_sha, inputs, digests, sboms, vex, spdx=None, assets=None, run=None, git_show=None, - asserts=None, artifacts_dir=None): + asserts=None, artifacts_dir=None, coverage=None): self.root = Path(root) self.tag = tag self.tag_sha = tag_sha @@ -49,6 +49,7 @@ def __init__(self, *, root, tag, tag_sha, inputs, digests, sboms, vex, self.vex = vex # parsed OpenVEX self.assets = assets # list[str] | None self.asserts = asserts or {} # arch -> parsed assert-image..json + self.coverage = coverage or {} # arch -> parsed vex-coverage..json self.artifacts_dir = Path(artifacts_dir) if artifacts_dir else None self.run = run or _run self.git_show = git_show or _git_show(self.root, tag_sha, self.run) @@ -330,6 +331,80 @@ def check_gate_script_hash(ctx, params): return errs +def check_input_hashes(ctx, params): + """builder-inputs records the sha256 of each input file, and it equals the sha256 of that + file at the tagged commit, for both architectures.""" + errs = [] + for field, tpl in params["files"].items(): + for a in ARCHES: + path = tpl.format(arch=a) + want = hashlib.sha256(ctx.git_show(path)).hexdigest() + got = (ctx.inputs.get(a) or {}).get(field) + if got != want: + errs.append(f"{a}: {field} {got!r} != sha256 of {path} at {ctx.tag_sha[:12]} ({want})") + return errs + + +def check_coverage_report(ctx, params): + """vex-coverage..json (written by check-vex-coverage.py in release.yml) says pass, is + internally consistent, and is tied by sha256 to the released CycloneDX SBOM, the OpenVEX file + at the tagged commit and the vulnerability scan stored with the release run.""" + errs = [] + vex_want = hashlib.sha256(ctx.git_show(params["vex_path"])).hexdigest() + for a in ARCHES: + doc = ctx.coverage.get(a) + if not isinstance(doc, dict): + errs.append(f"{a}: vex-coverage.{a}.json missing") + continue + if doc.get("verdict") != "pass": + errs.append(f"{a}: coverage verdict {doc.get('verdict')!r}, expected 'pass'") + if doc.get("scanner") != params["scanner"]: + errs.append(f"{a}: coverage scanner {doc.get('scanner')!r} != {params['scanner']!r}") + for f in ("scanner_version", "db_built"): + val = doc.get(f) + if not (isinstance(val, str) and val.strip()): + errs.append(f"{a}: coverage report {f} missing or empty") + if doc.get("problems") != []: + errs.append(f"{a}: coverage report lists problems: {doc.get('problems')!r}") + covered, uncovered = doc.get("covered"), doc.get("uncovered") + if uncovered != []: + errs.append(f"{a}: coverage report has uncovered findings: {uncovered!r}") + if not isinstance(covered, list): + errs.append(f"{a}: coverage report covered is not a list") + elif doc.get("findings") != len(covered): + errs.append(f"{a}: coverage findings {doc.get('findings')!r} != {len(covered)} covered findings") + sb = ctx.sboms.get(a) + n = len(sb.get("components", [])) if isinstance(sb, dict) else None + if n is None or doc.get("sbom_components") != n: + errs.append(f"{a}: coverage sbom_components {doc.get('sbom_components')!r} != {n!r} components in the released SBOM") + hashes = doc.get("input_sha256") + if not isinstance(hashes, dict): + errs.append(f"{a}: coverage report has no input_sha256") + continue + for key, name in (("scan", f"vuln-scan.{a}.json"), ("sbom", f"sbom.{a}.cdx.json")): + p = artifact_path(ctx, a, name) + if p is None or not p.exists(): + errs.append(f"{a}: {name} not found, cannot compare input_sha256.{key}") + elif hashes.get(key) != hashlib.sha256(p.read_bytes()).hexdigest(): + errs.append(f"{a}: coverage input_sha256.{key} does not match {name}") + if hashes.get("vex") != vex_want: + errs.append(f"{a}: coverage input_sha256.vex does not match {params['vex_path']} at {ctx.tag_sha[:12]}") + return errs + + +def check_coverage_attested(ctx, params): + """vex-coverage..json is the predicate of a verified attestation on the image digest, + made by the release workflow from the release tag.""" + errs = [] + sref = params["source_ref_template"].format(tag=ctx.tag) + for a in ARCHES: + d = _digest_ok(ctx, a, errs) + if d: + _predicate_equals(ctx, a, d, params, params["predicate_type"], sref, + ctx.coverage.get(a), f"vex-coverage.{a}.json", errs) + return errs + + def check_pinned_crates(ctx, params): sboms, errs = _per_arch(ctx, ctx.sboms.get, "SBOM") for a, s in sboms.items(): @@ -557,11 +632,13 @@ def f(arch, name): spdx = {x: jload(f(x, f"sbom.{x}.spdx.json")) for x in ARCHES} assets = a.assets_file.read_text().split() if a.assets_file else None asserts = {x: jload(f(x, f"assert-image.{x}.json")) for x in ARCHES} + coverage = {x: jload(f(x, f"vex-coverage.{x}.json")) for x in ARCHES} return Ctx(root=a.root, tag=a.tag, tag_sha=a.tag_sha, inputs=inputs, digests=digests, sboms={k: v for k, v in cdx.items() if v is not None}, spdx={k: v for k, v in spdx.items() if v is not None}, vex=jload(a.vex), assets=assets, asserts={k: v for k, v in asserts.items() if v is not None}, + coverage={k: v for k, v in coverage.items() if v is not None}, artifacts_dir=a.artifacts_dir) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 822a966..9b91159 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ name: release-image # gate require a successful build-environment-image run on main for exactly this commit # attest per arch: verify the build provenance, generate SBOMs with pinned Syft, run # assert-image.sh against the image and its SBOM (the facts the VEX relies on), and attest -# the CycloneDX SBOM, the SPDX SBOM, the VEX and the assertion results on the image digest +# the CycloneDX SBOM, the SPDX SBOM, the VEX, the assertion results and the VEX coverage report on the image digest # (signer: this workflow). The build log from the build run travels with them. # release load the published builder policy, check its sha256 against the value committed # below, run verify_builder_policy.py (one check per enforced rule), and only then @@ -34,10 +34,11 @@ env: VEX_PATH: environments/alpine-3.24-musl/vex/builder.openvex.json VEX_PREDICATE_TYPE: https://openvex.dev/ns/v0.2.0 ASSERT_PREDICATE_TYPE: https://github.com/patrickryankenneth/python-wheels-builder/predicate/assert-image/v1 - POLICY_TAG: policy-builder-image-v2.1 + COVERAGE_PREDICATE_TYPE: https://github.com/patrickryankenneth/python-wheels-builder/predicate/vex-coverage/v1 + POLICY_TAG: policy-builder-image-v2.2 # sha256 of policy/builder-image/policy.json. Update this whenever the policy changes # (tests/test_verify_builder_policy.py fails if it drifts). - POLICY_SHA256: a9d65504d7ebe4bc83269b6d4880877f04aa24e8810d670b580f21f7b4a8e29b + POLICY_SHA256: 3f41bc1fdba9b96de0dc3f7cf9c30c5fe40b1df8880cdaae1dc75ce2c0b56055 jobs: gate: @@ -240,6 +241,15 @@ jobs: predicate-path: out/assert-image.${{ matrix.arch }}.json push-to-registry: true + - name: Attest VEX coverage report + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4.2.2 + with: + subject-name: ${{ env.IMAGE }} + subject-digest: ${{ env.D }} + predicate-type: ${{ env.COVERAGE_PREDICATE_TYPE }} + predicate-path: out/vex-coverage.${{ matrix.arch }}.json + push-to-registry: true + - name: Upload SBOMs, digest and inputs uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: @@ -292,7 +302,7 @@ jobs: set -euo pipefail mkdir -p release-assets for a in x86_64 aarch64; do - for f in "builder-inputs.$a.json" "image.digest.$a" "sbom.$a.cdx.json" "sbom.$a.spdx.json" "assert-image.$a.json" "build-log.$a.txt.gz"; do + for f in "builder-inputs.$a.json" "image.digest.$a" "sbom.$a.cdx.json" "sbom.$a.spdx.json" "assert-image.$a.json" "build-log.$a.txt.gz" "vex-coverage.$a.json"; do cp "artifacts/sbom-$a/$f" release-assets/ done done diff --git a/.gitignore b/.gitignore index 6c56ff1..1f07f6b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ __pycache__/ .pytest_cache/ +out/ +*.pem diff --git a/policy/builder-image/POLICY.md b/policy/builder-image/POLICY.md index 710ad8a..1835fd3 100644 --- a/policy/builder-image/POLICY.md +++ b/policy/builder-image/POLICY.md @@ -1,4 +1,4 @@ -# Policy builder-image v2.1 +# Policy builder-image v2.2 @@ -207,7 +207,9 @@ Parameters: "sbom.aarch64.cdx.json", "sbom.aarch64.spdx.json", "sbom.x86_64.cdx.json", - "sbom.x86_64.spdx.json" + "sbom.x86_64.spdx.json", + "vex-coverage.aarch64.json", + "vex-coverage.x86_64.json" ] } ``` @@ -343,7 +345,11 @@ Parameters: "pinned bytes@1.10.1 present", "pinned rand@0.8.5 present", "SCCACHE_MAX_FRAME_LENGTH set in image", - "server seen on loopback; no other listeners" + "server seen on loopback; no other listeners", + "no readelf or objdump in image", + "no perl in image", + "no pip in image", + "no XCOFF/AIX target in binutils" ] } ``` @@ -387,6 +393,62 @@ Parameters: } ``` +### IMG-18 - Locked inputs match the committed files + +- Status: `enforced` +- Check: `check_input_hashes` +- Implemented by: build-environment-image.yml writes the hashes to builder-inputs.json; verify_builder_policy.py compares them to the files at the tagged commit + +For both architectures, builder-inputs.json records the sha256 of apk.lock. and of requirements-builder.txt, and each equals the sha256 of that file at the tagged commit. The image was therefore built against the package list and Python requirements committed at the release tag. + +Parameters: + +```json +{ + "files": { + "apk_lock_sha256": "environments/alpine-3.24-musl/apk.lock.{arch}", + "requirements_builder_sha256": "environments/alpine-3.24-musl/requirements-builder.txt" + } +} +``` + +### IMG-19 - VEX coverage report passes + +- Status: `enforced` +- Check: `check_coverage_report` +- Implemented by: release.yml runs check-vex-coverage.py (Grype scan of the image against the SBOM and VEX) and writes vex-coverage..json; verify_builder_policy.py checks the report against the released files + +For both architectures, the release carries vex-coverage..json with verdict pass: no problems, no vulnerability finding without a matching VEX statement, and a finding count equal to the covered findings. Its recorded sha256 values match the released CycloneDX SBOM, the OpenVEX file at the tagged commit and the vulnerability scan stored with the release run's artifacts. The scan is not a release asset and is not attested, so its hash only ties the report to that run's files. The report shows that every finding Grype reported at release time is fixed or has a VEX statement; it says nothing about vulnerabilities Grype does not know of. + +Parameters: + +```json +{ + "scanner": "grype", + "vex_path": "environments/alpine-3.24-musl/vex/builder.openvex.json" +} +``` + +### IMG-20 - VEX coverage report is attested + +- Status: `enforced` +- Check: `check_coverage_attested` +- Implemented by: release.yml attests vex-coverage..json on the image digest; verify_builder_policy.py compares the released file to the attested predicate + +For both architectures, vex-coverage..json is the predicate of a GitHub attestation on the image digest, with the predicate type named in the parameters, from the release workflow, from the release tag, at the tagged commit. + +Parameters: + +```json +{ + "image": "ghcr.io/patrickryankenneth/python-wheels-builder-alpine", + "predicate_type": "https://github.com/patrickryankenneth/python-wheels-builder/predicate/vex-coverage/v1", + "repo": "patrickryankenneth/python-wheels-builder", + "signer_workflow": "patrickryankenneth/python-wheels-builder/.github/workflows/release.yml", + "source_ref_template": "refs/tags/{tag}" +} +``` + ## Not guaranteed - The image is not reproducible, and apk packages, rustup archives and crate tarballs have no content-hash pins. diff --git a/policy/builder-image/policy.json b/policy/builder-image/policy.json index 9859864..0c5e042 100644 --- a/policy/builder-image/policy.json +++ b/policy/builder-image/policy.json @@ -14,7 +14,7 @@ "The assert-image results show that the listed checks passed on the image and its SBOM at release time. They check those properties only and are not a security review of the image.", "The build log is the build workflow's own output, attested as produced by that workflow. It is evidence for later inspection, not an independent record, and it is not a complete log of the release workflow run." ], - "policy_version": "v2.1", + "policy_version": "v2.2", "rules": [ { "check": "check_image_attested", @@ -173,7 +173,9 @@ "sbom.aarch64.cdx.json", "sbom.aarch64.spdx.json", "sbom.x86_64.cdx.json", - "sbom.x86_64.spdx.json" + "sbom.x86_64.spdx.json", + "vex-coverage.aarch64.json", + "vex-coverage.x86_64.json" ] }, "statement": "The draft release's asset set equals this list. SHA256SUMS and its Sigstore bundle are added by the maintainer when signing, after this check, and are checked by the publish guard.", @@ -284,7 +286,11 @@ "pinned bytes@1.10.1 present", "pinned rand@0.8.5 present", "SCCACHE_MAX_FRAME_LENGTH set in image", - "server seen on loopback; no other listeners" + "server seen on loopback; no other listeners", + "no readelf or objdump in image", + "no perl in image", + "no pip in image", + "no XCOFF/AIX target in binutils" ] }, "statement": "For both architectures, assert-image..json names this image digest, the tagged commit and the sha256 of the released CycloneDX SBOM, lists exactly the required checks, and every check is PASS. The checks are the ones assert-image.sh runs against the image and its SBOM, including the facts the VEX statements rely on.", @@ -319,6 +325,47 @@ "statement": "For both architectures, the release carries build-log..txt.gz, a non-empty gzip that mentions the image digest, and the file itself has a GitHub attestation from the named workflow, from refs/heads/main, at exactly the commit the release tag points to. The log is what the build workflow printed; it lets the build be inspected after Actions logs expire.", "status": "enforced", "title": "Build log is attested" + }, + { + "check": "check_input_hashes", + "id": "IMG-18", + "implemented_by": "build-environment-image.yml writes the hashes to builder-inputs.json; verify_builder_policy.py compares them to the files at the tagged commit", + "parameters": { + "files": { + "apk_lock_sha256": "environments/alpine-3.24-musl/apk.lock.{arch}", + "requirements_builder_sha256": "environments/alpine-3.24-musl/requirements-builder.txt" + } + }, + "statement": "For both architectures, builder-inputs.json records the sha256 of apk.lock. and of requirements-builder.txt, and each equals the sha256 of that file at the tagged commit. The image was therefore built against the package list and Python requirements committed at the release tag.", + "status": "enforced", + "title": "Locked inputs match the committed files" + }, + { + "check": "check_coverage_report", + "id": "IMG-19", + "implemented_by": "release.yml runs check-vex-coverage.py (Grype scan of the image against the SBOM and VEX) and writes vex-coverage..json; verify_builder_policy.py checks the report against the released files", + "parameters": { + "scanner": "grype", + "vex_path": "environments/alpine-3.24-musl/vex/builder.openvex.json" + }, + "statement": "For both architectures, the release carries vex-coverage..json with verdict pass: no problems, no vulnerability finding without a matching VEX statement, and a finding count equal to the covered findings. Its recorded sha256 values match the released CycloneDX SBOM, the OpenVEX file at the tagged commit and the vulnerability scan stored with the release run's artifacts. The scan is not a release asset and is not attested, so its hash only ties the report to that run's files. The report shows that every finding Grype reported at release time is fixed or has a VEX statement; it says nothing about vulnerabilities Grype does not know of.", + "status": "enforced", + "title": "VEX coverage report passes" + }, + { + "check": "check_coverage_attested", + "id": "IMG-20", + "implemented_by": "release.yml attests vex-coverage..json on the image digest; verify_builder_policy.py compares the released file to the attested predicate", + "parameters": { + "image": "ghcr.io/patrickryankenneth/python-wheels-builder-alpine", + "predicate_type": "https://github.com/patrickryankenneth/python-wheels-builder/predicate/vex-coverage/v1", + "repo": "patrickryankenneth/python-wheels-builder", + "signer_workflow": "patrickryankenneth/python-wheels-builder/.github/workflows/release.yml", + "source_ref_template": "refs/tags/{tag}" + }, + "statement": "For both architectures, vex-coverage..json is the predicate of a GitHub attestation on the image digest, with the predicate type named in the parameters, from the release workflow, from the release tag, at the tagged commit.", + "status": "enforced", + "title": "VEX coverage report is attested" } ], "schema": 2, diff --git a/tests/test_assert_image.py b/tests/test_assert_image.py index 0e5f896..cb8ac96 100644 --- a/tests/test_assert_image.py +++ b/tests/test_assert_image.py @@ -30,6 +30,10 @@ *"test -e /opt/cargo/bin/sccache-dist"*) [ "${STUB_DIST:-0}" = 1 ] && exit 0; exit 1 ;; *"test -x /opt/cargo/bin/sccache"*) [ "${STUB_SCCACHE:-1}" = 1 ] && exit 0; exit 1 ;; *"/listen-check.sh"*) [ "${STUB_LISTEN:-1}" = 1 ] && exit 0; exit 1 ;; + *"readelf"*) [ "${STUB_READELF:-0}" = 1 ] && exit 1; exit 0 ;; + *"command -v perl"*) [ "${STUB_PERL:-0}" = 1 ] && exit 1; exit 0 ;; + *"pip --version"*) [ "${STUB_PIP:-0}" = 1 ] && exit 1; exit 0 ;; + *"supported targets"*) [ "${STUB_XCOFF:-0}" = 1 ] && exit 1; exit 0 ;; esac exit 3 """ @@ -75,6 +79,10 @@ def harness_checks() -> list[str]: ("rand-pin-missing", "pinned rand@0.8.5 present", dict(sbom_doc=sbom(drop=["rand@0.8.5"]))), ("frame-env-missing", "SCCACHE_MAX_FRAME_LENGTH set in image", dict(sbom_doc=sbom(), STUB_FRAME=0)), ("unexpected-listener", "server seen on loopback; no other listeners", dict(sbom_doc=sbom(), STUB_LISTEN=0)), + ("readelf-or-objdump-present", "no readelf or objdump in image", dict(sbom_doc=sbom(), STUB_READELF=1)), + ("perl-present", "no perl in image", dict(sbom_doc=sbom(), STUB_PERL=1)), + ("pip-present", "no pip in image", dict(sbom_doc=sbom(), STUB_PIP=1)), + ("xcoff-target-present", "no XCOFF/AIX target in binutils", dict(sbom_doc=sbom(), STUB_XCOFF=1)), ] diff --git a/tests/test_policy_tool.py b/tests/test_policy_tool.py index e420f8a..b4ff910 100644 --- a/tests/test_policy_tool.py +++ b/tests/test_policy_tool.py @@ -49,4 +49,4 @@ def test_schema1_rejected(): def test_tag_name(): - assert f"policy-{POLICY['scope']}-{POLICY['policy_version']}" == "policy-builder-image-v2.1" \ No newline at end of file + assert f"policy-{POLICY['scope']}-{POLICY['policy_version']}" == "policy-builder-image-v2.2" \ No newline at end of file diff --git a/tests/test_verify_builder_policy.py b/tests/test_verify_builder_policy.py index 1108d64..0ae6682 100644 --- a/tests/test_verify_builder_policy.py +++ b/tests/test_verify_builder_policy.py @@ -44,7 +44,9 @@ def inputs(a): d.update({"source_commit": SHA, "assert_script_sha256": hashlib.sha256(SCRIPT).hexdigest(), "base_image_index_digest": "sha256:" + "e" * 64, "base_platform_manifest_digest": "sha256:" + "f" * 64, - "base_image_ref": "docker.io/library/alpine:3.24"}) + "base_image_ref": "docker.io/library/alpine:3.24", + "apk_lock_sha256": hashlib.sha256(SCRIPT).hexdigest(), + "requirements_builder_sha256": hashlib.sha256(SCRIPT).hexdigest()}) return {**d, "runner": {"image_os": "ubuntu24", "image_version": "1", "arch": P("IMG-6")["runner_arch"][a], "docker_version": "28", "buildx_version": "v0"}} @@ -54,8 +56,13 @@ def inputs(a): "FROM base AS final\nRUN true\n").encode() +VEX_BYTES = b'{"@context": "vex-fixture"}\n' + + def fake_show(path): - return CONTAINERFILE if path.endswith("Containerfile") else SCRIPT + if path.endswith("Containerfile"): return CONTAINERFILE + if path.endswith("builder.openvex.json"): return VEX_BYTES + return SCRIPT def good(tmp_path, run=None): @@ -87,12 +94,27 @@ def good(tmp_path, run=None): asserts[a] = {"image": f"{img}@{digs[a]}", "commit": SHA, "sbom_sha256": hashlib.sha256(raw).hexdigest(), "checks": [{"name": n, "result": "PASS"} for n in P("IMG-15")["required_checks"]]} run.by_key[(digs[a], P("IMG-16")["predicate_type"])] = copy.deepcopy(asserts[a]) + # VEX coverage report (IMG-19) is checked against the released SBOM, the scan from the same run + # and the VEX file at the tagged commit, and attested (IMG-20) + cov = {} + for a in v.ARCHES: + d = tmp_path / f"sbom-{a}" + scan_raw = json.dumps({"scan": a}).encode() + (d / f"vuln-scan.{a}.json").write_bytes(scan_raw) + cov[a] = {"verdict": "pass", "scanner": P("IMG-19")["scanner"], "scanner_version": "0.120.0", + "db_built": "2026-10-09T00:00:00Z", "sbom_components": len(cdx[a]["components"]), + "findings": 1, "uncovered": [], "problems": [], "vex_statements_without_finding": [], + "covered": [{"ids": ["X"], "package": "bytes", "version": "1.10.1", "status": "affected"}], + "input_sha256": {"scan": hashlib.sha256(scan_raw).hexdigest(), + "sbom": hashlib.sha256((d / f"sbom.{a}.cdx.json").read_bytes()).hexdigest(), + "vex": hashlib.sha256(VEX_BYTES).hexdigest()}} + run.by_key[(digs[a], P("IMG-20")["predicate_type"])] = copy.deepcopy(cov[a]) return v.Ctx(root=tmp_path, tag="v1", tag_sha=SHA, inputs=ins, digests=digs, sboms=cdx, spdx=spdx, vex=vex, assets=list(P("IMG-10")["expected_assets"]), run=run, git_show=fake_show, - asserts=asserts, artifacts_dir=tmp_path) + asserts=asserts, coverage=cov, artifacts_dir=tmp_path) def test_every_rule_has_a_check_and_a_bad_fixture(): @@ -162,6 +184,31 @@ def b_lg_empty(c): def b_lg_nodigest(c): with gzip.open(LOG(c, "aarch64"), "wt") as fh: fh.write("built, but no digest recorded\n") def b_lg_notgzip(c): LOG(c, "x86_64").write_bytes(b"not gzip") +def b_ih_lock(c): c.inputs["aarch64"]["apk_lock_sha256"] = "0" * 64 +def b_ih_req(c): c.inputs["x86_64"]["requirements_builder_sha256"] = "0" * 64 +def b_ih_missing(c): del c.inputs["x86_64"]["apk_lock_sha256"] +def b_ih_noarch(c): del c.inputs["aarch64"] +def b_ih_stale(c): c.git_show = lambda p: b"changed after the build\n" + +T20 = lambda: P("IMG-20")["predicate_type"] +def SCAN(c, a): return c.artifacts_dir / f"sbom-{a}" / f"vuln-scan.{a}.json" +def b_cv_fail(c): c.coverage["x86_64"]["verdict"] = "fail" +def b_cv_uncovered(c): c.coverage["aarch64"]["uncovered"] = [{"ids": ["Y"], "package": "p", "version": "1"}] +def b_cv_problems(c): c.coverage["x86_64"]["problems"] = ["vulnerability DB is 9 days old"] +def b_cv_scanner(c): c.coverage["x86_64"]["scanner"] = "other" +def b_cv_findings(c): c.coverage["aarch64"]["findings"] = 5 +def b_cv_count(c): c.coverage["x86_64"]["sbom_components"] += 1 +def b_cv_sbom(c): c.coverage["x86_64"]["input_sha256"]["sbom"] = "0" * 64 +def b_cv_vex(c): c.coverage["aarch64"]["input_sha256"]["vex"] = "0" * 64 +def b_cv_scan(c): c.coverage["x86_64"]["input_sha256"]["scan"] = "0" * 64 +def b_cv_scan_missing(c): SCAN(c, "x86_64").unlink() +def b_cv_nohashes(c): del c.coverage["aarch64"]["input_sha256"] +def b_cv_none(c): del c.coverage["aarch64"] +def b_ca_mismatch(c): c.run.by_key[K(c, "aarch64", T20())]["verdict"] = "fail" +def b_ca_none(c): c.run.by_key.pop(K(c, "x86_64", T20())) +def b_ca_swapped(c): + m, x, a = c.run.by_key, K(c, "x86_64", T20()), K(c, "aarch64", T20()) + m[x], m[a] = m[a], m[x] BAD = {"check_image_attested": [b_attest, b_commit], "check_inputs_attested": [b_attest, b_ia_mismatch, b_ia_none, b_ia_nodigest, b_ia_swapped], @@ -172,7 +219,11 @@ def b_lg_notgzip(c): LOG(c, "x86_64").write_bytes(b"not gzip") "check_tag_signed": [b_tag, b_tag_unverified, b_tagger], "check_actions_pinned": [b_pin], "check_release_assets": [b_assets], "check_assert_image_passed": [b_as_fail, b_as_dropped, b_as_extra, b_as_sbom, b_as_commit, b_as_image, b_as_none, b_as_swapped, b_ia_nodigest], "check_assert_attested": [b_attest, b_ia_nodigest, b_as_none, b_at_mismatch, b_at_none, b_at_swapped], - "check_build_log_attested": [b_attest, b_ia_nodigest, b_lg_missing, b_lg_empty, b_lg_nodigest, b_lg_notgzip]} + "check_build_log_attested": [b_attest, b_ia_nodigest, b_lg_missing, b_lg_empty, b_lg_nodigest, b_lg_notgzip], + "check_input_hashes": [b_ih_lock, b_ih_req, b_ih_missing, b_ih_noarch, b_ih_stale], + "check_coverage_report": [b_cv_fail, b_cv_uncovered, b_cv_problems, b_cv_scanner, b_cv_findings, b_cv_count, + b_cv_sbom, b_cv_vex, b_cv_scan, b_cv_scan_missing, b_cv_nohashes, b_cv_none], + "check_coverage_attested": [b_attest, b_ia_nodigest, b_cv_none, b_ca_mismatch, b_ca_none, b_ca_swapped]} @pytest.mark.parametrize("check,mutator", [(k, m) for k, ms in BAD.items() for m in ms]) @@ -285,4 +336,29 @@ def test_release_workflow_runs_assert_image_and_attests_the_results(): def test_build_workflow_attests_the_build_log(): wf = (ROOT / ".github/workflows/build-environment-image.yml").read_text() assert "subject-path: build-log.${{ matrix.arch }}.txt.gz" in wf - assert "build-log.${{ matrix.arch }}.txt.gz" in wf.split("Upload digest, inputs and build log")[1] \ No newline at end of file + assert "build-log.${{ matrix.arch }}.txt.gz" in wf.split("Upload digest, inputs and build log")[1] + +def test_release_workflow_stages_and_attests_the_coverage_report(): + wf = (ROOT / ".github/workflows/release.yml").read_text() + assert P("IMG-20")["predicate_type"] in wf + assert P("IMG-19")["vex_path"] in wf + assert "predicate-path: out/vex-coverage.${{ matrix.arch }}.json" in wf + assert '"vex-coverage.$a.json"' in wf + assert wf.index("check-vex-coverage.py") < wf.index("Attest VEX coverage report") + + +def test_check_vex_coverage_output_has_the_fields_the_verifier_reads(tmp_path): + import subprocess, sys + docs = {"scan": {"descriptor": {"name": "grype", "version": "0", "db": {"built": "2020-01-01T00:00:00Z"}}, "matches": []}, + "sbom": {"components": []}, "vex": {"statements": []}} + files = {} + for k, doc in docs.items(): + files[k] = tmp_path / f"{k}.json"; files[k].write_text(json.dumps(doc)) + out = tmp_path / "out.json" + subprocess.run([sys.executable, str(ROOT / ".github/scripts/check-vex-coverage.py"), + *map(str, files.values()), str(out)], capture_output=True) + rep = json.loads(out.read_text()) + for f in ("verdict", "scanner", "scanner_version", "db_built", "sbom_components", + "findings", "covered", "uncovered", "problems"): + assert f in rep, f + assert rep["input_sha256"] == {k: hashlib.sha256(p.read_bytes()).hexdigest() for k, p in files.items()}