From a580d86f6a1f38ccd2bc514c003a616bc354477b Mon Sep 17 00:00:00 2001 From: joey-huckabee <138994589+joey-huckabee@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:28:59 -0500 Subject: [PATCH] build: assemble the image from verified local bundles Replace the builder-stage DNF resolution with installation of the exact locked RPM closure. `scripts/build-image.sh` re-verifies the acquired bundle, preloads the digest-pinned UBI bases, and builds with `--pull=never` and `--network none`; `scripts/install-rpm-bundle.sh` imports the approved keys, checks every digest, signature, signer, and NEVRA, and compares the installed inventory with the lock-derived manifest. Add negative evidence: `tests/rpm-bundle-negative.py` mutates isolated copies of the real bundle to prove rejection of tampering, signature removal, signer mismatch, wrong version, wrong architecture, and missing or extra RPMs, and `tests/hermetic-build-negative.sh` proves the build rejects a wrong lock identity and an unavailable base. Native CI now builds with Podman and transfers the result into Docker by local archive for the existing compatibility smoke and scanner steps. --- .github/workflows/ci.yml | 65 +++++++++---- CHANGELOG.md | 12 +++ Containerfile | 50 ++++------ README.md | 23 ++++- compose.yaml | 3 - docs/ARTIFACT-ACQUISITION.md | 40 ++++++-- docs/CI.md | 15 ++- docs/CONTRIBUTOR-ENVIRONMENT.md | 6 +- docs/ROADMAP.md | 24 ++--- docs/RPM-PROVENANCE.md | 24 ++--- scripts/build-image.sh | 43 +++++++++ scripts/install-rpm-bundle.sh | 77 ++++++++++++++++ tests/hermetic-build-negative.sh | 41 ++++++++ tests/rpm-bundle-negative.py | 154 +++++++++++++++++++++++++++++++ 14 files changed, 479 insertions(+), 98 deletions(-) create mode 100644 scripts/build-image.sh create mode 100644 scripts/install-rpm-bundle.sh create mode 100644 tests/hermetic-build-negative.sh create mode 100644 tests/rpm-bundle-negative.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ef862e4..0c593ef 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,11 +78,9 @@ jobs: include: - architecture: amd64 runner: ubuntu-24.04 - platform: linux/amd64 machine: x86_64 - architecture: arm64 runner: ubuntu-24.04-arm - platform: linux/arm64 machine: aarch64 runs-on: ${{ matrix.runner }} timeout-minutes: 45 @@ -100,25 +98,58 @@ jobs: with: persist-credentials: false - - name: Set up Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - - name: Confirm native runner architecture env: EXPECTED_MACHINE: ${{ matrix.machine }} run: test "$(uname -m)" = "${EXPECTED_MACHINE}" - - name: Build test image - uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 - with: - context: . - file: Containerfile - platforms: ${{ matrix.platform }} - load: true - push: false - tags: ${{ env.TEST_IMAGE }} - cache-from: type=gha,scope=image-${{ matrix.architecture }} - cache-to: type=gha,mode=max,scope=image-${{ matrix.architecture }} + - name: Confirm hermetic-build tools + run: | + podman version + rpm --version + rpmsign --version + gpg --version | head -n 1 + + - name: Acquire architecture artifact bundle + run: >- + python scripts/artifacts.py acquire + --lock artifacts/locks/${{ matrix.architecture }}.json + --output .artifact-bundle/${{ matrix.architecture }} + + - name: Verify architecture artifact bundle + run: >- + bash scripts/verify-rpm-bundle.sh + artifacts/locks/${{ matrix.architecture }}.json + .artifact-bundle/${{ matrix.architecture }} + + - name: Reject invalid RPM bundles + run: >- + python tests/rpm-bundle-negative.py + --lock artifacts/locks/${{ matrix.architecture }}.json + --bundle .artifact-bundle/${{ matrix.architecture }} + + - name: Build without network access or image pulling + env: + PULL_BASES: "1" + run: >- + bash scripts/build-image.sh + ${{ matrix.architecture }} + "${TEST_IMAGE}" + + - name: Prove hermetic build boundaries fail closed + run: bash tests/hermetic-build-negative.sh ${{ matrix.architecture }} + + - name: Run native Podman restricted-runtime tests + env: + CONTAINER_RUNTIME: podman + IMAGE: ${{ env.TEST_IMAGE }} + run: bash tests/smoke.sh + + - name: Transfer image to Docker compatibility environment + run: | + podman save --format docker-archive --output /tmp/nginx-ubi-image.tar "${TEST_IMAGE}" + docker load --input /tmp/nginx-ubi-image.tar + rm --force /tmp/nginx-ubi-image.tar - name: Confirm loaded image architecture env: @@ -127,7 +158,7 @@ jobs: test "$(docker image inspect --format '{{.Architecture}}' "${TEST_IMAGE}")" = "${EXPECTED_ARCHITECTURE}" - - name: Run restricted-runtime smoke tests + - name: Run Docker compatibility restricted-runtime tests env: CONTAINER_RUNTIME: docker IMAGE: ${{ env.TEST_IMAGE }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 35330d4..7903fc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -80,6 +80,18 @@ but container releases use the upstream-derived format documented in artifact bundles, including exact inventory, digest, RPM signature, signer, NEVRA, architecture, and lock-manifest verification without storing source credentials or private trust material in the repository or image build. +- Migrated the development image to the selected official NGINX 1.30.4 RPM and + its exact architecture lock, with local-only RPM installation, installed + inventory comparison, digest-pinned base preloading, network-disabled + assembly, and an enforceable no-pull policy in local and native CI builds. +- Added hermetic-build rejection checks for a wrong lock identity and an + unavailable base, and removed repository configuration from the final + filesystem. +- Added native real-RPM negative tests for modified, unsigned, signer-mismatched, + wrong-version, wrong-architecture, missing, and unexpected bundle content; + bound production lock validation to the reviewed base images, NGINX seed, + and signing-key inputs; and required filenames to agree with RPM metadata + and official source URLs. - Expanded logging guidance with a field-by-field explanation of `$request`, a sensitive ClickHouse example, and safer variable choices. - Defined a source-independent pipeline contract that downloads and verifies diff --git a/Containerfile b/Containerfile index 2c3848a..4f7647d 100644 --- a/Containerfile +++ b/Containerfile @@ -3,35 +3,27 @@ ARG UBI_MINIMAL_IMAGE="registry.access.redhat.com/ubi9/ubi-minimal:9.8@sha256:7fbeae18dc9476399f565e68255f602a3374ea8614ba3d14843565131a13ff93" ARG UBI_MICRO_IMAGE="registry.access.redhat.com/ubi9/ubi-micro:9.8@sha256:f332c99eb8f798a8486821c91937f10ad64ee83d7e739303be2df051040918f6" +# The caller overrides this empty stage with a verified local named context. +# Keeping the fallback empty makes a missing bundle fail instead of pulling an +# image that happens to use the context name. +FROM scratch AS artifact_bundle + FROM ${UBI_MINIMAL_IMAGE} AS builder -ARG NGINX_MODULE_STREAM="1.26" -ARG NGINX_RPM_VERSION="2:1.26.3-9.module+el9.8.0+24599+8fde0ff7.3" +ARG ARTIFACT_LOCK_SHA256 + +COPY --from=artifact_bundle / /bundle/ +COPY --chmod=0755 scripts/install-rpm-bundle.sh /usr/local/bin/install-rpm-bundle -# Install the exact Red Hat NGINX build and its runtime dependency closure into -# a separate root. The UBI Micro final stage receives no package-management -# commands or builder caches. -# hadolint ignore=DL3041 -RUN microdnf install -y dnf \ - && mkdir -p /runtime \ - && dnf module enable -y \ - --installroot=/runtime \ - --releasever=9 \ - "nginx:${NGINX_MODULE_STREAM}" \ - && dnf install -y \ - --installroot=/runtime \ - --releasever=9 \ - --setopt=install_weak_deps=0 \ - --setopt=keepcache=0 \ - "nginx-core-${NGINX_RPM_VERSION}" \ - ca-certificates tzdata \ - && dnf clean all \ - && microdnf clean all \ +# Installation consumes only the complete local RPM closure. Network access +# and base-image pulling are disabled by the invoking build command. +RUN test -n "${ARTIFACT_LOCK_SHA256}" \ + && test "$(tr -d '\n' &2; exit 2 ;; +esac + +test "$("${runtime}" info --format '{{.Host.Arch}}')" = "${architecture}" +test "$(uname -m)" = "${machine}" +"${python}" scripts/artifacts.py validate-lock "${lock}" \ + --inputs artifacts/lock-inputs.json +bash scripts/verify-rpm-bundle.sh "${lock}" "${bundle}" + +builder=$("${python}" scripts/artifacts.py base-reference "${lock}" builder) +runtime_base=$("${python}" scripts/artifacts.py base-reference "${lock}" runtime) +if test "${PULL_BASES:-1}" = 1; then + "${runtime}" pull --platform "linux/${architecture}" "${builder}" + "${runtime}" pull --platform "linux/${architecture}" "${runtime_base}" +fi +"${runtime}" image exists "${builder}" +"${runtime}" image exists "${runtime_base}" + +lock_sha256=$(sha256sum "${lock}" | cut -d' ' -f1) +"${runtime}" build \ + --file Containerfile \ + --format docker \ + --network none \ + --pull=never \ + --build-context "artifact_bundle=${bundle}" \ + --build-arg "ARTIFACT_LOCK_SHA256=${lock_sha256}" \ + --build-arg "UBI_MINIMAL_IMAGE=${builder}" \ + --build-arg "UBI_MICRO_IMAGE=${runtime_base}" \ + --tag "${image}" \ + . diff --git a/scripts/install-rpm-bundle.sh b/scripts/install-rpm-bundle.sh new file mode 100644 index 0000000..8ced4b5 --- /dev/null +++ b/scripts/install-rpm-bundle.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +bundle=${1:?usage: install-rpm-bundle.sh BUNDLE ROOT} +root=${2:?usage: install-rpm-bundle.sh BUNDLE ROOT} + +test -f "${bundle}/key-manifest.tsv" +test -f "${bundle}/rpm-manifest.tsv" +test -d "${bundle}/keys" +test -d "${bundle}/rpms" +test ! -e "${root}" +test -z "$(find "${bundle}" -type l -print -quit)" +test "$(find "${bundle}" -mindepth 1 -maxdepth 1 -type f | wc -l)" -eq 3 +test "$(find "${bundle}" -mindepth 1 -maxdepth 1 -type d | wc -l)" -eq 2 +test -z "$(find "${bundle}/keys" "${bundle}/rpms" -mindepth 1 ! -type f -print -quit)" + +rpmdb=$(mktemp -d) +expected_packages=$(mktemp) +actual_packages=$(mktemp) +cleanup() { + rm -rf -- "${rpmdb}" "${expected_packages}" "${actual_packages}" +} +trap cleanup EXIT + +expected_key_count=0 +while IFS=$'\t' read -r filename fingerprint sha256; do + test "${filename}" != filename || continue + test -f "${bundle}/keys/${filename}" + printf '%s %s\n' "${sha256}" "${bundle}/keys/${filename}" \ + | sha256sum --check --status + rpm --define "_dbpath ${rpmdb}" --import "${bundle}/keys/${filename}" + expected_key_count=$((expected_key_count + 1)) +done < "${bundle}/key-manifest.tsv" +test "$(find "${bundle}/keys" -maxdepth 1 -type f | wc -l)" -eq "${expected_key_count}" + +expected_rpm_count=0 +while IFS=$'\t' read -r filename name epoch version release architecture source_rpm fingerprint sha256; do + test "${filename}" != filename || continue + package=${bundle}/rpms/${filename} + test -f "${package}" + printf '%s %s\n' "${sha256}" "${package}" | sha256sum --check --status + rpm_signature=$(rpm -qp --queryformat \ + $'%{RSAHEADER:pgpsig}\n%{DSAHEADER:pgpsig}\n%{SIGPGP:pgpsig}\n%{SIGGPG:pgpsig}\n' \ + "${package}" 2>/dev/null) + grep -Eqi "key ID [[:xdigit:]]*${fingerprint: -8}" <<< "${rpm_signature}" + signature_result=$(rpm --define "_dbpath ${rpmdb}" --checksig "${package}") + grep -Eq ': digests signatures OK$' <<< "${signature_result}" + actual=$(rpm -qp --queryformat $'%{NAME}\t%{EPOCHNUM}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\t%{SOURCERPM}' \ + "${package}" 2>/dev/null) + expected="${name}"$'\t'"${epoch}"$'\t'"${version}"$'\t'"${release}"$'\t'"${architecture}"$'\t'"${source_rpm}" + test "${actual}" = "${expected}" + printf '%s\t%s\t%s\t%s\t%s\n' \ + "${name}" "${epoch}" "${version}" "${release}" "${architecture}" \ + >> "${expected_packages}" + expected_rpm_count=$((expected_rpm_count + 1)) +done < "${bundle}/rpm-manifest.tsv" +test "$(find "${bundle}/rpms" -maxdepth 1 -type f -name '*.rpm' | wc -l)" \ + -eq "${expected_rpm_count}" +test "$(find "${bundle}/rpms" -maxdepth 1 -type f ! -name '*.rpm' | wc -l)" -eq 0 + +mkdir -p "${root}" +rpm --root "${root}" --initdb +rpm --root "${root}" --import "${bundle}"/keys/* +rpm --root "${root}" --install "${bundle}"/rpms/*.rpm + +rpm --root "${root}" --query --all \ + --queryformat $'%{NAME}\t%{EPOCHNUM}\t%{VERSION}\t%{RELEASE}\t%{ARCH}\n' \ + | awk -F $'\t' '$1 != "gpg-pubkey"' > "${actual_packages}" +sort -o "${expected_packages}" "${expected_packages}" +sort -o "${actual_packages}" "${actual_packages}" +test "$(sha256sum "${expected_packages}" | cut -d' ' -f1)" = \ + "$(sha256sum "${actual_packages}" | cut -d' ' -f1)" + +# Retain the verified package identity without retaining RPM tooling or inputs. +install -d -m 0755 "${root}/usr/share/nginx-ubi" +install -m 0444 "${bundle}/rpm-manifest.tsv" \ + "${root}/usr/share/nginx-ubi/rpm-manifest.tsv" diff --git a/tests/hermetic-build-negative.sh b/tests/hermetic-build-negative.sh new file mode 100644 index 0000000..20c7167 --- /dev/null +++ b/tests/hermetic-build-negative.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +architecture=${1:?usage: hermetic-build-negative.sh ARCHITECTURE} +runtime=${CONTAINER_RUNTIME:-podman} +python=${PYTHON:-python3} +bundle=${ARTIFACT_BUNDLE:-.artifact-bundle/${architecture}} +lock=artifacts/locks/${architecture}.json +builder=$("${python}" scripts/artifacts.py base-reference "${lock}" builder) +runtime_base=$("${python}" scripts/artifacts.py base-reference "${lock}" runtime) + +expect_build_failure() { + local label=$1 + shift + if "$@" >/dev/null 2>&1; then + echo "Hermetic build unexpectedly accepted ${label}" >&2 + return 1 + fi +} + +common_args=( + build + --file Containerfile + --format docker + --network none + --pull=never + --build-context "artifact_bundle=${bundle}" + --build-arg "UBI_MICRO_IMAGE=${runtime_base}" +) + +expect_build_failure "the wrong lock identity" \ + "${runtime}" "${common_args[@]}" \ + --build-arg "UBI_MINIMAL_IMAGE=${builder}" \ + --build-arg "ARTIFACT_LOCK_SHA256=$(printf '0%.0s' {1..64})" \ + . + +expect_build_failure "an unavailable base while pulling is forbidden" \ + "${runtime}" "${common_args[@]}" \ + --build-arg "UBI_MINIMAL_IMAGE=localhost/nginx-ubi-missing-base:never" \ + --build-arg "ARTIFACT_LOCK_SHA256=$(sha256sum "${lock}" | cut -d' ' -f1)" \ + . diff --git a/tests/rpm-bundle-negative.py b/tests/rpm-bundle-negative.py new file mode 100644 index 0000000..903e702 --- /dev/null +++ b/tests/rpm-bundle-negative.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Exercise fail-closed verification against a real acquired RPM bundle.""" + +from __future__ import annotations + +import argparse +import copy +import errno +import json +import os +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import artifacts # noqa: E402 + + +def reject(command: list[str], label: str) -> None: + result = subprocess.run( + command, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + if result.returncode == 0: + raise RuntimeError(f"verification unexpectedly accepted {label}") + print(f"rejected {label}") + + +def clone_bundle(source: Path, destination: Path) -> None: + def link_or_copy(source_path: str, destination_path: str) -> str: + try: + os.link(source_path, destination_path) + return destination_path + except OSError as exc: + if exc.errno != errno.EXDEV: + raise + return shutil.copy2(source_path, destination_path) + + shutil.copytree(source, destination, copy_function=link_or_copy) + for filename in ("LOCK-SHA256", "key-manifest.tsv", "rpm-manifest.tsv"): + path = destination / filename + contents = path.read_bytes() + path.unlink() + path.write_bytes(contents) + + +def detach(path: Path) -> None: + contents = path.read_bytes() + path.unlink() + path.write_bytes(contents) + + +def write_contract(lock: dict, lock_path: Path, bundle: Path) -> None: + lock_path.write_text(json.dumps(lock, indent=2) + "\n", encoding="utf-8") + manifests = artifacts.manifest_contents(lock, artifacts.sha256_file(lock_path), False) + for filename, contents in manifests.items(): + path = bundle / filename + if path.exists(): + path.unlink() + path.write_text(contents, encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--lock", required=True, type=Path) + parser.add_argument("--bundle", required=True, type=Path) + arguments = parser.parse_args() + + repository = Path(__file__).resolve().parents[1] + verifier = repository / "scripts" / "verify-rpm-bundle.sh" + original_lock = artifacts.validate_lock(arguments.lock) + subprocess.run( + ["bash", str(verifier), str(arguments.lock), str(arguments.bundle)], + check=True, + stdout=subprocess.DEVNULL, + ) + print("accepted the unchanged locked bundle") + nginx = next(item for item in original_lock["packages"] if item["name"] == "nginx") + first_package = original_lock["packages"][0] + + with tempfile.TemporaryDirectory(prefix="nginx-ubi-negative-") as temporary_name: + temporary = Path(temporary_name) + + def variant(label: str) -> tuple[dict, Path, Path]: + lock = copy.deepcopy(original_lock) + lock_path = temporary / f"{label}.json" + bundle = temporary / label + clone_bundle(arguments.bundle, bundle) + write_contract(lock, lock_path, bundle) + return lock, lock_path, bundle + + lock, lock_path, bundle = variant("tampered") + package = bundle / "rpms" / first_package["filename"] + detach(package) + with package.open("ab") as handle: + handle.write(b"tampered") + reject(["bash", str(verifier), str(lock_path), str(bundle)], "a tampered RPM") + + lock, lock_path, bundle = variant("missing") + (bundle / "rpms" / first_package["filename"]).unlink() + reject(["bash", str(verifier), str(lock_path), str(bundle)], "a missing RPM") + + lock, lock_path, bundle = variant("unexpected") + (bundle / "rpms" / "unexpected.rpm").write_bytes(b"unexpected") + reject(["bash", str(verifier), str(lock_path), str(bundle)], "an unexpected RPM") + + lock, lock_path, bundle = variant("wrong-version") + locked_nginx = next(item for item in lock["packages"] if item["name"] == "nginx") + lock["nginx_version"] = "1.30.3" + lock["nginx_rpm_version"] = f"1.30.3-{locked_nginx['release']}" + locked_nginx["version"] = "1.30.3" + locked_nginx["nevra"] = ( + f"nginx-{locked_nginx['epoch']}:1.30.3-{locked_nginx['release']}." + f"{locked_nginx['architecture']}" + ) + write_contract(lock, lock_path, bundle) + reject(["bash", str(verifier), str(lock_path), str(bundle)], "the wrong NGINX version") + + lock, lock_path, bundle = variant("wrong-architecture") + locked_nginx = next(item for item in lock["packages"] if item["name"] == "nginx") + locked_nginx["architecture"] = "aarch64" if lock["architecture"] == "amd64" else "x86_64" + write_contract(lock, lock_path, bundle) + reject(["bash", str(verifier), str(lock_path), str(bundle)], "the wrong RPM architecture") + + lock, lock_path, bundle = variant("wrong-signer") + locked_nginx = next(item for item in lock["packages"] if item["name"] == "nginx") + locked_nginx["signing_key_fingerprint"] = next( + item["fingerprint"] + for item in lock["signing_keys"] + if item["fingerprint"] != locked_nginx["signing_key_fingerprint"] + ) + write_contract(lock, lock_path, bundle) + reject(["bash", str(verifier), str(lock_path), str(bundle)], "a signer mismatch") + + lock, lock_path, bundle = variant("unsigned") + locked_nginx = next(item for item in lock["packages"] if item["name"] == "nginx") + package = bundle / "rpms" / nginx["filename"] + detach(package) + subprocess.run(["rpmsign", "--delsign", str(package)], check=True) + locked_nginx["size"] = package.stat().st_size + locked_nginx["sha256"] = artifacts.sha256_file(package) + write_contract(lock, lock_path, bundle) + reject(["bash", str(verifier), str(lock_path), str(bundle)], "an unsigned RPM") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())