diff --git a/.github/actions/build-guest-artifacts/action.yml b/.github/actions/build-guest-artifacts/action.yml index dd2e09731..c399e61a6 100644 --- a/.github/actions/build-guest-artifacts/action.yml +++ b/.github/actions/build-guest-artifacts/action.yml @@ -23,8 +23,30 @@ inputs: still builds them but leaves the cache unchanged required: false default: "true" + record-artifact: + description: >- + Name of the artifact in which to upload build/guest-artifacts.json, the + record of the selected components' exact cache keys and of the SHA-256 + of each of their files, so that the dev push that merges this run's + pull request can check them before it reuses them; "" records nothing + required: false + default: "" + reuse: + description: >- + "true", in the run of a pushed merge commit, installs every file of the + selected components, when one missed the cache, from the artifacts of + the latest pull_request run of this workflow for the merged head, from + this repository and never a fork, once they check out against that + run's record-artifact record, and builds them otherwise + required: false + default: "false" outputs: + cache-keys: + description: >- + The exact cache keys of the guest artifacts, as a JSON object by name, + from which a guest artifact record takes those of its components + value: ${{ toJSON(steps.guest-cache-keys.outputs) }} kernel-cache-key: description: Exact cache key for the Linux kernel artifacts value: ${{ steps.guest-cache-keys.outputs.kernel }} @@ -189,11 +211,40 @@ runs: key: ${{ steps.guest-cache-keys.outputs.azurelinux }} enableCrossOsArchive: true + # A push can't restore the caches that its pull request's runs saved, so + # it installs the selected components, when one missed the cache, from the + # artifacts of the latest run for the merged head, from this repository + # and never a fork, when that run recorded these keys and the files match + # the record, their provenance, and this checkout. Any other outcome + # builds them below, and the saves cache what it installed. + - name: Reuse the merged pull request's guest artifacts + id: reuse + if: >- + inputs.restore-only != 'true' && + inputs.reuse == 'true' && + inputs.record-artifact != '' + continue-on-error: true + shell: bash + env: + GH_TOKEN: ${{ github.token }} + COMPONENTS: ${{ inputs.components }} + GUEST_CACHE_KEYS: ${{ toJSON(steps.guest-cache-keys.outputs) }} + RECORD_ARTIFACT: ${{ inputs.record-artifact }} + run: >- + python3 scripts/nvx.py reuse-guest-artifacts + --repository "${GITHUB_REPOSITORY}" + --workflow-ref "${GITHUB_WORKFLOW_REF}" + --commit "${GITHUB_SHA}" + --components "${COMPONENTS}" + --record-artifact "${RECORD_ARTIFACT}" + --keys "${GUEST_CACHE_KEYS}" + - name: Build Linux kernel (Docker) if: >- inputs.restore-only != 'true' && steps.components.outputs.kernel == 'true' && - steps.cache-linux-kernel-zstd.outputs.cache-hit != 'true' + steps.cache-linux-kernel-zstd.outputs.cache-hit != 'true' && + steps.reuse.outputs.reused != 'true' shell: bash run: >- docker build @@ -261,7 +312,8 @@ runs: if: >- inputs.restore-only != 'true' && steps.components.outputs.alpine == 'true' && - steps.cache-alpine-image-zstd.outputs.cache-hit != 'true' + steps.cache-alpine-image-zstd.outputs.cache-hit != 'true' && + steps.reuse.outputs.reused != 'true' shell: bash run: >- docker build @@ -289,7 +341,8 @@ runs: if: >- inputs.restore-only != 'true' && steps.components.outputs.ubuntu == 'true' && - steps.cache-ubuntu-initramfs-zstd.outputs.cache-hit != 'true' + steps.cache-ubuntu-initramfs-zstd.outputs.cache-hit != 'true' && + steps.reuse.outputs.reused != 'true' shell: bash run: >- docker build @@ -298,11 +351,14 @@ runs: --output type=local,dest=build . + # A reused image passed this check in the run that built it, as a cached + # one did. - name: Verify Ubuntu artifact determinism (Docker) if: >- inputs.restore-only != 'true' && ( steps.components.outputs.ubuntu == 'true' && + steps.reuse.outputs.reused != 'true' && ( steps.cache-ubuntu-initramfs-zstd.outputs.cache-hit != 'true' || steps.cache-ubuntu-distro-zstd.outputs.cache-hit != 'true' @@ -333,7 +389,8 @@ runs: if: >- inputs.restore-only != 'true' && steps.components.outputs.azurelinux == 'true' && - steps.cache-azurelinux-initramfs-zstd.outputs.cache-hit != 'true' + steps.cache-azurelinux-initramfs-zstd.outputs.cache-hit != 'true' && + steps.reuse.outputs.reused != 'true' shell: bash env: AZURELINUX_IMAGE: ${{ steps.guest-versions.outputs.azurelinux-image }} @@ -365,7 +422,8 @@ runs: if: >- inputs.restore-only != 'true' && steps.components.outputs.ubuntu == 'true' && - steps.cache-ubuntu-distro-zstd.outputs.cache-hit != 'true' + steps.cache-ubuntu-distro-zstd.outputs.cache-hit != 'true' && + steps.reuse.outputs.reused != 'true' shell: bash run: >- docker build @@ -400,3 +458,26 @@ runs: -U 4f3c8bf7-1959-47b4-8811-62b17bbddaf4 \ -E lazy_itable_init=0,lazy_journal_init=0 \ build/ubuntu-smoke-scratch.ext4 + + # Uploaded beside the components' own artifacts, so that the dev push that + # merges this run's pull request can check them before it reuses them. + # The scratch template, which every run formats anew, is in no record. + - name: Record guest artifact keys and digests + if: inputs.restore-only != 'true' && inputs.record-artifact != '' + shell: bash + env: + COMPONENTS: ${{ inputs.components }} + GUEST_CACHE_KEYS: ${{ toJSON(steps.guest-cache-keys.outputs) }} + run: >- + python3 scripts/nvx.py record-guest-artifacts + --components "${COMPONENTS}" + --keys "${GUEST_CACHE_KEYS}" + + - name: Upload guest artifact record + if: inputs.restore-only != 'true' && inputs.record-artifact != '' + uses: actions/upload-artifact@v7 + with: + name: ${{ inputs.record-artifact }} + path: build/guest-artifacts.json + if-no-files-found: error + retention-days: 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c4783d3a4..541443697 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -377,6 +377,12 @@ jobs: }} runs-on: ubuntu-latest timeout-minutes: 60 + # A push reads its pull request's runs and downloads their artifacts. + permissions: + actions: read + contents: read + outputs: + guest-cache-keys: ${{ steps.guest-artifacts.outputs.cache-keys }} steps: - name: Checkout uses: actions/checkout@v5 @@ -384,13 +390,16 @@ jobs: persist-credentials: false # Full runs build the guest artifacts that miss the cache without saving - # them. + # them. A push installs those that its pull request's latest run built + # instead, when they check out. - name: Build guest artifacts id: guest-artifacts uses: ./.github/actions/build-guest-artifacts with: components: kernel alpine save-cache: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + record-artifact: kernel-alpine-record + reuse: ${{ github.event_name == 'push' }} # One artifact per component, so that each job downloads only the # components that it reads (doc/ci.md, "Guest artifact handoff"). @@ -434,6 +443,12 @@ jobs: }} runs-on: ubuntu-latest timeout-minutes: 60 + # A push reads its pull request's runs and downloads their artifacts. + permissions: + actions: read + contents: read + outputs: + guest-cache-keys: ${{ steps.guest-artifacts.outputs.cache-keys }} steps: - name: Checkout uses: actions/checkout@v5 @@ -441,13 +456,16 @@ jobs: persist-credentials: false # Full runs build the guest artifacts that miss the cache without saving - # them. + # them. A push installs those that its pull request's latest run built + # instead, when they check out. - name: Build guest artifacts id: guest-artifacts uses: ./.github/actions/build-guest-artifacts with: components: ubuntu azurelinux save-cache: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} + record-artifact: ubuntu-azurelinux-record + reuse: ${{ github.event_name == 'push' }} - name: Upload Ubuntu initramfs uses: actions/upload-artifact@v7 @@ -487,6 +505,67 @@ jobs: if-no-files-found: error retention-days: 1 + # A dev push reuses the guest artifact components that the pull request it + # merges built (see build-guest-artifacts). A same-repository pull request + # rehearses that selection and check on the artifacts that its own run + # uploaded, beside the jobs that use them, and installs nothing. + guest-artifact-reuse: + name: Rehearse guest artifact reuse + needs: [openvmm-changes, artifacts, distro-artifacts] + if: >- + ${{ + always() && + !cancelled() && + github.event_name == 'pull_request' && + github.event.pull_request.head.repo.full_name == github.repository && + needs.artifacts.result == 'success' && + needs.distro-artifacts.result == 'success' && + ( + needs.openvmm-changes.result != 'success' || + needs.openvmm-changes.outputs.run-guest-artifacts == 'true' + ) + }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + actions: read + contents: read + steps: + - name: Checkout + uses: actions/checkout@v5 + with: + persist-credentials: false + + - name: Rehearse kernel and Alpine artifact reuse + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GUEST_CACHE_KEYS: ${{ needs.artifacts.outputs.guest-cache-keys }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: >- + python3 scripts/nvx.py reuse-guest-artifacts --dry-run + --repository "${GITHUB_REPOSITORY}" + --workflow-ref "${GITHUB_WORKFLOW_REF}" + --head-sha "${HEAD_SHA}" + --components "kernel alpine" + --record-artifact kernel-alpine-record + --keys "${GUEST_CACHE_KEYS}" + + - name: Rehearse Ubuntu and Azure Linux artifact reuse + shell: bash + env: + GH_TOKEN: ${{ github.token }} + GUEST_CACHE_KEYS: ${{ needs.distro-artifacts.outputs.guest-cache-keys }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: >- + python3 scripts/nvx.py reuse-guest-artifacts --dry-run + --repository "${GITHUB_REPOSITORY}" + --workflow-ref "${GITHUB_WORKFLOW_REF}" + --head-sha "${HEAD_SHA}" + --components "ubuntu azurelinux" + --record-artifact ubuntu-azurelinux-record + --keys "${GUEST_CACHE_KEYS}" + # Beside the shared artifacts, so that a kernel rebuild doesn't delay the # jobs that don't use the debug kernel. debug-kernel: @@ -1702,6 +1781,7 @@ jobs: - artifacts - distro-artifacts - debug-kernel + - guest-artifact-reuse - build-openvmm-linux-gnu - build-openvmm-linux-musl - build-openvmm-windows-msvc @@ -1752,6 +1832,7 @@ jobs: ARTIFACTS_RESULT: ${{ needs.artifacts.result }} DISTRO_ARTIFACTS_RESULT: ${{ needs.distro-artifacts.result }} DEBUG_KERNEL_RESULT: ${{ needs.debug-kernel.result }} + GUEST_ARTIFACT_REUSE_RESULT: ${{ needs.guest-artifact-reuse.result }} BUILD_LINUX_GNU_RESULT: ${{ needs.build-openvmm-linux-gnu.result }} BUILD_LINUX_MUSL_RESULT: ${{ needs.build-openvmm-linux-musl.result }} BUILD_WINDOWS_MSVC_RESULT: ${{ needs.build-openvmm-windows-msvc.result }} diff --git a/doc/ci.md b/doc/ci.md index 1e9d16bb4..7238a15ca 100644 --- a/doc/ci.md +++ b/doc/ci.md @@ -487,6 +487,11 @@ release`, `Persist performance baseline`, and `Required status check` wait for the jobs whose results they check. Required handoffs between these jobs use same-run workflow artifacts; caches only skip rebuilding their inputs. +For a same-repository pull request, `guest-artifact-reuse` also waits for +`artifacts` and `distro-artifacts`, and rehearses the +[guest artifact reuse](#guest-artifact-reuse) of the push that merges it; +only `Required status check` waits for it. + The NVX CLI tests do not depend on the backend, so they run once per operating system in `nvx-cli-linux` and `nvx-cli-windows` rather than in every hypervisor job. Each checks out OpenVMM, whose pinned CPU profiles a test @@ -602,7 +607,81 @@ artifact job before. From October 19 to 21, 2026, before the guest image keys covered only the guest binaries' inputs, 159 of 200 guest artifact jobs missed the cache, nearly all for the Alpine, Ubuntu, and Azure Linux images; [#703](https://github.com/microsoft/nvx/issues/703) records the measurements -and how many of those misses the narrower keys remove. +and how many of those misses the narrower keys remove. A `dev` push installs +the components that its pull request's run built instead of building them, +when they check out; see [Guest artifact reuse](#guest-artifact-reuse). + +## Guest artifact reuse + +GitHub scopes the caches that a pull request's runs save to the pull request, +so the `dev` push that merges it cannot restore them, and used to rebuild +every guest image whose inputs the pull request had changed. From October 10 +to 21, 2026, when one job built every guest image, 50 of the 87 `dev` pushes +that ran it built at least one image, and the job took a median of 5.4 +minutes, and at most 11.3, instead of 0.3, which delayed every job that waits +for it. A push therefore reuses the components that its pull request's run +already built, when they check out. + +`artifacts` and `distro-artifacts` each record their components. The last +steps of +[`build-guest-artifacts`](../.github/actions/build-guest-artifacts/action.yml) +run [`record-guest-artifacts`](usage.md#record-guest-artifacts), which writes +the guest artifact record, `build/guest-artifacts.json`, with the exact keys of +the components' cache entries and the SHA-256 of each of their files, and +upload it for one day under the name that the action's `record-artifact` input +gives: `kernel-alpine-record` for the kernel and the Alpine initramfs, and +`ubuntu-azurelinux-record` for the Ubuntu and Azure Linux images. No job of the +run downloads a record. On a push, after the cache restores and before any +build, [`reuse-guest-artifacts`](usage.md#reuse-guest-artifacts) runs in each +of the two jobs when a file of its components is missing. It takes the merged +pull request's head from the second parent of the push's merge commit, and +the latest `pull_request` run of `ci.yml` for that head whose head repository +is `microsoft/nvx`, so never a fork's run. It downloads the job's record from +that run, unless the record has expired, which it does after one day, or lacks +the SHA-256 digest that GitHub records at upload, and compares its keys with +the push's own keys for the job's components, so the keys of the other job's +components never matter. Only when they are equal does it download the +components' artifacts, `guest-kernel` and `guest-alpine`, or `guest-ubuntu`, +`guest-ubuntu-distro`, and `guest-azurelinux`, under the same conditions. It +installs their files only when each artifact matches its digest, every file +matches the record, and the kernel and initramfs provenance, or the Ubuntu and +Azure Linux manifests, match the files and the push's checkout, as +[`package`](usage.md#package) requires. Then it replaces every file of the +job's components, not only the missing ones, because a cache restore that fails +partway can leave a truncated file behind and still report a miss; the files +that the cache restored have the same keys. The job then skips the builds of +its components and, in `distro-artifacts`, the Ubuntu determinism check, which +the run that built the images passed, as for a cache hit. It still saves the +components in its caches for later runs, records them, and uploads them as its +own artifacts, which its consumers download as before, so no job depends on +another run's artifact or on a cache. The debug kernel is never reused. + +Every other outcome builds the components that missed the cache, as before: a +merge commit without a same-repository pull request run, an expired record or +artifact or one without a digest, a run from before the records, other keys, +as after another merge changed the job's guest inputs, a failed request or +download, a file that does not check out, or a failed installation, which puts +back the files that it replaced. After two minutes, a timer stops a stalled +download or check in the same way, but never an installation, which it could +cut short with its files half put back. The step continues on error and never +fails the run. It reports why in the job summary and as a notice, or as a +warning for a fault. `artifacts`, `distro-artifacts`, and the rehearsal below +add `actions: read`, which reading runs and downloading their artifacts needs, +to the workflow's `contents: read`. + +Pull requests can't exercise the reuse itself, so the `guest-artifact-reuse` +job (`Rehearse guest artifact reuse`) rehearses it for each same-repository +pull request once both guest artifact jobs succeed, beside the jobs that wait +for them. It runs `reuse-guest-artifacts --dry-run` for the pull request's +head and each job's components, record, and keys, which selects the run in +progress, downloads the records and the artifacts that the two jobs just +uploaded, checks them as a push would, and reports in the job summary whether +a push would reuse them, without installing anything. A reason not to reuse +fails nothing, but it warns, because a run's own artifacts should always check +out. The change detector selects the rehearsal with the guest artifacts, and +`Required status check` requires it to succeed for a same-repository pull +request that builds them, so a pull request that breaks the reuse command +can't merge. ## Run concurrency diff --git a/doc/usage.md b/doc/usage.md index 5a600b7de..0e1da6145 100644 --- a/doc/usage.md +++ b/doc/usage.md @@ -80,6 +80,8 @@ stale build never ships or runs unnoticed. | `classify-ci-changes` | Select the CI suites that a list of changed files runs. | | `kernel-patch-identity` | Print the digest of the kernel patch paths and contents that keys the kernel cache. | | `guest-binary-source-identity` | Print the digest of the Rust guest binary source paths and contents that keys the guest image caches. | +| `record-guest-artifacts` | Record the cache keys and the SHA-256 of each file of guest artifact components, which a `dev` push checks before it reuses them. | +| `reuse-guest-artifacts` | Install guest artifact components that missed the cache from the latest same-repository pull request run for the merged head. | | `check-required-ci` | Validate required GitHub Actions job results. | | `report-nightly-ci` | Open, update, or close the report of failed nightly CI runs. | | `check-static-elf` | Check that executables are static ELF files without an interpreter or an executable stack. | @@ -165,6 +167,69 @@ crate's host side or tests rebuilds no guest image. Because the digest covers paths, and `hashFiles` hashes contents only, moving a module into its own `mod.rs` changes the keys even when no bytes change. +### `record-guest-artifacts` + +```text +python3 scripts/nvx.py record-guest-artifacts --components COMPONENTS + --keys JSON [--directory PATH] +``` + +Writes the guest artifact record, `guest-artifacts.json`, into the directory +of the guest artifacts, `build/` by default. `--components` lists, separated +by spaces, some of the `kernel`, `alpine`, `ubuntu`, and `azurelinux` +components of the CI guest build; the CI debug kernel and the Ubuntu sandbox +scratch template, which every run formats anew, are in no record. The record +holds the exact cache keys of those components' cache entries, from the JSON +object of keys by name that `--keys` gives, and the SHA-256 of each of their +files. It fails if a key or a file is missing. Each CI job that builds guest +images records them, and uploads the record beside the components' own +artifacts, so that the `dev` push that merges the pull request can check them +before it reuses them; see [Guest artifact reuse](ci.md#guest-artifact-reuse). + +### `reuse-guest-artifacts` + +```text +python3 scripts/nvx.py reuse-guest-artifacts + --repository OWNER/REPOSITORY + --workflow-ref REF + --record-artifact NAME + --components COMPONENTS + --keys JSON + (--commit SHA | --head-sha SHA) + [--dry-run] + [--destination PATH] +``` + +When the destination, `build/` by default, lacks a file of a component that +`--components` lists, as after a cache miss, replaces every file of those +components in it with the one from the artifacts of the latest `pull_request` +run of the workflow that `--workflow-ref`, a `GITHUB_WORKFLOW_REF` value, +names, for the pull request head: the second parent of the merge commit +`--commit`, or `--head-sha`. The run's head must be in `--repository` itself, +never a fork. +The command first downloads the run's `--record-artifact` artifact, which +holds the record that [`record-guest-artifacts`](#record-guest-artifacts) +wrote, and goes on only if the record gives exactly the cache keys of the +components' cache entries in `--keys`, a JSON object of keys by name. It then +downloads the artifacts in which CI hands over the components' files, +`guest-kernel`, `guest-alpine`, `guest-ubuntu`, `guest-ubuntu-distro`, or +`guest-azurelinux`, and installs the files only when no artifact has expired, +each matches the SHA-256 digest that GitHub recorded for it, every file +matches its SHA-256 in the record, and the kernel and initramfs provenance and +the Ubuntu and Azure Linux manifests match the files and this checkout's +sources. A failed installation puts back the files that it replaced. With +`--dry-run`, it checks the record and the artifacts in the same way, even when +the destination lacks no file, but installs nothing. + +It reads the GitHub token from `GH_TOKEN` or `GITHUB_TOKEN` and gives up on +the artifacts after two minutes, which a timer enforces on Linux until the +installation starts. It never fails for a reason not to reuse, so the guest +build proceeds: it prints the reason, as a GitHub notice when reuse is not +possible, such as for other keys or an expired artifact, and as a warning for +a fault, such as a failed request or a mismatched file, sets the `reused` step +output to `true` or `false`, and adds the outcome to the job summary; see +[Guest artifact reuse](ci.md#guest-artifact-reuse). + ### `classify-ci-changes` ```text diff --git a/scripts/nvx.py b/scripts/nvx.py index 9c2ab73b8..036760ab8 100644 --- a/scripts/nvx.py +++ b/scripts/nvx.py @@ -78,6 +78,12 @@ from nvx_tools.doctor import configure_parser as configure_doctor_parser from nvx_tools.doctor import host_cpu_signature from nvx_tools.elf import check_static_executables +from nvx_tools.guest_artifact_reuse import ( + parse_cache_keys, + parse_components, + record_guest_artifacts, + reuse_in_ci, +) from nvx_tools.guest_binaries import ( build_guest_binaries, check_guest_binaries, @@ -327,6 +333,30 @@ def command_guest_binary_source_identity(args: argparse.Namespace) -> None: print(guest_binary_source_sha256(args.root)) +def command_record_guest_artifacts(args: argparse.Namespace) -> None: + path = record_guest_artifacts( + args.directory, + parse_components(args.components), + parse_cache_keys(args.keys), + ) + print(f">> recorded the guest artifact keys and digests in {path}") + + +def command_reuse_guest_artifacts(args: argparse.Namespace) -> None: + reuse_in_ci( + repository=args.repository, + workflow_ref=args.workflow_ref, + record=args.record_artifact, + components=args.components, + keys=args.keys, + commit=args.commit, + head_sha=args.head_sha, + dry_run=args.dry_run, + destination=args.destination, + environment=os.environ, + ) + + def command_check_required_ci(args: argparse.Namespace) -> None: results = { job: os.environ.get(environment, "") @@ -890,6 +920,84 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: ) source_identity.set_defaults(handler=command_guest_binary_source_identity) + guest_record = subparsers.add_parser( + "record-guest-artifacts", + help=( + "record the cache keys and the SHA-256 of each file of guest " + "artifact components, which a dev push checks before it reuses them" + ), + ) + guest_record.add_argument( + "--components", + required=True, + help=( + "space-separated guest artifact components to record: kernel, " + "alpine, ubuntu, or azurelinux" + ), + ) + guest_record.add_argument( + "--keys", + required=True, + help="JSON object of the exact guest cache keys by name", + ) + guest_record.add_argument( + "--directory", + type=Path, + default=BuildConstants.BUILD_DIR, + help="directory of the guest artifacts (default: build)", + ) + guest_record.set_defaults(handler=command_record_guest_artifacts) + + guest_reuse = subparsers.add_parser( + "reuse-guest-artifacts", + help=( + "install guest artifact components that missed the cache from the " + "latest same-repository pull request run that built the merged head" + ), + ) + guest_reuse.add_argument("--repository", required=True, help="OWNER/REPOSITORY") + guest_reuse.add_argument( + "--workflow-ref", + required=True, + help="the run's GITHUB_WORKFLOW_REF, which names its workflow", + ) + guest_reuse.add_argument( + "--record-artifact", + required=True, + help="name of the artifact that holds the record of the components", + ) + guest_reuse.add_argument( + "--components", + required=True, + help=( + "space-separated guest artifact components to reuse: kernel, " + "alpine, ubuntu, or azurelinux" + ), + ) + guest_reuse.add_argument( + "--keys", + required=True, + help="JSON object of this run's exact guest cache keys by name", + ) + guest_reuse_head = guest_reuse.add_mutually_exclusive_group(required=True) + guest_reuse_head.add_argument( + "--commit", + help="the pushed merge commit, whose second parent is the pull request head", + ) + guest_reuse_head.add_argument("--head-sha", help="the pull request head itself") + guest_reuse.add_argument( + "--dry-run", + action="store_true", + help="check the artifacts as a push would, but install nothing", + ) + guest_reuse.add_argument( + "--destination", + type=Path, + default=BuildConstants.BUILD_DIR, + help="directory of the guest artifacts (default: build)", + ) + guest_reuse.set_defaults(handler=command_reuse_guest_artifacts) + required_ci = subparsers.add_parser( "check-required-ci", help="validate required GitHub Actions job results", diff --git a/scripts/nvx_tools/ci.py b/scripts/nvx_tools/ci.py index 180446159..b2f7120fc 100644 --- a/scripts/nvx_tools/ci.py +++ b/scripts/nvx_tools/ci.py @@ -69,6 +69,7 @@ "artifacts": "ARTIFACTS_RESULT", "distro-artifacts": "DISTRO_ARTIFACTS_RESULT", "debug-kernel": "DEBUG_KERNEL_RESULT", + "guest-artifact-reuse": "GUEST_ARTIFACT_REUSE_RESULT", "build-openvmm-linux-gnu": "BUILD_LINUX_GNU_RESULT", "build-openvmm-linux-musl": "BUILD_LINUX_MUSL_RESULT", "build-openvmm-windows-msvc": "BUILD_WINDOWS_MSVC_RESULT", @@ -123,6 +124,10 @@ REQUIRED_CI_DISTRO_ARTIFACT_JOB = "distro-artifacts" # GitHub-hosted, beside the shared artifacts; only the debug jobs wait for it. REQUIRED_CI_DEBUG_KERNEL_JOB = "debug-kernel" +# GitHub-hosted, after both guest artifact jobs of a same-repository pull +# request, whose artifacts it checks as the dev push that merges it would +# before it reuses them; only the required status check waits for it. +REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB = "guest-artifact-reuse" REQUIRED_CI_PLATFORM_JOBS = ( "platform-kvm", "platform-mshv", @@ -456,6 +461,11 @@ def required_ci_expected_results( "success" if run_guest_artifacts else "skipped" ), REQUIRED_CI_DEBUG_KERNEL_JOB: "success" if run_guest_artifacts else "skipped", + REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB: ( + "success" + if event_name == "pull_request" and same_repository and run_guest_artifacts + else "skipped" + ), "performance-gate": ( "success" if event_name == "pull_request" diff --git a/scripts/nvx_tools/guest_artifact_reuse.py b/scripts/nvx_tools/guest_artifact_reuse.py new file mode 100644 index 000000000..480b23195 --- /dev/null +++ b/scripts/nvx_tools/guest_artifact_reuse.py @@ -0,0 +1,919 @@ +"""Reuse of the guest artifact components that a pull request's CI run built. + +GitHub scopes the caches that a pull request's runs save to the pull request, +so the dev push that merges it misses them and would rebuild the guest images +that the pull request already built. Each guest artifact job therefore records +the exact cache keys of its components and the SHA-256 of each of their files +in a guest artifact record, which the build-guest-artifacts action uploads +beside the components' own artifacts. When a file of its components missed the +cache, the job of a push looks up the latest run of its workflow for the head +of the pull request that it merged, from the same repository and never from a +fork, and installs every file of its components from that run's artifacts once +each artifact matches the digest that GitHub recorded for it, the record's keys +equal the push's own, and the files match the record, their provenance, and +the push's sources. Any other outcome, or a time limit, leaves the components +to the action's builds, so reuse never fails a run and never installs an +unverified file. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import http.client +import json +import os +import re +import shutil +import signal +import sys +import tempfile +import threading +import time +import urllib.error +import urllib.parse +import urllib.request +import zipfile +from collections.abc import Generator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from types import FrameType +from typing import Protocol, cast + +from .common import ( + ScriptError, + credential_safe_opener, + require_file, + sha256_file, + strict_json_object, +) +from .release import ( + GITHUB_API_VERSION, + GUEST_ARTIFACT_COMPONENTS, + validate_guest_artifacts, +) + +RECORD_NAME = "guest-artifacts.json" +RECORD_FORMAT = 1 +# The outputs of the action's key step that key the cache entries of each +# component that a record covers: those that a package ships. +COMPONENT_CACHE_KEYS: Mapping[str, tuple[str, ...]] = { + "kernel": ("kernel",), + "alpine": ("alpine",), + "ubuntu": ("ubuntu", "ubuntu-distro"), + "azurelinux": ("azurelinux",), +} +# The workflow artifacts in which the guest artifact jobs hand over the files +# of each component that a record covers. +COMPONENT_ARTIFACTS: Mapping[str, tuple[str, ...]] = { + "kernel": ("guest-kernel",), + "alpine": ("guest-alpine",), + "ubuntu": ("guest-ubuntu", "guest-ubuntu-distro"), + "azurelinux": ("guest-azurelinux",), +} +GITHUB_API_URL = "https://api.github.com" +# A reuse takes seconds, and the cold builds that it replaces take from about +# 1.3 minutes, for the Alpine initramfs alone, to several minutes, so a reuse +# that stalls for longer than this saves little. +REUSE_TIMEOUT_SECONDS = 120.0 +_REQUEST_TIMEOUT_SECONDS = 30.0 +_CHUNK_BYTES = 1024 * 1024 +_RECORD_MAX_BYTES = 1024 * 1024 +_SHA256 = re.compile(r"[0-9a-f]{64}") +_SHA256_DIGEST = re.compile(r"sha256:([0-9a-f]{64})") +_REUSE = "Guest artifact reuse" +_REHEARSAL = "Guest artifact reuse rehearsal" + + +class ReuseRefused(ScriptError): + """A reason to build guest artifact components rather than reuse a run's. + + An expected reason, such as a push that merged no same-repository pull + request run or different cache keys, is a normal outcome; any other + suggests a fault, which CI reports as a warning. + """ + + def __init__(self, reason: str, *, expected: bool = False) -> None: + super().__init__(reason) + self.expected = expected + + +class GitHubApi(Protocol): + """The GitHub REST API requests that reuse makes.""" + + def get(self, endpoint: str) -> object: + """Return the JSON document at `endpoint`, relative to the API root.""" + ... + + def download(self, endpoint: str, destination: Path) -> None: + """Write the content at `endpoint` to `destination`.""" + ... + + +class RestApi: + """Reads the GitHub REST API with a token, until a monotonic deadline. + + Its opener drops the token when a download redirects to another origin, + as artifact downloads do to their storage. + """ + + def __init__( + self, + token: str, + deadline: float, + *, + url: str = GITHUB_API_URL, + ) -> None: + self._token = token + self._deadline = deadline + self._url = url.rstrip("/") + self._opener = credential_safe_opener() + + def _timeout(self, endpoint: str) -> float: + remaining = self._deadline - time.monotonic() + if remaining <= 0: + raise ReuseRefused(f"reuse ran out of time before GET {endpoint}") + return min(_REQUEST_TIMEOUT_SECONDS, remaining) + + def _request(self, endpoint: str) -> urllib.request.Request: + return urllib.request.Request( + f"{self._url}/{endpoint}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {self._token}", + "User-Agent": "nvx-guest-artifact-reuse", + "X-GitHub-Api-Version": GITHUB_API_VERSION, + }, + ) + + def get(self, endpoint: str) -> object: + try: + with self._opener.open( + self._request(endpoint), timeout=self._timeout(endpoint) + ) as response: + return json.load(response) + except (OSError, ValueError, http.client.HTTPException) as error: + raise _request_refusal(endpoint, error) from error + + def download(self, endpoint: str, destination: Path) -> None: + try: + with ( + self._opener.open( + self._request(endpoint), timeout=self._timeout(endpoint) + ) as response, + destination.open("wb") as output, + ): + while chunk := response.read(_CHUNK_BYTES): + output.write(chunk) + if time.monotonic() > self._deadline: + raise ReuseRefused( + f"reuse ran out of time while downloading {endpoint}" + ) + except (OSError, http.client.HTTPException) as error: + raise _request_refusal(endpoint, error) from error + + +def _request_refusal( + endpoint: str, + error: OSError | ValueError | http.client.HTTPException, +) -> ReuseRefused: + # A response that ends early, or a malformed one, raises an HTTPException, + # which is no OSError, but fails the request all the same. + if isinstance(error, urllib.error.HTTPError): + # An HTTP error holds the response open. + error.close() + return ReuseRefused(f"GET {endpoint} failed with HTTP {error.code}") + return ReuseRefused(f"GET {endpoint} failed: {error}") + + +@dataclass(frozen=True) +class PullRequestRun: + """A pull_request run of the workflow, from the repository itself.""" + + id: int + url: str + + +@dataclass(frozen=True) +class RunArtifact: + """An unexpired artifact of a workflow run.""" + + id: int + name: str + digest: str | None + + +@dataclass(frozen=True) +class ReuseRequest: + """What to reuse, and for which push or rehearsal. + + A push names its merge commit, whose second parent is the head of the + pull request that it merged; a rehearsal names that head itself. `keys` + gives, by name, the exact keys of the cache entries of `components`, and + `record` names the artifact of their record. + """ + + repository: str + workflow_ref: str + record: str + components: tuple[str, ...] + keys: Mapping[str, str] + commit: str | None = None + head_sha: str | None = None + dry_run: bool = False + + +@dataclass(frozen=True) +class ReuseOutcome: + """What reuse did, and why; `level` annotates a fallback to building.""" + + reused: bool + heading: str + status: str + detail: str + level: str | None = None + + def summary(self) -> str: + return f"### {self.heading}: {self.status}\n{self.detail}\n" + + +def _object(value: object, description: str) -> dict[str, object]: + if not isinstance(value, dict): + raise ReuseRefused(f"GitHub returned an invalid {description}") + return cast(dict[str, object], value) + + +def _objects(value: object, description: str) -> list[dict[str, object]]: + if not isinstance(value, list): + raise ReuseRefused(f"GitHub returned an invalid {description} list") + return [_object(item, description) for item in cast(list[object], value)] + + +def _text(document: dict[str, object], field: str, description: str) -> str: + value = document.get(field) + if not isinstance(value, str) or not value: + raise ReuseRefused(f"GitHub returned a {description} without a {field}") + return value + + +def _integer(document: dict[str, object], field: str, description: str) -> int: + value = document.get(field) + if not isinstance(value, int) or isinstance(value, bool): + raise ReuseRefused(f"GitHub returned a {description} without a {field}") + return value + + +def _repository_name(value: object) -> str | None: + """Return the full name of a repository document; a deleted fork's is + null.""" + if not isinstance(value, dict): + return None + name = cast(dict[str, object], value).get("full_name") + return name if isinstance(name, str) else None + + +def parse_components(text: str) -> tuple[str, ...]: + """Parse the space-separated guest artifact components of a record, as + the action's components input lists them.""" + names = set(text.split()) + unknown = sorted(names - COMPONENT_CACHE_KEYS.keys()) + if unknown: + raise ScriptError( + f"no guest artifact record covers {', '.join(unknown)}, only " + f"{', '.join(COMPONENT_CACHE_KEYS)}" + ) + if not names: + raise ScriptError("no guest artifact component is selected") + return tuple(name for name in COMPONENT_CACHE_KEYS if name in names) + + +def component_files(components: Sequence[str]) -> tuple[str, ...]: + """Return the names of the files of `components` in build/.""" + return tuple( + name + for component in components + for name in GUEST_ARTIFACT_COMPONENTS[component] + ) + + +def _cache_keys(document: object) -> dict[str, str]: + if not isinstance(document, dict) or not document: + raise ScriptError("cache keys must be a nonempty JSON object") + keys: dict[str, str] = {} + for name, value in cast(dict[str, object], document).items(): + if not name or not isinstance(value, str) or not value: + raise ScriptError(f"cache key {name!r} must be a nonempty string") + keys[name] = value + return keys + + +def parse_cache_keys(text: str) -> dict[str, str]: + """Parse a JSON object of exact cache keys by name.""" + try: + document: object = json.loads(text, object_pairs_hook=strict_json_object) + except ValueError as error: + raise ScriptError(f"cache keys are not JSON: {error}") from error + return _cache_keys(document) + + +def component_keys( + components: Sequence[str], + keys: Mapping[str, str], +) -> dict[str, str]: + """Return, by name, the exact cache keys among `keys` of the cache entries + of `components`.""" + selected: dict[str, str] = {} + for component in components: + for name in COMPONENT_CACHE_KEYS[component]: + if not keys.get(name): + raise ScriptError(f"the cache keys give no {name} key") + selected[name] = keys[name] + return dict(sorted(selected.items())) + + +def record_guest_artifacts( + directory: Path, + components: Sequence[str], + keys: Mapping[str, str], +) -> Path: + """Write the guest artifact record of `components` in `directory`: the + exact cache keys of their cache entries, from `keys`, and the SHA-256 of + each of their files, which a later run checks before it reuses them.""" + files = { + name: sha256_file(require_file(directory / name, f"guest artifact {name}")) + for name in component_files(components) + } + document = { + "format": RECORD_FORMAT, + "keys": component_keys(components, _cache_keys(dict(keys))), + "files": files, + } + path = directory / RECORD_NAME + path.write_text(json.dumps(document, indent=2) + "\n", encoding="utf-8") + return path + + +def _parse_record(contents: bytes) -> tuple[dict[str, str], dict[str, str]]: + """Return the cache keys and the file digests, by name, of a record.""" + try: + document: object = json.loads( + contents.decode("utf-8"), object_pairs_hook=strict_json_object + ) + if not isinstance(document, dict): + raise ScriptError("it is not a JSON object") + record = cast(dict[str, object], document) + if type(record.get("format")) is not int or record["format"] != RECORD_FORMAT: + raise ScriptError(f"its format is not {RECORD_FORMAT}") + keys = _cache_keys(record.get("keys")) + files = record.get("files") + if not isinstance(files, dict) or not files: + raise ScriptError("it lists no files") + digests: dict[str, str] = {} + for name, digest in cast(dict[str, object], files).items(): + if not isinstance(digest, str) or _SHA256.fullmatch(digest) is None: + raise ScriptError(f"it gives no SHA-256 of {name}") + digests[name] = digest + except (ScriptError, ValueError) as error: + raise ReuseRefused(f"the {RECORD_NAME} record is invalid: {error}") from error + return keys, digests + + +def workflow_path(workflow_ref: str, repository: str) -> str: + """Return the path of the workflow that `workflow_ref`, the + GITHUB_WORKFLOW_REF of a run of `repository`, names.""" + reference, separator, _ = workflow_ref.partition("@") + path = reference.removeprefix(f"{repository}/") + if ( + not separator + or path == reference + or PurePosixPath(path).parent != PurePosixPath(".github/workflows") + ): + raise ReuseRefused(f"{workflow_ref!r} names no workflow of {repository}") + return path + + +def merged_head(api: GitHubApi, repository: str, commit: str) -> str: + """Return the head of the pull request that the merge commit `commit` + merged: its second parent.""" + document = _object(api.get(f"repos/{repository}/git/commits/{commit}"), "commit") + parents = [ + _text(parent, "sha", "commit parent") + for parent in _objects(document.get("parents"), "commit parent") + ] + if len(parents) != 2: + raise ReuseRefused( + f"`{commit}` is not the merge commit of a pull request", expected=True + ) + return parents[1] + + +def latest_pull_request_run( + api: GitHubApi, + repository: str, + workflow: str, + head_sha: str, +) -> PullRequestRun: + """Return the latest pull_request run of `workflow` for `head_sha` whose + head is in `repository`; a fork's run never qualifies.""" + query = urllib.parse.urlencode( + {"event": "pull_request", "head_sha": head_sha, "per_page": 100} + ) + name = urllib.parse.quote(PurePosixPath(workflow).name, safe="") + document = _object( + api.get(f"repos/{repository}/actions/workflows/{name}/runs?{query}"), + "workflow run list", + ) + candidates: list[tuple[int, int, PullRequestRun]] = [] + other_repositories = 0 + for run in _objects(document.get("workflow_runs"), "workflow run"): + if ( + run.get("event") != "pull_request" + or run.get("head_sha") != head_sha + or _text(run, "path", "workflow run").partition("@")[0] != workflow + ): + continue + if ( + _repository_name(run.get("head_repository")) != repository + or _repository_name(run.get("repository")) != repository + ): + other_repositories += 1 + continue + run_id = _integer(run, "id", "workflow run") + candidates.append( + ( + _integer(run, "run_number", "workflow run"), + run_id, + PullRequestRun(run_id, _text(run, "html_url", "workflow run")), + ) + ) + if not candidates: + ignored = ( + f"; {other_repositories} from other repositories never qualify" + if other_repositories + else "" + ) + raise ReuseRefused( + f"no pull request run of `{workflow}` from {repository} built " + f"`{head_sha}`{ignored}", + expected=True, + ) + return max(candidates, key=lambda candidate: candidate[:2])[2] + + +def retained_artifact( + api: GitHubApi, + repository: str, + run: PullRequestRun, + name: str, +) -> RunArtifact: + """Return the newest unexpired artifact named `name` of `run`.""" + query = urllib.parse.urlencode({"name": name, "per_page": 100}) + document = _object( + api.get(f"repos/{repository}/actions/runs/{run.id}/artifacts?{query}"), + "artifact list", + ) + candidates: list[tuple[str, int, RunArtifact]] = [] + for artifact in _objects(document.get("artifacts"), "artifact"): + workflow_run = _object(artifact.get("workflow_run"), "artifact run") + if ( + artifact.get("name") != name + or artifact.get("expired") is not False + or workflow_run.get("id") != run.id + ): + continue + artifact_id = _integer(artifact, "id", "artifact") + digest = artifact.get("digest") + candidates.append( + ( + _text(artifact, "created_at", "artifact"), + artifact_id, + RunArtifact( + artifact_id, name, digest if isinstance(digest, str) else None + ), + ) + ) + if not candidates: + raise ReuseRefused( + f"[run {run.id}]({run.url}) has no unexpired `{name}` artifact", + expected=True, + ) + return max(candidates, key=lambda candidate: candidate[:2])[2] + + +def _download( + api: GitHubApi, + repository: str, + artifact: RunArtifact, + archive: Path, +) -> None: + # The digest that GitHub took when the run uploaded the artifact is the + # only check of the archive that doesn't come from inside it. + digest = _SHA256_DIGEST.fullmatch(artifact.digest or "") + if digest is None: + raise ReuseRefused( + f"artifact {artifact.id} has no SHA-256 digest, but {artifact.digest!r}" + ) + api.download(f"repos/{repository}/actions/artifacts/{artifact.id}/zip", archive) + if sha256_file(archive) != digest.group(1): + raise ReuseRefused( + f"artifact {artifact.id} does not match its digest {artifact.digest}" + ) + + +# zipfile reports a corrupt member as BadZipFile, a truncated one as EOFError, +# an unknown compression as NotImplementedError, and an encrypted one as +# RuntimeError. +_ZIP_ERRORS = ( + zipfile.BadZipFile, + EOFError, + NotImplementedError, + RuntimeError, + ValueError, +) + + +def _read_record(archive: Path, name: str) -> tuple[dict[str, str], dict[str, str]]: + """Return the cache keys and the file digests of the record that the + artifact `name`, downloaded as `archive`, holds.""" + try: + with zipfile.ZipFile(archive) as bundle: + members = [ + member for member in bundle.infolist() if member.filename == RECORD_NAME + ] + if len(members) != 1: + raise ReuseRefused(f"artifact `{name}` holds no single {RECORD_NAME}") + if members[0].file_size > _RECORD_MAX_BYTES: + raise ReuseRefused(f"the {RECORD_NAME} record is too large") + contents = bundle.read(members[0]) + except ScriptError: + raise + except _ZIP_ERRORS as error: + raise ReuseRefused( + f"artifact `{name}` is not a valid ZIP archive: {error}" + ) from error + return _parse_record(contents) + + +def _extract( + archive: Path, + name: str, + staging: Path, + digests: Mapping[str, str], + extracted: set[str], +) -> None: + """Extract into `staging` each file that `digests` names from the artifact + `name`, downloaded as `archive`, check it against its SHA-256 there, and + add its name to `extracted`.""" + try: + with zipfile.ZipFile(archive) as bundle: + for member in bundle.infolist(): + if member.is_dir() or member.filename not in digests: + continue + if member.filename in extracted: + raise ReuseRefused( + f"the artifacts hold {member.filename} more than once" + ) + digest = hashlib.sha256() + with ( + bundle.open(member) as source, + (staging / member.filename).open("wb") as output, + ): + while chunk := source.read(_CHUNK_BYTES): + output.write(chunk) + digest.update(chunk) + if digest.hexdigest() != digests[member.filename]: + raise ReuseRefused( + f"{member.filename} in artifact `{name}` does not match " + "the SHA-256 that the record gives" + ) + extracted.add(member.filename) + except ScriptError: + raise + except _ZIP_ERRORS as error: + raise ReuseRefused( + f"artifact `{name}` is not a valid ZIP archive: {error}" + ) from error + + +def _key_differences( + recorded: Mapping[str, str], + expected: Mapping[str, str], +) -> list[str]: + return sorted( + name + for name in recorded.keys() | expected.keys() + if recorded.get(name) != expected.get(name) + ) + + +def _install(staging: Path, destination: Path, names: Sequence[str]) -> None: + """Move the files `names` from `staging` into `destination`, in place of + any there, or leave `destination` as it was. + + Each move is noted before it happens, so that the rollback also undoes a + move that an exception interrupted right after the rename. + """ + replaced = staging / ".replaced" + replaced.mkdir() + moving: list[str] = [] + installing: list[str] = [] + try: + for name in names: + target = destination / name + if os.path.lexists(target): + moving.append(name) + os.replace(target, replaced / name) + installing.append(name) + os.replace(staging / name, target) + except BaseException as error: + # Whatever is at a target being installed is either its new file or + # nothing, since its old one, if any, has moved aside. + for name in installing: + (destination / name).unlink(missing_ok=True) + for name in moving: + if os.path.lexists(replaced / name): + os.replace(replaced / name, destination / name) + if isinstance(error, OSError): + raise ReuseRefused(f"the images could not be installed: {error}") from error + raise + + +@contextlib.contextmanager +def _time_limit(deadline: float | None) -> Generator[None]: + """Interrupt the work in the block with ReuseRefused once the + time.monotonic() value `deadline` passes. + + The request timeouts bound each wait for the network, but not a download + that keeps trickling in or the checks after it, so on POSIX an interval + timer raises wherever the main thread is. Elsewhere, only the request + timeouts and the download's own deadline checks apply. + """ + if deadline is None: + yield + return + seconds = deadline - time.monotonic() + if seconds <= 0: + raise ReuseRefused("reuse ran out of time") + if sys.platform == "win32": + yield + return + if threading.current_thread() is not threading.main_thread(): + yield + return + + def expire(_signal: int, _frame: FrameType | None) -> None: + if armed: + raise ReuseRefused("reuse ran out of time") + + armed = True + previous = signal.signal(signal.SIGALRM, expire) + signal.setitimer(signal.ITIMER_REAL, seconds) + try: + yield + finally: + # A signal that arrives once the block has ended must not raise in + # place of the restoration of the previous handler. + armed = False + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + +def _sentence(text: str) -> str: + text = text.rstrip(".") + return f"{text[:1].upper()}{text[1:]}" + + +def _fallback(dry_run: bool) -> str: + if dry_run: + return ( + "a push that merged this head would build the guest images that " + "missed its cache" + ) + return "the action builds the guest images that missed the cache" + + +def _series(names: Sequence[str]) -> str: + if len(names) < 3: + return " and ".join(names) + return f"{', '.join(names[:-1])}, and {names[-1]}" + + +def _quoted(names: Sequence[str]) -> list[str]: + return [f"`{name}`" for name in names] + + +def _heading(heading: str, components: Sequence[str]) -> str: + return f"{heading} ({', '.join(components)})" + + +def _checked_artifacts( + api: GitHubApi, + request: ReuseRequest, + staging: Path, +) -> str: + """Select the latest run for the pull request's head, download into + `staging` its record and the artifacts of the request's components, check + the components' files, and describe them.""" + workflow = workflow_path(request.workflow_ref, request.repository) + if request.head_sha is not None: + head_sha = request.head_sha + elif request.commit is not None: + head_sha = merged_head(api, request.repository, request.commit) + else: + raise ReuseRefused("neither a merge commit nor a head was given") + run = latest_pull_request_run(api, request.repository, workflow, head_sha) + source = ( + f"[run {run.id}]({run.url}), the latest run of `{workflow}` for pull " + f"request head `{head_sha}`" + ) + record = retained_artifact(api, request.repository, run, request.record) + archive = staging / f"{record.name}.zip" + _download(api, request.repository, record, archive) + keys, digests = _read_record(archive, record.name) + archive.unlink() + differences = _key_differences(keys, request.keys) + if differences: + raise ReuseRefused( + f"the `{record.name}` record of {source} gives other cache keys for " + f"{', '.join(differences)}", + expected=True, + ) + files = component_files(request.components) + if set(digests) != set(files): + raise ReuseRefused( + f"the `{record.name}` record of {source} describes other files than " + f"those of {_series(request.components)}" + ) + names = [ + name + for component in request.components + for name in COMPONENT_ARTIFACTS[component] + ] + extracted: set[str] = set() + for name in names: + artifact = retained_artifact(api, request.repository, run, name) + archive = staging / f"{artifact.name}.zip" + _download(api, request.repository, artifact, archive) + _extract(archive, artifact.name, staging, digests, extracted) + archive.unlink() + missing = [name for name in files if name not in extracted] + if missing: + raise ReuseRefused( + f"no artifact of {source} holds {_series(missing)}", + ) + try: + validate_guest_artifacts(staging, request.components) + except ReuseRefused: + raise + except (ScriptError, OSError) as error: + raise ReuseRefused( + f"the files of {source} do not match their provenance or this " + f"checkout: {error}" + ) from error + return f"the {_series(_quoted(names))} artifacts of {source}" + + +def reuse_guest_artifacts( + api: GitHubApi, + request: ReuseRequest, + destination: Path, + *, + deadline: float | None = None, +) -> ReuseOutcome: + """Install the files of the request's components from the artifacts of + the latest run for the pull request's head when `destination` lacks any + of them, or explain why not, giving up on the artifacts at the + time.monotonic() value `deadline`. + + A cache restore that misses leaves its files absent, but one that fails + partway can leave a truncated file and report a miss too, so reuse + replaces every file of the components with its checked copy rather than + only the absent ones; the files that the cache did restore have the same + keys. A rehearsal checks the artifacts in the same way but installs + nothing. + """ + heading = _heading(_REHEARSAL if request.dry_run else _REUSE, request.components) + files = component_files(request.components) + absent = [name for name in files if not (destination / name).exists()] + if not absent and not request.dry_run: + return ReuseOutcome( + False, + heading, + "not needed", + "The cache restored every file of these components.", + ) + try: + destination.mkdir(parents=True, exist_ok=True) + # Beside the destination, so that installing a file renames it. + staging = Path( + tempfile.mkdtemp(prefix=".guest-artifact-reuse-", dir=destination) + ) + try: + with _time_limit(deadline): + source = _checked_artifacts(api, request, staging) + if request.dry_run: + return ReuseOutcome( + False, + heading, + "would reuse", + "A push that merged this head would install every file of " + f"these components from {source}: the cache keys that its " + f"`{request.record}` record gives, the SHA-256 of each file, " + "and their provenance check out.", + ) + # Outside the time limit, whose timer must not cut a rollback short. + _install(staging, destination, files) + finally: + shutil.rmtree(staging, ignore_errors=True) + except (ScriptError, OSError) as error: + expected = isinstance(error, ReuseRefused) and error.expected + return ReuseOutcome( + False, + heading, + "build", + f"{_sentence(str(error))}, so {_fallback(request.dry_run)}.", + "notice" if expected and not request.dry_run else "warning", + ) + return ReuseOutcome( + True, + heading, + "reused", + f"Installed every file of these components from {source}: the cache " + f"keys that its `{request.record}` record gives, the SHA-256 of each " + "file, and their provenance check out.", + ) + + +def _escape_annotation(text: str, *, property_value: bool = False) -> str: + text = text.replace("%", "%25").replace("\r", "%0D").replace("\n", "%0A") + if property_value: + text = text.replace(":", "%3A").replace(",", "%2C") + return text + + +def report(outcome: ReuseOutcome, environment: Mapping[str, str]) -> None: + """Print `outcome`, annotate a fallback to building, and give CI the + `reused` step output and the job summary.""" + print(f">> {outcome.heading}: {outcome.status}. {outcome.detail}") + if outcome.level is not None: + title = _escape_annotation(outcome.heading, property_value=True) + print(f"::{outcome.level} title={title}::{_escape_annotation(outcome.detail)}") + output = environment.get("GITHUB_OUTPUT") + if output: + with Path(output).open("a", encoding="utf-8", newline="\n") as stream: + stream.write(f"reused={'true' if outcome.reused else 'false'}\n") + summary = environment.get("GITHUB_STEP_SUMMARY") + if summary: + with Path(summary).open("a", encoding="utf-8", newline="\n") as stream: + stream.write(outcome.summary()) + + +def reuse_in_ci( + *, + repository: str, + workflow_ref: str, + record: str, + components: str, + keys: str, + commit: str | None, + head_sha: str | None, + dry_run: bool, + destination: Path, + environment: Mapping[str, str], +) -> ReuseOutcome: + """Reuse the guest artifact `components`, as the action's components + input lists them, with the token in GH_TOKEN or GITHUB_TOKEN, giving up on + the artifacts after REUSE_TIMEOUT_SECONDS, and report the outcome; every + failure falls back to building.""" + token = environment.get("GH_TOKEN") or environment.get("GITHUB_TOKEN") + deadline = time.monotonic() + REUSE_TIMEOUT_SECONDS + heading = _REHEARSAL if dry_run else _REUSE + try: + selected = parse_components(components) + heading = _heading(heading, selected) + if not token: + raise ScriptError("no GitHub token is in GH_TOKEN or GITHUB_TOKEN") + request = ReuseRequest( + repository=repository, + workflow_ref=workflow_ref, + record=record, + components=selected, + keys=component_keys(selected, parse_cache_keys(keys)), + commit=commit, + head_sha=head_sha, + dry_run=dry_run, + ) + except ScriptError as error: + outcome = ReuseOutcome( + False, + heading, + "build", + f"{_sentence(str(error))}, so {_fallback(dry_run)}.", + "warning", + ) + else: + outcome = reuse_guest_artifacts( + RestApi(token, deadline), request, destination, deadline=deadline + ) + report(outcome, environment) + return outcome diff --git a/scripts/nvx_tools/release.py b/scripts/nvx_tools/release.py index 046faf75e..8866391cf 100644 --- a/scripts/nvx_tools/release.py +++ b/scripts/nvx_tools/release.py @@ -17,7 +17,7 @@ import urllib.request import uuid import zipfile -from collections.abc import Callable, Sequence +from collections.abc import Callable, Collection, Mapping, Sequence from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import cast @@ -71,6 +71,32 @@ # HEAD often has no release, as with CI's [skip ci] baseline commits. Each # commit searched costs one GitHub API request. _RELEASE_SEARCH_COMMITS = 20 +# The guest artifact components that a package ships, as the `components` +# input of the build-guest-artifacts action names them, and the files of each +# in build/. The action formats the Ubuntu sandbox scratch template in every +# run, and no package ships the CI debug kernel, so neither is one. +GUEST_ARTIFACT_COMPONENTS: Mapping[str, tuple[str, ...]] = { + "kernel": ( + KernelBuildConstants.BINARY_NAME, + KernelBuildConstants.CONFIG_NAME, + KernelBuildConstants.PROVENANCE_NAME, + ), + "alpine": ( + AlpineBuildConstants.INITRAMFS_NAME, + AlpineBuildConstants.PACKAGE_MANIFEST_NAME, + InitramfsBuildConstants.PROVENANCE_NAME, + ), + "ubuntu": ( + UbuntuBuildConstants.INITRAMFS_NAME, + UbuntuBuildConstants.PACKAGE_MANIFEST_NAME, + UbuntuBuildConstants.DISTRO_NAME, + UbuntuBuildConstants.DISTRO_MANIFEST_NAME, + ), + "azurelinux": ( + AzureLinuxBuildConstants.INITRAMFS_NAME, + AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME, + ), +} @dataclass(frozen=True) @@ -1021,9 +1047,8 @@ def _validate_linux_source_archive(path: Path) -> None: raise ScriptError(f"{path} has stale contents for {suffix}") -def _validate_azurelinux_manifest(manifest_path: Path) -> None: +def _validate_azurelinux_manifest(manifest_path: Path, initramfs: Path) -> None: manifest = _read_json_object(manifest_path, "Azure Linux initramfs manifest") - initramfs = artifact_path(AzureLinuxBuildConstants.INITRAMFS_NAME) if ( manifest.get("format") != AzureLinuxBuildConstants.PACKAGE_MANIFEST_VERSION or manifest.get("guest") != AzureLinuxBuildConstants.GUEST_NAME @@ -1062,33 +1087,47 @@ def _guest_release_inputs( azurelinux_manifests.append( artifact_path(AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME) ) - _validate_azurelinux_manifest(azurelinux_manifests[0]) + _validate_azurelinux_manifest( + azurelinux_manifests[0], + artifact_path(AzureLinuxBuildConstants.INITRAMFS_NAME), + ) expected_input_sha256 = converter_input_sha256(customization_files()) for artifact_name, manifest in zip( (UbuntuBuildConstants.INITRAMFS_NAME, UbuntuBuildConstants.DISTRO_NAME), ubuntu_manifests, strict=True, ): - artifact = artifact_path(artifact_name) - document = _read_json_object(manifest, f"{artifact_name} manifest") - mismatches = _manifest_mismatches( - document, - { - "format": UbuntuBuildConstants.PACKAGE_MANIFEST_VERSION, - "guest": UbuntuBuildConstants.GUEST_NAME, - "artifact": artifact.name, - "artifact_sha256": sha256_file(artifact), - "input_sha256": expected_input_sha256, - }, - ) - if mismatches: - raise ScriptError( - f"Ubuntu artifact manifest does not match {artifact_name}: " - f"{manifest.name} differs in {', '.join(mismatches)}" - ) + _validate_ubuntu_manifest( + artifact_path(artifact_name), manifest, expected_input_sha256 + ) return guest_names, alpine_manifests, ubuntu_manifests, azurelinux_manifests +def _validate_ubuntu_manifest( + artifact: Path, + manifest: Path, + input_sha256: str, +) -> None: + """Check that the Ubuntu manifest `manifest` describes `artifact` as built + from the inputs whose digest is `input_sha256`.""" + document = _read_json_object(manifest, f"{artifact.name} manifest") + mismatches = _manifest_mismatches( + document, + { + "format": UbuntuBuildConstants.PACKAGE_MANIFEST_VERSION, + "guest": UbuntuBuildConstants.GUEST_NAME, + "artifact": artifact.name, + "artifact_sha256": sha256_file(artifact), + "input_sha256": input_sha256, + }, + ) + if mismatches: + raise ScriptError( + f"Ubuntu artifact manifest does not match {artifact.name}: " + f"{manifest.name} differs in {', '.join(mismatches)}" + ) + + def _manifest_mismatches( document: dict[str, object], expected: dict[str, object], @@ -1290,6 +1329,52 @@ def validate_runtime_artifact_provenance() -> None: _validate_nvx_rs_provenance(nvx_rs_executable, nvx_rs_provenance_path, target=None) +def validate_guest_artifacts(directory: Path, components: Collection[str]) -> None: + """Check that the files of the guest artifact `components` in `directory` + match their provenance and manifests and were built from this checkout's + sources, as `package` checks those in build/.""" + unknown = sorted(set(components) - GUEST_ARTIFACT_COMPONENTS.keys()) + if unknown: + raise ScriptError(f"unknown guest artifact component: {', '.join(unknown)}") + if not components: + raise ScriptError("no guest artifact component to validate") + for component in components: + for name in GUEST_ARTIFACT_COMPONENTS[component]: + require_file(directory / name, f"guest artifact {name}") + if "kernel" in components: + _validate_kernel_provenance( + directory / KernelBuildConstants.BINARY_NAME, + directory / KernelBuildConstants.CONFIG_NAME, + directory / KernelBuildConstants.PROVENANCE_NAME, + ) + if "alpine" in components: + _validate_initramfs_provenance( + directory / AlpineBuildConstants.INITRAMFS_NAME, + directory / AlpineBuildConstants.PACKAGE_MANIFEST_NAME, + directory / InitramfsBuildConstants.PROVENANCE_NAME, + ) + if "ubuntu" in components: + input_sha256 = converter_input_sha256(customization_files()) + for artifact_name, manifest_name in ( + ( + UbuntuBuildConstants.INITRAMFS_NAME, + UbuntuBuildConstants.PACKAGE_MANIFEST_NAME, + ), + ( + UbuntuBuildConstants.DISTRO_NAME, + UbuntuBuildConstants.DISTRO_MANIFEST_NAME, + ), + ): + _validate_ubuntu_manifest( + directory / artifact_name, directory / manifest_name, input_sha256 + ) + if "azurelinux" in components: + _validate_azurelinux_manifest( + directory / AzureLinuxBuildConstants.PACKAGE_MANIFEST_NAME, + directory / AzureLinuxBuildConstants.INITRAMFS_NAME, + ) + + def _require_release_nvx_rs() -> tuple[Path, Path, dict[str, object]]: """Return the nvx-rs that a package ships, its provenance path, and its provenance, which must show a build for the release target of this diff --git a/scripts/test_nvx_tools.py b/scripts/test_nvx_tools.py index 8bfe01c1b..8b9a6e515 100644 --- a/scripts/test_nvx_tools.py +++ b/scripts/test_nvx_tools.py @@ -52,6 +52,7 @@ collect_ubuntu_sources, common, elf, + guest_artifact_reuse, guest_binaries, guests, microvm_durations, @@ -4242,6 +4243,1645 @@ def edit(manifest: dict[str, object], change: str, content: Path) -> None: run_checked.assert_not_called() +_REUSE_REPOSITORY = "microsoft/nvx" +_REUSE_WORKFLOW = ".github/workflows/ci.yml" +_REUSE_WORKFLOW_REF = f"{_REUSE_REPOSITORY}/{_REUSE_WORKFLOW}@refs/heads/dev" +_REUSE_MERGE = "a" * 40 +_REUSE_HEAD = "c" * 40 +_REUSE_KEYS = { + "kernel": "linux-kernel-v1-6.18.38-1111-2222", + "kernel-debug": "linux-kernel-debug-v1-6.18.38-3333-2222", + "alpine": "alpine-image-v2-3.24.1-4444-5555", + "ubuntu": "ubuntu-initramfs-v2-26.04.1-6666-5555", + "ubuntu-distro": "ubuntu-distro-v2-26.04.1-6666-5555", + "azurelinux": "azurelinux-initramfs-v2-3.0-7777-5555", +} +# The components of each guest artifact job in ci.yml. +_REUSE_JOBS = (("kernel", "alpine"), ("ubuntu", "azurelinux")) +# The artifact of each job's record. +_REUSE_RECORDS: dict[tuple[str, ...], str] = { + ("kernel", "alpine"): "kernel-alpine-record", + ("ubuntu", "azurelinux"): "ubuntu-azurelinux-record", +} +_REUSE_RUNS = ( + f"repos/{_REUSE_REPOSITORY}/actions/workflows/ci.yml/runs" + f"?event=pull_request&head_sha={_REUSE_HEAD}&per_page=100" +) +# The ID of each artifact of run 102 that the fixtures serve. +_REUSE_ARTIFACT_IDS = { + "kernel-alpine-record": 9, + "guest-kernel": 10, + "guest-alpine": 11, + "ubuntu-azurelinux-record": 19, + "guest-ubuntu": 20, + "guest-ubuntu-distro": 21, + "guest-azurelinux": 22, +} + + +def _reuse_list(name: str) -> str: + """Return the endpoint that lists the artifacts named `name` of run 102.""" + return ( + f"repos/{_REUSE_REPOSITORY}/actions/runs/102/artifacts?name={name}&per_page=100" + ) + + +def _reuse_download(name: str) -> str: + """Return the endpoint that downloads the artifact `name` of run 102.""" + return ( + f"repos/{_REUSE_REPOSITORY}/actions/artifacts/{_REUSE_ARTIFACT_IDS[name]}/zip" + ) + + +def _guest_uploads() -> dict[str, tuple[str, set[str]]]: + """Return the job that uploads each guest-* artifact of ci.yml, and the + names of the files in it, by artifact name.""" + workflow = yaml.safe_load( + (BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml").read_text( + encoding="utf-8" + ) + ) + uploads: dict[str, tuple[str, set[str]]] = {} + for job_name, job in workflow["jobs"].items(): + for step in job.get("steps", []): + arguments = step.get("with", {}) + if str(step.get("uses", "")).startswith("actions/upload-artifact@") and str( + arguments.get("name", "") + ).startswith("guest-"): + uploads[arguments["name"]] = ( + job_name, + {PurePosixPath(path).name for path in arguments["path"].split()}, + ) + return uploads + + +class _FakeGitHub: + """Answers reuse's GitHub API requests from canned documents.""" + + def __init__( + self, + documents: Mapping[str, object], + downloads: Mapping[str, bytes] | None = None, + ) -> None: + self.documents = dict(documents) + self.downloads = dict(downloads or {}) + self.requests: list[str] = [] + + def get(self, endpoint: str) -> object: + self.requests.append(endpoint) + if endpoint not in self.documents: + raise guest_artifact_reuse.ReuseRefused( + f"GET {endpoint} failed with HTTP 404" + ) + return self.documents[endpoint] + + def download(self, endpoint: str, destination: Path) -> None: + self.requests.append(endpoint) + if endpoint not in self.downloads: + raise guest_artifact_reuse.ReuseRefused( + f"GET {endpoint} failed with HTTP 404" + ) + destination.write_bytes(self.downloads[endpoint]) + + +def _reuse_run( + run_id: int, + run_number: int, + *, + head_repository: str | None = _REUSE_REPOSITORY, + event: str = "pull_request", + head_sha: str = _REUSE_HEAD, + path: str = _REUSE_WORKFLOW, +) -> dict[str, object]: + return { + "id": run_id, + "run_number": run_number, + "event": event, + "head_sha": head_sha, + "path": path, + "html_url": f"https://github.com/{_REUSE_REPOSITORY}/actions/runs/{run_id}", + "head_repository": ( + None if head_repository is None else {"full_name": head_repository} + ), + "repository": {"full_name": _REUSE_REPOSITORY}, + } + + +def _reuse_artifact( + artifact_id: int, + run_id: int, + *, + name: str, + expired: bool = False, + created_at: str = "2026-10-21T12:00:00Z", + digest: str | None = None, +) -> dict[str, object]: + return { + "id": artifact_id, + "name": name, + "expired": expired, + "created_at": created_at, + "digest": digest, + "workflow_run": {"id": run_id}, + } + + +def _reuse_zip(members: Mapping[str, bytes]) -> bytes: + """Return a ZIP archive of `members`, as an artifact download is.""" + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w", zipfile.ZIP_DEFLATED) as bundle: + for name, contents in members.items(): + bundle.writestr(name, contents) + return archive.getvalue() + + +def _reuse_request( + components: tuple[str, ...] = ("kernel", "alpine"), + *, + keys: Mapping[str, str] = _REUSE_KEYS, + dry_run: bool = False, +) -> guest_artifact_reuse.ReuseRequest: + return guest_artifact_reuse.ReuseRequest( + repository=_REUSE_REPOSITORY, + workflow_ref=_REUSE_WORKFLOW_REF, + record=_REUSE_RECORDS[components], + components=components, + keys=guest_artifact_reuse.component_keys(components, keys), + commit=None if dry_run else _REUSE_MERGE, + head_sha=_REUSE_HEAD if dry_run else None, + dry_run=dry_run, + ) + + +class GuestArtifactReuseTests(unittest.TestCase): + template: Path + kernel_inputs: dict[str, object] + _template_directory: tempfile.TemporaryDirectory[str] + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + # Valid guest artifacts and their provenance. + cls._template_directory = tempfile.TemporaryDirectory() + paths, cls.kernel_inputs, _revision = _write_release_fixture( + Path(cls._template_directory.name) + ) + cls.template = paths["build"] + + @classmethod + def tearDownClass(cls) -> None: + cls._template_directory.cleanup() + super().tearDownClass() + + @staticmethod + def _serve( + api: _FakeGitHub, + name: str, + contents: bytes, + *, + digest: str | None = None, + ) -> None: + """Serve `contents` as the artifact `name` of run 102, with the + SHA-256 digest of `contents` unless `digest` replaces it.""" + if digest is None: + digest = f"sha256:{hashlib.sha256(contents).hexdigest()}" + api.documents[_reuse_list(name)] = { + "artifacts": [ + _reuse_artifact( + _REUSE_ARTIFACT_IDS[name], 102, name=name, digest=digest + ) + ] + } + api.downloads[_reuse_download(name)] = contents + + def _fixture( + self, + root: Path, + components: tuple[str, ...] = ("kernel", "alpine"), + ) -> tuple[_FakeGitHub, dict[str, object], Path]: + """Return a fake GitHub API whose merge of _REUSE_HEAD and the latest + run for it lead to the record of `components` and to their artifacts, + as ci.yml uploads them, the kernel provenance inputs that the files + match, and a copy of the files and their record under `root`.""" + files = root / "files" + files.mkdir() + for name in guest_artifact_reuse.component_files(components): + shutil.copyfile(self.template / name, files / name) + record = guest_artifact_reuse.record_guest_artifacts( + files, components, _REUSE_KEYS + ) + api = _FakeGitHub( + { + f"repos/{_REUSE_REPOSITORY}/git/commits/{_REUSE_MERGE}": { + "sha": _REUSE_MERGE, + "parents": [{"sha": "b" * 40}, {"sha": _REUSE_HEAD}], + }, + _REUSE_RUNS: { + "workflow_runs": [ + _reuse_run(103, 1851, head_repository="fork/nvx"), + _reuse_run(102, 1850), + _reuse_run(101, 1849), + ] + }, + } + ) + self._serve( + api, + _REUSE_RECORDS[components], + _reuse_zip({record.name: record.read_bytes()}), + ) + uploads = _guest_uploads() + for component in components: + for name in guest_artifact_reuse.COMPONENT_ARTIFACTS[component]: + self._serve( + api, + name, + _reuse_zip( + { + member: (files / member).read_bytes() + for member in sorted(uploads[name][1]) + } + ), + ) + return api, self.kernel_inputs, files + + def _reuse( + self, + api: _FakeGitHub, + kernel_inputs: dict[str, object], + destination: Path, + request: guest_artifact_reuse.ReuseRequest | None = None, + ) -> guest_artifact_reuse.ReuseOutcome: + with patch.object( + release, "kernel_provenance_inputs", return_value=kernel_inputs + ): + outcome = guest_artifact_reuse.reuse_guest_artifacts( + api, request or _reuse_request(), destination + ) + # Reuse leaves no staging directory behind. + if destination.exists(): + self.assertEqual( + [path.name for path in destination.iterdir() if path.is_dir()], [] + ) + return outcome + + def _assert_built( + self, + outcome: guest_artifact_reuse.ReuseOutcome, + destination: Path, + reason: str, + level: str, + ) -> None: + self.assertFalse(outcome.reused) + self.assertEqual(outcome.status, "build") + self.assertEqual(outcome.level, level) + self.assertRegex(outcome.detail, re.compile(reason, re.IGNORECASE)) + self.assertTrue( + outcome.detail.endswith( + ", so the action builds the guest images that missed the cache." + ), + outcome.detail, + ) + installed = ( + [path.name for path in destination.iterdir()] + if destination.exists() + else [] + ) + self.assertEqual(installed, []) + + def test_record_names_the_cache_keys_and_the_digest_of_every_file(self): + for components in _REUSE_JOBS: + with ( + self.subTest(components=components), + tempfile.TemporaryDirectory() as temporary, + ): + directory = Path(temporary) + for name in guest_artifact_reuse.component_files(components): + shutil.copyfile(self.template / name, directory / name) + (directory / "ubuntu-smoke-scratch.ext4").write_bytes(b"scratch") + stdout = io.StringIO() + args = nvx.parse_args( + [ + "record-guest-artifacts", + "--components", + " ".join(reversed(components)), + "--keys", + json.dumps(_REUSE_KEYS, indent=2), + "--directory", + os.fspath(directory), + ] + ) + with contextlib.redirect_stdout(stdout): + args.handler(args) + + record = json.loads( + (directory / guest_artifact_reuse.RECORD_NAME).read_text( + encoding="utf-8" + ) + ) + # Only the keys of the components' own cache entries. + self.assertEqual( + record, + { + "format": 1, + "keys": { + name: _REUSE_KEYS[name] + for name in sorted( + name + for component in components + for name in guest_artifact_reuse.COMPONENT_CACHE_KEYS[ + component + ] + ) + }, + "files": { + name: common.sha256_file(directory / name) + for component in components + for name in release.GUEST_ARTIFACT_COMPONENTS[component] + }, + }, + ) + self.assertEqual(len(record["files"]), 6) + self.assertNotIn("kernel-debug", record["keys"]) + self.assertIn(guest_artifact_reuse.RECORD_NAME, stdout.getvalue()) + + first = guest_artifact_reuse.component_files(components)[1] + (directory / first).unlink() + with self.assertRaisesRegex( + common.ScriptError, f"guest artifact {re.escape(first)} not found" + ): + guest_artifact_reuse.record_guest_artifacts( + directory, components, _REUSE_KEYS + ) + + def test_record_requires_the_key_of_each_cache_entry(self): + keys = { + name: key for name, key in _REUSE_KEYS.items() if name != "ubuntu-distro" + } + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + components = ("ubuntu", "azurelinux") + for name in guest_artifact_reuse.component_files(components): + shutil.copyfile(self.template / name, directory / name) + guest_artifact_reuse.record_guest_artifacts( + directory, ("azurelinux",), keys + ) + with self.assertRaisesRegex( + common.ScriptError, "the cache keys give no ubuntu-distro key" + ): + guest_artifact_reuse.record_guest_artifacts(directory, components, keys) + + def test_record_components_are_those_that_a_package_ships(self): + self.assertEqual( + list(guest_artifact_reuse.COMPONENT_CACHE_KEYS), + list(release.GUEST_ARTIFACT_COMPONENTS), + ) + for text, expected in ( + ("kernel alpine", ("kernel", "alpine")), + (" alpine\nkernel kernel ", ("kernel", "alpine")), + ("azurelinux ubuntu", ("ubuntu", "azurelinux")), + ): + with self.subTest(components=text): + self.assertEqual(guest_artifact_reuse.parse_components(text), expected) + for text in ( + "", + " ", + "debug-kernel", + "kernel debug-kernel", + "Kernel", + "kernel,alpine", + ): + with ( + self.subTest(components=text), + self.assertRaises(common.ScriptError), + ): + guest_artifact_reuse.parse_components(text) + + def test_cache_keys_must_be_a_json_object_of_nonempty_strings(self): + self.assertEqual( + guest_artifact_reuse.parse_cache_keys(json.dumps(_REUSE_KEYS)), + _REUSE_KEYS, + ) + for text in ( + "", + "kernel", + "[]", + "{}", + '{"kernel": ""}', + '{"kernel": 1}', + '{"": "key"}', + '{"kernel": "a", "kernel": "b"}', + ): + with ( + self.subTest(keys=text), + self.assertRaises(common.ScriptError), + ): + guest_artifact_reuse.parse_cache_keys(text) + + def test_push_installs_every_file_of_its_components_when_one_missed(self): + for components, cached in ( + # The cache restored the kernel but not the Alpine initramfs. + ( + ("kernel", "alpine"), + ( + KernelBuildConstants.BINARY_NAME, + KernelBuildConstants.CONFIG_NAME, + KernelBuildConstants.PROVENANCE_NAME, + ), + ), + # A restore that failed partway left a truncated distro layer. + (("ubuntu", "azurelinux"), (UbuntuBuildConstants.DISTRO_NAME,)), + ): + with ( + self.subTest(components=components), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, files = self._fixture(root, components) + destination = root / "destination" + destination.mkdir() + for name in cached: + (destination / name).write_bytes(b"cached " + name.encode()) + # The files of the other components stay as they are. + (destination / "other").write_bytes(b"other") + + outcome = self._reuse( + api, kernel_inputs, destination, _reuse_request(components) + ) + + self.assertTrue(outcome.reused, outcome.detail) + self.assertEqual(outcome.status, "reused") + self.assertIsNone(outcome.level) + self.assertEqual( + outcome.heading, + f"Guest artifact reuse ({', '.join(components)})", + ) + self.assertIn( + "Installed every file of these components", outcome.detail + ) + self.assertIn( + "[run 102](https://github.com/microsoft/nvx/actions/runs/102)", + outcome.detail, + ) + artifacts = [ + name + for component in components + for name in guest_artifact_reuse.COMPONENT_ARTIFACTS[component] + ] + record = _REUSE_RECORDS[components] + self.assertEqual( + api.requests, + [ + f"repos/{_REUSE_REPOSITORY}/git/commits/{_REUSE_MERGE}", + _REUSE_RUNS, + _reuse_list(record), + _reuse_download(record), + *( + endpoint + for name in artifacts + for endpoint in (_reuse_list(name), _reuse_download(name)) + ), + ], + ) + # Every file of the components comes from the checked + # artifacts, and no other file. + names = guest_artifact_reuse.component_files(components) + self.assertEqual( + sorted(path.name for path in destination.iterdir()), + sorted((*names, "other")), + ) + for name in names: + with self.subTest(file=name): + self.assertEqual( + (destination / name).read_bytes(), + (files / name).read_bytes(), + ) + self.assertEqual((destination / "other").read_bytes(), b"other") + + def test_rehearsal_checks_the_artifacts_without_installing_them(self): + for components in _REUSE_JOBS: + with ( + self.subTest(components=components), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root, components) + destination = root / "destination" + destination.mkdir() + # A rehearsal checks the artifacts even when the cache + # restored every file. + for name in guest_artifact_reuse.component_files(components): + (destination / name).write_bytes(b"cached") + + outcome = self._reuse( + api, + kernel_inputs, + destination, + _reuse_request(components, dry_run=True), + ) + + self.assertFalse(outcome.reused) + self.assertEqual( + outcome.heading, + f"Guest artifact reuse rehearsal ({', '.join(components)})", + ) + self.assertEqual(outcome.status, "would reuse", outcome.detail) + self.assertIsNone(outcome.level) + # The rehearsal names the head, so it needs no merge commit. + self.assertEqual(api.requests[0], _REUSE_RUNS) + self.assertIn(_reuse_download(_REUSE_RECORDS[components]), api.requests) + for name in guest_artifact_reuse.component_files(components): + self.assertEqual((destination / name).read_bytes(), b"cached") + + # A rehearsal that its own run can't serve warns, even for a + # reason that a push would only note. + record = _REUSE_RECORDS[components] + api.documents[_reuse_list(record)] = { + "artifacts": [ + _reuse_artifact( + _REUSE_ARTIFACT_IDS[record], 102, name=record, expired=True + ) + ] + } + + outcome = self._reuse( + api, + kernel_inputs, + destination, + _reuse_request(components, dry_run=True), + ) + + self.assertFalse(outcome.reused) + self.assertEqual(outcome.status, "build") + self.assertEqual(outcome.level, "warning") + self.assertTrue( + outcome.detail.endswith( + f"has no unexpired `{record}` artifact, so a push that " + "merged this head would build the guest images that " + "missed its cache." + ), + outcome.detail, + ) + + def test_reuse_is_not_needed_when_the_cache_restored_every_file(self): + with tempfile.TemporaryDirectory() as temporary: + destination = Path(temporary) + # Those of the other components don't count. + for name in guest_artifact_reuse.component_files(("kernel", "alpine")): + (destination / name).write_bytes(b"cached") + api = _FakeGitHub({}) + + outcome = guest_artifact_reuse.reuse_guest_artifacts( + api, _reuse_request(), destination + ) + + self.assertFalse(outcome.reused) + self.assertEqual(outcome.status, "not needed") + self.assertIsNone(outcome.level) + self.assertEqual(api.requests, []) + + def test_only_same_repository_pull_request_runs_of_the_workflow_qualify(self): + def latest(*runs: dict[str, object]) -> guest_artifact_reuse.PullRequestRun: + api = _FakeGitHub({_REUSE_RUNS: {"workflow_runs": list(runs)}}) + return guest_artifact_reuse.latest_pull_request_run( + api, _REUSE_REPOSITORY, _REUSE_WORKFLOW, _REUSE_HEAD + ) + + ineligible = ( + _reuse_run(201, 1901, head_repository="fork/nvx"), + _reuse_run(202, 1902, head_repository=None), + _reuse_run(203, 1903, event="push"), + _reuse_run(204, 1904, event="workflow_dispatch"), + _reuse_run(205, 1905, head_sha="d" * 40), + _reuse_run(206, 1906, path=".github/workflows/adversarial.yml"), + ) + self.assertEqual( + latest(_reuse_run(101, 1849), *ineligible, _reuse_run(102, 1850)), + guest_artifact_reuse.PullRequestRun( + 102, "https://github.com/microsoft/nvx/actions/runs/102" + ), + ) + with self.assertRaises(guest_artifact_reuse.ReuseRefused) as raised: + latest(*ineligible) + self.assertTrue(raised.exception.expected) + self.assertIn( + f"no pull request run of `{_REUSE_WORKFLOW}` from microsoft/nvx built " + f"`{_REUSE_HEAD}`; 2 from other repositories never qualify", + str(raised.exception), + ) + with self.assertRaises(guest_artifact_reuse.ReuseRefused): + latest({"event": "pull_request", "head_sha": _REUSE_HEAD}) + + def test_fork_runs_never_supply_guest_artifacts(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + api.documents[_REUSE_RUNS] = { + "workflow_runs": [ + _reuse_run(102, 1850, head_repository="fork/nvx"), + ] + } + destination = root / "destination" + + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built( + outcome, destination, "from other repositories never qualify", "notice" + ) + self.assertEqual(len(api.requests), 2) + + def test_a_push_that_merged_no_pull_request_builds(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + api.documents[f"repos/{_REUSE_REPOSITORY}/git/commits/{_REUSE_MERGE}"] = { + "sha": _REUSE_MERGE, + "parents": [{"sha": "b" * 40}], + } + destination = root / "destination" + + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built( + outcome, + destination, + f"^`{_REUSE_MERGE}` is not the merge commit of a pull request", + "notice", + ) + self.assertEqual(len(api.requests), 1) + + def test_only_an_unexpired_artifact_of_the_run_qualifies(self): + run = guest_artifact_reuse.PullRequestRun(102, "https://example.invalid/102") + name = "kernel-alpine-record" + + def retained(*artifacts: dict[str, object]) -> guest_artifact_reuse.RunArtifact: + api = _FakeGitHub({_reuse_list(name): {"artifacts": list(artifacts)}}) + return guest_artifact_reuse.retained_artifact( + api, _REUSE_REPOSITORY, run, name + ) + + ineligible = ( + _reuse_artifact( + 1, 102, name=name, expired=True, created_at="2026-10-21T13:00:00Z" + ), + _reuse_artifact(2, 102, name="guest-kernel"), + _reuse_artifact(3, 101, name=name, created_at="2026-10-21T14:00:00Z"), + ) + self.assertEqual( + retained( + _reuse_artifact(4, 102, name=name, created_at="2026-10-21T11:00:00Z"), + *ineligible, + _reuse_artifact(5, 102, name=name, digest="sha256:" + "e" * 64), + ), + guest_artifact_reuse.RunArtifact(5, name, "sha256:" + "e" * 64), + ) + with self.assertRaises(guest_artifact_reuse.ReuseRefused) as raised: + retained(*ineligible) + self.assertTrue(raised.exception.expected) + self.assertEqual( + str(raised.exception), + "[run 102](https://example.invalid/102) has no unexpired " + "`kernel-alpine-record` artifact", + ) + + def test_reuse_needs_the_record_and_every_artifact_of_the_components(self): + for name in ( + "ubuntu-azurelinux-record", + "guest-ubuntu", + "guest-ubuntu-distro", + "guest-azurelinux", + ): + with ( + self.subTest(artifact=name), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + components = ("ubuntu", "azurelinux") + api, kernel_inputs, _files = self._fixture(root, components) + # Expired, as after a day, or never uploaded. + api.documents[_reuse_list(name)] = { + "artifacts": [ + _reuse_artifact( + _REUSE_ARTIFACT_IDS[name], 102, name=name, expired=True + ) + ] + } + destination = root / "destination" + + outcome = self._reuse( + api, kernel_inputs, destination, _reuse_request(components) + ) + + self._assert_built( + outcome, + destination, + re.escape(f"has no unexpired `{name}` artifact"), + "notice", + ) + self.assertNotIn(_reuse_download(name), api.requests) + + def test_reuse_requires_the_recorded_cache_keys_to_equal_its_own(self): + def record(keys: Mapping[str, str]) -> bytes: + document = { + "format": 1, + "keys": dict(keys), + "files": { + name: "0" * 64 + for name in guest_artifact_reuse.component_files( + ("kernel", "alpine") + ) + }, + } + return _reuse_zip({"guest-artifacts.json": json.dumps(document).encode()}) + + own = guest_artifact_reuse.component_keys(("kernel", "alpine"), _REUSE_KEYS) + for case, keys, record_keys, names in ( + ( + "changed", + dict(_REUSE_KEYS, alpine="alpine-image-v2-3.24.1-other-5555"), + own, + "alpine", + ), + ("missing", _REUSE_KEYS, {"alpine": own["alpine"]}, "kernel"), + ("extra", _REUSE_KEYS, dict(own, ubuntu=_REUSE_KEYS["ubuntu"]), "ubuntu"), + ("other", _REUSE_KEYS, {"kernel": "other"}, "alpine, kernel"), + ): + with ( + self.subTest(case=case), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + if record_keys is not own: + self._serve(api, "kernel-alpine-record", record(record_keys)) + destination = root / "destination" + + outcome = self._reuse( + api, kernel_inputs, destination, _reuse_request(keys=keys) + ) + + self._assert_built( + outcome, + destination, + f"gives other cache keys for {re.escape(names)},", + "notice", + ) + # Other keys refuse the components before any of their + # artifacts downloads. + self.assertNotIn(_reuse_download("guest-kernel"), api.requests) + + # A change in the key of another job's component does not matter. + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + destination = root / "destination" + + outcome = self._reuse( + api, + kernel_inputs, + destination, + _reuse_request(keys=dict(_REUSE_KEYS, ubuntu="other", azurelinux="")), + ) + + self.assertTrue(outcome.reused, outcome.detail) + + def test_reuse_refuses_files_that_do_not_match_the_record_or_artifact(self): + record_name = "kernel-alpine-record" + + def record(files: Path, update: Mapping[str, object]) -> bytes: + document = json.loads( + (files / guest_artifact_reuse.RECORD_NAME).read_bytes() + ) + document.update(update) + return _reuse_zip({"guest-artifacts.json": json.dumps(document).encode()}) + + def record_files( + files: Path, change: Callable[[dict[str, str]], object] + ) -> bytes: + document = json.loads( + (files / guest_artifact_reuse.RECORD_NAME).read_bytes() + ) + change(document["files"]) + return _reuse_zip({"guest-artifacts.json": json.dumps(document).encode()}) + + def members(files: Path, name: str) -> dict[str, bytes]: + return { + member: (files / member).read_bytes() + for member in sorted(_guest_uploads()[name][1]) + } + + cases: tuple[tuple[str, str, Callable[[Path], bytes], str, str], ...] = ( + ( + "tampered file", + "guest-alpine", + lambda files: _reuse_zip( + {**members(files, "guest-alpine"), "initramfs.cpio.gz": b"x"} + ), + "initramfs.cpio.gz in artifact `guest-alpine` does not match the " + "SHA-256 that the record gives", + "warning", + ), + ( + "missing file", + "guest-kernel", + lambda files: _reuse_zip( + { + name: contents + for name, contents in members(files, "guest-kernel").items() + if name != "vmlinux.config" + } + ), + "holds vmlinux.config", + "warning", + ), + ( + "file in two artifacts", + "guest-alpine", + lambda files: _reuse_zip( + { + **members(files, "guest-alpine"), + "vmlinux": (files / "vmlinux").read_bytes(), + } + ), + "the artifacts hold vmlinux more than once", + "warning", + ), + ( + "record artifact without a record", + record_name, + lambda _files: _reuse_zip({"other.json": b"{}"}), + "artifact `kernel-alpine-record` holds no single guest-artifacts.json", + "warning", + ), + ( + "record without a file", + record_name, + lambda files: record_files( + files, lambda digests: digests.pop("vmlinux.config") + ), + "describes other files than those of kernel and alpine", + "warning", + ), + ( + "record with a file of another component", + record_name, + lambda files: record_files( + files, + lambda digests: digests.update({"ubuntu-distro.erofs": "0" * 64}), + ), + "describes other files than those of kernel and alpine", + "warning", + ), + ( + "record without a digest", + record_name, + lambda files: record_files( + files, lambda digests: digests.update({"vmlinux": "MD5"}) + ), + "record is invalid: it gives no SHA-256 of vmlinux", + "warning", + ), + ( + "unknown record format", + record_name, + lambda files: record(files, {"format": 2}), + "record is invalid: its format is not 1", + "warning", + ), + ( + "boolean record format", + record_name, + lambda files: record(files, {"format": True}), + "record is invalid: its format is not 1", + "warning", + ), + ( + "malformed record", + record_name, + lambda _files: _reuse_zip({"guest-artifacts.json": b"{"}), + "record is invalid", + "warning", + ), + ( + "record not an archive", + record_name, + lambda _files: b"not a ZIP archive", + "artifact `kernel-alpine-record` is not a valid ZIP archive", + "warning", + ), + ( + "files not an archive", + "guest-kernel", + lambda _files: b"not a ZIP archive", + "artifact `guest-kernel` is not a valid ZIP archive", + "warning", + ), + ) + for case, name, contents, reason, level in cases: + with ( + self.subTest(case=case), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, files = self._fixture(root) + self._serve(api, name, contents(files)) + destination = root / "destination" + + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built(outcome, destination, re.escape(reason), level) + + for name in (record_name, "guest-kernel", "guest-alpine"): + with ( + self.subTest(digest_of=name), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + contents = api.downloads[_reuse_download(name)] + self._serve(api, name, contents, digest="sha256:" + "0" * 64) + destination = root / "destination" + + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built( + outcome, + destination, + re.escape( + f"artifact {_REUSE_ARTIFACT_IDS[name]} does not match its " + "digest sha256:" + "0" * 64 + ), + "warning", + ) + + # Without the digest that GitHub took at the upload, nothing but the + # archive itself would vouch for it, so reuse doesn't download it. + for digest in (None, "md5:" + "0" * 32, "sha256:00", "SHA256:" + "0" * 64): + for name in (record_name, "guest-alpine"): + with ( + self.subTest(digest=digest, artifact=name), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + api.documents[_reuse_list(name)] = { + "artifacts": [ + _reuse_artifact( + _REUSE_ARTIFACT_IDS[name], 102, name=name, digest=digest + ) + ] + } + destination = root / "destination" + + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built( + outcome, + destination, + re.escape( + f"artifact {_REUSE_ARTIFACT_IDS[name]} has no SHA-256 " + f"digest, but {digest!r}" + ), + "warning", + ) + self.assertNotIn(_reuse_download(name), api.requests) + + def test_reuse_requires_provenance_that_matches_this_checkout(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + components = ("ubuntu", "azurelinux") + api, kernel_inputs, files = self._fixture(root, components) + # A distro layer built from other inputs, recorded as such. + manifest = files / UbuntuBuildConstants.DISTRO_MANIFEST_NAME + document = json.loads(manifest.read_text(encoding="utf-8")) + document["input_sha256"] = "f" * 64 + manifest.write_text(json.dumps(document), encoding="utf-8") + record = guest_artifact_reuse.record_guest_artifacts( + files, components, _REUSE_KEYS + ) + self._serve( + api, + "ubuntu-azurelinux-record", + _reuse_zip({record.name: record.read_bytes()}), + ) + self._serve( + api, + "guest-ubuntu-distro", + _reuse_zip( + { + name: (files / name).read_bytes() + for name in sorted(_guest_uploads()["guest-ubuntu-distro"][1]) + } + ), + ) + destination = root / "destination" + + outcome = self._reuse( + api, kernel_inputs, destination, _reuse_request(components) + ) + + self._assert_built( + outcome, + destination, + "do not match their provenance or this checkout: Ubuntu artifact " + "manifest does not match ubuntu-distro.erofs", + "warning", + ) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + api, _kernel_inputs, _files = self._fixture(root) + destination = root / "destination" + + # The fixture's kernel provenance names inputs that this checkout + # lacks. + outcome = guest_artifact_reuse.reuse_guest_artifacts( + api, _reuse_request(), destination + ) + + self._assert_built( + outcome, + destination, + "kernel build provenance does not match", + "warning", + ) + + def test_reuse_installs_every_file_or_leaves_the_destination_as_it_was(self): + # The installation moves a restored file aside (calls 1 and 3) before + # it installs its replacement (2 and 4). Each rename fails before it + # happens, or right after it, before the installation can note it. + replace = os.replace + failure = OSError("disk full") + interruption = guest_artifact_reuse.ReuseRefused("interrupted") + for failing_call, error, after_rename in ( + *((number, failure, False) for number in (1, 2, 3, 4)), + *((number, interruption, False) for number in (1, 2, 3, 4)), + *((number, interruption, True) for number in (1, 2, 3, 4)), + ): + with ( + self.subTest( + failing_call=failing_call, + error=str(error), + after_rename=after_rename, + ), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + destination = root / "destination" + destination.mkdir() + restored = { + KernelBuildConstants.BINARY_NAME: b"cached kernel", + KernelBuildConstants.CONFIG_NAME: b"cached kernel config", + } + for name, contents in restored.items(): + (destination / name).write_bytes(contents) + calls: list[str] = [] + + def fail( + source: Path, + target: Path, + calls: list[str] = calls, + failing_call: int = failing_call, + error: BaseException = error, + after_rename: bool = after_rename, + ) -> None: + calls.append(Path(target).name) + if len(calls) != failing_call: + replace(source, target) + return + if after_rename: + replace(source, target) + raise error + + with patch.object(guest_artifact_reuse.os, "replace", fail): + outcome = self._reuse(api, kernel_inputs, destination) + + self.assertFalse(outcome.reused) + self.assertEqual(outcome.status, "build") + self.assertIn(str(error).lower(), outcome.detail.lower()) + self.assertEqual( + { + path.name: path.read_bytes() + for path in destination.iterdir() + if path.is_file() + }, + restored, + ) + + def test_time_limit_refuses_an_expired_deadline_at_once(self): + with guest_artifact_reuse._time_limit(None): + pass + with ( + self.assertRaisesRegex( + guest_artifact_reuse.ReuseRefused, "reuse ran out of time" + ), + guest_artifact_reuse._time_limit(time.monotonic() - 1), + ): + self.fail("an expired time limit ran its block") + + def test_time_limit_interrupts_work_that_outlasts_it(self): + if sys.platform == "win32": + self.skipTest("Windows has no interval timer") + previous = signal.getsignal(signal.SIGALRM) + start = time.monotonic() + with ( + self.assertRaisesRegex( + guest_artifact_reuse.ReuseRefused, "reuse ran out of time" + ), + guest_artifact_reuse._time_limit(time.monotonic() + 0.2), + ): + time.sleep(10) + self.assertLess(time.monotonic() - start, 5) + self.assertEqual(signal.getitimer(signal.ITIMER_REAL), (0.0, 0.0)) + self.assertEqual(signal.getsignal(signal.SIGALRM), previous) + + def test_time_limit_spares_the_installation(self): + if sys.platform == "win32": + self.skipTest("Windows has no interval timer") + install = guest_artifact_reuse._install + timers: dict[str, tuple[float, float]] = {} + + class TimedGitHub(_FakeGitHub): + def get(self, endpoint: str) -> object: + if sys.platform != "win32": + timers.setdefault("request", signal.getitimer(signal.ITIMER_REAL)) + return super().get(endpoint) + + def timed_install( + staging: Path, destination: Path, names: tuple[str, ...] + ) -> None: + if sys.platform != "win32": + timers["installation"] = signal.getitimer(signal.ITIMER_REAL) + install(staging, destination, names) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + fixture, kernel_inputs, _files = self._fixture(root) + api = TimedGitHub(fixture.documents, fixture.downloads) + with ( + patch.object( + release, "kernel_provenance_inputs", return_value=kernel_inputs + ), + patch.object(guest_artifact_reuse, "_install", timed_install), + ): + outcome = guest_artifact_reuse.reuse_guest_artifacts( + api, + _reuse_request(), + root / "destination", + deadline=time.monotonic() + 300, + ) + + self.assertTrue(outcome.reused, outcome.detail) + # The timer runs while reuse fetches and checks the artifact, but not + # while it installs the images, so it can't cut a rollback short. + self.assertGreater(timers["request"][0], 0) + self.assertEqual(timers["installation"], (0.0, 0.0)) + self.assertEqual(signal.getitimer(signal.ITIMER_REAL), (0.0, 0.0)) + + def test_time_limit_ignores_its_timer_once_its_block_ends(self): + if sys.platform == "win32": + self.skipTest("Windows has no interval timer") + previous = signal.getsignal(signal.SIGALRM) + with guest_artifact_reuse._time_limit(time.monotonic() + 300): + expire = signal.getsignal(signal.SIGALRM) + self.assertEqual(signal.getsignal(signal.SIGALRM), previous) + # A signal that arrived just before the timer stopped runs the handler + # after the block, where it must not raise in place of the restoration. + assert callable(expire) + expire(signal.SIGALRM, None) + + def test_reuse_cleans_up_when_its_time_limit_expires_as_the_checks_end(self): + @contextlib.contextmanager + def expiring(_deadline: float | None) -> Generator[None]: + yield + raise guest_artifact_reuse.ReuseRefused("reuse ran out of time") + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + destination = root / "destination" + with patch.object(guest_artifact_reuse, "_time_limit", expiring): + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built( + outcome, destination, "^Reuse ran out of time, so ", "warning" + ) + + def test_reuse_command_falls_back_when_reuse_outlasts_its_time_limit(self): + if sys.platform == "win32": + self.skipTest("Windows has no interval timer") + + class SlowGitHub(_FakeGitHub): + def get(self, endpoint: str) -> object: + time.sleep(10) + return super().get(endpoint) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + output = root / "output" + stdout = io.StringIO() + start = time.monotonic() + with ( + patch.object(guest_artifact_reuse, "REUSE_TIMEOUT_SECONDS", 0.2), + patch.object( + guest_artifact_reuse, "RestApi", return_value=SlowGitHub({}) + ), + contextlib.redirect_stdout(stdout), + ): + outcome = guest_artifact_reuse.reuse_in_ci( + repository=_REUSE_REPOSITORY, + workflow_ref=_REUSE_WORKFLOW_REF, + record="kernel-alpine-record", + components="kernel alpine", + keys=json.dumps(_REUSE_KEYS), + commit=_REUSE_MERGE, + head_sha=None, + dry_run=False, + destination=root / "destination", + environment={"GH_TOKEN": "token", "GITHUB_OUTPUT": str(output)}, + ) + + self.assertLess(time.monotonic() - start, 5) + self.assertFalse(outcome.reused) + self.assertEqual(outcome.level, "warning") + self.assertTrue( + outcome.detail.startswith("Reuse ran out of time, so "), + outcome.detail, + ) + self.assertEqual(output.read_text(encoding="utf-8"), "reused=false\n") + + def test_api_failures_fall_back_to_building(self): + for endpoint in ( + f"repos/{_REUSE_REPOSITORY}/git/commits/{_REUSE_MERGE}", + _REUSE_RUNS, + _reuse_list("kernel-alpine-record"), + _reuse_download("kernel-alpine-record"), + _reuse_list("guest-kernel"), + _reuse_download("guest-alpine"), + ): + with ( + self.subTest(endpoint=endpoint), + tempfile.TemporaryDirectory() as temporary, + ): + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + api.documents.pop(endpoint, None) + api.downloads.pop(endpoint, None) + destination = root / "destination" + + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built( + outcome, + destination, + re.escape(f"GET {endpoint} failed with HTTP 404"), + "warning", + ) + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + api.documents[_REUSE_RUNS] = {"workflow_runs": "invalid"} + destination = root / "destination" + + outcome = self._reuse(api, kernel_inputs, destination) + + self._assert_built( + outcome, + destination, + "GitHub returned an invalid workflow run list", + "warning", + ) + + def test_workflow_path_comes_from_the_workflow_ref(self): + for workflow_ref in ( + f"{_REUSE_REPOSITORY}/{_REUSE_WORKFLOW}@refs/heads/dev", + f"{_REUSE_REPOSITORY}/{_REUSE_WORKFLOW}@refs/pull/702/merge", + ): + with self.subTest(workflow_ref=workflow_ref): + self.assertEqual( + guest_artifact_reuse.workflow_path(workflow_ref, _REUSE_REPOSITORY), + _REUSE_WORKFLOW, + ) + for workflow_ref in ( + f"{_REUSE_REPOSITORY}/{_REUSE_WORKFLOW}", + f"fork/nvx/{_REUSE_WORKFLOW}@refs/heads/dev", + f"{_REUSE_REPOSITORY}/ci.yml@refs/heads/dev", + f"{_REUSE_REPOSITORY}/.github/workflows/nested/ci.yml@refs/heads/dev", + ): + with ( + self.subTest(workflow_ref=workflow_ref), + self.assertRaises(guest_artifact_reuse.ReuseRefused), + ): + guest_artifact_reuse.workflow_path(workflow_ref, _REUSE_REPOSITORY) + + def test_reuse_command_reports_its_outcome_and_never_fails(self): + base = [ + "reuse-guest-artifacts", + "--repository", + _REUSE_REPOSITORY, + "--workflow-ref", + _REUSE_WORKFLOW_REF, + "--record-artifact", + "kernel-alpine-record", + "--commit", + _REUSE_MERGE, + ] + keys = json.dumps(_REUSE_KEYS) + heading = "Guest artifact reuse (kernel, alpine)" + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + api, kernel_inputs, _files = self._fixture(root) + output = root / "output" + summary = root / "summary" + destination = root / "destination" + for case, components, keys_text, token, reused, level, title in ( + ("reused", "alpine kernel", keys, "token", "true", None, heading), + ("restored", "kernel alpine", keys, "token", "false", None, heading), + ("no token", "kernel alpine", keys, "", "false", "warning", heading), + ( + "invalid keys", + "kernel alpine", + "{", + "token", + "false", + "warning", + heading, + ), + ( + "missing key", + "kernel alpine", + json.dumps({"kernel": "key"}), + "token", + "false", + "warning", + heading, + ), + ( + "unknown component", + "kernel debug-kernel", + keys, + "token", + "false", + "warning", + "Guest artifact reuse", + ), + ): + with self.subTest(case=case): + output.write_text("", encoding="utf-8") + summary.write_text("", encoding="utf-8") + args = nvx.parse_args( + [ + *base, + "--components", + components, + "--keys", + keys_text, + "--destination", + os.fspath(destination), + ] + ) + stdout = io.StringIO() + with ( + patch.dict( + os.environ, + { + "GH_TOKEN": token, + "GITHUB_TOKEN": "", + "GITHUB_OUTPUT": os.fspath(output), + "GITHUB_STEP_SUMMARY": os.fspath(summary), + }, + ), + patch.object( + guest_artifact_reuse, "RestApi", return_value=api + ) as rest_api, + patch.object( + release, + "kernel_provenance_inputs", + return_value=kernel_inputs, + ), + contextlib.redirect_stdout(stdout), + ): + self.assertIsNone(args.handler(args)) + self.assertEqual( + output.read_text(encoding="utf-8"), f"reused={reused}\n" + ) + self.assertTrue( + summary.read_text(encoding="utf-8").startswith(f"### {title}: ") + ) + annotations = [ + line + for line in stdout.getvalue().splitlines() + if line.startswith("::") + ] + if level is None: + self.assertEqual(annotations, []) + else: + self.assertEqual(len(annotations), 1) + self.assertTrue( + annotations[0].startswith( + f"::{level} title={title.replace(',', '%2C')}::" + ), + annotations[0], + ) + rest_api.assert_not_called() + with ( + patch("sys.stderr", new_callable=io.StringIO), + self.assertRaises(SystemExit) as exit_context, + ): + nvx.parse_args(base[:-2] + ["--components", "kernel", "--keys", "{}"]) + self.assertEqual(exit_context.exception.code, 2) + # A rehearsal names the head instead of the merge commit. + with ( + patch("sys.stderr", new_callable=io.StringIO), + self.assertRaises(SystemExit) as exit_context, + ): + nvx.parse_args( + [ + *base, + "--components", + "kernel", + "--keys", + "{}", + "--head-sha", + _REUSE_HEAD, + "--dry-run", + ] + ) + self.assertEqual(exit_context.exception.code, 2) + args = nvx.parse_args( + [ + *base[:-2], + "--components", + "kernel", + "--keys", + "{}", + "--head-sha", + _REUSE_HEAD, + "--dry-run", + ] + ) + self.assertEqual( + (args.commit, args.head_sha, args.dry_run), (None, _REUSE_HEAD, True) + ) + + def test_reports_escape_annotations_and_append_outputs(self): + outcome = guest_artifact_reuse.ReuseOutcome( + False, + "Guest artifact reuse", + "build", + "100% of 2\nlines", + "warning", + ) + with tempfile.TemporaryDirectory() as temporary: + output = Path(temporary) / "output" + output.write_text("before=1\n", encoding="utf-8") + stdout = io.StringIO() + with contextlib.redirect_stdout(stdout): + guest_artifact_reuse.report( + outcome, {"GITHUB_OUTPUT": os.fspath(output)} + ) + self.assertEqual( + output.read_text(encoding="utf-8"), "before=1\nreused=false\n" + ) + self.assertIn( + "::warning title=Guest artifact reuse::100%25 of 2%0Alines\n", + stdout.getvalue(), + ) + self.assertEqual( + outcome.summary(), + "### Guest artifact reuse: build\n100% of 2\nlines\n", + ) + + def test_rest_api_reads_json_and_downloads_without_forwarding_the_token(self): + authorizations: dict[str, str | None] = {} + + class StorageHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: + authorizations["storage"] = self.headers.get("Authorization") + body = b"archive" + self.send_response(http.HTTPStatus.OK) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + pass + + storage = http.server.ThreadingHTTPServer(("127.0.0.1", 0), StorageHandler) + + class ApiHandler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: + authorizations[self.path] = self.headers.get("Authorization") + if self.path == "/repos/o/r/runs": + body = b'{"workflow_runs": []}' + self.send_response(http.HTTPStatus.OK) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + elif self.path == "/repos/o/r/zip": + self.send_response(http.HTTPStatus.FOUND) + self.send_header( + "Location", + f"http://127.0.0.1:{storage.server_address[1]}/blob", + ) + self.send_header("Content-Length", "0") + self.end_headers() + else: + self.send_error(http.HTTPStatus.NOT_FOUND) + + def log_message(self, format: str, *args: object) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), ApiHandler) + threads = [ + threading.Thread(target=instance.serve_forever, daemon=True) + for instance in (server, storage) + ] + for thread in threads: + thread.start() + try: + url = f"http://127.0.0.1:{server.server_address[1]}" + api = guest_artifact_reuse.RestApi("secret", time.monotonic() + 60, url=url) + self.assertEqual(api.get("repos/o/r/runs"), {"workflow_runs": []}) + with tempfile.TemporaryDirectory() as temporary: + archive = Path(temporary) / "artifact.zip" + api.download("repos/o/r/zip", archive) + self.assertEqual(archive.read_bytes(), b"archive") + self.assertEqual(authorizations["/repos/o/r/runs"], "Bearer secret") + self.assertEqual(authorizations["/repos/o/r/zip"], "Bearer secret") + self.assertIsNone(authorizations["storage"]) + with self.assertRaisesRegex( + guest_artifact_reuse.ReuseRefused, + "GET repos/o/r/missing failed with HTTP 404", + ): + api.get("repos/o/r/missing") + expired = guest_artifact_reuse.RestApi( + "secret", time.monotonic() - 1, url=url + ) + with self.assertRaisesRegex( + guest_artifact_reuse.ReuseRefused, "ran out of time" + ): + expired.get("repos/o/r/runs") + finally: + for instance in (server, storage): + instance.shutdown() + instance.server_close() + for thread in threads: + thread.join() + + def test_rest_api_refuses_responses_cut_short(self): + # A response that ends before its Content-Length, or within a chunk, + # raises http.client.IncompleteRead, which is no OSError, so it would + # fail the step instead of falling back to building. + class TruncatingHandler(http.server.BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_GET(self) -> None: + self.send_response(http.HTTPStatus.OK) + if self.path == "/repos/o/r/runs": + self.send_header("Content-Length", "100") + self.end_headers() + self.wfile.write(b'{"workflow_runs": ') + else: + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + self.wfile.write(b"64\r\n" + b"x" * 10) + self.wfile.flush() + self.close_connection = True + + def log_message(self, format: str, *args: object) -> None: + pass + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), TruncatingHandler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + api = guest_artifact_reuse.RestApi( + "secret", + time.monotonic() + 60, + url=f"http://127.0.0.1:{server.server_address[1]}", + ) + with self.assertRaisesRegex( + guest_artifact_reuse.ReuseRefused, + "^GET repos/o/r/runs failed: IncompleteRead", + ): + api.get("repos/o/r/runs") + with ( + tempfile.TemporaryDirectory() as temporary, + self.assertRaisesRegex( + guest_artifact_reuse.ReuseRefused, + "^GET repos/o/r/zip failed: IncompleteRead", + ), + ): + api.download("repos/o/r/zip", Path(temporary) / "artifact.zip") + finally: + server.shutdown() + server.server_close() + thread.join() + + class CiConfigurationTests(unittest.TestCase): def test_ci_needs_reference_declared_jobs(self): workflow = ( @@ -4827,6 +6467,9 @@ def test_required_ci_result_policy(self): ci.REQUIRED_CI_DEBUG_KERNEL_JOB, } platforms = set(ci.REQUIRED_CI_PLATFORM_JOBS) + # A same-repository pull request that builds the guest artifacts also + # rehearses their reuse. + rehearsal = {ci.REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB} microvm = microvm_tests | debug_kernel full_matrix = ( repository @@ -4852,14 +6495,14 @@ def test_required_ci_result_policy(self): "pull_request", True, (True, True, False, False, False, True), - repository | artifacts | builds | platforms | gate, + repository | artifacts | rehearsal | builds | platforms | gate, ), # Only the microVM tests change. ( "pull_request", True, (True, True, False, False, True, False), - repository | artifacts | builds | microvm, + repository | artifacts | rehearsal | builds | microvm, ), # Only the OpenVMM suites, which build OpenVMM themselves. ( @@ -4872,7 +6515,7 @@ def test_required_ci_result_policy(self): "pull_request", True, (True, False, False, True, False, False), - repository | artifacts | openvmm_artifact_tests, + repository | artifacts | rehearsal | openvmm_artifact_tests, ), ( "pull_request", @@ -4880,6 +6523,7 @@ def test_required_ci_result_policy(self): everything, repository | artifacts + | rehearsal | builds | openvmm_tests | openvmm_artifact_tests @@ -4892,7 +6536,13 @@ def test_required_ci_result_policy(self): "pull_request", True, (True, True, False, False, True, True), - repository | artifacts | builds | microvm | platforms | gate, + repository + | artifacts + | rehearsal + | builds + | microvm + | platforms + | gate, ), # Guest inputs reach the VMM tests but not the unit tests. ( @@ -4901,6 +6551,7 @@ def test_required_ci_result_policy(self): (True, True, False, True, True, True), repository | artifacts + | rehearsal | builds | openvmm_artifact_tests | microvm @@ -4919,7 +6570,7 @@ def test_required_ci_result_policy(self): "pull_request", True, (True, False, False, True, True, True), - repository | artifacts | openvmm_artifact_tests, + repository | artifacts | rehearsal | openvmm_artifact_tests, ), ("pull_request", False, (False,) * 6, always_successful), ( @@ -5074,6 +6725,7 @@ def test_ci_jobs_follow_their_change_detector_output(self): ci.REQUIRED_CI_ARTIFACT_JOB: "run-guest-artifacts", ci.REQUIRED_CI_DISTRO_ARTIFACT_JOB: "run-guest-artifacts", ci.REQUIRED_CI_DEBUG_KERNEL_JOB: "run-guest-artifacts", + ci.REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB: "run-guest-artifacts", **dict.fromkeys(ci.REQUIRED_CI_BUILD_JOBS, "run-openvmm-binaries"), **dict.fromkeys(ci.REQUIRED_CI_OPENVMM_TEST_JOBS, "run-openvmm-unit-tests"), **dict.fromkeys( @@ -5348,9 +7000,10 @@ def test_ci_full_runs_do_not_publish_persist_or_save_caches(self): with self.subTest(call=name): self.assertEqual(arguments.get("save-cache"), save_cache) # No other push-only job condition is allowed in this workflow: only - # its eight cache saves, the release, performance persistence, and the - # four save-cache arguments name the push event. - self.assertEqual(workflow.count("github.event_name == 'push'"), 14) + # its eight cache saves, the release, performance persistence, the + # four save-cache arguments, and the two reuse arguments name the push + # event. + self.assertEqual(workflow.count("github.event_name == 'push'"), 16) def test_ci_change_classification_selects_suites_by_input(self): guest, binaries, _, vmm, microvm, benchmarks = ci.CI_CHANGE_OUTPUTS @@ -7093,6 +8746,419 @@ def test_guest_artifact_restore_only_leaves_restore_and_scratch_steps_active(sel continue self.assertNotIn("restore-only", condition) + def test_guest_artifact_jobs_record_their_components(self): + github = BuildConstants.REPO_ROOT / ".github" + action = yaml.safe_load( + (github / "actions" / "build-guest-artifacts" / "action.yml").read_text( + encoding="utf-8" + ) + ) + steps = action["runs"]["steps"] + # The cache entries of each recorded component, as its restore steps + # name them, hold exactly its files under exactly the keys that its + # record gives. + restored: dict[str, tuple[set[str], set[str]]] = {} + for step in steps: + if not str(step.get("uses", "")).startswith("actions/cache/restore@"): + continue + [component] = re.findall( + r"steps\.components\.outputs\.([a-z-]+) == 'true'", step["if"] + ) + key = re.fullmatch( + r"\$\{\{ steps\.guest-cache-keys\.outputs\.([a-z-]+) \}\}", + step["with"]["key"], + ) + assert key is not None + keys, files = restored.setdefault(component, (set(), set())) + keys.add(key.group(1)) + files.update( + PurePosixPath(path).name for path in step["with"]["path"].split() + ) + self.assertEqual( + { + component: entry + for component, entry in restored.items() + if component != "debug-kernel" + }, + { + component: ( + set(keys), + set(release.GUEST_ARTIFACT_COMPONENTS[component]), + ) + for component, keys in guest_artifact_reuse.COMPONENT_CACHE_KEYS.items() + }, + ) + # The record follows every build, and restore-only, which builds + # nothing, records nothing. + record, upload = steps[-2:] + self.assertEqual(record["name"], "Record guest artifact keys and digests") + self.assertEqual( + _expression_conjuncts(record["if"]), + ["inputs.restore-only != 'true'", "inputs.record-artifact != ''"], + ) + self.assertEqual( + record["env"], + { + "COMPONENTS": "${{ inputs.components }}", + "GUEST_CACHE_KEYS": "${{ toJSON(steps.guest-cache-keys.outputs) }}", + }, + ) + self.assertEqual( + record["run"].split(), + [ + "python3", + "scripts/nvx.py", + "record-guest-artifacts", + "--components", + '"${COMPONENTS}"', + "--keys", + '"${GUEST_CACHE_KEYS}"', + ], + ) + self.assertEqual(upload["name"], "Upload guest artifact record") + self.assertEqual(upload["if"], record["if"]) + self.assertEqual(upload["uses"], "actions/upload-artifact@v7") + # It expires with the components' artifacts. + self.assertEqual( + upload["with"], + { + "name": "${{ inputs.record-artifact }}", + "path": f"build/{guest_artifact_reuse.RECORD_NAME}", + "if-no-files-found": "error", + "retention-days": 1, + }, + ) + + # Each job that builds recorded components records them under a name + # of its own, which the guest-* names of the handoff never take; the + # debug kernel job records nothing. + workflow = yaml.safe_load( + (github / "workflows" / "ci.yml").read_text(encoding="utf-8") + ) + records: dict[str, tuple[str, ...]] = {} + for job_name, job in workflow["jobs"].items(): + for step in job.get("steps", []): + if step.get("uses") != "./.github/actions/build-guest-artifacts": + continue + arguments = step["with"] + with self.subTest(job=job_name): + if arguments["components"].split() == ["debug-kernel"]: + self.assertNotIn("record-artifact", arguments) + continue + name = arguments["record-artifact"] + self.assertFalse(name.startswith("guest-"), name) + self.assertNotIn(name, records) + records[name] = guest_artifact_reuse.parse_components( + arguments["components"] + ) + self.assertEqual( + records, + { + "kernel-alpine-record": ("kernel", "alpine"), + "ubuntu-azurelinux-record": ("ubuntu", "azurelinux"), + }, + ) + self.assertEqual(tuple(records.values()), _REUSE_JOBS) + + def test_dev_pushes_reuse_the_guest_artifacts_of_their_pull_requests(self): + github = BuildConstants.REPO_ROOT / ".github" + workflow = yaml.safe_load( + (github / "workflows" / "ci.yml").read_text(encoding="utf-8") + ) + jobs = workflow["jobs"] + # Reading runs and downloading their artifacts needs actions: read, + # which no other job but the rehearsal and the release gets. + self.assertEqual(workflow["permissions"], {"contents": "read"}) + self.assertEqual( + { + name + for name, job in jobs.items() + if "actions" in (job.get("permissions") or {}) + }, + {"artifacts", "distro-artifacts", "guest-artifact-reuse", "release"}, + ) + reusing: set[str] = set() + for job_name, job in jobs.items(): + for step in job.get("steps", []): + if step.get("uses") != "./.github/actions/build-guest-artifacts": + continue + arguments = step["with"] + with self.subTest(job=job_name): + if "record-artifact" not in arguments: + self.assertNotIn("reuse", arguments) + continue + reusing.add(job_name) + self.assertEqual( + job["permissions"], {"actions": "read", "contents": "read"} + ) + # Only pushes reuse; pull requests and full runs build what + # misses the cache, as before. + self.assertEqual( + arguments["reuse"], "${{ github.event_name == 'push' }}" + ) + self.assertEqual(reusing, {"artifacts", "distro-artifacts"}) + + action = yaml.safe_load( + (github / "actions" / "build-guest-artifacts" / "action.yml").read_text( + encoding="utf-8" + ) + ) + self.assertEqual(action["inputs"]["reuse"]["default"], "false") + steps = action["runs"]["steps"] + position = next( + index for index, step in enumerate(steps) if step.get("id") == "reuse" + ) + reuse = steps[position] + self.assertEqual( + reuse["name"], "Reuse the merged pull request's guest artifacts" + ) + # A failure falls back to building rather than failing the run. + self.assertIs(reuse["continue-on-error"], True) + self.assertEqual( + _expression_conjuncts(reuse["if"]), + [ + "inputs.restore-only != 'true'", + "inputs.reuse == 'true'", + "inputs.record-artifact != ''", + ], + ) + self.assertEqual( + reuse["env"], + { + "GH_TOKEN": "${{ github.token }}", + "COMPONENTS": "${{ inputs.components }}", + "GUEST_CACHE_KEYS": "${{ toJSON(steps.guest-cache-keys.outputs) }}", + "RECORD_ARTIFACT": "${{ inputs.record-artifact }}", + }, + ) + self.assertEqual( + reuse["run"].split(), + [ + "python3", + "scripts/nvx.py", + "reuse-guest-artifacts", + "--repository", + '"${GITHUB_REPOSITORY}"', + "--workflow-ref", + '"${GITHUB_WORKFLOW_REF}"', + "--commit", + '"${GITHUB_SHA}"', + "--components", + '"${COMPONENTS}"', + "--record-artifact", + '"${RECORD_ARTIFACT}"', + "--keys", + '"${GUEST_CACHE_KEYS}"', + ], + ) + # Reuse fills in what the restores of the recorded components missed, + # before any of their builds, which then skip. The saves still cache + # what it installed, and the debug kernel never comes from it. + restores = [ + index + for index, step in enumerate(steps) + if str(step.get("uses", "")).startswith("actions/cache/restore@") + and "debug" not in step["name"] + ] + builds = [ + index + for index, step in enumerate(steps) + if "docker build" in str(step.get("run", "")) + and "debug" not in step["name"] + ] + + def required(expression: str) -> list[str]: + # The conditions that an expression ANDs, through parentheses. + conditions: list[str] = [] + for conjunct in _expression_conjuncts(expression): + if _outer_parentheses_enclose(conjunct): + conditions.extend(required(conjunct.strip()[1:-1])) + else: + conditions.append(conjunct) + return conditions + + self.assertEqual(len(restores), 5) + self.assertEqual(len(builds), 6) + self.assertLess(max(restores), position) + self.assertLess(position, min(builds)) + for index in builds: + with self.subTest(step=steps[index]["name"]): + self.assertIn( + "steps.reuse.outputs.reused != 'true'", + required(steps[index]["if"]), + ) + for step in steps: + if step is not reuse and ( + "debug" in step["name"] + or str(step.get("uses", "")).startswith("actions/") + ): + with self.subTest(step=step["name"]): + self.assertNotIn("steps.reuse", str(step.get("if", ""))) + + def test_pull_requests_rehearse_guest_artifact_reuse(self): + github = BuildConstants.REPO_ROOT / ".github" + workflow = yaml.safe_load( + (github / "workflows" / "ci.yml").read_text(encoding="utf-8") + ) + jobs = workflow["jobs"] + rehearsal = jobs[ci.REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB] + self.assertEqual(rehearsal["name"], "Rehearse guest artifact reuse") + self.assertEqual( + rehearsal["needs"], ["openvmm-changes", "artifacts", "distro-artifacts"] + ) + # Same-repository pull requests only, once both guest artifact jobs + # succeeded: a push reuses for real, and a fork's run never qualifies. + self.assertEqual( + _expression_conjuncts(rehearsal["if"]), + [ + "always()", + "!cancelled()", + "github.event_name == 'pull_request'", + "github.event.pull_request.head.repo.full_name == github.repository", + "needs.artifacts.result == 'success'", + "needs.distro-artifacts.result == 'success'", + "( needs.openvmm-changes.result != 'success' || " + "needs.openvmm-changes.outputs.run-guest-artifacts == 'true' )", + ], + ) + self.assertEqual(rehearsal["runs-on"], "ubuntu-latest") + self.assertEqual( + rehearsal["permissions"], {"actions": "read", "contents": "read"} + ) + # One step for each guest artifact job, which checks that job's + # record and artifacts with that job's keys. + steps = [step for step in rehearsal["steps"] if "run" in step] + records: dict[str, str] = {} + for job_name in ("artifacts", "distro-artifacts"): + job = jobs[job_name] + build = next( + step + for step in job["steps"] + if step.get("uses") == "./.github/actions/build-guest-artifacts" + ) + records[job_name] = build["with"]["record-artifact"] + # It compares the record with the keys that the action resolved. + self.assertEqual( + job["outputs"], + {"guest-cache-keys": "${{ steps.guest-artifacts.outputs.cache-keys }}"}, + ) + self.assertEqual(len(steps), 2) + for step, job_name in zip( + steps, ("artifacts", "distro-artifacts"), strict=True + ): + build = next( + step + for step in jobs[job_name]["steps"] + if step.get("uses") == "./.github/actions/build-guest-artifacts" + ) + with self.subTest(job=job_name): + self.assertEqual( + step["env"], + { + "GH_TOKEN": "${{ github.token }}", + "GUEST_CACHE_KEYS": ( + f"${{{{ needs.{job_name}.outputs.guest-cache-keys }}}}" + ), + "HEAD_SHA": "${{ github.event.pull_request.head.sha }}", + }, + ) + run = step["run"].split() + components = run[run.index("--components") + 1 :] + components = components[: components.index("--record-artifact")] + self.assertEqual( + " ".join(components).strip('"'), build["with"]["components"] + ) + self.assertEqual( + run, + [ + "python3", + "scripts/nvx.py", + "reuse-guest-artifacts", + "--dry-run", + "--repository", + '"${GITHUB_REPOSITORY}"', + "--workflow-ref", + '"${GITHUB_WORKFLOW_REF}"', + "--head-sha", + '"${HEAD_SHA}"', + "--components", + *components, + "--record-artifact", + records[job_name], + "--keys", + '"${GUEST_CACHE_KEYS}"', + ], + ) + action = yaml.safe_load( + (github / "actions" / "build-guest-artifacts" / "action.yml").read_text( + encoding="utf-8" + ) + ) + self.assertEqual( + action["outputs"]["cache-keys"]["value"], + "${{ toJSON(steps.guest-cache-keys.outputs) }}", + ) + # The required status check takes its result; no other job waits for + # it. + for name, job in jobs.items(): + with self.subTest(job=name): + if name == "required-status-check": + self.assertIn(ci.REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB, job["needs"]) + else: + self.assertNotIn( + ci.REQUIRED_CI_GUEST_ARTIFACT_REUSE_JOB, + str(job.get("needs", "")), + ) + # A change to the reuse, its record, or their commands rehearses it. + for path in ( + "scripts/nvx_tools/guest_artifact_reuse.py", + "scripts/nvx_tools/release.py", + "scripts/nvx.py", + ".github/actions/build-guest-artifacts/action.yml", + ): + with self.subTest(path=path): + self.assertTrue( + ci.classify_ci_changes("pull_request", [path]).run_guest_artifacts + ) + + def test_reuse_downloads_the_artifacts_of_each_recorded_component(self): + # The artifacts that reuse downloads for each component are those in + # which the job that builds it hands its files over, and together they + # hold every file of the component; the scratch template, which every + # run formats anew, is in none. + github = BuildConstants.REPO_ROOT / ".github" + workflow = yaml.safe_load( + (github / "workflows" / "ci.yml").read_text(encoding="utf-8") + ) + builders: dict[str, str] = {} + for job_name, job in workflow["jobs"].items(): + for step in job.get("steps", []): + if step.get("uses") == "./.github/actions/build-guest-artifacts": + for component in step["with"]["components"].split(): + builders[component] = job_name + uploads = _guest_uploads() + self.assertEqual( + set(guest_artifact_reuse.COMPONENT_ARTIFACTS), + set(release.GUEST_ARTIFACT_COMPONENTS), + ) + handed_over: set[str] = set() + for component, names in guest_artifact_reuse.COMPONENT_ARTIFACTS.items(): + with self.subTest(component=component): + files: list[str] = [] + for name in names: + job_name, members = uploads[name] + self.assertEqual(job_name, builders[component]) + files.extend(members) + handed_over.add(name) + self.assertEqual( + sorted(files), + sorted(release.GUEST_ARTIFACT_COMPONENTS[component]), + ) + self.assertEqual( + set(uploads) - handed_over, + {"guest-ubuntu-scratch", "guest-debug-kernel"}, + ) + def test_ci_runs_debug_kernel_on_kvm_for_prs_and_all_for_non_prs(self): workflow = ( BuildConstants.REPO_ROOT / ".github" / "workflows" / "ci.yml" @@ -12487,6 +14553,10 @@ def test_ci_guest_artifact_components_select_their_steps(self): "Select guest artifact components", "Resolve guest artifact versions", "Resolve guest cache keys", + # A record and its reuse cover every selected component. + "Reuse the merged pull request's guest artifacts", + "Record guest artifact keys and digests", + "Upload guest artifact record", } owners = ( ("debug kernel", "debug-kernel"), @@ -20780,6 +22850,103 @@ def test_package_rejects_stale_initramfs_package_manifest_provenance(self): paths["build"] / InitramfsBuildConstants.PROVENANCE_NAME, ) + def test_guest_artifact_validation_checks_the_files_of_each_component(self): + # The components hold every guest image file once. + files = [ + name + for names in release.GUEST_ARTIFACT_COMPONENTS.values() + for name in names + ] + self.assertEqual(len(files), len(set(files))) + self.assertEqual(set(files), set(DockerBuildConstants.ALL_GUEST_ARTIFACT_NAMES)) + components = list(release.GUEST_ARTIFACT_COMPONENTS) + with tempfile.TemporaryDirectory() as temporary: + paths, kernel_inputs, _revision = _write_release_fixture(Path(temporary)) + images = paths["build"] + with patch.object( + release, "kernel_provenance_inputs", return_value=kernel_inputs + ): + release.validate_guest_artifacts(images, components) + for component, name, message in ( + ("kernel", "vmlinux", "kernel build provenance does not match"), + ( + "kernel", + "vmlinux.config", + "kernel build provenance does not match", + ), + ( + "alpine", + "initramfs.cpio.gz", + "initramfs build provenance does not match", + ), + ( + "alpine", + "initramfs.cpio.gz.packages.json", + "initramfs build provenance does not match", + ), + ( + "ubuntu", + "initramfs-ubuntu.cpio.gz", + "Ubuntu artifact manifest does not match " + "initramfs-ubuntu.cpio.gz: initramfs-ubuntu.cpio.gz." + "packages.json differs in artifact_sha256", + ), + ( + "ubuntu", + "ubuntu-distro.erofs", + "Ubuntu artifact manifest does not match ubuntu-distro." + "erofs: ubuntu-distro.erofs.manifest.json differs in " + "artifact_sha256", + ), + ( + "azurelinux", + "initramfs-azurelinux.cpio.gz", + "Azure Linux initramfs manifest is invalid", + ), + ): + with self.subTest(file=name): + original = (images / name).read_bytes() + (images / name).write_bytes(b"stale " + original) + try: + with self.assertRaisesRegex( + common.ScriptError, re.escape(message) + ): + release.validate_guest_artifacts(images, [component]) + # Only its own component checks a file. + release.validate_guest_artifacts( + images, + [other for other in components if other != component], + ) + finally: + (images / name).write_bytes(original) + # A component needs only its own files. + for component, names in release.GUEST_ARTIFACT_COMPONENTS.items(): + with ( + self.subTest(component=component), + tempfile.TemporaryDirectory() as alone, + ): + for name in names: + shutil.copyfile(images / name, Path(alone) / name) + release.validate_guest_artifacts(Path(alone), [component]) + for selection, message in ( + ([], "no guest artifact component to validate"), + ( + ["kernel", "debug-kernel"], + "unknown guest artifact component: debug-kernel", + ), + ): + with ( + self.subTest(components=selection), + self.assertRaisesRegex(common.ScriptError, message), + ): + release.validate_guest_artifacts(images, selection) + (images / KernelBuildConstants.PROVENANCE_NAME).unlink() + with self.assertRaisesRegex( + common.ScriptError, + "guest artifact vmlinux.provenance.json not found", + ): + release.validate_guest_artifacts(images, ["kernel"]) + def test_guest_release_inputs_reject_stale_ubuntu_artifacts(self): for artifact_name, manifest_name in ( ("initramfs-ubuntu.cpio.gz", "initramfs-ubuntu.cpio.gz.packages.json"),