From a880683eb18e17e475a6ac9022b84bc24abb9c03 Mon Sep 17 00:00:00 2001 From: Bas van Gijzel Date: Mon, 3 Aug 2026 15:27:52 +0000 Subject: [PATCH 1/4] ci: wait for the in-flight mirror run before pulling node images On PRs that touch mirror-docker-images.yml, the mirror run and the edr-npm-release.yml docker jobs start in parallel, so the docker jobs could race ahead and fail their pulls with 'manifest unknown', needing a manual re-run once the mirror was green. The select-node-image action now polls the mirror workflow run for the same head SHA (when one exists) until it completes, same technique the HH3 regression benchmark uses to wait for edr-ci.yml. Closes #1593 --- .github/actions/select-node-image/action.yml | 44 +++++++++++++++++++ .github/workflows/edr-npm-release.yml | 5 +++ .github/workflows/mirror-docker-images.yml | 4 +- .../src/02_development/11_ci_docker_mirror.md | 2 +- 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.github/actions/select-node-image/action.yml b/.github/actions/select-node-image/action.yml index af3a95c7ea..96afdfac3b 100644 --- a/.github/actions/select-node-image/action.yml +++ b/.github/actions/select-node-image/action.yml @@ -17,6 +17,50 @@ outputs: runs: using: composite steps: + # On PRs/pushes that touch mirror-docker-images.yml, the mirror run and + # this workflow's docker jobs start in parallel, so a pull can race ahead + # of the mirror and fail with "manifest unknown". Same technique as + # waitForEdrCi in scripts/resolve-regression-trigger.cjs, except "no run + # for this SHA" means nothing to wait for (the common case) rather than + # "not started yet". Callers need `actions: read` to list workflow runs. + - name: Wait for in-flight mirror run + if: inputs.release-tag == '' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # For pull_request events the mirror run is recorded against the PR + # head SHA, not the merge commit that github.sha points at. + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + set -euo pipefail + DEADLINE=$(( $(date +%s) + 15 * 60 )) + while true; do + RUN=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/mirror-docker-images.yml/runs?head_sha=${HEAD_SHA}&per_page=1" \ + --jq '.workflow_runs[0] // empty') + if [ -z "$RUN" ]; then + echo "No mirror run for ${HEAD_SHA}; not waiting." + exit 0 + fi + STATUS=$(jq -r .status <<<"$RUN") + if [ "$STATUS" = "completed" ]; then + CONCLUSION=$(jq -r .conclusion <<<"$RUN") + URL=$(jq -r .html_url <<<"$RUN") + if [ "$CONCLUSION" = "success" ]; then + echo "Mirror run succeeded: $URL" + exit 0 + fi + echo "Mirror run for ${HEAD_SHA} concluded '$CONCLUSION': $URL" >&2 + exit 1 + fi + if [ "$(date +%s)" -ge "$DEADLINE" ]; then + echo "Timed out waiting for the mirror run for ${HEAD_SHA}" >&2 + exit 1 + fi + echo "Mirror run in progress (status: $STATUS); retrying in 30s..." + sleep 30 + done + - id: select shell: bash env: diff --git a/.github/workflows/edr-npm-release.yml b/.github/workflows/edr-npm-release.yml index f998a9a7e1..04653eaef5 100644 --- a/.github/workflows/edr-npm-release.yml +++ b/.github/workflows/edr-npm-release.yml @@ -32,6 +32,7 @@ jobs: permissions: contents: read packages: read + actions: read # select-node-image lists mirror workflow runs strategy: fail-fast: false matrix: @@ -279,6 +280,7 @@ jobs: permissions: contents: read packages: read + actions: read # select-node-image lists mirror workflow runs strategy: fail-fast: false matrix: @@ -331,6 +333,7 @@ jobs: permissions: contents: read packages: read + actions: read # select-node-image lists mirror workflow runs strategy: fail-fast: false matrix: @@ -381,6 +384,7 @@ jobs: permissions: contents: read packages: read + actions: read # select-node-image lists mirror workflow runs strategy: fail-fast: false matrix: @@ -436,6 +440,7 @@ jobs: permissions: contents: read packages: read + actions: read # select-node-image lists mirror workflow runs strategy: fail-fast: false matrix: diff --git a/.github/workflows/mirror-docker-images.yml b/.github/workflows/mirror-docker-images.yml index 659cc445f6..9b79416401 100644 --- a/.github/workflows/mirror-docker-images.yml +++ b/.github/workflows/mirror-docker-images.yml @@ -23,7 +23,9 @@ on: paths: - .github/workflows/mirror-docker-images.yml # Same-repo PRs touching this file mirror immediately, so a PR that adds a - # tag below (e.g. a new node version) can be tested before merge. + # tag below (e.g. a new node version) can be tested before merge. The + # edr-npm-release.yml docker jobs wait for this run before pulling (see the + # select-node-image action). pull_request: paths: - .github/workflows/mirror-docker-images.yml diff --git a/book/src/02_development/11_ci_docker_mirror.md b/book/src/02_development/11_ci_docker_mirror.md index 5593da270e..98d5e1a4ef 100644 --- a/book/src/02_development/11_ci_docker_mirror.md +++ b/book/src/02_development/11_ci_docker_mirror.md @@ -8,7 +8,7 @@ Release runs are the exception: when `check_commit` resolves a release tag, the Add the tag to the `TAGS` list in `mirror-docker-images.yml` in the same PR that changes the matrix in `edr-npm-release.yml`. The mirror workflow runs on same-repo PRs that touch it, so the new tag is mirrored — and the release matrix testable against it — before merge. A tag referenced in CI but missing from the mirror fails loudly with `manifest unknown`. -On such a PR the mirror job and the release-workflow jobs start in parallel, so on the first run the release jobs can race ahead and fail their pulls with `manifest unknown`. That's benign: re-run the failed jobs once the mirror job is green. +On such a PR the mirror job and the release-workflow docker jobs start in parallel. The docker jobs' "Select image source" step waits (up to 15 minutes) for the mirror run on the same commit to finish before pulling, so the new tag is in place by the time it's needed. If the mirror run fails, the docker jobs fail with a link to it instead of a `manifest unknown` pull error. On PRs that don't touch the mirror workflow no mirror run exists and the step doesn't wait. ## Access From 363d0086ca6ee2b3b3994918131f35eb3d3a2c53 Mon Sep 17 00:00:00 2001 From: Bas van Gijzel Date: Mon, 3 Aug 2026 17:54:39 +0000 Subject: [PATCH 2/4] ci: gate release runs on a single is_release output, assert cold builds check_commit's stringly tag output gated five different steps and jobs with three different expression shapes; a mistake in any of them would silently put a release build on the unsafe path (warm poisonable cache, mirror image in the supply chain). Derive an explicit is_release output from tag in one place, gate every consumer with the same expression, and prove the cold-build invariant at runtime: release builds now fail loudly if a cargo cache was restored. tag itself remains only as the npm dist-tag for publish. Raised by @anaPerezGhiglia in the #1587 review (#1593). --- .github/actions/select-node-image/action.yml | 10 ++--- .github/workflows/edr-npm-release.yml | 37 ++++++++++++++----- .../src/02_development/11_ci_docker_mirror.md | 2 +- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/.github/actions/select-node-image/action.yml b/.github/actions/select-node-image/action.yml index 96afdfac3b..2e75b0d18c 100644 --- a/.github/actions/select-node-image/action.yml +++ b/.github/actions/select-node-image/action.yml @@ -5,8 +5,8 @@ inputs: tag: description: node image tag, e.g. 24-bullseye-slim or 22-alpine required: true - release-tag: - description: check_commit's release tag; non-empty marks a release run + is-release: + description: check_commit's is_release output; 'true' marks a release run required: true outputs: @@ -24,7 +24,7 @@ runs: # for this SHA" means nothing to wait for (the common case) rather than # "not started yet". Callers need `actions: read` to list workflow runs. - name: Wait for in-flight mirror run - if: inputs.release-tag == '' + if: inputs.is-release != 'true' shell: bash env: GH_TOKEN: ${{ github.token }} @@ -65,9 +65,9 @@ runs: shell: bash env: TAG: ${{ inputs.tag }} - RELEASE_TAG: ${{ inputs.release-tag }} + IS_RELEASE: ${{ inputs.is-release }} run: | - if [ -n "$RELEASE_TAG" ]; then + if [ "$IS_RELEASE" = "true" ]; then echo "ref=node:${TAG}" >> "$GITHUB_OUTPUT" else echo "ref=ghcr.io/nomicfoundation/edr/mirror/node:${TAG}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/edr-npm-release.yml b/.github/workflows/edr-npm-release.yml index 04653eaef5..48b8ee4293 100644 --- a/.github/workflows/edr-npm-release.yml +++ b/.github/workflows/edr-npm-release.yml @@ -124,8 +124,8 @@ jobs: # Release commits build cold: napi codegen relies on proc-macro # expansion (napi-rs#1297), so warm-cache builds risk shipping # stale or empty index.js / index.d.ts. Don't enable for releases. - - name: Cache cargo # zizmor: ignore[cache-poisoning] release builds run cold (guarded below); PR/main caches never feed a publish - if: needs.check_commit.outputs.tag == '' + - name: Cache cargo # zizmor: ignore[cache-poisoning] release builds run cold (asserted below); PR/main caches never feed a publish + if: needs.check_commit.outputs.is_release != 'true' uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 with: path: | @@ -138,6 +138,18 @@ jobs: restore-keys: | ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}- + # Proves the cold-build invariant at runtime instead of trusting the + # skipped cache step: a broken gate fails the release loudly rather + # than shipping binaries built from a warm, poisonable cache. + - name: Assert release builds run cold + if: needs.check_commit.outputs.is_release == 'true' + shell: bash + run: | + if [ -e "${GITHUB_WORKSPACE}/target" ]; then + echo "target/ exists on a release build: the cargo cache gate is broken" >&2 + exit 1 + fi + - name: Install dependencies (non-Windows) if: ${{ !matrix.settings.docker_tag && runner.os != 'Windows' }} run: sfw pnpm install --frozen-lockfile --prefer-offline @@ -159,7 +171,7 @@ jobs: uses: ./.github/actions/select-node-image with: tag: ${{ matrix.settings.docker_tag }} - release-tag: ${{ needs.check_commit.outputs.tag }} + is-release: ${{ needs.check_commit.outputs.is_release }} - name: Build in docker (gnu) uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) @@ -318,7 +330,7 @@ jobs: uses: ./.github/actions/select-node-image with: tag: ${{ matrix.node }} - release-tag: ${{ needs.check_commit.outputs.tag }} + is-release: ${{ needs.check_commit.outputs.is_release }} - name: Test bindings # Setting CI=1 is important to make PNPM install non-interactive # https://github.com/pnpm/pnpm/issues/6615#issuecomment-1656945689 @@ -371,7 +383,7 @@ jobs: uses: ./.github/actions/select-node-image with: tag: ${{ matrix.node }}-alpine - release-tag: ${{ needs.check_commit.outputs.tag }} + is-release: ${{ needs.check_commit.outputs.is_release }} - name: Test bindings env: IMAGE: ${{ steps.select-image.outputs.ref }} @@ -417,7 +429,7 @@ jobs: uses: ./.github/actions/select-node-image with: tag: ${{ matrix.node }} - release-tag: ${{ needs.check_commit.outputs.tag }} + is-release: ${{ needs.check_commit.outputs.is_release }} - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64) test runner uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 with: @@ -472,7 +484,7 @@ jobs: uses: ./.github/actions/select-node-image with: tag: ${{ matrix.node }}-alpine - release-tag: ${{ needs.check_commit.outputs.tag }} + is-release: ${{ needs.check_commit.outputs.is_release }} - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64/musl) test runner uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 with: @@ -519,8 +531,13 @@ jobs: fi fi + # Single authority for release/non-release gating: every release-only or + # non-release-only step and job keys off `is_release`, derived here from + # `tag` so the two can't diverge. `tag` itself is only for the npm + # dist-tag (next/latest) in `publish`. outputs: tag: ${{ steps.check_commit.outputs.tag }} + is_release: ${{ steps.check_commit.outputs.tag != '' }} prepare: name: Prepare release bundle @@ -670,7 +687,7 @@ jobs: name: Cargo cooldown check runs-on: ubuntu-24.04 needs: check_commit - if: ${{ needs.check_commit.outputs.tag != '' }} + if: ${{ needs.check_commit.outputs.is_release == 'true' }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -683,7 +700,7 @@ jobs: name: Notify pre-deploy to Slack runs-on: ubuntu-24.04 needs: [check_commit, cooldown-check, prepare, review] - if: ${{ needs.check_commit.outputs.tag != '' }} + if: ${{ needs.check_commit.outputs.is_release == 'true' }} steps: - name: Notify pre-deploy uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 @@ -719,7 +736,7 @@ jobs: id-token: write # Needed for npm Trusted Publishing (OIDC) needs: [check_commit, notify-deploy, prepare, review] # Only run workflow if there is a tag defined - if: ${{ needs.check_commit.outputs.tag != '' }} + if: ${{ needs.check_commit.outputs.is_release == 'true' }} steps: - name: Notify deployment to Slack uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 diff --git a/book/src/02_development/11_ci_docker_mirror.md b/book/src/02_development/11_ci_docker_mirror.md index 98d5e1a4ef..41da292a72 100644 --- a/book/src/02_development/11_ci_docker_mirror.md +++ b/book/src/02_development/11_ci_docker_mirror.md @@ -2,7 +2,7 @@ CI pulls its Docker images from a GHCR mirror (`ghcr.io/nomicfoundation/edr/mirror/*`) instead of Docker Hub: Docker Hub rate-limits pulls, which fail intermittently on GitHub-hosted runners as a result. The mirror is maintained by `.github/workflows/mirror-docker-images.yml`, which re-copies the images weekly, on pushes to `main` and same-repo PRs that change the workflow itself, and on demand via `workflow_dispatch`. -Release runs are the exception: when `check_commit` resolves a release tag, the docker jobs in `edr-npm-release.yml` pull the official image straight from Docker Hub (the "Select image source" steps), so the mirror is never in the supply chain of published binaries — a tampered mirror tag can at most affect PR/branch CI, which publishes nothing. At one or two releases a week, Docker Hub's rate limits are not a concern for those runs. +Release runs are the exception: when `check_commit` sets its `is_release` output — the single authority for all release/non-release gating in `edr-npm-release.yml` — the docker jobs pull the official image straight from Docker Hub (the "Select image source" steps), so the mirror is never in the supply chain of published binaries — a tampered mirror tag can at most affect PR/branch CI, which publishes nothing. At one or two releases a week, Docker Hub's rate limits are not a concern for those runs. ## Adding a Node.js version (or any new tag) From 1e8df4bc4f4bcb3cf13d9e751c0446809f9fed86 Mon Sep 17 00:00:00 2001 From: Bas van Gijzel Date: Wed, 5 Aug 2026 12:29:03 +0000 Subject: [PATCH 3/4] ci: make the mirror wait best-effort; review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The wait step could hard-fail all five docker job families on any non-success mirror conclusion — including fork PRs, whose mirror run skips itself by design and concludes 'skipped', and a failed weekly re-sync sharing main's head SHA. The pull itself is the authoritative loud failure for a genuinely missing tag ('manifest unknown'), so warn and proceed on non-success or timeout instead of failing. Also from review: reword the cold-build assert comment (it's a tripwire against future edits to the cache gate, not a proof of the is_release derivation), log the non-release decision in check_commit instead of falling through silently, mention the wait in the action description, and trim reviewer-facing comments. --- .github/actions/select-node-image/action.yml | 30 +++++++++---------- .github/workflows/edr-npm-release.yml | 17 ++++++----- .github/workflows/mirror-docker-images.yml | 3 +- .../src/02_development/11_ci_docker_mirror.md | 2 +- 4 files changed, 27 insertions(+), 25 deletions(-) diff --git a/.github/actions/select-node-image/action.yml b/.github/actions/select-node-image/action.yml index 2e75b0d18c..1f48296513 100644 --- a/.github/actions/select-node-image/action.yml +++ b/.github/actions/select-node-image/action.yml @@ -1,5 +1,5 @@ name: Select node image -description: Resolves the node Docker image for a job. Normal runs use the GHCR mirror (see mirror-docker-images.yml); release runs pull the official Docker Hub image directly, keeping the mirror out of the supply chain of published binaries. +description: Resolves the node Docker image for a job, waiting (up to 15 minutes) for any in-flight mirror run first. Normal runs use the GHCR mirror (see mirror-docker-images.yml); release runs pull the official Docker Hub image directly, keeping the mirror out of the supply chain of published binaries. inputs: tag: @@ -17,12 +17,10 @@ outputs: runs: using: composite steps: - # On PRs/pushes that touch mirror-docker-images.yml, the mirror run and - # this workflow's docker jobs start in parallel, so a pull can race ahead - # of the mirror and fail with "manifest unknown". Same technique as - # waitForEdrCi in scripts/resolve-regression-trigger.cjs, except "no run - # for this SHA" means nothing to wait for (the common case) rather than - # "not started yet". Callers need `actions: read` to list workflow runs. + # On PRs/pushes that touch mirror-docker-images.yml, pulls can race ahead + # of the mirror run for the same SHA and fail with "manifest unknown"; no + # run means nothing to wait for. Best-effort: never fails the job — the + # pull itself is the authoritative failure for a genuinely missing tag. - name: Wait for in-flight mirror run if: inputs.is-release != 'true' shell: bash @@ -46,16 +44,18 @@ runs: if [ "$STATUS" = "completed" ]; then CONCLUSION=$(jq -r .conclusion <<<"$RUN") URL=$(jq -r .html_url <<<"$RUN") - if [ "$CONCLUSION" = "success" ]; then - echo "Mirror run succeeded: $URL" - exit 0 - fi - echo "Mirror run for ${HEAD_SHA} concluded '$CONCLUSION': $URL" >&2 - exit 1 + case "$CONCLUSION" in + success) echo "Mirror run succeeded: $URL" ;; + # The mirror job skips itself on fork PRs; the tags it would + # have re-copied already exist. + skipped) echo "Mirror run was skipped: $URL" ;; + *) echo "::warning::Mirror run for ${HEAD_SHA} concluded '$CONCLUSION' ($URL); proceeding anyway" ;; + esac + exit 0 fi if [ "$(date +%s)" -ge "$DEADLINE" ]; then - echo "Timed out waiting for the mirror run for ${HEAD_SHA}" >&2 - exit 1 + echo "::warning::Timed out waiting for the mirror run for ${HEAD_SHA}; proceeding anyway" + exit 0 fi echo "Mirror run in progress (status: $STATUS); retrying in 30s..." sleep 30 diff --git a/.github/workflows/edr-npm-release.yml b/.github/workflows/edr-npm-release.yml index 48b8ee4293..4e356c0bd9 100644 --- a/.github/workflows/edr-npm-release.yml +++ b/.github/workflows/edr-npm-release.yml @@ -138,9 +138,8 @@ jobs: restore-keys: | ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}- - # Proves the cold-build invariant at runtime instead of trusting the - # skipped cache step: a broken gate fails the release loudly rather - # than shipping binaries built from a warm, poisonable cache. + # Tripwire for future edits to the cache gate above: if it stops + # skipping release runs, fail before building from a warm cache. - name: Assert release builds run cold if: needs.check_commit.outputs.is_release == 'true' shell: bash @@ -520,6 +519,8 @@ jobs: then echo "pre-release commit: tag=next" echo "tag=next" >> "$GITHUB_OUTPUT" + else + echo "pre-release commit, but event is ${{ github.event_name }}, not push: not a release run" fi # matches with release commit elif git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+\s*"; @@ -528,13 +529,15 @@ jobs: then echo "release commit: tag=latest" echo "tag=latest" >> "$GITHUB_OUTPUT" + else + echo "release commit, but $GITHUB_REF is not main or hh2: not a release run" fi + else + echo "not a release commit: not a release run" fi - # Single authority for release/non-release gating: every release-only or - # non-release-only step and job keys off `is_release`, derived here from - # `tag` so the two can't diverge. `tag` itself is only for the npm - # dist-tag (next/latest) in `publish`. + # `is_release` is the single authority for release/non-release gating; + # `tag` is only the npm dist-tag (next/latest) for `publish`. outputs: tag: ${{ steps.check_commit.outputs.tag }} is_release: ${{ steps.check_commit.outputs.tag != '' }} diff --git a/.github/workflows/mirror-docker-images.yml b/.github/workflows/mirror-docker-images.yml index 9b79416401..ac8f18be52 100644 --- a/.github/workflows/mirror-docker-images.yml +++ b/.github/workflows/mirror-docker-images.yml @@ -24,8 +24,7 @@ on: - .github/workflows/mirror-docker-images.yml # Same-repo PRs touching this file mirror immediately, so a PR that adds a # tag below (e.g. a new node version) can be tested before merge. The - # edr-npm-release.yml docker jobs wait for this run before pulling (see the - # select-node-image action). + # edr-npm-release.yml docker jobs wait for this run before pulling. pull_request: paths: - .github/workflows/mirror-docker-images.yml diff --git a/book/src/02_development/11_ci_docker_mirror.md b/book/src/02_development/11_ci_docker_mirror.md index 41da292a72..6fecd7038b 100644 --- a/book/src/02_development/11_ci_docker_mirror.md +++ b/book/src/02_development/11_ci_docker_mirror.md @@ -8,7 +8,7 @@ Release runs are the exception: when `check_commit` sets its `is_release` output Add the tag to the `TAGS` list in `mirror-docker-images.yml` in the same PR that changes the matrix in `edr-npm-release.yml`. The mirror workflow runs on same-repo PRs that touch it, so the new tag is mirrored — and the release matrix testable against it — before merge. A tag referenced in CI but missing from the mirror fails loudly with `manifest unknown`. -On such a PR the mirror job and the release-workflow docker jobs start in parallel. The docker jobs' "Select image source" step waits (up to 15 minutes) for the mirror run on the same commit to finish before pulling, so the new tag is in place by the time it's needed. If the mirror run fails, the docker jobs fail with a link to it instead of a `manifest unknown` pull error. On PRs that don't touch the mirror workflow no mirror run exists and the step doesn't wait. +On such a PR the mirror job and the release-workflow docker jobs start in parallel. The docker jobs' "Select image source" step waits (up to 15 minutes) for the mirror run on the same commit to finish before pulling, so the new tag is in place by the time it's needed. The wait is best-effort and never fails the job — if the mirror run failed or was skipped, the jobs proceed with a warning, and a genuinely missing tag still fails the pull itself with `manifest unknown`. On PRs that don't touch the mirror workflow no mirror run exists and the step doesn't wait. ## Access From 47338ac4dff7a395299319db57a4cc34efa44b17 Mon Sep 17 00:00:00 2001 From: Bas van Gijzel Date: Mon, 3 Aug 2026 19:40:07 +0000 Subject: [PATCH 4/4] ci: split the npm pipeline into reusable build + thin CI/release callers edr-npm-release.yml did double duty as PR validation and release pipeline, with the mode threaded through job- and step-level is_release conditions that were easy to miss or drop. Move build/test/prepare/ review into edr-npm-build.yml (workflow_call, one typed 'release' input), called by a thin edr-npm-ci.yml with release: false and a thin edr-npm-release.yml with release: true. Each caller has exactly one condition, at its entry: CI skips release pushes, and the release workflow's jobs all hang off a single release_gate job, so release-only jobs no longer carry guards at all. The commit-message detection moves into a check-release-commit composite shared by both entries. workflow_dispatch on the release workflow is now a rehearsal: it forces the full release path (cold build, Docker Hub images, cooldown, environment approval) but publishes with --dry-run; real publishes happen only on push events. Raised by @anaPerezGhiglia in the #1587 review (#1593). --- .../actions/check-release-commit/action.yml | 44 ++ .github/actions/select-node-image/action.yml | 2 +- .github/workflows/edr-npm-build.yml | 637 ++++++++++++++++ .github/workflows/edr-npm-ci.yml | 55 ++ .github/workflows/edr-npm-release.yml | 714 ++---------------- .github/workflows/mirror-docker-images.yml | 4 +- .../src/02_development/11_ci_docker_mirror.md | 6 +- book/src/03_release.md | 12 + 8 files changed, 809 insertions(+), 665 deletions(-) create mode 100644 .github/actions/check-release-commit/action.yml create mode 100644 .github/workflows/edr-npm-build.yml create mode 100644 .github/workflows/edr-npm-ci.yml diff --git a/.github/actions/check-release-commit/action.yml b/.github/actions/check-release-commit/action.yml new file mode 100644 index 0000000000..f6dfad531f --- /dev/null +++ b/.github/actions/check-release-commit/action.yml @@ -0,0 +1,44 @@ +name: Check release commit +description: > + Detects (pre-)release commits from the HEAD commit message. Single authority + for the release/non-release decision, shared by edr-npm-ci.yml (which skips + itself on release pushes) and edr-npm-release.yml (which only proceeds on + them). Requires the repository to be checked out. + +outputs: + tag: + description: npm dist-tag ('next' for pre-releases, 'latest' for releases, empty otherwise) + value: ${{ steps.check.outputs.tag }} + is-release: + description: "'true' when the HEAD commit triggers a (pre-)release" + value: ${{ steps.check.outputs.tag != '' }} + +runs: + using: composite + steps: + - id: check + shell: bash + run: | + # matches with pre-release commits: edr release name but ends with - and some postfix like beta + if git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+-"; + then + if [ "$GITHUB_EVENT_NAME" = "push" ] + then + echo "pre-release commit: tag=next" + echo "tag=next" >> "$GITHUB_OUTPUT" + else + echo "pre-release commit, but event is $GITHUB_EVENT_NAME, not push: not a release run" + fi + # matches with release commit + elif git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+\s*"; + then + if [ "$GITHUB_REF" = "refs/heads/main" ] || [ "$GITHUB_REF" = "refs/heads/hh2" ] + then + echo "release commit: tag=latest" + echo "tag=latest" >> "$GITHUB_OUTPUT" + else + echo "release commit, but $GITHUB_REF is not main or hh2: not a release run" + fi + else + echo "not a release commit: not a release run" + fi diff --git a/.github/actions/select-node-image/action.yml b/.github/actions/select-node-image/action.yml index 1f48296513..2e7abd0d9d 100644 --- a/.github/actions/select-node-image/action.yml +++ b/.github/actions/select-node-image/action.yml @@ -6,7 +6,7 @@ inputs: description: node image tag, e.g. 24-bullseye-slim or 22-alpine required: true is-release: - description: check_commit's is_release output; 'true' marks a release run + description: edr-npm-build.yml's release input; 'true' marks a release run required: true outputs: diff --git a/.github/workflows/edr-npm-build.yml b/.github/workflows/edr-npm-build.yml new file mode 100644 index 0000000000..f1ed63988d --- /dev/null +++ b/.github/workflows/edr-npm-build.yml @@ -0,0 +1,637 @@ +# Reusable build+test+bundle pipeline for the EDR npm package, called by +# edr-npm-ci.yml (release: false) and edr-npm-release.yml (release: true). +# The `release` input is the only mode switch: release runs build cold and +# pull official Docker Hub images; non-release runs use the cargo cache and +# the GHCR mirror. Publishing lives in edr-npm-release.yml, never here. +name: EDR NPM build and test +env: + DEBUG: napi:* + APP_NAME: edr + NUMBER_OF_TARGETS: 7 +permissions: {} +on: + workflow_call: + inputs: + release: + description: release mode — cold builds, official Docker Hub images + type: boolean + required: true + outputs: + bundle-filename: + description: filename of the packed bundle artifact + value: ${{ jobs.prepare.outputs.filename }} + commit-sha: + description: HEAD commit the bindings were built from + value: ${{ jobs.build.outputs.commit_sha }} + +jobs: + build: + name: stable - ${{ matrix.settings.target }} - node@24 + runs-on: ${{ matrix.settings.host }} + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + settings: + - host: macos-15-intel + target: x86_64-apple-darwin + build: pnpm run build --strip + + - host: macos-14 + target: aarch64-apple-darwin + build: pnpm run build --strip + + - host: windows-2025 + target: x86_64-pc-windows-msvc + build: pnpm run build --strip + # NTFS small-file creates are ~5-10x slower than ext4/APFS, + # so napi-rs's pre-cargo `cargo metadata` step burns ~50s + # extracting ~500 .crate tarballs into registry/src/ on warm + # runs without this. Other platforms re-extract fast enough + # that caching it isn't worth the size hit. + cache_extra_paths: ~/.cargo/registry/src/ + + # docker_tag legs build in a node container resolved by the + # select-node-image action (GHCR mirror normally, Docker Hub on + # releases). New or changed tags must be added to the mirror + # workflow's tag list in the same PR. + - host: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + docker_tag: 24-bullseye-slim + flavor: gnu + + - host: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + docker_tag: 24-bullseye-slim + flavor: gnu + + - host: ubuntu-24.04 + target: x86_64-unknown-linux-musl + docker_tag: 24-alpine + flavor: musl + + - host: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + docker_tag: 24-alpine + flavor: musl + outputs: + commit_sha: ${{ steps.save-commit.outputs.commit_sha}} + defaults: + run: + working-directory: ./crates/edr_napi + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: socketdev/action@ba6de6cc0565af1f42295590380973573297e31f # v1.3.2 + with: + mode: firewall + + - name: Save HEAD commit sha + id: save-commit + shell: bash + run: | + COMMIT_SHA=$(git rev-parse HEAD) + echo "Current HEAD: $COMMIT_SHA" + echo "commit_sha=$COMMIT_SHA" >> $GITHUB_OUTPUT + + - name: Check number of targets + shell: bash + run: + | # zizmor: ignore[template-injection] strategy.job-total is a controlled integer, not external input + echo "Number of build jobs: ${{ strategy.job-total }}" + echo "Expected number of build jobs: $NUMBER_OF_TARGETS" + test ${{ strategy.job-total }} -eq "$NUMBER_OF_TARGETS" + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # latest main commit (2025-11-17) + if: ${{ !matrix.settings.docker_tag }} + with: + toolchain: stable + targets: ${{ matrix.settings.target }} + components: rust-src + + - name: Setup node and pnpm + if: ${{ !matrix.settings.docker_tag }} + uses: ./.github/actions/setup-node + with: + node-version: 24 + + # Release commits build cold: napi codegen relies on proc-macro + # expansion (napi-rs#1297), so warm-cache builds risk shipping + # stale or empty index.js / index.d.ts. Don't enable for releases. + - name: Cache cargo # zizmor: ignore[cache-poisoning] release builds run cold (asserted below); PR/main caches never feed a publish + if: ${{ !inputs.release }} + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + ${{ matrix.settings.cache_extra_paths }} + key: ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}- + + # Tripwire for future edits to the cache gate above: if it stops + # skipping release runs, fail before building from a warm cache. + - name: Assert release builds run cold + if: ${{ inputs.release }} + shell: bash + run: | + if [ -e "${GITHUB_WORKSPACE}/target" ]; then + echo "target/ exists on a release build: the cargo cache gate is broken" >&2 + exit 1 + fi + + - name: Install dependencies (non-Windows) + if: ${{ !matrix.settings.docker_tag && runner.os != 'Windows' }} + run: sfw pnpm install --frozen-lockfile --prefer-offline + + # SFW doesn't install dependencies correctly on Windows + # https://github.com/NomicFoundation/edr/issues/1199 + - name: Install dependencies (Windows) + if: ${{ !matrix.settings.docker_tag && runner.os == 'Windows' }} + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Build (non-docker) + if: ${{ !matrix.settings.docker_tag }} + run: ${{ matrix.settings.build }} + shell: bash + + - name: Select image source + id: select-image + if: ${{ matrix.settings.docker_tag }} + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.settings.docker_tag }} + is-release: ${{ inputs.release }} + + - name: Build in docker (gnu) + uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) + if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'gnu'}} + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # /home/runner/.cargo matches where the host's `Cache cargo` step writes + # (`~/.cargo` on GH-hosted Linux runners); docker -v doesn't expand `~` and + # GHA has no `runner.home` context, so the path is hardcoded. + options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" + run: | + set -e + + # Install Rust toolchain + apt-get update + apt-get install -y curl ca-certificates build-essential pkg-config + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + export PATH="/root/.cargo/bin:$PATH" + + npm i -g pnpm@11.17.0 --ignore-scripts + pnpm -v + npm i -g sfw + sfw pnpm install --frozen-lockfile --prefer-offline + pnpm run build --strip + + # SFW is not provided on musl versions of Linux + # https://github.com/NomicFoundation/edr/issues/1198 + - name: Build in docker (musl) + uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) + if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'musl' }} + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # See gnu step above for why /home/runner/.cargo is hardcoded. + options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" + run: | + set -e + + # Install Rust toolchain + apk add --no-cache build-base python3 make git pkgconfig openssl openssl-dev ca-certificates curl bash perl cmake + # Conditional symlink only for ARM builds + if [ "${{ matrix.settings.target }}" = "aarch64-unknown-linux-musl" ]; then + ln -sf /usr/bin/gcc /usr/bin/aarch64-linux-musl-gcc + fi + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + export PATH="/root/.cargo/bin:$PATH" + + npm i -g pnpm@11.17.0 --ignore-scripts + pnpm -v + pnpm install --frozen-lockfile --prefer-offline + pnpm run build --strip + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bindings-${{ matrix.settings.target }} + # The upload artifact action doesn't respect the working directory setting. Unclear if this is a bug or not + # https://github.com/actions/upload-artifact/issues/294 + path: ./crates/edr_napi/${{ env.APP_NAME }}.*.node + if-no-files-found: error + + test-macOS-windows-binding: + name: Test bindings on ${{ matrix.settings.target }} - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + strategy: + fail-fast: false + matrix: + settings: + - host: macos-15-intel + target: x86_64-apple-darwin + - host: macos-14 + target: aarch64-apple-darwin + - host: windows-2022 + target: x86_64-pc-windows-msvc + node: + - "22" + - "24" + - "26" + runs-on: ${{ matrix.settings.host }} + defaults: + run: + working-directory: ./crates/edr_napi + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-${{ matrix.settings.target }} + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Test bindings + run: pnpm testNoBuild + test-linux-x64-gnu-binding: + name: Test bindings on Linux-x64-gnu - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-x86_64-unknown-linux-gnu + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }} + is-release: ${{ inputs.release }} + - name: Test bindings + # Setting CI=1 is important to make PNPM install non-interactive + # https://github.com/pnpm/pnpm/issues/6615#issuecomment-1656945689 + env: + IMAGE: ${{ steps.select-image.outputs.ref }} + run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" bash -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" + test-linux-x64-musl-binding: + name: Test bindings on x86_64-unknown-linux-musl - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline --libc=musl + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-x86_64-unknown-linux-musl + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }}-alpine + is-release: ${{ inputs.release }} + - name: Test bindings + env: + IMAGE: ${{ steps.select-image.outputs.ref }} + run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" sh -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" + test-linux-aarch64-gnu-binding: + name: Test bindings on aarch64-unknown-linux-gnu - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-aarch64-unknown-linux-gnu + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Install dependencies + run: | + pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=glibc + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }} + is-release: ${{ inputs.release }} + - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64) test runner + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" + run: | + npm install -g pnpm@11.17.0 + set -e + pnpm testNoBuild + ls -la + test-linux-aarch64-musl-binding: + name: Test bindings on aarch64-unknown-linux-musl - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-aarch64-unknown-linux-musl + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=musl + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }}-alpine + is-release: ${{ inputs.release }} + - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64/musl) test runner + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" + run: | + npm install -g pnpm@11.17.0 + set -e + pnpm testNoBuild + + prepare: + name: Prepare release bundle + runs-on: ubuntu-24.04 + permissions: + contents: read + needs: + - build + - test-macOS-windows-binding + - test-linux-x64-gnu-binding + - test-linux-x64-musl-binding + - test-linux-aarch64-gnu-binding + - test-linux-aarch64-musl-binding + defaults: + run: + working-directory: ./crates/edr_napi + outputs: + filename: ${{ steps.pack.outputs.filename }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node + uses: ./.github/actions/setup-node + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + - name: Install sponge # needed for prepublish script + run: | + sudo apt-get update + sudo apt-get install -y moreutils + - name: Download all artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: ./crates/edr_napi/artifacts + - name: Run prepublish to update all `package.json` files + # we NEED to to this before pnpm artifacts, if not prepublish will try to publish + run: ../../scripts/prepublish.sh + - name: Move artifacts + # prepublish.sh (above) adds the platform packages to edr_napi's + # package.json but not the lockfile; skip pnpm 11's pre-run frozen + # deps check, which would otherwise fail on that intentional desync. + run: pnpm --config.verify-deps-before-run=false artifacts + - name: Copy coverage library into edr_napi crate + run: cp ../../data/contracts/coverage.sol ./coverage.sol + - name: Compile TypeScript helpers + # See "Move artifacts": package.json is intentionally out of sync with + # the lockfile here, so skip pnpm 11's pre-run frozen deps check. + run: pnpm --config.verify-deps-before-run=false exec tsc + - name: Create bundle with pnpm root & napi files + # create custom tar.gz because pnpm pack only packs root package and we need the platform-specific ones as well + id: pack + run: | + FILENAME="${{ env.APP_NAME }}_bundle.tar.gz" + NAPI_EXTRA_FILES=("npm/" "LICENSE" "README.md" "package.json" "Cargo.toml" "coverage.sol") + readarray -t FILES < <(cat package.json | jq -r .files[] ) + ALL_FILES=("${FILES[@]}" "${NAPI_EXTRA_FILES[@]}") + NAPI_FILES=() + for file in "${ALL_FILES[@]}"; do + NAPI_FILES+=("crates/edr_napi/${file}") + done + cd ../.. # cd to root dir + echo "Creating bundle with necessary files for publishing" + ROOT_FILES=("package.json" "Cargo.lock" "Cargo.toml" "pnpm-lock.yaml" "pnpm-workspace.yaml") + ALL_FILES=("${ROOT_FILES[@]}" "${NAPI_FILES[@]}") + echo "files to include in tar: ${ALL_FILES[*]}" + # Using --dereference since LICENSE file are symlinks + tar --dereference -czvf $FILENAME "${ALL_FILES[@]}" + echo "FILENAME=$FILENAME" + echo "filename=$FILENAME" >> $GITHUB_OUTPUT + - name: Upload packed tarball + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.pack.outputs.filename }} + path: ./${{ steps.pack.outputs.filename }} + retention-days: 2 + overwrite: true + if-no-files-found: error + + review: + name: Review release + runs-on: ubuntu-24.04 + permissions: + contents: read + needs: + - build + - prepare + steps: + - name: Download packed tarball + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.prepare.outputs.filename }} + path: . + - name: Decompress tarball + env: + FILENAME: ${{ needs.prepare.outputs.filename }} + run: tar -xvzf "$FILENAME" + - name: Inspect contents + run: tree . + - name: Check number of artifacts + shell: bash + run: | + # get number of artifacts with unique names + # shellcheck disable=SC2011 # .node names are fixed target triples, no special chars + NUMBER_OF_ARTIFACTS=$(ls -1q crates/edr_napi/npm/*/*.node | xargs -n 1 basename | sort | uniq | wc -l) + echo "Number of unique artifacts: $NUMBER_OF_ARTIFACTS" + echo "Expected number of unique artifacts: $NUMBER_OF_TARGETS" + test "$NUMBER_OF_ARTIFACTS" -eq "$NUMBER_OF_TARGETS" + - name: Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + - name: Setup node + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + id: setup-node + with: + node-version: 24 + # A publish workflow must not restore a poisonable cache. + package-manager-cache: false + - name: Validate that packages have everything for publishing + run: | + cd crates/edr_napi + # Check in every cross-platform package + for platform in ./npm/* ; do + cd $platform + pnpm pack --pack-destination "$RUNNER_TEMP" + cd - + done + # And in root edr package + pnpm pack --pack-destination "$RUNNER_TEMP" + - name: Checkout repository for comparison + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + path: remote-repository + persist-credentials: false + ref: ${{ needs.build.outputs.commit_sha }} + - name: Validate there were no modifications to `package.json` when building + run: | + git diff --color=always --exit-code --no-index remote-repository/package.json ./package.json + - name: Validate there were no modifications to `Cargo.toml` when building + run: | + git diff --color=always --exit-code --no-index remote-repository/Cargo.toml ./Cargo.toml + - name: Validate there were no modifications to `Cargo.lock` when building + run: | + git diff --color=always --exit-code --no-index remote-repository/Cargo.lock ./Cargo.lock + diff --git a/.github/workflows/edr-npm-ci.yml b/.github/workflows/edr-npm-ci.yml new file mode 100644 index 0000000000..4e361d008a --- /dev/null +++ b/.github/workflows/edr-npm-ci.yml @@ -0,0 +1,55 @@ +# Thin PR/branch validation entry: runs the reusable build workflow +# (edr-npm-build.yml) in non-release mode — warm cargo caches, GHCR-mirrored +# images. Release pushes are skipped here; edr-npm-release.yml owns those. +name: EDR NPM CI +permissions: {} +on: + push: + # keep branches and paths in sync with edr-npm-release.yml + branches: + - main + - prerelease + - hh2 + - feat/solidity-tests + tags-ignore: + - "**" + paths-ignore: + - "**/*.md" + - LICENSE + - "**/*.gitignore" + - .editorconfig + - docs/** + pull_request: null + workflow_dispatch: + +jobs: + check_commit: + name: Check commit + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.ref }} + + - id: check + uses: ./.github/actions/check-release-commit + + outputs: + is_release: ${{ steps.check.outputs.is-release }} + + build: + name: Build and test + needs: check_commit + # The single condition in this workflow: release pushes are handled by + # edr-npm-release.yml with a cold release-mode build instead. + if: needs.check_commit.outputs.is_release != 'true' + uses: ./.github/workflows/edr-npm-build.yml + with: + release: false + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs diff --git a/.github/workflows/edr-npm-release.yml b/.github/workflows/edr-npm-release.yml index 4e356c0bd9..258bb3bc75 100644 --- a/.github/workflows/edr-npm-release.yml +++ b/.github/workflows/edr-npm-release.yml @@ -1,13 +1,15 @@ +# Thin release pipeline: detects (pre-)release commits, runs the reusable +# build workflow (edr-npm-build.yml) in release mode, and publishes to npm. +# Non-release pushes skip everything after check_commit; PR/branch validation +# lives in edr-npm-ci.yml. name: EDR NPM release -env: - DEBUG: napi:* - APP_NAME: edr - NUMBER_OF_TARGETS: 7 permissions: {} on: push: # pre-releases will be triggered in any of these branches # production releases will only be triggered for `main` + # (keep branches and paths in sync with edr-npm-ci.yml, which skips + # release pushes so each push runs exactly one of the two workflows) branches: - main - prerelease @@ -21,483 +23,12 @@ on: - "**/*.gitignore" - .editorconfig - docs/** - pull_request: null + # Release rehearsal: forces the release path on any commit — cold build, + # official Docker Hub images, cooldown check, environment approval — but + # publishes with --dry-run. Real publishes happen only on push events. workflow_dispatch: jobs: - build: - name: stable - ${{ matrix.settings.target }} - node@24 - runs-on: ${{ matrix.settings.host }} - needs: check_commit - permissions: - contents: read - packages: read - actions: read # select-node-image lists mirror workflow runs - strategy: - fail-fast: false - matrix: - settings: - - host: macos-15-intel - target: x86_64-apple-darwin - build: pnpm run build --strip - - - host: macos-14 - target: aarch64-apple-darwin - build: pnpm run build --strip - - - host: windows-2025 - target: x86_64-pc-windows-msvc - build: pnpm run build --strip - # NTFS small-file creates are ~5-10x slower than ext4/APFS, - # so napi-rs's pre-cargo `cargo metadata` step burns ~50s - # extracting ~500 .crate tarballs into registry/src/ on warm - # runs without this. Other platforms re-extract fast enough - # that caching it isn't worth the size hit. - cache_extra_paths: ~/.cargo/registry/src/ - - # docker_tag legs build in a node container resolved by the - # select-node-image action (GHCR mirror normally, Docker Hub on - # releases). New or changed tags must be added to the mirror - # workflow's tag list in the same PR. - - host: ubuntu-24.04 - target: x86_64-unknown-linux-gnu - docker_tag: 24-bullseye-slim - flavor: gnu - - - host: ubuntu-24.04-arm - target: aarch64-unknown-linux-gnu - docker_tag: 24-bullseye-slim - flavor: gnu - - - host: ubuntu-24.04 - target: x86_64-unknown-linux-musl - docker_tag: 24-alpine - flavor: musl - - - host: ubuntu-24.04-arm - target: aarch64-unknown-linux-musl - docker_tag: 24-alpine - flavor: musl - outputs: - commit_sha: ${{ steps.save-commit.outputs.commit_sha}} - defaults: - run: - working-directory: ./crates/edr_napi - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - uses: socketdev/action@ba6de6cc0565af1f42295590380973573297e31f # v1.3.2 - with: - mode: firewall - - - name: Save HEAD commit sha - id: save-commit - shell: bash - run: | - COMMIT_SHA=$(git rev-parse HEAD) - echo "Current HEAD: $COMMIT_SHA" - echo "commit_sha=$COMMIT_SHA" >> $GITHUB_OUTPUT - - - name: Check number of targets - shell: bash - run: - | # zizmor: ignore[template-injection] strategy.job-total is a controlled integer, not external input - echo "Number of build jobs: ${{ strategy.job-total }}" - echo "Expected number of build jobs: $NUMBER_OF_TARGETS" - test ${{ strategy.job-total }} -eq "$NUMBER_OF_TARGETS" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # latest main commit (2025-11-17) - if: ${{ !matrix.settings.docker_tag }} - with: - toolchain: stable - targets: ${{ matrix.settings.target }} - components: rust-src - - - name: Setup node and pnpm - if: ${{ !matrix.settings.docker_tag }} - uses: ./.github/actions/setup-node - with: - node-version: 24 - - # Release commits build cold: napi codegen relies on proc-macro - # expansion (napi-rs#1297), so warm-cache builds risk shipping - # stale or empty index.js / index.d.ts. Don't enable for releases. - - name: Cache cargo # zizmor: ignore[cache-poisoning] release builds run cold (asserted below); PR/main caches never feed a publish - if: needs.check_commit.outputs.is_release != 'true' - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - ${{ matrix.settings.cache_extra_paths }} - key: ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}- - - # Tripwire for future edits to the cache gate above: if it stops - # skipping release runs, fail before building from a warm cache. - - name: Assert release builds run cold - if: needs.check_commit.outputs.is_release == 'true' - shell: bash - run: | - if [ -e "${GITHUB_WORKSPACE}/target" ]; then - echo "target/ exists on a release build: the cargo cache gate is broken" >&2 - exit 1 - fi - - - name: Install dependencies (non-Windows) - if: ${{ !matrix.settings.docker_tag && runner.os != 'Windows' }} - run: sfw pnpm install --frozen-lockfile --prefer-offline - - # SFW doesn't install dependencies correctly on Windows - # https://github.com/NomicFoundation/edr/issues/1199 - - name: Install dependencies (Windows) - if: ${{ !matrix.settings.docker_tag && runner.os == 'Windows' }} - run: pnpm install --frozen-lockfile --prefer-offline - - - name: Build (non-docker) - if: ${{ !matrix.settings.docker_tag }} - run: ${{ matrix.settings.build }} - shell: bash - - - name: Select image source - id: select-image - if: ${{ matrix.settings.docker_tag }} - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.settings.docker_tag }} - is-release: ${{ needs.check_commit.outputs.is_release }} - - - name: Build in docker (gnu) - uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) - if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'gnu'}} - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # /home/runner/.cargo matches where the host's `Cache cargo` step writes - # (`~/.cargo` on GH-hosted Linux runners); docker -v doesn't expand `~` and - # GHA has no `runner.home` context, so the path is hardcoded. - options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" - run: | - set -e - - # Install Rust toolchain - apt-get update - apt-get install -y curl ca-certificates build-essential pkg-config - curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable - export PATH="/root/.cargo/bin:$PATH" - - npm i -g pnpm@11.17.0 --ignore-scripts - pnpm -v - npm i -g sfw - sfw pnpm install --frozen-lockfile --prefer-offline - pnpm run build --strip - - # SFW is not provided on musl versions of Linux - # https://github.com/NomicFoundation/edr/issues/1198 - - name: Build in docker (musl) - uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) - if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'musl' }} - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # See gnu step above for why /home/runner/.cargo is hardcoded. - options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" - run: | - set -e - - # Install Rust toolchain - apk add --no-cache build-base python3 make git pkgconfig openssl openssl-dev ca-certificates curl bash perl cmake - # Conditional symlink only for ARM builds - if [ "${{ matrix.settings.target }}" = "aarch64-unknown-linux-musl" ]; then - ln -sf /usr/bin/gcc /usr/bin/aarch64-linux-musl-gcc - fi - curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable - export PATH="/root/.cargo/bin:$PATH" - - npm i -g pnpm@11.17.0 --ignore-scripts - pnpm -v - pnpm install --frozen-lockfile --prefer-offline - pnpm run build --strip - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bindings-${{ matrix.settings.target }} - # The upload artifact action doesn't respect the working directory setting. Unclear if this is a bug or not - # https://github.com/actions/upload-artifact/issues/294 - path: ./crates/edr_napi/${{ env.APP_NAME }}.*.node - if-no-files-found: error - - test-macOS-windows-binding: - name: Test bindings on ${{ matrix.settings.target }} - node@${{ matrix.node }} - needs: - - build - permissions: - contents: read - strategy: - fail-fast: false - matrix: - settings: - - host: macos-15-intel - target: x86_64-apple-darwin - - host: macos-14 - target: aarch64-apple-darwin - - host: windows-2022 - target: x86_64-pc-windows-msvc - node: - - "22" - - "24" - - "26" - runs-on: ${{ matrix.settings.host }} - defaults: - run: - working-directory: ./crates/edr_napi - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-${{ matrix.settings.target }} - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Test bindings - run: pnpm testNoBuild - test-linux-x64-gnu-binding: - name: Test bindings on Linux-x64-gnu - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - actions: read # select-node-image lists mirror workflow runs - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-x86_64-unknown-linux-gnu - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }} - is-release: ${{ needs.check_commit.outputs.is_release }} - - name: Test bindings - # Setting CI=1 is important to make PNPM install non-interactive - # https://github.com/pnpm/pnpm/issues/6615#issuecomment-1656945689 - env: - IMAGE: ${{ steps.select-image.outputs.ref }} - run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" bash -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" - test-linux-x64-musl-binding: - name: Test bindings on x86_64-unknown-linux-musl - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - actions: read # select-node-image lists mirror workflow runs - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline --libc=musl - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-x86_64-unknown-linux-musl - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }}-alpine - is-release: ${{ needs.check_commit.outputs.is_release }} - - name: Test bindings - env: - IMAGE: ${{ steps.select-image.outputs.ref }} - run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" sh -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" - test-linux-aarch64-gnu-binding: - name: Test bindings on aarch64-unknown-linux-gnu - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - actions: read # select-node-image lists mirror workflow runs - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-aarch64-unknown-linux-gnu - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Install dependencies - run: | - pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=glibc - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }} - is-release: ${{ needs.check_commit.outputs.is_release }} - - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64) test runner - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" - run: | - npm install -g pnpm@11.17.0 - set -e - pnpm testNoBuild - ls -la - test-linux-aarch64-musl-binding: - name: Test bindings on aarch64-unknown-linux-musl - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - actions: read # select-node-image lists mirror workflow runs - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-aarch64-unknown-linux-musl - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=musl - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }}-alpine - is-release: ${{ needs.check_commit.outputs.is_release }} - - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64/musl) test runner - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" - run: | - npm install -g pnpm@11.17.0 - set -e - pnpm testNoBuild - check_commit: name: Check commit runs-on: ubuntu-24.04 @@ -507,190 +38,50 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - ref: ${{ github.event.pull_request.head.ref }} - - name: Check if commit message is a release commit - id: check_commit - run: | - # matches with pre-release commits: edr release name but ends with - and some postfix like beta - if git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+-"; - then - if [ "${{ github.event_name }}" = "push" ] - then - echo "pre-release commit: tag=next" - echo "tag=next" >> "$GITHUB_OUTPUT" - else - echo "pre-release commit, but event is ${{ github.event_name }}, not push: not a release run" - fi - # matches with release commit - elif git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+\s*"; - then - if [ "$GITHUB_REF" == "refs/heads/main" ] || [ "$GITHUB_REF" == "refs/heads/hh2" ] - then - echo "release commit: tag=latest" - echo "tag=latest" >> "$GITHUB_OUTPUT" - else - echo "release commit, but $GITHUB_REF is not main or hh2: not a release run" - fi - else - echo "not a release commit: not a release run" - fi + - id: check + uses: ./.github/actions/check-release-commit - # `is_release` is the single authority for release/non-release gating; - # `tag` is only the npm dist-tag (next/latest) for `publish`. outputs: - tag: ${{ steps.check_commit.outputs.tag }} - is_release: ${{ steps.check_commit.outputs.tag != '' }} - - prepare: - name: Prepare release bundle + # workflow_dispatch is a rehearsal: force the release path with 'next' + # as a placeholder dist-tag (publish adds --dry-run, so it's never used). + tag: ${{ steps.check.outputs.tag || (github.event_name == 'workflow_dispatch' && 'next') || '' }} + is_release: ${{ steps.check.outputs.is-release == 'true' || github.event_name == 'workflow_dispatch' }} + + # The single release/non-release condition in this workflow: every other + # job hangs off this one via `needs`, so a non-release push skips the whole + # tree and nothing else needs a guard. + release_gate: + name: Release gate runs-on: ubuntu-24.04 - permissions: - contents: read - needs: - - build - - test-macOS-windows-binding - - test-linux-x64-gnu-binding - - test-linux-x64-musl-binding - - test-linux-aarch64-gnu-binding - - test-linux-aarch64-musl-binding - defaults: - run: - working-directory: ./crates/edr_napi - outputs: - filename: ${{ steps.pack.outputs.filename }} + needs: check_commit + if: needs.check_commit.outputs.is_release == 'true' steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node - uses: ./.github/actions/setup-node - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline - - name: Install sponge # needed for prepublish script - run: | - sudo apt-get update - sudo apt-get install -y moreutils - - name: Download all artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: ./crates/edr_napi/artifacts - - name: Run prepublish to update all `package.json` files - # we NEED to to this before pnpm artifacts, if not prepublish will try to publish - run: ../../scripts/prepublish.sh - - name: Move artifacts - # prepublish.sh (above) adds the platform packages to edr_napi's - # package.json but not the lockfile; skip pnpm 11's pre-run frozen - # deps check, which would otherwise fail on that intentional desync. - run: pnpm --config.verify-deps-before-run=false artifacts - - name: Copy coverage library into edr_napi crate - run: cp ../../data/contracts/coverage.sol ./coverage.sol - - name: Compile TypeScript helpers - # See "Move artifacts": package.json is intentionally out of sync with - # the lockfile here, so skip pnpm 11's pre-run frozen deps check. - run: pnpm --config.verify-deps-before-run=false exec tsc - - name: Create bundle with pnpm root & napi files - # create custom tar.gz because pnpm pack only packs root package and we need the platform-specific ones as well - id: pack - run: | - FILENAME="${{ env.APP_NAME }}_bundle.tar.gz" - NAPI_EXTRA_FILES=("npm/" "LICENSE" "README.md" "package.json" "Cargo.toml" "coverage.sol") - readarray -t FILES < <(cat package.json | jq -r .files[] ) - ALL_FILES=("${FILES[@]}" "${NAPI_EXTRA_FILES[@]}") - NAPI_FILES=() - for file in "${ALL_FILES[@]}"; do - NAPI_FILES+=("crates/edr_napi/${file}") - done - cd ../.. # cd to root dir - echo "Creating bundle with necessary files for publishing" - ROOT_FILES=("package.json" "Cargo.lock" "Cargo.toml" "pnpm-lock.yaml" "pnpm-workspace.yaml") - ALL_FILES=("${ROOT_FILES[@]}" "${NAPI_FILES[@]}") - echo "files to include in tar: ${ALL_FILES[*]}" - # Using --dereference since LICENSE file are symlinks - tar --dereference -czvf $FILENAME "${ALL_FILES[@]}" - echo "FILENAME=$FILENAME" - echo "filename=$FILENAME" >> $GITHUB_OUTPUT - - name: Upload packed tarball - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ steps.pack.outputs.filename }} - path: ./${{ steps.pack.outputs.filename }} - retention-days: 2 - overwrite: true - if-no-files-found: error + - name: Confirm release run + env: + TAG: ${{ needs.check_commit.outputs.tag }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' }} + run: echo "Release run confirmed (dist-tag $TAG, dry-run $DRY_RUN)" - review: - name: Review release - runs-on: ubuntu-24.04 + build: + name: Build and test + needs: release_gate + uses: ./.github/workflows/edr-npm-build.yml + with: + release: true permissions: contents: read - needs: - - build - - prepare - steps: - - name: Download packed tarball - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ needs.prepare.outputs.filename }} - path: . - - name: Decompress tarball - env: - FILENAME: ${{ needs.prepare.outputs.filename }} - run: tar -xvzf "$FILENAME" - - name: Inspect contents - run: tree . - - name: Check number of artifacts - shell: bash - run: | - # get number of artifacts with unique names - # shellcheck disable=SC2011 # .node names are fixed target triples, no special chars - NUMBER_OF_ARTIFACTS=$(ls -1q crates/edr_napi/npm/*/*.node | xargs -n 1 basename | sort | uniq | wc -l) - echo "Number of unique artifacts: $NUMBER_OF_ARTIFACTS" - echo "Expected number of unique artifacts: $NUMBER_OF_TARGETS" - test "$NUMBER_OF_ARTIFACTS" -eq "$NUMBER_OF_TARGETS" - - name: Setup pnpm - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - - name: Setup node - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - id: setup-node - with: - node-version: 24 - # A publish workflow must not restore a poisonable cache. - package-manager-cache: false - - name: Validate that packages have everything for publishing - run: | - cd crates/edr_napi - # Check in every cross-platform package - for platform in ./npm/* ; do - cd $platform - pnpm pack --pack-destination "$RUNNER_TEMP" - cd - - done - # And in root edr package - pnpm pack --pack-destination "$RUNNER_TEMP" - - name: Checkout repository for comparison - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - path: remote-repository - persist-credentials: false - ref: ${{ needs.build.outputs.commit_sha }} - - name: Validate there were no modifications to `package.json` when building - run: | - git diff --color=always --exit-code --no-index remote-repository/package.json ./package.json - - name: Validate there were no modifications to `Cargo.toml` when building - run: | - git diff --color=always --exit-code --no-index remote-repository/Cargo.toml ./Cargo.toml - - name: Validate there were no modifications to `Cargo.lock` when building - run: | - git diff --color=always --exit-code --no-index remote-repository/Cargo.lock ./Cargo.lock + packages: read + actions: read # select-node-image lists mirror workflow runs # Extra-defensive cooldown check: `main` branch and PRs already require passing this in CI, # but we re-check here to guarantee we never release with too-fresh dependencies. cooldown-check: name: Cargo cooldown check runs-on: ubuntu-24.04 - needs: check_commit - if: ${{ needs.check_commit.outputs.is_release == 'true' }} + needs: release_gate + permissions: + contents: read steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -702,8 +93,7 @@ jobs: notify-deploy: name: Notify pre-deploy to Slack runs-on: ubuntu-24.04 - needs: [check_commit, cooldown-check, prepare, review] - if: ${{ needs.check_commit.outputs.is_release == 'true' }} + needs: [cooldown-check, build] steps: - name: Notify pre-deploy uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 @@ -725,7 +115,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "*Commit:* <${{ github.event.head_commit.url || github.event.pull_request.html_url }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" + "text": "*Commit:* <${{ github.event.head_commit.url || format('{0}/{1}/commit/{2}', github.server_url, github.repository, github.sha) }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" } } ] @@ -737,9 +127,7 @@ jobs: runs-on: ubuntu-24.04 permissions: id-token: write # Needed for npm Trusted Publishing (OIDC) - needs: [check_commit, notify-deploy, prepare, review] - # Only run workflow if there is a tag defined - if: ${{ needs.check_commit.outputs.is_release == 'true' }} + needs: [check_commit, notify-deploy, build] steps: - name: Notify deployment to Slack uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 @@ -761,7 +149,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "*Commit:* <${{ github.event.head_commit.url || github.event.pull_request.html_url }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" + "text": "*Commit:* <${{ github.event.head_commit.url || format('{0}/{1}/commit/{2}', github.server_url, github.repository, github.sha) }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" } } ] @@ -769,11 +157,11 @@ jobs: - name: Download packed tarball uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ needs.prepare.outputs.filename }} + name: ${{ needs.build.outputs.bundle-filename }} path: . - name: Decompress tarball env: - FILENAME: ${{ needs.prepare.outputs.filename }} + FILENAME: ${{ needs.build.outputs.bundle-filename }} run: tar -xvzf "$FILENAME" - name: Setup pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 @@ -788,6 +176,7 @@ jobs: - name: Publish env: TAG: ${{ needs.check_commit.outputs.tag }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' }} run: | if [ "$TAG" = "next" ] then @@ -800,14 +189,21 @@ jobs: exit 1 fi + FLAGS=(--no-git-checks --tag "$TAG" --access public) + if [ "$DRY_RUN" = "true" ] + then + echo "Rehearsal run: publishing with --dry-run" + FLAGS+=(--dry-run) + fi + # We need to publish the contests of crates/edr_napi only cd ./crates/edr_napi # First publish every cross-platform package for platform in ./npm/* ; do cd $platform - pnpm publish --no-git-checks --tag "$TAG" --access public + pnpm publish "${FLAGS[@]}" cd - done # Finally Publish edr napi root package (because it depends on the cross-platform ones) - pnpm publish --no-git-checks --tag "$TAG" --access public + pnpm publish "${FLAGS[@]}" diff --git a/.github/workflows/mirror-docker-images.yml b/.github/workflows/mirror-docker-images.yml index ac8f18be52..0cce406b9c 100644 --- a/.github/workflows/mirror-docker-images.yml +++ b/.github/workflows/mirror-docker-images.yml @@ -24,7 +24,7 @@ on: - .github/workflows/mirror-docker-images.yml # Same-repo PRs touching this file mirror immediately, so a PR that adds a # tag below (e.g. a new node version) can be tested before merge. The - # edr-npm-release.yml docker jobs wait for this run before pulling. + # edr-npm-build.yml docker jobs wait for this run before pulling. pull_request: paths: - .github/workflows/mirror-docker-images.yml @@ -49,7 +49,7 @@ jobs: run: | set -euo pipefail # Keep in sync with the node versions and container images in - # edr-npm-release.yml; a tag missing here fails CI with "manifest + # edr-npm-build.yml; a tag missing here fails CI with "manifest # unknown" on the mirror pull. TAGS=(22 22-alpine 24 24-alpine 24-bullseye-slim 26 26-alpine) diff --git a/book/src/02_development/11_ci_docker_mirror.md b/book/src/02_development/11_ci_docker_mirror.md index 6fecd7038b..52149f3abb 100644 --- a/book/src/02_development/11_ci_docker_mirror.md +++ b/book/src/02_development/11_ci_docker_mirror.md @@ -2,13 +2,13 @@ CI pulls its Docker images from a GHCR mirror (`ghcr.io/nomicfoundation/edr/mirror/*`) instead of Docker Hub: Docker Hub rate-limits pulls, which fail intermittently on GitHub-hosted runners as a result. The mirror is maintained by `.github/workflows/mirror-docker-images.yml`, which re-copies the images weekly, on pushes to `main` and same-repo PRs that change the workflow itself, and on demand via `workflow_dispatch`. -Release runs are the exception: when `check_commit` sets its `is_release` output — the single authority for all release/non-release gating in `edr-npm-release.yml` — the docker jobs pull the official image straight from Docker Hub (the "Select image source" steps), so the mirror is never in the supply chain of published binaries — a tampered mirror tag can at most affect PR/branch CI, which publishes nothing. At one or two releases a week, Docker Hub's rate limits are not a concern for those runs. +Release runs are the exception: when `edr-npm-build.yml` runs in release mode (its `release` input — see the [release chapter](../03_release.md)), the docker jobs pull the official image straight from Docker Hub (the "Select image source" steps), so the mirror is never in the supply chain of published binaries — a tampered mirror tag can at most affect PR/branch CI, which publishes nothing. At one or two releases a week, Docker Hub's rate limits are not a concern for those runs. ## Adding a Node.js version (or any new tag) -Add the tag to the `TAGS` list in `mirror-docker-images.yml` in the same PR that changes the matrix in `edr-npm-release.yml`. The mirror workflow runs on same-repo PRs that touch it, so the new tag is mirrored — and the release matrix testable against it — before merge. A tag referenced in CI but missing from the mirror fails loudly with `manifest unknown`. +Add the tag to the `TAGS` list in `mirror-docker-images.yml` in the same PR that changes the matrix in `edr-npm-build.yml`. The mirror workflow runs on same-repo PRs that touch it, so the new tag is mirrored — and the release matrix testable against it — before merge. A tag referenced in CI but missing from the mirror fails loudly with `manifest unknown`. -On such a PR the mirror job and the release-workflow docker jobs start in parallel. The docker jobs' "Select image source" step waits (up to 15 minutes) for the mirror run on the same commit to finish before pulling, so the new tag is in place by the time it's needed. The wait is best-effort and never fails the job — if the mirror run failed or was skipped, the jobs proceed with a warning, and a genuinely missing tag still fails the pull itself with `manifest unknown`. On PRs that don't touch the mirror workflow no mirror run exists and the step doesn't wait. +On such a PR the mirror job and the build-workflow docker jobs start in parallel. The docker jobs' "Select image source" step waits (up to 15 minutes) for the mirror run on the same commit to finish before pulling, so the new tag is in place by the time it's needed. The wait is best-effort and never fails the job — if the mirror run failed or was skipped, the jobs proceed with a warning, and a genuinely missing tag still fails the pull itself with `manifest unknown`. On PRs that don't touch the mirror workflow no mirror run exists and the step doesn't wait. ## Access diff --git a/book/src/03_release.md b/book/src/03_release.md index f87dee8091..308bfd0261 100644 --- a/book/src/03_release.md +++ b/book/src/03_release.md @@ -3,3 +3,15 @@ Releasing the [EDR NPM package](../../crates/edr_napi/package.json) is handled by the [EDR NPM release](../../.github/workflows/edr-npm-release.yml) GitHub Action workflow. A new release is created automatically on commits to the `main` branch that follow the following format: `edr-0.1.0` for releases or `edr-0.1.0-alpha.1` for pre-releases. + +## Workflow structure + +The npm pipeline is split across three workflows: + +- [`edr-npm-build.yml`](../../.github/workflows/edr-npm-build.yml) — reusable (`workflow_call`) build+test+bundle pipeline: the 7-target build matrix, binding tests, bundle preparation, and the pre-publish review. Its single `release` boolean input is the only release/non-release mode switch: release runs build cold (no cargo cache, asserted at runtime) and pull official Docker Hub images instead of the [GHCR mirror](./02_development/11_ci_docker_mirror.md). +- [`edr-npm-ci.yml`](../../.github/workflows/edr-npm-ci.yml) — PR/branch validation: calls the build workflow with `release: false`. Skips itself on release pushes. +- [`edr-npm-release.yml`](../../.github/workflows/edr-npm-release.yml) — detects release commits ([`check-release-commit`](../../.github/actions/check-release-commit/action.yml)), then calls the build workflow with `release: true` and runs the release-only jobs: cargo cooldown check, Slack notifications, and the `edr-release` environment-protected publish. Everything hangs off a single `release_gate` job, so no individual job or step needs its own release condition. + +## Release rehearsal + +Dispatching `edr-npm-release.yml` manually (`workflow_dispatch`) runs the full release path on any commit — cold build, Docker Hub images, cooldown check, environment approval — but publishes with `--dry-run`, so nothing reaches the npm registry. Use this to validate release-path changes without cutting a release. Real publishes happen only on push events.