diff --git a/.github/actions/check-release-commit/action.yml b/.github/actions/check-release-commit/action.yml new file mode 100644 index 0000000000..f6dfad531f --- /dev/null +++ b/.github/actions/check-release-commit/action.yml @@ -0,0 +1,44 @@ +name: Check release commit +description: > + Detects (pre-)release commits from the HEAD commit message. Single authority + for the release/non-release decision, shared by edr-npm-ci.yml (which skips + itself on release pushes) and edr-npm-release.yml (which only proceeds on + them). Requires the repository to be checked out. + +outputs: + tag: + description: npm dist-tag ('next' for pre-releases, 'latest' for releases, empty otherwise) + value: ${{ steps.check.outputs.tag }} + is-release: + description: "'true' when the HEAD commit triggers a (pre-)release" + value: ${{ steps.check.outputs.tag != '' }} + +runs: + using: composite + steps: + - id: check + shell: bash + run: | + # matches with pre-release commits: edr release name but ends with - and some postfix like beta + if git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+-"; + then + if [ "$GITHUB_EVENT_NAME" = "push" ] + then + echo "pre-release commit: tag=next" + echo "tag=next" >> "$GITHUB_OUTPUT" + else + echo "pre-release commit, but event is $GITHUB_EVENT_NAME, not push: not a release run" + fi + # matches with release commit + elif git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+\s*"; + then + if [ "$GITHUB_REF" = "refs/heads/main" ] || [ "$GITHUB_REF" = "refs/heads/hh2" ] + then + echo "release commit: tag=latest" + echo "tag=latest" >> "$GITHUB_OUTPUT" + else + echo "release commit, but $GITHUB_REF is not main or hh2: not a release run" + fi + else + echo "not a release commit: not a release run" + fi diff --git a/.github/actions/select-node-image/action.yml b/.github/actions/select-node-image/action.yml index af3a95c7ea..2e7abd0d9d 100644 --- a/.github/actions/select-node-image/action.yml +++ b/.github/actions/select-node-image/action.yml @@ -1,12 +1,12 @@ name: Select node image -description: Resolves the node Docker image for a job. Normal runs use the GHCR mirror (see mirror-docker-images.yml); release runs pull the official Docker Hub image directly, keeping the mirror out of the supply chain of published binaries. +description: Resolves the node Docker image for a job, waiting (up to 15 minutes) for any in-flight mirror run first. Normal runs use the GHCR mirror (see mirror-docker-images.yml); release runs pull the official Docker Hub image directly, keeping the mirror out of the supply chain of published binaries. inputs: tag: description: node image tag, e.g. 24-bullseye-slim or 22-alpine required: true - release-tag: - description: check_commit's release tag; non-empty marks a release run + is-release: + description: edr-npm-build.yml's release input; 'true' marks a release run required: true outputs: @@ -17,13 +17,57 @@ outputs: runs: using: composite steps: + # On PRs/pushes that touch mirror-docker-images.yml, pulls can race ahead + # of the mirror run for the same SHA and fail with "manifest unknown"; no + # run means nothing to wait for. Best-effort: never fails the job — the + # pull itself is the authoritative failure for a genuinely missing tag. + - name: Wait for in-flight mirror run + if: inputs.is-release != 'true' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + # For pull_request events the mirror run is recorded against the PR + # head SHA, not the merge commit that github.sha points at. + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + run: | + set -euo pipefail + DEADLINE=$(( $(date +%s) + 15 * 60 )) + while true; do + RUN=$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/mirror-docker-images.yml/runs?head_sha=${HEAD_SHA}&per_page=1" \ + --jq '.workflow_runs[0] // empty') + if [ -z "$RUN" ]; then + echo "No mirror run for ${HEAD_SHA}; not waiting." + exit 0 + fi + STATUS=$(jq -r .status <<<"$RUN") + if [ "$STATUS" = "completed" ]; then + CONCLUSION=$(jq -r .conclusion <<<"$RUN") + URL=$(jq -r .html_url <<<"$RUN") + case "$CONCLUSION" in + success) echo "Mirror run succeeded: $URL" ;; + # The mirror job skips itself on fork PRs; the tags it would + # have re-copied already exist. + skipped) echo "Mirror run was skipped: $URL" ;; + *) echo "::warning::Mirror run for ${HEAD_SHA} concluded '$CONCLUSION' ($URL); proceeding anyway" ;; + esac + exit 0 + fi + if [ "$(date +%s)" -ge "$DEADLINE" ]; then + echo "::warning::Timed out waiting for the mirror run for ${HEAD_SHA}; proceeding anyway" + exit 0 + fi + echo "Mirror run in progress (status: $STATUS); retrying in 30s..." + sleep 30 + done + - id: select shell: bash env: TAG: ${{ inputs.tag }} - RELEASE_TAG: ${{ inputs.release-tag }} + IS_RELEASE: ${{ inputs.is-release }} run: | - if [ -n "$RELEASE_TAG" ]; then + if [ "$IS_RELEASE" = "true" ]; then echo "ref=node:${TAG}" >> "$GITHUB_OUTPUT" else echo "ref=ghcr.io/nomicfoundation/edr/mirror/node:${TAG}" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/edr-npm-build.yml b/.github/workflows/edr-npm-build.yml new file mode 100644 index 0000000000..f1ed63988d --- /dev/null +++ b/.github/workflows/edr-npm-build.yml @@ -0,0 +1,637 @@ +# Reusable build+test+bundle pipeline for the EDR npm package, called by +# edr-npm-ci.yml (release: false) and edr-npm-release.yml (release: true). +# The `release` input is the only mode switch: release runs build cold and +# pull official Docker Hub images; non-release runs use the cargo cache and +# the GHCR mirror. Publishing lives in edr-npm-release.yml, never here. +name: EDR NPM build and test +env: + DEBUG: napi:* + APP_NAME: edr + NUMBER_OF_TARGETS: 7 +permissions: {} +on: + workflow_call: + inputs: + release: + description: release mode — cold builds, official Docker Hub images + type: boolean + required: true + outputs: + bundle-filename: + description: filename of the packed bundle artifact + value: ${{ jobs.prepare.outputs.filename }} + commit-sha: + description: HEAD commit the bindings were built from + value: ${{ jobs.build.outputs.commit_sha }} + +jobs: + build: + name: stable - ${{ matrix.settings.target }} - node@24 + runs-on: ${{ matrix.settings.host }} + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + settings: + - host: macos-15-intel + target: x86_64-apple-darwin + build: pnpm run build --strip + + - host: macos-14 + target: aarch64-apple-darwin + build: pnpm run build --strip + + - host: windows-2025 + target: x86_64-pc-windows-msvc + build: pnpm run build --strip + # NTFS small-file creates are ~5-10x slower than ext4/APFS, + # so napi-rs's pre-cargo `cargo metadata` step burns ~50s + # extracting ~500 .crate tarballs into registry/src/ on warm + # runs without this. Other platforms re-extract fast enough + # that caching it isn't worth the size hit. + cache_extra_paths: ~/.cargo/registry/src/ + + # docker_tag legs build in a node container resolved by the + # select-node-image action (GHCR mirror normally, Docker Hub on + # releases). New or changed tags must be added to the mirror + # workflow's tag list in the same PR. + - host: ubuntu-24.04 + target: x86_64-unknown-linux-gnu + docker_tag: 24-bullseye-slim + flavor: gnu + + - host: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + docker_tag: 24-bullseye-slim + flavor: gnu + + - host: ubuntu-24.04 + target: x86_64-unknown-linux-musl + docker_tag: 24-alpine + flavor: musl + + - host: ubuntu-24.04-arm + target: aarch64-unknown-linux-musl + docker_tag: 24-alpine + flavor: musl + outputs: + commit_sha: ${{ steps.save-commit.outputs.commit_sha}} + defaults: + run: + working-directory: ./crates/edr_napi + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - uses: socketdev/action@ba6de6cc0565af1f42295590380973573297e31f # v1.3.2 + with: + mode: firewall + + - name: Save HEAD commit sha + id: save-commit + shell: bash + run: | + COMMIT_SHA=$(git rev-parse HEAD) + echo "Current HEAD: $COMMIT_SHA" + echo "commit_sha=$COMMIT_SHA" >> $GITHUB_OUTPUT + + - name: Check number of targets + shell: bash + run: + | # zizmor: ignore[template-injection] strategy.job-total is a controlled integer, not external input + echo "Number of build jobs: ${{ strategy.job-total }}" + echo "Expected number of build jobs: $NUMBER_OF_TARGETS" + test ${{ strategy.job-total }} -eq "$NUMBER_OF_TARGETS" + + - name: Install Rust toolchain + uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # latest main commit (2025-11-17) + if: ${{ !matrix.settings.docker_tag }} + with: + toolchain: stable + targets: ${{ matrix.settings.target }} + components: rust-src + + - name: Setup node and pnpm + if: ${{ !matrix.settings.docker_tag }} + uses: ./.github/actions/setup-node + with: + node-version: 24 + + # Release commits build cold: napi codegen relies on proc-macro + # expansion (napi-rs#1297), so warm-cache builds risk shipping + # stale or empty index.js / index.d.ts. Don't enable for releases. + - name: Cache cargo # zizmor: ignore[cache-poisoning] release builds run cold (asserted below); PR/main caches never feed a publish + if: ${{ !inputs.release }} + uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + ${{ matrix.settings.cache_extra_paths }} + key: ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}- + + # Tripwire for future edits to the cache gate above: if it stops + # skipping release runs, fail before building from a warm cache. + - name: Assert release builds run cold + if: ${{ inputs.release }} + shell: bash + run: | + if [ -e "${GITHUB_WORKSPACE}/target" ]; then + echo "target/ exists on a release build: the cargo cache gate is broken" >&2 + exit 1 + fi + + - name: Install dependencies (non-Windows) + if: ${{ !matrix.settings.docker_tag && runner.os != 'Windows' }} + run: sfw pnpm install --frozen-lockfile --prefer-offline + + # SFW doesn't install dependencies correctly on Windows + # https://github.com/NomicFoundation/edr/issues/1199 + - name: Install dependencies (Windows) + if: ${{ !matrix.settings.docker_tag && runner.os == 'Windows' }} + run: pnpm install --frozen-lockfile --prefer-offline + + - name: Build (non-docker) + if: ${{ !matrix.settings.docker_tag }} + run: ${{ matrix.settings.build }} + shell: bash + + - name: Select image source + id: select-image + if: ${{ matrix.settings.docker_tag }} + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.settings.docker_tag }} + is-release: ${{ inputs.release }} + + - name: Build in docker (gnu) + uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) + if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'gnu'}} + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # /home/runner/.cargo matches where the host's `Cache cargo` step writes + # (`~/.cargo` on GH-hosted Linux runners); docker -v doesn't expand `~` and + # GHA has no `runner.home` context, so the path is hardcoded. + options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" + run: | + set -e + + # Install Rust toolchain + apt-get update + apt-get install -y curl ca-certificates build-essential pkg-config + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + export PATH="/root/.cargo/bin:$PATH" + + npm i -g pnpm@11.17.0 --ignore-scripts + pnpm -v + npm i -g sfw + sfw pnpm install --frozen-lockfile --prefer-offline + pnpm run build --strip + + # SFW is not provided on musl versions of Linux + # https://github.com/NomicFoundation/edr/issues/1198 + - name: Build in docker (musl) + uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) + if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'musl' }} + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # See gnu step above for why /home/runner/.cargo is hardcoded. + options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" + run: | + set -e + + # Install Rust toolchain + apk add --no-cache build-base python3 make git pkgconfig openssl openssl-dev ca-certificates curl bash perl cmake + # Conditional symlink only for ARM builds + if [ "${{ matrix.settings.target }}" = "aarch64-unknown-linux-musl" ]; then + ln -sf /usr/bin/gcc /usr/bin/aarch64-linux-musl-gcc + fi + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable + export PATH="/root/.cargo/bin:$PATH" + + npm i -g pnpm@11.17.0 --ignore-scripts + pnpm -v + pnpm install --frozen-lockfile --prefer-offline + pnpm run build --strip + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: bindings-${{ matrix.settings.target }} + # The upload artifact action doesn't respect the working directory setting. Unclear if this is a bug or not + # https://github.com/actions/upload-artifact/issues/294 + path: ./crates/edr_napi/${{ env.APP_NAME }}.*.node + if-no-files-found: error + + test-macOS-windows-binding: + name: Test bindings on ${{ matrix.settings.target }} - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + strategy: + fail-fast: false + matrix: + settings: + - host: macos-15-intel + target: x86_64-apple-darwin + - host: macos-14 + target: aarch64-apple-darwin + - host: windows-2022 + target: x86_64-pc-windows-msvc + node: + - "22" + - "24" + - "26" + runs-on: ${{ matrix.settings.host }} + defaults: + run: + working-directory: ./crates/edr_napi + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-${{ matrix.settings.target }} + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Test bindings + run: pnpm testNoBuild + test-linux-x64-gnu-binding: + name: Test bindings on Linux-x64-gnu - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-x86_64-unknown-linux-gnu + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }} + is-release: ${{ inputs.release }} + - name: Test bindings + # Setting CI=1 is important to make PNPM install non-interactive + # https://github.com/pnpm/pnpm/issues/6615#issuecomment-1656945689 + env: + IMAGE: ${{ steps.select-image.outputs.ref }} + run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" bash -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" + test-linux-x64-musl-binding: + name: Test bindings on x86_64-unknown-linux-musl - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline --libc=musl + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-x86_64-unknown-linux-musl + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image + uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }}-alpine + is-release: ${{ inputs.release }} + - name: Test bindings + env: + IMAGE: ${{ steps.select-image.outputs.ref }} + run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" sh -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" + test-linux-aarch64-gnu-binding: + name: Test bindings on aarch64-unknown-linux-gnu - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-aarch64-unknown-linux-gnu + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Install dependencies + run: | + pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=glibc + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }} + is-release: ${{ inputs.release }} + - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64) test runner + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" + run: | + npm install -g pnpm@11.17.0 + set -e + pnpm testNoBuild + ls -la + test-linux-aarch64-musl-binding: + name: Test bindings on aarch64-unknown-linux-musl - node@${{ matrix.node }} + needs: + - build + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs + strategy: + fail-fast: false + matrix: + node: + - "22" + - "24" + - "26" + runs-on: ubuntu-24.04-arm + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node and pnpm + uses: ./.github/actions/setup-node + with: + node-version: ${{ matrix.node }} + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: bindings-aarch64-unknown-linux-musl + path: ./crates/edr_napi/ + - name: List packages + run: ls -R . + shell: bash + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=musl + - name: Select image source + id: select-image + uses: ./.github/actions/select-node-image + with: + tag: ${{ matrix.node }}-alpine + is-release: ${{ inputs.release }} + - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64/musl) test runner + uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 + with: + image: ${{ steps.select-image.outputs.ref }} + # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" + run: | + npm install -g pnpm@11.17.0 + set -e + pnpm testNoBuild + + prepare: + name: Prepare release bundle + runs-on: ubuntu-24.04 + permissions: + contents: read + needs: + - build + - test-macOS-windows-binding + - test-linux-x64-gnu-binding + - test-linux-x64-musl-binding + - test-linux-aarch64-gnu-binding + - test-linux-aarch64-musl-binding + defaults: + run: + working-directory: ./crates/edr_napi + outputs: + filename: ${{ steps.pack.outputs.filename }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + - name: Setup node + uses: ./.github/actions/setup-node + - name: Install dependencies + run: pnpm install --frozen-lockfile --prefer-offline + - name: Install sponge # needed for prepublish script + run: | + sudo apt-get update + sudo apt-get install -y moreutils + - name: Download all artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: ./crates/edr_napi/artifacts + - name: Run prepublish to update all `package.json` files + # we NEED to to this before pnpm artifacts, if not prepublish will try to publish + run: ../../scripts/prepublish.sh + - name: Move artifacts + # prepublish.sh (above) adds the platform packages to edr_napi's + # package.json but not the lockfile; skip pnpm 11's pre-run frozen + # deps check, which would otherwise fail on that intentional desync. + run: pnpm --config.verify-deps-before-run=false artifacts + - name: Copy coverage library into edr_napi crate + run: cp ../../data/contracts/coverage.sol ./coverage.sol + - name: Compile TypeScript helpers + # See "Move artifacts": package.json is intentionally out of sync with + # the lockfile here, so skip pnpm 11's pre-run frozen deps check. + run: pnpm --config.verify-deps-before-run=false exec tsc + - name: Create bundle with pnpm root & napi files + # create custom tar.gz because pnpm pack only packs root package and we need the platform-specific ones as well + id: pack + run: | + FILENAME="${{ env.APP_NAME }}_bundle.tar.gz" + NAPI_EXTRA_FILES=("npm/" "LICENSE" "README.md" "package.json" "Cargo.toml" "coverage.sol") + readarray -t FILES < <(cat package.json | jq -r .files[] ) + ALL_FILES=("${FILES[@]}" "${NAPI_EXTRA_FILES[@]}") + NAPI_FILES=() + for file in "${ALL_FILES[@]}"; do + NAPI_FILES+=("crates/edr_napi/${file}") + done + cd ../.. # cd to root dir + echo "Creating bundle with necessary files for publishing" + ROOT_FILES=("package.json" "Cargo.lock" "Cargo.toml" "pnpm-lock.yaml" "pnpm-workspace.yaml") + ALL_FILES=("${ROOT_FILES[@]}" "${NAPI_FILES[@]}") + echo "files to include in tar: ${ALL_FILES[*]}" + # Using --dereference since LICENSE file are symlinks + tar --dereference -czvf $FILENAME "${ALL_FILES[@]}" + echo "FILENAME=$FILENAME" + echo "filename=$FILENAME" >> $GITHUB_OUTPUT + - name: Upload packed tarball + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.pack.outputs.filename }} + path: ./${{ steps.pack.outputs.filename }} + retention-days: 2 + overwrite: true + if-no-files-found: error + + review: + name: Review release + runs-on: ubuntu-24.04 + permissions: + contents: read + needs: + - build + - prepare + steps: + - name: Download packed tarball + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ needs.prepare.outputs.filename }} + path: . + - name: Decompress tarball + env: + FILENAME: ${{ needs.prepare.outputs.filename }} + run: tar -xvzf "$FILENAME" + - name: Inspect contents + run: tree . + - name: Check number of artifacts + shell: bash + run: | + # get number of artifacts with unique names + # shellcheck disable=SC2011 # .node names are fixed target triples, no special chars + NUMBER_OF_ARTIFACTS=$(ls -1q crates/edr_napi/npm/*/*.node | xargs -n 1 basename | sort | uniq | wc -l) + echo "Number of unique artifacts: $NUMBER_OF_ARTIFACTS" + echo "Expected number of unique artifacts: $NUMBER_OF_TARGETS" + test "$NUMBER_OF_ARTIFACTS" -eq "$NUMBER_OF_TARGETS" + - name: Setup pnpm + uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 + - name: Setup node + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + id: setup-node + with: + node-version: 24 + # A publish workflow must not restore a poisonable cache. + package-manager-cache: false + - name: Validate that packages have everything for publishing + run: | + cd crates/edr_napi + # Check in every cross-platform package + for platform in ./npm/* ; do + cd $platform + pnpm pack --pack-destination "$RUNNER_TEMP" + cd - + done + # And in root edr package + pnpm pack --pack-destination "$RUNNER_TEMP" + - name: Checkout repository for comparison + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + path: remote-repository + persist-credentials: false + ref: ${{ needs.build.outputs.commit_sha }} + - name: Validate there were no modifications to `package.json` when building + run: | + git diff --color=always --exit-code --no-index remote-repository/package.json ./package.json + - name: Validate there were no modifications to `Cargo.toml` when building + run: | + git diff --color=always --exit-code --no-index remote-repository/Cargo.toml ./Cargo.toml + - name: Validate there were no modifications to `Cargo.lock` when building + run: | + git diff --color=always --exit-code --no-index remote-repository/Cargo.lock ./Cargo.lock + diff --git a/.github/workflows/edr-npm-ci.yml b/.github/workflows/edr-npm-ci.yml new file mode 100644 index 0000000000..4e361d008a --- /dev/null +++ b/.github/workflows/edr-npm-ci.yml @@ -0,0 +1,55 @@ +# Thin PR/branch validation entry: runs the reusable build workflow +# (edr-npm-build.yml) in non-release mode — warm cargo caches, GHCR-mirrored +# images. Release pushes are skipped here; edr-npm-release.yml owns those. +name: EDR NPM CI +permissions: {} +on: + push: + # keep branches and paths in sync with edr-npm-release.yml + branches: + - main + - prerelease + - hh2 + - feat/solidity-tests + tags-ignore: + - "**" + paths-ignore: + - "**/*.md" + - LICENSE + - "**/*.gitignore" + - .editorconfig + - docs/** + pull_request: null + workflow_dispatch: + +jobs: + check_commit: + name: Check commit + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + ref: ${{ github.event.pull_request.head.ref }} + + - id: check + uses: ./.github/actions/check-release-commit + + outputs: + is_release: ${{ steps.check.outputs.is-release }} + + build: + name: Build and test + needs: check_commit + # The single condition in this workflow: release pushes are handled by + # edr-npm-release.yml with a cold release-mode build instead. + if: needs.check_commit.outputs.is_release != 'true' + uses: ./.github/workflows/edr-npm-build.yml + with: + release: false + permissions: + contents: read + packages: read + actions: read # select-node-image lists mirror workflow runs diff --git a/.github/workflows/edr-npm-release.yml b/.github/workflows/edr-npm-release.yml index f998a9a7e1..258bb3bc75 100644 --- a/.github/workflows/edr-npm-release.yml +++ b/.github/workflows/edr-npm-release.yml @@ -1,13 +1,15 @@ +# Thin release pipeline: detects (pre-)release commits, runs the reusable +# build workflow (edr-npm-build.yml) in release mode, and publishes to npm. +# Non-release pushes skip everything after check_commit; PR/branch validation +# lives in edr-npm-ci.yml. name: EDR NPM release -env: - DEBUG: napi:* - APP_NAME: edr - NUMBER_OF_TARGETS: 7 permissions: {} on: push: # pre-releases will be triggered in any of these branches # production releases will only be triggered for `main` + # (keep branches and paths in sync with edr-npm-ci.yml, which skips + # release pushes so each push runs exactly one of the two workflows) branches: - main - prerelease @@ -21,467 +23,12 @@ on: - "**/*.gitignore" - .editorconfig - docs/** - pull_request: null + # Release rehearsal: forces the release path on any commit — cold build, + # official Docker Hub images, cooldown check, environment approval — but + # publishes with --dry-run. Real publishes happen only on push events. workflow_dispatch: jobs: - build: - name: stable - ${{ matrix.settings.target }} - node@24 - runs-on: ${{ matrix.settings.host }} - needs: check_commit - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: - settings: - - host: macos-15-intel - target: x86_64-apple-darwin - build: pnpm run build --strip - - - host: macos-14 - target: aarch64-apple-darwin - build: pnpm run build --strip - - - host: windows-2025 - target: x86_64-pc-windows-msvc - build: pnpm run build --strip - # NTFS small-file creates are ~5-10x slower than ext4/APFS, - # so napi-rs's pre-cargo `cargo metadata` step burns ~50s - # extracting ~500 .crate tarballs into registry/src/ on warm - # runs without this. Other platforms re-extract fast enough - # that caching it isn't worth the size hit. - cache_extra_paths: ~/.cargo/registry/src/ - - # docker_tag legs build in a node container resolved by the - # select-node-image action (GHCR mirror normally, Docker Hub on - # releases). New or changed tags must be added to the mirror - # workflow's tag list in the same PR. - - host: ubuntu-24.04 - target: x86_64-unknown-linux-gnu - docker_tag: 24-bullseye-slim - flavor: gnu - - - host: ubuntu-24.04-arm - target: aarch64-unknown-linux-gnu - docker_tag: 24-bullseye-slim - flavor: gnu - - - host: ubuntu-24.04 - target: x86_64-unknown-linux-musl - docker_tag: 24-alpine - flavor: musl - - - host: ubuntu-24.04-arm - target: aarch64-unknown-linux-musl - docker_tag: 24-alpine - flavor: musl - outputs: - commit_sha: ${{ steps.save-commit.outputs.commit_sha}} - defaults: - run: - working-directory: ./crates/edr_napi - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - uses: socketdev/action@ba6de6cc0565af1f42295590380973573297e31f # v1.3.2 - with: - mode: firewall - - - name: Save HEAD commit sha - id: save-commit - shell: bash - run: | - COMMIT_SHA=$(git rev-parse HEAD) - echo "Current HEAD: $COMMIT_SHA" - echo "commit_sha=$COMMIT_SHA" >> $GITHUB_OUTPUT - - - name: Check number of targets - shell: bash - run: - | # zizmor: ignore[template-injection] strategy.job-total is a controlled integer, not external input - echo "Number of build jobs: ${{ strategy.job-total }}" - echo "Expected number of build jobs: $NUMBER_OF_TARGETS" - test ${{ strategy.job-total }} -eq "$NUMBER_OF_TARGETS" - - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@efa25f7f19611383d5b0ccf2d1c8914531636bf9 # latest main commit (2025-11-17) - if: ${{ !matrix.settings.docker_tag }} - with: - toolchain: stable - targets: ${{ matrix.settings.target }} - components: rust-src - - - name: Setup node and pnpm - if: ${{ !matrix.settings.docker_tag }} - uses: ./.github/actions/setup-node - with: - node-version: 24 - - # Release commits build cold: napi codegen relies on proc-macro - # expansion (napi-rs#1297), so warm-cache builds risk shipping - # stale or empty index.js / index.d.ts. Don't enable for releases. - - name: Cache cargo # zizmor: ignore[cache-poisoning] release builds run cold (guarded below); PR/main caches never feed a publish - if: needs.check_commit.outputs.tag == '' - uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0 - with: - path: | - ~/.cargo/registry/index/ - ~/.cargo/registry/cache/ - ~/.cargo/git/db/ - target/ - ${{ matrix.settings.cache_extra_paths }} - key: ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}-${{ hashFiles('**/Cargo.lock') }} - restore-keys: | - ${{ matrix.settings.target }}-cargo-v5-${{ matrix.settings.host }}- - - - name: Install dependencies (non-Windows) - if: ${{ !matrix.settings.docker_tag && runner.os != 'Windows' }} - run: sfw pnpm install --frozen-lockfile --prefer-offline - - # SFW doesn't install dependencies correctly on Windows - # https://github.com/NomicFoundation/edr/issues/1199 - - name: Install dependencies (Windows) - if: ${{ !matrix.settings.docker_tag && runner.os == 'Windows' }} - run: pnpm install --frozen-lockfile --prefer-offline - - - name: Build (non-docker) - if: ${{ !matrix.settings.docker_tag }} - run: ${{ matrix.settings.build }} - shell: bash - - - name: Select image source - id: select-image - if: ${{ matrix.settings.docker_tag }} - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.settings.docker_tag }} - release-tag: ${{ needs.check_commit.outputs.tag }} - - - name: Build in docker (gnu) - uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) - if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'gnu'}} - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # /home/runner/.cargo matches where the host's `Cache cargo` step writes - # (`~/.cargo` on GH-hosted Linux runners); docker -v doesn't expand `~` and - # GHA has no `runner.home` context, so the path is hardcoded. - options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" - run: | - set -e - - # Install Rust toolchain - apt-get update - apt-get install -y curl ca-certificates build-essential pkg-config - curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable - export PATH="/root/.cargo/bin:$PATH" - - npm i -g pnpm@11.17.0 --ignore-scripts - pnpm -v - npm i -g sfw - sfw pnpm install --frozen-lockfile --prefer-offline - pnpm run build --strip - - # SFW is not provided on musl versions of Linux - # https://github.com/NomicFoundation/edr/issues/1198 - - name: Build in docker (musl) - uses: NomicFoundation/docker-run-action@63f044457cfb71a5c63fa589218c89a418565d9c # Fork of v3 with updated Docker (https://github.com/addnab/docker-run-action/issues/62) - if: ${{ matrix.settings.docker_tag && matrix.settings.flavor == 'musl' }} - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - # See gnu step above for why /home/runner/.cargo is hardcoded. - options: "--user 0:0 -v /home/runner/.cargo/git:/usr/local/cargo/git -v /home/runner/.cargo/registry:/usr/local/cargo/registry -v ${{ github.workspace }}:/build -w /build/crates/edr_napi" - run: | - set -e - - # Install Rust toolchain - apk add --no-cache build-base python3 make git pkgconfig openssl openssl-dev ca-certificates curl bash perl cmake - # Conditional symlink only for ARM builds - if [ "${{ matrix.settings.target }}" = "aarch64-unknown-linux-musl" ]; then - ln -sf /usr/bin/gcc /usr/bin/aarch64-linux-musl-gcc - fi - curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable - export PATH="/root/.cargo/bin:$PATH" - - npm i -g pnpm@11.17.0 --ignore-scripts - pnpm -v - pnpm install --frozen-lockfile --prefer-offline - pnpm run build --strip - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: bindings-${{ matrix.settings.target }} - # The upload artifact action doesn't respect the working directory setting. Unclear if this is a bug or not - # https://github.com/actions/upload-artifact/issues/294 - path: ./crates/edr_napi/${{ env.APP_NAME }}.*.node - if-no-files-found: error - - test-macOS-windows-binding: - name: Test bindings on ${{ matrix.settings.target }} - node@${{ matrix.node }} - needs: - - build - permissions: - contents: read - strategy: - fail-fast: false - matrix: - settings: - - host: macos-15-intel - target: x86_64-apple-darwin - - host: macos-14 - target: aarch64-apple-darwin - - host: windows-2022 - target: x86_64-pc-windows-msvc - node: - - "22" - - "24" - - "26" - runs-on: ${{ matrix.settings.host }} - defaults: - run: - working-directory: ./crates/edr_napi - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-${{ matrix.settings.target }} - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Test bindings - run: pnpm testNoBuild - test-linux-x64-gnu-binding: - name: Test bindings on Linux-x64-gnu - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-x86_64-unknown-linux-gnu - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }} - release-tag: ${{ needs.check_commit.outputs.tag }} - - name: Test bindings - # Setting CI=1 is important to make PNPM install non-interactive - # https://github.com/pnpm/pnpm/issues/6615#issuecomment-1656945689 - env: - IMAGE: ${{ steps.select-image.outputs.ref }} - run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" bash -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" - test-linux-x64-musl-binding: - name: Test bindings on x86_64-unknown-linux-musl - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline --libc=musl - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-x86_64-unknown-linux-musl - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Log in to GHCR # for mirror pulls; unused when a release run selects the Docker Hub image - uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }}-alpine - release-tag: ${{ needs.check_commit.outputs.tag }} - - name: Test bindings - env: - IMAGE: ${{ steps.select-image.outputs.ref }} - run: docker run --rm -e CI=1 -v "$(pwd)":/build -w /build/crates/edr_napi "$IMAGE" sh -c "npm install -g pnpm@11.17.0; pnpm testNoBuild" - test-linux-aarch64-gnu-binding: - name: Test bindings on aarch64-unknown-linux-gnu - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-aarch64-unknown-linux-gnu - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Install dependencies - run: | - pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=glibc - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }} - release-tag: ${{ needs.check_commit.outputs.tag }} - - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64) test runner - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 #v3 - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" - run: | - npm install -g pnpm@11.17.0 - set -e - pnpm testNoBuild - ls -la - test-linux-aarch64-musl-binding: - name: Test bindings on aarch64-unknown-linux-musl - node@${{ matrix.node }} - needs: - - build - - check_commit - permissions: - contents: read - packages: read - strategy: - fail-fast: false - matrix: - node: - - "22" - - "24" - - "26" - runs-on: ubuntu-24.04-arm - steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node and pnpm - uses: ./.github/actions/setup-node - with: - node-version: ${{ matrix.node }} - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: bindings-aarch64-unknown-linux-musl - path: ./crates/edr_napi/ - - name: List packages - run: ls -R . - shell: bash - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline --cpu=arm64 --libc=musl - - name: Select image source - id: select-image - uses: ./.github/actions/select-node-image - with: - tag: ${{ matrix.node }}-alpine - release-tag: ${{ needs.check_commit.outputs.tag }} - - name: Setup and run tests # zizmor: ignore[superfluous-actions] deliberate cross-arch (arm64/musl) test runner - uses: addnab/docker-run-action@4f65fabd2431ebc8d299f8e5a018d79a769ae185 # v3 - with: - image: ${{ steps.select-image.outputs.ref }} - # GHCR auth for mirror pulls; unused when a release run selects the Docker Hub image. - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - options: "--platform linux/arm64 -v ${{ github.workspace }}:/build -w /build/crates/edr_napi -e CI=1" - run: | - npm install -g pnpm@11.17.0 - set -e - pnpm testNoBuild - check_commit: name: Check commit runs-on: ubuntu-24.04 @@ -491,181 +38,50 @@ jobs: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false - ref: ${{ github.event.pull_request.head.ref }} - - name: Check if commit message is a release commit - id: check_commit - run: | - # matches with pre-release commits: edr release name but ends with - and some postfix like beta - if git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+-"; - then - if [ "${{ github.event_name }}" = "push" ] - then - echo "pre-release commit: tag=next" - echo "tag=next" >> "$GITHUB_OUTPUT" - fi - # matches with release commit - elif git log -1 --pretty=%B | grep "^edr-[0-9]\+\.[0-9]\+\.[0-9]\+\s*"; - then - if [ "$GITHUB_REF" == "refs/heads/main" ] || [ "$GITHUB_REF" == "refs/heads/hh2" ] - then - echo "release commit: tag=latest" - echo "tag=latest" >> "$GITHUB_OUTPUT" - fi - fi + - id: check + uses: ./.github/actions/check-release-commit outputs: - tag: ${{ steps.check_commit.outputs.tag }} - - prepare: - name: Prepare release bundle + # workflow_dispatch is a rehearsal: force the release path with 'next' + # as a placeholder dist-tag (publish adds --dry-run, so it's never used). + tag: ${{ steps.check.outputs.tag || (github.event_name == 'workflow_dispatch' && 'next') || '' }} + is_release: ${{ steps.check.outputs.is-release == 'true' || github.event_name == 'workflow_dispatch' }} + + # The single release/non-release condition in this workflow: every other + # job hangs off this one via `needs`, so a non-release push skips the whole + # tree and nothing else needs a guard. + release_gate: + name: Release gate runs-on: ubuntu-24.04 - permissions: - contents: read - needs: - - build - - test-macOS-windows-binding - - test-linux-x64-gnu-binding - - test-linux-x64-musl-binding - - test-linux-aarch64-gnu-binding - - test-linux-aarch64-musl-binding - defaults: - run: - working-directory: ./crates/edr_napi - outputs: - filename: ${{ steps.pack.outputs.filename }} + needs: check_commit + if: needs.check_commit.outputs.is_release == 'true' steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - persist-credentials: false - - name: Setup node - uses: ./.github/actions/setup-node - - name: Install dependencies - run: pnpm install --frozen-lockfile --prefer-offline - - name: Install sponge # needed for prepublish script - run: | - sudo apt-get update - sudo apt-get install -y moreutils - - name: Download all artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - path: ./crates/edr_napi/artifacts - - name: Run prepublish to update all `package.json` files - # we NEED to to this before pnpm artifacts, if not prepublish will try to publish - run: ../../scripts/prepublish.sh - - name: Move artifacts - # prepublish.sh (above) adds the platform packages to edr_napi's - # package.json but not the lockfile; skip pnpm 11's pre-run frozen - # deps check, which would otherwise fail on that intentional desync. - run: pnpm --config.verify-deps-before-run=false artifacts - - name: Copy coverage library into edr_napi crate - run: cp ../../data/contracts/coverage.sol ./coverage.sol - - name: Compile TypeScript helpers - # See "Move artifacts": package.json is intentionally out of sync with - # the lockfile here, so skip pnpm 11's pre-run frozen deps check. - run: pnpm --config.verify-deps-before-run=false exec tsc - - name: Create bundle with pnpm root & napi files - # create custom tar.gz because pnpm pack only packs root package and we need the platform-specific ones as well - id: pack - run: | - FILENAME="${{ env.APP_NAME }}_bundle.tar.gz" - NAPI_EXTRA_FILES=("npm/" "LICENSE" "README.md" "package.json" "Cargo.toml" "coverage.sol") - readarray -t FILES < <(cat package.json | jq -r .files[] ) - ALL_FILES=("${FILES[@]}" "${NAPI_EXTRA_FILES[@]}") - NAPI_FILES=() - for file in "${ALL_FILES[@]}"; do - NAPI_FILES+=("crates/edr_napi/${file}") - done - cd ../.. # cd to root dir - echo "Creating bundle with necessary files for publishing" - ROOT_FILES=("package.json" "Cargo.lock" "Cargo.toml" "pnpm-lock.yaml" "pnpm-workspace.yaml") - ALL_FILES=("${ROOT_FILES[@]}" "${NAPI_FILES[@]}") - echo "files to include in tar: ${ALL_FILES[*]}" - # Using --dereference since LICENSE file are symlinks - tar --dereference -czvf $FILENAME "${ALL_FILES[@]}" - echo "FILENAME=$FILENAME" - echo "filename=$FILENAME" >> $GITHUB_OUTPUT - - name: Upload packed tarball - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ steps.pack.outputs.filename }} - path: ./${{ steps.pack.outputs.filename }} - retention-days: 2 - overwrite: true - if-no-files-found: error + - name: Confirm release run + env: + TAG: ${{ needs.check_commit.outputs.tag }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' }} + run: echo "Release run confirmed (dist-tag $TAG, dry-run $DRY_RUN)" - review: - name: Review release - runs-on: ubuntu-24.04 + build: + name: Build and test + needs: release_gate + uses: ./.github/workflows/edr-npm-build.yml + with: + release: true permissions: contents: read - needs: - - build - - prepare - steps: - - name: Download packed tarball - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: ${{ needs.prepare.outputs.filename }} - path: . - - name: Decompress tarball - env: - FILENAME: ${{ needs.prepare.outputs.filename }} - run: tar -xvzf "$FILENAME" - - name: Inspect contents - run: tree . - - name: Check number of artifacts - shell: bash - run: | - # get number of artifacts with unique names - # shellcheck disable=SC2011 # .node names are fixed target triples, no special chars - NUMBER_OF_ARTIFACTS=$(ls -1q crates/edr_napi/npm/*/*.node | xargs -n 1 basename | sort | uniq | wc -l) - echo "Number of unique artifacts: $NUMBER_OF_ARTIFACTS" - echo "Expected number of unique artifacts: $NUMBER_OF_TARGETS" - test "$NUMBER_OF_ARTIFACTS" -eq "$NUMBER_OF_TARGETS" - - name: Setup pnpm - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 - - name: Setup node - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 - id: setup-node - with: - node-version: 24 - # A publish workflow must not restore a poisonable cache. - package-manager-cache: false - - name: Validate that packages have everything for publishing - run: | - cd crates/edr_napi - # Check in every cross-platform package - for platform in ./npm/* ; do - cd $platform - pnpm pack --pack-destination "$RUNNER_TEMP" - cd - - done - # And in root edr package - pnpm pack --pack-destination "$RUNNER_TEMP" - - name: Checkout repository for comparison - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - with: - path: remote-repository - persist-credentials: false - ref: ${{ needs.build.outputs.commit_sha }} - - name: Validate there were no modifications to `package.json` when building - run: | - git diff --color=always --exit-code --no-index remote-repository/package.json ./package.json - - name: Validate there were no modifications to `Cargo.toml` when building - run: | - git diff --color=always --exit-code --no-index remote-repository/Cargo.toml ./Cargo.toml - - name: Validate there were no modifications to `Cargo.lock` when building - run: | - git diff --color=always --exit-code --no-index remote-repository/Cargo.lock ./Cargo.lock + packages: read + actions: read # select-node-image lists mirror workflow runs # Extra-defensive cooldown check: `main` branch and PRs already require passing this in CI, # but we re-check here to guarantee we never release with too-fresh dependencies. cooldown-check: name: Cargo cooldown check runs-on: ubuntu-24.04 - needs: check_commit - if: ${{ needs.check_commit.outputs.tag != '' }} + needs: release_gate + permissions: + contents: read steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: @@ -677,8 +93,7 @@ jobs: notify-deploy: name: Notify pre-deploy to Slack runs-on: ubuntu-24.04 - needs: [check_commit, cooldown-check, prepare, review] - if: ${{ needs.check_commit.outputs.tag != '' }} + needs: [cooldown-check, build] steps: - name: Notify pre-deploy uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 @@ -700,7 +115,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "*Commit:* <${{ github.event.head_commit.url || github.event.pull_request.html_url }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" + "text": "*Commit:* <${{ github.event.head_commit.url || format('{0}/{1}/commit/{2}', github.server_url, github.repository, github.sha) }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" } } ] @@ -712,9 +127,7 @@ jobs: runs-on: ubuntu-24.04 permissions: id-token: write # Needed for npm Trusted Publishing (OIDC) - needs: [check_commit, notify-deploy, prepare, review] - # Only run workflow if there is a tag defined - if: ${{ needs.check_commit.outputs.tag != '' }} + needs: [check_commit, notify-deploy, build] steps: - name: Notify deployment to Slack uses: slackapi/slack-github-action@0d95c9a7becc1e6e297d76df9bc735c44f4cbcbc # v3.0.5 @@ -736,7 +149,7 @@ jobs: "type": "section", "text": { "type": "mrkdwn", - "text": "*Commit:* <${{ github.event.head_commit.url || github.event.pull_request.html_url }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" + "text": "*Commit:* <${{ github.event.head_commit.url || format('{0}/{1}/commit/{2}', github.server_url, github.repository, github.sha) }}|View commit>\n*Run:* <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|Open workflow run>" } } ] @@ -744,11 +157,11 @@ jobs: - name: Download packed tarball uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: ${{ needs.prepare.outputs.filename }} + name: ${{ needs.build.outputs.bundle-filename }} path: . - name: Decompress tarball env: - FILENAME: ${{ needs.prepare.outputs.filename }} + FILENAME: ${{ needs.build.outputs.bundle-filename }} run: tar -xvzf "$FILENAME" - name: Setup pnpm uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 @@ -763,6 +176,7 @@ jobs: - name: Publish env: TAG: ${{ needs.check_commit.outputs.tag }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' }} run: | if [ "$TAG" = "next" ] then @@ -775,14 +189,21 @@ jobs: exit 1 fi + FLAGS=(--no-git-checks --tag "$TAG" --access public) + if [ "$DRY_RUN" = "true" ] + then + echo "Rehearsal run: publishing with --dry-run" + FLAGS+=(--dry-run) + fi + # We need to publish the contests of crates/edr_napi only cd ./crates/edr_napi # First publish every cross-platform package for platform in ./npm/* ; do cd $platform - pnpm publish --no-git-checks --tag "$TAG" --access public + pnpm publish "${FLAGS[@]}" cd - done # Finally Publish edr napi root package (because it depends on the cross-platform ones) - pnpm publish --no-git-checks --tag "$TAG" --access public + pnpm publish "${FLAGS[@]}" diff --git a/.github/workflows/mirror-docker-images.yml b/.github/workflows/mirror-docker-images.yml index 659cc445f6..0cce406b9c 100644 --- a/.github/workflows/mirror-docker-images.yml +++ b/.github/workflows/mirror-docker-images.yml @@ -23,7 +23,8 @@ on: paths: - .github/workflows/mirror-docker-images.yml # Same-repo PRs touching this file mirror immediately, so a PR that adds a - # tag below (e.g. a new node version) can be tested before merge. + # tag below (e.g. a new node version) can be tested before merge. The + # edr-npm-build.yml docker jobs wait for this run before pulling. pull_request: paths: - .github/workflows/mirror-docker-images.yml @@ -48,7 +49,7 @@ jobs: run: | set -euo pipefail # Keep in sync with the node versions and container images in - # edr-npm-release.yml; a tag missing here fails CI with "manifest + # edr-npm-build.yml; a tag missing here fails CI with "manifest # unknown" on the mirror pull. TAGS=(22 22-alpine 24 24-alpine 24-bullseye-slim 26 26-alpine) diff --git a/book/src/02_development/11_ci_docker_mirror.md b/book/src/02_development/11_ci_docker_mirror.md index 5593da270e..52149f3abb 100644 --- a/book/src/02_development/11_ci_docker_mirror.md +++ b/book/src/02_development/11_ci_docker_mirror.md @@ -2,13 +2,13 @@ CI pulls its Docker images from a GHCR mirror (`ghcr.io/nomicfoundation/edr/mirror/*`) instead of Docker Hub: Docker Hub rate-limits pulls, which fail intermittently on GitHub-hosted runners as a result. The mirror is maintained by `.github/workflows/mirror-docker-images.yml`, which re-copies the images weekly, on pushes to `main` and same-repo PRs that change the workflow itself, and on demand via `workflow_dispatch`. -Release runs are the exception: when `check_commit` resolves a release tag, the docker jobs in `edr-npm-release.yml` pull the official image straight from Docker Hub (the "Select image source" steps), so the mirror is never in the supply chain of published binaries — a tampered mirror tag can at most affect PR/branch CI, which publishes nothing. At one or two releases a week, Docker Hub's rate limits are not a concern for those runs. +Release runs are the exception: when `edr-npm-build.yml` runs in release mode (its `release` input — see the [release chapter](../03_release.md)), the docker jobs pull the official image straight from Docker Hub (the "Select image source" steps), so the mirror is never in the supply chain of published binaries — a tampered mirror tag can at most affect PR/branch CI, which publishes nothing. At one or two releases a week, Docker Hub's rate limits are not a concern for those runs. ## Adding a Node.js version (or any new tag) -Add the tag to the `TAGS` list in `mirror-docker-images.yml` in the same PR that changes the matrix in `edr-npm-release.yml`. The mirror workflow runs on same-repo PRs that touch it, so the new tag is mirrored — and the release matrix testable against it — before merge. A tag referenced in CI but missing from the mirror fails loudly with `manifest unknown`. +Add the tag to the `TAGS` list in `mirror-docker-images.yml` in the same PR that changes the matrix in `edr-npm-build.yml`. The mirror workflow runs on same-repo PRs that touch it, so the new tag is mirrored — and the release matrix testable against it — before merge. A tag referenced in CI but missing from the mirror fails loudly with `manifest unknown`. -On such a PR the mirror job and the release-workflow jobs start in parallel, so on the first run the release jobs can race ahead and fail their pulls with `manifest unknown`. That's benign: re-run the failed jobs once the mirror job is green. +On such a PR the mirror job and the build-workflow docker jobs start in parallel. The docker jobs' "Select image source" step waits (up to 15 minutes) for the mirror run on the same commit to finish before pulling, so the new tag is in place by the time it's needed. The wait is best-effort and never fails the job — if the mirror run failed or was skipped, the jobs proceed with a warning, and a genuinely missing tag still fails the pull itself with `manifest unknown`. On PRs that don't touch the mirror workflow no mirror run exists and the step doesn't wait. ## Access diff --git a/book/src/03_release.md b/book/src/03_release.md index f87dee8091..308bfd0261 100644 --- a/book/src/03_release.md +++ b/book/src/03_release.md @@ -3,3 +3,15 @@ Releasing the [EDR NPM package](../../crates/edr_napi/package.json) is handled by the [EDR NPM release](../../.github/workflows/edr-npm-release.yml) GitHub Action workflow. A new release is created automatically on commits to the `main` branch that follow the following format: `edr-0.1.0` for releases or `edr-0.1.0-alpha.1` for pre-releases. + +## Workflow structure + +The npm pipeline is split across three workflows: + +- [`edr-npm-build.yml`](../../.github/workflows/edr-npm-build.yml) — reusable (`workflow_call`) build+test+bundle pipeline: the 7-target build matrix, binding tests, bundle preparation, and the pre-publish review. Its single `release` boolean input is the only release/non-release mode switch: release runs build cold (no cargo cache, asserted at runtime) and pull official Docker Hub images instead of the [GHCR mirror](./02_development/11_ci_docker_mirror.md). +- [`edr-npm-ci.yml`](../../.github/workflows/edr-npm-ci.yml) — PR/branch validation: calls the build workflow with `release: false`. Skips itself on release pushes. +- [`edr-npm-release.yml`](../../.github/workflows/edr-npm-release.yml) — detects release commits ([`check-release-commit`](../../.github/actions/check-release-commit/action.yml)), then calls the build workflow with `release: true` and runs the release-only jobs: cargo cooldown check, Slack notifications, and the `edr-release` environment-protected publish. Everything hangs off a single `release_gate` job, so no individual job or step needs its own release condition. + +## Release rehearsal + +Dispatching `edr-npm-release.yml` manually (`workflow_dispatch`) runs the full release path on any commit — cold build, Docker Hub images, cooldown check, environment approval — but publishes with `--dry-run`, so nothing reaches the npm registry. Use this to validate release-path changes without cutting a release. Real publishes happen only on push events.