release.yml

# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

name: Release Pipeline

# workflow_dispatch is capped at 10 inputs by GitHub (this block has 9).
# on_demand/release_version are workflow_call-only: the sole on-demand entry
# point is stage-release-rc.yml. The ops gate and the manual approval gate live
# HERE (authorize/approval jobs), so they bind every caller; stage-release-rc.yml
# repeats only the ops gate, as a fast-fail before its long ECR wait.
on:
  workflow_dispatch:
    inputs:
      commit_sha:
        description: 'Git commit SHA whose post-merge CI images to publish (full 40-char SHA).'
        required: true
        type: string
      rc_number:
        description: 'RC number (e.g., 0 for rc0). Leave empty to auto-increment.'
        required: false
        type: string
      nightly:
        description: 'Publish as nightly (nightly-dated NGC tags; dated pre-release Helm charts; skips RC). Must be run from main.'
        required: false
        type: boolean
        default: false
      skip_gitlab_pipeline:
        description: 'Skip the GitLab release automation pipeline trigger. Emergency use only.'
        required: false
        type: boolean
        default: false
      slack_thread_ts:
        description: 'Slack thread ts to reply into. Empty skips thread replies. Channel ID is read from secrets.SLACK_RELEASE_CHANNEL_ID.'
        required: false
        type: string
        default: ''
      containers:
        description: 'Container subset to publish (all|none|csv of vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,sidecar). Partial rc re-publishes supported.'
        required: false
        type: string
        default: all
      wheels:
        description: 'Wheel subset to stage (all|none|csv of ai-dynamo,ai-dynamo-runtime,kvbm).'
        required: false
        type: string
        default: all
      crates:
        description: 'Crate subset (all|none|csv of crate names, dependency-closed). Only staged on on-demand runs.'
        required: false
        type: string
        default: all
      helm:
        description: 'Helm chart subset to push (all|none|platform).'
        required: false
        type: string
        default: all
  workflow_call:
    inputs:
      commit_sha:
        required: true
        type: string
      rc_number:
        required: false
        type: string
      nightly:
        required: false
        type: boolean
        default: false
      skip_gitlab_pipeline:
        required: false
        type: boolean
        default: false
      slack_thread_ts:
        required: false
        type: string
        default: ''
      on_demand:
        required: false
        type: boolean
        default: false
      release_version:
        required: false
        type: string
        default: ''
      containers:
        required: false
        type: string
        default: all
      wheels:
        required: false
        type: string
        default: all
      crates:
        required: false
        type: string
        default: all
      helm:
        required: false
        type: string
        default: all
      # Nightly-only: the result of THIS nightly run's operator-build /
      # planner-build (needs.<job>.result — success/failure/skipped/cancelled).
      # The copy step layers this on top of the crane probe:
      #   non-empty and != success -> skip. A failed build tonight must never
      #     republish yesterday's image when the resolved SHA is unchanged
      #     (quiet-repo day) — SHA-keyed tags alone can't tell which run
      #     built them.
      #   empty (direct workflow_dispatch — no builds in this run) -> the
      #     probe alone decides, matching the runtimes: a manual nightly
      #     re-stage copies whatever nightly-built -nightly ECR tags exist
      #     for the SHA.
      operator_build_result:
        required: false
        type: string
        default: ''
      planner_build_result:
        required: false
        type: string
        default: ''
      frontend_build_result:
        required: false
        type: string
        default: ''
      vllm_efa_build_result:
        required: false
        type: string
        default: ''
      sglang_efa_build_result:
        required: false
        type: string
        default: ''
      trtllm_efa_build_result:
        required: false
        type: string
        default: ''
    outputs:
      # The exact NGC docker tag published for this run (nightly: <date>-<sha7>).
      # Informational for callers — recomputing the date can drift across a
      # UTC-midnight boundary; no current caller consumes it.
      ngc_version_tag:
        description: 'NGC docker version tag published by this run.'
        value: ${{ jobs.prepare-release.outputs.ngc_version_tag }}

# Note: workflow_dispatch can only be triggered from release/* branches (or main when nightly=true)
# This is enforced in the prepare-release job via branch validation. workflow_call paths
# (e.g. nightly-ci.yml) inherit the caller's ref — the same branch rules apply.

permissions:
  contents: read

env:
  ECR_REGISTRY: ${{ vars.ECR_REGISTRY }}
  ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
  AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}

jobs:
  # ============================================================================
  # GATE: Version Extraction
  # ============================================================================

  # OPS GATE — every run requires an active ai-dynamo/devops member, except the
  # unattended nightly (nightly=true AND a scheduled trigger). A manual nightly
  # (dispatch of this workflow or of nightly-ci.yml) is gated like any other run.
  # Lives here (not only in the callers) so it binds EVERY caller of this
  # reusable workflow — including a rogue workflow that calls it directly.
  #
  # The two steps are YAML anchors (&ops-token / &ops-check), repeated as the
  # first steps of EVERY job that reads automated-release secrets. "Re-run failed
  # jobs" reuses this job's earlier success, so a job-local check is the only
  # thing that re-verifies whoever triggered the re-run (github.triggering_actor).
  authorize:
    name: Ops gate
    runs-on: prod-default-small-v2
    environment: automated-release
    permissions:
      contents: read
    steps:
      - &ops-token
        name: Mint app token (ops gate)
        if: ${{ !(inputs.nightly && github.event_name == 'schedule') }}
        id: ops-token
        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
        with:
          app-id: ${{ secrets.RELEASE_APP_ID }}
          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
      - &ops-check
        name: Require release-ops team membership
        if: ${{ !(inputs.nightly && github.event_name == 'schedule') }}
        env:
          GH_TOKEN: ${{ steps.ops-token.outputs.token }}
          ORG: ${{ github.repository_owner }}
          TEAM: devops
          ACTOR: ${{ github.actor }}
          TRIGGERING_ACTOR: ${{ github.triggering_actor }}
          # The ORIGINAL trigger (the caller's, for a reusable workflow; unchanged
          # on re-runs). A push/PR-triggered run must never reach release secrets,
          # so a future test hook with a push trigger fails here instead.
          EVENT: ${{ github.event_name }}
        run: |
          set -euo pipefail
          case "${EVENT}" in
            workflow_dispatch|schedule) ;;
            *) echo "::error::The release pipeline only runs from a manual dispatch or the nightly schedule (got '${EVENT}')."; exit 1 ;;
          esac
          for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
            HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
              -H "Authorization: Bearer ${GH_TOKEN}" \
              -H "Accept: application/vnd.github+json" \
              "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
            # grep, not python3/jq: this step runs on every secret-using job's
            # runner, including the gitlab_ci_runners group.
            if [ "${HTTP}" != "200" ] || ! grep -Eq '"state"[[:space:]]*:[[:space:]]*"active"' "${RUNNER_TEMP}/membership.json"; then
              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this release job."
              exit 1
            fi
            echo "@${USER} authorized via ${ORG}/${TEAM}"
          done
      - name: Gate mode
        run: echo "ops gate applied=${{ !(inputs.nightly && github.event_name == 'schedule') }} (only scheduled nightly is exempt)"

  # APPROVAL GATE — operator runs also require a manual-release-approval reviewer.
  # Dynamic environment: empty (no gate) on the nightly path so unattended runs are
  # not blocked. Enforced by the environment's required-reviewers rule; no secrets
  # here, so its environment scope is independent of the RELEASE_APP secrets.
  approval:
    name: Approval gate
    needs: [authorize]
    runs-on: prod-default-small-v2
    environment: ${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && 'manual-release-approval' || '' }}
    permissions:
      contents: read
    steps:
      - name: Record approval
        env:
          SHA: ${{ inputs.commit_sha }}
          VER: ${{ inputs.release_version }}
          RC: ${{ inputs.rc_number }}
          ACTOR: ${{ github.actor }}
        run: echo "Release gate cleared for ${VER:-<branch-derived>} rc.${RC} @ ${SHA} by ${ACTOR}"

  prepare-release:
    name: Prepare Release
    needs: [approval]
    runs-on: prod-default-small-v2
    outputs:
      version: ${{ steps.extract.outputs.version }}
      commit_sha: ${{ steps.extract.outputs.commit_sha }}
      nightly: ${{ steps.extract.outputs.nightly }}
      on_demand: ${{ steps.extract.outputs.on_demand }}
      rc_number: ${{ steps.rc_tag.outputs.rc_number }}
      ngc_version_tag: ${{ steps.rc_tag.outputs.ngc_version_tag }}
      helm_chart_version: ${{ steps.rc_tag.outputs.helm_chart_version }}
      wheel_version: ${{ steps.rc_tag.outputs.wheel_version }}
      crate_version: ${{ steps.rc_tag.outputs.crate_version }}
    steps:
      - name: Extract version and validate inputs
        id: extract
        env:
          COMMIT_SHA: ${{ inputs.commit_sha }}
          BRANCH_NAME: ${{ github.ref_name }}
          NIGHTLY: ${{ inputs.nightly }}
          ON_DEMAND: ${{ inputs.on_demand }}
          RELEASE_VERSION: ${{ inputs.release_version }}
          CONTAINER_SUBSET: ${{ inputs.containers }}
          WHEEL_SUBSET: ${{ inputs.wheels }}
          CRATE_SUBSET: ${{ inputs.crates }}
          HELM_SUBSET: ${{ inputs.helm }}
          SLACK_THREAD_TS: ${{ inputs.slack_thread_ts }}
        run: |
          set -euo pipefail

          if ! [[ "${COMMIT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then
            echo "Error: commit_sha must be a full 40-character hex SHA (got: '${COMMIT_SHA}')"
            exit 1
          fi

          # A typo'd subset token would be silently skipped by every want()
          # filter and stage nothing — fail fast instead. Crate names are
          # graph-checked by stage_crates.py; only the charset is gated here.
          validate_subset() {
            local LABEL="$1" SPEC="$2" UNIVERSE="$3" TOK
            if [ "${SPEC}" = "all" ] || [ "${SPEC}" = "none" ] || [ -z "${SPEC}" ]; then
              return 0
            fi
            IFS=',' read -ra TOKS <<<"${SPEC}"
            for TOK in "${TOKS[@]}"; do
              case ",${UNIVERSE}," in
                *,"${TOK}",*) ;;
                *) echo "Error: unknown ${LABEL} token '${TOK}' (valid: ${UNIVERSE})"; exit 1 ;;
              esac
            done
          }
          validate_subset containers "${CONTAINER_SUBSET:-all}" "vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,sidecar"
          validate_subset wheels "${WHEEL_SUBSET:-all}" "ai-dynamo,ai-dynamo-runtime,kvbm"
          validate_subset helm "${HELM_SUBSET:-all}" "platform"
          if [[ ! "${CRATE_SUBSET:-all}" =~ ^[A-Za-z0-9_,-]*$ ]]; then
            echo "Error: crate subset may only contain [A-Za-z0-9_-] names separated by commas (got '${CRATE_SUBSET}')"
            exit 1
          fi
          if [[ -n "${SLACK_THREAD_TS}" && ! "${SLACK_THREAD_TS}" =~ ^[0-9]+\.[0-9]+$ ]]; then
            echo "Error: slack_thread_ts must look like a Slack ts '1234567890.123456' (got '${SLACK_THREAD_TS}')"
            exit 1
          fi

          if [[ "${ON_DEMAND}" == "true" ]]; then
            # Branch + version bump are created/validated by stage-release-rc.yml;
            # the version is passed explicitly rather than derived from the ref.
            if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
              echo "Error: on_demand=true requires release_version X.Y.Z[.devN|.postN] (got: '${RELEASE_VERSION}')"
              exit 1
            fi
            # SemVer has no post-release form: .postN crates/helm would sort
            # BEFORE X.Y.Z or collide with it on crates.io. Containers/wheels are fine.
            if [[ "${RELEASE_VERSION}" =~ \.post[0-9]+$ ]]; then
              if [ "${CRATE_SUBSET:-all}" != "none" ] || [ "${HELM_SUBSET:-all}" != "none" ]; then
                echo "Error: post releases (X.Y.Z.postN) cannot ship crates or helm charts; pass crates=none and helm=none"
                exit 1
              fi
            fi
            VERSION="${RELEASE_VERSION}"
          elif [[ "${NIGHTLY}" == "true" ]]; then
            if [[ "${BRANCH_NAME}" != "main" ]]; then
              echo "Error: nightly=true requires branch=main (got: ${BRANCH_NAME})"
              exit 1
            fi
            VERSION="nightly"
          else
            # Fully anchored — a prefix-only match would accept arbitrary trailing text.
            if [[ ! "$BRANCH_NAME" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
              echo "Error: workflow_dispatch must be triggered from a release/X.Y.Z[.devN|.postN] branch (or main with nightly=true)"
              echo "Current branch: $BRANCH_NAME"
              echo "Named releases are not dispatchable here — run stage-release-rc.yml with a BARE v<X.Y.Z>[-<name>] release_branch (a release/X.Y.Z-<name> branch is treated as a throwaway test branch there too)."
              exit 1
            fi
            VERSION="${BRANCH_NAME#release/}"
          fi

          echo "version=${VERSION}" >> $GITHUB_OUTPUT
          echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
          echo "nightly=${NIGHTLY}" >> $GITHUB_OUTPUT
          echo "on_demand=${ON_DEMAND}" >> $GITHUB_OUTPUT
          echo "Detected version: ${VERSION}"
          echo "Source commit SHA: ${COMMIT_SHA}"
          echo "Nightly: ${NIGHTLY}"

      # Checkout at the source commit with full tags so the RC-number step can
      # enumerate existing v<VERSION>-rc<N> tags and auto-increment. Also lets
      # downstream jobs (release-publish, stage-wheels-artifactory) consume the
      # computed rc_number/ngc_version_tag as needs.prepare-release.outputs.*
      # and run in parallel.
      - name: Checkout at source commit
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ steps.extract.outputs.commit_sha }}
          fetch-depth: 0
          fetch-tags: true

      - name: Determine next RC number
        id: rc_tag
        env:
          INPUT_RC_NUMBER: ${{ inputs.rc_number }}
          VERSION: ${{ steps.extract.outputs.version }}
          COMMIT_SHA: ${{ steps.extract.outputs.commit_sha }}
          NIGHTLY: ${{ steps.extract.outputs.nightly }}
          ON_DEMAND: ${{ steps.extract.outputs.on_demand }}
        run: |
          set -euo pipefail

          # Base version pinned in pyproject.toml at the source SHA. Same value
          # cargo and maturin stamp into the wheel filename / METADATA. The
          # optional .devN (pre-release) / .postN (post-release) tail covers dev
          # builds and post-release respins.
          BASE_VERSION=$(awk '/^version = / {gsub(/"/, "", $3); print $3; exit}' pyproject.toml)
          if [[ ! "${BASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
            echo "Error: cannot parse base version from pyproject.toml (got '${BASE_VERSION}')"
            exit 1
          fi
          # rc dispatch derives VERSION from the branch name but stamps wheels from
          # the tree at COMMIT_SHA — a mismatch ships wheels versioned differently
          # from the image tags (e.g. a SHA pasted from the wrong branch, or a
          # branch cut whose version bump hasn't landed yet).
          if [[ "${NIGHTLY}" != "true" && "${BASE_VERSION}" != "${VERSION}" ]]; then
            echo "Error: pyproject version '${BASE_VERSION}' at ${COMMIT_SHA:0:12} != release version '${VERSION}' — is the SHA on this release branch (post version-bump)?"
            exit 1
          fi

          if [[ "${NIGHTLY}" == "true" ]]; then
            DATE_UTC=$(date -u +%Y%m%d)
            SHORT_SHA="${COMMIT_SHA:0:7}"
            # No `nightly-` prefix on the tag: the NGC repo name itself carries
            # the `-nightly` suffix (e.g. vllm-runtime-nightly), so the tag stays
            # the bare date+sha (e.g. 20260520-abc1234).
            NGC_VERSION_TAG="${DATE_UTC}-${SHORT_SHA}"
            WHEEL_VERSION="${BASE_VERSION}.dev${DATE_UTC}"
            # SemVer pre-release: sorts below the train's rc and GA. The sha
            # keeps same-day re-runs unique on immutable ChartMuseum; its `g`
            # prefix keeps an all-digit sha from being invalid SemVer.
            HELM_CHART_VERSION="${BASE_VERSION}-dev.${DATE_UTC}.g${SHORT_SHA}"
            # Crate version: dotted SemVer pre-release identifiers so they sort
            # numerically (dev.20260828 < dev.20260901; an undotted -devYYYYMMDD
            # is one TEXT identifier and sorts wrong against any other shape,
            # e.g. -dev9 > -dev20260828). Date-only, matching the wheel's
            # .devYYYYMMDD: a same-day re-run at a new SHA is idempotently
            # SKIPPED by the immutable registry — the day's first staging wins.
            SEMVER_BASE=$(echo "${BASE_VERSION}" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+)\.(.+)/\1-\2/')
            CRATE_VERSION_NIGHTLY="${SEMVER_BASE}-dev.${DATE_UTC}"
            echo "rc_number=" >> $GITHUB_OUTPUT
            echo "ngc_version_tag=${NGC_VERSION_TAG}" >> $GITHUB_OUTPUT
            echo "helm_chart_version=${HELM_CHART_VERSION}" >> $GITHUB_OUTPUT
            echo "wheel_version=${WHEEL_VERSION}" >> $GITHUB_OUTPUT
            echo "crate_version=${CRATE_VERSION_NIGHTLY}" >> $GITHUB_OUTPUT
            echo "Nightly NGC tag: ${NGC_VERSION_TAG}"
            echo "Nightly wheel version: ${WHEEL_VERSION}"
            echo "Nightly Helm chart version: ${HELM_CHART_VERSION}"
            echo "Nightly crate version: ${CRATE_VERSION_NIGHTLY}"
            exit 0
          fi

          if [[ "${ON_DEMAND}" == "true" ]]; then
            # Internal staging carries an rc disambiguator: the crate/helm registries
            # are immutable, so each re-stage of the same version needs a distinct
            # internal version. GA stays clean — the GitLab trigger forwards
            # RELEASE_VERSION=${VERSION} + RC_NUMBER and promotes the rc-staged
            # artifacts to the clean version.
            #   NGC tag (docker string): ${VERSION}-rcN     (e.g. 0.1.0-rc0, 0.1.0.dev3-rc0)
            #   helm + crate (SemVer):   ${SEMVER_BASE}-rcN  where .devN/.postN become a
            #                            -devN/-postN pre-release (e.g. 0.1.0-rc0, 0.1.0-post3-rc0)
            #   wheel: clean version, disambiguated by the rcN upload subpath.
            # Passed through from stage-release-rc.yml's rc_number INPUT (default rc0;
            # NO GitHub tag is created). Require it (don't default to 0) so a broken
            # hand-off fails loud instead of silently colliding on rc0.
            RC_NUM="${INPUT_RC_NUMBER}"
            # RC_NUM is <round> or <round>.<sub> (dotted sub-increment). Both form a
            # valid, numerically-ordered SemVer pre-release when emitted as ...-rc.<RC_NUM>
            # (e.g. 1.3.0-rc.2 or 1.3.0-rc.2.1). The wheel subpath uses the glued rc<RC_NUM>
            # (a path segment, not a version) — no SemVer/PEP440 constraint there.
            if ! [[ "${RC_NUM}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
              echo "Error: on_demand requires rc_number as <round> or <round>.<sub> (got '${RC_NUM}')"
              exit 1
            fi
            # Tree at COMMIT_SHA must be stamped with the requested version, or
            # images/charts get tagged VERSION while their wheels carry BASE_VERSION.
            if [ "${BASE_VERSION}" != "${VERSION}" ]; then
              echo "Error: release_version '${VERSION}' does not match pyproject.toml version '${BASE_VERSION}' at ${COMMIT_SHA} (wrong commit_sha, or the version bump never landed)"
              exit 1
            fi
            SEMVER_BASE=$(echo "${VERSION}" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+)\.(.+)/\1-\2/')
            echo "rc_number=${RC_NUM}" >> $GITHUB_OUTPUT
            echo "ngc_version_tag=${VERSION}-rc.${RC_NUM}" >> $GITHUB_OUTPUT
            echo "helm_chart_version=${SEMVER_BASE}-rc.${RC_NUM}" >> $GITHUB_OUTPUT
            echo "wheel_version=${BASE_VERSION}" >> $GITHUB_OUTPUT
            echo "crate_version=${SEMVER_BASE}-rc.${RC_NUM}" >> $GITHUB_OUTPUT
            echo "On-demand internal staging (rc.${RC_NUM}): NGC ${VERSION}-rc.${RC_NUM}, helm ${SEMVER_BASE}-rc.${RC_NUM}, wheel ${BASE_VERSION} @ rc${RC_NUM}, crate ${SEMVER_BASE}-rc.${RC_NUM}"
            exit 0
          fi

          if [ -n "${INPUT_RC_NUMBER}" ]; then
            if ! [[ "${INPUT_RC_NUMBER}" =~ ^[0-9]+$ ]]; then
              echo "Error: rc_number must be a non-negative integer (got: ${INPUT_RC_NUMBER})"
              exit 1
            fi
            NEXT_RC="${INPUT_RC_NUMBER}"
            echo "Using provided RC number: ${NEXT_RC}"
          else
            echo "No RC number provided. Auto-incrementing..."
            RC_PATTERN="v${VERSION}-rc"
            EXISTING_RCS=$(git tag -l "${RC_PATTERN}*" | grep -E "^v${VERSION}-rc[0-9]+$" | sort -V || true)

            if [ -z "$EXISTING_RCS" ]; then
              NEXT_RC=0
              echo "No existing RC tags found. Starting with rc0."
            else
              LAST_RC=$(echo "$EXISTING_RCS" | tail -1)
              LAST_RC_NUM=${LAST_RC#v${VERSION}-rc}
              NEXT_RC=$((LAST_RC_NUM + 1))
              echo "Found existing RC tags:"
              echo "$EXISTING_RCS"
              echo "Last RC: ${LAST_RC}, Next RC number: ${NEXT_RC}"
            fi
          fi

          # Normalize version for Helm SemVer: X.Y.Z.suffix → X.Y.Z-suffix
          SEMVER_VERSION=$(echo "${VERSION}" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+)\.(.+)/\1-\2/')
          HELM_CHART_VERSION="${SEMVER_VERSION}-rc${NEXT_RC}"

          WHEEL_VERSION="${BASE_VERSION}"

          echo "rc_number=${NEXT_RC}" >> $GITHUB_OUTPUT
          echo "ngc_version_tag=${VERSION}rc${NEXT_RC}" >> $GITHUB_OUTPUT
          echo "helm_chart_version=${HELM_CHART_VERSION}" >> $GITHUB_OUTPUT
          echo "wheel_version=${WHEEL_VERSION}" >> $GITHUB_OUTPUT
          echo "RC number: ${NEXT_RC}"
          echo "Helm chart version: ${HELM_CHART_VERSION}"
          echo "RC wheel version: ${WHEEL_VERSION}"

  # ============================================================================
  # NGC PUBLISH: crane copy to NGC, Helm chart push
  # Sources images from ECR using SHA-based tags produced by post-merge CI.
  # ============================================================================

  release-publish:
    name: Publish to NGC
    needs: [prepare-release]
    # Skip only when neither containers nor helm are selected; nightly/rc default
    # to 'all' for both, so they always run unless explicitly narrowed.
    if: ${{ needs.prepare-release.result == 'success' && !(inputs.containers == 'none' && inputs.helm == 'none') }}
    runs-on: prod-builder-amd-v1 # TODO: needs to identify the correct runner here, definitely should not be prod-builder-amd-v1
    environment: automated-release
    outputs:
      # Comma-separated NGC image refs (repo:tag, no registry prefix) actually copied
      # to NGC — primary version tags + -efa variants only (the -cuda13 digest aliases
      # and floating :nightly tags are dropped: same image, not worth a second scan).
      # Forwarded to GitLab as STAGED_CONTAINERS for nspect registration.
      staged_containers: ${{ steps.copy_images.outputs.staged_containers }}
      # Comma-separated chart:version pairs actually pushed to the NGC helm
      # registry — helm_push covers rc/on-demand runs, nightly_helm_push covers
      # nightly (the two steps are mutually exclusive on env.NIGHTLY, so exactly
      # one contributes). Forwarded to GitLab for registration at the true
      # chart version.
      staged_helm: ${{ steps.helm_push.outputs.staged_helm || steps.nightly_helm_push.outputs.staged_helm }}
      failed_copy_count: ${{ steps.copy_images.outputs.failed_count }}
    env:
      VERSION: ${{ needs.prepare-release.outputs.version }}
      COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
      NIGHTLY: ${{ needs.prepare-release.outputs.nightly }}
      ON_DEMAND: ${{ needs.prepare-release.outputs.on_demand }}
      CONTAINER_SUBSET: ${{ inputs.containers }}
      HELM_SUBSET: ${{ inputs.helm }}
      OPERATOR_BUILD_RESULT: ${{ inputs.operator_build_result }}
      PLANNER_BUILD_RESULT: ${{ inputs.planner_build_result }}
      FRONTEND_BUILD_RESULT: ${{ inputs.frontend_build_result }}
      VLLM_EFA_BUILD_RESULT: ${{ inputs.vllm_efa_build_result }}
      SGLANG_EFA_BUILD_RESULT: ${{ inputs.sglang_efa_build_result }}
      TRTLLM_EFA_BUILD_RESULT: ${{ inputs.trtllm_efa_build_result }}
    steps:
      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
      - *ops-token
      - *ops-check
      - name: Checkout at source commit
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ needs.prepare-release.outputs.commit_sha }}

      - name: Setup crane
        env:
          CRANE_VERSION: v0.20.2
        run: |
          curl --retry 5 --retry-delay 2 \
            --connect-timeout 10 --max-time 120 \
            --fail --show-error -sL \
            "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" \
            | tar -xzf - crane
          sudo mv crane /usr/local/bin/
          crane version

      - name: Login to ECR
        run: |
          aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | docker login --username AWS --password-stdin "${ECR_REGISTRY}"

      - name: Login to NGC
        env:
          NGC_TOKEN: ${{ secrets.NGC_PUBLISH_TOKEN }}
          NGC_USERNAME: ${{ secrets.NGC_PUBLISH_USERNAME }}
        run: |
          echo "${NGC_TOKEN}" | docker login nvcr.io -u "${NGC_USERNAME}" --password-stdin
          echo "${NGC_TOKEN}" | crane auth login nvcr.io -u "${NGC_USERNAME}" --password-stdin

      - name: Inventory selected container images in ECR
        run: |
          set -uo pipefail
          # Report-only inventory of which selected container images are in ECR. Never fails.
          # Mirrors the copy step's subset gating, nightly rules, and source tags.
          NIGHTLY_TAG_SUFFIX=""
          [ "${NIGHTLY}" = "true" ] && NIGHTLY_TAG_SUFFIX="-nightly"

          SUBSET="${CONTAINER_SUBSET:-all}"
          want() {
            [ "${SUBSET}" = "all" ] && return 0
            case ",${SUBSET}," in *,"$1",*) return 0 ;; esac
            return 1
          }

          # token|ECR source tag — must match the "Copy images to NGC" sources.
          ENTRIES=(
            "vllm-runtime|${COMMIT_SHA}-vllm-runtime${NIGHTLY_TAG_SUFFIX}"
            "sglang-runtime|${COMMIT_SHA}-sglang-runtime${NIGHTLY_TAG_SUFFIX}"
            "trtllm-runtime|${COMMIT_SHA}-trtllm-runtime${NIGHTLY_TAG_SUFFIX}"
          )
          if [ "${NIGHTLY}" != "true" ]; then
            ENTRIES+=(
              "vllm-efa|${COMMIT_SHA}-vllm-runtime-efa"
              "sglang-efa|${COMMIT_SHA}-sglang-runtime-efa"
              "trtllm-efa|${COMMIT_SHA}-trtllm-runtime-efa"
              "frontend|${COMMIT_SHA}-dynamo-frontend"
              "operator|${COMMIT_SHA}-operator"
              "planner|${COMMIT_SHA}-dynamo-planner"
              "sidecar|${COMMIT_SHA}-dynamo-sidecar"
            )
          fi

          {
            echo "### Container image availability (ECR)"
            echo ""
            echo "Selection: \`${SUBSET}\`"
            echo ""
            echo "| Token | ECR tag | Status |"
            echo "|-------|---------|--------|"
          } >> "${GITHUB_STEP_SUMMARY}"

          avail=0; missing=0; selected=0
          echo "=== Container image availability (selection: ${SUBSET}) ==="
          for E in "${ENTRIES[@]}"; do
            IFS='|' read -r TOKEN TAG <<<"${E}"
            want "${TOKEN}" || continue
            selected=$((selected+1))
            REF="${ECR_REGISTRY}/${ECR_REPOSITORY}:${TAG}"
            if crane manifest "${REF}" >/dev/null 2>&1; then
              echo "  available: ${TOKEN} (${TAG})"
              echo "| ${TOKEN} | \`${TAG}\` | available |" >> "${GITHUB_STEP_SUMMARY}"
              avail=$((avail+1))
            else
              echo "  MISSING:   ${TOKEN} (${TAG})"
              echo "| ${TOKEN} | \`${TAG}\` | **missing** |" >> "${GITHUB_STEP_SUMMARY}"
              missing=$((missing+1))
            fi
          done

          if [ "${selected}" -eq 0 ]; then
            echo "No container images selected (subset='${SUBSET}')."
            echo "_No container images selected._" >> "${GITHUB_STEP_SUMMARY}"
          fi
          echo "Summary: ${avail} available, ${missing} missing (of ${selected} selected)."
          {
            echo ""
            echo "**${avail} available, ${missing} missing** (of ${selected} selected)."
          } >> "${GITHUB_STEP_SUMMARY}"

      - name: Copy images to NGC
        id: copy_images
        env:
          NGC_REGISTRY: nvcr.io
          NGC_ORG: ${{ secrets.NGC_PUBLISH_ORG }}
          NGC_VERSION_TAG: ${{ needs.prepare-release.outputs.ngc_version_tag }}
        run: |
          set -euo pipefail

          SUCCESSFUL_COPIES=()
          FAILED_COPIES=()

          # Nightly builds push ECR tags with a `-nightly` suffix so a missing
          # or failed nightly build can't fall through to the post-merge image
          # at the same SHA (which carries stable, non-dev wheel versions).
          NIGHTLY_TAG_SUFFIX=""
          if [ "${NIGHTLY}" = "true" ]; then
            NIGHTLY_TAG_SUFFIX="-nightly"
          fi

          echo "========================================"
          echo "Copying images from ECR to NGC (registry-to-registry)"
          echo "Source commit SHA: ${COMMIT_SHA}"
          echo "NGC Version Tag: ${NGC_VERSION_TAG}"
          echo "Nightly tag suffix: '${NIGHTLY_TAG_SUFFIX}'"
          echo "========================================"

          copy_image() {
            local SRC="$1" DST="$2" LABEL="$3"
            echo "----------------------------------------"
            echo "Copying: ${LABEL}"
            # crane copy preserves multi-arch manifest lists by default (no --platform needed).
            # Returns the real copy status — stage_optional() gates its *_STAGED
            # flags and floating-tag pushes on it. Under `set -e`, BARE call
            # sites must therefore carry `|| true` (a nonzero return would abort
            # the whole copy loop on the first failed image); condition-context
            # calls (if/elif) are exempt by shell rules.
            if crane copy "${SRC}" "${DST}"; then
              echo "  Copied: ${LABEL}"
              SUCCESSFUL_COPIES+=("${LABEL}")
              return 0
            else
              echo "  Warning: Failed to copy ${LABEL}, skipping..."
              FAILED_COPIES+=("${LABEL}")
              return 1
            fi
          }

          # On nightly, segregate published images into their own NGC repos by
          # appending `-nightly` to the repo name (e.g. vllm-runtime-nightly).
          # This keeps consumers from accidentally pulling a nightly image from
          # the stable repo.
          NGC_NAME_SUFFIX=""
          if [[ "${NIGHTLY}" == "true" ]]; then
            NGC_NAME_SUFFIX="-nightly"
          fi

          # Copy only the selected container subset; nightly/rc default to the full
          # set (SUBSET=all). Tokens: <fw>-runtime (bare + -cuda13 alias),
          # <fw>-efa, frontend, operator, planner.
          SUBSET="${CONTAINER_SUBSET:-all}"
          want() {
            [ "${SUBSET}" = "all" ] && return 0
            case ",${SUBSET}," in *,"$1",*) return 0 ;; esac
            return 1
          }

          # ---- CUDA 13 runtime images (vllm, sglang, trtllm) — DEFAULT ----
          echo ""
          echo "=== CUDA 13 Runtime Images (vllm, sglang, trtllm) — default ==="
          CUDA13_FRAMEWORKS=("vllm" "sglang" "trtllm")
          for FRAMEWORK in "${CUDA13_FRAMEWORKS[@]}"; do
            want "${FRAMEWORK}-runtime" || { echo "skip ${FRAMEWORK}-runtime (not selected)"; continue; }
            if [ "${FRAMEWORK}" = "trtllm" ]; then
              NGC_NAME="tensorrtllm-runtime${NGC_NAME_SUFFIX}"
              STABLE_NAME="tensorrtllm-runtime"
            else
              NGC_NAME="${FRAMEWORK}-runtime${NGC_NAME_SUFFIX}"
              STABLE_NAME="${FRAMEWORK}-runtime"
            fi

            SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-${FRAMEWORK}-runtime${NIGHTLY_TAG_SUFFIX}"
            TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${NGC_NAME}:${NGC_VERSION_TAG}"
            copy_image "${SOURCE}" "${TARGET}" "${NGC_NAME}:${NGC_VERSION_TAG}" || true

            # Explicit `-cuda13` alias on the same digest so consumers can
            # pin to cuda13 specifically. The bare tag above stays as the
            # implicit default and will move to cuda14+ when that flips.
            EXPLICIT_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${NGC_NAME}:${NGC_VERSION_TAG}-cuda13"
            copy_image "${SOURCE}" "${EXPLICIT_TARGET}" "${NGC_NAME}:${NGC_VERSION_TAG}-cuda13" || true

            # Mutable `:nightly` floating tag on the stable repo (e.g.
            # vllm-runtime:nightly). cuda13 is the default so no cuda suffix.
            if [[ "${NIGHTLY}" == "true" ]]; then
              MUTABLE_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${STABLE_NAME}:nightly"
              copy_image "${SOURCE}" "${MUTABLE_TARGET}" "${STABLE_NAME}:nightly" || true

              # Explicit `:nightly-cuda13` floating alias.
              EXPLICIT_MUTABLE_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${STABLE_NAME}:nightly-cuda13"
              copy_image "${SOURCE}" "${EXPLICIT_MUTABLE_TARGET}" "${STABLE_NAME}:nightly-cuda13" || true
            fi
          done

          # Nightly builds the runtime images above (vllm/sglang/trtllm
          # cuda13) plus operator and planner (operator-build / planner-build
          # in nightly-ci.yml — built there, not reused from post-merge, so
          # their compliance-<sha>-<token> artifacts land on the nightly run
          # id that GitLab's OSRB attach resolves, and their -nightly ECR
          # tags can't fall through to a post-merge image).
          # EFA/frontend are built by post-merge and other workflows
          # on release branches; for nightly we skip them so we don't try to
          # copy ECR tags that don't exist for the resolved SHA.
          if [[ "${NIGHTLY}" != "true" ]]; then
            # ---- EFA runtime images ----
            # All EFA runtime images are multi-arch (amd64+arm64).
            # crane copy preserves multi-arch manifests automatically.
            echo ""
            echo "=== EFA Runtime Images ==="

            # vllm EFA (CUDA 13 — default; no cuda suffix on ECR or NGC)
            if want vllm-efa; then
              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-vllm-runtime-efa"
              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/vllm-runtime:${NGC_VERSION_TAG}-efa"
              copy_image "${SOURCE}" "${TARGET}" "vllm-runtime:${NGC_VERSION_TAG}-efa" || true
            fi

            # sglang EFA (CUDA 13 — default; no cuda suffix on ECR or NGC)
            if want sglang-efa; then
              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-sglang-runtime-efa"
              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/sglang-runtime:${NGC_VERSION_TAG}-efa"
              copy_image "${SOURCE}" "${TARGET}" "sglang-runtime:${NGC_VERSION_TAG}-efa" || true
            fi

            # trtllm EFA (CUDA 13 — default; no cuda suffix on ECR or NGC)
            if want trtllm-efa; then
              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-trtllm-runtime-efa"
              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/tensorrtllm-runtime:${NGC_VERSION_TAG}-efa"
              copy_image "${SOURCE}" "${TARGET}" "tensorrtllm-runtime:${NGC_VERSION_TAG}-efa" || true
            fi

            # ---- Frontend image (already multi-arch from build-frontend-image workflow) ----
            if want frontend; then
              echo ""
              echo "=== Frontend Image ==="
              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-frontend"
              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-frontend:${NGC_VERSION_TAG}"
              copy_image "${SOURCE}" "${TARGET}" "dynamo-frontend:${NGC_VERSION_TAG}" || true
            fi

            # ---- Operator image (multi-arch manifest already built by post-merge operator-build) ----
            if want operator; then
              echo ""
              echo "=== Operator Image ==="
              OPERATOR_SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-operator"
              OPERATOR_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/kubernetes-operator:${NGC_VERSION_TAG}"
              copy_image "${OPERATOR_SOURCE}" "${OPERATOR_TARGET}" "kubernetes-operator:${NGC_VERSION_TAG}" || true
            fi

            # ---- Planner image (CPU-only, multi-arch from post-merge planner-pipeline) ----
            if want planner; then
              echo ""
              echo "=== Planner Image ==="
              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-planner"
              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-planner:${NGC_VERSION_TAG}"
              copy_image "${SOURCE}" "${TARGET}" "dynamo-planner:${NGC_VERSION_TAG}" || true
            fi

            # ---- Sidecar image (unified, multi-arch; built by post-merge's
            # sidecar-build as <sha>-dynamo-sidecar) ----
            if want sidecar; then
              echo ""
              echo "=== Sidecar Image ==="
              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-sidecar"
              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-sidecar:${NGC_VERSION_TAG}"
              copy_image "${SOURCE}" "${TARGET}" "dynamo-sidecar:${NGC_VERSION_TAG}" || true
            fi
          else
            # Optional nightly images — operator, planner, frontend and the
            # three runtime EFA variants — are built by THIS
            # nightly run (ordering-only needs of the release, deliberately
            # outside its success gate) and staged fail-soft: a build flake
            # must warn-skip here, not fail (a bare copy_image failure under
            # set -e would abort the step and skip the GitLab trigger for the
            # whole nightly).
            #
            # stage_optional gates each image in three layers:
            #   1. Caller-forwarded build result — non-success skips: SHA-keyed
            #      tags alone can't tell tonight's build from yesterday's when
            #      the resolved SHA is unchanged (quiet-repo day), and a failed
            #      build must never republish a previous run's image. An EMPTY
            #      result means a direct workflow_dispatch (no builds in this
            #      run): the probe alone decides, matching the runtimes — a
            #      manual re-stage copies whatever nightly-built tags exist.
            #   2. crane-manifest probe — push-integrity check on the ECR tag.
            #   3. copy_image — a failure with the source PRESENT points at
            #      the NGC target (write access / repo config).
            # On success it also repushes the image's mutable floating tag on
            # the stable repo (parity with the runtimes' :nightly aliases);
            # a floating-tag miss records in FAILED_COPIES but is non-fatal
            # under set -e (copy_image runs in an if-condition context).
            stage_optional() {
              local LABEL="$1" RESULT="$2" SRC="$3" DST="$4" FLOAT_DST="$5" FLOAT_LABEL="$6"
              echo ""
              echo "=== ${LABEL} (nightly) ==="
              if [ -n "${RESULT}" ] && [ "${RESULT}" != "success" ]; then
                echo "::warning::Skipped ${LABEL} — build result in this run: '${RESULT}'"
                FAILED_COPIES+=("${LABEL} (build ${RESULT})")
                return 1
              elif ! crane manifest "${SRC}" > /dev/null 2>&1; then
                echo "::warning::Skipped ${LABEL} — ${SRC} not found (no nightly build exists for this SHA)"
                FAILED_COPIES+=("${LABEL} (source missing)")
                return 1
              elif copy_image "${SRC}" "${DST}" "${LABEL}"; then
                if [ -n "${FLOAT_DST}" ]; then
                  if ! copy_image "${SRC}" "${FLOAT_DST}" "${FLOAT_LABEL}"; then
                    echo "::warning::Floating tag push failed for ${FLOAT_LABEL} (dated tag staged fine)"
                  fi
                fi
                return 0
              else
                echo "::warning::Copy failed for ${LABEL} — source exists; check NGC write access"
                return 1
              fi
            }
            ECR_BASE="${ECR_REGISTRY}/${ECR_REPOSITORY}"
            NGC_BASE="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo"

            OPERATOR_STAGED=false
            if want operator && stage_optional "kubernetes-operator-nightly:${NGC_VERSION_TAG}" "${OPERATOR_BUILD_RESULT}" \
                "${ECR_BASE}:${COMMIT_SHA}-operator${NIGHTLY_TAG_SUFFIX}" \
                "${NGC_BASE}/kubernetes-operator-nightly:${NGC_VERSION_TAG}" \
                "${NGC_BASE}/kubernetes-operator:nightly" "kubernetes-operator:nightly"; then OPERATOR_STAGED=true; fi

            PLANNER_STAGED=false
            if want planner && stage_optional "dynamo-planner-nightly:${NGC_VERSION_TAG}" "${PLANNER_BUILD_RESULT}" \
                "${ECR_BASE}:${COMMIT_SHA}-dynamo-planner${NIGHTLY_TAG_SUFFIX}" \
                "${NGC_BASE}/dynamo-planner-nightly:${NGC_VERSION_TAG}" \
                "${NGC_BASE}/dynamo-planner:nightly" "dynamo-planner:nightly"; then PLANNER_STAGED=true; fi

            FRONTEND_STAGED=false
            if want frontend && stage_optional "dynamo-frontend-nightly:${NGC_VERSION_TAG}" "${FRONTEND_BUILD_RESULT}" \
                "${ECR_BASE}:${COMMIT_SHA}-dynamo-frontend${NIGHTLY_TAG_SUFFIX}" \
                "${NGC_BASE}/dynamo-frontend-nightly:${NGC_VERSION_TAG}" \
                "${NGC_BASE}/dynamo-frontend:nightly" "dynamo-frontend:nightly"; then FRONTEND_STAGED=true; fi

            # EFA variants live in the runtime -nightly repos under a -efa tag
            # suffix, mirroring the RC convention (vllm-runtime:<ver>-efa).
            VLLM_EFA_STAGED=false
            if want vllm-efa && stage_optional "vllm-runtime-nightly:${NGC_VERSION_TAG}-efa" "${VLLM_EFA_BUILD_RESULT}" \
                "${ECR_BASE}:${COMMIT_SHA}-vllm-runtime-efa${NIGHTLY_TAG_SUFFIX}" \
                "${NGC_BASE}/vllm-runtime-nightly:${NGC_VERSION_TAG}-efa" \
                "${NGC_BASE}/vllm-runtime:nightly-efa" "vllm-runtime:nightly-efa"; then VLLM_EFA_STAGED=true; fi

            SGLANG_EFA_STAGED=false
            if want sglang-efa && stage_optional "sglang-runtime-nightly:${NGC_VERSION_TAG}-efa" "${SGLANG_EFA_BUILD_RESULT}" \
                "${ECR_BASE}:${COMMIT_SHA}-sglang-runtime-efa${NIGHTLY_TAG_SUFFIX}" \
                "${NGC_BASE}/sglang-runtime-nightly:${NGC_VERSION_TAG}-efa" \
                "${NGC_BASE}/sglang-runtime:nightly-efa" "sglang-runtime:nightly-efa"; then SGLANG_EFA_STAGED=true; fi

            TRTLLM_EFA_STAGED=false
            if want trtllm-efa && stage_optional "tensorrtllm-runtime-nightly:${NGC_VERSION_TAG}-efa" "${TRTLLM_EFA_BUILD_RESULT}" \
                "${ECR_BASE}:${COMMIT_SHA}-trtllm-runtime-efa${NIGHTLY_TAG_SUFFIX}" \
                "${NGC_BASE}/tensorrtllm-runtime-nightly:${NGC_VERSION_TAG}-efa" \
                "${NGC_BASE}/tensorrtllm-runtime:nightly-efa" "tensorrtllm-runtime:nightly-efa"; then TRTLLM_EFA_STAGED=true; fi
          fi

          # ---- Summary ----
          echo "successful_count=${#SUCCESSFUL_COPIES[@]}" >> $GITHUB_OUTPUT
          echo "failed_count=${#FAILED_COPIES[@]}" >> $GITHUB_OUTPUT
          echo "operator_staged=${OPERATOR_STAGED:-}" >> $GITHUB_OUTPUT
          echo "planner_staged=${PLANNER_STAGED:-}" >> $GITHUB_OUTPUT
          echo "frontend_staged=${FRONTEND_STAGED:-}" >> $GITHUB_OUTPUT
          echo "vllm_efa_staged=${VLLM_EFA_STAGED:-}" >> $GITHUB_OUTPUT
          echo "sglang_efa_staged=${SGLANG_EFA_STAGED:-}" >> $GITHUB_OUTPUT
          echo "trtllm_efa_staged=${TRTLLM_EFA_STAGED:-}" >> $GITHUB_OUTPUT

          # Staged container refs for nspect: repo:tag labels only (NO registry/org —
          # that's a secret; GitLab prefixes with its own NGC base). Keep primary version
          # tags + -efa; drop -cuda13 digest aliases and floating :nightly* (same image).
          STAGED_CTRS=""
          for L in "${SUCCESSFUL_COPIES[@]:-}"; do
            [ -z "${L}" ] && continue
            case "${L}" in *-cuda13|*:nightly|*:nightly-*) continue ;; esac
            [ -n "${STAGED_CTRS}" ] && STAGED_CTRS="${STAGED_CTRS},"
            STAGED_CTRS="${STAGED_CTRS}${L}"
          done
          echo "staged_containers=${STAGED_CTRS}" >> $GITHUB_OUTPUT
          {
            echo "### Staged containers (repo:tag)"
            echo ""
            echo "\`${STAGED_CTRS:-<none>}\`"
          } >> "${GITHUB_STEP_SUMMARY}"

          printf '%s\n' "${SUCCESSFUL_COPIES[@]}" > /tmp/successful_copies.txt
          printf '%s\n' "${FAILED_COPIES[@]}" > /tmp/failed_copies.txt 2>/dev/null || true

          echo "========================================"
          echo "NGC Publishing Summary:"
          echo "  Successful: ${#SUCCESSFUL_COPIES[@]}"
          echo "  Failed: ${#FAILED_COPIES[@]}"
          echo "========================================"

          if [ "${SUBSET}" = "none" ]; then
            echo "Container subset is 'none'; no images copied (helm-only or wheel/crate-only release)."
          elif [ ${#SUCCESSFUL_COPIES[@]} -eq 0 ]; then
            echo "ERROR: No images were successfully copied to NGC!"
            exit 1
          fi

      - name: Validate chart/image coherence (on-demand)
        if: ${{ env.NIGHTLY != 'true' && env.ON_DEMAND == 'true' && env.HELM_SUBSET != 'none' }}
        env:
          STAGED_CONTAINERS: ${{ steps.copy_images.outputs.staged_containers }}
          NGC_VERSION_TAG: ${{ needs.prepare-release.outputs.ngc_version_tag }}
        run: |
          set -euo pipefail
          # A published chart must reference only images published now (in
          # CONTAINER_SUBSET, and proven staged — copy_image failures are
          # non-fatal) or already resolvable at the exact repository the chart
          # references (the GA registry in values.yaml — NOT the staging org,
          # where only rc-suffixed tags exist).
          HELM="${HELM_SUBSET:-all}"
          CSET="${CONTAINER_SUBSET:-all}"
          want_helm() { [ "${HELM}" = "all" ] && return 0; case ",${HELM}," in *,"$1",*) return 0 ;; esac; return 1; }
          want_ctr()  { [ "${CSET}" = "all" ] && return 0; case ",${CSET}," in *,"$1",*) return 0 ;; esac; return 1; }
          fail=0
          # Each site: chart_token|container_token|values.yaml|image repo (as referenced by the chart)|staged image name
          SITES=(
            "platform|operator|deploy/helm/charts/platform/components/operator/values.yaml|nvcr.io/nvidia/ai-dynamo/kubernetes-operator|kubernetes-operator"
          )
          for SITE in "${SITES[@]}"; do
            IFS='|' read -r CHART CTOK VALUES REPO STAGED <<<"${SITE}"
            want_helm "${CHART}" || { echo "chart ${CHART} not published; skip coherence check"; continue; }
            if want_ctr "${CTOK}"; then
              # Selected: the copy step tolerates per-image failures, so require
              # the image in its successful-copy record before shipping a chart.
              if [[ ",${STAGED_CONTAINERS}," == *",${STAGED}:${NGC_VERSION_TAG},"* ]]; then
                echo "ok: ${CHART} image ${CTOK} staged this release (${STAGED}:${NGC_VERSION_TAG})"
              else
                echo "::error::chart ${CHART} needs ${CTOK}, which was selected but ${STAGED}:${NGC_VERSION_TAG} failed to copy — see 'Copy images to NGC'"
                fail=1
              fi
              continue
            fi
            # Image excluded: read the pinned tag the bump wrote and prove it exists
            # at the exact repository string the chart will deploy from.
            TAG=$(awk -v r="${REPO}" '
              $0 ~ "repository:[[:space:]]*"r {f=1; next}
              f && /tag:/ { gsub(/[",]/,""); print $2; exit }' "${VALUES}")
            if [ -z "${TAG}" ]; then
              echo "::error::could not read pinned tag for ${REPO} in ${VALUES}"; fail=1; continue
            fi
            if crane manifest "${REPO}:${TAG}" >/dev/null 2>&1; then
              echo "ok: ${CHART} references ${REPO}:${TAG} (exists)"
            else
              echo "::error::chart ${CHART} references ${REPO}:${TAG}, which is NOT in this release (${CTOK} excluded) and does NOT resolve"
              fail=1
            fi
          done
          [ "${fail}" -eq 0 ] || { echo "::error::chart/image coherence check failed"; exit 1; }

      - name: Package and push Helm charts to NGC
        id: helm_push
        # Skip entirely when no chart is selected — otherwise the step installs the
        # push plugin and adds credentialed repos just to push nothing.
        if: ${{ env.NIGHTLY != 'true' && env.HELM_SUBSET != 'none' }}
        env:
          NGC_HELM_REPO: https://helm.ngc.nvidia.com/${{ secrets.NGC_PUBLISH_ORG }}/ai-dynamo
          # OCI mirror of the same staging namespace. ChartMuseum and OCI are
          # SEPARATE registries in NGC — a chart in one is invisible from the
          # other — so a GA release that only cm-pushes leaves the OCI copy
          # missing, and the OCI publishing MR has nothing to promote. The
          # nightly step below has dual-pushed since it was written; this step
          # did not, which is why every GA chart through 1.4.0 is
          # ChartMuseum-only.
          NGC_HELM_OCI_REPO: oci://nvcr.io/${{ secrets.NGC_PUBLISH_ORG }}/ai-dynamo
          NGC_TOKEN: ${{ secrets.NGC_PUBLISH_TOKEN }}
          NGC_USERNAME: ${{ secrets.NGC_PUBLISH_USERNAME }}
          HELM_CHART_VERSION: ${{ needs.prepare-release.outputs.helm_chart_version }}
        run: |
          set -euo pipefail

          REPO_ALIAS="ngc-staging-dynamo"

          # Pinned: this plugin runs inside the NGC-credentialed publish job.
          helm plugin install https://github.com/chartmuseum/helm-push --version 0.10.4 || true

          echo "${NGC_TOKEN}" | helm repo add "${REPO_ALIAS}" \
            --username="${NGC_USERNAME}" \
            --password-stdin \
            "${NGC_HELM_REPO}" > /dev/null 2>&1

          helm repo add nats https://nats-io.github.io/k8s/helm/charts/ || true
          helm repo add bitnami https://charts.bitnami.com/bitnami || true

          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Helm Charts" >> $GITHUB_STEP_SUMMARY

          # dynamo-platform, mirroring the rcs/dynamo helm runbook.
          # Only the selected charts are pushed (token = chart dir name);
          # nightly/rc default to all (HELM_SUBSET defaults to 'all').
          STAGED_HELM=""
          for CHART_DIR in deploy/helm/charts/platform; do
            TOKEN="$(basename "${CHART_DIR}")"
            if [ "${HELM_SUBSET:-all}" != "all" ]; then
              case ",${HELM_SUBSET}," in *,"${TOKEN}",*) ;; *) echo "skip ${TOKEN} chart (not selected)"; continue ;; esac
            fi
            CHART_NAME=$(awk '/^name:/ {print $2; exit}' "${CHART_DIR}/Chart.yaml")
            ( cd "${CHART_DIR}" && helm dep build . )
            echo "Packaging ${CHART_NAME} with version ${HELM_CHART_VERSION}..."
            helm package \
              --version "${HELM_CHART_VERSION}" \
              --app-version "${HELM_CHART_VERSION}" \
              "${CHART_DIR}"
            CHART_FILE="${CHART_NAME}-${HELM_CHART_VERSION}.tgz"
            echo "Pushing ${CHART_FILE} to NGC Helm registry..."
            helm cm-push "${CHART_FILE}" "${REPO_ALIAS}"
            echo "- \`${CHART_NAME}:${HELM_CHART_VERSION}\` pushed to NGC Helm registry" >> $GITHUB_STEP_SUMMARY
            STAGED_HELM="${STAGED_HELM:+${STAGED_HELM},}${CHART_NAME}:${HELM_CHART_VERSION}"
            # Rewrite the output after every chart (later writes win) so a
            # failure on a subsequent chart still reports what actually landed.
            # Consumed by GitLab (STAGED_HELM) for exact nSpect chart registration.
            echo "staged_helm=${STAGED_HELM}" >> "${GITHUB_OUTPUT}"
          done

          # Same tarball, second registry. helm does not read docker's login for
          # OCI, so this needs its own registry login. Fail-soft: ChartMuseum is
          # still the primary distribution channel and has already succeeded by
          # this point, so an OCI hiccup must not fail the release. The GitLab
          # publish:helm-oci:ondemand job probes for this copy and fails loudly
          # if it is missing, so a soft failure here cannot ship silently.
          echo "Pushing ${CHART_FILE} to NGC OCI registry..."
          if echo "${NGC_TOKEN}" | helm registry login nvcr.io \
               --username "${NGC_USERNAME}" --password-stdin \
             && helm push "${CHART_FILE}" "${NGC_HELM_OCI_REPO}"; then
            echo "- \`${CHART_NAME}:${HELM_CHART_VERSION}\` pushed to NGC OCI (\`${NGC_HELM_OCI_REPO}/${CHART_NAME}\`)" >> $GITHUB_STEP_SUMMARY
          else
            echo "::warning::OCI push of ${CHART_FILE} failed — the ChartMuseum copy is unaffected. Re-run this job or push by hand before the OCI publishing MR."
            echo "- :warning: \`${CHART_NAME}:${HELM_CHART_VERSION}\` OCI push FAILED (ChartMuseum copy is fine)" >> $GITHUB_STEP_SUMMARY
          fi

          helm repo remove "${REPO_ALIAS}"

      # Nightly: package platform at the dated pre-release version
      # and dual-push to NGC staging (ChartMuseum + OCI). Values are rewritten
      # in the workspace only, onto the *-nightly repos at this run's tag.
      # Fail-soft per chart via the copy step's *_staged flags; fully fail-soft
      # overall (continue-on-error): a helm outage marks the step red but never
      # blocks the GitLab trigger, so containers/wheels still publish — charts
      # just skip registration that night (staged_helm stays empty).
      - name: Package and push nightly Helm charts to NGC
        id: nightly_helm_push
        if: env.NIGHTLY == 'true'
        continue-on-error: true
        env:
          NGC_HELM_REPO: https://helm.ngc.nvidia.com/${{ secrets.NGC_PUBLISH_ORG }}/ai-dynamo
          # OCI mirror of the same staging namespace.
          NGC_HELM_OCI_REPO: oci://nvcr.io/${{ secrets.NGC_PUBLISH_ORG }}/ai-dynamo
          NGC_TOKEN: ${{ secrets.NGC_PUBLISH_TOKEN }}
          NGC_USERNAME: ${{ secrets.NGC_PUBLISH_USERNAME }}
          HELM_CHART_VERSION: ${{ needs.prepare-release.outputs.helm_chart_version }}
          NGC_VERSION_TAG: ${{ needs.prepare-release.outputs.ngc_version_tag }}
          OPERATOR_STAGED: ${{ steps.copy_images.outputs.operator_staged }}
        run: |
          set -euo pipefail

          # Publish a chart only if its image staged tonight.
          CHARTS=""
          SKIPPED_HELM=""
          if [[ "${OPERATOR_STAGED}" == "true" ]]; then
            CHARTS="platform"
          else
            echo "::warning::Skipping platform chart: kubernetes-operator-nightly:${NGC_VERSION_TAG} was not staged this run"
            SKIPPED_HELM="platform"
          fi

          echo "staged_helm=" >> $GITHUB_OUTPUT
          echo "skipped_helm=${SKIPPED_HELM}" >> $GITHUB_OUTPUT
          if [[ -z "${CHARTS}" ]]; then
            echo "No nightly charts to publish: no chart image was staged this run."
            exit 0
          fi

          python3 .github/scripts/prepare_nightly_helm.py \
            --chart-version "${HELM_CHART_VERSION}" \
            --image-tag "${NGC_VERSION_TAG}" \
            --charts "${CHARTS}"

          REPO_ALIAS="ngc-staging-dynamo"

          helm plugin install https://github.com/chartmuseum/helm-push || true

          # --force-update refreshes the cached index when the alias already
          # exists on the runner, so the duplicate probe below never reads a
          # stale listing on re-runs.
          echo "${NGC_TOKEN}" | helm repo add "${REPO_ALIAS}" \
            --username="${NGC_USERNAME}" \
            --password-stdin \
            --force-update \
            "${NGC_HELM_REPO}" > /dev/null 2>&1

          # helm does not read docker's login for OCI pushes.
          echo "${NGC_TOKEN}" | helm registry login nvcr.io \
            --username "${NGC_USERNAME}" --password-stdin

          helm repo add nats https://nats-io.github.io/k8s/helm/charts/ || true
          helm repo add bitnami https://charts.bitnami.com/bitnami || true

          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Nightly Helm Charts" >> $GITHUB_STEP_SUMMARY

          STAGED_HELM=""
          for TOKEN in ${CHARTS//,/ }; do
            CHART_DIR="deploy/helm/charts/${TOKEN}"
            CHART_NAME=$(awk '/^name:/ {print $2}' "${CHART_DIR}/Chart.yaml")
            pushd "${CHART_DIR}"
            helm dep build .
            popd

            echo "Packaging ${CHART_NAME} with version ${HELM_CHART_VERSION}..."
            helm package \
              --version "${HELM_CHART_VERSION}" \
              --app-version "${HELM_CHART_VERSION}" \
              "${CHART_DIR}"
            CHART_FILE="${CHART_NAME}-${HELM_CHART_VERSION}.tgz"

            # OCI first: mutable, so same-SHA re-runs overwrite harmlessly;
            # ChartMuseum 409s on duplicates.
            echo "Pushing ${CHART_FILE} to NGC OCI registry..."
            helm push "${CHART_FILE}" "${NGC_HELM_OCI_REPO}"

            # Skip cm-push if already indexed so same-SHA re-runs stay green.
            SEARCH_OUT=$(helm search repo "${REPO_ALIAS}/${CHART_NAME}" \
                --version "${HELM_CHART_VERSION}" -o yaml)
            if grep -qF "version: ${HELM_CHART_VERSION}" <<< "${SEARCH_OUT}"; then
              echo "::notice::${CHART_FILE} already in the NGC Helm registry; skipping cm-push"
            else
              echo "Pushing ${CHART_FILE} to NGC Helm registry..."
              helm cm-push "${CHART_FILE}" "${REPO_ALIAS}"
            fi

            STAGED_HELM="${STAGED_HELM:+${STAGED_HELM},}${CHART_NAME}:${HELM_CHART_VERSION}"
            echo "- \`${CHART_NAME}:${HELM_CHART_VERSION}\` pushed to the NGC Helm registry and OCI (\`${CHART_NAME}\` in the staging org)" >> $GITHUB_STEP_SUMMARY
          done
          if [[ -n "${SKIPPED_HELM}" ]]; then
            echo "- Skipped (image not staged this run): \`${SKIPPED_HELM}\`" >> $GITHUB_STEP_SUMMARY
          fi

          echo "staged_helm=${STAGED_HELM}" >> $GITHUB_OUTPUT

          helm repo remove "${REPO_ALIAS}"

      - name: Create release summary
        env:
          NGC_VERSION_TAG: ${{ needs.prepare-release.outputs.ngc_version_tag }}
          HELM_CHART_VERSION: ${{ needs.prepare-release.outputs.helm_chart_version }}
          NIGHTLY_STAGED_HELM: ${{ steps.nightly_helm_push.outputs.staged_helm }}
          NIGHTLY_SKIPPED_HELM: ${{ steps.nightly_helm_push.outputs.skipped_helm }}
          SUCCESSFUL_COUNT: ${{ steps.copy_images.outputs.successful_count }}
          FAILED_COUNT: ${{ steps.copy_images.outputs.failed_count }}
          STAGED_CONTAINERS: ${{ steps.copy_images.outputs.staged_containers }}
          STAGED_HELM: ${{ steps.helm_push.outputs.staged_helm }}
          BRANCH_NAME: ${{ github.ref_name }}
          OPERATOR_STAGED: ${{ steps.copy_images.outputs.operator_staged }}
          PLANNER_STAGED: ${{ steps.copy_images.outputs.planner_staged }}
          FRONTEND_STAGED: ${{ steps.copy_images.outputs.frontend_staged }}
          VLLM_EFA_STAGED: ${{ steps.copy_images.outputs.vllm_efa_staged }}
          SGLANG_EFA_STAGED: ${{ steps.copy_images.outputs.sglang_efa_staged }}
          TRTLLM_EFA_STAGED: ${{ steps.copy_images.outputs.trtllm_efa_staged }}
        run: |
          if [[ "${NIGHTLY}" == "true" ]]; then
            echo "## Nightly Build Summary" >> $GITHUB_STEP_SUMMARY
          else
            echo "## Release Summary" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
          echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
          echo "| Version | ${VERSION} |" >> $GITHUB_STEP_SUMMARY
          echo "| NGC Version Tag | ${NGC_VERSION_TAG} |" >> $GITHUB_STEP_SUMMARY
          echo "| Source Commit SHA | ${COMMIT_SHA} |" >> $GITHUB_STEP_SUMMARY
          echo "| Branch | ${BRANCH_NAME} |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### NGC Publishing Results" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "- **Successful copies**: ${SUCCESSFUL_COUNT}" >> $GITHUB_STEP_SUMMARY
          echo "- **Failed copies**: ${FAILED_COUNT}" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          # Report what ACTUALLY landed, not a hardcoded full-release list: a partial
          # selection must not produce a summary claiming images it never published.
          echo "### Images published this run" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          # Actuals-driven list (a partial selection must not claim images it
          # never published); the nightly-only block below adds the optional
          # images with their per-build staged/skipped notes.
          if [ -n "${STAGED_CONTAINERS}" ]; then
            echo "${STAGED_CONTAINERS}" | tr ',' '\n' | sed 's/^/- `/; s/$/`/' >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            if [[ "${NIGHTLY}" == "true" ]]; then
              echo "_Each runtime image above also published as a \`-cuda13\` digest alias; floating tags are listed below._" >> $GITHUB_STEP_SUMMARY
            else
              echo "_Each runtime image above also published as a \`-cuda13\` digest alias._" >> $GITHUB_STEP_SUMMARY
            fi
          else
            echo "_No container images published (selection: \`${CONTAINER_SUBSET:-all}\`)._" >> $GITHUB_STEP_SUMMARY
          fi
          if [[ "${NIGHTLY}" == "true" ]]; then
            _note() { [[ "$1" == "true" ]] && echo "" || echo " — **NOT staged this run** (see copy-step warnings)"; }
            OP_NOTE=$(_note "${OPERATOR_STAGED}");   PL_NOTE=$(_note "${PLANNER_STAGED}")
            FE_NOTE=$(_note "${FRONTEND_STAGED}")
            VE_NOTE=$(_note "${VLLM_EFA_STAGED}");   SE_NOTE=$(_note "${SGLANG_EFA_STAGED}")
            TE_NOTE=$(_note "${TRTLLM_EFA_STAGED}")
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "Optional images (built by this nightly run, staged fail-soft):" >> $GITHUB_STEP_SUMMARY
            echo "- \`kubernetes-operator-nightly:${NGC_VERSION_TAG}\`${OP_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`dynamo-planner-nightly:${NGC_VERSION_TAG}\`${PL_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`dynamo-frontend-nightly:${NGC_VERSION_TAG}\`${FE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`vllm-runtime-nightly:${NGC_VERSION_TAG}-efa\`${VE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`sglang-runtime-nightly:${NGC_VERSION_TAG}-efa\`${SE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`tensorrtllm-runtime-nightly:${NGC_VERSION_TAG}-efa\`${TE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "Mutable floating tags (repushed every nightly):" >> $GITHUB_STEP_SUMMARY
            # Core runtime aliases stage whenever the (release-gating) runtime
            # build succeeded — never annotate them with the EFA build's fate;
            # the EFA alias is staged fail-soft and carries its own note.
            echo "- \`vllm-runtime:nightly\` / \`vllm-runtime:nightly-cuda13\`" >> $GITHUB_STEP_SUMMARY
            echo "- \`sglang-runtime:nightly\` / \`sglang-runtime:nightly-cuda13\`" >> $GITHUB_STEP_SUMMARY
            echo "- \`tensorrtllm-runtime:nightly\` / \`tensorrtllm-runtime:nightly-cuda13\`" >> $GITHUB_STEP_SUMMARY
            echo "- \`vllm-runtime:nightly-efa\`${VE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`sglang-runtime:nightly-efa\`${SE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`tensorrtllm-runtime:nightly-efa\`${TE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`kubernetes-operator:nightly\`${OP_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`dynamo-planner:nightly\`${PL_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "- \`dynamo-frontend:nightly\`${FE_NOTE}" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "Helm charts (staged fail-soft, dual-pushed to NGC ChartMuseum + OCI):" >> $GITHUB_STEP_SUMMARY
            if [[ -n "${NIGHTLY_STAGED_HELM}" ]]; then
              for ENTRY in ${NIGHTLY_STAGED_HELM//,/ }; do
                echo "- \`${ENTRY}\`" >> $GITHUB_STEP_SUMMARY
              done
            fi
            if [[ -n "${NIGHTLY_SKIPPED_HELM}" ]]; then
              echo "- Skipped (image not staged this run): \`${NIGHTLY_SKIPPED_HELM}\`" >> $GITHUB_STEP_SUMMARY
            fi
            if [[ -z "${NIGHTLY_STAGED_HELM}${NIGHTLY_SKIPPED_HELM}" ]]; then
              echo "- None (helm step failed or did not run — see its log)" >> $GITHUB_STEP_SUMMARY
            fi
          fi
          if [[ "${NIGHTLY}" != "true" ]]; then
            echo "" >> $GITHUB_STEP_SUMMARY
            echo "### Helm charts published this run" >> $GITHUB_STEP_SUMMARY
            echo "" >> $GITHUB_STEP_SUMMARY
            if [ -n "${STAGED_HELM}" ]; then
              echo "${STAGED_HELM}" | tr ',' '\n' | sed 's/^/- `/; s/$/`/' >> $GITHUB_STEP_SUMMARY
            else
              echo "_No charts published (selection: \`${HELM_SUBSET:-all}\`)._" >> $GITHUB_STEP_SUMMARY
            fi
          fi

      # Runs last so charts still publish first. rc and on-demand runs name an
      # explicit image set, so a partially-copied set must not report success and
      # let GitLab scan/promote images that never landed. Nightly stays warn-skip
      # by design: its optional images are gated per-build-result upstream and a
      # flake there must not block the GitLab trigger.
      - name: Fail on partially staged container selection
        if: ${{ env.NIGHTLY != 'true' && env.CONTAINER_SUBSET != 'none' }}
        env:
          FAILED_COUNT: ${{ steps.copy_images.outputs.failed_count }}
        run: |
          set -euo pipefail
          if [ "${FAILED_COUNT:-0}" -ne 0 ]; then
            echo "::error::${FAILED_COUNT} selected container image(s) failed to copy to NGC — see 'Copy images to NGC'. Once fixed, re-stage as a sub-round of the same rc (rc_round X.1, X.2, ... on stage-release-rc.yml; a direct rc dispatch still needs a new integer rc_number)."
            exit 1
          fi
          echo "All selected container images copied."

  # ============================================================================
  # ARTIFACTORY STAGING: extract wheels from the dynamo-runtime image in ECR
  # and upload them to JFrog Artifactory so consumers can pip-install them.
  # Runs in parallel with release-publish (both gated only on prepare-release).
  # ============================================================================

  stage-wheels-artifactory:
    name: Stage Wheels to Artifactory
    needs: [prepare-release]
    # Skip only when no wheels are selected; nightly/rc default to 'all'.
    if: ${{ needs.prepare-release.result == 'success' && inputs.wheels != 'none' }}
    runs-on: prod-tester-amd-v1
    environment: automated-release
    outputs:
      # Comma-separated wheel FILENAMES actually uploaded (NO Artifactory URL — that's a
      # secret; GitLab prefixes with its own WHEEL_URL_PREFIX). For nspect registration.
      staged_wheels: ${{ steps.upload_wheels.outputs.staged_wheels }}
    env:
      COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
      NIGHTLY: ${{ needs.prepare-release.outputs.nightly }}
      ON_DEMAND: ${{ needs.prepare-release.outputs.on_demand }}
      VERSION: ${{ needs.prepare-release.outputs.version }}
      RC_NUMBER: ${{ needs.prepare-release.outputs.rc_number }}
      WHEEL_SUBSET: ${{ inputs.wheels }}
    steps:
      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
      - *ops-token
      - *ops-check
      - name: Setup crane
        env:
          CRANE_VERSION: v0.20.2
        run: |
          curl --retry 5 --retry-delay 2 \
            --connect-timeout 10 --max-time 120 \
            --fail --show-error -sL \
            "https://github.com/google/go-containerregistry/releases/download/${CRANE_VERSION}/go-containerregistry_Linux_x86_64.tar.gz" \
            | tar -xzf - crane
          sudo mv crane /usr/local/bin/
          crane version

      - name: Login to ECR
        run: |
          aws ecr get-login-password --region ${AWS_DEFAULT_REGION} | crane auth login --username AWS --password-stdin "${ECR_REGISTRY}"

      - name: Extract wheels from dynamo-runtime image
        id: extract
        run: |
          set -euo pipefail

          # Nightly suffix ensures a failed nightly cannot fall through to
          # the stable image.
          NIGHTLY_TAG_SUFFIX=""
          if [ "${NIGHTLY}" = "true" ]; then
            NIGHTLY_TAG_SUFFIX="-nightly"
          fi
          ECR_IMAGE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-runtime${NIGHTLY_TAG_SUFFIX}"

          STAGE="${RUNNER_TEMP}/wheels"
          mkdir -p "${STAGE}/amd64" "${STAGE}/arm64"

          echo "Exporting amd64 filesystem from ${ECR_IMAGE}..."
          crane export --platform=linux/amd64 "${ECR_IMAGE}" - \
            | tar -x -C "${STAGE}/amd64" \
                --wildcards \
                'opt/dynamo/wheelhouse/ai_dynamo-*py3-none-any.whl' \
                'opt/dynamo/wheelhouse/ai_dynamo_runtime-*x86_64*.whl' \
                'opt/dynamo/wheelhouse/kvbm-*x86_64*.whl'

          echo "Exporting arm64 filesystem from ${ECR_IMAGE}..."
          crane export --platform=linux/arm64 "${ECR_IMAGE}" - \
            | tar -x -C "${STAGE}/arm64" \
                --wildcards \
                'opt/dynamo/wheelhouse/ai_dynamo_runtime-*aarch64*.whl' \
                'opt/dynamo/wheelhouse/kvbm-*aarch64*.whl'

          OUT="${STAGE}/upload"
          mkdir -p "${OUT}"

          shopt -s nullglob
          AMD64_WHEELS=("${STAGE}/amd64/opt/dynamo/wheelhouse/"*.whl)
          ARM64_WHEELS=("${STAGE}/arm64/opt/dynamo/wheelhouse/"*.whl)
          shopt -u nullglob

          if [ ${#AMD64_WHEELS[@]} -eq 0 ]; then
            echo "::error::No amd64 wheels extracted from ${ECR_IMAGE}"
            exit 1
          fi
          if [ ${#ARM64_WHEELS[@]} -eq 0 ]; then
            echo "::error::No arm64 wheels extracted from ${ECR_IMAGE}"
            exit 1
          fi

          cp "${AMD64_WHEELS[@]}" "${OUT}/"
          cp "${ARM64_WHEELS[@]}" "${OUT}/"

          echo "Wheels staged for upload:"
          ls -la "${OUT}/"

          echo "upload_dir=${OUT}" >> $GITHUB_OUTPUT

      - name: Inventory selected wheels
        env:
          UPLOAD_DIR: ${{ steps.extract.outputs.upload_dir }}
        run: |
          set -uo pipefail
          # Report-only inventory of which selected wheels were extracted. Never fails
          # (kvbm is built conditionally, so 'all' may legitimately omit it).
          WSUB="${WHEEL_SUBSET:-all}"
          want_wheel() {
            [ "${WSUB}" = "all" ] && return 0
            case ",${WSUB}," in *,"$1",*) return 0 ;; esac
            return 1
          }
          # token|filename glob (relative to the upload dir)
          ENTRIES=(
            "ai-dynamo|ai_dynamo-*-py3-none-any.whl"
            "ai-dynamo-runtime|ai_dynamo_runtime-*.whl"
            "kvbm|kvbm-*.whl"
          )

          {
            echo "### Wheel availability"
            echo ""
            echo "Selection: \`${WSUB}\`"
            echo ""
            echo "| Token | Wheels found | Status |"
            echo "|-------|--------------|--------|"
          } >> "${GITHUB_STEP_SUMMARY}"

          shopt -s nullglob
          avail=0; missing=0; selected=0
          echo "=== Wheel availability (selection: ${WSUB}) ==="
          for E in "${ENTRIES[@]}"; do
            IFS='|' read -r TOKEN GLOB <<<"${E}"
            want_wheel "${TOKEN}" || continue
            selected=$((selected+1))
            FILES=( "${UPLOAD_DIR}"/${GLOB} )
            if [ "${#FILES[@]}" -gt 0 ]; then
              echo "  available: ${TOKEN} -> ${FILES[*]##*/}"
              echo "| ${TOKEN} | ${#FILES[@]} | available |" >> "${GITHUB_STEP_SUMMARY}"
              avail=$((avail+1))
            else
              echo "  MISSING:   ${TOKEN} (no ${GLOB})"
              echo "| ${TOKEN} | 0 | **missing** |" >> "${GITHUB_STEP_SUMMARY}"
              missing=$((missing+1))
            fi
          done
          shopt -u nullglob

          echo "Summary: ${avail} available, ${missing} missing (of ${selected} selected)."
          {
            echo ""
            echo "**${avail} available, ${missing} missing** (of ${selected} selected)."
          } >> "${GITHUB_STEP_SUMMARY}"

      - name: Verify wheel versions match prepare-release expected tag
        env:
          EXPECTED_WHEEL_VERSION: ${{ needs.prepare-release.outputs.wheel_version }}
          UPLOAD_DIR: ${{ steps.extract.outputs.upload_dir }}
        run: |
          set -euo pipefail

          # Catches UTC-midnight drift between the build-time version stamp and
          # prepare-release's wheel_version. Every wheel that will be uploaded must agree,
          # including kvbm (optional: built conditionally, so it is checked only if present).
          WSUB="${WHEEL_SUBSET:-all}"
          want_wheel() {
            [ "${WSUB}" = "all" ] && return 0
            case ",${WSUB}," in *,"$1",*) return 0 ;; esac
            return 1
          }
          shopt -s nullglob
          AI_DYNAMO_WHEELS=("${UPLOAD_DIR}"/ai_dynamo-*-py3-none-any.whl)
          AI_DYNAMO_RUNTIME_WHEELS=("${UPLOAD_DIR}"/ai_dynamo_runtime-*.whl)
          KVBM_WHEELS=("${UPLOAD_DIR}"/kvbm-*.whl)
          shopt -u nullglob

          CHECK=()
          if want_wheel ai-dynamo; then
            [ ${#AI_DYNAMO_WHEELS[@]} -gt 0 ] || { echo "::error::Missing ai_dynamo wheel in ${UPLOAD_DIR}"; exit 1; }
            CHECK+=("${AI_DYNAMO_WHEELS[@]}")
          fi
          if want_wheel ai-dynamo-runtime; then
            [ ${#AI_DYNAMO_RUNTIME_WHEELS[@]} -gt 0 ] || { echo "::error::Missing ai_dynamo_runtime wheel in ${UPLOAD_DIR}"; exit 1; }
            CHECK+=("${AI_DYNAMO_RUNTIME_WHEELS[@]}")
          fi
          if want_wheel kvbm; then CHECK+=("${KVBM_WHEELS[@]}"); fi
          if [ ${#CHECK[@]} -eq 0 ]; then
            echo "No selected wheels to verify."
            exit 0
          fi

          echo "Expected wheel version: ${EXPECTED_WHEEL_VERSION}"

          mismatch=0
          for path in "${CHECK[@]}"; do
            file="$(basename "${path}")"
            # PEP 427: <dist>-<version>-<tags>.whl; version sits between the 1st and 2nd hyphen.
            rest="${file#*-}"
            version="${rest%%-*}"
            if [ -z "${version}" ]; then
              echo "::error::Empty version parsed from ${file}"
              exit 1
            fi
            echo "  ${file} -> ${version}"
            if [ "${version}" != "${EXPECTED_WHEEL_VERSION}" ]; then
              mismatch=1
            fi
          done

          if [ "${mismatch}" -ne 0 ]; then
            echo "::error::Wheel version mismatch — expected '${EXPECTED_WHEEL_VERSION}' for all wheels above. dynamo-pipeline and prepare-release likely computed dev_suffix on different UTC days."
            exit 1
          fi

      - name: Upload wheels to Artifactory
        id: upload_wheels
        env:
          ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
          ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
          ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }}
          UPLOAD_DIR: ${{ steps.extract.outputs.upload_dir }}
        run: |
          set -euo pipefail

          if [[ "${NIGHTLY}" == "true" ]]; then
            SUBPATH="nightly/${{ github.run_id }}"
          elif [[ "${ON_DEMAND}" == "true" ]]; then
            # Internal staging: rc subpath disambiguates re-stages of the same version
            # (the wheel filename keeps the clean version). GA promotion drops the rc.
            SUBPATH="release/${VERSION}/rc${RC_NUMBER}"
          else
            if [ -z "${RC_NUMBER}" ]; then
              echo "Error: RC_NUMBER is empty for a non-nightly release"
              exit 1
            fi
            SUBPATH="release/${VERSION}/rc${RC_NUMBER}"
          fi

          BASE_URL="${ARTIFACTORY_URL%/}/${SUBPATH}"
          echo "Uploading to: ${BASE_URL}/"

          echo "## Wheel Staging Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Wheel | Target |" >> $GITHUB_STEP_SUMMARY
          echo "|-------|--------|" >> $GITHUB_STEP_SUMMARY

          shopt -s nullglob
          WHEELS=("${UPLOAD_DIR}"/*.whl)
          if [ ${#WHEELS[@]} -eq 0 ]; then
            echo "Error: no wheels found in ${UPLOAD_DIR}"
            exit 1
          fi

          STAGED_WHEELS=""  # filenames actually uploaded (for nspect; URL stays secret)
          for WHEEL in "${WHEELS[@]}"; do
            WHEEL_FILENAME=$(basename "${WHEEL}")
            # Wheel subset filter (nightly/rc default to all: WHEEL_SUBSET=all).
            if [ "${WHEEL_SUBSET:-all}" != "all" ]; then
              case "${WHEEL_FILENAME}" in
                ai_dynamo_runtime-*) WTOK=ai-dynamo-runtime ;;
                ai_dynamo-*)         WTOK=ai-dynamo ;;
                kvbm-*)              WTOK=kvbm ;;
                *)                   WTOK="" ;;
              esac
              case ",${WHEEL_SUBSET}," in *,"${WTOK}",*) ;; *) echo "skip ${WHEEL_FILENAME} (not selected)"; continue ;; esac
            fi
            TARGET_URL="${BASE_URL}/${WHEEL_FILENAME}"
            BODY_FILE=$(mktemp)
            HTTP_CODE=$(curl -sSL -o "${BODY_FILE}" -w "%{http_code}" \
              --connect-timeout 30 --max-time 300 \
              --retry 3 --retry-delay 5 --retry-all-errors \
              -u "${ARTIFACTORY_USER}:${ARTIFACTORY_TOKEN}" \
              -T "${WHEEL}" \
              "${TARGET_URL}")
            if [ "${HTTP_CODE}" != "201" ]; then
              echo "::error::Artifactory deploy failed: HTTP ${HTTP_CODE} at ${TARGET_URL}"
              cat "${BODY_FILE}" || true
              exit 1
            fi
            echo "Uploaded: ${WHEEL_FILENAME} -> ${TARGET_URL}"
            echo "| \`${WHEEL_FILENAME}\` | ${TARGET_URL} |" >> $GITHUB_STEP_SUMMARY
            [ -n "${STAGED_WHEELS}" ] && STAGED_WHEELS="${STAGED_WHEELS},"
            STAGED_WHEELS="${STAGED_WHEELS}${WHEEL_FILENAME}"
            rm -f "${BODY_FILE}"
          done

          echo "staged_wheels=${STAGED_WHEELS}" >> "$GITHUB_OUTPUT"

          # A subset that matched no wheel in the image would otherwise upload
          # nothing and still report success.
          if [ -z "${STAGED_WHEELS}" ]; then
            echo "::error::wheel subset '${WHEEL_SUBSET:-all}' matched no wheels in the runtime image; nothing was uploaded"
            exit 1
          fi

  # ARTIFACTORY STAGING: publish workspace crates to the Artifactory cargo registry
  # (from ARTIFACTORY_CARGO_INDEX), dependency-ordered. Artifactory only, never crates.io.
  # cargo needs the native toolchain, so we `docker run` the ECR build image on
  # prod-builder-amd-v1 (the DinD builder release-publish uses). A job-level
  # `container:` cannot pull the image on a delayed release dispatch (registry
  # creds are only fresh in-step). On-demand and nightly; rc stays crate-less.

  stage-crates-artifactory:
    name: Stage Crates to Artifactory
    needs: [prepare-release]
    if: ${{ needs.prepare-release.result == 'success' && (needs.prepare-release.outputs.on_demand == 'true' || needs.prepare-release.outputs.nightly == 'true') && inputs.crates != 'none' && inputs.crates != '' }}
    runs-on: prod-builder-amd-v1
    environment: automated-release
    # Fail-soft: crate staging must never block the release. The job turns red
    # but the run continues. Consumers must read outputs.crates_status, not the
    # job result — continue-on-error masks it as 'success' (actions/runner#2495).
    continue-on-error: true
    outputs:
      # Comma-separated crate names actually staged to the cargo registry (set even on
      # partial failure). Forwarded to GitLab as STAGED_CRATES for nspect registration.
      staged_crates: ${{ steps.publish_crates.outputs.staged_crates }}
      # Comma-separated crate names that failed (or were skipped because a
      # dependency failed). Empty on full success.
      failed_crates: ${{ steps.publish_crates.outputs.failed_crates }}
      # success | partial | failed — the authoritative crate-staging outcome.
      crates_status: ${{ steps.publish_crates.outputs.crates_status }}
    env:
      COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
      NIGHTLY: ${{ needs.prepare-release.outputs.nightly }}
    steps:
      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
      - *ops-token
      - *ops-check
      - name: Checkout at source commit
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ needs.prepare-release.outputs.commit_sha }}

      # Use the workflow commit's copy of stage_crates.py, not the release
      # tree's: older release branches may predate the script.
      - name: Fetch crate-staging script from the workflow commit
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ github.sha }}
          path: .ondemand-tools
          sparse-checkout: .github/scripts
          sparse-checkout-cone-mode: false

      - name: Login to ECR
        run: |
          set -euo pipefail
          aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${ECR_REGISTRY}"
          # Nightly builds tag the toolchain image with the suffix BEFORE -test
          # (<sha>-dynamo-runtime-nightly-test, shared-build-image.yml); a missing
          # nightly build must not fall through to the post-merge image at the
          # same SHA (it carries stable, non-dev wheel versions).
          NIGHTLY_TAG_SUFFIX=""
          if [ "${NIGHTLY}" = "true" ]; then NIGHTLY_TAG_SUFFIX="-nightly"; fi
          echo "ECR_IMAGE=${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-runtime${NIGHTLY_TAG_SUFFIX}-test" >> "${GITHUB_ENV}"

      - name: Inventory crate build image in ECR
        env:
          CRATE_SUBSET: ${{ inputs.crates }}
        run: |
          set -uo pipefail
          # Report-only: is the crate build image in ECR? Never fails (publish step errors if absent).
          # Single source of truth: the tag the login step derived into ECR_IMAGE.
          TAG="${ECR_IMAGE##*:}"
          {
            echo "### Crate build image availability (ECR)"
            echo ""
            echo "Crate selection: \`${CRATE_SUBSET:-all}\`"
            echo ""
          } >> "${GITHUB_STEP_SUMMARY}"
          if aws ecr describe-images --repository-name "${ECR_REPOSITORY}" \
               --image-ids imageTag="${TAG}" --region "${AWS_DEFAULT_REGION}" >/dev/null 2>&1; then
            echo "available: build image ${ECR_REPOSITORY}:${TAG}"
            echo "- available: \`${ECR_REPOSITORY}:${TAG}\`" >> "${GITHUB_STEP_SUMMARY}"
          else
            echo "MISSING: build image ${ECR_REPOSITORY}:${TAG}"
            echo "- **missing**: \`${ECR_REPOSITORY}:${TAG}\` (crate publish will fail to pull)" >> "${GITHUB_STEP_SUMMARY}"
          fi

      - name: Publish crates to Artifactory cargo registry (inside build image)
        id: publish_crates
        env:
          ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }}
          ARTIFACTORY_CARGO_INDEX: ${{ secrets.ARTIFACTORY_CARGO_INDEX }}
          EXPECT_VERSION: ${{ needs.prepare-release.outputs.crate_version }}
          # Same rc-suffixed value: stage_crates.py rewrites every publishable crate's
          # version to this before publishing, so re-stages don't collide on the
          # immutable cargo registry. GA promotion (GitLab) republishes the clean version.
          STAGE_VERSION: ${{ needs.prepare-release.outputs.crate_version }}
          CRATE_SUBSET: ${{ inputs.crates }}
        run: |
          # The step shell is GitHub's implicit `bash -e {0}` (set -uo does not
          # clear -e): every fallible command below is guarded so the outputs are
          # written even when the publish fails partway; the real exit code is
          # raised at the end.
          set -uo pipefail
          # Mask the registry host: cargo error output embeds the index URL in
          # forms (https://<host>/...config.json) that exact-secret masking of
          # ARTIFACTORY_CARGO_INDEX does not catch.
          CARGO_HOST=$(printf '%s' "${ARTIFACTORY_CARGO_INDEX}" | sed -E 's#^sparse\+##; s#^https?://##; s#[/:].*$##')
          if [ -n "${CARGO_HOST}" ]; then echo "::add-mask::${CARGO_HOST}"; fi
          # Run stage_crates.py inside the ECR build image: repo mounted at /work, cargo uses
          # the image toolchain, root so it can write Cargo.lock/.cargo and the target dir.
          rc=0
          docker run --rm --user 0:0 \
            --network host \
            --ulimit nofile=65536:65536 \
            -v "${GITHUB_WORKSPACE}:/work" -w /work \
            -e HOME=/tmp \
            -e RUSTFLAGS="--cfg tokio_unstable" \
            -e CARGO_TARGET_DIR=/tmp/cargo-target \
            -e ARTIFACTORY_TOKEN \
            -e ARTIFACTORY_CARGO_INDEX \
            -e EXPECT_VERSION \
            -e STAGE_VERSION \
            -e CRATE_SUBSET \
            "${ECR_IMAGE}" \
            bash -lc 'python3 .ondemand-tools/.github/scripts/stage_crates.py --registry artifactory --only "${CRATE_SUBSET}"' \
            2>&1 | tee /tmp/stage_crates.log || rc=$?
          # `|| rc=$?` keeps a nonzero exit non-fatal: the step shell is `bash -e {0}`
          # (set -uo above does NOT clear -e), and errexit here would skip the
          # output-writing tail — crates_status could never be 'partial' and the
          # staged list would be lost on failure. pipefail makes rc docker's code.

          # stage_crates.py emits `STAGED_CRATE=<name>` and `FAILED_CRATE=<name> <reason>` lines.
          STAGED="$(sed -n 's/^STAGED_CRATE=//p' /tmp/stage_crates.log | paste -sd, - || true)"
          FAILED="$(sed -n 's/^FAILED_CRATE=\([^ ]*\).*/\1/p' /tmp/stage_crates.log | paste -sd, - || true)"
          # Idempotent skips: the registry is immutable, so an already-present
          # version keeps its FIRST content — an all-'already' run is a no-op
          # re-stage, not a fresh publish, and must be tellable from the summary.
          ALREADY="$(grep -c 'already on the registry (skip)' /tmp/stage_crates.log || true)"
          if [ "${rc}" -eq 0 ]; then
            STATUS="success"
          elif [ -n "${STAGED}" ]; then
            STATUS="partial"
          else
            STATUS="failed"
          fi
          {
            echo "staged_crates=${STAGED}"
            echo "failed_crates=${FAILED}"
            echo "crates_status=${STATUS}"
          } >> "${GITHUB_OUTPUT}"
          {
            echo "### Crate staging: ${STATUS}"
            echo ""
            echo "Staged: \`${STAGED:-<none>}\`"
            echo ""
            echo "Failed: \`${FAILED:-<none>}\`"
            if [ "${ALREADY:-0}" -gt 0 ]; then
              echo ""
              echo "Already present (idempotent skip, content unchanged from the first staging at this version): ${ALREADY}"
            fi
            if [ "${NIGHTLY}" = "true" ]; then
              echo ""
              echo "Nightly consumers must pin with '=' (e.g. \`= ${STAGE_VERSION}\`): a bare pre-release pin resolves upward to any later rc/GA on the index."
            fi
          } >> "${GITHUB_STEP_SUMMARY}"
          echo "Staged crates: ${STAGED:-<none>}"
          if [ "${rc}" -ne 0 ]; then
            echo "::error::Crate staging ${STATUS}: failed=[${FAILED:-unknown}] staged=[${STAGED:-none}] — the release continues; re-stage crates with a bumped rc_round after fixing."
          fi
          exit "${rc}"

  # Combined NGC + Artifactory (wheels + crates) staging reply on the thread.
  notify-slack-staging:
    name: Notify Slack — staging complete
    runs-on: prod-default-small-v2
    needs: [prepare-release, release-publish, stage-wheels-artifactory, stage-crates-artifactory]
    if: ${{ !cancelled() && inputs.slack_thread_ts != '' }}
    permissions:
      contents: read
    steps:
      - name: Post staging reply
        continue-on-error: true
        env:
          SLACK_BOT_TOKEN:  ${{ secrets.SLACK_RELEASE_BOT_TOKEN }}
          SLACK_CHANNEL_ID: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
          SLACK_THREAD_TS:  ${{ inputs.slack_thread_ts }}
          PREP_RESULT:      ${{ needs.prepare-release.result }}
          NGC_RESULT:       ${{ needs.release-publish.result }}
          NGC_FAILED_COPIES: ${{ needs.release-publish.outputs.failed_copy_count }}
          ART_RESULT:       ${{ needs.stage-wheels-artifactory.result }}
          # continue-on-error masks the job result; the output is authoritative.
          # Empty output = died before the publish step: treat as failed.
          CRATES_RESULT:    ${{ needs.stage-crates-artifactory.result == 'skipped' && 'skipped' || needs.stage-crates-artifactory.outputs.crates_status || 'failed' }}
          NGC_VERSION_TAG:  ${{ needs.prepare-release.outputs.ngc_version_tag }}
          WHEEL_VERSION:    ${{ needs.prepare-release.outputs.wheel_version }}
          CRATE_VERSION:    ${{ needs.prepare-release.outputs.crate_version }}
          RUN_URL:          ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        shell: bash
        run: |
          set -euo pipefail

          emoji_for() {
            case "$1" in
              success)   echo ":white_check_mark:" ;;
              partial)   echo ":warning:" ;;
              failure|failed) echo ":x:" ;;
              skipped)   echo ":fast_forward:" ;;
              cancelled) echo ":no_entry_sign:" ;;
              *)         echo ":grey_question:" ;;
            esac
          }
          NGC_EMOJI=$(emoji_for "${NGC_RESULT}")
          ART_EMOJI=$(emoji_for "${ART_RESULT}")
          CRATES_EMOJI=$(emoji_for "${CRATES_RESULT:-skipped}")

          # A deliberately skipped stage (subset 'none') is not an issue.
          ok() { case "$1" in success|skipped|'') return 0 ;; *) return 1 ;; esac; }
          if [ "${PREP_RESULT}" != "success" ]; then
            HEADER=":x: Staging failed before it started (prepare-release: ${PREP_RESULT})"
          elif ok "${NGC_RESULT}" && ok "${ART_RESULT}" && ok "${CRATES_RESULT:-}"; then
            # Copy failures can be non-fatal (optional-image warn-skips) —
            # a green job with skipped copies must not read as "complete".
            if [ -n "${NGC_FAILED_COPIES:-}" ] && [ "${NGC_FAILED_COPIES}" != "0" ]; then
              HEADER=":warning: Staging complete with ${NGC_FAILED_COPIES} skipped image copy(ies)"
            else
              HEADER=":package: Staging complete"
            fi
          else
            HEADER=":warning: Staging finished with issues"
          fi

          PAYLOAD=$(jq -n \
            --arg channel       "${SLACK_CHANNEL_ID}" \
            --arg thread_ts     "${SLACK_THREAD_TS}" \
            --arg header        "${HEADER}" \
            --arg ngc_emoji     "${NGC_EMOJI}" \
            --arg ngc_result    "${NGC_RESULT}" \
            --arg art_emoji     "${ART_EMOJI}" \
            --arg art_result    "${ART_RESULT}" \
            --arg crates_emoji  "${CRATES_EMOJI}" \
            --arg crates_result "${CRATES_RESULT:-skipped}" \
            --arg ngc_tag       "${NGC_VERSION_TAG}" \
            --arg wheel         "${WHEEL_VERSION}" \
            --arg crate_ver     "${CRATE_VERSION:-n/a}" \
            --arg run_url       "${RUN_URL}" \
            '{
              channel: $channel,
              thread_ts: $thread_ts,
              text: $header,
              blocks: [
                { type: "section",
                  text: { type: "mrkdwn", text: ("*" + $header + "*") } },
                { type: "section",
                  fields: [
                    { type: "mrkdwn",
                      text: ($ngc_emoji + " *NGC publish:* " + $ngc_result + "\n`" + $ngc_tag + "`") },
                    { type: "mrkdwn",
                      text: ($art_emoji + " *Artifactory wheels:* " + $art_result + "\n`" + $wheel + "`") },
                    { type: "mrkdwn",
                      text: ($crates_emoji + " *Artifactory crates:* " + $crates_result + "\n`" + $crate_ver + "`") }
                  ] },
                { type: "context",
                  elements: [ { type: "mrkdwn", text: ("<" + $run_url + "|GitHub Actions run>") } ] }
              ]
            }')

          RESPONSE=$(curl -fsSL -X POST https://slack.com/api/chat.postMessage \
            -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \
            -H "Content-type: application/json; charset=utf-8" \
            --data "${PAYLOAD}")

          OK=$(echo "${RESPONSE}" | jq -r '.ok')
          if [ "${OK}" != "true" ]; then
            ERR=$(echo "${RESPONSE}" | jq -r '.error // "unknown"')
            echo "::error::Slack chat.postMessage failed: ${ERR}"
            exit 1
          fi
          echo "Posted staging reply"

  # ============================================================================
  # GITLAB RELEASE AUTOMATION TRIGGER
  # Fires the GitLab release-automation pipeline once images and wheels are
  # staged. Variable shape consumed by
  # GITLAB/release-automation/projects/dynamo.yml. Skipped when
  # `skip_gitlab_pipeline` is set (emergency bypass).
  # ============================================================================

  trigger-gitlab-release-pipeline:
    name: Trigger GitLab Release Pipeline
    # Every SELECTED stage must succeed (a 'none' subset skips the stage and
    # counts as satisfied), so GitLab is never told to scan or promote something
    # that failed to stage. A failed rc is repaired by re-staging at a new rc
    # round. On-demand also requires crates to have FULLY staged: stage-crates is
    # continue-on-error (its job result is masked as success), so crates_status
    # is the authoritative signal. Nightly crates stay fail-soft (not gated).
    # !cancelled() overrides the implicit needs-success().
    needs: [prepare-release, release-publish, stage-wheels-artifactory, stage-crates-artifactory]
    if: >-
      ${{ !cancelled() && needs.prepare-release.result == 'success' && inputs.skip_gitlab_pipeline != true &&
          (needs.release-publish.result == 'success' || (inputs.containers == 'none' && inputs.helm == 'none')) &&
          (needs.stage-wheels-artifactory.result == 'success' || inputs.wheels == 'none') &&
          (needs.prepare-release.outputs.on_demand != 'true' || inputs.crates == 'none' || inputs.crates == '' ||
           needs.stage-crates-artifactory.outputs.crates_status == 'success') }}
    runs-on:
      group: gitlab_ci_runners
    environment: automated-release
    outputs:
      pipeline_id:  ${{ steps.trigger.outputs.pipeline_id }}
      pipeline_url: ${{ steps.trigger.outputs.pipeline_url }}
    steps:
      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
      - *ops-token
      - *ops-check
      - name: Trigger GitLab pipeline
        id: trigger
        env:
          # Both come from the automated-release environment. Storing the URL
          # as a secret (not hardcoded) keeps it masked in logs and lets it be
          # rotated without a code change.
          GITLAB_TRIGGER_URL:   ${{ secrets.GITLAB_PIPELINE_URL }}
          GITLAB_TRIGGER_TOKEN: ${{ secrets.GITLAB_TRIGGER_TOKEN }}
          # release-automation ref to run (var-overridable for testing a GitLab
          # dev branch). GitLab does NOT force dry-run on non-default refs, and
          # only main carries its dry-run guards — keep the default on main.
          GITLAB_REF:           ${{ vars.GITLAB_RELEASE_AUTOMATION_REF || 'main' }}
          VERSION:              ${{ needs.prepare-release.outputs.version }}
          NGC_VERSION_TAG:      ${{ needs.prepare-release.outputs.ngc_version_tag }}
          WHEEL_VERSION:        ${{ needs.prepare-release.outputs.wheel_version }}
          CRATE_VERSION:        ${{ needs.prepare-release.outputs.crate_version }}
          COMMIT_SHA:           ${{ needs.prepare-release.outputs.commit_sha }}
          NIGHTLY:              ${{ needs.prepare-release.outputs.nightly }}
          ON_DEMAND:            ${{ needs.prepare-release.outputs.on_demand }}
          RC_NUMBER:            ${{ needs.prepare-release.outputs.rc_number }}
          CONTAINER_SUBSET:     ${{ inputs.containers }}
          WHEEL_SUBSET:         ${{ inputs.wheels }}
          CRATE_SUBSET:         ${{ inputs.crates }}
          HELM_SUBSET:          ${{ inputs.helm }}
          # Artifacts ACTUALLY staged (names/filenames only — URLs/registry/org are secrets,
          # GitLab re-prefixes with its own bases). Forwarded so GitLab's nspect step
          # registers exactly what landed, not what was requested. Empty if a stage was
          # skipped or failed wholesale. STAGED_HELM covers both the rc/on-demand helm
          # step and the nightly helm step (chart:version pairs).
          STAGED_CRATES:        ${{ needs.stage-crates-artifactory.outputs.staged_crates }}
          STAGED_CONTAINERS:    ${{ needs.release-publish.outputs.staged_containers }}
          STAGED_WHEELS:        ${{ needs.stage-wheels-artifactory.outputs.staged_wheels }}
          STAGED_HELM:          ${{ needs.release-publish.outputs.staged_helm }}
          # success|partial|failed|skipped (crate staging is fail-soft; empty
          # output = died before the publish step, treated as failed).
          CRATES_STATUS:        ${{ needs.stage-crates-artifactory.result == 'skipped' && 'skipped' || needs.stage-crates-artifactory.outputs.crates_status || 'failed' }}
          # Forwarded so the GitLab pipeline can reply on the same thread.
          SLACK_THREAD_TS:      ${{ inputs.slack_thread_ts }}
          SLACK_CHANNEL_ID:     ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
        run: |
          set -euo pipefail

          if [ "${GITLAB_REF}" != "main" ]; then
            echo "::warning::GitLab pipeline ref override active: '${GITLAB_REF}' (vars.GITLAB_RELEASE_AUTOMATION_REF). This runs that ref's pipeline definition with production credentials; publish jobs remain manual-gated but are LIVE."
          fi

          # PIPELINE_TYPE / RELEASE_TYPE drive every rule in dynamo.yml.
          # ARTIFACTS gates per-artifact scans and promotions.
          # NIGHTLY_TAG carries the resolved nightly tag (date+shortsha) for
          # the GitLab pipeline summary; the nspect version-name is hardcoded
          # to "Dynamo Nightlies" inside dynamo.yml, so we don't pass it.
          RELEASE_TAG=""
          RELEASE_VERSION_VAR="${VERSION}"
          if [ "${NIGHTLY}" = "true" ]; then
            PIPELINE_TYPE="security"
            RELEASE_TYPE="nightly"
            ARTIFACTS="wheel,container"
            # helm only when charts were actually pushed (fail-soft).
            if [ -n "${STAGED_HELM}" ]; then
              ARTIFACTS="${ARTIFACTS},helm"
            fi
            # Nightly crates: staged fail-soft to the internal cargo registry at
            # the dated -dev.YYYYMMDD version. Never crates.io (immutable,
            # undeletable). Keyed on ACTUALS like helm above — a wholesale-failed
            # staging must not tell GitLab a crate artifact exists.
            if [ -n "${STAGED_CRATES}" ]; then
              ARTIFACTS="${ARTIFACTS},crate"
            fi
            NIGHTLY_TAG="${NGC_VERSION_TAG}"
            RC_TAG=""
            # VERSION is the literal 'nightly'; empty lets GitLab apply its default.
            RELEASE_VERSION_VAR=""
          elif [ "${ON_DEMAND}" = "true" ]; then
            # On-demand runs the security pipeline (scans + nSpect + manual-gated
            # publish); RELEASE_TYPE=ondemand (no hyphen) matches dynamo.yml's rules.
            PIPELINE_TYPE="security"
            RELEASE_TYPE="ondemand"
            # Coarse ARTIFACTS list keyed on what ACTUALLY staged (like nightly),
            # not on what was requested: GitLab must never scan or promote a type
            # that did not land. The job-level gate already requires every
            # selected stage to succeed; this is the second line of defense.
            ARTIFACTS=""
            [ -n "${STAGED_WHEELS}" ]     && ARTIFACTS="${ARTIFACTS},wheel"
            [ -n "${STAGED_CONTAINERS}" ] && ARTIFACTS="${ARTIFACTS},container"
            [ -n "${STAGED_CRATES}" ]     && ARTIFACTS="${ARTIFACTS},crate"
            [ -n "${STAGED_HELM}" ]       && ARTIFACTS="${ARTIFACTS},helm"
            ARTIFACTS="${ARTIFACTS#,}"
            # A selected type that staged nothing is a failed stage, even when its
            # job reported success (helm push is continue-on-error). Refuse, so
            # the operator re-stages as a sub-round (rc_round X.1) instead of a silent gap.
            MISSING=""
            for P in "wheel:${WHEEL_SUBSET:-all}:${STAGED_WHEELS}" \
                     "container:${CONTAINER_SUBSET:-all}:${STAGED_CONTAINERS}" \
                     "crate:${CRATE_SUBSET:-none}:${STAGED_CRATES}" \
                     "helm:${HELM_SUBSET:-all}:${STAGED_HELM}"; do
              IFS=':' read -r KIND SEL GOT <<<"${P}"
              if [ "${SEL}" != "none" ] && [ -z "${GOT}" ]; then MISSING="${MISSING} ${KIND}"; fi
            done
            if [ -n "${MISSING}" ]; then
              echo "::error::On-demand run selected but did not stage:${MISSING}. Refusing to trigger GitLab; fix and re-stage as a sub-round of the same rc (rc_round X.1, X.2, ...)."
              exit 1
            fi
            NIGHTLY_TAG=""
            RC_TAG=""
            RELEASE_TAG="v${VERSION}"
          else
            PIPELINE_TYPE="rc"
            RELEASE_TYPE="rc"
            # Subset-derived; defaults produce the historical "wheel,container,helm".
            ARTIFACTS=""
            [ "${WHEEL_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},wheel"
            [ "${CONTAINER_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},container"
            [ "${HELM_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},helm"
            ARTIFACTS="${ARTIFACTS#,}"
            NIGHTLY_TAG=""
            RC_TAG="v${VERSION}-rc${RC_NUMBER}"
          fi

          case "${ARTIFACTS}" in *crate*) ENABLE_CRATE_SCAN="true" ;; *) ENABLE_CRATE_SCAN="false" ;; esac

          # Pipe the trigger token via @file rather than inline form value so
          # it never appears in process listings or `set -x` output.
          # `trap` is registered BEFORE the write so an error between mktemp
          # and the write (or any later line) still cleans up the file.
          TOKEN_FILE=$(mktemp)
          trap 'rm -f "${TOKEN_FILE}"' EXIT
          chmod 600 "${TOKEN_FILE}"
          printf '%s' "${GITLAB_TRIGGER_TOKEN}" > "${TOKEN_FILE}"

          # --form-string everywhere: --form treats a leading '<'/'@' as a file
          # reference. Only the token deliberately uses --form "<file".
          RESPONSE=$(curl -fsSL \
            --request POST \
            --form "token=<${TOKEN_FILE}" \
            --form-string "ref=${GITLAB_REF}" \
            --form-string "variables[PIPELINE_TYPE]=${PIPELINE_TYPE}" \
            --form-string "variables[RELEASE_TYPE]=${RELEASE_TYPE}" \
            --form-string "variables[DRY_RUN]=false" \
            --form-string "variables[PROJECT]=dynamo" \
            --form-string "variables[NIGHTLY_TAG]=${NIGHTLY_TAG}" \
            --form-string "variables[RC_TAG]=${RC_TAG}" \
            --form-string "variables[RELEASE_TAG]=${RELEASE_TAG}" \
            --form-string "variables[RELEASE_VERSION]=${RELEASE_VERSION_VAR}" \
            --form-string "variables[WHEEL_VERSION]=${WHEEL_VERSION}" \
            --form-string "variables[CRATE_VERSION]=${CRATE_VERSION}" \
            --form-string "variables[CONTAINER_TAG]=${NGC_VERSION_TAG}" \
            --form-string "variables[RC_NUMBER]=${RC_NUMBER}" \
            --form-string "variables[COMMIT_SHA]=${COMMIT_SHA}" \
            --form-string "variables[GITHUB_RUN_ID]=${GITHUB_RUN_ID}" \
            --form-string "variables[ARTIFACTS]=${ARTIFACTS}" \
            --form-string "variables[CONTAINER_SUBSET]=${CONTAINER_SUBSET}" \
            --form-string "variables[WHEEL_SUBSET]=${WHEEL_SUBSET}" \
            --form-string "variables[CRATE_SUBSET]=${CRATE_SUBSET}" \
            --form-string "variables[HELM_SUBSET]=${HELM_SUBSET}" \
            --form-string "variables[STAGED_CRATES]=${STAGED_CRATES}" \
            --form-string "variables[STAGED_CONTAINERS]=${STAGED_CONTAINERS}" \
            --form-string "variables[STAGED_WHEELS]=${STAGED_WHEELS}" \
            --form-string "variables[STAGED_HELM]=${STAGED_HELM}" \
            --form-string "variables[CRATES_STATUS]=${CRATES_STATUS}" \
            --form-string "variables[ENABLE_PULSE_SCAN]=true" \
            --form-string "variables[ENABLE_WHEEL_SCAN]=true" \
            --form-string "variables[ENABLE_CRATE_SCAN]=${ENABLE_CRATE_SCAN}" \
            --form-string "variables[ENABLE_SONARQUBE]=true" \
            --form-string "variables[SONAR_BRANCH]=main" \
            --form-string "variables[NSPECT_REGISTERED]=false" \
            --form-string "variables[SLACK_THREAD_TS]=${SLACK_THREAD_TS}" \
            --form-string "variables[SLACK_CHANNEL_ID]=${SLACK_CHANNEL_ID}" \
            "${GITLAB_TRIGGER_URL}")

          PIPELINE_ID=$(echo "${RESPONSE}" | jq -r '.id // empty')
          PIPELINE_URL=$(echo "${RESPONSE}" | jq -r '.web_url // empty')
          if [ -z "${PIPELINE_ID}" ]; then
            # Don't echo full RESPONSE — GitLab echoes the submitted `variables`
            # block back, including SLACK_THREAD_TS / SLACK_CHANNEL_ID. Pull
            # just the error fields. `|| true` survives unparseable bodies.
            ERR=$(echo "${RESPONSE}" | jq -r '.message // .error_description // .error // "unparseable response"' 2>/dev/null || echo "unparseable response")
            echo "::error::Failed to trigger GitLab pipeline: ${ERR}"
            exit 1
          fi

          echo "pipeline_id=${PIPELINE_ID}"   >> "${GITHUB_OUTPUT}"
          echo "pipeline_url=${PIPELINE_URL}" >> "${GITHUB_OUTPUT}"
          echo "GitLab release pipeline triggered: pipeline ID ${PIPELINE_ID}"

          # web_url embeds the internal GitLab host (kept secret) — public
          # summary carries the numeric ID only; the URL goes to Slack.
          {
            echo "## GitLab Release Pipeline"
            echo ""
            echo "| Property | Value |"
            echo "|----------|-------|"
            echo "| Pipeline ID | ${PIPELINE_ID} |"
            echo "| PIPELINE_TYPE | ${PIPELINE_TYPE} |"
            echo "| RELEASE_TYPE | ${RELEASE_TYPE} |"
            echo "| Version | ${VERSION} |"
            echo "| NGC tag | ${NGC_VERSION_TAG} |"
            echo "| Commit | ${COMMIT_SHA} |"
            echo "| Artifacts | ${ARTIFACTS} |"
            echo "| Crate staging | ${CRATES_STATUS:-n/a} |"
          } >> "${GITHUB_STEP_SUMMARY}"

      - name: Post GitLab trigger reply on Slack thread
        if: ${{ inputs.slack_thread_ts != '' }}
        continue-on-error: true
        env:
          SLACK_BOT_TOKEN:  ${{ secrets.SLACK_RELEASE_BOT_TOKEN }}
          SLACK_CHANNEL_ID: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
          SLACK_THREAD_TS:  ${{ inputs.slack_thread_ts }}
          PIPELINE_ID:      ${{ steps.trigger.outputs.pipeline_id }}
          PIPELINE_URL:     ${{ steps.trigger.outputs.pipeline_url }}
          NGC_VERSION_TAG:  ${{ needs.prepare-release.outputs.ngc_version_tag }}
        shell: bash
        run: |
          set -euo pipefail

          PAYLOAD=$(jq -n \
            --arg channel   "${SLACK_CHANNEL_ID}" \
            --arg thread_ts "${SLACK_THREAD_TS}" \
            --arg pid       "${PIPELINE_ID}" \
            --arg purl      "${PIPELINE_URL}" \
            --arg ngc_tag   "${NGC_VERSION_TAG}" \
            '{
              channel: $channel,
              thread_ts: $thread_ts,
              text: ("GitLab release pipeline triggered: " + $pid),
              blocks: [
                { type: "section",
                  text: { type: "mrkdwn",
                          text: (":rocket: *GitLab release pipeline triggered*\n<" + $purl + "|Pipeline " + $pid + ">") } },
                { type: "context",
                  elements: [ { type: "mrkdwn",
                                text: ("NGC tag: `" + $ngc_tag + "`") } ] }
              ]
            }')

          RESPONSE=$(curl -fsSL -X POST https://slack.com/api/chat.postMessage \
            -H "Authorization: Bearer ${SLACK_BOT_TOKEN}" \
            -H "Content-type: application/json; charset=utf-8" \
            --data "${PAYLOAD}")

          OK=$(echo "${RESPONSE}" | jq -r '.ok')
          if [ "${OK}" != "true" ]; then
            ERR=$(echo "${RESPONSE}" | jq -r '.error // "unknown"')
            echo "::error::Slack chat.postMessage failed: ${ERR}"
            exit 1
          fi
          echo "Posted GitLab trigger reply"
  # ============================================================================
  # LINEAR RELEASE SYNC
  # ============================================================================

  linear-release-sync:
    name: Sync to Linear Release
    needs: [prepare-release, release-publish]
    # Skip on nightly: prepare-release sets version=nightly, which would create a
    # spurious "Dynamo vnightly" release in Linear. Also skip on-demand runs:
    # they re-stage an existing version (possibly a .devN test version) whose
    # Linear release is owned by the original RC flow. Only sync for RC/release runs.
    if: >-
      needs.release-publish.result == 'success' &&
      needs.prepare-release.outputs.nightly != 'true' &&
      needs.prepare-release.outputs.on_demand != 'true'
    runs-on: ubuntu-latest
    continue-on-error: true
    steps:
      - name: Checkout at source commit
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          ref: ${{ needs.prepare-release.outputs.commit_sha }}
          fetch-depth: 0

      - name: Sync issues to Linear release
        # Pinned to an audited commit SHA (not a mutable @v0 tag) since this step
        # receives LINEAR_ACCESS_KEY. Bump deliberately after reviewing upstream.
        uses: linear/linear-release-action@c0cb8354a362c24c6d3e0948f37fd66d07588e3f  # v0.14.5
        with:
          access_key: ${{ secrets.LINEAR_ACCESS_KEY }}
          version: ${{ needs.prepare-release.outputs.version }}
          name: Dynamo v${{ needs.prepare-release.outputs.version }}