diff --git a/.github/scripts/check-release-node-image.sh b/.github/scripts/check-release-node-image.sh new file mode 100755 index 0000000000..84364550cc --- /dev/null +++ b/.github/scripts/check-release-node-image.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Return 0 when both the immutable release tag and :latest identify the +# expected source commit, 1 when publication is missing/stale, and 2 when GHCR +# cannot provide an authoritative answer. +set -euo pipefail + +release_tag="${1:?release tag required}" +expected_sha="${2:?expected source SHA required}" +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY required}" + +[[ "$release_tag" =~ ^v[0-9]+$ ]] \ + || { echo "release tag must have the form vN" >&2; exit 2; } +[[ "$expected_sha" =~ ^[0-9a-f]{40}$ ]] \ + || { echo "expected source must be a lowercase 40-byte Git SHA" >&2; exit 2; } +[[ "$GITHUB_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] \ + || { echo "invalid GitHub repository: $GITHUB_REPOSITORY" >&2; exit 2; } + +image_repository=$(printf '%s' "$GITHUB_REPOSITORY" | tr '[:upper:]' '[:lower:]') +token=$(curl -fsSL --get \ + --data-urlencode "scope=repository:$image_repository:pull" \ + https://ghcr.io/token \ + | jq -er '.token // .access_token') \ + || { echo "could not obtain a GHCR pull token" >&2; exit 2; } + +tag_matches_release() { + local tag="$1" + local response http_code manifest jq_status + + response=$(curl -sS -w $'\n%{http_code}' \ + -H "Authorization: Bearer $token" \ + -H 'Accept: application/vnd.oci.image.index.v1+json' \ + "https://ghcr.io/v2/$image_repository/manifests/$tag") \ + || return 2 + http_code="${response##*$'\n'}" + manifest="${response%$'\n'*}" + + case "$http_code" in + 200) ;; + 404) return 1 ;; + *) + echo "GHCR returned HTTP $http_code for $image_repository:$tag" >&2 + return 2 + ;; + esac + + if jq -e \ + --arg revision "$expected_sha" \ + --arg version "$release_tag" ' + .annotations["org.opencontainers.image.revision"] == $revision and + .annotations["org.opencontainers.image.version"] == $version + ' <<<"$manifest" >/dev/null; then + return 0 + else + jq_status=$? + [[ "$jq_status" -eq 1 ]] && return 1 + echo "GHCR returned malformed metadata for $image_repository:$tag" >&2 + return 2 + fi +} + +for tag in "$release_tag" latest; do + status=0 + tag_matches_release "$tag" || status=$? + case "$status" in + 0) ;; + 1) + echo "$image_repository:$tag does not identify $release_tag at $expected_sha" >&2 + exit 1 + ;; + *) exit 2 ;; + esac +done + +echo "$image_repository:$release_tag and :latest identify $expected_sha" diff --git a/.github/scripts/test-check-release-node-image.sh b/.github/scripts/test-check-release-node-image.sh new file mode 100755 index 0000000000..1a6a61942f --- /dev/null +++ b/.github/scripts/test-check-release-node-image.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +repo_root=$(cd "$script_dir/../.." && pwd) +helper="$script_dir/check-release-node-image.sh" +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" + +cat > "$tmp/bin/curl" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail +url="${!#}" +if [[ "$url" == https://ghcr.io/token ]]; then + echo '{"token":"test-token"}' + exit 0 +fi +tag="${url##*/}" +case "$MOCK_SCENARIO:$tag" in + current:v443|current:latest) + printf '{"annotations":{"org.opencontainers.image.revision":"%s","org.opencontainers.image.version":"v443"}}\n200' "$EXPECTED_SHA" + ;; + missing:v443) + printf '{"errors":[]}\n404' + ;; + missing:latest) + printf '{}\n404' + ;; + stale-latest:v443) + printf '{"annotations":{"org.opencontainers.image.revision":"%s","org.opencontainers.image.version":"v443"}}\n200' "$EXPECTED_SHA" + ;; + stale-latest:latest) + printf '{"annotations":{"org.opencontainers.image.revision":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","org.opencontainers.image.version":"v442"}}\n200' + ;; + malformed:v443) + printf '{\n200' + ;; + unavailable:v443) + printf '{}\n503' + ;; + *) + echo "unexpected mock request: $MOCK_SCENARIO $url" >&2 + exit 2 + ;; +esac +MOCK +chmod +x "$tmp/bin/curl" + +export PATH="$tmp/bin:$PATH" +export GITHUB_REPOSITORY=RaoFoundation/subtensor +export EXPECTED_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + +MOCK_SCENARIO=current "$helper" v443 "$EXPECTED_SHA" + +for scenario in missing stale-latest; do + if MOCK_SCENARIO="$scenario" "$helper" v443 "$EXPECTED_SHA"; then + echo "$scenario publication unexpectedly reported current" >&2 + exit 1 + else + status=$? + [[ "$status" -eq 1 ]] || { echo "$scenario returned $status, expected 1" >&2; exit 1; } + fi +done + +for scenario in malformed unavailable; do + if MOCK_SCENARIO="$scenario" "$helper" v443 "$EXPECTED_SHA"; then + echo "$scenario registry response unexpectedly succeeded" >&2 + exit 1 + else + status=$? + [[ "$status" -eq 2 ]] || { echo "$scenario returned $status, expected 2" >&2; exit 1; } + fi +done + +docker_watcher="$repo_root/.github/workflows/watch-mainnet-docker.yml" +release_watcher="$repo_root/.github/workflows/watch-mainnet-release.yml" +docker_workflow="$repo_root/.github/workflows/docker.yml" + +grep -qF 'uses: ./.github/workflows/docker.yml' "$docker_watcher" +grep -qF 'expected_sha: ${{ needs.check.outputs.release_sha }}' "$docker_watcher" +! grep -qF 'environment: mainnet' "$docker_watcher" +! grep -qF 'gh workflow run docker.yml' "$release_watcher" +grep -qF 'workflow_call:' "$docker_workflow" +grep -qF 'index:org.opencontainers.image.revision=' "$docker_workflow" + +echo "release node image publication checks passed" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ed59a983eb..529e00a88d 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -1,13 +1,24 @@ name: Publish Docker Image +run-name: Docker image ${{ inputs.tag || github.ref_name }} from ${{ inputs.expected_sha || github.sha }} # Node images publish whenever main or a network mirror moves, on runtime # releases, and on demand. Main advances the development-facing :main tag. -# Releases cut by watch-mainnet-release.yml use the default GITHUB_TOKEN, which -# never emits `release: published`; the watcher therefore dispatches this -# workflow directly with the release tag. Release-version tags (vN) move +# Releases cut by automation use the default GITHUB_TOKEN, which never emits +# `release: published`; watch-mainnet-docker.yml therefore calls this workflow +# directly with the finalized release tag. Release-version tags (vN) move # :latest; branch tags do not. on: + workflow_call: + inputs: + tag: + description: "Immutable ref and Docker tag to publish" + required: true + type: string + expected_sha: + description: "Require the selected ref to resolve to this commit" + required: true + type: string release: types: [published] push: @@ -20,7 +31,7 @@ on: default: "" concurrency: - group: docker-${{ github.ref }} + group: docker-${{ inputs.expected_sha || github.ref }} cancel-in-progress: true permissions: @@ -37,11 +48,21 @@ jobs: with: # Push/release events carry the immutable promoted commit. Manual # dispatches may override it with a branch or tag. - ref: ${{ github.event.inputs.tag || github.sha }} + ref: ${{ inputs.tag || github.event.inputs.tag || github.sha }} - name: Resolve immutable source revision id: ref - run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" + env: + EXPECTED_SHA: ${{ inputs.expected_sha }} + run: | + sha=$(git rev-parse HEAD) + if [[ -n "$EXPECTED_SHA" ]]; then + [[ "$EXPECTED_SHA" =~ ^[0-9a-f]{40}$ ]] \ + || { echo "expected_sha must be a full lowercase commit SHA"; exit 1; } + [[ "$sha" == "$EXPECTED_SHA" ]] \ + || { echo "selected source $sha does not match expected $EXPECTED_SHA"; exit 1; } + fi + echo "sha=$sha" >> "$GITHUB_OUTPUT" binary: name: production binary (${{ matrix.platform.arch }}) @@ -124,7 +145,7 @@ jobs: steps: - name: Determine tag and image name env: - INPUT_TAG: ${{ github.event.inputs.tag || github.ref_name }} + INPUT_TAG: ${{ inputs.tag || github.event.inputs.tag || github.ref_name }} run: | # Docker tags cannot contain '/', so derive the tag by replacing any # disallowed characters — otherwise a ref like `feat/x` fails tag @@ -181,3 +202,6 @@ jobs: tags: | ${{ env.image }}:${{ env.tag }} ${{ env.latest_tag == 'true' && format('{0}:latest', env.image) || '' }} + annotations: | + index:org.opencontainers.image.revision=${{ needs.setup.outputs.sha }} + index:org.opencontainers.image.version=${{ env.tag }} diff --git a/.github/workflows/watch-mainnet-docker.yml b/.github/workflows/watch-mainnet-docker.yml new file mode 100644 index 0000000000..635c1b68a5 --- /dev/null +++ b/.github/workflows/watch-mainnet-docker.yml @@ -0,0 +1,118 @@ +name: Watch On-chain Runtime Docker + +# Docker publication is intentionally independent from GitHub release and +# package publication. The latter use the protected `mainnet` environment and +# may wait for approval; a finalized on-chain runtime must still produce its +# immutable image tag and advance :latest without human intervention. +on: + schedule: + - cron: "*/10 * * * *" + workflow_dispatch: + +concurrency: + group: watch-mainnet-docker + cancel-in-progress: false + +env: + MAINNET_HTTP: https://entrypoint-finney.opentensor.ai:443 + +permissions: + contents: read + actions: read + +jobs: + check: + name: Resolve finalized runtime image + runs-on: ubuntu-latest + outputs: + publish_needed: ${{ steps.resolve.outputs.publish_needed }} + release_tag: ${{ steps.resolve.outputs.release_tag }} + release_sha: ${{ steps.resolve.outputs.release_sha }} + steps: + - uses: actions/checkout@v4 + + - name: Validate release image publication contract + run: ./.github/scripts/test-check-release-node-image.sh + + - name: Resolve finalized runtime and registry state + id: resolve + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + local_spec=$(grep -Eo 'spec_version: *[0-9]+' runtime/src/lib.rs | head -n 1 | grep -Eo '[0-9]+') + : ${local_spec:?could not parse spec_version from runtime/src/lib.rs} + + finalized_head=$(curl -sf -H 'Content-Type: application/json' \ + -d '{"id":1,"jsonrpc":"2.0","method":"chain_getFinalizedHead","params":[]}' \ + "$MAINNET_HTTP" | jq -er '.result | strings | select(test("^0x[0-9a-f]{64}$"))') + runtime_request=$(jq -cn --arg block "$finalized_head" \ + '{id:1,jsonrpc:"2.0",method:"state_getRuntimeVersion",params:[$block]}') + chain_spec=$(curl -sf -H 'Content-Type: application/json' \ + -d "$runtime_request" "$MAINNET_HTTP" | jq -er '.result.specVersion') + code_hash_request=$(jq -cn --arg block "$finalized_head" \ + '{id:1,jsonrpc:"2.0",method:"state_getStorageHash",params:["0x3a636f6465",$block]}') + chain_code_hash=$(curl -sf -H 'Content-Type: application/json' \ + -d "$code_hash_request" "$MAINNET_HTTP" \ + | jq -er '.result | strings | select(test("^0x[0-9a-f]{64}$"))') + + [[ "$local_spec" =~ ^[0-9]+$ && "$chain_spec" =~ ^[0-9]+$ ]] \ + || { echo "spec_version values must be integers"; exit 1; } + if (( chain_spec > local_spec )); then + echo "main does not contain on-chain runtime $chain_spec yet" + echo "publish_needed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + if (( chain_spec <= 432 )); then + echo "runtime $chain_spec predates automated immutable image publication" + echo "publish_needed=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + release_tag="v${chain_spec}" + tag_json=$(gh api "repos/$GITHUB_REPOSITORY/git/ref/tags/$release_tag") + tag_type=$(jq -er '.object.type' <<<"$tag_json") + release_sha=$(jq -er '.object.sha' <<<"$tag_json") + [[ "$tag_type" == commit && "$release_sha" =~ ^[0-9a-f]{40}$ ]] \ + || { echo "$release_tag is not an immutable lightweight commit tag"; exit 1; } + ancestry=$(gh api \ + "repos/$GITHUB_REPOSITORY/compare/${release_sha}...main" --jq '.status') + case "$ancestry" in + ahead|identical) ;; + *) echo "$release_tag commit $release_sha is not an ancestor of main"; exit 1 ;; + esac + + image_status=0 + .github/scripts/check-release-node-image.sh \ + "$release_tag" "$release_sha" || image_status=$? + case "$image_status" in + 0) + echo "publish_needed=false" >> "$GITHUB_OUTPUT" + ;; + 1) + # Bind the Git tag to the exact runtime bytes observed at the + # finalized block before granting the image workflow write access. + .github/scripts/resolve-release-artifact.sh \ + "$chain_spec" "$release_sha" "$chain_code_hash" >/dev/null + echo "publish_needed=true" >> "$GITHUB_OUTPUT" + ;; + *) + echo "could not determine authoritative GHCR state" >&2 + exit 1 + ;; + esac + echo "release_tag=$release_tag" >> "$GITHUB_OUTPUT" + echo "release_sha=$release_sha" >> "$GITHUB_OUTPUT" + + publish: + name: Publish finalized runtime image + needs: check + if: needs.check.outputs.publish_needed == 'true' + permissions: + contents: read + packages: write + uses: ./.github/workflows/docker.yml + with: + tag: ${{ needs.check.outputs.release_tag }} + expected_sha: ${{ needs.check.outputs.release_sha }} diff --git a/.github/workflows/watch-mainnet-release.yml b/.github/workflows/watch-mainnet-release.yml index 2ac010518c..0b8c6971c7 100644 --- a/.github/workflows/watch-mainnet-release.yml +++ b/.github/workflows/watch-mainnet-release.yml @@ -9,17 +9,16 @@ name: Watch Mainnet Release # failed or partial upload: # 1. GitHub release v, with the srtool wasm + digest and # multisig call data from the release train attached as assets -# 2. Docker images via explicit dispatch of docker.yml and -# docker-localnet.yml (a release created with the default GITHUB_TOKEN -# does not emit `release: published`, so their `on: release` triggers -# never fire for releases cut by this workflow) -# 3. Python SDK + bittensor-core wheels to PyPI -# 4. Publishable Rust crates to crates.io -# 5. Production website/docs deployment on Vercel -# 6. Before finalizing the GitHub release, the `mainnet` branch is +# 2. Python SDK + bittensor-core wheels to PyPI +# 3. Publishable Rust crates to crates.io +# 4. Production website/docs deployment on Vercel +# 5. Before finalizing the GitHub release, the `mainnet` branch is # force-updated to the release-train commit, so it always contains the # code running on mainnet (the devnet and testnet branches are updated by # release-train.yml at deploy time) +# +# Docker is reconciled independently by watch-mainnet-docker.yml so protected +# environment approvals in this workflow cannot delay the node image. on: schedule: @@ -328,26 +327,6 @@ jobs: --prerelease=false \ --latest - publish-docker: - name: Dispatch Docker image publishing - needs: [check, release] - if: needs.check.outputs.release_needed == 'true' - runs-on: [self-hosted, fireactions-turbo-8] - permissions: - actions: write - steps: - # Dispatch at the release tag so both workflows check out and tag the - # exact released sha, even if main has moved on. - - name: Dispatch docker workflows at the release tag - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - tag="v${{ needs.check.outputs.spec_version }}" - gh workflow run docker.yml \ - --repo "$GITHUB_REPOSITORY" --ref "$tag" -f tag="$tag" - gh workflow run docker-localnet.yml \ - --repo "$GITHUB_REPOSITORY" --ref "$tag" -f branch-or-tag="$tag" - # Full platform matrix (manylinux x86_64/aarch64, macOS arm64/x86_64, # sdist) at the released commit; the SDK's committed .dev0 is stamped stable. build-core: