From 62e2ce679234e31829a39bbaa667a77f0734cd66 Mon Sep 17 00:00:00 2001 From: UnArbosFive Date: Tue, 4 Aug 2026 14:00:33 +0200 Subject: [PATCH] fix(ci): unblock TypeScript E2E fan-out --- .../scripts/select-shared-release-artifact.sh | 106 +++++++- .github/scripts/split-typescript-e2e-plan.sh | 48 ++++ .../test-classify-typescript-e2e-changes.sh | 15 ++ .../test-select-shared-release-artifact.sh | 175 +++++++++++++- .../scripts/test-split-typescript-e2e-plan.sh | 63 +++++ .github/workflows/typescript-e2e.yml | 226 ++++++++++++++---- .github/workflows/validate-sccache.yml | 3 + 7 files changed, 580 insertions(+), 56 deletions(-) create mode 100755 .github/scripts/split-typescript-e2e-plan.sh create mode 100755 .github/scripts/test-split-typescript-e2e-plan.sh diff --git a/.github/scripts/select-shared-release-artifact.sh b/.github/scripts/select-shared-release-artifact.sh index 6ffcb79257..4aaa4bcb1a 100755 --- a/.github/scripts/select-shared-release-artifact.sh +++ b/.github/scripts/select-shared-release-artifact.sh @@ -10,6 +10,9 @@ usage() { [[ $# -ge 1 && $# -le 2 ]] || usage output_file="$1" max_wait_seconds="${2:-360}" +producer_max_wait_seconds="${PRODUCER_MAX_WAIT_SECONDS:-420}" +producer_artifact_grace_seconds="${PRODUCER_ARTIFACT_GRACE_SECONDS:-30}" +poll_seconds="${SELECTOR_POLL_SECONDS:-15}" : "${GH_TOKEN:?GH_TOKEN must contain a short-lived Actions token}" : "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY must be set}" @@ -21,10 +24,18 @@ max_wait_seconds="${2:-360}" [[ "$GITHUB_SHA" =~ ^[0-9a-f]{40}$ ]] || usage [[ "$GITHUB_PR_HEAD_SHA" =~ ^[0-9a-f]{40}$ ]] || usage [[ "$max_wait_seconds" =~ ^[0-9]+$ ]] || usage +[[ "$producer_max_wait_seconds" =~ ^[0-9]+$ ]] || usage +[[ "$producer_artifact_grace_seconds" =~ ^[0-9]+$ ]] || usage +[[ "$poll_seconds" =~ ^[0-9]+$ ]] || usage artifact_name="node-subtensor-release-$GITHUB_SHA" workflow_path=.github/workflows/runtime-checks.yml started=$(date -u +%s) +producer_completed_seen= +producer_run_id= +producer_job_status= +producer_job_conclusion= +last_producer_state= write_miss() { { @@ -33,6 +44,28 @@ write_miss() { } >> "$output_file" } +find_latest_producer_run() { + local producer_runs + producer_runs=$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/$GITHUB_REPOSITORY/actions/runs?event=pull_request&head_sha=$GITHUB_PR_HEAD_SHA&per_page=100" \ + 2>/dev/null || true) + jq -er \ + --arg repository_id "$GITHUB_REPOSITORY_ID" \ + --arg sha "$GITHUB_PR_HEAD_SHA" \ + --arg workflow_path "$workflow_path" ' + [.workflow_runs[] + | select(.head_sha == $sha) + | select((.head_repository.id | tostring) == $repository_id) + | select(.path == $workflow_path) + | select(.event == "pull_request") + | select((.id | type) == "number" and .id > 0)] + | sort_by(.created_at) + | reverse + | (.[0].id // empty) + ' <<< "$producer_runs" 2>/dev/null || true +} + while true; do artifacts=$(gh api \ -H 'Accept: application/vnd.github+json' \ @@ -94,13 +127,76 @@ while true; do ) fi + if [[ -z "$producer_run_id" ]]; then + producer_run_id=$(find_latest_producer_run) + fi + + # Once discovered, latch the exact release-producing job. Transient API + # failures must not make us forget a producer and cross back to the shorter + # discovery timeout. + if [[ -n "$producer_run_id" ]]; then + producer_jobs=$(gh api \ + -H 'Accept: application/vnd.github+json' \ + "repos/$GITHUB_REPOSITORY/actions/runs/$producer_run_id/jobs?filter=latest&per_page=100" \ + 2>/dev/null || true) + producer_job=$( + jq -cer ' + [.jobs[] | select(.name == "build release node")] + | sort_by(.started_at // .created_at) + | reverse + | .[0] + ' <<< "$producer_jobs" 2>/dev/null || true + ) + current_status=$(jq -r '.status // empty' <<< "$producer_job" 2>/dev/null || true) + if [[ -n "$current_status" ]]; then + producer_job_status=$current_status + producer_job_conclusion=$(jq -r '.conclusion // empty' <<< "$producer_job" 2>/dev/null || true) + fi + fi + + producer_state=not-found + if [[ -n "$producer_run_id" ]]; then + producer_state=${producer_job_status:-job-pending} + [[ -z "$producer_job_conclusion" ]] || producer_state="$producer_state/$producer_job_conclusion" + fi + if [[ "$producer_state" != "$last_producer_state" ]]; then + echo "Runtime Checks release producer: $producer_state" + last_producer_state=$producer_state + fi + elapsed=$(($(date -u +%s) - started)) - if (( elapsed >= max_wait_seconds )); then + if [[ "$producer_job_status" == completed && "$producer_job_conclusion" == success ]]; then + [[ -n "$producer_completed_seen" ]] || producer_completed_seen=$(date -u +%s) + completed_elapsed=$(($(date -u +%s) - producer_completed_seen)) + if (( completed_elapsed >= producer_artifact_grace_seconds )); then + write_miss + echo "Runtime Checks release producer completed without a verified artifact after ${completed_elapsed}s of API propagation grace; using the local build fallback." + exit 0 + fi + elif [[ "$producer_job_status" == completed ]]; then + replacement_run_id=$(find_latest_producer_run) + if [[ -n "$replacement_run_id" && "$replacement_run_id" != "$producer_run_id" ]]; then + echo "Runtime Checks replaced producer run $producer_run_id with $replacement_run_id." + producer_run_id=$replacement_run_id + producer_job_status= + producer_job_conclusion= + producer_completed_seen= + continue + fi + write_miss + echo "Runtime Checks release producer completed (${producer_job_conclusion:-unknown}); using the local build fallback." + exit 0 + elif [[ -n "$producer_run_id" ]]; then + producer_completed_seen= + if (( elapsed >= producer_max_wait_seconds )); then + write_miss + echo "Runtime Checks release producer was still ${producer_job_status:-pending} after ${elapsed}s; using the local build fallback." + exit 0 + fi + elif (( elapsed >= max_wait_seconds )); then write_miss - echo "No exact-commit TypeScript release artifact appeared after ${elapsed}s; using the local build fallback." + echo "No exact-commit Runtime Checks producer appeared after ${elapsed}s; using the local build fallback." exit 0 fi - remaining=$((max_wait_seconds - elapsed)) - (( remaining > 10 )) || sleep "$remaining" - (( remaining <= 10 )) || sleep 10 + sleep "$poll_seconds" done diff --git a/.github/scripts/split-typescript-e2e-plan.sh b/.github/scripts/split-typescript-e2e-plan.sh new file mode 100755 index 0000000000..54622b11c2 --- /dev/null +++ b/.github/scripts/split-typescript-e2e-plan.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 OUTPUT_FILE" >&2 + exit 2 +fi + +output_file=$1 +: "${STATE_MATRIX:?STATE_MATRIX must contain the trusted state matrix}" +: "${BUILD_MATRIX:?BUILD_MATRIX must contain the trusted build matrix}" + +jq -e ' + type == "object" and + (.include | type == "array") and + all(.include[]; type == "object" and (.binary == "fast" or .binary == "release")) +' <<< "$STATE_MATRIX" >/dev/null +jq -e ' + type == "object" and + (.include | type == "array") and + all(.include[]; type == "object" and (.variant == "fast" or .variant == "release")) +' <<< "$BUILD_MATRIX" >/dev/null + +fast_state_matrix=$(jq -c '{include: [.include[] | select(.binary == "fast")]}' <<< "$STATE_MATRIX") +release_state_matrix=$(jq -c '{include: [.include[] | select(.binary == "release")]}' <<< "$STATE_MATRIX") +fast_state_count=$(jq '.include | length' <<< "$fast_state_matrix") +release_state_count=$(jq '.include | length' <<< "$release_state_matrix") +fast_build=$(jq -r 'any(.include[]; .variant == "fast")' <<< "$BUILD_MATRIX") +release_build=$(jq -r 'any(.include[]; .variant == "release")' <<< "$BUILD_MATRIX") + +if (( fast_state_count > 0 )) && [[ "$fast_build" != true ]]; then + echo "fast state jobs were planned without a fast binary build" >&2 + exit 1 +fi +if (( release_state_count > 0 )) && [[ "$release_build" != true ]]; then + echo "release state jobs were planned without a release binary build" >&2 + exit 1 +fi + +{ + echo "fast_state_count=$fast_state_count" + echo "fast_state_matrix=$fast_state_matrix" + echo "release_state_count=$release_state_count" + echo "release_state_matrix=$release_state_matrix" + echo "fast_build=$fast_build" + echo "release_build=$release_build" +} >> "$output_file" diff --git a/.github/scripts/test-classify-typescript-e2e-changes.sh b/.github/scripts/test-classify-typescript-e2e-changes.sh index 902b65f3e2..82a7eb1d84 100755 --- a/.github/scripts/test-classify-typescript-e2e-changes.sh +++ b/.github/scripts/test-classify-typescript-e2e-changes.sh @@ -264,9 +264,24 @@ fi grep -Fq 'ref: ${{ github.event_name == '\''pull_request'\'' && github.event.pull_request.base.sha || github.sha }}' "$workflow" grep -Fq '.trusted-e2e-filter/ts-tests/scripts/e2e-shard-plan.mjs' "$workflow" grep -Fq '.proposed-e2e-plan/ts-tests/e2e-shards.json' "$workflow" +grep -Fq '.trusted-e2e-filter/.github/scripts/split-typescript-e2e-plan.sh' "$workflow" +grep -Fq '.proposed-e2e-plan/.github/scripts/split-typescript-e2e-plan.sh' "$workflow" grep -Fq 'matrix: ${{ fromJSON(needs.changes.outputs.shield_matrix) }}' "$workflow" +grep -Fq 'matrix: ${{ fromJSON(needs.changes.outputs.fast_state_matrix) }}' "$workflow" +grep -Fq 'matrix: ${{ fromJSON(needs.changes.outputs.release_state_matrix) }}' "$workflow" +grep -Fq 'needs: [trusted-pr, changes, build-fast]' "$workflow" +grep -Fq 'needs: [trusted-pr, changes, build-release]' "$workflow" +grep -Fq 'needs: [trusted-pr, changes, select-release-artifact]' "$workflow" +grep -Fq 'needs: [trusted-pr, changes, build-fast, run-fast-e2e-tests, sharding-audit-fast]' "$workflow" +grep -Fq 'needs: [trusted-pr, changes, build-release, run-shield-tests, sharding-audit-release]' "$workflow" +grep -Fq 'fast_build: ${{ steps.lanes.outputs.fast_build }}' "$workflow" +grep -Fq 'release_build: ${{ steps.lanes.outputs.release_build }}' "$workflow" grep -Fq 'name: Audit canonical unsharded ${{ matrix.test }}' "$workflow" grep -Fq 'EVM_SELECTED: ${{ needs.changes.outputs.evm }}' "$workflow" grep -Fq 'SHIELD_SELECTED: ${{ needs.changes.outputs.shield }}' "$workflow" +if grep -Eq '^ build:$' "$workflow"; then + echo "TypeScript E2E consumers must not depend on a combined build matrix." >&2 + exit 1 +fi echo "typescript E2E change classifier tests passed" diff --git a/.github/scripts/test-select-shared-release-artifact.sh b/.github/scripts/test-select-shared-release-artifact.sh index 033363a118..15698d37d2 100755 --- a/.github/scripts/test-select-shared-release-artifact.sh +++ b/.github/scripts/test-select-shared-release-artifact.sh @@ -13,8 +13,44 @@ cat > "$tmp/bin/gh" <<'EOF' set -euo pipefail endpoint="${!#}" case "$endpoint" in - */actions/artifacts*) cat "$MOCK_ARTIFACTS" ;; + */actions/artifacts*) + calls=0 + [[ ! -f "$MOCK_ARTIFACT_CALLS" ]] || calls=$(cat "$MOCK_ARTIFACT_CALLS") + calls=$((calls + 1)) + echo "$calls" > "$MOCK_ARTIFACT_CALLS" + delay_calls=${MOCK_ARTIFACT_DELAY_CALLS:-0} + [[ "${MOCK_DELAY_ARTIFACT:-false}" != true ]] || delay_calls=1 + if (( calls <= delay_calls )); then + echo '{"artifacts":[]}' + else + cat "$MOCK_ARTIFACTS" + fi + ;; + */actions/runs/777/jobs*) + job_calls=0 + [[ ! -f "$MOCK_JOBS_CALLS" ]] || job_calls=$(cat "$MOCK_JOBS_CALLS") + job_calls=$((job_calls + 1)) + echo "$job_calls" > "$MOCK_JOBS_CALLS" + if [[ "${MOCK_FAIL_JOBS_ON_CALL:-0}" == "$job_calls" ]]; then + echo 'transient API failure' >&2 + exit 1 + fi + cat "$MOCK_JOBS" + ;; + */actions/runs/888/jobs*) cat "$MOCK_REPLACEMENT_JOBS" ;; + */actions/runs\?*) + run_calls=0 + [[ ! -f "$MOCK_RUNS_CALLS" ]] || run_calls=$(cat "$MOCK_RUNS_CALLS") + run_calls=$((run_calls + 1)) + echo "$run_calls" > "$MOCK_RUNS_CALLS" + if [[ -n "${MOCK_RUNS_AFTER_FIRST:-}" && "$run_calls" -gt 1 ]]; then + cat "$MOCK_RUNS_AFTER_FIRST" + else + cat "$MOCK_RUNS" + fi + ;; */actions/runs/777) cat "$MOCK_RUN" ;; + */actions/runs/888) cat "$MOCK_REPLACEMENT_RUN" ;; *) echo "unexpected endpoint: $endpoint" >&2; exit 2 ;; esac EOF @@ -27,7 +63,17 @@ export GITHUB_REPOSITORY_ID=608683796 export GITHUB_SHA=cccccccccccccccccccccccccccccccccccccccc export GITHUB_PR_HEAD_SHA=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa export MOCK_ARTIFACTS="$tmp/artifacts.json" +export MOCK_ARTIFACT_CALLS="$tmp/artifact-calls" export MOCK_RUN="$tmp/run.json" +export MOCK_RUNS="$tmp/runs.json" +export MOCK_JOBS="$tmp/jobs.json" +export MOCK_JOBS_CALLS="$tmp/job-calls" +export MOCK_RUNS_CALLS="$tmp/run-calls" +export MOCK_REPLACEMENT_JOBS="$tmp/replacement-jobs.json" +export MOCK_REPLACEMENT_RUN="$tmp/replacement-run.json" +export SELECTOR_POLL_SECONDS=0 +export PRODUCER_ARTIFACT_GRACE_SECONDS=0 +export PRODUCER_MAX_WAIT_SECONDS=10 cat > "$MOCK_ARTIFACTS" <<'EOF' {"artifacts":[{"id":123,"name":"node-subtensor-release-cccccccccccccccccccccccccccccccccccccccc","size_in_bytes":456,"digest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","expired":false,"created_at":"2026-07-17T00:00:00Z","workflow_run":{"id":777,"head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository_id":608683796}}]} @@ -35,8 +81,21 @@ EOF cat > "$MOCK_RUN" <<'EOF' {"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml"} EOF +cat > "$MOCK_RUNS" <<'EOF' +{"workflow_runs":[]} +EOF +cat > "$MOCK_JOBS" <<'EOF' +{"jobs":[]} +EOF +cat > "$MOCK_REPLACEMENT_JOBS" <<'EOF' +{"jobs":[]} +EOF +cat > "$MOCK_REPLACEMENT_RUN" <<'EOF' +{"id":888,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml"} +EOF : > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" "$selector" "$tmp/output" 0 >/dev/null grep -qx 'found=true' "$tmp/output" grep -qx 'artifact_id=123' "$tmp/output" @@ -50,6 +109,7 @@ jq '.artifacts[0].name = "node-subtensor-release-ddddddddddddddddddddddddddddddd "$MOCK_ARTIFACTS" > "$tmp/stale-merge-artifacts.json" export MOCK_ARTIFACTS="$tmp/stale-merge-artifacts.json" : > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" "$selector" "$tmp/output" 0 >/dev/null grep -qx 'found=false' "$tmp/output" @@ -59,6 +119,7 @@ export MOCK_ARTIFACTS="$tmp/artifacts.json" jq '.path = ".github/workflows/untrusted-producer.yml"' "$MOCK_RUN" > "$tmp/wrong-run.json" export MOCK_RUN="$tmp/wrong-run.json" : > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" "$selector" "$tmp/output" 0 >/dev/null grep -qx 'found=false' "$tmp/output" @@ -67,7 +128,119 @@ export MOCK_RUN="$tmp/run.json" jq '.artifacts[0].digest = "sha256:bad"' "$MOCK_ARTIFACTS" > "$tmp/bad-artifacts.json" export MOCK_ARTIFACTS="$tmp/bad-artifacts.json" : > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" "$selector" "$tmp/output" 0 >/dev/null grep -qx 'found=false' "$tmp/output" +# Once the exact Runtime Checks producer is active, do not let the ordinary +# producer-discovery timeout trigger a duplicate release build. The artifact +# may appear just after that boundary, as it did on PR #3035. +export MOCK_ARTIFACTS="$tmp/artifacts.json" +export MOCK_RUN="$tmp/run.json" +cat > "$MOCK_RUNS" <<'EOF' +{"workflow_runs":[{"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml","status":"in_progress","conclusion":null,"created_at":"2026-07-17T00:00:00Z"}]} +EOF +cat > "$MOCK_JOBS" <<'EOF' +{"jobs":[{"name":"build release node","status":"waiting","conclusion":null,"created_at":"2026-07-17T00:00:00Z"}]} +EOF +export MOCK_DELAY_ARTIFACT=true +: > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=true' "$tmp/output" +grep -qx 'artifact_id=123' "$tmp/output" +unset MOCK_DELAY_ARTIFACT + +# After a producer has been discovered, a transient jobs API failure must not +# forget it and re-enable the already-expired discovery timeout. +export MOCK_ARTIFACT_DELAY_CALLS=2 +export MOCK_FAIL_JOBS_ON_CALL=2 +: > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" +: > "$MOCK_JOBS_CALLS" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=true' "$tmp/output" +grep -qx 'artifact_id=123' "$tmp/output" +unset MOCK_ARTIFACT_DELAY_CALLS MOCK_FAIL_JOBS_ON_CALL + +# GitHub may report a successful producer before its artifact is visible. Give +# the artifact API a short propagation window instead of rebuilding at once. +cat > "$MOCK_RUNS" <<'EOF' +{"workflow_runs":[{"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml","status":"completed","conclusion":"success","created_at":"2026-07-17T00:00:00Z"}]} +EOF +cat > "$MOCK_JOBS" <<'EOF' +{"jobs":[{"name":"build release node","status":"completed","conclusion":"success","created_at":"2026-07-17T00:00:00Z"}]} +EOF +# Keep the behavioral test fast while still exercising a nonzero grace. +export PRODUCER_ARTIFACT_GRACE_SECONDS=1 +export MOCK_DELAY_ARTIFACT=true +: > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" +: > "$MOCK_JOBS_CALLS" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=true' "$tmp/output" +grep -qx 'artifact_id=123' "$tmp/output" +unset MOCK_DELAY_ARTIFACT +export PRODUCER_ARTIFACT_GRACE_SECONDS=0 + +# A completed producer without an artifact exits to the local fallback instead +# of polling until the longer active-producer ceiling. +cat > "$MOCK_ARTIFACTS" <<'EOF' +{"artifacts":[]} +EOF +cat > "$MOCK_RUNS" <<'EOF' +{"workflow_runs":[{"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml","status":"completed","conclusion":"failure","created_at":"2026-07-17T00:00:00Z"}]} +EOF +cat > "$MOCK_JOBS" <<'EOF' +{"jobs":[{"name":"build release node","status":"completed","conclusion":"failure","created_at":"2026-07-17T00:00:00Z"}]} +EOF +: > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=false' "$tmp/output" + +# A stuck active producer still has a hard ceiling, so a wedged Runtime Checks +# run cannot delay the local E2E release fallback forever. +cat > "$MOCK_RUNS" <<'EOF' +{"workflow_runs":[{"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml","status":"in_progress","conclusion":null,"created_at":"2026-07-17T00:00:00Z"}]} +EOF +cat > "$MOCK_JOBS" <<'EOF' +{"jobs":[{"name":"build release node","status":"requested","conclusion":null,"created_at":"2026-07-17T00:00:00Z"}]} +EOF +export PRODUCER_MAX_WAIT_SECONDS=0 +: > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=false' "$tmp/output" +export PRODUCER_MAX_WAIT_SECONDS=10 + +# A cancelled producer may have been superseded by a newer Runtime Checks run +# for the same PR head (for example after a label change). Follow the newer run +# instead of immediately starting a duplicate local build. +cat > "$MOCK_ARTIFACTS" <<'EOF' +{"artifacts":[{"id":124,"name":"node-subtensor-release-cccccccccccccccccccccccccccccccccccccccc","size_in_bytes":456,"digest":"sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","expired":false,"created_at":"2026-07-17T00:01:00Z","workflow_run":{"id":888,"head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository_id":608683796}}]} +EOF +cat > "$MOCK_RUNS" <<'EOF' +{"workflow_runs":[{"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml","status":"completed","conclusion":"cancelled","created_at":"2026-07-17T00:00:00Z"}]} +EOF +cat > "$tmp/replacement-runs.json" <<'EOF' +{"workflow_runs":[{"id":888,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml","status":"in_progress","conclusion":null,"created_at":"2026-07-17T00:01:00Z"},{"id":777,"event":"pull_request","head_sha":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","head_repository":{"id":608683796},"path":".github/workflows/runtime-checks.yml","status":"completed","conclusion":"cancelled","created_at":"2026-07-17T00:00:00Z"}]} +EOF +cat > "$MOCK_JOBS" <<'EOF' +{"jobs":[{"name":"build release node","status":"completed","conclusion":"cancelled","created_at":"2026-07-17T00:00:00Z"}]} +EOF +cat > "$MOCK_REPLACEMENT_JOBS" <<'EOF' +{"jobs":[{"name":"build release node","status":"waiting","conclusion":null,"created_at":"2026-07-17T00:01:00Z"}]} +EOF +export MOCK_RUNS_AFTER_FIRST="$tmp/replacement-runs.json" +export MOCK_ARTIFACT_DELAY_CALLS=2 +: > "$tmp/output" +: > "$MOCK_ARTIFACT_CALLS" +: > "$MOCK_RUNS_CALLS" +"$selector" "$tmp/output" 0 >/dev/null +grep -qx 'found=true' "$tmp/output" +grep -qx 'artifact_id=124' "$tmp/output" +grep -qx 'run_id=888' "$tmp/output" +unset MOCK_RUNS_AFTER_FIRST MOCK_ARTIFACT_DELAY_CALLS + echo "shared release artifact selector tests passed" diff --git a/.github/scripts/test-split-typescript-e2e-plan.sh b/.github/scripts/test-split-typescript-e2e-plan.sh new file mode 100755 index 0000000000..c833de356e --- /dev/null +++ b/.github/scripts/test-split-typescript-e2e-plan.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +splitter="$script_dir/split-typescript-e2e-plan.sh" +output=$(mktemp) +trap 'rm -f "$output"' EXIT + +value() { + sed -n "s/^$1=//p" "$output" +} + +run_split() { + : > "$output" + STATE_MATRIX=$1 BUILD_MATRIX=$2 "$splitter" "$output" +} + +mixed_state='{"include":[{"test":"evm-a","binary":"fast"},{"test":"dev","binary":"release"},{"test":"staking-a","binary":"fast"}]}' +mixed_build='{"include":[{"variant":"release","flags":""},{"variant":"fast","flags":"--features fast-runtime"}]}' +run_split "$mixed_state" "$mixed_build" +[[ "$(value fast_state_count)" == 2 ]] +[[ "$(value release_state_count)" == 1 ]] +[[ "$(value fast_build)" == true ]] +[[ "$(value release_build)" == true ]] +jq -e '.include | length == 2 and all(.[]; .binary == "fast")' <<< "$(value fast_state_matrix)" >/dev/null +jq -e '.include | length == 1 and all(.[]; .binary == "release")' <<< "$(value release_state_matrix)" >/dev/null + +fast_state='{"include":[{"test":"evm-a","binary":"fast"}]}' +fast_build='{"include":[{"variant":"fast","flags":"--features fast-runtime"}]}' +run_split "$fast_state" "$fast_build" +[[ "$(value fast_state_count)" == 1 ]] +[[ "$(value release_state_count)" == 0 ]] +[[ "$(value fast_build)" == true ]] +[[ "$(value release_build)" == false ]] + +release_state='{"include":[{"test":"dev","binary":"release"}]}' +release_build='{"include":[{"variant":"release","flags":""}]}' +run_split "$release_state" "$release_build" +[[ "$(value fast_state_count)" == 0 ]] +[[ "$(value release_state_count)" == 1 ]] +[[ "$(value fast_build)" == false ]] +[[ "$(value release_build)" == true ]] + +run_split '{"include":[]}' '{"include":[]}' +[[ "$(value fast_state_count)" == 0 ]] +[[ "$(value release_state_count)" == 0 ]] +[[ "$(value fast_build)" == false ]] +[[ "$(value release_build)" == false ]] + +if STATE_MATRIX='{"include":[{"test":"future","binary":"unknown"}]}' \ + BUILD_MATRIX="$mixed_build" "$splitter" "$output" >/dev/null 2>&1; then + echo "expected an unknown binary lane to fail closed" >&2 + exit 1 +fi + +if STATE_MATRIX="$fast_state" BUILD_MATRIX="$release_build" \ + "$splitter" "$output" >/dev/null 2>&1; then + echo "expected a missing fast build to fail closed" >&2 + exit 1 +fi + +echo "TypeScript E2E lane split tests passed" diff --git a/.github/workflows/typescript-e2e.yml b/.github/workflows/typescript-e2e.yml index 65a6ff9c4e..0927c7687c 100644 --- a/.github/workflows/typescript-e2e.yml +++ b/.github/workflows/typescript-e2e.yml @@ -37,12 +37,14 @@ jobs: evm: ${{ steps.filter.outputs.evm }} shield: ${{ steps.filter.outputs.shield }} topology_audit: ${{ steps.filter.outputs.topology_audit }} - state_count: ${{ steps.plan.outputs.state_count }} - state_matrix: ${{ steps.plan.outputs.state_matrix }} shield_count: ${{ steps.plan.outputs.shield_count }} shield_matrix: ${{ steps.plan.outputs.shield_matrix }} - build_count: ${{ steps.plan.outputs.build_count }} - build_matrix: ${{ steps.plan.outputs.build_matrix }} + fast_state_count: ${{ steps.lanes.outputs.fast_state_count }} + fast_state_matrix: ${{ steps.lanes.outputs.fast_state_matrix }} + release_state_count: ${{ steps.lanes.outputs.release_state_count }} + release_state_matrix: ${{ steps.lanes.outputs.release_state_matrix }} + fast_build: ${{ steps.lanes.outputs.fast_build }} + release_build: ${{ steps.lanes.outputs.release_build }} runtime_release: ${{ steps.filter.outputs.runtime_release }} steps: # Routing decisions for pull requests must come from the trusted base @@ -56,6 +58,7 @@ jobs: .github/scripts/classify-typescript-e2e-changes.sh .github/scripts/classify-runtime-changes.sh .github/scripts/extract-pull-file-paths.sh + .github/scripts/split-typescript-e2e-plan.sh ts-tests/scripts/e2e-shard-plan.mjs path: .trusted-e2e-filter persist-credentials: false @@ -71,6 +74,7 @@ jobs: sparse-checkout: | ts-tests/e2e-shards.json ts-tests/scripts/e2e-shard-plan.mjs + .github/scripts/split-typescript-e2e-plan.sh sparse-checkout-cone-mode: false path: .proposed-e2e-plan persist-credentials: false @@ -169,6 +173,23 @@ jobs: "$GITHUB_OUTPUT" \ "$EVM" "$STAKING" "$COLDKEY_SWAP" "$DEV" "$SUBNETS" "$SHIELD" + # Keep the trusted planner's stable output contract while separating the + # fast and release dependency lanes. This rollout therefore works on the + # PR that introduces it, before the updated workflow reaches main. + - name: Split planned jobs by binary + id: lanes + env: + STATE_MATRIX: ${{ steps.plan.outputs.state_matrix }} + BUILD_MATRIX: ${{ steps.plan.outputs.build_matrix }} + run: | + set -euo pipefail + splitter=.trusted-e2e-filter/.github/scripts/split-typescript-e2e-plan.sh + if [[ ! -x "$splitter" ]]; then + # One-time bootstrap. Future PRs execute the trusted base helper. + splitter=.proposed-e2e-plan/.github/scripts/split-typescript-e2e-plan.sh + fi + "$splitter" "$GITHUB_OUTPUT" + typescript-formatting: runs-on: ubuntu-latest steps: @@ -202,45 +223,72 @@ jobs: cd ts-tests pnpm run fmt - # Build each variant required by the selected suites exactly once and share - # it as an artifact. Ambiguous/shared changes select every suite and therefore - # still build both production release and fast-runtime binaries. - build: - runs-on: [self-hosted, fireactions-turbo-8] - # PR HEAD executes here; never attach sccache-writer / pass R2 write keys. - environment: - name: sccache-reader - deployment: false + # Keep each artifact producer as a separate job so consumers of the fast + # binary do not wait for the unrelated release lane. Runtime Checks remains + # the canonical producer for runtime-affecting release builds. + select-release-artifact: + name: Select Runtime release artifact + runs-on: ubuntu-latest needs: [trusted-pr, changes] - if: needs.changes.outputs.build_count != '0' - timeout-minutes: 60 - strategy: - matrix: ${{ fromJSON(needs.changes.outputs.build_matrix) }} - env: - RUST_BACKTRACE: full + if: needs.changes.outputs.release_build == 'true' + timeout-minutes: 8 permissions: contents: read actions: read + outputs: + found: ${{ steps.shared.outputs.found }} + artifact_id: ${{ steps.shared.outputs.artifact_id }} + digest: ${{ steps.shared.outputs.digest }} + size: ${{ steps.shared.outputs.size }} + waited_seconds: ${{ steps.shared.outputs.waited_seconds }} steps: - name: Check-out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Select exact-merge Runtime release artifact id: shared - if: >- - github.event_name == 'pull_request' && - matrix.variant == 'release' && - needs.changes.outputs.runtime_release == 'true' continue-on-error: true env: GH_TOKEN: ${{ github.token }} GITHUB_REPOSITORY_ID: ${{ github.repository_id }} GITHUB_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: .github/scripts/select-shared-release-artifact.sh "$GITHUB_OUTPUT" + RUNTIME_RELEASE: ${{ needs.changes.outputs.runtime_release }} + run: | + if [[ "$GITHUB_EVENT_NAME" == pull_request && "$RUNTIME_RELEASE" == true ]]; then + .github/scripts/select-shared-release-artifact.sh "$GITHUB_OUTPUT" + else + { + echo 'found=false' + echo 'waited_seconds=0' + } >> "$GITHUB_OUTPUT" + fi + + build-release: + name: build (release) + runs-on: [self-hosted, fireactions-turbo-8] + # PR HEAD executes here; never attach sccache-writer / pass R2 write keys. + environment: + name: sccache-reader + deployment: false + needs: [trusted-pr, changes, select-release-artifact] + if: >- + always() && + needs.trusted-pr.result == 'success' && + needs.changes.result == 'success' && + needs.changes.outputs.release_build == 'true' + timeout-minutes: 60 + env: + RUST_BACKTRACE: full + permissions: + contents: read + actions: read + steps: + - name: Check-out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - name: Reuse verified Runtime release build id: reuse - if: steps.shared.outputs.found == 'true' + if: needs.select-release-artifact.outputs.found == 'true' env: GH_TOKEN: ${{ github.token }} GITHUB_REPOSITORY_ID: ${{ github.repository_id }} @@ -249,10 +297,10 @@ jobs: output=$(mktemp) trap 'rm -f "$output"' EXIT if .github/scripts/download-artifact.sh \ - "${{ steps.shared.outputs.artifact_id }}" \ + "${{ needs.select-release-artifact.outputs.artifact_id }}" \ "node-subtensor-release-${GITHUB_SHA}" \ - "${{ steps.shared.outputs.digest }}" \ - "${{ steps.shared.outputs.size }}" \ + "${{ needs.select-release-artifact.outputs.digest }}" \ + "${{ needs.select-release-artifact.outputs.size }}" \ target/release "$output" && [[ -s target/release/node-subtensor ]] && [[ -s target/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm ]]; then @@ -262,7 +310,7 @@ jobs: { echo "### E2E release node" echo "- Reused exact-merge Runtime Checks artifact" - echo "- Producer wait: ${{ steps.shared.outputs.waited_seconds }}s" + echo "- Producer wait: ${{ needs.select-release-artifact.outputs.waited_seconds }}s" } >> "$GITHUB_STEP_SUMMARY" else rm -rf target/release @@ -274,21 +322,59 @@ jobs: if: steps.reuse.outputs.reused != 'true' uses: ./.github/actions/rust-setup with: - cache-key: e2e-${{ matrix.variant }} + cache-key: e2e-release sccache-credential-mode: reader - - name: Build node-subtensor (${{ matrix.variant }}) + - name: Build node-subtensor (release) if: steps.reuse.outputs.reused != 'true' - run: cargo build --profile release ${{ matrix.flags }} -p node-subtensor + run: cargo build --profile release -p node-subtensor - - name: Report ${{ matrix.variant }} compiler cache + - name: Report release compiler cache if: always() && steps.reuse.outputs.reused != 'true' - run: .github/scripts/sccache-report.sh "E2E ${{ matrix.variant }} compiler cache" + run: .github/scripts/sccache-report.sh "E2E release compiler cache" + + - name: Upload binary + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: node-subtensor-release + path: target/release/node-subtensor + if-no-files-found: error + retention-days: 3 + + build-fast: + name: build (fast, --features fast-runtime) + runs-on: [self-hosted, fireactions-turbo-8] + environment: + name: sccache-reader + deployment: false + needs: [trusted-pr, changes] + if: needs.changes.outputs.fast_build == 'true' + timeout-minutes: 60 + env: + RUST_BACKTRACE: full + permissions: + contents: read + steps: + - name: Check-out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Set up Rust build environment + uses: ./.github/actions/rust-setup + with: + cache-key: e2e-fast + sccache-credential-mode: reader + + - name: Build node-subtensor (fast) + run: cargo build --profile release --features fast-runtime -p node-subtensor + + - name: Report fast compiler cache + if: always() + run: .github/scripts/sccache-report.sh "E2E fast compiler cache" - name: Upload binary uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 with: - name: node-subtensor-${{ matrix.variant }} + name: node-subtensor-fast path: target/release/node-subtensor if-no-files-found: error retention-days: 3 @@ -298,16 +384,16 @@ jobs: # file runtime without changing coverage. Shield retains the production-like # multi-node topology below. Seven state workers plus five Shield workers use # 12 of the 32 turbo-8 runners, so two full PRs still leave eight slots free. - run-e2e-tests: - needs: [trusted-pr, changes, build] - if: needs.changes.outputs.state_count != '0' + run-fast-e2e-tests: + needs: [trusted-pr, changes, build-fast] + if: needs.changes.outputs.fast_state_count != '0' runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 30 strategy: fail-fast: false - max-parallel: 7 - matrix: ${{ fromJSON(needs.changes.outputs.state_matrix) }} + max-parallel: 6 + matrix: ${{ fromJSON(needs.changes.outputs.fast_state_matrix) }} name: "typescript-e2e-${{ matrix.test }}" @@ -322,13 +408,33 @@ jobs: test: ${{ matrix.test }} additional-test: ${{ matrix.additional_test }} + run-release-e2e-tests: + needs: [trusted-pr, changes, build-release] + if: needs.changes.outputs.release_state_count != '0' + runs-on: [self-hosted, fireactions-turbo-8] + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.changes.outputs.release_state_matrix) }} + name: "typescript-e2e-${{ matrix.test }}" + steps: + - name: Check-out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Run E2E suite + uses: ./.github/actions/run-typescript-e2e + with: + binary: ${{ matrix.binary }} + test: ${{ matrix.test }} + additional-test: ${{ matrix.additional_test }} + # Keep the historical EVM check context stable while the implementation is # split into balanced _a/_b/_c jobs. This also reports success when path routing # intentionally skips state E2E, avoiding a permanently Expected context. evm-result: name: typescript-e2e-zombienet_evm if: always() - needs: [trusted-pr, changes, build, run-e2e-tests, sharding-audit] + needs: [trusted-pr, changes, build-fast, run-fast-e2e-tests, sharding-audit-fast] runs-on: ubuntu-latest steps: - name: Check EVM shard prerequisites @@ -336,10 +442,10 @@ jobs: TRUSTED_RESULT: ${{ needs.trusted-pr.result }} CHANGES_RESULT: ${{ needs.changes.result }} EVM_SELECTED: ${{ needs.changes.outputs.evm }} - BUILD_RESULT: ${{ needs.build.result }} - STATE_RESULT: ${{ needs.run-e2e-tests.result }} + BUILD_RESULT: ${{ needs.build-fast.result }} + STATE_RESULT: ${{ needs.run-fast-e2e-tests.result }} TOPOLOGY_AUDIT: ${{ needs.changes.outputs.topology_audit }} - AUDIT_RESULT: ${{ needs.sharding-audit.result }} + AUDIT_RESULT: ${{ needs.sharding-audit-fast.result }} run: | if [[ "$TRUSTED_RESULT" != "success" || "$CHANGES_RESULT" != "success" ]]; then echo "EVM E2E routing failed: trusted=$TRUSTED_RESULT changes=$CHANGES_RESULT" >&2 @@ -370,7 +476,7 @@ jobs: # slot duration. A 250 ms runtime outruns the asynchronous key-rotation # pipeline and can finalize a wrapper without unshielding its inner call. run-shield-tests: - needs: [trusted-pr, changes, build] + needs: [trusted-pr, changes, build-release] if: needs.changes.outputs.shield_count != '0' runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 30 @@ -391,9 +497,9 @@ jobs: # Only topology/matrix changes pay for this audit. It proves that the fast # file shards remain equivalent to the original canonical environments and # catches hidden cross-file state or concurrency assumptions before merge. - sharding-audit: + sharding-audit-fast: name: Audit canonical unsharded ${{ matrix.test }} - needs: [trusted-pr, changes, build] + needs: [trusted-pr, changes, build-fast] if: needs.changes.outputs.topology_audit == 'true' runs-on: [self-hosted, fireactions-turbo-8] timeout-minutes: 60 @@ -405,6 +511,26 @@ jobs: binary: fast - test: zombienet_staking binary: fast + steps: + - name: Check-out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + + - name: Run canonical environment + uses: ./.github/actions/run-typescript-e2e + with: + binary: ${{ matrix.binary }} + test: ${{ matrix.test }} + + sharding-audit-release: + name: Audit canonical unsharded ${{ matrix.test }} + needs: [trusted-pr, changes, build-release] + if: needs.changes.outputs.topology_audit == 'true' + runs-on: [self-hosted, fireactions-turbo-8] + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + include: - test: zombienet_shield binary: release steps: @@ -420,7 +546,7 @@ jobs: shield-result: name: typescript-e2e-zombienet_shield if: always() - needs: [trusted-pr, changes, build, run-shield-tests, sharding-audit] + needs: [trusted-pr, changes, build-release, run-shield-tests, sharding-audit-release] runs-on: ubuntu-latest steps: - name: Check Shield shard results @@ -428,10 +554,10 @@ jobs: TRUSTED_RESULT: ${{ needs.trusted-pr.result }} CHANGES_RESULT: ${{ needs.changes.result }} SHIELD_SELECTED: ${{ needs.changes.outputs.shield }} - BUILD_RESULT: ${{ needs.build.result }} + BUILD_RESULT: ${{ needs.build-release.result }} SHIELD_RESULT: ${{ needs.run-shield-tests.result }} TOPOLOGY_AUDIT: ${{ needs.changes.outputs.topology_audit }} - AUDIT_RESULT: ${{ needs.sharding-audit.result }} + AUDIT_RESULT: ${{ needs.sharding-audit-release.result }} run: | if [[ "$TRUSTED_RESULT" != "success" || "$CHANGES_RESULT" != "success" ]]; then echo "Shield E2E routing failed: trusted=$TRUSTED_RESULT changes=$CHANGES_RESULT" >&2 diff --git a/.github/workflows/validate-sccache.yml b/.github/workflows/validate-sccache.yml index 72c10e1250..3563d406e8 100644 --- a/.github/workflows/validate-sccache.yml +++ b/.github/workflows/validate-sccache.yml @@ -24,6 +24,8 @@ on: - ".github/scripts/test-download-artifact.sh" - ".github/scripts/select-shared-release-artifact.sh" - ".github/scripts/test-select-shared-release-artifact.sh" + - ".github/scripts/split-typescript-e2e-plan.sh" + - ".github/scripts/test-split-typescript-e2e-plan.sh" - ".github/scripts/prewarm-exact-runtime.sh" - ".github/scripts/test-prewarm-exact-runtime.sh" - ".github/scripts/benchmark-sccache-paired.sh" @@ -80,6 +82,7 @@ jobs: .github/scripts/test-snapshot-artifact.sh .github/scripts/test-download-artifact.sh .github/scripts/test-select-shared-release-artifact.sh + .github/scripts/test-split-typescript-e2e-plan.sh .github/scripts/test-r2-artifact-mirror.py .github/scripts/test-r2-sccache-warmset.py .github/scripts/test-classify-bittensor-e2e-changes.sh