From 1aab9f5eee4985e315149bc54af0ebf88037b14a Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Fri, 30 Oct 2026 22:01:01 -0400 Subject: [PATCH 1/4] =?UTF-8?q?=E2=9C=A8=20feat:=20bind=20candidate=20prod?= =?UTF-8?q?ucer=20identity=20to=20trusted=20CI=20jobs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- .github/workflows/go-security-analysis.yml | 37 ++- .github/workflows/v2-ci.yml | 32 +- .github/workflows/v2-tests.yml | 70 +++-- ...ed-integration-candidate-producer-trust.md | 1 + src/docs/integration-orchestration.md | 50 ++++ src/pkg/github/integration_producer_keys.go | 149 +++++++++ src/pkg/github/integration_producer_token.go | 168 +++++++++++ .../github/integration_producer_token_test.go | 283 ++++++++++++++++++ .../integration_producer_workflows_test.go | 126 ++++++++ 9 files changed, 877 insertions(+), 39 deletions(-) create mode 100644 changelog.d/added-integration-candidate-producer-trust.md create mode 100644 src/pkg/github/integration_producer_keys.go create mode 100644 src/pkg/github/integration_producer_token.go create mode 100644 src/pkg/github/integration_producer_token_test.go create mode 100644 src/pkg/github/integration_producer_workflows_test.go diff --git a/.github/workflows/go-security-analysis.yml b/.github/workflows/go-security-analysis.yml index c8c09925b7..f134fffa8f 100644 --- a/.github/workflows/go-security-analysis.yml +++ b/.github/workflows/go-security-analysis.yml @@ -46,6 +46,12 @@ name: Go Security Analysis # Same shape as dashboard/openapi.json in v2-tests.yml: guarded by a job whose # filter excluded it. on: + workflow_call: + inputs: + integration_candidate: + description: Run an installed immutable Integration-aware candidate with isolated read-only technical jobs. + type: boolean + default: false merge_group: types: [checks_requested] push: @@ -75,8 +81,8 @@ permissions: # Same discipline as v2-ci.yml: cancel superseded PR runs, never cancel a # push-to-branch run (keyed on run_id so each gets its own group). concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: ${{ inputs.integration_candidate && 'integration-security' || github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ !inputs.integration_candidate && github.event_name == 'pull_request' }} # Build-cache isolation for the self-hosted pool (see # .github/scripts/go-cache-guard.sh). `job` gives each job its own GOCACHE @@ -89,7 +95,7 @@ env: jobs: required-context-scope: name: required context scope - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 outputs: run-go-security: ${{ steps.scope.outputs.run-go-security }} @@ -101,6 +107,11 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -euo pipefail + if [ "${{ inputs.integration_candidate }}" = true ] && + { [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [[ "$GITHUB_REF" != refs/tags/hive-integration/* ]]; }; then + echo "::error::Integration candidates require an explicitly dispatched immutable candidate tag" + exit 1 + fi if [ "${{ github.event_name }}" != "pull_request" ]; then echo "run-go-security=true" >> "$GITHUB_OUTPUT" exit 0 @@ -148,10 +159,12 @@ jobs: name: govulncheck (known vulnerabilities) # Required on v5: keep on the self-hosted pool so a GitHub-hosted runner # outage does not make otherwise-mergeable PRs impossible to merge. - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -187,10 +200,12 @@ jobs: name: gosec (insecure patterns) # Required on v5: keep on the self-hosted pool so a GitHub-hosted runner # outage does not make otherwise-mergeable PRs impossible to merge. - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} # cache: false — the hive runners mount a persistent GOCACHE/GOMODCACHE; # setup-go's post-step would otherwise tar that multi-GB shared cache @@ -249,10 +264,12 @@ jobs: name: golangci-lint # Required on v5: keep on the self-hosted pool so a GitHub-hosted runner # outage does not make otherwise-mergeable PRs impossible to merge. - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 25 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -287,10 +304,12 @@ jobs: name: action pins resolve needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') }} - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} # A fabricated 40-hex SHA looks exactly like a correct pin and is only # caught when the workflow RUNS. tagged-release.yml shipped one (#4908), so every @@ -307,7 +326,7 @@ jobs: name: NOTICE matches the module graph needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') }} - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 20 # WHY THIS JOB EXISTS: a CNCF General Technical Review asks how the # project ensures third-party attribution stays correct and complete. A @@ -337,6 +356,8 @@ jobs: # exactly as it is on every run after. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} - name: Exercise NOTICE drift diagnostics run: src/scripts/test-check-notice-drift.sh diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index 7bd2efda1c..e0afbfd617 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -1,6 +1,12 @@ name: v2 CI on: + workflow_call: + inputs: + integration_candidate: + description: Run an installed immutable Integration-aware candidate with isolated read-only technical jobs. + type: boolean + default: false merge_group: types: [checks_requested] push: @@ -40,8 +46,8 @@ permissions: # non-PR group on run_id gives every such run its own group, so they can # neither cancel each other nor queue behind one another. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: ${{ inputs.integration_candidate && 'integration-ci' || github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ !inputs.integration_candidate && github.event_name == 'pull_request' }} # A slow cache-segment download must never cost more than a recompile # (see v2-tests.yml — observed 16-minute restore steps on some nodes). @@ -57,7 +63,7 @@ env: jobs: required-context-scope: name: required context scope - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 outputs: run-ci: ${{ steps.scope.outputs.run-ci }} @@ -69,6 +75,11 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -euo pipefail + if [ "${{ inputs.integration_candidate }}" = true ] && + { [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [[ "$GITHUB_REF" != refs/tags/hive-integration/* ]]; }; then + echo "::error::Integration candidates require an explicitly dispatched immutable candidate tag" + exit 1 + fi if [ "${{ github.event_name }}" != "pull_request" ]; then echo "run-ci=true" >> "$GITHUB_OUTPUT" exit 0 @@ -115,13 +126,15 @@ jobs: if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} # Required on v5: keep on the self-hosted pool so a GitHub-hosted runner # outage does not make otherwise-mergeable PRs impossible to merge. - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 50 defaults: run: working-directory: src steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} - name: Ensure PyYAML on self-hosted runners working-directory: . run: | @@ -1237,7 +1250,7 @@ jobs: working-directory: . run: bash src/deploy/test_runner_labels_portable.sh create-issue-on-failure: - if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/v2' + if: "!inputs.integration_candidate && failure() && github.event_name == 'push' && github.ref == 'refs/heads/v2'" runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }} timeout-minutes: 10 needs: build-and-test @@ -1305,7 +1318,7 @@ jobs: name: docker (${{ matrix.name }}) needs: required-context-scope if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') }} - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 45 strategy: fail-fast: false @@ -1323,12 +1336,13 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + persist-credentials: ${{ !inputs.integration_candidate }} # Dockerfile.hub does `COPY .git` to stamp the build with a git # describe; a shallow clone makes that describe wrong, not absent. fetch-depth: ${{ matrix.name == 'hub' && 0 || 1 }} - name: Skip non-hive images on push - if: github.event_name != 'pull_request' && matrix.name != 'hive' + if: "!inputs.integration_candidate && github.event_name != 'pull_request' && matrix.name != 'hive'" run: echo "skip=true" >> "$GITHUB_ENV" - name: Set up Docker Buildx @@ -1387,6 +1401,8 @@ jobs: # No `needs: build-and-test` — see the note above the `docker` job. steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: ${{ !inputs.integration_candidate }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 @@ -1510,7 +1526,7 @@ jobs: echo "OK: hive execs in-place under containerd/overlayfs (no copy, no NET_ADMIN) AND reports the current commit ($REPORTED_COMMIT)." nightly-release: - if: github.event_name == 'push' && github.ref == 'refs/heads/v2' + if: "!inputs.integration_candidate && github.event_name == 'push' && github.ref == 'refs/heads/v2'" runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }} timeout-minutes: 60 needs: [build-and-test, docker] diff --git a/.github/workflows/v2-tests.yml b/.github/workflows/v2-tests.yml index 05cbb86e45..2140960609 100644 --- a/.github/workflows/v2-tests.yml +++ b/.github/workflows/v2-tests.yml @@ -1,6 +1,12 @@ name: v2 Tests on: + workflow_call: + inputs: + integration_candidate: + description: Run an installed immutable Integration-aware candidate with isolated read-only technical jobs. + type: boolean + default: false merge_group: types: [checks_requested] schedule: @@ -79,8 +85,8 @@ env: # group on run_id gives every such run its own group, so they can neither # cancel each other nor queue behind one another. concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: ${{ inputs.integration_candidate && 'integration-tests' || github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ !inputs.integration_candidate && github.event_name == 'pull_request' }} # ── Sharding layout (#4118) ────────────────────────────────────────────────── # The suite is split so PR wall-clock ≈ the slowest single piece, not the sum. @@ -208,7 +214,7 @@ concurrency: jobs: required-context-scope: name: required context scope - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 outputs: run-tests: ${{ steps.scope.outputs.run-tests }} @@ -220,6 +226,11 @@ jobs: PR_NUMBER: ${{ github.event.pull_request.number }} run: | set -euo pipefail + if [ "${{ inputs.integration_candidate }}" = true ] && + { [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [[ "$GITHUB_REF" != refs/tags/hive-integration/* ]]; }; then + echo "::error::Integration candidates require an explicitly dispatched immutable candidate tag" + exit 1 + fi if [ "${{ github.event_name }}" != "pull_request" ]; then echo "run-tests=true" >> "$GITHUB_OUTPUT" exit 0 @@ -276,7 +287,7 @@ jobs: fail-fast: false matrix: idx: [1, 2, 3, 4] - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 20 defaults: run: @@ -302,6 +313,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: steps.select.outputs.run == 'true' + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -391,7 +404,7 @@ jobs: id: prepare-cgo-toolchain env: HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race - HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !inputs.integration_candidate && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} run: | bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ --require gcc --for '-race tests' \ @@ -557,7 +570,7 @@ jobs: - { lane: scheduler, pkg: pkg/scheduler, idx: 1, total: 3 } - { lane: scheduler, pkg: pkg/scheduler, idx: 2, total: 3 } - { lane: scheduler, pkg: pkg/scheduler, idx: 3, total: 3 } - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 20 defaults: run: @@ -591,6 +604,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: steps.select.outputs.run == 'true' + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -680,7 +695,7 @@ jobs: id: prepare-cgo-toolchain env: HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race - HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !inputs.integration_candidate && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} run: | bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ --require gcc --for '-race tests' \ @@ -842,7 +857,7 @@ jobs: fail-fast: false matrix: idx: [1, 2, 3, 4, 5] - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 25 defaults: run: @@ -868,6 +883,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: steps.select.outputs.run == 'true' + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -933,7 +950,7 @@ jobs: id: prepare-cgo-toolchain env: HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race - HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !inputs.integration_candidate && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} run: | bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ --require gcc --for '-race tests' \ @@ -1091,7 +1108,7 @@ jobs: list-packages: needs: required-context-scope if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 15 outputs: matrix: ${{ steps.partition.outputs.matrix }} @@ -1112,6 +1129,7 @@ jobs: # present for the affected-package diff below. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + persist-credentials: ${{ !inputs.integration_candidate }} fetch-depth: 2 # Populate the shared node-local Go tool cache under the runner pods' @@ -1165,7 +1183,7 @@ jobs: id: prepare-cgo-toolchain env: HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race - HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !inputs.integration_candidate && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} run: | bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ --require gcc --for '-race tests' \ @@ -1402,7 +1420,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.list-packages.outputs.matrix) }} - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 25 defaults: run: @@ -1428,6 +1446,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: steps.select.outputs.run == 'true' + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -1517,7 +1537,7 @@ jobs: id: prepare-cgo-toolchain env: HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race - HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !inputs.integration_candidate && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} run: | bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ --require gcc --for '-race tests' \ @@ -1623,7 +1643,7 @@ jobs: fail-fast: false matrix: idx: [1, 2, 3] - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 20 defaults: run: @@ -1649,6 +1669,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: steps.select.outputs.run == 'true' + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -1738,7 +1760,7 @@ jobs: id: prepare-cgo-toolchain env: HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race - HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !inputs.integration_candidate && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} run: | bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ --require gcc --for '-race tests' \ @@ -1863,7 +1885,7 @@ jobs: # the surviving run still fail-closes on any non-success shard result. # Required on v5: keep on the self-hosted pool so a GitHub-hosted runner # outage does not make otherwise-mergeable PRs impossible to merge. - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 needs: [required-context-scope, list-packages, test-hub, test-sliced, test-agent, test-rest, test-dashboard-shuffle] steps: @@ -2005,10 +2027,10 @@ jobs: # still parsed for the two states a profile cannot express: FAIL and # `? ... [no test files]` — both hard failures, as before. coverage: - if: ${{ always() && (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'release-gate/'))) }} + if: ${{ always() && (inputs.integration_candidate || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'release-gate/'))) }} # Required on v5: keep on the self-hosted pool so a GitHub-hosted runner # outage does not make otherwise-mergeable PRs impossible to merge. - runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} + runs-on: ${{ !inputs.integration_candidate && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 30 needs: [required-context-scope, list-packages, test-hub, test-sliced, test-agent, test-rest] steps: @@ -2049,6 +2071,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 if: ${{ steps.cov-scope.outputs.score == 'true' }} + with: + persist-credentials: ${{ !inputs.integration_candidate }} # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a @@ -2459,7 +2483,7 @@ jobs: echo "All packages meet their coverage floor" >> "$GITHUB_STEP_SUMMARY" patch-coverage: - if: ${{ github.event_name == 'pull_request' && needs.coverage.result == 'success' && needs.required-context-scope.outputs.run-tests == 'true' && !cancelled() }} + if: ${{ !inputs.integration_candidate && github.event_name == 'pull_request' && needs.coverage.result == 'success' && needs.required-context-scope.outputs.run-tests == 'true' && !cancelled() }} needs: [coverage, required-context-scope] runs-on: ubuntu-latest timeout-minutes: 10 @@ -2500,7 +2524,7 @@ jobs: --pr "$PR_NUMBER" \ --advisory create-issue-on-failure: - if: (failure() || cancelled()) && github.event_name == 'schedule' + if: "!inputs.integration_candidate && (failure() || cancelled()) && github.event_name == 'schedule'" runs-on: ${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) && fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') || 'ubuntu-latest' }} timeout-minutes: 10 needs: [list-packages, test-hub, test-sliced, test-agent, test-rest, test-dashboard-shuffle] @@ -2557,7 +2581,7 @@ jobs: }); create-issues-on-flakes: - if: always() && github.event_name == 'push' && (github.ref == 'refs/heads/v5' || github.ref == 'refs/heads/v6') && needs.test-hub.result == 'success' && needs.test-sliced.result == 'success' && needs.test-agent.result == 'success' && needs.test-rest.result == 'success' && needs.test-dashboard-shuffle.result == 'success' + if: "!inputs.integration_candidate && always() && github.event_name == 'push' && (github.ref == 'refs/heads/v5' || github.ref == 'refs/heads/v6') && needs.test-hub.result == 'success' && needs.test-sliced.result == 'success' && needs.test-agent.result == 'success' && needs.test-rest.result == 'success' && needs.test-dashboard-shuffle.result == 'success'" runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }} timeout-minutes: 10 needs: [test-hub, test-sliced, test-agent, test-rest, test-dashboard-shuffle] @@ -2706,7 +2730,7 @@ jobs: # objects each. Push/schedule only: PR-scoped cache saves are invisible to # other branches and would only thrash the repo's 10GB cache budget. warm-build-cache: - if: github.event_name != 'pull_request' && !contains(vars.HIVE_RUNNER_LABELS, 'hive-runners') + if: "!inputs.integration_candidate && github.event_name != 'pull_request' && !contains(vars.HIVE_RUNNER_LABELS, 'hive-runners')" runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }} timeout-minutes: 40 defaults: @@ -2757,7 +2781,7 @@ jobs: id: prepare-cgo-toolchain env: HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race - HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !inputs.integration_candidate && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} run: | bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ --require gcc --for '-race tests' \ diff --git a/changelog.d/added-integration-candidate-producer-trust.md b/changelog.d/added-integration-candidate-producer-trust.md new file mode 100644 index 0000000000..b8937aa5b8 --- /dev/null +++ b/changelog.d/added-integration-candidate-producer-trust.md @@ -0,0 +1 @@ +- Prepare Integration-aware candidate CI with default-false reusable technical workflows, isolated read-only execution and job-bound producer identity verification; this source support does not activate candidate execution or replace required CI/review policy. diff --git a/src/docs/integration-orchestration.md b/src/docs/integration-orchestration.md index 27cbedde0e..d728234d85 100644 --- a/src/docs/integration-orchestration.md +++ b/src/docs/integration-orchestration.md @@ -184,6 +184,56 @@ listing. Reported minutes per merged PR are an observed-window aggregate, not a claim that every individual change's eventual cost is known. Unfinished execution and unattributed jobs remain explicit. +## Candidate producer trust boundary + +The reusable technical workflows accept a typed `integration_candidate` input +that defaults to false. False preserves ordinary PR, push and native +`merge_group` behavior. Candidate calls require an explicitly dispatched +`refs/tags/hive-integration/` ref, run on disposable GitHub-hosted runners with +read-only technical-job permissions, do not persist checkout credentials, and +retain full test/coverage requirements without manual-run toolchain soft failure. +Their distinct concurrency groups do not cancel another reusable technical +workflow in the same caller run. Candidate code never runs in issue/release +writers; no repository secrets may be inherited by a candidate caller. + +These call seams and the producer verifier are source support, not an installed +producer, execution controller, runtime opt-in, protection migration or +execution-readiness claim. Existing authoring enforce mode does not activate +them. The current eleven-context destination contract still needs genuine +candidate-aware producers; author-head green statuses must not be copied. + +The producer identity protocol uses +[GitHub OIDC claims](https://docs.github.com/en/actions/reference/security/oidc) +including the actual finalizer `check_run_id`, run/attempt, repository/actor +identities, workflow SHA/ref and hosted runner environment. A custom audience +binds the SHA-256 digest of the exact payload bytes. Only the trusted finalizer +may obtain signing identity; untrusted technical jobs receive no such +permission. Verification uses only GitHub's fixed issuer/public signing keys, +without installation credentials or token-selected key URLs. Refreshes are +bounded/coalesced, and no cache mutex spans remote I/O. Missing issuer keys or +claims are unknown, not passing validation. + +A signed producer identity is not proof that all CI passed. The consumer must +separately establish exact subject/tree/ordered-head membership, installed +reviewed workflow/action closure, real successful finalizer and required +technical jobs, immutable artifact identity/digest and complete observations. +Signed issuance must fall within the independently observed finalizer interval; +a historical attestation is not a currently valid bearer login or live +promotion authorization. Rotation/durable verification history and the actual +isolated provider execution remain required lifecycle work. + +Executing untrusted candidate code from a default-ref dispatch can expose that +ref's Actions-cache namespace even with a read-only GitHub token. The intended +producer therefore requires controlled immutable candidate refs and verified +trusted definitions, not an arbitrary author workflow. Publication with a +trusted control workflow's `GITHUB_TOKEN` suppresses push-triggered workflows; +[GitHub documents explicit dispatch and certain approval-required PR-event +exceptions](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/trigger-a-workflow). +The installation's trigger permissions, controlled-ref protections and real +candidate-required-context compatibility must be verified before opt-in. This +feature does not grant those permissions, change protections or install a live +pilot. + The append-only journal uses local cross-process fencing, suffix replay and fsync; no remote reads occur under its lock. Replay is bounded to 256 MiB, 100,000 records and 1 MiB per record. At a bound or corruption, new observations diff --git a/src/pkg/github/integration_producer_keys.go b/src/pkg/github/integration_producer_keys.go new file mode 100644 index 0000000000..503576e03f --- /dev/null +++ b/src/pkg/github/integration_producer_keys.go @@ -0,0 +1,149 @@ +package github + +import ( + "context" + "crypto/rsa" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "math/big" + "net/http" + "sync" + "time" +) + +const ( + integrationProducerMaxKeyID = 128 + integrationProducerMaxKeys = 64 + integrationProducerMaxKeyBytes = 256 << 10 + integrationProducerMinRSABits = 2048 + integrationProducerMaxRSABits = 8192 + integrationProducerMaxExponent = 1<<31 - 1 + integrationProducerKeyTTL = time.Hour + integrationProducerRefreshDelay = time.Minute + integrationProducerHTTPTimeout = 10 * time.Second + integrationProducerJWKS = integrationProducerIssuer + "/.well-known/jwks" +) + +// IntegrationProducerVerifier caches only public keys from GitHub's fixed +// issuer. It never uses the forge client or sends an installation credential. +// Construction performs no I/O; disabled execution need not construct it. +type IntegrationProducerVerifier struct { + mu sync.Mutex + client *http.Client + keys map[string]*rsa.PublicKey + refreshedAt time.Time + attemptedAt time.Time + refreshing chan struct{} +} + +func NewIntegrationProducerVerifier() *IntegrationProducerVerifier { + return &IntegrationProducerVerifier{client: &http.Client{ + Timeout: integrationProducerHTTPTimeout, + CheckRedirect: func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + }, + }} +} + +func (v *IntegrationProducerVerifier) key(ctx context.Context, kid string) (*rsa.PublicKey, error) { + for { + now := time.Now() + v.mu.Lock() + key := v.keys[kid] + if key != nil && now.Sub(v.refreshedAt) < integrationProducerKeyTTL { + v.mu.Unlock() + return key, nil + } + if pending := v.refreshing; pending != nil { + v.mu.Unlock() + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-pending: + continue + } + } + if now.Sub(v.attemptedAt) < integrationProducerRefreshDelay { + v.mu.Unlock() + return nil, fmt.Errorf("integration producer key unavailable; issuer refresh bounded") + } + v.attemptedAt = now + v.refreshing = make(chan struct{}) + v.mu.Unlock() + + keys, err := v.fetch(ctx) + v.mu.Lock() + if err == nil { + v.keys, v.refreshedAt = keys, time.Now() + } + close(v.refreshing) + v.refreshing = nil + v.mu.Unlock() + if err != nil { + return nil, err + } + } +} + +func (v *IntegrationProducerVerifier) fetch(ctx context.Context) (map[string]*rsa.PublicKey, error) { + if v.client == nil { + return nil, fmt.Errorf("integration producer issuer transport unavailable") + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, integrationProducerJWKS, nil) + if err != nil { + return nil, err + } + response, err := v.client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("integration producer issuer returned HTTP %d", response.StatusCode) + } + body, err := io.ReadAll(io.LimitReader(response.Body, integrationProducerMaxKeyBytes+1)) + if err != nil || len(body) > integrationProducerMaxKeyBytes { + return nil, fmt.Errorf("integration producer key response unavailable or oversized") + } + var document struct { + Keys []struct { + KID string `json:"kid"` + KTY string `json:"kty"` + Use string `json:"use"` + Alg string `json:"alg"` + N string `json:"n"` + E string `json:"e"` + } `json:"keys"` + } + if err := json.Unmarshal(body, &document); err != nil || len(document.Keys) > integrationProducerMaxKeys { + return nil, fmt.Errorf("integration producer signing key set invalid") + } + keys := make(map[string]*rsa.PublicKey) + for _, wire := range document.Keys { + if wire.KTY != "RSA" || wire.Use != "sig" || wire.Alg != "RS256" { + continue + } + if wire.KID == "" || len(wire.KID) > integrationProducerMaxKeyID || keys[wire.KID] != nil { + return nil, fmt.Errorf("integration producer signing key identity invalid or duplicated") + } + modulus, nerr := base64.RawURLEncoding.DecodeString(wire.N) + exponent, eerr := base64.RawURLEncoding.DecodeString(wire.E) + n, e := new(big.Int).SetBytes(modulus), new(big.Int).SetBytes(exponent) + if !e.IsInt64() { + return nil, fmt.Errorf("integration producer RSA exponent invalid") + } + exponentValue := e.Int64() + if nerr != nil || eerr != nil || n.BitLen() < integrationProducerMinRSABits || + n.BitLen() > integrationProducerMaxRSABits || exponentValue < 3 || + exponentValue > integrationProducerMaxExponent || e.Bit(0) == 0 { + return nil, fmt.Errorf("integration producer RSA signing key invalid") + } + keys[wire.KID] = &rsa.PublicKey{N: n, E: int(exponentValue)} + } + if len(keys) == 0 { + return nil, fmt.Errorf("integration producer issuer supplied no usable signing keys") + } + return keys, nil +} diff --git a/src/pkg/github/integration_producer_token.go b/src/pkg/github/integration_producer_token.go new file mode 100644 index 0000000000..49df62c214 --- /dev/null +++ b/src/pkg/github/integration_producer_token.go @@ -0,0 +1,168 @@ +package github + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +const ( + integrationProducerIssuer = "https://token.actions.githubusercontent.com" + integrationProducerAudience = "hive-integration-producer-v1/" + integrationProducerRefPrefix = "refs/tags/hive-integration/" + integrationProducerMaxToken = 16 << 10 + integrationProducerMaxPayload = 256 << 10 + integrationProducerClockSkew = 2 * time.Minute + integrationProducerMaxLifetime = time.Hour + integrationProducerDigestBytes = 64 +) + +// IntegrationProducerBinding must come from the enrolled policy, exact +// candidate and authoritative run/attempt/job observations, not the artifact. +type IntegrationProducerBinding struct { + Repo string + RepositoryID int64 + ActorID int64 + RunID int64 + RunAttempt int64 + FinalizerCheckRunID int64 + WorkflowPath string + WorkflowSHA string + SubjectID string + PayloadSHA256 string + JobStartedAt time.Time + JobCompletedAt time.Time + ObservedAt time.Time +} + +// IntegrationProducerIdentity authenticates a manifest's producer only. It is +// not passing CI, current eligibility, promotion permission or a capacity grant. +type IntegrationProducerIdentity struct { + Issuer string + KeyID string + IssuedAt time.Time + RunID int64 + RunAttempt int64 + FinalizerCheckRunID int64 + PayloadSHA256 string +} + +type integrationProducerClaims struct { + jwt.RegisteredClaims + Repository string `json:"repository"` + RepositoryID json.RawMessage `json:"repository_id"` + ActorID json.RawMessage `json:"actor_id"` + RunID json.RawMessage `json:"run_id"` + RunAttempt json.RawMessage `json:"run_attempt"` + CheckRunID json.RawMessage `json:"check_run_id"` + WorkflowRef string `json:"workflow_ref"` + WorkflowSHA string `json:"workflow_sha"` + Ref string `json:"ref"` + RefType string `json:"ref_type"` + EventName string `json:"event_name"` + RunnerEnvironment string `json:"runner_environment"` + Environment string `json:"environment"` +} + +func integrationProducerDigest(value string) bool { + return len(value) == integrationProducerDigestBytes && strings.Trim(value, "0123456789abcdef") == "" && + strings.Trim(value, "0") != "" +} + +func (b IntegrationProducerBinding) validate() error { + parts := strings.Split(b.Repo, "/") + if !integrationProducerDigest(b.SubjectID) || !integrationProducerDigest(b.PayloadSHA256) || + !integrationOID(b.WorkflowSHA) || b.WorkflowSHA != strings.ToLower(b.WorkflowSHA) || + strings.Trim(b.WorkflowSHA, "0") == "" || + b.RepositoryID <= 0 || b.ActorID <= 0 || b.RunID <= 0 || b.RunAttempt <= 0 || b.FinalizerCheckRunID <= 0 || + len(parts) != 2 || parts[0] == "" || parts[1] == "" || strings.ContainsAny(b.Repo, "?#%\\ \t\r\n") || + !strings.HasPrefix(b.WorkflowPath, ".github/workflows/") || + strings.Contains(b.WorkflowPath, "..") || strings.ContainsAny(b.WorkflowPath, "@ \t\r\n") || + !strings.HasSuffix(b.WorkflowPath, ".yml") || + b.JobStartedAt.IsZero() || b.JobCompletedAt.Before(b.JobStartedAt) || + b.ObservedAt.IsZero() || b.JobCompletedAt.After(b.ObservedAt.Add(integrationProducerClockSkew)) { + return fmt.Errorf("incomplete authoritative integration producer binding") + } + return nil +} + +func integrationProducerID(raw json.RawMessage, expected int64) bool { + var text string + if err := json.Unmarshal(raw, &text); err != nil { + var number json.Number + if err := json.Unmarshal(raw, &number); err != nil { + return false + } + text = string(number) + } + return text == strconv.FormatInt(expected, 10) +} + +// Verify authenticates the exact payload digest using GitHub's fixed issuer. +// JWT expiry is not a permanent evidence expiry: the signed issuance must fall +// inside the independently observed finalizer interval. Live eligibility and +// evidence freshness must still be checked separately before any promotion. +func (v *IntegrationProducerVerifier) Verify(ctx context.Context, raw string, payload []byte, binding IntegrationProducerBinding) (IntegrationProducerIdentity, error) { + if err := binding.validate(); err != nil { + return IntegrationProducerIdentity{}, err + } + if len(payload) == 0 || len(payload) > integrationProducerMaxPayload { + return IntegrationProducerIdentity{}, fmt.Errorf("integration producer payload unavailable or oversized") + } + digest := sha256.Sum256(payload) + if hex.EncodeToString(digest[:]) != binding.PayloadSHA256 { + return IntegrationProducerIdentity{}, fmt.Errorf("integration producer payload does not match the bound digest") + } + if v == nil || raw == "" || len(raw) > integrationProducerMaxToken { + return IntegrationProducerIdentity{}, fmt.Errorf("integration producer token unavailable or oversized") + } + claims := new(integrationProducerClaims) + keyID := "" + token, err := jwt.ParseWithClaims(raw, claims, func(token *jwt.Token) (any, error) { + kid, ok := token.Header["kid"].(string) + if !ok || kid == "" || len(kid) > integrationProducerMaxKeyID { + return nil, fmt.Errorf("integration producer signing key identity unavailable") + } + keyID = kid + return v.key(ctx, kid) + }, jwt.WithValidMethods([]string{jwt.SigningMethodRS256.Alg()}), jwt.WithoutClaimsValidation()) + if err != nil { + return IntegrationProducerIdentity{}, fmt.Errorf("integration producer signature rejected: %w", err) + } + if token == nil || !token.Valid { + return IntegrationProducerIdentity{}, fmt.Errorf("integration producer signature unavailable") + } + ref := integrationProducerRefPrefix + binding.SubjectID + if claims.Issuer != integrationProducerIssuer || len(claims.Audience) != 1 || + claims.Audience[0] != integrationProducerAudience+binding.PayloadSHA256 || + claims.Repository != binding.Repo || !integrationProducerID(claims.RepositoryID, binding.RepositoryID) || + !integrationProducerID(claims.ActorID, binding.ActorID) || !integrationProducerID(claims.RunID, binding.RunID) || + !integrationProducerID(claims.RunAttempt, binding.RunAttempt) || + !integrationProducerID(claims.CheckRunID, binding.FinalizerCheckRunID) || + claims.WorkflowRef != binding.Repo+"/"+binding.WorkflowPath+"@"+ref || + claims.WorkflowSHA != binding.WorkflowSHA || claims.Ref != ref || claims.RefType != "tag" || + claims.EventName != "workflow_dispatch" || claims.RunnerEnvironment != "github-hosted" || claims.Environment != "" { + return IntegrationProducerIdentity{}, fmt.Errorf("integration producer identity does not match the authoritative candidate finalizer") + } + if claims.IssuedAt == nil || claims.NotBefore == nil || claims.ExpiresAt == nil || + claims.IssuedAt.Before(binding.JobStartedAt.Add(-integrationProducerClockSkew)) || + claims.IssuedAt.After(binding.JobCompletedAt.Add(integrationProducerClockSkew)) || + claims.NotBefore.Before(claims.IssuedAt.Add(-integrationProducerClockSkew)) || + claims.NotBefore.After(claims.IssuedAt.Add(integrationProducerClockSkew)) || + !claims.ExpiresAt.After(claims.IssuedAt.Time) || + claims.ExpiresAt.Sub(claims.IssuedAt.Time) > integrationProducerMaxLifetime { + return IntegrationProducerIdentity{}, fmt.Errorf("integration producer issuance is outside the authoritative finalizer interval") + } + return IntegrationProducerIdentity{ + Issuer: claims.Issuer, KeyID: keyID, IssuedAt: claims.IssuedAt.Time, + RunID: binding.RunID, RunAttempt: binding.RunAttempt, FinalizerCheckRunID: binding.FinalizerCheckRunID, + PayloadSHA256: binding.PayloadSHA256, + }, nil +} diff --git a/src/pkg/github/integration_producer_token_test.go b/src/pkg/github/integration_producer_token_test.go new file mode 100644 index 0000000000..9e6887fd09 --- /dev/null +++ b/src/pkg/github/integration_producer_token_test.go @@ -0,0 +1,283 @@ +package github + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "math/big" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" +) + +type integrationProducerTransport func(*http.Request) (*http.Response, error) + +const producerIdentityPayload = `{"version":1,"fixture":"producer identity, not CI success"}` + +func (f integrationProducerTransport) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + +func producerIdentityFixture(t *testing.T) (*rsa.PrivateKey, IntegrationProducerBinding, jwt.MapClaims) { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, integrationProducerMinRSABits) + if err != nil { + t.Fatal(err) + } + // Deliberately historical: this attests durable evidence, not bearer-token + // login or a decision that current eligibility is still satisfied. + issued := time.Date(2026, time.October, 1, 12, 0, 0, 0, time.UTC) + binding := IntegrationProducerBinding{ + Repo: "owner/repo", RepositoryID: 1, ActorID: 2, RunID: 3, RunAttempt: 1, FinalizerCheckRunID: 4, + WorkflowPath: ".github/workflows/integration-candidate-validation.yml", + WorkflowSHA: strings.Repeat("a", 40), SubjectID: strings.Repeat("b", integrationProducerDigestBytes), + PayloadSHA256: fmt.Sprintf("%x", sha256.Sum256([]byte(producerIdentityPayload))), + JobStartedAt: issued.Add(-time.Minute), JobCompletedAt: issued.Add(time.Minute), ObservedAt: issued.Add(time.Hour), + } + ref := integrationProducerRefPrefix + binding.SubjectID + return key, binding, jwt.MapClaims{ + "iss": integrationProducerIssuer, "aud": integrationProducerAudience + binding.PayloadSHA256, + "sub": "repo:owner/repo:ref:" + ref, "iat": issued.Unix(), "nbf": issued.Unix(), "exp": issued.Add(time.Minute).Unix(), + "repository": binding.Repo, "repository_id": "1", "actor_id": "2", "run_id": "3", "run_attempt": "1", "check_run_id": "4", + "workflow_ref": binding.Repo + "/" + binding.WorkflowPath + "@" + ref, "workflow_sha": binding.WorkflowSHA, + "ref": ref, "ref_type": "tag", "event_name": "workflow_dispatch", "runner_environment": "github-hosted", + } +} + +func signProducerToken(t *testing.T, key *rsa.PrivateKey, claims jwt.MapClaims) string { + t.Helper() + token := jwt.NewWithClaims(jwt.SigningMethodRS256, claims) + token.Header["kid"] = "github-key" + raw, err := token.SignedString(key) + if err != nil { + t.Fatal(err) + } + return raw +} + +func TestIntegrationProducerAuthenticatesExactHistoricalFinalizer(t *testing.T) { + key, binding, claims := producerIdentityFixture(t) + verifier := NewIntegrationProducerVerifier() + verifier.keys, verifier.refreshedAt = map[string]*rsa.PublicKey{"github-key": &key.PublicKey}, time.Now() + for _, checkID := range []any{"4", json.Number("4")} { + claims["check_run_id"] = checkID + identity, err := verifier.Verify(context.Background(), signProducerToken(t, key, claims), []byte(producerIdentityPayload), binding) + if err != nil || identity.FinalizerCheckRunID != binding.FinalizerCheckRunID || + identity.PayloadSHA256 != binding.PayloadSHA256 || identity.KeyID != "github-key" { + t.Fatalf("exact historical producer: %+v %v", identity, err) + } + } +} + +func TestIntegrationProducerRejectsCrossSubjectRunAndJobClaims(t *testing.T) { + key, binding, original := producerIdentityFixture(t) + verifier := NewIntegrationProducerVerifier() + verifier.keys, verifier.refreshedAt = map[string]*rsa.PublicKey{"github-key": &key.PublicKey}, time.Now() + for _, test := range []struct { + name string + claim string + value any + }{ + {"issuer", "iss", "https://attacker.invalid"}, + {"payload", "aud", integrationProducerAudience + strings.Repeat("d", integrationProducerDigestBytes)}, + {"extra-audience", "aud", []string{integrationProducerAudience + binding.PayloadSHA256, "another-audience"}}, + {"repository", "repository", "other/repo"}, + {"repository-id", "repository_id", "9"}, + {"actor", "actor_id", "9"}, + {"run", "run_id", "9"}, + {"attempt", "run_attempt", "2"}, + {"untrusted-job", "check_run_id", "9"}, + {"missing-job", "check_run_id", nil}, + {"fractional-job", "check_run_id", 4.5}, + {"exponent-job", "check_run_id", json.Number("4e0")}, + {"workflow", "workflow_ref", "owner/repo/.github/workflows/untrusted.yml@" + integrationProducerRefPrefix + binding.SubjectID}, + {"workflow-sha", "workflow_sha", strings.Repeat("d", 40)}, + {"ref", "ref", "refs/heads/v5"}, + {"ref-type", "ref_type", "branch"}, + {"event", "event_name", "pull_request"}, + {"runner", "runner_environment", "self-hosted"}, + {"environment", "environment", "production"}, + {"missing-issuance", "iat", nil}, + {"missing-not-before", "nbf", nil}, + {"missing-expiry", "exp", nil}, + {"early-issuance", "iat", binding.JobStartedAt.Add(-time.Hour).Unix()}, + {"late-issuance", "iat", binding.JobCompletedAt.Add(time.Hour).Unix()}, + {"not-yet-valid", "nbf", binding.JobCompletedAt.Add(time.Hour).Unix()}, + {"unbounded-not-before", "nbf", binding.JobStartedAt.Add(-time.Hour).Unix()}, + {"expired-at-issuance", "exp", binding.JobStartedAt.Unix()}, + {"excessive-lifetime", "exp", binding.JobCompletedAt.Add(integrationProducerMaxLifetime).Unix()}, + } { + t.Run(test.name, func(t *testing.T) { + claims := make(jwt.MapClaims, len(original)) + for name, value := range original { + claims[name] = value + } + claims[test.claim] = test.value + if _, err := verifier.Verify(context.Background(), signProducerToken(t, key, claims), []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("accepted a different, unbound or temporally invalid producer") + } + }) + } +} + +func TestIntegrationProducerRejectsUntrustedSignatureAndAlgorithms(t *testing.T) { + key, binding, claims := producerIdentityFixture(t) + verifier := NewIntegrationProducerVerifier() + verifier.keys, verifier.refreshedAt = map[string]*rsa.PublicKey{"github-key": &key.PublicKey}, time.Now() + other, err := rsa.GenerateKey(rand.Reader, integrationProducerMinRSABits) + if err != nil { + t.Fatal(err) + } + if _, err := verifier.Verify(context.Background(), signProducerToken(t, other, claims), []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("forged producer signature accepted") + } + token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims) + token.Header["kid"] = "github-key" + raw, err := token.SignedString([]byte("not-a-github-signing-key")) + if err != nil { + t.Fatal(err) + } + if _, err := verifier.Verify(context.Background(), raw, []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("RSA/HMAC algorithm confusion accepted") + } + if _, err := verifier.Verify(context.Background(), strings.Repeat("x", integrationProducerMaxToken+1), []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("unbounded token accepted") + } +} + +func producerJWKS(t *testing.T, key *rsa.PrivateKey) string { + t.Helper() + wire := map[string]string{ + "kid": "github-key", "kty": "RSA", "use": "sig", "alg": "RS256", + "n": base64.RawURLEncoding.EncodeToString(key.N.Bytes()), + "e": base64.RawURLEncoding.EncodeToString(big.NewInt(int64(key.E)).Bytes()), + } + body, err := json.Marshal(map[string]any{"keys": []map[string]string{wire}}) + if err != nil { + t.Fatal(err) + } + return string(body) +} + +func TestIntegrationProducerUsesOnlyBoundedUnauthenticatedIssuerKeys(t *testing.T) { + key, binding, claims := producerIdentityFixture(t) + raw, body := signProducerToken(t, key, claims), producerJWKS(t, key) + verifier := NewIntegrationProducerVerifier() + calls := atomic.Int32{} + entered, release := make(chan struct{}), make(chan struct{}) + verifier.client.Transport = integrationProducerTransport(func(request *http.Request) (*http.Response, error) { + calls.Add(1) + if request.URL.String() != integrationProducerJWKS || request.Header.Get("Authorization") != "" { + t.Error("issuer transport leaked authority or trusted a token-selected URL") + } + close(entered) + <-release + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body))}, nil + }) + const parallelRequests = 8 + var waiting sync.WaitGroup + verify := func() { + defer waiting.Done() + if _, err := verifier.Verify(context.Background(), raw, []byte(producerIdentityPayload), binding); err != nil { + t.Error(err) + } + } + waiting.Add(1) + go verify() + <-entered + if !verifier.mu.TryLock() { + close(release) + waiting.Wait() + t.Fatal("remote issuer I/O held the cache mutex") + } + verifier.mu.Unlock() + for range parallelRequests - 1 { + waiting.Add(1) + go verify() + } + close(release) + waiting.Wait() + if calls.Load() != 1 { + t.Fatalf("parallel observation caused issuer poll fanout: %d", calls.Load()) + } + for range parallelRequests { + if _, err := verifier.key(context.Background(), "unknown-key"); err == nil { + t.Fatal("unknown key accepted") + } + } + if calls.Load() != 1 { + t.Fatal("unknown-key flood escaped refresh bounds") + } +} + +func TestIntegrationProducerIssuerFailuresNeverBecomeIdentity(t *testing.T) { + key, binding, claims := producerIdentityFixture(t) + for _, body := range []string{ + `invalid`, `{"keys":[]}`, `{"keys":[{"kid":"k","kty":"EC","use":"sig","alg":"ES256"}]}`, + `{"keys":[{"kid":"k","kty":"RSA","use":"sig","alg":"RS256","n":"AA","e":"Aw"}]}`, + strings.Repeat(" ", integrationProducerMaxKeyBytes+1), + } { + verifier := NewIntegrationProducerVerifier() + verifier.client.Transport = integrationProducerTransport(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body))}, nil + }) + if _, err := verifier.Verify(context.Background(), signProducerToken(t, key, claims), []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("missing or malformed issuer keys became trusted identity") + } + } + verifier := NewIntegrationProducerVerifier() + verifier.client.Transport = integrationProducerTransport(func(*http.Request) (*http.Response, error) { + return nil, fmt.Errorf("issuer unavailable") + }) + if _, err := verifier.Verify(context.Background(), signProducerToken(t, key, claims), []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("unavailable issuer became trusted identity") + } +} + +func TestIntegrationProducerInvalidBindingsDoNotFetchKeys(t *testing.T) { + _, binding, _ := producerIdentityFixture(t) + verifier := NewIntegrationProducerVerifier() + verifier.client.Transport = integrationProducerTransport(func(*http.Request) (*http.Response, error) { + t.Fatal("invalid binding performed remote I/O") + return nil, fmt.Errorf("unexpected network") + }) + for _, change := range []func(*IntegrationProducerBinding){ + func(b *IntegrationProducerBinding) { b.SubjectID = "" }, + func(b *IntegrationProducerBinding) { b.PayloadSHA256 = "" }, + func(b *IntegrationProducerBinding) { b.WorkflowSHA = "short" }, + func(b *IntegrationProducerBinding) { b.RepositoryID = 0 }, + func(b *IntegrationProducerBinding) { b.FinalizerCheckRunID = 0 }, + func(b *IntegrationProducerBinding) { b.Repo = "/repo" }, + func(b *IntegrationProducerBinding) { b.WorkflowPath = "../untrusted.yml" }, + func(b *IntegrationProducerBinding) { b.JobCompletedAt = b.JobStartedAt.Add(-time.Minute) }, + func(b *IntegrationProducerBinding) { b.ObservedAt = time.Time{} }, + } { + bad := binding + change(&bad) + if _, err := verifier.Verify(context.Background(), "invalid", []byte(producerIdentityPayload), bad); err == nil { + t.Fatal("invalid binding accepted") + } + for _, payload := range [][]byte{nil, []byte("replaced manifest"), make([]byte, integrationProducerMaxPayload+1)} { + if _, err := verifier.Verify(context.Background(), "token", payload, binding); err == nil { + t.Fatal("missing, substituted or unbounded payload accepted") + } + } + } + if _, err := verifier.Verify(context.Background(), "", []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("missing token accepted") + } + var missing *IntegrationProducerVerifier + if _, err := missing.Verify(context.Background(), "token", []byte(producerIdentityPayload), binding); err == nil { + t.Fatal("missing verifier accepted") + } +} diff --git a/src/pkg/github/integration_producer_workflows_test.go b/src/pkg/github/integration_producer_workflows_test.go new file mode 100644 index 0000000000..3de3e4088c --- /dev/null +++ b/src/pkg/github/integration_producer_workflows_test.go @@ -0,0 +1,126 @@ +package github + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +type integrationWorkflowContract struct { + On struct { + Call struct { + Inputs map[string]struct { + Type string `yaml:"type"` + Default bool `yaml:"default"` + } `yaml:"inputs"` + } `yaml:"workflow_call"` + Group struct { + Types []string `yaml:"types"` + } `yaml:"merge_group"` + } `yaml:"on"` + Permissions map[string]string `yaml:"permissions"` + Concurrency struct { + Group string `yaml:"group"` + Cancel string `yaml:"cancel-in-progress"` + } `yaml:"concurrency"` + Jobs map[string]struct { + If string `yaml:"if"` + RunsOn string `yaml:"runs-on"` + Permissions map[string]string `yaml:"permissions"` + Steps []struct { + Uses string `yaml:"uses"` + Run string `yaml:"run"` + Env map[string]string `yaml:"env"` + With map[string]string `yaml:"with"` + } `yaml:"steps"` + } `yaml:"jobs"` +} + +func TestIntegrationProducerWorkflowIsolationAndLegacySeams(t *testing.T) { + _, current, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("repository source path unavailable") + } + root := filepath.Clean(filepath.Join(filepath.Dir(current), "../../..")) + for _, file := range []struct { + name string + concurrency string + }{ + {"v2-ci.yml", "integration-ci"}, + {"v2-tests.yml", "integration-tests"}, + {"go-security-analysis.yml", "integration-security"}, + } { + t.Run(file.name, func(t *testing.T) { + body, err := os.ReadFile(filepath.Join(root, ".github/workflows", file.name)) + if err != nil { + t.Fatal(err) + } + var workflow integrationWorkflowContract + if err := yaml.Unmarshal(body, &workflow); err != nil { + t.Fatal(err) + } + flag, exists := workflow.On.Call.Inputs["integration_candidate"] + if !exists || flag.Type != "boolean" || flag.Default { + t.Fatal("candidate execution is not a typed default-false call") + } + if len(workflow.On.Group.Types) != 1 || workflow.On.Group.Types[0] != "checks_requested" { + t.Fatal("landed native merge_group trigger changed") + } + if len(workflow.Permissions) != 1 || workflow.Permissions["contents"] != "read" { + t.Fatal("technical workflow inherited additional authority") + } + if !strings.Contains(workflow.Concurrency.Group, "'"+file.concurrency+"' || github.workflow") || + !strings.Contains(workflow.Concurrency.Group, "github.run_id") || + !strings.Contains(workflow.Concurrency.Cancel, "!inputs.integration_candidate && github.event_name == 'pull_request'") { + t.Fatal("candidate callees collide or change legacy/native concurrency") + } + for name, job := range workflow.Jobs { + writes := false + for _, permission := range job.Permissions { + writes = writes || permission == "write" + } + excluded := strings.Contains(job.If, "!inputs.integration_candidate") + if writes && !excluded { + t.Errorf("job %s can acquire write authority for a candidate", name) + } + if excluded { + continue + } + if job.RunsOn != "ubuntu-latest" && !strings.Contains(job.RunsOn, "!inputs.integration_candidate &&") { + t.Errorf("job %s may execute candidate code on an operator/self-hosted runner", name) + } + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/checkout@") { + if step.With["persist-credentials"] != "${{ !inputs.integration_candidate }}" { + t.Errorf("job %s leaves checkout credentials in candidate code", name) + } + if step.With["ref"] != "" { + t.Errorf("job %s does not test the actual event commit", name) + } + } + if softFail := step.Env["HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL"]; softFail != "" && + !strings.Contains(softFail, "!inputs.integration_candidate &&") { + t.Errorf("job %s permits manual-run toolchain soft failure", name) + } + } + } + scope := workflow.Jobs["required-context-scope"] + var scopeScript strings.Builder + for _, step := range scope.Steps { + scopeScript.WriteString(step.Run) + } + if !strings.Contains(scopeScript.String(), `"$GITHUB_EVENT_NAME" != workflow_dispatch`) || + !strings.Contains(scopeScript.String(), `"$GITHUB_REF" != refs/tags/hive-integration/*`) { + t.Fatal("candidate calls are not restricted to the installed dispatch/ref protocol") + } + if file.name == "v2-tests.yml" && + !strings.Contains(workflow.Jobs["coverage"].If, "inputs.integration_candidate || github.event_name == 'merge_group'") { + t.Fatal("candidate or native coverage was dropped") + } + }) + } +} From 2a166b4e65f1e71a3b09d3050538c336a2b34004 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Fri, 30 Oct 2026 22:54:02 -0400 Subject: [PATCH 2/4] =?UTF-8?q?=E2=9C=A8=20feat:=20materialize=20controlle?= =?UTF-8?q?d=20integration=20prefixes=20with=20isolated=20CI=20receipts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- .github/integration-producer-policy.json | 18 + .github/scripts/generate-integration-ci.py | 166 +++ .github/scripts/integration-materialize.cjs | 311 +++++ .../scripts/integration-materialize.test.cjs | 347 ++++++ .../integration-candidate-prepare.yml | 99 ++ .github/workflows/integration-candidate.yml | 189 ++++ ...gration-technical-go-security-analysis.yml | 211 ++++ .../workflows/integration-technical-v2-ci.yml | 534 +++++++++ .../integration-technical-v2-tests.yml | 1002 +++++++++++++++++ .github/workflows/v2-ci.yml | 8 + ...ed-integration-candidate-producer-trust.md | 2 +- src/docs/integration-orchestration.md | 118 +- src/pkg/github/integration_producer_token.go | 71 +- .../github/integration_producer_token_test.go | 38 +- .../integration_producer_workflows_test.go | 62 +- 15 files changed, 3157 insertions(+), 19 deletions(-) create mode 100644 .github/integration-producer-policy.json create mode 100644 .github/scripts/generate-integration-ci.py create mode 100644 .github/scripts/integration-materialize.cjs create mode 100644 .github/scripts/integration-materialize.test.cjs create mode 100644 .github/workflows/integration-candidate-prepare.yml create mode 100644 .github/workflows/integration-candidate.yml create mode 100644 .github/workflows/integration-technical-go-security-analysis.yml create mode 100644 .github/workflows/integration-technical-v2-ci.yml create mode 100644 .github/workflows/integration-technical-v2-tests.yml diff --git a/.github/integration-producer-policy.json b/.github/integration-producer-policy.json new file mode 100644 index 0000000000..4ba18e6d6a --- /dev/null +++ b/.github/integration-producer-policy.json @@ -0,0 +1,18 @@ +{ + "schema": 1, + "enabled": false, + "authority": "forge-single-attempt-v1", + "actorIDs": [], + "scopes": [], + "maxMembers": 1, + "protectedPaths": [ + ".github", + "src/.golangci.yml", + "src/scripts", + "bin" + ], + "committer": { + "name": "github-actions[bot]", + "email": "41898282+github-actions[bot]@users.noreply.github.com" + } +} diff --git a/.github/scripts/generate-integration-ci.py b/.github/scripts/generate-integration-ci.py new file mode 100644 index 0000000000..656546e1b8 --- /dev/null +++ b/.github/scripts/generate-integration-ci.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +"""Generate readonly candidate callees without changing ordinary/native CI.""" + +import argparse +import copy +import difflib +import re +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +SOURCES = { + "v2-ci.yml": { + "create-issue-on-failure", + "nightly-release", + }, + "v2-tests.yml": { + "patch-coverage", + "create-issue-on-failure", + "create-issues-on-flakes", + "warm-build-cache", + }, + "go-security-analysis.yml": set(), +} +FLAG = "inputs.integration_candidate" +PROFILE_VARIABLES = { + "vars.HIVE_GO_BUILD_CACHE": "'job'", + "vars.HIVE_RUNNER_LABELS": "'[\"ubuntu-latest\"]'", + "vars.HIVE_AFFECTED_TEST_SELECTION": "'false'", +} +HEADER = ( + "# Generated by .github/scripts/generate-integration-ci.py; do not edit.\n" + "# Candidate-only technical CI. Ordinary and native CI remain in the source.\n" +) + + +class WorkflowLoader(yaml.SafeLoader): + pass + + +# GitHub's `on` is a string, not YAML 1.1's boolean spelling. +WorkflowLoader.yaml_implicit_resolvers = { + key: [ + (tag, pattern) + for tag, pattern in resolvers + if tag != "tag:yaml.org,2002:bool" + ] + for key, resolvers in yaml.SafeLoader.yaml_implicit_resolvers.items() +} +WorkflowLoader.add_implicit_resolver( + "tag:yaml.org,2002:bool", re.compile(r"^(?:true|false)$", re.I), list("tTfF") +) + + +class WorkflowDumper(yaml.SafeDumper): + def ignore_aliases(self, _data): + return True + + +def represent_string(dumper, value): + return dumper.represent_scalar( + "tag:yaml.org,2002:str", value, style="|" if "\n" in value else None + ) + + +WorkflowDumper.add_representer(str, represent_string) + + +def candidate_only(value): + if isinstance(value, str): + value = value.replace(FLAG, "true") + value = re.sub( + r"\bvars\.[A-Za-z0-9_]+", + lambda match: PROFILE_VARIABLES.get(match.group(0), match.group(0)), + value, + ) + if re.search(r"\bvars\.", value) or re.search(r"\$\{\{[^}]*\bvars\b", value): + raise ValueError("unqualified mutable workflow variable in candidate profile") + return value + if isinstance(value, list): + return [candidate_only(item) for item in value] + if isinstance(value, dict): + return {key: candidate_only(item) for key, item in value.items()} + return value + + +def render(source, document): + if source not in SOURCES: + raise ValueError(f"unsupported source {source}") + document = copy.deepcopy(document) + if set(document) != { + "name", "on", "permissions", "concurrency", "env", "jobs" + }: + raise ValueError(f"{source}: source workflow structure needs review") + candidate_input = ( + document["on"].get("workflow_call", {}).get("inputs", {}).get( + "integration_candidate" + ) + ) + if not isinstance(candidate_input, dict) or ( + candidate_input.get("type") != "boolean" + or candidate_input.get("default") is not False + ): + raise ValueError(f"{source}: candidate entrypoint contract changed") + if document["permissions"] != {"contents": "read"}: + raise ValueError(f"{source}: readonly root permission contract changed") + jobs = document["jobs"] + for name in SOURCES[source]: + if name not in jobs or f"!{FLAG}" not in str(jobs[name].get("if", "")): + raise ValueError(f"{source}: excluded job {name} lost its guard") + del jobs[name] + for name, job in jobs.items(): + if any(value not in {"read", "none"} for value in job.get("permissions", {}).values()): + raise ValueError(f"{source}: active job {name} requests write authority") + runner = job.get("runs-on", "") + if runner != "ubuntu-latest" and FLAG not in runner: + raise ValueError(f"{source}: active job {name} lacks candidate runner isolation") + job["runs-on"] = "ubuntu-latest" + needs = job.get("needs", []) + if isinstance(needs, str): + needs = [needs] + if any(dependency not in jobs for dependency in needs): + raise ValueError(f"{source}: active job {name} depends on an excluded job") + for step in job.get("steps", []): + if str(step.get("uses", "")).startswith("actions/checkout@"): + checkout = step.setdefault("with", {}) + if "ref" in checkout: + raise ValueError(f"{source}: active job {name} overrides the tested subject") + checkout["persist-credentials"] = False + document["name"] = f"Integration candidate / {document['name']}" + document["on"] = {"workflow_call": None} + document["concurrency"]["cancel-in-progress"] = False + document = candidate_only(document) + return HEADER + yaml.dump( + document, Dumper=WorkflowDumper, sort_keys=False, width=10000 + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + different = False + for source in SOURCES: + original = ROOT / ".github" / "workflows" / source + output = original.with_name(f"integration-technical-{source}") + expected = render(source, yaml.load(original.read_text(), Loader=WorkflowLoader)) + if args.check: + actual = output.read_text() if output.exists() else "" + if actual != expected: + different = True + print("".join(difflib.unified_diff( + actual.splitlines(True), expected.splitlines(True), + fromfile=str(output), tofile=f"{output} (generated)" + ))) + else: + output.write_text(expected) + print(output.relative_to(ROOT)) + if different: + raise SystemExit("Run python3 .github/scripts/generate-integration-ci.py") + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/integration-materialize.cjs b/.github/scripts/integration-materialize.cjs new file mode 100644 index 0000000000..4d5f2d3066 --- /dev/null +++ b/.github/scripts/integration-materialize.cjs @@ -0,0 +1,311 @@ +'use strict'; + +const crypto = require('node:crypto'); +const {execFileSync} = require('node:child_process'); + +const PROTOCOL = 'integration-prefix-v1'; +const MAX_MEMBERS = 8; +const MAX_REQUEST_BYTES = 32 * 1024; +const MAX_GIT_OUTPUT_BYTES = 1024 * 1024; +const GIT_TIMEOUT_MS = 120000; +const SHA = /^[a-f0-9]{40}$/; +const DIGEST = /^[a-f0-9]{64}$/; +const DECIMAL_ID = /^[1-9][0-9]*$/; +const SAFE_POLICY_PATH = /^[a-zA-Z0-9_.-]+(?:\/[a-zA-Z0-9_.-]+)*$/; + +function canonical(value) { + if (value === null || typeof value !== 'object') return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; +} + +function digest(value) { + return crypto.createHash('sha256').update(value).digest('hex'); +} + +function objectID(value) { + return typeof value === 'string' && SHA.test(value) && !/^0+$/.test(value); +} + +function reject(code, message) { + const error = new Error(`${code}: ${message}`); + error.reason = code; + throw error; +} + +function parseRequest(raw) { + if (typeof raw !== 'string' || Buffer.byteLength(raw, 'utf8') > MAX_REQUEST_BYTES) { + reject('request_size', 'bounded request required'); + } + let request; + try { request = JSON.parse(raw); } catch { reject('request_json', 'invalid request JSON'); } + const fields = ['schema', 'protocol', 'engine', 'repo', 'base', 'baseSHA', + 'policy', 'generation', 'members', 'createdAt']; + if (!request || request.schema !== 1 || request.protocol !== PROTOCOL || + Object.keys(request).some(key => !fields.includes(key)) || + request.engine !== 'hive' || typeof request.repo !== 'string' || + typeof request.base !== 'string' || !request.base.length || + !objectID(request.baseSHA) || typeof request.policy !== 'string' || + !DIGEST.test(request.policy) || /^0+$/.test(request.policy) || + !Number.isSafeInteger(request.generation) || request.generation < 1 || + !Array.isArray(request.members) || !request.members.length || + request.members.length > MAX_MEMBERS || + typeof request.createdAt !== 'string' || + !Number.isFinite(Date.parse(request.createdAt)) || + new Date(request.createdAt).toISOString() !== request.createdAt) { + reject('request_identity', 'invalid versioned preparation identity'); + } + const seen = new Set(); + const heads = new Set(); + for (const member of request.members) { + if (!member || !Number.isSafeInteger(member.pr) || member.pr < 1 || + Object.keys(member).some(key => !['pr', 'head'].includes(key)) || + !objectID(member.head) || seen.has(member.pr) || heads.has(member.head)) { + reject('request_member', 'invalid or duplicate ordered member'); + } + seen.add(member.pr); + heads.add(member.head); + } + return request; +} + +function authorize(request, policyText, actorID, repo, sourceEvent, sourceRef, defaultBranch, eventAction) { + let policy; + try { policy = JSON.parse(policyText); } catch { reject('policy_json', 'invalid installed producer policy'); } + const installedEvent = sourceEvent === 'workflow_dispatch' || + (sourceEvent === 'repository_dispatch' && eventAction === 'hive.integration.prepare.v1'); + if (!installedEvent || sourceRef !== `refs/heads/${defaultBranch}`) { + reject('source_ref', 'materializer must use its trusted default-branch dispatch'); + } + if (!policy || policy.schema !== 1 || policy.enabled !== true || + policy.authority !== 'forge-single-attempt-v1' || + !Array.isArray(policy.actorIDs) || policy.actorIDs.length !== 1 || !DECIMAL_ID.test(actorID) || + !policy.actorIDs.includes(actorID) || !Array.isArray(policy.scopes) || + request.repo !== repo || digest(policyText) !== request.policy || + !policy.scopes.some(scope => scope && scope.repo === repo && scope.base === request.base)) { + reject('producer_not_enrolled', 'disabled, unauthorized actor, policy, or destination'); + } + if (!Number.isSafeInteger(policy.maxMembers) || policy.maxMembers < 1 || + policy.maxMembers > MAX_MEMBERS || request.members.length > policy.maxMembers || + !Array.isArray(policy.protectedPaths) || !policy.protectedPaths.includes('.github') || + !policy.protectedPaths.length || + policy.protectedPaths.some(path => typeof path !== 'string' || + !SAFE_POLICY_PATH.test(path) || path.split('/').some(part => part === '.' || part === '..' || part === '.git'))) { + reject('producer_bounds', 'invalid installed bounds or protected policy closure'); + } + const committer = policy.committer; + if (!committer || typeof committer.name !== 'string' || !committer.name.trim() || + typeof committer.email !== 'string' || !/^[^\s<>@\r\n]+@[^\s<>@\r\n]+$/.test(committer.email) || + /[\r\n<>]/.test(committer.name)) { + reject('producer_committer', 'explicit valid DCO identity required'); + } + return policy; +} + +function gitRunner(token) { + if (typeof token !== 'string' || !token) reject('producer_token', 'trusted job credential missing'); + const header = `AUTHORIZATION: basic ${Buffer.from(`x-access-token:${token}`).toString('base64')}`; + const env = { + ...process.env, + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_CONFIG_SYSTEM: '/dev/null', + GIT_CONFIG_COUNT: '1', + GIT_CONFIG_KEY_0: 'http.https://github.com/.extraheader', + GIT_CONFIG_VALUE_0: header, + GIT_TERMINAL_PROMPT: '0', + GIT_TRACE: '0', + GIT_TRACE_CURL: '0', + GIT_CURL_VERBOSE: '0', + }; + return (args, extraEnv = {}) => execFileSync( + 'git', ['-c', 'core.hooksPath=/dev/null', '--literal-pathspecs', ...args], + {env: {...env, ...extraEnv}, encoding: 'utf8', timeout: GIT_TIMEOUT_MS, + maxBuffer: MAX_GIT_OUTPUT_BYTES, stdio: ['ignore', 'pipe', 'pipe']} + ).trim(); +} + +function makeSubject(request, members, prefixes, workflowTree, preparedBy) { + const subject = { + schema: 1, protocol: PROTOCOL, engine: 'hive', repo: request.repo, + base: request.base, baseSHA: request.baseSHA, policy: request.policy, + workflowTree, generation: request.generation, members, prefixes, + }; + const {generation, ...immutable} = subject; + subject.id = digest(canonical(immutable)); + subject.preparedBy = preparedBy; + return subject; +} + +async function claimCandidateDispatch({github, source, subject, execution, committer, now}) { + const key = digest(`integration-ci-initial-v1/${subject.id}`); + const ref = `heads/hive-integration-intents/${key}`; + let existing; + try { existing = await github.rest.git.getRef({...source, ref}); } catch (error) { + if (error.status !== 404) return {state: 'unknown', key, ref}; + } + if (existing) return {state: 'observer', key, ref, commit: existing.data.object?.sha}; + const owner = {...execution, nonce: crypto.randomBytes(16).toString('hex')}; + const record = {schema: 1, kind: 'integration-ci-single-attempt-v1', key, subjectID: subject.id, owner}; + const last = subject.prefixes.at(-1); + let claimCommit; + try { + const {data: commit} = await github.rest.git.createCommit({ + ...source, tree: last.tree, parents: [last.commit], + message: `Integration CI intent ${key}\n\n${canonical(record)}\n\nSigned-off-by: ${committer.name} <${committer.email}>\n`, + author: {...committer, date: now.toISOString()}, + committer: {...committer, date: now.toISOString()}, + }); + claimCommit = commit.sha; + if (!objectID(claimCommit)) return {state: 'unknown', key, ref}; + const acknowledgement = await github.rest.git.createRef({...source, ref: `refs/${ref}`, sha: claimCommit}); + if (acknowledgement.status !== 201 || acknowledgement.data?.ref !== `refs/${ref}` || + acknowledgement.data.object?.sha !== claimCommit) { + return {state: 'unknown', key, ref, commit: claimCommit, owner}; + } + return {state: 'claimed', key, ref, commit: claimCommit, owner}; + } catch { + try { + const {data: current} = await github.rest.git.getRef({...source, ref}); + return {state: claimCommit && current.object?.sha === claimCommit ? 'claimed' : 'observer', + key, ref, commit: current.object?.sha, owner}; + } catch { return {state: 'unknown', key, ref, commit: claimCommit, owner}; } + } +} + +async function prepare({github, context, requestText, policyText, actorID, execution, git, now = new Date()}) { + if (context.serverUrl !== 'https://github.com' || context.apiUrl !== 'https://api.github.com') { + reject('provider_unsupported', 'this installed producer supports public GitHub only; no credential forwarding'); + } + const request = parseRequest(requestText); + const repo = `${context.repo.owner}/${context.repo.repo}`; + const policy = authorize(request, policyText, actorID, repo, context.eventName, + context.ref, context.payload.repository.default_branch, context.payload.action); + const maxRequestAgeMs = 15 * 60 * 1000; + const futureClockAllowanceMs = 2 * 60 * 1000; + const age = now.getTime() - Date.parse(request.createdAt); + if (age > maxRequestAgeMs || age < -futureClockAllowanceMs) { + reject('request_expired', 'request is outside the bounded preparation interval'); + } + if (!execution || !DECIMAL_ID.test(execution.runID) || !DECIMAL_ID.test(execution.attempt)) { + reject('executor_unknown', 'actual trusted job run/attempt is required; no TTL takeover'); + } + git(['check-ref-format', `refs/heads/${request.base}`]); + const sourceSHA = context.sha; + if (!objectID(sourceSHA)) reject('source_sha', 'trusted source commit is unknown'); + const source = {owner: context.repo.owner, repo: context.repo.repo}; + const {data: destination} = await github.rest.git.getRef({...source, ref: `heads/${request.base}`}); + if (destination.object?.sha !== request.baseSHA) { + reject('base_changed', 'requested destination is no longer its exact starting base'); + } + const protectedObjects = new Map(policy.protectedPaths.map(path => + [path, git(['ls-tree', sourceSHA, '--', path])])); + if ([...protectedObjects.values()].some(value => !value)) { + reject('policy_closure_unknown', 'protected path missing from trusted source'); + } + const workflowTree = git(['rev-parse', '--verify', `${sourceSHA}:.github`]); + if (!objectID(workflowTree)) reject('policy_closure_unknown', 'trusted workflow tree missing'); + const gitVersion = git(['--version']); + const requestDigest = digest(canonical(request)); + const preparedBy = {sourceSHA, requestDigest, gitVersion}; + const remote = `https://github.com/${repo}.git`; + const prepared = []; + const members = []; + const prefixes = []; + let parent = request.baseSHA; + const commitEnv = { + GIT_AUTHOR_NAME: policy.committer.name, + GIT_AUTHOR_EMAIL: policy.committer.email, + GIT_COMMITTER_NAME: policy.committer.name, + GIT_COMMITTER_EMAIL: policy.committer.email, + }; + let failure = null; + try { + git(['fetch', '--quiet', '--no-tags', '--depth=1', remote, request.baseSHA]); + for (const member of request.members) { + const {data: pr} = await github.rest.pulls.get({...source, pull_number: member.pr}); + if (pr.state !== 'open' || pr.draft || pr.head?.sha !== member.head || + pr.head?.repo?.full_name !== repo || pr.base?.repo?.full_name !== repo || + pr.base?.ref !== request.base) { + reject('member_changed', `member #${member.pr} changed, forked, closed, or became draft`); + } + const {data: comparison} = await github.rest.repos.compareCommitsWithBasehead({ + ...source, basehead: `${parent}...${member.head}`, per_page: 1, + }); + const mergeBase = comparison.merge_base_commit?.sha; + if (!objectID(mergeBase)) reject('merge_base_unknown', `no authoritative merge base for #${member.pr}`); + if (mergeBase === member.head) reject('member_integrated', `#${member.pr} already occurs in the prefix`); + git(['fetch', '--quiet', '--no-tags', '--depth=1', remote, member.head, mergeBase]); + let tree; + try { + tree = git(['merge-tree', '--write-tree', `--merge-base=${mergeBase}`, parent, member.head]).split('\n')[0]; + } catch (error) { + if (error.status === 1) { + reject('materialization_conflict', `ordered prefix through #${member.pr} needs owner-first repair`); + } + reject('materialization_unknown', `Git execution for #${member.pr} is unavailable; not a proven textual conflict`); + } + if (!objectID(tree)) reject('materialization_unknown', 'merge did not return an exact tree'); + const timestamp = Math.max(...[parent, member.head].map(sha => { + const value = git(['show', '-s', '--format=%ct', sha]); + if (!DECIMAL_ID.test(value) || !Number.isSafeInteger(Number(value))) { + reject('commit_time_unknown', 'deterministic parent commit timestamp unavailable'); + } + return Number(value); + })); + const date = new Date((timestamp + 1) * 1000); + if (!Number.isFinite(date.getTime())) reject('commit_time_unknown', 'parent commit timestamp is out of range'); + const prefixKey = digest(canonical({protocol: PROTOCOL, repo, base: request.base, + baseSHA: request.baseSHA, policy: request.policy, workflowTree, + members: [...members, member]})); + const message = `Integration candidate ${prefixKey}\n\nSigned-off-by: ${policy.committer.name} <${policy.committer.email}>\n`; + const commit = git(['-c', 'commit.gpgsign=false', 'commit-tree', tree, + '-p', parent, '-p', member.head, '-m', message], + {...commitEnv, GIT_AUTHOR_DATE: date.toISOString(), GIT_COMMITTER_DATE: date.toISOString()}); + if (!objectID(commit)) reject('materialization_unknown', 'merge did not return an exact commit'); + for (const [path, expected] of protectedObjects) { + if (git(['ls-tree', commit, '--', path]) !== expected) { + reject('policy_change', `candidate changes installed policy closure ${path}; separate qualification required`); + } + } + members.push({...member}); + prefixes.push({commit, tree}); + const subject = makeSubject(request, [...members], [...prefixes], workflowTree, preparedBy); + // GitHub-token publication suppresses unrelated push-triggered workflows. + // No candidate tree is checked out and no author or destination ref is changed. + const result = {subject, publication: 'unresolved', dispatch: 'not_requested'}; + prepared.push(result); + git(['push', '--quiet', remote, `${commit}:refs/tags/hive-integration/${subject.id}`]); + result.publication = 'accepted'; + result.ownership = await claimCandidateDispatch({github, source, subject, execution, committer: policy.committer, now}); + if (result.ownership.state === 'unknown') { + reject('dispatch_authority_unknown', 'shared dispatch intent is unresolved; no TTL steal or mutation'); + } + if (result.ownership.state === 'observer') { + result.dispatch = 'observer'; + parent = commit; + continue; + } + subject.dispatchClaim = result.ownership; + const manifest = canonical(subject); + if (Buffer.byteLength(manifest, 'utf8') > MAX_REQUEST_BYTES) reject('subject_size', 'candidate manifest exceeds dispatch bounds'); + result.dispatch = 'unresolved'; + try { + const acknowledgement = await github.rest.actions.createWorkflowDispatch({ + ...source, workflow_id: 'integration-candidate.yml', + ref: `hive-integration/${subject.id}`, inputs: {manifest}, + }); + if (acknowledgement.status !== 204) throw new Error('unknown dispatch acknowledgement'); + result.dispatch = 'accepted'; + } catch { + reject('dispatch_unresolved', `candidate ${subject.id} dispatch has no authoritative acknowledgement; reconcile, never blindly retry`); + } + parent = commit; + } + } catch (error) { + failure = {reason: error.reason || 'provider_unknown', message: error.reason ? error.message : 'Provider/materializer failed; outcome requires bounded reconciliation'}; + } + return {schema: 1, protocol: 'integration-materializer-receipt-v1', + request, prepared, failure, preparedBy}; +} + +module.exports = {authorize, canonical, claimCandidateDispatch, digest, gitRunner, makeSubject, parseRequest, prepare}; diff --git a/.github/scripts/integration-materialize.test.cjs b/.github/scripts/integration-materialize.test.cjs new file mode 100644 index 0000000000..fcf63da68a --- /dev/null +++ b/.github/scripts/integration-materialize.test.cjs @@ -0,0 +1,347 @@ +'use strict'; + +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const {execFileSync} = require('node:child_process'); +const test = require('node:test'); +const {canonical, digest, makeSubject, parseRequest, prepare} = require('./integration-materialize.cjs'); + +const oid = value => value.repeat(40); +const clock = new Date('2026-10-31T01:00:00.000Z'); +const PARENT_EPOCH = 1793400000; + +function fixture(remote = new Map()) { + const policyText = JSON.stringify({ + schema: 1, enabled: true, authority: 'forge-single-attempt-v1', actorIDs: ['42'], + scopes: [{repo: 'owner/repo', base: 'v5'}], maxMembers: 8, + protectedPaths: ['.github', 'src/scripts'], + committer: {name: 'integration-bot', email: 'bot@example.invalid'}, + }); + const request = { + schema: 1, protocol: 'integration-prefix-v1', engine: 'hive', + repo: 'owner/repo', base: 'v5', baseSHA: oid('1'), + policy: digest(policyText), generation: 1, createdAt: clock.toISOString(), + members: [{pr: 10, head: oid('2')}, {pr: 11, head: oid('3')}], + }; + const context = { + serverUrl: 'https://github.com', apiUrl: 'https://api.github.com', + repo: {owner: 'owner', repo: 'repo'}, sha: oid('9'), + ref: 'refs/heads/v5', eventName: 'workflow_dispatch', + payload: {repository: {default_branch: 'v5'}}, + }; + const calls = []; + let commits = 0; + const git = (args, env = {}) => { + calls.push({type: 'git', args, env}); + const operation = args.includes('commit-tree') ? 'commit-tree' : args[0]; + switch (operation) { + case 'ls-tree': return `040000 tree ${oid('a')}\t${args.at(-1)}`; + case 'rev-parse': return oid('a'); + case '--version': return 'git version 2.48.1'; + case 'show': return String(PARENT_EPOCH + (args.at(-1) === oid('5') ? 1 : 0)); + case 'merge-tree': return commits ? oid('8') : oid('7'); + case 'commit-tree': return ++commits === 1 ? oid('5') : oid('6'); + case 'fetch': + case 'push': + case 'check-ref-format': return ''; + default: throw new Error(`unexpected Git operation ${operation}`); + } + }; + const github = {rest: { + git: { + getRef: async ({ref}) => { + if (ref === 'heads/v5') return {data: {object: {sha: request.baseSHA}}}; + if (remote.has(ref)) return {data: {object: {sha: remote.get(ref)}}}; + const error = new Error('ref absent'); error.status = 404; throw error; + }, + createCommit: async input => ({data: {sha: crypto.createHash('sha1').update(input.message).digest('hex')}}), + createRef: async ({ref, sha}) => { + const name = ref.replace(/^refs\//, ''); + if (remote.has(name)) { const error = new Error('ref exists'); error.status = 422; throw error; } + remote.set(name, sha); + return {status: 201, data: {ref, object: {sha}}}; + }, + }, + pulls: {get: async ({pull_number}) => { + const member = request.members.find(item => item.pr === pull_number); + return {data: {state: 'open', draft: false, + head: {sha: member.head, repo: {full_name: 'owner/repo'}}, + base: {ref: 'v5', repo: {full_name: 'owner/repo'}}}}; + }}, + repos: {compareCommitsWithBasehead: async input => { + calls.push({type: 'compare', input}); + return {data: {merge_base_commit: {sha: oid('4')}}}; + }}, + actions: {createWorkflowDispatch: async input => { + calls.push({type: 'dispatch', input}); + return {status: 204}; + }}, + }}; + return { + policyText, request, context, calls, git, github, + args() { return {github, context, requestText: JSON.stringify(request), + policyText: this.policyText, actorID: '42', execution: {runID: '100', attempt: '1'}, + git: this.git, now: clock}; }, + }; +} + +test('ordered prefixes dispatch as soon as published, without candidate checkout/base writes', async () => { + const f = fixture(); + const result = await prepare(f.args()); + assert.equal(result.failure, null); + assert.equal(result.prepared.length, 2); + const [first, second] = result.prepared; + assert.deepEqual(first.subject.members, f.request.members.slice(0, 1)); + assert.deepEqual(second.subject.members, f.request.members); + assert.equal(second.subject.prefixes[0].commit, first.subject.prefixes[0].commit); + assert.equal(first.publication, 'accepted'); + assert.equal(second.dispatch, 'accepted'); + const calls = f.calls; + const firstDispatch = calls.findIndex(call => call.type === 'dispatch'); + const secondMerge = calls.findIndex(call => call.type === 'git' && + call.args[0] === 'merge-tree' && call.args.includes(oid('3'))); + assert.ok(firstDispatch < secondMerge, 'CI must overlap later materialization'); + const comparisons = calls.filter(call => call.type === 'compare'); + assert.equal(comparisons[0].input.basehead, `${oid('1')}...${oid('2')}`); + assert.equal(comparisons[1].input.basehead, `${oid('5')}...${oid('3')}`); + let candidateNumber = 0; + for (const call of calls.filter(call => call.type === 'git')) { + assert.ok(!call.args.includes('checkout')); + assert.ok(!call.args.includes('--force')); + if (call.args[0] === 'push') { + assert.match(call.args.at(-1), /^[a-f0-9]{40}:refs\/tags\/hive-integration\/[a-f0-9]{64}$/); + } + if (call.args.includes('commit-tree')) { + const index = call.args.indexOf('commit-tree'); + assert.equal(call.args[index + 2], '-p'); + assert.equal(call.args[index + 4], '-p'); + assert.match(call.args.at(-1), /Signed-off-by: integration-bot /); + assert.equal(call.env.GIT_COMMITTER_DATE, new Date((PARENT_EPOCH + ++candidateNumber) * 1000).toISOString()); + } + } +}); + +test('unchanged subject identity does not depend on observer/materializer source instance', () => { + const f = fixture(), prefixes = [{commit: oid('5'), tree: oid('7')}]; + const first = makeSubject(f.request, f.request.members.slice(0, 1), prefixes, oid('a'), {sourceSHA: oid('9')}); + const replay = makeSubject(f.request, f.request.members.slice(0, 1), prefixes, oid('a'), {sourceSHA: oid('b')}); + assert.equal(first.id, replay.id); + const changed = makeSubject(f.request, [f.request.members[1]], prefixes, oid('a'), {}); + assert.notEqual(first.id, changed.id, 'base+A+B evidence cannot become base+B evidence'); + assert.equal(canonical({z: 1, a: ['x', {b: true}]}), '{"a":["x",{"b":true}],"z":1}'); +}); + +test('independent hives share one exact-subject dispatch owner; no TTL steal or rerun', async () => { + const remote = new Map(), first = fixture(remote), second = fixture(remote); + const original = await prepare(first.args()); + assert.equal(original.failure, null); + const args = second.args(); + args.execution = {runID: '200', attempt: '1'}; + second.request.generation = 2; + second.request.createdAt = new Date(clock.getTime() + 120000).toISOString(); + args.requestText = JSON.stringify(second.request); + args.now = new Date(second.request.createdAt); + const observed = await prepare(args); + assert.equal(observed.failure, null); + assert.deepEqual(observed.prepared.map(item => item.subject.id), original.prepared.map(item => item.subject.id)); + assert.ok(observed.prepared.every(item => item.dispatch === 'observer')); + assert.ok(!second.calls.some(call => call.type === 'dispatch')); + assert.equal(remote.size, 2); +}); + +test('default disabled policy, changed actor/scope/source and stale request cannot start Git effects', async () => { + const cases = [ + args => { args.policyText = args.policyText.replace('"enabled":true', '"enabled":false'); }, + args => { args.actorID = '43'; }, + args => { args.context.ref = 'refs/heads/untrusted'; }, + args => { args.context.eventName = 'pull_request'; }, + args => { args.now = new Date(clock.getTime() + 16 * 60 * 1000); }, + ]; + for (const modify of cases) { + const f = fixture(), args = f.args(); + modify(args); + await assert.rejects(prepare(args)); + assert.equal(f.calls.length, 0); + } +}); + +test('changed base/member and policy closure never publish or dispatch candidate work', async () => { + for (const reason of ['base', 'member', 'closure']) { + const f = fixture(); + if (reason === 'base') f.github.rest.git.getRef = async () => ({data: {object: {sha: oid('b')}}}); + if (reason === 'member') f.github.rest.pulls.get = async () => ({data: {state: 'closed'}}); + if (reason === 'closure') { + const original = f.git; + f.git = (args, env) => args[0] === 'ls-tree' && args[1] !== f.context.sha ? + '120000 blob changed\t.github' : original(args, env); + } + let result; + try { result = await prepare(f.args()); } catch (error) { + assert.equal(reason, 'base'); + assert.equal(error.reason, 'base_changed'); + } + if (result) assert.ok(result.failure); + assert.ok(!f.calls.some(call => call.type === 'dispatch' || + (call.type === 'git' && call.args[0] === 'push'))); + } +}); + +test('a later textual conflict retains exact earlier published/accepted prefix', async () => { + const f = fixture(), original = f.git; + f.git = (args, env) => { + if (args[0] === 'merge-tree' && args.includes(oid('3'))) { + const error = new Error('conflict'); + error.status = 1; + throw error; + } + return original(args, env); + }; + const result = await prepare(f.args()); + assert.equal(result.failure.reason, 'materialization_conflict'); + assert.equal(result.prepared.length, 1); + assert.equal(result.prepared[0].dispatch, 'accepted'); + assert.equal(f.calls.filter(call => call.type === 'dispatch').length, 1); +}); + +test('ambiguous publication/dispatch is retained for reconciliation, never blindly retried', async () => { + for (const operation of ['push', 'dispatch']) { + const f = fixture(), original = f.git; + let attempts = 0; + if (operation === 'push') f.git = (args, env) => { + if (args[0] === 'push') { attempts++; throw new Error('lost acknowledgement'); } + return original(args, env); + }; + else f.github.rest.actions.createWorkflowDispatch = async () => { + attempts++; + throw new Error('lost acknowledgement'); + }; + const result = await prepare(f.args()); + assert.ok(result.failure); + assert.equal(result.prepared.length, 1); + assert.equal(attempts, 1); + assert.equal(result.prepared[0][operation === 'push' ? 'publication' : 'dispatch'], 'unresolved'); + } +}); + +test('malformed, extra, duplicate, zero and oversized request identities fail closed', () => { + const f = fixture(); + for (const raw of [ + 'null', '[]', '{}', '{broken', ' '.repeat(33 * 1024), + JSON.stringify({...f.request, injected: 'not a supported field'}), + JSON.stringify({...f.request, baseSHA: oid('0')}), + JSON.stringify({...f.request, members: [f.request.members[0], f.request.members[0]]}), + JSON.stringify({...f.request, members: [f.request.members[0], {pr: 12, head: f.request.members[0].head}]}), + JSON.stringify({...f.request, members: [{pr: 1, head: oid('2'), extra: true}]}), + ]) assert.throws(() => parseRequest(raw)); +}); + +test('create-ref lost acknowledgement resolves only the exact own pre-effect claim', async () => { + const f = fixture(), createRef = f.github.rest.git.createRef; + f.github.rest.git.createRef = async input => { + await createRef(input); + throw new Error('lost create-ref acknowledgement'); + }; + const result = await prepare(f.args()); + assert.equal(result.failure, null); + assert.ok(result.prepared.every(item => item.ownership.state === 'claimed')); + assert.equal(f.calls.filter(call => call.type === 'dispatch').length, 2); +}); + +test('foreign winners and unreadable/malformed claim acknowledgements never authorize dispatch', async () => { + for (const reason of ['foreign', 'read_unavailable', 'ack_malformed']) { + const remote = new Map(), f = fixture(remote); + if (reason === 'foreign') f.github.rest.git.createRef = async ({ref}) => { + remote.set(ref.replace(/^refs\//, ''), oid('b')); + const error = new Error('another writer won'); error.status = 422; throw error; + }; + if (reason === 'read_unavailable') { + const getRef = f.github.rest.git.getRef; + f.github.rest.git.getRef = async input => { + if (input.ref === 'heads/v5') return getRef(input); + const error = new Error('unreadable'); error.status = 503; throw error; + }; + } + if (reason === 'ack_malformed') f.github.rest.git.createRef = async () => ({status: 201, data: {}}); + const result = await prepare(f.args()); + assert.ok(!f.calls.some(call => call.type === 'dispatch'))); + if (reason === 'foreign') { + assert.equal(result.failure, null); + assert.ok(result.prepared.every(item => item.dispatch === 'observer')); + } else { + assert.equal(result.failure.reason, 'dispatch_authority_unknown'); + assert.equal(result.prepared.length, 1); + } + } +}); + +test('real Git plumbing produces exact deterministic prefixes and retains an actual earlier prefix on conflict', async () => { + const directory = path.join(process.cwd(), `.integration-contract-${crypto.randomBytes(16).toString('hex')}`); + fs.mkdirSync(directory); + const environment = { + ...process.env, GIT_CONFIG_GLOBAL: '/dev/null', GIT_CONFIG_SYSTEM: '/dev/null', + GIT_AUTHOR_NAME: 'fixture', GIT_COMMITTER_NAME: 'fixture', + GIT_AUTHOR_EMAIL: 'fixture@example.invalid', GIT_COMMITTER_EMAIL: 'fixture@example.invalid', + GIT_AUTHOR_DATE: clock.toISOString(), GIT_COMMITTER_DATE: clock.toISOString(), + GIT_TERMINAL_PROMPT: '0', + }; + const git = (args, env = {}) => execFileSync('git', + ['-c', 'core.hooksPath=/dev/null', '--literal-pathspecs', ...args], + {cwd: directory, env: {...environment, ...env}, encoding: 'utf8', + timeout: 30000, maxBuffer: 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe']}).trim(); + const write = (file, value) => fs.writeFileSync(path.join(directory, file), `${value}\n`); + try { + git(['init', '--quiet']); + for (const folder of ['.github', 'src/scripts', 'app']) fs.mkdirSync(path.join(directory, folder), {recursive: true}); + write('.github/policy', 'trusted'); + write('src/scripts/guard.sh', 'trusted'); + write('app/a.txt', 'base A'); + write('app/b.txt', 'base B'); + git(['add', '.']); git(['commit', '--quiet', '-m', 'base']); + const base = git(['rev-parse', 'HEAD']); + write('app/a.txt', 'author A'); + git(['add', '.']); git(['commit', '--quiet', '-m', 'head A']); + const headA = git(['rev-parse', 'HEAD']); + git(['checkout', '--quiet', '--detach', base]); + write('app/b.txt', 'author B'); + git(['add', '.']); git(['commit', '--quiet', '-m', 'head B']); + const headB = git(['rev-parse', 'HEAD']); + git(['checkout', '--quiet', '--detach', base]); + write('app/a.txt', 'conflicting author C'); + git(['add', '.']); git(['commit', '--quiet', '-m', 'head C']); + const headC = git(['rev-parse', 'HEAD']); + git(['checkout', '--quiet', '--detach', base]); + const f = fixture(); + f.request.baseSHA = base; + f.request.members = [{pr: 10, head: headA}, {pr: 11, head: headB}]; + f.context.sha = base; + f.git = (args, env) => { + f.calls.push({type: 'git', args}); + if (args[0] === 'fetch' || args[0] === 'push') return ''; + return git(args, env); + }; + f.github.rest.repos.compareCommitsWithBasehead = async ({basehead}) => ({ + data: {merge_base_commit: {sha: git(['merge-base', ...basehead.split('...')])}}, + }); + const result = await prepare(f.args()); + assert.equal(result.failure, null); + const [first, second] = result.prepared; + const a = first.subject.prefixes.at(-1), b = second.subject.prefixes.at(-1); + assert.equal(git(['show', '-s', '--format=%P', a.commit]), `${base} ${headA}`); + assert.equal(git(['show', '-s', '--format=%P', b.commit]), `${a.commit} ${headB}`); + assert.equal(git(['rev-parse', `${b.commit}^{tree}`]), b.tree); + assert.equal(git(['show', `${b.commit}:app/a.txt`]), 'author A'); + assert.equal(git(['show', `${b.commit}:app/b.txt`]), 'author B'); + const replay = await prepare(f.args()); + assert.deepEqual(replay.prepared.map(item => item.subject.id), result.prepared.map(item => item.subject.id)); + assert.ok(replay.prepared.every(item => item.dispatch === 'observer')); + f.request.members[1].head = headC; + const conflict = await prepare(f.args()); + assert.equal(conflict.failure.reason, 'materialization_conflict'); + assert.equal(conflict.prepared.length, 1); + assert.equal(conflict.prepared[0].subject.id, first.subject.id); + } finally { + fs.rmSync(directory, {recursive: true}); + } +}); diff --git a/.github/workflows/integration-candidate-prepare.yml b/.github/workflows/integration-candidate-prepare.yml new file mode 100644 index 0000000000..25c700bf85 --- /dev/null +++ b/.github/workflows/integration-candidate-prepare.yml @@ -0,0 +1,99 @@ +name: Prepare integration candidates + +on: + repository_dispatch: + types: [hive.integration.prepare.v1] + workflow_dispatch: + inputs: + request: + description: Bounded versioned request from an explicitly enrolled integration controller. + required: true + type: string + +permissions: + contents: read + +concurrency: + group: integration-preparation-${{ github.run_id }} + cancel-in-progress: false + +jobs: + materialize: + name: trusted integration materializer + if: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + actions: write + outputs: + receipt: ${{ steps.prepare.outputs.receipt }} + steps: + - name: Checkout trusted control code, never candidate code + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + persist-credentials: false + - name: Materialize and publish bounded immutable prefixes + id: prepare + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + INTEGRATION_REQUEST: ${{ inputs.request || github.event.client_payload.request }} + GH_TOKEN: ${{ github.token }} + with: + retries: 0 + script: | + const fs = require('node:fs'); + const {canonical, gitRunner, prepare} = require('./.github/scripts/integration-materialize.cjs'); + const receipt = await prepare({ + github, context, + requestText: process.env.INTEGRATION_REQUEST, + policyText: fs.readFileSync('.github/integration-producer-policy.json', 'utf8'), + actorID: process.env.GITHUB_ACTOR_ID, + execution: {runID: process.env.GITHUB_RUN_ID, attempt: process.env.GITHUB_RUN_ATTEMPT}, + git: gitRunner(process.env.GH_TOKEN), + }); + core.setOutput('receipt', Buffer.from(canonical(receipt), 'utf8').toString('base64')); + if (receipt.failure) core.warning(`${receipt.failure.reason}: prefix preparation is partial/rejected; no promotion is authorized`); + core.info(`Prepared ${receipt.prepared.length} exact subjects; dispatch acknowledgement is not CI completion.`); + + receipt: + name: integration materializer receipt + needs: materialize + if: ${{ needs.materialize.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + id-token: write + steps: + - name: Attest trusted materialization without checkout or candidate execution + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + MATERIALIZATION_RECEIPT: ${{ needs.materialize.outputs.receipt }} + with: + script: | + const crypto = require('node:crypto'); + const fs = require('node:fs'); + const MAX_RECEIPT_BYTES = 256 * 1024; + const payload = Buffer.from(process.env.MATERIALIZATION_RECEIPT || '', 'base64'); + if (!payload.length || payload.length > MAX_RECEIPT_BYTES) throw new Error('Materialization receipt is missing/oversized'); + const receipt = JSON.parse(payload); + if (receipt.schema !== 1 || receipt.protocol !== 'integration-materializer-receipt-v1') { + throw new Error('Unknown materialization receipt version'); + } + const digest = crypto.createHash('sha256').update(payload).digest('hex'); + const token = await core.getIDToken(`hive-integration-materializer-v1/${digest}`); + core.setSecret(token); + fs.mkdirSync('.integration-producer', {recursive: true}); + fs.writeFileSync('.integration-producer/materialization.json', JSON.stringify({ + schema: 1, payload: payload.toString('base64'), token, + }), {flag: 'wx', mode: 0o600}); + core.info(`Recorded exact preparation provenance; ${receipt.failure ? 'partial/rejected' : 'completed'} is not promotion authorization.`); + - name: Publish immutable attempt-specific materialization receipt + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: integration-materializer-receipt-${{ github.run_attempt }} + path: .integration-producer/materialization.json + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/integration-candidate.yml b/.github/workflows/integration-candidate.yml new file mode 100644 index 0000000000..cd61280736 --- /dev/null +++ b/.github/workflows/integration-candidate.yml @@ -0,0 +1,189 @@ +name: Integration candidate +run-name: Integration ${{ fromJSON(inputs.manifest).id }} ${{ fromJSON(inputs.manifest).dispatchClaim.owner.nonce }} + +on: + workflow_dispatch: + inputs: + manifest: + description: Exact versioned subject emitted by an enrolled trusted materializer. + required: true + type: string + +permissions: + contents: read + +concurrency: + group: integration-candidate-${{ github.run_id }} + cancel-in-progress: false + +jobs: + subject: + name: integration subject + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Validate immutable ordered subject before assigning technical jobs + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SUBJECT_MANIFEST: ${{ inputs.manifest }} + with: + script: | + const crypto = require('node:crypto'); + const MAX_MANIFEST_BYTES = 32 * 1024; + const MAX_CANDIDATE_MEMBERS = 8; + const EXPECTED_PARENTS = 2; + const fail = message => { throw new Error(message); }; + const sha = value => typeof value === 'string' && /^[a-f0-9]{40}$/.test(value) && !/^0+$/.test(value); + const digest = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value) && !/^0+$/.test(value); + const raw = process.env.SUBJECT_MANIFEST || ''; + if (Buffer.byteLength(raw, 'utf8') > MAX_MANIFEST_BYTES) fail('Candidate manifest exceeds the bounded protocol'); + const subject = JSON.parse(raw); + const canonical = value => { + if (value === null || typeof value !== 'object') return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; + }; + if (subject.schema !== 1 || subject.protocol !== 'integration-prefix-v1' || + subject.engine !== 'hive' || !digest(subject.id) || + subject.repo !== `${context.repo.owner}/${context.repo.repo}` || + !sha(subject.baseSHA) || !digest(subject.policy) || + !sha(subject.workflowTree) || !Number.isSafeInteger(subject.generation) || + subject.generation < 1 || typeof subject.base !== 'string' || !subject.base.length) { + fail('Invalid integration subject identity; no candidate jobs admitted'); + } + const {id, generation, preparedBy, dispatchClaim, ...immutable} = subject; + if (crypto.createHash('sha256').update(canonical(immutable)).digest('hex') !== id) { + fail('Subject ID does not bind its exact immutable membership, commits, trees and policy'); + } + const claimKey = crypto.createHash('sha256').update(`integration-ci-initial-v1/${subject.id}`).digest('hex'); + if (!dispatchClaim || dispatchClaim.state !== 'claimed' || + dispatchClaim.key !== claimKey || dispatchClaim.ref !== `heads/hive-integration-intents/${claimKey}` || + !sha(dispatchClaim.commit) || !dispatchClaim.owner || + typeof dispatchClaim.owner.nonce !== 'string' || !/^[a-f0-9]{32}$/.test(dispatchClaim.owner.nonce) || + !/^[1-9][0-9]*$/.test(dispatchClaim.owner.runID) || !/^[1-9][0-9]*$/.test(dispatchClaim.owner.attempt)) { + fail('Candidate has no exact shared single-attempt dispatch owner'); + } + if (context.eventName !== 'workflow_dispatch' || + context.ref !== `refs/tags/hive-integration/${subject.id}` || + !Array.isArray(subject.members) || !Array.isArray(subject.prefixes) || + subject.members.length < 1 || subject.members.length > MAX_CANDIDATE_MEMBERS || + subject.members.length !== subject.prefixes.length) { + fail('Candidate dispatch/ref or bounded ordered membership mismatch'); + } + const numbers = new Set(); + const heads = new Set(); + const variables = {owner: context.repo.owner, name: context.repo.repo, + claimRef: `refs/${dispatchClaim.ref}`}; + const declarations = ['$owner:String!', '$name:String!', '$policy:String!', '$claimRef:String!']; + const objects = []; + for (let i = 0; i < subject.members.length; i++) { + const member = subject.members[i], prefix = subject.prefixes[i]; + if (!member || !Number.isSafeInteger(member.pr) || member.pr < 1 || + numbers.has(member.pr) || heads.has(member.head) || !sha(member.head) || + !prefix || !sha(prefix.commit) || !sha(prefix.tree)) { + fail('Invalid or duplicate ordered candidate member'); + } + numbers.add(member.pr); + heads.add(member.head); + declarations.push(`$prefix${i}:String!`); + variables[`prefix${i}`] = prefix.commit; + objects.push(`prefix${i}:object(expression:$prefix${i}){... on Commit{oid tree{oid} parents(first:${EXPECTED_PARENTS + 1}){nodes{oid}}}}`); + } + if (subject.prefixes.at(-1).commit !== context.sha) { + fail('Dispatch does not execute the actual tested candidate commit'); + } + variables.policy = `${context.sha}:.github`; + const query = `query(${declarations.join(',')}){repository(owner:$owner,name:$name){nameWithOwner claim:ref(qualifiedName:$claimRef){target{oid ... on Commit{message}}} policy:object(expression:$policy){... on Tree{oid}} ${objects.join(' ')}}}`; + const {repository} = await github.graphql(query, variables); + if (repository?.nameWithOwner !== subject.repo || repository.policy?.oid !== subject.workflowTree) { + fail('Repository or producer workflow closure changed'); + } + if (repository.claim?.target?.oid !== dispatchClaim.commit) { + fail('Shared dispatch authority changed or is unavailable; no TTL takeover'); + } + const records = (repository.claim.target.message || '').split('\n').filter(line => line.startsWith('{')); + if (records.length !== 1) fail('Shared dispatch authority record is malformed'); + const owner = JSON.parse(records[0]); + if (owner.schema !== 1 || owner.kind !== 'integration-ci-single-attempt-v1' || + owner.key !== claimKey || owner.subjectID !== subject.id || + owner.owner?.nonce !== dispatchClaim.owner.nonce || + owner.owner?.runID !== dispatchClaim.owner.runID || + owner.owner?.attempt !== dispatchClaim.owner.attempt) { + fail('Shared dispatch owner does not match the candidate execution'); + } + let parent = subject.baseSHA; + for (let i = 0; i < subject.members.length; i++) { + const actual = repository[`prefix${i}`], expected = subject.prefixes[i]; + const parents = actual?.parents?.nodes; + if (actual?.oid !== expected.commit || actual.tree?.oid !== expected.tree || + !Array.isArray(parents) || parents.length !== EXPECTED_PARENTS || + parents[0].oid !== parent || parents[1].oid !== subject.members[i].head) { + fail(`Candidate prefix ${i + 1} is not its exact ordered two-parent subject`); + } + parent = actual.oid; + } + core.info(`Validated immutable subject ${subject.id}; this is not promotion authorization.`); + core.info(`Manifest SHA256 ${crypto.createHash('sha256').update(raw).digest('hex')}`); + + ci: + needs: subject + permissions: + contents: read + uses: ./.github/workflows/integration-technical-v2-ci.yml + + tests: + needs: subject + permissions: + contents: read + uses: ./.github/workflows/integration-technical-v2-tests.yml + + security: + needs: subject + permissions: + contents: read + uses: ./.github/workflows/integration-technical-go-security-analysis.yml + + receipt: + name: integration producer receipt + needs: [subject, ci, tests, security] + if: ${{ always() && needs.subject.result == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + id-token: write + steps: + - name: Bind exact receipt bytes to this isolated finalizer job + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + env: + SUBJECT_MANIFEST: ${{ inputs.manifest }} + TECHNICAL_OUTCOMES: ${{ toJSON(needs) }} + with: + script: | + const crypto = require('node:crypto'); + const fs = require('node:fs'); + const subject = JSON.parse(process.env.SUBJECT_MANIFEST); + const needs = JSON.parse(process.env.TECHNICAL_OUTCOMES); + const payload = Buffer.from(JSON.stringify({ + schema: 1, + protocol: 'integration-producer-receipt-v1', + subject, + runID: process.env.GITHUB_RUN_ID, + runAttempt: process.env.GITHUB_RUN_ATTEMPT, + outcomes: {ci: needs.ci.result, tests: needs.tests.result, security: needs.security.result}, + }), 'utf8'); + const digest = crypto.createHash('sha256').update(payload).digest('hex'); + const token = await core.getIDToken(`hive-integration-producer-v1/${digest}`); + core.setSecret(token); + fs.mkdirSync('.integration-producer', {recursive: true}); + fs.writeFileSync('.integration-producer/receipt.json', JSON.stringify({ + schema: 1, payload: payload.toString('base64'), token, + }), {flag: 'wx', mode: 0o600}); + core.info(`Recorded producer identity for ${subject.id}; passing CI and live policy require independent verification.`); + - name: Publish immutable attempt-specific producer receipt + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: integration-producer-receipt-${{ github.run_attempt }} + path: .integration-producer/receipt.json + if-no-files-found: error + retention-days: 30 diff --git a/.github/workflows/integration-technical-go-security-analysis.yml b/.github/workflows/integration-technical-go-security-analysis.yml new file mode 100644 index 0000000000..0710fed719 --- /dev/null +++ b/.github/workflows/integration-technical-go-security-analysis.yml @@ -0,0 +1,211 @@ +# Generated by .github/scripts/generate-integration-ci.py; do not edit. +# Candidate-only technical CI. Ordinary and native CI remain in the source. +name: Integration candidate / Go Security Analysis +'on': + workflow_call: null +permissions: + contents: read +concurrency: + group: ${{ true && 'integration-security' || github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: false +env: + GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} +jobs: + required-context-scope: + name: required context scope + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + run-go-security: ${{ steps.scope.outputs.run-go-security }} + steps: + - name: Decide whether Go security checks need real scans + id: scope + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + if [ "${{ true }}" = true ] && + { [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [[ "$GITHUB_REF" != refs/tags/hive-integration/* ]]; }; then + echo "::error::Integration candidates require an explicitly dispatched immutable candidate tag" + exit 1 + fi + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "run-go-security=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + : > changed-files.txt + page=1 + api_root="${GITHUB_API_URL:-https://api.github.com}" + while :; do + curl -fsSL -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" "${api_root}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?per_page=100&page=${page}" -o changed-files-page.json + python3 - <<'PYFILES' >> changed-files.txt + import json + with open('changed-files-page.json', encoding='utf-8') as f: + for item in json.load(f): + print(item['filename']) + PYFILES + count=$(python3 - <<'PYCOUNT' + import json + with open('changed-files-page.json', encoding='utf-8') as f: + print(len(json.load(f))) + PYCOUNT + ) + [ "$count" -lt 100 ] && break + page=$((page + 1)) + done + rm -f changed-files-page.json + run=false + while IFS= read -r path; do + case "$path" in + CHANGELOG.md|changelog.d/*|src/*|NOTICE|.github/workflows/*|.github/scripts/*|.github/release-lines.yml) + run=true + ;; + esac + done < changed-files.txt + echo "run-go-security=${run}" >> "$GITHUB_OUTPUT" + if [ "$run" = true ]; then + echo "Go security checks will run real scans." + else + echo "No Go security paths changed; required scan contexts will be skipped/success." + fi + govulncheck: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: govulncheck (known vulnerabilities) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry + - name: Install govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@v1.1.4 + - name: Scan + working-directory: src + run: govulncheck ./... ./crux/... + gosec: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: gosec (insecure patterns) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry + - name: Install gosec + run: go install github.com/securego/gosec/v2/cmd/gosec@v2.29.0 + - name: Scan + working-directory: src + continue-on-error: true + run: gosec -exclude-generated ./... ./crux/... + golangci-lint: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: golangci-lint + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry + - uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a + with: + version: v2.13.1 + working-directory: src + args: ./... ./crux/... + skip-cache: true + action-pins: + name: action pins resolve + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Action-pins gate self-test + run: bash src/scripts/test-check-action-pins.sh + - name: Check every pinned action SHA exists + env: + GH_TOKEN: ${{ github.token }} + run: src/scripts/check-action-pins.sh .github/workflows + notice-drift: + name: NOTICE matches the module graph + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-go-security == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Exercise NOTICE drift diagnostics + run: src/scripts/test-check-notice-drift.sh + - name: Exercise generated-NOTICE validation + run: src/scripts/test-validate-generated-notice.sh + - name: Warm shared Go tool cache (serialize the cold-cache population) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry + - name: Regenerate NOTICE from the current module graph + run: src/scripts/generate-notice.sh /tmp/NOTICE.generated + - name: Publish the regenerated NOTICE for the autofix workflow + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: notice-generated + path: /tmp/NOTICE.generated + retention-days: 1 + if-no-files-found: error + - name: Record the PR number for the autofix workflow + if: github.event_name == 'pull_request' + run: echo "${{ github.event.pull_request.number }}" > /tmp/pr-number.txt + - name: Publish the PR number for the autofix workflow + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: notice-pr-number + path: /tmp/pr-number.txt + retention-days: 1 + if-no-files-found: error + - name: Fail if the committed NOTICE is stale + run: src/scripts/check-notice-drift.sh NOTICE /tmp/NOTICE.generated diff --git a/.github/workflows/integration-technical-v2-ci.yml b/.github/workflows/integration-technical-v2-ci.yml new file mode 100644 index 0000000000..c2354fbfe7 --- /dev/null +++ b/.github/workflows/integration-technical-v2-ci.yml @@ -0,0 +1,534 @@ +# Generated by .github/scripts/generate-integration-ci.py; do not edit. +# Candidate-only technical CI. Ordinary and native CI remain in the source. +name: Integration candidate / v2 CI +'on': + workflow_call: null +permissions: + contents: read +concurrency: + group: ${{ true && 'integration-ci' || github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: false +env: + ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' + GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} +jobs: + required-context-scope: + name: required context scope + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + run-ci: ${{ steps.scope.outputs.run-ci }} + steps: + - name: Decide whether build-and-test needs the real CI lane + id: scope + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + if [ "${{ true }}" = true ] && + { [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [[ "$GITHUB_REF" != refs/tags/hive-integration/* ]]; }; then + echo "::error::Integration candidates require an explicitly dispatched immutable candidate tag" + exit 1 + fi + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "run-ci=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + : > changed-files.txt + page=1 + api_root="${GITHUB_API_URL:-https://api.github.com}" + while :; do + curl -fsSL -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" "${api_root}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?per_page=100&page=${page}" -o changed-files-page.json + python3 - <<'PYFILES' >> changed-files.txt + import json + with open('changed-files-page.json', encoding='utf-8') as f: + for item in json.load(f): + print(item['filename']) + PYFILES + count=$(python3 - <<'PYCOUNT' + import json + with open('changed-files-page.json', encoding='utf-8') as f: + print(len(json.load(f))) + PYCOUNT + ) + [ "$count" -lt 100 ] && break + page=$((page + 1)) + done + rm -f changed-files-page.json + run=false + while IFS= read -r path; do + case "$path" in + CHANGELOG.md|changelog.d/*|src/*|bin/*|config/*|README.md|docs/*|.github/scripts/*|.github/workflows/*) + run=true + ;; + esac + done < changed-files.txt + echo "run-ci=${run}" >> "$GITHUB_OUTPUT" + if [ "$run" = true ]; then + echo "v2 build-and-test will run." + else + echo "No v2 CI paths changed; build-and-test will be reported as skipped/success." + fi + build-and-test: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 50 + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Ensure PyYAML on self-hosted runners + working-directory: . + run: | + python3 -c "import yaml" 2>/dev/null && exit 0 + # A stale or unreachable third-party apt index (e.g. a PPA 503) must not block the install. + sudo apt-get update -qq || true + sudo apt-get install -y -qq python3-yaml && exit 0 + # Noble (Python 3.12) refuses --user installs under PEP 668 without --break-system-packages. + python3 -m pip install --user -q --break-system-packages pyyaml \ + || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } + - name: Check readonly candidate CI generation + working-directory: . + run: python3 .github/scripts/generate-integration-ci.py --check + - name: Warm shared Go tool cache (serialize the cold-cache population) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry + - name: Restore warm Go build cache + if: ${{ !contains('["ubuntu-latest"]', 'hive-runners') }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + - name: Build + run: go build ./cmd/hive/ + - name: Build crux (root host and standalone module) + run: | + go build -o /dev/null ./cmd/crux + (cd crux && GOWORK=off go build -o /dev/null ./cmd/crux) + - name: Crux module smoke (crux serve over a unix socket) + working-directory: src/crux + run: GOWORK=off bash scripts/smoke.sh + - name: CI install tool apt cache tests + working-directory: . + run: bash .github/scripts/test-ci-install-tool-apt-cache.sh + - name: Go cache guard tests + working-directory: . + run: bash .github/scripts/test-go-cache-guard.sh + - name: Go tool cache guard tests + working-directory: . + run: bash .github/scripts/test-go-toolcache-guard.sh + - name: Runner cache prune tests + working-directory: . + run: python3 .github/scripts/test-runner-cache-prune.py + - name: Patch coverage script tests + working-directory: . + run: bash .github/scripts/test-patch-coverage.sh + - name: CI infra classifier tests + working-directory: . + run: python3 .github/scripts/test-ci-infra.py + - name: Vet + run: go vet ./... ./crux/... + - name: Price table staleness guard + run: go run ./internal/tools/pricetable --check-age 60 + - name: Entrypoint runtime-config migration tests + run: bash deploy/test_entrypoint_runtime_config.sh + - name: Entrypoint NET_ADMIN ambient-cap tests (#3760) + run: bash deploy/test_entrypoint_netadmin_ambient.sh + - name: Entrypoint dangling key_file tests (#4368) + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash deploy/test_entrypoint_dangling_keyfile.sh + - name: Entrypoint standalone terminal-key fallback tests (#6489) + run: bash deploy/test_entrypoint_terminal_key_fallback.sh + - name: Entrypoint /data ownership invariant (#5369) + run: bash deploy/test_entrypoint_data_ownership.sh + - name: Entrypoint UID-isolation migration (#5525) + run: bash deploy/test_entrypoint_uid_isolation.sh + - name: Entrypoint stable agent UID allocation (#9592) + run: bash deploy/test_entrypoint_uid_stable.sh + - name: Entrypoint replica agent names (#13166) + run: bash deploy/test_entrypoint_replica_names.sh + - name: Entrypoint token-access audit log not agent-writable (#6287) + run: bash deploy/test_entrypoint_token_access_audit.sh + - name: arm64 lane probes PR code, not the published image (#5370) + run: bash deploy/test_arm64_lane_probes_pr_code.sh + - name: PR Verifier reports on the PR, not on registry reachability (#6085) + run: bash deploy/test_pr_verifier_advisory.sh + - name: Supply-chain pin guard + run: bash deploy/test_supply_chain_pins.sh + - name: Docker release-tag publication policy (#4804) + run: bash scripts/test-publish-image-tags.sh + - name: Stable soak promotion policy (#5974) + run: bash scripts/test-promote-stable.sh + - name: Issue co-author trailer helper (#6588) + run: bash scripts/test-issue-coauthor.sh + - name: Issue confirm-fixed command (#9746) + run: bash scripts/test-issue-confirm-fixed.sh + - name: Release push retry/deferral state machine (#5142) + run: bash scripts/test-release-push-retry.sh + - name: release_sha ancestry guard (#6419) + run: bash scripts/test-check-release-sha-ancestry.sh + - name: Changelog fragment compiler properties (#5675) + run: bash scripts/test-compile-changelog.sh + - name: Release notes extraction fallback (#5934) + run: bash scripts/test-extract-release-notes.sh + - name: Standalone image-reference source of truth (#4206) + run: bash deploy/test_standalone_image_refs.sh + - name: Docker-socket containment guard (#3865) + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash deploy/test_watchtower_socket_contract.sh + - name: Standalone service deployment contract (#4204) + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash deploy/test_standalone_service_contract.sh + - name: Quadlet published-port boundary (#4375) + run: bash deploy/test_quadlet_port_boundary.sh + - name: Quadlet config-coupling contract (#4367) + run: bash deploy/test_quadlet_config_contract.sh + - name: Documented Compose quick start contract (#4477) + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash deploy/test_compose_quickstart_contract.sh + - name: Docker/Podman contract parity (#4404) + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash deploy/test_standalone_runtime_parity.sh + - name: gh-wrapper gate tests + working-directory: . + run: bash bin/test_gh_wrapper_gates.sh + - name: go-wrapper gate tests + working-directory: . + run: bash bin/go-wrapper.test.sh + - name: enumerate-actionable contract tests + working-directory: . + run: bash bin/test_enumerate_actionable.sh + - name: issue-classifier contract tests (tiers, lanes, clustering) + working-directory: . + run: bash bin/test_issue_classifier.sh + - name: hive-gh compact projection contract tests + working-directory: . + run: bash bin/test_hive_gh.sh + - name: kick-governor contract tests + working-directory: . + run: bash bin/test_kick_governor.sh + - name: merge-gate contract tests + working-directory: . + run: bash bin/test_merge_gate.sh + - name: gh-rate-check contract tests + working-directory: . + run: bash bin/test_gh_rate_check.sh + - name: dibs cutover preflight contract (#5925) + working-directory: . + run: bash bin/test_dibs_cutover_preflight.sh + - name: dibs cutover verification contract (#5925) + working-directory: . + run: bash bin/test_dibs_cutover_verify.sh + - name: GA4 anomaly detector drop-arm contract (#6430) + working-directory: . + run: bash bin/test_ga4_anomaly_detector.sh + - name: git-credential-hive helper tests (#4289) + working-directory: . + run: bash bin/test_git_credential_hive.sh + - name: gh-auth native-install guard (#3842) + working-directory: . + run: bash bin/test_gh_auth_native_no_cat.sh + - name: agent shell token scrub (#4045) + working-directory: . + run: bash bin/test_agent_env_scrub.sh + - name: Standalone runtime selection contract (#4205) + working-directory: . + run: bash bin/test_hive_standalone_runtime.sh + - name: Podman preflight contract (#4207) + working-directory: . + run: bash bin/test_hive_podman_preflight.sh + - name: Podman ownership and cleanup contract (#4210) + working-directory: . + run: bash bin/test_hive_podman_cleanup.sh + - name: Podman teardown through the ownership contract (#4326) + working-directory: . + run: bash bin/test_hive_podman_teardown.sh + - name: Podman SELinux, mount, secret, and port preflight (#4209) + working-directory: . + run: bash bin/test_hive_podman_preflight_host.sh + - name: Podman subordinate-ID, graphroot, and network preflight (#4208) + working-directory: . + run: bash bin/test_hive_podman_preflight_ids.sh + - name: Podman Quadlet lifecycle probe (#4377) + working-directory: . + run: bash bin/test_hive_podman_lifecycle_probe.sh + - name: SELinux label reader is resolved, never assumed (#4490) + working-directory: . + run: bash src/deploy/test_selinux_label_reader.sh + - name: Podman one-command installer (#4470) + working-directory: . + run: bash bin/test_hive_podman_setup.sh + - name: hivectl contributor bootstrap (#8200) + working-directory: . + run: bash bin/test_hivectl_bootstrap.sh + - name: Podman Quadlet update and rollback (#4378) + working-directory: . + run: bash bin/test_hive_podman_update.sh + - name: gh-wrapper author-gate tests (#3072, + working-directory: . + run: bash bin/gh-wrapper.test.sh + - name: repo-toolchain manifest tests (#7925) + working-directory: . + run: bash bin/repo-toolchain.test.sh + - name: hive-open-issue helper tests + working-directory: . + run: bash bin/test_hive_open_issue.sh + - name: hive-open-pr helper tests + working-directory: . + run: bash bin/test_hive_open_pr.sh + - name: hive-review helper tests + working-directory: . + run: bash bin/test_hive_review.sh + - name: Shared CI baseline classifier tests + working-directory: . + run: bash bin/test_hive_baseline_check.sh + - name: Every bin/ test suite is wired into something (#4363) + working-directory: . + run: bash bin/test_bin_suites_wired.sh + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 + with: + node-version: '22' + - name: Guard shared node tool cache (repair a half-written entry) + if: ${{ always() }} + run: | + set -u + command -v npm >/dev/null 2>&1 && exit 0 + for marker in "${RUNNER_TOOL_CACHE:-/nonexistent}"/node/*/x64.complete; do + [ -e "$marker" ] || continue + dir="${marker%.complete}" + [ -x "$dir/bin/npm" ] && continue + echo "::warning::node tool cache $dir has a .complete marker but no executable bin/npm; removing the marker so the next job on $RUNNER_NAME rebuilds it" + rm -f "$marker" + done + echo "::error::npm is not on PATH after actions/setup-node on $RUNNER_NAME (half-written shared tool cache, see #10518); marker removed, re-run this job" + exit 1 + - name: Proxy Tests + run: cd proxy && npm ci && npm test + - name: Dashboard helper tests + working-directory: . + run: node --test src/pkg/dashboard/*.test.js + - name: Commons web page tests + working-directory: . + run: node --test src/pkg/hivectl/commands/*.test.js + - name: Integration candidate materializer contracts + working-directory: . + run: node --test .github/scripts/integration-materialize.test.cjs + - name: Contributor Relay Syntax + Tests + working-directory: . + run: | + bash bin/contributor-agent.test.sh + node --check bin/contributor-relay.js + node --check bin/lib/pane-classifier.js + bash -n bin/contributor-relay.sh + node bin/contributor-relay.test.js + node --test bin/mlx-backend.test.js + - name: vibe-kanban mirror tests + working-directory: . + run: | + node --check bin/vibe-kanban-mirror.js + node bin/vibe-kanban-mirror.test.js + - name: Backend smoke failure-issue attribution + working-directory: . + run: bash bin/test_backend_smoke_attribution.sh + - name: Backend smoke pins this repository's contributor image + working-directory: . + run: bash bin/test_backend_smoke_image_owner.sh + - name: Backend smoke (keyless subset) + working-directory: . + run: bash bin/test_backend_smoke.sh + - name: Install just + run: | + set -euo pipefail + JUST_VERSION=1.58.0 + JUST_SHA256=4a5cc2f53e6f0f8c59092a6cc38291eb729d46a7dd95d3ae582008881b84931d + curl -fsSL -o /tmp/just.tgz "https://github.com/casey/just/releases/download/${JUST_VERSION}/just-${JUST_VERSION}-x86_64-unknown-linux-musl.tar.gz" + echo "${JUST_SHA256} /tmp/just.tgz" | sha256sum -c - + tar -xzf /tmp/just.tgz -C /tmp just + sudo install -m 0755 /tmp/just /usr/local/bin/just + just --version + - name: Contributor K8s Workload Generation + working-directory: . + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash src/deploy/test_contribute_k8s_workload.sh + - name: Terminal scrollback legibility (#4399) + working-directory: . + run: bash src/deploy/test_terminal_scrollback_status.sh + - name: ttyd keeps --url-arg when credentialed (#4593) + working-directory: . + run: bash src/deploy/test_ttyd_url_arg.sh + - name: ttyd finds the default tmux socket under an arbitrary container UID + working-directory: . + run: bash src/deploy/test_ttyd_default_socket_uid.sh + - name: hive-discord.service does not execute planted code (#5435) + working-directory: . + run: bash src/deploy/test_hive_discord_unit_contract.sh + - name: hive-snapshot.service does not execute planted code (#5483) + working-directory: . + run: bash src/deploy/test_hive_snapshot_unit_contract.sh + - name: hive-deploy.service does not root-install planted code (#5936) + working-directory: . + run: bash src/deploy/test_hive_deploy_unit_contract.sh + - name: Changelog reminder never blocks a merge (#4440) + working-directory: . + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash src/deploy/test_changelog_reminder.sh + - name: Contributor identity relocation (#4408) + working-directory: . + run: bash src/deploy/test_contribute_move.sh + - name: In-container attach hints name the real runtime (#5145) + working-directory: . + run: bash src/deploy/test_attach_hint_runtime.sh + - name: Isolated contributor Compose contract (#9188) + working-directory: . + run: | + # hive-runners lack python3-venv (ensurepip); the job's first step + # already guarantees PyYAML for the system python3, so fall back to it. + if python3 -m venv "$RUNNER_TEMP/isolated-contributor-tests" 2>/dev/null; then + "$RUNNER_TEMP/isolated-contributor-tests/bin/pip" install PyYAML==6.0.2 + PY="$RUNNER_TEMP/isolated-contributor-tests/bin/python" + else + python3 -c "import yaml" + PY=python3 + fi + "$PY" src/deploy/test_isolated_contributor_compose.py + - name: Contributor containers enforce resource limits (#6485) + working-directory: . + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash src/deploy/test_contributor_container_resources.sh + - name: Contributor containers stage only the Claude credential and settings (#7836) + working-directory: . + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash src/deploy/test_contributor_claude_staging.sh + - name: A skip is fatal where the caller guarantees the precondition (#5388) + working-directory: . + run: bash src/deploy/test_skip_discipline.sh + - name: Entrypoint system gitconfig contract + working-directory: . + env: + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: bash src/deploy/test_entrypoint_system_gitconfig.sh + - name: Entrypoint Copilot token symlink contract (#5529) + working-directory: . + run: bash src/deploy/test_entrypoint_symlinks.sh + - name: Entrypoint permission-guard survivability (#5730) + working-directory: . + run: bash src/deploy/test_entrypoint_perm_guard.sh + - name: hivectl extraction route contract (#5646) + working-directory: . + run: bash src/deploy/test_hivectl_extraction_contract.sh + - name: Entrypoint iptables append failure handling (#6003) + working-directory: . + run: bash src/deploy/test_entrypoint_iptables_gate.sh + - name: Entrypoint egress ruleset result checks (#6003) + working-directory: . + run: bash src/deploy/test_entrypoint_egress_ruleset.sh + - name: Entrypoint netfilter extension preflight (#6003) + working-directory: . + run: bash src/deploy/test_entrypoint_xt_module_preflight.sh + - name: Every src/deploy/ test suite is wired into a workflow (#5529) + working-directory: . + run: bash src/deploy/test_deploy_suites_wired.sh + - name: Runner labels are fork-portable (#6083) + working-directory: . + run: bash src/deploy/test_runner_labels_portable.sh + docker: + name: docker (${{ matrix.name }}) + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - name: hive + file: src/Dockerfile + smoke: 'true' + - name: hub + file: src/Dockerfile.hub + smoke: 'false' + - name: contributor + file: src/Dockerfile.contributor + smoke: 'false' + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + fetch-depth: ${{ matrix.name == 'hub' && 0 || 1 }} + - name: Skip non-hive images on push + if: '!true && github.event_name != ''pull_request'' && matrix.name != ''hive''' + run: echo "skip=true" >> "$GITHUB_ENV" + - name: Set up Docker Buildx + if: env.skip != 'true' + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 + with: + driver-opts: network=host + - name: Resolve base image registry (mirror.gcr.io with docker.io fallback) + if: env.skip != 'true' + uses: ./.github/actions/docker-base-registry + - name: Build Docker image + if: env.skip != 'true' + run: docker buildx build --load --network host --build-arg BASE_REGISTRY="$BASE_REGISTRY" -f "${{ matrix.file }}" -t "hive-ci-${{ matrix.name }}:test" . + - name: Smoke test + if: env.skip != 'true' && matrix.smoke == 'true' + shell: bash + run: "set -euo pipefail\nname=\"hive-v2-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}\"\ncleanup() {\n docker rm -f -v \"$name\" >/dev/null 2>&1 || true\n}\ntrap cleanup EXIT\n# #3760: the binary carries NO file capability anymore, so it must exec\n# under docker's DEFAULT cap set (no --cap-add) \u2014 a self-hosted spoke\n# has exactly that bounding set. NET_ADMIN, when present, is raised by\n# the entrypoint as an ambient cap; the raw `--version` here needs none.\n# The old `--help || true` form could not fail and asserted nothing.\ntimeout --signal=TERM --kill-after=10s 60s \\\n docker run --rm --name \"$name\" --network none hive-ci-hive:test --version\n" + overlayfs-exec-guard: + needs: required-context-scope + if: ${{ always() && (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-ci == 'true') }} + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 + - name: Resolve runner ref name + id: ref_name + run: | + set -euo pipefail + : "${GITHUB_REF_NAME:?GITHUB_REF_NAME is required}" + echo "value=$GITHUB_REF_NAME" >> "$GITHUB_OUTPUT" + - name: Resolve base image registry (mirror.gcr.io with docker.io fallback) + uses: ./.github/actions/docker-base-registry + - name: Build image via buildx (OCI archive, no attestations) + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc + with: + context: . + file: src/Dockerfile + platforms: linux/amd64 + provenance: false + sbom: false + outputs: type=oci,oci-mediatypes=true,dest=/tmp/hive-oci.tar,tar=true + cache-from: type=gha,scope=tmux-linux-amd64 + build-args: | + BASE_REGISTRY=${{ env.BASE_REGISTRY }} + GIT_HASH=${{ github.sha }} + GIT_BRANCH=${{ steps.ref_name.outputs.value }} + - name: Run under containerd/overlayfs and assert in-place exec + shell: bash + run: "set -euo pipefail\n# containerd ships on GitHub ubuntu runners; ensure it is up and using\n# the overlayfs snapshotter (its default) \u2014 that is what makes the\n# metacopy redirect observable.\nsudo systemctl start containerd 2>/dev/null || sudo containerd >/tmp/containerd.log 2>&1 &\nfor i in $(seq 1 30); do\n if sudo ctr version >/dev/null 2>&1; then break; fi\n sleep 1\ndone\nsudo ctr version\n# Confirm the overlayfs snapshotter is present (the failing runtime).\nsudo ctr plugins ls | grep -i overlayfs || {\n echo \"overlayfs snapshotter not available on this runner\" >&2; exit 1; }\n\nIMG=\"hive-3760-guard:test\"\n# Import the OCI archive BuildKit produced into containerd. Resolve\n# the linux/amd64 platform (the archive is single-arch, but pin it so\n# the run below has a concrete manifest to unpack).\nsudo ctr images import --platform linux/amd64 --index-name \"$IMG\" /tmp/hive-oci.tar \\\n || sudo ctr images import --index-name \"$IMG\" /tmp/hive-oci.tar\nsudo ctr images ls | grep \"$IMG\"\n\n# The assertion: exec /usr/local/bin/hive AT ITS IMAGE PATH, no copy,\n# under containerd's overlayfs snapshotter. --net-host keeps the run\n# simple; we deliberately do NOT add NET_ADMIN \u2014 a self-hosted spoke\n# has the docker/containerd default bounding set. Override the\n# entrypoint so we test the raw binary exec (the #3760 symptom is a\n# bare EPERM at execve of the binary, not an entrypoint failure).\nset +e\nout=$(sudo timeout 90s ctr run --rm --snapshotter overlayfs \\\n --net-host \\\n \"$IMG\" hive-3760-inplace \\\n /usr/local/bin/hive --version 2>&1)\nrc=$?\nset -e\necho \"----- hive --version (in-place, containerd/overlayfs) -----\"\necho \"$out\"\necho \"exit=$rc\"\nif [ $rc -ne 0 ]; then\n echo \"::error::#3760 regression: /usr/local/bin/hive failed to exec IN PLACE under containerd/overlayfs (rc=$rc). This is the BuildKit metacopy-redirect crash-loop \u2014 a plain build-side attestation/OCI-index change or a removed in-stage binary rewrite has reintroduced it.\" >&2\n exit 1\nfi\ncase \"$out\" in\n *\"Operation not permitted\"*|*EPERM*)\n echo \"::error::#3760 regression: hive execve returned Operation not permitted at its image path under containerd/overlayfs.\" >&2\n exit 1 ;;\nesac\n\n# STALE-BINARY GUARD. The exec check above passes even for an OLD hive\n# binary \u2014 a stale binary works fine, it just reports the WRONG commit.\n# A branch-persistent GHA cache once let BuildKit serve a cached\n# final-stage `COPY --from=builder /hive` layer, so the manifest digest\n# advanced but /usr/local/bin/hive was an old build (reported commit\n# 4d9c11c while HEAD had the #3760 fix). Assert the built binary's\n# embedded commit matches THIS build's commit so a stale-layer cache hit\n# is caught directly, not just an exec failure. `hive --version` prints\n# `hive 3.0.0 (commit , branch )`; main.gitShort is the\n# 7-char short SHA stamped from GIT_HASH=${{ github.sha }} at build.\nEXPECTED_SHORT=\"${GITHUB_SHA::7}\"\nREPORTED_COMMIT=$(printf '%s\\n' \"$out\" | sed -n 's/.*commit \\([0-9a-f][0-9a-f]*\\).*/\\1/p' | head -n1)\necho \"expected commit=$EXPECTED_SHORT reported commit=$REPORTED_COMMIT\"\nif [ -z \"$REPORTED_COMMIT\" ]; then\n echo \"::error::stale-binary guard: could not parse commit from 'hive --version' output \u2014 version stamping is broken or the output format changed.\" >&2\n exit 1\nfi\nif [ \"$REPORTED_COMMIT\" != \"$EXPECTED_SHORT\" ]; then\n echo \"::error::stale-binary guard: shipped hive reports commit '$REPORTED_COMMIT' but this build is '$EXPECTED_SHORT'. The binary is STALE \u2014 a cached final-stage COPY --from=builder layer shipped an old /usr/local/bin/hive (the bug that let #3760's fix miss the published image). Check no-cache-filters covers the runtime stage in docker.yml.\" >&2\n exit 1\nfi\necho \"OK: hive execs in-place under containerd/overlayfs (no copy, no NET_ADMIN) AND reports the current commit ($REPORTED_COMMIT).\"\n" diff --git a/.github/workflows/integration-technical-v2-tests.yml b/.github/workflows/integration-technical-v2-tests.yml new file mode 100644 index 0000000000..5c5ef59cc6 --- /dev/null +++ b/.github/workflows/integration-technical-v2-tests.yml @@ -0,0 +1,1002 @@ +# Generated by .github/scripts/generate-integration-ci.py; do not edit. +# Candidate-only technical CI. Ordinary and native CI remain in the source. +name: Integration candidate / v2 Tests +'on': + workflow_call: null +permissions: + contents: read +env: + CGO_ENABLED: '1' + KUBERNETES_SERVICE_HOST: '' + KUBERNETES_SERVICE_PORT: '' + ACTIONS_CACHE_SEGMENT_DOWNLOAD_TIMEOUT_MINS: '1' + GO_CACHE_GUARD_BUILD_CACHE: ${{ 'job' || 'job' }} +concurrency: + group: ${{ true && 'integration-tests' || github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} + cancel-in-progress: false +jobs: + required-context-scope: + name: required context scope + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + run-tests: ${{ steps.scope.outputs.run-tests }} + steps: + - name: Decide whether the test and coverage gates need the real suite + id: scope + env: + GITHUB_TOKEN: ${{ github.token }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -euo pipefail + if [ "${{ true }}" = true ] && + { [ "$GITHUB_EVENT_NAME" != workflow_dispatch ] || [[ "$GITHUB_REF" != refs/tags/hive-integration/* ]]; }; then + echo "::error::Integration candidates require an explicitly dispatched immutable candidate tag" + exit 1 + fi + if [ "${{ github.event_name }}" != "pull_request" ]; then + echo "run-tests=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + : > changed-files.txt + page=1 + api_root="${GITHUB_API_URL:-https://api.github.com}" + while :; do + curl -fsSL -H "Authorization: Bearer ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" "${api_root}/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}/files?per_page=100&page=${page}" -o changed-files-page.json + python3 - <<'PYFILES' >> changed-files.txt + import json + with open('changed-files-page.json', encoding='utf-8') as f: + for item in json.load(f): + print(item['filename']) + PYFILES + count=$(python3 - <<'PYCOUNT' + import json + with open('changed-files-page.json', encoding='utf-8') as f: + print(len(json.load(f))) + PYCOUNT + ) + [ "$count" -lt 100 ] && break + page=$((page + 1)) + done + rm -f changed-files-page.json + run=false + while IFS= read -r path; do + case "$path" in + src/*|bin/*|config/*|Justfile|dashboard/openapi.json|CHANGELOG.md|changelog.d/*|.github/workflows/*|.github/scripts/*) + run=true + ;; + esac + done < changed-files.txt + echo "run-tests=${run}" >> "$GITHUB_OUTPUT" + if [ "$run" = true ]; then + echo "v2 test/coverage gates will run real shards." + else + echo "No v2 test paths changed; required test/coverage contexts will report success without shards." + fi + test-hub: + needs: + - required-context-scope + - list-packages + if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: test (hub ${{ matrix.idx }}/4) + strategy: + fail-fast: false + matrix: + idx: + - 1 + - 2 + - 3 + - 4 + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - name: Affected-package selection + id: select + working-directory: . + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + RUN_LANE: ${{ needs.list-packages.outputs.run-hub }} + run: | + if [ "$SELECTION" = partial ] && [ "$RUN_LANE" = false ]; then + echo "Affected-package selection: pkg/hub is not affected by this diff; reporting success without running tests. Push and scheduled runs test it in full." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: steps.select.outputs.run == 'true' + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + if: steps.select.outputs.run == 'true' + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry + - name: Install gotestsum + if: steps.select.outputs.run == 'true' + run: | + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + go install gotest.tools/gotestsum@v1.13.0 + "$(go env GOPATH)/bin/gotestsum" --version + - name: Restore warm Go build cache + if: ${{ steps.select.outputs.run == 'true' && (!contains('["ubuntu-latest"]', 'hive-runners')) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 + if: steps.select.outputs.run == 'true' + with: + node-version: '22' + - name: Guard shared node tool cache (repair a half-written entry) + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: | + set -u + command -v npm >/dev/null 2>&1 && exit 0 + for marker in "${RUNNER_TOOL_CACHE:-/nonexistent}"/node/*/x64.complete; do + [ -e "$marker" ] || continue + dir="${marker%.complete}" + [ -x "$dir/bin/npm" ] && continue + echo "::warning::node tool cache $dir has a .complete marker but no executable bin/npm; removing the marker so the next job on $RUNNER_NAME rebuilds it" + rm -f "$marker" + done + echo "::error::npm is not on PATH after actions/setup-node on $RUNNER_NAME (half-written shared tool cache, see #10518); marker removed, re-run this job" + exit 1 + - name: Restore cgo toolchain package cache + if: steps.select.outputs.run == 'true' + id: restore-cgo-apt-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + restore-keys: | + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev- + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}- + hive-cgo-race-apt- + - name: Prepare cgo toolchain for race tests + if: steps.select.outputs.run == 'true' + id: prepare-cgo-toolchain + env: + HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !true && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require gcc --for '-race tests' \ + --apt 'gcc libc6-dev' --apk 'gcc musl-dev' --verify 'gcc --version' + - name: Save cgo toolchain package cache + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.restore-cgo-apt-cache.outputs.cache-hit != 'true' && steps.prepare-cgo-toolchain.outputs.apt-cache-populated == 'true') }} + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + - name: Test slice of pkg/hub + if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} + env: + IDX: ${{ matrix.idx }} + TOTAL: ${{ strategy.job-total }} + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: "set -o pipefail\nROOT=\"$PWD\"\nexport LC_ALL=C # Regex length guards below must count bytes.\nexport TEST_PACKAGE=github.com/hivecommons/hive/pkg/hub\nexport TEST_DIR=\"$ROOT/pkg/hub\"\nexport TEST_BINARY=\"$ROOT/hub.test\"\ngo test -c -race -cover -o \"$TEST_BINARY\" ./pkg/hub\n(cd \"$TEST_DIR\" && \"$TEST_BINARY\" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt\nif [ ! -s all-tests.txt ]; then\n echo \"::error::go test -list produced no test functions for pkg/hub \u2014 refusing to run an empty slice (fail closed)\"\n exit 1\nfi\nawk -v n=\"$TOTAL\" -v i=\"$IDX\" 'NR % n == i - 1' all-tests.txt > slice.txt\necho \"slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions\"\nif [ ! -s slice.txt ]; then\n echo \"::error::empty hub slice \u2014 failing closed\"\n exit 1\nfi\nexport CHUNK_DIR TEST_SHUFFLE=off\nCHUNK_DIR=$(mktemp -d \"$ROOT/test-chunks.XXXXXX\")\nsplit -l 1000 slice.txt \"$CHUNK_DIR/chunk-\"\nif [ \"$GITHUB_EVENT_NAME\" != \"pull_request\" ]; then TEST_SHUFFLE=on; fi\n# raw-command preserves one JUnit/flake report across all chunks.\n# gotestsum appends -test.run and package arguments on reruns.\ncat > \"$CHUNK_DIR/run.sh\" <<'RUNNER'\n#!/usr/bin/env bash\nset -uo pipefail\ncd \"$TEST_DIR\" || exit 1\nrun_chunk() {\n local RUN=\"$1\" PROFILE=\"$2\"\n # MAX_ARG_STRLEN is 131072 bytes; never approach it, even if a\n # future test has an exceptionally long name. Guard before exec.\n if [ \"${#RUN}\" -gt 120000 ]; then\n echo \"::error::chunk regex exceeds 120000 bytes; reduce chunk size\" >&2\n return 1\n fi\n go tool test2json -t -p \"$TEST_PACKAGE\" \"$TEST_BINARY\" \\\n -test.short -test.count=1 -test.v=test2json -test.timeout=10m \\\n -test.shuffle=\"$TEST_SHUFFLE\" \"$RUN\" -test.coverprofile=\"$PROFILE\"\n}\nif [ \"$#\" -gt 0 ]; then\n # Reruns must not overwrite the first-pass coverage.\n run_chunk \"$1\" \"$CHUNK_DIR/rerun.out\"\n exit $?\nfi\nstatus=0\nfor chunk in \"$CHUNK_DIR\"/chunk-*; do\n RUN=\"-test.run=^($(paste -sd'|' \"$chunk\"))$\"\n run_chunk \"$RUN\" \"$chunk.out\" || status=1\ndone\nexit \"$status\"\nRUNNER\nstatus=0\nif gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \\\n --rerun-fails-report=flaky-tests-hub-\"$IDX\".txt \\\n --junitfile junit-hub-\"$IDX\".xml --raw-command -- bash \"$CHUNK_DIR/run.sh\" \\\n 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-hub-\"$IDX\".log; then\n :\nelse\n status=1\nfi\n# The test binary runs under test2json, not the `go test` driver, so\n# nothing prints the `ok ` / `FAIL ` summary line the\n# coverage scorer classifies packages by (#14135). Append it.\nif [ \"$status\" -eq 0 ]; then\n printf 'ok \\t%s\\t(slice %s/%s)\\n' \"$TEST_PACKAGE\" \"$IDX\" \"$TOTAL\" >> test-hub-\"$IDX\".log\nelse\n printf 'FAIL\\t%s\\t(slice %s/%s)\\n' \"$TEST_PACKAGE\" \"$IDX\" \"$TOTAL\" >> test-hub-\"$IDX\".log\nfi\n# Sum first-pass counters for identical blocks; never count reruns.\n# Do this even after test failure so the coverage gate has evidence.\nawk 'FNR == 1 { next } { key=$1 \" \" $2; hits[key]+=$3 }\n END { print \"mode: atomic\"; for (key in hits) print key, hits[key] }' \\\n \"$CHUNK_DIR\"/chunk-*.out > coverage-hub-\"$IDX\".out\nexit \"$status\"\n" + - name: Surface flaky reruns + id: flaky-reruns + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-flake-summary.sh" "flaky-tests-hub-${{ matrix.idx }}.txt" "hub ${{ matrix.idx }}/4" + - name: Upload flaky rerun report + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.flaky-reruns.outputs.has-flakes == 'true') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: flaky-tests-hub-${{ matrix.idx }} + path: src/flaky-tests-hub-${{ matrix.idx }}.txt + if-no-files-found: error + retention-days: 14 + - name: Slowest tests in this slice + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/slowest-tests.sh" "hub ${{ matrix.idx }}/4" test-hub-${{ matrix.idx }}.log + - name: Upload shard coverage artifacts + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: test-shard-hub-${{ matrix.idx }} + path: | + src/coverage-hub-${{ matrix.idx }}.out + src/test-hub-${{ matrix.idx }}.log + if-no-files-found: warn + retention-days: 1 + test-sliced: + needs: + - required-context-scope + - list-packages + if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: test (${{ matrix.lane }} ${{ matrix.idx }}/${{ matrix.total }}) + strategy: + fail-fast: false + matrix: + include: + - lane: dashboard + pkg: pkg/dashboard + idx: 1 + total: 4 + - lane: dashboard + pkg: pkg/dashboard + idx: 2 + total: 4 + - lane: dashboard + pkg: pkg/dashboard + idx: 3 + total: 4 + - lane: dashboard + pkg: pkg/dashboard + idx: 4 + total: 4 + - lane: scheduler + pkg: pkg/scheduler + idx: 1 + total: 3 + - lane: scheduler + pkg: pkg/scheduler + idx: 2 + total: 3 + - lane: scheduler + pkg: pkg/scheduler + idx: 3 + total: 3 + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - name: Affected-package selection + id: select + working-directory: . + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + LANE: ${{ matrix.lane }} + PKG: ${{ matrix.pkg }} + RUN_DASHBOARD: ${{ needs.list-packages.outputs.run-dashboard }} + RUN_SCHEDULER: ${{ needs.list-packages.outputs.run-scheduler }} + run: | + case "$LANE" in + dashboard) RUN_LANE=$RUN_DASHBOARD ;; + scheduler) RUN_LANE=$RUN_SCHEDULER ;; + *) RUN_LANE=true ;; + esac + if [ "$SELECTION" = partial ] && [ "$RUN_LANE" = false ]; then + echo "Affected-package selection: $PKG is not affected by this diff; reporting success without running tests. Push and scheduled runs test it in full." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: steps.select.outputs.run == 'true' + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + if: steps.select.outputs.run == 'true' + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry + - name: Install gotestsum + if: steps.select.outputs.run == 'true' + run: | + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + go install gotest.tools/gotestsum@v1.13.0 + "$(go env GOPATH)/bin/gotestsum" --version + - name: Restore warm Go build cache + if: ${{ steps.select.outputs.run == 'true' && (!contains('["ubuntu-latest"]', 'hive-runners')) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 + if: steps.select.outputs.run == 'true' + with: + node-version: '22' + - name: Guard shared node tool cache (repair a half-written entry) + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: | + set -u + command -v npm >/dev/null 2>&1 && exit 0 + for marker in "${RUNNER_TOOL_CACHE:-/nonexistent}"/node/*/x64.complete; do + [ -e "$marker" ] || continue + dir="${marker%.complete}" + [ -x "$dir/bin/npm" ] && continue + echo "::warning::node tool cache $dir has a .complete marker but no executable bin/npm; removing the marker so the next job on $RUNNER_NAME rebuilds it" + rm -f "$marker" + done + echo "::error::npm is not on PATH after actions/setup-node on $RUNNER_NAME (half-written shared tool cache, see #10518); marker removed, re-run this job" + exit 1 + - name: Restore cgo toolchain package cache + if: steps.select.outputs.run == 'true' + id: restore-cgo-apt-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + restore-keys: | + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev- + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}- + hive-cgo-race-apt- + - name: Prepare cgo toolchain for race tests + if: steps.select.outputs.run == 'true' + id: prepare-cgo-toolchain + env: + HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !true && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require gcc --for '-race tests' \ + --apt 'gcc libc6-dev' --apk 'gcc musl-dev' --verify 'gcc --version' + - name: Save cgo toolchain package cache + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.restore-cgo-apt-cache.outputs.cache-hit != 'true' && steps.prepare-cgo-toolchain.outputs.apt-cache-populated == 'true') }} + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + - name: Test slice of ${{ matrix.pkg }} + if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} + env: + IDX: ${{ matrix.idx }} + TOTAL: ${{ matrix.total }} + LANE: ${{ matrix.lane }} + PKG: ${{ matrix.pkg }} + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: "set -o pipefail\nROOT=\"$PWD\"\nexport LC_ALL=C # Regex length guards below must count bytes.\nexport TEST_PACKAGE=\"github.com/hivecommons/hive/$PKG\"\nexport TEST_DIR=\"$ROOT/$PKG\"\nexport TEST_BINARY=\"$ROOT/$LANE.test\"\ngo test -c -race -cover -o \"$TEST_BINARY\" \"./$PKG\"\n(cd \"$TEST_DIR\" && \"$TEST_BINARY\" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt\nif [ ! -s all-tests.txt ]; then\n echo \"::error::go test -list produced no test functions for $PKG \u2014 refusing to run an empty slice (fail closed)\"\n exit 1\nfi\nawk -v n=\"$TOTAL\" -v i=\"$IDX\" 'NR % n == i - 1' all-tests.txt > slice.txt\necho \"slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions\"\nif [ ! -s slice.txt ]; then\n echo \"::error::empty $LANE slice \u2014 failing closed\"\n exit 1\nfi\nexport CHUNK_DIR TEST_SHUFFLE=off\nCHUNK_DIR=$(mktemp -d \"$ROOT/test-chunks.XXXXXX\")\nsplit -l 1000 slice.txt \"$CHUNK_DIR/chunk-\"\nif [ \"$GITHUB_EVENT_NAME\" != \"pull_request\" ]; then TEST_SHUFFLE=on; fi\n# raw-command preserves one JUnit/flake report across all chunks.\n# gotestsum appends -test.run and package arguments on reruns.\ncat > \"$CHUNK_DIR/run.sh\" <<'RUNNER'\n#!/usr/bin/env bash\nset -uo pipefail\ncd \"$TEST_DIR\" || exit 1\nrun_chunk() {\n local RUN=\"$1\" PROFILE=\"$2\"\n # MAX_ARG_STRLEN is 131072 bytes; never approach it, even if a\n # future test has an exceptionally long name. Guard before exec.\n if [ \"${#RUN}\" -gt 120000 ]; then\n echo \"::error::chunk regex exceeds 120000 bytes; reduce chunk size\" >&2\n return 1\n fi\n go tool test2json -t -p \"$TEST_PACKAGE\" \"$TEST_BINARY\" \\\n -test.short -test.count=1 -test.v=test2json -test.timeout=10m \\\n -test.shuffle=\"$TEST_SHUFFLE\" \"$RUN\" -test.coverprofile=\"$PROFILE\"\n}\nif [ \"$#\" -gt 0 ]; then\n # Reruns must not overwrite the first-pass coverage.\n run_chunk \"$1\" \"$CHUNK_DIR/rerun.out\"\n exit $?\nfi\nstatus=0\nfor chunk in \"$CHUNK_DIR\"/chunk-*; do\n RUN=\"-test.run=^($(paste -sd'|' \"$chunk\"))$\"\n run_chunk \"$RUN\" \"$chunk.out\" || status=1\ndone\nexit \"$status\"\nRUNNER\nstatus=0\nif gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \\\n --rerun-fails-report=flaky-tests-\"$LANE\"-\"$IDX\".txt \\\n --junitfile junit-\"$LANE\"-\"$IDX\".xml --raw-command -- bash \"$CHUNK_DIR/run.sh\" \\\n 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-\"$LANE\"-\"$IDX\".log; then\n :\nelse\n status=1\nfi\n# The test binary runs under test2json, not the `go test` driver, so\n# nothing prints the `ok ` / `FAIL ` summary line the\n# coverage scorer classifies packages by (#14135). Append it.\nif [ \"$status\" -eq 0 ]; then\n printf 'ok \\t%s\\t(slice %s/%s)\\n' \"$TEST_PACKAGE\" \"$IDX\" \"$TOTAL\" >> test-\"$LANE\"-\"$IDX\".log\nelse\n printf 'FAIL\\t%s\\t(slice %s/%s)\\n' \"$TEST_PACKAGE\" \"$IDX\" \"$TOTAL\" >> test-\"$LANE\"-\"$IDX\".log\nfi\n# Sum first-pass counters for identical blocks; never count reruns.\n# Do this even after test failure so the coverage gate has evidence.\nawk 'FNR == 1 { next } { key=$1 \" \" $2; hits[key]+=$3 }\n END { print \"mode: atomic\"; for (key in hits) print key, hits[key] }' \\\n \"$CHUNK_DIR\"/chunk-*.out > coverage-\"$LANE\"-\"$IDX\".out\nexit \"$status\"\n" + - name: Surface flaky reruns + id: flaky-reruns + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-flake-summary.sh" "flaky-tests-${{ matrix.lane }}-${{ matrix.idx }}.txt" "${{ matrix.lane }} ${{ matrix.idx }}/${{ matrix.total }}" + - name: Upload flaky rerun report + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.flaky-reruns.outputs.has-flakes == 'true') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: flaky-tests-${{ matrix.lane }}-${{ matrix.idx }} + path: src/flaky-tests-${{ matrix.lane }}-${{ matrix.idx }}.txt + if-no-files-found: error + retention-days: 14 + - name: Slowest tests in this slice + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/slowest-tests.sh" "${{ matrix.lane }} ${{ matrix.idx }}/${{ matrix.total }}" test-${{ matrix.lane }}-${{ matrix.idx }}.log + - name: Upload shard coverage artifacts + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: test-shard-${{ matrix.lane }}-${{ matrix.idx }} + path: | + src/coverage-${{ matrix.lane }}-${{ matrix.idx }}.out + src/test-${{ matrix.lane }}-${{ matrix.idx }}.log + if-no-files-found: warn + retention-days: 1 + test-agent: + needs: + - required-context-scope + - list-packages + if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: test (agent ${{ matrix.idx }}/5) + strategy: + fail-fast: false + matrix: + idx: + - 1 + - 2 + - 3 + - 4 + - 5 + runs-on: ubuntu-latest + timeout-minutes: 25 + defaults: + run: + working-directory: src + steps: + - name: Affected-package selection + id: select + working-directory: . + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + RUN_LANE: ${{ needs.list-packages.outputs.run-agent }} + run: | + if [ "$SELECTION" = partial ] && [ "$RUN_LANE" = false ]; then + echo "Affected-package selection: crux/agent is not affected by this diff; reporting success without running tests. Push and scheduled runs test it in full." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: steps.select.outputs.run == 'true' + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + if: steps.select.outputs.run == 'true' + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry + - name: Install gotestsum + if: steps.select.outputs.run == 'true' + run: | + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + go install gotest.tools/gotestsum@v1.13.0 + "$(go env GOPATH)/bin/gotestsum" --version + - name: Restore warm Go build cache + if: ${{ steps.select.outputs.run == 'true' && (!contains('["ubuntu-latest"]', 'hive-runners')) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + - name: Restore cgo toolchain package cache + if: steps.select.outputs.run == 'true' + id: restore-cgo-apt-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + restore-keys: | + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev- + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}- + hive-cgo-race-apt- + - name: Prepare cgo toolchain for race tests + if: steps.select.outputs.run == 'true' + id: prepare-cgo-toolchain + env: + HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !true && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require gcc --for '-race tests' \ + --apt 'gcc libc6-dev' --apk 'gcc musl-dev' --verify 'gcc --version' + - name: Save cgo toolchain package cache + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.restore-cgo-apt-cache.outputs.cache-hit != 'true' && steps.prepare-cgo-toolchain.outputs.apt-cache-populated == 'true') }} + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + - name: Install tmux (precondition for the behavioural gate) + if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require tmux --for 'the behavioural gate' \ + --apt tmux --apk tmux --verify 'tmux -V' + - name: Test slice of crux/agent + if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} + env: + IDX: ${{ matrix.idx }} + TOTAL: ${{ strategy.job-total }} + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: "set -o pipefail\nROOT=\"$PWD\"\nexport LC_ALL=C # Regex length guards below must count bytes.\nexport TEST_PACKAGE=github.com/hivecommons/hive/crux/agent\nexport TEST_DIR=\"$ROOT/crux/agent\"\nexport TEST_BINARY=\"$ROOT/agent.test\"\ngo test -c -race -cover -o \"$TEST_BINARY\" ./crux/agent\n# Include Test/Example/Fuzz, but not opt-in benchmarks.\n(cd \"$TEST_DIR\" && \"$TEST_BINARY\" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt\nif [ ! -s all-tests.txt ]; then\n echo \"::error::go test -list produced no test functions for crux/agent \u2014 refusing to run an empty slice (fail closed)\"\n exit 1\nfi\nawk -v n=\"$TOTAL\" -v i=\"$IDX\" 'NR % n == i - 1' all-tests.txt > slice.txt\necho \"slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions\"\nif [ ! -s slice.txt ]; then\n echo \"::error::empty agent slice \u2014 failing closed\"\n exit 1\nfi\nexport CHUNK_DIR TEST_SHUFFLE=off\nCHUNK_DIR=$(mktemp -d \"$ROOT/test-chunks.XXXXXX\")\nsplit -l 1000 slice.txt \"$CHUNK_DIR/chunk-\"\nif [ \"$GITHUB_EVENT_NAME\" != \"pull_request\" ]; then TEST_SHUFFLE=on; fi\n# raw-command preserves one JUnit/flake report across all chunks.\n# gotestsum appends -test.run and package arguments on reruns.\ncat > \"$CHUNK_DIR/run.sh\" <<'RUNNER'\n#!/usr/bin/env bash\nset -uo pipefail\ncd \"$TEST_DIR\" || exit 1\nrun_chunk() {\n local RUN=\"$1\" PROFILE=\"$2\"\n # MAX_ARG_STRLEN is 131072 bytes; never approach it, even if a\n # future test has an exceptionally long name. Guard before exec.\n if [ \"${#RUN}\" -gt 120000 ]; then\n echo \"::error::chunk regex exceeds 120000 bytes; reduce chunk size\" >&2\n return 1\n fi\n go tool test2json -t -p \"$TEST_PACKAGE\" \"$TEST_BINARY\" \\\n -test.short -test.count=1 -test.v=test2json -test.timeout=10m \\\n -test.shuffle=\"$TEST_SHUFFLE\" \"$RUN\" -test.coverprofile=\"$PROFILE\"\n}\nif [ \"$#\" -gt 0 ]; then\n # Reruns must not overwrite the first-pass coverage.\n run_chunk \"$1\" \"$CHUNK_DIR/rerun.out\"\n exit $?\nfi\nstatus=0\nfor chunk in \"$CHUNK_DIR\"/chunk-*; do\n RUN=\"-test.run=^($(paste -sd'|' \"$chunk\"))$\"\n run_chunk \"$RUN\" \"$chunk.out\" || status=1\ndone\nexit \"$status\"\nRUNNER\nstatus=0\nif gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \\\n --rerun-fails-report=flaky-tests-agent-\"$IDX\".txt \\\n --junitfile junit-agent-\"$IDX\".xml --raw-command -- bash \"$CHUNK_DIR/run.sh\" \\\n 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-agent-\"$IDX\".log; then\n :\nelse\n status=1\nfi\n# The test binary runs under test2json, not the `go test` driver, so\n# nothing prints the `ok ` / `FAIL ` summary line the\n# coverage scorer classifies packages by (#14135). Append it.\nif [ \"$status\" -eq 0 ]; then\n printf 'ok \\t%s\\t(slice %s/%s)\\n' \"$TEST_PACKAGE\" \"$IDX\" \"$TOTAL\" >> test-agent-\"$IDX\".log\nelse\n printf 'FAIL\\t%s\\t(slice %s/%s)\\n' \"$TEST_PACKAGE\" \"$IDX\" \"$TOTAL\" >> test-agent-\"$IDX\".log\nfi\n# Sum first-pass counters for identical blocks; never count reruns.\n# Do this even after test failure so the coverage gate has evidence.\nawk 'FNR == 1 { next } { key=$1 \" \" $2; hits[key]+=$3 }\n END { print \"mode: atomic\"; for (key in hits) print key, hits[key] }' \\\n \"$CHUNK_DIR\"/chunk-*.out > coverage-agent-\"$IDX\".out\nexit \"$status\"\n" + - name: Surface flaky reruns + id: flaky-reruns + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-flake-summary.sh" "flaky-tests-agent-${{ matrix.idx }}.txt" "agent ${{ matrix.idx }}/5" + - name: Upload flaky rerun report + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.flaky-reruns.outputs.has-flakes == 'true') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: flaky-tests-agent-${{ matrix.idx }} + path: src/flaky-tests-agent-${{ matrix.idx }}.txt + if-no-files-found: error + retention-days: 14 + - name: Slowest tests in this slice + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/slowest-tests.sh" "agent ${{ matrix.idx }}/5" test-agent-${{ matrix.idx }}.log + - name: Upload shard coverage artifacts + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: test-shard-agent-${{ matrix.idx }} + path: | + src/coverage-agent-${{ matrix.idx }}.out + src/test-agent-${{ matrix.idx }}.log + if-no-files-found: warn + retention-days: 1 + list-packages: + needs: required-context-scope + if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 15 + outputs: + matrix: ${{ steps.partition.outputs.matrix }} + selection: ${{ steps.select.outputs.selection }} + selected: ${{ steps.select.outputs.selected }} + run-hub: ${{ steps.select.outputs.run-hub }} + run-agent: ${{ steps.select.outputs.run-agent }} + run-dashboard: ${{ steps.select.outputs.run-dashboard }} + run-scheduler: ${{ steps.select.outputs.run-scheduler }} + crux-scope: ${{ steps.select.outputs.crux-scope }} + defaults: + run: + working-directory: src + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + fetch-depth: 2 + - name: Warm shared Go tool cache (serialize the cold-cache population) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + with: + cache: false + go-version-file: src/go.mod + - name: Guard shared Go cache (fall back when the runner volume is full) + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + uses: ./.github/actions/go-mod-download-retry + - name: Restore warm Go build cache + if: ${{ !contains('["ubuntu-latest"]', 'hive-runners') }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + - name: Restore cgo toolchain package cache + id: restore-cgo-apt-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + restore-keys: | + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev- + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}- + hive-cgo-race-apt- + - name: Prepare cgo toolchain for race tests + id: prepare-cgo-toolchain + env: + HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !true && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require gcc --for '-race tests' \ + --apt 'gcc libc6-dev' --apk 'gcc musl-dev' --verify 'gcc --version' + - name: Save cgo toolchain package cache + if: always() && steps.restore-cgo-apt-cache.outputs.cache-hit != 'true' && steps.prepare-cgo-toolchain.outputs.apt-cache-populated == 'true' + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + - name: Select affected packages + id: select + env: + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + SELECTION_SWITCH: ${{ 'false' }} + run: "set -uo pipefail\nsel=../affected-pkgs.txt\nif [ \"$GITHUB_EVENT_NAME\" != pull_request ]; then\n echo \"$GITHUB_EVENT_NAME run: full suite (only pull requests use affected-package selection).\"\n echo all > \"$sel\"\nelif [ \"${SELECTION_SWITCH:-}\" = off ]; then\n echo \"HIVE_AFFECTED_TEST_SELECTION=off: full suite.\"\n echo all > \"$sel\"\nelif [ -z \"${PR_HEAD_SHA:-}\" ] || [ \"$(git rev-parse -q --verify 'HEAD^2' || true)\" != \"$PR_HEAD_SHA\" ]; then\n echo \"::notice::checkout is not the PR merge commit of $PR_HEAD_SHA; running the full suite\"\n echo all > \"$sel\"\nelse\n bash \"$GITHUB_WORKSPACE/.github/scripts/affected-go-packages.sh\" 'HEAD^1' HEAD \"$sel\" || echo all > \"$sel\"\nfi\n[ -f \"$sel\" ] || echo all > \"$sel\"\n# src/go.work puts `go list -m` in workspace mode, where it prints\n# every workspace module; the selection file is keyed by the root\n# module's import paths, so ask for that module alone.\nmod=$(GOWORK=off go list -m)\ncrux_scope=full\nif [ \"$(head -n1 \"$sel\")\" = all ]; then\n selection=full\n run_hub=true run_agent=true run_dashboard=true run_scheduler=true\nelse\n selection=partial\n has() { grep -Fxq \"$mod/$1\" \"$sel\" && echo true || echo false; }\n run_hub=$(has pkg/hub) run_agent=$(has crux/agent) run_dashboard=$(has pkg/dashboard) run_scheduler=$(has pkg/scheduler)\n # Crux boundary filter (#13536) \u2014 see the header comment. Only a\n # partial selection is ever narrowed; the script answers `full`\n # for every fail-safe path and for anything it cannot read, and\n # `full` leaves the selection exactly as the dependency analysis\n # computed it.\n crux_scope=$(bash \"$GITHUB_WORKSPACE/.github/scripts/crux-scope.sh\" 'HEAD^1' HEAD) || crux_scope=full\n case \"$crux_scope\" in\n crux-only)\n run_hub=false run_dashboard=false run_scheduler=false run_agent=true\n # Rest buckets: the selected Crux packages plus the always-on\n # tree-scanning packages (mirrors ALWAYS_PATTERNS in\n # affected-go-packages.sh); a failure here falls back to the\n # unnarrowed selection.\n if go list ./crux/... \\\n ./internal/... ./pkg/config ./pkg/rotation > ../crux-pkgs.txt 2>/dev/null \\\n && [ -s ../crux-pkgs.txt ]; then\n grep -Fx -f ../crux-pkgs.txt \"$sel\" > \"$sel.crux\" || true\n echo \"crux-only diff: narrowing $(grep -c . \"$sel\" || true) selected packages to $(grep -c . \"$sel.crux\" || true) Crux/always-on packages; hub, dashboard and scheduler slices and the dashboard shuffle lane will not run\"\n mv \"$sel.crux\" \"$sel\"\n else\n echo \"::warning::crux-only diff but go list of the Crux set failed; keeping the unnarrowed selection\"\n crux_scope=mixed\n fi\n ;;\n none)\n # Drop crux/agent from the selection too: the coverage job\n # requires every selected package to have produced a result,\n # and the agent slices are the only lane that tests it.\n run_agent=false\n grep -Fxv \"$mod/crux/agent\" \"$sel\" > \"$sel.nocrux\" || true\n mv \"$sel.nocrux\" \"$sel\"\n echo \"no Crux package changed: agent slices will not run\"\n ;;\n mixed|full) ;;\n *)\n echo \"::warning::unexpected crux-scope verdict '$crux_scope'; treating as mixed\"\n crux_scope=mixed\n ;;\n esac\nfi\n{\n echo \"selection=$selection\"\n echo \"selected=$(paste -sd' ' \"$sel\")\"\n echo \"run-hub=$run_hub\"\n echo \"run-agent=$run_agent\"\n echo \"run-dashboard=$run_dashboard\"\n echo \"run-scheduler=$run_scheduler\"\n echo \"crux-scope=$crux_scope\"\n} >> \"$GITHUB_OUTPUT\"\n{\n echo \"## Go test selection\"\n echo \"\"\n if [ \"$selection\" = full ]; then\n echo \"Full suite.\"\n else\n echo \"Affected packages only ($(grep -c . \"$sel\" || true)); crux-scope=$crux_scope; hub=$run_hub agent=$run_agent dashboard=$run_dashboard scheduler=$run_scheduler.\"\n echo \"\"\n sed 's/^/- /' \"$sel\"\n fi\n} >> \"$GITHUB_STEP_SUMMARY\"\necho \"selection=$selection crux-scope=$crux_scope hub=$run_hub agent=$run_agent dashboard=$run_dashboard scheduler=$run_scheduler\"\n" + - name: Partition packages into balanced buckets + id: partition + run: "R=4\n# Drop only the monolithic packages that have function-sliced jobs\n# of their own: pkg/hub (test-hub), pkg/dashboard and pkg/scheduler\n# (test-sliced). All their subpackages belong in the rest buckets\n# so coverage floors and the no-test-files gate apply to them too.\n# pkg/hubbackup, pkg/dashboard/webstatic et al. stay here.\n# ./internal/... carries the shared test helpers (internal/testutil)\n# and their contract tests, such as the time.Sleep ratchet in\n# sleep_ratchet_test.go. ./crux/... is the nested Crux module\n# (#13536), reachable from src/ through src/go.work; its manager\n# package crux/agent has the dedicated agent slices.\ngo list ./pkg/... ./cmd/... ./internal/... ./crux/... \\\n | awk '$0 == \"github.com/hivecommons/hive/pkg/hub\" { next }\n $0 == \"github.com/hivecommons/hive/pkg/dashboard\" { next }\n $0 == \"github.com/hivecommons/hive/pkg/scheduler\" { next }\n { print }' \\\n | grep -Ev '^github\\.com/hivecommons/hive/crux/agent$' \\\n | LC_ALL=C sort > rest-pkgs.txt\nif [ ! -s rest-pkgs.txt ]; then\n echo \"::error::go list produced no rest packages \u2014 failing closed\"\n exit 1\nfi\n# Affected-package selection narrows membership only after the\n# fail-closed check above ran on the full list. The bucket count\n# stays R, so every `test (rest i/R)` context still reports; a\n# bucket left empty succeeds without running (see test-rest).\nif [ \"$(head -n1 ../affected-pkgs.txt)\" != all ]; then\n grep -Fx -f ../affected-pkgs.txt rest-pkgs.txt > rest-selected.txt || true\n echo \"affected-package selection: $(wc -l < rest-selected.txt) of $(wc -l < rest-pkgs.txt) rest packages\"\n mv rest-selected.txt rest-pkgs.txt\nfi\n\n# Weights = measured -short -race package wall seconds on the\n# hive-runners fleet, mean of two v5 push runs (40383615331 and\n# 40382244011, 2026-10-26, rounded). Re-measure with: for each\n# `test (rest i/R)` job log, grep the\n# `ok github.com/hivecommons/hive/ s` lines. Anything\n# under ~5s is noise and takes the 1s default. Packages inside a\n# bucket run concurrently under one go test, so a bucket's wall\n# clock is its heaviest package plus compile time \u2014 the table\n# mainly has to keep the big ones apart, and the sum-based greedy\n# below does that while spreading the compile load. pkg/hub/spoke\n# is the floor now that pkg/dashboard and pkg/scheduler are sliced\n# (test-sliced above): the widest bucket can never be faster than\n# it. pkg/dashboard and pkg/scheduler are listed so a stale\n# exclusion above still yields a sane partition. pkg/github is\n# ~1s with ~1% coverage on push runs but 70s on pull requests (PR\n# 14407's run 40387365403), where it is weighed \u2014 measure it from a\n# PR run, not a push run.\nweight() {\n case \"$1\" in\n */pkg/dashboard) echo 219 ;;\n */pkg/scheduler) echo 199 ;;\n */pkg/hub/spoke) echo 85 ;;\n */pkg/config) echo 82 ;;\n */pkg/github) echo 70 ;;\n */cmd/hive) echo 69 ;;\n */crux/proxy) echo 56 ;;\n */pkg/dashboard/webstatic) echo 35 ;;\n */pkg/hubbackup) echo 30 ;;\n */pkg/advisory) echo 29 ;;\n */internal/testutil) echo 22 ;;\n */pkg/ioscan) echo 16 ;;\n */pkg/hivectl) echo 15 ;;\n */pkg/tui) echo 14 ;;\n */pkg/discord) echo 12 ;;\n */pkg/fleetreport) echo 11 ;;\n */pkg/hivectl/commands) echo 10 ;;\n */pkg/stageengine) echo 10 ;;\n */pkg/apphealth) echo 9 ;;\n */pkg/rotation) echo 9 ;;\n */pkg/beads) echo 8 ;;\n */pkg/github/automerge) echo 8 ;;\n */crux/api/callback) echo 6 ;;\n */crux/inference) echo 6 ;;\n */cmd/hive-backup) echo 5 ;;\n */pkg/mint) echo 5 ;;\n */pkg/policies) echo 5 ;;\n *) echo 1 ;;\n esac\n}\n\n# Deterministic greedy LPT: stable-sort by weight desc (name asc on\n# ties), assign each package to the currently lightest bucket\n# (lowest index on ties).\nwhile read -r p; do\n echo \"$(weight \"$p\") $p\"\ndone < rest-pkgs.txt | sort -k1,1nr -k2,2 > weighted.txt\n\ndeclare -a LOAD BUCKET\nfor ((i=0; i> \"$GITHUB_OUTPUT\"\n" + test-rest: + if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: test (rest ${{ matrix.idx }}/${{ matrix.total }}) + needs: + - required-context-scope + - list-packages + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.list-packages.outputs.matrix) }} + runs-on: ubuntu-latest + timeout-minutes: 25 + defaults: + run: + working-directory: src + steps: + - name: Affected-package selection + id: select + working-directory: . + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + PKGS: ${{ matrix.pkgs }} + run: | + if [ "$SELECTION" = partial ] && [ -z "$PKGS" ]; then + echo "Affected-package selection: no affected package landed in this bucket; reporting success without running tests. Push and scheduled runs test it in full." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: steps.select.outputs.run == 'true' + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + if: steps.select.outputs.run == 'true' + with: + cache: false + go-version-file: src/go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 + if: steps.select.outputs.run == 'true' + with: + node-version: '22' + - name: Guard shared node tool cache (repair a half-written entry) + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: | + set -u + command -v npm >/dev/null 2>&1 && exit 0 + for marker in "${RUNNER_TOOL_CACHE:-/nonexistent}"/node/*/x64.complete; do + [ -e "$marker" ] || continue + dir="${marker%.complete}" + [ -x "$dir/bin/npm" ] && continue + echo "::warning::node tool cache $dir has a .complete marker but no executable bin/npm; removing the marker so the next job on $RUNNER_NAME rebuilds it" + rm -f "$marker" + done + echo "::error::npm is not on PATH after actions/setup-node on $RUNNER_NAME (half-written shared tool cache, see #10518); marker removed, re-run this job" + exit 1 + - name: Guard shared Go cache (fall back when the runner volume is full) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry + - name: Install gotestsum + if: steps.select.outputs.run == 'true' + run: | + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + go install gotest.tools/gotestsum@v1.13.0 + "$(go env GOPATH)/bin/gotestsum" --version + - name: Restore warm Go build cache + if: ${{ steps.select.outputs.run == 'true' && (!contains('["ubuntu-latest"]', 'hive-runners')) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + - name: Restore cgo toolchain package cache + if: steps.select.outputs.run == 'true' + id: restore-cgo-apt-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + restore-keys: | + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev- + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}- + hive-cgo-race-apt- + - name: Prepare cgo toolchain for race tests + if: steps.select.outputs.run == 'true' + id: prepare-cgo-toolchain + env: + HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !true && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require gcc --for '-race tests' \ + --apt 'gcc libc6-dev' --apk 'gcc musl-dev' --verify 'gcc --version' + - name: Save cgo toolchain package cache + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.restore-cgo-apt-cache.outputs.cache-hit != 'true' && steps.prepare-cgo-toolchain.outputs.apt-cache-populated == 'true') }} + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + - name: Test + if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} + env: + PKGS: ${{ matrix.pkgs }} + HIVE_TEST_REQUIRE_BEHAVIOURAL: '1' + run: "set -o pipefail\nif [ -z \"$PKGS\" ]; then\n echo \"::error::empty package bucket \u2014 failing closed\"\n exit 1\nfi\n# -timeout=20m: kept from when pkg/dashboard lived in these buckets\n# and nearly exhausted go test's default 10m on loaded PR runners\n# (#12113); a loaded runner can still stretch pkg/hub/spoke.\nSHUFFLE=\"\"\nif [ \"$GITHUB_EVENT_NAME\" != \"pull_request\" ]; then SHUFFLE=\"-shuffle=on\"; fi\n# shellcheck disable=SC2086 # PKGS and SHUFFLE are deliberate word-split lists\nbash \"$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-preserve-coverprofile.sh\" \\\n gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \\\n --rerun-fails-report=flaky-tests-rest-${{ matrix.idx }}.txt \\\n --packages=\"$PKGS\" --junitfile junit-rest-${{ matrix.idx }}.xml \\\n -- -short -race -count=1 -timeout=20m -v $SHUFFLE -coverprofile=coverage-rest-${{ matrix.idx }}.out \\\n 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-rest-${{ matrix.idx }}.log\n" + - name: Surface flaky reruns + id: flaky-reruns + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-flake-summary.sh" "flaky-tests-rest-${{ matrix.idx }}.txt" "rest ${{ matrix.idx }}/${{ matrix.total }}" + - name: Upload flaky rerun report + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.flaky-reruns.outputs.has-flakes == 'true') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: flaky-tests-rest-${{ matrix.idx }} + path: src/flaky-tests-rest-${{ matrix.idx }}.txt + if-no-files-found: error + retention-days: 14 + - name: Slowest tests in this bucket + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/slowest-tests.sh" "rest ${{ matrix.idx }}/${{ matrix.total }}" test-rest-${{ matrix.idx }}.log + - name: Upload shard coverage artifacts + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: test-shard-rest-${{ matrix.idx }} + path: | + src/coverage-rest-${{ matrix.idx }}.out + src/test-rest-${{ matrix.idx }}.log + if-no-files-found: warn + retention-days: 1 + test-dashboard-shuffle: + needs: + - required-context-scope + - list-packages + if: ${{ (needs.required-context-scope.result != 'success' || needs.required-context-scope.outputs.run-tests == 'true') && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + name: test (dashboard shuffle ${{ matrix.idx }}/3) + strategy: + fail-fast: false + matrix: + idx: + - 1 + - 2 + - 3 + runs-on: ubuntu-latest + timeout-minutes: 20 + defaults: + run: + working-directory: src + steps: + - name: Affected-package selection + id: select + working-directory: . + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + RUN_LANE: ${{ needs.list-packages.outputs.run-dashboard }} + run: | + if [ "$SELECTION" = partial ] && [ "$RUN_LANE" = false ]; then + echo "Affected-package selection: pkg/dashboard is not affected by this diff; reporting success without running tests. Push and scheduled runs test it in full." + echo "run=false" >> "$GITHUB_OUTPUT" + else + echo "run=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: steps.select.outputs.run == 'true' + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + if: steps.select.outputs.run == 'true' + with: + cache: false + go-version-file: src/go.mod + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 + if: steps.select.outputs.run == 'true' + with: + node-version: '22' + - name: Guard shared node tool cache (repair a half-written entry) + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: | + set -u + command -v npm >/dev/null 2>&1 && exit 0 + for marker in "${RUNNER_TOOL_CACHE:-/nonexistent}"/node/*/x64.complete; do + [ -e "$marker" ] || continue + dir="${marker%.complete}" + [ -x "$dir/bin/npm" ] && continue + echo "::warning::node tool cache $dir has a .complete marker but no executable bin/npm; removing the marker so the next job on $RUNNER_NAME rebuilds it" + rm -f "$marker" + done + echo "::error::npm is not on PATH after actions/setup-node on $RUNNER_NAME (half-written shared tool cache, see #10518); marker removed, re-run this job" + exit 1 + - name: Guard shared Go cache (fall back when the runner volume is full) + if: steps.select.outputs.run == 'true' + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh" + - name: Download Go modules (retry transient proxy resets) + if: steps.select.outputs.run == 'true' + uses: ./.github/actions/go-mod-download-retry + - name: Install gotestsum + if: steps.select.outputs.run == 'true' + run: | + echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH" + go install gotest.tools/gotestsum@v1.13.0 + "$(go env GOPATH)/bin/gotestsum" --version + - name: Restore warm Go build cache + if: ${{ steps.select.outputs.run == 'true' && (!contains('["ubuntu-latest"]', 'hive-runners')) }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: ~/.cache/go-build + key: hive-go-build-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + restore-keys: | + hive-go-build-${{ runner.os }}-${{ runner.arch }}- + - name: Restore cgo toolchain package cache + if: steps.select.outputs.run == 'true' + id: restore-cgo-apt-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + restore-keys: | + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev- + hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}- + hive-cgo-race-apt- + - name: Prepare cgo toolchain for race tests + if: steps.select.outputs.run == 'true' + id: prepare-cgo-toolchain + env: + HIVE_CI_APT_CACHE_DIR: ${{ github.workspace }}/.ci/apt-cache/cgo-race + HIVE_CI_MISSING_TOOLCHAIN_SOFT_FAIL: ${{ !true && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && '1' || '0' }} + run: | + bash "$GITHUB_WORKSPACE/.github/scripts/ci-install-tool.sh" \ + --require gcc --for '-race tests' \ + --apt 'gcc libc6-dev' --apk 'gcc musl-dev' --verify 'gcc --version' + - name: Save cgo toolchain package cache + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.restore-cgo-apt-cache.outputs.cache-hit != 'true' && steps.prepare-cgo-toolchain.outputs.apt-cache-populated == 'true') }} + continue-on-error: true + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 + with: + path: .ci/apt-cache/cgo-race + key: hive-cgo-race-apt-${{ runner.os }}-${{ runner.arch }}-gcc-libc6-dev-v2 + - name: Test pkg/dashboard in shuffled order + if: ${{ steps.select.outputs.run == 'true' && (steps.prepare-cgo-toolchain.outputs.toolchain-ready == 'true') }} + env: + IDX: ${{ matrix.idx }} + TOTAL: ${{ strategy.job-total }} + run: "set -o pipefail\n# Sliced like test-hub (NR mod N \u2014 pairwise disjoint, jointly\n# exhaustive) so no single job carries the whole ~6-minute lane;\n# -shuffle still varies order within each slice every run.\nROOT=\"$PWD\"\nexport TEST_PACKAGE=github.com/hivecommons/hive/pkg/dashboard\nexport TEST_DIR=\"$ROOT/pkg/dashboard\"\nexport TEST_BINARY=\"$ROOT/dashboard.test\"\ngo test -c -race -o \"$TEST_BINARY\" ./pkg/dashboard\n(cd \"$TEST_DIR\" && \"$TEST_BINARY\" -test.short -test.list '.*') | grep -E '^(Test|Example|Fuzz)' | LC_ALL=C sort > all-tests.txt\nif [ ! -s all-tests.txt ]; then\n echo \"::error::go test -list produced no test functions for pkg/dashboard \u2014 refusing to run an empty slice (fail closed)\"\n exit 1\nfi\nawk -v n=\"$TOTAL\" -v i=\"$IDX\" 'NR % n == i - 1' all-tests.txt > slice.txt\necho \"slice $IDX/$TOTAL: $(wc -l < slice.txt) of $(wc -l < all-tests.txt) test functions\"\nif [ ! -s slice.txt ]; then\n echo \"::error::empty dashboard slice \u2014 failing closed\"\n exit 1\nfi\nexport CHUNK_DIR\nCHUNK_DIR=$(mktemp -d \"$ROOT/test-chunks.XXXXXX\")\nsplit -l 1000 slice.txt \"$CHUNK_DIR/chunk-\"\n# raw-command preserves one JUnit/flake report across all chunks.\n# gotestsum appends -test.run and package arguments on reruns.\ncat > \"$CHUNK_DIR/run.sh\" <<'RUNNER'\n#!/usr/bin/env bash\nset -uo pipefail\nexport LC_ALL=C # ${#RUN} must count bytes, not Unicode characters.\ncd \"$TEST_DIR\" || exit 1\nrun_chunk() {\n local RUN=\"$1\"\n # MAX_ARG_STRLEN is 131072 bytes; fail clearly before execve.\n if [ \"${#RUN}\" -gt 120000 ]; then\n echo \"::error::chunk regex exceeds 120000 bytes; reduce chunk size\" >&2\n return 1\n fi\n # Bound t.Parallel to avoid exhausting httptest ephemeral ports.\n go tool test2json -t -p \"$TEST_PACKAGE\" \"$TEST_BINARY\" \\\n -test.short -test.count=1 -test.v=test2json -test.timeout=10m \\\n -test.shuffle=on -test.parallel=4 \"$RUN\"\n}\nif [ \"$#\" -gt 0 ]; then\n run_chunk \"$1\"\n exit $?\nfi\nstatus=0\nfor chunk in \"$CHUNK_DIR\"/chunk-*; do\n RUN=\"-test.run=^($(paste -sd'|' \"$chunk\"))$\"\n run_chunk \"$RUN\" || status=1\ndone\nexit \"$status\"\nRUNNER\nif gotestsum --format=standard-verbose --rerun-fails=2 --rerun-fails-max-failures=5 \\\n --rerun-fails-report=flaky-tests-dashboard-shuffle-\"$IDX\".txt \\\n --junitfile junit-dashboard-shuffle-\"$IDX\".xml --raw-command -- bash \"$CHUNK_DIR/run.sh\" \\\n 2>&1 | grep --line-buffered -vE '^=== (RUN|PAUSE|CONT) ' | tee test-dashboard-shuffle-\"$IDX\".log; then\n exit 0\nfi\n# Each chunk prints its own -test.shuffle seed immediately before\n# its tests; do not report the first chunk's seed for a later failure.\necho \"::error::pkg/dashboard failed (slice $IDX/$TOTAL); see the failing chunk's -test.shuffle seed in test-dashboard-shuffle-$IDX.log. Reproduce the failing test with: go test ./pkg/dashboard -short -race -count=1 -p 1 -parallel 4 -run '^TestName$' -shuffle=\"\nexit 1\n" + - name: Surface flaky reruns + id: flaky-reruns + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/ci-gotestsum-flake-summary.sh" "flaky-tests-dashboard-shuffle-${{ matrix.idx }}.txt" "dashboard shuffle ${{ matrix.idx }}/3" + - name: Upload flaky rerun report + if: ${{ steps.select.outputs.run == 'true' && (always() && steps.flaky-reruns.outputs.has-flakes == 'true') }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: flaky-tests-dashboard-shuffle-${{ matrix.idx }} + path: src/flaky-tests-dashboard-shuffle-${{ matrix.idx }}.txt + if-no-files-found: error + retention-days: 14 + - name: Slowest tests in this lane + if: ${{ steps.select.outputs.run == 'true' && (always()) }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/slowest-tests.sh" "dashboard shuffle ${{ matrix.idx }}/3" test-dashboard-shuffle-${{ matrix.idx }}.log + - name: Upload shuffle log + if: ${{ steps.select.outputs.run == 'true' && (failure()) }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 + with: + name: test-dashboard-shuffle-log-${{ matrix.idx }} + path: src/test-dashboard-shuffle-${{ matrix.idx }}.log + if-no-files-found: warn + retention-days: 1 + test: + if: ${{ always() && !cancelled() && !(github.event_name == 'pull_request' && startsWith(github.head_ref, 'release-gate/')) }} + runs-on: ubuntu-latest + timeout-minutes: 10 + needs: + - required-context-scope + - list-packages + - test-hub + - test-sliced + - test-agent + - test-rest + - test-dashboard-shuffle + steps: + - name: Check shard results + env: + LIST_PACKAGES_RESULT: ${{ needs.list-packages.result }} + TEST_HUB_RESULT: ${{ needs.test-hub.result }} + TEST_SLICED_RESULT: ${{ needs.test-sliced.result }} + TEST_AGENT_RESULT: ${{ needs.test-agent.result }} + TEST_REST_RESULT: ${{ needs.test-rest.result }} + TEST_DASHBOARD_SHUFFLE_RESULT: ${{ needs.test-dashboard-shuffle.result }} + RUN_TESTS: ${{ needs.required-context-scope.outputs.run-tests }} + SCOPE_RESULT: ${{ needs.required-context-scope.result }} + SELECTION: ${{ needs.list-packages.outputs.selection }} + CRUX_SCOPE: ${{ needs.list-packages.outputs.crux-scope }} + RUN_HUB: ${{ needs.list-packages.outputs.run-hub }} + RUN_AGENT: ${{ needs.list-packages.outputs.run-agent }} + RUN_DASHBOARD: ${{ needs.list-packages.outputs.run-dashboard }} + RUN_SCHEDULER: ${{ needs.list-packages.outputs.run-scheduler }} + run: "set -euo pipefail\n\nif [ \"${SCOPE_RESULT:-}\" != success ]; then\n echo \"::error::required-context-scope failed; refusing to report test success without a diff verdict\"\n exit 1\nfi\n\nif [ \"${RUN_TESTS:-}\" != true ]; then\n echo \"No v2 test paths changed; reporting required test context success without running shards.\"\n exit 0\nfi\n\n# Lanes with nothing selected still ran (and succeeded) \u2014 see the\n# Affected-package selection step in each shard \u2014 so the fail-closed\n# result check below is unchanged by selection.\necho \"Go test selection: ${SELECTION:-} (partial = affected packages only on this PR; push and scheduled runs are always full)\"\n# Crux boundary filter (#13536): the lanes allowed to report success\n# without testing are exactly those whose run flag list-packages set\n# to false. Print that allowlist here so the verdict is auditable\n# from the aggregate log; a lane absent from it that did not test\n# would have failed its own selection step logic, and any shard\n# result other than success still fails this job below.\nif [ \"${SELECTION:-}\" = partial ]; then\n allow=()\n [ \"${RUN_HUB:-true}\" = false ] && allow+=(test-hub)\n [ \"${RUN_AGENT:-true}\" = false ] && allow+=(test-agent)\n [ \"${RUN_DASHBOARD:-true}\" = false ] && allow+=(test-dashboard-shuffle \"test-sliced (dashboard)\")\n [ \"${RUN_SCHEDULER:-true}\" = false ] && allow+=(\"test-sliced (scheduler)\")\n echo \"Crux scope: ${CRUX_SCOPE:-}; lanes allowed to succeed without testing: ${allow[*]:-}\"\nfi\n\nshard_names=(\n list-packages\n test-hub\n test-sliced\n test-agent\n test-rest\n test-dashboard-shuffle\n)\nshard_results=(\n \"${LIST_PACKAGES_RESULT:-}\"\n \"${TEST_HUB_RESULT:-}\"\n \"${TEST_SLICED_RESULT:-}\"\n \"${TEST_AGENT_RESULT:-}\"\n \"${TEST_REST_RESULT:-}\"\n \"${TEST_DASHBOARD_SHUFFLE_RESULT:-}\"\n)\n\nfailures=()\nskipped=()\ncancelled=()\nunknown=()\n\nfor i in \"${!shard_names[@]}\"; do\n name=${shard_names[$i]}\n result=${shard_results[$i]}\n printf '%-23s %s\\n' \"${name}:\" \"${result:-}\"\n case \"$result\" in\n success) ;;\n failure) failures+=(\"$name\") ;;\n skipped) skipped+=(\"$name\") ;;\n cancelled) cancelled+=(\"$name\") ;;\n *) unknown+=(\"$name=${result:-}\") ;;\n esac\ndone\n\njoin_by_comma() {\n local IFS=', '\n echo \"$*\"\n}\n\nappend_summary() {\n if [ -n \"${GITHUB_STEP_SUMMARY:-}\" ]; then\n {\n echo \"## v2 shard aggregate\"\n echo \"\"\n echo \"$1\"\n } >> \"$GITHUB_STEP_SUMMARY\"\n fi\n}\n\nif [ \"${#failures[@]}\" -gt 0 ] || [ \"${#skipped[@]}\" -gt 0 ] || [ \"${#unknown[@]}\" -gt 0 ]; then\n if [ \"${#failures[@]}\" -gt 0 ]; then\n echo \"::error::Failing shard(s): $(join_by_comma \"${failures[@]}\")\"\n fi\n if [ \"${#skipped[@]}\" -gt 0 ]; then\n echo \"::error::Skipped shard(s): $(join_by_comma \"${skipped[@]}\"). A skipped shard means tests may not have run.\"\n fi\n if [ \"${#unknown[@]}\" -gt 0 ]; then\n echo \"::error::Unknown shard result(s): $(join_by_comma \"${unknown[@]}\")\"\n fi\n if [ \"${#cancelled[@]}\" -gt 0 ]; then\n echo \"Cancelled shard(s) were also present: $(join_by_comma \"${cancelled[@]}\"). They are not treated as infrastructure-only because another shard failed, skipped, or reported an unknown state.\"\n fi\n append_summary \"\u274C At least one shard failed, skipped, or reported an unknown state. Cancelled shards, if any, were not allowed to mask that result.\"\n exit 1\nfi\n\nif [ \"${#cancelled[@]}\" -gt 0 ]; then\n message=\"Infrastructure cancellation: shard(s) did not complete: $(join_by_comma \"${cancelled[@]}\"). No test failure was observed, but this run is not complete test evidence for those shard(s).\"\n echo \"::warning title=Infrastructure cancellation::$message\"\n append_summary \"\u26A0\uFE0F $message\"\n exit 0\nfi\n\nappend_summary \"\u2705 All shards succeeded.\"\necho \"All shards succeeded.\"\n" + coverage: + if: ${{ always() && (true || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && !startsWith(github.head_ref, 'release-gate/'))) }} + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: + - required-context-scope + - list-packages + - test-hub + - test-sliced + - test-agent + - test-rest + steps: + - name: Report coverage not required for this diff + if: ${{ needs.required-context-scope.result == 'success' && needs.required-context-scope.outputs.run-tests != 'true' }} + run: echo "No v2 coverage paths changed; reporting required coverage context success." + - name: Fail if scope failed or a required test shard did not complete + if: ${{ needs.required-context-scope.result != 'success' || (needs.required-context-scope.outputs.run-tests == 'true' && (needs.test-hub.result != 'success' || needs.test-sliced.result != 'success' || needs.test-agent.result != 'success' || needs.test-rest.result != 'success')) }} + run: | + if [ "${{ needs.required-context-scope.result }}" != success ]; then + echo "::error::required-context-scope failed; refusing to report coverage success without a diff verdict" + exit 1 + fi + echo "::error::Coverage requires successful test shards (hub=${{ needs.test-hub.result }}, sliced=${{ needs.test-sliced.result }}, agent=${{ needs.test-agent.result }}, rest=${{ needs.test-rest.result }})" + exit 1 + - name: Decide what coverage to score + id: cov-scope + if: ${{ needs.required-context-scope.outputs.run-tests == 'true' }} + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + SELECTED: ${{ needs.list-packages.outputs.selected }} + run: | + if [ "$SELECTION" = partial ] && [ -z "${SELECTED// /}" ]; then + echo "Affected-package selection chose no Go package; no coverage to score. Push and scheduled runs score the full suite." + echo "score=false" >> "$GITHUB_OUTPUT" + else + echo "Coverage scoring: ${SELECTION:-} selection." + echo "score=true" >> "$GITHUB_OUTPUT" + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + if: ${{ steps.cov-scope.outputs.score == 'true' }} + with: + persist-credentials: false + - name: Warm shared Go tool cache (serialize the cold-cache population) + if: ${{ steps.cov-scope.outputs.score == 'true' }} + run: bash "$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh" + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e + if: ${{ steps.cov-scope.outputs.score == 'true' }} + with: + go-version-file: src/go.mod + cache-dependency-path: | + src/go.sum + src/crux/go.sum + - name: Download Go modules (retry transient proxy resets) + if: ${{ steps.cov-scope.outputs.score == 'true' }} + uses: ./.github/actions/go-mod-download-retry + - name: List packages present in checkout + if: ${{ steps.cov-scope.outputs.score == 'true' }} + working-directory: src + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + run: | + # The coverage scorer consumes shard artifacts, but the package set it + # scores must still be bounded by the checked-out tree for this run. + # This mirrors coverage-hourly.yml, whose `go test` command can only + # emit packages that exist in the checkout. + go list ./pkg/... ./cmd/... ./internal/... ./crux/... | LC_ALL=C sort -u > ../existing-pkgs.txt + if [ ! -s ../existing-pkgs.txt ]; then + echo "::error::go list produced no packages for coverage scoring" + exit 1 + fi + # On a partial selection the shard logs only cover the selected + # packages, so take the no-test-files gate from the whole tree + # instead (cheap: go list only). Same definition as the `?` / + # `coverage: 0.0%` lines a full run classifies below. + : > ../no-test-pkgs.txt + if [ "$SELECTION" = partial ]; then + go list -f '{{if not (or .TestGoFiles .XTestGoFiles)}}{{.ImportPath}}{{end}}' ./pkg/... ./cmd/... ./crux/... \ + | { grep . || true; } | LC_ALL=C sort -u > ../no-test-pkgs.txt + fi + - name: Download shard coverage artifacts + if: ${{ steps.cov-scope.outputs.score == 'true' }} + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 + with: + pattern: test-shard-* + path: shards + merge-multiple: true + - name: Check per-package coverage + if: ${{ steps.cov-scope.outputs.score == 'true' }} + working-directory: shards + env: + SELECTION: ${{ needs.list-packages.outputs.selection }} + SELECTED: ${{ needs.list-packages.outputs.selected }} + run: "THRESHOLD=90\nFAILED=\"\"\nBROKEN=\"\"\nREPORT=\"\"\n\n# Per-package threshold overrides. A handful of packages are heavily\n# coupled to the live runtime (fixed /data,/proc,/var/run paths, real\n# github.com/platform.claude.com device-flow, a live tmux agent buffer,\n# an in-cluster k8s serviceaccount) and cannot be exercised hermetically\n# in CI without invasive production seams. They get a documented lower\n# floor so the gate still catches regressions below their real ceiling,\n# rather than blocking forever on untestable code. Raise these toward 90\n# as seams are added; do not add new entries without justification here.\n#\n# KEEP IN SYNC with coverage-hourly.yml, which applies the same floors\n# on its hourly scheduled run.\ndeclare -A PKG_THRESHOLD=(\n [dashboard]=84 # 86.2% on 2026-09-01; the residue is the inception tmux watcher + copilot/claude device-flow + /data file I/O + k8s\n [hub]=87 # kubectl-shelling + live-GitHub paths; see #2355. Was 89 against v2's 89.9%; v4 measures 87.6%.\n [crux/agent]=89 # pre-existing shortfall inherited from #2350, NOT a new regression. See #2355. Keyed crux/ since the module cut (#13536)\n\n # \u2500\u2500 Floors recorded when the gate moved from v2 to v4 (#3903) \u2500\u2500\u2500\u2500\u2500\n # A RATCHET, NOT A TARGET \u2014 see coverage-hourly.yml for the full\n # rationale. Raise these toward 90 as tests land; never raise one to\n # paper over a drop.\n #\n # \u2500\u2500 Ratchet turned for #4570 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n # sandbox/ioscan/pushbroker: tests added in the same PR, so the\n # floors move with the gain (67.6\u219298.5, 71.0\u219297.1, 72.8\u2192100.0).\n # intent/tracing/dashboard: NO code change \u2014 these had drifted far\n # above floors recorded at d89121f5 and were no longer catching\n # anything. Re-measured on v4 at cd8aee5d and moved up to what\n # they actually hold, with ~1 point of headroom.\n # scheduler (90.4) and escalation (93.2) now clear the 90% default\n # and no longer need an entry at all.\n [intent]=87\n [tracing]=85\n [review]=80\n [retro]=82\n [github]=88 # 88.9% on 2026-10-05\n [hivectl]=88\n [knowledge]=89\n [crux/proxy]=89\n [ioscan]=96\n [crux/sandbox]=98\n [crux/pushbroker]=99\n [crux/kubejob]=90 # new in #6311; sandbox.Launcher over the in-cluster API, tested against httptest\n\n # \u2500\u2500 Stale module-path parser repaired on v5 (#6066) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n # These packages were pre-existing below-default floors that became\n # visible again when the v2-tests coverage classifier was corrected\n # from the old github.com/kubestellar/hive module path to\n # github.com/hivecommons/hive. Record their current ceilings so the\n # parser fix can gate new packages (pkg/effects, pkg/hub/spoke)\n # without turning unrelated backlog into a blocker.\n [convergence/mutation]=86 # 86.7% on PR #6076\n [convergence/audit]=92 # measured 95.6% on v5 after #9930 (in-package Run/reconcile/publish tests, #9928); was 6 (seed tests only, #9688)\n [dashboard/webstatic]=88 # 88.9% on PR #6076\n [github/automerge]=88 # 88.6% on PR #6076\n [mint]=89 # 89.1% on PR #6076\n [turn]=87 # 87.1% on PR #6076\n\n # \u2500\u2500 Ratchet turned for #7146 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n # spoke was unfloored and flapped across the 90 default (run\n # 35026215085 at 89.6%, green runs either side, no spoke change\n # between). #7130 covered the timing-dependent paths rather than\n # lowering the floor; spoke now holds a flat 96.3% with zero variance\n # over five runs. See coverage-hourly.yml for the full rationale.\n [spoke]=95 # 96.3% on 2026-09-16 after #7130; was unfloored/flapping\n\n # \u2500\u2500 ./cmd/... floors (#4701) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n # See coverage-hourly.yml for the rationale. cmd keys are prefixed\n # (\"cmd/\") so cmd/hivectl can never collide with\n # pkg/hivectl's floor. Recorded just below what v4 measures after\n # #4700/#4719 so the ratchet can turn as CLI tests land.\n # \u2500\u2500 Ratchet turned 2026-09-01 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n # cmd/bd, cmd/hive, cmd/hive-backup, dashboard, github: NO code\n # change \u2014 each had drifted 4\u201320 points above its recorded floor\n # (cmd/hive most of all: 18.8 \u2192 37.5 as eval-cycle helpers grew\n # tests) and the gate was no longer catching a drop. Re-measured\n # on the 2026-09-01 hourly run and moved to ~2\u20134 points under\n # what they hold. cmd/apiproxy is unchanged at 13.9%.\n # \u2500\u2500 Ratchet turned for #7407 \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n # NO code change \u2014 re-measured on 2026-09-17 with `go test -cover\n # ./cmd/... -short -count=1` (three runs, zero variance) and moved\n # to ~2-3 points under the lower of the linux/darwin measurements.\n # See coverage-hourly.yml for the full rationale and both numbers.\n [cmd/bd]=92 # 95.7% on 2026-09-17; was 75\n [cmd/hive]=42 # 44.8% on 2026-09-17; bulk is main()/run-loop wiring; was 35\n [cmd/apiproxy]=92 # 95.6% on 2026-09-17; was 13\n [cmd/hive-backup]=85 # 90.4% on 2026-09-17; was 78\n [cmd/hivectl]=97 # 100.0% on 2026-09-17; was exempt as an untested main\n [cmd/flue-fixture]=75 # 80.0% on 2026-09-22 (#8361): main() is the single uncovered statement; run() and signalContext() are tested in-process and the binary is smoke-tested as a child process, which yields no profile\n [cmd/crux]=75 # 80.0% on 2026-10-23 (#13536): same shape as flue-fixture; main() is the one uncovered statement, run() is tested in-process. floor_key maps BOTH ./cmd/crux (root host) and ./crux/cmd/crux (the module's own binary, #14138) here; the module one measured 80.0% on 2026-10-25\n [crux/cli]=92 # 97.8% on 2026-10-23 (#13536): the crux CLI against a fake crux.Runtime plus a real unix control socket; the residue is Main() and write-error branches\n\n # \u2500\u2500 Crux module (#13536, Stage 2 step 4) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n # Packages of the nested module at src/crux are keyed \"crux/\"\n # (floor_key below): crux/agent, crux/proxy, crux/sandbox,\n # crux/pushbroker, crux/kubejob and crux/cli above carried their\n # pkg/ floors across the move unchanged. The module root\n # package is keyed \"crux\". Every other crux/ takes the\n # default.\n)\n\n# Thin flag-parsing mains with no test files, wrapping packages the\n# gate already floors. KEEP IN SYNC with coverage-hourly.yml. Any NEW\n# untested cmd package still hard-fails.\n#\n# Empty since #7407: cmd/hivectl, the only entry, gained tests in\n# a3f9842c, so no \"[no test files]\" line is emitted for it and the\n# exemption could never match. The mechanism itself is unchanged.\nUNTESTED_CMD_MAINS=''\n\n# Merge every shard profile: per-block counts are summed, so the N\n# partial crux/agent profiles combine into the package's true\n# coverage. Blocks are keyed by (location, statement count); package\n# = the directory of the block's file. Emits \"pkg pct\" lines plus a\n# TOTAL over ./pkg/... \u2014 the same statements-covered/statements\n# definition `go test -cover` prints, so floors are unchanged.\n#\n# Packages are selected by their \"/pkg/\" path segment rather than by\n# a hard-coded module prefix. The previous version matched the literal\n# module path, so the kubestellar -> hivecommons org rename silently\n# emptied every filter and disabled the whole gate. A segment match\n# survives any future rename, and only this repository's own packages\n# appear in these profiles, so it cannot over-match. The nested Crux\n# module's packages carry a \"/crux\" segment (github.com//hive/crux\n# and .../crux/) and get a second, informational total.\nawk '\n FNR == 1 { next } # skip each file'\\''s \"mode:\" line\n {\n file = $1; sub(/:[^:]*$/, \"\", file)\n pkg = file; sub(/\\/[^\\/]+$/, \"\", pkg)\n key = pkg \"|\" $1 \"|\" $2\n stmts[key] = $2; owner[key] = pkg\n count[key] += $3\n }\n END {\n for (k in count) {\n p = owner[k]\n tot[p] += stmts[k]\n if (count[k] > 0) cov[p] += stmts[k]\n }\n gt = 0; gc = 0; xt = 0; xc = 0\n for (p in tot) {\n printf \"%s %.1f\\n\", p, cov[p] * 100 / tot[p]\n if (p ~ /\\/pkg\\//) {\n gt += tot[p]; gc += cov[p]\n } else if (p ~ /\\/crux(\\/|$)/ && p !~ /\\/cmd\\//) {\n xt += tot[p]; xc += cov[p]\n }\n }\n if (gt > 0) printf \"TOTAL %.1f\\n\", gc * 100 / gt\n if (xt > 0) printf \"CRUX_TOTAL %.1f\\n\", xc * 100 / xt\n }\n' coverage-*.out > pkg-pcts.txt\n\nTOTAL=$(awk '$1 == \"TOTAL\" {print $2 \"%\"}' pkg-pcts.txt)\nCRUX_TOTAL=$(awk '$1 == \"CRUX_TOTAL\" {print $2 \"%\"}' pkg-pcts.txt)\necho \"Total coverage (./pkg/...): ${TOTAL:-unavailable}\"\necho \"Total coverage (./crux/...): ${CRUX_TOTAL:-unavailable}\"\ + \n\n# Partial selection (#12454): every number here covers the selected\n# packages only. Per-package floors still apply unchanged to each of\n# them; the total is informational and not comparable to the\n# repo-wide figure, so it is labelled rather than compared.\nPARTIAL=false\nSELECTED_SCORED=\"\"\nif [ \"$SELECTION\" = partial ]; then\n PARTIAL=true\n for p in $SELECTED; do\n case \"$p\" in */pkg/*|*/cmd/*|*/crux|*/crux/*) ;; *) continue ;; esac\n grep -Fxq \"$p\" ../existing-pkgs.txt && SELECTED_SCORED=\"$SELECTED_SCORED $p\"\n done\n echo \"Partial selection: scoring only the $(wc -w <<<\"$SELECTED_SCORED\") selected ./pkg/..., ./cmd/... and ./crux/... packages; the totals above are over the selected packages and are not comparable to the repo-wide totals.\"\n if [ -z \"$TOTAL\" ] && ! grep -q '/pkg/' <<<\"$SELECTED_SCORED\"; then\n TOTAL=\"n/a (no ./pkg/... package selected)\"\n fi\nfi\n\n# Fail closed. Every step below is a filter, and an empty filter\n# result looks exactly like \"nothing was wrong\". This gate silently\n# scored zero packages for the whole window in which it still matched\n# the pre-rename module path github.com/kubestellar/hive, while the\n# module had become github.com/hivecommons/hive: TOTAL was\n# \"unavailable\", pkg-states.txt was empty, the scoring loop never ran\n# once, and the job reported success on every PR. A gate that cannot\n# see anything must fail, not pass.\nif [ -z \"$TOTAL\" ]; then\n echo \"COVERAGE GATE FAILED: no coverage data parsed from the shard artifacts.\"\n echo \"coverage-*.out was present but produced no ./pkg/... totals.\"\n exit 1\nfi\n# Same guard for the Crux module on a full run: its shards (the\n# agent slices and the rest buckets holding ./crux/...) must have\n# produced profiles, or the module's floors are vacuously met.\nif [ \"$SELECTION\" != partial ] && [ -z \"$CRUX_TOTAL\" ]; then\n echo \"COVERAGE GATE FAILED: no coverage data parsed for the Crux module (./crux/...).\"\n exit 1\nfi\n\n# Classify every package exactly once from the combined logs. With\n# gotestsum reruns, a package can have an initial FAIL line followed\n# by a final ok line in the same successful shard; score the final\n# successful rerun rather than treating the surfaced flake as a\n# coverage-blocking broken package.\n# ./cmd/... is scored too (#4701): the shards have run cmd tests\n# since the partition change, but coverage floors previously applied\n# to ./pkg/... only, so a CLI package could lose all its tests\n# without the gate noticing. The TOTAL line remains ./pkg/...-only\n# for trend continuity.\ncat test-*.log > cover-run.log\nawk '\n NR == FNR {\n exists[$1] = 1\n next\n }\n $1 == \"ok\" || $1 == \"FAIL\" || $1 == \"?\" {\n if (!exists[$2]) next\n if ($2 ~ /\\/(pkg|cmd)\\// || $2 ~ /\\/crux(\\/|$)/) {\n seen[$2] = 1\n if ($1 == \"ok\") ok[$2] = 1\n else if ($1 == \"FAIL\") fail[$2] = 1\n else if ($1 == \"?\") no_tests[$2] = 1\n }\n next\n }\n /^\\t.*\\t\\tcoverage: 0\\.0% of statements$/ {\n if (exists[$1] && ($1 ~ /\\/(pkg|cmd)\\// || $1 ~ /\\/crux(\\/|$)/)) {\n seen[$1] = 1\n no_tests[$1] = 1\n }\n }\n END {\n for (pkg in seen) {\n if (ok[pkg]) print \"ok\", pkg\n else if (fail[pkg]) print \"FAIL\", pkg\n else if (no_tests[pkg]) print \"?\", pkg\n }\n }\n ' ../existing-pkgs.txt cover-run.log \\\n | LC_ALL=C sort -u > pkg-states.txt\n\n# Second fail-closed guard, for the same reason as the TOTAL one:\n# an empty pkg-states.txt means the scoring loop never executes and\n# every floor is vacuously satisfied.\nif [ ! -s pkg-states.txt ] && { [ \"$PARTIAL\" != true ] || [ -n \"$SELECTED_SCORED\" ]; }; then\n echo \"COVERAGE GATE FAILED: no package pass/fail states parsed from test-*.log.\"\n echo \"No package could be scored against its floor.\"\n exit 1\nfi\n\n# Partial selection: every selected package must have been run and\n# classified \u2014 a selected package missing here was never tested.\n# The no-test-files gate is taken from the whole tree (see \"List\n# packages present in checkout\") so an unselected untested package\n# still fails exactly as it would on a full run.\nif [ \"$PARTIAL\" = true ]; then\n MISSING=\"\"\n for p in $SELECTED_SCORED; do\n awk -v p=\"$p\" '$2 == p { f = 1 } END { exit !f }' pkg-states.txt || MISSING=\"$MISSING $p\"\n done\n if [ -n \"$MISSING\" ]; then\n echo \"COVERAGE GATE FAILED: selected package(s) produced no test result:$MISSING\"\n exit 1\n fi\n while read -r p; do\n [ -n \"$p\" ] || continue\n awk -v p=\"$p\" '$2 == p { f = 1 } END { exit !f }' pkg-states.txt || echo \"? $p\" >> pkg-states.txt\n done < ../no-test-pkgs.txt\nfi\n\n# Floor-map key: cmd packages get a \"cmd/\" prefix so cmd/hivectl can\n# never resolve to pkg/hivectl's floor; pkg packages are keyed by\n# their full path below pkg/ so a nested package can never alias the\n# top-level one sharing its name (v5 has both pkg/spoke and\n# pkg/hub/spoke); the nested Crux module's packages are keyed\n# \"crux/\" (the module root package is \"crux\") so crux/config\n# can never alias pkg/config. See coverage-hourly.yml for the full\n# rationale.\nfloor_key() {\n case \"$1\" in\n */cmd/*) echo \"cmd/${1##*/}\" ;;\n */pkg/*) echo \"${1##*/pkg/}\" ;;\n */crux/*) echo \"crux/${1##*/crux/}\" ;;\n */crux) echo \"crux\" ;;\n *) echo \"${1##*/}\" ;;\n esac\n}\n\nwhile read -r state pkgpath; do\n pkg=$(floor_key \"$pkgpath\")\n case \"$state\" in\n ok)\n # Package states are collapsed above, so an ok means either\n # the package passed immediately or gotestsum reran its flakes\n # to a successful final result.\n pct=$(awk -v p=\"$pkgpath\" '$1 == p {print $2}' pkg-pcts.txt)\n if [ -z \"$pct\" ]; then continue; fi\n int_pct=${pct%.*}\n pkg_threshold=${PKG_THRESHOLD[$pkg]:-$THRESHOLD}\n if [ \"$int_pct\" -lt \"$pkg_threshold\" ]; then\n FAILED=\"${FAILED}${pkg}: ${pct}% (floor ${pkg_threshold}%)\\n\"\n fi\n REPORT=\"${REPORT}${pkg}: ${pct}%\\n\"\n ;;\n FAIL)\n BROKEN=\"${BROKEN}${pkg}: TESTS FAILING\\n\"\n REPORT=\"${REPORT}${pkg}: FAIL\\n\"\n ;;\n \\?)\n case \" $UNTESTED_CMD_MAINS \" in\n *\" $pkg \"*)\n REPORT=\"${REPORT}${pkg}: no tests (exempt thin main)\\n\"\n ;;\n *)\n BROKEN=\"${BROKEN}${pkg}: NO TEST FILES\\n\"\n REPORT=\"${REPORT}${pkg}: no tests\\n\"\n ;;\n esac\n ;;\n esac\ndone < pkg-states.txt\n\n{\n echo \"## Per-package coverage\"\n echo \"\"\n echo \"Total (./pkg/...): ${TOTAL:-unavailable}\"\n echo \"Total (./crux/...): ${CRUX_TOTAL:-unavailable}\"\n echo \"\"\n echo -e \"${REPORT}\"\n} >> \"$GITHUB_STEP_SUMMARY\"\n\nif [ -n \"$BROKEN\" ] || [ -n \"$FAILED\" ]; then\n {\n [ -n \"$BROKEN\" ] && echo -e \"### Packages that cannot be scored\\n${BROKEN}\"\n [ -n \"$FAILED\" ] && echo -e \"### Packages below their floor\\n${FAILED}\"\n } >> \"$GITHUB_STEP_SUMMARY\"\n echo \"\"\n echo \"COVERAGE GATE FAILED:\"\n [ -n \"$BROKEN\" ] && echo -e \"${BROKEN}\"\n [ -n \"$FAILED\" ] && echo -e \"${FAILED}\"\n exit 1\nfi\necho \"All packages meet their coverage floor\" >> \"$GITHUB_STEP_SUMMARY\"\n" diff --git a/.github/workflows/v2-ci.yml b/.github/workflows/v2-ci.yml index e0afbfd617..b42d8910bb 100644 --- a/.github/workflows/v2-ci.yml +++ b/.github/workflows/v2-ci.yml @@ -146,6 +146,10 @@ jobs: python3 -m pip install --user -q --break-system-packages pyyaml \ || { curl -sS https://bootstrap.pypa.io/get-pip.py | python3 - --user -q --break-system-packages && python3 -m pip install --user -q --break-system-packages pyyaml; } + - name: Check readonly candidate CI generation + working-directory: . + run: python3 .github/scripts/generate-integration-ci.py --check + # Populate the shared node-local Go tool cache under the runner pods' # node-wide lock before setup-go, so parallel jobs on a cold cache (a # src/go.mod bump) stop clobbering each other's rm -rf + copy (#11175, @@ -906,6 +910,10 @@ jobs: working-directory: . run: node --test src/pkg/hivectl/commands/*.test.js + - name: Integration candidate materializer contracts + working-directory: . + run: node --test .github/scripts/integration-materialize.test.cjs + # bin/contributor-relay.js used to carry the source under a misleading # .sh name (kubestellar/hive#6429), which needed a copy trick for # node --check. It is a real .js file now; check it and its lib/ module diff --git a/changelog.d/added-integration-candidate-producer-trust.md b/changelog.d/added-integration-candidate-producer-trust.md index b8937aa5b8..1d2082918d 100644 --- a/changelog.d/added-integration-candidate-producer-trust.md +++ b/changelog.d/added-integration-candidate-producer-trust.md @@ -1 +1 @@ -- Prepare Integration-aware candidate CI with default-false reusable technical workflows, isolated read-only execution and job-bound producer identity verification; this source support does not activate candidate execution or replace required CI/review policy. +- Add a default-disabled trusted integration-candidate materializer, create-only initial-dispatch ownership and isolated job-bound CI receipt workflows derived from read-only validation profiles; no execution controller, runtime activation or required CI/review replacement is enabled. diff --git a/src/docs/integration-orchestration.md b/src/docs/integration-orchestration.md index d728234d85..15b90592af 100644 --- a/src/docs/integration-orchestration.md +++ b/src/docs/integration-orchestration.md @@ -260,10 +260,122 @@ exposes nonlinear machine/feedback tradeoffs; and [ICST 2026 PRediBatch](https://conf.researchr.org/details/icst-2026/icst-2026-industry/8/Predictive-Pull-Request-Batching-to-Accelerate-Merge-Pipelines-in-Continuous-Integrat) reports a BMW-data simulation, not Hive production results. These support checking bounded work-in-progress, anti-starvation and invalidation/failed-execution cost, -not skipping validation. This implementation supplies neither ML prioritization, -custom batching, speculative group construction, batch bisection nor another -merge authority. Native group observations only describe groups GitHub created. +not skipping validation. The bounded authoring/admission layer supplies neither ML prioritization, +custom batching, batch bisection nor another merge authority. Native group +observations only describe groups GitHub created. The separately opt-in +candidate producer source described below constructs exact speculative prefixes; +it is not an installed promotion controller. There is no automatic live pilot, adaptive concurrency growth, runner procurement or claim of 5,000 merged PRs/day. That is an empirical capacity target requiring observed quality and retained outcomes, not a code milestone. + +## Optional candidate producer protocol: source phase + +The reference source now includes a real preparation/validation workflow graph, +not just a green-status copier: + +- `integration-candidate-prepare.yml` accepts an explicitly installed + `hive.integration.prepare.v1` repository-dispatch transport, or an authorized + manual workflow dispatch. It only uses trusted default-branch control code. + The checked-in `.github/integration-producer-policy.json` is **disabled**, has + no authorized actor and has no enrolled repo/base. Installation is not + activation; existing authoring `enforce` does not enable a writer. +- The trusted materializer never checks out or executes candidate code. It + fetches exact objects, obtains the actual merge base for each prior synthetic + prefix and next head, uses `git merge-tree`, and creates two-parent commits + with deterministic metadata and actual bot DCO sign-off. It refuses changed + installed policy/harness paths, different heads, forks and moved starting + bases. Clean earlier prefixes are retained if a later prefix conflicts. +- Each prefix is published as a controlled tag using the trusted job's + `GITHUB_TOKEN`, then validation is dispatched immediately rather than waiting + for the tail. The tag is never force-updated; author and destination refs are + not changed. Default-ref dispatch does **not** execute candidate code. +- `integration-candidate.yml` checks the actual commit/tree/two-parent ordered + lineage in one bounded GraphQL query before assigning technical jobs. Its + three callee workflows are mechanically derived from the landed CI/test/ + security sources, with writer jobs absent, hosted runners and checkout + credentials explicitly disabled. Mutable runner/cache/selection variables + are fixed in the qualified candidate profile; test selection is disabled. +- An isolated finalizer, with no checkout or candidate execution, signs exact + receipt bytes using job-bound GitHub OIDC and publishes an immutable + attempt-specific artifact. Candidate validation and trusted materialization + use distinct audiences/ref/event profiles. A receipt authenticates provenance: + **its successful publication is not passing CI or promotion authorization**. + +Ordinary PR/push/native-group workflows retain their typed default-false path. +To update source workflow definitions, regenerate the call-only artifacts with +`python3 .github/scripts/generate-integration-ci.py`. Canonical CI, not a local +guard run, checks generation and the materializer/isolation contracts. This +avoids granting extra permissions to a virtual caller merely because skipped +legacy reporter/release jobs still declare writes. + +### Cross-hive ownership and explicit limits + +The producer uses an atomic create-only forge ref for each exact initial +candidate CI-dispatch key. The intent records the trusted workflow run/attempt +and a random invocation nonce before the dispatch RPC. A competing materializer +becomes an observer; unreadable or ambiguous intent becomes unknown. The +candidate preflight rechecks that exact ref/owner before technical execution. +Reservations expiring, a process restarting or an elapsed cooldown do not +authorize replacing this intent. There is no force update, TTL steal or automatic +failover. These create-only records are audit/attempt ownership, not permanent +source-file locks. + +This is narrowly **initial candidate validation**, not an implementation of +legacy PR CI rerun authority. The independent CI-reporting reconciler supplies +verified head/run/attempt/job observations and `owner-action-required`; an +observation is not mutation permission. Neither the local journal/flock, +post-effect comments nor destination CAS coordinates independent hive storage. +No second legacy rerun owner is installed here. + +The initial supported execution model must explicitly enroll its designated +requesting principal and trusted workflow executor; other hives observe. +Namespace protections, all participating writer routing and credential custody +remain installation prerequisites. Cooperative create-only ownership does not +prevent an administrator deleting an intent or an unsupported writer ignoring +the protocol. GitHub's Actions dispatch/rerun RPC does not consume a caller +fencing/idempotency token: forge CAS alone cannot fence a paused RPC sender. +Ambiguous effects must be reconciled, not automatically reassigned after a TTL. +Bootstrap repository-dispatch/control-job overhead is real execution cost; +duplicate event delivery is not claimed to be free or exactly-once. + +[GitHub documents installation-token Contents:write for repository dispatch](https://docs.github.com/en/rest/repos/repos#create-a-repository-dispatch-event) +and [its trusted default-branch SHA/ref](https://docs.github.com/en/actions/reference/workflows-and-actions/events-that-trigger-workflows#repository_dispatch). +This is an explicitly selected transport, **not** fallback after an Actions +permission error. No App Actions-write grant is requested. Only the trusted +materializer job has the minimum job-scoped Contents/Actions writes needed to +publish controlled refs and dispatch validation; technical jobs are read-only, +and only isolated finalizers have OIDC permission. + +### Remaining readiness gates + +Source contract tests are not isolated provider execution proof. The actual +artifact/run/job consumer, durable verification/key history, execution +controller and shared promotion authority handover are still required. Existing +eleven protected-branch contexts do not automatically become satisfied by +prefixed reusable jobs. No copied DCO/review greens, branch-rule bypass, +signature/linear-history downgrade or unproved conditional no-op is permitted. +Protection-compatible exact-commit promotion must be verified using the actual +designated actor and equivalent gates in an isolated installation first. + +No producer, execution mode, rules, permissions, repository variable, image or +runtime has been activated by this source work. Source SHA, image publication, +deployed runtime, desired mode and effective writer are distinct facts. + +| Metric | Current claim | +| --- | --- | +| Eligible-to-start / CI-to-promotion / end-to-end p50 and p95 | Unknown: no engine benchmark | +| Successful integrations/day attributable to this engine | Unknown: execution is OFF | +| Runner-minutes/success, canceled/rebuild waste, failure recovery | Unknown: actual job accounting/controlled comparison required | +| Global merges observed by the overnight collector | Existing-flow baseline, not autonomous attribution to this feature | + +### Decision-to-evidence-to-test mapping + +| Decision | Source/result and limitation | Required check | +| --- | --- | --- | +| Parallel exact prefixes, not textual independence | [Full FSE 2023 paper](https://arxiv.org/html/2308.13129): historical simulation, no merge conflicts/rerun testing, compile time excluded; not Hive production proof | Exact ordered parents/trees; actual conflict handling; publication precedes later materialization | +| No fixed batch-fill delay, ML or speculative proof transfer | PRediBatch primary abstract reports simulation; batching saves compute only under workload/recovery assumptions | Identical workload/resource/policy comparison; record discarded execution and rebuild cost | +| Preserve documented promoted prefixes; invalidate changed dependent subjects | [Zuul speculative execution](https://zuul-ci.org/docs/zuul/latest/gating.html) and GitHub merge-group semantics; provider behavior is not Hive parity | Prefix identity stable across observer/lease generations; never recast A+B proof as B-only | +| Fixed bounded admission before adaptation | Little's-law capacity accounting describes average WIP, not tail latency or fairness | Assigned-job wall minutes, shared-job dedup, unknown completeness, aging/fairness and explicit limits | +| Trusted control, isolated technical jobs/finalizers | GitHub trigger/cache/OIDC contracts; provider receipts and protections still need isolated verification | No candidate checkout in privileged/finalizer jobs; exact job-bound identities, no skipped writer permissions, ambiguous claims never retried blindly | diff --git a/src/pkg/github/integration_producer_token.go b/src/pkg/github/integration_producer_token.go index 49df62c214..232f315a50 100644 --- a/src/pkg/github/integration_producer_token.go +++ b/src/pkg/github/integration_producer_token.go @@ -14,14 +14,16 @@ import ( ) const ( - integrationProducerIssuer = "https://token.actions.githubusercontent.com" - integrationProducerAudience = "hive-integration-producer-v1/" - integrationProducerRefPrefix = "refs/tags/hive-integration/" - integrationProducerMaxToken = 16 << 10 - integrationProducerMaxPayload = 256 << 10 - integrationProducerClockSkew = 2 * time.Minute - integrationProducerMaxLifetime = time.Hour - integrationProducerDigestBytes = 64 + integrationProducerIssuer = "https://token.actions.githubusercontent.com" + integrationProducerAudience = "hive-integration-producer-v1/" + integrationMaterializerAudience = "hive-integration-materializer-v1/" + integrationProducerRefPrefix = "refs/tags/hive-integration/" + integrationProducerMaxToken = 16 << 10 + integrationProducerMaxPayload = 256 << 10 + integrationProducerClockSkew = 2 * time.Minute + integrationProducerMaxLifetime = time.Hour + integrationProducerDigestBytes = 64 + integrationProducerMaxDefaultBranch = 255 ) // IntegrationProducerBinding must come from the enrolled policy, exact @@ -45,6 +47,7 @@ type IntegrationProducerBinding struct { // IntegrationProducerIdentity authenticates a manifest's producer only. It is // not passing CI, current eligibility, promotion permission or a capacity grant. type IntegrationProducerIdentity struct { + Role string Issuer string KeyID string IssuedAt time.Time @@ -54,6 +57,19 @@ type IntegrationProducerIdentity struct { PayloadSHA256 string } +// IntegrationMaterializerBinding describes the independently observed trusted +// default-branch finalizer. SubjectID is the exact preparation request digest. +// EventName is the explicitly installed transport, never an API-error fallback. +type IntegrationMaterializerBinding struct { + IntegrationProducerBinding + DefaultBranch string + EventName string +} + +type integrationProducerProfile struct { + role, audience, ref, refType, event string +} + type integrationProducerClaims struct { jwt.RegisteredClaims Repository string `json:"repository"` @@ -110,6 +126,35 @@ func integrationProducerID(raw json.RawMessage, expected int64) bool { // inside the independently observed finalizer interval. Live eligibility and // evidence freshness must still be checked separately before any promotion. func (v *IntegrationProducerVerifier) Verify(ctx context.Context, raw string, payload []byte, binding IntegrationProducerBinding) (IntegrationProducerIdentity, error) { + return v.verify(ctx, raw, payload, binding, integrationProducerProfile{ + role: "candidate", audience: integrationProducerAudience, + ref: integrationProducerRefPrefix + binding.SubjectID, refType: "tag", event: "workflow_dispatch", + }) +} + +// VerifyMaterializer authenticates a trusted materialization receipt, not +// candidate CI success, shared mutation ownership or promotion permission. +func (v *IntegrationProducerVerifier) VerifyMaterializer(ctx context.Context, raw string, payload []byte, binding IntegrationMaterializerBinding) (IntegrationProducerIdentity, error) { + branch := binding.DefaultBranch + if branch == "" || branch == "@" || len(branch) > integrationProducerMaxDefaultBranch || strings.Contains(branch, "..") || + strings.Contains(branch, "@{") || strings.ContainsAny(branch, "\x00\x7f ~^:?*[\\\t\r\n") { + return IntegrationProducerIdentity{}, fmt.Errorf("authoritative materializer default branch unavailable") + } + for _, part := range strings.Split(branch, "/") { + if part == "" || strings.HasPrefix(part, ".") || strings.HasSuffix(part, ".lock") { + return IntegrationProducerIdentity{}, fmt.Errorf("authoritative materializer default branch invalid") + } + } + if binding.EventName != "workflow_dispatch" && binding.EventName != "repository_dispatch" { + return IntegrationProducerIdentity{}, fmt.Errorf("materializer transport is not explicitly supported") + } + return v.verify(ctx, raw, payload, binding.IntegrationProducerBinding, integrationProducerProfile{ + role: "materializer", audience: integrationMaterializerAudience, + ref: "refs/heads/" + branch, refType: "branch", event: binding.EventName, + }) +} + +func (v *IntegrationProducerVerifier) verify(ctx context.Context, raw string, payload []byte, binding IntegrationProducerBinding, profile integrationProducerProfile) (IntegrationProducerIdentity, error) { if err := binding.validate(); err != nil { return IntegrationProducerIdentity{}, err } @@ -139,16 +184,15 @@ func (v *IntegrationProducerVerifier) Verify(ctx context.Context, raw string, pa if token == nil || !token.Valid { return IntegrationProducerIdentity{}, fmt.Errorf("integration producer signature unavailable") } - ref := integrationProducerRefPrefix + binding.SubjectID if claims.Issuer != integrationProducerIssuer || len(claims.Audience) != 1 || - claims.Audience[0] != integrationProducerAudience+binding.PayloadSHA256 || + claims.Audience[0] != profile.audience+binding.PayloadSHA256 || claims.Repository != binding.Repo || !integrationProducerID(claims.RepositoryID, binding.RepositoryID) || !integrationProducerID(claims.ActorID, binding.ActorID) || !integrationProducerID(claims.RunID, binding.RunID) || !integrationProducerID(claims.RunAttempt, binding.RunAttempt) || !integrationProducerID(claims.CheckRunID, binding.FinalizerCheckRunID) || - claims.WorkflowRef != binding.Repo+"/"+binding.WorkflowPath+"@"+ref || - claims.WorkflowSHA != binding.WorkflowSHA || claims.Ref != ref || claims.RefType != "tag" || - claims.EventName != "workflow_dispatch" || claims.RunnerEnvironment != "github-hosted" || claims.Environment != "" { + claims.WorkflowRef != binding.Repo+"/"+binding.WorkflowPath+"@"+profile.ref || + claims.WorkflowSHA != binding.WorkflowSHA || claims.Ref != profile.ref || claims.RefType != profile.refType || + claims.EventName != profile.event || claims.RunnerEnvironment != "github-hosted" || claims.Environment != "" { return IntegrationProducerIdentity{}, fmt.Errorf("integration producer identity does not match the authoritative candidate finalizer") } if claims.IssuedAt == nil || claims.NotBefore == nil || claims.ExpiresAt == nil || @@ -161,6 +205,7 @@ func (v *IntegrationProducerVerifier) Verify(ctx context.Context, raw string, pa return IntegrationProducerIdentity{}, fmt.Errorf("integration producer issuance is outside the authoritative finalizer interval") } return IntegrationProducerIdentity{ + Role: profile.role, Issuer: claims.Issuer, KeyID: keyID, IssuedAt: claims.IssuedAt.Time, RunID: binding.RunID, RunAttempt: binding.RunAttempt, FinalizerCheckRunID: binding.FinalizerCheckRunID, PayloadSHA256: binding.PayloadSHA256, diff --git a/src/pkg/github/integration_producer_token_test.go b/src/pkg/github/integration_producer_token_test.go index 9e6887fd09..dd018656df 100644 --- a/src/pkg/github/integration_producer_token_test.go +++ b/src/pkg/github/integration_producer_token_test.go @@ -39,7 +39,7 @@ func producerIdentityFixture(t *testing.T) (*rsa.PrivateKey, IntegrationProducer issued := time.Date(2026, time.October, 1, 12, 0, 0, 0, time.UTC) binding := IntegrationProducerBinding{ Repo: "owner/repo", RepositoryID: 1, ActorID: 2, RunID: 3, RunAttempt: 1, FinalizerCheckRunID: 4, - WorkflowPath: ".github/workflows/integration-candidate-validation.yml", + WorkflowPath: ".github/workflows/integration-candidate.yml", WorkflowSHA: strings.Repeat("a", 40), SubjectID: strings.Repeat("b", integrationProducerDigestBytes), PayloadSHA256: fmt.Sprintf("%x", sha256.Sum256([]byte(producerIdentityPayload))), JobStartedAt: issued.Add(-time.Minute), JobCompletedAt: issued.Add(time.Minute), ObservedAt: issued.Add(time.Hour), @@ -79,6 +79,42 @@ func TestIntegrationProducerAuthenticatesExactHistoricalFinalizer(t *testing.T) } } +func TestIntegrationMaterializerHasDistinctRoleRefAndInstalledTransport(t *testing.T) { + key, candidate, claims := producerIdentityFixture(t) + candidate.WorkflowPath = ".github/workflows/integration-candidate-prepare.yml" + binding := IntegrationMaterializerBinding{ + IntegrationProducerBinding: candidate, DefaultBranch: "v5", EventName: "repository_dispatch", + } + claims["aud"] = integrationMaterializerAudience + binding.PayloadSHA256 + claims["ref"], claims["ref_type"] = "refs/heads/v5", "branch" + claims["sub"] = "repo:owner/repo:ref:refs/heads/v5" + claims["workflow_ref"] = binding.Repo + "/" + binding.WorkflowPath + "@refs/heads/v5" + verifier := NewIntegrationProducerVerifier() + verifier.keys, verifier.refreshedAt = map[string]*rsa.PublicKey{"github-key": &key.PublicKey}, time.Now() + for _, event := range []string{"workflow_dispatch", "repository_dispatch"} { + binding.EventName, claims["event_name"] = event, event + raw := signProducerToken(t, key, claims) + got, err := verifier.VerifyMaterializer(context.Background(), raw, []byte(producerIdentityPayload), binding) + if err != nil || got.Role != "materializer" { + t.Fatalf("installed %s materializer: %+v %v", event, got, err) + } + if _, err := verifier.Verify(context.Background(), raw, []byte(producerIdentityPayload), candidate); err == nil { + t.Fatal("materialization identity became candidate CI identity") + } + } + for _, bad := range []IntegrationMaterializerBinding{ + {IntegrationProducerBinding: candidate, DefaultBranch: "v5", EventName: "pull_request"}, + {IntegrationProducerBinding: candidate, DefaultBranch: "../v5", EventName: "repository_dispatch"}, + {IntegrationProducerBinding: candidate, DefaultBranch: "v6", EventName: "repository_dispatch"}, + {IntegrationProducerBinding: candidate, DefaultBranch: "v5.lock", EventName: "repository_dispatch"}, + {IntegrationProducerBinding: candidate, DefaultBranch: "v5", EventName: "workflow_dispatch"}, + } { + if _, err := verifier.VerifyMaterializer(context.Background(), signProducerToken(t, key, claims), []byte(producerIdentityPayload), bad); err == nil { + t.Fatalf("unsupported/ref-mismatched materializer accepted: %+v", bad) + } + } +} + func TestIntegrationProducerRejectsCrossSubjectRunAndJobClaims(t *testing.T) { key, binding, original := producerIdentityFixture(t) verifier := NewIntegrationProducerVerifier() diff --git a/src/pkg/github/integration_producer_workflows_test.go b/src/pkg/github/integration_producer_workflows_test.go index 3de3e4088c..7c69205e26 100644 --- a/src/pkg/github/integration_producer_workflows_test.go +++ b/src/pkg/github/integration_producer_workflows_test.go @@ -40,11 +40,71 @@ type integrationWorkflowContract struct { } `yaml:"jobs"` } -func TestIntegrationProducerWorkflowIsolationAndLegacySeams(t *testing.T) { +func integrationWorkflowSourcePath(t *testing.T) string { + t.Helper() _, current, _, ok := runtime.Caller(0) if !ok { t.Fatal("repository source path unavailable") } + return current +} + +func TestIntegrationProducerGeneratedReadOnlyEntrypoints(t *testing.T) { + _, current, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("repository source path unavailable") + } + root := filepath.Clean(filepath.Join(filepath.Dir(current), "../../..", ".github/workflows")) + for _, file := range []string{ + "integration-technical-v2-ci.yml", + "integration-technical-v2-tests.yml", + "integration-technical-go-security-analysis.yml", + } { + t.Run(file, func(t *testing.T) { + body, err := os.ReadFile(filepath.Join(root, file)) + if err != nil { + t.Fatal(err) + } + var workflow integrationWorkflowContract + var document map[string]yaml.Node + if err := yaml.Unmarshal(body, &workflow); err != nil { + t.Fatal(err) + } + if err := yaml.Unmarshal(body, &document); err != nil { + t.Fatal(err) + } + on := document["on"] + if on.Kind != yaml.MappingNode || len(on.Content) != 2 || on.Content[0].Value != "workflow_call" { + t.Fatal("candidate-only callee acquired automatic PR/push/group/schedule triggers") + } + if len(workflow.Permissions) != 1 || workflow.Permissions["contents"] != "read" || + len(workflow.Jobs) == 0 || workflow.Concurrency.Cancel != "false" { + t.Fatal("candidate-only callee lacks a noncanceling readonly graph") + } + if strings.Contains(string(body), "inputs.integration_candidate") || strings.Contains(string(body), "vars.") { + t.Fatal("candidate-only profile is mutable or can be switched to legacy behavior") + } + for name, job := range workflow.Jobs { + if job.RunsOn != "ubuntu-latest" { + t.Errorf("job %s can acquire an operator/self-hosted runner", name) + } + for _, permission := range job.Permissions { + if permission != "read" && permission != "none" { + t.Errorf("job %s still declares writer authority (even if skipped)", name) + } + } + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/checkout@") && + (step.With["persist-credentials"] != "false" || step.With["ref"] != "") { + t.Errorf("job %s does not isolate checkout credentials/exact event commit", name) + } + } + } + }) + } +} +func TestIntegrationProducerWorkflowIsolationAndLegacySeams(t *testing.T) { + current := integrationWorkflowSourcePath(t) root := filepath.Clean(filepath.Join(filepath.Dir(current), "../../..")) for _, file := range []struct { name string From 13d23158c89469f21bc15b453185af0fc5390e8f Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Fri, 30 Oct 2026 23:05:51 -0400 Subject: [PATCH 3/4] =?UTF-8?q?=E2=9C=A8=20feat:=20observe=20exact=20candi?= =?UTF-8?q?date=20and=20materializer=20run=20artifact=20provenance?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- .../integration-candidate-prepare.yml | 1 + .github/workflows/integration-candidate.yml | 1 + src/docs/integration-orchestration.md | 12 +- src/pkg/forgestate/integration_candidate.go | 90 ++++++ .../forgestate/integration_candidate_test.go | 53 ++++ .../github/integration_candidate_objects.go | 136 +++++++++ .../integration_candidate_observation.go | 278 ++++++++++++++++++ .../integration_candidate_observation_test.go | 250 ++++++++++++++++ .../integration_materializer_observation.go | 252 ++++++++++++++++ .../github/integration_producer_archive.go | 105 +++++++ src/pkg/github/integration_producer_token.go | 19 +- .../github/integration_producer_token_test.go | 4 +- 12 files changed, 1195 insertions(+), 6 deletions(-) create mode 100644 src/pkg/forgestate/integration_candidate.go create mode 100644 src/pkg/forgestate/integration_candidate_test.go create mode 100644 src/pkg/github/integration_candidate_objects.go create mode 100644 src/pkg/github/integration_candidate_observation.go create mode 100644 src/pkg/github/integration_candidate_observation_test.go create mode 100644 src/pkg/github/integration_materializer_observation.go create mode 100644 src/pkg/github/integration_producer_archive.go diff --git a/.github/workflows/integration-candidate-prepare.yml b/.github/workflows/integration-candidate-prepare.yml index 25c700bf85..2de0f77302 100644 --- a/.github/workflows/integration-candidate-prepare.yml +++ b/.github/workflows/integration-candidate-prepare.yml @@ -95,5 +95,6 @@ jobs: with: name: integration-materializer-receipt-${{ github.run_attempt }} path: .integration-producer/materialization.json + include-hidden-files: true if-no-files-found: error retention-days: 30 diff --git a/.github/workflows/integration-candidate.yml b/.github/workflows/integration-candidate.yml index cd61280736..d2e19050e7 100644 --- a/.github/workflows/integration-candidate.yml +++ b/.github/workflows/integration-candidate.yml @@ -185,5 +185,6 @@ jobs: with: name: integration-producer-receipt-${{ github.run_attempt }} path: .integration-producer/receipt.json + include-hidden-files: true if-no-files-found: error retention-days: 30 diff --git a/src/docs/integration-orchestration.md b/src/docs/integration-orchestration.md index 15b90592af..c6e989d11f 100644 --- a/src/docs/integration-orchestration.md +++ b/src/docs/integration-orchestration.md @@ -350,9 +350,15 @@ and only isolated finalizers have OIDC permission. ### Remaining readiness gates -Source contract tests are not isolated provider execution proof. The actual -artifact/run/job consumer, durable verification/key history, execution -controller and shared promotion authority handover are still required. Existing +Source contract tests are not isolated provider execution proof. Read-only +candidate and materializer observers now bind authoritative run/attempt/job/check +metadata, immutable artifact digest, exact signed bytes and enrolled ordered +commit/tree/workflow closure. Their source tests include changed attempts, +partial publication and mismatched producers/subjects; they do not prove an +installed provider execution. Actual job observations include materialization +and finalizer overhead, not just local recorder time. Durable verification/key +history, controller wiring and shared promotion authority handover are still +required. Existing eleven protected-branch contexts do not automatically become satisfied by prefixed reusable jobs. No copied DCO/review greens, branch-rule bypass, signature/linear-history downgrade or unproved conditional no-op is permitted. diff --git a/src/pkg/forgestate/integration_candidate.go b/src/pkg/forgestate/integration_candidate.go new file mode 100644 index 0000000000..e68674071f --- /dev/null +++ b/src/pkg/forgestate/integration_candidate.go @@ -0,0 +1,90 @@ +package forgestate + +import "time" + +// IntegrationCandidateSubject identifies one immutable, ordered tested prefix. +// Generation and producer/dispatch metadata do not change its immutable core. +type IntegrationCandidateSubject struct { + Schema int `json:"schema"` + Protocol string `json:"protocol"` + Engine string `json:"engine"` + Repo string `json:"repo"` + Base string `json:"base"` + BaseSHA string `json:"baseSHA"` + Policy string `json:"policy"` + WorkflowTree string `json:"workflowTree"` + Generation int64 `json:"generation"` + Members []IntegrationCandidateMember `json:"members"` + Prefixes []IntegrationCandidatePrefix `json:"prefixes"` + ID string `json:"id"` + PreparedBy IntegrationPreparationSource `json:"preparedBy"` + Claim *IntegrationDispatchClaim `json:"dispatchClaim,omitempty"` +} + +type IntegrationCandidateMember struct { + PR int `json:"pr"` + Head string `json:"head"` +} + +type IntegrationCandidatePrefix struct { + Commit string `json:"commit"` + Tree string `json:"tree"` +} + +type IntegrationPreparationSource struct { + SourceSHA string `json:"sourceSHA"` + RequestDigest string `json:"requestDigest"` + GitVersion string `json:"gitVersion"` +} + +type IntegrationDispatchClaim struct { + State string `json:"state"` + Key string `json:"key"` + Ref string `json:"ref"` + Commit string `json:"commit"` + Owner IntegrationDispatchOwner `json:"owner"` +} + +type IntegrationDispatchOwner struct { + RunID string `json:"runID"` + Attempt string `json:"attempt"` + Nonce string `json:"nonce"` +} + +// SameCore never transfers evidence between different base/member/parent/tree +// identities. Reobserving an identical prefix with a new lease is not a change. +func (s IntegrationCandidateSubject) SameCore(other IntegrationCandidateSubject) bool { + if s.Schema != other.Schema || s.Protocol != other.Protocol || s.Engine != other.Engine || + s.Repo != other.Repo || s.Base != other.Base || s.BaseSHA != other.BaseSHA || + s.Policy != other.Policy || s.WorkflowTree != other.WorkflowTree || s.ID != other.ID || + len(s.Members) != len(other.Members) || len(s.Prefixes) != len(other.Prefixes) { + return false + } + for i := range s.Members { + if s.Members[i] != other.Members[i] { + return false + } + } + for i := range s.Prefixes { + if s.Prefixes[i] != other.Prefixes[i] { + return false + } + } + return true +} + +// IntegrationRunnerJob is an actual assigned-runner observation, not an +// estimate from local execution or queued/run-duration timestamps. +type IntegrationRunnerJob struct { + Repo string `json:"repo"` + RunID int64 `json:"run_id"` + Attempt int `json:"attempt"` + JobID int64 `json:"job_id"` + RunnerID int64 `json:"runner_id"` + Name string `json:"name"` + Status string `json:"status"` + Conclusion string `json:"conclusion"` + StartedAt time.Time `json:"started_at"` + CompletedAt time.Time `json:"completed_at"` + ObservedAt time.Time `json:"observed_at"` +} diff --git a/src/pkg/forgestate/integration_candidate_test.go b/src/pkg/forgestate/integration_candidate_test.go new file mode 100644 index 0000000000..8bb07f8e87 --- /dev/null +++ b/src/pkg/forgestate/integration_candidate_test.go @@ -0,0 +1,53 @@ +package forgestate + +import "testing" + +func TestIntegrationCandidateCoreExcludesLeaseNotTestedIdentity(t *testing.T) { + first := IntegrationCandidateSubject{ + Schema: 1, Protocol: "integration-prefix-v1", Engine: "hive", Repo: "owner/repo", + Base: "v5", BaseSHA: "base", Policy: "policy", WorkflowTree: "workflow", ID: "subject", Generation: 1, + Members: []IntegrationCandidateMember{{PR: 1, Head: "A"}, {PR: 2, Head: "B"}}, + Prefixes: []IntegrationCandidatePrefix{{Commit: "base+A", Tree: "treeA"}, {Commit: "base+A+B", Tree: "treeAB"}}, + } + copySubject := func() IntegrationCandidateSubject { + out := first + out.Members = append([]IntegrationCandidateMember(nil), first.Members...) + out.Prefixes = append([]IntegrationCandidatePrefix(nil), first.Prefixes...) + return out + } + reobserved := copySubject() + reobserved.Generation++ + reobserved.PreparedBy = IntegrationPreparationSource{SourceSHA: "new-observer"} + reobserved.Claim = &IntegrationDispatchClaim{State: "observer"} + if !first.SameCore(reobserved) { + t.Fatal("an unchanged documented prefix lost identity after a lease/provenance observation") + } + for _, change := range []struct { + name string + mutate func(*IntegrationCandidateSubject) + }{ + {"schema", func(s *IntegrationCandidateSubject) { s.Schema++ }}, + {"protocol", func(s *IntegrationCandidateSubject) { s.Protocol += "-changed" }}, + {"engine", func(s *IntegrationCandidateSubject) { s.Engine = "native" }}, + {"repository", func(s *IntegrationCandidateSubject) { s.Repo = "other/repo" }}, + {"base-name", func(s *IntegrationCandidateSubject) { s.Base = "other" }}, + {"base-object", func(s *IntegrationCandidateSubject) { s.BaseSHA = "other-base" }}, + {"policy", func(s *IntegrationCandidateSubject) { s.Policy = "other-policy" }}, + {"producer-closure", func(s *IntegrationCandidateSubject) { s.WorkflowTree = "other-harness" }}, + {"subject", func(s *IntegrationCandidateSubject) { s.ID = "other-subject" }}, + {"member-head", func(s *IntegrationCandidateSubject) { s.Members[0].Head = "new-A" }}, + {"member-order", func(s *IntegrationCandidateSubject) { s.Members[0], s.Members[1] = s.Members[1], s.Members[0] }}, + {"tested-commit", func(s *IntegrationCandidateSubject) { s.Prefixes[1].Commit = "other-commit" }}, + {"tested-tree", func(s *IntegrationCandidateSubject) { s.Prefixes[1].Tree = "other-tree" }}, + {"remove-predecessor", func(s *IntegrationCandidateSubject) { s.Members = s.Members[1:]; s.Prefixes = s.Prefixes[1:] }}, + {"drop-prefix", func(s *IntegrationCandidateSubject) { s.Prefixes = s.Prefixes[:1] }}, + } { + t.Run(change.name, func(t *testing.T) { + changed := copySubject() + change.mutate(&changed) + if first.SameCore(changed) { + t.Fatal("different ordered tested identity reused evidence") + } + }) + } +} diff --git a/src/pkg/github/integration_candidate_objects.go b/src/pkg/github/integration_candidate_objects.go new file mode 100644 index 0000000000..9ee9447572 --- /dev/null +++ b/src/pkg/github/integration_candidate_objects.go @@ -0,0 +1,136 @@ +package github + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/forgestate" +) + +func (c *Client) verifyIntegrationProducerClosure(ctx context.Context, repo, commit, tree string) error { + parts := strings.Split(repo, "/") + var response struct { + Repository struct { + Name string `json:"nameWithOwner"` + Closure struct { + OID string `json:"oid"` + } `json:"closure"` + } `json:"repository"` + } + err := c.graphQL(ctx, `query($owner:String!,$name:String!,$closure:String!){ +repository(owner:$owner,name:$name){nameWithOwner closure:object(expression:$closure){... on Tree{oid}}}}`, + map[string]any{"owner": parts[0], "name": parts[1], "closure": commit + ":.github"}, &response) + if err != nil { + return err + } + if response.Repository.Name != repo || response.Repository.Closure.OID != tree { + return fmt.Errorf("actual producer source workflow closure differs from enrollment") + } + return nil +} + +func (c *Client) integrationCandidateArtifact(ctx context.Context, repo string, run *gh.WorkflowRun) (int64, string, error) { + return c.integrationProducerArtifact(ctx, repo, run, integrationReceiptArtifact) +} + +func (c *Client) integrationProducerArtifact(ctx context.Context, repo string, run *gh.WorkflowRun, artifactPrefix string) (int64, string, error) { + var listing struct { + TotalCount int `json:"total_count"` + Artifacts []struct { + ID int64 `json:"id"` + Name string `json:"name"` + Expired bool `json:"expired"` + Size int64 `json:"size_in_bytes"` + Digest string `json:"digest"` + Run struct { + ID int64 `json:"id"` + Head string `json:"head_sha"` + } `json:"workflow_run"` + } `json:"artifacts"` + } + path := fmt.Sprintf("repos/%s/actions/runs/%d/artifacts?per_page=%d", repo, run.GetID(), integrationGroupMaxEntries) + if err := c.integrationCandidateRead(ctx, path, &listing); err != nil { + return 0, "", err + } + if listing.TotalCount != len(listing.Artifacts) { + return 0, "", fmt.Errorf("artifact cohort incomplete") + } + expectedName := fmt.Sprintf("%s%d", artifactPrefix, run.GetRunAttempt()) + var id int64 + digest := "" + for _, artifact := range listing.Artifacts { + if artifact.Name != expectedName { + continue + } + if id != 0 || artifact.ID <= 0 || artifact.Expired || artifact.Size <= 0 || + artifact.Size > integrationArtifactMaxBytes || artifact.Run.ID != run.GetID() || + artifact.Run.Head != run.GetHeadSHA() { + return 0, "", fmt.Errorf("producer artifact identity expired, ambiguous or unbound") + } + id, digest = artifact.ID, artifact.Digest + } + if id == 0 { + return 0, "", fmt.Errorf("immutable attempt-specific receipt artifact unavailable") + } + return id, digest, nil +} + +func (c *Client) verifyIntegrationCandidateObjects(ctx context.Context, subject forgestate.IntegrationCandidateSubject) error { + parts := strings.Split(subject.Repo, "/") + last := subject.Prefixes[len(subject.Prefixes)-1] + declarations := []string{"$owner:String!", "$name:String!", "$closure:String!"} + fields := make([]string, 0, len(subject.Prefixes)) + variables := map[string]any{"owner": parts[0], "name": parts[1], "closure": last.Commit + ":.github"} + for i, prefix := range subject.Prefixes { + key := fmt.Sprintf("prefix%d", i) + declarations = append(declarations, "$"+key+":String!") + variables[key] = prefix.Commit + fields = append(fields, key+":object(expression:$"+key+"){... on Commit{oid tree{oid} parents(first:3){nodes{oid}}}}") + } + query := "query(" + strings.Join(declarations, ",") + "){repository(owner:$owner,name:$name){nameWithOwner closure:object(expression:$closure){... on Tree{oid}} " + strings.Join(fields, " ") + "}}" + var response struct { + Repository map[string]json.RawMessage `json:"repository"` + } + if err := c.graphQL(ctx, query, variables, &response); err != nil { + return err + } + var name string + var closure struct { + OID string `json:"oid"` + } + if err := json.Unmarshal(response.Repository["nameWithOwner"], &name); err != nil { + return err + } + if err := json.Unmarshal(response.Repository["closure"], &closure); err != nil { + return err + } + if name != subject.Repo || closure.OID != subject.WorkflowTree { + return fmt.Errorf("actual candidate repository or installed workflow closure changed") + } + parent := subject.BaseSHA + for i, expected := range subject.Prefixes { + var actual struct { + OID string `json:"oid"` + Tree struct { + OID string `json:"oid"` + } `json:"tree"` + Parents struct { + Nodes []struct { + OID string `json:"oid"` + } `json:"nodes"` + } `json:"parents"` + } + if err := json.Unmarshal(response.Repository[fmt.Sprintf("prefix%d", i)], &actual); err != nil { + return err + } + if actual.OID != expected.Commit || actual.Tree.OID != expected.Tree || len(actual.Parents.Nodes) != 2 || + actual.Parents.Nodes[0].OID != parent || actual.Parents.Nodes[1].OID != subject.Members[i].Head { + return fmt.Errorf("actual ordered candidate prefix %d commit/tree/parents changed", i+1) + } + parent = expected.Commit + } + return nil +} diff --git a/src/pkg/github/integration_candidate_observation.go b/src/pkg/github/integration_candidate_observation.go new file mode 100644 index 0000000000..20e1f18229 --- /dev/null +++ b/src/pkg/github/integration_candidate_observation.go @@ -0,0 +1,278 @@ +package github + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/forgestate" +) + +const ( + integrationCandidateWorkflow = ".github/workflows/integration-candidate.yml" + integrationCandidateFinalizer = "integration producer receipt" + integrationCandidateMaxMembers = 8 +) + +// IntegrationCandidateExpectation comes from enrollment and the durable +// materializer observation, never from assertions inside a downloaded artifact. +type IntegrationCandidateExpectation struct { + Subject forgestate.IntegrationCandidateSubject + RepositoryID, ActorID, WorkflowID, CheckAppID int64 + RunID int64 + Attempt int + RequiredJobs []string +} + +// IntegrationCandidateObservation keeps verified producer provenance separate +// from technical success and live promotion eligibility. +type IntegrationCandidateObservation struct { + SubjectID string + State, Detail string + ObservedAt time.Time + RunID int64 + Attempt int + WorkflowID, SuiteID, ArtifactID int64 + ArtifactDigest string + Identity IntegrationProducerIdentity + Payload []byte `json:"-"` + Token string `json:"-"` + CheckIDs []int64 + Jobs []forgestate.IntegrationRunnerJob + JobCompleteness string +} + +type integrationCandidateReceipt struct { + Schema int `json:"schema"` + Protocol string `json:"protocol"` + Subject forgestate.IntegrationCandidateSubject `json:"subject"` + RunID string `json:"runID"` + RunAttempt string `json:"runAttempt"` + Outcomes struct { + CI string `json:"ci"` + Tests string `json:"tests"` + Security string `json:"security"` + } `json:"outcomes"` +} + +func (e IntegrationCandidateExpectation) validate() error { + s := e.Subject + parts := strings.Split(s.Repo, "/") + if s.Schema != 1 || s.Protocol != "integration-prefix-v1" || s.Engine != "hive" || + len(parts) != 2 || parts[0] == "" || parts[1] == "" || + strings.ContainsAny(s.Repo, "?#%\\ \t\r\n") || s.Base == "" || + !integrationProducerDigest(s.ID) || !integrationProducerDigest(s.Policy) || + !integrationOID(s.BaseSHA) || !integrationOID(s.WorkflowTree) || + e.RepositoryID <= 0 || e.ActorID <= 0 || e.WorkflowID <= 0 || e.CheckAppID <= 0 || + e.RunID <= 0 || e.Attempt < 1 || len(s.Members) == 0 || + len(s.Members) > integrationCandidateMaxMembers || len(s.Prefixes) != len(s.Members) || + len(e.RequiredJobs) == 0 || len(e.RequiredJobs) >= integrationGroupMaxEntries { + return fmt.Errorf("incomplete enrolled candidate expectation") + } + members := make(map[int]bool) + heads := make(map[string]bool) + for i, member := range s.Members { + prefix := s.Prefixes[i] + if member.PR <= 0 || members[member.PR] || heads[member.Head] || + !integrationOID(member.Head) || !integrationOID(prefix.Commit) || !integrationOID(prefix.Tree) { + return fmt.Errorf("candidate ordered member/prefix identity invalid") + } + members[member.PR], heads[member.Head] = true, true + } + names := make(map[string]bool) + for _, name := range e.RequiredJobs { + if name == "" || names[name] || name == integrationCandidateFinalizer { + return fmt.Errorf("qualified required-job cohort is empty or ambiguous") + } + names[name] = true + } + return nil +} + +func (c *Client) integrationCandidateRead(ctx context.Context, path string, out any) error { + request, err := c.client.NewRequest("GET", path, nil) + if err != nil { + return err + } + response, err := c.client.Do(ctx, request, out) + if err != nil { + return err + } + if response != nil && response.NextPage != 0 { + return fmt.Errorf("candidate observation cohort truncated; completeness unknown") + } + return nil +} + +// ReadIntegrationCandidateObservation performs bounded cohort reads, not +// per-job polling. The controller must coalesce events/reconciliation by exact +// run/attempt and persist verified receipts before acknowledging observations. +func (c *Client) ReadIntegrationCandidateObservation(ctx context.Context, expected IntegrationCandidateExpectation, verifier *IntegrationProducerVerifier, now time.Time) (IntegrationCandidateObservation, error) { + out := IntegrationCandidateObservation{ + SubjectID: expected.Subject.ID, RunID: expected.RunID, Attempt: expected.Attempt, + State: "unknown", ObservedAt: now, JobCompleteness: "unknown", + } + if err := expected.validate(); err != nil { + return out, err + } + if c == nil || c.client == nil || now.IsZero() { + return out, fmt.Errorf("candidate observer unavailable") + } + repo := expected.Subject.Repo + var run gh.WorkflowRun + runPath := fmt.Sprintf("repos/%s/actions/runs/%d", repo, expected.RunID) + if err := c.integrationCandidateRead(ctx, runPath, &run); err != nil { + return out, err + } + last := expected.Subject.Prefixes[len(expected.Subject.Prefixes)-1] + if run.GetRunAttempt() != expected.Attempt { + out.State, out.Detail = "superseded", "authoritative latest workflow attempt changed" + return out, nil + } + if run.GetID() != expected.RunID || run.GetWorkflowID() != expected.WorkflowID || + run.GetCheckSuiteID() <= 0 || run.GetEvent() != "workflow_dispatch" || + run.GetPath() != integrationCandidateWorkflow || run.GetHeadSHA() != last.Commit || + run.GetActor().GetID() != expected.ActorID || + run.GetRepository().GetID() != expected.RepositoryID || run.GetRepository().GetFullName() != repo || + run.GetHeadRepository().GetFullName() != repo { + return out, fmt.Errorf("candidate workflow/actor/repository/head not bound to enrollment") + } + out.WorkflowID, out.SuiteID = run.GetWorkflowID(), run.GetCheckSuiteID() + var jobs struct { + TotalCount int `json:"total_count"` + Jobs []*gh.WorkflowJob `json:"jobs"` + } + path := fmt.Sprintf("%s/attempts/%d/jobs?per_page=%d", runPath, expected.Attempt, integrationGroupMaxEntries) + if err := c.integrationCandidateRead(ctx, path, &jobs); err != nil { + return out, err + } + if jobs.TotalCount == 0 || jobs.TotalCount != len(jobs.Jobs) { + return out, fmt.Errorf("job cohort incomplete") + } + for _, job := range jobs.Jobs { + if job == nil || job.GetID() <= 0 || job.GetRunID() != expected.RunID || job.GetHeadSHA() != last.Commit { + return out, fmt.Errorf("job observation not bound to candidate run/head") + } + out.Jobs = append(out.Jobs, forgestate.IntegrationRunnerJob{ + Repo: repo, RunID: expected.RunID, Attempt: expected.Attempt, + JobID: job.GetID(), RunnerID: job.GetRunnerID(), Name: job.GetName(), + Status: job.GetStatus(), Conclusion: job.GetConclusion(), + StartedAt: job.GetStartedAt().Time, CompletedAt: job.GetCompletedAt().Time, ObservedAt: now, + }) + } + out.JobCompleteness = "complete" + if run.GetStatus() != "completed" { + out.State, out.Detail = "waiting", "authoritative candidate workflow is waiting" + if run.GetStatus() == "in_progress" { + out.State, out.Detail = "running", "authoritative candidate workflow is running" + } + return out, nil + } + parts := strings.Split(repo, "/") + checks, response, err := c.client.Checks.ListCheckRunsForRef(ctx, parts[0], parts[1], last.Commit, + &gh.ListCheckRunsOptions{Filter: gh.String("all"), ListOptions: gh.ListOptions{PerPage: integrationGroupMaxEntries}}) + if err != nil { + return out, err + } + if checks == nil || response != nil && response.NextPage != 0 { + return out, fmt.Errorf("candidate check cohort truncated") + } + policy := config.IntegrationPolicy{Jobs: append(append([]string(nil), expected.RequiredJobs...), integrationCandidateFinalizer), AppID: expected.CheckAppID} + proof := IntegrationWorkflowProof{} + if err := bindIntegrationWorkflowJobs(policy, &run, jobs.Jobs, checks.CheckRuns, &proof); err != nil { + return out, err + } + var finalizer *gh.WorkflowJob + for _, job := range jobs.Jobs { + if job.GetName() != integrationCandidateFinalizer { + continue + } + if finalizer != nil { + return out, fmt.Errorf("ambiguous candidate receipt finalizer") + } + finalizer = job + } + if finalizer == nil || finalizer.GetStatus() != "completed" || finalizer.GetConclusion() != "success" { + return out, fmt.Errorf("isolated producer finalizer did not complete successfully") + } + finalizerProof := IntegrationWorkflowProof{} + if err := bindIntegrationWorkflowJobs(config.IntegrationPolicy{Jobs: []string{integrationCandidateFinalizer}, AppID: expected.CheckAppID}, + &run, jobs.Jobs, checks.CheckRuns, &finalizerProof); err != nil || len(finalizerProof.CheckIDs) != 1 { + return out, fmt.Errorf("finalizer's actual check identity unavailable: %v", err) + } + finalizerPassed := false + for _, check := range checks.CheckRuns { + if check.GetID() == finalizerProof.CheckIDs[0] { + finalizerPassed = check.GetStatus() == "completed" && check.GetConclusion() == "success" + } + } + if !finalizerPassed { + return out, fmt.Errorf("actual finalizer check did not succeed") + } + artifactID, artifactDigest, err := c.integrationCandidateArtifact(ctx, repo, &run) + if err != nil { + return out, err + } + archive, err := c.downloadIntegrationReceipt(ctx, repo, artifactID, artifactDigest) + if err != nil { + return out, err + } + payload, token, err := decodeIntegrationReceipt(archive) + if err != nil { + return out, err + } + var receipt integrationCandidateReceipt + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&receipt); err != nil { + return out, err + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return out, fmt.Errorf("candidate receipt has trailing data") + } + if receipt.Schema != 1 || receipt.Protocol != "integration-producer-receipt-v1" || + receipt.RunID != strconv.FormatInt(expected.RunID, 10) || + receipt.RunAttempt != strconv.Itoa(expected.Attempt) || !receipt.Subject.SameCore(expected.Subject) { + return out, fmt.Errorf("receipt does not bind the enrolled exact immutable subject and attempt") + } + digest := sha256.Sum256(payload) + out.Identity, err = verifier.Verify(ctx, token, payload, IntegrationProducerBinding{ + Repo: repo, RepositoryID: expected.RepositoryID, ActorID: expected.ActorID, + RunID: expected.RunID, RunAttempt: int64(expected.Attempt), FinalizerCheckRunID: finalizerProof.CheckIDs[0], + WorkflowPath: integrationCandidateWorkflow, WorkflowSHA: last.Commit, + SubjectID: expected.Subject.ID, PayloadSHA256: hex.EncodeToString(digest[:]), + JobStartedAt: finalizer.GetStartedAt().Time, JobCompletedAt: finalizer.GetCompletedAt().Time, ObservedAt: now, + }) + if err != nil { + return out, err + } + if err := c.verifyIntegrationCandidateObjects(ctx, expected.Subject); err != nil { + return out, err + } + var current gh.WorkflowRun + if err := c.integrationCandidateRead(ctx, runPath, ¤t); err != nil { + return out, err + } + if current.GetRunAttempt() != expected.Attempt || current.GetHeadSHA() != last.Commit || + current.GetStatus() != run.GetStatus() || current.GetConclusion() != run.GetConclusion() { + out.State, out.Detail = "superseded", "run changed during receipt verification" + return out, nil + } + out.ArtifactID, out.ArtifactDigest, out.Payload, out.Token = artifactID, artifactDigest, payload, token + out.CheckIDs = proof.CheckIDs + out.State, out.Detail = "rejected", "actual technical workflow/jobs rejected; provenance alone is not green" + if proof.Passed && receipt.Outcomes.CI == "success" && receipt.Outcomes.Tests == "success" && receipt.Outcomes.Security == "success" { + out.State, out.Detail = "passed", "exact candidate technical proof verified; author review and live promotion gates remain separate" + } + return out, nil +} diff --git a/src/pkg/github/integration_candidate_observation_test.go b/src/pkg/github/integration_candidate_observation_test.go new file mode 100644 index 0000000000..08802d9c56 --- /dev/null +++ b/src/pkg/github/integration_candidate_observation_test.go @@ -0,0 +1,250 @@ +package github + +import ( + "archive/zip" + "bytes" + "context" + "crypto/rsa" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/forgestate" +) + +func TestIntegrationCandidateObserverBindsActualRunJobCheckArtifactAndOrderedSubject(t *testing.T) { + for _, scenario := range []string{"passed", "rejected", "latest_attempt", "changed_during_read", "wrong_app", "wrong_tree", "missing_job", "digest", "unknown_job_cohort", "wrong_receipt_subject", "materialized", "materialized_partial", "materialized_wrong_closure"} { + t.Run(scenario, func(t *testing.T) { + key, binding, claims := producerIdentityFixture(t) + subject := forgestate.IntegrationCandidateSubject{ + Schema: 1, Protocol: "integration-prefix-v1", Engine: "hive", Repo: binding.Repo, + Base: "v5", BaseSHA: strings.Repeat("1", 40), Policy: strings.Repeat("c", 64), + WorkflowTree: strings.Repeat("d", 40), Generation: 1, ID: binding.SubjectID, + Members: []forgestate.IntegrationCandidateMember{{PR: 10, Head: strings.Repeat("2", 40)}}, + Prefixes: []forgestate.IntegrationCandidatePrefix{{Commit: binding.WorkflowSHA, Tree: strings.Repeat("3", 40)}}, + } + expected := IntegrationCandidateExpectation{Subject: subject, RepositoryID: 1, ActorID: 2, + WorkflowID: 7, CheckAppID: 9, RunID: 3, Attempt: 1, RequiredJobs: []string{"ci / build-and-test"}} + materializer := strings.HasPrefix(scenario, "materialized") + names := []string{"ci / build-and-test", integrationCandidateFinalizer} + workflowPath, event, artifactPrefix, filename := integrationCandidateWorkflow, "workflow_dispatch", integrationReceiptArtifact, integrationReceiptFile + preparation := IntegrationPreparationRequest{ + Schema: 1, Protocol: subject.Protocol, Engine: subject.Engine, Repo: subject.Repo, + Base: subject.Base, BaseSHA: subject.BaseSHA, Policy: subject.Policy, Generation: 1, + Members: subject.Members, CreatedAt: binding.JobStartedAt.Format("2006-01-02T15:04:05.000Z"), + } + receiptSubject := subject + if scenario == "wrong_receipt_subject" { + receiptSubject.BaseSHA = strings.Repeat("4", 40) + } + payload, err := json.Marshal(map[string]any{ + "schema": 1, "protocol": "integration-producer-receipt-v1", "subject": receiptSubject, + "runID": "3", "runAttempt": "1", "outcomes": map[string]string{"ci": "success", "tests": "success", "security": "success"}, + }) + if err != nil { + t.Fatal(err) + } + if materializer { + workflowPath, event = integrationMaterializerWorkflow, "repository_dispatch" + artifactPrefix, filename = integrationMaterializerArtifact, "materialization.json" + names = []string{integrationMaterializerJob, integrationMaterializerFinalizer} + source := forgestate.IntegrationPreparationSource{SourceSHA: binding.WorkflowSHA, RequestDigest: binding.SubjectID, GitVersion: "fixture"} + subject.PreparedBy = source + prepared := IntegrationPreparedSubject{Subject: subject, Publication: "accepted", Dispatch: "accepted"} + body := IntegrationMaterializationReceipt{Schema: 1, Protocol: "integration-materializer-receipt-v1", + Request: preparation, PreparedBy: source, Prepared: []IntegrationPreparedSubject{prepared}} + if scenario == "materialized_partial" { + body.Prepared[0].Publication, body.Prepared[0].Dispatch = "unresolved", "not_requested" + body.Failure = &IntegrationPreparationFailure{Reason: "provider_unknown"} + } + payload, err = json.Marshal(body) + if err != nil { + t.Fatal(err) + } + claims["workflow_ref"] = binding.Repo + "/" + workflowPath + "@refs/heads/v5" + claims["ref"], claims["ref_type"], claims["event_name"] = "refs/heads/v5", "branch", event + } + binding.PayloadSHA256 = fmt.Sprintf("%x", sha256.Sum256(payload)) + claims["aud"] = integrationProducerAudience + binding.PayloadSHA256 + if materializer { + claims["aud"] = integrationMaterializerAudience + binding.PayloadSHA256 + } + envelope, err := json.Marshal(integrationReceiptEnvelope{Schema: 1, + Payload: base64.StdEncoding.EncodeToString(payload), Token: signProducerToken(t, key, claims)}) + if err != nil { + t.Fatal(err) + } + var buffer bytes.Buffer + archive := zip.NewWriter(&buffer) + file, err := archive.Create(filename) + if err != nil { + t.Fatal(err) + } + if _, err := file.Write(envelope); err != nil { + t.Fatal(err) + } + if err := archive.Close(); err != nil { + t.Fatal(err) + } + digest := fmt.Sprintf("sha256:%x", sha256.Sum256(buffer.Bytes())) + if scenario == "digest" { + digest = "sha256:" + strings.Repeat("f", 64) + } + reads := 0 + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + emit := func(value any) { + t.Helper() + if err := json.NewEncoder(w).Encode(value); err != nil { + t.Error(err) + } + } + switch r.URL.Path { + case "/repos/owner/repo/actions/runs/3": + reads++ + attempt := 1 + if scenario == "latest_attempt" || scenario == "changed_during_read" && reads > 1 { + attempt = 2 + } + conclusion := "success" + if scenario == "rejected" { + conclusion = "failure" + } + emit(map[string]any{"id": 3, "run_attempt": attempt, "workflow_id": 7, "check_suite_id": 8, + "event": event, "path": workflowPath, + "head_sha": binding.WorkflowSHA, "actor": map[string]int{"id": 2}, + "repository": map[string]any{"id": 1, "full_name": "owner/repo"}, + "head_repository": map[string]string{"full_name": "owner/repo"}, + "status": "completed", "conclusion": conclusion}) + case "/repos/owner/repo/actions/runs/3/attempts/1/jobs": + jobs := []map[string]any{} + for i, name := range names { + if scenario == "missing_job" && i == 0 { + continue + } + checkID := 5 + if i == 1 { + checkID = 4 + } + result := "success" + if scenario == "rejected" && i == 0 { + result = "failure" + } + jobs = append(jobs, map[string]any{"id": 80 + i, "run_id": 3, "head_sha": binding.WorkflowSHA, + "name": name, "runner_id": 100 + i, "status": "completed", "conclusion": result, + "started_at": binding.JobStartedAt, "completed_at": binding.JobCompletedAt, + "check_run_url": fmt.Sprintf("%s/repos/owner/repo/check-runs/%d", server.URL, checkID)}) + } + count := len(jobs) + if scenario == "unknown_job_cohort" { + count++ + } + emit(map[string]any{"total_count": count, "jobs": jobs}) + case "/repos/owner/repo/commits/" + binding.WorkflowSHA + "/check-runs": + app := 9 + if scenario == "wrong_app" { + app = 10 + } + checks := []map[string]any{} + for i, name := range names { + id, result := 5, "success" + if i == 1 { + id = 4 + } + if scenario == "rejected" && i == 0 { + result = "failure" + } + checks = append(checks, map[string]any{"id": id, "name": name, "head_sha": binding.WorkflowSHA, + "status": "completed", "conclusion": result, "check_suite": map[string]int{"id": 8}, "app": map[string]int{"id": app}}) + } + emit(map[string]any{"total_count": len(checks), "check_runs": checks}) + case "/repos/owner/repo/actions/runs/3/artifacts": + emit(map[string]any{"total_count": 1, "artifacts": []any{map[string]any{"id": 10, + "name": artifactPrefix + "1", "size_in_bytes": buffer.Len(), "digest": digest, + "workflow_run": map[string]any{"id": 3, "head_sha": binding.WorkflowSHA}}}}) + case "/repos/owner/repo/actions/artifacts/10/zip": + w.Header().Set("Location", server.URL+"/asset") + w.WriteHeader(http.StatusFound) + case "/asset": + if r.Header.Get("Authorization") != "" { + t.Error("forge credential leaked to artifact storage") + } + if _, err := w.Write(buffer.Bytes()); err != nil { + t.Error(err) + } + case "/graphql": + tree := subject.Prefixes[0].Tree + closure := subject.WorkflowTree + if scenario == "materialized_wrong_closure" { + closure = strings.Repeat("e", 40) + } + if scenario == "wrong_tree" { + tree = strings.Repeat("4", 40) + } + emit(map[string]any{"data": map[string]any{"repository": map[string]any{ + "nameWithOwner": "owner/repo", "closure": map[string]string{"oid": closure}, + "prefix0": map[string]any{"oid": binding.WorkflowSHA, "tree": map[string]string{"oid": tree}, + "parents": map[string]any{"nodes": []any{map[string]string{"oid": subject.BaseSHA}, map[string]string{"oid": subject.Members[0].Head}}}}, + }}}) + default: + t.Errorf("unexpected provider request %s %s", r.Method, r.URL) + http.NotFound(w, r) + } + })) + defer server.Close() + client := gh.NewClient(server.Client()) + client.BaseURL, err = url.Parse(server.URL + "/") + if err != nil { + t.Fatal(err) + } + observer := &Client{client: client} + verifier := NewIntegrationProducerVerifier() + verifier.keys, verifier.refreshedAt = map[string]*rsa.PublicKey{"github-key": &key.PublicKey}, time.Now() + if materializer { + result, err := observer.ReadIntegrationMaterializerObservation(context.Background(), IntegrationMaterializerExpectation{ + Request: preparation, RequestDigest: binding.SubjectID, DefaultBranch: "v5", EventName: event, + WorkflowTree: subject.WorkflowTree, RepositoryID: 1, ActorID: 2, WorkflowID: 7, CheckAppID: 9, RunID: 3, Attempt: 1, + }, verifier, binding.ObservedAt) + if scenario == "materialized_wrong_closure" { + if err == nil || result.Evidence.State != "unknown" { + t.Fatalf("untrusted control closure accepted: %+v %v", result, err) + } + return + } + if err != nil || result.Evidence.State != "passed" || len(result.Receipt.Prepared) != 1 || + result.Evidence.Identity.Role != "materializer" || result.Evidence.Identity.FinalizerCheckRunID != 4 { + t.Fatalf("materialization provenance: %+v %v", result, err) + } + if scenario == "materialized_partial" && result.Receipt.Failure == nil { + t.Fatal("partial preparation lost") + } + return + } + result, err := observer.ReadIntegrationCandidateObservation(context.Background(), expected, verifier, binding.ObservedAt) + switch scenario { + case "passed", "rejected": + if err != nil || result.State != scenario || result.ArtifactID != 10 || + result.Identity.FinalizerCheckRunID != 4 || len(result.Jobs) != 2 || result.JobCompleteness != "complete" { + t.Fatalf("actual observation: %+v %v", result, err) + } + case "latest_attempt", "changed_during_read": + if err != nil || result.State != "superseded" { + t.Fatalf("attempt race: %+v %v", result, err) + } + default: + if err == nil || result.State != "unknown" { + t.Fatalf("unverified subject accepted: %+v %v", result, err) + } + } + }) + } +} diff --git a/src/pkg/github/integration_materializer_observation.go b/src/pkg/github/integration_materializer_observation.go new file mode 100644 index 0000000000..794954ec59 --- /dev/null +++ b/src/pkg/github/integration_materializer_observation.go @@ -0,0 +1,252 @@ +package github + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "reflect" + "strings" + "time" + + gh "github.com/google/go-github/v72/github" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/forgestate" +) + +const ( + integrationMaterializerWorkflow = ".github/workflows/integration-candidate-prepare.yml" + integrationMaterializerFinalizer = "integration materializer receipt" + integrationMaterializerJob = "trusted integration materializer" + integrationMaterializerArtifact = "integration-materializer-receipt-" +) + +// IntegrationPreparationRequest is versioned independently of the older +// single-PR direct attestation. Members are ordered and never inferred by paths. +type IntegrationPreparationRequest struct { + Schema int `json:"schema"` + Protocol string `json:"protocol"` + Engine string `json:"engine"` + Repo string `json:"repo"` + Base string `json:"base"` + BaseSHA string `json:"baseSHA"` + Policy string `json:"policy"` + Generation int64 `json:"generation"` + Members []forgestate.IntegrationCandidateMember `json:"members"` + CreatedAt string `json:"createdAt"` +} + +type IntegrationMaterializationReceipt struct { + Schema int `json:"schema"` + Protocol string `json:"protocol"` + Request IntegrationPreparationRequest `json:"request"` + Prepared []IntegrationPreparedSubject `json:"prepared"` + Failure *IntegrationPreparationFailure `json:"failure"` + PreparedBy forgestate.IntegrationPreparationSource `json:"preparedBy"` +} + +type IntegrationPreparedSubject struct { + Subject forgestate.IntegrationCandidateSubject `json:"subject"` + Publication string `json:"publication"` + Dispatch string `json:"dispatch"` + Ownership forgestate.IntegrationDispatchClaim `json:"ownership"` +} + +type IntegrationPreparationFailure struct { + Reason string `json:"reason"` + Message string `json:"message"` +} + +// IntegrationMaterializerExpectation is installed policy plus an exact durable +// bootstrap request. EventName selects a transport; an API error never does. +type IntegrationMaterializerExpectation struct { + Request IntegrationPreparationRequest + RequestDigest, DefaultBranch, EventName, WorkflowTree string + RepositoryID, ActorID, WorkflowID, CheckAppID, RunID int64 + Attempt int +} + +type IntegrationMaterializerObservation struct { + Receipt IntegrationMaterializationReceipt + Evidence IntegrationCandidateObservation +} + +func (c *Client) ReadIntegrationMaterializerObservation(ctx context.Context, expected IntegrationMaterializerExpectation, verifier *IntegrationProducerVerifier, now time.Time) (IntegrationMaterializerObservation, error) { + out := IntegrationMaterializerObservation{Evidence: IntegrationCandidateObservation{ + SubjectID: expected.RequestDigest, RunID: expected.RunID, Attempt: expected.Attempt, + State: "unknown", ObservedAt: now, JobCompleteness: "unknown", + }} + repo := expected.Request.Repo + parts := strings.Split(repo, "/") + if c == nil || c.client == nil || now.IsZero() || + expected.Request.Schema != 1 || expected.Request.Protocol != "integration-prefix-v1" || expected.Request.Engine != "hive" || + expected.Request.Generation < 1 || expected.Request.Base == "" || !integrationOID(expected.Request.BaseSHA) || + !integrationProducerDigest(expected.Request.Policy) || len(expected.Request.Members) == 0 || + len(expected.Request.Members) > integrationCandidateMaxMembers || + len(parts) != 2 || parts[0] == "" || parts[1] == "" || + strings.ContainsAny(repo, "?#%\\ \t\r\n") || expected.RepositoryID <= 0 || + expected.ActorID <= 0 || expected.WorkflowID <= 0 || expected.CheckAppID <= 0 || + expected.RunID <= 0 || expected.Attempt < 1 || !integrationProducerDigest(expected.RequestDigest) || + !integrationOID(expected.WorkflowTree) || expected.DefaultBranch == "" || + expected.EventName != "workflow_dispatch" && expected.EventName != "repository_dispatch" { + return out, fmt.Errorf("incomplete enrolled materializer observation") + } + runPath := fmt.Sprintf("repos/%s/actions/runs/%d", repo, expected.RunID) + var run gh.WorkflowRun + if err := c.integrationCandidateRead(ctx, runPath, &run); err != nil { + return out, err + } + if run.GetRunAttempt() != expected.Attempt { + out.Evidence.State, out.Evidence.Detail = "superseded", "materializer attempt changed" + return out, nil + } + if run.GetID() != expected.RunID || run.GetWorkflowID() != expected.WorkflowID || run.GetCheckSuiteID() <= 0 || + run.GetPath() != integrationMaterializerWorkflow || run.GetEvent() != expected.EventName || + run.GetRepository().GetID() != expected.RepositoryID || run.GetRepository().GetFullName() != repo || + run.GetHeadRepository().GetFullName() != repo || run.GetActor().GetID() != expected.ActorID || !integrationOID(run.GetHeadSHA()) { + return out, fmt.Errorf("materializer workflow/actor/source identity changed") + } + var jobs struct { + TotalCount int `json:"total_count"` + Jobs []*gh.WorkflowJob `json:"jobs"` + } + if err := c.integrationCandidateRead(ctx, fmt.Sprintf("%s/attempts/%d/jobs?per_page=%d", runPath, expected.Attempt, integrationGroupMaxEntries), &jobs); err != nil { + return out, err + } + if jobs.TotalCount == 0 || jobs.TotalCount != len(jobs.Jobs) { + return out, fmt.Errorf("materializer job cohort incomplete") + } + var finalizer *gh.WorkflowJob + for _, job := range jobs.Jobs { + if job == nil || job.GetID() <= 0 || job.GetRunID() != expected.RunID || job.GetHeadSHA() != run.GetHeadSHA() { + return out, fmt.Errorf("materializer job not bound to actual run/head") + } + out.Evidence.Jobs = append(out.Evidence.Jobs, forgestate.IntegrationRunnerJob{ + Repo: repo, RunID: expected.RunID, Attempt: expected.Attempt, JobID: job.GetID(), RunnerID: job.GetRunnerID(), + Name: job.GetName(), Status: job.GetStatus(), Conclusion: job.GetConclusion(), + StartedAt: job.GetStartedAt().Time, CompletedAt: job.GetCompletedAt().Time, ObservedAt: now, + }) + if job.GetName() == integrationMaterializerFinalizer { + if finalizer != nil { + return out, fmt.Errorf("ambiguous materializer finalizer") + } + finalizer = job + } + } + out.Evidence.JobCompleteness = "complete" + if run.GetStatus() != "completed" { + out.Evidence.State, out.Evidence.Detail = "waiting", "materializer waiting" + if run.GetStatus() == "in_progress" { + out.Evidence.State, out.Evidence.Detail = "running", "materializer running" + } + return out, nil + } + checks, response, err := c.client.Checks.ListCheckRunsForRef(ctx, parts[0], parts[1], run.GetHeadSHA(), + &gh.ListCheckRunsOptions{Filter: gh.String("all"), ListOptions: gh.ListOptions{PerPage: integrationGroupMaxEntries}}) + if err != nil { + return out, err + } + if checks == nil || response != nil && response.NextPage != 0 { + return out, fmt.Errorf("materializer check cohort truncated") + } + proof := IntegrationWorkflowProof{} + if err := bindIntegrationWorkflowJobs(config.IntegrationPolicy{Jobs: []string{integrationMaterializerJob, integrationMaterializerFinalizer}, AppID: expected.CheckAppID}, + &run, jobs.Jobs, checks.CheckRuns, &proof); err != nil { + return out, err + } + if !proof.Passed || finalizer == nil || len(proof.CheckIDs) != 2 { + return out, fmt.Errorf("trusted materializer/finalizer did not succeed") + } + id, digest, err := c.integrationProducerArtifact(ctx, repo, &run, integrationMaterializerArtifact) + if err != nil { + return out, err + } + archive, err := c.downloadIntegrationReceipt(ctx, repo, id, digest) + if err != nil { + return out, err + } + payload, token, err := decodeIntegrationEnvelope(archive, "materialization.json") + if err != nil { + return out, err + } + decoder := json.NewDecoder(bytes.NewReader(payload)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&out.Receipt); err != nil { + return out, err + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return out, fmt.Errorf("materializer receipt has trailing data") + } + if out.Receipt.Schema != 1 || out.Receipt.Protocol != "integration-materializer-receipt-v1" || + !reflect.DeepEqual(out.Receipt.Request, expected.Request) || + out.Receipt.PreparedBy.SourceSHA != run.GetHeadSHA() || out.Receipt.PreparedBy.RequestDigest != expected.RequestDigest || + len(out.Receipt.Prepared) > len(expected.Request.Members) { + return out, fmt.Errorf("materializer receipt differs from exact durable request") + } + payloadDigest := sha256.Sum256(payload) + out.Evidence.Identity, err = verifier.VerifyMaterializer(ctx, token, payload, IntegrationMaterializerBinding{ + IntegrationProducerBinding: IntegrationProducerBinding{ + Repo: repo, RepositoryID: expected.RepositoryID, ActorID: expected.ActorID, + RunID: expected.RunID, RunAttempt: int64(expected.Attempt), FinalizerCheckRunID: proof.CheckIDs[1], + WorkflowPath: integrationMaterializerWorkflow, WorkflowSHA: run.GetHeadSHA(), SubjectID: expected.RequestDigest, + PayloadSHA256: hex.EncodeToString(payloadDigest[:]), JobStartedAt: finalizer.GetStartedAt().Time, + JobCompletedAt: finalizer.GetCompletedAt().Time, ObservedAt: now, + }, + DefaultBranch: expected.DefaultBranch, EventName: expected.EventName, + }) + if err != nil { + return out, err + } + if err := c.verifyIntegrationProducerClosure(ctx, repo, run.GetHeadSHA(), expected.WorkflowTree); err != nil { + return out, err + } + var published *forgestate.IntegrationCandidateSubject + for i, prepared := range out.Receipt.Prepared { + s := prepared.Subject + if s.Schema != expected.Request.Schema || s.Protocol != expected.Request.Protocol || s.Engine != expected.Request.Engine || + s.Repo != repo || s.Base != expected.Request.Base || s.BaseSHA != expected.Request.BaseSHA || + s.Policy != expected.Request.Policy || s.WorkflowTree != expected.WorkflowTree || + s.Generation != expected.Request.Generation || !integrationProducerDigest(s.ID) || + !reflect.DeepEqual(s.Members, expected.Request.Members[:i+1]) || len(s.Prefixes) != i+1 { + return out, fmt.Errorf("materializer changed ordered prepared prefix %d", i+1) + } + if i > 0 && !reflect.DeepEqual(s.Prefixes[:i], out.Receipt.Prepared[i-1].Subject.Prefixes) { + return out, fmt.Errorf("materializer changed predecessor prefix identity") + } + if prepared.Publication != "accepted" && prepared.Publication != "unresolved" { + return out, fmt.Errorf("unknown prefix publication state") + } + switch prepared.Dispatch { + case "accepted", "unresolved", "observer", "not_requested": + default: + return out, fmt.Errorf("unknown prefix dispatch state") + } + if prepared.Publication == "accepted" { + copySubject := s + published = ©Subject + } + } + if published != nil { + if err := c.verifyIntegrationCandidateObjects(ctx, *published); err != nil { + return out, err + } + } + var current gh.WorkflowRun + if err := c.integrationCandidateRead(ctx, runPath, ¤t); err != nil { + return out, err + } + if current.GetRunAttempt() != expected.Attempt || current.GetHeadSHA() != run.GetHeadSHA() || + current.GetStatus() != "completed" || current.GetConclusion() != run.GetConclusion() { + out.Evidence.State, out.Evidence.Detail = "superseded", "materializer changed during verification" + return out, nil + } + out.Evidence.WorkflowID, out.Evidence.SuiteID = expected.WorkflowID, run.GetCheckSuiteID() + out.Evidence.ArtifactID, out.Evidence.ArtifactDigest = id, digest + out.Evidence.Payload, out.Evidence.Token, out.Evidence.CheckIDs = payload, token, proof.CheckIDs + out.Evidence.State, out.Evidence.Detail = "passed", "trusted materialization provenance verified; not candidate CI success or promotion permission" + return out, nil +} diff --git a/src/pkg/github/integration_producer_archive.go b/src/pkg/github/integration_producer_archive.go new file mode 100644 index 0000000000..e11c5d10f7 --- /dev/null +++ b/src/pkg/github/integration_producer_archive.go @@ -0,0 +1,105 @@ +package github + +import ( + "archive/zip" + "bytes" + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + integrationReceiptFile = "receipt.json" + integrationReceiptArtifact = "integration-producer-receipt-" + integrationReceiptEnvelopeLimit = 384 << 10 +) + +type integrationReceiptEnvelope struct { + Schema int `json:"schema"` + Payload string `json:"payload"` + Token string `json:"token"` +} + +func decodeIntegrationReceipt(archive []byte) ([]byte, string, error) { + return decodeIntegrationEnvelope(archive, integrationReceiptFile) +} + +func decodeIntegrationEnvelope(archive []byte, filename string) ([]byte, string, error) { + reader, err := zip.NewReader(bytes.NewReader(archive), int64(len(archive))) + if err != nil { + return nil, "", err + } + if len(reader.File) != 1 || reader.File[0].Name != filename || + reader.File[0].UncompressedSize64 > integrationReceiptEnvelopeLimit { + return nil, "", fmt.Errorf("producer artifact requires one bounded receipt.json") + } + file, err := reader.File[0].Open() + if err != nil { + return nil, "", err + } + defer file.Close() + var envelope integrationReceiptEnvelope + decoder := json.NewDecoder(io.LimitReader(file, integrationReceiptEnvelopeLimit+1)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&envelope); err != nil { + return nil, "", err + } + var trailing any + if err := decoder.Decode(&trailing); err != io.EOF { + return nil, "", fmt.Errorf("producer envelope has trailing content") + } + payload, err := base64.StdEncoding.Strict().DecodeString(envelope.Payload) + if err != nil || envelope.Schema != 1 || len(envelope.Token) == 0 || + len(envelope.Token) > integrationProducerMaxToken || len(payload) == 0 || + len(payload) > integrationProducerMaxPayload { + return nil, "", fmt.Errorf("producer envelope identity or payload bounds rejected") + } + return payload, envelope.Token, nil +} + +func (c *Client) downloadIntegrationReceipt(ctx context.Context, repo string, artifactID int64, digest string) ([]byte, error) { + if !strings.HasPrefix(digest, "sha256:") || !integrationProducerDigest(strings.TrimPrefix(digest, "sha256:")) { + return nil, fmt.Errorf("immutable artifact digest unavailable; completeness unknown") + } + parts := strings.Split(repo, "/") + location, _, err := c.client.Actions.DownloadArtifact(ctx, parts[0], parts[1], artifactID, 0) + if err != nil { + return nil, err + } + if location == nil || location.User != nil || + location.Scheme != "https" && !(c.client.BaseURL.Scheme == "http" && + location.Scheme == "http" && location.Host == c.client.BaseURL.Host) { + return nil, fmt.Errorf("producer artifact download requires HTTPS") + } + request, err := http.NewRequestWithContext(ctx, http.MethodGet, location.String(), nil) + if err != nil { + return nil, err + } + // Storage URLs never receive the authenticated forge/App transport. + client := &http.Client{Timeout: integrationArtifactTimeout, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + return http.ErrUseLastResponse + }} + response, err := client.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("producer artifact download HTTP %d", response.StatusCode) + } + archive, err := io.ReadAll(io.LimitReader(response.Body, integrationArtifactMaxBytes+1)) + if err != nil || len(archive) > integrationArtifactMaxBytes { + return nil, fmt.Errorf("producer artifact archive unavailable or oversized") + } + actual := sha256.Sum256(archive) + if "sha256:"+hex.EncodeToString(actual[:]) != digest { + return nil, fmt.Errorf("producer artifact immutable digest mismatch") + } + return archive, nil +} diff --git a/src/pkg/github/integration_producer_token.go b/src/pkg/github/integration_producer_token.go index 232f315a50..5ad139d34a 100644 --- a/src/pkg/github/integration_producer_token.go +++ b/src/pkg/github/integration_producer_token.go @@ -3,6 +3,7 @@ package github import ( "context" "crypto/sha256" + "crypto/x509" "encoding/hex" "encoding/json" "fmt" @@ -55,6 +56,9 @@ type IntegrationProducerIdentity struct { RunAttempt int64 FinalizerCheckRunID int64 PayloadSHA256 string + SigningKeySPKI []byte `json:"-"` + SigningKeySHA256 string + VerifiedAt time.Time } // IntegrationMaterializerBinding describes the independently observed trusted @@ -170,13 +174,22 @@ func (v *IntegrationProducerVerifier) verify(ctx context.Context, raw string, pa } claims := new(integrationProducerClaims) keyID := "" + var publicKey []byte token, err := jwt.ParseWithClaims(raw, claims, func(token *jwt.Token) (any, error) { kid, ok := token.Header["kid"].(string) if !ok || kid == "" || len(kid) > integrationProducerMaxKeyID { return nil, fmt.Errorf("integration producer signing key identity unavailable") } keyID = kid - return v.key(ctx, kid) + key, err := v.key(ctx, kid) + if err != nil { + return nil, err + } + publicKey, err = x509.MarshalPKIXPublicKey(key) + if err != nil { + return nil, err + } + return key, nil }, jwt.WithValidMethods([]string{jwt.SigningMethodRS256.Alg()}), jwt.WithoutClaimsValidation()) if err != nil { return IntegrationProducerIdentity{}, fmt.Errorf("integration producer signature rejected: %w", err) @@ -204,10 +217,12 @@ func (v *IntegrationProducerVerifier) verify(ctx context.Context, raw string, pa claims.ExpiresAt.Sub(claims.IssuedAt.Time) > integrationProducerMaxLifetime { return IntegrationProducerIdentity{}, fmt.Errorf("integration producer issuance is outside the authoritative finalizer interval") } + keyDigest := sha256.Sum256(publicKey) return IntegrationProducerIdentity{ Role: profile.role, Issuer: claims.Issuer, KeyID: keyID, IssuedAt: claims.IssuedAt.Time, RunID: binding.RunID, RunAttempt: binding.RunAttempt, FinalizerCheckRunID: binding.FinalizerCheckRunID, - PayloadSHA256: binding.PayloadSHA256, + PayloadSHA256: binding.PayloadSHA256, + SigningKeySPKI: publicKey, SigningKeySHA256: hex.EncodeToString(keyDigest[:]), VerifiedAt: binding.ObservedAt, }, nil } diff --git a/src/pkg/github/integration_producer_token_test.go b/src/pkg/github/integration_producer_token_test.go index dd018656df..9d02b4ca53 100644 --- a/src/pkg/github/integration_producer_token_test.go +++ b/src/pkg/github/integration_producer_token_test.go @@ -73,7 +73,9 @@ func TestIntegrationProducerAuthenticatesExactHistoricalFinalizer(t *testing.T) claims["check_run_id"] = checkID identity, err := verifier.Verify(context.Background(), signProducerToken(t, key, claims), []byte(producerIdentityPayload), binding) if err != nil || identity.FinalizerCheckRunID != binding.FinalizerCheckRunID || - identity.PayloadSHA256 != binding.PayloadSHA256 || identity.KeyID != "github-key" { + identity.PayloadSHA256 != binding.PayloadSHA256 || identity.KeyID != "github-key" || + len(identity.SigningKeySPKI) == 0 || !integrationProducerDigest(identity.SigningKeySHA256) || + identity.VerifiedAt != binding.ObservedAt { t.Fatalf("exact historical producer: %+v %v", identity, err) } } From 6fa177529ccddf8c23fa60d83a3e2b9b205a13b0 Mon Sep 17 00:00:00 2001 From: Andy Anderson Date: Fri, 30 Oct 2026 23:17:59 -0400 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=90=9B=20fix:=20repair=20materializer?= =?UTF-8?q?=20contract=20assertion=20syntax=20from=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Andy Anderson --- .github/scripts/integration-materialize.test.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/integration-materialize.test.cjs b/.github/scripts/integration-materialize.test.cjs index fcf63da68a..882910f1fb 100644 --- a/.github/scripts/integration-materialize.test.cjs +++ b/.github/scripts/integration-materialize.test.cjs @@ -265,7 +265,7 @@ test('foreign winners and unreadable/malformed claim acknowledgements never auth } if (reason === 'ack_malformed') f.github.rest.git.createRef = async () => ({status: 201, data: {}}); const result = await prepare(f.args()); - assert.ok(!f.calls.some(call => call.type === 'dispatch'))); + assert.ok(!f.calls.some(call => call.type === 'dispatch')); if (reason === 'foreign') { assert.equal(result.failure, null); assert.ok(result.prepared.every(item => item.dispatch === 'observer'));