diff --git a/.github/workflows/base-check.yml b/.github/workflows/base-check.yml index 9925033..bb40d51 100644 --- a/.github/workflows/base-check.yml +++ b/.github/workflows/base-check.yml @@ -80,6 +80,24 @@ on: artifact-name: type: string default: acceptance-base-check + share-build: + description: "Opt in: reuse the head build that run-specs.yml (called with share-build: true in the SAME run, and ordered before this with needs:) uploaded, instead of rebuilding head. Falls back to building when the artifact is missing. Skipped on fork PRs. See docs/adopting.md." + type: boolean + default: false + share-build-artifact-name: + description: "The artifact-name that run-specs.yml was called with (its default is acceptance-results)." + type: string + default: acceptance-results + outputs: + verdict: + description: "pass, red-on-base-green-on-head, fail, passes-on-base, not-run or skipped. In mode report a failure still leaves the jobs green: read this." + value: ${{ jobs.verdict.outputs.verdict || (jobs.detect.outputs.has_specs == 'false' && 'skipped' || 'fail') }} + report-only: + description: "true when mode is report." + value: ${{ jobs.detect.outputs.report_only || 'false' }} + build-reused: + description: "true when the head build was taken from run-specs instead of being rebuilt." + value: ${{ jobs.head-run.outputs.build_reused || 'false' }} secrets: ACCEPTANCE_USER: required: false @@ -106,6 +124,7 @@ jobs: has_specs: ${{ steps.diff.outputs.has_specs }} issues: ${{ steps.diff.outputs.issues }} base_sha: ${{ steps.diff.outputs.base_sha }} + report_only: ${{ steps.diff.outputs.report_only }} steps: - name: Only for pull_request env: @@ -158,6 +177,8 @@ jobs: - name: Changed acceptance specs id: diff working-directory: ${{ inputs.working-directory }} + env: + MODE: ${{ inputs.mode }} run: | # --relative: paths are relative to the working directory, and only it is considered. git diff --relative --name-status HEAD^1 HEAD -- 'acceptance/*/spec.json' > "$RUNNER_TEMP/changed-specs.txt" @@ -166,6 +187,7 @@ jobs: { echo "issues=$ISSUES" echo "base_sha=$(git rev-parse HEAD^1)" + if [ "$MODE" = "report" ]; then echo "report_only=true"; else echo "report_only=false"; fi if [ -n "$ISSUES" ]; then echo "has_specs=true"; else echo "has_specs=false"; fi } >> "$GITHUB_OUTPUT" @@ -178,6 +200,8 @@ jobs: timeout-minutes: ${{ inputs.timeout-minutes }} env: CHANGED_ISSUES: ${{ needs.detect.outputs.issues }} + outputs: + build_reused: ${{ steps.unpack.outputs.reused }} steps: - name: Only for pull_request env: @@ -250,8 +274,29 @@ jobs: fi { echo "skip=$skip"; echo "base_url=$base_url"; } >> "$GITHUB_OUTPUT" + # Opt-in reuse (share-build): take the head build run-specs uploaded in this same run. A missing + # artifact or a bad archive falls back to the normal build below. Never on forks. + - name: Fetch the head build shared by run-specs + id: share + if: inputs.share-build && inputs.build-command != '' && steps.decide.outputs.skip != 'true' && github.event.pull_request.head.repo.full_name == github.repository + continue-on-error: true + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: ${{ inputs.share-build-artifact-name }}-build-${{ github.run_id }} + path: ${{ runner.temp }}/shared-build + + - name: Unpack the shared head build + id: unpack + if: steps.share.outcome == 'success' + continue-on-error: true + working-directory: ${{ inputs.working-directory }} + run: | + bash "$RUNNER_TEMP/acceptance-scripts/share-build.sh" unpack "$RUNNER_TEMP/shared-build/acceptance-build-share.tar" + echo "reused=true" >> "$GITHUB_OUTPUT" + echo "::notice::Head build reused from run-specs (not rebuilt)." + - name: Build - if: inputs.build-command != '' && steps.decide.outputs.skip != 'true' + if: inputs.build-command != '' && steps.decide.outputs.skip != 'true' && steps.unpack.outputs.reused != 'true' working-directory: ${{ inputs.working-directory }} env: BUILD_COMMAND: ${{ inputs.build-command }} @@ -529,6 +574,8 @@ jobs: if: always() && needs.detect.outputs.has_specs == 'true' runs-on: ${{ inputs.runs-on }} timeout-minutes: 5 + outputs: + verdict: ${{ steps.verdict.outputs.verdict }} steps: - name: Only for pull_request env: @@ -591,6 +638,7 @@ jobs: name: ${{ inputs.artifact-name }}-base-${{ github.run_id }} path: base-results - name: Each changed spec is red on base and green on head + id: verdict working-directory: ${{ inputs.working-directory }} env: PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} @@ -608,5 +656,9 @@ jobs: --mode "$MODE" \ --out "$RUNNER_TEMP/spec-first-verdict.json" > "$RUNNER_TEMP/spec-first.md" || rc=$? cat "$RUNNER_TEMP/spec-first.md" + # One clear verdict line first (job summary, annotation title, outputs), then the per-spec detail. + # It describes the result and never changes the exit code below. + node "$RUNNER_TEMP/acceptance-scripts/acceptance-verdict.mjs" --kind base-check \ + --input "$RUNNER_TEMP/spec-first-verdict.json" --mode "$MODE" grep -v '^::' "$RUNNER_TEMP/spec-first.md" >> "$GITHUB_STEP_SUMMARY" || true exit "$rc" diff --git a/.github/workflows/comment.yml b/.github/workflows/comment.yml new file mode 100644 index 0000000..879502e --- /dev/null +++ b/.github/workflows/comment.yml @@ -0,0 +1,165 @@ +name: acceptance comment + +# Reusable workflow (on: workflow_call). Posts ONE sticky pull request comment that summarises the +# acceptance jobs in a small table, found by a hidden marker and edited in place on every push (never a +# new comment per push). It exists because `mode: report` keeps every job green even when a spec is red, +# so the check marks hide the result; the comment shows it. +# +# Call it from a `pull_request` workflow ONLY, pinned to a full commit SHA, as its own job that needs the +# others and is the ONLY job granted `pull-requests: write`: +# comment: +# needs: [convention, specs, spec-first] +# if: always() +# permissions: { contents: read, pull-requests: write } +# uses: spiculedata/acceptance-spec/.github/workflows/comment.yml@<40-char-sha> +# with: { comment: true, convention-verdict: ..., run-specs-verdict: ..., base-check-verdict: ..., report-only: ... } +# +# Trust model: this is the only workflow here with a write scope, and it is the least powerful job that +# can hold it. It does NOT check out the caller's repository or run any PR code or PR text: it fetches +# only this repository's scripts at the pinned ref, reads only the verdict strings its caller passes in, +# and uses the token for two calls (list comments, create or update one). Fork PRs get a read-only token, +# so the job is skipped for them. The jobs that DO execute PR code (build, start, specs) never get a +# write scope. + +on: + workflow_call: + inputs: + comment: + description: "Opt in. Nothing is posted unless this is true." + type: boolean + default: false + convention-verdict: + description: "needs..outputs.verdict. Empty: that row is left out." + type: string + default: "" + run-specs-verdict: + description: "needs..outputs.verdict. Empty: that row is left out." + type: string + default: "" + base-check-verdict: + description: "needs..outputs.verdict. Empty: that row is left out." + type: string + default: "" + report-only: + description: "true when the jobs ran with mode: report (adds a note to the comment)." + type: string + default: "false" + heading: + description: "Heading of the comment." + type: string + default: "Acceptance results" + runs-on: + type: string + default: ubuntu-latest + node-version: + type: string + default: "22" + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + comment: + name: acceptance comment + # Same-repository PRs only: a fork's token is read-only and the call would only fail. + if: inputs.comment && github.event.pull_request.head.repo.full_name == github.repository + runs-on: ${{ inputs.runs-on }} + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + concurrency: + group: acceptance-comment-${{ github.event.pull_request.number }} + cancel-in-progress: false + steps: + - name: Only for pull_request + env: + EVENT_NAME: ${{ github.event_name }} + run: | + if [ "$EVENT_NAME" != "pull_request" ]; then + echo "::error::acceptance-spec workflows must be called from a pull_request workflow (got $EVENT_NAME). Use the plain pull_request trigger." + exit 1 + fi + + # job.workflow_repository / job.workflow_sha identify the REUSABLE workflow file that defines + # this job, i.e. this repository at the SHA the caller pinned in `uses:`. See README, "How the scripts are loaded". + - name: Guard the scripts ref + env: + WF_REPO: ${{ job.workflow_repository }} + WF_SHA: ${{ job.workflow_sha }} + run: | + if [ -z "$WF_REPO" ] || ! printf '%s' "$WF_SHA" | grep -Eq '^[0-9a-f]{40}$'; then + echo "::error::job.workflow_repository / job.workflow_sha are not populated; cannot load the scripts at the pinned ref." + exit 1 + fi + echo "acceptance-spec scripts: $WF_REPO @ $WF_SHA" + + # This is the only checkout in this workflow: this repository's scripts, never the caller's code. + - name: Fetch the acceptance-spec scripts (same pinned ref as this workflow) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ job.workflow_repository }} + ref: ${{ job.workflow_sha }} + path: .acceptance-spec-src + sparse-checkout: scripts + persist-credentials: false + + - name: Move the scripts out of the workspace + env: + WF_SHA: ${{ job.workflow_sha }} + run: | + set -euo pipefail + [ "$(git -C .acceptance-spec-src rev-parse HEAD)" = "$WF_SHA" ] + mv .acceptance-spec-src/scripts "$RUNNER_TEMP/acceptance-scripts" + rm -rf .acceptance-spec-src + + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: ${{ inputs.node-version }} + + - name: Render the comment + id: render + env: + CONVENTION_VERDICT: ${{ inputs.convention-verdict }} + RUN_SPECS_VERDICT: ${{ inputs.run-specs-verdict }} + BASE_CHECK_VERDICT: ${{ inputs.base-check-verdict }} + REPORT_ONLY: ${{ inputs.report-only }} + HEADING: ${{ inputs.heading }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + rc=0 + node "$RUNNER_TEMP/acceptance-scripts/acceptance-comment.mjs" --out "$RUNNER_TEMP/comment.md" || rc=$? + if [ "$rc" -eq 3 ]; then echo "post=false" >> "$GITHUB_OUTPUT"; exit 0; fi + if [ "$rc" -ne 0 ]; then exit "$rc"; fi + echo "post=true" >> "$GITHUB_OUTPUT" + cat "$RUNNER_TEMP/comment.md" >> "$GITHUB_STEP_SUMMARY" + + # Find-and-update: the comment is found by its hidden marker AND by its author (the Actions bot), so + # a look-alike comment from anyone else is never edited. A failed call is a warning, not a red job: + # the comment is a convenience, the verdicts are also in each job summary. + - name: Create or update the sticky comment + if: steps.render.outputs.post == 'true' + env: + GH_TOKEN: ${{ github.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + set -uo pipefail + MARKER="$(node "$RUNNER_TEMP/acceptance-scripts/acceptance-comment.mjs" --marker)" + export MARKER + id=$(gh api --paginate "repos/$REPO/issues/$PR_NUMBER/comments" \ + --jq '.[] | select(.user.login == "github-actions[bot]" and (.body | contains(env.MARKER))) | .id' | sed -n 1p) || id="" + if [ -n "$id" ]; then + gh api --method PATCH "repos/$REPO/issues/comments/$id" -F "body=@$RUNNER_TEMP/comment.md" > /dev/null \ + && echo "Updated comment $id." \ + || echo "::warning::Could not update the acceptance comment (is pull-requests: write granted to this job?)." + else + gh api --method POST "repos/$REPO/issues/$PR_NUMBER/comments" -F "body=@$RUNNER_TEMP/comment.md" > /dev/null \ + && echo "Created the comment." \ + || echo "::warning::Could not create the acceptance comment (is pull-requests: write granted to this job?)." + fi diff --git a/.github/workflows/convention.yml b/.github/workflows/convention.yml index 545a0f9..363fc88 100644 --- a/.github/workflows/convention.yml +++ b/.github/workflows/convention.yml @@ -32,6 +32,13 @@ on: node-version: type: string default: "22" + outputs: + verdict: + description: "pass, fail, or skipped (nothing owed). In mode report a fail still leaves the job green: read this." + value: ${{ jobs.convention.outputs.verdict || 'fail' }} + report-only: + description: "true when mode is report." + value: ${{ jobs.convention.outputs.report-only || 'false' }} permissions: contents: read @@ -45,6 +52,9 @@ jobs: name: acceptance convention runs-on: ${{ inputs.runs-on }} timeout-minutes: 5 + outputs: + verdict: ${{ steps.verdict.outputs.verdict }} + report-only: ${{ steps.verdict.outputs.report-only }} steps: - name: Only for pull_request env: @@ -109,4 +119,17 @@ jobs: PR_LABELS: ${{ join(github.event.pull_request.labels.*.name, ',') }} ACCEPTANCE_ENFORCE_FROM_ISSUE: ${{ inputs.enforce-from-issue }} ACCEPTANCE_MODE: ${{ inputs.mode }} - run: node "$RUNNER_TEMP/acceptance-scripts/acceptance-check.mjs" --changed-files "$RUNNER_TEMP/changed-files.txt" + run: | + rc=0 + node "$RUNNER_TEMP/acceptance-scripts/acceptance-check.mjs" --changed-files "$RUNNER_TEMP/changed-files.txt" \ + --json "$RUNNER_TEMP/convention-result.json" || rc=$? + exit "$rc" + + # One clear verdict line (job summary, annotation title, workflow output), even when the check above + # failed or, in mode report, only warned. It never changes the job's result. + - name: Verdict + id: verdict + if: always() + env: + MODE: ${{ inputs.mode }} + run: node "$RUNNER_TEMP/acceptance-scripts/acceptance-verdict.mjs" --kind convention --input "$RUNNER_TEMP/convention-result.json" --mode "$MODE" diff --git a/.github/workflows/run-specs.yml b/.github/workflows/run-specs.yml index 61161a3..91fb10a 100644 --- a/.github/workflows/run-specs.yml +++ b/.github/workflows/run-specs.yml @@ -83,6 +83,21 @@ on: description: "Name of the uploaded results artifact." type: string default: acceptance-results + share-build: + description: "Opt in: upload the head build output (share-build-paths) as a workflow artifact so base-check, called in the same run with share-build: true, does not rebuild head. Skipped on fork PRs. See docs/adopting.md." + type: boolean + default: false + share-build-paths: + description: "Newline-separated build outputs, relative to working-directory (for example dist), to share. Required when share-build is true." + type: string + default: "" + outputs: + verdict: + description: "pass, fail, not-run or skipped. In mode report a fail still leaves the job green: read this." + value: ${{ jobs.specs.outputs.verdict || 'fail' }} + report-only: + description: "true when mode is report." + value: ${{ jobs.specs.outputs.report-only || 'false' }} secrets: ACCEPTANCE_USER: description: "Account for auth session / basic specs. Use a low-privilege account on a disposable instance." @@ -105,6 +120,9 @@ jobs: name: acceptance specs runs-on: ${{ inputs.runs-on }} timeout-minutes: ${{ inputs.timeout-minutes }} + outputs: + verdict: ${{ steps.verdict.outputs.verdict }} + report-only: ${{ steps.verdict.outputs.report-only }} steps: - name: Only for pull_request env: @@ -191,6 +209,26 @@ jobs: BUILD_TOKEN: ${{ secrets.BUILD_TOKEN }} run: bash -eo pipefail -c "$BUILD_COMMAND" + # Opt-in reuse (share-build): hand the head build to base-check in the same run. Never on forks, + # whose token is read-only and whose build did not run anyway. + - name: Pack the head build for base-check + id: share + if: inputs.share-build && inputs.build-command != '' && steps.decide.outputs.skip != 'true' && github.event.pull_request.head.repo.full_name == github.repository + working-directory: ${{ inputs.working-directory }} + env: + SHARE_PATHS: ${{ inputs.share-build-paths }} + run: bash "$RUNNER_TEMP/acceptance-scripts/share-build.sh" pack "$RUNNER_TEMP/acceptance-build-share.tar" + + - name: Upload the head build for base-check + if: steps.share.outcome == 'success' + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: ${{ inputs.artifact-name }}-build-${{ github.run_id }} + path: ${{ runner.temp }}/acceptance-build-share.tar + if-no-files-found: error + overwrite: true + retention-days: 1 + - name: Start the app if: inputs.start-command != '' && steps.decide.outputs.skip != 'true' env: @@ -227,11 +265,20 @@ jobs: if: always() run: bash "$RUNNER_TEMP/acceptance-scripts/stop-server.sh" + # One clear verdict line (job summary, annotation title, workflow output). In mode report the job + # stays green on a red spec, so this is where the result is visible. It never changes the job result. + - name: Verdict + id: verdict + if: always() + env: + MODE: ${{ inputs.mode }} + run: node "$RUNNER_TEMP/acceptance-scripts/acceptance-verdict.mjs" --kind run-specs --input "$RUNNER_TEMP/acceptance-results.json" --mode "$MODE" + - name: Summarise if: always() run: | { - echo '## Acceptance specs' + echo '### Results' echo echo '```json' cat "$RUNNER_TEMP/acceptance-results.json" 2>/dev/null || echo '{}' diff --git a/.github/workflows/selftest.yml b/.github/workflows/selftest.yml index 140ff02..75704a7 100644 --- a/.github/workflows/selftest.yml +++ b/.github/workflows/selftest.yml @@ -26,34 +26,86 @@ jobs: enforce-from-issue: "999999" mode: enforce + # The fixture has a build step (a copy into dist/) so the share-build path is exercised: run-specs + # uploads dist/, base-check (ordered after it) takes it instead of rebuilding head. run-specs: name: selftest / run-specs uses: ./.github/workflows/run-specs.yml with: working-directory: fixtures/consumer - start-command: PORT=18080 node server.mjs + build-command: mkdir -p dist && cp server.mjs dist/server.mjs + start-command: PORT=18080 node dist/server.mjs base-url: http://127.0.0.1:18080 ready-url: http://127.0.0.1:18080/api/healthz ready-timeout: 30 base-path: /api timeout-minutes: 10 + share-build: true + share-build-paths: dist base-check: name: selftest / base-check + needs: [run-specs] uses: ./.github/workflows/base-check.yml with: working-directory: fixtures/consumer - start-command: PORT=18080 node server.mjs + build-command: mkdir -p dist && cp server.mjs dist/server.mjs + start-command: PORT=18080 node dist/server.mjs base-url: http://127.0.0.1:18080 ready-url: http://127.0.0.1:18080/api/healthz ready-timeout: 30 base-path: /api timeout-minutes: 10 + share-build: true + + # A spec that is red on head. mode: report keeps every job green, which is exactly the situation the + # verdict output is for: `assert-verdicts` below checks the output says fail while the job is green. + red-run-specs: + name: selftest / red run-specs + uses: ./.github/workflows/run-specs.yml + with: + working-directory: fixtures/red-on-head + mode: report + artifact-name: acceptance-results-red + + red-base-check: + name: selftest / red base-check + uses: ./.github/workflows/base-check.yml + with: + working-directory: fixtures/red-on-head + mode: report + artifact-name: acceptance-base-check-red + + green-base-check: + name: selftest / green-on-base base-check + uses: ./.github/workflows/base-check.yml + with: + working-directory: fixtures/green-on-base + mode: report + artifact-name: acceptance-base-check-green + + # The sticky comment, for real, on this repository's own same-repository PRs (a fork PR's token is + # read-only, so the job is skipped there). Only this job holds pull-requests: write. + comment: + name: selftest / comment + needs: [convention, run-specs, base-check] + if: always() + permissions: + contents: read + pull-requests: write + uses: ./.github/workflows/comment.yml + with: + comment: true + heading: Acceptance self-test (fixture) + convention-verdict: ${{ needs.convention.outputs.verdict }} + run-specs-verdict: ${{ needs.run-specs.outputs.verdict }} + base-check-verdict: ${{ needs.base-check.outputs.verdict }} + report-only: "false" # Aggregate for this file, so one name says whether the self-tests are green. selftest: name: selftest - needs: [convention, run-specs, base-check] + needs: [convention, run-specs, base-check, red-run-specs, red-base-check, green-base-check, comment] if: always() runs-on: ubuntu-latest steps: @@ -62,10 +114,14 @@ jobs: CONVENTION: ${{ needs.convention.result }} RUN_SPECS: ${{ needs.run-specs.result }} BASE_CHECK: ${{ needs.base-check.result }} + RED_RUN_SPECS: ${{ needs.red-run-specs.result }} + RED_BASE_CHECK: ${{ needs.red-base-check.result }} + GREEN_BASE_CHECK: ${{ needs.green-base-check.result }} + COMMENT: ${{ needs.comment.result }} run: | set -euo pipefail fail=0 - for entry in "convention=$CONVENTION" "run-specs=$RUN_SPECS" "base-check=$BASE_CHECK"; do + for entry in "convention=$CONVENTION" "run-specs=$RUN_SPECS" "base-check=$BASE_CHECK" "red-run-specs=$RED_RUN_SPECS" "red-base-check=$RED_BASE_CHECK" "green-base-check=$GREEN_BASE_CHECK" "comment=$COMMENT"; do name="${entry%%=*}" result="${entry#*=}" case "$result" in @@ -74,3 +130,40 @@ jobs: esac done exit "$fail" + + # The verdicts the callers see. A red spec in report mode leaves its job green, so this is the proof + # that the verdict output, not the job result, carries it. base-check only does real work on a PR that + # adds or changes a spec in its fixture, so on any other PR `skipped` is the expected verdict there. + assert-verdicts: + name: selftest / verdicts + needs: [convention, run-specs, base-check, red-run-specs, red-base-check, green-base-check] + if: always() + runs-on: ubuntu-latest + steps: + - name: Each verdict is what the fixture should produce + env: + RUN_SPECS: ${{ needs.run-specs.outputs.verdict }} + RED_RUN_SPECS: ${{ needs.red-run-specs.outputs.verdict }} + RED_RUN_SPECS_RESULT: ${{ needs.red-run-specs.result }} + RED_RUN_SPECS_REPORT_ONLY: ${{ needs.red-run-specs.outputs.report-only }} + BASE_CHECK: ${{ needs.base-check.outputs.verdict }} + BUILD_REUSED: ${{ needs.base-check.outputs.build-reused }} + RED_BASE_CHECK: ${{ needs.red-base-check.outputs.verdict }} + GREEN_BASE_CHECK: ${{ needs.green-base-check.outputs.verdict }} + run: | + fail=0 + expect() { # name actual allowed... + local name="$1" actual="$2"; shift 2 + for ok in "$@"; do if [ "$actual" = "$ok" ]; then echo "ok: $name=$actual"; return; fi; done + echo "::error::$name=$actual, expected one of: $*"; fail=1 + } + expect run-specs "$RUN_SPECS" pass + expect red-run-specs "$RED_RUN_SPECS" fail + expect "red-run-specs job result (report mode keeps it green)" "$RED_RUN_SPECS_RESULT" success + expect red-run-specs-report-only "$RED_RUN_SPECS_REPORT_ONLY" true + expect base-check "$BASE_CHECK" red-on-base-green-on-head skipped + expect red-base-check "$RED_BASE_CHECK" fail skipped + expect green-base-check "$GREEN_BASE_CHECK" passes-on-base skipped + # When base-check did real work, head must have been taken from run-specs, not rebuilt. + if [ "$BASE_CHECK" != "skipped" ]; then expect build-reused "$BUILD_REUSED" true; fi + exit "$fail" diff --git a/README.md b/README.md index 34e4f55..fa0a88c 100644 --- a/README.md +++ b/README.md @@ -11,16 +11,17 @@ holds no secrets, and consumers pass anything sensitive through their own reposi Licensed under the Apache License 2.0 (see `LICENSE`). -## The three workflows +## The workflows -All three are GitHub reusable workflows (`on: workflow_call`). Call them from a `pull_request` workflow, +The first three do the work; the fourth, `comment.yml`, is optional. All are GitHub reusable workflows (`on: workflow_call`). Call them from a `pull_request` workflow, never `pull_request_target`, pinned to a **full commit SHA**: | Workflow | What it does | |---|---| | `convention.yml` | Fails a PR that closes issue N (N >= `enforce-from-issue`) without `acceptance/N/spec.json`. A human-applied `acceptance-waived` label waives a docs-only or tests-only PR, and the waiver is verified against the changed files. `mode: report` only warns. | | `run-specs.yml` | Builds the caller's app, starts it, and runs every `acceptance//spec.json` against it. `command` specs always run; `api` specs run only when a base URL is available, otherwise they are `not-run`, never `pass`. | -| `base-check.yml` | Red on base, green on head: every spec a PR adds or changes must fail on the base commit's build and pass on the PR's build. A spec that already passes on base fails the check unless a human applied `spec-already-green`. Self-contained (builds and runs both sides) and caches the base build on the base SHA. | +| `base-check.yml` | Red on base, green on head: every spec a PR adds or changes must fail on the base commit's build and pass on the PR's build. A spec that already passes on base fails the check unless a human applied `spec-already-green`. A spec that is not green on the PR build fails it with "not green on the PR build yet: implement the feature or fix the spec". Self-contained (builds and runs both sides) and caches the base build on the base SHA. | +| `comment.yml` | Optional. Posts one sticky PR comment summarising the three verdicts in a small table (see "Seeing the result"). The only workflow here with a write scope, and only when a caller opts in. | ### Copy-paste caller for a generic repository @@ -74,6 +75,52 @@ jobs: mode: report ``` +### Seeing the result + +In `mode: report` every job concludes green even when a spec is red, so the check marks alone hide it. +Each of the three workflows therefore reports a verdict, three ways, without changing any job's result: + +* **Job summary.** One heading line, for example `## Acceptance specs: fail`, then the detail. +* **Annotation.** `::notice` (pass, proven, skipped) or `::warning` (anything else) titled + `acceptance : `; it shows on the check run's page. +* **Workflow outputs** `verdict` and `report-only` (`true` or `false`). `verdict` is one of: + +| `verdict` | Meaning | +|---|---| +| `pass` | Everything that ran passed (`convention`, `run-specs`), or every changed spec was excused by `spec-already-green`. | +| `red-on-base-green-on-head` | `base-check` only: every changed spec fails on the base and passes on the PR. The proof. | +| `fail` | Something is red or could not be proven (also: the job failed before it could report). | +| `passes-on-base` | `base-check` only: the only problem is specs that already pass on the base branch. | +| `not-run` | It could not run here (for example a fork PR without the build token). Never a pass. | +| `skipped` | Nothing to check: no spec changed, no closing issue, or no specs. | + +`base-check` also has the output `build-reused` (see `share-build` in `docs/adopting.md`). + +Job names are not changed (callers key required checks on them); the verdict is not in the name. + +**Sticky comment (opt in).** Add a job that calls `comment.yml`. It reads only the verdict strings, checks +out none of the caller's code, and is the only job that gets `pull-requests: write`. It finds its comment +by a hidden marker (and by the Actions bot as author) and edits it in place on every push, so there is one +comment per PR. Fork PRs are skipped (read-only token). + +```yaml + comment: + needs: [convention, specs, spec-first] + if: always() + permissions: + contents: read + pull-requests: write # this job only + uses: spiculedata/acceptance-spec/.github/workflows/comment.yml@ + with: + comment: true + convention-verdict: ${{ needs.convention.outputs.verdict }} + run-specs-verdict: ${{ needs.specs.outputs.verdict }} + base-check-verdict: ${{ needs.spec-first.outputs.verdict }} + report-only: ${{ needs.specs.outputs.report-only }} +``` + +Leave a verdict input out (or empty) to leave that row out. Callers that do not add this job are unaffected. + ### Inputs Common to `run-specs.yml` and `base-check.yml` unless marked. @@ -95,6 +142,9 @@ Common to `run-specs.yml` and `base-check.yml` unless marked. | `runs-on` | `ubuntu-latest` | Runner label. | | `timeout-minutes` | `25` | Job timeout (the two run workflows). | | `artifact-name` | `acceptance-results` / `acceptance-base-check` | Prefix of the uploaded result artifacts. | +| `share-build`, `share-build-paths` | `false`, empty | `run-specs` only. Opt in: upload the head build output (`share-build-paths`, newline separated, relative to `working-directory`) as a workflow artifact for `base-check`. See `docs/adopting.md`. | +| `share-build`, `share-build-artifact-name` | `false`, `acceptance-results` | `base-check` only. Opt in: reuse that artifact instead of rebuilding head (falls back to building). Order the job after `run-specs` with `needs:`. | +| `comment`, `*-verdict`, `report-only`, `heading` | `false`, empty | `comment.yml` only; see above. | | `base-cache-paths` | empty | `base-check` only. Newline-separated build outputs, relative to `working-directory`, cached on the base SHA. Empty: the base is always built. | | `enforce-from-issue` | required | `convention` only. Issues numbered below this owe no spec. Empty or not a number enforces nothing. | @@ -134,7 +184,7 @@ Kept from saiku: - `pull_request` only. The workflows refuse to run on any other event, and a guard test fails if a workflow here gains `pull_request_target` or `workflow_run`. -- `permissions: contents: read`, no write scope, no `GITHUB_TOKEN` handed to specs. The first +- `permissions: contents: read`, no write scope anywhere except the opt-in `comment.yml` job (`pull-requests: write`, no checkout of the caller's code, no PR text, skipped on forks; a guard test enforces this), no `GITHUB_TOKEN` handed to specs. The first `actions/checkout` persists no credentials. - Third-party actions are pinned to full commit SHAs (the same pins saiku uses); a test fails otherwise. - Spec commands run with an allow-listed environment (`PATH`, `HOME`, `TMPDIR`, `LANG`, `LC_*`, `CI`, diff --git a/docs/adopting.md b/docs/adopting.md index 8b3d188..6b691bb 100644 --- a/docs/adopting.md +++ b/docs/adopting.md @@ -29,7 +29,10 @@ A checklist per repository. Do it in one PR, in report-only mode first. 9. **Enforce.** Switch `convention` to `enforce`, then `run-specs` and `base-check`. Making any of them a required check is the repository's own decision (branch protection or rulesets); verify it in the repository's settings rather than assuming it. -10. **Bump later** by replacing the SHA in every `uses:` line (see the README). +10. **Make report mode readable (optional).** In `mode: report` red does not turn a check red. Add the + `comment` job from the README ("Seeing the result") to get one sticky PR comment with the three verdicts. + Give `pull-requests: write` to that job only, never to the jobs that build or run specs. +11. **Bump later** by replacing the SHA in every `uses:` line (see the README). ## After the first adoption: confirm the cross-repository `uses:` @@ -37,3 +40,43 @@ The first caller in another repository should confirm, in its `convention` job l `acceptance-spec scripts: spiculedata/acceptance-spec @ `, and that the "Fetch the acceptance-spec scripts" step succeeded with the repository's default token. If `job.workflow_sha` is empty the job stops with an explicit error rather than running unpinned scripts. + +## Optional: reuse the head build (`share-build`) + +`run-specs` and `base-check` each build the PR head, so the head is built twice. With `share-build`, +`run-specs` uploads the head build output once and `base-check` unpacks it instead of rebuilding head +(the base commit is still built by `base-check`). Default off; callers that do not set it are unchanged. + +```yaml + specs: + uses: spiculedata/acceptance-spec/.github/workflows/run-specs.yml@ + with: + # ...as before... + share-build: true + share-build-paths: | + dist + spec-first: + needs: [specs] # required: the artifact must exist before base-check starts + uses: spiculedata/acceptance-spec/.github/workflows/base-check.yml@ + with: + # ...as before... + share-build: true +``` + +Trade-offs, read before turning it on: + +* **Ordering.** `base-check` has to wait for `run-specs` (`needs:`), so the two no longer run in parallel. You + save one head build of runner time; wall-clock time can stay the same or grow, because the base build now + starts after `run-specs` finishes. Measure it on a few PRs. +* **Size and retention.** The build output is uploaded as one tar artifact named + `-build-` with a one day retention. It counts against the repository's artifact + storage while it lives, and anyone who can read the repository's Actions artifacts can download it. Share + only `share-build-paths` that are safe to expose (no baked-in tokens, no environment dumps). Large outputs + (hundreds of MB) cost upload and download time and may not pay for themselves. +* **Same run only.** Nothing is cached across runs; a re-run job gets the artifact of its own run. +* **Safe fallback.** If the artifact is missing, cannot be downloaded or fails to unpack, `base-check` logs it and + builds head as usual (a failed download shows as an annotation on the run, the job stays green). The + `build-reused` output of `base-check` says which happened. +* **Forks.** Never shared on fork PRs (read-only token); each workflow builds for itself. +* **Same inputs.** Use the same `working-directory`, `build-command` and `java-version` on both, and set + `share-build-artifact-name` on `base-check` if you changed `artifact-name` on `run-specs`. diff --git a/docs/spec-format.md b/docs/spec-format.md index a834df0..eb4d785 100644 --- a/docs/spec-format.md +++ b/docs/spec-format.md @@ -101,9 +101,15 @@ documentation (`docs/`, `*.md`, `.github/ISSUE_TEMPLATE/`) or a test (`src/test/ held as if the label were absent, and the check names the disqualifying files. **Spec first.** Commit `acceptance/N/spec.json` before the implementation. `base-check.yml` runs every spec a -PR adds or changes against the base commit's build and the PR's build. Per spec: red on base and green on head -passes; passes on base fails (it proves nothing); fails on head fails; a base run that errored or recorded -nothing fails; `not-run` on either side is reported `not-run`. The human-applied `spec-already-green` label +PR adds or changes against the base commit's build and the PR's build. Per spec, the message says which case you are in: + +* red on base and green on head: **proven** (verdict `red-on-base-green-on-head`); +* red on head: "not green on the PR build yet: implement the feature or fix the spec" (verdict `fail`; if it + passes on base, the message adds that this PR may have broken it); +* green on base: "passes on the base branch: it proves nothing about this change" (verdict `passes-on-base`); +* a base run that errored or recorded nothing fails; `not-run` on either side is reported `not-run`. + +The human-applied `spec-already-green` label excuses specs that legitimately pass on base (a regression guard for behaviour that already works). It never rescues a spec that fails on head. It is read from the event payload: apply it before pushing, or close and reopen the PR. Agents must not apply it. diff --git a/fixtures/consumer/acceptance/3/spec.json b/fixtures/consumer/acceptance/3/spec.json new file mode 100644 index 0000000..d782b75 --- /dev/null +++ b/fixtures/consumer/acceptance/3/spec.json @@ -0,0 +1,33 @@ +{ + "issue": 3, + "title": "the gadgets endpoint lists a cog", + "kind": "api", + "criteria": [ + { + "id": "C1", + "given": "any caller", + "then": "GET /api/gadgets answers 200 with a list that names a cog" + } + ], + "steps": [ + { + "name": "gadgets are listed", + "criterion": "C1", + "request": { + "method": "GET", + "path": "/gadgets" + }, + "expect": { + "status": 200, + "json": { + "$": { + "type": "array" + }, + "[0].name": { + "equals": "cog" + } + } + } + } + ] +} diff --git a/fixtures/consumer/server.mjs b/fixtures/consumer/server.mjs index 905e2ed..1523242 100644 --- a/fixtures/consumer/server.mjs +++ b/fixtures/consumer/server.mjs @@ -1,6 +1,7 @@ // A tiny loopback "application under test" for the self-tests. No dependencies, no network // beyond 127.0.0.1. PORT defaults to 18080; WIDGETS=off makes /widgets answer 404 (a "base" build -// that does not have the feature yet). +// that does not have the feature yet). /api/gadgets is the feature acceptance/3 proves: the base branch +// of the PR that introduced it answers 404 there, so the spec was red on base and is green on head. import { createServer } from 'node:http'; const port = Number(process.env.PORT ?? 18080); @@ -13,6 +14,7 @@ const server = createServer((req, res) => { }; if (req.url === '/api/healthz') return send(200, { status: 'ok' }); if (req.url === '/api/widgets' && widgetsEnabled) return send(200, [{ id: 1, name: 'sprocket' }]); + if (req.url === '/api/gadgets') return send(200, [{ id: 1, name: 'cog' }]); return send(404, { error: 'no route' }); }); diff --git a/fixtures/green-on-base/acceptance/1/check.mjs b/fixtures/green-on-base/acceptance/1/check.mjs new file mode 100644 index 0000000..dcbbff6 --- /dev/null +++ b/fixtures/green-on-base/acceptance/1/check.mjs @@ -0,0 +1 @@ +process.exit(0); diff --git a/fixtures/green-on-base/acceptance/1/spec.json b/fixtures/green-on-base/acceptance/1/spec.json new file mode 100644 index 0000000..f234271 --- /dev/null +++ b/fixtures/green-on-base/acceptance/1/spec.json @@ -0,0 +1,16 @@ +{ + "issue": 1, + "title": "a command spec that is green on base (proves nothing)", + "kind": "command", + "criteria": [ + { + "id": "C1", + "given": "any build", + "then": "the command exits 0" + } + ], + "command": [ + "node", + "check.mjs" + ] +} diff --git a/fixtures/red-on-head/acceptance/1/check.mjs b/fixtures/red-on-head/acceptance/1/check.mjs new file mode 100644 index 0000000..6cee2e1 --- /dev/null +++ b/fixtures/red-on-head/acceptance/1/check.mjs @@ -0,0 +1 @@ +process.exit(1); diff --git a/fixtures/red-on-head/acceptance/1/spec.json b/fixtures/red-on-head/acceptance/1/spec.json new file mode 100644 index 0000000..f710503 --- /dev/null +++ b/fixtures/red-on-head/acceptance/1/spec.json @@ -0,0 +1,16 @@ +{ + "issue": 1, + "title": "a command spec that is red on head (and on base)", + "kind": "command", + "criteria": [ + { + "id": "C1", + "given": "any build", + "then": "the command exits 1" + } + ], + "command": [ + "node", + "check.mjs" + ] +} diff --git a/scripts/acceptance-base-check.mjs b/scripts/acceptance-base-check.mjs index c8f4261..bdcbe77 100644 --- a/scripts/acceptance-base-check.mjs +++ b/scripts/acceptance-base-check.mjs @@ -65,37 +65,55 @@ const verdictOf = (results, issue) => { export function decideSpec({ issue, head, base, baseJobResult = 'success', excused = false }) { const spec = `acceptance/${issue}/spec.json`; if (!head) { - return { issue, verdict: 'fail', message: `${spec}: no result from the PR build, so "green on head" is unproven. Check the head-side run.` }; + return { issue, verdict: 'fail', code: 'no-head-result', message: `${spec}: no result from the PR build, so "green on head" is unproven. Check the head-side run.` }; } if (head.verdict === 'fail') { - return { issue, verdict: 'fail', message: `${spec}: fails on the PR build too. Fix the implementation (or the spec) until it is green on head.` }; + const regression = base?.verdict === 'pass' ? ' It passes on the base branch, so this PR may have broken it.' : ''; + return { + issue, + verdict: 'fail', + code: 'not-green-on-head', + message: `${spec}: not green on the PR build yet: implement the feature or fix the spec.${regression}` + }; } if (head.verdict !== 'pass') { - return { issue, verdict: 'not-run', message: `${spec}: not run on the PR build${head.reason ? ` (${head.reason})` : ''}; red-on-base is unproven.` }; + return { issue, verdict: 'not-run', code: 'not-run', message: `${spec}: not run on the PR build${head.reason ? ` (${head.reason})` : ''}; red-on-base is unproven.` }; } if (baseJobResult === 'failure' || baseJobResult === 'cancelled') { - return { issue, verdict: 'fail', message: `${spec}: the base-side run did not complete (${baseJobResult}), so "red on base" is unproven. Re-run the failed jobs.` }; + return { issue, verdict: 'fail', code: 'base-incomplete', message: `${spec}: the base-side run did not complete (${baseJobResult}), so "red on base" is unproven. Re-run the failed jobs.` }; } if (!base) { - return { issue, verdict: 'fail', message: `${spec}: no result from the base build although the base run completed.` }; + return { issue, verdict: 'fail', code: 'no-base-result', message: `${spec}: no result from the base build although the base run completed.` }; } if (base.verdict === 'fail') { - return { issue, verdict: 'pass', message: `${spec}: red on base, green on head.` }; + return { issue, verdict: 'pass', code: 'red-on-base-green-on-head', message: `${spec}: red on base, green on head.` }; } if (base.verdict === 'pass') { if (excused) { - return { issue, verdict: 'excused', message: `${spec}: already passes on base; excused by the "${ESCAPE_LABEL}" label (applied by a human).` }; + return { issue, verdict: 'excused', code: 'excused', message: `${spec}: passes on the base branch; excused by the "${ESCAPE_LABEL}" label (applied by a human).` }; } return { issue, verdict: 'fail', + code: 'passes-on-base', message: - `${spec}: PASSES on the base branch's build, so it proves nothing about this change. ` + - `Write the spec so it fails without the fix (commit it first), or, if it is a legitimate regression guard for behaviour that already works, ` + - `have a maintainer add the "${ESCAPE_LABEL}" label and then push or close and reopen the PR.` + `${spec}: passes on the base branch: it proves nothing about this change (label ${ESCAPE_LABEL}, applied by a human, excuses it). ` + + `Otherwise write the spec so it fails without the fix and commit it first; then push or close and reopen the PR.` }; } - return { issue, verdict: 'not-run', message: `${spec}: not run on the base build${base.reason ? ` (${base.reason})` : ''}; red-on-base is unproven.` }; + return { issue, verdict: 'not-run', code: 'not-run', message: `${spec}: not run on the base build${base.reason ? ` (${base.reason})` : ''}; red-on-base is unproven.` }; +} + +/** + * The verdict a caller sees (workflow output, sticky comment): one of pass, red-on-base-green-on-head, + * fail, passes-on-base, not-run, skipped. `passes-on-base` is used only when that is the sole reason + * for failing; any other failure (or a mix) is `fail`. + */ +export function callerVerdictOf(verdict, specs) { + if (verdict === 'none') return 'skipped'; + if (verdict === 'not-run') return 'not-run'; + if (verdict === 'fail') return specs.filter((s) => s.verdict === 'fail').every((s) => s.code === 'passes-on-base') ? 'passes-on-base' : 'fail'; + return specs.some((s) => s.code === 'red-on-base-green-on-head') ? 'red-on-base-green-on-head' : 'pass'; } /** Fold the changed specs into one report. `verdict` is fail > not-run > pass; nothing changed => `none`. */ @@ -114,12 +132,12 @@ export function evaluate({ changedIssues, headResults, baseResults, baseJobResul if (specs.length === 0) verdict = 'none'; else if (specs.some((s) => s.verdict === 'fail')) verdict = 'fail'; else if (specs.some((s) => s.verdict === 'not-run')) verdict = 'not-run'; - return { schemaVersion: 1, verdict, escapeLabel: excused, specs }; + return { schemaVersion: 1, verdict, callerVerdict: callerVerdictOf(verdict, specs), escapeLabel: excused, specs }; } /** Markdown for the job summary, one line per spec. */ export function renderReport(report) { - const lines = [`## acceptance / spec fails on base: ${report.verdict}`, '']; + const lines = ['### Per-spec detail', '']; if (report.specs.length === 0) lines.push('No `acceptance//spec.json` was added or changed by this PR; nothing to prove.'); for (const s of report.specs) lines.push(`- **${s.verdict}** ${s.message}`); if (report.verdict === 'not-run') { diff --git a/scripts/acceptance-base-check.test.mjs b/scripts/acceptance-base-check.test.mjs index 70c16d8..4bba011 100644 --- a/scripts/acceptance-base-check.test.mjs +++ b/scripts/acceptance-base-check.test.mjs @@ -11,7 +11,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import test from 'node:test'; -import { ESCAPE_LABEL, changedSpecIssues, decideSpec, evaluate, isExcused, main, renderReport } from './acceptance-base-check.mjs'; +import { ESCAPE_LABEL, callerVerdictOf, changedSpecIssues, decideSpec, evaluate, isExcused, main, renderReport } from './acceptance-base-check.mjs'; import { runSpecs } from './acceptance-runner.mjs'; const repoRoot = join(import.meta.dirname, '..'); @@ -73,7 +73,7 @@ test('a spec that passes on base fails, names the spec and says how to proceed', const d = decideSpec({ issue: 5, head: pass, base: pass }); assert.equal(d.verdict, 'fail'); assert.match(d.message, /acceptance\/5\/spec\.json/); - assert.match(d.message, /PASSES on the base/); + assert.match(d.message, /passes on the base branch: it proves nothing/); assert.match(d.message, /spec-already-green/); }); @@ -132,8 +132,8 @@ test('evaluate: any failing spec fails the PR; the others are still reported', ( assert.equal(report.verdict, 'fail'); assert.deepEqual(report.specs.map((s) => s.verdict), ['pass', 'fail', 'pass']); const text = renderReport(report); - assert.match(text, /acceptance \/ spec fails on base: fail/); - assert.match(text, /acceptance\/2\/spec\.json: PASSES on the base/); + assert.match(text, /Per-spec detail/); + assert.match(text, /acceptance\/2\/spec\.json: passes on the base branch: it proves nothing/); }); test('evaluate: fail beats not-run, not-run beats pass, no specs is none', () => { @@ -184,7 +184,7 @@ test('main exits 1 and annotates the failing spec when a spec passes on base', ( const cap = capture(); const code = main(['--changed', join(dir, 'changed.txt'), '--head', join(dir, 'head.json'), '--base', join(dir, 'base.json'), '--out', join(dir, 'verdict.json')], cap.io); assert.equal(code, 1); - assert.match(cap.out(), /::error::acceptance\/77\/spec\.json: PASSES on the base/); + assert.match(cap.out(), /::error::acceptance\/77\/spec\.json: passes on the base branch: it proves nothing/); assert.equal(JSON.parse(readFileSync(join(dir, 'verdict.json'), 'utf8')).verdict, 'fail'); }); @@ -283,7 +283,7 @@ test('end to end with the runner: a spec written after the fact (green on both b const headResults = await runSpecs({ repoRoot: root, baseUrl: head.url, issues: [4242], env: {} }); const report = evaluate({ changedIssues: [4242], headResults, baseResults, baseJobResult: 'success', labels: '' }); assert.equal(report.verdict, 'fail'); - assert.match(report.specs[0].message, /acceptance\/4242\/spec\.json: PASSES on the base/); + assert.match(report.specs[0].message, /acceptance\/4242\/spec\.json: passes on the base branch: it proves nothing/); } finally { base.server.close(); head.server.close(); @@ -307,7 +307,54 @@ test('--mode report downgrades a failing proof to a warning and exits 0', () => const args = [cli, '--changed', join(dir, 'changed.txt'), '--head', join(dir, 'head.json'), '--base', join(dir, 'base.json')]; const report = spawnSync(process.execPath, [...args, '--mode', 'report'], { encoding: 'utf8' }); assert.equal(report.status, 0, report.stdout); - assert.match(report.stdout, /::warning::report-only: acceptance\/77\/spec\.json: PASSES on the base/); + assert.match(report.stdout, /::warning::report-only: acceptance\/77\/spec\.json: passes on the base branch: it proves nothing/); assert.equal(spawnSync(process.execPath, args, { encoding: 'utf8' }).status, 1); assert.equal(spawnSync(process.execPath, [...args, '--mode', 'x'], { encoding: 'utf8' }).status, 2); }); + +// ------------------------------------------------------------------ wording and caller verdicts + +test('red on head, whatever the base: "not green on the PR build yet", never "fails on the PR build too"', () => { + for (const base of [fail, notRun('x'), null]) { + const d = decideSpec({ issue: 5, head: fail, base }); + assert.equal(d.code, 'not-green-on-head'); + assert.match(d.message, /not green on the PR build yet: implement the feature or fix the spec\./); + assert.doesNotMatch(d.message, /too/); + assert.doesNotMatch(d.message, /passes on the base/); + } +}); + +test('red on head but green on base is flagged as a possible regression', () => { + const d = decideSpec({ issue: 5, head: fail, base: pass }); + assert.match(d.message, /not green on the PR build yet/); + assert.match(d.message, /passes on the base branch, so this PR may have broken it/); +}); + +test('passes on base names the label, says a human applies it, and carries its own code', () => { + const d = decideSpec({ issue: 5, head: pass, base: pass }); + assert.equal(d.code, 'passes-on-base'); + assert.match(d.message, /passes on the base branch: it proves nothing about this change \(label spec-already-green, applied by a human, excuses it\)/); +}); + +test('red on base and green on head carries the success code', () => { + assert.equal(decideSpec({ issue: 5, head: pass, base: fail }).code, 'red-on-base-green-on-head'); +}); + +const callerVerdict = (changedIssues, head, base, labels = '') => + evaluate({ changedIssues, headResults: results(head), baseResults: results(base), baseJobResult: 'success', labels }).callerVerdict; + +test('callerVerdict: one of pass, red-on-base-green-on-head, fail, passes-on-base, not-run, skipped', () => { + assert.equal(callerVerdict([], [], []), 'skipped'); + assert.equal(callerVerdict([1], [[1, 'pass']], [[1, 'fail']]), 'red-on-base-green-on-head'); + assert.equal(callerVerdict([1], [[1, 'fail']], [[1, 'fail']]), 'fail'); + assert.equal(callerVerdict([1], [[1, 'pass']], [[1, 'pass']]), 'passes-on-base'); + assert.equal(callerVerdict([1], [[1, 'pass']], [[1, 'pass']], 'spec-already-green'), 'pass'); + assert.equal(callerVerdict([1, 2], [[1, 'pass'], [2, 'pass']], [[1, 'pass'], [2, 'fail']], 'spec-already-green'), 'red-on-base-green-on-head'); + assert.equal(callerVerdict([1], [[1, 'not-run']], [[1, 'fail']]), 'not-run'); +}); + +test('callerVerdict: passes-on-base only when it is the sole failure; a mix is fail', () => { + assert.equal(callerVerdict([1, 2], [[1, 'pass'], [2, 'fail']], [[1, 'pass'], [2, 'fail']]), 'fail'); + assert.equal(callerVerdict([1, 2], [[1, 'pass'], [2, 'pass']], [[1, 'pass'], [2, 'pass']]), 'passes-on-base'); + assert.equal(callerVerdictOf('none', []), 'skipped'); +}); diff --git a/scripts/acceptance-comment.mjs b/scripts/acceptance-comment.mjs new file mode 100644 index 0000000..f6a944a --- /dev/null +++ b/scripts/acceptance-comment.mjs @@ -0,0 +1,87 @@ +#!/usr/bin/env node +// Renders the ONE sticky pull request comment that summarises the three acceptance jobs. It only +// builds text; comment.yml finds the existing comment by MARKER and updates it, or creates it once. +// The verdicts come from the callers' `needs..outputs.verdict`; nothing from the PR is read. +// +// node acceptance-comment.mjs --out FILE (inputs from env, see below) +// node acceptance-comment.mjs --marker (print the hidden marker) +// +// env: CONVENTION_VERDICT, RUN_SPECS_VERDICT, BASE_CHECK_VERDICT (empty = that job is not used), +// REPORT_ONLY (true|false), HEAD_SHA, RUN_URL, HEADING + +import { writeFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +import { MEANINGS, VERDICTS } from './acceptance-verdict.mjs'; + +/** Hidden marker the comment is found by. Nothing else may contain it. */ +export const MARKER = ''; + +const ROWS = [ + ['convention', 'Convention (a closing PR ships its spec)'], + ['runSpecs', 'Specs on this PR\'s build'], + ['baseCheck', 'Spec-first proof (red on base, green on this PR)'] +]; + +const WORDS = { + pass: 'Pass', + 'red-on-base-green-on-head': 'Proven', + fail: 'Fail', + 'passes-on-base': 'Passes on base: proves nothing', + 'not-run': 'Not run', + skipped: 'Nothing to check' +}; + +const clean = (value) => { + const v = String(value ?? '').trim(); + return v === '' || VERDICTS.includes(v) ? v : 'fail'; +}; + +/** Markdown for the comment. Rows whose verdict is empty are left out. */ +export function renderComment({ verdicts, reportOnly = false, headSha = '', runUrl = '', heading = 'Acceptance results' }) { + const rows = ROWS.filter(([key]) => clean(verdicts?.[key]) !== ''); + const lines = [MARKER, `### ${String(heading).replace(/[\r\n<>]/g, ' ')}`, '']; + lines.push('| Check | Result | What it means |', '| --- | --- | --- |'); + for (const [key, name] of rows) { + const verdict = clean(verdicts[key]); + lines.push(`| ${name} | **${WORDS[verdict]}** | ${MEANINGS[verdict]} |`); + } + if (reportOnly) { + lines.push('', 'Report-only mode: a red result here does not turn the checks red. Read this table, not the check marks.'); + } + const foot = []; + if (/^[0-9a-f]{7,40}$/.test(headSha)) foot.push(`commit ${headSha.slice(0, 7)}`); + if (/^https:\/\/[^\s<>"]+$/.test(runUrl)) foot.push(`[workflow run](${runUrl})`); + if (foot.length > 0) lines.push('', `Updated for ${foot.join(', ')}. This comment is edited in place, never repeated.`); + return `${lines.join('\n')}\n`; +} + +/** True when there is at least one row to show. */ +export const hasRows = (verdicts) => ROWS.some(([key]) => clean(verdicts?.[key]) !== ''); + +export function main(argv, env = process.env, io = { stdout: process.stdout, stderr: process.stderr }) { + if (argv[0] === '--marker') { + io.stdout.write(`${MARKER}\n`); + return 0; + } + if (argv[0] !== '--out' || !argv[1]) { + io.stderr.write('usage: acceptance-comment.mjs --out FILE | --marker\n'); + return 2; + } + const verdicts = { convention: env.CONVENTION_VERDICT, runSpecs: env.RUN_SPECS_VERDICT, baseCheck: env.BASE_CHECK_VERDICT }; + if (!hasRows(verdicts)) { + io.stdout.write('::notice::No acceptance verdicts were passed to comment.yml; nothing to post.\n'); + return 3; + } + const text = renderComment({ + verdicts, + reportOnly: env.REPORT_ONLY === 'true', + headSha: env.HEAD_SHA, + runUrl: env.RUN_URL, + heading: env.HEADING || undefined + }); + writeFileSync(argv[1], text); + return 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) process.exit(main(process.argv.slice(2))); diff --git a/scripts/acceptance-verdict.mjs b/scripts/acceptance-verdict.mjs new file mode 100644 index 0000000..3fa0190 --- /dev/null +++ b/scripts/acceptance-verdict.mjs @@ -0,0 +1,117 @@ +#!/usr/bin/env node +// One clear verdict per reusable workflow. Each workflow calls this once, after its real work, to +// * append a single verdict heading to the job summary, +// * set the `verdict` and `report-only` job outputs (exposed as workflow_call outputs), and +// * print one annotation whose title carries the verdict (it shows on the check run's page). +// It only DESCRIBES: it never changes an exit code, so `mode: report` semantics are untouched. +// +// node acceptance-verdict.mjs --kind convention|run-specs|base-check --input FILE --mode enforce|report +// +// --input is the JSON the workflow already produced (acceptance-check --json, the runner's --out, +// or acceptance-base-check --out). A missing or unreadable file is verdict `fail`: no result is +// never a pass. GITHUB_OUTPUT and GITHUB_STEP_SUMMARY are the usual runner files. + +import { appendFileSync, readFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +/** Everything a caller may see in the `verdict` output. */ +export const VERDICTS = ['pass', 'red-on-base-green-on-head', 'fail', 'passes-on-base', 'not-run', 'skipped']; + +const GOOD = new Set(['pass', 'red-on-base-green-on-head']); +const QUIET = new Set(['skipped']); + +const KINDS = { + convention: { title: 'Acceptance convention' }, + 'run-specs': { title: 'Acceptance specs' }, + 'base-check': { title: 'Acceptance spec-first proof' } +}; + +/** Plain-words meaning of each verdict, shared by the job summary and the sticky comment. */ +export const MEANINGS = { + pass: 'Everything that ran passed.', + 'red-on-base-green-on-head': 'Proven: each new or changed spec fails on the base branch and passes on this PR.', + fail: 'Something is red or could not be proven. Read the job for the reason.', + 'passes-on-base': 'A spec already passes on the base branch, so it proves nothing about this change.', + 'not-run': 'It could not run here, so nothing is proven (for example a fork PR without the build token).', + skipped: 'Nothing to check on this PR.' +}; + +/** Convention result (acceptance-check --json) -> verdict. */ +export function conventionVerdict(result) { + if (!result || !Array.isArray(result.problems)) return 'fail'; + if (result.problems.length > 0) return 'fail'; + if (result.reason === 'no-closing-issue' || result.waived || String(result.reason ?? '').startsWith('grandfathered')) return 'skipped'; + return 'pass'; +} + +/** Runner results (acceptance-runner --out) -> verdict. */ +export function runSpecsVerdict(results) { + if (!results || !Array.isArray(results.specs)) return 'fail'; + if (results.specs.length === 0) return 'skipped'; + return ['pass', 'fail', 'not-run'].includes(results.verdict) ? results.verdict : 'fail'; +} + +/** Base-check report (acceptance-base-check --out) -> verdict. */ +export function baseCheckVerdict(report) { + return VERDICTS.includes(report?.callerVerdict) ? report.callerVerdict : 'fail'; +} + +const DECIDERS = { convention: conventionVerdict, 'run-specs': runSpecsVerdict, 'base-check': baseCheckVerdict }; + +/** The verdict for `kind` from its input JSON (null when the file is absent). */ +export function decideVerdict(kind, input) { + const decide = DECIDERS[kind]; + if (!decide) throw new Error(`unknown kind "${kind}"`); + return decide(input); +} + +/** One heading line for the job summary. */ +export function summaryHeading(kind, verdict, reportOnly) { + const note = reportOnly && !GOOD.has(verdict) && !QUIET.has(verdict) ? ' This is report-only mode: the check itself stays green.' : ''; + return `## ${KINDS[kind].title}: ${verdict}\n\n${MEANINGS[verdict]}${note}\n`; +} + +/** The annotation line: a `::notice` when fine, a `::warning` otherwise, always titled with the verdict. */ +export function annotation(kind, verdict) { + const level = GOOD.has(verdict) || QUIET.has(verdict) ? 'notice' : 'warning'; + return `::${level} title=acceptance ${kind}: ${verdict}::${MEANINGS[verdict]}`; +} + +function readJson(file) { + try { + return JSON.parse(readFileSync(file, 'utf8')); + } catch { + return null; + } +} + +function parseArgs(argv) { + const opts = { mode: 'enforce' }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (arg === '--kind') opts.kind = argv[++i]; + else if (arg === '--input') opts.input = argv[++i]; + else if (arg === '--mode') opts.mode = argv[++i]; + else return { error: `unknown argument "${arg}"` }; + } + if (!KINDS[opts.kind]) return { error: '--kind must be convention, run-specs or base-check' }; + if (opts.mode !== 'enforce' && opts.mode !== 'report') return { error: '--mode must be enforce or report' }; + return { opts }; +} + +/** CLI entry. Returns the exit code (0, or 2 on bad arguments). */ +export function main(argv, env = process.env, io = { stdout: process.stdout, stderr: process.stderr }) { + const { opts, error } = parseArgs(argv); + if (error) { + io.stderr.write(`acceptance-verdict: ${error}\n`); + return 2; + } + const verdict = decideVerdict(opts.kind, opts.input ? readJson(opts.input) : null); + const reportOnly = opts.mode === 'report'; + if (env.GITHUB_OUTPUT) appendFileSync(env.GITHUB_OUTPUT, `verdict=${verdict}\nreport-only=${reportOnly}\n`); + if (env.GITHUB_STEP_SUMMARY) appendFileSync(env.GITHUB_STEP_SUMMARY, `${summaryHeading(opts.kind, verdict, reportOnly)}\n`); + io.stdout.write(`${annotation(opts.kind, verdict)}\n`); + return 0; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) process.exit(main(process.argv.slice(2))); diff --git a/scripts/acceptance-verdict.test.mjs b/scripts/acceptance-verdict.test.mjs new file mode 100644 index 0000000..6b73643 --- /dev/null +++ b/scripts/acceptance-verdict.test.mjs @@ -0,0 +1,166 @@ +// Unit tests for the visible verdicts: acceptance-verdict.mjs (summary, outputs, annotation), +// acceptance-comment.mjs (the sticky comment text) and share-build.sh (the shared head build). +// Run with `node --test "scripts/*.test.mjs"`. + +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; + +import { MARKER, hasRows, main as commentMain, renderComment } from './acceptance-comment.mjs'; +import { VERDICTS, annotation, baseCheckVerdict, conventionVerdict, decideVerdict, main as verdictMain, runSpecsVerdict, summaryHeading } from './acceptance-verdict.mjs'; + +const scripts = import.meta.dirname; +const tmp = () => mkdtempSync(join(tmpdir(), 'acceptance-verdict-')); +const sink = () => { + let out = ''; + return { stdout: { write: (s) => (out += s) }, stderr: { write: (s) => (out += s) }, text: () => out }; +}; + +// ------------------------------------------------------------------ verdict mapping + +test('run-specs: pass, fail, not-run and "no specs" map to verdicts; a missing result is fail', () => { + assert.equal(runSpecsVerdict({ verdict: 'pass', specs: [{ issue: 1 }] }), 'pass'); + assert.equal(runSpecsVerdict({ verdict: 'fail', specs: [{ issue: 1 }] }), 'fail'); + assert.equal(runSpecsVerdict({ verdict: 'not-run', specs: [{ issue: 1 }] }), 'not-run'); + assert.equal(runSpecsVerdict({ verdict: 'not-run', specs: [] }), 'skipped'); + assert.equal(runSpecsVerdict(null), 'fail'); + assert.equal(runSpecsVerdict({ verdict: 'weird', specs: [{}] }), 'fail'); +}); + +test('convention: problems fail, not-applicable and waived skip, otherwise pass; a missing result is fail', () => { + assert.equal(conventionVerdict({ problems: ['x'], reason: 'required' }), 'fail'); + assert.equal(conventionVerdict({ problems: [], reason: 'no-closing-issue' }), 'skipped'); + assert.equal(conventionVerdict({ problems: [], reason: 'grandfathered: below cutoff' }), 'skipped'); + assert.equal(conventionVerdict({ problems: [], waived: true, reason: 'label' }), 'skipped'); + assert.equal(conventionVerdict({ problems: [], reason: 'required' }), 'pass'); + assert.equal(conventionVerdict(null), 'fail'); +}); + +test('base-check: the report carries the caller verdict; anything unknown is fail', () => { + for (const v of VERDICTS) assert.equal(baseCheckVerdict({ callerVerdict: v }), v); + assert.equal(baseCheckVerdict({}), 'fail'); + assert.equal(baseCheckVerdict(null), 'fail'); + assert.throws(() => decideVerdict('nope', {})); +}); + +test('the annotation title carries the verdict; only good or quiet verdicts are notices', () => { + assert.match(annotation('base-check', 'red-on-base-green-on-head'), /^::notice title=acceptance base-check: red-on-base-green-on-head::/); + assert.match(annotation('run-specs', 'fail'), /^::warning title=acceptance run-specs: fail::/); + assert.match(annotation('convention', 'skipped'), /^::notice /); + assert.match(annotation('run-specs', 'not-run'), /^::warning /); +}); + +test('the summary heading is one line with the verdict, and says report-only keeps the check green', () => { + assert.match(summaryHeading('run-specs', 'fail', true), /^## Acceptance specs: fail\n/); + assert.match(summaryHeading('run-specs', 'fail', true), /report-only mode: the check itself stays green/); + assert.doesNotMatch(summaryHeading('run-specs', 'fail', false), /report-only/); + assert.doesNotMatch(summaryHeading('run-specs', 'pass', true), /report-only/); +}); + +test('main writes the outputs and the summary, prints the annotation, and exits 0 even on a red verdict in report mode', () => { + const dir = tmp(); + const input = join(dir, 'r.json'); + writeFileSync(input, JSON.stringify({ verdict: 'fail', specs: [{ issue: 1, verdict: 'fail' }] })); + const env = { GITHUB_OUTPUT: join(dir, 'out'), GITHUB_STEP_SUMMARY: join(dir, 'sum') }; + const io = sink(); + assert.equal(verdictMain(['--kind', 'run-specs', '--input', input, '--mode', 'report'], env, io), 0); + assert.equal(readFileSync(env.GITHUB_OUTPUT, 'utf8'), 'verdict=fail\nreport-only=true\n'); + assert.match(readFileSync(env.GITHUB_STEP_SUMMARY, 'utf8'), /^## Acceptance specs: fail/); + assert.match(io.text(), /::warning title=acceptance run-specs: fail::/); +}); + +test('main with no results file is a fail verdict, never a pass; bad arguments exit 2', () => { + const dir = tmp(); + const env = { GITHUB_OUTPUT: join(dir, 'out') }; + assert.equal(verdictMain(['--kind', 'run-specs', '--input', join(dir, 'missing.json'), '--mode', 'enforce'], env, sink()), 0); + assert.equal(readFileSync(env.GITHUB_OUTPUT, 'utf8'), 'verdict=fail\nreport-only=false\n'); + assert.equal(verdictMain(['--kind', 'bogus'], {}, sink()), 2); + assert.equal(verdictMain(['--kind', 'run-specs', '--mode', 'maybe'], {}, sink()), 2); +}); + +// ------------------------------------------------------------------ sticky comment text + +const verdicts = { convention: 'pass', runSpecs: 'fail', baseCheck: 'passes-on-base' }; + +test('the comment starts with the hidden marker and has one plain-words row per job', () => { + const text = renderComment({ verdicts, reportOnly: true, headSha: 'da497650b97d0a0764d23a5d6fb969e54aacd82c', runUrl: 'https://example.invalid/run/1' }); + assert.ok(text.startsWith(`${MARKER}\n`)); + assert.equal(text.split(MARKER).length, 2, 'the marker appears exactly once'); + assert.match(text, /\| Convention \(a closing PR ships its spec\) \| \*\*Pass\*\* \|/); + assert.match(text, /\| Specs on this PR's build \| \*\*Fail\*\* \|/); + assert.match(text, /\| Spec-first proof \(red on base, green on this PR\) \| \*\*Passes on base: proves nothing\*\* \|/); + assert.match(text, /Report-only mode: a red result here does not turn the checks red/); + assert.match(text, /commit da49765/); + assert.match(text, /\[workflow run\]\(https:\/\/example\.invalid\/run\/1\)/); +}); + +test('the comment omits rows for jobs that are not used, and the report-only note when enforcing', () => { + const text = renderComment({ verdicts: { runSpecs: 'pass', baseCheck: '' }, reportOnly: false }); + assert.doesNotMatch(text, /Convention|Spec-first/); + assert.doesNotMatch(text, /Report-only/); + assert.match(text, /Specs on this PR/); +}); + +test('the comment never echoes unvetted text: unknown verdicts read as Fail, odd urls and shas are dropped', () => { + const text = renderComment({ verdicts: { runSpecs: '' }, headSha: 'not-a-sha', runUrl: 'javascript:alert(1)', heading: 'Hi\n' }); + assert.match(text, /\*\*Fail\*\*/); + assert.doesNotMatch(text, /script|javascript|not-a-sha|/); +}); + +test('hasRows is false when no verdict was passed, and the CLI then posts nothing (exit 3)', () => { + assert.equal(hasRows({}), false); + const dir = tmp(); + const io = sink(); + assert.equal(commentMain(['--out', join(dir, 'c.md')], {}, io), 3); + assert.equal(existsSync(join(dir, 'c.md')), false); + assert.equal(commentMain(['--out', join(dir, 'c.md')], { RUN_SPECS_VERDICT: 'pass', REPORT_ONLY: 'false' }, sink()), 0); + assert.match(readFileSync(join(dir, 'c.md'), 'utf8'), /Specs on this PR/); + const marker = sink(); + assert.equal(commentMain(['--marker'], {}, marker), 0); + assert.equal(marker.text().trim(), MARKER); +}); + +// ------------------------------------------------------------------ share-build.sh + +const share = (args, cwd, env = {}) => + spawnSync('bash', [join(scripts, 'share-build.sh'), ...args], { cwd, encoding: 'utf8', env: { PATH: process.env.PATH, ...env } }); + +test('share-build pack/unpack round-trips paths and the executable bit', () => { + const src = tmp(); + mkdirSync(join(src, 'dist', 'bin'), { recursive: true }); + writeFileSync(join(src, 'dist', 'bin', 'run.sh'), '#!/bin/sh\n', { mode: 0o755 }); + writeFileSync(join(src, 'app.jar'), 'jar'); + const tar = join(tmp(), 'b.tar'); + const packed = share(['pack', tar], src, { SHARE_PATHS: 'dist\napp.jar\n' }); + assert.equal(packed.status, 0, packed.stdout + packed.stderr); + const dest = tmp(); + const unpacked = share(['unpack', tar], dest); + assert.equal(unpacked.status, 0, unpacked.stdout + unpacked.stderr); + assert.equal(readFileSync(join(dest, 'app.jar'), 'utf8'), 'jar'); + assert.ok(statSync(join(dest, 'dist', 'bin', 'run.sh')).mode & 0o100, 'executable bit kept'); +}); + +test('share-build pack refuses empty, missing, absolute and climbing paths', () => { + const src = tmp(); + const tar = join(tmp(), 'b.tar'); + for (const paths of ['', 'missing', '/etc/passwd', '../x', 'a/../../x', '-rf']) { + const r = share(['pack', tar], src, { SHARE_PATHS: paths }); + assert.equal(r.status, 1, `paths=${JSON.stringify(paths)}: ${r.stdout}`); + assert.match(r.stdout, /::error::/); + } +}); + +test('share-build unpack refuses an archive with an unsafe member', () => { + const dir = tmp(); + writeFileSync(join(dir, 'f'), 'x'); + const tar = join(dir, 'evil.tar'); + // -P keeps the leading slash so the archive really carries an absolute member name. + const made = spawnSync('tar', ['-cPf', tar, join(dir, 'f')]); + assert.equal(made.status, 0); + const r = share(['unpack', tar], tmp()); + assert.equal(r.status, 1); + assert.match(r.stdout, /unsafe path/); +}); diff --git a/scripts/e2e-fixture.test.mjs b/scripts/e2e-fixture.test.mjs index 498c17b..b562f7b 100644 --- a/scripts/e2e-fixture.test.mjs +++ b/scripts/e2e-fixture.test.mjs @@ -91,7 +91,7 @@ test('base-check flow: red on a base without the feature, green on head -> pass; const afterTheFact = verdict(headOut); assert.equal(afterTheFact.status, 1); - assert.match(afterTheFact.stdout, /PASSES on the base/); + assert.match(afterTheFact.stdout, /passes on the base branch: it proves nothing/); } finally { base.stop(); head.stop(); diff --git a/scripts/share-build.sh b/scripts/share-build.sh new file mode 100755 index 0000000..dc9a945 --- /dev/null +++ b/scripts/share-build.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Pack or unpack the head build output that run-specs shares with base-check (same workflow run). +# A tar keeps the paths and file modes exactly; upload-artifact alone would drop the common root +# and the executable bit. Run it with the working directory as the cwd. +# +# SHARE_PATHS="dist +# target/app.jar" share-build.sh pack OUT.tar +# share-build.sh unpack IN.tar +# +# Paths are newline separated, relative to the cwd, must exist, and may not climb out of it. +set -euo pipefail + +mode="${1:-}" +file="${2:-}" +if [ -z "$file" ]; then + echo "::error::usage: share-build.sh pack|unpack FILE" >&2 + exit 2 +fi + +case "$mode" in + pack) + paths=() + while IFS= read -r p; do + [ -n "$p" ] || continue + case "$p" in + /* | -* | .. | ../* | */.. | */../*) + echo "::error::share-build-paths: '$p' must be a relative path inside the working directory." + exit 1 + ;; + esac + if [ ! -e "$p" ]; then + echo "::error::share-build-paths: '$p' does not exist after the build." + exit 1 + fi + paths+=("$p") + done <<< "${SHARE_PATHS:-}" + if [ "${#paths[@]}" -eq 0 ]; then + echo "::error::share-build is true but share-build-paths is empty." + exit 1 + fi + tar -cf "$file" -- "${paths[@]}" + echo "Packed ${#paths[@]} path(s), $(wc -c < "$file" | tr -d ' ') bytes." + ;; + unpack) + # Refuse anything that would land outside the cwd (absolute or parent-relative member names). + if tar -tf "$file" | grep -Eq '^/|(^|/)\.\.(/|$)'; then + echo "::error::the shared build archive has an unsafe path; not unpacking." + exit 1 + fi + tar -xf "$file" --no-same-owner + echo "Unpacked the shared build." + ;; + *) + echo "::error::usage: share-build.sh pack|unpack FILE" >&2 + exit 2 + ;; +esac diff --git a/scripts/workflows.test.mjs b/scripts/workflows.test.mjs index 5c8e9c4..4fd17c3 100644 --- a/scripts/workflows.test.mjs +++ b/scripts/workflows.test.mjs @@ -7,7 +7,11 @@ import { join } from 'node:path'; import test from 'node:test'; const dir = join(import.meta.dirname, '..', '.github', 'workflows'); -const REUSABLE = ['convention.yml', 'run-specs.yml', 'base-check.yml']; +const REUSABLE = ['convention.yml', 'run-specs.yml', 'base-check.yml', 'comment.yml']; +// The jobs that build, start or run PR code. None of them may ever hold a write scope. +const PR_CODE_WORKFLOWS = ['convention.yml', 'run-specs.yml', 'base-check.yml']; +// comment.yml posts the sticky comment, and selftest.yml is its only caller here. +const COMMENT_WRITERS = ['comment.yml', 'selftest.yml']; const ALL = readdirSync(dir).filter((f) => f.endsWith('.yml')); const read = (file) => readFileSync(join(dir, file), 'utf8'); @@ -30,11 +34,62 @@ test('no workflow uses pull_request_target or workflow_run, or any write scope', for (const file of ALL) { const text = code(file); assert.doesNotMatch(text, /^\s*(pull_request_target|workflow_run)\s*:/m, file); - assert.doesNotMatch(text, /(?:contents|issues|pull-requests|actions|checks|statuses|id-token|packages|deployments): write/, file); + assert.doesNotMatch(text, /(?:contents|issues|actions|checks|statuses|id-token|packages|deployments): write/, file); assert.match(text, /^permissions:\n {2}contents: read\n/m, `${file} declares contents: read at the top level`); + if (!COMMENT_WRITERS.includes(file)) assert.doesNotMatch(text, /pull-requests/, `${file} has no pull-requests scope`); } }); +// The one deliberate exception to "no write scope": the sticky comment. `pull-requests: write` may appear +// only at JOB level (never top level) in comment.yml and in the one job of selftest.yml that calls it, and +// the workflows that execute PR code never mention it. Callers grant it to their `comment` job alone. +test('pull-requests: write exists only on the comment job, never at the top level, never near PR code', () => { + for (const file of COMMENT_WRITERS) { + const lines = code(file).split('\n'); + const hits = lines.map((l, i) => [l, i]).filter(([l]) => /^\s*pull-requests: write\s*$/.test(l)); + assert.equal(hits.length, 1, `${file}: exactly one pull-requests: write`); + assert.ok(/^ {6}pull-requests: write$/.test(hits[0][0]), `${file}: it sits in a job-level permissions block`); + assert.match(code(file), /^permissions:\n {2}contents: read\n(?! {2}pull-requests)/m, `${file}: the top level stays read-only`); + } + for (const file of PR_CODE_WORKFLOWS) assert.doesNotMatch(code(file), /pull-requests/, file); +}); + +test('comment.yml never checks out or runs the caller\'s code, and uses the token only to post the comment', () => { + const text = code('comment.yml'); + const checkouts = [...text.matchAll(/actions\/checkout@[0-9a-f]{40}[^\n]*\n((?: {8,}[^\n]*\n)+)/g)]; + assert.equal(checkouts.length, 1, 'one checkout'); + assert.match(checkouts[0][1], /repository: \$\{\{ job\.workflow_repository \}\}/, 'the checkout is of the acceptance-spec repository, not the caller\'s'); + assert.doesNotMatch(text, /working-directory|build-command|start-command|BUILD_TOKEN|secrets\./, 'no PR code is built, started or given secrets'); + assert.match(text, /if: inputs\.comment && github\.event\.pull_request\.head\.repo\.full_name == github\.repository/, 'opt-in, and same-repository PRs only'); + assert.equal([...text.matchAll(/github\.token/g)].length, 1, 'the token appears once'); + assert.match(text, /GH_TOKEN: \$\{\{ github\.token \}\}/); + assert.match(text, /select\(\.user\.login == "github-actions\[bot\]"/, 'only the bot\'s own comment is edited'); + assert.match(text, /acceptance-comment\.mjs" --marker/, 'found by the hidden marker'); +}); + +test('every reusable workflow exposes verdict and report-only as workflow_call outputs', () => { + for (const file of PR_CODE_WORKFLOWS) { + const text = code(file); + assert.match(text, /^ {4}outputs:\n {6}verdict:\n/m, `${file}: verdict output`); + assert.match(text, /^ {6}report-only:\n/m, `${file}: report-only output`); + assert.match(text, /acceptance-verdict\.mjs" --kind/, `${file}: writes the verdict line`); + } +}); + +test('share-build is opt-in (default false), skipped on forks, and falls back to building', () => { + const run = code('run-specs.yml'); + const base = code('base-check.yml'); + for (const text of [run, base]) { + assert.match(text, /share-build:\n {8}description: [^\n]*\n {8}type: boolean\n {8}default: false\n/); + assert.match(text, /github\.event\.pull_request\.head\.repo\.full_name == github\.repository/); + } + assert.match(run, /retention-days: 1\n/); + // the consumer is advisory: a missing artifact does not fail the job, and Build runs unless the unpack succeeded + assert.match(base, /id: share\n[^\n]*\n {8}continue-on-error: true\n {8}uses: actions\/download-artifact@[0-9a-f]{40}/); + assert.match(base, /steps\.unpack\.outputs\.reused != 'true'/); + assert.match(base, /name: \$\{\{ inputs\.share-build-artifact-name \}\}-build-\$\{\{ github\.run_id \}\}/, 'same run only'); +}); + test('the reusable workflows are workflow_call only, and refuse to run on anything but pull_request', () => { for (const file of REUSABLE) { const text = code(file); @@ -50,7 +105,9 @@ test('the only secrets the reusable workflows touch are the three declared ones, const used = [...new Set([...text.matchAll(/secrets\.([A-Za-z_]+)/g)].map((m) => m[1]))].sort(); for (const name of used) assert.ok(['ACCEPTANCE_PASSWORD', 'ACCEPTANCE_USER', 'BUILD_TOKEN'].includes(name), `${file} uses secrets.${name}`); assert.doesNotMatch(text, /secrets: inherit|secrets\[/, file); - assert.doesNotMatch(text, /GITHUB_TOKEN|github\.token/, file); + // comment.yml is the one place that uses the token (checked by its own test); the others never see it. + if (file !== 'comment.yml') assert.doesNotMatch(text, /GITHUB_TOKEN|github\.token/, file); + else assert.doesNotMatch(text, /GITHUB_TOKEN|secrets\./, file); } }); @@ -80,7 +137,7 @@ test('untrusted PR text and caller inputs reach scripts through env, never inter }); test('the scripts are loaded from the same pinned ref as the workflow, via job.workflow_repository / job.workflow_sha', () => { - for (const file of ['convention.yml', 'run-specs.yml', 'base-check.yml']) { + for (const file of REUSABLE) { const text = code(file); assert.match(text, /repository: \$\{\{ job\.workflow_repository \}\}/, file); assert.match(text, /ref: \$\{\{ job\.workflow_sha \}\}/, file);