diff --git a/.github/scripts/test-pr-verifier-settle.py b/.github/scripts/test-pr-verifier-settle.py new file mode 100755 index 0000000000..a667a3e08a --- /dev/null +++ b/.github/scripts/test-pr-verifier-settle.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""PR Verifier never leaves `PR Type` / `PR Issue` in progress (#18764). + +The verifier image creates both check runs in_progress and then PATCHes the +result. On #18667 that PATCH returned 404, the two required checks stayed +in_progress for ever and the `verify PR contents` job still went green. The +"Settle PR Type and PR Issue check runs" step in pr-verifier.yml must: + + * conclude every unfinished PR Type / PR Issue run of THIS run's check + suite as `failure` with the reason, falling back to creating a completed + `failure` run of the same name when the PATCH fails too; + * fail the job whenever the verifier could not write a result; + * leave finished check runs (the verifier's real verdict) alone. + +This test extracts that step's script from the workflow and runs it with a +stubbed `gh` (real `jq` applies the step's own --jq filters to fixtures), so +no network call is made. +""" +import json +import os +import pathlib +import shutil +import stat +import subprocess +import tempfile +import textwrap +import unittest + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github/workflows/pr-verifier.yml" +STEP_NAME = "Settle PR Type and PR Issue check runs" + +GH_STUB = textwrap.dedent( + r""" + #!/usr/bin/env bash + # Minimal `gh api` stand-in driven by files in $STUB_DIR. + set -u + method=GET path="" jq_expr="" prev="" + for arg in "$@"; do + case "$prev" in + -X) method="$arg"; prev=""; continue ;; + --jq) jq_expr="$arg"; prev=""; continue ;; + -f|-F) prev=""; continue ;; + esac + case "$arg" in + -X|--jq|-f|-F) prev="$arg" ;; + repos/*) path="$arg" ;; + esac + done + printf '%s %s\n' "$method" "$path" >> "$STUB_DIR/calls.log" + case "$method $path" in + "GET repos/o/r/actions/runs/77") + [ -f "$STUB_DIR/run-fails" ] && { echo "HTTP 502" >&2; exit 1; } + body='{"check_suite_id": 555}' ;; + "GET repos/o/r/check-suites/555/check-runs"*) + body="$(cat "$STUB_DIR/check-runs.json")" ;; + "PATCH repos/o/r/check-runs/"*) + [ -f "$STUB_DIR/patch-404" ] && { echo "HTTP 404: Not Found" >&2; exit 1; } + body='{}' ;; + "POST repos/o/r/check-runs") + [ -f "$STUB_DIR/post-fails" ] && { echo "HTTP 403" >&2; exit 1; } + body='{}' ;; + *) + echo "unexpected gh call: $method $path" >&2; exit 99 ;; + esac + if [ -n "$jq_expr" ]; then + printf '%s' "$body" | jq -r "$jq_expr" + else + printf '%s\n' "$body" + fi + """ +).lstrip() + + +def load_steps(): + data = yaml.safe_load(WORKFLOW.read_text()) + return data["jobs"]["verify"]["steps"] + + +def check_run(run_id, name, status, conclusion=None): + return { + "id": run_id, + "name": name, + "status": status, + "conclusion": conclusion, + "head_sha": "c" * 40, + } + + +class SettleStep(unittest.TestCase): + def setUp(self): + steps = load_steps() + matches = [s for s in steps if s.get("name") == STEP_NAME] + self.assertEqual(len(matches), 1, f"pr-verifier.yml must have one {STEP_NAME!r} step") + self.step = matches[0] + self.steps = steps + self.tmp = pathlib.Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, self.tmp) + bindir = self.tmp / "bin" + bindir.mkdir() + gh = bindir / "gh" + gh.write_text(GH_STUB) + gh.chmod(gh.stat().st_mode | stat.S_IEXEC) + self.bindir = bindir + (self.tmp / "calls.log").write_text("") + self.log = self.tmp / "pr-verifier.log" + + def run_step(self, check_runs, verifier_log="2 plugins ran\n", flags=()): + (self.tmp / "check-runs.json").write_text(json.dumps({"check_runs": check_runs})) + for flag in flags: + (self.tmp / flag).write_text("") + self.log.write_text(verifier_log) + env = dict(os.environ) + env.update( + PATH=f"{self.bindir}:{env.get('PATH', '')}", + STUB_DIR=str(self.tmp), + GITHUB_REPOSITORY="o/r", + GITHUB_RUN_ID="77", + GH_TOKEN="x", + VERIFIER_LOG=str(self.log), + ) + proc = subprocess.run( + ["bash", "-e", "-c", self.step["run"]], + env=env, + capture_output=True, + text=True, + timeout=60, + ) + calls = (self.tmp / "calls.log").read_text().splitlines() + writes = [c for c in calls if not c.startswith("GET ")] + return proc, writes + + # --- contract ------------------------------------------------------- + + def test_step_is_not_advisory_and_runs_after_a_failed_verifier(self): + self.assertFalse(self.step.get("continue-on-error", False), + "a verifier that cannot write its result must fail the job") + cond = self.step.get("if", "") + self.assertIn("steps.verifier.outcome == 'failure'", cond) + self.assertIn("steps.verifier.outcome == 'success'", cond) + + def test_verifier_output_is_kept_for_the_settle_step(self): + verifier = next(s for s in self.steps if s.get("id") == "verifier") + self.assertIn("set -o pipefail", verifier["run"]) + self.assertIn('tee "${RUNNER_TEMP}/pr-verifier.log"', verifier["run"]) + self.assertEqual(self.step["env"]["VERIFIER_LOG"], "${{ runner.temp }}/pr-verifier.log") + names = [s.get("name") for s in self.steps] + self.assertGreater(names.index(STEP_NAME), names.index("Verifier action")) + + # --- behaviour ------------------------------------------------------ + + def test_404_on_update_concludes_stuck_checks_and_fails_the_job(self): + log = ( + "::error::unable to update check run with results: PATCH " + "https://api.github.com/repos/o/r/check-runs/11: 404 Not Found []\n" + "::error::unable to update check run with results: PATCH " + "https://api.github.com/repos/o/r/check-runs/12: 404 Not Found []\n" + "2 plugins ran\n::error::2 plugins had errors\n" + ) + proc, writes = self.run_step( + [check_run(11, "PR Type", "in_progress"), check_run(12, "PR Issue", "in_progress"), + check_run(13, "verify PR contents", "in_progress")], + verifier_log=log, + ) + self.assertNotEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(writes, ["PATCH repos/o/r/check-runs/11", "PATCH repos/o/r/check-runs/12"]) + self.assertIn("::error::PR Type (check run 11)", proc.stdout) + self.assertIn("unable to update check run with results", proc.stdout) + + def test_when_the_patch_also_fails_a_failed_run_replaces_it(self): + proc, writes = self.run_step( + [check_run(11, "PR Type", "in_progress"), check_run(12, "PR Issue", "completed", "success")], + flags=("patch-404",), + ) + self.assertNotEqual(proc.returncode, 0) + self.assertEqual(writes, ["PATCH repos/o/r/check-runs/11", "POST repos/o/r/check-runs"]) + self.assertIn("created a failed PR Type check run", proc.stdout) + + def test_unreplaceable_stuck_check_still_fails_the_job(self): + proc, _ = self.run_step( + [check_run(11, "PR Type", "queued")], flags=("patch-404", "post-fails"), + ) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("could neither be updated nor replaced", proc.stdout) + + def test_finished_verdicts_are_left_alone(self): + proc, writes = self.run_step( + [check_run(11, "PR Type", "completed", "failure"), check_run(12, "PR Issue", "completed", "success")], + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(writes, []) + + def test_write_error_without_a_stuck_run_still_fails_the_job(self): + proc, writes = self.run_step( + [], verifier_log="::error::unable to create check run: POST ...: 403 Forbidden []\n", + ) + self.assertNotEqual(proc.returncode, 0) + self.assertEqual(writes, []) + self.assertIn("could not write a check run result", proc.stdout) + + def test_unreadable_suite_fails_instead_of_passing(self): + proc, writes = self.run_step([check_run(11, "PR Type", "in_progress")], flags=("run-fails",)) + self.assertNotEqual(proc.returncode, 0) + self.assertEqual(writes, []) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/.github/workflows/pr-auto-update-test.yml b/.github/workflows/pr-auto-update-test.yml index 118e3ec806..9bc8943c47 100644 --- a/.github/workflows/pr-auto-update-test.yml +++ b/.github/workflows/pr-auto-update-test.yml @@ -19,6 +19,7 @@ on: - '.github/workflows/pr-verifier.yml' # Any workflow edit can change a required check's concurrency (#18578). - '.github/scripts/test-required-check-concurrency.py' + - '.github/scripts/test-pr-verifier-settle.py' - '.github/workflows/*.yml' push: branches: [v5, v6] @@ -39,6 +40,7 @@ on: - '.github/workflows/pr-verifier.yml' # Any workflow edit can change a required check's concurrency (#18578). - '.github/scripts/test-required-check-concurrency.py' + - '.github/scripts/test-pr-verifier-settle.py' - '.github/workflows/*.yml' concurrency: @@ -71,3 +73,5 @@ jobs: run: python3 .github/scripts/test-native-queue-workflows.py - name: Required checks are never cancelled by same-head re-checks (#18578) run: python3 .github/scripts/test-required-check-concurrency.py + - name: PR Verifier never leaves PR Type / PR Issue in progress (#18764) + run: python3 .github/scripts/test-pr-verifier-settle.py diff --git a/.github/workflows/pr-verifier.yml b/.github/workflows/pr-verifier.yml index e3fe5dcc90..5daefe08f3 100644 --- a/.github/workflows/pr-verifier.yml +++ b/.github/workflows/pr-verifier.yml @@ -40,7 +40,8 @@ on: # - events that re-check the same head (labeled, unlabeled, edited, # reopened, ready_for_review) also carry github.run_id: each runs in its # own group and can neither cancel nor be cancelled. The wrapper is green -# whatever the label set (every verdict step is continue-on-error), and +# whatever the label set (every verdict step is continue-on-error; only a +# verifier that could not WRITE its checks fails it, #18764), and # the label types stay for #17036, not for the verdict. # - runs on a superseded commit finish (10 min cap) rather than being # cancelled; their verdict lands on that old SHA, which never merges. @@ -141,7 +142,10 @@ jobs: # pull_request_target run it stands in for (see dispatch_event). VERIFIER_EVENT_NAME: ${{ github.event_name == 'workflow_dispatch' && 'pull_request_target' || github.event_name }} VERIFIER_EVENT_PATH: ${{ steps.dispatch_event.outputs.event_path || github.event_path }} + # The output is kept so the settle step below can name the reason + # when the verifier could not write a check run (#18764). run: | + set -o pipefail docker run --rm \ -e INPUT_GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \ -e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \ @@ -154,8 +158,76 @@ jobs: -e GITHUB_WORKSPACE="/github/workspace" \ -v "$VERIFIER_EVENT_PATH:/github/event.json:ro" \ -v "$GITHUB_WORKSPACE:/github/workspace:ro" \ - "$PR_VERIFIER_IMAGE" + "$PR_VERIFIER_IMAGE" 2>&1 | tee "${RUNNER_TEMP}/pr-verifier.log" - name: Report verifier failure if: steps.verifier.outcome == 'failure' run: | echo "::warning::PR Verifier returned a failure. This wrapper is advisory: inspect the individual PR checks and verifier log; a green wrapper does not prove those checks passed." + # `PR Type` and `PR Issue` are required, and the verifier creates them + # in_progress and then PATCHes the result. When that PATCH fails (404 on + # #18667, both attempts of run 44069761061) the check runs stayed + # in_progress for ever, the PR could never merge, and this job went + # green (#18764). A required check must end in a definite state, so: + # - every PR Type / PR Issue check run in THIS run's check suite that + # is not completed is concluded `failure` with the reason; if that + # PATCH fails too, a completed `failure` run of the same name is + # created on the same head (the newest run of a name is the one + # GitHub and Tide score), and + # - this job FAILS whenever the verifier could not write a result. + # Only this run's suite is touched: a sibling run on the same head owns + # its own check runs. A verdict the verifier did write (pass or fail) + # is never changed here. .github/scripts/test-pr-verifier-settle.py + # exercises this step with a stubbed gh. + - name: Settle PR Type and PR Issue check runs + if: steps.verifier.outcome == 'success' || steps.verifier.outcome == 'failure' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERIFIER_LOG: ${{ runner.temp }}/pr-verifier.log + run: | + set -uo pipefail + names_filter='.name == "PR Type" or .name == "PR Issue"' + write_errors="" + if [ -f "$VERIFIER_LOG" ]; then + write_errors="$(grep -E 'unable to (create|get|reset|update|duplicate) check run' "$VERIFIER_LOG" || true)" + fi + if ! suite="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq '.check_suite_id')" \ + || ! [[ "$suite" =~ ^[0-9]+$ ]]; then + echo "::error::Could not read this run's check suite, so PR Type / PR Issue may be left in progress. Re-run this job (#18764)." + exit 1 + fi + if ! stuck="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/check-suites/${suite}/check-runs?filter=all&per_page=100" \ + --jq ".check_runs[] | select(${names_filter}) | select(.status != \"completed\") | [.id, .name, .head_sha] | @tsv")"; then + echo "::error::Could not list this run's PR Type / PR Issue check runs, so they may be left in progress. Re-run this job (#18764)." + exit 1 + fi + reason="PR Verifier could not record its result for this check, so it is marked failed instead of being left in progress." + if [ -n "$write_errors" ]; then + reason="${reason} Verifier said: $(printf '%s' "$write_errors" | sed -E 's/^.*(unable to)/\1/' | tr '\n' ' ')" + fi + reason="${reason} Re-run the PR Verifier workflow, or edit the PR title, to get a real result (#18764)." + failed=0 + while IFS=$'\t' read -r id name head_sha; do + [ -n "$id" ] || continue + failed=1 + completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + if gh api -X PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${id}" \ + -f status=completed -f conclusion=failure -f completed_at="$completed_at" \ + -f "output[title]=PR Verifier could not write its result" -f "output[summary]=$reason" >/dev/null; then + echo "::error::${name} (check run ${id}) was left in progress by the verifier; concluded it as failure. ${reason}" + elif gh api -X POST "repos/${GITHUB_REPOSITORY}/check-runs" \ + -f name="$name" -f head_sha="$head_sha" -f status=completed -f conclusion=failure \ + -f completed_at="$completed_at" \ + -f "output[title]=PR Verifier could not write its result" -f "output[summary]=$reason" >/dev/null; then + echo "::error::${name} (check run ${id}) was left in progress and could not be updated; created a failed ${name} check run on ${head_sha} in its place. ${reason}" + else + echo "::error::${name} (check run ${id}) is stuck in progress and could neither be updated nor replaced. ${reason}" + fi + done <<<"$stuck" + if [ -n "$write_errors" ] && [ "$failed" -eq 0 ]; then + failed=1 + echo "::error::PR Verifier could not write a check run result: $(printf '%s' "$write_errors" | tr '\n' ' ')(#18764)" + fi + if [ "$failed" -ne 0 ]; then + exit 1 + fi + echo "PR Type / PR Issue check runs from this run all reached a conclusion."