Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
210 changes: 210 additions & 0 deletions .github/scripts/test-pr-verifier-settle.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
#!/usr/bin/env python3
"""PR Verifier never leaves `PR Type` / `PR Issue` in progress (#18764).

The verifier image creates both check runs in_progress and then PATCHes the
result. On #18667 that PATCH returned 404, the two required checks stayed
in_progress for ever and the `verify PR contents` job still went green. The
"Settle PR Type and PR Issue check runs" step in pr-verifier.yml must:

* conclude every unfinished PR Type / PR Issue run of THIS run's check
suite as `failure` with the reason, falling back to creating a completed
`failure` run of the same name when the PATCH fails too;
* fail the job whenever the verifier could not write a result;
* leave finished check runs (the verifier's real verdict) alone.

This test extracts that step's script from the workflow and runs it with a
stubbed `gh` (real `jq` applies the step's own --jq filters to fixtures), so
no network call is made.
"""
import json
import os
import pathlib
import shutil
import stat
import subprocess
import tempfile
import textwrap
import unittest

import yaml

ROOT = pathlib.Path(__file__).resolve().parents[2]
WORKFLOW = ROOT / ".github/workflows/pr-verifier.yml"
STEP_NAME = "Settle PR Type and PR Issue check runs"

GH_STUB = textwrap.dedent(
r"""
#!/usr/bin/env bash
# Minimal `gh api` stand-in driven by files in $STUB_DIR.
set -u
method=GET path="" jq_expr="" prev=""
for arg in "$@"; do
case "$prev" in
-X) method="$arg"; prev=""; continue ;;
--jq) jq_expr="$arg"; prev=""; continue ;;
-f|-F) prev=""; continue ;;
esac
case "$arg" in
-X|--jq|-f|-F) prev="$arg" ;;
repos/*) path="$arg" ;;
esac
done
printf '%s %s\n' "$method" "$path" >> "$STUB_DIR/calls.log"
case "$method $path" in
"GET repos/o/r/actions/runs/77")
[ -f "$STUB_DIR/run-fails" ] && { echo "HTTP 502" >&2; exit 1; }
body='{"check_suite_id": 555}' ;;
"GET repos/o/r/check-suites/555/check-runs"*)
body="$(cat "$STUB_DIR/check-runs.json")" ;;
"PATCH repos/o/r/check-runs/"*)
[ -f "$STUB_DIR/patch-404" ] && { echo "HTTP 404: Not Found" >&2; exit 1; }
body='{}' ;;
"POST repos/o/r/check-runs")
[ -f "$STUB_DIR/post-fails" ] && { echo "HTTP 403" >&2; exit 1; }
body='{}' ;;
*)
echo "unexpected gh call: $method $path" >&2; exit 99 ;;
esac
if [ -n "$jq_expr" ]; then
printf '%s' "$body" | jq -r "$jq_expr"
else
printf '%s\n' "$body"
fi
"""
).lstrip()


def load_steps():
data = yaml.safe_load(WORKFLOW.read_text())
return data["jobs"]["verify"]["steps"]


def check_run(run_id, name, status, conclusion=None):
return {
"id": run_id,
"name": name,
"status": status,
"conclusion": conclusion,
"head_sha": "c" * 40,
}


class SettleStep(unittest.TestCase):
def setUp(self):
steps = load_steps()
matches = [s for s in steps if s.get("name") == STEP_NAME]
self.assertEqual(len(matches), 1, f"pr-verifier.yml must have one {STEP_NAME!r} step")
self.step = matches[0]
self.steps = steps
self.tmp = pathlib.Path(tempfile.mkdtemp())
self.addCleanup(shutil.rmtree, self.tmp)
bindir = self.tmp / "bin"
bindir.mkdir()
gh = bindir / "gh"
gh.write_text(GH_STUB)
gh.chmod(gh.stat().st_mode | stat.S_IEXEC)
self.bindir = bindir
(self.tmp / "calls.log").write_text("")
self.log = self.tmp / "pr-verifier.log"

def run_step(self, check_runs, verifier_log="2 plugins ran\n", flags=()):
(self.tmp / "check-runs.json").write_text(json.dumps({"check_runs": check_runs}))
for flag in flags:
(self.tmp / flag).write_text("")
self.log.write_text(verifier_log)
env = dict(os.environ)
env.update(
PATH=f"{self.bindir}:{env.get('PATH', '')}",
STUB_DIR=str(self.tmp),
GITHUB_REPOSITORY="o/r",
GITHUB_RUN_ID="77",
GH_TOKEN="x",
VERIFIER_LOG=str(self.log),
)
proc = subprocess.run(
["bash", "-e", "-c", self.step["run"]],
env=env,
capture_output=True,
text=True,
timeout=60,
)
calls = (self.tmp / "calls.log").read_text().splitlines()
writes = [c for c in calls if not c.startswith("GET ")]
return proc, writes

# --- contract -------------------------------------------------------

def test_step_is_not_advisory_and_runs_after_a_failed_verifier(self):
self.assertFalse(self.step.get("continue-on-error", False),
"a verifier that cannot write its result must fail the job")
cond = self.step.get("if", "")
self.assertIn("steps.verifier.outcome == 'failure'", cond)
self.assertIn("steps.verifier.outcome == 'success'", cond)

def test_verifier_output_is_kept_for_the_settle_step(self):
verifier = next(s for s in self.steps if s.get("id") == "verifier")
self.assertIn("set -o pipefail", verifier["run"])
self.assertIn('tee "${RUNNER_TEMP}/pr-verifier.log"', verifier["run"])
self.assertEqual(self.step["env"]["VERIFIER_LOG"], "${{ runner.temp }}/pr-verifier.log")
names = [s.get("name") for s in self.steps]
self.assertGreater(names.index(STEP_NAME), names.index("Verifier action"))

# --- behaviour ------------------------------------------------------

def test_404_on_update_concludes_stuck_checks_and_fails_the_job(self):
log = (
"::error::unable to update check run with results: PATCH "
"https://api.github.com/repos/o/r/check-runs/11: 404 Not Found []\n"
"::error::unable to update check run with results: PATCH "
"https://api.github.com/repos/o/r/check-runs/12: 404 Not Found []\n"
"2 plugins ran\n::error::2 plugins had errors\n"
)
proc, writes = self.run_step(
[check_run(11, "PR Type", "in_progress"), check_run(12, "PR Issue", "in_progress"),
check_run(13, "verify PR contents", "in_progress")],
verifier_log=log,
)
self.assertNotEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertEqual(writes, ["PATCH repos/o/r/check-runs/11", "PATCH repos/o/r/check-runs/12"])
self.assertIn("::error::PR Type (check run 11)", proc.stdout)
self.assertIn("unable to update check run with results", proc.stdout)

def test_when_the_patch_also_fails_a_failed_run_replaces_it(self):
proc, writes = self.run_step(
[check_run(11, "PR Type", "in_progress"), check_run(12, "PR Issue", "completed", "success")],
flags=("patch-404",),
)
self.assertNotEqual(proc.returncode, 0)
self.assertEqual(writes, ["PATCH repos/o/r/check-runs/11", "POST repos/o/r/check-runs"])
self.assertIn("created a failed PR Type check run", proc.stdout)

def test_unreplaceable_stuck_check_still_fails_the_job(self):
proc, _ = self.run_step(
[check_run(11, "PR Type", "queued")], flags=("patch-404", "post-fails"),
)
self.assertNotEqual(proc.returncode, 0)
self.assertIn("could neither be updated nor replaced", proc.stdout)

def test_finished_verdicts_are_left_alone(self):
proc, writes = self.run_step(
[check_run(11, "PR Type", "completed", "failure"), check_run(12, "PR Issue", "completed", "success")],
)
self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr)
self.assertEqual(writes, [])

def test_write_error_without_a_stuck_run_still_fails_the_job(self):
proc, writes = self.run_step(
[], verifier_log="::error::unable to create check run: POST ...: 403 Forbidden []\n",
)
self.assertNotEqual(proc.returncode, 0)
self.assertEqual(writes, [])
self.assertIn("could not write a check run result", proc.stdout)

def test_unreadable_suite_fails_instead_of_passing(self):
proc, writes = self.run_step([check_run(11, "PR Type", "in_progress")], flags=("run-fails",))
self.assertNotEqual(proc.returncode, 0)
self.assertEqual(writes, [])


if __name__ == "__main__":
unittest.main(verbosity=2)
4 changes: 4 additions & 0 deletions .github/workflows/pr-auto-update-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ on:
- '.github/workflows/pr-verifier.yml'
# Any workflow edit can change a required check's concurrency (#18578).
- '.github/scripts/test-required-check-concurrency.py'
- '.github/scripts/test-pr-verifier-settle.py'
- '.github/workflows/*.yml'
push:
branches: [v5, v6]
Expand All @@ -39,6 +40,7 @@ on:
- '.github/workflows/pr-verifier.yml'
# Any workflow edit can change a required check's concurrency (#18578).
- '.github/scripts/test-required-check-concurrency.py'
- '.github/scripts/test-pr-verifier-settle.py'
- '.github/workflows/*.yml'

concurrency:
Expand Down Expand Up @@ -71,3 +73,5 @@ jobs:
run: python3 .github/scripts/test-native-queue-workflows.py
- name: Required checks are never cancelled by same-head re-checks (#18578)
run: python3 .github/scripts/test-required-check-concurrency.py
- name: PR Verifier never leaves PR Type / PR Issue in progress (#18764)
run: python3 .github/scripts/test-pr-verifier-settle.py
76 changes: 74 additions & 2 deletions .github/workflows/pr-verifier.yml
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ on:
# - events that re-check the same head (labeled, unlabeled, edited,
# reopened, ready_for_review) also carry github.run_id: each runs in its
# own group and can neither cancel nor be cancelled. The wrapper is green
# whatever the label set (every verdict step is continue-on-error), and
# whatever the label set (every verdict step is continue-on-error; only a
# verifier that could not WRITE its checks fails it, #18764), and
# the label types stay for #17036, not for the verdict.
# - runs on a superseded commit finish (10 min cap) rather than being
# cancelled; their verdict lands on that old SHA, which never merges.
Expand Down Expand Up @@ -141,7 +142,10 @@ jobs:
# pull_request_target run it stands in for (see dispatch_event).
VERIFIER_EVENT_NAME: ${{ github.event_name == 'workflow_dispatch' && 'pull_request_target' || github.event_name }}
VERIFIER_EVENT_PATH: ${{ steps.dispatch_event.outputs.event_path || github.event_path }}
# The output is kept so the settle step below can name the reason
# when the verifier could not write a check run (#18764).
run: |
set -o pipefail
docker run --rm \
-e INPUT_GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
-e GITHUB_TOKEN="${{ secrets.GITHUB_TOKEN }}" \
Expand All @@ -154,8 +158,76 @@ jobs:
-e GITHUB_WORKSPACE="/github/workspace" \
-v "$VERIFIER_EVENT_PATH:/github/event.json:ro" \
-v "$GITHUB_WORKSPACE:/github/workspace:ro" \
"$PR_VERIFIER_IMAGE"
"$PR_VERIFIER_IMAGE" 2>&1 | tee "${RUNNER_TEMP}/pr-verifier.log"
- name: Report verifier failure
if: steps.verifier.outcome == 'failure'
run: |
echo "::warning::PR Verifier returned a failure. This wrapper is advisory: inspect the individual PR checks and verifier log; a green wrapper does not prove those checks passed."
# `PR Type` and `PR Issue` are required, and the verifier creates them
# in_progress and then PATCHes the result. When that PATCH fails (404 on
# #18667, both attempts of run 44069761061) the check runs stayed
# in_progress for ever, the PR could never merge, and this job went
# green (#18764). A required check must end in a definite state, so:
# - every PR Type / PR Issue check run in THIS run's check suite that
# is not completed is concluded `failure` with the reason; if that
# PATCH fails too, a completed `failure` run of the same name is
# created on the same head (the newest run of a name is the one
# GitHub and Tide score), and
# - this job FAILS whenever the verifier could not write a result.
# Only this run's suite is touched: a sibling run on the same head owns
# its own check runs. A verdict the verifier did write (pass or fail)
# is never changed here. .github/scripts/test-pr-verifier-settle.py
# exercises this step with a stubbed gh.
- name: Settle PR Type and PR Issue check runs
if: steps.verifier.outcome == 'success' || steps.verifier.outcome == 'failure'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
VERIFIER_LOG: ${{ runner.temp }}/pr-verifier.log
run: |
set -uo pipefail
names_filter='.name == "PR Type" or .name == "PR Issue"'
write_errors=""
if [ -f "$VERIFIER_LOG" ]; then
write_errors="$(grep -E 'unable to (create|get|reset|update|duplicate) check run' "$VERIFIER_LOG" || true)"
fi
if ! suite="$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" --jq '.check_suite_id')" \
|| ! [[ "$suite" =~ ^[0-9]+$ ]]; then
echo "::error::Could not read this run's check suite, so PR Type / PR Issue may be left in progress. Re-run this job (#18764)."
exit 1
fi
if ! stuck="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/check-suites/${suite}/check-runs?filter=all&per_page=100" \
--jq ".check_runs[] | select(${names_filter}) | select(.status != \"completed\") | [.id, .name, .head_sha] | @tsv")"; then
echo "::error::Could not list this run's PR Type / PR Issue check runs, so they may be left in progress. Re-run this job (#18764)."
exit 1
fi
reason="PR Verifier could not record its result for this check, so it is marked failed instead of being left in progress."
if [ -n "$write_errors" ]; then
reason="${reason} Verifier said: $(printf '%s' "$write_errors" | sed -E 's/^.*(unable to)/\1/' | tr '\n' ' ')"
fi
reason="${reason} Re-run the PR Verifier workflow, or edit the PR title, to get a real result (#18764)."
failed=0
while IFS=$'\t' read -r id name head_sha; do
[ -n "$id" ] || continue
failed=1
completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
if gh api -X PATCH "repos/${GITHUB_REPOSITORY}/check-runs/${id}" \
-f status=completed -f conclusion=failure -f completed_at="$completed_at" \
-f "output[title]=PR Verifier could not write its result" -f "output[summary]=$reason" >/dev/null; then
echo "::error::${name} (check run ${id}) was left in progress by the verifier; concluded it as failure. ${reason}"
elif gh api -X POST "repos/${GITHUB_REPOSITORY}/check-runs" \
-f name="$name" -f head_sha="$head_sha" -f status=completed -f conclusion=failure \
-f completed_at="$completed_at" \
-f "output[title]=PR Verifier could not write its result" -f "output[summary]=$reason" >/dev/null; then
echo "::error::${name} (check run ${id}) was left in progress and could not be updated; created a failed ${name} check run on ${head_sha} in its place. ${reason}"
else
echo "::error::${name} (check run ${id}) is stuck in progress and could neither be updated nor replaced. ${reason}"
fi
done <<<"$stuck"
if [ -n "$write_errors" ] && [ "$failed" -eq 0 ]; then
failed=1
echo "::error::PR Verifier could not write a check run result: $(printf '%s' "$write_errors" | tr '\n' ' ')(#18764)"
fi
if [ "$failed" -ne 0 ]; then
exit 1
fi
echo "PR Type / PR Issue check runs from this run all reached a conclusion."
Loading