diff --git a/.github/scripts/test-required-check-concurrency.py b/.github/scripts/test-required-check-concurrency.py new file mode 100644 index 0000000000..a61c87366d --- /dev/null +++ b/.github/scripts/test-required-check-concurrency.py @@ -0,0 +1,419 @@ +#!/usr/bin/env python3 +"""Required checks must never be CANCELLED by a re-check of the same head (#18578). + +GitHub scores every check-run of a required name on a commit, so one cancelled +sibling keeps a PR BLOCKED beside a green survivor and Tide jams on it +(#11250, #18578). Two concurrency behaviours produce such siblings: + + * cancel-in-progress: a new run in the group cancels the one in flight; + * queue replacement: a group holds at most one pending run, and the next run + to queue cancels it, even with cancel-in-progress false. + +So any event that re-checks the SAME head (a label flip, an edit, a reopen) +must land in a concurrency group of its own. This test parses every workflow +under .github/workflows/, finds the jobs that produce a required context on +v5, and evaluates their concurrency expressions for a push followed by a +burst of label events on one head SHA. It fails if any two of those runs +share a group. + +The expression evaluator below covers the subset of GitHub's expression +syntax these workflows use (contexts, literals, ==, !=, !, &&, ||, +parentheses, contains/fromJSON/format/startsWith/endsWith). An unsupported +construct fails loudly rather than passing. +""" +import json +import pathlib +import re +import unittest + +import yaml + +ROOT = pathlib.Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / ".github/workflows" + +# Mirror of the v5 branch protection's required contexts, read with +# `gh api repos/hivecommons/hive/branches/v5/protection/required_status_checks`. +# Keep in step when a context is added; contexts produced by an app (DCO) and +# not by a workflow job are simply never matched. +REQUIRED_CONTEXTS = { + "gate", + "DCO", + "build-and-test", + "test", + "coverage", + "golangci-lint", + "gosec (insecure patterns)", + "govulncheck (known vulnerabilities)", + "changelog-fragment-guard", + "verify PR contents", + "Sign-off survives the squash", +} + +PR_EVENTS = ("pull_request", "pull_request_target") +# Events that re-check a head SHA that already has (or is getting) a run. +RECHECK_ACTIONS = ("labeled", "unlabeled") +# GitHub's default activity types when `types:` is omitted. +DEFAULT_PR_TYPES = ["opened", "synchronize", "reopened"] + +PR_NUMBER = 4242 +HEAD_SHA = "a" * 40 +MERGE_SHA = "b" * 40 + + +# --- minimal GitHub expression evaluator ------------------------------------ + +TOKEN = re.compile(r""" + \s*(?: + (?P-?\d+(?:\.\d+)?) + | (?P'(?:[^']|'')*') + | (?P==|!=|&&|\|\||<=|>=|[!<>()\[\],.*]) + | (?P[A-Za-z_][A-Za-z0-9_-]*) + )""", re.X) + + +class ExprError(ValueError): + pass + + +def tokenize(src): + pos, out = 0, [] + while pos < len(src): + if src[pos:].strip() == "": + break + m = TOKEN.match(src, pos) + if not m or m.end() == pos: + raise ExprError(f"cannot tokenize {src[pos:]!r}") + pos = m.end() + kind = m.lastgroup + out.append((kind, m.group(kind))) + return out + + +def truthy(v): + return not (v is None or v is False or v == 0 or v == "") + + +def to_str(v): + if v is None: + return "" + if v is True: + return "true" + if v is False: + return "false" + if isinstance(v, (dict, list)): + return json.dumps(v) + return str(v) + + +def loose_eq(a, b): + if type(a) is type(b): + return a == b + if isinstance(a, str) and isinstance(b, str): + return a.lower() == b.lower() + return to_str(a) == to_str(b) + + +def fn_format(fmt, *args): + return re.sub(r"\{(\d+)\}", lambda m: to_str(args[int(m.group(1))]), fmt) + + +FUNCTIONS = { + "contains": lambda hay, needle: ( + any(loose_eq(x, needle) for x in hay) if isinstance(hay, list) + else to_str(needle).lower() in to_str(hay).lower()), + "fromjson": lambda s: json.loads(s), + "format": fn_format, + "startswith": lambda s, p: to_str(s).lower().startswith(to_str(p).lower()), + "endswith": lambda s, p: to_str(s).lower().endswith(to_str(p).lower()), +} + + +class Parser: + def __init__(self, src, ctx): + self.toks, self.i, self.ctx = tokenize(src), 0, ctx + + def peek(self, value=None): + if self.i >= len(self.toks): + return None + tok = self.toks[self.i] + return tok if value is None or tok[1] == value else None + + def take(self, value=None): + tok = self.peek(value) + if tok is None: + raise ExprError(f"expected {value!r} at token {self.i}") + self.i += 1 + return tok + + def parse(self): + v = self.or_() + if self.i != len(self.toks): + raise ExprError(f"trailing tokens: {self.toks[self.i:]}") + return v + + def or_(self): + v = self.and_() + while self.peek("||"): + self.take() + rhs = self.and_() + v = v if truthy(v) else rhs + return v + + def and_(self): + v = self.cmp() + while self.peek("&&"): + self.take() + rhs = self.cmp() + v = rhs if truthy(v) else v + return v + + def cmp(self): + v = self.unary() + while self.peek("==") or self.peek("!="): + op = self.take()[1] + rhs = self.unary() + v = loose_eq(v, rhs) if op == "==" else not loose_eq(v, rhs) + return v + + def unary(self): + if self.peek("!"): + self.take() + return not truthy(self.unary()) + return self.postfix() + + def postfix(self): + v = self.primary() + while True: + if self.peek("."): + self.take() + name = self.take()[1] + v = v.get(name) if isinstance(v, dict) else None + elif self.peek("["): + self.take() + key = self.or_() + self.take("]") + v = v.get(key) if isinstance(v, dict) else None + else: + return v + + def primary(self): + kind, val = self.take() + if val == "(": + v = self.or_() + self.take(")") + return v + if kind == "num": + return float(val) if "." in val else int(val) + if kind == "str": + return val[1:-1].replace("''", "'") + if kind != "ident": + raise ExprError(f"unexpected token {val!r}") + if val in ("true", "false"): + return val == "true" + if val == "null": + return None + if self.peek("("): + fn = FUNCTIONS.get(val.lower()) + if fn is None: + raise ExprError(f"unsupported function {val}()") + self.take("(") + args = [] + while not self.peek(")"): + args.append(self.or_()) + if not self.peek(")"): + self.take(",") + self.take(")") + return fn(*args) + if val not in self.ctx: + raise ExprError(f"unsupported context {val!r}") + return self.ctx[val] + + +def evaluate(template, ctx): + """Evaluate a workflow value: a plain scalar or a string with ${{ }} parts.""" + if not isinstance(template, str): + return template + parts = re.split(r"\$\{\{(.*?)\}\}", template) + if len(parts) == 3 and parts[0].strip() == "" and parts[2].strip() == "": + return Parser(parts[1], ctx).parse() + out = [] + for idx, part in enumerate(parts): + out.append(part if idx % 2 == 0 else to_str(Parser(part, ctx).parse())) + return "".join(out) + + +# --- workflow inspection ----------------------------------------------------- + +def load(path): + data = yaml.safe_load(path.read_text()) + # PyYAML's YAML 1.1 loader reads the key `on` as True. + events = data.get("on", data.get(True)) or {} + if isinstance(events, str): + events = {events: None} + elif isinstance(events, list): + events = {e: None for e in events} + return data, events + + +def pr_types(events, name): + cfg = events.get(name) + if name not in events: + return [] + if isinstance(cfg, dict) and "types" in cfg: + return list(cfg["types"]) + return list(DEFAULT_PR_TYPES) + + +def context_for(workflow_name, event_name, action, run_id): + return { + "github": { + "workflow": workflow_name, + "event_name": event_name, + "ref": f"refs/pull/{PR_NUMBER}/merge", + "sha": MERGE_SHA, + "run_id": run_id, + "repository": "hivecommons/hive", + "event": { + "action": action, + "number": PR_NUMBER, + "pull_request": { + "number": PR_NUMBER, + "head": {"sha": HEAD_SHA, "repo": {"full_name": "hivecommons/hive"}}, + "base": {"ref": "v5"}, + }, + }, + }, + "inputs": {}, + "vars": {}, + } + + +def same_head_runs(data, events): + """Yield (event_name, action, run_id) for a push then a burst of re-checks.""" + run_id = 1000 + for event_name in PR_EVENTS: + types = pr_types(events, event_name) + if not any(a in types for a in RECHECK_ACTIONS): + continue + first = "synchronize" if "synchronize" in types else types[0] + yield event_name, first, run_id + for action in RECHECK_ACTIONS: + if action in types: + # Two of each: lgtm + approved land together on approval. + for _ in range(2): + run_id += 1 + yield event_name, action, run_id + + +def collisions(workflow_name, concurrency, runs): + """Return a list of human-readable problems for one concurrency block.""" + if concurrency is None: + return [] + if isinstance(concurrency, str): + concurrency = {"group": concurrency} + problems, seen = [], {} + for event_name, action, run_id in runs: + ctx = context_for(workflow_name, event_name, action, run_id) + group = to_str(evaluate(concurrency["group"], ctx)) + if group in seen: + other = seen[group] + cancel = truthy(evaluate(concurrency.get("cancel-in-progress", False), ctx)) + how = "cancels it in flight" if cancel else "replaces it in the queue" + problems.append( + f"{event_name}/{action} (run {run_id}) shares group {group!r} with " + f"{other[0]}/{other[1]} (run {other[2]}) on the same head SHA and {how}") + else: + seen[group] = (event_name, action, run_id) + return problems + + +def required_jobs(data): + for key, job in (data.get("jobs") or {}).items(): + name = job.get("name", key) if isinstance(job, dict) else key + if name in REQUIRED_CONTEXTS: + yield name, job + + +def audit(path): + data, events = load(path) + jobs = list(required_jobs(data)) + if not jobs: + return None, [] + runs = list(same_head_runs(data, events)) + if not runs: + return [n for n, _ in jobs], [] + name = data.get("name", path.name) + problems = collisions(name, data.get("concurrency"), runs) + for job_name, job in jobs: + problems += [f"job {job_name!r}: {p}" + for p in collisions(name, job.get("concurrency"), runs)] + return [n for n, _ in jobs], problems + + +class RequiredCheckConcurrency(unittest.TestCase): + def test_same_head_rechecks_never_share_a_group(self): + audited = {} + for path in sorted(WORKFLOWS.glob("*.yml")): + with self.subTest(workflow=path.name): + contexts, problems = audit(path) + if contexts: + audited[path.name] = contexts + self.assertEqual(problems, [], f"{path.name}: see #18578") + # Positive control: the label-triggered required checks this guard + # exists for must actually have been found and evaluated. + for file, context in ( + ("pr-verifier.yml", "verify PR contents"), + ("dco-squash-attribution.yml", "Sign-off survives the squash"), + ("changelog-fragment-guard.yml", "changelog-fragment-guard"), + ): + self.assertIn(context, audited.get(file, []), file) + + def test_guard_flags_the_pre_18578_shapes(self): + events = {"pull_request_target": {"types": ["opened", "synchronize", "labeled", "unlabeled"]}} + runs = list(same_head_runs({}, events)) + # pr-verifier.yml before #18578: per-PR group, cancel on PR events. + old_verifier = { + "group": "${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.ref }}", + "cancel-in-progress": "${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }}", + } + problems = collisions("PR Verifier", old_verifier, runs) + self.assertTrue(problems) + self.assertIn("cancels it in flight", problems[0]) + # Head SHA in the group alone still lets label runs on that SHA cancel + # each other. + sha_only = { + "group": "${{ github.workflow }}-${{ github.event.pull_request.number }}-${{ github.event.pull_request.head.sha }}", + "cancel-in-progress": True, + } + self.assertTrue(collisions("PR Verifier", sha_only, runs)) + # changelog-fragment-guard before #18578: no cancel on labels, but a + # shared group still drops queued runs. + queued = { + "group": "changelog-fragment-guard-${{ github.event.pull_request.number }}", + "cancel-in-progress": "${{ github.event.action == 'synchronize' }}", + } + problems = collisions("changelog-fragment-guard", queued, runs) + self.assertTrue(problems) + self.assertIn("replaces it in the queue", problems[0]) + + def test_head_moving_events_still_share_the_per_sha_group(self): + # The run_id suffix is for re-checks only: the push-triggered run keeps + # the plain per-PR, per-SHA group. + data, _ = load(WORKFLOWS / "pr-verifier.yml") + ctx = context_for("PR Verifier", "pull_request_target", "synchronize", 1) + self.assertEqual(evaluate(data["concurrency"]["group"], ctx), + f"PR Verifier-{PR_NUMBER}-{HEAD_SHA}") + ctx = context_for("PR Verifier", "pull_request_target", "labeled", 7) + self.assertEqual(evaluate(data["concurrency"]["group"], ctx), + f"PR Verifier-{PR_NUMBER}-{HEAD_SHA}-labeled-7") + self.assertIs(data["concurrency"]["cancel-in-progress"], False) + + def test_evaluator_rejects_what_it_cannot_model(self): + with self.assertRaises(ExprError): + evaluate("${{ hashFiles('x') }}", context_for("w", "pull_request", "opened", 1)) + with self.assertRaises(ExprError): + evaluate("${{ needs.a.outputs.b }}", context_for("w", "pull_request", "opened", 1)) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/changelog-fragment-guard.yml b/.github/workflows/changelog-fragment-guard.yml index 850a0da40f..235d88096b 100644 --- a/.github/workflows/changelog-fragment-guard.yml +++ b/.github/workflows/changelog-fragment-guard.yml @@ -72,8 +72,14 @@ permissions: # was left with CANCELLED check-runs for this *required* context. GitHub # scores every run of a required name, so one cancelled sibling kept the PR # at mergeStateStatus=BLOCKED with a green survivor beside it (#11250). +# cancel-in-progress false is not enough on its own: a queued run in a shared +# group is cancelled by the next run to queue, so a burst of three or four +# labels still left CANCELLED siblings (#18578). labeled/unlabeled/reopened +# runs therefore get their own group (github.run_id) and can neither cancel +# nor be cancelled; .github/scripts/test-required-check-concurrency.py +# enforces it. concurrency: - group: changelog-fragment-guard-${{ github.event.pull_request.number }} + group: changelog-fragment-guard-${{ github.event.pull_request.number }}${{ contains(fromJSON('["labeled","unlabeled","reopened"]'), github.event.action) && format('-{0}-{1}', github.event.action, github.run_id) || '' }} cancel-in-progress: ${{ github.event.action == 'synchronize' }} jobs: @@ -88,8 +94,8 @@ jobs: # a malformed fragment (#14629), which then broke the release `decide` # job on v5 and held every open v5 PR behind a red base. The guard is # cheap (git, grep, echo); re-running it on each label event keeps the - # reported verdict truthful. Label runs are not cancelled by concurrency - # (see above), so they cannot leave CANCELLED check-runs either. + # reported verdict truthful. Label runs each get their own concurrency + # group (see above), so they cannot leave CANCELLED check-runs either. # Forward-merge sync PRs (v5-topup.yml / v6-topup.yml, and the hand-rolled # ones maintainers open when those conflict) live on sync/* branches — # or on /sync-vN-to-vM when an agent opens the top-up, since the diff --git a/.github/workflows/dco-squash-attribution.yml b/.github/workflows/dco-squash-attribution.yml index 0445602975..0594c12bd6 100644 --- a/.github/workflows/dco-squash-attribution.yml +++ b/.github/workflows/dco-squash-attribution.yml @@ -22,9 +22,24 @@ on: types: [opened, synchronize, reopened, labeled, unlabeled] +# Required context: never CANCEL a run on the head SHA (#18578). GitHub scores +# every check-run of a required name on a commit, so one cancelled sibling +# leaves the PR BLOCKED beside a green survivor and Tide jams on it (#11250). +# The verdict does depend on labels (merge-method/merge), so label events keep +# re-running the job in full, but: +# - the group carries the head SHA, so runs on different commits never meet; +# - labeled/unlabeled/reopened runs also carry github.run_id, so each runs +# in its own group: a burst of label flips can neither cancel the run in +# flight nor have its queued runs cancelled by the next one to queue +# (GitHub does that to pending runs even with cancel-in-progress false). +# Every run reads the label set from its own event payload, and the +# newest run carries the newest set. +# - runs on a superseded commit finish (5 min cap) rather than being +# cancelled; their verdict lands on that old SHA, which never merges. +# .github/scripts/test-required-check-concurrency.py enforces this. concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}${{ contains(fromJSON('["labeled","unlabeled","reopened"]'), github.event.action) && format('-{0}-{1}', github.event.action, github.run_id) || '' }} + cancel-in-progress: false permissions: contents: read diff --git a/.github/workflows/pr-auto-update-test.yml b/.github/workflows/pr-auto-update-test.yml index 011d3c6c78..118e3ec806 100644 --- a/.github/workflows/pr-auto-update-test.yml +++ b/.github/workflows/pr-auto-update-test.yml @@ -17,6 +17,9 @@ on: - '.github/workflows/v2-tests.yml' - '.github/workflows/go-security-analysis.yml' - '.github/workflows/pr-verifier.yml' + # Any workflow edit can change a required check's concurrency (#18578). + - '.github/scripts/test-required-check-concurrency.py' + - '.github/workflows/*.yml' push: branches: [v5, v6] paths: @@ -34,6 +37,9 @@ on: - '.github/workflows/v2-tests.yml' - '.github/workflows/go-security-analysis.yml' - '.github/workflows/pr-verifier.yml' + # Any workflow edit can change a required check's concurrency (#18578). + - '.github/scripts/test-required-check-concurrency.py' + - '.github/workflows/*.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -63,3 +69,5 @@ jobs: sudo apt-get install -y -qq python3-yaml - name: Check native queue workflow contracts run: python3 .github/scripts/test-native-queue-workflows.py + - name: Required checks are never cancelled by same-head re-checks (#18578) + run: python3 .github/scripts/test-required-check-concurrency.py diff --git a/.github/workflows/pr-verifier.yml b/.github/workflows/pr-verifier.yml index c01821d511..e3fe5dcc90 100644 --- a/.github/workflows/pr-verifier.yml +++ b/.github/workflows/pr-verifier.yml @@ -19,7 +19,7 @@ name: PR Verifier # and the run's check lands on the branch tip, i.e. the pushed head. Only # same-repo branches are ever dispatched (the autofix never pushes to # forks); such a run uses the GitHub-hosted ARM runner like fork PRs do. -# The concurrency group below bounds the cost. +# The concurrency group below bounds the cost without cancelling anything. on: pull_request_target: types: [opened, edited, synchronize, reopened, ready_for_review, labeled, unlabeled] @@ -30,9 +30,24 @@ on: required: true type: string +# Required context: never CANCEL a run on the head SHA (#18578). GitHub scores +# every check-run of a required name on a commit, so one cancelled sibling +# leaves the PR BLOCKED beside a green survivor and Tide jams on it (#11250). +# A per-PR group with cancel-in-progress let every label flip (lgtm, approved, +# size, stage) cancel the run in flight, and a queued run in a shared group is +# cancelled by the next one to queue even with cancel-in-progress false. So: +# - the group carries the head SHA, so runs on different commits never meet; +# - events that re-check the same head (labeled, unlabeled, edited, +# reopened, ready_for_review) also carry github.run_id: each runs in its +# own group and can neither cancel nor be cancelled. The wrapper is green +# whatever the label set (every verdict step is continue-on-error), and +# the label types stay for #17036, not for the verdict. +# - runs on a superseded commit finish (10 min cap) rather than being +# cancelled; their verdict lands on that old SHA, which never merges. +# .github/scripts/test-required-check-concurrency.py enforces this. concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.ref }} - cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_target' }} + group: ${{ github.workflow }}-${{ github.event.pull_request.number || inputs.pr_number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}${{ contains(fromJSON('["labeled","unlabeled","edited","reopened","ready_for_review"]'), github.event.action) && format('-{0}-{1}', github.event.action, github.run_id) || '' }} + cancel-in-progress: false permissions: checks: write diff --git a/changelog.d/fixed-18578-required-check-label-cancel.md b/changelog.d/fixed-18578-required-check-label-cancel.md new file mode 100644 index 0000000000..2ce7cb508b --- /dev/null +++ b/changelog.d/fixed-18578-required-check-label-cancel.md @@ -0,0 +1 @@ +- CI: label flips no longer cancel the required `verify PR contents`, `Sign-off survives the squash` and `changelog-fragment-guard` checks. Their concurrency groups now carry the head SHA, and label, edit and reopen re-checks each run in their own group, so a burst of `lgtm`/`approved`/size labels can neither cancel the run in flight nor drop a queued run. Those cancelled runs used to leave PRs BLOCKED and jam Tide. A new guard, `.github/scripts/test-required-check-concurrency.py`, evaluates every required-check workflow's concurrency for a label burst and fails on any shared group (#18578).