From 428e3eba0b8ba687a91a66e9f9bed1698e6c9f92 Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Sun, 8 Nov 2026 07:39:09 -0500 Subject: [PATCH] Refuse a merge-gate run that examined nothing it could verify On 2026-11-03 the gate ran with a token that carried no checks: read, because bot/policy.json had not been widened yet, and six of the ten org jobs still reported success: those orgs had no open Dependabot pull requests, so they never called the API at all. Only the four orgs that happened to have pull requests went red, which read as four isolated org problems rather than one blind control plane. Had the branch sweep left nothing open anywhere, the run would have been ten green jobs over a gate that could not read a single check. A quiet run now proves it could have read a check. With nothing examined the gate exercises the same check-runs endpoint against a real commit and exits non-zero if that read fails, while distinguishing the two cases that are not failures: an account with no repository to probe, and an empty repository answering 409. Where pull requests were examined and every one of them errored, the summary names it as a single systemic failure. This is the "assert non-vacuity first" rule the estate already applies to its other discovery-driven checks, and the same shape as verify-release-publish.py, whose clean answer is also an empty result. --- dependabot/merge.py | 62 ++++++++++++++++++++++++++++++++++ dependabot/tests/test_merge.py | 51 ++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/dependabot/merge.py b/dependabot/merge.py index 5aba4ec..1620a1d 100644 --- a/dependabot/merge.py +++ b/dependabot/merge.py @@ -176,6 +176,47 @@ def floor_for(gh, repo): return len(check_runs(gh, repo, last["head"]["sha"])) +def checks_readable(gh, owner, kind): + """Positive control for the `checks: read` this gate depends on. + + A run with no open Dependabot pull requests examines nothing, so its exit code reports + only that the search came back empty -- never that the gate *could* have read a check had + there been one. On 2026-11-03 that distinction was the whole story: six of ten org jobs + reported `0 open Dependabot pull requests` and went green while the minted token carried no + `checks: read` at all, because the ceiling in bot/policy.json had not been widened yet. The + run read as four isolated org failures instead of a control plane that could not read a + single check, and had the branch sweep left nothing open anywhere it would have been ten + green jobs over a gate that was entirely blind. + + So exercise the same endpoint check_runs() uses, against a real commit. Returns + (ok, detail): True verified, False the permission is genuinely missing, and None when the + account simply has no repository to probe -- which is not a permission failure and must not + be reported as one. + """ + listing = f"{'orgs' if kind == 'org' else 'users'}/{owner}/repos?per_page=5&sort=pushed" + try: + repos = gh.api(listing) or [] + except Exception as exc: + return False, f"cannot list repositories: {str(exc)[:200]}" + + last = None + for repo in repos: + full, branch = repo.get("full_name"), repo.get("default_branch") + if not full or not branch: + continue + try: + check_runs(gh, full, branch) + return True, f"verified on {full}@{branch}" + except Exception as exc: + detail = str(exc)[:200] + # An empty repository answers 409 and proves nothing either way; keep looking. + if "empty" in detail.lower() or "409" in detail: + last = f"{full} is empty" + continue + return False, f"cannot read check runs on {full}@{branch}: {detail}" + return None, last or "no repository with a default branch to probe" + + def dependabot_prs(gh, owner, kind): qualifier = "org" if kind == "org" else "user" q = f"{qualifier}:{owner}+is:pr+is:open+author:app/dependabot" @@ -228,6 +269,27 @@ def main(argv=None, gh=None): mode = "apply" if opts.apply else "dry run" text = (f"## Dependabot merge gate: {opts.owner} ({mode})\n\n{len(rows)} open Dependabot pull requests.\n\n" + common.markdown_table(["repo", "PR", "action", "reason"], rows)) + + # Non-vacuity. A run that examined nothing, or whose every examined pull request errored, + # has established nothing about this gate's own access -- so it must not exit 0 on the + # strength of an empty result. See checks_readable() for the run this is written against. + examined = [r for r in rows if r[2] != "skip"] + if not examined: + ok, detail = checks_readable(gh, opts.owner, kind) + if ok is False: + errors += 1 + text += (f"\n**Nothing examined, and the checks API is not readable — {detail}.** " + "This is a gate failure, not an idle week: with no readable checks every " + "pull request would be refused, so a quiet run cannot be trusted as green.\n") + elif ok is None: + text += f"\nNothing examined; no positive control available ({detail}).\n" + else: + text += f"\nNothing examined; checks API confirmed readable ({detail}).\n" + elif errors and errors == len(examined): + text += (f"\n**Every one of the {len(examined)} pull requests examined errored.** Treat this " + "as one gate failure rather than N unrelated ones — a systemic cause (a token " + "missing `checks: read`, say) presents exactly this way.\n") + print(text) common.write_summary(text) return 1 if errors else 0 diff --git a/dependabot/tests/test_merge.py b/dependabot/tests/test_merge.py index c68d191..6a90df1 100644 --- a/dependabot/tests/test_merge.py +++ b/dependabot/tests/test_merge.py @@ -164,5 +164,56 @@ def test_client_account_refused(self): merge.main(["--owner", "adelsaiq"], gh=FakeGh({})) +class NonVacuityTest(unittest.TestCase): + """A quiet run must prove it could have read a check, not merely that it read none. + + The gate went 6/10 green on 2026-11-03 while holding no `checks: read`, because those six + orgs had no open Dependabot pull requests and so never called the API. These cases pin the + distinction between 'nothing to do' and 'cannot see anything'. + """ + + EMPTY_SEARCH = {"search/issues": [{"items": []}]} + + def repos_route(self, **extra): + routes = dict(self.EMPTY_SEARCH) + routes["orgs/laranail/repos"] = [{"full_name": "laranail/emojis", "default_branch": "main"}] + routes.update(extra) + return routes + + def test_quiet_run_passes_when_the_control_verifies(self): + gh = FakeGh(self.repos_route(**{"repos/laranail/emojis/commits/main/check-runs": [{"check_runs": []}]})) + self.assertEqual(merge.main(["--owner", "laranail"], gh=gh), 0) + + def test_quiet_run_FAILS_when_checks_are_unreadable(self): + # No route for the check-runs path, so FakeGh raises -- the shape of a 403. + gh = FakeGh(self.repos_route()) + self.assertEqual(merge.main(["--owner", "laranail"], gh=gh), 1) + + def test_quiet_run_passes_when_there_is_nothing_to_probe(self): + # An account with no repositories cannot be probed; that is not a permission failure. + gh = FakeGh(dict(self.EMPTY_SEARCH, **{"orgs/laranail/repos": []})) + self.assertEqual(merge.main(["--owner", "laranail"], gh=gh), 0) + + def test_an_empty_repository_is_not_read_as_a_permission_failure(self): + class EmptyRepoGh(FakeGh): + def api(self, path, paginate=False): + if "/check-runs" in path: + raise common.GhError("gh: Git Repository is empty. (HTTP 409)") + return super().api(path, paginate=paginate) + + self.assertEqual(merge.main(["--owner", "laranail"], gh=EmptyRepoGh(self.repos_route())), 0) + + def test_control_reports_absence_not_just_presence(self): + # The negative half: a control that can only ever say "readable" proves nothing. + ok, detail = merge.checks_readable(FakeGh(self.repos_route()), "laranail", "org") + self.assertIs(ok, False) + self.assertIn("cannot read check runs", detail) + + def test_every_examined_pr_erroring_is_called_out_as_systemic(self): + gh = FakeGh({"search/issues": [{"items": [ + {"repository_url": "https://api.github.com/repos/laranail/emojis", "number": 7}]}]}) + self.assertEqual(merge.main(["--owner", "laranail"], gh=gh), 1) + + if __name__ == "__main__": unittest.main()