From 6e63141a947c9699084c7065760fa3f8e701c703 Mon Sep 17 00:00:00 2001 From: Imani Manyara Date: Sun, 8 Nov 2026 08:32:15 -0500 Subject: [PATCH] Probe a private repository, or the control verifies the wrong thing checks: read is enforced on private repositories only, so the positive control added in #23 can report "readable" against a public repo while the gate is blind to every repository it actually works on. The estate's Dependabot pull requests sit overwhelmingly in private repos, so that is not a corner case. The 2026-11-03 run is the proof, and it was misread when #23 was written. The simtabi job examined one pull request and it passed -- on the public simtabi/cli-kit-js, with the same token that answered 403 for every private repository across four other orgs. So "every pull request the gate examined errored" was wrong: the one in a public repo succeeded, and the discriminator was visibility, not the org. Measured against the control as merged: for simtabi and ichava it picks a public repo, so for those two it would have passed while private repos were unreadable. The other six pass only by luck of which repository was pushed most recently, which changes constantly. Sort private-first over a wide enough page, name the visibility that was verified, and where only a public repository was available say that the read does not exercise checks: read. An account with no private repository has no private blindness to detect, so a public probe is complete for it. --- dependabot/merge.py | 22 +++++++++++++++++++--- dependabot/tests/test_merge.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/dependabot/merge.py b/dependabot/merge.py index 1620a1d..ab64702 100644 --- a/dependabot/merge.py +++ b/dependabot/merge.py @@ -192,13 +192,25 @@ def checks_readable(gh, owner, kind): (ok, detail): True verified, False the permission is genuinely missing, and None when the account simply has no repository to probe -- which is not a permission failure and must not be reported as one. + + The probe must prefer a PRIVATE repository, because `checks: read` is only enforced on one. + The same 2026-11-03 run proves it: `simtabi` examined one pull request and it passed, on the + public `simtabi/cli-kit-js`, with the very token that answered 403 for every private repo in + the other four orgs. A control that happens to pick a public repo therefore reports + "readable" while the gate is blind to every repository it actually works on -- the estate's + Dependabot pull requests sit overwhelmingly in private ones. Sorting private-first over a + wide-enough page closes that; where an account has only public repositories there is no + private blindness to detect and a public probe is complete for it. """ - listing = f"{'orgs' if kind == 'org' else 'users'}/{owner}/repos?per_page=5&sort=pushed" + listing = f"{'orgs' if kind == 'org' else 'users'}/{owner}/repos?per_page=30&sort=pushed" try: repos = gh.api(listing) or [] except Exception as exc: return False, f"cannot list repositories: {str(exc)[:200]}" + # False sorts before True, so private repositories come first. + repos = sorted(repos, key=lambda r: not r.get("private")) + last = None for repo in repos: full, branch = repo.get("full_name"), repo.get("default_branch") @@ -206,7 +218,8 @@ def checks_readable(gh, owner, kind): continue try: check_runs(gh, full, branch) - return True, f"verified on {full}@{branch}" + scope = "private" if repo.get("private") else "public" + return True, f"verified on {full}@{branch} ({scope})" except Exception as exc: detail = str(exc)[:200] # An empty repository answers 409 and proves nothing either way; keep looking. @@ -284,7 +297,10 @@ def main(argv=None, gh=None): elif ok is None: text += f"\nNothing examined; no positive control available ({detail}).\n" else: - text += f"\nNothing examined; checks API confirmed readable ({detail}).\n" + caveat = "" if "(private)" in detail else ( + " Only a public repository was available to probe, which does not exercise " + "`checks: read` — that is enforced on private repositories only.") + text += f"\nNothing examined; checks API confirmed readable ({detail}).{caveat}\n" elif errors and errors == len(examined): text += (f"\n**Every one of the {len(examined)} pull requests examined errored.** Treat this " "as one gate failure rather than N unrelated ones — a systemic cause (a token " diff --git a/dependabot/tests/test_merge.py b/dependabot/tests/test_merge.py index 6a90df1..d1867bf 100644 --- a/dependabot/tests/test_merge.py +++ b/dependabot/tests/test_merge.py @@ -203,6 +203,40 @@ def api(self, path, paginate=False): self.assertEqual(merge.main(["--owner", "laranail"], gh=EmptyRepoGh(self.repos_route())), 0) + def test_control_prefers_a_private_repository(self): + """`checks: read` is enforced on private repos only, so probing a public one can + report 'readable' while the gate is blind to every repo it actually works on. That is + what happened on 2026-11-03: simtabi's single pull request passed on the public + cli-kit-js with the same token that 403'd on four orgs' private repos.""" + routes = dict(self.EMPTY_SEARCH) + routes["orgs/laranail/repos"] = [ + {"full_name": "laranail/pub", "default_branch": "main", "private": False}, + {"full_name": "laranail/priv", "default_branch": "main", "private": True}, + ] + # Only the PUBLIC repo is readable -- the private one 403s, as in the real run. + routes["repos/laranail/pub/commits/main/check-runs"] = [{"check_runs": []}] + gh = FakeGh(routes) + ok, detail = merge.checks_readable(gh, "laranail", "org") + self.assertIs(ok, False, "a public-only read must not pass as verification") + self.assertIn("laranail/priv", detail) + + def test_control_names_the_visibility_it_verified(self): + routes = dict(self.EMPTY_SEARCH) + routes["orgs/laranail/repos"] = [{"full_name": "laranail/priv", "default_branch": "main", "private": True}] + routes["repos/laranail/priv/commits/main/check-runs"] = [{"check_runs": []}] + ok, detail = merge.checks_readable(FakeGh(routes), "laranail", "org") + self.assertIs(ok, True) + self.assertIn("(private)", detail) + + def test_a_public_only_account_still_verifies(self): + # No private repository means no private blindness to detect; a public probe is complete. + routes = dict(self.EMPTY_SEARCH) + routes["orgs/laranail/repos"] = [{"full_name": "laranail/pub", "default_branch": "main", "private": False}] + routes["repos/laranail/pub/commits/main/check-runs"] = [{"check_runs": []}] + ok, detail = merge.checks_readable(FakeGh(routes), "laranail", "org") + self.assertIs(ok, True) + self.assertIn("(public)", detail) + def test_control_reports_absence_not_just_presence(self): # The negative half: a control that can only ever say "readable" proves nothing. ok, detail = merge.checks_readable(FakeGh(self.repos_route()), "laranail", "org")