From 85289953172dd10527080b3eab963fe65817fd40 Mon Sep 17 00:00:00 2001 From: Joris Wouter Jonkers Date: Thu, 22 Oct 2026 12:53:07 +0200 Subject: [PATCH] feat: add a reusable PR title and agent attribution check --- .github/workflows/ci.yml | 8 ++ .github/workflows/pr-title-check.yml | 45 ++++++++ README.md | 1 + release-please-config.json | 4 + scripts/pr_title_check.py | 152 +++++++++++++++++++++++++++ tests/test_pr_title_check.py | 146 +++++++++++++++++++++++++ 6 files changed, 356 insertions(+) create mode 100644 .github/workflows/pr-title-check.yml create mode 100644 scripts/pr_title_check.py create mode 100644 tests/test_pr_title_check.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0205b0..de679d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,6 +128,14 @@ jobs: .venv-tests/bin/python -m coverage run --branch --source=scripts.quarantine_check -m unittest tests.test_quarantine_check .venv-tests/bin/python -m coverage report --show-missing --fail-under=100 + # A wrong verdict either blocks every pull request in the estate or lets agent attribution + # through, so every line and branch of it is held. + - name: Run PR title check tests with branch coverage + if: ${{ !cancelled() }} + run: | + .venv-tests/bin/python -m coverage run --branch --source=scripts.pr_title_check -m unittest tests.test_pr_title_check + .venv-tests/bin/python -m coverage report --show-missing --fail-under=100 + # --------------------------------------------------------------------------- # Terminal aggregator. This is the single required check in the ruleset. # It runs even when an upstream job fails/cancels (if: always()) and then diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml new file mode 100644 index 0000000..3837bcb --- /dev/null +++ b/.github/workflows/pr-title-check.yml @@ -0,0 +1,45 @@ +name: PR Title Check + +# Fails a pull request whose title or commits are not Conventional Commits, or whose body or +# commits credit a coding agent (scripts/pr_title_check.py). Call it from a workflow triggered +# by pull_request with the types opened, edited, reopened and synchronize, so a retitle re-runs +# it, and list the calling job in Pipeline Complete's needs. + +on: + workflow_call: {} + +permissions: + contents: read + +jobs: + check: + name: PR title + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + # The pull request's head, not the merge commit, so the range holds only its own commits. + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ github.event.pull_request.head.sha }} + fetch-depth: 0 + persist-credentials: false + + # Checkout github-workflows at this workflow file's own released version; release-please + # keeps the literal in step with the tag this file ships in. + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: JorisJonkers-dev/github-workflows + ref: v0.34.3 # x-release-please-version + path: .github-workflows + persist-credentials: false + + - name: Check the title, the body and every commit + env: + PR_TITLE: ${{ github.event.pull_request.title }} + PR_BODY: ${{ github.event.pull_request.body }} + BASE_REF: ${{ github.event.pull_request.base.ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + base="$(git merge-base "origin/${BASE_REF}" "${HEAD_SHA}")" + python3 .github-workflows/scripts/pr_title_check.py --range "${base}..${HEAD_SHA}" diff --git a/README.md b/README.md index a8ef60f..801f36e 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ should call released tags instead of branches. | `nix-ci.yml` | Nix formatter, Statix, Deadnix, flake check, and optional validation. | | `jvm-ci.yml` | Gradle static lint/test workflow that fails Gradle warnings and deprecations. | | `quarantine-check.yml` | Fails the build on a quarantined test whose quarantine has expired, names no issue, or lasts more than 14 days. The marker is `quarantine: until YYYY-MM-DD`, in the reason of whatever skips the test. | +| `pr-title-check.yml` | Fails a pull request whose title or commits are not Conventional Commits, or whose body or commits credit a coding agent (a Co-Authored-By trailer, a "generated with" banner, or a session link). | | `mutation-report.yml` | Mutates the lines a pull request changes (gremlins for Go, Stryker for TypeScript and Vue) and keeps one comment listing the mutants that survived. Never fails the build. | | `docker-image-ci.yml` | Build a Docker image without publishing. | | `container-publish.yml` | Publish a GHCR image with release and sha tags, labelled `org.opencontainers.image.version` with the software inside it: the digest-pinned upstream base's tag, our release for our own software, or the `software-version` input. | diff --git a/release-please-config.json b/release-please-config.json index 3b5d06b..529752a 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -81,6 +81,10 @@ { "type": "generic", "path": ".github/workflows/publish-fragment.yml" + }, + { + "type": "generic", + "path": ".github/workflows/pr-title-check.yml" } ] } diff --git a/scripts/pr_title_check.py b/scripts/pr_title_check.py new file mode 100644 index 0000000..71f5724 --- /dev/null +++ b/scripts/pr_title_check.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Fail a pull request whose title or commits are not Conventional Commits, or that credits a coding agent. + +release-please reads the squash-merged title, so a title it cannot parse is a change the +changelog never shows. The estate also carries no agent attribution anywhere (workspace +CLAUDE.md), so the pull request body and every commit message in the range are scanned for a +Co-Authored-By trailer naming a coding agent, a "generated with" banner naming one, or a link +back to an agent session. Ported from deploy-kit's scripts/check-pr-title.ts, so every +repository applies the same rules. + + pr_title_check.py --title "feat(ci): add a workflow" [--body TEXT] [--range BASE..HEAD] + +The title and body fall back to PR_TITLE and PR_BODY in the environment, which is how the +reusable workflow passes them without quoting them through a shell. +""" + +from __future__ import annotations + +import argparse +import os +import re +import subprocess + +# The Conventional Commits types release-please understands. +COMMIT_TYPES = ("feat", "fix", "docs", "style", "refactor", "perf", "test", "build", "ci", "chore", "revert") + +CONVENTIONAL = re.compile(rf"^({'|'.join(COMMIT_TYPES)})(\([a-z0-9._/-]+\))?!?: .+") +# Lines release-please and GitHub write themselves, which follow no type. +GENERATED = re.compile(r"^(Merge\s|Revert\s|chore\(main\): release\s|Release\s)") + +# Brand terms only: Cody, Devin and Jules are names real people carry, so an agent known by a +# plain name is matched by its bot account instead. +AGENT_MARKERS = ( + "claude", + "anthropic", + "copilot", + "chatgpt", + "openai", + "codex", + "cursor", + "windsurf", + "aider", + "codeium", + "tabnine", + "devin-ai-integration", + "devin.ai", + "google-labs-jules", + "jules[bot]", + "amazon-q-developer", + "codewhisperer", +) + +SESSION_HOSTS = ( + "claude.ai", + "chatgpt.com", + "chat.openai.com", + "devin.ai", + "cursor.sh", + "windsurf.com", + "windsurf.ai", + "aider.chat", + "jules.google.com", +) + +CO_AUTHOR = re.compile(r"^co-authored-by:\s*(.+)$", re.IGNORECASE) +BANNER = re.compile(r"\bgenerated\s+(with|by|using)\b", re.IGNORECASE) +# A host matches only at a host boundary: after the scheme and any subdomains, and before `/`, +# `:`, whitespace or the end. Unanchored, `https://example.com/claude.ai/x` would match, and an +# unescaped dot would let `chatXopenai.com` match `chat.openai.com`. +SESSION_LINK = re.compile( + r"https?://(?:[a-z0-9-]+\.)*(?:" + "|".join(map(re.escape, SESSION_HOSTS)) + r")(?=[/:\s]|$)", + re.IGNORECASE, +) + + +def failure_reason(subject: str) -> str | None: + """Why a subject line is not a Conventional Commit, or None when it is one.""" + if GENERATED.match(subject) or CONVENTIONAL.match(subject): + return None + return ( + f'"{subject}" is not a Conventional Commit. Expected type(scope)!: subject, ' + f"e.g. feat(ci): add a workflow, with type one of {', '.join(COMMIT_TYPES)}." + ) + + +def agent_marker(text: str) -> str | None: + lower = text.lower() + return next((marker for marker in AGENT_MARKERS if marker in lower), None) + + +def attribution_findings(text: str) -> list[str]: + """Every line of text that credits a coding agent, each with the reason.""" + findings = [] + for raw in text.splitlines(): + line = raw.strip() + co_author = CO_AUTHOR.match(line) + marker = agent_marker(co_author.group(1)) if co_author else None + if marker: + findings.append(f'a Co-Authored-By trailer names a coding agent ("{marker}"); credit a person or remove it: {line}') + continue + marker = agent_marker(line) if BANNER.search(line) else None + if marker: + findings.append(f'a "generated with" banner names a coding agent ("{marker}"); remove it: {line}') + continue + if SESSION_LINK.search(line): + findings.append(f"a link back to a coding agent session; remove it: {line}") + return findings + + +def commit_messages(commit_range: str) -> list[str]: + """The full message of every commit in the range, newest first.""" + out = subprocess.run( + ["git", "log", "--format=%B%x00", commit_range], check=True, capture_output=True, text=True + ).stdout + return [message.strip("\n") for message in out.split("\0") if message.strip("\n")] + + +def check(title: str, body: str, messages: list[str]) -> list[str]: + """Every failure in the title, the body and the commit messages. Empty means clean.""" + failures = [] + if (why := failure_reason(title)) is not None: + failures.append(f"pull request title: {why}") + failures += [f"pull request body: {finding}" for finding in attribution_findings(body)] + for message in messages: + subject = message.splitlines()[0] + if (why := failure_reason(subject)) is not None: + failures.append(f'commit "{subject}": {why}') + failures += [f'commit "{subject}": {finding}' for finding in attribution_findings(message)] + return failures + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Check a pull request's title, body and commits.") + parser.add_argument("--title", default=os.environ.get("PR_TITLE", "")) + parser.add_argument("--body", default=os.environ.get("PR_BODY", "")) + parser.add_argument("--range", dest="commit_range", help="BASE..HEAD, the pull request's commits") + args = parser.parse_args(argv) + if not args.title: + parser.error("no title: pass --title or set PR_TITLE") + + messages = commit_messages(args.commit_range) if args.commit_range else [] + failures = check(args.title, args.body, messages) + for failure in failures: + print(f"::error::{failure}") + if failures: + return 1 + print(f"title, body and {len(messages)} commit(s) are conventional and carry no agent attribution") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/tests/test_pr_title_check.py b/tests/test_pr_title_check.py new file mode 100644 index 0000000..3ff0833 --- /dev/null +++ b/tests/test_pr_title_check.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import io +import os +import subprocess +import tempfile +import unittest +from contextlib import redirect_stderr, redirect_stdout +from unittest import mock + +from scripts.pr_title_check import attribution_findings, check, commit_messages, failure_reason, main + + +class ReadingATitle(unittest.TestCase): + def test_a_typed_subject_is_accepted(self) -> None: + self.assertIsNone(failure_reason("feat: add a workflow")) + + def test_a_scope_and_a_breaking_mark_are_accepted(self) -> None: + self.assertIsNone(failure_reason("fix(ci/release)!: drop the old input")) + + def test_every_type_release_please_reads_is_accepted(self) -> None: + for kind in ("feat", "fix", "docs", "style", "refactor", "perf", "test", "build", "ci", "chore", "revert"): + self.assertIsNone(failure_reason(f"{kind}: something"), kind) + + def test_release_and_merge_lines_are_accepted(self) -> None: + for subject in ("chore(main): release 0.35.0", "Merge branch 'main' into x", 'Revert "feat: x"', "Release 1.0"): + self.assertIsNone(failure_reason(subject), subject) + + def test_an_untyped_subject_is_refused_with_the_shape_expected(self) -> None: + reason = failure_reason("Add a workflow") + self.assertIn('"Add a workflow" is not a Conventional Commit', reason) + self.assertIn("feat(ci): add a workflow", reason) + + def test_an_unknown_type_is_refused(self) -> None: + self.assertIsNotNone(failure_reason("feature: add a workflow")) + + def test_an_uppercase_scope_is_refused(self) -> None: + self.assertIsNotNone(failure_reason("feat(CI): add a workflow")) + + def test_a_type_without_a_subject_is_refused(self) -> None: + self.assertIsNotNone(failure_reason("feat: ")) + + +class FindingAttribution(unittest.TestCase): + def test_a_co_author_trailer_naming_an_agent_is_found(self) -> None: + [finding] = attribution_findings("Co-Authored-By: Claude ") + self.assertIn('Co-Authored-By trailer names a coding agent ("claude")', finding) + + def test_a_human_co_author_is_not_found(self) -> None: + self.assertEqual(attribution_findings("Co-authored-by: Jules Verne "), []) + + def test_a_generated_with_banner_naming_an_agent_is_found(self) -> None: + [finding] = attribution_findings("🤖 Generated with Copilot") + self.assertIn('"generated with" banner names a coding agent ("copilot")', finding) + + def test_a_generated_by_line_naming_no_agent_is_not_found(self) -> None: + self.assertEqual(attribution_findings("Types generated by ogen from the OpenAPI file."), []) + + def test_a_session_link_is_found(self) -> None: + [finding] = attribution_findings("See https://claude.ai/code/session_123 for the session.") + self.assertIn("link back to a coding agent session", finding) + + def test_a_session_host_on_a_subdomain_is_found(self) -> None: + self.assertEqual(len(attribution_findings("https://app.devin.ai/sessions/1")), 1) + + def test_a_session_host_inside_another_hosts_path_is_not_found(self) -> None: + self.assertEqual(attribution_findings("https://example.com/claude.ai/x"), []) + + def test_a_host_one_character_off_is_not_found(self) -> None: + self.assertEqual(attribution_findings("https://chatXopenai.com/"), []) + + def test_ordinary_text_is_not_found(self) -> None: + self.assertEqual(attribution_findings("## Summary\n\nAdds a workflow.\n\n- one\n"), []) + + +class CheckingAPullRequest(unittest.TestCase): + def test_a_clean_pull_request_has_no_failures(self) -> None: + self.assertEqual(check("feat: x", "Body.", ["feat: x\n\nBody.", "fix: y"]), []) + + def test_each_failure_names_where_it_was_found(self) -> None: + failures = check( + "Add x", + "Generated with Claude Code", + ["wip", "fix: y\n\nCo-Authored-By: Codex "], + ) + self.assertEqual(len(failures), 4) + self.assertTrue(failures[0].startswith("pull request title: ")) + self.assertTrue(failures[1].startswith("pull request body: ")) + self.assertTrue(failures[2].startswith('commit "wip": ')) + self.assertTrue(failures[3].startswith('commit "fix: y": a Co-Authored-By')) + + +class ReadingCommits(unittest.TestCase): + def setUp(self) -> None: + self.dir = tempfile.TemporaryDirectory() + self.addCleanup(self.dir.cleanup) + self.cwd = os.getcwd() + os.chdir(self.dir.name) + self.addCleanup(os.chdir, self.cwd) + + def git(*args: str) -> None: + subprocess.run(["git", *args], check=True, capture_output=True) + + git("init", "-q", "-b", "main") + git("config", "user.email", "test@example.com") + git("config", "user.name", "Test") + git("commit", "-q", "--allow-empty", "-m", "chore: base") + git("commit", "-q", "--allow-empty", "-m", "feat: one\n\nWith a body.\n\nAnd a blank line.") + git("commit", "-q", "--allow-empty", "-m", "fix: two") + + def test_every_message_in_the_range_is_read_whole(self) -> None: + self.assertEqual( + commit_messages("HEAD~2..HEAD"), + ["fix: two", "feat: one\n\nWith a body.\n\nAnd a blank line."], + ) + + def test_main_passes_a_clean_range(self) -> None: + out = io.StringIO() + with redirect_stdout(out): + self.assertEqual(main(["--title", "feat: x", "--range", "HEAD~2..HEAD"]), 0) + self.assertIn("2 commit(s)", out.getvalue()) + + +class RunningTheCommand(unittest.TestCase): + def test_failures_are_reported_as_error_annotations(self) -> None: + out = io.StringIO() + with redirect_stdout(out): + self.assertEqual(main(["--title", "Add x"]), 1) + self.assertTrue(out.getvalue().startswith("::error::pull request title: ")) + + def test_title_and_body_are_read_from_the_environment(self) -> None: + out = io.StringIO() + env = {"PR_TITLE": "docs: x", "PR_BODY": "Generated with ChatGPT"} + with mock.patch.dict(os.environ, env), redirect_stdout(out): + self.assertEqual(main([]), 1) + self.assertIn("pull request body: ", out.getvalue()) + + def test_a_missing_title_is_a_usage_error(self) -> None: + with mock.patch.dict(os.environ, {"PR_TITLE": ""}), redirect_stderr(io.StringIO()): + with self.assertRaises(SystemExit) as raised: + main([]) + self.assertEqual(raised.exception.code, 2) + + +if __name__ == "__main__": + unittest.main()