From 4ef00ec41f51aa74904a7d20fde4bebdff021ed8 Mon Sep 17 00:00:00 2001 From: Joe Harris <57046+joeharris76@users.noreply.github.com> Date: Fri, 10 Jul 2026 08:38:21 -0400 Subject: [PATCH 1/2] fix(ci): retarget seed-corpus PR from main to develop `seed-corpus.yml` opened its automated bundle PR with `--base main`. That was already broken: the head it pushes is `chore/seed-corpus--`, and `validate-main-pr.yml` rejects any PR into `main` whose head is not a `vX.Y.Z` release branch. It would also hard-fail outright once the main -> release rename (PR #1027) removes the `main` branch. `develop` is the intended base: `sync-results-data-to-published.yml` triggers on pushes to `develop` under `results-data/bundles/**` and documents "a maintainer-run seed corpus refresh from `seed-corpus.yml`" as its motivating case. Correct the PR body's "Merging" section to match that flow -- it claimed the merge triggers `docs`, which only fires on pushes to the release branch. The workflow is `workflow_dispatch`-only and `release-cut` curates it out of the release tree, so no CI job covers this. Add unit tests pinning the base to `develop` and encoding why a `chore/` head can never target the release branch. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/seed-corpus.yml | 7 +- .../workflows/test_seed_corpus_pr_base.py | 74 +++++++++++++++++++ 2 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 tests/unit/workflows/test_seed_corpus_pr_base.py diff --git a/.github/workflows/seed-corpus.yml b/.github/workflows/seed-corpus.yml index 8d2fcb64b..d3febe3c9 100644 --- a/.github/workflows/seed-corpus.yml +++ b/.github/workflows/seed-corpus.yml @@ -277,11 +277,12 @@ jobs: ### Merging - Merge triggers the \`docs\` workflow which rebuilds the read model and - redeploys the results explorer. + Merge into \`develop\` triggers \`sync-results-data-to-published\`, which + opens a draft PR mirroring these bundles onto the \`published-results\` + corpus branch. EOF )" \ - --base main \ + --base develop \ --head "${{ steps.commit.outputs.branch }}" - name: No changes to commit diff --git a/tests/unit/workflows/test_seed_corpus_pr_base.py b/tests/unit/workflows/test_seed_corpus_pr_base.py new file mode 100644 index 000000000..9b4573bdf --- /dev/null +++ b/tests/unit/workflows/test_seed_corpus_pr_base.py @@ -0,0 +1,74 @@ +"""Guardrails for the seed-corpus workflow's PR base branch. + +`seed-corpus.yml` is `workflow_dispatch`-only and `release-cut` curates it out +of the release tree, so no CI job ever exercises its `gh pr create` call. These +tests stand in for that missing coverage: they pin the base branch to `develop` +and encode why the release-only branch can never be a valid target. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + +pytestmark = [ + pytest.mark.unit, + pytest.mark.fast, +] + +REPO_ROOT = Path(__file__).resolve().parents[3] +WORKFLOW_PATH = REPO_ROOT / ".github" / "workflows" / "seed-corpus.yml" + +# Mirrors the head-branch gate on the release-only branch (`validate-main-pr.yml`, +# renamed to `validate-release-pr.yml` by the main -> release rename). Duplicated +# rather than parsed out of that workflow so this test does not break when the +# rename lands. Only `vX.Y.Z` heads may merge into the release branch. +RELEASE_HEAD_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9.+-]+)?$") + + +def _create_pr_steps() -> dict[str, str]: + """Map step name -> `run` script for the workflow's `create-pr` job.""" + workflow = yaml.safe_load(WORKFLOW_PATH.read_text(encoding="utf-8")) + steps = workflow["jobs"]["create-pr"]["steps"] + return {step["name"]: step.get("run", "") for step in steps if "name" in step} + + +def _rendered_head_branch(commit_script: str) -> str: + """Render the `BRANCH=` template the commit step pushes as the PR head.""" + match = re.search(r'BRANCH="([^"]+)"', commit_script) + assert match is not None, "could not find the BRANCH= assignment in the commit step" + branch = match.group(1) + branch = branch.replace("${{ github.run_number }}", "123") + return branch.replace("${DATE}", "20260710") + + +def test_seed_corpus_pr_targets_develop() -> None: + """The seed corpus PR opens against `develop`, the default branch. + + `main` is release-only and rejects non-`vX.Y.Z` heads; it also disappears + once the main -> release rename lands, which would hard-fail `gh pr create`. + """ + run = _create_pr_steps()["Open pull request"] + + assert "--base develop" in run + assert "--base main" not in run + assert "--base release" not in run + + +def test_seed_corpus_head_branch_can_never_target_the_release_branch() -> None: + """The pushed head is a `chore/` branch, so the release branch would reject it.""" + steps = _create_pr_steps() + head = _rendered_head_branch(steps["Create PR branch and commit"]) + + assert head.startswith("chore/seed-corpus-") + assert not RELEASE_HEAD_RE.match(head) + + +def test_release_head_regex_matches_real_release_branches() -> None: + """Guard the mirrored regex itself, so the test above cannot pass vacuously.""" + assert RELEASE_HEAD_RE.match("v0.3.1") + assert RELEASE_HEAD_RE.match("v1.2.3-rc.1") + assert not RELEASE_HEAD_RE.match("chore/seed-corpus-20260710-123") From 044e3459a4a14e3d6fe6653d5c1ea41cc3dba275 Mon Sep 17 00:00:00 2001 From: Joe Harris <57046+joeharris76@users.noreply.github.com> Date: Fri, 10 Jul 2026 09:14:34 -0400 Subject: [PATCH 2/2] fix(ci): unescape backticks in seed-corpus PR body The body is built with a quoted heredoc (`<<'EOF'`), so the shell performs no expansion inside it. The `\`` escapes were therefore not protecting anything -- they rendered as literal backslashes in the PR body. Drop them and add a guard test. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/seed-corpus.yml | 10 +++++----- tests/unit/workflows/test_seed_corpus_pr_base.py | 13 +++++++++++++ 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/seed-corpus.yml b/.github/workflows/seed-corpus.yml index d3febe3c9..baaab25fa 100644 --- a/.github/workflows/seed-corpus.yml +++ b/.github/workflows/seed-corpus.yml @@ -264,21 +264,21 @@ jobs: --body "$(cat <<'EOF' ## Seed corpus update - Automated PR from the \`seed-corpus\` workflow (run [${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})). + Automated PR from the `seed-corpus` workflow (run [${{ github.run_number }}](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})). ### Bundles included - New or updated result bundles are committed under \`results-data/bundles/\`. + New or updated result bundles are committed under `results-data/bundles/`. Review the diff to confirm expected benchmark × platform × scale coverage. ### Trust label - All bundles in this PR carry the \`maintainer-run\` / \`public-curated\` trust label. + All bundles in this PR carry the `maintainer-run` / `public-curated` trust label. ### Merging - Merge into \`develop\` triggers \`sync-results-data-to-published\`, which - opens a draft PR mirroring these bundles onto the \`published-results\` + Merge into `develop` triggers `sync-results-data-to-published`, which + opens a draft PR mirroring these bundles onto the `published-results` corpus branch. EOF )" \ diff --git a/tests/unit/workflows/test_seed_corpus_pr_base.py b/tests/unit/workflows/test_seed_corpus_pr_base.py index 9b4573bdf..1fd2f2c2b 100644 --- a/tests/unit/workflows/test_seed_corpus_pr_base.py +++ b/tests/unit/workflows/test_seed_corpus_pr_base.py @@ -67,6 +67,19 @@ def test_seed_corpus_head_branch_can_never_target_the_release_branch() -> None: assert not RELEASE_HEAD_RE.match(head) +def test_seed_corpus_pr_body_does_not_escape_backticks() -> None: + """The PR body heredoc is quoted (`<<'EOF'`), so nothing inside is expanded. + + Backslash-escaping a backtick there is not protection, it is a literal + backslash in the rendered PR body. + """ + run = _create_pr_steps()["Open pull request"] + body = run.split("<<'EOF'", 1)[1].split("\nEOF", 1)[0] + + assert "`" in body, "sanity: the body is expected to contain backticks" + assert "\\`" not in body + + def test_release_head_regex_matches_real_release_branches() -> None: """Guard the mirrored regex itself, so the test above cannot pass vacuously.""" assert RELEASE_HEAD_RE.match("v0.3.1")