diff --git a/.github/workflows/update-v1-tag.yml b/.github/workflows/update-v1-tag.yml index 7e30c7534..a8ae4e09a 100644 --- a/.github/workflows/update-v1-tag.yml +++ b/.github/workflows/update-v1-tag.yml @@ -11,7 +11,9 @@ # image under test. When the candidate fails, the same suites boot again with # the tests at the current v1 on the same digest (the baseline), and # scripts/v1_gate_compare.py moves v1 only if every candidate failure also -# fails at v1. Anything it cannot explain holds v1. +# fails at v1. Anything it cannot explain holds v1. The failures it accepts +# come from the image or the VM, so scripts/v1_gate_preexisting.py files each +# one as an issue for the hive to quarantine or fix (#1143, #1153). # # The range starts at v1, not at the push's previous commit: if a newer push # cancels a gated run, the next run still covers the cancelled change. @@ -226,7 +228,9 @@ jobs: with: ref: ${{ needs.plan.outputs.sha }} persist-credentials: false - sparse-checkout: scripts/v1_gate_compare.py + sparse-checkout: | + scripts/v1_gate_compare.py + scripts/v1_gate_preexisting.py sparse-checkout-cone-mode: false - name: Download candidate and baseline results @@ -242,7 +246,7 @@ jobs: run: | set -euo pipefail python3 scripts/v1_gate_compare.py artifacts "${IMAGE}" "${BASELINE_IMAGE}" \ - --summary compare.md | tee -a "$GITHUB_OUTPUT" + --summary compare.md --preexisting preexisting.json | tee -a "$GITHUB_OUTPUT" cat compare.md >> "$GITHUB_STEP_SUMMARY" - name: Move or hold v1 @@ -342,3 +346,49 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" echo "::error::v1 held at ${PREVIOUS:-none}: ${WHY}" exit 1 + + # A failure that also fails at v1 comes from the image or the VM, not + # from the merged change, so it never holds v1. Left alone it keeps every + # later run red, the PR VM check included, until someone notices it by + # hand (#1143, #1153). File each one for the hive, once while an issue + # with the same title is open. Runs after a hold too: the failures it + # files are explained by v1 either way. Never fails the job. + - name: File pre-existing failures + if: always() && steps.compare.outcome == 'success' + continue-on-error: true + run: | + set -euo pipefail + [ -s preexisting.json ] || exit 0 + python3 scripts/v1_gate_preexisting.py artifacts "${IMAGE}" preexisting.json "${RUN_URL}" > issues.jsonl + [ -s issues.jsonl ] || exit 0 + OPEN_TITLES=$(gh issue list --state open --limit 1000 --json title --jq '.[].title') + : > filed.md + while IFS= read -r ISSUE; do + TITLE=$(jq -r .title <<<"${ISSUE}") + if grep -qxF -- "${TITLE}" <<<"${OPEN_TITLES}"; then + echo "Already open: ${TITLE}" + continue + fi + jq -r .body <<<"${ISSUE}" > body.md + URL=$(gh issue create --title "${TITLE}" --body-file body.md) + NUM=${URL##*/} + # One issue for many failures is a maintainer's call (likely the + # image as a whole); a single scenario is ready for the hive. + if [ "$(jq -r .needs_human <<<"${ISSUE}")" = "true" ]; then + LABELS=(kind/bug needs-human) + else + LABELS=(kind/bug triage/accepted) + fi + for LABEL in "${LABELS[@]}"; do + gh issue edit "${NUM}" --add-label "${LABEL}" > /dev/null \ + || echo "::warning::could not label #${NUM} ${LABEL}" + done + echo "- #${NUM} ${TITLE}" >> filed.md + done < issues.jsonl + if [ -s filed.md ]; then + { + echo "### Pre-existing failures filed for the hive" + echo + cat filed.md + } >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/docs/skills/ci-ops/ops/references/candidate-vm-validation.md b/docs/skills/ci-ops/ops/references/candidate-vm-validation.md index 53204270f..237105172 100644 --- a/docs/skills/ci-ops/ops/references/candidate-vm-validation.md +++ b/docs/skills/ci-ops/ops/references/candidate-vm-validation.md @@ -116,6 +116,20 @@ and `.../check-runs`. Also `priority/critical-urgent`; - the issue has been open for more than 24 hours. Also `priority/critical-urgent`. +4. Files the failures it let through. A scenario that fails at both the + merged SHA and `v1` comes from the image or the VM, so it never holds + `v1`, but it keeps every later run red (the PR VM check included) until + someone deals with it. The step "File pre-existing failures" runs + `scripts/v1_gate_preexisting.py` on the `--preexisting` output of + `v1_gate_compare.py` and opens one issue per scenario, titled + `e2e: fails on , also at v1: :: `, + labelled `kind/bug` and `triage/accepted`, for the hive to skip in VMs, + quarantine or fix. It files a title only when no open issue has it, so a + failure seen on every run is filed once. More than five at once becomes one + issue, `e2e: many scenarios fail on , also at v1`, labelled + `needs-human`: that many points at the image as a whole. The step runs + after a hold too, and never fails the run. Before this, #1143 and #1153 + were found by hand. `lifecycle` is planned but not booted (`UNGATED_SUITES`): its upgrade scenarios need a second, newer image to upgrade to (#706, #1023), and on one diff --git a/scripts/v1_gate_compare.py b/scripts/v1_gate_compare.py index 9ee4bc66e..f8d2961a7 100755 --- a/scripts/v1_gate_compare.py +++ b/scripts/v1_gate_compare.py @@ -22,13 +22,17 @@ digest, so their suffixes differ. Usage: - python3 scripts/v1_gate_compare.py ARTIFACT_DIR CANDIDATE_IMAGE BASELINE_IMAGE [--summary FILE] + python3 scripts/v1_gate_compare.py ARTIFACT_DIR CANDIDATE_IMAGE BASELINE_IMAGE [--summary FILE] [--preexisting FILE] Prints ``verdict=move`` or ``verdict=hold`` and ``infrastructure=true`` or ``infrastructure=false`` for ``$GITHUB_OUTPUT``. ``infrastructure=true`` means no failed suite produced test results on either side: the test machines most likely never booted (a runner or install problem, as in #1123), so the hold is not a test regression and needs a maintainer rather than a contributor. + +``--preexisting FILE`` writes the failures that also fail at ``v1`` as JSON +(``{"suite": ["Feature :: Scenario", ...]}``) for +``scripts/v1_gate_preexisting.py``, which files them for the hive. """ from __future__ import annotations @@ -164,10 +168,15 @@ def main(argv: list[str]) -> int: parser.add_argument("candidate_image") parser.add_argument("baseline_image") parser.add_argument("--summary", type=Path) + parser.add_argument("--preexisting", type=Path) args = parser.parse_args(argv[1:]) result = compare(args.artifacts, args.candidate_image, args.baseline_image) if args.summary: args.summary.write_text(summary_markdown(result), encoding="utf-8") + if args.preexisting: + args.preexisting.write_text( + json.dumps(result["preexisting"], indent=2) + "\n", encoding="utf-8" + ) print(f"verdict={result['verdict']}") print(f"infrastructure={'true' if result['infrastructure'] else 'false'}") return 0 diff --git a/scripts/v1_gate_preexisting.py b/scripts/v1_gate_preexisting.py new file mode 100644 index 000000000..743340597 --- /dev/null +++ b/scripts/v1_gate_preexisting.py @@ -0,0 +1,211 @@ +#!/usr/bin/env python3 +"""Turn failures the v1 gate accepted as pre-existing into issues for the hive. + +When a candidate failure also fails with the tests at ``v1`` on the same image +digest, ``scripts/v1_gate_compare.py`` lets ``v1`` move: the failure comes from +the image (or the VM), not from the merged change. Left alone, such a failure +keeps every later run red, including the PR VM check, until someone notices +it by hand (#1143, #1153). This script writes one issue per failing scenario +so the hive quarantines or fixes it without a person finding it first. + +Usage: + python3 scripts/v1_gate_preexisting.py ARTIFACT_DIR CANDIDATE_IMAGE PREEXISTING_JSON RUN_URL + +``PREEXISTING_JSON`` is the ``--preexisting`` file from ``v1_gate_compare.py`` +(``{"suite": ["Feature :: Scenario", ...]}``). Prints one JSON object per line, +``{"title", "body", "needs_human"}``. The workflow files each title that has no +open issue yet, so a failure seen on every run is filed once. + +More than ``MAX_ISSUES`` scenarios at once is treated as one image-wide +problem (a broken build, a missing package set) and becomes a single issue for +a maintainer, rather than a flood of per-scenario issues. Its title leaves +out the count, so a later run with a different count finds it still open. +""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path + +# v1_gate_compare.py is a sibling script, not a package. Add this directory so +# the import works both when run as ``python3 scripts/v1_gate_preexisting.py`` +# and when imported as a module (pytest). +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from v1_gate_compare import FAILED_STATUSES, artifact_suffix # noqa: E402 + +MAX_ISSUES = 5 +TITLE_LIMIT = 200 + + +def image_name(image: str) -> str: + """The image without its digest: ghcr.io/org/name:tag@sha256:... -> org/name:tag.""" + name = image.split("@", 1)[0] + return name.split("/", 1)[1] if name.startswith("ghcr.io/") else name + + +def image_repo(image: str) -> str: + """The GitHub repository that builds the image, for ghcr.io images.""" + name = image.split("@", 1)[0] + if not name.startswith("ghcr.io/"): + return "" + path = name[len("ghcr.io/") :].rsplit(":", 1)[0] + return path if path.count("/") == 1 else "" + + +def repo_path(location: str) -> str: + """behave's file:line, relative to the repo. + + Some suites run from a checkout inside the guest and report it as a prefix + ("bluefin-tests/tests/smoke/...:55"); others report "tests/...". + """ + if location.startswith("tests/"): + return location + at = location.find("/tests/") + return location[at + 1 :] if at >= 0 else location + + +def locations(artifacts: Path, image: str, suite: str) -> dict[str, str]: + """Map "Feature :: Scenario" -> the scenario's file:line from behave's JSON.""" + results = ( + artifacts / f"e2e-results-{artifact_suffix(image)}-{suite}" / "results.json" + ) + try: + features = json.loads(results.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + found = {} + for feature in features if isinstance(features, list) else []: + for element in feature.get("elements", []): + if ( + element.get("type") == "scenario" + and element.get("status") in FAILED_STATUSES + ): + key = f"{feature.get('name', '?')} :: {element.get('name', '?')}" + found[key] = repo_path(element.get("location", "")) + return found + + +def title_for(image: str, suite: str, scenario: str) -> str: + title = f"e2e: {suite} fails on {image_name(image)}, also at v1: {scenario}" + return title if len(title) <= TITLE_LIMIT else title[: TITLE_LIMIT - 1] + "…" + + +def scenario_body( + image: str, suite: str, scenario: str, location: str, run_url: str +) -> str: + repo = image_repo(image) + where = ( + f"`{location}`" + if location + else "the feature file (search for the scenario name)" + ) + image_issue = ( + f"an issue in `{repo}` (a maintainer files it if the hive cannot open issues there)" + if repo + else "an issue in the image's own repository (a maintainer files it)" + ) + return f"""**Summary:** the scenario "{scenario}" in suite `{suite}` fails on `{image}` with the tests at the merged commit **and** with the tests at `v1`, so the post-merge `v1` gate accepted it as a pre-existing failure. Its cause is in the image or the VM, not in a testsuite change. Until it is handled, every run that boots `{suite}` stays red, including the PR VM check. + +Filed automatically by the `v1` gate (`.github/workflows/update-v1-tag.yml`, step "File pre-existing failures"). + +## Evidence + +- Scenario: {where} +- Image: `{image}` +- Run: {run_url} (job "Boot merged tests" and job "Boot v1 tests (baseline)", suite `{suite}`). The step summary lists it under "already present at v1"; the failing step and logs are in the `e2e-*` artifacts. + +## Ask + +1. Find out why it fails on this image: read the scenario's failing step and the journal, screenshot and coredump output in the run's artifacts. +2. Decide: + - **VM-only** (the VM lacks hardware or a service real machines have): skip it in VMs the way the suite already handles VM-only cases (for example `IGNORED_FAILED_UNITS_IN_VM` in `tests/smoke/features/steps/system_health_steps.py`, or an existing VM tag). Keep it on bare metal. + - **Image bug** (it would fail on real hardware too): tag the scenario `@quarantine` with a comment naming the cause, the way #1147 did, and say in the PR that a maintainer must file {image_issue}. + - **Test bug** (the test is wrong for this image): fix the test. +3. Don't raise a timeout to hide it, and don't delete the scenario. + +## Acceptance criteria + +- This scenario no longer fails in `{suite}` on the current `{image_name(image)}`: skipped in VMs, quarantined or fixed, with the reason next to the scenario. Other failures in the suite have their own issues. + +## Test plan + +- Unit tests and `behave --dry-run` (AGENTS.md gates). +- The PR VM check boots `{suite}` on the PR; link its run. If it does not run, a maintainer starts `manual.yml` with `suites={suite}` from the fix branch. + +## Out of scope + +- Changing the image itself, or `.github/workflows/e2e.yml`. +""" + + +def many_body(image: str, failures: list[tuple[str, str]], run_url: str) -> str: + listed = "\n".join(f"- `{suite}`: {scenario}" for suite, scenario in failures) + return f"""**Needs a maintainer:** {len(failures)} scenarios fail on `{image}` with the tests at the merged commit **and** at `v1`. That many at once points at the image as a whole (a broken or stale build, missing packages, a changed desktop), not at single tests, so the `v1` gate filed one issue instead of one per scenario. + +Filed automatically by the `v1` gate (`.github/workflows/update-v1-tag.yml`, step "File pre-existing failures"). + +Run: {run_url} + +{listed} + +Decide whether the image or the tests need to change. Quarantining all of these would hide a broken image; once the cause is known, file per-scenario issues for what the hive should fix. +""" + + +def issues(artifacts: Path, image: str, preexisting: dict, run_url: str) -> list[dict]: + failures = [ + (suite, scenario) + for suite in sorted(preexisting) + for scenario in sorted(preexisting[suite]) + ] + if not failures: + return [] + if len(failures) > MAX_ISSUES: + title = f"e2e: many scenarios fail on {image_name(image)}, also at v1" + return [ + { + "title": title, + "body": many_body(image, failures, run_url), + "needs_human": True, + } + ] + found: dict[str, dict[str, str]] = {} + out = [] + for suite, scenario in failures: + if suite not in found: + found[suite] = locations(artifacts, image, suite) + out.append( + { + "title": title_for(image, suite, scenario), + "body": scenario_body( + image, suite, scenario, found[suite].get(scenario, ""), run_url + ), + "needs_human": False, + } + ) + return out + + +def main(argv: list[str]) -> int: + if len(argv) != 5: + print(__doc__, file=sys.stderr) + return 2 + artifacts, image, preexisting_file, run_url = ( + Path(argv[1]), + argv[2], + Path(argv[3]), + argv[4], + ) + try: + preexisting = json.loads(preexisting_file.read_text(encoding="utf-8")) + except (OSError, ValueError): + preexisting = {} + for issue in issues(artifacts, image, preexisting, run_url): + print(json.dumps(issue)) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv)) diff --git a/scripts/vm_validation_plan.py b/scripts/vm_validation_plan.py index ec4ac7855..6aec09346 100644 --- a/scripts/vm_validation_plan.py +++ b/scripts/vm_validation_plan.py @@ -85,6 +85,7 @@ ("scripts/v1_gate.py", ()), ("scripts/v1_gate_compare.py", ()), ("scripts/v1_gate_pins.py", ()), + ("scripts/v1_gate_preexisting.py", ()), ("scripts/vm_validation_plan.py", ()), # Shared helpers: route by the path that imports them. ("tests/shared/kde_*", ("kde-smoke",)), diff --git a/tests/unit/test_v1_gate_preexisting.py b/tests/unit/test_v1_gate_preexisting.py new file mode 100644 index 000000000..a128e53ec --- /dev/null +++ b/tests/unit/test_v1_gate_preexisting.py @@ -0,0 +1,283 @@ +"""Unit tests for filing the v1 gate's pre-existing failures (scripts/v1_gate_preexisting.py).""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import yaml + +from scripts.v1_gate_compare import artifact_suffix +from scripts.v1_gate_preexisting import ( + MAX_ISSUES, + TITLE_LIMIT, + image_name, + image_repo, + issues, +) + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "scripts" / "v1_gate_preexisting.py" +COMPARE = REPO_ROOT / "scripts" / "v1_gate_compare.py" +WORKFLOW = REPO_ROOT / ".github" / "workflows" / "update-v1-tag.yml" + +DIGEST = "sha256:" + "ab" * 32 +CANDIDATE = f"ghcr.io/projectbluefin/dakota:testing@{DIGEST}" +BASELINE = f"ghcr.io/projectbluefin/dakota@{DIGEST}" +RUN = "https://example.invalid/run/1" +POWER = "GNOME Settings panels :: Power settings panel is accessible" + + +def write_suite(root: Path, image: str, suite: str, conclusion: str, scenarios: dict): + """Write e2e.yml's metadata and behave results for one suite job.""" + suffix = artifact_suffix(image) + meta = root / f"e2e-metadata-{suffix}-{suite}" + meta.mkdir(parents=True) + (meta / "e2e-metadata.json").write_text(json.dumps({"conclusion": conclusion})) + results = root / f"e2e-results-{suffix}-{suite}" + results.mkdir(parents=True) + features: dict[str, list] = {} + for key, (status, location) in scenarios.items(): + feature, name = key.split(" :: ") + features.setdefault(feature, []).append( + {"type": "scenario", "name": name, "status": status, "location": location} + ) + (results / "results.json").write_text( + json.dumps([{"name": f, "elements": e} for f, e in features.items()]) + ) + + +def test_image_name_and_repo(): + assert image_name(CANDIDATE) == "projectbluefin/dakota:testing" + assert image_repo(CANDIDATE) == "projectbluefin/dakota" + assert image_repo("quay.io/fedora/fedora-bootc:42") == "" + + +def test_one_issue_per_scenario_with_its_location(tmp_path): + loc = "tests/smoke/features/gnome_settings.feature:55" + write_suite(tmp_path, CANDIDATE, "smoke-b", "failure", {POWER: ("failed", loc)}) + out = issues(tmp_path, CANDIDATE, {"smoke-b": [POWER]}, RUN) + assert len(out) == 1 + issue = out[0] + assert issue["title"] == ( + f"e2e: smoke-b fails on projectbluefin/dakota:testing, also at v1: {POWER}" + ) + assert issue["needs_human"] is False + assert f"`{loc}`" in issue["body"] + assert RUN in issue["body"] + assert "`projectbluefin/dakota`" in issue["body"] + assert "@quarantine" in issue["body"] + + +def test_location_is_relative_to_the_repo(tmp_path): + # Real smoke-b output (run 43035684640) carries the guest checkout's name. + write_suite( + tmp_path, + CANDIDATE, + "smoke-b", + "failure", + { + POWER: ( + "failed", + "bluefin-tests/tests/smoke/features/gnome_settings.feature:55", + ) + }, + ) + body = issues(tmp_path, CANDIDATE, {"smoke-b": [POWER]}, RUN)[0]["body"] + assert "`tests/smoke/features/gnome_settings.feature:55`" in body + assert "bluefin-tests/" not in body + + +def test_missing_results_still_files_without_a_location(tmp_path): + out = issues(tmp_path, CANDIDATE, {"smoke-b": [POWER]}, RUN) + assert len(out) == 1 + assert "search for the scenario name" in out[0]["body"] + + +def test_nothing_to_file(): + assert issues(Path("/nonexistent"), CANDIDATE, {}, RUN) == [] + assert issues(Path("/nonexistent"), CANDIDATE, {"smoke-b": []}, RUN) == [] + + +def test_many_failures_become_one_issue_for_a_maintainer(): + many = {"smoke-a": [f"F :: s{i}" for i in range(MAX_ISSUES)], "common": ["F :: c"]} + out = issues(Path("/nonexistent"), CANDIDATE, many, RUN) + assert len(out) == 1 + assert out[0]["needs_human"] is True + assert out[0]["title"] == ( + "e2e: many scenarios fail on projectbluefin/dakota:testing, also at v1" + ) + assert "- `common`: F :: c" in out[0]["body"] + assert f"{MAX_ISSUES + 1} scenarios fail" in out[0]["body"] + # A later run with a different count must match the same open title. + more = {"smoke-a": [f"F :: s{i}" for i in range(MAX_ISSUES + 4)]} + assert ( + issues(Path("/nonexistent"), CANDIDATE, more, RUN)[0]["title"] + == out[0]["title"] + ) + + +def test_exactly_the_limit_is_still_per_scenario(): + limit = {"smoke-a": [f"F :: s{i}" for i in range(MAX_ISSUES)]} + out = issues(Path("/nonexistent"), CANDIDATE, limit, RUN) + assert len(out) == MAX_ISSUES + assert not any(i["needs_human"] for i in out) + + +def test_long_titles_are_cut_the_same_way_every_time(): + long = "F :: " + "x" * 400 + first = issues(Path("/nonexistent"), CANDIDATE, {"smoke-a": [long]}, RUN)[0][ + "title" + ] + again = issues(Path("/nonexistent"), CANDIDATE, {"smoke-a": [long]}, RUN)[0][ + "title" + ] + assert len(first) == TITLE_LIMIT + assert first == again + + +def test_compare_writes_preexisting_and_the_cli_files_it(tmp_path): + art = tmp_path / "artifacts" + loc = "tests/smoke/features/gnome_settings.feature:55" + write_suite(art, CANDIDATE, "smoke-b", "failure", {POWER: ("failed", loc)}) + write_suite(art, BASELINE, "smoke-b", "failure", {POWER: ("failed", loc)}) + pre = tmp_path / "preexisting.json" + out = subprocess.run( + [ + sys.executable, + str(COMPARE), + str(art), + CANDIDATE, + BASELINE, + "--preexisting", + str(pre), + ], + check=True, + capture_output=True, + text=True, + ) + assert "verdict=move" in out.stdout + assert json.loads(pre.read_text()) == {"smoke-b": [POWER]} + + filed = subprocess.run( + [sys.executable, str(SCRIPT), str(art), CANDIDATE, str(pre), RUN], + check=True, + capture_output=True, + text=True, + ) + lines = [json.loads(line) for line in filed.stdout.splitlines()] + assert [line["title"] for line in lines] == [ + f"e2e: smoke-b fails on projectbluefin/dakota:testing, also at v1: {POWER}" + ] + assert f"`{loc}`" in lines[0]["body"] + + +def test_new_failures_are_not_preexisting(tmp_path): + art = tmp_path / "artifacts" + write_suite(art, CANDIDATE, "smoke-b", "failure", {POWER: ("failed", "a:1")}) + write_suite(art, BASELINE, "smoke-b", "success", {POWER: ("passed", "a:1")}) + pre = tmp_path / "preexisting.json" + subprocess.run( + [ + sys.executable, + str(COMPARE), + str(art), + CANDIDATE, + BASELINE, + "--preexisting", + str(pre), + ], + check=True, + capture_output=True, + text=True, + ) + assert json.loads(pre.read_text()) == {} + + +# The workflow step, run for real with a stubbed gh: it files each new title +# once, labels it, and skips titles that already have an open issue. +STUB_GH = """#!/bin/bash +echo "gh $*" >> "$GH_LOG" +case "$1 $2" in + "issue list") printf '%s\\n' "$STUB_OPEN" ;; + "issue create") echo "https://example.invalid/issues/$((1200 + $(grep -c 'issue create' "$GH_LOG")))" ;; +esac +exit 0 +""" + + +def _step_script() -> str: + workflow = yaml.safe_load(WORKFLOW.read_text()) + steps = workflow["jobs"]["publish"]["steps"] + return next( + s["run"] for s in steps if s.get("name") == "File pre-existing failures" + ) + + +def _run_step(tmp_path: Path, preexisting: dict, stub_open: str = "") -> list[str]: + work = tmp_path / "work" + (work / "scripts").mkdir(parents=True) + for name in ("v1_gate_compare.py", "v1_gate_preexisting.py"): + (work / "scripts" / name).write_text((REPO_ROOT / "scripts" / name).read_text()) + (work / "artifacts").mkdir() + (work / "preexisting.json").write_text(json.dumps(preexisting)) + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + gh = bin_dir / "gh" + gh.write_text(STUB_GH) + gh.chmod(0o755) + log = tmp_path / "gh.log" + log.write_text("") + script = tmp_path / "step.sh" + script.write_text(_step_script()) + env = { + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "GH_LOG": str(log), + "GITHUB_STEP_SUMMARY": str(tmp_path / "summary.md"), + "IMAGE": CANDIDATE, + "RUN_URL": RUN, + "STUB_OPEN": stub_open, + } + subprocess.run( + ["bash", str(script)], + cwd=work, + env=env, + check=True, + capture_output=True, + text=True, + ) + return log.read_text().splitlines() + + +def test_step_files_and_labels_a_new_failure(tmp_path): + calls = _run_step(tmp_path, {"smoke-b": [POWER]}) + creates = [c for c in calls if c.startswith("gh issue create")] + assert len(creates) == 1 + assert "--add-label kind/bug" in " ".join(calls) + assert "--add-label triage/accepted" in " ".join(calls) + assert "needs-human" not in " ".join(calls) + assert "Pre-existing failures filed" in (tmp_path / "summary.md").read_text() + + +def test_step_skips_a_failure_that_already_has_an_open_issue(tmp_path): + title = f"e2e: smoke-b fails on projectbluefin/dakota:testing, also at v1: {POWER}" + calls = _run_step( + tmp_path, {"smoke-b": [POWER]}, stub_open=f"Something else\n{title}" + ) + assert not any(c.startswith("gh issue create") for c in calls) + + +def test_step_does_nothing_without_preexisting_failures(tmp_path): + calls = _run_step(tmp_path, {}) + assert calls == [] + + +def test_step_sends_many_failures_to_a_maintainer(tmp_path): + many = {"smoke-a": [f"F :: s{i}" for i in range(MAX_ISSUES + 1)]} + calls = _run_step(tmp_path, many) + assert len([c for c in calls if c.startswith("gh issue create")]) == 1 + assert "--add-label needs-human" in " ".join(calls) + assert "triage/accepted" not in " ".join(calls)