diff --git a/.github/workflows/update-v1-tag.yml b/.github/workflows/update-v1-tag.yml index 9ed35b084..23fe17bea 100644 --- a/.github/workflows/update-v1-tag.yml +++ b/.github/workflows/update-v1-tag.yml @@ -241,6 +241,7 @@ jobs: - name: Move or hold v1 env: COMPARE: ${{ steps.compare.outputs.verdict }} + INFRA: ${{ steps.compare.outputs.infrastructure }} run: | set -euo pipefail HELD=$(gh issue list --state open --search "\"${HELD_TITLE}\" in:title" \ @@ -281,7 +282,23 @@ jobs: else WHY="${REASON}" fi - BODY=$(printf '%s\n\n%s\n\n%s\n\n%s\n' \ + # A hold that no test change explains, or one that has lasted a day, + # is a maintainer's problem, not contributor work: say so first and + # label the issue so it is parked for a person (#1123 sat 36 hours as + # a "good first issue" while the runner could not boot any VM). + ESCALATE="" + if [ "${INFRA:-false}" = "true" ]; then + ESCALATE="no failed suite produced test results on either side, so the test machines most likely never booted (a runner or install problem). This is not a test regression." + elif [ -n "${HELD}" ]; then + OPENED=$(gh issue view "${HELD}" --json createdAt --jq .createdAt) + if [ $(( $(date +%s) - $(date -d "${OPENED}" +%s) )) -gt 86400 ]; then + ESCALATE="v1 has been held for more than 24 hours." + fi + fi + HEADLINE="" + if [ -n "${ESCALATE}" ]; then HEADLINE="**Needs a maintainer:** ${ESCALATE}"; fi + BODY=$(printf '%s\n\n%s\n\n%s\n\n%s\n\n%s\n' \ + "${HEADLINE}" \ "v1 stays at \`${PREVIOUS:-none}\` and was **not** moved to \`${SHA}\`." \ "Why: ${WHY}." \ "Run: ${RUN_URL}" \ @@ -289,7 +306,14 @@ jobs: if [ -n "${HELD}" ]; then gh issue comment "${HELD}" --body "${BODY}" else - gh issue create --title "${HELD_TITLE}" --body "${BODY}" + HELD=$(gh issue create --title "${HELD_TITLE}" --body "${BODY}" | grep -oE '[0-9]+$') + fi + if [ -n "${ESCALATE}" ] && [ -n "${HELD}" ]; then + gh issue edit "${HELD}" --add-label needs-human --add-label priority/critical-urgent \ + || echo "::warning::could not label #${HELD} for a maintainer" + for LABEL in "good first issue" "help wanted"; do + gh issue edit "${HELD}" --remove-label "${LABEL}" > /dev/null 2>&1 || true + done fi { echo "### v1 held at \`${PREVIOUS:-none}\`" diff --git a/docs/skills/ci-ops/ops/references/candidate-vm-validation.md b/docs/skills/ci-ops/ops/references/candidate-vm-validation.md index 325aa22c5..18da7c4bc 100644 --- a/docs/skills/ci-ops/ops/references/candidate-vm-validation.md +++ b/docs/skills/ci-ops/ops/references/candidate-vm-validation.md @@ -97,7 +97,14 @@ and `.../check-runs`. or `no_hosted_lane`). 3. On a hold or a failed gate, fails the run and opens (or comments on) the issue **"v1 tag held by the post-merge VM gate"** naming the held SHA. The - next successful move closes it. + next successful move closes it. Two holds are a maintainer's problem rather + than contributor work, so the comment starts with **Needs a maintainer** and + the issue gets `needs-human` and `priority/critical-urgent` (and loses + `good first issue` / `help wanted`): + - no failed suite produced test results on either side + (`v1_gate_compare.py` prints `infrastructure=true`): the test machines + most likely never booted, as when `ubuntu-latest` moved to 26.04 (#1123); + - the issue has been open for more than 24 hours. `lifecycle` is planned but not booted (`UNGATED_SUITES`): its upgrade scenarios need a second, newer image to upgrade to (#706, #1023), and on one diff --git a/scripts/v1_gate_compare.py b/scripts/v1_gate_compare.py index d8d381bc3..9ee4bc66e 100755 --- a/scripts/v1_gate_compare.py +++ b/scripts/v1_gate_compare.py @@ -24,7 +24,11 @@ Usage: python3 scripts/v1_gate_compare.py ARTIFACT_DIR CANDIDATE_IMAGE BASELINE_IMAGE [--summary FILE] -Prints ``verdict=move`` or ``verdict=hold`` for ``$GITHUB_OUTPUT``. +Prints ``verdict=move`` or ``verdict=hold`` and ``infrastructure=true`` or +``infrastructure=false`` for ``$GITHUB_OUTPUT``. ``infrastructure=true`` means +no failed suite produced test results on either side: the test machines most +likely never booted (a runner or install problem, as in #1123), so the hold is +not a test regression and needs a maintainer rather than a contributor. """ from __future__ import annotations @@ -113,11 +117,36 @@ def compare(artifacts: Path, candidate_image: str, baseline_image: str) -> dict: "verdict": "hold" if problems else "move", "problems": problems, "preexisting": preexisting, + "infrastructure": nothing_ran(candidate, baseline), } +def nothing_ran( + candidate: dict[str, tuple[str, set[str] | None]], + baseline: dict[str, tuple[str, set[str] | None]], +) -> bool: + """True when every failed candidate suite, and the same suite at v1, has no results.""" + failed = [suite for suite, (conclusion, _) in candidate.items() if conclusion != "success"] + if not failed: + return False + for suite in failed: + base = baseline.get(suite) + if candidate[suite][1] is not None or base is None: + return False + if base[0] == "success" or base[1] is not None: + return False + return True + + def summary_markdown(result: dict) -> str: lines = ["### Candidate failures compared with v1", ""] + if result.get("infrastructure"): + lines.append( + "**No failed suite produced test results on either side:** the test " + "machines most likely never booted (a runner or install problem), " + "not a test regression." + ) + lines.append("") if result["problems"]: lines.append("**Not explained by v1:**") lines.extend(f"- {p}" for p in result["problems"]) @@ -140,6 +169,7 @@ def main(argv: list[str]) -> int: if args.summary: args.summary.write_text(summary_markdown(result), encoding="utf-8") print(f"verdict={result['verdict']}") + print(f"infrastructure={'true' if result['infrastructure'] else 'false'}") return 0 diff --git a/tests/unit/test_v1_gate_compare.py b/tests/unit/test_v1_gate_compare.py index 917ea57fb..c69183fa1 100644 --- a/tests/unit/test_v1_gate_compare.py +++ b/tests/unit/test_v1_gate_compare.py @@ -92,6 +92,43 @@ def test_no_candidate_artifacts_hold_v1(tmp_path): assert compare(tmp_path, CANDIDATE, BASELINE)["verdict"] == "hold" +def test_no_results_on_either_side_is_infrastructure(tmp_path): + # #1123: the VM never booted, so neither side wrote results.json. + for suite in ("smoke-a", "common"): + write_suite(tmp_path, CANDIDATE, suite, "failure", None) + write_suite(tmp_path, BASELINE, suite, "failure", None) + result = compare(tmp_path, CANDIDATE, BASELINE) + assert result["verdict"] == "hold" + assert result["infrastructure"] is True + assert "never booted" in summary_markdown(result) + + +def test_results_on_one_side_is_not_infrastructure(tmp_path): + write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", None) + write_suite(tmp_path, BASELINE, "smoke-a", "failure", {"A": "failed"}) + assert compare(tmp_path, CANDIDATE, BASELINE)["infrastructure"] is False + + +def test_failed_scenarios_are_not_infrastructure(tmp_path): + write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", {"A": "failed"}) + write_suite(tmp_path, BASELINE, "smoke-a", "failure", {"A": "passed"}) + result = compare(tmp_path, CANDIDATE, BASELINE) + assert result["infrastructure"] is False + assert "never booted" not in summary_markdown(result) + + +def test_one_suite_with_results_is_not_infrastructure(tmp_path): + write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", None) + write_suite(tmp_path, BASELINE, "smoke-a", "failure", None) + write_suite(tmp_path, CANDIDATE, "common", "failure", {"A": "failed"}) + write_suite(tmp_path, BASELINE, "common", "failure", {"A": "failed"}) + assert compare(tmp_path, CANDIDATE, BASELINE)["infrastructure"] is False + + +def test_no_candidate_artifacts_is_not_infrastructure(tmp_path): + assert compare(tmp_path, CANDIDATE, BASELINE)["infrastructure"] is False + + def test_summary_lists_preexisting_and_new(tmp_path): write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", {"A": "failed"}) write_suite(tmp_path, BASELINE, "smoke-a", "failure", {"A": "failed"}) @@ -118,5 +155,5 @@ def test_cli_prints_verdict_and_writes_summary(tmp_path): text=True, check=True, ).stdout - assert out.strip() == "verdict=move" + assert out.splitlines() == ["verdict=move", "infrastructure=false"] assert summary.read_text().startswith("### Candidate failures compared with v1")