Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 26 additions & 2 deletions .github/workflows/update-v1-tag.yml
Original file line number Diff line number Diff line change
Expand Up @@ -241,6 +241,7 @@ jobs:
- name: Move or hold v1
env:
COMPARE: ${{ steps.compare.outputs.verdict }}
INFRA: ${{ steps.compare.outputs.infrastructure }}
run: |
set -euo pipefail
HELD=$(gh issue list --state open --search "\"${HELD_TITLE}\" in:title" \
Expand Down Expand Up @@ -281,15 +282,38 @@ jobs:
else
WHY="${REASON}"
fi
BODY=$(printf '%s\n\n%s\n\n%s\n\n%s\n' \
# A hold that no test change explains, or one that has lasted a day,
# is a maintainer's problem, not contributor work: say so first and
# label the issue so it is parked for a person (#1123 sat 36 hours as
# a "good first issue" while the runner could not boot any VM).
ESCALATE=""
if [ "${INFRA:-false}" = "true" ]; then
ESCALATE="no failed suite produced test results on either side, so the test machines most likely never booted (a runner or install problem). This is not a test regression."
elif [ -n "${HELD}" ]; then
OPENED=$(gh issue view "${HELD}" --json createdAt --jq .createdAt)
if [ $(( $(date +%s) - $(date -d "${OPENED}" +%s) )) -gt 86400 ]; then
ESCALATE="v1 has been held for more than 24 hours."
fi
fi
HEADLINE=""
if [ -n "${ESCALATE}" ]; then HEADLINE="**Needs a maintainer:** ${ESCALATE}"; fi
BODY=$(printf '%s\n\n%s\n\n%s\n\n%s\n\n%s\n' \
"${HEADLINE}" \
"v1 stays at \`${PREVIOUS:-none}\` and was **not** moved to \`${SHA}\`." \
"Why: ${WHY}." \
"Run: ${RUN_URL}" \
"To release it: fix main (the next push re-runs the gate), or after validating by hand run **Update v1 tag** on main with \`force\` and a reason.")
if [ -n "${HELD}" ]; then
gh issue comment "${HELD}" --body "${BODY}"
else
gh issue create --title "${HELD_TITLE}" --body "${BODY}"
HELD=$(gh issue create --title "${HELD_TITLE}" --body "${BODY}" | grep -oE '[0-9]+$')
fi
if [ -n "${ESCALATE}" ] && [ -n "${HELD}" ]; then
gh issue edit "${HELD}" --add-label needs-human --add-label priority/critical-urgent \
|| echo "::warning::could not label #${HELD} for a maintainer"
for LABEL in "good first issue" "help wanted"; do
gh issue edit "${HELD}" --remove-label "${LABEL}" > /dev/null 2>&1 || true
done
fi
{
echo "### v1 held at \`${PREVIOUS:-none}\`"
Expand Down
9 changes: 8 additions & 1 deletion docs/skills/ci-ops/ops/references/candidate-vm-validation.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,14 @@ and `.../check-runs`.
or `no_hosted_lane`).
3. On a hold or a failed gate, fails the run and opens (or comments on) the
issue **"v1 tag held by the post-merge VM gate"** naming the held SHA. The
next successful move closes it.
next successful move closes it. Two holds are a maintainer's problem rather
than contributor work, so the comment starts with **Needs a maintainer** and
the issue gets `needs-human` and `priority/critical-urgent` (and loses
`good first issue` / `help wanted`):
- no failed suite produced test results on either side
(`v1_gate_compare.py` prints `infrastructure=true`): the test machines
most likely never booted, as when `ubuntu-latest` moved to 26.04 (#1123);
- the issue has been open for more than 24 hours.

`lifecycle` is planned but not booted (`UNGATED_SUITES`): its upgrade
scenarios need a second, newer image to upgrade to (#706, #1023), and on one
Expand Down
32 changes: 31 additions & 1 deletion scripts/v1_gate_compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,11 @@
Usage:
python3 scripts/v1_gate_compare.py ARTIFACT_DIR CANDIDATE_IMAGE BASELINE_IMAGE [--summary FILE]

Prints ``verdict=move`` or ``verdict=hold`` for ``$GITHUB_OUTPUT``.
Prints ``verdict=move`` or ``verdict=hold`` and ``infrastructure=true`` or
``infrastructure=false`` for ``$GITHUB_OUTPUT``. ``infrastructure=true`` means
no failed suite produced test results on either side: the test machines most
likely never booted (a runner or install problem, as in #1123), so the hold is
not a test regression and needs a maintainer rather than a contributor.
"""

from __future__ import annotations
Expand Down Expand Up @@ -113,11 +117,36 @@ def compare(artifacts: Path, candidate_image: str, baseline_image: str) -> dict:
"verdict": "hold" if problems else "move",
"problems": problems,
"preexisting": preexisting,
"infrastructure": nothing_ran(candidate, baseline),
}


def nothing_ran(
candidate: dict[str, tuple[str, set[str] | None]],
baseline: dict[str, tuple[str, set[str] | None]],
) -> bool:
"""True when every failed candidate suite, and the same suite at v1, has no results."""
failed = [suite for suite, (conclusion, _) in candidate.items() if conclusion != "success"]
if not failed:
return False
for suite in failed:
base = baseline.get(suite)
if candidate[suite][1] is not None or base is None:
return False
if base[0] == "success" or base[1] is not None:
return False
return True


def summary_markdown(result: dict) -> str:
lines = ["### Candidate failures compared with v1", ""]
if result.get("infrastructure"):
lines.append(
"**No failed suite produced test results on either side:** the test "
"machines most likely never booted (a runner or install problem), "
"not a test regression."
)
lines.append("")
if result["problems"]:
lines.append("**Not explained by v1:**")
lines.extend(f"- {p}" for p in result["problems"])
Expand All @@ -140,6 +169,7 @@ def main(argv: list[str]) -> int:
if args.summary:
args.summary.write_text(summary_markdown(result), encoding="utf-8")
print(f"verdict={result['verdict']}")
print(f"infrastructure={'true' if result['infrastructure'] else 'false'}")
return 0


Expand Down
39 changes: 38 additions & 1 deletion tests/unit/test_v1_gate_compare.py
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,43 @@ def test_no_candidate_artifacts_hold_v1(tmp_path):
assert compare(tmp_path, CANDIDATE, BASELINE)["verdict"] == "hold"


def test_no_results_on_either_side_is_infrastructure(tmp_path):
# #1123: the VM never booted, so neither side wrote results.json.
for suite in ("smoke-a", "common"):
write_suite(tmp_path, CANDIDATE, suite, "failure", None)
write_suite(tmp_path, BASELINE, suite, "failure", None)
result = compare(tmp_path, CANDIDATE, BASELINE)
assert result["verdict"] == "hold"
assert result["infrastructure"] is True
assert "never booted" in summary_markdown(result)


def test_results_on_one_side_is_not_infrastructure(tmp_path):
write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", None)
write_suite(tmp_path, BASELINE, "smoke-a", "failure", {"A": "failed"})
assert compare(tmp_path, CANDIDATE, BASELINE)["infrastructure"] is False


def test_failed_scenarios_are_not_infrastructure(tmp_path):
write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", {"A": "failed"})
write_suite(tmp_path, BASELINE, "smoke-a", "failure", {"A": "passed"})
result = compare(tmp_path, CANDIDATE, BASELINE)
assert result["infrastructure"] is False
assert "never booted" not in summary_markdown(result)


def test_one_suite_with_results_is_not_infrastructure(tmp_path):
write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", None)
write_suite(tmp_path, BASELINE, "smoke-a", "failure", None)
write_suite(tmp_path, CANDIDATE, "common", "failure", {"A": "failed"})
write_suite(tmp_path, BASELINE, "common", "failure", {"A": "failed"})
assert compare(tmp_path, CANDIDATE, BASELINE)["infrastructure"] is False


def test_no_candidate_artifacts_is_not_infrastructure(tmp_path):
assert compare(tmp_path, CANDIDATE, BASELINE)["infrastructure"] is False


def test_summary_lists_preexisting_and_new(tmp_path):
write_suite(tmp_path, CANDIDATE, "smoke-a", "failure", {"A": "failed"})
write_suite(tmp_path, BASELINE, "smoke-a", "failure", {"A": "failed"})
Expand All @@ -118,5 +155,5 @@ def test_cli_prints_verdict_and_writes_summary(tmp_path):
text=True,
check=True,
).stdout
assert out.strip() == "verdict=move"
assert out.splitlines() == ["verdict=move", "infrastructure=false"]
assert summary.read_text().startswith("### Candidate failures compared with v1")
Loading