From 34c1ccfee695a421d3e6c5464f106f8744fb0370 Mon Sep 17 00:00:00 2001 From: quality <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:58:15 -0500 Subject: [PATCH 1/3] =?UTF-8?q?=F0=9F=8C=B1=20ci:=20mutation=20report=20ke?= =?UTF-8?q?eps=20its=20own=20time=20budget=20and=20says=20when=20it=20is?= =?UTF-8?q?=20partial?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A 20-minute runner kill of the advisory gremlins step wrote no comment.md, so XXL PRs kept the comment from an earlier head (#17596) or got none (#17572), and the text could not tell a clean run from a timeout. The run phase now bounds each package by the remaining budget, always writes the report, names the head SHA, and lists the packages it did not finish. test-mutation-report.sh pins the three shapes with a stubbed gremlins; TRUSTED_SCRIPTS_TREE re-pinned. Refs #17602 Signed-off-by: quality <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .github/scripts/integration-governance.cjs | 2 +- .../fixed-17602-mutation-report-budget.md | 1 + src/scripts/mutation-report.sh | 47 +++++- src/scripts/test-mutation-report.sh | 148 ++++++++++++++++++ 4 files changed, 194 insertions(+), 4 deletions(-) create mode 100644 changelog.d/fixed-17602-mutation-report-budget.md create mode 100755 src/scripts/test-mutation-report.sh diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 3450a885d5..fd132e13bb 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '7c365d4a9df3670491f8751464340e2dceb43e77'; +const TRUSTED_SCRIPTS_TREE = 'e4047bc25a5e4ee3078f772edc5974770d0fb07b'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/changelog.d/fixed-17602-mutation-report-budget.md b/changelog.d/fixed-17602-mutation-report-budget.md new file mode 100644 index 0000000000..dcc3bd5e1c --- /dev/null +++ b/changelog.d/fixed-17602-mutation-report-budget.md @@ -0,0 +1 @@ +- **Advisory mutation report survives its time budget** — `mutation-report.sh` now runs gremlins under its own `MUTATION_BUDGET_SECONDS` budget (17 min by default, under the job's 20‑minute limit), so a size/XXL PR no longer ends with the runner killing the step before anything is written. The `` comment is always posted, names the PR head it reports on, and when the budget ran out it says which changed packages were left unfinished or not started instead of a stale report from an earlier push, or none, standing in for a clean result. ([#17602](https://github.com/hivecommons/hive/issues/17602)) diff --git a/src/scripts/mutation-report.sh b/src/scripts/mutation-report.sh index 5187c11e66..eaab6117ee 100755 --- a/src/scripts/mutation-report.sh +++ b/src/scripts/mutation-report.sh @@ -4,6 +4,11 @@ # upserts ONE PR comment listing surviving (LIVED) # mutants in changed files. Never fails. # +# The run phase keeps its own time budget (MUTATION_BUDGET_SECONDS, default +# 17 min) under the job's timeout-minutes so comment.md is always written: a +# job killed by the runner writes nothing, and the PR keeps the comment from +# an earlier head (or none) with no sign the report is stale or partial. +# # Two phases so the token never shares an environment with PR-controlled code: # mutation-report.sh run runs gremlins (no GH_TOKEN) and writes # $out_dir/comment.md @@ -52,8 +57,13 @@ fi base_ref="${BASE_REF:?BASE_REF is required}" coefficient="${MUTATION_TIMEOUT_COEFFICIENT:-3}" +budget="${MUTATION_BUDGET_SECONDS:-1020}" +deadline=$(( $(date +%s) + budget )) mkdir -p "$out_dir" out_abs=$(cd "$out_dir" && pwd) +# On a pull_request checkout HEAD is the synthetic merge commit; HEAD^2 is the +# PR head the comment should name so a reader can tell a stale report apart. +head_sha=$(git rev-parse --short HEAD^2 2>/dev/null || git rev-parse --short HEAD) changed=$(git diff --name-only --diff-filter=d "origin/${base_ref}...HEAD" -- 'src/*.go' \ | grep -vE '(_test\.go$|(^|/)testdata/|\.pb\.go$|_gen\.go$|_generated\.go$|(^|/)mock)' || true) @@ -94,15 +104,36 @@ pkgs="${roots%$'\n'}" # package-relative paths; otherwise every mutant would be skipped, or (with no # --diff) the whole package is mutated and the job times out (#17463). Coverage # chdirs to the module root, so the output path must be absolute. +# Each package gets whatever is left of the budget; one that outlives it is +# killed (exit 124) and recorded as unfinished, and the packages after it are +# recorded as not started, so the comment says what was and was not mutated. +unfinished="$out_dir/unfinished.txt" +: > "$unfinished" i=0 +total=0 while IFS= read -r pkg; do [ -n "$pkg" ] || continue + total=$((total + 1)) i=$((i + 1)) printf '%s\n' "$pkg" > "$out_dir/pkg-$i.dir" - echo "::group::gremlins unleash ./$pkg --diff origin/${base_ref}" + remaining=$(( deadline - $(date +%s) )) + if [ "$remaining" -le 0 ]; then + echo "time budget exhausted before $pkg; not started" + printf '%s\tnot started\n' "$pkg" >> "$unfinished" + continue + fi + echo "::group::gremlins unleash ./$pkg --diff origin/${base_ref} (budget ${remaining}s)" (cd "src/$pkg" && GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=diff.relative GIT_CONFIG_VALUE_0=true \ + timeout --kill-after=10 "$remaining" \ gremlins unleash --timeout-coefficient "$coefficient" --diff "origin/${base_ref}" \ - --output "$out_abs/pkg-$i.json") || echo "gremlins exited non-zero for $pkg (advisory)" + --output "$out_abs/pkg-$i.json") + rc=$? + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + echo "gremlins hit the time budget in $pkg after ${remaining}s; results for it are partial or missing" + printf '%s\tunfinished\n' "$pkg" >> "$unfinished" + elif [ "$rc" -ne 0 ]; then + echo "gremlins exited non-zero ($rc) for $pkg (advisory)" + fi echo "::endgroup::" done <<< "$pkgs" @@ -132,14 +163,24 @@ done < "$survivors" echo "$marker" echo "### Mutation testing (advisory)" echo + echo "Head \`${head_sha}\`." + echo if [ -s "$filtered" ]; then echo "gremlins found **$(wc -l < "$filtered" | tr -d ' ') surviving mutant(s)** in changed files: the code ran under tests, but changing it did not fail any test." echo echo "| File | Line | Mutation |" echo "|---|---|---|" sort -t$'\t' -k1,1 -k2,2n "$filtered" | head -n 100 | awk -F'\t' '{printf "| `%s` | %s | %s |\n", $1, $2, $3}' + elif [ -s "$unfinished" ]; then + echo "No surviving mutants reported in the packages that finished." else - echo "No surviving mutants in changed files (or the run hit its time budget before reporting any)." + echo "No surviving mutants in changed files." + fi + if [ -s "$unfinished" ]; then + echo + echo "⏱ **Time budget (${budget}s) exhausted: $(wc -l < "$unfinished" | tr -d ' ') of ${total} changed package(s) were not fully mutated**, so this report is partial:" + echo + awk -F'\t' '{printf "- `src/%s` — %s\n", $1, $2}' "$unfinished" fi echo echo "_Advisory only; this job never blocks the PR (#17349)._" diff --git a/src/scripts/test-mutation-report.sh b/src/scripts/test-mutation-report.sh new file mode 100755 index 0000000000..4cfb6cbf0f --- /dev/null +++ b/src/scripts/test-mutation-report.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# test-mutation-report.sh — exercises the run phase of mutation-report.sh +# against a stubbed gremlins and a throwaway git repository (#17349). +# +# The stub is the point: gremlins is slow and nondeterministic, and the cases +# that matter here (survivors in a changed file, a clean run, a package that +# outlives the time budget) are shapes of its output and its runtime, not +# anything a real mutation run can be asked to produce on demand. +# +# The budget section is the one that must never regress: a run that hits its +# time budget must still write comment.md, and that comment must say which +# packages were not mutated. Before #17601's sibling PRs were triaged, the job +# timeout killed the step first, nothing was written, and the PR kept the +# comment from an earlier head with no sign it was stale. +set -u -o pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="${HERE}/mutation-report.sh" +TMP_ROOT="${HERE}/../.test-tmp" +mkdir -p "$TMP_ROOT" +TMP="$(mktemp -d "$TMP_ROOT/mutation-report.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +fail=0 +pass() { echo " ok: $*"; } +bad() { echo " FAIL: $*"; fail=1; } + +# --- the gremlins stub ------------------------------------------------------- +# +# Honours only `unleash ... --output `. STUB_MODE picks the behaviour: +# lived one LIVED mutant in the package's changed file and one in a file +# the PR did not touch (must be filtered out) +# clean a KILLED mutant only +# hang sleep past any budget without writing output; the script's +# `timeout` must kill it +# STUB_CALLS records every package directory it was run from. +STUB_DIR="$TMP/stub" +mkdir -p "$STUB_DIR" +cat > "$STUB_DIR/gremlins" <<'STUB' +#!/usr/bin/env bash +set -u +pwd >> "$STUB_CALLS" +out="" +while [ $# -gt 0 ]; do + case "$1" in --output) out="$2"; shift ;; esac + shift +done +case "${STUB_MODE:-clean}" in + hang) sleep 30; exit 0 ;; + lived) + cat > "$out" <<'JSON' +{"files":[ + {"file_name":"./changed.go","mutations":[{"line":3,"type":"CONDITIONALS_NEGATION","status":"LIVED"}]}, + {"file_name":"./untouched.go","mutations":[{"line":1,"type":"ARITHMETIC_BASE","status":"LIVED"}]} +]} +JSON + ;; + *) + cat > "$out" <<'JSON' +{"files":[{"file_name":"./changed.go","mutations":[{"line":3,"type":"CONDITIONALS_NEGATION","status":"KILLED"}]}]} +JSON + ;; +esac +exit 0 +STUB +chmod +x "$STUB_DIR/gremlins" + +# --- the repository ---------------------------------------------------------- +# +# origin/main has two packages; the PR head edits one non-test file in each. +REPO="$TMP/repo" +mkdir -p "$REPO" +git -C "$REPO" init -q -b main +git -C "$REPO" config user.email t@example.com +git -C "$REPO" config user.name t +for p in alpha beta; do + mkdir -p "$REPO/src/pkg/$p" + printf 'package %s\n\nvar X = 1\n' "$p" > "$REPO/src/pkg/$p/changed.go" + printf 'package %s\n' "$p" > "$REPO/src/pkg/$p/untouched.go" +done +git -C "$REPO" add -A && git -C "$REPO" commit -q -m base +git -C "$REPO" update-ref refs/remotes/origin/main HEAD +git -C "$REPO" checkout -q -b pr +for p in alpha beta; do + printf 'package %s\n\nvar X = 2\n' "$p" > "$REPO/src/pkg/$p/changed.go" +done +printf 'package alpha\n' > "$REPO/src/pkg/alpha/changed_test.go" +git -C "$REPO" add -A && git -C "$REPO" commit -q -m change +HEAD_SHORT=$(git -C "$REPO" rev-parse --short HEAD) + +run_phase() { + # $1 = STUB_MODE, $2 = MUTATION_BUDGET_SECONDS; the report lands in $OUT, + # the exit status in $RC. + OUT="$TMP/out-$1-$2" + rm -rf "$OUT" + export STUB_CALLS="$TMP/calls-$1-$2" + : > "$STUB_CALLS" + (cd "$REPO" && PATH="$STUB_DIR:$PATH" STUB_MODE="$1" \ + MUTATION_BUDGET_SECONDS="$2" MUTATION_OUT_DIR="$OUT" BASE_REF=main \ + bash "$SCRIPT" run >"$OUT.log" 2>&1) + RC=$? +} + +echo "survivors in changed files" +run_phase lived 600 +[ "$RC" = 0 ] || bad "run phase exited $RC" +BODY="$OUT/comment.md" +[ -s "$BODY" ] || bad "no comment.md written" +grep -qF '' "$BODY" && pass "comment starts with the upsert marker" || bad "marker missing" +grep -qF "Head \`${HEAD_SHORT}\`" "$BODY" && pass "comment names the head it reports on" || bad "head sha missing: $(cat "$BODY")" +grep -qF '**2 surviving mutant(s)**' "$BODY" && pass "one survivor per changed package counted" || bad "survivor count wrong: $(cat "$BODY")" +grep -qF '| `src/pkg/alpha/changed.go` | 3 | CONDITIONALS_NEGATION |' "$BODY" && pass "survivor re-anchored to a repo-relative path" || bad "alpha survivor row missing" +grep -qF 'untouched.go' "$BODY" && bad "survivor in an untouched file leaked into the report" || pass "survivors in untouched files filtered out" +grep -qF 'Time budget' "$BODY" && bad "complete run flagged as partial" || pass "complete run carries no budget note" +[ "$(wc -l < "$STUB_CALLS" | tr -d ' ')" = 2 ] && pass "gremlins ran once per changed package" || bad "gremlins calls: $(cat "$STUB_CALLS")" + +echo "clean run" +run_phase clean 600 +BODY="$OUT/comment.md" +grep -qxF 'No surviving mutants in changed files.' "$BODY" && pass "clean run says so without hedging" || bad "clean message wrong: $(cat "$BODY")" +grep -qF 'time budget' "$BODY" && bad "clean run still hedges about the time budget" || pass "clean run is not conflated with a timeout" + +echo "time budget exhausted" +run_phase hang 2 +[ "$RC" = 0 ] || bad "run phase exited $RC after the budget" +BODY="$OUT/comment.md" +[ -s "$BODY" ] || bad "budget run wrote no comment.md — the stale-comment bug is back" +grep -qF 'Time budget (2s) exhausted' "$BODY" && pass "partial report is labelled as partial" || bad "budget note missing: $(cat "$BODY")" +grep -qF '2 of 2 changed package(s) were not fully mutated' "$BODY" && pass "unfinished count covers both packages" || bad "unfinished count wrong: $(cat "$BODY")" +grep -qF -- '- `src/pkg/alpha` — unfinished' "$BODY" && pass "package killed by the budget is listed as unfinished" || bad "alpha not listed as unfinished" +grep -qF -- '- `src/pkg/beta` — not started' "$BODY" && pass "package after the budget is listed as not started" || bad "beta not listed as not started" +grep -qF 'No surviving mutants reported in the packages that finished' "$BODY" && pass "no-survivor line is scoped to finished packages" || bad "scoped no-survivor line missing" +[ "$(wc -l < "$STUB_CALLS" | tr -d ' ')" = 1 ] && pass "gremlins was not started once the budget was gone" || bad "gremlins calls after budget: $(cat "$STUB_CALLS")" +grep -qF 'hit the time budget in pkg/alpha' "$OUT.log" && pass "log names the package that outlived the budget" || bad "log line missing: $(cat "$OUT.log")" + +echo "test files only" +git -C "$REPO" checkout -q -b tests-only origin/main +printf 'package alpha\n' > "$REPO/src/pkg/alpha/new_test.go" +git -C "$REPO" add -A && git -C "$REPO" commit -q -m tests +run_phase clean 600 +[ -f "$OUT/comment.md" ] && bad "tests-only diff produced a report" || pass "tests-only diff writes no report" +grep -qF 'No changed non-test Go files' "$OUT.log" && pass "tests-only diff is skipped explicitly" || bad "skip message missing" + +if [ "$fail" -ne 0 ]; then + echo "FAILED" + exit 1 +fi +echo "all tests passed" From fba8d52ad03abe884ac95bea44cd7335b743ec7a Mon Sep 17 00:00:00 2001 From: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:07:00 -0500 Subject: [PATCH 2/3] =?UTF-8?q?=F0=9F=8C=B1=20ci:=20mutation=20report=20re?= =?UTF-8?q?view=20fixes=20=E2=80=94=20cite=20#17602,=20drop=20redundant=20?= =?UTF-8?q?total=20counter?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix review findings on #17604: the test header rationale now points at #17602 (the mutation-advisory timeout issue) rather than the unrelated #17601, and the package loop reuses `i` instead of a duplicate `total` counter that was always equal to it. Hive-Fix-Attempt: 1/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: ci-maintainer <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/scripts/mutation-report.sh | 4 +--- src/scripts/test-mutation-report.sh | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/scripts/mutation-report.sh b/src/scripts/mutation-report.sh index eaab6117ee..14d05fdf95 100755 --- a/src/scripts/mutation-report.sh +++ b/src/scripts/mutation-report.sh @@ -110,10 +110,8 @@ pkgs="${roots%$'\n'}" unfinished="$out_dir/unfinished.txt" : > "$unfinished" i=0 -total=0 while IFS= read -r pkg; do [ -n "$pkg" ] || continue - total=$((total + 1)) i=$((i + 1)) printf '%s\n' "$pkg" > "$out_dir/pkg-$i.dir" remaining=$(( deadline - $(date +%s) )) @@ -178,7 +176,7 @@ done < "$survivors" fi if [ -s "$unfinished" ]; then echo - echo "⏱ **Time budget (${budget}s) exhausted: $(wc -l < "$unfinished" | tr -d ' ') of ${total} changed package(s) were not fully mutated**, so this report is partial:" + echo "⏱ **Time budget (${budget}s) exhausted: $(wc -l < "$unfinished" | tr -d ' ') of ${i} changed package(s) were not fully mutated**, so this report is partial:" echo awk -F'\t' '{printf "- `src/%s` — %s\n", $1, $2}' "$unfinished" fi diff --git a/src/scripts/test-mutation-report.sh b/src/scripts/test-mutation-report.sh index 4cfb6cbf0f..3ca76d6c85 100755 --- a/src/scripts/test-mutation-report.sh +++ b/src/scripts/test-mutation-report.sh @@ -9,7 +9,7 @@ # # The budget section is the one that must never regress: a run that hits its # time budget must still write comment.md, and that comment must say which -# packages were not mutated. Before #17601's sibling PRs were triaged, the job +# packages were not mutated. Before #17602's sibling PRs were triaged, the job # timeout killed the step first, nothing was written, and the PR kept the # comment from an earlier head with no sign it was stale. set -u -o pipefail From e56e36099507bda39ad1d34714397aa829f607a6 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:18:49 -0500 Subject: [PATCH 3/3] ci: re-pin TRUSTED_SCRIPTS_TREE to the v6-merged src/scripts tree The runtime-tests contract compares the pin against the merge ref's src/scripts tree, so the pin must follow v6, not the branch tip alone. Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- .github/scripts/integration-governance.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index fd132e13bb..6cbea58e7e 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = 'e4047bc25a5e4ee3078f772edc5974770d0fb07b'; +const TRUSTED_SCRIPTS_TREE = '5d965ad41a5898673930799efca57d03fe31a52d'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) =>