Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/scripts/integration-governance.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000;
const MAX_AUTHOR_BATCH = 50;
// This literal is inside the enrolled .github closure. A caller-supplied
// preparedBy.sourceSHA must never choose a new executable verifier tree.
const TRUSTED_SCRIPTS_TREE = '7c365d4a9df3670491f8751464340e2dceb43e77';
const TRUSTED_SCRIPTS_TREE = '5d965ad41a5898673930799efca57d03fe31a52d';
const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4';
const sha = value => crypto.createHash('sha256').update(value).digest('hex');
const git = (args, cwd = process.cwd()) =>
Expand Down
1 change: 1 addition & 0 deletions changelog.d/fixed-17602-mutation-report-budget.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- **Advisory mutation report survives its time budget** — `mutation-report.sh` now runs gremlins under its own `MUTATION_BUDGET_SECONDS` budget (17 min by default, under the job's 20‑minute limit), so a size/XXL PR no longer ends with the runner killing the step before anything is written. The `<!-- hive-mutation -->` comment is always posted, names the PR head it reports on, and when the budget ran out it says which changed packages were left unfinished or not started instead of a stale report from an earlier push, or none, standing in for a clean result. ([#17602](https://github.com/hivecommons/hive/issues/17602))
45 changes: 42 additions & 3 deletions src/scripts/mutation-report.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@
# upserts ONE <!-- hive-mutation --> PR comment listing surviving (LIVED)
# mutants in changed files. Never fails.
#
# The run phase keeps its own time budget (MUTATION_BUDGET_SECONDS, default
# 17 min) under the job's timeout-minutes so comment.md is always written: a
# job killed by the runner writes nothing, and the PR keeps the comment from
# an earlier head (or none) with no sign the report is stale or partial.
#
# Two phases so the token never shares an environment with PR-controlled code:
# mutation-report.sh run runs gremlins (no GH_TOKEN) and writes
# $out_dir/comment.md
Expand Down Expand Up @@ -52,8 +57,13 @@ fi

base_ref="${BASE_REF:?BASE_REF is required}"
coefficient="${MUTATION_TIMEOUT_COEFFICIENT:-3}"
budget="${MUTATION_BUDGET_SECONDS:-1020}"
deadline=$(( $(date +%s) + budget ))
mkdir -p "$out_dir"
out_abs=$(cd "$out_dir" && pwd)
# On a pull_request checkout HEAD is the synthetic merge commit; HEAD^2 is the
# PR head the comment should name so a reader can tell a stale report apart.
head_sha=$(git rev-parse --short HEAD^2 2>/dev/null || git rev-parse --short HEAD)

changed=$(git diff --name-only --diff-filter=d "origin/${base_ref}...HEAD" -- 'src/*.go' \
| grep -vE '(_test\.go$|(^|/)testdata/|\.pb\.go$|_gen\.go$|_generated\.go$|(^|/)mock)' || true)
Expand Down Expand Up @@ -94,15 +104,34 @@ pkgs="${roots%$'\n'}"
# package-relative paths; otherwise every mutant would be skipped, or (with no
# --diff) the whole package is mutated and the job times out (#17463). Coverage
# chdirs to the module root, so the output path must be absolute.
# Each package gets whatever is left of the budget; one that outlives it is
# killed (exit 124) and recorded as unfinished, and the packages after it are
# recorded as not started, so the comment says what was and was not mutated.
unfinished="$out_dir/unfinished.txt"
: > "$unfinished"
i=0
while IFS= read -r pkg; do
[ -n "$pkg" ] || continue
i=$((i + 1))
printf '%s\n' "$pkg" > "$out_dir/pkg-$i.dir"
echo "::group::gremlins unleash ./$pkg --diff origin/${base_ref}"
remaining=$(( deadline - $(date +%s) ))
if [ "$remaining" -le 0 ]; then
echo "time budget exhausted before $pkg; not started"
printf '%s\tnot started\n' "$pkg" >> "$unfinished"
continue
fi
echo "::group::gremlins unleash ./$pkg --diff origin/${base_ref} (budget ${remaining}s)"
(cd "src/$pkg" && GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=diff.relative GIT_CONFIG_VALUE_0=true \
timeout --kill-after=10 "$remaining" \
gremlins unleash --timeout-coefficient "$coefficient" --diff "origin/${base_ref}" \
--output "$out_abs/pkg-$i.json") || echo "gremlins exited non-zero for $pkg (advisory)"
--output "$out_abs/pkg-$i.json")
rc=$?
if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then
echo "gremlins hit the time budget in $pkg after ${remaining}s; results for it are partial or missing"
printf '%s\tunfinished\n' "$pkg" >> "$unfinished"
elif [ "$rc" -ne 0 ]; then
echo "gremlins exited non-zero ($rc) for $pkg (advisory)"
fi
echo "::endgroup::"
done <<< "$pkgs"

Expand Down Expand Up @@ -132,14 +161,24 @@ done < "$survivors"
echo "$marker"
echo "### Mutation testing (advisory)"
echo
echo "Head \`${head_sha}\`."
echo
if [ -s "$filtered" ]; then
echo "gremlins found **$(wc -l < "$filtered" | tr -d ' ') surviving mutant(s)** in changed files: the code ran under tests, but changing it did not fail any test."
echo
echo "| File | Line | Mutation |"
echo "|---|---|---|"
sort -t$'\t' -k1,1 -k2,2n "$filtered" | head -n 100 | awk -F'\t' '{printf "| `%s` | %s | %s |\n", $1, $2, $3}'
elif [ -s "$unfinished" ]; then
echo "No surviving mutants reported in the packages that finished."
else
echo "No surviving mutants in changed files (or the run hit its time budget before reporting any)."
echo "No surviving mutants in changed files."
fi
if [ -s "$unfinished" ]; then
echo
echo "⏱ **Time budget (${budget}s) exhausted: $(wc -l < "$unfinished" | tr -d ' ') of ${i} changed package(s) were not fully mutated**, so this report is partial:"
echo
awk -F'\t' '{printf "- `src/%s` — %s\n", $1, $2}' "$unfinished"
fi
echo
echo "_Advisory only; this job never blocks the PR (#17349)._"
Expand Down
148 changes: 148 additions & 0 deletions src/scripts/test-mutation-report.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
#!/usr/bin/env bash
# test-mutation-report.sh — exercises the run phase of mutation-report.sh
# against a stubbed gremlins and a throwaway git repository (#17349).
#
# The stub is the point: gremlins is slow and nondeterministic, and the cases
# that matter here (survivors in a changed file, a clean run, a package that
# outlives the time budget) are shapes of its output and its runtime, not
# anything a real mutation run can be asked to produce on demand.
#
# The budget section is the one that must never regress: a run that hits its
# time budget must still write comment.md, and that comment must say which
# packages were not mutated. Before #17602's sibling PRs were triaged, the job
# timeout killed the step first, nothing was written, and the PR kept the
# comment from an earlier head with no sign it was stale.
set -u -o pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT="${HERE}/mutation-report.sh"
TMP_ROOT="${HERE}/../.test-tmp"
mkdir -p "$TMP_ROOT"
TMP="$(mktemp -d "$TMP_ROOT/mutation-report.XXXXXX")"
trap 'rm -rf "$TMP"' EXIT

fail=0
pass() { echo " ok: $*"; }
bad() { echo " FAIL: $*"; fail=1; }

# --- the gremlins stub -------------------------------------------------------
#
# Honours only `unleash ... --output <path>`. STUB_MODE picks the behaviour:
# lived one LIVED mutant in the package's changed file and one in a file
# the PR did not touch (must be filtered out)
# clean a KILLED mutant only
# hang sleep past any budget without writing output; the script's
# `timeout` must kill it
# STUB_CALLS records every package directory it was run from.
STUB_DIR="$TMP/stub"
mkdir -p "$STUB_DIR"
cat > "$STUB_DIR/gremlins" <<'STUB'
#!/usr/bin/env bash
set -u
pwd >> "$STUB_CALLS"
out=""
while [ $# -gt 0 ]; do
case "$1" in --output) out="$2"; shift ;; esac
shift
done
case "${STUB_MODE:-clean}" in
hang) sleep 30; exit 0 ;;
lived)
cat > "$out" <<'JSON'
{"files":[
{"file_name":"./changed.go","mutations":[{"line":3,"type":"CONDITIONALS_NEGATION","status":"LIVED"}]},
{"file_name":"./untouched.go","mutations":[{"line":1,"type":"ARITHMETIC_BASE","status":"LIVED"}]}
]}
JSON
;;
*)
cat > "$out" <<'JSON'
{"files":[{"file_name":"./changed.go","mutations":[{"line":3,"type":"CONDITIONALS_NEGATION","status":"KILLED"}]}]}
JSON
;;
esac
exit 0
STUB
chmod +x "$STUB_DIR/gremlins"

# --- the repository ----------------------------------------------------------
#
# origin/main has two packages; the PR head edits one non-test file in each.
REPO="$TMP/repo"
mkdir -p "$REPO"
git -C "$REPO" init -q -b main
git -C "$REPO" config user.email t@example.com
git -C "$REPO" config user.name t
for p in alpha beta; do
mkdir -p "$REPO/src/pkg/$p"
printf 'package %s\n\nvar X = 1\n' "$p" > "$REPO/src/pkg/$p/changed.go"
printf 'package %s\n' "$p" > "$REPO/src/pkg/$p/untouched.go"
done
git -C "$REPO" add -A && git -C "$REPO" commit -q -m base
git -C "$REPO" update-ref refs/remotes/origin/main HEAD
git -C "$REPO" checkout -q -b pr
for p in alpha beta; do
printf 'package %s\n\nvar X = 2\n' "$p" > "$REPO/src/pkg/$p/changed.go"
done
printf 'package alpha\n' > "$REPO/src/pkg/alpha/changed_test.go"
git -C "$REPO" add -A && git -C "$REPO" commit -q -m change
HEAD_SHORT=$(git -C "$REPO" rev-parse --short HEAD)

run_phase() {
# $1 = STUB_MODE, $2 = MUTATION_BUDGET_SECONDS; the report lands in $OUT,
# the exit status in $RC.
OUT="$TMP/out-$1-$2"
rm -rf "$OUT"
export STUB_CALLS="$TMP/calls-$1-$2"
: > "$STUB_CALLS"
(cd "$REPO" && PATH="$STUB_DIR:$PATH" STUB_MODE="$1" \
MUTATION_BUDGET_SECONDS="$2" MUTATION_OUT_DIR="$OUT" BASE_REF=main \
bash "$SCRIPT" run >"$OUT.log" 2>&1)
RC=$?
}

echo "survivors in changed files"
run_phase lived 600
[ "$RC" = 0 ] || bad "run phase exited $RC"
BODY="$OUT/comment.md"
[ -s "$BODY" ] || bad "no comment.md written"
grep -qF '<!-- hive-mutation -->' "$BODY" && pass "comment starts with the upsert marker" || bad "marker missing"
grep -qF "Head \`${HEAD_SHORT}\`" "$BODY" && pass "comment names the head it reports on" || bad "head sha missing: $(cat "$BODY")"
grep -qF '**2 surviving mutant(s)**' "$BODY" && pass "one survivor per changed package counted" || bad "survivor count wrong: $(cat "$BODY")"
grep -qF '| `src/pkg/alpha/changed.go` | 3 | CONDITIONALS_NEGATION |' "$BODY" && pass "survivor re-anchored to a repo-relative path" || bad "alpha survivor row missing"
grep -qF 'untouched.go' "$BODY" && bad "survivor in an untouched file leaked into the report" || pass "survivors in untouched files filtered out"
grep -qF 'Time budget' "$BODY" && bad "complete run flagged as partial" || pass "complete run carries no budget note"
[ "$(wc -l < "$STUB_CALLS" | tr -d ' ')" = 2 ] && pass "gremlins ran once per changed package" || bad "gremlins calls: $(cat "$STUB_CALLS")"

echo "clean run"
run_phase clean 600
BODY="$OUT/comment.md"
grep -qxF 'No surviving mutants in changed files.' "$BODY" && pass "clean run says so without hedging" || bad "clean message wrong: $(cat "$BODY")"
grep -qF 'time budget' "$BODY" && bad "clean run still hedges about the time budget" || pass "clean run is not conflated with a timeout"

echo "time budget exhausted"
run_phase hang 2
[ "$RC" = 0 ] || bad "run phase exited $RC after the budget"
BODY="$OUT/comment.md"
[ -s "$BODY" ] || bad "budget run wrote no comment.md — the stale-comment bug is back"
grep -qF 'Time budget (2s) exhausted' "$BODY" && pass "partial report is labelled as partial" || bad "budget note missing: $(cat "$BODY")"
grep -qF '2 of 2 changed package(s) were not fully mutated' "$BODY" && pass "unfinished count covers both packages" || bad "unfinished count wrong: $(cat "$BODY")"
grep -qF -- '- `src/pkg/alpha` — unfinished' "$BODY" && pass "package killed by the budget is listed as unfinished" || bad "alpha not listed as unfinished"
grep -qF -- '- `src/pkg/beta` — not started' "$BODY" && pass "package after the budget is listed as not started" || bad "beta not listed as not started"
grep -qF 'No surviving mutants reported in the packages that finished' "$BODY" && pass "no-survivor line is scoped to finished packages" || bad "scoped no-survivor line missing"
[ "$(wc -l < "$STUB_CALLS" | tr -d ' ')" = 1 ] && pass "gremlins was not started once the budget was gone" || bad "gremlins calls after budget: $(cat "$STUB_CALLS")"
grep -qF 'hit the time budget in pkg/alpha' "$OUT.log" && pass "log names the package that outlived the budget" || bad "log line missing: $(cat "$OUT.log")"

echo "test files only"
git -C "$REPO" checkout -q -b tests-only origin/main
printf 'package alpha\n' > "$REPO/src/pkg/alpha/new_test.go"
git -C "$REPO" add -A && git -C "$REPO" commit -q -m tests
run_phase clean 600
[ -f "$OUT/comment.md" ] && bad "tests-only diff produced a report" || pass "tests-only diff writes no report"
grep -qF 'No changed non-test Go files' "$OUT.log" && pass "tests-only diff is skipped explicitly" || bad "skip message missing"

if [ "$fail" -ne 0 ]; then
echo "FAILED"
exit 1
fi
echo "all tests passed"
Loading