From 97adb732d92f96ca23c0cf55c740ebc0d30633f2 Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 10 Nov 2026 18:07:08 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20mutation-advisory:=20bound=20the?= =?UTF-8?q?=20gremlins=20run=20to=20an=20in-script=20time=20budget=20so=20?= =?UTF-8?q?the=20job=20is=20never=20cancelled?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-mutation-advisory-budget.md | 1 + src/scripts/mutation-report.sh | 31 +++++++- src/scripts/test-mutation-report.sh | 76 +++++++++++++++++++ 3 files changed, 106 insertions(+), 2 deletions(-) create mode 100644 changelog.d/fixed-mutation-advisory-budget.md create mode 100755 src/scripts/test-mutation-report.sh diff --git a/changelog.d/fixed-mutation-advisory-budget.md b/changelog.d/fixed-mutation-advisory-budget.md new file mode 100644 index 0000000000..a626b13450 --- /dev/null +++ b/changelog.d/fixed-mutation-advisory-budget.md @@ -0,0 +1 @@ +- CI: the advisory mutation job now bounds its `gremlins` runs with an in-script time budget (`MUTATION_BUDGET_SECONDS`, default 900s) instead of running into the job's 20-minute `timeout-minutes`, which GitHub reports as a `cancelled` check-run that lost the report and blocked the merge gate. Packages that do not fit are skipped and named in the PR comment, the partial report is kept, and the job finishes `success` (hivecommons/hive#17605). diff --git a/src/scripts/mutation-report.sh b/src/scripts/mutation-report.sh index 5187c11e66..5711929d87 100755 --- a/src/scripts/mutation-report.sh +++ b/src/scripts/mutation-report.sh @@ -52,6 +52,10 @@ fi base_ref="${BASE_REF:?BASE_REF is required}" coefficient="${MUTATION_TIMEOUT_COEFFICIENT:-3}" +# Wall-clock budget for all gremlins runs. Stays under the job's +# timeout-minutes (20) minus checkout/install, because GitHub reports a +# timeout-minutes kill as a `cancelled` check-run and the report is lost (#17605). +budget="${MUTATION_BUDGET_SECONDS:-900}" mkdir -p "$out_dir" out_abs=$(cd "$out_dir" && pwd) @@ -95,14 +99,30 @@ pkgs="${roots%$'\n'}" # --diff) the whole package is mutated and the job times out (#17463). Coverage # chdirs to the module root, so the output path must be absolute. i=0 +skipped="" +cut_off="" +start=$SECONDS while IFS= read -r pkg; do [ -n "$pkg" ] || continue i=$((i + 1)) + remaining=$((budget - (SECONDS - start))) + if [ "$remaining" -le 0 ]; then + echo "Time budget (${budget}s) exhausted; skipping $pkg" + skipped+="${skipped:+, }\`$pkg\`" + continue + fi printf '%s\n' "$pkg" > "$out_dir/pkg-$i.dir" echo "::group::gremlins unleash ./$pkg --diff origin/${base_ref}" (cd "src/$pkg" && GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=diff.relative GIT_CONFIG_VALUE_0=true \ - gremlins unleash --timeout-coefficient "$coefficient" --diff "origin/${base_ref}" \ - --output "$out_abs/pkg-$i.json") || echo "gremlins exited non-zero for $pkg (advisory)" + timeout "$remaining" gremlins unleash --timeout-coefficient "$coefficient" --diff "origin/${base_ref}" \ + --output "$out_abs/pkg-$i.json") + rc=$? + if [ "$rc" -eq 124 ]; then + echo "gremlins hit the time budget in $pkg; its report is partial (advisory)" + cut_off+="${cut_off:+, }\`$pkg\`" + elif [ "$rc" -ne 0 ]; then + echo "gremlins exited non-zero for $pkg (advisory)" + fi echo "::endgroup::" done <<< "$pkgs" @@ -141,6 +161,13 @@ done < "$survivors" else echo "No surviving mutants in changed files (or the run hit its time budget before reporting any)." fi + if [ -n "$skipped$cut_off" ]; then + echo + note="_Time budget (${budget}s) exhausted:" + [ -z "$skipped" ] || note+=" skipped $skipped;" + [ -z "$cut_off" ] || note+=" $cut_off was cut off mid-run;" + echo "${note%;}._" + fi echo echo "_Advisory only; this job never blocks the PR (#17349)._" } > "$body" diff --git a/src/scripts/test-mutation-report.sh b/src/scripts/test-mutation-report.sh new file mode 100755 index 0000000000..a34c85016c --- /dev/null +++ b/src/scripts/test-mutation-report.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# test-mutation-report.sh — exercises the in-script time budget of +# mutation-report.sh (#17605) with a fake `gremlins` on PATH. +set -u -o pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPORT="${REPORT:-${HERE}/mutation-report.sh}" +TMP_ROOT="${HERE}/../.test-tmp" +mkdir -p "$TMP_ROOT" +TMP="$(mktemp -d "$TMP_ROOT/mutation-report.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +fail=0 +pass() { echo " ok: $*"; } +bad() { echo " FAIL: $*"; fail=1; } + +repo="$TMP/repo" +mkdir -p "$repo/src/foo" "$TMP/bin" +cd "$repo" || exit 1 +git init -q +git config user.name 'Test Author' +git config user.email 'test@example.com' +echo base > README.md +git add README.md +git commit -q -m base +git update-ref refs/remotes/origin/base HEAD +printf 'package foo\n\nfunc Foo() int { return 1 }\n' > src/foo/foo.go +git add src/foo/foo.go +git commit -q -m 'add foo' + +# $1 = sleep seconds, $2 = whether the fake writes a report +make_gremlins() { + cat > "$TMP/bin/gremlins" < "\$2" + shift + done +fi +EOF + chmod +x "$TMP/bin/gremlins" +} + +run_report() { # $1 = budget; sets rc, elapsed + rm -rf "$TMP/out" + local t0=$SECONDS + PATH="$TMP/bin:$PATH" BASE_REF=base MUTATION_BUDGET_SECONDS="$1" \ + MUTATION_OUT_DIR="$TMP/out" bash "$REPORT" run >"$TMP/run.log" 2>&1 + rc=$? + elapsed=$((SECONDS - t0)) +} + +echo "case 1: gremlins outlives the budget" +make_gremlins 20 nowrite +run_report 1 +[ "$rc" -eq 0 ] && pass "exit 0" || bad "exit code $rc" +[ "$elapsed" -lt 10 ] && pass "cut off after ${elapsed}s" || bad "ran ${elapsed}s; timeout did not fire" +grep -q 'Time budget (1s) exhausted' "$TMP/out/comment.md" 2>/dev/null \ + && pass "comment mentions budget" || bad "comment lacks budget line" +grep -q 'Advisory only' "$TMP/out/comment.md" 2>/dev/null \ + && pass "advisory footer kept" || bad "advisory footer missing" + +echo "case 2: gremlins finishes within the budget" +make_gremlins 0 write +run_report 60 +[ "$rc" -eq 0 ] && pass "exit 0" || bad "exit code $rc" +[ -s "$TMP/out/comment.md" ] && pass "comment written" || bad "comment missing" +if grep -q 'Time budget' "$TMP/out/comment.md" 2>/dev/null; then + bad "unexpected budget line" +else + pass "no budget line" +fi + +[ "$fail" -eq 0 ] && echo "all passed" || { echo "FAILED"; exit 1; }