Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-mutation-advisory-budget.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- CI: the advisory mutation job now bounds its `gremlins` runs with an in-script time budget (`MUTATION_BUDGET_SECONDS`, default 900s) instead of running into the job's 20-minute `timeout-minutes`, which GitHub reports as a `cancelled` check-run that lost the report and blocked the merge gate. Packages that do not fit are skipped and named in the PR comment, the partial report is kept, and the job finishes `success` (hivecommons/hive#17605).
31 changes: 29 additions & 2 deletions src/scripts/mutation-report.sh
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,10 @@ fi

base_ref="${BASE_REF:?BASE_REF is required}"
coefficient="${MUTATION_TIMEOUT_COEFFICIENT:-3}"
# Wall-clock budget for all gremlins runs. Stays under the job's
# timeout-minutes (20) minus checkout/install, because GitHub reports a
# timeout-minutes kill as a `cancelled` check-run and the report is lost (#17605).
budget="${MUTATION_BUDGET_SECONDS:-900}"
mkdir -p "$out_dir"
out_abs=$(cd "$out_dir" && pwd)

Expand Down Expand Up @@ -95,14 +99,30 @@ pkgs="${roots%$'\n'}"
# --diff) the whole package is mutated and the job times out (#17463). Coverage
# chdirs to the module root, so the output path must be absolute.
i=0
skipped=""
cut_off=""
start=$SECONDS
while IFS= read -r pkg; do
[ -n "$pkg" ] || continue
i=$((i + 1))
remaining=$((budget - (SECONDS - start)))
if [ "$remaining" -le 0 ]; then
echo "Time budget (${budget}s) exhausted; skipping $pkg"
skipped+="${skipped:+, }\`$pkg\`"
continue
fi
printf '%s\n' "$pkg" > "$out_dir/pkg-$i.dir"
echo "::group::gremlins unleash ./$pkg --diff origin/${base_ref}"
(cd "src/$pkg" && GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=diff.relative GIT_CONFIG_VALUE_0=true \
gremlins unleash --timeout-coefficient "$coefficient" --diff "origin/${base_ref}" \
--output "$out_abs/pkg-$i.json") || echo "gremlins exited non-zero for $pkg (advisory)"
timeout "$remaining" gremlins unleash --timeout-coefficient "$coefficient" --diff "origin/${base_ref}" \
--output "$out_abs/pkg-$i.json")
rc=$?
if [ "$rc" -eq 124 ]; then
echo "gremlins hit the time budget in $pkg; its report is partial (advisory)"
cut_off+="${cut_off:+, }\`$pkg\`"
elif [ "$rc" -ne 0 ]; then
echo "gremlins exited non-zero for $pkg (advisory)"
fi
echo "::endgroup::"
done <<< "$pkgs"

Expand Down Expand Up @@ -141,6 +161,13 @@ done < "$survivors"
else
echo "No surviving mutants in changed files (or the run hit its time budget before reporting any)."
fi
if [ -n "$skipped$cut_off" ]; then
echo
note="_Time budget (${budget}s) exhausted:"
[ -z "$skipped" ] || note+=" skipped $skipped;"
[ -z "$cut_off" ] || note+=" $cut_off was cut off mid-run;"
echo "${note%;}._"
fi
echo
echo "_Advisory only; this job never blocks the PR (#17349)._"
} > "$body"
Expand Down
76 changes: 76 additions & 0 deletions src/scripts/test-mutation-report.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
#!/usr/bin/env bash
# test-mutation-report.sh — exercises the in-script time budget of
# mutation-report.sh (#17605) with a fake `gremlins` on PATH.
set -u -o pipefail

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPORT="${REPORT:-${HERE}/mutation-report.sh}"
TMP_ROOT="${HERE}/../.test-tmp"
mkdir -p "$TMP_ROOT"
TMP="$(mktemp -d "$TMP_ROOT/mutation-report.XXXXXX")"
trap 'rm -rf "$TMP"' EXIT

fail=0
pass() { echo " ok: $*"; }
bad() { echo " FAIL: $*"; fail=1; }

repo="$TMP/repo"
mkdir -p "$repo/src/foo" "$TMP/bin"
cd "$repo" || exit 1
git init -q
git config user.name 'Test Author'
git config user.email 'test@example.com'
echo base > README.md
git add README.md
git commit -q -m base
git update-ref refs/remotes/origin/base HEAD
printf 'package foo\n\nfunc Foo() int { return 1 }\n' > src/foo/foo.go
git add src/foo/foo.go
git commit -q -m 'add foo'

# $1 = sleep seconds, $2 = whether the fake writes a report
make_gremlins() {
cat > "$TMP/bin/gremlins" <<EOF
#!/usr/bin/env bash
sleep $1
if [ "$2" = write ]; then
while [ \$# -gt 0 ]; do
[ "\$1" = --output ] && echo '{"files":[]}' > "\$2"
shift
done
fi
EOF
chmod +x "$TMP/bin/gremlins"
}

run_report() { # $1 = budget; sets rc, elapsed
rm -rf "$TMP/out"
local t0=$SECONDS
PATH="$TMP/bin:$PATH" BASE_REF=base MUTATION_BUDGET_SECONDS="$1" \
MUTATION_OUT_DIR="$TMP/out" bash "$REPORT" run >"$TMP/run.log" 2>&1
rc=$?
elapsed=$((SECONDS - t0))
}

echo "case 1: gremlins outlives the budget"
make_gremlins 20 nowrite
run_report 1
[ "$rc" -eq 0 ] && pass "exit 0" || bad "exit code $rc"
[ "$elapsed" -lt 10 ] && pass "cut off after ${elapsed}s" || bad "ran ${elapsed}s; timeout did not fire"
grep -q 'Time budget (1s) exhausted' "$TMP/out/comment.md" 2>/dev/null \
&& pass "comment mentions budget" || bad "comment lacks budget line"
grep -q 'Advisory only' "$TMP/out/comment.md" 2>/dev/null \
&& pass "advisory footer kept" || bad "advisory footer missing"

echo "case 2: gremlins finishes within the budget"
make_gremlins 0 write
run_report 60
[ "$rc" -eq 0 ] && pass "exit 0" || bad "exit code $rc"
[ -s "$TMP/out/comment.md" ] && pass "comment written" || bad "comment missing"
if grep -q 'Time budget' "$TMP/out/comment.md" 2>/dev/null; then
bad "unexpected budget line"
else
pass "no budget line"
fi

[ "$fail" -eq 0 ] && echo "all passed" || { echo "FAILED"; exit 1; }
Loading