From fbf8c932d7741d2f47fad1422a555fe1ff47f002 Mon Sep 17 00:00:00 2001 From: saiththerobo Date: Tue, 3 Nov 2026 17:24:42 +0000 Subject: [PATCH 1/2] tests/packages: run the package suites side by side, the slow ones first CI's test job took 934 s on #389, and three suites ran most of it in a row: omamobile-audio 316 s, omamobile-shell 263 s and omamobile-modem-sit 138 s, against about 217 s for the other suites together. The runner has four CPUs. The suites now run up to OMAMOBILE_TEST_JOBS at once (default: the CPUs, at most 8; 1 is the old order), audio, shell and modem-sit first. Each suite's output is kept together and printed when it ends, with the same ok / not ok lines and summary as before. Locally, with four jobs, every package suite took 308 s; the four that failed failed the same way one at a time (python-dbus and python-gobject missing on that machine), none because of the others. Co-Authored-By: Claude Opus 5.5 --- tests/packages | 56 ++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/tests/packages b/tests/packages index ddffc3de1..f585ba43b 100755 --- a/tests/packages +++ b/tests/packages @@ -1,23 +1,63 @@ #!/bin/bash # SPDX-License-Identifier: MIT -# Every package's own tests (packages/*/test/all), one after another, with a -# summary at the end; CI runs it on every pull request. Each suite skips -# what its machine can't run (no qmllint, no Docker, no phone) by itself. +# Every package's own tests (packages/*/test/all), with a summary at the +# end; CI runs it on every pull request. Each suite skips what its machine +# can't run (no qmllint, no Docker, no phone) by itself. +# +# The suites run side by side, up to OMAMOBILE_TEST_JOBS at once (default: +# the CPUs, at most 8; 1 runs them one after another as before), the slow +# ones first, since three of them (audio, shell, modem-sit) took three +# quarters of CI's time when run in a row. Each suite's output is kept +# together and printed when it ends, with its ok or not ok line. # # tests/packages [NAME...] (default: every package with test/all) set -uo pipefail cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." if (($#)); then names=("$@"); else + # The slowest first, so the run isn't left waiting on one at the end. names=() - for t in packages/*/test/all; do names+=("$(basename "$(dirname "$(dirname "$t")")")"); done + for n in omamobile-audio omamobile-shell omamobile-modem-sit; do + [[ -x packages/$n/test/all ]] && names+=("$n") + done + for t in packages/*/test/all; do + n=$(basename "$(dirname "$(dirname "$t")")") + [[ " ${names[*]} " == *" $n "* ]] || names+=("$n") + done fi +jobs=${OMAMOBILE_TEST_JOBS:-$(nproc 2>/dev/null || echo 1)} +((jobs > 8)) && jobs=8 +((jobs < 1)) && jobs=1 +out=$(mktemp -d); trap 'rm -rf "$out"' EXIT failed=() + +# One suite, its output to $out/NAME.log and its result to $out/NAME.rc. +run() { + local n=$1 t=packages/$1/test/all + if [[ ! -x $t ]]; then echo "not ok - $n has no test/all" > "$out/$n.log"; echo 1 > "$out/$n.rc"; return; fi + { echo "# $n"; "$t"; } > "$out/$n.log" 2>&1 + echo $? > "$out/$n.rc" +} +# Print the suites that have ended, each in one piece. +report() { + local f n rc + for f in "$out"/*.rc; do + [[ -e $f ]] || continue + n=$(basename "$f" .rc); rc=$(<"$f") + cat "$out/$n.log" + if [[ $rc == 0 ]]; then echo "ok - $n"; else echo "not ok - $n"; failed+=("$n"); fi + rm -f "$f" "$out/$n.log" + done +} + +running=0 for n in "${names[@]}"; do - t=packages/$n/test/all - [[ -x $t ]] || { echo "not ok - $n has no test/all"; failed+=("$n"); continue; } - echo "# $n" - if "$t"; then echo "ok - $n"; else echo "not ok - $n"; failed+=("$n"); fi + if ((running >= jobs)); then wait -n; running=$((running - 1)); report; fi + run "$n" & + running=$((running + 1)) done +wait +report + if ((${#failed[@]})); then echo "failed: ${failed[*]}" >&2 exit 1 From 7eb4ff8e3a515161f0afbdadcc44cf462a82bc52 Mon Sep 17 00:00:00 2001 From: saiththerobo Date: Wed, 4 Nov 2026 07:08:10 +0000 Subject: [PATCH 2/2] tests/packages: a result only once written, a missing one fails, a time limit per suite Review on #410: `echo $? > NAME.rc` created the file before the number was in it, so a report running at that moment read an empty result, printed not ok and removed it; the result is now written aside and moved into place. A suite whose subshell died before writing a result passed silently, where the old runner counted it as failed; every name is now checked at the end and reported "not ok - NAME (no result)". And since output is held until a suite ends, a hung suite showed nothing in CI: each suite now stops after OMAMOBILE_TEST_TIMEOUT seconds (default 1800, the slowest takes about 320 s) and fails with its log printed. With OMAMOBILE_TEST_JOBS=1 the slow suites still go first, so the comment says "one at a time" rather than "as before". Co-Authored-By: Claude Opus 5.5 --- tests/packages | 36 ++++++++++++++++++++++++++---------- 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/tests/packages b/tests/packages index f585ba43b..d9b6476ca 100755 --- a/tests/packages +++ b/tests/packages @@ -5,10 +5,12 @@ # can't run (no qmllint, no Docker, no phone) by itself. # # The suites run side by side, up to OMAMOBILE_TEST_JOBS at once (default: -# the CPUs, at most 8; 1 runs them one after another as before), the slow -# ones first, since three of them (audio, shell, modem-sit) took three -# quarters of CI's time when run in a row. Each suite's output is kept -# together and printed when it ends, with its ok or not ok line. +# the CPUs, at most 8; 1 runs them one at a time), the slow ones first, +# since three of them (audio, shell, modem-sit) took three quarters of CI's +# time when run in a row. Each suite's output is kept together and printed +# when it ends, with its ok or not ok line. A suite that runs longer than +# OMAMOBILE_TEST_TIMEOUT seconds (default 1800) is stopped and fails, and a +# suite that leaves no result fails too. # # tests/packages [NAME...] (default: every package with test/all) set -uo pipefail @@ -24,18 +26,25 @@ if (($#)); then names=("$@"); else [[ " ${names[*]} " == *" $n "* ]] || names+=("$n") done fi +tmo=${OMAMOBILE_TEST_TIMEOUT:-1800} jobs=${OMAMOBILE_TEST_JOBS:-$(nproc 2>/dev/null || echo 1)} ((jobs > 8)) && jobs=8 ((jobs < 1)) && jobs=1 out=$(mktemp -d); trap 'rm -rf "$out"' EXIT -failed=() +failed=() reported=() -# One suite, its output to $out/NAME.log and its result to $out/NAME.rc. +# One suite, its output to $out/NAME.log and its result to $out/NAME.rc, +# which appears only once it holds the number (written aside, then moved). run() { - local n=$1 t=packages/$1/test/all - if [[ ! -x $t ]]; then echo "not ok - $n has no test/all" > "$out/$n.log"; echo 1 > "$out/$n.rc"; return; fi - { echo "# $n"; "$t"; } > "$out/$n.log" 2>&1 - echo $? > "$out/$n.rc" + local n=$1 t=packages/$1/test/all rc + if [[ ! -x $t ]]; then + echo "not ok - $n has no test/all" > "$out/$n.log"; rc=1 + else + { echo "# $n"; timeout -k 10 "$tmo" "$t"; } > "$out/$n.log" 2>&1 + rc=$? + ((rc == 124)) && echo "# $n: stopped after ${tmo}s" >> "$out/$n.log" + fi + echo "$rc" > "$out/$n.rc.tmp" && mv "$out/$n.rc.tmp" "$out/$n.rc" } # Print the suites that have ended, each in one piece. report() { @@ -45,6 +54,7 @@ report() { n=$(basename "$f" .rc); rc=$(<"$f") cat "$out/$n.log" if [[ $rc == 0 ]]; then echo "ok - $n"; else echo "not ok - $n"; failed+=("$n"); fi + reported+=("$n") rm -f "$f" "$out/$n.log" done } @@ -57,6 +67,12 @@ for n in "${names[@]}"; do done wait report +# A suite whose subshell died before it wrote a result. +for n in "${names[@]}"; do + [[ " ${reported[*]} " == *" $n "* ]] && continue + [[ -e $out/$n.log ]] && cat "$out/$n.log" + echo "not ok - $n (no result)"; failed+=("$n") +done if ((${#failed[@]})); then echo "failed: ${failed[*]}" >&2