From 4f5cde6c35e8a9ee537a91e410447c07148e3940 Mon Sep 17 00:00:00 2001 From: Derek Ross Date: Mon, 2 Nov 2026 01:27:02 -0500 Subject: [PATCH 1/2] omamobile-audio 0.5.9: the watchdog tests no longer race the runner The watchdog tests kept failing on GitHub's runners and passing locally, and #315, #321 and #325 each patched one case. Two causes, both in the test setup, neither in what the phone enforces: - A lost update in the fake mixer. The JSON backend's heartbeat read loads the whole state file, bumps the heartbeat and saves the file; test/fake-amixer's cset does the same. When a test's cset landed between the watchdog's load and save, the save put the old coil temperature back, so the watchdog never saw the jump. A tight loop lost 38 of 40 writes. Both now hold a flock on .lock around load-modify-save (0 of 40 lost). - Events timed by sleeps. Helpers waited for "watchdog: watching" and then slept a fixed time (or slept from the start, for the kmsg, heartbeat-freeze, feedback and earpiece cases) before changing the tree, while the watchdog's grace, heartbeat timeout and settle window ran on the real clock. On a slow runner the change landed before the baseline, after the last round, or (kmsg) before the log was opened. The start line itself was never late: log() already flushes. The watchdog loop takes a test-only OMAMOBILE_WATCH_HOOK: a command run with the round number before each round, and with it set the clock is round x interval. test/all's at ROUND COMMAND queues an event, so every case that changed the tree mid-watch (jump, c9 jump-off, c9settle's step/rise/cold, heartbeat freeze, both kmsg cases, feedback, earpiece run) now lands at a known round relative to the baseline, however slow the machine. The unacknowledged-pause case also checks the note is logged now, so it can't pass by missing the record. Unset on the phone, the loop runs exactly as before: the real clock, nothing run, the same thresholds and timing. The JSON backend is tests-only. The version moves because speaker_dsp.py ships. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BeEcdVeSfyLCbijF8XQF1d --- packages/omamobile-audio/lib/speaker_dsp.py | 30 +++++++- packages/omamobile-audio/test/all | 80 +++++++++++---------- packages/omamobile-audio/test/fake-amixer | 5 ++ packages/omamobile-audio/version | 2 +- 4 files changed, 79 insertions(+), 38 deletions(-) diff --git a/packages/omamobile-audio/lib/speaker_dsp.py b/packages/omamobile-audio/lib/speaker_dsp.py index 39cd349ba..a395a0102 100644 --- a/packages/omamobile-audio/lib/speaker_dsp.py +++ b/packages/omamobile-audio/lib/speaker_dsp.py @@ -218,6 +218,16 @@ class JsonCtl: def __init__(self, path): self.path = path + def _locked(self): + # The heartbeat read and write_bytes load, change and save the whole + # file, and test/fake-amixer does the same under the same lock. + # Without it a save here put back a value a test had just written (a + # coil temperature step lost, so the watchdog saw no jump). + import fcntl + f = open(f'{self.path}.lock', 'a') + fcntl.flock(f, fcntl.LOCK_EX) + return f + def _load(self): try: with open(self.path) as f: @@ -237,6 +247,10 @@ def names(self): return list(self._load()) def read(self, name): + with self._locked(): + return self._read(name) + + def _read(self, name): state = self._load() c = state.get(name) if c is None: @@ -262,6 +276,10 @@ def read_bytes(self, name): return bytes(c['v']) if c and c['type'] == 'BYTES' else None def write_bytes(self, name, data): + with self._locked(): + return self._write_bytes(name, data) + + def _write_bytes(self, name, data): state = self._load() c = state.get(name) if not c or c['type'] != 'BYTES' or len(c['v']) != len(data): @@ -1215,9 +1233,19 @@ def watch_loop(args, amps, devs, kmsg, pcm, digital_ceil, analog_ceil, temp_max, failures = 0 n = 0 acked = None + # Tests only (test/all): OMAMOBILE_WATCH_HOOK is a command run with the + # round number before each round, and the clock is then the round count + # times the interval, so a test's event lands at a known round and the + # grace, heartbeat and settle times count rounds, however slow the + # machine. Unset on the phone: the real clock, nothing run. + hook = os.environ.get('OMAMOBILE_WATCH_HOOK') while args.iterations == 0 or n < args.iterations: n += 1 - now = time.monotonic() + if hook: + subprocess.run([hook, str(n)], check=True) + now = n * args.interval + else: + now = time.monotonic() # The mode (C9k): taken at the start of a round and answered in the # ack file, which omamobile-speaker-protection waits for before it # changes anything the new mode allows and the old one does not. diff --git a/packages/omamobile-audio/test/all b/packages/omamobile-audio/test/all index e69270fed..1ecc7c021 100755 --- a/packages/omamobile-audio/test/all +++ b/packages/omamobile-audio/test/all @@ -335,13 +335,31 @@ rc=0; prep=(OMAMOBILE_ASOUND=$r/none); sp start > "$r/out" 2>&1 || rc=$?; prep=( pass "a hang marker from the last boot keeps the amplifiers off" # --- the watchdog ---------------------------------------------------------------- -watchrun() { # name: run the watchdog briefly on a healthy started tree; $1 hook runs before +watchrun() { # run the watchdog briefly on a healthy started tree, with the events of at() prep=(OMAMOBILE_WATCH_INTERVAL=0.05 OMAMOBILE_WATCH_GRACE=0.1 OMAMOBILE_WATCH_HB_TIMEOUT=0.3 OMAMOBILE_WATCH_ITERATIONS=${WATCH_N:-20} "${wprep[@]}") + [[ ! -e $r/events ]] || prep+=(OMAMOBILE_WATCH_HOOK="$tmp/watch-hook" WATCH_EVENTS="$r/events" WATCH_R="$r") rc=0; sp watch > "$r/watch.out" 2>&1 || rc=$? prep=(); wprep=() } wprep=() +# An event during a watch: `at ROUND COMMAND` runs COMMAND (amx, kmsg, any +# shell) just before the watchdog's round ROUND, inside the watchdog's own +# loop (OMAMOBILE_WATCH_HOOK), and the watchdog's clock is then ROUND times +# the interval. So "after the stream's baseline" or "before the settle time" +# is a round number, not a sleep that a slow CI runner outran: the stream +# starts in round 1 (time 0), round k is (k-1) x 0.05 s into it. +at() { printf '%s %s\n' "$1" "$2" >> "$r/events"; } +cat > "$tmp/watch-hook" <<'EOF' +#!/bin/bash +set -euo pipefail +r=$WATCH_R +while read -r k cmd; do + [[ $k != "$1" ]] || eval "$cmd" +done < "$WATCH_EVENTS" +EOF +chmod +x "$tmp/watch-hook" +export -f amx kmsg startok() { run "$1"; ((rc == 0)) || { cat "$r/out" >&2; fail "$1: start"; }; } startok w-idle; watchrun ((rc == 0)) && ! muted || { cat "$r/watch.out" >&2; fail "watchdog: idle and healthy, nothing happens"; } @@ -364,8 +382,8 @@ startok w-hb8-log; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; ((rc == 0)) && ! muted && grep -c 'stuck' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: OMAMOBILE_SPK_HEARTBEAT=0 logs a stuck heartbeat in c8"; } # the C9a case: the heartbeat moves, then stops while the stream still runs startok w-hb-freeze; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status" -( sleep 0.5; touch "$r/freeze" ) & -wprep=(FAKE_HB_FREEZE_FILE=$r/freeze); WATCH_N=40 watchrun; wait +at 10 'touch "$r/freeze"' # the heartbeat stops 0.45 s into the stream, after the grace +wprep=(FAKE_HB_FREEZE_FILE=$r/freeze); WATCH_N=40 watchrun ((rc == 1)) && muted || { cat "$r/watch.out" >&2; fail "watchdog: a heartbeat that stops mid-stream mutes"; } startok w-hb9-off; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_HEARTBEAT=0); watchrun ((rc == 1)) && muted || fail "watchdog: the heartbeat cannot be switched off in stage c9" @@ -373,13 +391,13 @@ startok w-hb9; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wpre ((rc == 1)) && muted || fail "watchdog: a stuck heartbeat mutes in stage c9" # a kernel log error after the watchdog started startok w-kmsg -( sleep 0.4; kmsg 1.0 'cs35l41 spi6.1: Wake failed, re-enter hibernate: -42' ) & -WATCH_N=40 watchrun; wait +at 8 "kmsg 1.0 'cs35l41 spi6.1: Wake failed, re-enter hibernate: -42'" +WATCH_N=40 watchrun ((rc == 1)) && muted || { cat "$r/watch.out" >&2; fail "watchdog: a failed wake in the kernel log mutes"; } startok w-kmsg-warn -( sleep 0.4; kmsg 1.0 'cs35l41 spi6.0: Failed to set mailbox cmd 1 (status 0)' ) & -WATCH_N=20 watchrun; wait -((rc == 0)) && ! muted || fail "watchdog: an unacknowledged pause is logged only" +at 8 "kmsg 1.0 'cs35l41 spi6.0: Failed to set mailbox cmd 1 (status 0)'" +WATCH_N=20 watchrun +((rc == 0)) && ! muted && grep -c 'watchdog: note: .*mailbox cmd 1' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: an unacknowledged pause is logged only"; } pass "watchdog: mutes on DSP errors, bypass, levels, runtime PM, a stuck heartbeat (c9) and kernel log failures" # C9j: the coil-estimate jump rule is for analog above 10. At the everyday level the @@ -387,12 +405,9 @@ pass "watchdog: mutes on DSP errors, bypass, levels, runtime PM, a stuck heartbe # level drop, so a jump there must not mute; OMAMOBILE_SPK_TEMP_JUMP brings it back. jump() { # name [wprep...]: watchdog on a running stream; the bottom's estimate goes 25 -> 40 C a moment in local n=$1; shift; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wprep=("$@") - # the jump comes once the watchdog has its first reading (its start line, - # then a few intervals), not at a fixed time: a slow runner started it late - rm -f "$r/watch.out" - ( for _ in $(seq 1 100); do grep -c 'watchdog: watching' "$r/watch.out" >/dev/null 2>&1 && break; sleep 0.05; done - sleep 0.3; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null ) & - WATCH_N=60 watchrun; wait + # in round 8 (0.35 s in): after the baseline of a 0.01 s settle (round 2) + at 8 'amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null' + WATCH_N=30 watchrun } startok w-jump8; jump w-jump8 ((rc == 0)) && ! muted && grep -c 'jump rule off' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: at analog 10 or below a 15 C jump of the estimate does not mute (C9j)"; } @@ -423,11 +438,12 @@ pass "watchdog: stock media delta at stream start, both amplifiers, or mute" # the amplifiers' V/I feedback stream (C9d-7): with each speaker stream when asked fbstate() { cat "$r/asound/card0/pcm15c/sub0/status"; } +export -f fbstate startok w-fb-off; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; watchrun ((rc == 0)) && [[ $(fbstate) == closed ]] && ! grep -c NoHost1 "$r/cset.log" >/dev/null || { cat "$r/watch.out" >&2; fail "feedback: off by default"; } startok w-fb; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status" -( sleep 0.6; [[ $(fbstate) == 'state: RUNNING' ]] && touch "$r/fb-seen"; echo closed > "$r/asound/card0/pcm5p/sub0/status" ) & -wprep=(OMAMOBILE_SPK_FEEDBACK=1); WATCH_N=30 watchrun; wait +at 12 'if [[ $(fbstate) == "state: RUNNING" ]]; then touch "$r/fb-seen"; fi; echo closed > "$r/asound/card0/pcm5p/sub0/status"' +wprep=(OMAMOBILE_SPK_FEEDBACK=1); WATCH_N=30 watchrun ((rc == 0)) && ! muted && [[ -e $r/fb-seen && $(fbstate) == closed ]] || { cat "$r/watch.out" >&2; fail "feedback: runs during the stream, stops after it"; } [[ $(grep NoHost1 "$r/cset.log") == $'NoHost1 TX Mixer TDM_0_TX=1\nNoHost1 TX Mixer TDM_0_TX=0' ]] || { cat "$r/cset.log" >&2; fail "feedback: the mixer path on, then off, as stock"; } [[ ! -e $r/run/speaker-feedback.pid ]] || fail "feedback: no pid file left" @@ -488,37 +504,29 @@ c9 c9-ok $(val "Top DRE Switch") == on && $(val "Bottom DRE Switch") == on && $(cat "$r/module/snd_soc_cs35l41/parameters/boost_en") == Y ]] || { cat "$r/out" >&2; fail "stage c9 at stock levels, boost on"; } echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status" -# as jump(): after the watchdog's first reading, not at a fixed time -rm -f "$r/watch.out" -( for _ in $(seq 1 100); do grep -c 'watchdog: watching' "$r/watch.out" >/dev/null 2>&1 && break; sleep 0.05; done - sleep 0.3; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null ) & -wprep=(OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_DIGITAL=817 OMAMOBILE_SPK_ANALOG=17 FAKE_ANALOG_MAX=17 OMAMOBILE_SPK_TEMP_SETTLE=0.01); WATCH_N=60 watchrun; wait +# as jump(): in round 8, after the baseline of a 0.01 s settle (round 2) +at 8 'amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null' +wprep=(OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_DIGITAL=817 OMAMOBILE_SPK_ANALOG=17 FAKE_ANALOG_MAX=17 OMAMOBILE_SPK_TEMP_SETTLE=0.01); WATCH_N=30 watchrun ((rc == 0)) && ! muted && grep -c 'jump rule off' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 the jump rule is off by default, so a 15 C rise plays on (the backstop stays)"; } # C9d-8: a stream's first estimates are the firmware's 25 C reference until # it has measured the coil, often warmer, a step of about a second: the # estimate at the settle time is the baseline, rises above it are held to # the jump. -c9settle() { # name settle steps...: steps are delay_s:temp_bytes, applied in order +c9settle() { # name settle steps...: steps are round:temp_bytes (at()); round k is (k-1) x 0.05 s into the stream local n=$1 st=$2; shift 2; local steps=("$@") c9 "$n"; ((rc == 0)) || { cat "$r/out" >&2; fail "$n: start"; } echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status" amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" "${C9_START:-0x00,0x06,0x40,0x00}" >/dev/null - rm -f "$r/watch.out" - ( for _ in $(seq 1 100); do grep -c 'watchdog: watching' "$r/watch.out" >/dev/null 2>&1 && break; sleep 0.05; done - for s in "${steps[@]}"; do sleep "${s%%:*}"; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" "${s#*:}" >/dev/null; done ) & + local s + for s in "${steps[@]}"; do at "${s%%:*}" "amx cset 'name=Bottom DSP1 Protection cd CSPL_TEMPERATURE' ${s#*:} >/dev/null"; done wprep=(OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_DIGITAL=817 OMAMOBILE_SPK_ANALOG=17 FAKE_ANALOG_MAX=17 OMAMOBILE_SPK_TEMP_SETTLE=$st OMAMOBILE_SPK_TEMP_JUMP=${JUMP:-5}) - # 200 polls (10 s at 0.05 s): on a slow CI runner the steps above arrive - # seconds late, and 80 polls let the watchdog finish before the rise came. - WATCH_N=200 watchrun; wait + WATCH_N=60 watchrun } -c9settle c9-step 1.5 0.2:0x00,0x0a,0x00,0x00 # 25 -> 40 C: the measured value, a step +c9settle c9-step 1.5 5:0x00,0x0a,0x00,0x00 # 25 -> 40 C at 0.2 s: the measured value, a step before the baseline (1.5 s, round 31) ((rc == 0)) && ! muted && grep -c 's into a stream' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 the step from the reference to the measured coil does not mute"; } -# The settle window leaves a second either side: on a loaded machine (CI, the -# nightly publish) the step to 33 C once landed after a 0.6 s baseline was -# taken, so the watchdog saw 25 -> 41 and the test failed for timing alone. -C9_START=0x00,0x08,0x40,0x00 c9settle c9-rise 1.0 2.0:0x00,0x0a,0x40,0x00 # starts at the measured 33 C (so the baseline never depends on when the watchdog's output shows), then 33 -> 41 C +C9_START=0x00,0x08,0x40,0x00 c9settle c9-rise 1.0 41:0x00,0x0a,0x40,0x00 # the baseline the measured 33 C (1.0 s, round 21), then 33 -> 41 C at 2.0 s ((rc == 1)) && muted && grep -c 'jumped 8.0 C (33.0 -> 41.0)' "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 a rise of 8 C above the measured value mutes"; } -c9settle c9-cold 0.4 1.5:0x00,0x07,0xc0,0x00 # no step: the baseline at 0.4 s, then 25 -> 31 C +c9settle c9-cold 0.4 31:0x00,0x07,0xc0,0x00 # no step: the baseline 25 C at 0.4 s (round 9), then 25 -> 31 C at 1.5 s ((rc == 1)) && muted && grep -c 'jumped 6.0 C' "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 a 6 C rise after the settle window mutes"; } pass "watchdog: in stage c9 the jump rule starts from the measured coil (C9d-8)" # auto (the default stage): c9 at the everyday loud levels when the phone has @@ -574,8 +582,8 @@ grep -c 'earpiece mode: Top through ASP or DSP at digital <= 737, analog <= 6' " media; run e-w-run; earset; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status" dspset Bottom CSPL_STATE 1; dspset Top CAL_SET_STATUS 1 echo '4,100,200,300' > "$r/fw/cirrus/cs35l41-cheetah-top-media-delta.txt"; echo '4,100,200,301' > "$r/fw/cirrus/cs35l41-cheetah-bottom-media-delta.txt" -( sleep 0.4; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null ) & -wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_TEMP_JUMP=5 OMAMOBILE_SPK_MEDIA_DELTA=1 FAKE_DELTA_LOG=$r/delta.log); ewatch; wait +at 8 'amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null' +wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_TEMP_JUMP=5 OMAMOBILE_SPK_MEDIA_DELTA=1 FAKE_DELTA_LOG=$r/delta.log); ewatch ((rc == 0)) && ! muted && [[ ! -e $r/delta.log ]] || { cat "$r/watch.out" >&2; fail "earpiece watchdog: no DSP rule while neither DSP is audible (heartbeat, CSPL, calibration, jump, no media delta)"; } ewfail() { # name, setup...: the watchdog mutes, and says $want local n=$1; shift diff --git a/packages/omamobile-audio/test/fake-amixer b/packages/omamobile-audio/test/fake-amixer index f56293be4..9268f4dbc 100755 --- a/packages/omamobile-audio/test/fake-amixer +++ b/packages/omamobile-audio/test/fake-amixer @@ -8,6 +8,7 @@ # FAKE_DIGITAL_MAX Digital PCM Volume max (817 with the kernel limit) # FAKE_ANALOG_MAX Analog PCM Volume max (10 with the kernel limit) # FAKE_NO_DSP=1 the firmware does not load (no DSP controls) +import fcntl import json import os import sys @@ -117,6 +118,10 @@ def main(argv): print("amixer: Invalid card number ''.", file=sys.stderr) return 1 args = args[2:] + # The watchdog's JSON backend changes this file too (the heartbeat): + # under the same lock, so neither puts back a value the other just wrote + lock = open(f'{state_path}.lock', 'a') + fcntl.flock(lock, fcntl.LOCK_EX) c = load() cmd = args[0] if cmd == 'controls': diff --git a/packages/omamobile-audio/version b/packages/omamobile-audio/version index 659914ae9..416bfb0a2 100644 --- a/packages/omamobile-audio/version +++ b/packages/omamobile-audio/version @@ -1 +1 @@ -0.5.8 +0.5.9 From c8bddaa8d42c329ce539ced9bfa092398feff8fb Mon Sep 17 00:00:00 2001 From: Derek Ross Date: Mon, 2 Nov 2026 02:04:04 -0500 Subject: [PATCH 2/2] omamobile-audio 0.5.9: the watch test hook only with the JSON backend Review of #327: OMAMOBILE_WATCH_HOOK was honoured whenever it was set, so a stray line in /etc/omamobile/speaker.conf could have changed the real watchdog's clock or run a command every round as root. watch_hook() now honours it only with the tests' JSON control backend (OMAMOBILE_CTL_BACKEND=json:...), and otherwise ignores it and logs that. A unit-level test checks the selection with no backend, alsa and json, and that the probe command never ran (the real backend needs a card, so the watchdog itself can't run there). The kmsg and heartbeat-freeze cases now check their mute reason, as the others do, so an unrelated earlier mute can't pass them. The stuck heartbeat message prints tenths ("for 0.3 s", not "for 0 s"). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01BeEcdVeSfyLCbijF8XQF1d --- packages/omamobile-audio/lib/speaker_dsp.py | 26 +++++++++++++++------ packages/omamobile-audio/test/all | 21 ++++++++++++++--- 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/packages/omamobile-audio/lib/speaker_dsp.py b/packages/omamobile-audio/lib/speaker_dsp.py index a395a0102..26baffc5a 100644 --- a/packages/omamobile-audio/lib/speaker_dsp.py +++ b/packages/omamobile-audio/lib/speaker_dsp.py @@ -1219,6 +1219,23 @@ def cmd_watch(args): feedback.stop() +def watch_hook(): + """Tests only (test/all): OMAMOBILE_WATCH_HOOK is a command run with the + round number before each round, and the clock is then the round count + times the interval, so a test's event lands at a known round and the + grace, heartbeat and settle times count rounds, however slow the + machine. Honoured only with the tests' JSON control backend, so a stray + line in speaker.conf can never change the real watchdog's clock or run + a command as root: there it is ignored, and said so.""" + hook = os.environ.get('OMAMOBILE_WATCH_HOOK') + if not hook: + return None + if not os.environ.get('OMAMOBILE_CTL_BACKEND', '').startswith('json:'): + log('watchdog: OMAMOBILE_WATCH_HOOK ignored: tests only (the JSON control backend)') + return None + return hook + + def watch_loop(args, amps, devs, kmsg, pcm, digital_ceil, analog_ceil, temp_max, temp_jump, strict_hb, media_delta, feedback): since = None @@ -1233,12 +1250,7 @@ def watch_loop(args, amps, devs, kmsg, pcm, digital_ceil, analog_ceil, temp_max, failures = 0 n = 0 acked = None - # Tests only (test/all): OMAMOBILE_WATCH_HOOK is a command run with the - # round number before each round, and the clock is then the round count - # times the interval, so a test's event lands at a known round and the - # grace, heartbeat and settle times count rounds, however slow the - # machine. Unset on the phone: the real clock, nothing run. - hook = os.environ.get('OMAMOBILE_WATCH_HOOK') + hook = watch_hook() while args.iterations == 0 or n < args.iterations: n += 1 if hook: @@ -1350,7 +1362,7 @@ def watch_loop(args, amps, devs, kmsg, pcm, digital_ceil, analog_ceil, temp_max, if last is None or hb != last[0]: beat[amp] = (hb, now) elif strict and now - last[1] >= args.heartbeat_timeout: - msg = f'{amp}: HALO_HEARTBEAT stuck at {hb} for {now - last[1]:.0f} s' + msg = f'{amp}: HALO_HEARTBEAT stuck at {hb} for {now - last[1]:.1f} s' if strict_hb: problems.append(msg) elif not warned: diff --git a/packages/omamobile-audio/test/all b/packages/omamobile-audio/test/all index 1ecc7c021..166ebba0a 100755 --- a/packages/omamobile-audio/test/all +++ b/packages/omamobile-audio/test/all @@ -345,7 +345,7 @@ watchrun() { # run the watchdog briefly on a healthy started tree, with the eve wprep=() # An event during a watch: `at ROUND COMMAND` runs COMMAND (amx, kmsg, any # shell) just before the watchdog's round ROUND, inside the watchdog's own -# loop (OMAMOBILE_WATCH_HOOK), and the watchdog's clock is then ROUND times +# loop (OMAMOBILE_WATCH_HOOK, honoured with the JSON backend sp() sets), and the watchdog's clock is then ROUND times # the interval. So "after the stream's baseline" or "before the settle time" # is a round number, not a sleep that a slow CI runner outran: the stream # starts in round 1 (time 0), round k is (k-1) x 0.05 s into it. @@ -384,7 +384,7 @@ startok w-hb8-log; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; startok w-hb-freeze; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status" at 10 'touch "$r/freeze"' # the heartbeat stops 0.45 s into the stream, after the grace wprep=(FAKE_HB_FREEZE_FILE=$r/freeze); WATCH_N=40 watchrun -((rc == 1)) && muted || { cat "$r/watch.out" >&2; fail "watchdog: a heartbeat that stops mid-stream mutes"; } +((rc == 1)) && muted && grep -c "HALO_HEARTBEAT stuck" "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: a heartbeat that stops mid-stream mutes"; } startok w-hb9-off; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_HEARTBEAT=0); watchrun ((rc == 1)) && muted || fail "watchdog: the heartbeat cannot be switched off in stage c9" startok w-hb9; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_STAGE=c9); watchrun @@ -393,13 +393,28 @@ startok w-hb9; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wpre startok w-kmsg at 8 "kmsg 1.0 'cs35l41 spi6.1: Wake failed, re-enter hibernate: -42'" WATCH_N=40 watchrun -((rc == 1)) && muted || { cat "$r/watch.out" >&2; fail "watchdog: a failed wake in the kernel log mutes"; } +((rc == 1)) && muted && grep -c "Wake failed" "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: a failed wake in the kernel log mutes"; } startok w-kmsg-warn at 8 "kmsg 1.0 'cs35l41 spi6.0: Failed to set mailbox cmd 1 (status 0)'" WATCH_N=20 watchrun ((rc == 0)) && ! muted && grep -c 'watchdog: note: .*mailbox cmd 1' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: an unacknowledged pause is logged only"; } pass "watchdog: mutes on DSP errors, bypass, levels, runtime PM, a stuck heartbeat (c9) and kernel log failures" +# the test hook is honoured only with the JSON control backend: on the phone +# (alsa-lib, no OMAMOBILE_CTL_BACKEND) a stray OMAMOBILE_WATCH_HOOK in +# speaker.conf neither runs nor changes the clock. Unit level: the real +# backend needs a card, so watch_loop's own selection is called directly. +printf '#!/bin/sh\ntouch "%s"\n' "$tmp/hook-ran" > "$tmp/hook-probe"; chmod +x "$tmp/hook-probe" +out=$(env -u OMAMOBILE_CTL_BACKEND OMAMOBILE_WATCH_HOOK="$tmp/hook-probe" python3 -I -c ' +import sys; sys.path.insert(0, sys.argv[1]); import speaker_dsp as d +print("real:", d.watch_hook()) +import os; os.environ["OMAMOBILE_CTL_BACKEND"] = "alsa"; print("alsa:", d.watch_hook()) +os.environ["OMAMOBILE_CTL_BACKEND"] = "json:/x"; print("json:", d.watch_hook() == os.environ["OMAMOBILE_WATCH_HOOK"]) +' "$here/lib") +[[ $out == *"real: None"*"alsa: None"*"json: True"* && $(grep -c 'OMAMOBILE_WATCH_HOOK ignored' <<<"$out") == 2 && ! -e $tmp/hook-ran ]] || + { echo "$out" >&2; fail "watchdog: the test hook is ignored, and said so, without the JSON backend"; } +pass "watchdog: the test hook only with the tests' JSON control backend" + # C9j: the coil-estimate jump rule is for analog above 10. At the everyday level the # firmware's estimate is open-loop and the bottom amplifier's jumps 10-15 C on any # level drop, so a jump there must not mute; OMAMOBILE_SPK_TEMP_JUMP brings it back.