Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 42 additions & 2 deletions packages/omamobile-audio/lib/speaker_dsp.py
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,16 @@ class JsonCtl:
def __init__(self, path):
self.path = path

def _locked(self):
# The heartbeat read and write_bytes load, change and save the whole
# file, and test/fake-amixer does the same under the same lock.
# Without it a save here put back a value a test had just written (a
# coil temperature step lost, so the watchdog saw no jump).
import fcntl
f = open(f'{self.path}.lock', 'a')
fcntl.flock(f, fcntl.LOCK_EX)
return f

def _load(self):
try:
with open(self.path) as f:
Expand All @@ -237,6 +247,10 @@ def names(self):
return list(self._load())

def read(self, name):
with self._locked():
return self._read(name)

def _read(self, name):
state = self._load()
c = state.get(name)
if c is None:
Expand All @@ -262,6 +276,10 @@ def read_bytes(self, name):
return bytes(c['v']) if c and c['type'] == 'BYTES' else None

def write_bytes(self, name, data):
with self._locked():
return self._write_bytes(name, data)

def _write_bytes(self, name, data):
state = self._load()
c = state.get(name)
if not c or c['type'] != 'BYTES' or len(c['v']) != len(data):
Expand Down Expand Up @@ -1201,6 +1219,23 @@ def cmd_watch(args):
feedback.stop()


def watch_hook():
"""Tests only (test/all): OMAMOBILE_WATCH_HOOK is a command run with the
round number before each round, and the clock is then the round count
times the interval, so a test's event lands at a known round and the
grace, heartbeat and settle times count rounds, however slow the
machine. Honoured only with the tests' JSON control backend, so a stray
line in speaker.conf can never change the real watchdog's clock or run
a command as root: there it is ignored, and said so."""
hook = os.environ.get('OMAMOBILE_WATCH_HOOK')
if not hook:
return None
if not os.environ.get('OMAMOBILE_CTL_BACKEND', '').startswith('json:'):
log('watchdog: OMAMOBILE_WATCH_HOOK ignored: tests only (the JSON control backend)')
return None
return hook


def watch_loop(args, amps, devs, kmsg, pcm, digital_ceil, analog_ceil, temp_max, temp_jump,
strict_hb, media_delta, feedback):
since = None
Expand All @@ -1215,9 +1250,14 @@ def watch_loop(args, amps, devs, kmsg, pcm, digital_ceil, analog_ceil, temp_max,
failures = 0
n = 0
acked = None
hook = watch_hook()
while args.iterations == 0 or n < args.iterations:
n += 1
now = time.monotonic()
if hook:
subprocess.run([hook, str(n)], check=True)
now = n * args.interval
else:
now = time.monotonic()
# The mode (C9k): taken at the start of a round and answered in the
# ack file, which omamobile-speaker-protection waits for before it
# changes anything the new mode allows and the old one does not.
Expand Down Expand Up @@ -1322,7 +1362,7 @@ def watch_loop(args, amps, devs, kmsg, pcm, digital_ceil, analog_ceil, temp_max,
if last is None or hb != last[0]:
beat[amp] = (hb, now)
elif strict and now - last[1] >= args.heartbeat_timeout:
msg = f'{amp}: HALO_HEARTBEAT stuck at {hb} for {now - last[1]:.0f} s'
msg = f'{amp}: HALO_HEARTBEAT stuck at {hb} for {now - last[1]:.1f} s'
if strict_hb:
problems.append(msg)
elif not warned:
Expand Down
99 changes: 61 additions & 38 deletions packages/omamobile-audio/test/all
Original file line number Diff line number Diff line change
Expand Up @@ -335,13 +335,31 @@ rc=0; prep=(OMAMOBILE_ASOUND=$r/none); sp start > "$r/out" 2>&1 || rc=$?; prep=(
pass "a hang marker from the last boot keeps the amplifiers off"

# --- the watchdog ----------------------------------------------------------------
watchrun() { # name: run the watchdog briefly on a healthy started tree; $1 hook runs before
watchrun() { # run the watchdog briefly on a healthy started tree, with the events of at()
prep=(OMAMOBILE_WATCH_INTERVAL=0.05 OMAMOBILE_WATCH_GRACE=0.1 OMAMOBILE_WATCH_HB_TIMEOUT=0.3
OMAMOBILE_WATCH_ITERATIONS=${WATCH_N:-20} "${wprep[@]}")
[[ ! -e $r/events ]] || prep+=(OMAMOBILE_WATCH_HOOK="$tmp/watch-hook" WATCH_EVENTS="$r/events" WATCH_R="$r")
rc=0; sp watch > "$r/watch.out" 2>&1 || rc=$?
prep=(); wprep=()
}
wprep=()
# An event during a watch: `at ROUND COMMAND` runs COMMAND (amx, kmsg, any
# shell) just before the watchdog's round ROUND, inside the watchdog's own
# loop (OMAMOBILE_WATCH_HOOK, honoured with the JSON backend sp() sets), and the watchdog's clock is then ROUND times
# the interval. So "after the stream's baseline" or "before the settle time"
# is a round number, not a sleep that a slow CI runner outran: the stream
# starts in round 1 (time 0), round k is (k-1) x 0.05 s into it.
at() { printf '%s %s\n' "$1" "$2" >> "$r/events"; }
cat > "$tmp/watch-hook" <<'EOF'
#!/bin/bash
set -euo pipefail
r=$WATCH_R
while read -r k cmd; do
[[ $k != "$1" ]] || eval "$cmd"
done < "$WATCH_EVENTS"
EOF
chmod +x "$tmp/watch-hook"
export -f amx kmsg
startok() { run "$1"; ((rc == 0)) || { cat "$r/out" >&2; fail "$1: start"; }; }
startok w-idle; watchrun
((rc == 0)) && ! muted || { cat "$r/watch.out" >&2; fail "watchdog: idle and healthy, nothing happens"; }
Expand All @@ -364,35 +382,47 @@ startok w-hb8-log; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status";
((rc == 0)) && ! muted && grep -c 'stuck' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: OMAMOBILE_SPK_HEARTBEAT=0 logs a stuck heartbeat in c8"; }
# the C9a case: the heartbeat moves, then stops while the stream still runs
startok w-hb-freeze; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"
( sleep 0.5; touch "$r/freeze" ) &
wprep=(FAKE_HB_FREEZE_FILE=$r/freeze); WATCH_N=40 watchrun; wait
((rc == 1)) && muted || { cat "$r/watch.out" >&2; fail "watchdog: a heartbeat that stops mid-stream mutes"; }
at 10 'touch "$r/freeze"' # the heartbeat stops 0.45 s into the stream, after the grace
wprep=(FAKE_HB_FREEZE_FILE=$r/freeze); WATCH_N=40 watchrun
((rc == 1)) && muted && grep -c "HALO_HEARTBEAT stuck" "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: a heartbeat that stops mid-stream mutes"; }
startok w-hb9-off; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_HEARTBEAT=0); watchrun
((rc == 1)) && muted || fail "watchdog: the heartbeat cannot be switched off in stage c9"
startok w-hb9; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_STAGE=c9); watchrun
((rc == 1)) && muted || fail "watchdog: a stuck heartbeat mutes in stage c9"
# a kernel log error after the watchdog started
startok w-kmsg
( sleep 0.4; kmsg 1.0 'cs35l41 spi6.1: Wake failed, re-enter hibernate: -42' ) &
WATCH_N=40 watchrun; wait
((rc == 1)) && muted || { cat "$r/watch.out" >&2; fail "watchdog: a failed wake in the kernel log mutes"; }
at 8 "kmsg 1.0 'cs35l41 spi6.1: Wake failed, re-enter hibernate: -42'"
WATCH_N=40 watchrun
((rc == 1)) && muted && grep -c "Wake failed" "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: a failed wake in the kernel log mutes"; }
startok w-kmsg-warn
( sleep 0.4; kmsg 1.0 'cs35l41 spi6.0: Failed to set mailbox cmd 1 (status 0)' ) &
WATCH_N=20 watchrun; wait
((rc == 0)) && ! muted || fail "watchdog: an unacknowledged pause is logged only"
at 8 "kmsg 1.0 'cs35l41 spi6.0: Failed to set mailbox cmd 1 (status 0)'"
WATCH_N=20 watchrun
((rc == 0)) && ! muted && grep -c 'watchdog: note: .*mailbox cmd 1' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: an unacknowledged pause is logged only"; }
pass "watchdog: mutes on DSP errors, bypass, levels, runtime PM, a stuck heartbeat (c9) and kernel log failures"

# the test hook is honoured only with the JSON control backend: on the phone
# (alsa-lib, no OMAMOBILE_CTL_BACKEND) a stray OMAMOBILE_WATCH_HOOK in
# speaker.conf neither runs nor changes the clock. Unit level: the real
# backend needs a card, so watch_loop's own selection is called directly.
printf '#!/bin/sh\ntouch "%s"\n' "$tmp/hook-ran" > "$tmp/hook-probe"; chmod +x "$tmp/hook-probe"
out=$(env -u OMAMOBILE_CTL_BACKEND OMAMOBILE_WATCH_HOOK="$tmp/hook-probe" python3 -I -c '
import sys; sys.path.insert(0, sys.argv[1]); import speaker_dsp as d
print("real:", d.watch_hook())
import os; os.environ["OMAMOBILE_CTL_BACKEND"] = "alsa"; print("alsa:", d.watch_hook())
os.environ["OMAMOBILE_CTL_BACKEND"] = "json:/x"; print("json:", d.watch_hook() == os.environ["OMAMOBILE_WATCH_HOOK"])
' "$here/lib")
[[ $out == *"real: None"*"alsa: None"*"json: True"* && $(grep -c 'OMAMOBILE_WATCH_HOOK ignored' <<<"$out") == 2 && ! -e $tmp/hook-ran ]] ||
{ echo "$out" >&2; fail "watchdog: the test hook is ignored, and said so, without the JSON backend"; }
pass "watchdog: the test hook only with the tests' JSON control backend"

# C9j: the coil-estimate jump rule is for analog above 10. At the everyday level the
# firmware's estimate is open-loop and the bottom amplifier's jumps 10-15 C on any
# level drop, so a jump there must not mute; OMAMOBILE_SPK_TEMP_JUMP brings it back.
jump() { # name [wprep...]: watchdog on a running stream; the bottom's estimate goes 25 -> 40 C a moment in
local n=$1; shift; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; wprep=("$@")
# the jump comes once the watchdog has its first reading (its start line,
# then a few intervals), not at a fixed time: a slow runner started it late
rm -f "$r/watch.out"
( for _ in $(seq 1 100); do grep -c 'watchdog: watching' "$r/watch.out" >/dev/null 2>&1 && break; sleep 0.05; done
sleep 0.3; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null ) &
WATCH_N=60 watchrun; wait
# in round 8 (0.35 s in): after the baseline of a 0.01 s settle (round 2)
at 8 'amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null'
WATCH_N=30 watchrun
}
startok w-jump8; jump w-jump8
((rc == 0)) && ! muted && grep -c 'jump rule off' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: at analog 10 or below a 15 C jump of the estimate does not mute (C9j)"; }
Expand Down Expand Up @@ -423,11 +453,12 @@ pass "watchdog: stock media delta at stream start, both amplifiers, or mute"

# the amplifiers' V/I feedback stream (C9d-7): with each speaker stream when asked
fbstate() { cat "$r/asound/card0/pcm15c/sub0/status"; }
export -f fbstate
startok w-fb-off; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"; watchrun
((rc == 0)) && [[ $(fbstate) == closed ]] && ! grep -c NoHost1 "$r/cset.log" >/dev/null || { cat "$r/watch.out" >&2; fail "feedback: off by default"; }
startok w-fb; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"
( sleep 0.6; [[ $(fbstate) == 'state: RUNNING' ]] && touch "$r/fb-seen"; echo closed > "$r/asound/card0/pcm5p/sub0/status" ) &
wprep=(OMAMOBILE_SPK_FEEDBACK=1); WATCH_N=30 watchrun; wait
at 12 'if [[ $(fbstate) == "state: RUNNING" ]]; then touch "$r/fb-seen"; fi; echo closed > "$r/asound/card0/pcm5p/sub0/status"'
wprep=(OMAMOBILE_SPK_FEEDBACK=1); WATCH_N=30 watchrun
((rc == 0)) && ! muted && [[ -e $r/fb-seen && $(fbstate) == closed ]] || { cat "$r/watch.out" >&2; fail "feedback: runs during the stream, stops after it"; }
[[ $(grep NoHost1 "$r/cset.log") == $'NoHost1 TX Mixer TDM_0_TX=1\nNoHost1 TX Mixer TDM_0_TX=0' ]] || { cat "$r/cset.log" >&2; fail "feedback: the mixer path on, then off, as stock"; }
[[ ! -e $r/run/speaker-feedback.pid ]] || fail "feedback: no pid file left"
Expand Down Expand Up @@ -488,37 +519,29 @@ c9 c9-ok
$(val "Top DRE Switch") == on && $(val "Bottom DRE Switch") == on &&
$(cat "$r/module/snd_soc_cs35l41/parameters/boost_en") == Y ]] || { cat "$r/out" >&2; fail "stage c9 at stock levels, boost on"; }
echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"
# as jump(): after the watchdog's first reading, not at a fixed time
rm -f "$r/watch.out"
( for _ in $(seq 1 100); do grep -c 'watchdog: watching' "$r/watch.out" >/dev/null 2>&1 && break; sleep 0.05; done
sleep 0.3; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null ) &
wprep=(OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_DIGITAL=817 OMAMOBILE_SPK_ANALOG=17 FAKE_ANALOG_MAX=17 OMAMOBILE_SPK_TEMP_SETTLE=0.01); WATCH_N=60 watchrun; wait
# as jump(): in round 8, after the baseline of a 0.01 s settle (round 2)
at 8 'amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null'
wprep=(OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_DIGITAL=817 OMAMOBILE_SPK_ANALOG=17 FAKE_ANALOG_MAX=17 OMAMOBILE_SPK_TEMP_SETTLE=0.01); WATCH_N=30 watchrun
((rc == 0)) && ! muted && grep -c 'jump rule off' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 the jump rule is off by default, so a 15 C rise plays on (the backstop stays)"; }
# C9d-8: a stream's first estimates are the firmware's 25 C reference until
# it has measured the coil, often warmer, a step of about a second: the
# estimate at the settle time is the baseline, rises above it are held to
# the jump.
c9settle() { # name settle steps...: steps are delay_s:temp_bytes, applied in order
c9settle() { # name settle steps...: steps are round:temp_bytes (at()); round k is (k-1) x 0.05 s into the stream
local n=$1 st=$2; shift 2; local steps=("$@")
c9 "$n"; ((rc == 0)) || { cat "$r/out" >&2; fail "$n: start"; }
echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"
amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" "${C9_START:-0x00,0x06,0x40,0x00}" >/dev/null
rm -f "$r/watch.out"
( for _ in $(seq 1 100); do grep -c 'watchdog: watching' "$r/watch.out" >/dev/null 2>&1 && break; sleep 0.05; done
for s in "${steps[@]}"; do sleep "${s%%:*}"; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" "${s#*:}" >/dev/null; done ) &
local s
for s in "${steps[@]}"; do at "${s%%:*}" "amx cset 'name=Bottom DSP1 Protection cd CSPL_TEMPERATURE' ${s#*:} >/dev/null"; done
wprep=(OMAMOBILE_SPK_STAGE=c9 OMAMOBILE_SPK_DIGITAL=817 OMAMOBILE_SPK_ANALOG=17 FAKE_ANALOG_MAX=17 OMAMOBILE_SPK_TEMP_SETTLE=$st OMAMOBILE_SPK_TEMP_JUMP=${JUMP:-5})
# 200 polls (10 s at 0.05 s): on a slow CI runner the steps above arrive
# seconds late, and 80 polls let the watchdog finish before the rise came.
WATCH_N=200 watchrun; wait
WATCH_N=60 watchrun
}
c9settle c9-step 1.5 0.2:0x00,0x0a,0x00,0x00 # 25 -> 40 C: the measured value, a step
c9settle c9-step 1.5 5:0x00,0x0a,0x00,0x00 # 25 -> 40 C at 0.2 s: the measured value, a step before the baseline (1.5 s, round 31)
((rc == 0)) && ! muted && grep -c 's into a stream' "$r/watch.out" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 the step from the reference to the measured coil does not mute"; }
# The settle window leaves a second either side: on a loaded machine (CI, the
# nightly publish) the step to 33 C once landed after a 0.6 s baseline was
# taken, so the watchdog saw 25 -> 41 and the test failed for timing alone.
C9_START=0x00,0x08,0x40,0x00 c9settle c9-rise 1.0 2.0:0x00,0x0a,0x40,0x00 # starts at the measured 33 C (so the baseline never depends on when the watchdog's output shows), then 33 -> 41 C
C9_START=0x00,0x08,0x40,0x00 c9settle c9-rise 1.0 41:0x00,0x0a,0x40,0x00 # the baseline the measured 33 C (1.0 s, round 21), then 33 -> 41 C at 2.0 s
((rc == 1)) && muted && grep -c 'jumped 8.0 C (33.0 -> 41.0)' "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 a rise of 8 C above the measured value mutes"; }
c9settle c9-cold 0.4 1.5:0x00,0x07,0xc0,0x00 # no step: the baseline at 0.4 s, then 25 -> 31 C
c9settle c9-cold 0.4 31:0x00,0x07,0xc0,0x00 # no step: the baseline 25 C at 0.4 s (round 9), then 25 -> 31 C at 1.5 s
((rc == 1)) && muted && grep -c 'jumped 6.0 C' "$r/run/speaker-protection" >/dev/null || { cat "$r/watch.out" >&2; fail "watchdog: in stage c9 a 6 C rise after the settle window mutes"; }
pass "watchdog: in stage c9 the jump rule starts from the measured coil (C9d-8)"
# auto (the default stage): c9 at the everyday loud levels when the phone has
Expand Down Expand Up @@ -574,8 +597,8 @@ grep -c 'earpiece mode: Top through ASP or DSP at digital <= 737, analog <= 6' "
media; run e-w-run; earset; echo 'state: RUNNING' > "$r/asound/card0/pcm5p/sub0/status"
dspset Bottom CSPL_STATE 1; dspset Top CAL_SET_STATUS 1
echo '4,100,200,300' > "$r/fw/cirrus/cs35l41-cheetah-top-media-delta.txt"; echo '4,100,200,301' > "$r/fw/cirrus/cs35l41-cheetah-bottom-media-delta.txt"
( sleep 0.4; amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null ) &
wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_TEMP_JUMP=5 OMAMOBILE_SPK_MEDIA_DELTA=1 FAKE_DELTA_LOG=$r/delta.log); ewatch; wait
at 8 'amx cset "name=Bottom DSP1 Protection cd CSPL_TEMPERATURE" 0x00,0x0a,0x00,0x00 >/dev/null'
wprep=(FAKE_HB_STUCK=1 OMAMOBILE_SPK_TEMP_JUMP=5 OMAMOBILE_SPK_MEDIA_DELTA=1 FAKE_DELTA_LOG=$r/delta.log); ewatch
((rc == 0)) && ! muted && [[ ! -e $r/delta.log ]] || { cat "$r/watch.out" >&2; fail "earpiece watchdog: no DSP rule while neither DSP is audible (heartbeat, CSPL, calibration, jump, no media delta)"; }
ewfail() { # name, setup...: the watchdog mutes, and says $want
local n=$1; shift
Expand Down
5 changes: 5 additions & 0 deletions packages/omamobile-audio/test/fake-amixer
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
# FAKE_DIGITAL_MAX Digital PCM Volume max (817 with the kernel limit)
# FAKE_ANALOG_MAX Analog PCM Volume max (10 with the kernel limit)
# FAKE_NO_DSP=1 the firmware does not load (no DSP controls)
import fcntl
import json
import os
import sys
Expand Down Expand Up @@ -117,6 +118,10 @@ def main(argv):
print("amixer: Invalid card number ''.", file=sys.stderr)
return 1
args = args[2:]
# The watchdog's JSON backend changes this file too (the heartbeat):
# under the same lock, so neither puts back a value the other just wrote
lock = open(f'{state_path}.lock', 'a')
fcntl.flock(lock, fcntl.LOCK_EX)
c = load()
cmd = args[0]
if cmd == 'controls':
Expand Down
2 changes: 1 addition & 1 deletion packages/omamobile-audio/version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
0.5.8
0.5.9