diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index abbd59e..7e3de1c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,8 +1,8 @@ name: Tests -# The forked docker-compose Update Module runs every retina-node install on -# every node. These tests run the real module under /bin/sh (dash on Ubuntu, -# as on the nodes) against a stub docker and jq, with real tar payloads. +# The forked docker-compose Update Module, retina-env-guard and the stack-health +# inventory script run as root on every node. These tests run the real scripts +# under /bin/sh (dash on Ubuntu, as on the nodes) against a stub docker. on: push: @@ -18,7 +18,10 @@ jobs: with: python-version: "3.12" - run: pip install 'pytest>=8.0' - - name: Module is executable - run: test -x plugins/playbooks/board_support/roles/mender/files/update-modules/docker-compose + - name: Scripts are executable + run: | + test -x plugins/playbooks/board_support/roles/mender/files/update-modules/docker-compose + test -x plugins/playbooks/os_setup/roles/radar_data_bootstrap/files/retina-env-guard + test -x configuration/mender/inventory/mender-inventory-retina-stack - name: Tests - run: pytest tests/update-modules/ + run: pytest tests/ diff --git a/configuration/mender/inventory/mender-inventory-retina-stack b/configuration/mender/inventory/mender-inventory-retina-stack new file mode 100755 index 0000000..87c2b71 --- /dev/null +++ b/configuration/mender/inventory/mender-inventory-retina-stack @@ -0,0 +1,63 @@ +#!/bin/sh + +# Reports the retina-node stack's health from outside the stack. +# +# retina-telemetry runs inside the same compose project, so a node whose stack +# is down goes silent instead of reporting unhealthy. ret9573ecda sat with no +# radar for four weeks that way. mender-updated runs this every inventory poll +# (600 s) whether or not a single container is up. +# +# retina_stack up | degraded | down | absent +# retina_stack_running project containers running +# retina_stack_restarting project containers in a restart loop +# retina_blah2 blah2's state (running, restarting, exited, missing) +# retina_env_ok manifests/.env parses (false: NUL bytes) +# retina_compose_ok compose can load the project +# retina_mode radar | spectrum | sdrconnect (retina-gui's mode.txt) +# +# In spectrum and sdrconnect modes blah2 is stopped on purpose, so it does not +# make the stack degraded there. + +MANIFESTS="${RETINA_STACK_MANIFESTS:-/data/mender-docker-compose/current/manifests}" +MODE_FILE="${RETINA_STACK_MODE_FILE:-/data/retina-gui/mode.txt}" + +if [ ! -f "$MANIFESTS/docker-compose.yaml" ]; then + echo retina_stack=absent + exit 0 +fi + +mode=radar +[ -s "$MODE_FILE" ] && mode=$(cat "$MODE_FILE") + +env_ok=true +if [ -f "$MANIFESTS/.env" ] \ + && [ "$(tr -d '\000' < "$MANIFESTS/.env" | wc -c)" -ne "$(wc -c < "$MANIFESTS/.env")" ]; then + env_ok=false +fi + +compose_ok=true +(cd "$MANIFESTS" && timeout 20 docker compose -p retina-node config -q > /dev/null 2>&1) || compose_ok=false + +count() { + timeout 10 docker ps -q --filter label=com.docker.compose.project=retina-node --filter "status=$1" 2>/dev/null | wc -l +} +running=$(count running) +restarting=$(count restarting) +blah2=$(timeout 10 docker inspect -f '{{.State.Status}}' blah2 2>/dev/null) || blah2=missing + +state=up +if [ "$running" -eq 0 ]; then + state=down +elif [ "$restarting" -gt 0 ] || [ "$env_ok" = false ] || [ "$compose_ok" = false ] \ + || { [ "$mode" = radar ] && [ "$blah2" != running ]; }; then + state=degraded +fi + +echo "retina_stack=$state" +echo "retina_stack_running=$running" +echo "retina_stack_restarting=$restarting" +echo "retina_blah2=$blah2" +echo "retina_env_ok=$env_ok" +echo "retina_compose_ok=$compose_ok" +echo "retina_mode=$mode" +exit 0 diff --git a/plugins/playbooks/os_setup/roles/radar_data_bootstrap/files/retina-env-guard b/plugins/playbooks/os_setup/roles/radar_data_bootstrap/files/retina-env-guard new file mode 100755 index 0000000..3589c54 --- /dev/null +++ b/plugins/playbooks/os_setup/roles/radar_data_bootstrap/files/retina-env-guard @@ -0,0 +1,40 @@ +#!/bin/sh +# Set aside a compose .env that compose cannot parse, before retina-node.service +# starts the stack. +# +# The config-merger writes manifests/.env seconds after every boot, and a power +# cut during that write has left it entirely NUL (ret9573ecda, 2026-08-27). +# Compose then refuses to load the project at all, including the +# `run config-merger` that would regenerate the file, so the node stays down +# through every reboot until someone removes it by hand. Moving it aside lets +# compose fall back to defaults for one start, and the config-merger writes a +# good one from user.yml. The retina-node artifact's preflight does the same +# during an install; this covers the boots in between. +# +# Best effort: nothing is deleted, and it always exits 0. + +MANIFESTS="${RETINA_ENV_GUARD_MANIFESTS:-/data/mender-docker-compose/current/manifests}" +env_file="$MANIFESTS/.env" + +[ -f "$env_file" ] || exit 0 + +reason="" +if [ "$(tr -d '\000' < "$env_file" | wc -c)" -ne "$(wc -c < "$env_file")" ]; then + reason="contains NUL bytes" +elif compose_err=$(cd "$MANIFESTS" && docker compose -p retina-node config -q 2>&1); then + : +else + case "$compose_err" in + *"$env_file"*|*"/.env:"*) reason="compose cannot parse it" ;; + esac +fi +[ -n "$reason" ] || exit 0 + +aside="$env_file.corrupt-$(date -u +%Y%m%dT%H%M%SZ)" +if mv "$env_file" "$aside"; then + sync + echo "retina-env-guard: set aside $env_file ($reason) as $aside; the config-merger regenerates it" +else + echo "retina-env-guard: could not set aside $env_file ($reason)" +fi +exit 0 diff --git a/plugins/playbooks/os_setup/roles/radar_data_bootstrap/tasks/main.yml b/plugins/playbooks/os_setup/roles/radar_data_bootstrap/tasks/main.yml index 48b9608..a528c57 100644 --- a/plugins/playbooks/os_setup/roles/radar_data_bootstrap/tasks/main.yml +++ b/plugins/playbooks/os_setup/roles/radar_data_bootstrap/tasks/main.yml @@ -93,6 +93,17 @@ ADSBLOL_ENABLED=false ADSBLOL_RADIUS=40 +# 3a. Boot guard: set aside a compose .env that compose cannot parse, so a +# power cut during the config-merger's write cannot keep the stack down +# through every reboot. See the script's header. +- name: Install retina-env-guard + copy: + src: retina-env-guard + dest: /usr/local/sbin/retina-env-guard + mode: '0755' + owner: root + group: root + # 3. Create systemd service for retina-node auto-start after OS updates - name: Create retina-node auto-start service copy: @@ -113,6 +124,8 @@ RemainAfterExit=yes WorkingDirectory={{ retina_node_manifests_path }} Environment="RETINA_NODE_PATH={{ retina_node_manifests_path }}" + # "-": the guard is best effort and must never stop the stack starting. + ExecStartPre=-/usr/local/sbin/retina-env-guard ExecStart=/bin/bash -c '/usr/bin/docker compose -p retina-node run --rm config-merger && /usr/bin/docker compose -p retina-node up -d --remove-orphans' StartLimitBurst=3 StartLimitIntervalSec=300 diff --git a/plugins/playbooks/os_setup/roles/radar_packages/tasks/main.yml b/plugins/playbooks/os_setup/roles/radar_packages/tasks/main.yml index 9340f91..98a8b9a 100644 --- a/plugins/playbooks/os_setup/roles/radar_packages/tasks/main.yml +++ b/plugins/playbooks/os_setup/roles/radar_packages/tasks/main.yml @@ -592,6 +592,9 @@ # one instance blocked while cron keeps starting more every 5 minutes. FIRST_CHAR=$(curl -s --max-time 10 127.0.0.1:3000/api/map | head -c1) TIMESTAMP=$(curl -s --max-time 10 127.0.0.1:3000/api/map | head -c23 | tail -c10) + # No answer, or not a number, reads as stale. Left empty, the subtraction + # below failed with "operand expected" whenever blah2-api was down. + case "$TIMESTAMP" in ''|*[!0-9]*) TIMESTAMP=0 ;; esac CURR_TIMESTAMP=$(date +%s) DIFF_TIMESTAMP=$(($CURR_TIMESTAMP-$TIMESTAMP)) diff --git a/tests/stack-health/test_stack_health.py b/tests/stack-health/test_stack_health.py new file mode 100644 index 0000000..e99b1cd --- /dev/null +++ b/tests/stack-health/test_stack_health.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +""" +Tests for retina-env-guard (retina-node.service ExecStartPre) and the +mender-inventory-retina-stack inventory script. + +Both run as root on every node: the guard at every boot, the inventory script +every 600 s. Each test runs the real script under /bin/sh with a stub docker. +""" + +import os +import shutil +import subprocess +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +GUARD = os.path.join(REPO, 'plugins', 'playbooks', 'os_setup', 'roles', 'radar_data_bootstrap', + 'files', 'retina-env-guard') +INVENTORY = os.path.join(REPO, 'configuration', 'mender', 'inventory', 'mender-inventory-retina-stack') + +STUB_DOCKER = r'''#!/bin/sh +echo "$*" >> "$STUB/calls" +case "$1" in + compose) + if [ -f "$STUB/compose_error" ]; then cat "$STUB/compose_error" >&2; exit 1; fi + exit 0 ;; + ps) + for a; do case "$a" in status=*) s=${a#status=} ;; esac; done + n=$(cat "$STUB/$s" 2>/dev/null || echo 0) + i=0; while [ "$i" -lt "$n" ]; do echo "c$i"; i=$((i+1)); done + exit 0 ;; + inspect) + [ -f "$STUB/blah2" ] || exit 1 + cat "$STUB/blah2"; exit 0 ;; +esac +exit 0 +''' + + +class Harness(unittest.TestCase): + + def setUp(self): + self.dir = tempfile.mkdtemp() + self.stub = os.path.join(self.dir, 'stub') + self.bin = os.path.join(self.dir, 'bin') + self.manifests = os.path.join(self.dir, 'current', 'manifests') + for d in (self.stub, self.bin, self.manifests): + os.makedirs(d) + docker = os.path.join(self.bin, 'docker') + self.write(docker, STUB_DOCKER) + os.chmod(docker, 0o755) + self.write(os.path.join(self.manifests, 'docker-compose.yaml'), 'services: {}\n') + self.mode_file = os.path.join(self.dir, 'mode.txt') + + def tearDown(self): + shutil.rmtree(self.dir) + + def write(self, path, text, mode='w'): + with open(path, mode) as f: + f.write(text) + + def env(self): + return dict(os.environ, PATH=f"{self.bin}:{os.environ['PATH']}", STUB=self.stub, + RETINA_ENV_GUARD_MANIFESTS=self.manifests, + RETINA_STACK_MANIFESTS=self.manifests, + RETINA_STACK_MODE_FILE=self.mode_file) + + def run_script(self, script): + result = subprocess.run(['/bin/sh', script], env=self.env(), capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + return result.stdout + + def set_env(self, content): + self.write(os.path.join(self.manifests, '.env'), content, + mode='wb' if isinstance(content, bytes) else 'w') + + def set_stub(self, name, value): + self.write(os.path.join(self.stub, name), str(value)) + + +class TestEnvGuard(Harness): + + def test_a_nul_filled_env_is_set_aside(self): + # ret9573ecda: 294 NUL bytes since 2026-08-27, stack down on every boot. + self.set_env(b'\x00' * 294) + + out = self.run_script(GUARD) + + names = os.listdir(self.manifests) + self.assertNotIn('.env', names) + self.assertEqual(len([n for n in names if n.startswith('.env.corrupt-')]), 1) + self.assertIn('NUL bytes', out) + + def test_an_env_compose_cannot_parse_is_set_aside(self): + env = os.path.join(self.manifests, '.env') + self.set_env('BAD LINE\n') + self.write(os.path.join(self.stub, 'compose_error'), f'failed to read {env}: line 1: unexpected character\n') + + self.assertIn('compose cannot parse it', self.run_script(GUARD)) + self.assertFalse(os.path.exists(env)) + + def test_a_good_env_is_left_alone(self): + self.set_env('RECEIVER_LAT=51.5\n') + self.assertEqual(self.run_script(GUARD), '') + self.assertTrue(os.path.exists(os.path.join(self.manifests, '.env'))) + + def test_an_unrelated_compose_error_leaves_env_alone(self): + self.set_env('RECEIVER_LAT=51.5\n') + self.write(os.path.join(self.stub, 'compose_error'), 'service "x" has neither an image nor a build\n') + self.run_script(GUARD) + self.assertTrue(os.path.exists(os.path.join(self.manifests, '.env'))) + + def test_no_env_and_no_composition_are_fine(self): + self.run_script(GUARD) + shutil.rmtree(os.path.join(self.dir, 'current')) + self.run_script(GUARD) + + +class TestInventory(Harness): + + def attrs(self): + out = self.run_script(INVENTORY) + return dict(line.split('=', 1) for line in out.split()) + + def healthy(self): + self.set_env('A=1\n') + self.set_stub('running', 8) + self.set_stub('restarting', 0) + self.set_stub('blah2', 'running') + + def test_a_healthy_stack_is_up(self): + self.healthy() + self.assertEqual(self.attrs(), { + 'retina_stack': 'up', 'retina_stack_running': '8', 'retina_stack_restarting': '0', + 'retina_blah2': 'running', 'retina_env_ok': 'true', 'retina_compose_ok': 'true', + 'retina_mode': 'radar'}) + + def test_nothing_running_is_down(self): + self.healthy() + self.set_stub('running', 0) + self.assertEqual(self.attrs()['retina_stack'], 'down') + + def test_the_ret9573ecda_state_is_reported_degraded(self): + # Old containers kept alive by Docker's restart policy, blah2 aborting + # on a zeroed config.yml, and a NUL .env that compose cannot load. + self.set_env(b'\x00' * 294) + self.write(os.path.join(self.stub, 'compose_error'), 'failed to read .env\n') + self.set_stub('running', 4) + self.set_stub('restarting', 3) + self.set_stub('blah2', 'restarting') + + attrs = self.attrs() + + self.assertEqual(attrs['retina_stack'], 'degraded') + self.assertEqual(attrs['retina_env_ok'], 'false') + self.assertEqual(attrs['retina_compose_ok'], 'false') + self.assertEqual(attrs['retina_blah2'], 'restarting') + + def test_blah2_stopped_in_spectrum_mode_is_not_degraded(self): + self.healthy() + self.set_stub('blah2', 'exited') + self.write(self.mode_file, 'spectrum') + attrs = self.attrs() + self.assertEqual(attrs['retina_stack'], 'up') + self.assertEqual(attrs['retina_mode'], 'spectrum') + + def test_blah2_stopped_in_radar_mode_is_degraded(self): + self.healthy() + self.set_stub('blah2', 'exited') + self.assertEqual(self.attrs()['retina_stack'], 'degraded') + + def test_a_missing_blah2_is_reported(self): + self.healthy() + os.remove(os.path.join(self.stub, 'blah2')) + self.assertEqual(self.attrs()['retina_blah2'], 'missing') + + def test_no_composition_is_absent(self): + shutil.rmtree(os.path.join(self.dir, 'current')) + self.assertEqual(self.attrs(), {'retina_stack': 'absent'}) + + def test_every_line_is_key_equals_value(self): + # mender-updated rejects the whole script's output on a malformed line. + self.healthy() + for line in self.run_script(INVENTORY).splitlines(): + key, _, value = line.partition('=') + self.assertRegex(key, r'^[a-z0-9_]+$') + self.assertTrue(value) + + +if __name__ == '__main__': + unittest.main()